1use std::path::{Path, PathBuf};
18
19use serde::{Deserialize, Serialize};
20
21use crate::adapters::{all_config_dir_names, all_tool_vocabulary};
22use crate::core::{ConditionsRecord, RunRecord, ToolInvocation};
23use crate::pipeline::error::PipelineError;
24use crate::pipeline::guard_denials::collect_guard_denials;
25use crate::pipeline::io::{now_iso8601, write_json};
26use crate::pipeline::slots::{run_key, run_slots};
27use crate::sandbox::policy::classify_bash_with_cwd;
28use crate::sandbox::{is_shell_tool, is_under, is_write_tool, path_arg};
29use crate::validation::{SchemaName, validate_against_schema};
30
31fn is_read_tool(name: &str) -> bool {
34 all_tool_vocabulary().read_tools.iter().any(|t| t == name)
35}
36
37const LIVE_SOURCE_REASON: &str =
38 "reads the live skill source instead of its staged copy — the arm may be contaminated";
39
40#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
43pub struct StrayFinding {
44 pub tool: String,
45 #[serde(skip_serializing_if = "Option::is_none")]
46 pub path: Option<String>,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 pub command: Option<String>,
49 pub ordinal: u32,
50 pub reason: String,
51}
52
53#[derive(Debug, Clone, PartialEq, Eq, Default)]
55pub struct RunFindings {
56 pub violations: Vec<StrayFinding>,
57 pub warnings: Vec<StrayFinding>,
58}
59
60fn command_of(inv: &ToolInvocation) -> &str {
62 inv.args
63 .as_ref()
64 .and_then(|a| a.get("command"))
65 .and_then(|v| v.as_str())
66 .unwrap_or("")
67}
68
69pub fn detect_stray_writes(
72 invocations: &[ToolInvocation],
73 eval_root: &str,
74 invocation_cwd: &Path,
75) -> RunFindings {
76 let mut findings = RunFindings::default();
77
78 for inv in invocations {
79 if is_write_tool(&inv.name) {
80 if let Some(p) = inv.args.as_ref().and_then(path_arg)
81 && !is_under(p, eval_root, invocation_cwd)
82 {
83 findings.violations.push(StrayFinding {
84 tool: inv.name.clone(),
85 path: Some(p.to_string()),
86 command: None,
87 ordinal: inv.ordinal,
88 reason: "writes outside the run's task environment".to_string(),
89 });
90 }
91 continue;
92 }
93
94 if is_shell_tool(&inv.name) {
95 let command = command_of(inv);
96 if let Some(classification) = classify_bash_with_cwd(
97 command,
98 std::slice::from_ref(&eval_root.to_string()),
99 invocation_cwd,
100 ) {
101 findings.warnings.push(StrayFinding {
102 tool: inv.name.clone(),
103 path: None,
104 command: Some(command.to_string()),
105 ordinal: inv.ordinal,
106 reason: classification.reason.to_string(),
107 });
108 }
109 }
110 }
111
112 findings
113}
114
115fn absolutize(p: &Path) -> PathBuf {
117 std::path::absolute(p).unwrap_or_else(|_| p.to_path_buf())
118}
119
120fn path_relative(from: &Path, to: &Path) -> String {
124 let from_comps: Vec<_> = from.components().collect();
125 let to_comps: Vec<_> = to.components().collect();
126 let mut i = 0;
127 while i < from_comps.len() && i < to_comps.len() && from_comps[i] == to_comps[i] {
128 i += 1;
129 }
130 let mut parts: Vec<String> = vec!["..".to_string(); from_comps.len() - i];
131 for c in &to_comps[i..] {
132 parts.push(c.as_os_str().to_string_lossy().into_owned());
133 }
134 parts.join("/")
135}
136
137fn is_leading_boundary(b: u8) -> bool {
140 b.is_ascii_whitespace() || matches!(b, b'\'' | b'"' | b'=' | b':' | b'(' | b'/')
141}
142
143fn is_trailing_boundary(b: u8) -> bool {
146 b == b'/' || b.is_ascii_whitespace() || matches!(b, b'\'' | b'"' | b')')
147}
148
149fn references_bare_rel(command: &str, rel: &str, config_dirs: &[String]) -> bool {
155 if rel.is_empty() {
156 return false;
157 }
158 let bytes = command.as_bytes();
159 let mut search_from = 0;
160 while let Some(off) = command[search_from..].find(rel) {
161 let start = search_from + off;
162 let end = start + rel.len();
163
164 let leading_ok = start == 0 || is_leading_boundary(bytes[start - 1]);
165 let lookbehind_ok = start == 0 || {
168 let before = &command[..start - 1];
169 !config_dirs.iter().any(|dir| before.ends_with(dir.as_str()))
170 };
171 let trailing_ok = end == command.len() || is_trailing_boundary(bytes[end]);
172
173 if leading_ok && lookbehind_ok && trailing_ok {
174 return true;
175 }
176 search_from = start + 1;
177 }
178 false
179}
180
181pub fn detect_live_source_reads(
185 invocations: &[ToolInvocation],
186 live_skill_dir: &Path,
187 repo_root: &Path,
188) -> Vec<StrayFinding> {
189 let mut findings = Vec::new();
190 let live_dir = absolutize(live_skill_dir);
191 let live_dir_str = live_dir.to_string_lossy();
192 let rel = path_relative(repo_root, &live_dir);
193 let rel_usable = !rel.starts_with("..");
194 let config_dirs = all_config_dir_names();
195
196 for inv in invocations {
197 if is_read_tool(&inv.name) {
198 if let Some(p) = inv.args.as_ref().and_then(path_arg)
199 && is_under(p, &live_dir_str, repo_root)
200 {
201 findings.push(StrayFinding {
202 tool: inv.name.clone(),
203 path: Some(p.to_string()),
204 command: None,
205 ordinal: inv.ordinal,
206 reason: LIVE_SOURCE_REASON.to_string(),
207 });
208 }
209 continue;
210 }
211
212 if is_shell_tool(&inv.name) {
213 let command = command_of(inv);
214 if command.contains(live_dir_str.as_ref())
215 || (rel_usable && references_bare_rel(command, &rel, &config_dirs))
216 {
217 findings.push(StrayFinding {
218 tool: inv.name.clone(),
219 path: None,
220 command: Some(command.to_string()),
221 ordinal: inv.ordinal,
222 reason: LIVE_SOURCE_REASON.to_string(),
223 });
224 }
225 }
226 }
227
228 findings
229}
230
231#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
235pub struct RunReport {
236 pub eval_id: String,
237 pub condition: String,
238 #[serde(skip_serializing_if = "Option::is_none")]
240 pub run_index: Option<u32>,
241 pub violations: Vec<StrayFinding>,
242 pub warnings: Vec<StrayFinding>,
243 pub live_source_reads: Vec<StrayFinding>,
244}
245
246#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
248pub struct Totals {
249 pub violations: usize,
250 pub warnings: usize,
251 pub live_source_reads: usize,
252}
253
254#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
256pub struct StrayWritesReport {
257 pub generated: String,
258 pub iteration: u32,
259 pub totals: Totals,
260 pub runs: Vec<RunReport>,
261 #[serde(skip)]
265 pub invocations_inspected: usize,
266 #[serde(skip)]
270 pub guard_denials: usize,
271}
272
273#[derive(Debug, Deserialize)]
275struct DispatchEnvelope {
276 tasks: Option<Vec<DispatchRef>>,
277}
278
279#[derive(Debug, Deserialize)]
280struct DispatchRef {
281 eval_id: String,
282 condition: String,
283 #[serde(default)]
284 run_index: Option<u32>,
285 #[serde(default)]
286 eval_root: Option<String>,
287}
288
289pub fn detect_stray_writes_report(
293 iteration_dir: &Path,
294 iteration: u32,
295 live_skill_dir: &Path,
296 repo_root: &Path,
297) -> Result<StrayWritesReport, PipelineError> {
298 let conditions_path = iteration_dir.join("conditions.json");
299 if !conditions_path.exists() {
300 return Err(PipelineError::Message(format!(
301 "missing: {}",
302 conditions_path.display()
303 )));
304 }
305 let conditions: ConditionsRecord =
306 serde_json::from_str(&std::fs::read_to_string(&conditions_path)?)?;
307 let condition_names: Vec<String> = conditions
308 .conditions
309 .iter()
310 .map(|c| c.name.clone())
311 .collect();
312
313 let allowed_roots_by_key = eval_roots_by_key(iteration_dir);
314 let guard_denials = collect_guard_denials(iteration_dir, iteration, repo_root)?;
315
316 let mut runs = Vec::new();
317 let mut totals = Totals {
318 violations: 0,
319 warnings: 0,
320 live_source_reads: 0,
321 };
322 let mut invocations_inspected = 0usize;
323
324 let mut eval_dirs: Vec<String> = std::fs::read_dir(iteration_dir)?
325 .flatten()
326 .filter_map(|e| {
327 let name = e.file_name().to_string_lossy().into_owned();
328 name.starts_with("eval-").then_some(name)
329 })
330 .collect();
331 eval_dirs.sort();
332
333 for dir_name in &eval_dirs {
334 let eval_id = dir_name.strip_prefix("eval-").unwrap_or(dir_name);
335 for cond in &condition_names {
336 let cond_dir = iteration_dir.join(dir_name).join(cond);
337 for slot in run_slots(&cond_dir) {
338 let run_path = slot.dir.join("run.json");
339 if !run_path.exists() {
340 continue;
341 }
342 let source = run_path.to_string_lossy();
343 let run: RunRecord = validate_against_schema(
344 SchemaName::RunRecord,
345 &serde_json::from_str(&std::fs::read_to_string(&run_path)?)?,
346 &source,
347 )?;
348
349 let eval_root = allowed_roots_by_key.get(&run_key(eval_id, cond, slot.run_index));
350
351 invocations_inspected += run.tool_invocations.len();
352 let findings = match eval_root {
357 Some(dir) => detect_stray_writes(&run.tool_invocations, dir, Path::new(dir)),
358 None => {
359 let run_label = slot
360 .run_index
361 .map(|k| format!(" run-{k}"))
362 .unwrap_or_default();
363 eprintln!(
364 "⚠ {eval_id}/{cond}{run_label}: no eval_root in dispatch.json — \
365 skipping out-of-bounds write classification (boundary unknown)"
366 );
367 RunFindings::default()
368 }
369 };
370 let live_reads =
371 detect_live_source_reads(&run.tool_invocations, live_skill_dir, repo_root);
372
373 totals.violations += findings.violations.len();
374 totals.warnings += findings.warnings.len();
375 totals.live_source_reads += live_reads.len();
376
377 if !findings.violations.is_empty()
378 || !findings.warnings.is_empty()
379 || !live_reads.is_empty()
380 {
381 runs.push(RunReport {
382 eval_id: eval_id.to_string(),
383 condition: cond.clone(),
384 run_index: slot.run_index,
385 violations: findings.violations,
386 warnings: findings.warnings,
387 live_source_reads: live_reads,
388 });
389 }
390 }
391 }
392 }
393
394 let report = StrayWritesReport {
395 generated: now_iso8601(),
396 iteration,
397 totals,
398 runs,
399 invocations_inspected,
400 guard_denials: guard_denials.total_denials,
401 };
402
403 let out_path = iteration_dir.join("stray-writes.json");
404 validate_against_schema::<serde_json::Value>(
405 SchemaName::StrayWrites,
406 &serde_json::to_value(&report)?,
407 &out_path.to_string_lossy(),
408 )?;
409 write_json(&out_path, &report)?;
410
411 Ok(report)
412}
413
414fn eval_roots_by_key(iteration_dir: &Path) -> std::collections::HashMap<String, String> {
417 let mut out = std::collections::HashMap::new();
418 if let Ok(raw) = std::fs::read_to_string(iteration_dir.join("dispatch.json"))
419 && let Ok(env) = serde_json::from_str::<DispatchEnvelope>(&raw)
420 {
421 for t in env.tasks.unwrap_or_default() {
422 if let Some(dir) = t.eval_root {
423 out.insert(run_key(&t.eval_id, &t.condition, t.run_index), dir);
424 }
425 }
426 }
427 out
428}
429
430#[cfg(test)]
431mod tests {
432 use super::*;
433 use serde_json::json;
434
435 const ALLOWED_ROOT: &str = "/work/iteration-1/env-g1-with_skill";
436 const REPO: &str = "/work/repo";
437 const LIVE_SKILL: &str = "/work/repo/skills/mr-review";
438
439 fn inv(name: &str, args: serde_json::Value, ordinal: u32) -> ToolInvocation {
441 ToolInvocation {
442 name: name.to_string(),
443 args: Some(args),
444 result: None,
445 ordinal,
446 }
447 }
448
449 fn repo() -> &'static Path {
450 Path::new(REPO)
451 }
452
453 fn live() -> &'static Path {
454 Path::new(LIVE_SKILL)
455 }
456
457 #[test]
460 fn a_write_inside_the_task_environment_is_clean() {
461 let f = detect_stray_writes(
462 &[inv(
463 "Write",
464 json!({"file_path": format!("{ALLOWED_ROOT}/answer.md")}),
465 0,
466 )],
467 ALLOWED_ROOT,
468 repo(),
469 );
470 assert!(f.violations.is_empty());
471 assert!(f.warnings.is_empty());
472 }
473
474 #[test]
475 fn a_relative_write_resolves_from_the_task_environment() {
476 let f = detect_stray_writes(
477 &[inv("Edit", json!({"file_path": "src/lib.rs"}), 0)],
478 ALLOWED_ROOT,
479 Path::new(ALLOWED_ROOT),
480 );
481 assert!(f.violations.is_empty());
482 }
483
484 #[test]
485 fn a_write_outside_the_task_environment_is_a_violation() {
486 let f = detect_stray_writes(
487 &[inv(
488 "Write",
489 json!({"file_path": format!("{REPO}/runner/run.ts")}),
490 2,
491 )],
492 ALLOWED_ROOT,
493 repo(),
494 );
495 assert_eq!(f.violations.len(), 1);
496 assert_eq!(f.violations[0].tool, "Write");
497 assert_eq!(
498 f.violations[0].path.as_deref(),
499 Some(&*format!("{REPO}/runner/run.ts"))
500 );
501 assert_eq!(f.violations[0].ordinal, 2);
502 }
503
504 #[test]
505 fn edit_multiedit_notebookedit_outside_the_task_environment_is_a_violation() {
506 let f = detect_stray_writes(
507 &[
508 inv("Edit", json!({"file_path": "/etc/hosts"}), 0),
509 inv("NotebookEdit", json!({"notebook_path": "/tmp/x.ipynb"}), 1),
510 ],
511 ALLOWED_ROOT,
512 repo(),
513 );
514 let mut tools: Vec<&str> = f.violations.iter().map(|v| v.tool.as_str()).collect();
515 tools.sort();
516 assert_eq!(tools, vec!["Edit", "NotebookEdit"]);
517 }
518
519 #[test]
520 fn an_install_command_is_a_warning() {
521 let f = detect_stray_writes(
522 &[inv("Bash", json!({"command": "npm install left-pad"}), 0)],
523 ALLOWED_ROOT,
524 repo(),
525 );
526 assert_eq!(f.warnings.len(), 1);
527 assert_eq!(f.warnings[0].tool, "Bash");
528 assert!(f.warnings[0].reason.to_lowercase().contains("install"));
529 }
530
531 #[test]
532 fn a_codex_command_execution_install_is_a_warning() {
533 let f = detect_stray_writes(
534 &[inv(
535 "command_execution",
536 json!({"command": "npm install left-pad"}),
537 0,
538 )],
539 ALLOWED_ROOT,
540 repo(),
541 );
542 assert_eq!(f.warnings.len(), 1);
543 assert_eq!(f.warnings[0].tool, "command_execution");
544 assert!(f.warnings[0].reason.to_lowercase().contains("install"));
545 }
546
547 #[test]
548 fn a_codex_file_change_outside_the_task_environment_is_a_violation() {
549 let f = detect_stray_writes(
550 &[inv(
551 "file_change",
552 json!({"path": format!("{REPO}/src/app.ts")}),
553 4,
554 )],
555 ALLOWED_ROOT,
556 repo(),
557 );
558 assert_eq!(f.violations.len(), 1);
559 assert_eq!(f.violations[0].tool, "file_change");
560 assert_eq!(
561 f.violations[0].path.as_deref(),
562 Some(&*format!("{REPO}/src/app.ts"))
563 );
564 assert_eq!(f.violations[0].ordinal, 4);
565 }
566
567 #[test]
568 fn a_mutating_bash_scoped_to_the_task_environment_is_not_flagged() {
569 let f = detect_stray_writes(
570 &[inv(
571 "Bash",
572 json!({"command": format!("echo hi > {ALLOWED_ROOT}/log.txt")}),
573 0,
574 )],
575 ALLOWED_ROOT,
576 repo(),
577 );
578 assert!(f.warnings.is_empty());
579 }
580
581 #[test]
582 fn a_relative_redirection_resolves_from_the_task_environment() {
583 let f = detect_stray_writes(
584 &[inv(
585 "Bash",
586 json!({"command": "printf done > final-message.md"}),
587 0,
588 )],
589 ALLOWED_ROOT,
590 Path::new(ALLOWED_ROOT),
591 );
592 assert!(f.warnings.is_empty(), "{:?}", f.warnings);
593 }
594
595 #[test]
596 fn git_worktree_add_is_a_warning() {
597 let f = detect_stray_writes(
598 &[inv(
599 "Bash",
600 json!({"command": "git worktree add ../wt -b scratch"}),
601 0,
602 )],
603 ALLOWED_ROOT,
604 repo(),
605 );
606 assert_eq!(f.warnings.len(), 1);
607 assert!(f.warnings[0].reason.to_lowercase().contains("worktree"));
608 }
609
610 #[test]
611 fn creating_a_path_under_dot_claude_is_a_warning() {
612 let f = detect_stray_writes(
613 &[inv("Bash", json!({"command": "mkdir -p .claude/foo"}), 0)],
614 ALLOWED_ROOT,
615 repo(),
616 );
617 assert_eq!(f.warnings.len(), 1);
618 assert!(f.warnings[0].reason.to_lowercase().contains("config dir"));
619 }
620
621 #[test]
622 fn creating_a_path_under_dot_codex_is_a_warning() {
623 let f = detect_stray_writes(
624 &[inv(
625 "Bash",
626 json!({"command": "cp evil.json .codex/hooks.json"}),
627 0,
628 )],
629 ALLOWED_ROOT,
630 repo(),
631 );
632 assert_eq!(f.warnings.len(), 1);
633 assert!(f.warnings[0].reason.to_lowercase().contains("config dir"));
634 }
635
636 #[test]
637 fn read_only_tools_are_never_flagged() {
638 let f = detect_stray_writes(
639 &[
640 inv("Read", json!({"file_path": "/anywhere"}), 0),
641 inv("Grep", json!({"pattern": "x"}), 1),
642 inv("Bash", json!({"command": "ls -la /"}), 2),
643 ],
644 ALLOWED_ROOT,
645 repo(),
646 );
647 assert!(f.violations.is_empty());
648 assert!(f.warnings.is_empty());
649 }
650
651 #[test]
654 fn a_read_of_the_live_skill_md_is_flagged() {
655 let f = detect_live_source_reads(
656 &[inv(
657 "Read",
658 json!({"file_path": format!("{LIVE_SKILL}/SKILL.md")}),
659 1,
660 )],
661 live(),
662 repo(),
663 );
664 assert_eq!(f.len(), 1);
665 assert_eq!(f[0].tool, "Read");
666 assert_eq!(
667 f[0].path.as_deref(),
668 Some(&*format!("{LIVE_SKILL}/SKILL.md"))
669 );
670 assert_eq!(f[0].ordinal, 1);
671 assert!(f[0].reason.to_lowercase().contains("live skill source"));
672 }
673
674 #[test]
675 fn a_read_of_a_staged_eval_copy_is_not_flagged() {
676 let f = detect_live_source_reads(
677 &[inv(
678 "Read",
679 json!({"file_path": format!("{REPO}/.claude/skills/slow-powers-eval-1-old_skill__mr-review/SKILL.md")}),
680 0,
681 )],
682 live(),
683 repo(),
684 );
685 assert!(f.is_empty());
686 }
687
688 #[test]
689 fn a_relative_read_resolving_under_the_live_dir_is_flagged() {
690 let f = detect_live_source_reads(
691 &[inv(
692 "Read",
693 json!({"file_path": "skills/mr-review/SKILL.md"}),
694 0,
695 )],
696 live(),
697 repo(),
698 );
699 assert_eq!(f.len(), 1);
700 }
701
702 #[test]
703 fn a_grep_scoped_to_the_live_dir_is_flagged() {
704 let f = detect_live_source_reads(
705 &[inv("Grep", json!({"pattern": "x", "path": LIVE_SKILL}), 2)],
706 live(),
707 repo(),
708 );
709 assert_eq!(f.len(), 1);
710 assert_eq!(f[0].tool, "Grep");
711 }
712
713 #[test]
714 fn a_bash_referencing_the_live_dir_relatively_is_flagged() {
715 let f = detect_live_source_reads(
716 &[inv(
717 "Bash",
718 json!({"command": "cat skills/mr-review/SKILL.md"}),
719 3,
720 )],
721 live(),
722 repo(),
723 );
724 assert_eq!(f.len(), 1);
725 assert_eq!(f[0].tool, "Bash");
726 assert_eq!(
727 f[0].command.as_deref(),
728 Some("cat skills/mr-review/SKILL.md")
729 );
730 }
731
732 #[test]
733 fn a_codex_command_referencing_the_live_dir_relatively_is_flagged() {
734 let f = detect_live_source_reads(
735 &[inv(
736 "command_execution",
737 json!({"command": "cat skills/mr-review/SKILL.md"}),
738 3,
739 )],
740 live(),
741 repo(),
742 );
743 assert_eq!(f.len(), 1);
744 assert_eq!(f[0].tool, "command_execution");
745 assert_eq!(
746 f[0].command.as_deref(),
747 Some("cat skills/mr-review/SKILL.md")
748 );
749 }
750
751 #[test]
752 fn a_bash_referencing_the_live_dir_absolutely_is_flagged() {
753 let f = detect_live_source_reads(
754 &[inv(
755 "Bash",
756 json!({"command": format!("grep -r trigger {LIVE_SKILL}/")}),
757 0,
758 )],
759 live(),
760 repo(),
761 );
762 assert_eq!(f.len(), 1);
763 }
764
765 #[test]
766 fn a_bash_referencing_a_staged_copy_under_dot_claude_skills_is_not_flagged() {
767 let f = detect_live_source_reads(
768 &[inv(
769 "Bash",
770 json!({"command": "cat .claude/skills/mr-review/SKILL.md"}),
771 0,
772 )],
773 live(),
774 repo(),
775 );
776 assert!(f.is_empty());
777 }
778
779 #[test]
780 fn a_bash_referencing_a_staged_copy_under_dot_agents_skills_is_not_flagged() {
781 let f = detect_live_source_reads(
782 &[inv(
783 "Bash",
784 json!({"command": "cat .agents/skills/mr-review/SKILL.md"}),
785 0,
786 )],
787 live(),
788 repo(),
789 );
790 assert!(f.is_empty());
791 }
792
793 #[test]
794 fn a_bash_referencing_a_staged_copy_under_dot_opencode_skills_is_not_flagged() {
795 let f = detect_live_source_reads(
796 &[inv(
797 "Bash",
798 json!({"command": "cat .opencode/skills/mr-review/SKILL.md"}),
799 0,
800 )],
801 live(),
802 repo(),
803 );
804 assert!(f.is_empty());
805 }
806
807 #[test]
808 fn unrelated_reads_and_commands_are_not_flagged() {
809 let f = detect_live_source_reads(
810 &[
811 inv(
812 "Read",
813 json!({"file_path": format!("{ALLOWED_ROOT}/x.md")}),
814 0,
815 ),
816 inv("Bash", json!({"command": "ls .eval-magic"}), 1),
817 inv(
819 "Write",
820 json!({"file_path": format!("{LIVE_SKILL}/SKILL.md")}),
821 2,
822 ),
823 ],
824 live(),
825 repo(),
826 );
827 assert!(f.is_empty());
828 }
829}