eval_magic/pipeline/
diff_scope.rs

1//! Baseline capture and deterministic final-environment diff metrics.
2//!
3//! A baseline snapshots every file in a task's private `eval_root` after
4//! framework setup. Measurement compares the completed task with that snapshot,
5//! excluding only the task's `.eval-magic-outputs` subtree.
6
7use std::collections::{BTreeSet, HashMap};
8use std::fs;
9use std::path::{Path, PathBuf};
10
11use serde::{Deserialize, Serialize};
12use similar::{Algorithm, DiffTag, capture_diff_slices};
13use walkdir::{DirEntry, WalkDir};
14
15use crate::pipeline::error::PipelineError;
16use crate::pipeline::io::write_json;
17
18const BASELINE_DIR: &str = "diff-scope-baseline";
19const BASELINE_MANIFEST: &str = "manifest.json";
20const BASELINE_FILES: &str = "files";
21const RESULT_FILE: &str = "diff-scope.json";
22
23#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
24pub struct DiffScopeMetrics {
25    pub files_touched: u64,
26    pub lines_added: u64,
27    pub lines_removed: u64,
28    pub hunks: u64,
29}
30
31impl DiffScopeMetrics {
32    pub fn lines_changed(self) -> u64 {
33        self.lines_added.saturating_add(self.lines_removed)
34    }
35}
36
37#[derive(Debug, Serialize, Deserialize)]
38struct BaselineManifest {
39    preexisting_files: Vec<String>,
40}
41
42#[derive(Debug, Deserialize)]
43struct DispatchFile {
44    #[serde(default)]
45    tasks: Vec<DispatchTask>,
46}
47
48#[derive(Debug, Deserialize)]
49struct DispatchTask {
50    eval_id: String,
51    condition: String,
52    #[serde(default)]
53    run_index: Option<u32>,
54    eval_root: Option<String>,
55    run_record_path: String,
56}
57
58#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
59pub struct DiffScopeSummary {
60    pub measured: usize,
61    pub reused: usize,
62    pub missing_baseline: usize,
63    pub shared_environment: usize,
64}
65
66#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
67struct FileDiff {
68    lines_added: u64,
69    lines_removed: u64,
70    hunks: u64,
71}
72
73fn diff_bytes(old: &[u8], new: &[u8]) -> FileDiff {
74    let old_lines: Vec<&[u8]> = old.split_inclusive(|byte| *byte == b'\n').collect();
75    let new_lines: Vec<&[u8]> = new.split_inclusive(|byte| *byte == b'\n').collect();
76    let mut result = FileDiff::default();
77    let mut in_hunk = false;
78
79    for operation in capture_diff_slices(Algorithm::Myers, &old_lines, &new_lines) {
80        let (tag, old_range, new_range) = operation.as_tag_tuple();
81        match tag {
82            DiffTag::Equal => in_hunk = false,
83            DiffTag::Delete => {
84                if !in_hunk {
85                    result.hunks += 1;
86                    in_hunk = true;
87                }
88                result.lines_removed += old_range.len() as u64;
89            }
90            DiffTag::Insert => {
91                if !in_hunk {
92                    result.hunks += 1;
93                    in_hunk = true;
94                }
95                result.lines_added += new_range.len() as u64;
96            }
97            DiffTag::Replace => {
98                if !in_hunk {
99                    result.hunks += 1;
100                    in_hunk = true;
101                }
102                result.lines_removed += old_range.len() as u64;
103                result.lines_added += new_range.len() as u64;
104            }
105        }
106    }
107    result
108}
109
110pub fn capture_iteration_baselines(iteration_dir: &Path) -> Result<(), PipelineError> {
111    let dispatch_path = iteration_dir.join("dispatch.json");
112    let dispatch: DispatchFile = serde_json::from_str(&fs::read_to_string(&dispatch_path)?)?;
113    for task in dispatch.tasks {
114        let eval_root = task.eval_root.ok_or_else(|| {
115            PipelineError::Message(format!(
116                "dispatch task in {} has no eval_root for diff-scope capture",
117                dispatch_path.display()
118            ))
119        })?;
120        let run_dir = Path::new(&task.run_record_path).parent().ok_or_else(|| {
121            PipelineError::Message(format!(
122                "diff-scope task has no run directory in run_record_path: {}",
123                task.run_record_path
124            ))
125        })?;
126        fs::create_dir_all(run_dir)?;
127        capture_task_baseline(Path::new(&eval_root), run_dir)?;
128    }
129    Ok(())
130}
131
132pub fn measure_iteration_diff_scopes(
133    iteration_dir: &Path,
134) -> Result<DiffScopeSummary, PipelineError> {
135    let dispatch_path = iteration_dir.join("dispatch.json");
136    // Legacy and hand-authored iterations may contain run records without a
137    // dispatch manifest. They remain gradeable; only an explicit diff_scope
138    // assertion makes the missing measurement a finalize-time error.
139    if !dispatch_path.exists() {
140        return Ok(DiffScopeSummary::default());
141    }
142    let dispatch: DispatchFile = serde_json::from_str(&fs::read_to_string(&dispatch_path)?)?;
143    let root_counts: HashMap<String, usize> = dispatch
144        .tasks
145        .iter()
146        .filter_map(|task| task.eval_root.as_deref())
147        .fold(HashMap::new(), |mut counts, root| {
148            *counts.entry(root.to_string()).or_default() += 1;
149            counts
150        });
151    let mut summary = DiffScopeSummary::default();
152
153    for task in dispatch.tasks {
154        let run_record_path = Path::new(&task.run_record_path);
155        let run_dir = run_record_path.parent().ok_or_else(|| {
156            PipelineError::Message(format!(
157                "diff-scope task has no run directory in run_record_path: {}",
158                task.run_record_path
159            ))
160        })?;
161        if !run_record_path.exists() {
162            continue;
163        }
164        let result_path = run_dir.join(RESULT_FILE);
165        if result_path.exists() {
166            let value = serde_json::from_str(&fs::read_to_string(&result_path)?)?;
167            crate::validation::validate_against_schema::<DiffScopeMetrics>(
168                crate::validation::SchemaName::DiffScope,
169                &value,
170                &result_path.to_string_lossy(),
171            )?;
172            summary.reused += 1;
173            continue;
174        }
175        let run_label = task
176            .run_index
177            .map(|run| format!("/run-{run}"))
178            .unwrap_or_default();
179        let Some(eval_root) = task.eval_root.as_deref() else {
180            eprintln!(
181                "warn: {}/{}{run_label} has no eval_root — diff-scope unavailable; rebuild the iteration to capture metrics",
182                task.eval_id, task.condition
183            );
184            summary.missing_baseline += 1;
185            continue;
186        };
187        if root_counts.get(eval_root).copied().unwrap_or_default() != 1 {
188            eprintln!(
189                "warn: {}/{}{run_label} shares eval_root with another task — diff-scope unavailable; rebuild the iteration for task-scoped environments",
190                task.eval_id, task.condition
191            );
192            summary.shared_environment += 1;
193            continue;
194        }
195        if !run_dir.join(BASELINE_DIR).join(BASELINE_MANIFEST).exists() {
196            eprintln!(
197                "warn: {}/{}{run_label} has no pre-dispatch baseline — diff-scope unavailable; rebuild the iteration to capture metrics",
198                task.eval_id, task.condition
199            );
200            summary.missing_baseline += 1;
201            continue;
202        }
203        if run_dir.join("command-checks").exists() {
204            return Err(PipelineError::Message(format!(
205                "cannot capture diff scope for {}/{}{run_label} after command checks have run; rebuild the iteration",
206                task.eval_id, task.condition
207            )));
208        }
209
210        let metrics = measure_task_diff(Path::new(eval_root), run_dir)?;
211        crate::validation::validate_against_schema::<DiffScopeMetrics>(
212            crate::validation::SchemaName::DiffScope,
213            &serde_json::to_value(metrics)?,
214            &result_path.to_string_lossy(),
215        )?;
216        write_json(&result_path, &metrics)?;
217        summary.measured += 1;
218    }
219    Ok(summary)
220}
221
222fn capture_task_baseline(eval_root: &Path, run_dir: &Path) -> Result<(), PipelineError> {
223    let baseline_dir = run_dir.join(BASELINE_DIR);
224    if baseline_dir.exists() {
225        fs::remove_dir_all(&baseline_dir)?;
226    }
227    let file_snapshot = baseline_dir.join(BASELINE_FILES);
228    fs::create_dir_all(&file_snapshot)?;
229
230    let excluded_roots = [
231        eval_root.join(".eval-magic-outputs"),
232        eval_root.join(".git"),
233    ];
234    let mut preexisting_paths = walk_files(eval_root, &excluded_roots)?;
235    preexisting_paths.sort();
236    let preexisting_files = preexisting_paths
237        .iter()
238        .map(|path| relative_key(eval_root, path))
239        .collect::<Result<Vec<_>, _>>()?;
240    write_json(
241        &baseline_dir.join(BASELINE_MANIFEST),
242        &BaselineManifest { preexisting_files },
243    )?;
244
245    for source in preexisting_paths {
246        let relative = source.strip_prefix(eval_root).map_err(|_| {
247            PipelineError::Message(format!(
248                "diff-scope baseline path {} is outside {}",
249                source.display(),
250                eval_root.display()
251            ))
252        })?;
253        copy_entry(&source, &file_snapshot.join(relative))?;
254    }
255    Ok(())
256}
257
258fn measure_task_diff(eval_root: &Path, run_dir: &Path) -> Result<DiffScopeMetrics, PipelineError> {
259    let baseline_dir = run_dir.join(BASELINE_DIR);
260    let manifest: BaselineManifest =
261        serde_json::from_str(&fs::read_to_string(baseline_dir.join(BASELINE_MANIFEST))?)?;
262    let file_snapshot = baseline_dir.join(BASELINE_FILES);
263    let mut candidates = BTreeSet::new();
264
265    for relative in manifest.preexisting_files {
266        if fs::symlink_metadata(file_snapshot.join(&relative)).is_err() {
267            return Err(PipelineError::Message(format!(
268                "diff-scope baseline is incomplete: missing snapshot for {relative}"
269            )));
270        }
271        candidates.insert(relative);
272    }
273    let excluded_roots = [
274        eval_root.join(".eval-magic-outputs"),
275        eval_root.join(".git"),
276    ];
277    for path in walk_files(eval_root, &excluded_roots)? {
278        candidates.insert(relative_key(eval_root, &path)?);
279    }
280
281    let mut metrics = DiffScopeMetrics::default();
282    for relative in candidates {
283        let before = file_snapshot.join(&relative);
284        let after = eval_root.join(&relative);
285        let old = file_content(&before)?;
286        let new = file_content(&after)?;
287        if old == new {
288            continue;
289        }
290        metrics.files_touched += 1;
291        let diff = match (old, new) {
292            (FileContent::Regular(old), FileContent::Regular(new)) => Some(diff_bytes(&old, &new)),
293            (FileContent::Missing, FileContent::Regular(new)) => Some(diff_bytes(&[], &new)),
294            (FileContent::Regular(old), FileContent::Missing) => Some(diff_bytes(&old, &[])),
295            _ => None,
296        };
297        if let Some(diff) = diff {
298            metrics.lines_added += diff.lines_added;
299            metrics.lines_removed += diff.lines_removed;
300            metrics.hunks += diff.hunks;
301        }
302    }
303    Ok(metrics)
304}
305
306#[derive(Debug, PartialEq, Eq)]
307enum FileContent {
308    Missing,
309    Regular(Vec<u8>),
310    Symlink(PathBuf),
311}
312
313fn file_content(path: &Path) -> Result<FileContent, PipelineError> {
314    let metadata = match fs::symlink_metadata(path) {
315        Ok(metadata) => metadata,
316        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
317            return Ok(FileContent::Missing);
318        }
319        Err(error) => return Err(error.into()),
320    };
321    if metadata.file_type().is_symlink() {
322        return Ok(FileContent::Symlink(fs::read_link(path)?));
323    }
324    if metadata.is_file() {
325        return Ok(FileContent::Regular(fs::read(path)?));
326    }
327    Ok(FileContent::Missing)
328}
329
330fn walk_files(root: &Path, excluded_roots: &[PathBuf]) -> Result<Vec<PathBuf>, PipelineError> {
331    if !root.exists() {
332        return Ok(Vec::new());
333    }
334    WalkDir::new(root)
335        .follow_links(false)
336        .into_iter()
337        .filter_entry(|entry| !is_excluded(entry, excluded_roots))
338        .filter_map(|entry| match entry {
339            Ok(entry) if entry.file_type().is_file() || entry.file_type().is_symlink() => {
340                Some(Ok(entry.into_path()))
341            }
342            Ok(_) => None,
343            Err(error) => Some(Err(PipelineError::Message(format!(
344                "could not walk diff-scope path under {}: {error}",
345                root.display()
346            )))),
347        })
348        .collect()
349}
350
351fn is_excluded(entry: &DirEntry, excluded_roots: &[PathBuf]) -> bool {
352    excluded_roots
353        .iter()
354        .any(|excluded| entry.path().starts_with(excluded))
355}
356
357fn relative_key(root: &Path, path: &Path) -> Result<String, PipelineError> {
358    let relative = path.strip_prefix(root).map_err(|_| {
359        PipelineError::Message(format!(
360            "diff-scope path {} is outside {}",
361            path.display(),
362            root.display()
363        ))
364    })?;
365    Ok(relative
366        .components()
367        .map(|component| component.as_os_str().to_string_lossy())
368        .collect::<Vec<_>>()
369        .join("/"))
370}
371
372fn copy_entry(source: &Path, destination: &Path) -> Result<(), PipelineError> {
373    let metadata = fs::symlink_metadata(source)?;
374    if metadata.file_type().is_symlink() {
375        if let Some(parent) = destination.parent() {
376            fs::create_dir_all(parent)?;
377        }
378        let target = fs::read_link(source)?;
379        #[cfg(unix)]
380        std::os::unix::fs::symlink(target, destination)?;
381        #[cfg(windows)]
382        if source.metadata().is_ok_and(|metadata| metadata.is_dir()) {
383            std::os::windows::fs::symlink_dir(target, destination)?;
384        } else {
385            std::os::windows::fs::symlink_file(target, destination)?;
386        }
387    } else if metadata.is_dir() {
388        fs::create_dir_all(destination)?;
389        for entry in fs::read_dir(source)? {
390            let entry = entry?;
391            copy_entry(&entry.path(), &destination.join(entry.file_name()))?;
392        }
393    } else {
394        if let Some(parent) = destination.parent() {
395            fs::create_dir_all(parent)?;
396        }
397        fs::copy(source, destination)?;
398    }
399    Ok(())
400}
401
402#[cfg(test)]
403mod tests {
404    use super::*;
405    use std::fs;
406
407    #[test]
408    fn byte_line_diff_counts_changes_and_zero_context_hunks() {
409        let diff = diff_bytes(b"old\nsame\nbefore\n", b"new\nsame\nafter\n");
410        assert_eq!(diff.lines_added, 2);
411        assert_eq!(diff.lines_removed, 2);
412        assert_eq!(diff.hunks, 2);
413    }
414
415    #[test]
416    fn byte_line_diff_handles_empty_trailing_newline_and_non_utf8_inputs() {
417        assert_eq!(diff_bytes(b"", b""), FileDiff::default());
418
419        let trailing = diff_bytes(b"value", b"value\n");
420        assert_eq!(trailing.lines_added, 1);
421        assert_eq!(trailing.lines_removed, 1);
422        assert_eq!(trailing.hunks, 1);
423
424        let binary = diff_bytes(&[0xff, b'\n'], &[0xfe, b'\n']);
425        assert_eq!(binary.lines_added, 1);
426        assert_eq!(binary.lines_removed, 1);
427        assert_eq!(binary.hunks, 1);
428    }
429
430    #[test]
431    fn lines_changed_saturates_untrusted_artifact_totals() {
432        let metrics = DiffScopeMetrics {
433            lines_added: u64::MAX,
434            lines_removed: 1,
435            ..DiffScopeMetrics::default()
436        };
437        assert_eq!(metrics.lines_changed(), u64::MAX);
438    }
439
440    #[test]
441    fn baseline_measurement_counts_all_task_changes_except_framework_outputs() {
442        let temp = tempfile::TempDir::new().unwrap();
443        let eval_root = temp.path().join("env");
444        let run_dir = temp.path().join("run");
445        let outputs_dir = eval_root.join(".eval-magic-outputs/eval-e1/with_skill");
446        fs::create_dir_all(eval_root.join("src")).unwrap();
447        fs::create_dir_all(&outputs_dir).unwrap();
448        fs::write(eval_root.join("src/changed.txt"), "old\nsame\n").unwrap();
449        fs::write(eval_root.join("src/deleted.txt"), "gone\n").unwrap();
450        fs::write(eval_root.join("framework.txt"), "before\n").unwrap();
451
452        capture_task_baseline(&eval_root, &run_dir).unwrap();
453
454        fs::write(eval_root.join("src/changed.txt"), "new\nsame\n").unwrap();
455        fs::remove_file(eval_root.join("src/deleted.txt")).unwrap();
456        fs::write(eval_root.join("framework.txt"), "after\n").unwrap();
457        fs::write(eval_root.join("notes.txt"), "one\ntwo\n").unwrap();
458        fs::write(outputs_dir.join("final-message.md"), "ignored\n").unwrap();
459        fs::write(
460            eval_root.join(".eval-magic-outputs/agent-created.txt"),
461            "also ignored\n",
462        )
463        .unwrap();
464
465        let metrics = measure_task_diff(&eval_root, &run_dir).unwrap();
466        assert_eq!(
467            metrics,
468            DiffScopeMetrics {
469                files_touched: 4,
470                lines_added: 4,
471                lines_removed: 3,
472                hunks: 4,
473            }
474        );
475    }
476
477    #[test]
478    fn baseline_ignores_only_runner_owned_root_git_metadata() {
479        let temp = tempfile::TempDir::new().unwrap();
480        let eval_root = temp.path().join("env");
481        let run_dir = temp.path().join("run");
482        fs::create_dir_all(eval_root.join(".git")).unwrap();
483        fs::create_dir_all(eval_root.join("vendor/.git")).unwrap();
484        fs::write(eval_root.join(".git/config"), "root-before\n").unwrap();
485        fs::write(eval_root.join("vendor/.git/config"), "nested-before\n").unwrap();
486        fs::write(eval_root.join("source.txt"), "before\n").unwrap();
487
488        capture_task_baseline(&eval_root, &run_dir).unwrap();
489
490        fs::write(eval_root.join(".git/config"), "root-after\n").unwrap();
491        fs::write(eval_root.join("vendor/.git/config"), "nested-after\n").unwrap();
492        fs::write(eval_root.join("source.txt"), "after\n").unwrap();
493
494        assert_eq!(
495            measure_task_diff(&eval_root, &run_dir).unwrap(),
496            DiffScopeMetrics {
497                files_touched: 2,
498                lines_added: 2,
499                lines_removed: 2,
500                hunks: 2,
501            }
502        );
503    }
504}
eval_magic/pipeline/diff_scope.rs

eval_magic/pipeline/
diff_scope.rs
