eval_magic/pipeline/
diff_scope.rs

1//! Baseline capture and deterministic final-environment diff metrics.
2//!
3//! A baseline snapshots every file in a task's private `eval_root` after
4//! framework setup. Measurement compares the completed task with that snapshot,
5//! excluding only the task's `.eval-magic-outputs` subtree.
6
7use std::collections::{BTreeSet, HashMap};
8use std::fs;
9use std::path::{Path, PathBuf};
10
11use serde::{Deserialize, Serialize};
12use similar::{Algorithm, DiffTag, capture_diff_slices};
13use walkdir::{DirEntry, WalkDir};
14
15use crate::pipeline::error::PipelineError;
16use crate::pipeline::io::write_json;
17
18const BASELINE_DIR: &str = "diff-scope-baseline";
19const BASELINE_MANIFEST: &str = "manifest.json";
20const BASELINE_FILES: &str = "files";
21const RESULT_FILE: &str = "diff-scope.json";
22
23#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
24pub struct DiffScopeMetrics {
25    pub files_touched: u64,
26    pub lines_added: u64,
27    pub lines_removed: u64,
28    pub hunks: u64,
29}
30
31impl DiffScopeMetrics {
32    pub fn lines_changed(self) -> u64 {
33        self.lines_added.saturating_add(self.lines_removed)
34    }
35}
36
37#[derive(Debug, Serialize, Deserialize)]
38struct BaselineManifest {
39    preexisting_files: Vec<String>,
40}
41
42#[derive(Debug, Deserialize)]
43struct DispatchFile {
44    #[serde(default)]
45    tasks: Vec<DispatchTask>,
46}
47
48#[derive(Debug, Deserialize)]
49struct DispatchTask {
50    eval_id: String,
51    condition: String,
52    #[serde(default)]
53    run_index: Option<u32>,
54    eval_root: Option<String>,
55    run_record_path: String,
56}
57
58#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
59pub struct DiffScopeSummary {
60    pub measured: usize,
61    pub reused: usize,
62    pub missing_baseline: usize,
63    pub shared_environment: usize,
64}
65
66#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
67struct FileDiff {
68    lines_added: u64,
69    lines_removed: u64,
70    hunks: u64,
71}
72
73fn diff_bytes(old: &[u8], new: &[u8]) -> FileDiff {
74    let old_lines: Vec<&[u8]> = old.split_inclusive(|byte| *byte == b'\n').collect();
75    let new_lines: Vec<&[u8]> = new.split_inclusive(|byte| *byte == b'\n').collect();
76    let mut result = FileDiff::default();
77    let mut in_hunk = false;
78
79    for operation in capture_diff_slices(Algorithm::Myers, &old_lines, &new_lines) {
80        let (tag, old_range, new_range) = operation.as_tag_tuple();
81        match tag {
82            DiffTag::Equal => in_hunk = false,
83            DiffTag::Delete => {
84                if !in_hunk {
85                    result.hunks += 1;
86                    in_hunk = true;
87                }
88                result.lines_removed += old_range.len() as u64;
89            }
90            DiffTag::Insert => {
91                if !in_hunk {
92                    result.hunks += 1;
93                    in_hunk = true;
94                }
95                result.lines_added += new_range.len() as u64;
96            }
97            DiffTag::Replace => {
98                if !in_hunk {
99                    result.hunks += 1;
100                    in_hunk = true;
101                }
102                result.lines_removed += old_range.len() as u64;
103                result.lines_added += new_range.len() as u64;
104            }
105        }
106    }
107    result
108}
109
110pub fn capture_iteration_baselines(iteration_dir: &Path) -> Result<(), PipelineError> {
111    let dispatch_path = iteration_dir.join("dispatch.json");
112    let dispatch: DispatchFile = serde_json::from_str(&fs::read_to_string(&dispatch_path)?)?;
113    for task in dispatch.tasks {
114        let eval_root = task.eval_root.ok_or_else(|| {
115            PipelineError::Message(format!(
116                "dispatch task in {} has no eval_root for diff-scope capture",
117                dispatch_path.display()
118            ))
119        })?;
120        let run_dir = Path::new(&task.run_record_path).parent().ok_or_else(|| {
121            PipelineError::Message(format!(
122                "diff-scope task has no run directory in run_record_path: {}",
123                task.run_record_path
124            ))
125        })?;
126        fs::create_dir_all(run_dir)?;
127        capture_task_baseline(Path::new(&eval_root), run_dir)?;
128    }
129    Ok(())
130}
131
132pub fn measure_iteration_diff_scopes(
133    iteration_dir: &Path,
134) -> Result<DiffScopeSummary, PipelineError> {
135    let dispatch_path = iteration_dir.join("dispatch.json");
136    // Legacy and hand-authored iterations may contain run records without a
137    // dispatch manifest. They remain gradeable; only an explicit diff_scope
138    // assertion makes the missing measurement a finalize-time error.
139    if !dispatch_path.exists() {
140        return Ok(DiffScopeSummary::default());
141    }
142    let dispatch: DispatchFile = serde_json::from_str(&fs::read_to_string(&dispatch_path)?)?;
143    let root_counts: HashMap<String, usize> = dispatch
144        .tasks
145        .iter()
146        .filter_map(|task| task.eval_root.as_deref())
147        .fold(HashMap::new(), |mut counts, root| {
148            *counts.entry(root.to_string()).or_default() += 1;
149            counts
150        });
151    let mut summary = DiffScopeSummary::default();
152
153    for task in dispatch.tasks {
154        let run_record_path = Path::new(&task.run_record_path);
155        let run_dir = run_record_path.parent().ok_or_else(|| {
156            PipelineError::Message(format!(
157                "diff-scope task has no run directory in run_record_path: {}",
158                task.run_record_path
159            ))
160        })?;
161        if !run_record_path.exists() {
162            continue;
163        }
164        let result_path = run_dir.join(RESULT_FILE);
165        if result_path.exists() {
166            let value = serde_json::from_str(&fs::read_to_string(&result_path)?)?;
167            crate::validation::validate_against_schema::<DiffScopeMetrics>(
168                crate::validation::SchemaName::DiffScope,
169                &value,
170                &result_path.to_string_lossy(),
171            )?;
172            summary.reused += 1;
173            continue;
174        }
175        let run_label = task
176            .run_index
177            .map(|run| format!("/run-{run}"))
178            .unwrap_or_default();
179        let Some(eval_root) = task.eval_root.as_deref() else {
180            eprintln!(
181                "warn: {}/{}{run_label} has no eval_root — diff-scope unavailable; rebuild the iteration to capture metrics",
182                task.eval_id, task.condition
183            );
184            summary.missing_baseline += 1;
185            continue;
186        };
187        if root_counts.get(eval_root).copied().unwrap_or_default() != 1 {
188            eprintln!(
189                "warn: {}/{}{run_label} shares eval_root with another task — diff-scope unavailable; rebuild the iteration for task-scoped environments",
190                task.eval_id, task.condition
191            );
192            summary.shared_environment += 1;
193            continue;
194        }
195        if !run_dir.join(BASELINE_DIR).join(BASELINE_MANIFEST).exists() {
196            eprintln!(
197                "warn: {}/{}{run_label} has no pre-dispatch baseline — diff-scope unavailable; rebuild the iteration to capture metrics",
198                task.eval_id, task.condition
199            );
200            summary.missing_baseline += 1;
201            continue;
202        }
203        if run_dir.join("command-checks").exists() {
204            return Err(PipelineError::Message(format!(
205                "cannot capture diff scope for {}/{}{run_label} after command checks have run; rebuild the iteration",
206                task.eval_id, task.condition
207            )));
208        }
209
210        let metrics = measure_task_diff(Path::new(eval_root), run_dir)?;
211        crate::validation::validate_against_schema::<DiffScopeMetrics>(
212            crate::validation::SchemaName::DiffScope,
213            &serde_json::to_value(metrics)?,
214            &result_path.to_string_lossy(),
215        )?;
216        write_json(&result_path, &metrics)?;
217        summary.measured += 1;
218    }
219    Ok(summary)
220}
221
222fn capture_task_baseline(eval_root: &Path, run_dir: &Path) -> Result<(), PipelineError> {
223    let baseline_dir = run_dir.join(BASELINE_DIR);
224    if baseline_dir.exists() {
225        fs::remove_dir_all(&baseline_dir)?;
226    }
227    let file_snapshot = baseline_dir.join(BASELINE_FILES);
228    fs::create_dir_all(&file_snapshot)?;
229
230    let framework_outputs = eval_root.join(".eval-magic-outputs");
231    let mut preexisting_paths = walk_files(eval_root, Some(&framework_outputs))?;
232    preexisting_paths.sort();
233    let preexisting_files = preexisting_paths
234        .iter()
235        .map(|path| relative_key(eval_root, path))
236        .collect::<Result<Vec<_>, _>>()?;
237    write_json(
238        &baseline_dir.join(BASELINE_MANIFEST),
239        &BaselineManifest { preexisting_files },
240    )?;
241
242    for source in preexisting_paths {
243        let relative = source.strip_prefix(eval_root).map_err(|_| {
244            PipelineError::Message(format!(
245                "diff-scope baseline path {} is outside {}",
246                source.display(),
247                eval_root.display()
248            ))
249        })?;
250        copy_entry(&source, &file_snapshot.join(relative))?;
251    }
252    Ok(())
253}
254
255fn measure_task_diff(eval_root: &Path, run_dir: &Path) -> Result<DiffScopeMetrics, PipelineError> {
256    let baseline_dir = run_dir.join(BASELINE_DIR);
257    let manifest: BaselineManifest =
258        serde_json::from_str(&fs::read_to_string(baseline_dir.join(BASELINE_MANIFEST))?)?;
259    let file_snapshot = baseline_dir.join(BASELINE_FILES);
260    let mut candidates = BTreeSet::new();
261
262    for relative in manifest.preexisting_files {
263        if fs::symlink_metadata(file_snapshot.join(&relative)).is_err() {
264            return Err(PipelineError::Message(format!(
265                "diff-scope baseline is incomplete: missing snapshot for {relative}"
266            )));
267        }
268        candidates.insert(relative);
269    }
270    let framework_outputs = eval_root.join(".eval-magic-outputs");
271    for path in walk_files(eval_root, Some(&framework_outputs))? {
272        candidates.insert(relative_key(eval_root, &path)?);
273    }
274
275    let mut metrics = DiffScopeMetrics::default();
276    for relative in candidates {
277        let before = file_snapshot.join(&relative);
278        let after = eval_root.join(&relative);
279        let old = file_content(&before)?;
280        let new = file_content(&after)?;
281        if old == new {
282            continue;
283        }
284        metrics.files_touched += 1;
285        let diff = match (old, new) {
286            (FileContent::Regular(old), FileContent::Regular(new)) => Some(diff_bytes(&old, &new)),
287            (FileContent::Missing, FileContent::Regular(new)) => Some(diff_bytes(&[], &new)),
288            (FileContent::Regular(old), FileContent::Missing) => Some(diff_bytes(&old, &[])),
289            _ => None,
290        };
291        if let Some(diff) = diff {
292            metrics.lines_added += diff.lines_added;
293            metrics.lines_removed += diff.lines_removed;
294            metrics.hunks += diff.hunks;
295        }
296    }
297    Ok(metrics)
298}
299
300#[derive(Debug, PartialEq, Eq)]
301enum FileContent {
302    Missing,
303    Regular(Vec<u8>),
304    Symlink(PathBuf),
305}
306
307fn file_content(path: &Path) -> Result<FileContent, PipelineError> {
308    let metadata = match fs::symlink_metadata(path) {
309        Ok(metadata) => metadata,
310        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
311            return Ok(FileContent::Missing);
312        }
313        Err(error) => return Err(error.into()),
314    };
315    if metadata.file_type().is_symlink() {
316        return Ok(FileContent::Symlink(fs::read_link(path)?));
317    }
318    if metadata.is_file() {
319        return Ok(FileContent::Regular(fs::read(path)?));
320    }
321    Ok(FileContent::Missing)
322}
323
324fn walk_files(root: &Path, excluded_root: Option<&Path>) -> Result<Vec<PathBuf>, PipelineError> {
325    if !root.exists() {
326        return Ok(Vec::new());
327    }
328    WalkDir::new(root)
329        .follow_links(false)
330        .into_iter()
331        .filter_entry(|entry| !is_excluded(entry, excluded_root))
332        .filter_map(|entry| match entry {
333            Ok(entry) if entry.file_type().is_file() || entry.file_type().is_symlink() => {
334                Some(Ok(entry.into_path()))
335            }
336            Ok(_) => None,
337            Err(error) => Some(Err(PipelineError::Message(format!(
338                "could not walk diff-scope path under {}: {error}",
339                root.display()
340            )))),
341        })
342        .collect()
343}
344
345fn is_excluded(entry: &DirEntry, excluded_root: Option<&Path>) -> bool {
346    excluded_root.is_some_and(|excluded| entry.path().starts_with(excluded))
347}
348
349fn relative_key(root: &Path, path: &Path) -> Result<String, PipelineError> {
350    let relative = path.strip_prefix(root).map_err(|_| {
351        PipelineError::Message(format!(
352            "diff-scope path {} is outside {}",
353            path.display(),
354            root.display()
355        ))
356    })?;
357    Ok(relative
358        .components()
359        .map(|component| component.as_os_str().to_string_lossy())
360        .collect::<Vec<_>>()
361        .join("/"))
362}
363
364fn copy_entry(source: &Path, destination: &Path) -> Result<(), PipelineError> {
365    let metadata = fs::symlink_metadata(source)?;
366    if metadata.file_type().is_symlink() {
367        if let Some(parent) = destination.parent() {
368            fs::create_dir_all(parent)?;
369        }
370        let target = fs::read_link(source)?;
371        #[cfg(unix)]
372        std::os::unix::fs::symlink(target, destination)?;
373        #[cfg(windows)]
374        if source.metadata().is_ok_and(|metadata| metadata.is_dir()) {
375            std::os::windows::fs::symlink_dir(target, destination)?;
376        } else {
377            std::os::windows::fs::symlink_file(target, destination)?;
378        }
379    } else if metadata.is_dir() {
380        fs::create_dir_all(destination)?;
381        for entry in fs::read_dir(source)? {
382            let entry = entry?;
383            copy_entry(&entry.path(), &destination.join(entry.file_name()))?;
384        }
385    } else {
386        if let Some(parent) = destination.parent() {
387            fs::create_dir_all(parent)?;
388        }
389        fs::copy(source, destination)?;
390    }
391    Ok(())
392}
393
394#[cfg(test)]
395mod tests {
396    use super::*;
397    use std::fs;
398
399    #[test]
400    fn byte_line_diff_counts_changes_and_zero_context_hunks() {
401        let diff = diff_bytes(b"old\nsame\nbefore\n", b"new\nsame\nafter\n");
402        assert_eq!(diff.lines_added, 2);
403        assert_eq!(diff.lines_removed, 2);
404        assert_eq!(diff.hunks, 2);
405    }
406
407    #[test]
408    fn byte_line_diff_handles_empty_trailing_newline_and_non_utf8_inputs() {
409        assert_eq!(diff_bytes(b"", b""), FileDiff::default());
410
411        let trailing = diff_bytes(b"value", b"value\n");
412        assert_eq!(trailing.lines_added, 1);
413        assert_eq!(trailing.lines_removed, 1);
414        assert_eq!(trailing.hunks, 1);
415
416        let binary = diff_bytes(&[0xff, b'\n'], &[0xfe, b'\n']);
417        assert_eq!(binary.lines_added, 1);
418        assert_eq!(binary.lines_removed, 1);
419        assert_eq!(binary.hunks, 1);
420    }
421
422    #[test]
423    fn lines_changed_saturates_untrusted_artifact_totals() {
424        let metrics = DiffScopeMetrics {
425            lines_added: u64::MAX,
426            lines_removed: 1,
427            ..DiffScopeMetrics::default()
428        };
429        assert_eq!(metrics.lines_changed(), u64::MAX);
430    }
431
432    #[test]
433    fn baseline_measurement_counts_all_task_changes_except_framework_outputs() {
434        let temp = tempfile::TempDir::new().unwrap();
435        let eval_root = temp.path().join("env");
436        let run_dir = temp.path().join("run");
437        let outputs_dir = eval_root.join(".eval-magic-outputs/eval-e1/with_skill");
438        fs::create_dir_all(eval_root.join("src")).unwrap();
439        fs::create_dir_all(&outputs_dir).unwrap();
440        fs::write(eval_root.join("src/changed.txt"), "old\nsame\n").unwrap();
441        fs::write(eval_root.join("src/deleted.txt"), "gone\n").unwrap();
442        fs::write(eval_root.join("framework.txt"), "before\n").unwrap();
443
444        capture_task_baseline(&eval_root, &run_dir).unwrap();
445
446        fs::write(eval_root.join("src/changed.txt"), "new\nsame\n").unwrap();
447        fs::remove_file(eval_root.join("src/deleted.txt")).unwrap();
448        fs::write(eval_root.join("framework.txt"), "after\n").unwrap();
449        fs::write(eval_root.join("notes.txt"), "one\ntwo\n").unwrap();
450        fs::write(outputs_dir.join("final-message.md"), "ignored\n").unwrap();
451        fs::write(
452            eval_root.join(".eval-magic-outputs/agent-created.txt"),
453            "also ignored\n",
454        )
455        .unwrap();
456
457        let metrics = measure_task_diff(&eval_root, &run_dir).unwrap();
458        assert_eq!(
459            metrics,
460            DiffScopeMetrics {
461                files_touched: 4,
462                lines_added: 4,
463                lines_removed: 3,
464                hunks: 4,
465            }
466        );
467    }
468}
eval_magic/pipeline/diff_scope.rs

eval_magic/pipeline/
diff_scope.rs
