eval_magic/pipeline/grade/
command_check.rs

1//! Runner-owned command-check grading.
2
3use std::collections::{BTreeMap, HashMap};
4use std::fs;
5use std::path::{Component, Path};
6use std::process::{Command, ExitStatus};
7
8use regex::Regex;
9use serde::{Deserialize, Serialize};
10
11use crate::core::{Assertion, AssertionCommandCheck, EvalsConfig};
12use crate::pipeline::error::PipelineError;
13use crate::pipeline::io::write_json;
14use crate::validation::{SchemaName, validate_against_schema};
15
16const DIAGNOSTIC_LIMIT: usize = 2 * 1024;
17
18/// The schema-gated intermediate result persisted before finalize converts it
19/// into a normal [`crate::core::AssertionResult`].
20#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
21pub struct CommandCheckResult {
22    pub id: String,
23    pub passed: bool,
24    pub evidence: String,
25    pub expected_exit_code: i32,
26    pub actual_exit_code: Option<i32>,
27    pub stdout: String,
28    pub stderr: String,
29    #[serde(skip_serializing_if = "Option::is_none")]
30    pub cells: Option<Vec<CommandCheckCellResult>>,
31}
32
33/// The result of one environment-matrix cell.
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
35pub struct CommandCheckCellResult {
36    pub env: BTreeMap<String, String>,
37    pub passed: bool,
38    pub evidence: String,
39    pub actual_exit_code: Option<i32>,
40    pub stdout: String,
41    pub stderr: String,
42}
43
44#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
45pub struct CommandCheckSummary {
46    pub executed: usize,
47    pub reused: usize,
48    pub failed: usize,
49}
50
51#[derive(Debug, Deserialize)]
52struct DispatchFile {
53    #[serde(default)]
54    tasks: Vec<DispatchTask>,
55}
56
57#[derive(Debug, Deserialize)]
58struct DispatchTask {
59    eval_id: String,
60    condition: String,
61    eval_root: Option<String>,
62    run_record_path: String,
63}
64
65/// Inject held-out setup files and execute all command checks in declaration
66/// order for every matching dispatch task.
67pub fn grade_command_checks(
68    iteration_dir: &Path,
69    evals: &EvalsConfig,
70    skill_dir: &Path,
71    overwrite: bool,
72) -> Result<CommandCheckSummary, PipelineError> {
73    let has_command_checks = evals.evals.iter().any(|eval| {
74        eval.assertions
75            .as_deref()
76            .unwrap_or(&[])
77            .iter()
78            .any(|assertion| matches!(assertion, Assertion::CommandCheck(_)))
79    });
80    if !has_command_checks {
81        return Ok(CommandCheckSummary::default());
82    }
83
84    let dispatch_path = iteration_dir.join("dispatch.json");
85    let dispatch: DispatchFile =
86        serde_json::from_str(&fs::read_to_string(&dispatch_path).map_err(|error| {
87            PipelineError::Message(format!(
88                "could not read {} for command_check grading: {error}",
89                dispatch_path.display()
90            ))
91        })?)?;
92    let root_counts: HashMap<&str, usize> = dispatch
93        .tasks
94        .iter()
95        .filter_map(|task| task.eval_root.as_deref())
96        .fold(HashMap::new(), |mut counts, root| {
97            *counts.entry(root).or_default() += 1;
98            counts
99        });
100    let mut summary = CommandCheckSummary::default();
101
102    for task in &dispatch.tasks {
103        let Some(eval) = evals.evals.iter().find(|eval| eval.id == task.eval_id) else {
104            continue;
105        };
106        let checks: Vec<&AssertionCommandCheck> = eval
107            .assertions
108            .as_deref()
109            .unwrap_or(&[])
110            .iter()
111            .filter_map(|assertion| match assertion {
112                Assertion::CommandCheck(check) => Some(check),
113                _ => None,
114            })
115            .collect();
116        if checks.is_empty() {
117            continue;
118        }
119
120        let eval_root = task
121            .eval_root
122            .as_deref()
123            .ok_or_else(|| isolation_error(task, "does not record an eval_root"))?;
124        if root_counts.get(eval_root).copied().unwrap_or_default() != 1 {
125            return Err(isolation_error(
126                task,
127                "shares eval_root with another dispatch task",
128            ));
129        }
130        let eval_root = Path::new(eval_root);
131        let run_dir = Path::new(&task.run_record_path).parent().ok_or_else(|| {
132            PipelineError::Message(format!(
133                "command_check task '{}'/{} has no run directory in run_record_path",
134                task.eval_id, task.condition
135            ))
136        })?;
137        let results_dir = run_dir.join("command-checks");
138
139        for check in checks {
140            validate_assertion_id(&check.id)?;
141            let result_path = results_dir.join(format!("{}.json", check.id));
142            if result_path.exists() && !overwrite {
143                let value = serde_json::from_str(&fs::read_to_string(&result_path)?)?;
144                validate_against_schema::<CommandCheckResult>(
145                    SchemaName::CommandCheck,
146                    &value,
147                    &result_path.to_string_lossy(),
148                )?;
149                summary.reused += 1;
150                continue;
151            }
152
153            inject_setup_files(check, skill_dir, eval_root)?;
154            let result = execute_command_check(check, eval_root)?;
155            if !result.passed {
156                summary.failed += 1;
157            }
158            fs::create_dir_all(&results_dir)?;
159            validate_against_schema::<CommandCheckResult>(
160                SchemaName::CommandCheck,
161                &serde_json::to_value(&result)?,
162                &result_path.to_string_lossy(),
163            )?;
164            write_json(&result_path, &result)?;
165            summary.executed += 1;
166        }
167    }
168
169    Ok(summary)
170}
171
172fn isolation_error(task: &DispatchTask, detail: &str) -> PipelineError {
173    PipelineError::Message(format!(
174        "command_check task '{}'/{} {detail}; command checks require task-scoped environments. Build and dispatch a fresh iteration with this evals.json before grading.",
175        task.eval_id, task.condition
176    ))
177}
178
179fn validate_assertion_id(id: &str) -> Result<(), PipelineError> {
180    let components: Vec<_> = Path::new(id).components().collect();
181    if !matches!(components.as_slice(), [Component::Normal(_)]) {
182        return Err(PipelineError::Message(format!(
183            "command_check assertion id must be one path-safe component: {id}"
184        )));
185    }
186    Ok(())
187}
188
189fn inject_setup_files(
190    check: &AssertionCommandCheck,
191    skill_dir: &Path,
192    eval_root: &Path,
193) -> Result<(), PipelineError> {
194    for relative in check.setup_files.as_deref().unwrap_or(&[]) {
195        validate_setup_relative(relative)?;
196        let source = skill_dir.join("evals").join(relative);
197        if !source.exists() {
198            return Err(PipelineError::Message(format!(
199                "command-check setup file not found during grading: {}",
200                source.display()
201            )));
202        }
203        let destination = eval_root.join(relative);
204        if let Some(parent) = destination.parent() {
205            fs::create_dir_all(parent)?;
206        }
207        copy_entry(&source, &destination)?;
208    }
209    Ok(())
210}
211
212fn validate_setup_relative(relative: &str) -> Result<(), PipelineError> {
213    let unsafe_component = Path::new(relative).components().any(|component| {
214        matches!(
215            component,
216            Component::ParentDir | Component::RootDir | Component::Prefix(_)
217        )
218    });
219    if unsafe_component {
220        return Err(PipelineError::Message(format!(
221            "command_check setup path must be relative and stay within the task environment: {relative}"
222        )));
223    }
224    Ok(())
225}
226
227fn copy_entry(source: &Path, destination: &Path) -> Result<(), PipelineError> {
228    if fs::metadata(source)?.is_dir() {
229        fs::create_dir_all(destination)?;
230        for entry in fs::read_dir(source)? {
231            let entry = entry?;
232            copy_entry(&entry.path(), &destination.join(entry.file_name()))?;
233        }
234    } else {
235        fs::copy(source, destination)?;
236    }
237    Ok(())
238}
239
240/// Execute one trusted command-check assertion through the platform shell.
241pub(super) fn execute_command_check(
242    assertion: &AssertionCommandCheck,
243    eval_root: &Path,
244) -> Result<CommandCheckResult, PipelineError> {
245    validate_command_environment(assertion)?;
246
247    let Some(matrix) = &assertion.matrix else {
248        let env = assertion.env.clone().unwrap_or_default();
249        let cell = execute_command_check_cell(assertion, eval_root, env)?;
250        return Ok(CommandCheckResult {
251            id: assertion.id.clone(),
252            passed: cell.passed,
253            evidence: cell.evidence,
254            expected_exit_code: assertion.expect_exit_code,
255            actual_exit_code: cell.actual_exit_code,
256            stdout: cell.stdout,
257            stderr: cell.stderr,
258            cells: None,
259        });
260    };
261
262    let cells = matrix_environments(assertion)
263        .into_iter()
264        .map(|env| execute_command_check_cell(assertion, eval_root, env))
265        .collect::<Result<Vec<_>, _>>()?;
266    let passed_count = cells.iter().filter(|cell| cell.passed).count();
267    let passed = passed_count == cells.len();
268    let mut evidence = format!("{passed_count}/{} matrix cells passed", cells.len());
269    if !passed {
270        let failed_cells = cells
271            .iter()
272            .filter(|cell| !cell.passed)
273            .map(|cell| {
274                let label = matrix
275                    .keys()
276                    .map(|name| format!("{name}={}", cell.env[name]))
277                    .collect::<Vec<_>>()
278                    .join(",");
279                format!("{label} ({})", cell.evidence)
280            })
281            .collect::<Vec<_>>()
282            .join("; ");
283        evidence.push_str("; failed cells: ");
284        evidence.push_str(&failed_cells);
285    }
286
287    Ok(CommandCheckResult {
288        id: assertion.id.clone(),
289        passed,
290        evidence,
291        expected_exit_code: assertion.expect_exit_code,
292        actual_exit_code: None,
293        stdout: String::new(),
294        stderr: String::new(),
295        cells: Some(cells),
296    })
297}
298
299fn validate_command_environment(assertion: &AssertionCommandCheck) -> Result<(), PipelineError> {
300    for (name, value) in assertion.env.as_ref().into_iter().flatten() {
301        validate_command_environment_name(&assertion.id, "env", name)?;
302        validate_command_environment_value(&assertion.id, "env", name, value)?;
303    }
304    for (name, values) in assertion.matrix.as_ref().into_iter().flatten() {
305        validate_command_environment_name(&assertion.id, "matrix", name)?;
306        for value in values {
307            validate_command_environment_value(&assertion.id, "matrix", name, value)?;
308        }
309    }
310    Ok(())
311}
312
313fn validate_command_environment_name(
314    assertion_id: &str,
315    field: &str,
316    name: &str,
317) -> Result<(), PipelineError> {
318    if name.is_empty() || name.contains('=') || name.contains('\0') {
319        return Err(PipelineError::Message(format!(
320            "command_check '{assertion_id}': {field} environment variable name must be non-empty and contain neither '=' nor NUL: {name:?}"
321        )));
322    }
323    Ok(())
324}
325
326fn validate_command_environment_value(
327    assertion_id: &str,
328    field: &str,
329    name: &str,
330    value: &str,
331) -> Result<(), PipelineError> {
332    if value.contains('\0') {
333        return Err(PipelineError::Message(format!(
334            "command_check '{assertion_id}': {field} environment variable {name:?} value must not contain NUL"
335        )));
336    }
337    Ok(())
338}
339
340fn matrix_environments(assertion: &AssertionCommandCheck) -> Vec<BTreeMap<String, String>> {
341    let mut cells = vec![assertion.env.clone().unwrap_or_default()];
342    for (name, values) in assertion.matrix.as_ref().into_iter().flatten() {
343        let mut expanded = Vec::new();
344        for cell in cells {
345            for value in values {
346                let mut env = cell.clone();
347                env.insert(name.clone(), value.clone());
348                expanded.push(env);
349            }
350        }
351        cells = expanded;
352    }
353    cells
354}
355
356fn execute_command_check_cell(
357    assertion: &AssertionCommandCheck,
358    eval_root: &Path,
359    env: BTreeMap<String, String>,
360) -> Result<CommandCheckCellResult, PipelineError> {
361    #[cfg(unix)]
362    let mut command = {
363        let mut command = Command::new("sh");
364        command.arg("-c").arg(&assertion.command);
365        command
366    };
367    #[cfg(windows)]
368    let mut command = {
369        let mut command = Command::new("cmd");
370        command.arg("/C").arg(&assertion.command);
371        command
372    };
373
374    command.current_dir(eval_root);
375    command.envs(&env);
376    let output = command.output();
377
378    let output = output.map_err(|error| {
379        PipelineError::Message(format!(
380            "could not launch the platform shell for command_check '{}': {error}",
381            assertion.id
382        ))
383    })?;
384    let complete_stdout = String::from_utf8_lossy(&output.stdout);
385    let complete_stderr = String::from_utf8_lossy(&output.stderr);
386    let actual_exit_code = output.status.code();
387    let mut failures = Vec::new();
388
389    match actual_exit_code {
390        Some(actual) if actual != assertion.expect_exit_code => failures.push(format!(
391            "expected exit code {}, got {actual}",
392            assertion.expect_exit_code
393        )),
394        Some(_) => {}
395        None => failures.push(termination_evidence(&output.status)),
396    }
397
398    if let Some(pattern) = &assertion.expect_stdout {
399        match Regex::new(pattern) {
400            Ok(regex) if !regex.is_match(&complete_stdout) => {
401                failures.push(format!(
402                    "stdout did not match expect_stdout regex {pattern:?}"
403                ));
404            }
405            Ok(_) => {}
406            Err(error) => {
407                failures.push(format!("invalid expect_stdout regex {pattern:?}: {error}"))
408            }
409        }
410    }
411
412    let passed = failures.is_empty();
413    let evidence = if passed {
414        match &assertion.expect_stdout {
415            Some(pattern) => format!(
416                "exit code matched {}; stdout matched regex {pattern:?}",
417                assertion.expect_exit_code
418            ),
419            None => format!("exit code matched {}", assertion.expect_exit_code),
420        }
421    } else {
422        failures.join("; ")
423    };
424
425    Ok(CommandCheckCellResult {
426        env,
427        passed,
428        evidence,
429        actual_exit_code,
430        stdout: truncate_diagnostic(&complete_stdout),
431        stderr: truncate_diagnostic(&complete_stderr),
432    })
433}
434
435#[cfg(unix)]
436fn termination_evidence(status: &ExitStatus) -> String {
437    use std::os::unix::process::ExitStatusExt;
438    match status.signal() {
439        Some(signal) => format!("command terminated by signal {signal}"),
440        None => "command terminated without an exit code".to_string(),
441    }
442}
443
444#[cfg(windows)]
445fn termination_evidence(_status: &ExitStatus) -> String {
446    "command terminated without an exit code".to_string()
447}
448
449fn truncate_diagnostic(value: &str) -> String {
450    if value.len() <= DIAGNOSTIC_LIMIT {
451        return value.to_string();
452    }
453    let mut end = DIAGNOSTIC_LIMIT;
454    while !value.is_char_boundary(end) {
455        end -= 1;
456    }
457    value[..end].to_string()
458}
459
460#[cfg(test)]
461mod tests;
eval_magic/pipeline/grade/command_check.rs

eval_magic/pipeline/grade/
command_check.rs
