harness_lock_core/
tasks.rs

1//! Eval packs: the tasks a stack is measured against.
2
3use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5
6use serde::{Deserialize, Serialize};
7
8use crate::error::{Error, Result};
9
10#[derive(Debug, Clone, Serialize, Deserialize, Default)]
11#[serde(default)]
12pub struct Task {
13    pub id: String,
14    pub prompt: String,
15    pub fixtures: Option<PathBuf>,
16    #[serde(default)]
17    pub tools: Vec<String>,
18    #[serde(default)]
19    pub expected: BTreeMap<String, serde_json::Value>,
20    #[serde(default)]
21    pub graders: Vec<String>,
22    #[serde(default)]
23    pub tags: Vec<String>,
24    #[serde(rename = "timeout-seconds", default = "default_timeout")]
25    pub timeout_seconds: u64,
26    #[serde(skip)]
27    pub prompt_text: String,
28    #[serde(skip)]
29    pub pack_dir: Option<PathBuf>,
30}
31
32fn default_timeout() -> u64 {
33    600
34}
35
36impl Task {
37    pub fn expected_artifacts(&self) -> Vec<String> {
38        self.expected
39            .get("artifacts")
40            .and_then(serde_json::Value::as_array)
41            .map(|a| {
42                a.iter()
43                    .filter_map(|v| v.as_str().map(|s| s.to_string()))
44                    .collect()
45            })
46            .unwrap_or_default()
47    }
48
49    pub fn load_prompt_text(&mut self) {
50        if !self.prompt_text.is_empty() {
51            return;
52        }
53        if let (Some(dir), false) = (&self.pack_dir, self.prompt.is_empty()) {
54            let p = dir.join(&self.prompt);
55            if let Ok(t) = std::fs::read_to_string(&p) {
56                self.prompt_text = t;
57            }
58        }
59    }
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize, Default)]
63#[serde(default)]
64pub struct PackHeader {
65    pub name: String,
66    pub version: String,
67    pub domain: String,
68    pub license: String,
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, Default)]
72#[serde(default)]
73pub struct PackFile {
74    pub pack: PackHeader,
75    pub task: Vec<Task>,
76}
77
78#[derive(Debug, Clone)]
79pub struct EvalPack {
80    pub name: String,
81    pub version: String,
82    pub domain: String,
83    pub path: PathBuf,
84    pub hash: String,
85    pub tasks: Vec<Task>,
86}
87
88/// Marks a spot in a scaffolded eval pack that still needs filling in.
89pub const PLACEHOLDER: &str = "TODO(hlock)";
90
91/// Below this many tasks the holdout split is too small to say much.
92pub const MIN_TASKS: usize = 10;
93
94/// A problem [`EvalPack::check`] found. `level` is `error` (the pack isn't usable
95/// yet) or `warn` (it runs, but the evidence it produces is weak).
96#[derive(Debug, Clone, Serialize)]
97pub struct PackIssue {
98    pub level: String,
99    pub task: Option<String>,
100    pub message: String,
101}
102
103impl PackIssue {
104    fn error(task: Option<&str>, message: String) -> PackIssue {
105        PackIssue { level: "error".into(), task: task.map(str::to_string), message }
106    }
107
108    fn warn(task: Option<&str>, message: String) -> PackIssue {
109        PackIssue { level: "warn".into(), task: task.map(str::to_string), message }
110    }
111}
112
113#[deprecated(note = "renamed to `EvalPack`")]
114pub type TaskPack = EvalPack;
115
116#[derive(Debug, Clone)]
117pub struct Grade {
118    pub grader: String,
119    pub success: bool,
120    pub score: f64,
121}
122
123impl EvalPack {
124    pub fn load(source: &str, base: Option<&Path>) -> Result<EvalPack> {
125        let p = PathBuf::from(source);
126        let p = if p.is_absolute() {
127            p
128        } else if let Some(b) = base {
129            b.join(&p)
130        } else {
131            p
132        };
133        let pack_path = p.join("pack.toml");
134        let text = std::fs::read_to_string(&pack_path)
135            .map_err(|e| Error::EvalPack(format!("missing pack.toml at {}: {e}", p.display())))?;
136        let file: PackFile = toml::from_str(&text)
137            .map_err(|e| Error::EvalPack(format!("invalid pack.toml: {e}")))?;
138        let mut tasks = file.task;
139        for t in tasks.iter_mut() {
140            t.pack_dir = Some(p.clone());
141            t.load_prompt_text();
142        }
143        let hash = crate::hash::sha256_hex(text.as_bytes());
144        Ok(EvalPack {
145            name: file.pack.name,
146            version: file.pack.version,
147            domain: file.pack.domain,
148            path: p,
149            hash,
150            tasks,
151        })
152    }
153
154    /// Check that the pack is ready to measure stacks with: required fields set,
155    /// prompts and fixtures present, no unfilled [`PLACEHOLDER`]s, and tasks that
156    /// can fail.
157    pub fn check(&self) -> Vec<PackIssue> {
158        let mut issues = Vec::new();
159        for (key, value) in [("name", &self.name), ("version", &self.version), ("domain", &self.domain)] {
160            if value.trim().is_empty() {
161                issues.push(PackIssue::error(None, format!("[pack] {key} is missing")));
162            }
163        }
164        if let Ok(text) = std::fs::read_to_string(self.path.join("pack.toml")) {
165            let lines: Vec<String> = text.lines().enumerate()
166                .filter(|(_, l)| l.contains(PLACEHOLDER))
167                .map(|(i, _)| (i + 1).to_string())
168                .collect();
169            if !lines.is_empty() {
170                let shown = if lines.len() > 5 { format!("{}, …", lines[..5].join(", ")) } else { lines.join(", ") };
171                let word = if lines.len() == 1 { "line" } else { "lines" };
172                issues.push(PackIssue::error(None, format!(
173                    "pack.toml still has {} {PLACEHOLDER} ({word} {shown})", lines.len()
174                )));
175            }
176        }
177        if self.tasks.is_empty() {
178            issues.push(PackIssue::error(None, "the pack has no [[task]] entries".into()));
179            return issues;
180        }
181        let mut seen = std::collections::BTreeSet::new();
182        for t in &self.tasks {
183            let id = Some(t.id.as_str()).filter(|s| !s.is_empty());
184            if t.id.is_empty() {
185                issues.push(PackIssue::error(None, format!("a task has no id (prompt {:?})", t.prompt)));
186            } else if !seen.insert(t.id.clone()) {
187                issues.push(PackIssue::error(id, format!("duplicate task id {:?}", t.id)));
188            }
189            if t.prompt.is_empty() {
190                issues.push(PackIssue::error(id, "no prompt file".into()));
191            } else if !self.path.join(&t.prompt).is_file() {
192                issues.push(PackIssue::error(id, format!("prompt file {} not found", t.prompt)));
193            } else if t.prompt_text.trim().is_empty() {
194                issues.push(PackIssue::error(id, format!("prompt file {} is empty", t.prompt)));
195            } else {
196                let n = t.prompt_text.matches(PLACEHOLDER).count();
197                if n > 0 {
198                    issues.push(PackIssue::error(id, format!("{} still has {n} {PLACEHOLDER}", t.prompt)));
199                }
200            }
201            let fixtures = t.fixtures.as_ref().map(|f| self.path.join(f));
202            if let Some(dir) = &fixtures {
203                if !dir.is_dir() {
204                    issues.push(PackIssue::error(id, format!("fixtures directory {} not found", t.fixtures.as_ref().unwrap().display())));
205                }
206            }
207            let artifacts = t.expected_artifacts();
208            if artifacts.is_empty() && t.graders.is_empty() {
209                issues.push(PackIssue::warn(id, "no expected artifacts or graders, so it always passes".into()));
210            }
211            if let Some(dir) = fixtures.as_ref().filter(|d| d.is_dir()) {
212                for a in artifacts.iter().filter(|a| dir.join(a).exists()) {
213                    issues.push(PackIssue::warn(id, format!("expected artifact {a} is already in the fixtures, so it passes without any work")));
214                }
215            }
216            for g in t.graders.iter().filter(|g| !g.starts_with("artifact:") && !g.starts_with("tests:")) {
217                issues.push(PackIssue::warn(id, format!("grader {g:?} isn't supported yet; the task is graded on its artifacts")));
218            }
219        }
220        if self.tasks.len() < MIN_TASKS {
221            let (_, holdout) = self.split(0.7, 0);
222            issues.push(PackIssue::warn(None, format!(
223                "{} tasks leaves {} for the holdout split; add tasks until there are at least {MIN_TASKS}",
224                self.tasks.len(), holdout.len()
225            )));
226        }
227        issues
228    }
229
230    /// Deterministic search/holdout split over task ids.
231    pub fn split(&self, search_frac: f64, seed: u64) -> (Vec<Task>, Vec<Task>) {
232        let mut tasks: Vec<&Task> = self.tasks.iter().collect();
233        tasks.sort_by(|a, b| a.id.cmp(&b.id));
234        let n_search = ((tasks.len() as f64) * search_frac).round().max(1.0) as usize;
235        let mut idx: Vec<usize> = (0..tasks.len()).collect();
236        // xorshift-ish deterministic shuffle from seed
237        let mut state = seed;
238        for i in (1..idx.len()).rev() {
239            state ^= state << 13;
240            state ^= state >> 7;
241            state ^= state << 17;
242            let j = (state as usize) % (i + 1);
243            idx.swap(i, j);
244        }
245        let search: Vec<Task> = idx[..n_search].iter().map(|&i| tasks[i].clone()).collect();
246        let holdout: Vec<Task> = idx[n_search..].iter().map(|&i| tasks[i].clone()).collect();
247        (search, holdout)
248    }
249
250    /// Grade a task against its workspace. v0.1 uses artifact checks by default.
251    pub fn grade(&self, task: &Task, workspace: &Path) -> Vec<Grade> {
252        let expected = task.expected_artifacts();
253        if task.graders.is_empty() || task.graders.iter().all(|g| g.starts_with("artifact:")) {
254            let missing: Vec<String> = expected
255                .iter()
256                .filter(|a| !workspace.join(a).exists())
257                .cloned()
258                .collect();
259            let success = missing.is_empty();
260            return vec![Grade {
261                grader: format!("artifact:{}", expected.join(",")),
262                success,
263                score: if success { 1.0 } else { 0.0 },
264            }];
265        }
266        // For v0.1 the only supported extra grader type is "tests:" which we
267        // treat as a failed grade unless an artifact check passes too. This
268        // mirrors the reference (deterministic) grader set.
269        let expected_ok = expected.iter().all(|a| workspace.join(a).exists());
270        vec![Grade {
271            grader: "tests:check".into(),
272            success: expected_ok,
273            score: if expected_ok { 1.0 } else { 0.0 },
274        }]
275    }
276}
harness_lock_core/tasks.rs

harness_lock_core/
tasks.rs
