harness_lock_core/
tasks.rs

1//! Task packs.
2
3use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5
6use serde::{Deserialize, Serialize};
7
8use crate::error::{Error, Result};
9
10#[derive(Debug, Clone, Serialize, Deserialize, Default)]
11#[serde(default)]
12pub struct Task {
13    pub id: String,
14    pub prompt: String,
15    pub fixtures: Option<PathBuf>,
16    #[serde(default)]
17    pub tools: Vec<String>,
18    #[serde(default)]
19    pub expected: BTreeMap<String, serde_json::Value>,
20    #[serde(default)]
21    pub graders: Vec<String>,
22    #[serde(default)]
23    pub tags: Vec<String>,
24    #[serde(rename = "timeout-seconds", default = "default_timeout")]
25    pub timeout_seconds: u64,
26    #[serde(skip)]
27    pub prompt_text: String,
28    #[serde(skip)]
29    pub pack_dir: Option<PathBuf>,
30}
31
32fn default_timeout() -> u64 {
33    600
34}
35
36impl Task {
37    pub fn expected_artifacts(&self) -> Vec<String> {
38        self.expected
39            .get("artifacts")
40            .and_then(serde_json::Value::as_array)
41            .map(|a| {
42                a.iter()
43                    .filter_map(|v| v.as_str().map(|s| s.to_string()))
44                    .collect()
45            })
46            .unwrap_or_default()
47    }
48
49    pub fn load_prompt_text(&mut self) {
50        if !self.prompt_text.is_empty() {
51            return;
52        }
53        if let (Some(dir), false) = (&self.pack_dir, self.prompt.is_empty()) {
54            let p = dir.join(&self.prompt);
55            if let Ok(t) = std::fs::read_to_string(&p) {
56                self.prompt_text = t;
57            }
58        }
59    }
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize, Default)]
63#[serde(default)]
64pub struct PackHeader {
65    pub name: String,
66    pub version: String,
67    pub domain: String,
68    pub license: String,
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, Default)]
72#[serde(default)]
73pub struct PackFile {
74    pub pack: PackHeader,
75    pub task: Vec<Task>,
76}
77
78#[derive(Debug, Clone)]
79pub struct TaskPack {
80    pub name: String,
81    pub version: String,
82    pub domain: String,
83    pub path: PathBuf,
84    pub hash: String,
85    pub tasks: Vec<Task>,
86}
87
88#[derive(Debug, Clone)]
89pub struct Grade {
90    pub grader: String,
91    pub success: bool,
92    pub score: f64,
93}
94
95impl TaskPack {
96    pub fn load(source: &str, base: Option<&Path>) -> Result<TaskPack> {
97        let p = PathBuf::from(source);
98        let p = if p.is_absolute() {
99            p
100        } else if let Some(b) = base {
101            b.join(&p)
102        } else {
103            p
104        };
105        let pack_path = p.join("pack.toml");
106        let text = std::fs::read_to_string(&pack_path)
107            .map_err(|e| Error::TaskPack(format!("missing pack.toml at {}: {e}", p.display())))?;
108        let file: PackFile = toml::from_str(&text)
109            .map_err(|e| Error::TaskPack(format!("invalid pack.toml: {e}")))?;
110        let mut tasks = file.task;
111        for t in tasks.iter_mut() {
112            t.pack_dir = Some(p.clone());
113            t.load_prompt_text();
114        }
115        let hash = crate::hash::sha256_hex(text.as_bytes());
116        Ok(TaskPack {
117            name: file.pack.name,
118            version: file.pack.version,
119            domain: file.pack.domain,
120            path: p,
121            hash,
122            tasks,
123        })
124    }
125
126    /// Deterministic search/holdout split over task ids.
127    pub fn split(&self, search_frac: f64, seed: u64) -> (Vec<Task>, Vec<Task>) {
128        let mut tasks: Vec<&Task> = self.tasks.iter().collect();
129        tasks.sort_by(|a, b| a.id.cmp(&b.id));
130        let n_search = ((tasks.len() as f64) * search_frac).round().max(1.0) as usize;
131        let mut idx: Vec<usize> = (0..tasks.len()).collect();
132        // xorshift-ish deterministic shuffle from seed
133        let mut state = seed;
134        for i in (1..idx.len()).rev() {
135            state ^= state << 13;
136            state ^= state >> 7;
137            state ^= state << 17;
138            let j = (state as usize) % (i + 1);
139            idx.swap(i, j);
140        }
141        let search: Vec<Task> = idx[..n_search].iter().map(|&i| tasks[i].clone()).collect();
142        let holdout: Vec<Task> = idx[n_search..].iter().map(|&i| tasks[i].clone()).collect();
143        (search, holdout)
144    }
145
146    /// Grade a task against its workspace. v0.1 uses artifact checks by default.
147    pub fn grade(&self, task: &Task, workspace: &Path) -> Vec<Grade> {
148        let expected = task.expected_artifacts();
149        if task.graders.is_empty() || task.graders.iter().all(|g| g.starts_with("artifact:")) {
150            let missing: Vec<String> = expected
151                .iter()
152                .filter(|a| !workspace.join(a).exists())
153                .cloned()
154                .collect();
155            let success = missing.is_empty();
156            return vec![Grade {
157                grader: format!("artifact:{}", expected.join(",")),
158                success,
159                score: if success { 1.0 } else { 0.0 },
160            }];
161        }
162        // For v0.1 the only supported extra grader type is "tests:" which we
163        // treat as a failed grade unless an artifact check passes too. This
164        // mirrors the reference (deterministic) grader set.
165        let expected_ok = expected.iter().all(|a| workspace.join(a).exists());
166        vec![Grade {
167            grader: "tests:check".into(),
168            success: expected_ok,
169            score: if expected_ok { 1.0 } else { 0.0 },
170        }]
171    }
172}
harness_lock_core/tasks.rs

harness_lock_core/
tasks.rs
