harness_lock_core/
contract.rs

1//! The execution contract for code harnesses (SPEC §7.8.3): how hlock runs a
2//! framework agent or a custom script declared under `[harnesses.<id>]`.
3//!
4//! hlock passes everything through environment variables (`HLOCK_PROMPT`,
5//! `HLOCK_MODEL`, `HLOCK_LLM_BASE_URL`, `HLOCK_WORKSPACE`, `HLOCK_MCP_CONFIG`,
6//! `HLOCK_SKILLS_DIR`, `HLOCK_LOCK`, `HLOCK_OUTPUT`, `HLOCK_SEED`) and reads an
7//! optional JSON result from `HLOCK_OUTPUT`; without one, exit code 0 means
8//! success. A Python `entrypoint` runs through a small shim (`entry_shim.py`,
9//! passed with `python -c`), so both kinds share one path and the agent's
10//! interpreter needs nothing from hlock installed. [`CommandAdapter`] plugs this into the resolver,
11//! so `resolve` evaluates code harnesses for real.
12
13use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::time::{Duration, Instant};
17
18use serde_json::{json, Map, Value};
19
20use crate::adapters::{HarnessAdapter, RunOutcome};
21use crate::error::{Error, Result};
22use crate::lockfile::{Lockfile, Tool};
23use crate::profiles::{python_for, Ctx, HarnessProfile};
24use crate::search::Stack;
25use crate::tasks::Task;
26
27/// The Python shim that calls `module:callable(prompt, ctx)` and writes the result.
28pub const ENTRY_SHIM: &str = include_str!("entry_shim.py");
29
30/// A ready-to-run code-harness invocation.
31#[derive(Debug, Clone)]
32pub struct Invocation {
33    pub argv: Vec<String>,
34    pub env: BTreeMap<String, String>,
35    pub cwd: PathBuf,
36    /// Where the harness may write its JSON result.
37    pub output: PathBuf,
38}
39
40/// The result of one run.
41#[derive(Debug, Clone, Default, serde::Serialize)]
42pub struct RunResult {
43    pub success: bool,
44    pub exit_code: Option<i32>,
45    pub output: Option<Value>,
46    pub cost_usd: f64,
47    pub steps: u64,
48    pub latency_s: f64,
49    pub trace: Value,
50    pub log: String,
51}
52
53fn subst(s: &str, vars: &[(&str, &str)]) -> String {
54    vars.iter().fold(s.to_string(), |acc, (k, v)| acc.replace(&format!("{{{k}}}"), v))
55}
56
57/// Replace `${VAR}` placeholders in a string with environment values, when set.
58fn fill_env(s: &str, ctx: &Ctx) -> String {
59    let mut out = s.to_string();
60    let mut start = 0;
61    while let Some(i) = out[start..].find("${") {
62        let i = start + i;
63        let Some(j) = out[i..].find('}') else { break };
64        let name = out[i + 2..i + j].to_string();
65        match ctx.var(&name) {
66            Some(v) => {
67                let v = v.to_string();
68                out.replace_range(i..i + j + 1, &v);
69                start = i + v.len();
70            }
71            None => start = i + j + 1,
72        }
73    }
74    out
75}
76
77/// A standard `mcpServers` document for locked MCP servers (secrets filled from the environment).
78pub fn mcp_config(tools: &[Tool], ctx: &Ctx) -> Value {
79    let mut servers = Map::new();
80    for t in tools.iter().filter(|t| t.kind == "mcp" && t.enabled != Some(false)) {
81        let fill_map = |m: &BTreeMap<String, String>| -> Value {
82            Value::Object(m.iter().map(|(k, v)| (k.clone(), json!(fill_env(v, ctx)))).collect())
83        };
84        let entry = match (t.transport.as_deref(), &t.command, &t.url) {
85            (Some("http") | Some("sse"), _, Some(url)) => {
86                let mut e = json!({"type": t.transport, "url": fill_env(url, ctx)});
87                if !t.headers.is_empty() {
88                    e["headers"] = fill_map(&t.headers);
89                }
90                e
91            }
92            (_, Some(cmd), _) => {
93                let mut e = json!({"command": cmd, "args": t.args.iter().map(|a| fill_env(a, ctx)).collect::<Vec<_>>()});
94                if !t.env.is_empty() {
95                    e["env"] = fill_map(&t.env);
96                }
97                e
98            }
99            _ => continue,
100        };
101        servers.insert(t.name.clone(), entry);
102    }
103    json!({ "mcpServers": servers })
104}
105
106/// Build the invocation for a code harness. `model` is `provider:model`; when
107/// `None` it comes from the lock's first LLM role.
108pub fn invocation(
109    profile: &HarnessProfile,
110    lock: Option<&Lockfile>,
111    lock_path: Option<&Path>,
112    model: Option<&str>,
113    prompt: &str,
114    workspace: &Path,
115    seed: u64,
116    ctx: &Ctx,
117) -> Result<Invocation> {
118    // the harness runs inside the workspace, so every path it gets must be absolute
119    let workspace = &if workspace.is_absolute() { workspace.to_path_buf() } else { ctx.cwd.join(workspace) };
120    let hl = workspace.join(".hlock");
121    std::fs::create_dir_all(&hl)?;
122    let output = hl.join("result.json");
123    let _ = std::fs::remove_file(&output);
124
125    let llm = lock.and_then(|l| l.llm.iter().find(|x| x.role == "default").or_else(|| l.llm.first()));
126    let model = model.map(str::to_string).or_else(|| llm.map(|l| format!("{}:{}", l.provider, l.model))).unwrap_or_default();
127    let base_url = llm.and_then(|l| l.endpoint.clone()).or_else(|| ctx.var("HL_LLM_BASE_URL").map(str::to_string));
128
129    let mut env: BTreeMap<String, String> = profile.run.env.clone();
130    env.insert("HLOCK_PROMPT".into(), prompt.into());
131    env.insert("HLOCK_MODEL".into(), model.clone());
132    env.insert("HLOCK_WORKSPACE".into(), workspace.to_string_lossy().into_owned());
133    env.insert("HLOCK_OUTPUT".into(), output.to_string_lossy().into_owned());
134    env.insert("HLOCK_SEED".into(), seed.to_string());
135    env.insert("HLOCK_HARNESS".into(), profile.id.clone());
136    if let Some(b) = &base_url {
137        env.insert("HLOCK_LLM_BASE_URL".into(), b.clone());
138    }
139    if let Some(p) = lock_path {
140        let abs = if p.is_absolute() { p.to_path_buf() } else { ctx.cwd.join(p) };
141        env.insert("HLOCK_LOCK".into(), abs.to_string_lossy().into_owned());
142    }
143    if let Some(l) = lock {
144        if l.tool.iter().any(|t| t.kind == "mcp") {
145            let path = hl.join("mcp.json");
146            std::fs::write(&path, serde_json::to_string_pretty(&mcp_config(&l.tool, ctx))?)?;
147            env.insert("HLOCK_MCP_CONFIG".into(), path.to_string_lossy().into_owned());
148        }
149        let lock_dir = lock_path.and_then(Path::parent).map(|d| if d.is_absolute() { d.to_path_buf() } else { ctx.cwd.join(d) }).unwrap_or_else(|| ctx.cwd.clone());
150        let skills: Vec<(String, PathBuf)> = l
151            .skill
152            .iter()
153            .filter_map(|s| {
154                let dir = s.vendored.as_ref().map(|v| lock_dir.join(v)).or_else(|| s.path.as_ref().map(|p| ctx.expand(p, None)))?;
155                dir.join("SKILL.md").is_file().then(|| (s.name.clone(), dir))
156            })
157            .collect();
158        if !skills.is_empty() {
159            let dir = hl.join("skills");
160            let _ = std::fs::remove_dir_all(&dir);
161            for (name, src) in skills {
162                copy_dir(&src, &dir.join(name));
163            }
164            env.insert("HLOCK_SKILLS_DIR".into(), dir.to_string_lossy().into_owned());
165        }
166    }
167
168    let vars = [("prompt", prompt), ("model", model.as_str()), ("workspace", &*workspace.to_string_lossy())];
169    let argv: Vec<String> = if let Some(ep) = &profile.run.entrypoint {
170        let py = python_for(profile.run.python.as_deref(), ctx).ok_or_else(|| Error::Profile("no Python interpreter found (set HL_PYTHON)".into()))?;
171        // the project directory must be importable
172        let pp = match ctx.var("PYTHONPATH") {
173            Some(old) => format!("{}{}{old}", ctx.cwd.display(), if cfg!(windows) { ";" } else { ":" }),
174            None => ctx.cwd.to_string_lossy().into_owned(),
175        };
176        env.insert("PYTHONPATH".into(), pp);
177        vec![py.to_string_lossy().into_owned(), "-c".into(), ENTRY_SHIM.into(), ep.clone()]
178    } else if !profile.run.command.is_empty() {
179        let mut a: Vec<String> = profile.run.command.iter().map(|x| subst(x, &vars)).collect();
180        if a[0].contains('/') && !Path::new(&a[0]).is_absolute() {
181            a[0] = ctx.cwd.join(&a[0]).to_string_lossy().into_owned();
182        }
183        a
184    } else {
185        return Err(Error::Profile(format!("harness {:?} has no entrypoint or command", profile.id)));
186    };
187    Ok(Invocation { argv, env, cwd: workspace.to_path_buf(), output })
188}
189
190/// Run an invocation to completion, capturing output, with a timeout.
191pub fn run(inv: &Invocation, ctx: &Ctx, timeout: Duration) -> Result<RunResult> {
192    let start = Instant::now();
193    let mut child = Command::new(&inv.argv[0])
194        .args(&inv.argv[1..])
195        .current_dir(&inv.cwd)
196        .env_clear()
197        .envs(&ctx.env)
198        .envs(&inv.env)
199        .stdin(Stdio::null())
200        .stdout(Stdio::piped())
201        .stderr(Stdio::piped())
202        .spawn()
203        .map_err(|e| Error::Profile(format!("cannot run {}: {e}", inv.argv[0])))?;
204    let mut timed_out = false;
205    loop {
206        if child.try_wait()?.is_some() {
207            break;
208        }
209        if start.elapsed() > timeout {
210            let _ = child.kill();
211            timed_out = true;
212            break;
213        }
214        std::thread::sleep(Duration::from_millis(20));
215    }
216    let out = child.wait_with_output()?;
217    let mut log = String::from_utf8_lossy(&out.stdout).into_owned();
218    log.push_str(&String::from_utf8_lossy(&out.stderr));
219    let parsed: Option<Value> = std::fs::read_to_string(&inv.output).ok().and_then(|t| serde_json::from_str(&t).ok());
220    let exit_ok = out.status.success() && !timed_out;
221    let success = parsed.as_ref().and_then(|v| v.get("success")).and_then(Value::as_bool).unwrap_or(exit_ok) && !timed_out;
222    if timed_out {
223        log.push_str(&format!("\n[hlock] timed out after {}s", timeout.as_secs()));
224    }
225    Ok(RunResult {
226        success,
227        exit_code: out.status.code(),
228        cost_usd: parsed.as_ref().and_then(|v| v["cost_usd"].as_f64()).unwrap_or(0.0),
229        steps: parsed.as_ref().and_then(|v| v["steps"].as_u64()).unwrap_or(1),
230        trace: parsed.as_ref().and_then(|v| v.get("trace").cloned()).unwrap_or_else(|| json!([])),
231        latency_s: (start.elapsed().as_secs_f64() * 1e3).round() / 1e3,
232        output: parsed,
233        log,
234    })
235}
236
237fn copy_dir(src: &Path, dst: &Path) {
238    let _ = std::fs::create_dir_all(dst);
239    for e in std::fs::read_dir(src).into_iter().flatten().flatten() {
240        let (from, to) = (e.path(), dst.join(e.file_name()));
241        if from.is_dir() {
242            copy_dir(&from, &to);
243        } else {
244            let _ = std::fs::copy(&from, &to);
245        }
246    }
247}
248
249/// Runs trials through a code harness's entrypoint, for `resolve` and `tune`.
250pub struct CommandAdapter {
251    profile: HarnessProfile,
252    ctx: Ctx,
253}
254
255impl CommandAdapter {
256    pub fn new(profile: HarnessProfile, ctx: Ctx) -> Self {
257        CommandAdapter { profile, ctx }
258    }
259}
260
261impl HarnessAdapter for CommandAdapter {
262    fn name(&self) -> &str {
263        &self.profile.id
264    }
265
266    fn run(&self, stack: &Stack, task: &Task, workspace: &Path, seed: u64) -> Result<RunOutcome> {
267        if let (Some(fixtures), Some(pack_dir)) = (&task.fixtures, &task.pack_dir) {
268            let src = pack_dir.join(fixtures);
269            if src.exists() {
270                copy_dir(&src, workspace);
271            }
272        }
273        let model = stack.llm.get("planner").or_else(|| stack.llm.values().next()).cloned();
274        let inv = invocation(&self.profile, None, None, model.as_deref(), &task.prompt_text, workspace, seed, &self.ctx)?;
275        let r = run(&inv, &self.ctx, Duration::from_secs(task.timeout_seconds.max(1)))?;
276        let mut trace = r.trace.clone();
277        if let Value::Array(a) = &mut trace {
278            let tail: Vec<&str> = r.log.lines().rev().take(20).collect::<Vec<_>>().into_iter().rev().collect();
279            a.push(json!({"type": "harness.exit", "harness": self.profile.id, "code": r.exit_code, "latency_s": r.latency_s, "log_tail": tail.join("\n")}));
280        }
281        Ok(RunOutcome { success: r.success, cost_usd: r.cost_usd, latency_s: r.latency_s, steps: r.steps, trace })
282    }
283}
284
285#[cfg(test)]
286mod tests {
287    use super::*;
288
289    #[test]
290    fn substitution_and_env_fill() {
291        assert_eq!(subst("--model={model} {prompt}", &[("model", "a:b"), ("prompt", "hi")]), "--model=a:b hi");
292        let ctx = Ctx { home: "/h".into(), cwd: "/w".into(), env: [("TOK".to_string(), "x".to_string())].into(), path: vec![] };
293        assert_eq!(fill_env("Bearer ${TOK} ${MISSING}", &ctx), "Bearer x ${MISSING}");
294    }
295
296    #[test]
297    fn mcp_document() {
298        let ctx = Ctx { home: "/h".into(), cwd: "/w".into(), env: [("HLOCK_SECRET_T".to_string(), "s3".to_string())].into(), path: vec![] };
299        let tools = vec![
300            Tool { name: "gh".into(), kind: "mcp".into(), transport: Some("stdio".into()), command: Some("npx".into()), args: vec!["pkg".into()], env: [("T".to_string(), "${HLOCK_SECRET_T}".to_string())].into(), ..Default::default() },
301            Tool { name: "remote".into(), kind: "mcp".into(), transport: Some("http".into()), url: Some("https://x/mcp".into()), ..Default::default() },
302            Tool { name: "off".into(), kind: "mcp".into(), command: Some("x".into()), enabled: Some(false), ..Default::default() },
303        ];
304        let v = mcp_config(&tools, &ctx);
305        assert_eq!(v["mcpServers"]["gh"]["env"]["T"], "s3");
306        assert_eq!(v["mcpServers"]["remote"]["type"], "http");
307        assert!(v["mcpServers"].get("off").is_none());
308    }
309}
harness_lock_core/contract.rs

harness_lock_core/
contract.rs
