harness_lock_core/
resolver.rs

1//! The resolver: turns a manifest into an `harness.lock`.
2
3use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5
6use serde_json::json;
7
8use crate::adapters::{HarnessAdapter, ReferenceAdapter};
9use crate::error::{Error, Result};
10use crate::lockfile::{Context, Decision, Evidence, Harness, Instructions, LLM, Lockfile, Policy, Provenance, Skill, Tool, Verify};
11use crate::manifest::{HarnessCandidate, Manifest};
12use crate::profiles::{probe_harness, Ctx, Registry};
13use crate::search::{select_winner, Stack, StackScore, TrialResult};
14use crate::store::Store;
15use crate::tasks::EvalPack;
16use crate::worker::{Backend, PythonAdapter};
17
18/// Aggregated metrics container produced by the resolver.
19
20pub struct Resolver {
21    pub manifest: Manifest,
22    pub workdir: PathBuf,
23    pub store: Store,
24    pub backend: Backend,
25    pack_by_task: BTreeMap<String, crate::tasks::EvalPack>,
26    adapter: Box<dyn HarnessAdapter>,
27    holdout: Vec<crate::tasks::Task>,
28    /// Target harness from the CLI (`--harness`), overriding `[target] harness`.
29    pub harness_override: Option<String>,
30    /// Code harnesses (`[harnesses.<id>]`) that trials run through their entrypoint.
31    code_harnesses: BTreeMap<String, crate::profiles::HarnessProfile>,
32}
33
34impl Resolver {
35    /// Create a resolver that runs on the local, in-process reference harness.
36    pub fn new(manifest: Manifest, workdir: impl Into<PathBuf>) -> Self {
37        Self::with_backend(manifest, workdir, Backend::Local)
38    }
39
40    /// Create a resolver for a specific backend. `devbox`/`cluster` delegate trials
41    /// to the Python worker (which may lift them to real Flyte tasks); `local`
42    /// keeps the deterministic offline reference harness.
43    pub fn with_backend(manifest: Manifest, workdir: impl Into<PathBuf>, backend: Backend) -> Self {
44        let workdir: PathBuf = workdir.into();
45        let store = Store::new(workdir.join("store"));
46        let adapter: Box<dyn HarnessAdapter> = if backend.needs_worker() {
47            Box::new(PythonAdapter::new())
48        } else {
49            Box::new(ReferenceAdapter::default())
50        };
51        Resolver {
52            manifest,
53            workdir,
54            store,
55            backend,
56            pack_by_task: BTreeMap::new(),
57            adapter,
58            holdout: Vec::new(),
59            harness_override: None,
60            code_harnesses: BTreeMap::new(),
61        }
62    }
63
64    /// Load the manifest's eval packs and return the search-split tasks.
65    pub fn load_packs(&mut self) -> Vec<crate::tasks::Task> {
66        let mut search: Vec<crate::tasks::Task> = Vec::new();
67        let base = self.manifest.root();
68        for ref_ in self.manifest.eval_packs() {
69            match EvalPack::load(&ref_, Some(&base)) {
70                Ok(pack) => {
71                    let errors = pack.check().iter().filter(|i| i.level == "error").count();
72                    if errors > 0 {
73                        eprintln!("warn: eval pack {ref_} has {errors} problem(s); run `hlock evals check {ref_}`");
74                    }
75                    let (s, h) = pack.split(self.manifest.search_frac(), self.manifest.budget.random_seed);
76                    for t in pack.tasks.clone() {
77                        self.pack_by_task.insert(t.id.clone(), pack.clone());
78                    }
79                    search.extend(s);
80                    self.holdout.extend(h);
81                }
82                Err(e) => eprintln!("warn: skipping eval pack {ref_}: {e}"),
83            }
84        }
85        search
86    }
87
88    /// The holdout-split tasks (populated by [`load_packs`]).
89    pub fn holdout(&self) -> &[crate::tasks::Task] {
90        &self.holdout
91    }
92
93    /// Cartesian product over the manifest's candidate axes.
94    pub fn enumerate_stacks(&self) -> Vec<Stack> {
95        let c = &self.manifest.candidates;
96        let llm_combos = llm_combos(&c.llm);
97        let decisions = if c.decision.is_empty() {
98            vec!["none".to_string()]
99        } else {
100            c.decision.clone()
101        };
102        let harnesses = harness_combos(&c.harness);
103
104        let mut stacks = Vec::new();
105        for llm in &llm_combos {
106            for decision in &decisions {
107                for (harness, pattern) in &harnesses {
108                    stacks.push(Stack {
109                        llm: llm.clone(),
110                        harness: harness.clone(),
111                        pattern: pattern.clone(),
112                        decision: decision.clone(),
113                        tools: self.manifest.tools.keys().cloned().collect(),
114                        skills: self.manifest.skills.keys().cloned().collect(),
115                        instructions: instructions_files(&self.manifest.instructions),
116                        max_steps: 40,
117                        max_cost_usd_per_task: None,
118                        retries: policy_axis(&self.manifest.candidates.policy, "retries", 0),
119                        temperature: policy_axis_float(&self.manifest.candidates.policy, "temperature"),
120                        tool_timeout_seconds: None,
121                    });
122                }
123            }
124        }
125        if stacks.len() > 200 {
126            stacks.truncate(200);
127        }
128        stacks
129    }
130
131    /// Run one trial (copying fixtures, running the harness, grading).
132    pub fn run_trial(&self, stack: &Stack, task: &crate::tasks::Task, seed: u64, ws_root: &Path) -> TrialResult {
133        let ws = ws_root.join(format!("{}_{}_{}", &stack.hash()[..10.min(stack.hash().len())], task.id, seed));
134        let _ = std::fs::remove_dir_all(&ws);
135        let _ = std::fs::create_dir_all(&ws);
136
137        let outcome = match self.code_harnesses.get(&stack.harness) {
138            Some(p) => crate::contract::CommandAdapter::new(p.clone(), Ctx::from_process()).run(stack, task, &ws, seed),
139            None => self.adapter.run(stack, task, &ws, seed),
140        };
141        let outcome = outcome.unwrap_or_else(|_| {
142            crate::adapters::RunOutcome {
143                success: false,
144                cost_usd: 0.0,
145                latency_s: 0.0,
146                steps: 0,
147                trace: json!([]),
148            }
149        });
150        let grades = self
151            .pack_by_task
152            .get(&task.id)
153            .map(|p| p.grade(task, &ws))
154            .unwrap_or_default();
155        let success = outcome.success && grades.iter().all(|g| g.success);
156        let score = grades.iter().map(|g| g.score).sum::<f64>() / grades.len().max(1) as f64;
157
158        // persist the trace to the store (content-addressed)
159        let trace = json!({
160            "trial": {"stack": stack.hash(), "task": task.id, "seed": seed},
161            "events": outcome.trace,
162            "result": {"artifacts": task.expected_artifacts()},
163            "metrics": {"steps": outcome.steps, "cost_usd": outcome.cost_usd, "latency_s": outcome.latency_s},
164            "grades": grades.iter().map(|g| json!({"success": g.success, "score": g.score})).collect::<Vec<_>>(),
165        });
166        let _ = self.store.put_value(&trace);
167
168        TrialResult {
169            stack_hash: stack.hash(),
170            task_id: task.id.clone(),
171            seed,
172            success,
173            score,
174            cost_usd: outcome.cost_usd,
175            latency_s: outcome.latency_s,
176            steps: outcome.steps,
177        }
178    }
179
180    /// Resolve: enumerate, run, aggregate, select, and write `harness.lock`.
181    pub fn resolve(&mut self) -> Result<Lockfile> {
182        let lock = self.resolve_lock()?;
183        lock.save(std::path::Path::new(crate::LOCKFILE_PATH))?;
184        Ok(lock)
185    }
186
187    /// Resolve and return the winning lock without writing it.
188    pub fn resolve_lock(&mut self) -> Result<Lockfile> {
189        let registry = Registry::load_for(&Ctx::from_process(), Some(&self.manifest))?;
190        apply_target(&mut self.manifest, self.harness_override.as_deref(), &registry)?;
191        self.code_harnesses = registry
192            .harnesses
193            .iter()
194            .filter(|h| h.kind == "custom" && (h.run.entrypoint.is_some() || !h.run.command.is_empty()))
195            .map(|h| (h.id.clone(), h.clone()))
196            .collect();
197        let search_tasks = self.load_packs();
198        let stacks = self.enumerate_stacks();
199        let seeds = self.manifest.seeds();
200        let ws_root = self.workdir.join("runs");
201        let _ = std::fs::create_dir_all(&ws_root);
202
203        let mut results_by_stack: BTreeMap<String, Vec<TrialResult>> = BTreeMap::new();
204        for stack in &stacks {
205            for task in &search_tasks {
206                for seed in 0..seeds {
207                    let tr = self.run_trial(stack, task, seed, &ws_root);
208                    results_by_stack.entry(stack.hash()).or_default().push(tr);
209                }
210            }
211        }
212        // respect max-trials globally
213        let mut n = 0usize;
214        let max = self.manifest.budget.max_trials.unwrap_or(usize::MAX);
215        for v in results_by_stack.values_mut() {
216            if n >= max {
217                v.clear();
218            } else {
219                v.truncate(max.saturating_sub(n));
220                n += v.len();
221            }
222        }
223
224        let mut scores: Vec<StackScore> = Vec::new();
225        for stack in &stacks {
226            if let Some(trials) = results_by_stack.get(&stack.hash()) {
227                if !trials.is_empty() {
228                    scores.push(StackScore {
229                        stack: stack.clone(),
230                        trials: trials.clone(),
231                        n: trials.len(),
232                    });
233                }
234            }
235        }
236
237        let winner = select_winner(
238            scores.clone(),
239            &self.manifest.objective.primary,
240            &self.manifest.objective.constraints,
241            &self.manifest.objective.select,
242        )
243        .ok_or_else(|| Error::Tuning("no viable stack found".into()))?;
244
245        let lock = self.build_resolve(&winner, &scores);
246        let _ = self.write_report(&lock, &scores);
247        Ok(lock)
248    }
249
250    fn build_resolve(&self, winner: &StackScore, _scores: &[StackScore]) -> Lockfile {
251        let s = &winner.stack;
252        let llms = s
253            .llm
254            .iter()
255            .map(|(role, spec)| {
256                let (provider, model) = match spec.split_once(':') {
257                    Some((p, m)) if !m.is_empty() => (p.to_string(), m.to_string()),
258                    _ => ("reference".to_string(), spec.clone()),
259                };
260                LLM {
261                    role: role.clone(),
262                    provider,
263                    model,
264                    resolved_from: Some(spec.clone()),
265                    params: BTreeMap::new(),
266                    fingerprint: Some(format!("sha256:{}", crate::hash::sha256_hex(spec.as_bytes()))),
267                    ..Default::default()
268                }
269            })
270            .collect();
271
272        let mut success_rate = BTreeMap::new();
273        success_rate.insert("mean".to_string(), roundf(winner.success_rate()));
274        let mut cost = BTreeMap::new();
275        cost.insert("mean".to_string(), roundf(winner.mean_cost()));
276        let mut latency = BTreeMap::new();
277        latency.insert("p50".to_string(), roundf(winner.mean_latency()));
278        latency.insert("p95".to_string(), roundf(winner.p95_latency()));
279
280        let mut stack_map = BTreeMap::new();
281        stack_map.insert("name".to_string(), serde_json::json!(self.manifest.name()));
282        stack_map.insert("selection".to_string(), serde_json::json!(self.manifest.objective.select));
283        let mut search_map = BTreeMap::new();
284        search_map.insert("strategy".to_string(), serde_json::json!(self.manifest.budget.strategy));
285        search_map.insert("candidates".to_string(), serde_json::json!(winner.n));
286
287        Lockfile {
288            manifest_hash: Some(self.manifest.manifest_hash()),
289            stack: stack_map,
290            llm: llms,
291            decision: Decision {
292                kind: s.decision.clone(),
293                decisions: if s.decision == "none" { vec![] } else { vec!["retry".into(), "stop".into()] },
294                ..Default::default()
295            },
296            harness: self.pin_harness(&s.harness, &s.pattern),
297            tool: s.tools.iter().map(|t| Tool { name: t.clone(), kind: "builtin".into(), ..Default::default() }).collect(),
298            skill: s.skills.iter().map(|sk| Skill { name: sk.clone(), origin: "first-party".into(), editable: true, evidence_digest: Some(String::new()), ..Default::default() }).collect(),
299            instructions: s.instructions.iter().map(|i| Instructions { name: i.clone(), scope: "project".into(), ..Default::default() }).collect(),
300            policy: Policy {
301                retries: 1,
302                max_steps: 40,
303                human_approval: "never".into(),
304                ..Default::default()
305            },
306            context: Context { isolation: self.manifest.context.get("isolation").and_then(serde_json::Value::as_str).unwrap_or("locked-only").into(), ..Default::default() },
307            evidence: Evidence {
308                eval_pack: Some(self.manifest.eval_packs().join(",")),
309                split: "search".into(),
310                trials: winner.n,
311                success_rate,
312                cost_usd_per_task: cost,
313                latency_s_per_task: latency,
314                ..Default::default()
315            },
316            verify: Verify::default(),
317            provenance: Provenance {
318                backend: self.manifest.backend_target(),
319                search: search_map,
320                ..Default::default()
321            },
322            ..Default::default()
323        }
324    }
325
326    /// The `[harness]` table for the winner. A harness with a profile gets the
327    /// version and install source of the copy on this machine, and `adapter`
328    /// records what actually ran the trials when that differs from the harness.
329    fn pin_harness(&self, name: &str, pattern: &str) -> Harness {
330        let mut h = Harness { name: name.to_string(), pattern: Some(pattern.to_string()), ..Default::default() };
331        let ctx = Ctx::from_process();
332        let Ok(registry) = Registry::load_for(&ctx, Some(&self.manifest)) else { return h };
333        if let Some(profile) = registry.harness(name) {
334            let probe = probe_harness(profile, &ctx);
335            h.version = probe.version;
336            h.source = probe.source;
337            h.adapter = Some(if self.code_harnesses.contains_key(name) { name.to_string() } else { self.adapter.name().to_string() });
338        }
339        h
340    }
341
342    fn write_report(&self, lock: &Lockfile, _scores: &[StackScore]) -> Result<()> {
343        let report = self.workdir.join("report.md");
344        std::fs::write(&report, format!("# harness-lock report\n\n{}\n", lock.summary()))?;
345        Ok(())
346    }
347}
348
349/// Narrow the harness candidates to the target harness (`explicit`, else the
350/// manifest's `[target] harness`). With no harness candidates, or only the
351/// default `reference`, the target becomes the sole candidate; otherwise it must
352/// be one of them. Returns the canonical target id.
353pub fn apply_target(manifest: &mut Manifest, explicit: Option<&str>, registry: &Registry) -> Result<Option<String>> {
354    let Some(name) = explicit.map(str::to_string).or_else(|| manifest.target_harness()) else {
355        return Ok(None);
356    };
357    if name == "reference" {
358        manifest.candidates.harness.retain(|h| h.name == "reference");
359        return Ok(Some(name));
360    }
361    let profile = registry.require_harness(&name)?;
362    let cands = &mut manifest.candidates.harness;
363    let only_default = cands.iter().all(|h| h.name == "reference");
364    if only_default {
365        let pattern = cands.first().map(|h| h.pattern.clone()).unwrap_or_default();
366        *cands = vec![HarnessCandidate { name: profile.id.clone(), pattern, ..Default::default() }];
367    } else {
368        let before: Vec<String> = cands.iter().map(|h| h.name.clone()).collect();
369        cands.retain(|h| profile.matches(&h.name));
370        if cands.is_empty() {
371            return Err(Error::Manifest(format!(
372                "target harness {:?} is not one of [candidates] harness ({})",
373                profile.id,
374                before.join(", ")
375            )));
376        }
377        for h in cands.iter_mut() {
378            h.name = profile.id.clone();
379        }
380    }
381    Ok(Some(profile.id.clone()))
382}
383
384fn roundf(v: f64) -> f64 {
385    (v * 1e4).round() / 1e4
386}
387
388fn llm_combos(cands: &BTreeMap<String, crate::manifest::LLMCandidate>) -> Vec<BTreeMap<String, String>> {
389    if cands.is_empty() {
390        let mut m = BTreeMap::new();
391        m.insert("planner".to_string(), "reference:qwen-7b".to_string());
392        return vec![m];
393    }
394    let roles: Vec<String> = cands.keys().cloned().collect();
395    let mut per_role: Vec<Vec<(String, String)>> = Vec::new();
396    for role in &roles {
397        let opts = cands.get(role).map(|c| c.any_of.clone()).unwrap_or_default();
398        let opts = if opts.is_empty() {
399            vec!["reference:qwen-7b".to_string()]
400        } else {
401            opts
402        };
403        per_role.push(opts.into_iter().map(|o| (role.clone(), o)).collect());
404    }
405    // cartesian product
406    let mut combos: Vec<BTreeMap<String, String>> = vec![BTreeMap::new()];
407    for options in per_role {
408        let mut next = Vec::new();
409        for c in &combos {
410            for (role, model) in &options {
411                let mut cc = c.clone();
412                cc.insert(role.clone(), model.clone());
413                next.push(cc);
414            }
415        }
416        combos = next;
417    }
418    combos
419}
420
421fn harness_combos(cands: &[HarnessCandidate]) -> Vec<(String, String)> {
422    if cands.is_empty() {
423        return vec![("reference".to_string(), "single-agent".to_string())];
424    }
425    let mut out = Vec::new();
426    for h in cands {
427        let patterns = if h.pattern.is_empty() { vec!["single-agent".to_string()] } else { h.pattern.clone() };
428        for p in patterns {
429            out.push((h.name.clone(), p));
430        }
431    }
432    out
433}
434
435fn instructions_files(ins: &BTreeMap<String, serde_json::Value>) -> Vec<String> {
436    ins.get("files")
437        .and_then(serde_json::Value::as_array)
438        .map(|a| a.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect())
439        .unwrap_or_default()
440}
441
442fn policy_axis(policy: &BTreeMap<String, Vec<serde_json::Value>>, key: &str, default: u32) -> u32 {
443    policy
444        .get(key)
445        .and_then(|v| v.first())
446        .and_then(serde_json::Value::as_u64)
447        .map(|n| n as u32)
448        .unwrap_or(default)
449}
450
451fn policy_axis_float(policy: &BTreeMap<String, Vec<serde_json::Value>>, key: &str) -> Option<f64> {
452    policy
453        .get(key)
454        .and_then(|v| v.first())
455        .and_then(serde_json::Value::as_f64)
456}
harness_lock_core/resolver.rs

harness_lock_core/
resolver.rs
