harness-lock-core 0.1.0

Core library for harness-lock: manifest, lockfile, store, tuning.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

//! Harness adapters — a baseline (reference) adapter for the local loop.

use std::fs;
use std::path::Path;

use serde_json::json;

use crate::error::Result;
use crate::providers::ReferenceProvider;
use crate::search::Stack;
use crate::tasks::Task;

/// The resolved outcome of running one task through a harness.
pub struct RunOutcome {
    pub success: bool,
    pub cost_usd: f64,
    pub latency_s: f64,
    pub steps: u64,
    /// Trace events (a serde array) for content-addressed storage.
    pub trace: serde_json::Value,
}

/// A harness adapter runs one `(stack, task, seed)` and produces a `RunOutcome`.
pub trait HarnessAdapter {
    fn name(&self) -> &str;
    fn run(&self, stack: &Stack, task: &Task, workspace: &Path, seed: u64) -> Result<RunOutcome>;
}

/// Baseline/offline harness that drives the deterministic reference provider.
pub struct ReferenceAdapter {
    provider: ReferenceProvider,
}

impl Default for ReferenceAdapter {
    fn default() -> Self {
        ReferenceAdapter {
            provider: ReferenceProvider::default(),
        }
    }
}

impl HarnessAdapter for ReferenceAdapter {
    fn name(&self) -> &str {
        "reference"
    }

    fn run(&self, stack: &Stack, task: &Task, workspace: &Path, seed: u64) -> Result<RunOutcome> {
        // copy fixtures into the workspace
        if let (Some(fixtures), Some(pack_dir)) = (&task.fixtures, &task.pack_dir) {
            let src = pack_dir.join(fixtures);
            if src.exists() {
                copy_dir(&src, workspace);
            }
        }
        let model_spec = stack
            .llm
            .get("planner")
            .cloned()
            .unwrap_or_else(|| "reference:qwen-7b".to_string());
        let model = self.provider.resolve_alias(&model_spec).1;
        let expected = task.expected_artifacts();

        let mut provider = self.provider.clone();
        provider.seed = seed;
        let mut cost = 0.0;
        let mut latency = 0.0;
        let mut success = false;
        let mut steps = 0u64;
        let mut events = Vec::new();

        for step in 0..stack.max_steps {
            // agent-level retries: attempt this step up to 1+retries times
            let max_attempts = (stack.retries + 1) as usize;
            let mut any_ok = false;
            for _attempt in 0..max_attempts {
                let r = provider.complete(&model, 1);
                cost += r.cost_usd;
                latency += r.latency_ms / 1000.0;
                events.push(json!({
                    "type": "llm.call",
                    "role": "planner",
                    "model": model.clone(),
                    "usage": {"input_tokens": r.usage.input_tokens, "output_tokens": r.usage.output_tokens},
                    "cost_usd": r.cost_usd,
                    "latency_ms": r.latency_ms,
                }));
                if r.content.starts_with("ANSWER_OK") {
                    any_ok = true;
                    break;
                }
            }
            if any_ok {
                for art in &expected {
                    let out = workspace.join(art);
                    if let Some(parent) = out.parent() {
                        let _ = fs::create_dir_all(parent);
                    }
                    let _ = fs::write(&out, format!("artifact for {}", task.id));
                }
                success = true;
                steps = step as u64 + 1;
                break;
            }
            if let Some(max_cost) = stack.max_cost_usd_per_task {
                if cost > max_cost {
                    steps = step as u64 + 1;
                    break;
                }
            }
            steps = step as u64 + 1;
        }

        Ok(RunOutcome {
            success,
            cost_usd: (cost * 1e6).round() / 1e6,
            latency_s: (latency * 1e3).round() / 1e3,
            steps,
            trace: json!(events),
        })
    }
}

fn copy_dir(src: &Path, dst: &Path) {
    let _ = fs::create_dir_all(dst);
    if let Ok(rd) = fs::read_dir(src) {
        for entry in rd.flatten() {
            let from = entry.path();
            let to = dst.join(entry.file_name());
            if from.is_dir() {
                copy_dir(&from, &to);
            } else if let Ok(data) = fs::read(&from) {
                let _ = fs::write(&to, data);
            }
        }
    }
}