harness-lock-core 0.3.1

Core library for hlock (also published as harness-lock): manifest, lockfile, store, tuning.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

//! End-to-end resolver test: build a temp eval pack, run `Resolver::resolve`, and check
//! the produced `harness.lock`.

use std::fs;
use std::path::{Path, PathBuf};

use harness_lock_core::{Manifest, Resolver};

fn make_pack(root: &Path) -> PathBuf {
    let taskdir = root.join("t1");
    fs::create_dir_all(&taskdir).unwrap();
    fs::write(
        taskdir.join("prompt.md"),
        "Produce an executive summary as summary.md",
    )
    .unwrap();
    let pack = root.join("pack.toml");
    fs::write(
        &pack,
        r#"
[pack]
name = "test/pack"
version = "0.1.0"
license = "Apache-2.0"
domain = "demo"

[[task]]
id = "summarize"
prompt = "t1/prompt.md"
tools = ["filesystem"]
expected = { artifacts = ["summary.md"] }
"#,
    )
    .unwrap();
    root.to_path_buf()
}

fn make_manifest(pack: &Path) -> Manifest {
    let toml = format!(
        r#"
[project]
name = "resolvertest"

# the old [tasks] name for [evals] still parses
[tasks]
seeds = 2

[tasks.packs]
# (empty, replaced below)

[candidates.llm.planner]
any_of = ["reference:qwen-7b", "reference:qwen-32b"]

[candidates]
decision = ["none", "rules:default"]
harness = ["reference"]

[objective]
primary = "success-rate"
select = "min-cost"

[objective.constraints]
success-rate = ">=0.5"

[budget]
strategy = "grid"
max_trials = 200

[backend]
target = "local"
"#
    );
    let mut m = Manifest::from_str(&toml).unwrap();
    m.evals.insert(
        "packs".to_string(),
        serde_json::json!([pack.to_string_lossy().to_string()]),
    );
    m
}

#[test]
fn tune_writes_a_resolve() {
    let base = std::env::temp_dir().join(format!("hlock-resolver-{}", std::process::id()));
    let _ = fs::remove_dir_all(&base);
    fs::create_dir_all(&base).unwrap();
    let cwd = std::env::current_dir().unwrap();
    std::env::set_current_dir(&base).unwrap();

    let pack = make_pack(&base);
    let manifest = make_manifest(&pack);
    let mh = manifest.manifest_hash();
    let mut resolver = Resolver::new(manifest, base.join(".harnesslock"));
    let lock = resolver.resolve().unwrap();

    // lock is written at cwd
    let lock_path = base.join("harness.lock");
    assert!(lock_path.exists(), "harness.lock should be written at cwd");
    assert_eq!(lock.harness.name, "reference");
    assert!(lock.evidence.trials >= 2);

    // reload and confirm it round-trips
    let reloaded = harness_lock_core::Lockfile::load(&lock_path).unwrap();
    assert_eq!(reloaded.summary(), lock.summary());
    assert!(reloaded.check_manifest(&mh).is_empty());

    std::env::set_current_dir(&cwd).unwrap();
    let _ = fs::remove_dir_all(&base);
}

#[test]
fn manifest_accepts_harness_list() {
    let m = Manifest::from_str(
        r#"
[project]
name = "a"
[candidates.llm.planner]
any_of = ["reference:qwen-7b"]
[candidates]
harness = ["reference"]
"#,
    )
    .unwrap();
    assert_eq!(m.candidates.harness.len(), 1);
    assert_eq!(m.candidates.harness[0].name, "reference");
}