newt-agent-py 0.8.2

Python bindings for newt-agent
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

"""Smoke tests for newt_agent.eval."""

from __future__ import annotations

import tempfile
from pathlib import Path

import newt_agent.eval as evalmod
from newt_agent.acp_worker import TaskReply


def _write_case(dir: Path, name: str, body: str) -> Path:
    case_dir = dir / name
    (case_dir / "workspace").mkdir(parents=True)
    (case_dir / "case.toml").write_text(body)
    return case_dir


def test_testcase_load_dir_roundtrips() -> None:
    with tempfile.TemporaryDirectory() as d:
        body = """
name = "demo"
description = "demo case"
language = "rust"
prompt = "do a thing"
evaluators = ["diff_nonempty"]
expected_patterns = ["hello"]

[mock_response]
content = "diff goes here"
"""
        case_dir = _write_case(Path(d), "001-demo", body)
        case = evalmod.TestCase.load_dir(str(case_dir))
        assert case.name == "demo"
        assert case.is_rust()
        assert case.evaluators == ["diff_nonempty"]
        assert case.expected_patterns == ["hello"]
        assert case.mock_response.content == "diff goes here"


def test_load_all_sorts_by_name() -> None:
    with tempfile.TemporaryDirectory() as d:
        for n in ["002-bravo", "001-alpha"]:
            _write_case(
                Path(d),
                n,
                f"""
name = "{n}"
description = ""
language = "rust"
prompt = ""
evaluators = []

[mock_response]
content = ""
""",
            )
        cases = evalmod.TestCase.load_all(d)
        assert [c.name for c in cases] == ["001-alpha", "002-bravo"]


def test_eval_result_pass_fail_helpers() -> None:
    p = evalmod.EvalResult.pass_("ev", "ok")
    assert p.passed and p.score == 1.0
    f = evalmod.EvalResult.fail("ev", "boom")
    assert not f.passed and f.score == 0.0


def test_diff_nonempty_evaluator_passes_on_diff() -> None:
    case = _make_inline_case()
    reply = TaskReply("test-model", "content", "+real\n", False)
    ctx = evalmod.EvalContext(case, "/tmp", "/tmp", reply)
    ev = evalmod.DiffNonemptyEvaluator()
    res = ev.evaluate(ctx)
    assert res.passed


def test_diff_nonempty_evaluator_fails_on_empty() -> None:
    case = _make_inline_case()
    reply = TaskReply("test-model", "content", "", False)
    ctx = evalmod.EvalContext(case, "/tmp", "/tmp", reply)
    ev = evalmod.DiffNonemptyEvaluator()
    res = ev.evaluate(ctx)
    assert not res.passed


def test_default_evaluator_names_includes_five() -> None:
    names = evalmod.default_evaluator_names()
    assert set(names) == {
        "diff_nonempty",
        "diff_applies",
        "rust_compiles",
        "tests_pass",
        "pattern_match",
    }


def test_evaluator_known() -> None:
    assert evalmod.evaluator_known("diff_nonempty")
    assert not evalmod.evaluator_known("nope")


def test_runner_config_builders() -> None:
    cfg = evalmod.RunnerConfig("/tmp/newt")
    assert cfg.worker_bin == "/tmp/newt"
    assert cfg.mock_endpoint is None
    cfg.with_mock_endpoint("http://127.0.0.1:8080")
    cfg.with_model("llama3.1:8b")
    cfg.with_timeout_ms(5000)
    cfg.with_coder_mode(True)
    assert cfg.mock_endpoint == "http://127.0.0.1:8080"
    assert cfg.model_override == "llama3.1:8b"
    assert cfg.timeout_ms == 5000
    assert cfg.coder_mode


def test_scorecard_renders_table() -> None:
    s = evalmod.Scorecard()
    s.push("case-a", [evalmod.EvalResult.pass_("diff_nonempty", "ok")])
    s.push("case-b", [evalmod.EvalResult.fail("diff_nonempty", "boom")])
    table = s.render_table()
    assert "case-a" in table
    assert "case-b" in table
    assert "FAIL" in table


def _make_inline_case() -> "evalmod.TestCase":
    # Load a one-off case from disk so the TestCase has a valid `case_dir`.
    import tempfile

    d = tempfile.mkdtemp()
    case_dir = _write_case(
        Path(d),
        "inline",
        """
name = "inline"
description = ""
language = "rust"
prompt = ""
evaluators = []

[mock_response]
content = ""
""",
    )
    return evalmod.TestCase.load_dir(str(case_dir))