eval_core/
scorer.rs

1//! Scoring one expectation against a run's result, behind the [`Scorer`] trait — plus the
2//! batteries-included [`BuiltinScorer`] that needs NO host scoring code at all.
3//!
4//! The host implements [`Scorer`] over the SAME `World` its [`Harness`](crate::Harness) produces. For
5//! each of a case's `expect` predicates the runner calls [`Scorer::score`], collecting the returned
6//! `(label, passed)` pairs into the case's [`CaseOutcome::predicates`](crate::report::CaseOutcome). A
7//! case passes iff the run succeeded AND every predicate's `passed` is `true`.
8//!
9//! `score` now also receives the run's [`RunArtifacts`], so a scorer can assert on
10//! what the agent DID (tool calls, params, final text) and not only on the post-run world. The common
11//! case — "score the built-in [`Expectation`]s over the artifacts, ignoring
12//! the world" — is [`BuiltinScorer`], which a host gets for free via [`run_suite`](crate::run_suite).
13
14use crate::expect::Expectation;
15use crate::harness::RunArtifacts;
16
17/// Scores one expectation against the world AND artifacts a case produced. The host implements this for a
18/// custom predicate/world; for the built-in assertions use [`BuiltinScorer`].
19///
20/// `score` returns BOTH a human-readable label and the pass/fail in one call (the existing AetherCore
21/// scorer kept these as a paired `score()`/`label()`; folding them avoids re-deriving the label and
22/// guarantees the label always matches the verdict). The label is shown in the report's per-predicate
23/// diagnostics, so make it identify WHICH predicate it is (e.g. `"SolidPlaced(>= 4)"`).
24pub trait Scorer {
25    /// The world type to score against — must be the same world the paired
26    /// [`Harness::World`](crate::Harness::World) produces (the runner enforces
27    /// `Scorer<World = Harness::World>`).
28    type World;
29    /// The host's predicate type — matches the element type of [`EvalCase::expect`](crate::EvalCase::expect).
30    type Expect;
31
32    /// Score one `expect` predicate against the post-run `world` and the run's `artifacts`.
33    ///
34    /// Returns `(label, passed)`: a human-readable label for per-predicate diagnostics, and whether the
35    /// predicate held. Must be side-effect-free with respect to `world` (it takes a shared reference);
36    /// the runner may call it for several predicates over the same world. A scorer that only inspects
37    /// the world can ignore `artifacts` (and vice versa).
38    fn score(
39        &self,
40        expect: &Self::Expect,
41        artifacts: &RunArtifacts,
42        world: &Self::World,
43    ) -> (String, bool);
44}
45
46/// The batteries-included scorer: evaluates the built-in [`Expectation`](crate::expect) assertions over
47/// a run's [`RunArtifacts`], ignoring the world entirely (`World = ()`).
48///
49/// This is what removes the "implement a `Scorer`" step for the common case. Paired with the
50/// [`Agent`](crate::Agent) trait + [`run_suite`](crate::run_suite), a host scores tool-call / parameter /
51/// final-text assertions with zero scoring code of its own.
52///
53/// A malformed regex in a [`FinalTextMatches`](crate::expect::Expectation::FinalTextMatches) expectation
54/// can't surface a `Result` through the infallible [`Scorer::score`] signature, so it is reported as a
55/// FAILED predicate whose label names the regex error (rather than panicking or being silently dropped).
56#[derive(Debug, Default, Clone, Copy)]
57pub struct BuiltinScorer;
58
59impl Scorer for BuiltinScorer {
60    type World = ();
61    type Expect = Expectation;
62
63    fn score(&self, expect: &Expectation, artifacts: &RunArtifacts, _world: &()) -> (String, bool) {
64        match expect.evaluate(artifacts) {
65            Ok(result) => result,
66            // A bad regex is an authoring error; surface it as a clearly-labeled failed predicate so the
67            // report shows WHICH expectation is malformed instead of hiding it behind the infallible API.
68            Err(err) => (format!("{} [invalid: {err}]", expect.label()), false),
69        }
70    }
71}
72
73#[cfg(test)]
74mod tests {
75    use super::*;
76    use crate::harness::ToolCall;
77    use serde_json::json;
78
79    fn artifacts() -> RunArtifacts {
80        RunArtifacts {
81            tool_calls: vec![ToolCall::new(
82                "calculator",
83                json!({"op": "add", "a": 2, "b": 2}),
84            )],
85            final_text: Some("The answer is 4".to_owned()),
86            ..RunArtifacts::default()
87        }
88    }
89
90    #[test]
91    fn builtin_scorer_evaluates_over_artifacts() {
92        let art = artifacts();
93        let (label, passed) = BuiltinScorer.score(
94            &Expectation::CalledToolWith {
95                tool: "calculator".into(),
96                args: json!({"op": "add"}),
97            },
98            &art,
99            &(),
100        );
101        assert!(passed);
102        assert!(label.starts_with("CalledToolWith("));
103
104        let (_, passed) = BuiltinScorer.score(
105            &Expectation::FinalNumberEquals {
106                value: 4.0,
107                tolerance: 0.0,
108            },
109            &art,
110            &(),
111        );
112        assert!(passed);
113    }
114
115    #[test]
116    fn builtin_scorer_reports_bad_regex_as_failed_labeled_predicate() {
117        let (label, passed) = BuiltinScorer.score(
118            &Expectation::FinalTextMatches { regex: "(".into() },
119            &artifacts(),
120            &(),
121        );
122        assert!(!passed, "a malformed regex scores as a failed predicate");
123        assert!(
124            label.contains("invalid:"),
125            "label names the regex error: {label}"
126        );
127    }
128}
eval_core/scorer.rs

eval_core/
scorer.rs
