assay_core/
metrics_api.rs

1use crate::model::{Expected, LlmResponse, TestCase};
2use async_trait::async_trait;
3
4/// Whether a metric actually evaluated anything for this test.
5///
6/// Orthogonal to `passed`, and deliberately not folded into it. Assertion-based verification
7/// settled this decades ago: an assertion with zero attempts is a coverage hole with no
8/// verification value, so every assert is paired with a *companion cover* confirming it was
9/// genuinely exercised rather than vacuously passed. `passed` answers "did the check hold";
10/// this answers "was there a check to hold".
11///
12/// The distinction is load-bearing here because the runner evaluates all thirteen registered
13/// metrics against every test, and a metric that does not handle a test's `Expected` variant used
14/// to return `pass(1.0)` — indistinguishable from a metric that ran and was satisfied.
15///
16/// `NotExercised` is a status and never a failure. Over-eager vacuity detection earns a
17/// suppression and takes real findings with it (Beer et al. on temporal antecedent failure), so
18/// this reports rather than decides.
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum Exercised {
21    /// The metric does not handle this test's `Expected` variant. Nothing was checked.
22    NotApplicable,
23    /// The metric applies, but the data it needs never appeared in this run, so its antecedent
24    /// never fired. A `trace_must_not_call_tool` naming a tool the agent never had is the shape:
25    /// syntactically perfect, permanently vacuous for this trace.
26    NotExercised,
27    /// Genuinely evaluated against the response.
28    Exercised,
29}
30
31impl Exercised {
32    /// The stable string for this value in `details["metrics"][…]["exercised"]`.
33    ///
34    /// A vocabulary, not a `Debug` rendering: it reaches `run.json`, so it is an interface. It
35    /// lives on the enum rather than beside the writer because there is now a reader —
36    /// [`crate::report::exercised`] — and a reader with its own copy of `"not_exercised"` would
37    /// match nothing the day the spelling moved, reporting a clean run instead of a broken one.
38    pub const fn label(self) -> &'static str {
39        match self {
40            Self::Exercised => "exercised",
41            Self::NotApplicable => "not_applicable",
42            Self::NotExercised => "not_exercised",
43        }
44    }
45}
46
47#[derive(Debug, Clone)]
48pub struct MetricResult {
49    pub score: f64,
50    pub passed: bool,
51    pub unstable: bool,
52    /// Additive on purpose. Existing `passed` readers keep working, and callers that want to know
53    /// whether the number means anything now have somewhere to look.
54    pub exercised: Exercised,
55    pub details: serde_json::Value,
56}
57
58impl MetricResult {
59    pub fn pass(score: f64) -> Self {
60        Self {
61            score,
62            passed: true,
63            unstable: false,
64            exercised: Exercised::Exercised,
65            details: serde_json::json!({}),
66        }
67    }
68    pub fn fail(score: f64, msg: &str) -> Self {
69        Self {
70            score,
71            passed: false,
72            unstable: false,
73            exercised: Exercised::Exercised,
74            details: serde_json::json!({"message": msg}),
75        }
76    }
77    pub fn unstable(score: f64, msg: &str) -> Self {
78        Self {
79            score,
80            passed: false,
81            unstable: true,
82            exercised: Exercised::Exercised,
83            details: serde_json::json!({"message": msg}),
84        }
85    }
86
87    /// This metric does not handle the test's `Expected` variant.
88    ///
89    /// `passed` stays true and the score stays 1.0 so that no existing reader starts failing tests
90    /// over a metric that was never asked to run. What changes is that the result now says so, and
91    /// the runner no longer lets it set the test's score.
92    pub fn not_applicable() -> Self {
93        Self {
94            score: 1.0,
95            passed: true,
96            unstable: false,
97            exercised: Exercised::NotApplicable,
98            details: serde_json::json!({"exercised": "not_applicable"}),
99        }
100    }
101
102    /// This metric applies, but the run produced nothing for it to check.
103    ///
104    /// `reason` names the missing antecedent, because "not exercised" without it is the same
105    /// unactionable silence the dimension exists to remove.
106    pub fn not_exercised(reason: &str) -> Self {
107        Self {
108            score: 1.0,
109            passed: true,
110            unstable: false,
111            exercised: Exercised::NotExercised,
112            details: serde_json::json!({"exercised": "not_exercised", "reason": reason}),
113        }
114    }
115
116    /// Whether this result's `score` and `passed` describe an actual evaluation.
117    pub fn is_exercised(&self) -> bool {
118        matches!(self.exercised, Exercised::Exercised)
119    }
120}
121
122#[async_trait]
123pub trait Metric: Send + Sync {
124    fn name(&self) -> &'static str;
125    async fn evaluate(
126        &self,
127        tc: &TestCase,
128        expected: &Expected,
129        resp: &LlmResponse,
130    ) -> anyhow::Result<MetricResult>;
131}
assay_core/metrics_api.rs

assay_core/
metrics_api.rs
