assay_core/agent_assertions/
mod.rs

1pub mod cover;
2pub mod matchers;
3pub mod model;
4
5use crate::errors::diagnostic::Diagnostic;
6use crate::storage::Store;
7
8pub struct EpisodeGraph {
9    pub episode_id: String,
10    pub steps: Vec<crate::storage::rows::StepRow>,
11    pub tool_calls: Vec<crate::storage::rows::ToolCallRow>,
12}
13
14/// The failures, and the assertions that could not have failed.
15///
16/// Two lists rather than one: a diagnostic says a check did not hold, a cover says a check was
17/// never put to the test. Folding the second into the first would make a coverage observation
18/// fail a build, which is the thing #1949's own design forbids.
19pub struct AssertionOutcome {
20    pub diagnostics: Vec<Diagnostic>,
21    pub not_exercised: Vec<cover::AssertionCover>,
22}
23
24/// The failures alone, for callers with no response to hand.
25///
26/// Delegates rather than duplicating: one evaluation, two entry points. `meta` is `Null`, so the
27/// companion cover sees no declared tool list and — by the asymmetry in [`cover`] — reports
28/// nothing. That is the correct answer for a caller that cannot supply the metadata, and it is
29/// why this shortcut is safe to keep for the assertion tests and the published API.
30///
31/// A production caller that has an `LlmResponse` should use [`verify_assertions_with_meta`]; this
32/// one silently has no coverage signal to give.
33pub fn verify_assertions(
34    store: &Store,
35    run_id: i64,
36    test_id: &str,
37    assertions: &[model::TraceAssertion],
38) -> anyhow::Result<Vec<Diagnostic>> {
39    verify_assertions_with_meta(store, run_id, test_id, assertions, &serde_json::Value::Null)
40        .map(|o| o.diagnostics)
41}
42
43pub fn verify_assertions_with_meta(
44    store: &Store,
45    run_id: i64,
46    test_id: &str,
47    assertions: &[model::TraceAssertion],
48    meta: &serde_json::Value,
49) -> anyhow::Result<AssertionOutcome> {
50    let finish = |graph: &EpisodeGraph| -> anyhow::Result<AssertionOutcome> {
51        let tools = cover::ToolAvailability::observe(meta, graph);
52        Ok(AssertionOutcome {
53            diagnostics: matchers::evaluate(graph, assertions)?,
54            not_exercised: cover::evaluate_cover(graph, &tools, assertions),
55        })
56    };
57
58    let graph_res = store.get_episode_graph(run_id, test_id);
59    match graph_res {
60        Ok(graph) => finish(&graph),
61        Err(e) => {
62            // FALLBACK 1: Unit Test Mode (Policy Validation)
63            // If assertions have explicit `test_args`, `test_trace`, etc., we don't need a real episode.
64            // Check if ALL assertions are unit tests.
65            #[expect(
66                clippy::wildcard_enum_match_arm,
67                reason = "an assertion kind with no test-input field cannot be a unit test; a new kind that carries one must be named above or its unit-test form is not recognised"
68            )]
69            let is_unit_test = assertions.iter().all(|a| match a {
70                model::TraceAssertion::ArgsValid { test_args, .. } => test_args.is_some(),
71                model::TraceAssertion::SequenceValid {
72                    test_trace,
73                    test_trace_raw,
74                    ..
75                } => test_trace.is_some() || test_trace_raw.is_some(),
76                model::TraceAssertion::ToolBlocklist {
77                    test_tool_calls, ..
78                } => test_tool_calls.is_some(),
79                _ => false,
80            });
81
82            if is_unit_test {
83                // Construct dummy graph
84                let dummy = EpisodeGraph {
85                    episode_id: "unit_test_mock".into(),
86                    steps: vec![],
87                    tool_calls: vec![],
88                };
89                return finish(&dummy);
90            }
91
92            // FALLBACK 2 (PR-406): If no episode found for this run_id,
93            // try to find the LATEST episode for this test_id regardless of run_id.
94            // This supports the "Demo Flow": Record -> Ingest (Run A) -> Verify (Run B)
95            if e.to_string().contains("E_TRACE_EPISODE_MISSING") {
96                match store.get_latest_episode_graph_by_test_id(test_id) {
97                    Ok(latest_graph) => return finish(&latest_graph),
98                    Err(fallback_err) => {
99                        return Err(anyhow::anyhow!("E_TRACE_EPISODE_MISSING: Primary query failed ({}), Fallback failed: {}", e, fallback_err));
100                    }
101                }
102            }
103
104            // Check if error is ambiguous or missing
105            // For now, return Err to platform, but ideally convert to Diagnostic
106            Err(e)
107        }
108    }
109}
assay_core/agent_assertions/mod.rs

assay_core/agent_assertions/
mod.rs
