Skip to main content

paladin_eval/
scenario.rs

1//! The `.eval.yaml` scenario file format (OBS-04, D-28).
2//!
3//! A scenario file is hand-written, structured input describing one deterministic
4//! evaluation run: which graph to compile, what the mocked LLM says, and what to assert
5//! once the run finishes. [`Scenario::from_path`] parses either YAML (`.yaml`/`.yml`) or
6//! JSON, validating [`EVAL_SCHEMA_VERSION`] before returning so an unrecognised newer
7//! format is a typed error, never a silent misparse (X-04).
8//!
9//! # Safety (T-28-05-01)
10//!
11//! The format carries only structured, deserialized assertion and script parameters --
12//! no shell command, no path that is executed, no dynamically-loaded code. A
13//! `custom(fn)` assertion (the Rust-API-only escape hatch a later plan's assertion
14//! library adds) is never deserialized from a file; this module has no variant for it.
15//!
16//! # A note on hand-rolled (de)serialization for three types (Rule 1 deviation)
17//!
18//! [`Times`], [`ScriptEntry`] and [`Assertion`] implement [`Serialize`]/[`Deserialize`]
19//! by hand instead of `#[derive(Serialize, Deserialize)]`. `serde_yaml` 0.9 (this
20//! workspace's already-pinned, deprecated-upstream version -- the same limitation
21//! `src/application/cli/config/paladin_config.rs:847` documents for a narrower case)
22//! cannot deserialize ANY externally-tagged enum variant that carries data through its
23//! derived `Deserializer::deserialize_enum` path: a data-carrying variant's
24//! conventional single-key-map wire form (`exact: 3`, `text: hello`,
25//! `final_state_field_equals: {field: ..., value: ...}`) fails with `"invalid type:
26//! map, expected a YAML tag starting with '!'"` for EVERY variant shape (newtype of a
27//! primitive, newtype of a string, struct variant) -- confirmed with an isolated
28//! reproduction against this exact dependency version before writing this workaround.
29//! Unit-only enums ([`StoreKind`], [`LlmErrorKind`], [`RunStatusValue`]) are unaffected
30//! (a bare-string variant never reaches the broken code path) and keep the ordinary
31//! derive. The workaround: implement `Deserialize` by hand using
32//! `Deserializer::deserialize_any` + a `Visitor` that reads the single-key map (or, for
33//! [`Assertion::FinalStateSnapshot`], a bare string) via `MapAccess` directly --
34//! `deserialize_any` never invokes `serde_yaml`'s broken `deserialize_enum`, so this
35//! reads correctly under BOTH `serde_yaml` and `serde_json`. `Serialize` is
36//! hand-written alongside it (via `Serializer::serialize_map`) so writing one of these
37//! types back out produces the SAME conventional map form under both formats, rather
38//! than `serde_yaml`'s own derived `Serialize` YAML-tag output (`!exact 3`) that its own
39//! derived `Deserialize` would have required -- internally consistent with itself, but
40//! not what D-28's hand-authored file format or the `schemars`-derived JSON Schema
41//! describe. The `#[serde(...)]` container/variant attributes stay on each type
42//! purely for `schemars::JsonSchema` to read (it parses them independently of whether
43//! `Serialize`/`Deserialize` are actually derived) so the golden schema continues to
44//! describe exactly the shape these manual impls produce and consume.
45
46use std::collections::BTreeMap;
47use std::fmt;
48use std::path::{Path, PathBuf};
49
50use schemars::JsonSchema;
51use serde::de::{self, MapAccess, Visitor};
52use serde::ser::SerializeMap;
53use serde::{Deserialize, Deserializer, Serialize, Serializer};
54use thiserror::Error;
55
56use paladin_ports::output::llm_port::LlmError;
57
58/// The scenario schema version this build of `paladin-eval` understands.
59///
60/// A scenario document whose own `schema_version` does not equal this string fails to
61/// parse with [`ScenarioError::UnsupportedSchemaVersion`] naming the version found,
62/// rather than being silently misinterpreted under the current shape (X-04).
63pub const EVAL_SCHEMA_VERSION: &str = "1";
64
65/// A parsed, validated `.eval.yaml` (or `.eval.json`) scenario document (D-28).
66///
67/// # Example
68///
69/// ```rust
70/// use paladin_eval::scenario::{Scenario, StoreKind};
71///
72/// let yaml = r#"
73/// schema_version: "1"
74/// target:
75///   graph_doc: "fixtures/linear.json"
76/// cases:
77///   - name: happy_path
78///     assertions:
79///       - run_status: completed
80/// "#;
81///
82/// let scenario: Scenario = serde_yaml::from_str(yaml).unwrap();
83/// assert_eq!(scenario.store, StoreKind::InMemory);
84/// assert_eq!(scenario.cases.len(), 1);
85/// ```
86#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
87#[serde(deny_unknown_fields)]
88pub struct Scenario {
89    /// The scenario schema version this document was authored against. Checked against
90    /// [`EVAL_SCHEMA_VERSION`] by [`Scenario::from_path`] and [`Scenario::validate_schema_version`].
91    pub schema_version: String,
92    /// What this scenario runs: a `WarGraphDoc` file or a Rust-registered constructor
93    /// (D-31).
94    pub target: ScenarioTarget,
95    /// Which `WaypointPort`/state store backs the run. Defaults to
96    /// [`StoreKind::InMemory`] when absent.
97    #[serde(default)]
98    pub store: StoreKind,
99    /// The scenario-wide scripted LLM behaviour (global sequence, per-node sequences,
100    /// prompt-match rules). Defaults to an empty script.
101    #[serde(default)]
102    pub llm: LlmScript,
103    /// One or more cases run against [`Scenario::target`], each with its own
104    /// assertions.
105    pub cases: Vec<Case>,
106    /// Live-mode options (D-35): whether content-bearing assertions are allowed to run
107    /// against a real provider instead of being skipped.
108    #[serde(default)]
109    pub live: LiveOptions,
110    /// The name of a host-registered `EngineRegistries` constructor
111    /// (`ScenarioRunner::register_registries`) this scenario's [`ScenarioTarget::GraphDoc`]
112    /// target compiles through, if it needs one beyond the harness default.
113    #[serde(default)]
114    pub registries: Option<String>,
115}
116
117impl Scenario {
118    /// Parse a [`Scenario`] from a file, dispatching on its extension: `.yaml`/`.yml`
119    /// parse as YAML, every other extension (including `.json`) parses as JSON.
120    ///
121    /// Validates [`Scenario::schema_version`] against [`EVAL_SCHEMA_VERSION`] before
122    /// returning -- a document from a newer, incompatible format version is a typed
123    /// [`ScenarioError::UnsupportedSchemaVersion`], never a value silently misread under
124    /// today's shape (X-04).
125    ///
126    /// # Errors
127    ///
128    /// - [`ScenarioError::Io`] if the file cannot be read.
129    /// - [`ScenarioError::Parse`] if the contents do not deserialize.
130    /// - [`ScenarioError::UnsupportedSchemaVersion`] if `schema_version` is not
131    ///   [`EVAL_SCHEMA_VERSION`].
132    pub fn from_path(path: &Path) -> Result<Self, ScenarioError> {
133        let contents = std::fs::read_to_string(path).map_err(|source| ScenarioError::Io {
134            path: path.to_path_buf(),
135            source,
136        })?;
137
138        let is_yaml = matches!(
139            path.extension().and_then(|ext| ext.to_str()),
140            Some("yaml") | Some("yml")
141        );
142
143        let scenario: Scenario = if is_yaml {
144            serde_yaml::from_str(&contents).map_err(|source| ScenarioError::Parse {
145                path: path.to_path_buf(),
146                source: ParseSource::Yaml(source),
147            })?
148        } else {
149            serde_json::from_str(&contents).map_err(|source| ScenarioError::Parse {
150                path: path.to_path_buf(),
151                source: ParseSource::Json(source),
152            })?
153        };
154
155        scenario.validate_schema_version()?;
156        Ok(scenario)
157    }
158
159    /// Confirm [`Scenario::schema_version`] equals [`EVAL_SCHEMA_VERSION`], or return a
160    /// typed [`ScenarioError::UnsupportedSchemaVersion`] naming the version found.
161    ///
162    /// Called by [`Scenario::from_path`]; exposed separately so a caller holding an
163    /// already-deserialized [`Scenario`] (e.g. from an embedded fixture) can run the
164    /// same check without a round trip through the filesystem.
165    ///
166    /// # Errors
167    ///
168    /// [`ScenarioError::UnsupportedSchemaVersion`] if [`Scenario::schema_version`] is
169    /// not [`EVAL_SCHEMA_VERSION`].
170    pub fn validate_schema_version(&self) -> Result<(), ScenarioError> {
171        if self.schema_version != EVAL_SCHEMA_VERSION {
172            return Err(ScenarioError::UnsupportedSchemaVersion {
173                found: self.schema_version.clone(),
174            });
175        }
176        Ok(())
177    }
178}
179
180/// What a [`Scenario`] runs (D-31): a `WarGraphDoc` file on disk, or a Rust closure a
181/// host test binary registered by name.
182///
183/// Untagged so both documented spellings parse directly with no wrapping "type" key --
184/// `{ graph_doc: <path> }` or `{ registered: <name> }` -- unambiguous because the two
185/// variants' field names never overlap.
186#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
187#[serde(untagged)]
188pub enum ScenarioTarget {
189    /// Compile the named `WarGraphDoc` file (JSON or YAML) through the scenario's
190    /// (optional) named registries.
191    GraphDoc {
192        /// Path to the `WarGraphDoc` file, resolved relative to the scenario file's own
193        /// directory by the runner.
194        graph_doc: PathBuf,
195    },
196    /// Build the graph from a Rust closure a host test binary registered under this
197    /// name (`ScenarioRunner::register_graph`).
198    Registered {
199        /// The name a host test binary passed to `ScenarioRunner::register_graph`.
200        registered: String,
201    },
202}
203
204/// Which state store backs a scenario run (D-28). Defaults to [`StoreKind::InMemory`].
205#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
206#[serde(rename_all = "snake_case")]
207pub enum StoreKind {
208    /// An in-memory `WaypointPort`, discarded when the run ends. The default.
209    #[default]
210    InMemory,
211    /// A temp-file SQLite `WaypointPort`, needed by any case that simulates a crash
212    /// (`interrupt_after_superstep`) and resumes over the same store.
213    SqliteTemp,
214}
215
216/// The scenario-wide (or per-case) scripted LLM behaviour (D-28, D-30).
217///
218/// Resolution order, applied per call by `ScenarioLlm`: [`LlmScript::match_rules`]
219/// first (first match wins, does not advance a sequence), then
220/// [`LlmScript::per_node`] for the node the call is attributed to, then
221/// [`LlmScript::global`].
222#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq)]
223#[serde(deny_unknown_fields)]
224pub struct LlmScript {
225    /// The default sequence consulted when no per-node sequence exists for a call's
226    /// node, consumed one entry per call (cycling is `ScenarioLlm`'s decision, not the
227    /// format's).
228    #[serde(default)]
229    pub global: Vec<ScriptEntry>,
230    /// Per-node sequences, keyed by the Paladin node id. A call routed to a node with
231    /// an entry here consumes THIS sequence and leaves [`LlmScript::global`] untouched.
232    #[serde(default)]
233    pub per_node: BTreeMap<String, Vec<ScriptEntry>>,
234    /// Prompt-substring match rules, checked in declaration order before either
235    /// sequence; the wire key is `match` (a Rust keyword, hence the field rename).
236    #[serde(default, rename = "match")]
237    pub match_rules: Vec<MatchRule>,
238}
239
240/// One scripted response to an incoming LLM call (D-28, D-30).
241///
242/// Externally tagged so the three documented spellings are `{ text: "..." }`,
243/// `{ tool_call: { name, arguments } }` and `{ error: <kind> }` -- exactly D-28's `{
244/// text | tool_call: {name, arguments} | error: <LlmError kind> }`.
245///
246/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
247/// hand-rolled (de)serialization" doc section for why.
248#[derive(Debug, Clone, JsonSchema, PartialEq)]
249#[serde(rename_all = "snake_case")]
250pub enum ScriptEntry {
251    /// `text`: a plain success completion.
252    Text(String),
253    /// `tool_call`: a success completion carrying a [`FunctionCall`](paladin_ports::output::llm_port::FunctionCall)-shaped
254    /// tool invocation.
255    ToolCall {
256        /// The tool/function name the mocked model "calls".
257        name: String,
258        /// The JSON-encoded call arguments.
259        arguments: serde_json::Value,
260    },
261    /// `error`: the call returns this [`LlmErrorKind`] instead of a completion.
262    Error(LlmErrorKind),
263}
264
265/// The wire shape of [`ScriptEntry::ToolCall`]'s content, as its own plain struct so it
266/// can be handed to `MapAccess::next_value` (deserializing) and
267/// `SerializeMap::serialize_entry` (serializing) directly. Plain structs are unaffected
268/// by the `serde_yaml` enum-deserialization bug this module's manual impls work around.
269#[derive(Debug, Clone, Serialize, Deserialize)]
270#[serde(deny_unknown_fields)]
271struct ToolCallFields {
272    name: String,
273    arguments: serde_json::Value,
274}
275
276const SCRIPT_ENTRY_VARIANTS: &[&str] = &["text", "tool_call", "error"];
277
278impl Serialize for ScriptEntry {
279    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
280    where
281        S: Serializer,
282    {
283        let mut map = serializer.serialize_map(Some(1))?;
284        match self {
285            ScriptEntry::Text(text) => map.serialize_entry("text", text)?,
286            ScriptEntry::ToolCall { name, arguments } => map.serialize_entry(
287                "tool_call",
288                &ToolCallFields {
289                    name: name.clone(),
290                    arguments: arguments.clone(),
291                },
292            )?,
293            ScriptEntry::Error(kind) => map.serialize_entry("error", kind)?,
294        }
295        map.end()
296    }
297}
298
299impl<'de> Deserialize<'de> for ScriptEntry {
300    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
301    where
302        D: Deserializer<'de>,
303    {
304        struct ScriptEntryVisitor;
305
306        impl<'de> Visitor<'de> for ScriptEntryVisitor {
307            type Value = ScriptEntry;
308
309            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310                write!(f, "a single-key map: `text`, `tool_call` or `error`")
311            }
312
313            fn visit_map<A>(self, mut map: A) -> Result<ScriptEntry, A::Error>
314            where
315                A: MapAccess<'de>,
316            {
317                let key: String = map.next_key()?.ok_or_else(|| {
318                    de::Error::custom("expected exactly one key (`text`, `tool_call` or `error`)")
319                })?;
320                let entry = match key.as_str() {
321                    "text" => ScriptEntry::Text(map.next_value()?),
322                    "tool_call" => {
323                        let fields: ToolCallFields = map.next_value()?;
324                        ScriptEntry::ToolCall {
325                            name: fields.name,
326                            arguments: fields.arguments,
327                        }
328                    }
329                    "error" => ScriptEntry::Error(map.next_value()?),
330                    other => {
331                        return Err(de::Error::unknown_variant(other, SCRIPT_ENTRY_VARIANTS));
332                    }
333                };
334                Ok(entry)
335            }
336        }
337
338        deserializer.deserialize_any(ScriptEntryVisitor)
339    }
340}
341
342/// A closed vocabulary of [`paladin_ports::output::llm_port::LlmError`] shapes a
343/// scenario file can script (D-28, D-30).
344///
345/// [`LlmError`] itself does not implement [`Deserialize`], so this mirrors its
346/// vocabulary for the file format the same way `WarGraphDoc`'s `NodeKindDoc` mirrors
347/// `NodeSpec` -- [`LlmErrorKind::to_llm_error`] is the one place a scripted kind becomes
348/// a real [`LlmError`].
349///
350/// `#[non_exhaustive]`: a future `LlmError` variant may need its own scriptable kind.
351#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
352#[serde(rename_all = "snake_case")]
353#[non_exhaustive]
354pub enum LlmErrorKind {
355    /// Maps to [`LlmError::NetworkError`].
356    Network,
357    /// Maps to [`LlmError::AuthenticationError`].
358    Authentication,
359    /// Maps to [`LlmError::InvalidPrompt`].
360    InvalidPrompt,
361    /// Maps to [`LlmError::RateLimitExceeded`].
362    RateLimit,
363    /// Maps to [`LlmError::UsageLimitExceeded`].
364    UsageLimit,
365    /// Maps to [`LlmError::ModelNotAvailable`].
366    ModelNotAvailable,
367    /// Maps to [`LlmError::TokenLimitExceeded`].
368    TokenLimit,
369    /// Maps to [`LlmError::EmptyCompletion`].
370    EmptyCompletion,
371    /// Maps to [`LlmError::ProcessingError`].
372    Processing,
373    /// Maps to [`LlmError::Timeout`].
374    Timeout,
375    /// A generic transient failure (maps to [`LlmError::NetworkError`]) for a scenario
376    /// that only cares about the retry/fallback behaviour a transient error drives, not
377    /// which specific variant produced it.
378    Transient,
379}
380
381impl LlmErrorKind {
382    /// Build the concrete [`LlmError`] this scripted kind represents.
383    ///
384    /// `node` and `call_index` are folded into the error's own message so a failure
385    /// rendered from a captured request log can point at exactly which scripted call
386    /// produced it.
387    ///
388    /// # Examples
389    ///
390    /// ```rust
391    /// use paladin_eval::scenario::LlmErrorKind;
392    ///
393    /// let error = LlmErrorKind::RateLimit.to_llm_error("planner", 2);
394    /// assert!(error.to_string().contains("Rate limit"));
395    /// ```
396    pub fn to_llm_error(
397        &self,
398        node: &str,
399        call_index: usize,
400    ) -> paladin_ports::output::llm_port::LlmError {
401        match self {
402            LlmErrorKind::Network => LlmError::NetworkError(format!(
403                "scripted network error (node {node}, call {call_index})"
404            )),
405            LlmErrorKind::Authentication => LlmError::AuthenticationError(format!(
406                "scripted authentication error (node {node}, call {call_index})"
407            )),
408            LlmErrorKind::InvalidPrompt => LlmError::InvalidPrompt(format!(
409                "scripted invalid prompt (node {node}, call {call_index})"
410            )),
411            LlmErrorKind::RateLimit => LlmError::rate_limited(None),
412            LlmErrorKind::UsageLimit => LlmError::UsageLimitExceeded {
413                provider: "scenario".to_string(),
414                regain_hint: None,
415            },
416            LlmErrorKind::ModelNotAvailable => LlmError::ModelNotAvailable(format!(
417                "scripted model-not-available (node {node}, call {call_index})"
418            )),
419            LlmErrorKind::TokenLimit => LlmError::TokenLimitExceeded,
420            LlmErrorKind::EmptyCompletion => LlmError::empty_completion(format!(
421                "scripted empty completion (node {node}, call {call_index})"
422            )),
423            LlmErrorKind::Processing => LlmError::ProcessingError(format!(
424                "scripted processing error (node {node}, call {call_index})"
425            )),
426            LlmErrorKind::Timeout => {
427                LlmError::Timeout(format!("scripted timeout (node {node}, call {call_index})"))
428            }
429            LlmErrorKind::Transient => LlmError::NetworkError(format!(
430                "scripted transient error (node {node}, call {call_index})"
431            )),
432        }
433    }
434}
435
436/// A prompt-substring match rule (D-28), checked before either script sequence.
437#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
438#[serde(deny_unknown_fields)]
439pub struct MatchRule {
440    /// If the incoming prompt contains this substring, [`MatchRule::response`] is
441    /// returned and neither sequence is advanced.
442    pub prompt_contains: String,
443    /// The response returned when [`MatchRule::prompt_contains`] matches.
444    pub response: ScriptEntry,
445}
446
447/// One test case within a [`Scenario`] (D-28, D-34).
448#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
449#[serde(deny_unknown_fields)]
450pub struct Case {
451    /// This case's name, used in `libtest-mimic` trial naming and failure output.
452    pub name: String,
453    /// Initial input fields, by name, handed to the run.
454    #[serde(default)]
455    pub input: BTreeMap<String, serde_json::Value>,
456    /// Simulate a crash after this superstep completes: the runner drops the engine
457    /// and builds a fresh one over the same [`StoreKind::SqliteTemp`] store to resume
458    /// (D-34's E2E-1 technique). Requires `store: sqlite_temp`.
459    #[serde(default)]
460    pub interrupt_after_superstep: Option<u64>,
461    /// Scripted Parley responses, consumed in order whenever the run raises a Parley
462    /// (D-34's E2E-2 technique).
463    #[serde(default)]
464    pub parley_responses: Vec<ParleyScript>,
465    /// A case-specific LLM script overriding [`Scenario::llm`] for this case only.
466    #[serde(default)]
467    pub llm: Option<LlmScript>,
468    /// The assertions evaluated against the captured trace records and the final
469    /// `Battlefield` once this case's run finishes.
470    pub assertions: Vec<Assertion>,
471}
472
473/// One scripted response to a Parley the run raises (D-28, D-34).
474#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
475#[serde(deny_unknown_fields)]
476pub struct ParleyScript {
477    /// The Parley kind this response answers (mirrors
478    /// `paladin_core::platform::container::parley::ParleyKind`'s wire vocabulary).
479    pub kind: String,
480    /// The value handed back as the Parley's resolution.
481    pub value: serde_json::Value,
482}
483
484/// Live-mode options (D-35): whether a scenario opts in to running content-bearing
485/// assertions against a real provider instead of skipping them.
486#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
487#[serde(deny_unknown_fields)]
488pub struct LiveOptions {
489    /// When `true` and the runner is invoked with `--live`, content-bearing assertions
490    /// (`final_state_field_equals`, `final_state_field_matches`, `field_json_path_equals`,
491    /// `final_state_snapshot`) run instead of being skipped. Defaults to `false`.
492    #[serde(default)]
493    pub allow_content_assertions: bool,
494}
495
496/// A bound on how many times [`Assertion::NodeExecuted`] expects its node to have run.
497///
498/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
499/// hand-rolled (de)serialization" doc section for why.
500#[derive(Debug, Clone, Copy, JsonSchema, PartialEq, Eq)]
501#[serde(rename_all = "snake_case")]
502pub enum Times {
503    /// The node ran exactly this many times.
504    Exact(u32),
505    /// The node ran at least this many times.
506    Min(u32),
507    /// The node ran at most this many times.
508    Max(u32),
509}
510
511const TIMES_VARIANTS: &[&str] = &["exact", "min", "max"];
512
513impl Serialize for Times {
514    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
515    where
516        S: Serializer,
517    {
518        let mut map = serializer.serialize_map(Some(1))?;
519        match self {
520            Times::Exact(v) => map.serialize_entry("exact", v)?,
521            Times::Min(v) => map.serialize_entry("min", v)?,
522            Times::Max(v) => map.serialize_entry("max", v)?,
523        }
524        map.end()
525    }
526}
527
528impl<'de> Deserialize<'de> for Times {
529    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
530    where
531        D: Deserializer<'de>,
532    {
533        struct TimesVisitor;
534
535        impl<'de> Visitor<'de> for TimesVisitor {
536            type Value = Times;
537
538            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
539                write!(f, "a single-key map: `exact`, `min` or `max`")
540            }
541
542            fn visit_map<A>(self, mut map: A) -> Result<Times, A::Error>
543            where
544                A: MapAccess<'de>,
545            {
546                let key: String = map.next_key()?.ok_or_else(|| {
547                    de::Error::custom("expected exactly one key (`exact`, `min` or `max`)")
548                })?;
549                let times = match key.as_str() {
550                    "exact" => Times::Exact(map.next_value()?),
551                    "min" => Times::Min(map.next_value()?),
552                    "max" => Times::Max(map.next_value()?),
553                    other => return Err(de::Error::unknown_variant(other, TIMES_VARIANTS)),
554                };
555                Ok(times)
556            }
557        }
558
559        deserializer.deserialize_any(TimesVisitor)
560    }
561}
562
563/// The vocabulary [`Assertion::RunStatus`] compares against.
564///
565/// Mirrors every status of `paladin_core::platform::container::trace::RunFinishStatus`
566/// except `Cancelled`.
567/// That core type carries no `schemars::JsonSchema` derive (ADR-0015: `paladin-core`
568/// does not take the `schemars` dependency), so this crate mirrors the vocabulary
569/// locally -- exactly the pattern `WarGraphDoc`'s `NodeKindDoc` uses for `NodeSpec`.
570///
571/// A run that finished `cancelled` matches no `run_status` value until the next minor,
572/// which adds `Cancelled` together with a schema refresh. A variant cannot be added in
573/// v0.11.1, because this enum and [`Assertion`] are exhaustive public enums of a
574/// published crate and the patch-level semver gate treats either change as major
575/// (FIXES-04, re-homed).
576#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
577#[serde(rename_all = "snake_case")]
578pub enum RunStatusValue {
579    /// The run completed normally.
580    Completed,
581    /// The run failed.
582    Failed,
583    /// The run was halted.
584    Halted,
585    /// The run suspended awaiting Parley input.
586    AwaitingInput,
587}
588
589/// One assertion evaluated against a case's captured trace records and final
590/// `Battlefield` (D-28, D-29, PRD 07 OBS-FR-12).
591///
592/// A closed, externally tagged enum -- an unrecognised tag is a deserialize error
593/// naming the tag, never a silently-ignored no-op (X-04's "closed, not silently
594/// permissive" rule). `custom(fn)` (a later plan's Rust-API-only escape hatch) has no
595/// variant here and can never be deserialized from a file (T-28-05-01).
596///
597/// Every variant name below is PRD 07 OBS-FR-12's verbatim, snake-cased wire name.
598///
599/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
600/// hand-rolled (de)serialization" doc section for why.
601#[derive(Debug, Clone, JsonSchema, PartialEq)]
602#[serde(rename_all = "snake_case", deny_unknown_fields)]
603pub enum Assertion {
604    /// `final_state_field_equals`: the named final-`Battlefield` field equals `value`
605    /// exactly.
606    FinalStateFieldEquals {
607        /// The `Battlefield` field name.
608        field: String,
609        /// The expected value.
610        value: serde_json::Value,
611    },
612    /// `final_state_field_matches`: the named final-`Battlefield` field, rendered as a
613    /// string, matches the `pattern` regular expression.
614    FinalStateFieldMatches {
615        /// The `Battlefield` field name.
616        field: String,
617        /// The regular expression the field's rendered value must match.
618        pattern: String,
619    },
620    /// `field_json_path_equals`: the value at `path` (a JSON Pointer / JSONPath-style
621    /// path into the final `Battlefield`) equals `value`.
622    FieldJsonPathEquals {
623        /// The path into the final `Battlefield`'s JSON representation.
624        path: String,
625        /// The expected value at `path`.
626        value: serde_json::Value,
627    },
628    /// `node_executed`: the named node ran the number of times `times` bounds.
629    NodeExecuted {
630        /// The node id.
631        node: String,
632        /// The execution-count bound.
633        times: Times,
634    },
635    /// `node_not_executed`: the named node never ran.
636    NodeNotExecuted {
637        /// The node id.
638        node: String,
639    },
640    /// `edge_fired`: the edge `from -> to` fired at least once (needs `EdgeEvaluated`
641    /// records, D-04).
642    EdgeFired {
643        /// The source node id.
644        from: String,
645        /// The destination node id.
646        to: String,
647    },
648    /// `route_taken`: the sequence of node ids in `NodeStarted` order contains this
649    /// list as a subsequence.
650    RouteTaken(Vec<String>),
651    /// `run_status`: the run's terminal status equals this value. A cancelled run matches
652    /// no value (see [`RunStatusValue`]).
653    RunStatus(RunStatusValue),
654    /// `total_tokens_max`: the run's total token usage does not exceed this bound.
655    TotalTokensMax(u64),
656    /// `supersteps_max`: the run's total superstep count does not exceed this bound.
657    SuperstepsMax(u64),
658    /// `parley_raised`: a Parley of `kind` was raised by `node`.
659    ParleyRaised {
660        /// The Parley kind (mirrors `ParleyKind`'s wire vocabulary).
661        kind: String,
662        /// The node that raised it.
663        node: String,
664    },
665    /// `final_state_snapshot`: the final `Battlefield` matches a blessed snapshot file
666    /// (`--bless` regenerates it, D-33).
667    FinalStateSnapshot,
668}
669
670// ---------------------------------------------------------------------------
671// Assertion: hand-rolled Serialize/Deserialize (see this module's top-level doc note).
672// ---------------------------------------------------------------------------
673
674/// Wire shape of [`Assertion::FinalStateFieldEquals`]'s content.
675#[derive(Debug, Clone, Serialize, Deserialize)]
676#[serde(deny_unknown_fields)]
677struct FieldEqualsFields {
678    field: String,
679    value: serde_json::Value,
680}
681
682/// Wire shape of [`Assertion::FinalStateFieldMatches`]'s content.
683#[derive(Debug, Clone, Serialize, Deserialize)]
684#[serde(deny_unknown_fields)]
685struct FieldMatchesFields {
686    field: String,
687    pattern: String,
688}
689
690/// Wire shape of [`Assertion::FieldJsonPathEquals`]'s content.
691#[derive(Debug, Clone, Serialize, Deserialize)]
692#[serde(deny_unknown_fields)]
693struct JsonPathFields {
694    path: String,
695    value: serde_json::Value,
696}
697
698/// Wire shape of [`Assertion::NodeExecuted`]'s content.
699#[derive(Debug, Clone, Serialize, Deserialize)]
700#[serde(deny_unknown_fields)]
701struct NodeExecutedFields {
702    node: String,
703    times: Times,
704}
705
706/// Wire shape of [`Assertion::NodeNotExecuted`]'s content.
707#[derive(Debug, Clone, Serialize, Deserialize)]
708#[serde(deny_unknown_fields)]
709struct NodeOnlyFields {
710    node: String,
711}
712
713/// Wire shape of [`Assertion::EdgeFired`]'s content.
714#[derive(Debug, Clone, Serialize, Deserialize)]
715#[serde(deny_unknown_fields)]
716struct EdgeFields {
717    from: String,
718    to: String,
719}
720
721/// Wire shape of [`Assertion::ParleyRaised`]'s content.
722#[derive(Debug, Clone, Serialize, Deserialize)]
723#[serde(deny_unknown_fields)]
724struct ParleyRaisedFields {
725    kind: String,
726    node: String,
727}
728
729const ASSERTION_VARIANTS: &[&str] = &[
730    "final_state_field_equals",
731    "final_state_field_matches",
732    "field_json_path_equals",
733    "node_executed",
734    "node_not_executed",
735    "edge_fired",
736    "route_taken",
737    "run_status",
738    "total_tokens_max",
739    "supersteps_max",
740    "parley_raised",
741    "final_state_snapshot",
742];
743
744impl Serialize for Assertion {
745    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
746    where
747        S: Serializer,
748    {
749        // `final_state_snapshot` is the one unit variant -- a bare string, not a
750        // single-key map, matching the default external-tagging representation a
751        // derived impl would also have produced for a unit variant.
752        if let Assertion::FinalStateSnapshot = self {
753            return serializer.serialize_str("final_state_snapshot");
754        }
755
756        let mut map = serializer.serialize_map(Some(1))?;
757        match self {
758            Assertion::FinalStateFieldEquals { field, value } => map.serialize_entry(
759                "final_state_field_equals",
760                &FieldEqualsFields {
761                    field: field.clone(),
762                    value: value.clone(),
763                },
764            )?,
765            Assertion::FinalStateFieldMatches { field, pattern } => map.serialize_entry(
766                "final_state_field_matches",
767                &FieldMatchesFields {
768                    field: field.clone(),
769                    pattern: pattern.clone(),
770                },
771            )?,
772            Assertion::FieldJsonPathEquals { path, value } => map.serialize_entry(
773                "field_json_path_equals",
774                &JsonPathFields {
775                    path: path.clone(),
776                    value: value.clone(),
777                },
778            )?,
779            Assertion::NodeExecuted { node, times } => map.serialize_entry(
780                "node_executed",
781                &NodeExecutedFields {
782                    node: node.clone(),
783                    times: *times,
784                },
785            )?,
786            Assertion::NodeNotExecuted { node } => {
787                map.serialize_entry("node_not_executed", &NodeOnlyFields { node: node.clone() })?
788            }
789            Assertion::EdgeFired { from, to } => map.serialize_entry(
790                "edge_fired",
791                &EdgeFields {
792                    from: from.clone(),
793                    to: to.clone(),
794                },
795            )?,
796            Assertion::RouteTaken(nodes) => map.serialize_entry("route_taken", nodes)?,
797            Assertion::RunStatus(status) => map.serialize_entry("run_status", status)?,
798            Assertion::TotalTokensMax(v) => map.serialize_entry("total_tokens_max", v)?,
799            Assertion::SuperstepsMax(v) => map.serialize_entry("supersteps_max", v)?,
800            Assertion::ParleyRaised { kind, node } => map.serialize_entry(
801                "parley_raised",
802                &ParleyRaisedFields {
803                    kind: kind.clone(),
804                    node: node.clone(),
805                },
806            )?,
807            Assertion::FinalStateSnapshot => unreachable!("handled by the early return above"),
808        }
809        map.end()
810    }
811}
812
813impl<'de> Deserialize<'de> for Assertion {
814    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
815    where
816        D: Deserializer<'de>,
817    {
818        struct AssertionVisitor;
819
820        impl<'de> Visitor<'de> for AssertionVisitor {
821            type Value = Assertion;
822
823            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
824                write!(
825                    f,
826                    "`final_state_snapshot` or a single-key map naming one of the other eleven assertion kinds"
827                )
828            }
829
830            fn visit_str<E>(self, v: &str) -> Result<Assertion, E>
831            where
832                E: de::Error,
833            {
834                match v {
835                    "final_state_snapshot" => Ok(Assertion::FinalStateSnapshot),
836                    other => Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
837                }
838            }
839
840            fn visit_map<A>(self, mut map: A) -> Result<Assertion, A::Error>
841            where
842                A: MapAccess<'de>,
843            {
844                let key: String = map.next_key()?.ok_or_else(|| {
845                    de::Error::custom("expected exactly one key naming the assertion kind")
846                })?;
847                let assertion = match key.as_str() {
848                    "final_state_field_equals" => {
849                        let fields: FieldEqualsFields = map.next_value()?;
850                        Assertion::FinalStateFieldEquals {
851                            field: fields.field,
852                            value: fields.value,
853                        }
854                    }
855                    "final_state_field_matches" => {
856                        let fields: FieldMatchesFields = map.next_value()?;
857                        Assertion::FinalStateFieldMatches {
858                            field: fields.field,
859                            pattern: fields.pattern,
860                        }
861                    }
862                    "field_json_path_equals" => {
863                        let fields: JsonPathFields = map.next_value()?;
864                        Assertion::FieldJsonPathEquals {
865                            path: fields.path,
866                            value: fields.value,
867                        }
868                    }
869                    "node_executed" => {
870                        let fields: NodeExecutedFields = map.next_value()?;
871                        Assertion::NodeExecuted {
872                            node: fields.node,
873                            times: fields.times,
874                        }
875                    }
876                    "node_not_executed" => {
877                        let fields: NodeOnlyFields = map.next_value()?;
878                        Assertion::NodeNotExecuted { node: fields.node }
879                    }
880                    "edge_fired" => {
881                        let fields: EdgeFields = map.next_value()?;
882                        Assertion::EdgeFired {
883                            from: fields.from,
884                            to: fields.to,
885                        }
886                    }
887                    "route_taken" => Assertion::RouteTaken(map.next_value()?),
888                    "run_status" => Assertion::RunStatus(map.next_value()?),
889                    "total_tokens_max" => Assertion::TotalTokensMax(map.next_value()?),
890                    "supersteps_max" => Assertion::SuperstepsMax(map.next_value()?),
891                    "parley_raised" => {
892                        let fields: ParleyRaisedFields = map.next_value()?;
893                        Assertion::ParleyRaised {
894                            kind: fields.kind,
895                            node: fields.node,
896                        }
897                    }
898                    "final_state_snapshot" => {
899                        // Tolerate `{ final_state_snapshot: null }` / `{
900                        // final_state_snapshot: {} }` alongside the bare-string form
901                        // `visit_str` handles -- either spelling of the unit variant
902                        // parses.
903                        let _: serde::de::IgnoredAny = map.next_value()?;
904                        Assertion::FinalStateSnapshot
905                    }
906                    other => return Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
907                };
908                Ok(assertion)
909            }
910        }
911
912        deserializer.deserialize_any(AssertionVisitor)
913    }
914}
915
916/// The underlying parse failure wrapped by [`ScenarioError::Parse`] -- either a YAML or
917/// a JSON deserialize error, depending on which format [`Scenario::from_path`]
918/// dispatched to.
919#[derive(Debug, Error)]
920pub enum ParseSource {
921    /// A YAML deserialize failure (`.yaml`/`.yml` files).
922    #[error(transparent)]
923    Yaml(#[from] serde_yaml::Error),
924    /// A JSON deserialize failure (every other extension).
925    #[error(transparent)]
926    Json(#[from] serde_json::Error),
927}
928
929/// Errors [`Scenario::from_path`] and [`Scenario::validate_schema_version`] can return.
930///
931/// `#[non_exhaustive]`: a future variant (e.g. a semantic validation failure beyond
932/// `schema_version`) can be added without a semver-major bump.
933#[derive(Debug, Error)]
934#[non_exhaustive]
935pub enum ScenarioError {
936    /// The document's `schema_version` is not [`EVAL_SCHEMA_VERSION`].
937    #[error(
938        "unsupported scenario schema_version {found:?}; this build of paladin-eval understands {EVAL_SCHEMA_VERSION:?}"
939    )]
940    UnsupportedSchemaVersion {
941        /// The `schema_version` string found in the document.
942        found: String,
943    },
944    /// The document's contents did not deserialize.
945    #[error("failed to parse scenario file {path}: {source}", path = path.display())]
946    Parse {
947        /// The file that failed to parse.
948        path: PathBuf,
949        /// The underlying YAML or JSON deserialize error.
950        #[source]
951        source: ParseSource,
952    },
953    /// The scenario file could not be read from disk.
954    #[error("failed to read scenario file {path}: {source}", path = path.display())]
955    Io {
956        /// The file that could not be read.
957        path: PathBuf,
958        /// The underlying I/O error.
959        #[source]
960        source: std::io::Error,
961    },
962}
963
964#[cfg(test)]
965mod tests {
966    use super::*;
967
968    fn minimal_yaml() -> &'static str {
969        r#"
970schema_version: "1"
971target:
972  graph_doc: "fixtures/linear.json"
973cases:
974  - name: happy_path
975    input:
976      foo: "bar"
977    assertions:
978      - final_state_field_equals:
979          field: "status"
980          value: "done"
981"#
982    }
983
984    #[test]
985    fn minimal_scenario_parses() {
986        let scenario: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses");
987        assert_eq!(scenario.schema_version, "1");
988        assert_eq!(
989            scenario.target,
990            ScenarioTarget::GraphDoc {
991                graph_doc: PathBuf::from("fixtures/linear.json"),
992            }
993        );
994        assert_eq!(scenario.store, StoreKind::InMemory);
995        assert_eq!(scenario.llm, LlmScript::default());
996        assert_eq!(scenario.cases.len(), 1);
997        assert_eq!(scenario.cases[0].name, "happy_path");
998        assert_eq!(scenario.cases[0].assertions.len(), 1);
999    }
1000
1001    #[test]
1002    fn json_and_yaml_agree() {
1003        let from_yaml: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses yaml");
1004
1005        let json = serde_json::json!({
1006            "schema_version": "1",
1007            "target": { "graph_doc": "fixtures/linear.json" },
1008            "cases": [
1009                {
1010                    "name": "happy_path",
1011                    "input": { "foo": "bar" },
1012                    "assertions": [
1013                        { "final_state_field_equals": { "field": "status", "value": "done" } }
1014                    ]
1015                }
1016            ]
1017        });
1018        let from_json: Scenario =
1019            serde_json::from_value(json).expect("parses the equivalent json document");
1020
1021        assert_eq!(from_yaml, from_json);
1022    }
1023
1024    #[test]
1025    fn registered_target_parses() {
1026        let yaml = r#"
1027schema_version: "1"
1028target:
1029  registered: "e2e1"
1030cases: []
1031"#;
1032        let scenario: Scenario = serde_yaml::from_str(yaml).expect("parses");
1033        assert_eq!(
1034            scenario.target,
1035            ScenarioTarget::Registered {
1036                registered: "e2e1".to_string(),
1037            }
1038        );
1039    }
1040
1041    #[test]
1042    fn unknown_schema_version_is_typed() {
1043        let yaml = r#"
1044schema_version: "2"
1045target:
1046  graph_doc: "fixtures/linear.json"
1047cases: []
1048"#;
1049        let scenario: Scenario = serde_yaml::from_str(yaml).expect("still deserializes");
1050        let result = scenario.validate_schema_version();
1051        match result {
1052            Err(ScenarioError::UnsupportedSchemaVersion { found }) => {
1053                assert_eq!(found, "2");
1054            }
1055            other => panic!("expected UnsupportedSchemaVersion, got {other:?}"),
1056        }
1057    }
1058
1059    #[test]
1060    fn unknown_assertion_kind_is_an_error() {
1061        let result: Result<Assertion, _> = serde_yaml::from_str("made_up_kind: {}");
1062        let err = result.expect_err("an unrecognised assertion tag must fail to deserialize");
1063        let message = err.to_string();
1064        assert!(
1065            message.contains("made_up_kind"),
1066            "error message {message:?} does not name the unrecognised tag"
1067        );
1068    }
1069
1070    #[test]
1071    fn times_variants_round_trip() {
1072        for (yaml, expected) in [
1073            ("exact: 3", Times::Exact(3)),
1074            ("min: 1", Times::Min(1)),
1075            ("max: 5", Times::Max(5)),
1076        ] {
1077            let parsed: Times = serde_yaml::from_str(yaml).expect("parses");
1078            assert_eq!(parsed, expected);
1079        }
1080    }
1081
1082    #[test]
1083    fn script_entry_spellings_round_trip() {
1084        let text: ScriptEntry = serde_yaml::from_str("text: hello").unwrap();
1085        assert_eq!(text, ScriptEntry::Text("hello".to_string()));
1086
1087        let tool_call: ScriptEntry =
1088            serde_yaml::from_str("tool_call:\n  name: search\n  arguments: {}").unwrap();
1089        assert_eq!(
1090            tool_call,
1091            ScriptEntry::ToolCall {
1092                name: "search".to_string(),
1093                arguments: serde_json::json!({}),
1094            }
1095        );
1096
1097        let error: ScriptEntry = serde_yaml::from_str("error: transient").unwrap();
1098        assert_eq!(error, ScriptEntry::Error(LlmErrorKind::Transient));
1099    }
1100}