paladin_eval/scenario.rs
1//! The `.eval.yaml` scenario file format (OBS-04, D-28).
2//!
3//! A scenario file is hand-written, structured input describing one deterministic
4//! evaluation run: which graph to compile, what the mocked LLM says, and what to assert
5//! once the run finishes. [`Scenario::from_path`] parses either YAML (`.yaml`/`.yml`) or
6//! JSON, validating [`EVAL_SCHEMA_VERSION`] before returning so an unrecognised newer
7//! format is a typed error, never a silent misparse (X-04).
8//!
9//! # Safety (T-28-05-01)
10//!
11//! The format carries only structured, deserialized assertion and script parameters --
12//! no shell command, no path that is executed, no dynamically-loaded code. A
13//! `custom(fn)` assertion (the Rust-API-only escape hatch a later plan's assertion
14//! library adds) is never deserialized from a file; this module has no variant for it.
15//!
16//! # A note on hand-rolled (de)serialization for three types (Rule 1 deviation)
17//!
18//! [`Times`], [`ScriptEntry`] and [`Assertion`] implement [`Serialize`]/[`Deserialize`]
19//! by hand instead of `#[derive(Serialize, Deserialize)]`. `serde_yaml` 0.9 (this
20//! workspace's already-pinned, deprecated-upstream version -- the same limitation
21//! `src/application/cli/config/paladin_config.rs:847` documents for a narrower case)
22//! cannot deserialize ANY externally-tagged enum variant that carries data through its
23//! derived `Deserializer::deserialize_enum` path: a data-carrying variant's
24//! conventional single-key-map wire form (`exact: 3`, `text: hello`,
25//! `final_state_field_equals: {field: ..., value: ...}`) fails with `"invalid type:
26//! map, expected a YAML tag starting with '!'"` for EVERY variant shape (newtype of a
27//! primitive, newtype of a string, struct variant) -- confirmed with an isolated
28//! reproduction against this exact dependency version before writing this workaround.
29//! Unit-only enums ([`StoreKind`], [`LlmErrorKind`], [`RunStatusValue`]) are unaffected
30//! (a bare-string variant never reaches the broken code path) and keep the ordinary
31//! derive. The workaround: implement `Deserialize` by hand using
32//! `Deserializer::deserialize_any` + a `Visitor` that reads the single-key map (or, for
33//! [`Assertion::FinalStateSnapshot`], a bare string) via `MapAccess` directly --
34//! `deserialize_any` never invokes `serde_yaml`'s broken `deserialize_enum`, so this
35//! reads correctly under BOTH `serde_yaml` and `serde_json`. `Serialize` is
36//! hand-written alongside it (via `Serializer::serialize_map`) so writing one of these
37//! types back out produces the SAME conventional map form under both formats, rather
38//! than `serde_yaml`'s own derived `Serialize` YAML-tag output (`!exact 3`) that its own
39//! derived `Deserialize` would have required -- internally consistent with itself, but
40//! not what D-28's hand-authored file format or the `schemars`-derived JSON Schema
41//! describe. The `#[serde(...)]` container/variant attributes stay on each type
42//! purely for `schemars::JsonSchema` to read (it parses them independently of whether
43//! `Serialize`/`Deserialize` are actually derived) so the golden schema continues to
44//! describe exactly the shape these manual impls produce and consume.
45
46use std::collections::BTreeMap;
47use std::fmt;
48use std::path::{Path, PathBuf};
49
50use schemars::JsonSchema;
51use serde::de::{self, MapAccess, Visitor};
52use serde::ser::SerializeMap;
53use serde::{Deserialize, Deserializer, Serialize, Serializer};
54use thiserror::Error;
55
56use paladin_ports::output::llm_port::LlmError;
57
58/// The scenario schema version this build of `paladin-eval` understands.
59///
60/// A scenario document whose own `schema_version` does not equal this string fails to
61/// parse with [`ScenarioError::UnsupportedSchemaVersion`] naming the version found,
62/// rather than being silently misinterpreted under the current shape (X-04).
63pub const EVAL_SCHEMA_VERSION: &str = "1";
64
65/// A parsed, validated `.eval.yaml` (or `.eval.json`) scenario document (D-28).
66///
67/// # Example
68///
69/// ```rust
70/// use paladin_eval::scenario::{Scenario, StoreKind};
71///
72/// let yaml = r#"
73/// schema_version: "1"
74/// target:
75/// graph_doc: "fixtures/linear.json"
76/// cases:
77/// - name: happy_path
78/// assertions:
79/// - run_status: completed
80/// "#;
81///
82/// let scenario: Scenario = serde_yaml::from_str(yaml).unwrap();
83/// assert_eq!(scenario.store, StoreKind::InMemory);
84/// assert_eq!(scenario.cases.len(), 1);
85/// ```
86#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
87#[serde(deny_unknown_fields)]
88pub struct Scenario {
89 /// The scenario schema version this document was authored against. Checked against
90 /// [`EVAL_SCHEMA_VERSION`] by [`Scenario::from_path`] and [`Scenario::validate_schema_version`].
91 pub schema_version: String,
92 /// What this scenario runs: a `WarGraphDoc` file or a Rust-registered constructor
93 /// (D-31).
94 pub target: ScenarioTarget,
95 /// Which `WaypointPort`/state store backs the run. Defaults to
96 /// [`StoreKind::InMemory`] when absent.
97 #[serde(default)]
98 pub store: StoreKind,
99 /// The scenario-wide scripted LLM behaviour (global sequence, per-node sequences,
100 /// prompt-match rules). Defaults to an empty script.
101 #[serde(default)]
102 pub llm: LlmScript,
103 /// One or more cases run against [`Scenario::target`], each with its own
104 /// assertions.
105 pub cases: Vec<Case>,
106 /// Live-mode options (D-35): whether content-bearing assertions are allowed to run
107 /// against a real provider instead of being skipped.
108 #[serde(default)]
109 pub live: LiveOptions,
110 /// The name of a host-registered `EngineRegistries` constructor
111 /// (`ScenarioRunner::register_registries`) this scenario's [`ScenarioTarget::GraphDoc`]
112 /// target compiles through, if it needs one beyond the harness default.
113 #[serde(default)]
114 pub registries: Option<String>,
115}
116
117impl Scenario {
118 /// Parse a [`Scenario`] from a file, dispatching on its extension: `.yaml`/`.yml`
119 /// parse as YAML, every other extension (including `.json`) parses as JSON.
120 ///
121 /// Validates [`Scenario::schema_version`] against [`EVAL_SCHEMA_VERSION`] before
122 /// returning -- a document from a newer, incompatible format version is a typed
123 /// [`ScenarioError::UnsupportedSchemaVersion`], never a value silently misread under
124 /// today's shape (X-04).
125 ///
126 /// # Errors
127 ///
128 /// - [`ScenarioError::Io`] if the file cannot be read.
129 /// - [`ScenarioError::Parse`] if the contents do not deserialize.
130 /// - [`ScenarioError::UnsupportedSchemaVersion`] if `schema_version` is not
131 /// [`EVAL_SCHEMA_VERSION`].
132 pub fn from_path(path: &Path) -> Result<Self, ScenarioError> {
133 let contents = std::fs::read_to_string(path).map_err(|source| ScenarioError::Io {
134 path: path.to_path_buf(),
135 source,
136 })?;
137
138 let is_yaml = matches!(
139 path.extension().and_then(|ext| ext.to_str()),
140 Some("yaml") | Some("yml")
141 );
142
143 let scenario: Scenario = if is_yaml {
144 serde_yaml::from_str(&contents).map_err(|source| ScenarioError::Parse {
145 path: path.to_path_buf(),
146 source: ParseSource::Yaml(source),
147 })?
148 } else {
149 serde_json::from_str(&contents).map_err(|source| ScenarioError::Parse {
150 path: path.to_path_buf(),
151 source: ParseSource::Json(source),
152 })?
153 };
154
155 scenario.validate_schema_version()?;
156 Ok(scenario)
157 }
158
159 /// Confirm [`Scenario::schema_version`] equals [`EVAL_SCHEMA_VERSION`], or return a
160 /// typed [`ScenarioError::UnsupportedSchemaVersion`] naming the version found.
161 ///
162 /// Called by [`Scenario::from_path`]; exposed separately so a caller holding an
163 /// already-deserialized [`Scenario`] (e.g. from an embedded fixture) can run the
164 /// same check without a round trip through the filesystem.
165 ///
166 /// # Errors
167 ///
168 /// [`ScenarioError::UnsupportedSchemaVersion`] if [`Scenario::schema_version`] is
169 /// not [`EVAL_SCHEMA_VERSION`].
170 pub fn validate_schema_version(&self) -> Result<(), ScenarioError> {
171 if self.schema_version != EVAL_SCHEMA_VERSION {
172 return Err(ScenarioError::UnsupportedSchemaVersion {
173 found: self.schema_version.clone(),
174 });
175 }
176 Ok(())
177 }
178}
179
180/// What a [`Scenario`] runs (D-31): a `WarGraphDoc` file on disk, or a Rust closure a
181/// host test binary registered by name.
182///
183/// Untagged so both documented spellings parse directly with no wrapping "type" key --
184/// `{ graph_doc: <path> }` or `{ registered: <name> }` -- unambiguous because the two
185/// variants' field names never overlap.
186#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
187#[serde(untagged)]
188pub enum ScenarioTarget {
189 /// Compile the named `WarGraphDoc` file (JSON or YAML) through the scenario's
190 /// (optional) named registries.
191 GraphDoc {
192 /// Path to the `WarGraphDoc` file, resolved relative to the scenario file's own
193 /// directory by the runner.
194 graph_doc: PathBuf,
195 },
196 /// Build the graph from a Rust closure a host test binary registered under this
197 /// name (`ScenarioRunner::register_graph`).
198 Registered {
199 /// The name a host test binary passed to `ScenarioRunner::register_graph`.
200 registered: String,
201 },
202}
203
204/// Which state store backs a scenario run (D-28). Defaults to [`StoreKind::InMemory`].
205#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
206#[serde(rename_all = "snake_case")]
207pub enum StoreKind {
208 /// An in-memory `WaypointPort`, discarded when the run ends. The default.
209 #[default]
210 InMemory,
211 /// A temp-file SQLite `WaypointPort`, needed by any case that simulates a crash
212 /// (`interrupt_after_superstep`) and resumes over the same store.
213 SqliteTemp,
214}
215
216/// The scenario-wide (or per-case) scripted LLM behaviour (D-28, D-30).
217///
218/// Resolution order, applied per call by `ScenarioLlm`: [`LlmScript::match_rules`]
219/// first (first match wins, does not advance a sequence), then
220/// [`LlmScript::per_node`] for the node the call is attributed to, then
221/// [`LlmScript::global`].
222#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq)]
223#[serde(deny_unknown_fields)]
224pub struct LlmScript {
225 /// The default sequence consulted when no per-node sequence exists for a call's
226 /// node, consumed one entry per call (cycling is `ScenarioLlm`'s decision, not the
227 /// format's).
228 #[serde(default)]
229 pub global: Vec<ScriptEntry>,
230 /// Per-node sequences, keyed by the Paladin node id. A call routed to a node with
231 /// an entry here consumes THIS sequence and leaves [`LlmScript::global`] untouched.
232 #[serde(default)]
233 pub per_node: BTreeMap<String, Vec<ScriptEntry>>,
234 /// Prompt-substring match rules, checked in declaration order before either
235 /// sequence; the wire key is `match` (a Rust keyword, hence the field rename).
236 #[serde(default, rename = "match")]
237 pub match_rules: Vec<MatchRule>,
238}
239
240/// One scripted response to an incoming LLM call (D-28, D-30).
241///
242/// Externally tagged so the three documented spellings are `{ text: "..." }`,
243/// `{ tool_call: { name, arguments } }` and `{ error: <kind> }` -- exactly D-28's `{
244/// text | tool_call: {name, arguments} | error: <LlmError kind> }`.
245///
246/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
247/// hand-rolled (de)serialization" doc section for why.
248#[derive(Debug, Clone, JsonSchema, PartialEq)]
249#[serde(rename_all = "snake_case")]
250pub enum ScriptEntry {
251 /// `text`: a plain success completion.
252 Text(String),
253 /// `tool_call`: a success completion carrying a [`FunctionCall`](paladin_ports::output::llm_port::FunctionCall)-shaped
254 /// tool invocation.
255 ToolCall {
256 /// The tool/function name the mocked model "calls".
257 name: String,
258 /// The JSON-encoded call arguments.
259 arguments: serde_json::Value,
260 },
261 /// `error`: the call returns this [`LlmErrorKind`] instead of a completion.
262 Error(LlmErrorKind),
263}
264
265/// The wire shape of [`ScriptEntry::ToolCall`]'s content, as its own plain struct so it
266/// can be handed to `MapAccess::next_value` (deserializing) and
267/// `SerializeMap::serialize_entry` (serializing) directly. Plain structs are unaffected
268/// by the `serde_yaml` enum-deserialization bug this module's manual impls work around.
269#[derive(Debug, Clone, Serialize, Deserialize)]
270#[serde(deny_unknown_fields)]
271struct ToolCallFields {
272 name: String,
273 arguments: serde_json::Value,
274}
275
276const SCRIPT_ENTRY_VARIANTS: &[&str] = &["text", "tool_call", "error"];
277
278impl Serialize for ScriptEntry {
279 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
280 where
281 S: Serializer,
282 {
283 let mut map = serializer.serialize_map(Some(1))?;
284 match self {
285 ScriptEntry::Text(text) => map.serialize_entry("text", text)?,
286 ScriptEntry::ToolCall { name, arguments } => map.serialize_entry(
287 "tool_call",
288 &ToolCallFields {
289 name: name.clone(),
290 arguments: arguments.clone(),
291 },
292 )?,
293 ScriptEntry::Error(kind) => map.serialize_entry("error", kind)?,
294 }
295 map.end()
296 }
297}
298
299impl<'de> Deserialize<'de> for ScriptEntry {
300 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
301 where
302 D: Deserializer<'de>,
303 {
304 struct ScriptEntryVisitor;
305
306 impl<'de> Visitor<'de> for ScriptEntryVisitor {
307 type Value = ScriptEntry;
308
309 fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310 write!(f, "a single-key map: `text`, `tool_call` or `error`")
311 }
312
313 fn visit_map<A>(self, mut map: A) -> Result<ScriptEntry, A::Error>
314 where
315 A: MapAccess<'de>,
316 {
317 let key: String = map.next_key()?.ok_or_else(|| {
318 de::Error::custom("expected exactly one key (`text`, `tool_call` or `error`)")
319 })?;
320 let entry = match key.as_str() {
321 "text" => ScriptEntry::Text(map.next_value()?),
322 "tool_call" => {
323 let fields: ToolCallFields = map.next_value()?;
324 ScriptEntry::ToolCall {
325 name: fields.name,
326 arguments: fields.arguments,
327 }
328 }
329 "error" => ScriptEntry::Error(map.next_value()?),
330 other => {
331 return Err(de::Error::unknown_variant(other, SCRIPT_ENTRY_VARIANTS));
332 }
333 };
334 Ok(entry)
335 }
336 }
337
338 deserializer.deserialize_any(ScriptEntryVisitor)
339 }
340}
341
342/// A closed vocabulary of [`paladin_ports::output::llm_port::LlmError`] shapes a
343/// scenario file can script (D-28, D-30).
344///
345/// [`LlmError`] itself does not implement [`Deserialize`], so this mirrors its
346/// vocabulary for the file format the same way `WarGraphDoc`'s `NodeKindDoc` mirrors
347/// `NodeSpec` -- [`LlmErrorKind::to_llm_error`] is the one place a scripted kind becomes
348/// a real [`LlmError`].
349///
350/// `#[non_exhaustive]`: a future `LlmError` variant may need its own scriptable kind.
351#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
352#[serde(rename_all = "snake_case")]
353#[non_exhaustive]
354pub enum LlmErrorKind {
355 /// Maps to [`LlmError::NetworkError`].
356 Network,
357 /// Maps to [`LlmError::AuthenticationError`].
358 Authentication,
359 /// Maps to [`LlmError::InvalidPrompt`].
360 InvalidPrompt,
361 /// Maps to [`LlmError::RateLimitExceeded`].
362 RateLimit,
363 /// Maps to [`LlmError::UsageLimitExceeded`].
364 UsageLimit,
365 /// Maps to [`LlmError::ModelNotAvailable`].
366 ModelNotAvailable,
367 /// Maps to [`LlmError::TokenLimitExceeded`].
368 TokenLimit,
369 /// Maps to [`LlmError::EmptyCompletion`].
370 EmptyCompletion,
371 /// Maps to [`LlmError::ProcessingError`].
372 Processing,
373 /// Maps to [`LlmError::Timeout`].
374 Timeout,
375 /// A generic transient failure (maps to [`LlmError::NetworkError`]) for a scenario
376 /// that only cares about the retry/fallback behaviour a transient error drives, not
377 /// which specific variant produced it.
378 Transient,
379}
380
381impl LlmErrorKind {
382 /// Build the concrete [`LlmError`] this scripted kind represents.
383 ///
384 /// `node` and `call_index` are folded into the error's own message so a failure
385 /// rendered from a captured request log can point at exactly which scripted call
386 /// produced it.
387 ///
388 /// # Examples
389 ///
390 /// ```rust
391 /// use paladin_eval::scenario::LlmErrorKind;
392 ///
393 /// let error = LlmErrorKind::RateLimit.to_llm_error("planner", 2);
394 /// assert!(error.to_string().contains("Rate limit"));
395 /// ```
396 pub fn to_llm_error(
397 &self,
398 node: &str,
399 call_index: usize,
400 ) -> paladin_ports::output::llm_port::LlmError {
401 match self {
402 LlmErrorKind::Network => LlmError::NetworkError(format!(
403 "scripted network error (node {node}, call {call_index})"
404 )),
405 LlmErrorKind::Authentication => LlmError::AuthenticationError(format!(
406 "scripted authentication error (node {node}, call {call_index})"
407 )),
408 LlmErrorKind::InvalidPrompt => LlmError::InvalidPrompt(format!(
409 "scripted invalid prompt (node {node}, call {call_index})"
410 )),
411 LlmErrorKind::RateLimit => LlmError::rate_limited(None),
412 LlmErrorKind::UsageLimit => LlmError::UsageLimitExceeded {
413 provider: "scenario".to_string(),
414 regain_hint: None,
415 },
416 LlmErrorKind::ModelNotAvailable => LlmError::ModelNotAvailable(format!(
417 "scripted model-not-available (node {node}, call {call_index})"
418 )),
419 LlmErrorKind::TokenLimit => LlmError::TokenLimitExceeded,
420 LlmErrorKind::EmptyCompletion => LlmError::empty_completion(format!(
421 "scripted empty completion (node {node}, call {call_index})"
422 )),
423 LlmErrorKind::Processing => LlmError::ProcessingError(format!(
424 "scripted processing error (node {node}, call {call_index})"
425 )),
426 LlmErrorKind::Timeout => {
427 LlmError::Timeout(format!("scripted timeout (node {node}, call {call_index})"))
428 }
429 LlmErrorKind::Transient => LlmError::NetworkError(format!(
430 "scripted transient error (node {node}, call {call_index})"
431 )),
432 }
433 }
434}
435
436/// A prompt-substring match rule (D-28), checked before either script sequence.
437#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
438#[serde(deny_unknown_fields)]
439pub struct MatchRule {
440 /// If the incoming prompt contains this substring, [`MatchRule::response`] is
441 /// returned and neither sequence is advanced.
442 pub prompt_contains: String,
443 /// The response returned when [`MatchRule::prompt_contains`] matches.
444 pub response: ScriptEntry,
445}
446
447/// One test case within a [`Scenario`] (D-28, D-34).
448#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
449#[serde(deny_unknown_fields)]
450pub struct Case {
451 /// This case's name, used in `libtest-mimic` trial naming and failure output.
452 pub name: String,
453 /// Initial input fields, by name, handed to the run.
454 #[serde(default)]
455 pub input: BTreeMap<String, serde_json::Value>,
456 /// Simulate a crash after this superstep completes: the runner drops the engine
457 /// and builds a fresh one over the same [`StoreKind::SqliteTemp`] store to resume
458 /// (D-34's E2E-1 technique). Requires `store: sqlite_temp`.
459 #[serde(default)]
460 pub interrupt_after_superstep: Option<u64>,
461 /// Scripted Parley responses, consumed in order whenever the run raises a Parley
462 /// (D-34's E2E-2 technique).
463 #[serde(default)]
464 pub parley_responses: Vec<ParleyScript>,
465 /// A case-specific LLM script overriding [`Scenario::llm`] for this case only.
466 #[serde(default)]
467 pub llm: Option<LlmScript>,
468 /// The assertions evaluated against the captured trace records and the final
469 /// `Battlefield` once this case's run finishes.
470 pub assertions: Vec<Assertion>,
471}
472
473/// One scripted response to a Parley the run raises (D-28, D-34).
474#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
475#[serde(deny_unknown_fields)]
476pub struct ParleyScript {
477 /// The Parley kind this response answers (mirrors
478 /// `paladin_core::platform::container::parley::ParleyKind`'s wire vocabulary).
479 pub kind: String,
480 /// The value handed back as the Parley's resolution.
481 pub value: serde_json::Value,
482}
483
484/// Live-mode options (D-35): whether a scenario opts in to running content-bearing
485/// assertions against a real provider instead of skipping them.
486#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
487#[serde(deny_unknown_fields)]
488pub struct LiveOptions {
489 /// When `true` and the runner is invoked with `--live`, content-bearing assertions
490 /// (`final_state_field_equals`, `final_state_field_matches`, `field_json_path_equals`,
491 /// `final_state_snapshot`) run instead of being skipped. Defaults to `false`.
492 #[serde(default)]
493 pub allow_content_assertions: bool,
494}
495
496/// A bound on how many times [`Assertion::NodeExecuted`] expects its node to have run.
497///
498/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
499/// hand-rolled (de)serialization" doc section for why.
500#[derive(Debug, Clone, Copy, JsonSchema, PartialEq, Eq)]
501#[serde(rename_all = "snake_case")]
502pub enum Times {
503 /// The node ran exactly this many times.
504 Exact(u32),
505 /// The node ran at least this many times.
506 Min(u32),
507 /// The node ran at most this many times.
508 Max(u32),
509}
510
511const TIMES_VARIANTS: &[&str] = &["exact", "min", "max"];
512
513impl Serialize for Times {
514 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
515 where
516 S: Serializer,
517 {
518 let mut map = serializer.serialize_map(Some(1))?;
519 match self {
520 Times::Exact(v) => map.serialize_entry("exact", v)?,
521 Times::Min(v) => map.serialize_entry("min", v)?,
522 Times::Max(v) => map.serialize_entry("max", v)?,
523 }
524 map.end()
525 }
526}
527
528impl<'de> Deserialize<'de> for Times {
529 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
530 where
531 D: Deserializer<'de>,
532 {
533 struct TimesVisitor;
534
535 impl<'de> Visitor<'de> for TimesVisitor {
536 type Value = Times;
537
538 fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
539 write!(f, "a single-key map: `exact`, `min` or `max`")
540 }
541
542 fn visit_map<A>(self, mut map: A) -> Result<Times, A::Error>
543 where
544 A: MapAccess<'de>,
545 {
546 let key: String = map.next_key()?.ok_or_else(|| {
547 de::Error::custom("expected exactly one key (`exact`, `min` or `max`)")
548 })?;
549 let times = match key.as_str() {
550 "exact" => Times::Exact(map.next_value()?),
551 "min" => Times::Min(map.next_value()?),
552 "max" => Times::Max(map.next_value()?),
553 other => return Err(de::Error::unknown_variant(other, TIMES_VARIANTS)),
554 };
555 Ok(times)
556 }
557 }
558
559 deserializer.deserialize_any(TimesVisitor)
560 }
561}
562
563/// The vocabulary [`Assertion::RunStatus`] compares against.
564///
565/// Mirrors every status of `paladin_core::platform::container::trace::RunFinishStatus`
566/// except `Cancelled`.
567/// That core type carries no `schemars::JsonSchema` derive (ADR-0015: `paladin-core`
568/// does not take the `schemars` dependency), so this crate mirrors the vocabulary
569/// locally -- exactly the pattern `WarGraphDoc`'s `NodeKindDoc` uses for `NodeSpec`.
570///
571/// A run that finished `cancelled` matches no `run_status` value until the next minor,
572/// which adds `Cancelled` together with a schema refresh. A variant cannot be added in
573/// v0.11.1, because this enum and [`Assertion`] are exhaustive public enums of a
574/// published crate and the patch-level semver gate treats either change as major
575/// (FIXES-04, re-homed).
576#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
577#[serde(rename_all = "snake_case")]
578pub enum RunStatusValue {
579 /// The run completed normally.
580 Completed,
581 /// The run failed.
582 Failed,
583 /// The run was halted.
584 Halted,
585 /// The run suspended awaiting Parley input.
586 AwaitingInput,
587}
588
589/// One assertion evaluated against a case's captured trace records and final
590/// `Battlefield` (D-28, D-29, PRD 07 OBS-FR-12).
591///
592/// A closed, externally tagged enum -- an unrecognised tag is a deserialize error
593/// naming the tag, never a silently-ignored no-op (X-04's "closed, not silently
594/// permissive" rule). `custom(fn)` (a later plan's Rust-API-only escape hatch) has no
595/// variant here and can never be deserialized from a file (T-28-05-01).
596///
597/// Every variant name below is PRD 07 OBS-FR-12's verbatim, snake-cased wire name.
598///
599/// `Serialize`/`Deserialize` are hand-written -- see this module's top-level "A note on
600/// hand-rolled (de)serialization" doc section for why.
601#[derive(Debug, Clone, JsonSchema, PartialEq)]
602#[serde(rename_all = "snake_case", deny_unknown_fields)]
603pub enum Assertion {
604 /// `final_state_field_equals`: the named final-`Battlefield` field equals `value`
605 /// exactly.
606 FinalStateFieldEquals {
607 /// The `Battlefield` field name.
608 field: String,
609 /// The expected value.
610 value: serde_json::Value,
611 },
612 /// `final_state_field_matches`: the named final-`Battlefield` field, rendered as a
613 /// string, matches the `pattern` regular expression.
614 FinalStateFieldMatches {
615 /// The `Battlefield` field name.
616 field: String,
617 /// The regular expression the field's rendered value must match.
618 pattern: String,
619 },
620 /// `field_json_path_equals`: the value at `path` (a JSON Pointer / JSONPath-style
621 /// path into the final `Battlefield`) equals `value`.
622 FieldJsonPathEquals {
623 /// The path into the final `Battlefield`'s JSON representation.
624 path: String,
625 /// The expected value at `path`.
626 value: serde_json::Value,
627 },
628 /// `node_executed`: the named node ran the number of times `times` bounds.
629 NodeExecuted {
630 /// The node id.
631 node: String,
632 /// The execution-count bound.
633 times: Times,
634 },
635 /// `node_not_executed`: the named node never ran.
636 NodeNotExecuted {
637 /// The node id.
638 node: String,
639 },
640 /// `edge_fired`: the edge `from -> to` fired at least once (needs `EdgeEvaluated`
641 /// records, D-04).
642 EdgeFired {
643 /// The source node id.
644 from: String,
645 /// The destination node id.
646 to: String,
647 },
648 /// `route_taken`: the sequence of node ids in `NodeStarted` order contains this
649 /// list as a subsequence.
650 RouteTaken(Vec<String>),
651 /// `run_status`: the run's terminal status equals this value. A cancelled run matches
652 /// no value (see [`RunStatusValue`]).
653 RunStatus(RunStatusValue),
654 /// `total_tokens_max`: the run's total token usage does not exceed this bound.
655 TotalTokensMax(u64),
656 /// `supersteps_max`: the run's total superstep count does not exceed this bound.
657 SuperstepsMax(u64),
658 /// `parley_raised`: a Parley of `kind` was raised by `node`.
659 ParleyRaised {
660 /// The Parley kind (mirrors `ParleyKind`'s wire vocabulary).
661 kind: String,
662 /// The node that raised it.
663 node: String,
664 },
665 /// `final_state_snapshot`: the final `Battlefield` matches a blessed snapshot file
666 /// (`--bless` regenerates it, D-33).
667 FinalStateSnapshot,
668}
669
670// ---------------------------------------------------------------------------
671// Assertion: hand-rolled Serialize/Deserialize (see this module's top-level doc note).
672// ---------------------------------------------------------------------------
673
674/// Wire shape of [`Assertion::FinalStateFieldEquals`]'s content.
675#[derive(Debug, Clone, Serialize, Deserialize)]
676#[serde(deny_unknown_fields)]
677struct FieldEqualsFields {
678 field: String,
679 value: serde_json::Value,
680}
681
682/// Wire shape of [`Assertion::FinalStateFieldMatches`]'s content.
683#[derive(Debug, Clone, Serialize, Deserialize)]
684#[serde(deny_unknown_fields)]
685struct FieldMatchesFields {
686 field: String,
687 pattern: String,
688}
689
690/// Wire shape of [`Assertion::FieldJsonPathEquals`]'s content.
691#[derive(Debug, Clone, Serialize, Deserialize)]
692#[serde(deny_unknown_fields)]
693struct JsonPathFields {
694 path: String,
695 value: serde_json::Value,
696}
697
698/// Wire shape of [`Assertion::NodeExecuted`]'s content.
699#[derive(Debug, Clone, Serialize, Deserialize)]
700#[serde(deny_unknown_fields)]
701struct NodeExecutedFields {
702 node: String,
703 times: Times,
704}
705
706/// Wire shape of [`Assertion::NodeNotExecuted`]'s content.
707#[derive(Debug, Clone, Serialize, Deserialize)]
708#[serde(deny_unknown_fields)]
709struct NodeOnlyFields {
710 node: String,
711}
712
713/// Wire shape of [`Assertion::EdgeFired`]'s content.
714#[derive(Debug, Clone, Serialize, Deserialize)]
715#[serde(deny_unknown_fields)]
716struct EdgeFields {
717 from: String,
718 to: String,
719}
720
721/// Wire shape of [`Assertion::ParleyRaised`]'s content.
722#[derive(Debug, Clone, Serialize, Deserialize)]
723#[serde(deny_unknown_fields)]
724struct ParleyRaisedFields {
725 kind: String,
726 node: String,
727}
728
729const ASSERTION_VARIANTS: &[&str] = &[
730 "final_state_field_equals",
731 "final_state_field_matches",
732 "field_json_path_equals",
733 "node_executed",
734 "node_not_executed",
735 "edge_fired",
736 "route_taken",
737 "run_status",
738 "total_tokens_max",
739 "supersteps_max",
740 "parley_raised",
741 "final_state_snapshot",
742];
743
744impl Serialize for Assertion {
745 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
746 where
747 S: Serializer,
748 {
749 // `final_state_snapshot` is the one unit variant -- a bare string, not a
750 // single-key map, matching the default external-tagging representation a
751 // derived impl would also have produced for a unit variant.
752 if let Assertion::FinalStateSnapshot = self {
753 return serializer.serialize_str("final_state_snapshot");
754 }
755
756 let mut map = serializer.serialize_map(Some(1))?;
757 match self {
758 Assertion::FinalStateFieldEquals { field, value } => map.serialize_entry(
759 "final_state_field_equals",
760 &FieldEqualsFields {
761 field: field.clone(),
762 value: value.clone(),
763 },
764 )?,
765 Assertion::FinalStateFieldMatches { field, pattern } => map.serialize_entry(
766 "final_state_field_matches",
767 &FieldMatchesFields {
768 field: field.clone(),
769 pattern: pattern.clone(),
770 },
771 )?,
772 Assertion::FieldJsonPathEquals { path, value } => map.serialize_entry(
773 "field_json_path_equals",
774 &JsonPathFields {
775 path: path.clone(),
776 value: value.clone(),
777 },
778 )?,
779 Assertion::NodeExecuted { node, times } => map.serialize_entry(
780 "node_executed",
781 &NodeExecutedFields {
782 node: node.clone(),
783 times: *times,
784 },
785 )?,
786 Assertion::NodeNotExecuted { node } => {
787 map.serialize_entry("node_not_executed", &NodeOnlyFields { node: node.clone() })?
788 }
789 Assertion::EdgeFired { from, to } => map.serialize_entry(
790 "edge_fired",
791 &EdgeFields {
792 from: from.clone(),
793 to: to.clone(),
794 },
795 )?,
796 Assertion::RouteTaken(nodes) => map.serialize_entry("route_taken", nodes)?,
797 Assertion::RunStatus(status) => map.serialize_entry("run_status", status)?,
798 Assertion::TotalTokensMax(v) => map.serialize_entry("total_tokens_max", v)?,
799 Assertion::SuperstepsMax(v) => map.serialize_entry("supersteps_max", v)?,
800 Assertion::ParleyRaised { kind, node } => map.serialize_entry(
801 "parley_raised",
802 &ParleyRaisedFields {
803 kind: kind.clone(),
804 node: node.clone(),
805 },
806 )?,
807 Assertion::FinalStateSnapshot => unreachable!("handled by the early return above"),
808 }
809 map.end()
810 }
811}
812
813impl<'de> Deserialize<'de> for Assertion {
814 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
815 where
816 D: Deserializer<'de>,
817 {
818 struct AssertionVisitor;
819
820 impl<'de> Visitor<'de> for AssertionVisitor {
821 type Value = Assertion;
822
823 fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
824 write!(
825 f,
826 "`final_state_snapshot` or a single-key map naming one of the other eleven assertion kinds"
827 )
828 }
829
830 fn visit_str<E>(self, v: &str) -> Result<Assertion, E>
831 where
832 E: de::Error,
833 {
834 match v {
835 "final_state_snapshot" => Ok(Assertion::FinalStateSnapshot),
836 other => Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
837 }
838 }
839
840 fn visit_map<A>(self, mut map: A) -> Result<Assertion, A::Error>
841 where
842 A: MapAccess<'de>,
843 {
844 let key: String = map.next_key()?.ok_or_else(|| {
845 de::Error::custom("expected exactly one key naming the assertion kind")
846 })?;
847 let assertion = match key.as_str() {
848 "final_state_field_equals" => {
849 let fields: FieldEqualsFields = map.next_value()?;
850 Assertion::FinalStateFieldEquals {
851 field: fields.field,
852 value: fields.value,
853 }
854 }
855 "final_state_field_matches" => {
856 let fields: FieldMatchesFields = map.next_value()?;
857 Assertion::FinalStateFieldMatches {
858 field: fields.field,
859 pattern: fields.pattern,
860 }
861 }
862 "field_json_path_equals" => {
863 let fields: JsonPathFields = map.next_value()?;
864 Assertion::FieldJsonPathEquals {
865 path: fields.path,
866 value: fields.value,
867 }
868 }
869 "node_executed" => {
870 let fields: NodeExecutedFields = map.next_value()?;
871 Assertion::NodeExecuted {
872 node: fields.node,
873 times: fields.times,
874 }
875 }
876 "node_not_executed" => {
877 let fields: NodeOnlyFields = map.next_value()?;
878 Assertion::NodeNotExecuted { node: fields.node }
879 }
880 "edge_fired" => {
881 let fields: EdgeFields = map.next_value()?;
882 Assertion::EdgeFired {
883 from: fields.from,
884 to: fields.to,
885 }
886 }
887 "route_taken" => Assertion::RouteTaken(map.next_value()?),
888 "run_status" => Assertion::RunStatus(map.next_value()?),
889 "total_tokens_max" => Assertion::TotalTokensMax(map.next_value()?),
890 "supersteps_max" => Assertion::SuperstepsMax(map.next_value()?),
891 "parley_raised" => {
892 let fields: ParleyRaisedFields = map.next_value()?;
893 Assertion::ParleyRaised {
894 kind: fields.kind,
895 node: fields.node,
896 }
897 }
898 "final_state_snapshot" => {
899 // Tolerate `{ final_state_snapshot: null }` / `{
900 // final_state_snapshot: {} }` alongside the bare-string form
901 // `visit_str` handles -- either spelling of the unit variant
902 // parses.
903 let _: serde::de::IgnoredAny = map.next_value()?;
904 Assertion::FinalStateSnapshot
905 }
906 other => return Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
907 };
908 Ok(assertion)
909 }
910 }
911
912 deserializer.deserialize_any(AssertionVisitor)
913 }
914}
915
916/// The underlying parse failure wrapped by [`ScenarioError::Parse`] -- either a YAML or
917/// a JSON deserialize error, depending on which format [`Scenario::from_path`]
918/// dispatched to.
919#[derive(Debug, Error)]
920pub enum ParseSource {
921 /// A YAML deserialize failure (`.yaml`/`.yml` files).
922 #[error(transparent)]
923 Yaml(#[from] serde_yaml::Error),
924 /// A JSON deserialize failure (every other extension).
925 #[error(transparent)]
926 Json(#[from] serde_json::Error),
927}
928
929/// Errors [`Scenario::from_path`] and [`Scenario::validate_schema_version`] can return.
930///
931/// `#[non_exhaustive]`: a future variant (e.g. a semantic validation failure beyond
932/// `schema_version`) can be added without a semver-major bump.
933#[derive(Debug, Error)]
934#[non_exhaustive]
935pub enum ScenarioError {
936 /// The document's `schema_version` is not [`EVAL_SCHEMA_VERSION`].
937 #[error(
938 "unsupported scenario schema_version {found:?}; this build of paladin-eval understands {EVAL_SCHEMA_VERSION:?}"
939 )]
940 UnsupportedSchemaVersion {
941 /// The `schema_version` string found in the document.
942 found: String,
943 },
944 /// The document's contents did not deserialize.
945 #[error("failed to parse scenario file {path}: {source}", path = path.display())]
946 Parse {
947 /// The file that failed to parse.
948 path: PathBuf,
949 /// The underlying YAML or JSON deserialize error.
950 #[source]
951 source: ParseSource,
952 },
953 /// The scenario file could not be read from disk.
954 #[error("failed to read scenario file {path}: {source}", path = path.display())]
955 Io {
956 /// The file that could not be read.
957 path: PathBuf,
958 /// The underlying I/O error.
959 #[source]
960 source: std::io::Error,
961 },
962}
963
964#[cfg(test)]
965mod tests {
966 use super::*;
967
968 fn minimal_yaml() -> &'static str {
969 r#"
970schema_version: "1"
971target:
972 graph_doc: "fixtures/linear.json"
973cases:
974 - name: happy_path
975 input:
976 foo: "bar"
977 assertions:
978 - final_state_field_equals:
979 field: "status"
980 value: "done"
981"#
982 }
983
984 #[test]
985 fn minimal_scenario_parses() {
986 let scenario: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses");
987 assert_eq!(scenario.schema_version, "1");
988 assert_eq!(
989 scenario.target,
990 ScenarioTarget::GraphDoc {
991 graph_doc: PathBuf::from("fixtures/linear.json"),
992 }
993 );
994 assert_eq!(scenario.store, StoreKind::InMemory);
995 assert_eq!(scenario.llm, LlmScript::default());
996 assert_eq!(scenario.cases.len(), 1);
997 assert_eq!(scenario.cases[0].name, "happy_path");
998 assert_eq!(scenario.cases[0].assertions.len(), 1);
999 }
1000
1001 #[test]
1002 fn json_and_yaml_agree() {
1003 let from_yaml: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses yaml");
1004
1005 let json = serde_json::json!({
1006 "schema_version": "1",
1007 "target": { "graph_doc": "fixtures/linear.json" },
1008 "cases": [
1009 {
1010 "name": "happy_path",
1011 "input": { "foo": "bar" },
1012 "assertions": [
1013 { "final_state_field_equals": { "field": "status", "value": "done" } }
1014 ]
1015 }
1016 ]
1017 });
1018 let from_json: Scenario =
1019 serde_json::from_value(json).expect("parses the equivalent json document");
1020
1021 assert_eq!(from_yaml, from_json);
1022 }
1023
1024 #[test]
1025 fn registered_target_parses() {
1026 let yaml = r#"
1027schema_version: "1"
1028target:
1029 registered: "e2e1"
1030cases: []
1031"#;
1032 let scenario: Scenario = serde_yaml::from_str(yaml).expect("parses");
1033 assert_eq!(
1034 scenario.target,
1035 ScenarioTarget::Registered {
1036 registered: "e2e1".to_string(),
1037 }
1038 );
1039 }
1040
1041 #[test]
1042 fn unknown_schema_version_is_typed() {
1043 let yaml = r#"
1044schema_version: "2"
1045target:
1046 graph_doc: "fixtures/linear.json"
1047cases: []
1048"#;
1049 let scenario: Scenario = serde_yaml::from_str(yaml).expect("still deserializes");
1050 let result = scenario.validate_schema_version();
1051 match result {
1052 Err(ScenarioError::UnsupportedSchemaVersion { found }) => {
1053 assert_eq!(found, "2");
1054 }
1055 other => panic!("expected UnsupportedSchemaVersion, got {other:?}"),
1056 }
1057 }
1058
1059 #[test]
1060 fn unknown_assertion_kind_is_an_error() {
1061 let result: Result<Assertion, _> = serde_yaml::from_str("made_up_kind: {}");
1062 let err = result.expect_err("an unrecognised assertion tag must fail to deserialize");
1063 let message = err.to_string();
1064 assert!(
1065 message.contains("made_up_kind"),
1066 "error message {message:?} does not name the unrecognised tag"
1067 );
1068 }
1069
1070 #[test]
1071 fn times_variants_round_trip() {
1072 for (yaml, expected) in [
1073 ("exact: 3", Times::Exact(3)),
1074 ("min: 1", Times::Min(1)),
1075 ("max: 5", Times::Max(5)),
1076 ] {
1077 let parsed: Times = serde_yaml::from_str(yaml).expect("parses");
1078 assert_eq!(parsed, expected);
1079 }
1080 }
1081
1082 #[test]
1083 fn script_entry_spellings_round_trip() {
1084 let text: ScriptEntry = serde_yaml::from_str("text: hello").unwrap();
1085 assert_eq!(text, ScriptEntry::Text("hello".to_string()));
1086
1087 let tool_call: ScriptEntry =
1088 serde_yaml::from_str("tool_call:\n name: search\n arguments: {}").unwrap();
1089 assert_eq!(
1090 tool_call,
1091 ScriptEntry::ToolCall {
1092 name: "search".to_string(),
1093 arguments: serde_json::json!({}),
1094 }
1095 );
1096
1097 let error: ScriptEntry = serde_yaml::from_str("error: transient").unwrap();
1098 assert_eq!(error, ScriptEntry::Error(LlmErrorKind::Transient));
1099 }
1100}