Skip to main content

paladin_eval/
runner.rs

1//! `ScenarioRunner` -- drives a [`crate::scenario::Scenario`] case through a real
2//! [`WarEngine`], captures its trace record stream, and evaluates its assertions
3//! (D-29, D-31, D-32, D-34).
4//!
5//! # What a case run touches (D-29)
6//!
7//! A case run compiles or resolves the scenario's [`ScenarioTarget`] into a real
8//! [`WarGraph`], substitutes a [`ScenarioLlm`] for every Paladin node's `LlmPort`
9//! slot, attaches a capturing [`TraceSink`] through [`WarEngine::with_trace_sink`],
10//! drives the run to a terminal [`RunOutcome`] (answering any raised Parley from
11//! [`crate::scenario::Case::parley_responses`] in order), and evaluates every
12//! [`crate::scenario::Assertion`] through [`crate::assertion::evaluate`] against
13//! exactly the captured records, the final `Battlefield`, and the `RunOutcome` --
14//! nothing else.
15//!
16//! # Two targets, one substitution point (D-31)
17//!
18//! A [`ScenarioTarget::GraphDoc`] compiles through the [`EngineRegistries`] the
19//! scenario names (or an empty default); a [`ScenarioTarget::Registered`] resolves
20//! against a host-registered [`GraphConstructor`] closure, receiving a
21//! [`ScriptedPorts`] handle so code-built graphs with `Function` nodes, worker
22//! templates, custom edge evaluators and Aegis handlers can reach a per-node
23//! [`ScenarioLlm`] directly -- the same substitution a `graph_doc` target's Paladin
24//! nodes receive automatically by walking the compiled [`WarGraph`]'s own node
25//! table.
26//!
27//! # Simulated crash-and-resume (D-28, D-34)
28//!
29//! A case with `interrupt_after_superstep` set reuses the EXACT technique
30//! `tests/integration/e2e_crash_resume_test.rs` documents: a throwaway "control"
31//! run drives the scenario to completion over its own temp-file SQLite store, the
32//! first N Waypoints are copied into a fresh "resumed" store, and a brand new
33//! engine resumes from there. This is deterministic (no `CancellationToken` race
34//! against an async trace sink) at the cost of one documented limitation: a
35//! scenario's SHARED `llm.global` sequence is consumed in call order across every
36//! node, so the resumed run's fresh cursor does not "fast-forward" past the calls
37//! the already-completed (seeded) nodes made pre-crash. Author an
38//! `interrupt_after_superstep` case's LLM script with PER-NODE sequences
39//! (`llm.per_node`) for deterministic replay -- each node's own cursor is
40//! independent of which OTHER nodes already ran, so a fresh root reproduces the
41//! same per-node sequence regardless of the crash point.
42//!
43//! # The `evals` harness target (D-32)
44//!
45//! [`crate::eval_scenarios!`] expands to a `fn main()` for a `harness = false` `[[test]]`
46//! target: it globs the given pattern at RUNTIME (so `evals/` growing between
47//! builds needs no manifest change) and hands one `libtest_mimic::Trial` per
48//! `(file, case)` pair to the custom harness, named `<file-stem>::<case>` -- so
49//! `cargo test --test evals <filter>` filters exactly like any other Rust test.
50//! Every trial runs its case on a FRESH single-threaded Tokio runtime (never a
51//! shared one), so one case's runtime state can never leak into another's.
52
53use std::collections::HashMap;
54use std::path::{Path, PathBuf};
55use std::sync::{Arc, Mutex};
56use std::time::Duration;
57
58use async_trait::async_trait;
59use libtest_mimic::{Arguments, Failed, Trial};
60use serde_json::Value;
61use thiserror::Error;
62use uuid::Uuid;
63
64use paladin_battalion::engine::graph_doc::WarGraphDoc;
65use paladin_battalion::engine::{
66    EngineError, EngineRegistries, NodeSpec, RunOutcome, WarEngine, WarGraph,
67};
68use paladin_core::platform::container::battlefield::{Battlefield, FieldName, StateDelta};
69use paladin_core::platform::container::paladin::Paladin;
70use paladin_core::platform::container::paladin_error::PaladinError;
71use paladin_core::platform::container::parley::ParleyResponse;
72use paladin_core::platform::container::prompt::{PromptItem, PromptType, UserPrompt};
73use paladin_core::platform::container::trace::TraceRecord;
74use paladin_core::platform::container::waypoint::{ThreadId, Waypoint};
75use paladin_ports::output::llm_port::{LlmPort, LlmRequest};
76use paladin_ports::output::paladin_port::{
77    PaladinPort, PaladinResult, PaladinStream, PaladinStreamChunk, StopReason,
78};
79use paladin_ports::output::trace_sink_port::{TraceSink, TraceSinkError};
80use paladin_ports::output::waypoint_port::{WaypointError, WaypointPort};
81use paladin_storage::waypoint::in_memory::InMemoryWaypointStore;
82use paladin_storage::waypoint::sqlite::SqliteWaypointStore;
83
84use crate::assertion::{self, AssertionContext, AssertionFailure, AssertionOutcome};
85use crate::scenario::{Assertion, Case, ParleyScript, Scenario, ScenarioTarget, StoreKind};
86use crate::scripted_llm::ScenarioLlm;
87
88// ---------------------------------------------------------------------------
89// ScriptedPorts / GraphConstructor / RegistriesFactory (D-31)
90// ---------------------------------------------------------------------------
91
92/// The per-node scripted-port handle a [`GraphConstructor`] receives (D-31).
93///
94/// Wraps ONE scenario's resolved [`crate::scenario::LlmScript`] as a
95/// [`ScenarioLlm`] "root": [`ScriptedPorts::for_node`] hands out a
96/// [`ScenarioLlm::for_node`] clone sharing that root's cursors, call counters
97/// and captured-request log, so a code-built `Function` node can call
98/// `.generate()` directly with the exact same per-node routing a compiled
99/// `graph_doc`'s Paladin nodes receive automatically.
100pub struct ScriptedPorts {
101    root: ScenarioLlm,
102}
103
104impl ScriptedPorts {
105    /// Return a [`ScenarioLlm`] attributed to `node_id`, sharing this handle's
106    /// underlying script, cursors and captured-request log.
107    pub fn for_node(&self, node_id: impl Into<String>) -> ScenarioLlm {
108        self.root.for_node(node_id)
109    }
110}
111
112/// A host-registered Rust closure building a [`WarGraph`] from a resolved
113/// [`ScriptedPorts`] handle (D-31), registered via
114/// [`ScenarioRunner::register_graph`].
115pub type GraphConstructor = Arc<dyn Fn(&ScriptedPorts) -> WarGraph + Send + Sync>;
116
117/// A host-registered Rust closure building an [`EngineRegistries`] bundle,
118/// registered via [`ScenarioRunner::register_registries`] and resolved by a
119/// `graph_doc` target naming it in [`crate::scenario::Scenario::registries`].
120pub type RegistriesFactory = Arc<dyn Fn() -> EngineRegistries + Send + Sync>;
121
122// ---------------------------------------------------------------------------
123// RunOptions
124// ---------------------------------------------------------------------------
125
126/// Per-run options a caller of [`ScenarioRunner::run_case`] controls (D-32,
127/// D-33). The `evals` harness (`eval_scenarios!`) always uses
128/// [`RunOptions::default`] -- `bless: false`, `live: false` -- so a plain
129/// `cargo test --test evals` never blesses a snapshot and never enters live
130/// mode.
131#[derive(Debug, Clone, Copy, Default)]
132pub struct RunOptions {
133    /// When `true` and the case carries a `final_state_snapshot` assertion,
134    /// (re)write the blessed snapshot file from the final `Battlefield`
135    /// before evaluating assertions -- so the freshly-blessed file always
136    /// passes its own comparison.
137    pub bless: bool,
138    /// Reserved for the gated live-model mode; inert until the runner's own
139    /// `LiveMode` gate exists. `paladin-cli eval run`'s `--live` flag threads
140    /// its value through here.
141    pub live: bool,
142}
143
144// ---------------------------------------------------------------------------
145// CaseReport / Verdict
146// ---------------------------------------------------------------------------
147
148/// One evaluated [`Assertion`]'s result within a [`CaseReport`].
149#[derive(Debug)]
150pub struct AssertionResult {
151    /// The verdict this assertion reached.
152    pub verdict: Verdict,
153}
154
155/// The outcome of evaluating one [`Assertion`] (D-29, D-35).
156#[derive(Debug)]
157pub enum Verdict {
158    /// The assertion held.
159    Passed,
160    /// The assertion did not hold; carries the actionable failure.
161    Failed(AssertionFailure),
162    /// A content-bearing assertion (D-35) was skipped because the run was
163    /// live and the scenario did not opt in (`live.allow_content_assertions`)
164    /// -- carries the skip reason. Never folded into
165    /// [`CaseReport::passed`]'s pass count: a skip is neither a pass nor a
166    /// failure.
167    Skipped(String),
168}
169
170/// What a [`ScenarioRunner::run_case`] call produced for one
171/// [`crate::scenario::Case`] (D-29, D-32).
172pub struct CaseReport {
173    /// This case's name (`Case::name`), used in trial naming and CLI output.
174    pub case_name: String,
175    /// Whether the case ran at all, or errored before assertions could be
176    /// evaluated (an unresolvable target, a compile failure, an engine
177    /// error, ...).
178    pub outcome: CaseOutcome,
179}
180
181/// The two shapes a [`CaseReport::outcome`] can take.
182pub enum CaseOutcome {
183    /// The case's engine run reached a terminal [`RunOutcome`] and every
184    /// assertion was evaluated against it.
185    Ran {
186        /// The captured `TraceRecord` stream, in `seq` order.
187        records: Vec<TraceRecord>,
188        /// The final `Battlefield` assertions were evaluated against.
189        final_state: Battlefield,
190        /// One [`AssertionResult`] per [`Case::assertions`] entry, in order.
191        results: Vec<AssertionResult>,
192    },
193    /// The case never reached a point where assertions could be evaluated
194    /// (a resolution, compile, or engine-level failure -- never itself an
195    /// assertion failure).
196    Errored(String),
197}
198
199impl CaseReport {
200    /// Whether every assertion in this report passed. A case that
201    /// [`CaseOutcome::Errored`] never passes.
202    pub fn passed(&self) -> bool {
203        match &self.outcome {
204            CaseOutcome::Ran { results, .. } => !results
205                .iter()
206                .any(|r| matches!(r.verdict, Verdict::Failed(_))),
207            CaseOutcome::Errored(_) => false,
208        }
209    }
210
211    /// The captured trace record stream, or an empty slice for an errored
212    /// case.
213    pub fn records(&self) -> &[TraceRecord] {
214        match &self.outcome {
215            CaseOutcome::Ran { records, .. } => records,
216            CaseOutcome::Errored(_) => &[],
217        }
218    }
219
220    /// Render every failing assertion's [`AssertionFailure::render_failure`]
221    /// text, concatenated, or a one-line description of the runner-level
222    /// error for an errored case.
223    pub fn render_failures(&self) -> String {
224        match &self.outcome {
225            CaseOutcome::Errored(message) => {
226                format!(
227                    "case {:?} errored before assertions could run: {message}",
228                    self.case_name
229                )
230            }
231            CaseOutcome::Ran { results, .. } => {
232                let mut out = String::new();
233                for result in results {
234                    if let Verdict::Failed(failure) = &result.verdict {
235                        out.push_str(&failure.render_failure());
236                    }
237                }
238                if out.is_empty() {
239                    "(no failing assertions)".to_string()
240                } else {
241                    out
242                }
243            }
244        }
245    }
246}
247
248// ---------------------------------------------------------------------------
249// ScenarioRunner
250// ---------------------------------------------------------------------------
251
252/// Drives [`Scenario`] cases through a real [`WarEngine`] (D-29, D-31, D-32).
253///
254/// Holds two host-registered name tables -- [`GraphConstructor`]s
255/// ([`ScenarioRunner::register_graph`]) and [`RegistriesFactory`]s
256/// ([`ScenarioRunner::register_registries`]) -- consulted by
257/// [`ScenarioRunner::run_case`] when a scenario's target or `registries` field
258/// names one. `Clone`: every registration is `Arc`-backed, so cloning is cheap
259/// and each [`ScenarioRunner::trials`] `Trial` closure can own its own copy.
260#[derive(Clone, Default)]
261pub struct ScenarioRunner {
262    graphs: HashMap<String, GraphConstructor>,
263    registries: HashMap<String, RegistriesFactory>,
264}
265
266impl ScenarioRunner {
267    /// Construct an empty runner: no graphs, no registries registered.
268    pub fn new() -> Self {
269        Self::default()
270    }
271
272    /// Register a [`GraphConstructor`] under `name`, resolved by a
273    /// [`ScenarioTarget::Registered { registered: name }`](ScenarioTarget::Registered)
274    /// target.
275    pub fn register_graph(&mut self, name: impl Into<String>, ctor: GraphConstructor) -> &mut Self {
276        self.graphs.insert(name.into(), ctor);
277        self
278    }
279
280    /// Register a [`RegistriesFactory`] under `name`, resolved by a
281    /// `graph_doc` target naming it in [`Scenario::registries`].
282    pub fn register_registries(
283        &mut self,
284        name: impl Into<String>,
285        factory: RegistriesFactory,
286    ) -> &mut Self {
287        self.registries.insert(name.into(), factory);
288        self
289    }
290
291    fn resolve_registries(&self, name: Option<&str>) -> Result<EngineRegistries, RunnerError> {
292        match name {
293            None => Ok(EngineRegistries::new()),
294            Some(name) => {
295                let factory =
296                    self.registries
297                        .get(name)
298                        .ok_or_else(|| RunnerError::UnknownRegistries {
299                            name: name.to_string(),
300                            registered: self.registries.keys().cloned().collect(),
301                        })?;
302                Ok((factory)())
303            }
304        }
305    }
306
307    /// Resolve `scenario.target` into a real [`WarGraph`] plus the
308    /// [`ScenarioLlm`] router keyed by each Paladin node's OWN `PaladinData`
309    /// name (D-31) -- the identity [`PaladinPort::execute`] receives, mapped
310    /// to a [`ScenarioLlm::for_node`] clone tagged with that node's GRAPH id
311    /// (the identity a scenario's `llm.per_node` map is keyed by).
312    fn resolve_target(
313        &self,
314        scenario_path: &Path,
315        scenario: &Scenario,
316    ) -> Result<(WarGraph, HashMap<String, ScenarioLlm>), RunnerError> {
317        match &scenario.target {
318            ScenarioTarget::GraphDoc { graph_doc } => {
319                let path = resolve_relative(scenario_path, graph_doc);
320                let doc = load_graph_doc(&path)?;
321                let registries = self.resolve_registries(scenario.registries.as_deref())?;
322                let graph = doc
323                    .compile(&registries)
324                    .map_err(|source| RunnerError::Compile {
325                        path: path.clone(),
326                        source: Box::new(source),
327                    })?;
328                let root = ScenarioLlm::new(scenario.llm.clone());
329                let router = build_llm_router(&graph, &root);
330                Ok((graph, router))
331            }
332            ScenarioTarget::Registered { registered } => {
333                let ctor =
334                    self.graphs
335                        .get(registered)
336                        .ok_or_else(|| RunnerError::UnknownGraph {
337                            name: registered.clone(),
338                            registered: self.graphs.keys().cloned().collect(),
339                        })?;
340                let root = ScenarioLlm::new(scenario.llm.clone());
341                let scripted_ports = ScriptedPorts { root: root.clone() };
342                let graph = (ctor)(&scripted_ports);
343                let router = build_llm_router(&graph, &root);
344                Ok((graph, router))
345            }
346        }
347    }
348
349    /// Run one [`Case`] of `scenario` (parsed from `scenario_path`) and
350    /// evaluate its assertions. Never panics on a resolution/engine failure
351    /// -- reported as [`CaseOutcome::Errored`] instead.
352    pub async fn run_case(
353        &self,
354        scenario_path: &Path,
355        scenario: &Scenario,
356        case: &Case,
357        options: RunOptions,
358    ) -> CaseReport {
359        match self
360            .run_case_inner(scenario_path, scenario, case, options)
361            .await
362        {
363            Ok(report) => report,
364            Err(err) => CaseReport {
365                case_name: case.name.clone(),
366                outcome: CaseOutcome::Errored(err.to_string()),
367            },
368        }
369    }
370
371    async fn run_case_inner(
372        &self,
373        scenario_path: &Path,
374        scenario: &Scenario,
375        case: &Case,
376        options: RunOptions,
377    ) -> Result<CaseReport, RunnerError> {
378        // D-35: the runner ITSELF enforces the three-way live gate, not
379        // only a caller that remembers to check first -- `options.live` is
380        // `false` on every path `eval_scenarios!`'s own harness takes, so
381        // this is unreachable for a plain `cargo test --test evals`.
382        if options.live {
383            check_live_mode(true)?;
384        }
385
386        let mut initial = StateDelta::new();
387        for (key, value) in &case.input {
388            let field = FieldName::new(key).map_err(|source| RunnerError::InvalidField {
389                field: key.clone(),
390                message: source.to_string(),
391            })?;
392            initial.set_raw(field, value.clone());
393        }
394
395        let thread = ThreadId::new(format!("scenario-{}", Uuid::new_v4()))
396            .map_err(|source| RunnerError::InvalidThreadId(source.to_string()))?;
397
398        let (records, outcome, final_state) =
399            if let Some(superstep) = case.interrupt_after_superstep {
400                if scenario.store != StoreKind::SqliteTemp {
401                    return Err(RunnerError::InterruptRequiresSqliteTemp {
402                        case: case.name.clone(),
403                    });
404                }
405                self.run_interrupted(
406                    scenario_path,
407                    scenario,
408                    case,
409                    &thread,
410                    &initial,
411                    superstep,
412                    options,
413                )
414                .await?
415            } else {
416                let (graph, router) = self.resolve_target(scenario_path, scenario)?;
417                let port = build_port(router, options)?;
418                match scenario.store {
419                    StoreKind::InMemory => {
420                        let store = Arc::new(InMemoryWaypointStore::new());
421                        run_plain(
422                            store,
423                            &graph,
424                            port,
425                            thread.clone(),
426                            initial,
427                            &case.parley_responses,
428                        )
429                        .await?
430                    }
431                    StoreKind::SqliteTemp => {
432                        let store = Arc::new(SqliteWaypointStore::new(&temp_db_url("case")).await?);
433                        run_plain(
434                            store,
435                            &graph,
436                            port,
437                            thread.clone(),
438                            initial,
439                            &case.parley_responses,
440                        )
441                        .await?
442                    }
443                }
444            };
445
446        let snapshot_path = snapshot_file_path(scenario_path, &case.name);
447        if options.bless
448            && case
449                .assertions
450                .iter()
451                .any(|a| matches!(a, Assertion::FinalStateSnapshot))
452        {
453            let value = battlefield_snapshot_value(&final_state);
454            // Trailing newline: keeps blessed snapshots hook-clean (`end-of-file-fixer`
455            // runs over all files in CI); comparison is JSON-based, so it is inert.
456            let mut text =
457                serde_json::to_string_pretty(&value).unwrap_or_else(|_| value.to_string());
458            text.push('\n');
459            if let Some(parent) = snapshot_path.parent() {
460                let _ = std::fs::create_dir_all(parent);
461            }
462            std::fs::write(&snapshot_path, text).map_err(|source| RunnerError::Io {
463                path: snapshot_path.clone(),
464                source,
465            })?;
466        }
467
468        let ctx = AssertionContext::new(&records, &final_state, &outcome)
469            .with_snapshot_path(snapshot_path);
470
471        let mut results = Vec::with_capacity(case.assertions.len());
472        for assertion in &case.assertions {
473            let verdict = if should_skip_for_live(
474                assertion,
475                options.live,
476                scenario.live.allow_content_assertions,
477            ) {
478                Verdict::Skipped("content assertion, live mode".to_string())
479            } else {
480                match assertion::evaluate(assertion, &ctx) {
481                    AssertionOutcome::Passed => Verdict::Passed,
482                    AssertionOutcome::Failed(failure) => Verdict::Failed(failure),
483                }
484            };
485            results.push(AssertionResult { verdict });
486        }
487
488        Ok(CaseReport {
489            case_name: case.name.clone(),
490            outcome: CaseOutcome::Ran {
491                records,
492                final_state,
493                results,
494            },
495        })
496    }
497
498    /// The simulated-crash driver (D-28, D-34): a throwaway "control" run
499    /// drives `scenario`'s case to completion over its own temp-file SQLite
500    /// store; every Waypoint up to and including `interrupt_superstep` is
501    /// copied into a FRESH "resumed" store; a brand new engine resumes from
502    /// there. See this module's own doc comment for the per-node-vs-global
503    /// LLM script caveat.
504    #[allow(clippy::too_many_arguments)]
505    async fn run_interrupted(
506        &self,
507        scenario_path: &Path,
508        scenario: &Scenario,
509        case: &Case,
510        thread: &ThreadId,
511        initial: &StateDelta,
512        interrupt_superstep: u64,
513        options: RunOptions,
514    ) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
515        // Control run: throwaway store and sink, driven to completion so its
516        // Waypoint history is ground truth for the seed below.
517        let (control_graph, control_router) = self.resolve_target(scenario_path, scenario)?;
518        let control_port = build_port(control_router, options)?;
519        let control_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("control")).await?);
520        let control_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
521        let control_engine = WarEngine::new(control_port, Arc::clone(&control_store))
522            .with_trace_sink(control_sink.clone());
523        let control_outcome = control_engine
524            .start(&control_graph, thread.clone(), initial.clone())
525            .await?;
526        drive_parleys(
527            &control_engine,
528            &control_graph,
529            thread,
530            control_outcome,
531            &case.parley_responses,
532        )
533        .await?;
534
535        // Seed a fresh store with every Waypoint up to and including the
536        // interrupt superstep -- "the process died right after superstep N".
537        let resumed_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("resumed")).await?);
538        let mut history = full_history(control_store.as_ref(), thread).await?;
539        history.sort_by_key(|wp| wp.superstep);
540        for waypoint in history
541            .iter()
542            .filter(|wp| wp.superstep <= interrupt_superstep)
543        {
544            resumed_store.save(waypoint).await?;
545        }
546
547        // Resumed run: a FRESH graph/router (fresh ScenarioLlm cursors) and
548        // the REAL capturing sink whose records feed this case's assertions.
549        let (resumed_graph, resumed_router) = self.resolve_target(scenario_path, scenario)?;
550        let resumed_port = build_port(resumed_router, options)?;
551        let resumed_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
552        let resumed_engine = WarEngine::new(resumed_port, Arc::clone(&resumed_store))
553            .with_trace_sink(resumed_sink.clone());
554        let resumed_outcome = resumed_engine
555            .resume(&resumed_graph, thread.clone())
556            .await?;
557        let resumed_outcome = drive_parleys(
558            &resumed_engine,
559            &resumed_graph,
560            thread,
561            resumed_outcome,
562            &case.parley_responses,
563        )
564        .await?;
565
566        // The dispatcher's consumer task drains asynchronously -- give it a
567        // moment before reading the captured records back out (established
568        // house pattern, see `engine::hooks`'s own tests).
569        tokio::time::sleep(Duration::from_millis(50)).await;
570        let records = resumed_sink.records();
571        let final_state =
572            resolve_battlefield(resumed_store.as_ref(), thread, &resumed_outcome).await?;
573        Ok((records, resumed_outcome, final_state))
574    }
575
576    /// Glob `pattern` at RUNTIME and build one [`Trial`] per `(file, case)`
577    /// pair, named `<file-stem>::<case>` (D-32). A scenario file that fails
578    /// to parse becomes a single failing `<file-stem>::parse_error` trial
579    /// rather than aborting discovery for every other file. Each trial's
580    /// runner drives its case on a FRESH single-threaded Tokio runtime.
581    pub fn trials(&self, pattern: &str) -> Vec<Trial> {
582        let mut paths: Vec<PathBuf> = match glob::glob(pattern) {
583            Ok(entries) => entries.filter_map(Result::ok).collect(),
584            Err(_) => Vec::new(),
585        };
586        paths.sort();
587
588        let mut trials = Vec::new();
589        for path in paths {
590            let stem = path
591                .file_stem()
592                .and_then(|s| s.to_str())
593                .unwrap_or("scenario")
594                .to_string();
595            match Scenario::from_path(&path) {
596                Ok(scenario) => {
597                    let scenario = Arc::new(scenario);
598                    for case in &scenario.cases {
599                        let trial_name = format!("{stem}::{}", case.name);
600                        let runner = self.clone();
601                        let scenario = Arc::clone(&scenario);
602                        let case = case.clone();
603                        let path = path.clone();
604                        trials.push(Trial::test(trial_name, move || {
605                            let rt = tokio::runtime::Builder::new_current_thread()
606                                .enable_all()
607                                .build()
608                                .map_err(|e| Failed::from(e.to_string()))?;
609                            let report = rt.block_on(runner.run_case(
610                                &path,
611                                &scenario,
612                                &case,
613                                RunOptions::default(),
614                            ));
615                            if report.passed() {
616                                Ok(())
617                            } else {
618                                Err(Failed::from(report.render_failures()))
619                            }
620                        }));
621                    }
622                }
623                Err(err) => {
624                    trials.push(Trial::test(format!("{stem}::parse_error"), move || {
625                        Err(Failed::from(err.to_string()))
626                    }));
627                }
628            }
629        }
630        trials
631    }
632}
633
634/// The `fn main()` body [`crate::eval_scenarios!`] expands to (D-32): parses
635/// `libtest_mimic::Arguments` from the process's own CLI args, builds a
636/// default (nothing registered) [`ScenarioRunner`]'s trials over `pattern`,
637/// and exits with the harness's own exit code.
638pub fn run_eval_harness_main(pattern: &str) -> ! {
639    let args = Arguments::from_args();
640    let runner = ScenarioRunner::new();
641    let trials = runner.trials(pattern);
642    libtest_mimic::run(&args, trials).exit()
643}
644
645/// Like [`run_eval_harness_main`], but calls `configure` on a fresh
646/// [`ScenarioRunner`] before building trials (D-31, D-34, plan 28-16): lets a
647/// host `[[test]] harness = false` binary register its own
648/// [`ScenarioRunner::register_graph`]/[`ScenarioRunner::register_registries`]
649/// constructors before the pattern is globbed, so a `registered` target
650/// resolves instead of reporting [`RunnerError::UnknownGraph`].
651pub fn run_eval_harness_main_with(pattern: &str, configure: impl FnOnce(&mut ScenarioRunner)) -> ! {
652    let args = Arguments::from_args();
653    let mut runner = ScenarioRunner::new();
654    configure(&mut runner);
655    let trials = runner.trials(pattern);
656    libtest_mimic::run(&args, trials).exit()
657}
658
659/// Expands to a `fn main()` for a `harness = false` `[[test]]` target (D-32):
660/// globs `$pattern` at runtime and hands one `libtest_mimic::Trial` per
661/// `(file, case)` pair to the custom harness, so `cargo test --test <name>
662/// <filter>` filters exactly like any other Rust test.
663///
664/// A second, optional argument registers graph constructors and registries
665/// factories before the pattern is globbed (D-31, D-34) -- needed by any
666/// `registered` (as opposed to `graph_doc`) scenario target.
667///
668/// # Examples
669///
670/// ```ignore
671/// // tests/evals.rs -- no registered targets
672/// paladin_eval::eval_scenarios!("evals/**/*.eval.yaml");
673/// ```
674///
675/// ```ignore
676/// // tests/evals.rs -- with registered targets
677/// paladin_eval::eval_scenarios!("evals/**/*.eval.yaml", |runner: &mut paladin_eval::ScenarioRunner| {
678///     runner.register_graph("my-graph", std::sync::Arc::new(|_ports| my_graph()));
679/// });
680/// ```
681#[macro_export]
682macro_rules! eval_scenarios {
683    ($pattern:expr) => {
684        fn main() {
685            $crate::runner::run_eval_harness_main($pattern)
686        }
687    };
688    ($pattern:expr, $configure:expr) => {
689        fn main() {
690            $crate::runner::run_eval_harness_main_with($pattern, $configure)
691        }
692    };
693}
694
695// ---------------------------------------------------------------------------
696// ScenarioPaladinPort -- the PaladinPort bridging every Paladin node's
697// LlmPort slot to its substituted ScenarioLlm (D-31).
698// ---------------------------------------------------------------------------
699
700/// Where a [`ScenarioPaladinPort`] reaches for its `LlmPort` (D-31, D-35).
701enum LlmSource {
702    /// Per-node scripted routing (the default, mock mode).
703    Scripted(HashMap<String, ScenarioLlm>),
704    /// A single real provider, resolved by [`resolve_live_provider`] once
705    /// [`check_live_mode`] has already passed (D-35): live mode uses ONE
706    /// provider for the whole run (routing by each node's own `model` is
707    /// the provider's own job, exactly as `LlmRequest::model` already
708    /// carries), never a per-node substitution the way scripted mode does.
709    Live(Arc<dyn LlmPort>),
710}
711
712/// A [`PaladinPort`] implementation routing each call to the [`ScenarioLlm`]
713/// substituted for that Paladin's own node (D-31): looked up by the
714/// `Paladin`'s own `PaladinData.name` -- the only identity
715/// [`PaladinPort::execute`] receives.
716struct ScenarioPaladinPort {
717    llm: LlmSource,
718}
719
720impl ScenarioPaladinPort {
721    fn scripted(router: HashMap<String, ScenarioLlm>) -> Self {
722        Self {
723            llm: LlmSource::Scripted(router),
724        }
725    }
726
727    fn live(provider: Arc<dyn LlmPort>) -> Self {
728        Self {
729            llm: LlmSource::Live(provider),
730        }
731    }
732}
733
734#[async_trait]
735impl PaladinPort for ScenarioPaladinPort {
736    async fn execute(&self, paladin: &Paladin, input: &str) -> Result<PaladinResult, PaladinError> {
737        let prompt = PromptItem::new(PromptType::User(UserPrompt {
738            query: input.to_string(),
739            context: None,
740        }))
741        .map_err(|source| PaladinError::ConfigurationError(source.to_string()))?;
742        let request = LlmRequest::new(paladin.data.model.clone(), prompt);
743
744        let response = match &self.llm {
745            LlmSource::Scripted(router) => {
746                let llm = router.get(&paladin.data.name).ok_or_else(|| {
747                    PaladinError::ConfigurationError(format!(
748                        "ScenarioPaladinPort: no ScenarioLlm registered for paladin {:?} \
749                         (known: {:?})",
750                        paladin.data.name,
751                        router.keys().collect::<Vec<_>>()
752                    ))
753                })?;
754                llm.generate(request).await
755            }
756            LlmSource::Live(provider) => provider.generate(request).await,
757        }
758        // The runner converts the real `LlmError` with
759        // `paladin_battalion::llm_failure::to_paladin_error`, which keeps the
760        // adapter's own transience: a scripted `error:` entry's
761        // `LlmErrorKind::Transient` survives as `Transience::Transient`, so a
762        // scenario's Aegis retry policy under the DEFAULT `TransientOnly`
763        // predicate can actually retry it (D-34's E2E-3 dogfood scenario
764        // depends on this). A string-only conversion would classify every
765        // failure `Transience::Unknown`, which `TransientOnly` never retries.
766        .map_err(|source| paladin_battalion::llm_failure::to_paladin_error(&source))?;
767
768        Ok(PaladinResult {
769            output: response.content,
770            usage: response.usage.clone(),
771            execution_time_ms: 0,
772            loop_count: 1,
773            stop_reason: StopReason::Completed,
774            ..Default::default()
775        })
776    }
777
778    async fn execute_stream(
779        &self,
780        paladin: &Paladin,
781        input: &str,
782    ) -> Result<PaladinStream, PaladinError> {
783        let result = self.execute(paladin, input).await?;
784        let (tx, rx) = tokio::sync::mpsc::channel(1);
785        let _ = tx
786            .send(Ok(PaladinStreamChunk {
787                text: result.output,
788                is_final: true,
789                metadata: None,
790            }))
791            .await;
792        Ok(rx)
793    }
794
795    fn validate(&self, _paladin: &Paladin) -> Result<(), PaladinError> {
796        Ok(())
797    }
798}
799
800fn build_port(
801    router: HashMap<String, ScenarioLlm>,
802    options: RunOptions,
803) -> Result<Arc<dyn PaladinPort>, RunnerError> {
804    if options.live {
805        let provider = resolve_live_provider()?;
806        Ok(Arc::new(ScenarioPaladinPort::live(provider)))
807    } else {
808        Ok(Arc::new(ScenarioPaladinPort::scripted(router)))
809    }
810}
811
812/// Build the [`ScenarioLlm`] router keyed by each Paladin node's OWN
813/// `PaladinData.name` (D-31): the identity [`PaladinPort::execute`]
814/// receives, mapped to a [`ScenarioLlm::for_node`] clone tagged with that
815/// node's GRAPH id -- the identity a scenario's `llm.per_node` map is keyed
816/// by (see e.g. `crates/paladin-battalion/tests/fixtures/graph_docs/linear.json`,
817/// whose Paladin nodes are authored `name: "Start"` under graph node id
818/// `"start"`, proving the two identities are not interchangeable).
819fn build_llm_router(graph: &WarGraph, root: &ScenarioLlm) -> HashMap<String, ScenarioLlm> {
820    let mut router = HashMap::new();
821    for node_id in graph.node_order() {
822        if let Some(NodeSpec::Paladin { paladin, .. }) = graph.node(node_id) {
823            router.insert(paladin.data.name.clone(), root.for_node(node_id.as_str()));
824        }
825    }
826    router
827}
828
829// ---------------------------------------------------------------------------
830// CapturingSink -- the TraceSink whose records feed AssertionContext (D-29).
831// ---------------------------------------------------------------------------
832
833/// A [`TraceSink`] that captures every [`TraceRecord`] it receives, in
834/// arrival order, for later evaluation against a case's assertions (D-29).
835struct CapturingSink {
836    records: Mutex<Vec<TraceRecord>>,
837}
838
839impl CapturingSink {
840    fn new() -> Self {
841        Self {
842            records: Mutex::new(Vec::new()),
843        }
844    }
845
846    fn records(&self) -> Vec<TraceRecord> {
847        self.records
848            .lock()
849            .expect("CapturingSink records lock poisoned")
850            .clone()
851    }
852}
853
854#[async_trait]
855impl TraceSink for CapturingSink {
856    async fn on_event(&self, record: TraceRecord) -> Result<(), TraceSinkError> {
857        self.records
858            .lock()
859            .expect("CapturingSink records lock poisoned")
860            .push(record);
861        Ok(())
862    }
863}
864
865// ---------------------------------------------------------------------------
866// Engine driving helpers
867// ---------------------------------------------------------------------------
868
869async fn run_plain<W: WaypointPort + 'static>(
870    store: Arc<W>,
871    graph: &WarGraph,
872    port: Arc<dyn PaladinPort>,
873    thread: ThreadId,
874    initial: StateDelta,
875    parley_responses: &[ParleyScript],
876) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
877    let sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
878    let engine = WarEngine::new(port, Arc::clone(&store)).with_trace_sink(sink.clone());
879    let outcome = engine.start(graph, thread.clone(), initial).await?;
880    let outcome = drive_parleys(&engine, graph, &thread, outcome, parley_responses).await?;
881
882    tokio::time::sleep(Duration::from_millis(50)).await;
883    let records = sink.records();
884    let final_state = resolve_battlefield(store.as_ref(), &thread, &outcome).await?;
885    Ok((records, outcome, final_state))
886}
887
888/// Answer every Parley a run raises, in order, from `parley_responses`
889/// (D-34) -- stopping WITHOUT error if fewer responses remain than the
890/// current `AwaitingInput` has outstanding requests, so a scenario that
891/// deliberately wants to assert `run_status: awaiting_input` (by supplying
892/// no further responses) is a supported, non-error terminal state.
893async fn drive_parleys<W: WaypointPort + 'static>(
894    engine: &WarEngine<W>,
895    graph: &WarGraph,
896    thread: &ThreadId,
897    mut outcome: RunOutcome,
898    parley_responses: &[ParleyScript],
899) -> Result<RunOutcome, RunnerError> {
900    let mut idx = 0usize;
901    while let RunOutcome::AwaitingInput { parleys, .. } = &outcome {
902        if idx + parleys.len() > parley_responses.len() {
903            break;
904        }
905        let mut responses = Vec::with_capacity(parleys.len());
906        for request in parleys {
907            let script = &parley_responses[idx];
908            idx += 1;
909            responses.push(ParleyResponse {
910                parley_id: request.parley_id,
911                kind: request.kind.clone(),
912                prompt: request.prompt.clone(),
913                value: script.value.clone(),
914                responded_by: Some("paladin-eval".to_string()),
915                responded_at: chrono::Utc::now(),
916                defaulted: false,
917            });
918        }
919        outcome = engine.resume_with(graph, thread.clone(), responses).await?;
920    }
921    Ok(outcome)
922}
923
924/// Resolve the [`Battlefield`] a terminal [`RunOutcome`] leaves behind:
925/// `Completed` carries it inline, every other terminal outcome's Battlefield
926/// is read back from its own persisted `Waypoint`.
927async fn resolve_battlefield<W: WaypointPort>(
928    store: &W,
929    thread: &ThreadId,
930    outcome: &RunOutcome,
931) -> Result<Battlefield, RunnerError> {
932    match outcome {
933        RunOutcome::Completed { final_state, .. } => Ok(final_state.clone()),
934        RunOutcome::Halted { waypoint, .. } | RunOutcome::AwaitingInput { waypoint, .. } => {
935            let loaded = store
936                .get(thread, waypoint)
937                .await?
938                .ok_or(RunnerError::NoFinalState)?;
939            Ok(loaded.battlefield)
940        }
941        RunOutcome::Failed {
942            waypoint: Some(waypoint),
943            ..
944        } => {
945            let loaded = store
946                .get(thread, waypoint)
947                .await?
948                .ok_or(RunnerError::NoFinalState)?;
949            Ok(loaded.battlefield)
950        }
951        RunOutcome::Failed { waypoint: None, .. } => Err(RunnerError::NoFinalState),
952    }
953}
954
955async fn full_history<W: WaypointPort>(
956    store: &W,
957    thread: &ThreadId,
958) -> Result<Vec<Waypoint>, RunnerError> {
959    let summaries = store.history(thread, None, None).await?;
960    let mut waypoints = Vec::with_capacity(summaries.len());
961    for summary in summaries {
962        if let Some(waypoint) = store.get(thread, &summary.waypoint_id).await? {
963            waypoints.push(waypoint);
964        }
965    }
966    Ok(waypoints)
967}
968
969fn temp_db_url(label: &str) -> String {
970    let path = std::env::temp_dir().join(format!("paladin-eval-{label}-{}.sqlite", Uuid::new_v4()));
971    format!("sqlite://{}", path.display())
972}
973
974fn resolve_relative(scenario_path: &Path, target: &Path) -> PathBuf {
975    if target.is_absolute() {
976        target.to_path_buf()
977    } else {
978        scenario_path
979            .parent()
980            .unwrap_or_else(|| Path::new("."))
981            .join(target)
982    }
983}
984
985fn load_graph_doc(path: &Path) -> Result<WarGraphDoc, RunnerError> {
986    let contents = std::fs::read_to_string(path).map_err(|source| RunnerError::Io {
987        path: path.to_path_buf(),
988        source,
989    })?;
990    let is_yaml = matches!(
991        path.extension().and_then(|ext| ext.to_str()),
992        Some("yaml") | Some("yml")
993    );
994    let doc: WarGraphDoc = if is_yaml {
995        serde_yaml::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
996            path: path.to_path_buf(),
997            message: source.to_string(),
998        })?
999    } else {
1000        serde_json::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
1001            path: path.to_path_buf(),
1002            message: source.to_string(),
1003        })?
1004    };
1005    Ok(doc)
1006}
1007
1008fn snapshot_file_path(scenario_path: &Path, case_name: &str) -> PathBuf {
1009    let dir = scenario_path.parent().unwrap_or_else(|| Path::new("."));
1010    let stem = scenario_path
1011        .file_stem()
1012        .and_then(|s| s.to_str())
1013        .unwrap_or("scenario");
1014    dir.join(format!("{stem}.{case_name}.snap.json"))
1015}
1016
1017/// Render a final `Battlefield` the same way `--bless` blesses it: every
1018/// declared field's raw JSON value, by name. Deliberately duplicated from
1019/// `crate::assertion`'s private, identically-shaped helper rather than
1020/// reaching into that module (this plan's own `files_modified` does not
1021/// touch `assertion.rs`) -- both read exactly `Battlefield::schema().fields`
1022/// and `Battlefield::get_raw`, the same two public accessors, so the two
1023/// stay trivially in agreement.
1024fn battlefield_snapshot_value(battlefield: &Battlefield) -> Value {
1025    let mut obj = serde_json::Map::new();
1026    for field in &battlefield.schema().fields {
1027        if let Some(value) = battlefield.get_raw(&field.name) {
1028            obj.insert(field.name.as_str().to_string(), value.clone());
1029        }
1030    }
1031    Value::Object(obj)
1032}
1033
1034// ---------------------------------------------------------------------------
1035// LiveMode (D-35)
1036// ---------------------------------------------------------------------------
1037//
1038// Promotion path: a scenario runs script-mocked, deterministic, and free
1039// (`ScenarioLlm`, the default) in CI on every `cargo test --test evals` and
1040// every plain `paladin-cli eval run`. Live mode is a DELIBERATE, three-times
1041// opt-in pre-release smoke check against a real provider -- never the
1042// default, never entered by accident, and never silently substituted when
1043// any one of the three gates below is missing.
1044
1045/// The environment variable [`check_live_mode`] requires to be set (to any
1046/// non-blank value) alongside `--live` before live mode is entered (D-35).
1047/// No workflow in this repository ever sets it -- default CI never spends a
1048/// provider credit.
1049pub const PALADIN_EVAL_LIVE_ENV: &str = "PALADIN_EVAL_LIVE";
1050
1051/// Why [`check_live_mode`] refused to enter live mode, or why
1052/// `resolve_live_provider` (private: the runner's own internal resolution
1053/// step) could not build a real provider once the gate passed (D-35). Never
1054/// converted into an implicit fallback to scripted mocks -- every variant
1055/// here is surfaced to the caller as a refusal.
1056#[derive(Debug, Error)]
1057pub enum LiveModeError {
1058    /// `--live` (or the equivalent `RunOptions::live`) was not set.
1059    #[error("live mode requires --live")]
1060    FlagNotSet,
1061    /// [`PALADIN_EVAL_LIVE_ENV`] is unset or blank.
1062    #[error(
1063        "live mode requires the {PALADIN_EVAL_LIVE_ENV} environment variable to be set to a \
1064         non-blank value (deliberately never set by any CI workflow in this repository)"
1065    )]
1066    EnvNotSet,
1067    /// No provider compiled into this build of `paladin-eval` has a
1068    /// configured credential (ADR-0012): `paladin_llm::provider_factory::
1069    /// LlmProviderFactory::get_default_provider()` returned `None`.
1070    #[error(
1071        "live mode requires a configured provider credential; none of paladin-llm's compiled-in \
1072         providers has one set (see ADR-0012)"
1073    )]
1074    NoProviderKey,
1075    /// The gate passed, but this build of `paladin-eval` was not compiled
1076    /// with its own `live` feature (which enables real `paladin-llm`
1077    /// provider adapters) -- refuses rather than silently falling back to a
1078    /// scripted mock.
1079    #[error(
1080        "live mode's gate passed, but this build of paladin-eval was not compiled with its own \
1081         `live` feature (enables real paladin-llm provider adapters); rebuild with \
1082         `--features live`"
1083    )]
1084    FeatureNotCompiled,
1085    /// The gate passed and the `live` feature is compiled in, but
1086    /// constructing the resolved provider adapter itself failed.
1087    #[error("failed to construct the live provider: {0}")]
1088    ProviderConstruction(String),
1089}
1090
1091/// The three-way live-mode gate (D-35): `--live` AND
1092/// [`PALADIN_EVAL_LIVE_ENV`] AND a configured provider credential, ALL
1093/// three, or [`LiveModeError`] naming exactly which is missing. Called by
1094/// [`ScenarioRunner::run_case`] itself whenever `options.live` is `true` --
1095/// never only by a caller that remembers to check first.
1096pub fn check_live_mode(live_flag: bool) -> Result<(), LiveModeError> {
1097    if !live_flag {
1098        return Err(LiveModeError::FlagNotSet);
1099    }
1100    let env_set = std::env::var(PALADIN_EVAL_LIVE_ENV)
1101        .map(|v| !v.trim().is_empty())
1102        .unwrap_or(false);
1103    if !env_set {
1104        return Err(LiveModeError::EnvNotSet);
1105    }
1106    if paladin_llm::provider_factory::LlmProviderFactory::get_default_provider().is_none() {
1107        return Err(LiveModeError::NoProviderKey);
1108    }
1109    Ok(())
1110}
1111
1112/// Whether `assertion` is content-bearing (D-35): compares rendered LLM
1113/// output rather than structural execution facts. Skipped in live mode
1114/// unless the scenario's own `live.allow_content_assertions` opts in --
1115/// every other assertion kind (route, status, node counts, edges, limits,
1116/// Parley) is structural and always runs in live mode regardless.
1117fn is_content_assertion(assertion: &Assertion) -> bool {
1118    matches!(
1119        assertion,
1120        Assertion::FinalStateFieldEquals { .. }
1121            | Assertion::FinalStateFieldMatches { .. }
1122            | Assertion::FieldJsonPathEquals { .. }
1123            | Assertion::FinalStateSnapshot
1124    )
1125}
1126
1127/// The pure skip-decision [`ScenarioRunner::run_case`]'s assertion loop
1128/// applies (D-35): `true` only when the run is live, `assertion` is
1129/// content-bearing, and the scenario did not opt in. A structural assertion
1130/// (`is_content_assertion` false) is never skipped regardless of `live` --
1131/// it always evaluates, live or scripted.
1132fn should_skip_for_live(assertion: &Assertion, live: bool, allow_content_assertions: bool) -> bool {
1133    live && is_content_assertion(assertion) && !allow_content_assertions
1134}
1135
1136/// Resolve a real provider once [`check_live_mode`] has already passed
1137/// (D-35): live mode uses `paladin_llm::provider_factory`'s first available
1138/// (configured-credential) provider for the whole run. Compiled only when
1139/// this crate's own `live` feature is enabled -- see
1140/// [`LiveModeError::FeatureNotCompiled`] for the refusal a build without it
1141/// gives instead.
1142#[cfg(feature = "live")]
1143fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
1144    let provider_name = paladin_llm::provider_factory::LlmProviderFactory::get_default_provider()
1145        .ok_or(LiveModeError::NoProviderKey)?;
1146    paladin_llm::provider_factory::LlmProviderFactory::new()
1147        .create(&provider_name)
1148        .map_err(|source| LiveModeError::ProviderConstruction(source.to_string()))
1149}
1150
1151/// See the `#[cfg(feature = "live")]` sibling above -- this build was not
1152/// compiled with `paladin-eval`'s own `live` feature, so live mode's gate
1153/// may pass (a credential IS configured) while this crate still cannot
1154/// build the real adapter itself.
1155#[cfg(not(feature = "live"))]
1156fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
1157    Err(LiveModeError::FeatureNotCompiled)
1158}
1159
1160// ---------------------------------------------------------------------------
1161// RunnerError
1162// ---------------------------------------------------------------------------
1163
1164/// Errors [`ScenarioRunner::run_case`] can encounter resolving, compiling or
1165/// running a case -- always folded into [`CaseOutcome::Errored`] by
1166/// [`ScenarioRunner::run_case`] itself, never propagated as a panic.
1167#[derive(Debug, Error)]
1168pub enum RunnerError {
1169    /// A file (a `graph_doc` target, or a snapshot write) could not be read
1170    /// or written.
1171    #[error("failed to access {path}: {source}")]
1172    Io {
1173        /// The file that could not be accessed.
1174        path: PathBuf,
1175        /// The underlying I/O error.
1176        #[source]
1177        source: std::io::Error,
1178    },
1179    /// A `graph_doc` target's file did not deserialize as a `WarGraphDoc`.
1180    #[error("failed to parse WarGraphDoc {path}: {message}")]
1181    ParseGraphDoc {
1182        /// The file that failed to parse.
1183        path: PathBuf,
1184        /// The underlying parse error's message.
1185        message: String,
1186    },
1187    /// A `graph_doc` target's document failed `WarGraphDoc::compile`.
1188    #[error("failed to compile WarGraphDoc {path}: {source}")]
1189    Compile {
1190        /// The file that failed to compile.
1191        path: PathBuf,
1192        /// The underlying compile error. Boxed (`clippy::result_large_err`):
1193        /// `CompileError`'s largest variant is well over the lint's
1194        /// threshold, and every `Result<_, RunnerError>` in this module pays
1195        /// that size on its happy path too if left unboxed.
1196        #[source]
1197        source: Box<paladin_battalion::engine::graph_doc::CompileError>,
1198    },
1199    /// A `registered` target names a [`GraphConstructor`] no
1200    /// [`ScenarioRunner::register_graph`] call registered.
1201    #[error("scenario names unregistered graph constructor {name:?}; registered: {registered:?}")]
1202    UnknownGraph {
1203        /// The unresolved name.
1204        name: String,
1205        /// Every name this runner does have registered.
1206        registered: Vec<String>,
1207    },
1208    /// A scenario's `registries` field names a [`RegistriesFactory`] no
1209    /// [`ScenarioRunner::register_registries`] call registered.
1210    #[error("scenario names unregistered registries {name:?}; registered: {registered:?}")]
1211    UnknownRegistries {
1212        /// The unresolved name.
1213        name: String,
1214        /// Every name this runner does have registered.
1215        registered: Vec<String>,
1216    },
1217    /// `interrupt_after_superstep` was set on a case whose scenario does not
1218    /// use `store: sqlite_temp`.
1219    #[error("case {case:?}: interrupt_after_superstep requires store: sqlite_temp")]
1220    InterruptRequiresSqliteTemp {
1221        /// The offending case's name.
1222        case: String,
1223    },
1224    /// The `WaypointPort` backing this run reported an error.
1225    #[error("waypoint store error: {0}")]
1226    Waypoint(#[from] WaypointError),
1227    /// The engine itself reported an error (never a `RunOutcome::Failed`,
1228    /// which is a normal terminal outcome an assertion can check).
1229    #[error("engine error: {0}")]
1230    Engine(#[from] EngineError),
1231    /// A case's `input` map named a field whose name is not a valid
1232    /// `FieldName`.
1233    #[error("invalid field name {field:?}: {message}")]
1234    InvalidField {
1235        /// The offending field name.
1236        field: String,
1237        /// The underlying validation error's message.
1238        message: String,
1239    },
1240    /// A terminal `RunOutcome` left no persisted Waypoint to read the final
1241    /// `Battlefield` back from.
1242    #[error("run finished with no persisted final state to evaluate assertions against")]
1243    NoFinalState,
1244    /// The generated `ThreadId` for this run failed validation.
1245    #[error("invalid thread id: {0}")]
1246    InvalidThreadId(String),
1247    /// `options.live` was set but the three-way live-mode gate refused
1248    /// (D-35) -- never a silent fallback to scripted mocks.
1249    #[error("live mode: {0}")]
1250    Live(#[from] LiveModeError),
1251}
1252
1253#[cfg(test)]
1254mod tests {
1255    use super::*;
1256    use crate::scenario::{
1257        Case as ScenarioCase, LlmScript, RunStatusValue, Scenario as ScenarioDoc, ScenarioTarget,
1258        ScriptEntry, StoreKind as ScenarioStoreKind, Times,
1259    };
1260    use paladin_battalion::engine::graph::{EdgeSpec, EngineLimits};
1261    use paladin_battalion::engine::node::{NodeContext, StateNode, StateNodeError};
1262    use paladin_battalion::engine::{InputMapping, NodeSpec as EngineNodeSpec};
1263    use paladin_core::base::entity::Entity;
1264    use paladin_core::platform::container::battlefield::{
1265        BattlefieldSchema, DispatchRule, FieldSpec,
1266    };
1267    use paladin_core::platform::container::directive::Directive;
1268    use paladin_core::platform::container::paladin::{MaxLoops, PaladinData, PaladinStatus};
1269    use paladin_core::platform::container::waypoint::NodeId;
1270    use std::collections::BTreeMap;
1271
1272    fn field(name: &str) -> FieldName {
1273        FieldName::new(name).expect("valid field name")
1274    }
1275
1276    fn make_paladin(name: &str, model: &str) -> Paladin {
1277        let data = PaladinData {
1278            system_prompt: format!("{name} prompt"),
1279            name: name.to_string(),
1280            user_name: String::new(),
1281            model: model.to_string(),
1282            temperature: 0.7,
1283            max_loops: MaxLoops::Fixed(1),
1284            stop_words: vec![],
1285            status: PaladinStatus::Idle,
1286            vision_enabled: false,
1287            ..Default::default()
1288        };
1289        Entity::new(data, Some(name.to_string()))
1290    }
1291
1292    fn minimal_scenario(
1293        target: ScenarioTarget,
1294        llm: LlmScript,
1295        assertions: Vec<Assertion>,
1296    ) -> ScenarioDoc {
1297        ScenarioDoc {
1298            schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
1299            target,
1300            store: ScenarioStoreKind::InMemory,
1301            llm,
1302            cases: vec![ScenarioCase {
1303                name: "happy_path".to_string(),
1304                input: BTreeMap::from([("topic".to_string(), serde_json::json!("widgets"))]),
1305                interrupt_after_superstep: None,
1306                parley_responses: vec![],
1307                llm: None,
1308                assertions,
1309            }],
1310            live: Default::default(),
1311            registries: None,
1312        }
1313    }
1314
1315    fn linear_fixture_path() -> PathBuf {
1316        // Cross-crate fixture reuse (D-31 note in the module doc): the exact
1317        // document `crates/paladin-battalion`'s own tests compile, proving a
1318        // `graph_doc` target's relative-path resolution works against a
1319        // fixture this crate does not own a copy of.
1320        PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1321            .join("../paladin-battalion/tests/fixtures/graph_docs/linear.json")
1322    }
1323
1324    #[tokio::test]
1325    async fn runner_executes_a_graph_doc_case() {
1326        let scenario = minimal_scenario(
1327            ScenarioTarget::GraphDoc {
1328                graph_doc: PathBuf::from(
1329                    "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1330                ),
1331            },
1332            LlmScript {
1333                global: vec![
1334                    ScriptEntry::Text("s1".to_string()),
1335                    ScriptEntry::Text("s2".to_string()),
1336                    ScriptEntry::Text("s3".to_string()),
1337                ],
1338                ..Default::default()
1339            },
1340            vec![
1341                Assertion::NodeExecuted {
1342                    node: "start".to_string(),
1343                    times: Times::Exact(1),
1344                },
1345                Assertion::RunStatus(RunStatusValue::Completed),
1346            ],
1347        );
1348
1349        let runner = ScenarioRunner::new();
1350        // The scenario's own `target.graph_doc` path is authored relative to
1351        // this crate's own manifest dir (a stand-in "scenario file" location)
1352        // -- `linear_fixture_path()` below proves the SAME resolved path.
1353        let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1354        let report = runner
1355            .run_case(
1356                &scenario_path,
1357                &scenario,
1358                &scenario.cases[0],
1359                RunOptions::default(),
1360            )
1361            .await;
1362
1363        assert!(
1364            report.passed(),
1365            "expected the case to pass: {}",
1366            report.render_failures()
1367        );
1368        assert!(linear_fixture_path().exists(), "sanity: fixture must exist");
1369    }
1370
1371    #[tokio::test]
1372    async fn runner_reports_a_failing_assertion_with_its_rendering() {
1373        let scenario = minimal_scenario(
1374            ScenarioTarget::GraphDoc {
1375                graph_doc: PathBuf::from(
1376                    "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1377                ),
1378            },
1379            LlmScript {
1380                global: vec![
1381                    ScriptEntry::Text("s1".to_string()),
1382                    ScriptEntry::Text("s2".to_string()),
1383                    ScriptEntry::Text("s3".to_string()),
1384                ],
1385                ..Default::default()
1386            },
1387            vec![Assertion::NodeExecuted {
1388                node: "start".to_string(),
1389                times: Times::Exact(5),
1390            }],
1391        );
1392
1393        let runner = ScenarioRunner::new();
1394        let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1395        let report = runner
1396            .run_case(
1397                &scenario_path,
1398                &scenario,
1399                &scenario.cases[0],
1400                RunOptions::default(),
1401            )
1402            .await;
1403
1404        assert!(!report.passed());
1405        let rendered = report.render_failures();
1406        assert!(
1407            rendered.contains("expected: exactly 5"),
1408            "rendering must be the assertion's own render_failure() output: {rendered:?}"
1409        );
1410    }
1411
1412    #[tokio::test]
1413    async fn runner_executes_a_registered_target() {
1414        let mut runner = ScenarioRunner::new();
1415        runner.register_graph(
1416            "solo",
1417            Arc::new(|ports: &ScriptedPorts| {
1418                let schema = BattlefieldSchema::new(vec![FieldSpec::new(
1419                    field("out"),
1420                    DispatchRule::LastWrite,
1421                    None,
1422                    false,
1423                )]);
1424                let mut graph = WarGraph::new(schema, EngineLimits::default());
1425                let node = NodeId::new("solo");
1426                let llm = ports.for_node("solo");
1427                graph.add_node(
1428                    node.clone(),
1429                    EngineNodeSpec::Function(Arc::new(RegisteredEchoNode {
1430                        llm,
1431                        output_field: field("out"),
1432                    })),
1433                );
1434                graph.add_entry(node);
1435                graph
1436            }),
1437        );
1438
1439        let mut scenario = minimal_scenario(
1440            ScenarioTarget::Registered {
1441                registered: "solo".to_string(),
1442            },
1443            LlmScript {
1444                per_node: BTreeMap::from([(
1445                    "solo".to_string(),
1446                    vec![ScriptEntry::Text("registered-output".to_string())],
1447                )]),
1448                ..Default::default()
1449            },
1450            vec![Assertion::FinalStateFieldEquals {
1451                field: "out".to_string(),
1452                value: serde_json::json!("registered-output"),
1453            }],
1454        );
1455        // The "solo" graph's own schema declares only `out` -- unlike
1456        // `minimal_scenario`'s default `{"topic": "widgets"}` input, which
1457        // targets the `linear.json` fixture's schema instead.
1458        scenario.cases[0].input = BTreeMap::new();
1459
1460        let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1461        let report = runner
1462            .run_case(
1463                &scenario_path,
1464                &scenario,
1465                &scenario.cases[0],
1466                RunOptions::default(),
1467            )
1468            .await;
1469        assert!(
1470            report.passed(),
1471            "expected the registered target's Function node to receive the scripted port: {}",
1472            report.render_failures()
1473        );
1474    }
1475
1476    struct RegisteredEchoNode {
1477        llm: ScenarioLlm,
1478        output_field: FieldName,
1479    }
1480
1481    #[async_trait::async_trait]
1482    impl StateNode for RegisteredEchoNode {
1483        async fn run(
1484            &self,
1485            _state: &Battlefield,
1486            _ctx: &NodeContext,
1487        ) -> Result<Directive, StateNodeError> {
1488            let request = LlmRequest::new(
1489                "scenario-model",
1490                PromptItem::new(PromptType::User(UserPrompt {
1491                    query: "go".to_string(),
1492                    context: None,
1493                }))
1494                .expect("valid prompt"),
1495            );
1496            let response = self
1497                .llm
1498                .generate(request)
1499                .await
1500                .map_err(|e| StateNodeError(e.to_string()))?;
1501            let mut delta = StateDelta::new();
1502            delta
1503                .set(self.output_field.clone(), response.content)
1504                .map_err(|e| StateNodeError(e.to_string()))?;
1505            Ok(delta.into())
1506        }
1507    }
1508
1509    #[tokio::test]
1510    async fn interrupt_and_resume_reproduces_the_control_run() {
1511        // A two-Paladin-node chain with PER-NODE scripts (see this module's
1512        // own doc comment on the global-vs-per-node caveat).
1513        fn build_graph() -> WarGraph {
1514            let schema = BattlefieldSchema::new(vec![
1515                FieldSpec::new(
1516                    field("topic"),
1517                    DispatchRule::LastWrite,
1518                    Some(serde_json::json!("widgets")),
1519                    false,
1520                ),
1521                FieldSpec::new(field("step_one"), DispatchRule::LastWrite, None, false),
1522                FieldSpec::new(field("step_two"), DispatchRule::LastWrite, None, false),
1523            ]);
1524            let mut graph = WarGraph::new(schema, EngineLimits::default());
1525            let first = NodeId::new("first");
1526            let second = NodeId::new("second");
1527            graph.add_node(
1528                first.clone(),
1529                EngineNodeSpec::paladin(
1530                    make_paladin("First", "gpt-4"),
1531                    InputMapping::new("{topic}"),
1532                    field("step_one"),
1533                ),
1534            );
1535            graph.add_node(
1536                second.clone(),
1537                EngineNodeSpec::paladin(
1538                    make_paladin("Second", "gpt-4"),
1539                    InputMapping::new("{step_one}"),
1540                    field("step_two"),
1541                ),
1542            );
1543            graph.add_edge(EdgeSpec {
1544                from: first.clone(),
1545                to: second,
1546                condition: None,
1547            });
1548            graph.add_entry(first);
1549            graph
1550        }
1551
1552        let mut runner = ScenarioRunner::new();
1553        runner.register_graph("chain", Arc::new(|_ports: &ScriptedPorts| build_graph()));
1554
1555        let llm = LlmScript {
1556            per_node: BTreeMap::from([
1557                (
1558                    "first".to_string(),
1559                    vec![ScriptEntry::Text("one".to_string())],
1560                ),
1561                (
1562                    "second".to_string(),
1563                    vec![ScriptEntry::Text("two".to_string())],
1564                ),
1565            ]),
1566            ..Default::default()
1567        };
1568
1569        let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1570
1571        // Control: no interruption.
1572        let control_scenario = ScenarioDoc {
1573            schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
1574            target: ScenarioTarget::Registered {
1575                registered: "chain".to_string(),
1576            },
1577            store: ScenarioStoreKind::SqliteTemp,
1578            llm: llm.clone(),
1579            cases: vec![ScenarioCase {
1580                name: "control".to_string(),
1581                input: BTreeMap::new(),
1582                interrupt_after_superstep: None,
1583                parley_responses: vec![],
1584                llm: None,
1585                assertions: vec![],
1586            }],
1587            live: Default::default(),
1588            registries: None,
1589        };
1590        let (control_graph, control_router) = runner
1591            .resolve_target(&scenario_path, &control_scenario)
1592            .expect("resolve control target");
1593        let control_port =
1594            build_port(control_router, RunOptions::default()).expect("scripted port");
1595        let store = Arc::new(InMemoryWaypointStore::new());
1596        let thread = ThreadId::new(format!("control-{}", Uuid::new_v4())).expect("valid thread id");
1597        let (_records, _outcome, control_final_state) = run_plain(
1598            store,
1599            &control_graph,
1600            control_port,
1601            thread,
1602            StateDelta::new(),
1603            &[],
1604        )
1605        .await
1606        .expect("control run succeeds");
1607
1608        // Interrupted: crash after superstep 1 (only `first` has completed).
1609        let interrupted_scenario = ScenarioDoc {
1610            store: ScenarioStoreKind::SqliteTemp,
1611            cases: vec![ScenarioCase {
1612                name: "interrupted".to_string(),
1613                interrupt_after_superstep: Some(1),
1614                ..control_scenario.cases[0].clone()
1615            }],
1616            ..control_scenario
1617        };
1618        let report = runner
1619            .run_case(
1620                &scenario_path,
1621                &interrupted_scenario,
1622                &interrupted_scenario.cases[0],
1623                RunOptions::default(),
1624            )
1625            .await;
1626
1627        match &report.outcome {
1628            CaseOutcome::Ran { final_state, .. } => {
1629                assert_eq!(
1630                    *final_state, control_final_state,
1631                    "resumed final Battlefield must equal the control run's"
1632                );
1633            }
1634            CaseOutcome::Errored(message) => panic!("interrupted case errored: {message}"),
1635        }
1636    }
1637
1638    #[test]
1639    fn evals_target_discovers_one_trial_per_case() {
1640        let dir = std::env::temp_dir().join(format!("paladin-eval-trials-{}", Uuid::new_v4()));
1641        std::fs::create_dir_all(&dir).expect("create temp dir");
1642
1643        // Deliberately `alpha.yaml`/`beta.yaml`, not the `.eval.yaml` double
1644        // extension the real `evals/` tree uses: `Path::file_stem` only
1645        // strips the LAST extension, so a `<name>.eval.yaml` file's own
1646        // trial-name stem would be `<name>.eval` -- a single extension here
1647        // keeps this test's expected names exactly `alpha`/`beta`.
1648        std::fs::write(
1649            dir.join("alpha.yaml"),
1650            r#"
1651schema_version: "1"
1652target:
1653  registered: "nowhere"
1654cases:
1655  - name: case_one
1656    assertions: []
1657  - name: case_two
1658    assertions: []
1659"#,
1660        )
1661        .expect("write alpha fixture");
1662        std::fs::write(
1663            dir.join("beta.yaml"),
1664            r#"
1665schema_version: "1"
1666target:
1667  registered: "nowhere"
1668cases:
1669  - name: only_case
1670    assertions: []
1671"#,
1672        )
1673        .expect("write beta fixture");
1674
1675        let runner = ScenarioRunner::new();
1676        let pattern = dir.join("*.yaml");
1677        let trials = runner.trials(pattern.to_str().expect("valid utf8 path"));
1678
1679        assert_eq!(trials.len(), 3, "one trial per (file, case) pair");
1680        let names: Vec<&str> = trials.iter().map(Trial::name).collect();
1681        assert!(names.contains(&"alpha::case_one"));
1682        assert!(names.contains(&"alpha::case_two"));
1683        assert!(names.contains(&"beta::only_case"));
1684
1685        let filter = Arguments {
1686            filter: Some("case_one".to_string()),
1687            ..Default::default()
1688        };
1689        let selected: Vec<&Trial> = trials
1690            .iter()
1691            .filter(|t| !filter.is_filtered_out(t))
1692            .collect();
1693        assert_eq!(selected.len(), 1, "a name filter must select a single case");
1694        assert_eq!(selected[0].name(), "alpha::case_one");
1695
1696        let _ = std::fs::remove_dir_all(&dir);
1697    }
1698
1699    // -- Task 3: the LiveMode gate (D-35) -----------------------------------
1700    //
1701    // These exercise only the GATE and CLASSIFICATION logic, never a real
1702    // provider call: `live_mode_requires_flag_and_env_and_keys` is the only
1703    // test in this module that reads/writes `PALADIN_EVAL_LIVE_ENV`, so it
1704    // owns that env var exclusively (no other test depends on its value) --
1705    // the `unsafe` blocks are edition-2024's `std::env::set_var`/`remove_var`
1706    // requirement, not a signal these mutations are broadly unsafe here.
1707
1708    #[test]
1709    fn live_mode_requires_flag_and_env_and_keys() {
1710        // SAFETY: this is the ONLY test in this module (or crate) reading or
1711        // writing PALADIN_EVAL_LIVE_ENV; no other test's outcome depends on
1712        // its value, so this process-wide mutation cannot race a concurrent
1713        // test's own assertion.
1714        unsafe {
1715            std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
1716        }
1717
1718        assert!(matches!(
1719            check_live_mode(false),
1720            Err(LiveModeError::FlagNotSet)
1721        ));
1722        assert!(matches!(
1723            check_live_mode(true),
1724            Err(LiveModeError::EnvNotSet)
1725        ));
1726
1727        // SAFETY: see above.
1728        unsafe {
1729            std::env::set_var(PALADIN_EVAL_LIVE_ENV, "1");
1730        }
1731        // The third stage depends on which providers are compiled in. This
1732        // crate's own `paladin-llm` edge is `default-features = false,
1733        // features = ["mock"]`, so a crate-local test build has no provider
1734        // and the gate refuses with `NoProviderKey` (ADR-0012: never a mock
1735        // fallback). In a WORKSPACE build cargo's feature unification can
1736        // compile real providers in (e.g. `--features llm-all` brings Ollama,
1737        // which needs no key), and then `get_default_provider()` is `Some`
1738        // and the gate correctly passes -- assert whichever this build
1739        // actually is, so the test is honest under every feature set the CI
1740        // matrix runs.
1741        let compiled_provider =
1742            paladin_llm::provider_factory::LlmProviderFactory::get_default_provider();
1743        match compiled_provider {
1744            None => assert!(matches!(
1745                check_live_mode(true),
1746                Err(LiveModeError::NoProviderKey)
1747            )),
1748            Some(_) => assert!(check_live_mode(true).is_ok()),
1749        }
1750
1751        // SAFETY: see above.
1752        unsafe {
1753            std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
1754        }
1755    }
1756
1757    fn content_assertion_fixtures() -> Vec<Assertion> {
1758        vec![
1759            Assertion::FinalStateFieldEquals {
1760                field: "status".to_string(),
1761                value: serde_json::json!("done"),
1762            },
1763            Assertion::FinalStateFieldMatches {
1764                field: "status".to_string(),
1765                pattern: "^do".to_string(),
1766            },
1767            Assertion::FieldJsonPathEquals {
1768                path: "/status".to_string(),
1769                value: serde_json::json!("done"),
1770            },
1771            Assertion::FinalStateSnapshot,
1772        ]
1773    }
1774
1775    fn structural_assertion_fixtures() -> Vec<Assertion> {
1776        vec![
1777            Assertion::NodeExecuted {
1778                node: "worker".to_string(),
1779                times: Times::Exact(1),
1780            },
1781            Assertion::NodeNotExecuted {
1782                node: "idle".to_string(),
1783            },
1784            Assertion::EdgeFired {
1785                from: "a".to_string(),
1786                to: "b".to_string(),
1787            },
1788            Assertion::RouteTaken(vec!["a".to_string()]),
1789            Assertion::RunStatus(RunStatusValue::Completed),
1790            Assertion::TotalTokensMax(10),
1791            Assertion::SuperstepsMax(10),
1792            Assertion::ParleyRaised {
1793                kind: "approval".to_string(),
1794                node: "worker".to_string(),
1795            },
1796        ]
1797    }
1798
1799    #[test]
1800    fn content_assertions_are_skipped_in_live_mode_without_opt_in() {
1801        for assertion in content_assertion_fixtures() {
1802            assert!(
1803                should_skip_for_live(&assertion, true, false),
1804                "{assertion:?} must be skipped in live mode without the scenario's opt-in"
1805            );
1806        }
1807    }
1808
1809    #[test]
1810    fn content_assertions_run_in_live_mode_with_opt_in() {
1811        for assertion in content_assertion_fixtures() {
1812            assert!(
1813                !should_skip_for_live(&assertion, true, true),
1814                "{assertion:?} must run in live mode once the scenario opts in"
1815            );
1816        }
1817    }
1818
1819    #[test]
1820    fn structural_assertions_always_run_in_live_mode() {
1821        for assertion in structural_assertion_fixtures() {
1822            assert!(
1823                !should_skip_for_live(&assertion, true, false),
1824                "{assertion:?} is structural and must always run in live mode"
1825            );
1826            assert!(
1827                !should_skip_for_live(&assertion, true, true),
1828                "{assertion:?} is structural and must always run in live mode, opt-in or not"
1829            );
1830        }
1831        // Sanity: never skipped outside live mode either, regardless of kind.
1832        for assertion in content_assertion_fixtures()
1833            .into_iter()
1834            .chain(structural_assertion_fixtures())
1835        {
1836            assert!(!should_skip_for_live(&assertion, false, false));
1837        }
1838    }
1839
1840    #[tokio::test]
1841    async fn default_test_run_never_enters_live_mode() {
1842        // A plain RunOptions::default() (`live: false`, matching every
1843        // eval_scenarios! harness trial) must run entirely on scripted
1844        // mocks -- proven by SUCCEEDING here with no PALADIN_EVAL_LIVE_ENV
1845        // set and no provider credential available at all: if this code
1846        // path ever silently entered live mode, `check_live_mode` would
1847        // refuse (no env var, no key) and this case would error instead of
1848        // passing.
1849        let scenario = minimal_scenario(
1850            ScenarioTarget::GraphDoc {
1851                graph_doc: PathBuf::from(
1852                    "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1853                ),
1854            },
1855            LlmScript {
1856                global: vec![
1857                    ScriptEntry::Text("s1".to_string()),
1858                    ScriptEntry::Text("s2".to_string()),
1859                    ScriptEntry::Text("s3".to_string()),
1860                ],
1861                ..Default::default()
1862            },
1863            vec![Assertion::RunStatus(RunStatusValue::Completed)],
1864        );
1865
1866        let runner = ScenarioRunner::new();
1867        let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1868        assert!(
1869            !RunOptions::default().live,
1870            "the harness's own default options must never carry live: true"
1871        );
1872        let report = runner
1873            .run_case(
1874                &scenario_path,
1875                &scenario,
1876                &scenario.cases[0],
1877                RunOptions::default(),
1878            )
1879            .await;
1880        assert!(
1881            report.passed(),
1882            "a plain (non-live) run must succeed entirely on scripted mocks: {}",
1883            report.render_failures()
1884        );
1885    }
1886}