1use std::collections::HashMap;
54use std::path::{Path, PathBuf};
55use std::sync::{Arc, Mutex};
56use std::time::Duration;
57
58use async_trait::async_trait;
59use libtest_mimic::{Arguments, Failed, Trial};
60use serde_json::Value;
61use thiserror::Error;
62use uuid::Uuid;
63
64use paladin_battalion::engine::graph_doc::WarGraphDoc;
65use paladin_battalion::engine::{
66 EngineError, EngineRegistries, NodeSpec, RunOutcome, WarEngine, WarGraph,
67};
68use paladin_core::platform::container::battlefield::{Battlefield, FieldName, StateDelta};
69use paladin_core::platform::container::paladin::Paladin;
70use paladin_core::platform::container::paladin_error::PaladinError;
71use paladin_core::platform::container::parley::ParleyResponse;
72use paladin_core::platform::container::prompt::{PromptItem, PromptType, UserPrompt};
73use paladin_core::platform::container::trace::TraceRecord;
74use paladin_core::platform::container::waypoint::{ThreadId, Waypoint};
75use paladin_ports::output::llm_port::{LlmPort, LlmRequest};
76use paladin_ports::output::paladin_port::{
77 PaladinPort, PaladinResult, PaladinStream, PaladinStreamChunk, StopReason,
78};
79use paladin_ports::output::trace_sink_port::{TraceSink, TraceSinkError};
80use paladin_ports::output::waypoint_port::{WaypointError, WaypointPort};
81use paladin_storage::waypoint::in_memory::InMemoryWaypointStore;
82use paladin_storage::waypoint::sqlite::SqliteWaypointStore;
83
84use crate::assertion::{self, AssertionContext, AssertionFailure, AssertionOutcome};
85use crate::scenario::{Assertion, Case, ParleyScript, Scenario, ScenarioTarget, StoreKind};
86use crate::scripted_llm::ScenarioLlm;
87
88pub struct ScriptedPorts {
101 root: ScenarioLlm,
102}
103
104impl ScriptedPorts {
105 pub fn for_node(&self, node_id: impl Into<String>) -> ScenarioLlm {
108 self.root.for_node(node_id)
109 }
110}
111
112pub type GraphConstructor = Arc<dyn Fn(&ScriptedPorts) -> WarGraph + Send + Sync>;
116
117pub type RegistriesFactory = Arc<dyn Fn() -> EngineRegistries + Send + Sync>;
121
122#[derive(Debug, Clone, Copy, Default)]
132pub struct RunOptions {
133 pub bless: bool,
138 pub live: bool,
142}
143
144#[derive(Debug)]
150pub struct AssertionResult {
151 pub verdict: Verdict,
153}
154
155#[derive(Debug)]
157pub enum Verdict {
158 Passed,
160 Failed(AssertionFailure),
162 Skipped(String),
168}
169
170pub struct CaseReport {
173 pub case_name: String,
175 pub outcome: CaseOutcome,
179}
180
181pub enum CaseOutcome {
183 Ran {
186 records: Vec<TraceRecord>,
188 final_state: Battlefield,
190 results: Vec<AssertionResult>,
192 },
193 Errored(String),
197}
198
199impl CaseReport {
200 pub fn passed(&self) -> bool {
203 match &self.outcome {
204 CaseOutcome::Ran { results, .. } => !results
205 .iter()
206 .any(|r| matches!(r.verdict, Verdict::Failed(_))),
207 CaseOutcome::Errored(_) => false,
208 }
209 }
210
211 pub fn records(&self) -> &[TraceRecord] {
214 match &self.outcome {
215 CaseOutcome::Ran { records, .. } => records,
216 CaseOutcome::Errored(_) => &[],
217 }
218 }
219
220 pub fn render_failures(&self) -> String {
224 match &self.outcome {
225 CaseOutcome::Errored(message) => {
226 format!(
227 "case {:?} errored before assertions could run: {message}",
228 self.case_name
229 )
230 }
231 CaseOutcome::Ran { results, .. } => {
232 let mut out = String::new();
233 for result in results {
234 if let Verdict::Failed(failure) = &result.verdict {
235 out.push_str(&failure.render_failure());
236 }
237 }
238 if out.is_empty() {
239 "(no failing assertions)".to_string()
240 } else {
241 out
242 }
243 }
244 }
245 }
246}
247
248#[derive(Clone, Default)]
261pub struct ScenarioRunner {
262 graphs: HashMap<String, GraphConstructor>,
263 registries: HashMap<String, RegistriesFactory>,
264}
265
266impl ScenarioRunner {
267 pub fn new() -> Self {
269 Self::default()
270 }
271
272 pub fn register_graph(&mut self, name: impl Into<String>, ctor: GraphConstructor) -> &mut Self {
276 self.graphs.insert(name.into(), ctor);
277 self
278 }
279
280 pub fn register_registries(
283 &mut self,
284 name: impl Into<String>,
285 factory: RegistriesFactory,
286 ) -> &mut Self {
287 self.registries.insert(name.into(), factory);
288 self
289 }
290
291 fn resolve_registries(&self, name: Option<&str>) -> Result<EngineRegistries, RunnerError> {
292 match name {
293 None => Ok(EngineRegistries::new()),
294 Some(name) => {
295 let factory =
296 self.registries
297 .get(name)
298 .ok_or_else(|| RunnerError::UnknownRegistries {
299 name: name.to_string(),
300 registered: self.registries.keys().cloned().collect(),
301 })?;
302 Ok((factory)())
303 }
304 }
305 }
306
307 fn resolve_target(
313 &self,
314 scenario_path: &Path,
315 scenario: &Scenario,
316 ) -> Result<(WarGraph, HashMap<String, ScenarioLlm>), RunnerError> {
317 match &scenario.target {
318 ScenarioTarget::GraphDoc { graph_doc } => {
319 let path = resolve_relative(scenario_path, graph_doc);
320 let doc = load_graph_doc(&path)?;
321 let registries = self.resolve_registries(scenario.registries.as_deref())?;
322 let graph = doc
323 .compile(®istries)
324 .map_err(|source| RunnerError::Compile {
325 path: path.clone(),
326 source: Box::new(source),
327 })?;
328 let root = ScenarioLlm::new(scenario.llm.clone());
329 let router = build_llm_router(&graph, &root);
330 Ok((graph, router))
331 }
332 ScenarioTarget::Registered { registered } => {
333 let ctor =
334 self.graphs
335 .get(registered)
336 .ok_or_else(|| RunnerError::UnknownGraph {
337 name: registered.clone(),
338 registered: self.graphs.keys().cloned().collect(),
339 })?;
340 let root = ScenarioLlm::new(scenario.llm.clone());
341 let scripted_ports = ScriptedPorts { root: root.clone() };
342 let graph = (ctor)(&scripted_ports);
343 let router = build_llm_router(&graph, &root);
344 Ok((graph, router))
345 }
346 }
347 }
348
349 pub async fn run_case(
353 &self,
354 scenario_path: &Path,
355 scenario: &Scenario,
356 case: &Case,
357 options: RunOptions,
358 ) -> CaseReport {
359 match self
360 .run_case_inner(scenario_path, scenario, case, options)
361 .await
362 {
363 Ok(report) => report,
364 Err(err) => CaseReport {
365 case_name: case.name.clone(),
366 outcome: CaseOutcome::Errored(err.to_string()),
367 },
368 }
369 }
370
371 async fn run_case_inner(
372 &self,
373 scenario_path: &Path,
374 scenario: &Scenario,
375 case: &Case,
376 options: RunOptions,
377 ) -> Result<CaseReport, RunnerError> {
378 if options.live {
383 check_live_mode(true)?;
384 }
385
386 let mut initial = StateDelta::new();
387 for (key, value) in &case.input {
388 let field = FieldName::new(key).map_err(|source| RunnerError::InvalidField {
389 field: key.clone(),
390 message: source.to_string(),
391 })?;
392 initial.set_raw(field, value.clone());
393 }
394
395 let thread = ThreadId::new(format!("scenario-{}", Uuid::new_v4()))
396 .map_err(|source| RunnerError::InvalidThreadId(source.to_string()))?;
397
398 let (records, outcome, final_state) =
399 if let Some(superstep) = case.interrupt_after_superstep {
400 if scenario.store != StoreKind::SqliteTemp {
401 return Err(RunnerError::InterruptRequiresSqliteTemp {
402 case: case.name.clone(),
403 });
404 }
405 self.run_interrupted(
406 scenario_path,
407 scenario,
408 case,
409 &thread,
410 &initial,
411 superstep,
412 options,
413 )
414 .await?
415 } else {
416 let (graph, router) = self.resolve_target(scenario_path, scenario)?;
417 let port = build_port(router, options)?;
418 match scenario.store {
419 StoreKind::InMemory => {
420 let store = Arc::new(InMemoryWaypointStore::new());
421 run_plain(
422 store,
423 &graph,
424 port,
425 thread.clone(),
426 initial,
427 &case.parley_responses,
428 )
429 .await?
430 }
431 StoreKind::SqliteTemp => {
432 let store = Arc::new(SqliteWaypointStore::new(&temp_db_url("case")).await?);
433 run_plain(
434 store,
435 &graph,
436 port,
437 thread.clone(),
438 initial,
439 &case.parley_responses,
440 )
441 .await?
442 }
443 }
444 };
445
446 let snapshot_path = snapshot_file_path(scenario_path, &case.name);
447 if options.bless
448 && case
449 .assertions
450 .iter()
451 .any(|a| matches!(a, Assertion::FinalStateSnapshot))
452 {
453 let value = battlefield_snapshot_value(&final_state);
454 let mut text =
457 serde_json::to_string_pretty(&value).unwrap_or_else(|_| value.to_string());
458 text.push('\n');
459 if let Some(parent) = snapshot_path.parent() {
460 let _ = std::fs::create_dir_all(parent);
461 }
462 std::fs::write(&snapshot_path, text).map_err(|source| RunnerError::Io {
463 path: snapshot_path.clone(),
464 source,
465 })?;
466 }
467
468 let ctx = AssertionContext::new(&records, &final_state, &outcome)
469 .with_snapshot_path(snapshot_path);
470
471 let mut results = Vec::with_capacity(case.assertions.len());
472 for assertion in &case.assertions {
473 let verdict = if should_skip_for_live(
474 assertion,
475 options.live,
476 scenario.live.allow_content_assertions,
477 ) {
478 Verdict::Skipped("content assertion, live mode".to_string())
479 } else {
480 match assertion::evaluate(assertion, &ctx) {
481 AssertionOutcome::Passed => Verdict::Passed,
482 AssertionOutcome::Failed(failure) => Verdict::Failed(failure),
483 }
484 };
485 results.push(AssertionResult { verdict });
486 }
487
488 Ok(CaseReport {
489 case_name: case.name.clone(),
490 outcome: CaseOutcome::Ran {
491 records,
492 final_state,
493 results,
494 },
495 })
496 }
497
498 #[allow(clippy::too_many_arguments)]
505 async fn run_interrupted(
506 &self,
507 scenario_path: &Path,
508 scenario: &Scenario,
509 case: &Case,
510 thread: &ThreadId,
511 initial: &StateDelta,
512 interrupt_superstep: u64,
513 options: RunOptions,
514 ) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
515 let (control_graph, control_router) = self.resolve_target(scenario_path, scenario)?;
518 let control_port = build_port(control_router, options)?;
519 let control_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("control")).await?);
520 let control_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
521 let control_engine = WarEngine::new(control_port, Arc::clone(&control_store))
522 .with_trace_sink(control_sink.clone());
523 let control_outcome = control_engine
524 .start(&control_graph, thread.clone(), initial.clone())
525 .await?;
526 drive_parleys(
527 &control_engine,
528 &control_graph,
529 thread,
530 control_outcome,
531 &case.parley_responses,
532 )
533 .await?;
534
535 let resumed_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("resumed")).await?);
538 let mut history = full_history(control_store.as_ref(), thread).await?;
539 history.sort_by_key(|wp| wp.superstep);
540 for waypoint in history
541 .iter()
542 .filter(|wp| wp.superstep <= interrupt_superstep)
543 {
544 resumed_store.save(waypoint).await?;
545 }
546
547 let (resumed_graph, resumed_router) = self.resolve_target(scenario_path, scenario)?;
550 let resumed_port = build_port(resumed_router, options)?;
551 let resumed_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
552 let resumed_engine = WarEngine::new(resumed_port, Arc::clone(&resumed_store))
553 .with_trace_sink(resumed_sink.clone());
554 let resumed_outcome = resumed_engine
555 .resume(&resumed_graph, thread.clone())
556 .await?;
557 let resumed_outcome = drive_parleys(
558 &resumed_engine,
559 &resumed_graph,
560 thread,
561 resumed_outcome,
562 &case.parley_responses,
563 )
564 .await?;
565
566 tokio::time::sleep(Duration::from_millis(50)).await;
570 let records = resumed_sink.records();
571 let final_state =
572 resolve_battlefield(resumed_store.as_ref(), thread, &resumed_outcome).await?;
573 Ok((records, resumed_outcome, final_state))
574 }
575
576 pub fn trials(&self, pattern: &str) -> Vec<Trial> {
582 let mut paths: Vec<PathBuf> = match glob::glob(pattern) {
583 Ok(entries) => entries.filter_map(Result::ok).collect(),
584 Err(_) => Vec::new(),
585 };
586 paths.sort();
587
588 let mut trials = Vec::new();
589 for path in paths {
590 let stem = path
591 .file_stem()
592 .and_then(|s| s.to_str())
593 .unwrap_or("scenario")
594 .to_string();
595 match Scenario::from_path(&path) {
596 Ok(scenario) => {
597 let scenario = Arc::new(scenario);
598 for case in &scenario.cases {
599 let trial_name = format!("{stem}::{}", case.name);
600 let runner = self.clone();
601 let scenario = Arc::clone(&scenario);
602 let case = case.clone();
603 let path = path.clone();
604 trials.push(Trial::test(trial_name, move || {
605 let rt = tokio::runtime::Builder::new_current_thread()
606 .enable_all()
607 .build()
608 .map_err(|e| Failed::from(e.to_string()))?;
609 let report = rt.block_on(runner.run_case(
610 &path,
611 &scenario,
612 &case,
613 RunOptions::default(),
614 ));
615 if report.passed() {
616 Ok(())
617 } else {
618 Err(Failed::from(report.render_failures()))
619 }
620 }));
621 }
622 }
623 Err(err) => {
624 trials.push(Trial::test(format!("{stem}::parse_error"), move || {
625 Err(Failed::from(err.to_string()))
626 }));
627 }
628 }
629 }
630 trials
631 }
632}
633
634pub fn run_eval_harness_main(pattern: &str) -> ! {
639 let args = Arguments::from_args();
640 let runner = ScenarioRunner::new();
641 let trials = runner.trials(pattern);
642 libtest_mimic::run(&args, trials).exit()
643}
644
645pub fn run_eval_harness_main_with(pattern: &str, configure: impl FnOnce(&mut ScenarioRunner)) -> ! {
652 let args = Arguments::from_args();
653 let mut runner = ScenarioRunner::new();
654 configure(&mut runner);
655 let trials = runner.trials(pattern);
656 libtest_mimic::run(&args, trials).exit()
657}
658
659#[macro_export]
682macro_rules! eval_scenarios {
683 ($pattern:expr) => {
684 fn main() {
685 $crate::runner::run_eval_harness_main($pattern)
686 }
687 };
688 ($pattern:expr, $configure:expr) => {
689 fn main() {
690 $crate::runner::run_eval_harness_main_with($pattern, $configure)
691 }
692 };
693}
694
695enum LlmSource {
702 Scripted(HashMap<String, ScenarioLlm>),
704 Live(Arc<dyn LlmPort>),
710}
711
712struct ScenarioPaladinPort {
717 llm: LlmSource,
718}
719
720impl ScenarioPaladinPort {
721 fn scripted(router: HashMap<String, ScenarioLlm>) -> Self {
722 Self {
723 llm: LlmSource::Scripted(router),
724 }
725 }
726
727 fn live(provider: Arc<dyn LlmPort>) -> Self {
728 Self {
729 llm: LlmSource::Live(provider),
730 }
731 }
732}
733
734#[async_trait]
735impl PaladinPort for ScenarioPaladinPort {
736 async fn execute(&self, paladin: &Paladin, input: &str) -> Result<PaladinResult, PaladinError> {
737 let prompt = PromptItem::new(PromptType::User(UserPrompt {
738 query: input.to_string(),
739 context: None,
740 }))
741 .map_err(|source| PaladinError::ConfigurationError(source.to_string()))?;
742 let request = LlmRequest::new(paladin.data.model.clone(), prompt);
743
744 let response = match &self.llm {
745 LlmSource::Scripted(router) => {
746 let llm = router.get(&paladin.data.name).ok_or_else(|| {
747 PaladinError::ConfigurationError(format!(
748 "ScenarioPaladinPort: no ScenarioLlm registered for paladin {:?} \
749 (known: {:?})",
750 paladin.data.name,
751 router.keys().collect::<Vec<_>>()
752 ))
753 })?;
754 llm.generate(request).await
755 }
756 LlmSource::Live(provider) => provider.generate(request).await,
757 }
758 .map_err(|source| paladin_battalion::llm_failure::to_paladin_error(&source))?;
767
768 Ok(PaladinResult {
769 output: response.content,
770 usage: response.usage.clone(),
771 execution_time_ms: 0,
772 loop_count: 1,
773 stop_reason: StopReason::Completed,
774 ..Default::default()
775 })
776 }
777
778 async fn execute_stream(
779 &self,
780 paladin: &Paladin,
781 input: &str,
782 ) -> Result<PaladinStream, PaladinError> {
783 let result = self.execute(paladin, input).await?;
784 let (tx, rx) = tokio::sync::mpsc::channel(1);
785 let _ = tx
786 .send(Ok(PaladinStreamChunk {
787 text: result.output,
788 is_final: true,
789 metadata: None,
790 }))
791 .await;
792 Ok(rx)
793 }
794
795 fn validate(&self, _paladin: &Paladin) -> Result<(), PaladinError> {
796 Ok(())
797 }
798}
799
800fn build_port(
801 router: HashMap<String, ScenarioLlm>,
802 options: RunOptions,
803) -> Result<Arc<dyn PaladinPort>, RunnerError> {
804 if options.live {
805 let provider = resolve_live_provider()?;
806 Ok(Arc::new(ScenarioPaladinPort::live(provider)))
807 } else {
808 Ok(Arc::new(ScenarioPaladinPort::scripted(router)))
809 }
810}
811
812fn build_llm_router(graph: &WarGraph, root: &ScenarioLlm) -> HashMap<String, ScenarioLlm> {
820 let mut router = HashMap::new();
821 for node_id in graph.node_order() {
822 if let Some(NodeSpec::Paladin { paladin, .. }) = graph.node(node_id) {
823 router.insert(paladin.data.name.clone(), root.for_node(node_id.as_str()));
824 }
825 }
826 router
827}
828
829struct CapturingSink {
836 records: Mutex<Vec<TraceRecord>>,
837}
838
839impl CapturingSink {
840 fn new() -> Self {
841 Self {
842 records: Mutex::new(Vec::new()),
843 }
844 }
845
846 fn records(&self) -> Vec<TraceRecord> {
847 self.records
848 .lock()
849 .expect("CapturingSink records lock poisoned")
850 .clone()
851 }
852}
853
854#[async_trait]
855impl TraceSink for CapturingSink {
856 async fn on_event(&self, record: TraceRecord) -> Result<(), TraceSinkError> {
857 self.records
858 .lock()
859 .expect("CapturingSink records lock poisoned")
860 .push(record);
861 Ok(())
862 }
863}
864
865async fn run_plain<W: WaypointPort + 'static>(
870 store: Arc<W>,
871 graph: &WarGraph,
872 port: Arc<dyn PaladinPort>,
873 thread: ThreadId,
874 initial: StateDelta,
875 parley_responses: &[ParleyScript],
876) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
877 let sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
878 let engine = WarEngine::new(port, Arc::clone(&store)).with_trace_sink(sink.clone());
879 let outcome = engine.start(graph, thread.clone(), initial).await?;
880 let outcome = drive_parleys(&engine, graph, &thread, outcome, parley_responses).await?;
881
882 tokio::time::sleep(Duration::from_millis(50)).await;
883 let records = sink.records();
884 let final_state = resolve_battlefield(store.as_ref(), &thread, &outcome).await?;
885 Ok((records, outcome, final_state))
886}
887
888async fn drive_parleys<W: WaypointPort + 'static>(
894 engine: &WarEngine<W>,
895 graph: &WarGraph,
896 thread: &ThreadId,
897 mut outcome: RunOutcome,
898 parley_responses: &[ParleyScript],
899) -> Result<RunOutcome, RunnerError> {
900 let mut idx = 0usize;
901 while let RunOutcome::AwaitingInput { parleys, .. } = &outcome {
902 if idx + parleys.len() > parley_responses.len() {
903 break;
904 }
905 let mut responses = Vec::with_capacity(parleys.len());
906 for request in parleys {
907 let script = &parley_responses[idx];
908 idx += 1;
909 responses.push(ParleyResponse {
910 parley_id: request.parley_id,
911 kind: request.kind.clone(),
912 prompt: request.prompt.clone(),
913 value: script.value.clone(),
914 responded_by: Some("paladin-eval".to_string()),
915 responded_at: chrono::Utc::now(),
916 defaulted: false,
917 });
918 }
919 outcome = engine.resume_with(graph, thread.clone(), responses).await?;
920 }
921 Ok(outcome)
922}
923
924async fn resolve_battlefield<W: WaypointPort>(
928 store: &W,
929 thread: &ThreadId,
930 outcome: &RunOutcome,
931) -> Result<Battlefield, RunnerError> {
932 match outcome {
933 RunOutcome::Completed { final_state, .. } => Ok(final_state.clone()),
934 RunOutcome::Halted { waypoint, .. } | RunOutcome::AwaitingInput { waypoint, .. } => {
935 let loaded = store
936 .get(thread, waypoint)
937 .await?
938 .ok_or(RunnerError::NoFinalState)?;
939 Ok(loaded.battlefield)
940 }
941 RunOutcome::Failed {
942 waypoint: Some(waypoint),
943 ..
944 } => {
945 let loaded = store
946 .get(thread, waypoint)
947 .await?
948 .ok_or(RunnerError::NoFinalState)?;
949 Ok(loaded.battlefield)
950 }
951 RunOutcome::Failed { waypoint: None, .. } => Err(RunnerError::NoFinalState),
952 }
953}
954
955async fn full_history<W: WaypointPort>(
956 store: &W,
957 thread: &ThreadId,
958) -> Result<Vec<Waypoint>, RunnerError> {
959 let summaries = store.history(thread, None, None).await?;
960 let mut waypoints = Vec::with_capacity(summaries.len());
961 for summary in summaries {
962 if let Some(waypoint) = store.get(thread, &summary.waypoint_id).await? {
963 waypoints.push(waypoint);
964 }
965 }
966 Ok(waypoints)
967}
968
969fn temp_db_url(label: &str) -> String {
970 let path = std::env::temp_dir().join(format!("paladin-eval-{label}-{}.sqlite", Uuid::new_v4()));
971 format!("sqlite://{}", path.display())
972}
973
974fn resolve_relative(scenario_path: &Path, target: &Path) -> PathBuf {
975 if target.is_absolute() {
976 target.to_path_buf()
977 } else {
978 scenario_path
979 .parent()
980 .unwrap_or_else(|| Path::new("."))
981 .join(target)
982 }
983}
984
985fn load_graph_doc(path: &Path) -> Result<WarGraphDoc, RunnerError> {
986 let contents = std::fs::read_to_string(path).map_err(|source| RunnerError::Io {
987 path: path.to_path_buf(),
988 source,
989 })?;
990 let is_yaml = matches!(
991 path.extension().and_then(|ext| ext.to_str()),
992 Some("yaml") | Some("yml")
993 );
994 let doc: WarGraphDoc = if is_yaml {
995 serde_yaml::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
996 path: path.to_path_buf(),
997 message: source.to_string(),
998 })?
999 } else {
1000 serde_json::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
1001 path: path.to_path_buf(),
1002 message: source.to_string(),
1003 })?
1004 };
1005 Ok(doc)
1006}
1007
1008fn snapshot_file_path(scenario_path: &Path, case_name: &str) -> PathBuf {
1009 let dir = scenario_path.parent().unwrap_or_else(|| Path::new("."));
1010 let stem = scenario_path
1011 .file_stem()
1012 .and_then(|s| s.to_str())
1013 .unwrap_or("scenario");
1014 dir.join(format!("{stem}.{case_name}.snap.json"))
1015}
1016
1017fn battlefield_snapshot_value(battlefield: &Battlefield) -> Value {
1025 let mut obj = serde_json::Map::new();
1026 for field in &battlefield.schema().fields {
1027 if let Some(value) = battlefield.get_raw(&field.name) {
1028 obj.insert(field.name.as_str().to_string(), value.clone());
1029 }
1030 }
1031 Value::Object(obj)
1032}
1033
1034pub const PALADIN_EVAL_LIVE_ENV: &str = "PALADIN_EVAL_LIVE";
1050
1051#[derive(Debug, Error)]
1057pub enum LiveModeError {
1058 #[error("live mode requires --live")]
1060 FlagNotSet,
1061 #[error(
1063 "live mode requires the {PALADIN_EVAL_LIVE_ENV} environment variable to be set to a \
1064 non-blank value (deliberately never set by any CI workflow in this repository)"
1065 )]
1066 EnvNotSet,
1067 #[error(
1071 "live mode requires a configured provider credential; none of paladin-llm's compiled-in \
1072 providers has one set (see ADR-0012)"
1073 )]
1074 NoProviderKey,
1075 #[error(
1080 "live mode's gate passed, but this build of paladin-eval was not compiled with its own \
1081 `live` feature (enables real paladin-llm provider adapters); rebuild with \
1082 `--features live`"
1083 )]
1084 FeatureNotCompiled,
1085 #[error("failed to construct the live provider: {0}")]
1088 ProviderConstruction(String),
1089}
1090
1091pub fn check_live_mode(live_flag: bool) -> Result<(), LiveModeError> {
1097 if !live_flag {
1098 return Err(LiveModeError::FlagNotSet);
1099 }
1100 let env_set = std::env::var(PALADIN_EVAL_LIVE_ENV)
1101 .map(|v| !v.trim().is_empty())
1102 .unwrap_or(false);
1103 if !env_set {
1104 return Err(LiveModeError::EnvNotSet);
1105 }
1106 if paladin_llm::provider_factory::LlmProviderFactory::get_default_provider().is_none() {
1107 return Err(LiveModeError::NoProviderKey);
1108 }
1109 Ok(())
1110}
1111
1112fn is_content_assertion(assertion: &Assertion) -> bool {
1118 matches!(
1119 assertion,
1120 Assertion::FinalStateFieldEquals { .. }
1121 | Assertion::FinalStateFieldMatches { .. }
1122 | Assertion::FieldJsonPathEquals { .. }
1123 | Assertion::FinalStateSnapshot
1124 )
1125}
1126
1127fn should_skip_for_live(assertion: &Assertion, live: bool, allow_content_assertions: bool) -> bool {
1133 live && is_content_assertion(assertion) && !allow_content_assertions
1134}
1135
1136#[cfg(feature = "live")]
1143fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
1144 let provider_name = paladin_llm::provider_factory::LlmProviderFactory::get_default_provider()
1145 .ok_or(LiveModeError::NoProviderKey)?;
1146 paladin_llm::provider_factory::LlmProviderFactory::new()
1147 .create(&provider_name)
1148 .map_err(|source| LiveModeError::ProviderConstruction(source.to_string()))
1149}
1150
1151#[cfg(not(feature = "live"))]
1156fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
1157 Err(LiveModeError::FeatureNotCompiled)
1158}
1159
1160#[derive(Debug, Error)]
1168pub enum RunnerError {
1169 #[error("failed to access {path}: {source}")]
1172 Io {
1173 path: PathBuf,
1175 #[source]
1177 source: std::io::Error,
1178 },
1179 #[error("failed to parse WarGraphDoc {path}: {message}")]
1181 ParseGraphDoc {
1182 path: PathBuf,
1184 message: String,
1186 },
1187 #[error("failed to compile WarGraphDoc {path}: {source}")]
1189 Compile {
1190 path: PathBuf,
1192 #[source]
1197 source: Box<paladin_battalion::engine::graph_doc::CompileError>,
1198 },
1199 #[error("scenario names unregistered graph constructor {name:?}; registered: {registered:?}")]
1202 UnknownGraph {
1203 name: String,
1205 registered: Vec<String>,
1207 },
1208 #[error("scenario names unregistered registries {name:?}; registered: {registered:?}")]
1211 UnknownRegistries {
1212 name: String,
1214 registered: Vec<String>,
1216 },
1217 #[error("case {case:?}: interrupt_after_superstep requires store: sqlite_temp")]
1220 InterruptRequiresSqliteTemp {
1221 case: String,
1223 },
1224 #[error("waypoint store error: {0}")]
1226 Waypoint(#[from] WaypointError),
1227 #[error("engine error: {0}")]
1230 Engine(#[from] EngineError),
1231 #[error("invalid field name {field:?}: {message}")]
1234 InvalidField {
1235 field: String,
1237 message: String,
1239 },
1240 #[error("run finished with no persisted final state to evaluate assertions against")]
1243 NoFinalState,
1244 #[error("invalid thread id: {0}")]
1246 InvalidThreadId(String),
1247 #[error("live mode: {0}")]
1250 Live(#[from] LiveModeError),
1251}
1252
1253#[cfg(test)]
1254mod tests {
1255 use super::*;
1256 use crate::scenario::{
1257 Case as ScenarioCase, LlmScript, RunStatusValue, Scenario as ScenarioDoc, ScenarioTarget,
1258 ScriptEntry, StoreKind as ScenarioStoreKind, Times,
1259 };
1260 use paladin_battalion::engine::graph::{EdgeSpec, EngineLimits};
1261 use paladin_battalion::engine::node::{NodeContext, StateNode, StateNodeError};
1262 use paladin_battalion::engine::{InputMapping, NodeSpec as EngineNodeSpec};
1263 use paladin_core::base::entity::Entity;
1264 use paladin_core::platform::container::battlefield::{
1265 BattlefieldSchema, DispatchRule, FieldSpec,
1266 };
1267 use paladin_core::platform::container::directive::Directive;
1268 use paladin_core::platform::container::paladin::{MaxLoops, PaladinData, PaladinStatus};
1269 use paladin_core::platform::container::waypoint::NodeId;
1270 use std::collections::BTreeMap;
1271
1272 fn field(name: &str) -> FieldName {
1273 FieldName::new(name).expect("valid field name")
1274 }
1275
1276 fn make_paladin(name: &str, model: &str) -> Paladin {
1277 let data = PaladinData {
1278 system_prompt: format!("{name} prompt"),
1279 name: name.to_string(),
1280 user_name: String::new(),
1281 model: model.to_string(),
1282 temperature: 0.7,
1283 max_loops: MaxLoops::Fixed(1),
1284 stop_words: vec![],
1285 status: PaladinStatus::Idle,
1286 vision_enabled: false,
1287 ..Default::default()
1288 };
1289 Entity::new(data, Some(name.to_string()))
1290 }
1291
1292 fn minimal_scenario(
1293 target: ScenarioTarget,
1294 llm: LlmScript,
1295 assertions: Vec<Assertion>,
1296 ) -> ScenarioDoc {
1297 ScenarioDoc {
1298 schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
1299 target,
1300 store: ScenarioStoreKind::InMemory,
1301 llm,
1302 cases: vec![ScenarioCase {
1303 name: "happy_path".to_string(),
1304 input: BTreeMap::from([("topic".to_string(), serde_json::json!("widgets"))]),
1305 interrupt_after_superstep: None,
1306 parley_responses: vec![],
1307 llm: None,
1308 assertions,
1309 }],
1310 live: Default::default(),
1311 registries: None,
1312 }
1313 }
1314
1315 fn linear_fixture_path() -> PathBuf {
1316 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1321 .join("../paladin-battalion/tests/fixtures/graph_docs/linear.json")
1322 }
1323
1324 #[tokio::test]
1325 async fn runner_executes_a_graph_doc_case() {
1326 let scenario = minimal_scenario(
1327 ScenarioTarget::GraphDoc {
1328 graph_doc: PathBuf::from(
1329 "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1330 ),
1331 },
1332 LlmScript {
1333 global: vec![
1334 ScriptEntry::Text("s1".to_string()),
1335 ScriptEntry::Text("s2".to_string()),
1336 ScriptEntry::Text("s3".to_string()),
1337 ],
1338 ..Default::default()
1339 },
1340 vec![
1341 Assertion::NodeExecuted {
1342 node: "start".to_string(),
1343 times: Times::Exact(1),
1344 },
1345 Assertion::RunStatus(RunStatusValue::Completed),
1346 ],
1347 );
1348
1349 let runner = ScenarioRunner::new();
1350 let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1354 let report = runner
1355 .run_case(
1356 &scenario_path,
1357 &scenario,
1358 &scenario.cases[0],
1359 RunOptions::default(),
1360 )
1361 .await;
1362
1363 assert!(
1364 report.passed(),
1365 "expected the case to pass: {}",
1366 report.render_failures()
1367 );
1368 assert!(linear_fixture_path().exists(), "sanity: fixture must exist");
1369 }
1370
1371 #[tokio::test]
1372 async fn runner_reports_a_failing_assertion_with_its_rendering() {
1373 let scenario = minimal_scenario(
1374 ScenarioTarget::GraphDoc {
1375 graph_doc: PathBuf::from(
1376 "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1377 ),
1378 },
1379 LlmScript {
1380 global: vec![
1381 ScriptEntry::Text("s1".to_string()),
1382 ScriptEntry::Text("s2".to_string()),
1383 ScriptEntry::Text("s3".to_string()),
1384 ],
1385 ..Default::default()
1386 },
1387 vec![Assertion::NodeExecuted {
1388 node: "start".to_string(),
1389 times: Times::Exact(5),
1390 }],
1391 );
1392
1393 let runner = ScenarioRunner::new();
1394 let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1395 let report = runner
1396 .run_case(
1397 &scenario_path,
1398 &scenario,
1399 &scenario.cases[0],
1400 RunOptions::default(),
1401 )
1402 .await;
1403
1404 assert!(!report.passed());
1405 let rendered = report.render_failures();
1406 assert!(
1407 rendered.contains("expected: exactly 5"),
1408 "rendering must be the assertion's own render_failure() output: {rendered:?}"
1409 );
1410 }
1411
1412 #[tokio::test]
1413 async fn runner_executes_a_registered_target() {
1414 let mut runner = ScenarioRunner::new();
1415 runner.register_graph(
1416 "solo",
1417 Arc::new(|ports: &ScriptedPorts| {
1418 let schema = BattlefieldSchema::new(vec![FieldSpec::new(
1419 field("out"),
1420 DispatchRule::LastWrite,
1421 None,
1422 false,
1423 )]);
1424 let mut graph = WarGraph::new(schema, EngineLimits::default());
1425 let node = NodeId::new("solo");
1426 let llm = ports.for_node("solo");
1427 graph.add_node(
1428 node.clone(),
1429 EngineNodeSpec::Function(Arc::new(RegisteredEchoNode {
1430 llm,
1431 output_field: field("out"),
1432 })),
1433 );
1434 graph.add_entry(node);
1435 graph
1436 }),
1437 );
1438
1439 let mut scenario = minimal_scenario(
1440 ScenarioTarget::Registered {
1441 registered: "solo".to_string(),
1442 },
1443 LlmScript {
1444 per_node: BTreeMap::from([(
1445 "solo".to_string(),
1446 vec![ScriptEntry::Text("registered-output".to_string())],
1447 )]),
1448 ..Default::default()
1449 },
1450 vec![Assertion::FinalStateFieldEquals {
1451 field: "out".to_string(),
1452 value: serde_json::json!("registered-output"),
1453 }],
1454 );
1455 scenario.cases[0].input = BTreeMap::new();
1459
1460 let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1461 let report = runner
1462 .run_case(
1463 &scenario_path,
1464 &scenario,
1465 &scenario.cases[0],
1466 RunOptions::default(),
1467 )
1468 .await;
1469 assert!(
1470 report.passed(),
1471 "expected the registered target's Function node to receive the scripted port: {}",
1472 report.render_failures()
1473 );
1474 }
1475
1476 struct RegisteredEchoNode {
1477 llm: ScenarioLlm,
1478 output_field: FieldName,
1479 }
1480
1481 #[async_trait::async_trait]
1482 impl StateNode for RegisteredEchoNode {
1483 async fn run(
1484 &self,
1485 _state: &Battlefield,
1486 _ctx: &NodeContext,
1487 ) -> Result<Directive, StateNodeError> {
1488 let request = LlmRequest::new(
1489 "scenario-model",
1490 PromptItem::new(PromptType::User(UserPrompt {
1491 query: "go".to_string(),
1492 context: None,
1493 }))
1494 .expect("valid prompt"),
1495 );
1496 let response = self
1497 .llm
1498 .generate(request)
1499 .await
1500 .map_err(|e| StateNodeError(e.to_string()))?;
1501 let mut delta = StateDelta::new();
1502 delta
1503 .set(self.output_field.clone(), response.content)
1504 .map_err(|e| StateNodeError(e.to_string()))?;
1505 Ok(delta.into())
1506 }
1507 }
1508
1509 #[tokio::test]
1510 async fn interrupt_and_resume_reproduces_the_control_run() {
1511 fn build_graph() -> WarGraph {
1514 let schema = BattlefieldSchema::new(vec![
1515 FieldSpec::new(
1516 field("topic"),
1517 DispatchRule::LastWrite,
1518 Some(serde_json::json!("widgets")),
1519 false,
1520 ),
1521 FieldSpec::new(field("step_one"), DispatchRule::LastWrite, None, false),
1522 FieldSpec::new(field("step_two"), DispatchRule::LastWrite, None, false),
1523 ]);
1524 let mut graph = WarGraph::new(schema, EngineLimits::default());
1525 let first = NodeId::new("first");
1526 let second = NodeId::new("second");
1527 graph.add_node(
1528 first.clone(),
1529 EngineNodeSpec::paladin(
1530 make_paladin("First", "gpt-4"),
1531 InputMapping::new("{topic}"),
1532 field("step_one"),
1533 ),
1534 );
1535 graph.add_node(
1536 second.clone(),
1537 EngineNodeSpec::paladin(
1538 make_paladin("Second", "gpt-4"),
1539 InputMapping::new("{step_one}"),
1540 field("step_two"),
1541 ),
1542 );
1543 graph.add_edge(EdgeSpec {
1544 from: first.clone(),
1545 to: second,
1546 condition: None,
1547 });
1548 graph.add_entry(first);
1549 graph
1550 }
1551
1552 let mut runner = ScenarioRunner::new();
1553 runner.register_graph("chain", Arc::new(|_ports: &ScriptedPorts| build_graph()));
1554
1555 let llm = LlmScript {
1556 per_node: BTreeMap::from([
1557 (
1558 "first".to_string(),
1559 vec![ScriptEntry::Text("one".to_string())],
1560 ),
1561 (
1562 "second".to_string(),
1563 vec![ScriptEntry::Text("two".to_string())],
1564 ),
1565 ]),
1566 ..Default::default()
1567 };
1568
1569 let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1570
1571 let control_scenario = ScenarioDoc {
1573 schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
1574 target: ScenarioTarget::Registered {
1575 registered: "chain".to_string(),
1576 },
1577 store: ScenarioStoreKind::SqliteTemp,
1578 llm: llm.clone(),
1579 cases: vec![ScenarioCase {
1580 name: "control".to_string(),
1581 input: BTreeMap::new(),
1582 interrupt_after_superstep: None,
1583 parley_responses: vec![],
1584 llm: None,
1585 assertions: vec![],
1586 }],
1587 live: Default::default(),
1588 registries: None,
1589 };
1590 let (control_graph, control_router) = runner
1591 .resolve_target(&scenario_path, &control_scenario)
1592 .expect("resolve control target");
1593 let control_port =
1594 build_port(control_router, RunOptions::default()).expect("scripted port");
1595 let store = Arc::new(InMemoryWaypointStore::new());
1596 let thread = ThreadId::new(format!("control-{}", Uuid::new_v4())).expect("valid thread id");
1597 let (_records, _outcome, control_final_state) = run_plain(
1598 store,
1599 &control_graph,
1600 control_port,
1601 thread,
1602 StateDelta::new(),
1603 &[],
1604 )
1605 .await
1606 .expect("control run succeeds");
1607
1608 let interrupted_scenario = ScenarioDoc {
1610 store: ScenarioStoreKind::SqliteTemp,
1611 cases: vec![ScenarioCase {
1612 name: "interrupted".to_string(),
1613 interrupt_after_superstep: Some(1),
1614 ..control_scenario.cases[0].clone()
1615 }],
1616 ..control_scenario
1617 };
1618 let report = runner
1619 .run_case(
1620 &scenario_path,
1621 &interrupted_scenario,
1622 &interrupted_scenario.cases[0],
1623 RunOptions::default(),
1624 )
1625 .await;
1626
1627 match &report.outcome {
1628 CaseOutcome::Ran { final_state, .. } => {
1629 assert_eq!(
1630 *final_state, control_final_state,
1631 "resumed final Battlefield must equal the control run's"
1632 );
1633 }
1634 CaseOutcome::Errored(message) => panic!("interrupted case errored: {message}"),
1635 }
1636 }
1637
1638 #[test]
1639 fn evals_target_discovers_one_trial_per_case() {
1640 let dir = std::env::temp_dir().join(format!("paladin-eval-trials-{}", Uuid::new_v4()));
1641 std::fs::create_dir_all(&dir).expect("create temp dir");
1642
1643 std::fs::write(
1649 dir.join("alpha.yaml"),
1650 r#"
1651schema_version: "1"
1652target:
1653 registered: "nowhere"
1654cases:
1655 - name: case_one
1656 assertions: []
1657 - name: case_two
1658 assertions: []
1659"#,
1660 )
1661 .expect("write alpha fixture");
1662 std::fs::write(
1663 dir.join("beta.yaml"),
1664 r#"
1665schema_version: "1"
1666target:
1667 registered: "nowhere"
1668cases:
1669 - name: only_case
1670 assertions: []
1671"#,
1672 )
1673 .expect("write beta fixture");
1674
1675 let runner = ScenarioRunner::new();
1676 let pattern = dir.join("*.yaml");
1677 let trials = runner.trials(pattern.to_str().expect("valid utf8 path"));
1678
1679 assert_eq!(trials.len(), 3, "one trial per (file, case) pair");
1680 let names: Vec<&str> = trials.iter().map(Trial::name).collect();
1681 assert!(names.contains(&"alpha::case_one"));
1682 assert!(names.contains(&"alpha::case_two"));
1683 assert!(names.contains(&"beta::only_case"));
1684
1685 let filter = Arguments {
1686 filter: Some("case_one".to_string()),
1687 ..Default::default()
1688 };
1689 let selected: Vec<&Trial> = trials
1690 .iter()
1691 .filter(|t| !filter.is_filtered_out(t))
1692 .collect();
1693 assert_eq!(selected.len(), 1, "a name filter must select a single case");
1694 assert_eq!(selected[0].name(), "alpha::case_one");
1695
1696 let _ = std::fs::remove_dir_all(&dir);
1697 }
1698
1699 #[test]
1709 fn live_mode_requires_flag_and_env_and_keys() {
1710 unsafe {
1715 std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
1716 }
1717
1718 assert!(matches!(
1719 check_live_mode(false),
1720 Err(LiveModeError::FlagNotSet)
1721 ));
1722 assert!(matches!(
1723 check_live_mode(true),
1724 Err(LiveModeError::EnvNotSet)
1725 ));
1726
1727 unsafe {
1729 std::env::set_var(PALADIN_EVAL_LIVE_ENV, "1");
1730 }
1731 let compiled_provider =
1742 paladin_llm::provider_factory::LlmProviderFactory::get_default_provider();
1743 match compiled_provider {
1744 None => assert!(matches!(
1745 check_live_mode(true),
1746 Err(LiveModeError::NoProviderKey)
1747 )),
1748 Some(_) => assert!(check_live_mode(true).is_ok()),
1749 }
1750
1751 unsafe {
1753 std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
1754 }
1755 }
1756
1757 fn content_assertion_fixtures() -> Vec<Assertion> {
1758 vec![
1759 Assertion::FinalStateFieldEquals {
1760 field: "status".to_string(),
1761 value: serde_json::json!("done"),
1762 },
1763 Assertion::FinalStateFieldMatches {
1764 field: "status".to_string(),
1765 pattern: "^do".to_string(),
1766 },
1767 Assertion::FieldJsonPathEquals {
1768 path: "/status".to_string(),
1769 value: serde_json::json!("done"),
1770 },
1771 Assertion::FinalStateSnapshot,
1772 ]
1773 }
1774
1775 fn structural_assertion_fixtures() -> Vec<Assertion> {
1776 vec![
1777 Assertion::NodeExecuted {
1778 node: "worker".to_string(),
1779 times: Times::Exact(1),
1780 },
1781 Assertion::NodeNotExecuted {
1782 node: "idle".to_string(),
1783 },
1784 Assertion::EdgeFired {
1785 from: "a".to_string(),
1786 to: "b".to_string(),
1787 },
1788 Assertion::RouteTaken(vec!["a".to_string()]),
1789 Assertion::RunStatus(RunStatusValue::Completed),
1790 Assertion::TotalTokensMax(10),
1791 Assertion::SuperstepsMax(10),
1792 Assertion::ParleyRaised {
1793 kind: "approval".to_string(),
1794 node: "worker".to_string(),
1795 },
1796 ]
1797 }
1798
1799 #[test]
1800 fn content_assertions_are_skipped_in_live_mode_without_opt_in() {
1801 for assertion in content_assertion_fixtures() {
1802 assert!(
1803 should_skip_for_live(&assertion, true, false),
1804 "{assertion:?} must be skipped in live mode without the scenario's opt-in"
1805 );
1806 }
1807 }
1808
1809 #[test]
1810 fn content_assertions_run_in_live_mode_with_opt_in() {
1811 for assertion in content_assertion_fixtures() {
1812 assert!(
1813 !should_skip_for_live(&assertion, true, true),
1814 "{assertion:?} must run in live mode once the scenario opts in"
1815 );
1816 }
1817 }
1818
1819 #[test]
1820 fn structural_assertions_always_run_in_live_mode() {
1821 for assertion in structural_assertion_fixtures() {
1822 assert!(
1823 !should_skip_for_live(&assertion, true, false),
1824 "{assertion:?} is structural and must always run in live mode"
1825 );
1826 assert!(
1827 !should_skip_for_live(&assertion, true, true),
1828 "{assertion:?} is structural and must always run in live mode, opt-in or not"
1829 );
1830 }
1831 for assertion in content_assertion_fixtures()
1833 .into_iter()
1834 .chain(structural_assertion_fixtures())
1835 {
1836 assert!(!should_skip_for_live(&assertion, false, false));
1837 }
1838 }
1839
1840 #[tokio::test]
1841 async fn default_test_run_never_enters_live_mode() {
1842 let scenario = minimal_scenario(
1850 ScenarioTarget::GraphDoc {
1851 graph_doc: PathBuf::from(
1852 "../paladin-battalion/tests/fixtures/graph_docs/linear.json",
1853 ),
1854 },
1855 LlmScript {
1856 global: vec![
1857 ScriptEntry::Text("s1".to_string()),
1858 ScriptEntry::Text("s2".to_string()),
1859 ScriptEntry::Text("s3".to_string()),
1860 ],
1861 ..Default::default()
1862 },
1863 vec![Assertion::RunStatus(RunStatusValue::Completed)],
1864 );
1865
1866 let runner = ScenarioRunner::new();
1867 let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
1868 assert!(
1869 !RunOptions::default().live,
1870 "the harness's own default options must never carry live: true"
1871 );
1872 let report = runner
1873 .run_case(
1874 &scenario_path,
1875 &scenario,
1876 &scenario.cases[0],
1877 RunOptions::default(),
1878 )
1879 .await;
1880 assert!(
1881 report.passed(),
1882 "a plain (non-live) run must succeed entirely on scripted mocks: {}",
1883 report.render_failures()
1884 );
1885 }
1886}