use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use async_trait::async_trait;
use libtest_mimic::{Arguments, Failed, Trial};
use serde_json::Value;
use thiserror::Error;
use uuid::Uuid;
use paladin_battalion::engine::graph_doc::WarGraphDoc;
use paladin_battalion::engine::{
EngineError, EngineRegistries, NodeSpec, RunOutcome, WarEngine, WarGraph,
};
use paladin_core::platform::container::battlefield::{Battlefield, FieldName, StateDelta};
use paladin_core::platform::container::paladin::Paladin;
use paladin_core::platform::container::paladin_error::PaladinError;
use paladin_core::platform::container::parley::ParleyResponse;
use paladin_core::platform::container::prompt::{PromptItem, PromptType, UserPrompt};
use paladin_core::platform::container::trace::TraceRecord;
use paladin_core::platform::container::waypoint::{ThreadId, Waypoint};
use paladin_ports::output::llm_port::{LlmPort, LlmRequest};
use paladin_ports::output::paladin_port::{
PaladinPort, PaladinResult, PaladinStream, PaladinStreamChunk, StopReason,
};
use paladin_ports::output::trace_sink_port::{TraceSink, TraceSinkError};
use paladin_ports::output::waypoint_port::{WaypointError, WaypointPort};
use paladin_storage::waypoint::in_memory::InMemoryWaypointStore;
use paladin_storage::waypoint::sqlite::SqliteWaypointStore;
use crate::assertion::{self, AssertionContext, AssertionFailure, AssertionOutcome};
use crate::scenario::{Assertion, Case, ParleyScript, Scenario, ScenarioTarget, StoreKind};
use crate::scripted_llm::ScenarioLlm;
pub struct ScriptedPorts {
root: ScenarioLlm,
}
impl ScriptedPorts {
pub fn for_node(&self, node_id: impl Into<String>) -> ScenarioLlm {
self.root.for_node(node_id)
}
}
pub type GraphConstructor = Arc<dyn Fn(&ScriptedPorts) -> WarGraph + Send + Sync>;
pub type RegistriesFactory = Arc<dyn Fn() -> EngineRegistries + Send + Sync>;
#[derive(Debug, Clone, Copy, Default)]
pub struct RunOptions {
pub bless: bool,
pub live: bool,
}
#[derive(Debug)]
pub struct AssertionResult {
pub verdict: Verdict,
}
#[derive(Debug)]
pub enum Verdict {
Passed,
Failed(AssertionFailure),
Skipped(String),
}
pub struct CaseReport {
pub case_name: String,
pub outcome: CaseOutcome,
}
pub enum CaseOutcome {
Ran {
records: Vec<TraceRecord>,
final_state: Battlefield,
results: Vec<AssertionResult>,
},
Errored(String),
}
impl CaseReport {
pub fn passed(&self) -> bool {
match &self.outcome {
CaseOutcome::Ran { results, .. } => !results
.iter()
.any(|r| matches!(r.verdict, Verdict::Failed(_))),
CaseOutcome::Errored(_) => false,
}
}
pub fn records(&self) -> &[TraceRecord] {
match &self.outcome {
CaseOutcome::Ran { records, .. } => records,
CaseOutcome::Errored(_) => &[],
}
}
pub fn render_failures(&self) -> String {
match &self.outcome {
CaseOutcome::Errored(message) => {
format!(
"case {:?} errored before assertions could run: {message}",
self.case_name
)
}
CaseOutcome::Ran { results, .. } => {
let mut out = String::new();
for result in results {
if let Verdict::Failed(failure) = &result.verdict {
out.push_str(&failure.render_failure());
}
}
if out.is_empty() {
"(no failing assertions)".to_string()
} else {
out
}
}
}
}
}
#[derive(Clone, Default)]
pub struct ScenarioRunner {
graphs: HashMap<String, GraphConstructor>,
registries: HashMap<String, RegistriesFactory>,
}
impl ScenarioRunner {
pub fn new() -> Self {
Self::default()
}
pub fn register_graph(&mut self, name: impl Into<String>, ctor: GraphConstructor) -> &mut Self {
self.graphs.insert(name.into(), ctor);
self
}
pub fn register_registries(
&mut self,
name: impl Into<String>,
factory: RegistriesFactory,
) -> &mut Self {
self.registries.insert(name.into(), factory);
self
}
fn resolve_registries(&self, name: Option<&str>) -> Result<EngineRegistries, RunnerError> {
match name {
None => Ok(EngineRegistries::new()),
Some(name) => {
let factory =
self.registries
.get(name)
.ok_or_else(|| RunnerError::UnknownRegistries {
name: name.to_string(),
registered: self.registries.keys().cloned().collect(),
})?;
Ok((factory)())
}
}
}
fn resolve_target(
&self,
scenario_path: &Path,
scenario: &Scenario,
) -> Result<(WarGraph, HashMap<String, ScenarioLlm>), RunnerError> {
match &scenario.target {
ScenarioTarget::GraphDoc { graph_doc } => {
let path = resolve_relative(scenario_path, graph_doc);
let doc = load_graph_doc(&path)?;
let registries = self.resolve_registries(scenario.registries.as_deref())?;
let graph = doc
.compile(®istries)
.map_err(|source| RunnerError::Compile {
path: path.clone(),
source: Box::new(source),
})?;
let root = ScenarioLlm::new(scenario.llm.clone());
let router = build_llm_router(&graph, &root);
Ok((graph, router))
}
ScenarioTarget::Registered { registered } => {
let ctor =
self.graphs
.get(registered)
.ok_or_else(|| RunnerError::UnknownGraph {
name: registered.clone(),
registered: self.graphs.keys().cloned().collect(),
})?;
let root = ScenarioLlm::new(scenario.llm.clone());
let scripted_ports = ScriptedPorts { root: root.clone() };
let graph = (ctor)(&scripted_ports);
let router = build_llm_router(&graph, &root);
Ok((graph, router))
}
}
}
pub async fn run_case(
&self,
scenario_path: &Path,
scenario: &Scenario,
case: &Case,
options: RunOptions,
) -> CaseReport {
match self
.run_case_inner(scenario_path, scenario, case, options)
.await
{
Ok(report) => report,
Err(err) => CaseReport {
case_name: case.name.clone(),
outcome: CaseOutcome::Errored(err.to_string()),
},
}
}
async fn run_case_inner(
&self,
scenario_path: &Path,
scenario: &Scenario,
case: &Case,
options: RunOptions,
) -> Result<CaseReport, RunnerError> {
if options.live {
check_live_mode(true)?;
}
let mut initial = StateDelta::new();
for (key, value) in &case.input {
let field = FieldName::new(key).map_err(|source| RunnerError::InvalidField {
field: key.clone(),
message: source.to_string(),
})?;
initial.set_raw(field, value.clone());
}
let thread = ThreadId::new(format!("scenario-{}", Uuid::new_v4()))
.map_err(|source| RunnerError::InvalidThreadId(source.to_string()))?;
let (records, outcome, final_state) =
if let Some(superstep) = case.interrupt_after_superstep {
if scenario.store != StoreKind::SqliteTemp {
return Err(RunnerError::InterruptRequiresSqliteTemp {
case: case.name.clone(),
});
}
self.run_interrupted(
scenario_path,
scenario,
case,
&thread,
&initial,
superstep,
options,
)
.await?
} else {
let (graph, router) = self.resolve_target(scenario_path, scenario)?;
let port = build_port(router, options)?;
match scenario.store {
StoreKind::InMemory => {
let store = Arc::new(InMemoryWaypointStore::new());
run_plain(
store,
&graph,
port,
thread.clone(),
initial,
&case.parley_responses,
)
.await?
}
StoreKind::SqliteTemp => {
let store = Arc::new(SqliteWaypointStore::new(&temp_db_url("case")).await?);
run_plain(
store,
&graph,
port,
thread.clone(),
initial,
&case.parley_responses,
)
.await?
}
}
};
let snapshot_path = snapshot_file_path(scenario_path, &case.name);
if options.bless
&& case
.assertions
.iter()
.any(|a| matches!(a, Assertion::FinalStateSnapshot))
{
let value = battlefield_snapshot_value(&final_state);
let mut text =
serde_json::to_string_pretty(&value).unwrap_or_else(|_| value.to_string());
text.push('\n');
if let Some(parent) = snapshot_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
std::fs::write(&snapshot_path, text).map_err(|source| RunnerError::Io {
path: snapshot_path.clone(),
source,
})?;
}
let ctx = AssertionContext::new(&records, &final_state, &outcome)
.with_snapshot_path(snapshot_path);
let mut results = Vec::with_capacity(case.assertions.len());
for assertion in &case.assertions {
let verdict = if should_skip_for_live(
assertion,
options.live,
scenario.live.allow_content_assertions,
) {
Verdict::Skipped("content assertion, live mode".to_string())
} else {
match assertion::evaluate(assertion, &ctx) {
AssertionOutcome::Passed => Verdict::Passed,
AssertionOutcome::Failed(failure) => Verdict::Failed(failure),
}
};
results.push(AssertionResult { verdict });
}
Ok(CaseReport {
case_name: case.name.clone(),
outcome: CaseOutcome::Ran {
records,
final_state,
results,
},
})
}
#[allow(clippy::too_many_arguments)]
async fn run_interrupted(
&self,
scenario_path: &Path,
scenario: &Scenario,
case: &Case,
thread: &ThreadId,
initial: &StateDelta,
interrupt_superstep: u64,
options: RunOptions,
) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
let (control_graph, control_router) = self.resolve_target(scenario_path, scenario)?;
let control_port = build_port(control_router, options)?;
let control_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("control")).await?);
let control_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
let control_engine = WarEngine::new(control_port, Arc::clone(&control_store))
.with_trace_sink(control_sink.clone());
let control_outcome = control_engine
.start(&control_graph, thread.clone(), initial.clone())
.await?;
drive_parleys(
&control_engine,
&control_graph,
thread,
control_outcome,
&case.parley_responses,
)
.await?;
let resumed_store = Arc::new(SqliteWaypointStore::new(&temp_db_url("resumed")).await?);
let mut history = full_history(control_store.as_ref(), thread).await?;
history.sort_by_key(|wp| wp.superstep);
for waypoint in history
.iter()
.filter(|wp| wp.superstep <= interrupt_superstep)
{
resumed_store.save(waypoint).await?;
}
let (resumed_graph, resumed_router) = self.resolve_target(scenario_path, scenario)?;
let resumed_port = build_port(resumed_router, options)?;
let resumed_sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
let resumed_engine = WarEngine::new(resumed_port, Arc::clone(&resumed_store))
.with_trace_sink(resumed_sink.clone());
let resumed_outcome = resumed_engine
.resume(&resumed_graph, thread.clone())
.await?;
let resumed_outcome = drive_parleys(
&resumed_engine,
&resumed_graph,
thread,
resumed_outcome,
&case.parley_responses,
)
.await?;
tokio::time::sleep(Duration::from_millis(50)).await;
let records = resumed_sink.records();
let final_state =
resolve_battlefield(resumed_store.as_ref(), thread, &resumed_outcome).await?;
Ok((records, resumed_outcome, final_state))
}
pub fn trials(&self, pattern: &str) -> Vec<Trial> {
let mut paths: Vec<PathBuf> = match glob::glob(pattern) {
Ok(entries) => entries.filter_map(Result::ok).collect(),
Err(_) => Vec::new(),
};
paths.sort();
let mut trials = Vec::new();
for path in paths {
let stem = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("scenario")
.to_string();
match Scenario::from_path(&path) {
Ok(scenario) => {
let scenario = Arc::new(scenario);
for case in &scenario.cases {
let trial_name = format!("{stem}::{}", case.name);
let runner = self.clone();
let scenario = Arc::clone(&scenario);
let case = case.clone();
let path = path.clone();
trials.push(Trial::test(trial_name, move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|e| Failed::from(e.to_string()))?;
let report = rt.block_on(runner.run_case(
&path,
&scenario,
&case,
RunOptions::default(),
));
if report.passed() {
Ok(())
} else {
Err(Failed::from(report.render_failures()))
}
}));
}
}
Err(err) => {
trials.push(Trial::test(format!("{stem}::parse_error"), move || {
Err(Failed::from(err.to_string()))
}));
}
}
}
trials
}
}
pub fn run_eval_harness_main(pattern: &str) -> ! {
let args = Arguments::from_args();
let runner = ScenarioRunner::new();
let trials = runner.trials(pattern);
libtest_mimic::run(&args, trials).exit()
}
pub fn run_eval_harness_main_with(pattern: &str, configure: impl FnOnce(&mut ScenarioRunner)) -> ! {
let args = Arguments::from_args();
let mut runner = ScenarioRunner::new();
configure(&mut runner);
let trials = runner.trials(pattern);
libtest_mimic::run(&args, trials).exit()
}
#[macro_export]
macro_rules! eval_scenarios {
($pattern:expr) => {
fn main() {
$crate::runner::run_eval_harness_main($pattern)
}
};
($pattern:expr, $configure:expr) => {
fn main() {
$crate::runner::run_eval_harness_main_with($pattern, $configure)
}
};
}
enum LlmSource {
Scripted(HashMap<String, ScenarioLlm>),
Live(Arc<dyn LlmPort>),
}
struct ScenarioPaladinPort {
llm: LlmSource,
}
impl ScenarioPaladinPort {
fn scripted(router: HashMap<String, ScenarioLlm>) -> Self {
Self {
llm: LlmSource::Scripted(router),
}
}
fn live(provider: Arc<dyn LlmPort>) -> Self {
Self {
llm: LlmSource::Live(provider),
}
}
}
#[async_trait]
impl PaladinPort for ScenarioPaladinPort {
async fn execute(&self, paladin: &Paladin, input: &str) -> Result<PaladinResult, PaladinError> {
let prompt = PromptItem::new(PromptType::User(UserPrompt {
query: input.to_string(),
context: None,
}))
.map_err(|source| PaladinError::ConfigurationError(source.to_string()))?;
let request = LlmRequest::new(paladin.data.model.clone(), prompt);
let response = match &self.llm {
LlmSource::Scripted(router) => {
let llm = router.get(&paladin.data.name).ok_or_else(|| {
PaladinError::ConfigurationError(format!(
"ScenarioPaladinPort: no ScenarioLlm registered for paladin {:?} \
(known: {:?})",
paladin.data.name,
router.keys().collect::<Vec<_>>()
))
})?;
llm.generate(request).await
}
LlmSource::Live(provider) => provider.generate(request).await,
}
.map_err(|source| paladin_battalion::llm_failure::to_paladin_error(&source))?;
Ok(PaladinResult {
output: response.content,
usage: response.usage.clone(),
execution_time_ms: 0,
loop_count: 1,
stop_reason: StopReason::Completed,
..Default::default()
})
}
async fn execute_stream(
&self,
paladin: &Paladin,
input: &str,
) -> Result<PaladinStream, PaladinError> {
let result = self.execute(paladin, input).await?;
let (tx, rx) = tokio::sync::mpsc::channel(1);
let _ = tx
.send(Ok(PaladinStreamChunk {
text: result.output,
is_final: true,
metadata: None,
}))
.await;
Ok(rx)
}
fn validate(&self, _paladin: &Paladin) -> Result<(), PaladinError> {
Ok(())
}
}
fn build_port(
router: HashMap<String, ScenarioLlm>,
options: RunOptions,
) -> Result<Arc<dyn PaladinPort>, RunnerError> {
if options.live {
let provider = resolve_live_provider()?;
Ok(Arc::new(ScenarioPaladinPort::live(provider)))
} else {
Ok(Arc::new(ScenarioPaladinPort::scripted(router)))
}
}
fn build_llm_router(graph: &WarGraph, root: &ScenarioLlm) -> HashMap<String, ScenarioLlm> {
let mut router = HashMap::new();
for node_id in graph.node_order() {
if let Some(NodeSpec::Paladin { paladin, .. }) = graph.node(node_id) {
router.insert(paladin.data.name.clone(), root.for_node(node_id.as_str()));
}
}
router
}
struct CapturingSink {
records: Mutex<Vec<TraceRecord>>,
}
impl CapturingSink {
fn new() -> Self {
Self {
records: Mutex::new(Vec::new()),
}
}
fn records(&self) -> Vec<TraceRecord> {
self.records
.lock()
.expect("CapturingSink records lock poisoned")
.clone()
}
}
#[async_trait]
impl TraceSink for CapturingSink {
async fn on_event(&self, record: TraceRecord) -> Result<(), TraceSinkError> {
self.records
.lock()
.expect("CapturingSink records lock poisoned")
.push(record);
Ok(())
}
}
async fn run_plain<W: WaypointPort + 'static>(
store: Arc<W>,
graph: &WarGraph,
port: Arc<dyn PaladinPort>,
thread: ThreadId,
initial: StateDelta,
parley_responses: &[ParleyScript],
) -> Result<(Vec<TraceRecord>, RunOutcome, Battlefield), RunnerError> {
let sink: Arc<CapturingSink> = Arc::new(CapturingSink::new());
let engine = WarEngine::new(port, Arc::clone(&store)).with_trace_sink(sink.clone());
let outcome = engine.start(graph, thread.clone(), initial).await?;
let outcome = drive_parleys(&engine, graph, &thread, outcome, parley_responses).await?;
tokio::time::sleep(Duration::from_millis(50)).await;
let records = sink.records();
let final_state = resolve_battlefield(store.as_ref(), &thread, &outcome).await?;
Ok((records, outcome, final_state))
}
async fn drive_parleys<W: WaypointPort + 'static>(
engine: &WarEngine<W>,
graph: &WarGraph,
thread: &ThreadId,
mut outcome: RunOutcome,
parley_responses: &[ParleyScript],
) -> Result<RunOutcome, RunnerError> {
let mut idx = 0usize;
while let RunOutcome::AwaitingInput { parleys, .. } = &outcome {
if idx + parleys.len() > parley_responses.len() {
break;
}
let mut responses = Vec::with_capacity(parleys.len());
for request in parleys {
let script = &parley_responses[idx];
idx += 1;
responses.push(ParleyResponse {
parley_id: request.parley_id,
kind: request.kind.clone(),
prompt: request.prompt.clone(),
value: script.value.clone(),
responded_by: Some("paladin-eval".to_string()),
responded_at: chrono::Utc::now(),
defaulted: false,
});
}
outcome = engine.resume_with(graph, thread.clone(), responses).await?;
}
Ok(outcome)
}
async fn resolve_battlefield<W: WaypointPort>(
store: &W,
thread: &ThreadId,
outcome: &RunOutcome,
) -> Result<Battlefield, RunnerError> {
match outcome {
RunOutcome::Completed { final_state, .. } => Ok(final_state.clone()),
RunOutcome::Halted { waypoint, .. } | RunOutcome::AwaitingInput { waypoint, .. } => {
let loaded = store
.get(thread, waypoint)
.await?
.ok_or(RunnerError::NoFinalState)?;
Ok(loaded.battlefield)
}
RunOutcome::Failed {
waypoint: Some(waypoint),
..
} => {
let loaded = store
.get(thread, waypoint)
.await?
.ok_or(RunnerError::NoFinalState)?;
Ok(loaded.battlefield)
}
RunOutcome::Failed { waypoint: None, .. } => Err(RunnerError::NoFinalState),
}
}
async fn full_history<W: WaypointPort>(
store: &W,
thread: &ThreadId,
) -> Result<Vec<Waypoint>, RunnerError> {
let summaries = store.history(thread, None, None).await?;
let mut waypoints = Vec::with_capacity(summaries.len());
for summary in summaries {
if let Some(waypoint) = store.get(thread, &summary.waypoint_id).await? {
waypoints.push(waypoint);
}
}
Ok(waypoints)
}
fn temp_db_url(label: &str) -> String {
let path = std::env::temp_dir().join(format!("paladin-eval-{label}-{}.sqlite", Uuid::new_v4()));
format!("sqlite://{}", path.display())
}
fn resolve_relative(scenario_path: &Path, target: &Path) -> PathBuf {
if target.is_absolute() {
target.to_path_buf()
} else {
scenario_path
.parent()
.unwrap_or_else(|| Path::new("."))
.join(target)
}
}
fn load_graph_doc(path: &Path) -> Result<WarGraphDoc, RunnerError> {
let contents = std::fs::read_to_string(path).map_err(|source| RunnerError::Io {
path: path.to_path_buf(),
source,
})?;
let is_yaml = matches!(
path.extension().and_then(|ext| ext.to_str()),
Some("yaml") | Some("yml")
);
let doc: WarGraphDoc = if is_yaml {
serde_yaml::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
path: path.to_path_buf(),
message: source.to_string(),
})?
} else {
serde_json::from_str(&contents).map_err(|source| RunnerError::ParseGraphDoc {
path: path.to_path_buf(),
message: source.to_string(),
})?
};
Ok(doc)
}
fn snapshot_file_path(scenario_path: &Path, case_name: &str) -> PathBuf {
let dir = scenario_path.parent().unwrap_or_else(|| Path::new("."));
let stem = scenario_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("scenario");
dir.join(format!("{stem}.{case_name}.snap.json"))
}
fn battlefield_snapshot_value(battlefield: &Battlefield) -> Value {
let mut obj = serde_json::Map::new();
for field in &battlefield.schema().fields {
if let Some(value) = battlefield.get_raw(&field.name) {
obj.insert(field.name.as_str().to_string(), value.clone());
}
}
Value::Object(obj)
}
pub const PALADIN_EVAL_LIVE_ENV: &str = "PALADIN_EVAL_LIVE";
#[derive(Debug, Error)]
pub enum LiveModeError {
#[error("live mode requires --live")]
FlagNotSet,
#[error(
"live mode requires the {PALADIN_EVAL_LIVE_ENV} environment variable to be set to a \
non-blank value (deliberately never set by any CI workflow in this repository)"
)]
EnvNotSet,
#[error(
"live mode requires a configured provider credential; none of paladin-llm's compiled-in \
providers has one set (see ADR-0012)"
)]
NoProviderKey,
#[error(
"live mode's gate passed, but this build of paladin-eval was not compiled with its own \
`live` feature (enables real paladin-llm provider adapters); rebuild with \
`--features live`"
)]
FeatureNotCompiled,
#[error("failed to construct the live provider: {0}")]
ProviderConstruction(String),
}
pub fn check_live_mode(live_flag: bool) -> Result<(), LiveModeError> {
if !live_flag {
return Err(LiveModeError::FlagNotSet);
}
let env_set = std::env::var(PALADIN_EVAL_LIVE_ENV)
.map(|v| !v.trim().is_empty())
.unwrap_or(false);
if !env_set {
return Err(LiveModeError::EnvNotSet);
}
if paladin_llm::provider_factory::LlmProviderFactory::get_default_provider().is_none() {
return Err(LiveModeError::NoProviderKey);
}
Ok(())
}
fn is_content_assertion(assertion: &Assertion) -> bool {
matches!(
assertion,
Assertion::FinalStateFieldEquals { .. }
| Assertion::FinalStateFieldMatches { .. }
| Assertion::FieldJsonPathEquals { .. }
| Assertion::FinalStateSnapshot
)
}
fn should_skip_for_live(assertion: &Assertion, live: bool, allow_content_assertions: bool) -> bool {
live && is_content_assertion(assertion) && !allow_content_assertions
}
#[cfg(feature = "live")]
fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
let provider_name = paladin_llm::provider_factory::LlmProviderFactory::get_default_provider()
.ok_or(LiveModeError::NoProviderKey)?;
paladin_llm::provider_factory::LlmProviderFactory::new()
.create(&provider_name)
.map_err(|source| LiveModeError::ProviderConstruction(source.to_string()))
}
#[cfg(not(feature = "live"))]
fn resolve_live_provider() -> Result<Arc<dyn LlmPort>, LiveModeError> {
Err(LiveModeError::FeatureNotCompiled)
}
#[derive(Debug, Error)]
pub enum RunnerError {
#[error("failed to access {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("failed to parse WarGraphDoc {path}: {message}")]
ParseGraphDoc {
path: PathBuf,
message: String,
},
#[error("failed to compile WarGraphDoc {path}: {source}")]
Compile {
path: PathBuf,
#[source]
source: Box<paladin_battalion::engine::graph_doc::CompileError>,
},
#[error("scenario names unregistered graph constructor {name:?}; registered: {registered:?}")]
UnknownGraph {
name: String,
registered: Vec<String>,
},
#[error("scenario names unregistered registries {name:?}; registered: {registered:?}")]
UnknownRegistries {
name: String,
registered: Vec<String>,
},
#[error("case {case:?}: interrupt_after_superstep requires store: sqlite_temp")]
InterruptRequiresSqliteTemp {
case: String,
},
#[error("waypoint store error: {0}")]
Waypoint(#[from] WaypointError),
#[error("engine error: {0}")]
Engine(#[from] EngineError),
#[error("invalid field name {field:?}: {message}")]
InvalidField {
field: String,
message: String,
},
#[error("run finished with no persisted final state to evaluate assertions against")]
NoFinalState,
#[error("invalid thread id: {0}")]
InvalidThreadId(String),
#[error("live mode: {0}")]
Live(#[from] LiveModeError),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::scenario::{
Case as ScenarioCase, LlmScript, RunStatusValue, Scenario as ScenarioDoc, ScenarioTarget,
ScriptEntry, StoreKind as ScenarioStoreKind, Times,
};
use paladin_battalion::engine::graph::{EdgeSpec, EngineLimits};
use paladin_battalion::engine::node::{NodeContext, StateNode, StateNodeError};
use paladin_battalion::engine::{InputMapping, NodeSpec as EngineNodeSpec};
use paladin_core::base::entity::Entity;
use paladin_core::platform::container::battlefield::{
BattlefieldSchema, DispatchRule, FieldSpec,
};
use paladin_core::platform::container::directive::Directive;
use paladin_core::platform::container::paladin::{MaxLoops, PaladinData, PaladinStatus};
use paladin_core::platform::container::waypoint::NodeId;
use std::collections::BTreeMap;
fn field(name: &str) -> FieldName {
FieldName::new(name).expect("valid field name")
}
fn make_paladin(name: &str, model: &str) -> Paladin {
let data = PaladinData {
system_prompt: format!("{name} prompt"),
name: name.to_string(),
user_name: String::new(),
model: model.to_string(),
temperature: 0.7,
max_loops: MaxLoops::Fixed(1),
stop_words: vec![],
status: PaladinStatus::Idle,
vision_enabled: false,
..Default::default()
};
Entity::new(data, Some(name.to_string()))
}
fn minimal_scenario(
target: ScenarioTarget,
llm: LlmScript,
assertions: Vec<Assertion>,
) -> ScenarioDoc {
ScenarioDoc {
schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
target,
store: ScenarioStoreKind::InMemory,
llm,
cases: vec![ScenarioCase {
name: "happy_path".to_string(),
input: BTreeMap::from([("topic".to_string(), serde_json::json!("widgets"))]),
interrupt_after_superstep: None,
parley_responses: vec![],
llm: None,
assertions,
}],
live: Default::default(),
registries: None,
}
}
fn linear_fixture_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../paladin-battalion/tests/fixtures/graph_docs/linear.json")
}
#[tokio::test]
async fn runner_executes_a_graph_doc_case() {
let scenario = minimal_scenario(
ScenarioTarget::GraphDoc {
graph_doc: PathBuf::from(
"../paladin-battalion/tests/fixtures/graph_docs/linear.json",
),
},
LlmScript {
global: vec![
ScriptEntry::Text("s1".to_string()),
ScriptEntry::Text("s2".to_string()),
ScriptEntry::Text("s3".to_string()),
],
..Default::default()
},
vec![
Assertion::NodeExecuted {
node: "start".to_string(),
times: Times::Exact(1),
},
Assertion::RunStatus(RunStatusValue::Completed),
],
);
let runner = ScenarioRunner::new();
let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
let report = runner
.run_case(
&scenario_path,
&scenario,
&scenario.cases[0],
RunOptions::default(),
)
.await;
assert!(
report.passed(),
"expected the case to pass: {}",
report.render_failures()
);
assert!(linear_fixture_path().exists(), "sanity: fixture must exist");
}
#[tokio::test]
async fn runner_reports_a_failing_assertion_with_its_rendering() {
let scenario = minimal_scenario(
ScenarioTarget::GraphDoc {
graph_doc: PathBuf::from(
"../paladin-battalion/tests/fixtures/graph_docs/linear.json",
),
},
LlmScript {
global: vec![
ScriptEntry::Text("s1".to_string()),
ScriptEntry::Text("s2".to_string()),
ScriptEntry::Text("s3".to_string()),
],
..Default::default()
},
vec![Assertion::NodeExecuted {
node: "start".to_string(),
times: Times::Exact(5),
}],
);
let runner = ScenarioRunner::new();
let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
let report = runner
.run_case(
&scenario_path,
&scenario,
&scenario.cases[0],
RunOptions::default(),
)
.await;
assert!(!report.passed());
let rendered = report.render_failures();
assert!(
rendered.contains("expected: exactly 5"),
"rendering must be the assertion's own render_failure() output: {rendered:?}"
);
}
#[tokio::test]
async fn runner_executes_a_registered_target() {
let mut runner = ScenarioRunner::new();
runner.register_graph(
"solo",
Arc::new(|ports: &ScriptedPorts| {
let schema = BattlefieldSchema::new(vec![FieldSpec::new(
field("out"),
DispatchRule::LastWrite,
None,
false,
)]);
let mut graph = WarGraph::new(schema, EngineLimits::default());
let node = NodeId::new("solo");
let llm = ports.for_node("solo");
graph.add_node(
node.clone(),
EngineNodeSpec::Function(Arc::new(RegisteredEchoNode {
llm,
output_field: field("out"),
})),
);
graph.add_entry(node);
graph
}),
);
let mut scenario = minimal_scenario(
ScenarioTarget::Registered {
registered: "solo".to_string(),
},
LlmScript {
per_node: BTreeMap::from([(
"solo".to_string(),
vec![ScriptEntry::Text("registered-output".to_string())],
)]),
..Default::default()
},
vec![Assertion::FinalStateFieldEquals {
field: "out".to_string(),
value: serde_json::json!("registered-output"),
}],
);
scenario.cases[0].input = BTreeMap::new();
let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
let report = runner
.run_case(
&scenario_path,
&scenario,
&scenario.cases[0],
RunOptions::default(),
)
.await;
assert!(
report.passed(),
"expected the registered target's Function node to receive the scripted port: {}",
report.render_failures()
);
}
struct RegisteredEchoNode {
llm: ScenarioLlm,
output_field: FieldName,
}
#[async_trait::async_trait]
impl StateNode for RegisteredEchoNode {
async fn run(
&self,
_state: &Battlefield,
_ctx: &NodeContext,
) -> Result<Directive, StateNodeError> {
let request = LlmRequest::new(
"scenario-model",
PromptItem::new(PromptType::User(UserPrompt {
query: "go".to_string(),
context: None,
}))
.expect("valid prompt"),
);
let response = self
.llm
.generate(request)
.await
.map_err(|e| StateNodeError(e.to_string()))?;
let mut delta = StateDelta::new();
delta
.set(self.output_field.clone(), response.content)
.map_err(|e| StateNodeError(e.to_string()))?;
Ok(delta.into())
}
}
#[tokio::test]
async fn interrupt_and_resume_reproduces_the_control_run() {
fn build_graph() -> WarGraph {
let schema = BattlefieldSchema::new(vec![
FieldSpec::new(
field("topic"),
DispatchRule::LastWrite,
Some(serde_json::json!("widgets")),
false,
),
FieldSpec::new(field("step_one"), DispatchRule::LastWrite, None, false),
FieldSpec::new(field("step_two"), DispatchRule::LastWrite, None, false),
]);
let mut graph = WarGraph::new(schema, EngineLimits::default());
let first = NodeId::new("first");
let second = NodeId::new("second");
graph.add_node(
first.clone(),
EngineNodeSpec::paladin(
make_paladin("First", "gpt-4"),
InputMapping::new("{topic}"),
field("step_one"),
),
);
graph.add_node(
second.clone(),
EngineNodeSpec::paladin(
make_paladin("Second", "gpt-4"),
InputMapping::new("{step_one}"),
field("step_two"),
),
);
graph.add_edge(EdgeSpec {
from: first.clone(),
to: second,
condition: None,
});
graph.add_entry(first);
graph
}
let mut runner = ScenarioRunner::new();
runner.register_graph("chain", Arc::new(|_ports: &ScriptedPorts| build_graph()));
let llm = LlmScript {
per_node: BTreeMap::from([
(
"first".to_string(),
vec![ScriptEntry::Text("one".to_string())],
),
(
"second".to_string(),
vec![ScriptEntry::Text("two".to_string())],
),
]),
..Default::default()
};
let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
let control_scenario = ScenarioDoc {
schema_version: crate::scenario::EVAL_SCHEMA_VERSION.to_string(),
target: ScenarioTarget::Registered {
registered: "chain".to_string(),
},
store: ScenarioStoreKind::SqliteTemp,
llm: llm.clone(),
cases: vec![ScenarioCase {
name: "control".to_string(),
input: BTreeMap::new(),
interrupt_after_superstep: None,
parley_responses: vec![],
llm: None,
assertions: vec![],
}],
live: Default::default(),
registries: None,
};
let (control_graph, control_router) = runner
.resolve_target(&scenario_path, &control_scenario)
.expect("resolve control target");
let control_port =
build_port(control_router, RunOptions::default()).expect("scripted port");
let store = Arc::new(InMemoryWaypointStore::new());
let thread = ThreadId::new(format!("control-{}", Uuid::new_v4())).expect("valid thread id");
let (_records, _outcome, control_final_state) = run_plain(
store,
&control_graph,
control_port,
thread,
StateDelta::new(),
&[],
)
.await
.expect("control run succeeds");
let interrupted_scenario = ScenarioDoc {
store: ScenarioStoreKind::SqliteTemp,
cases: vec![ScenarioCase {
name: "interrupted".to_string(),
interrupt_after_superstep: Some(1),
..control_scenario.cases[0].clone()
}],
..control_scenario
};
let report = runner
.run_case(
&scenario_path,
&interrupted_scenario,
&interrupted_scenario.cases[0],
RunOptions::default(),
)
.await;
match &report.outcome {
CaseOutcome::Ran { final_state, .. } => {
assert_eq!(
*final_state, control_final_state,
"resumed final Battlefield must equal the control run's"
);
}
CaseOutcome::Errored(message) => panic!("interrupted case errored: {message}"),
}
}
#[test]
fn evals_target_discovers_one_trial_per_case() {
let dir = std::env::temp_dir().join(format!("paladin-eval-trials-{}", Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create temp dir");
std::fs::write(
dir.join("alpha.yaml"),
r#"
schema_version: "1"
target:
registered: "nowhere"
cases:
- name: case_one
assertions: []
- name: case_two
assertions: []
"#,
)
.expect("write alpha fixture");
std::fs::write(
dir.join("beta.yaml"),
r#"
schema_version: "1"
target:
registered: "nowhere"
cases:
- name: only_case
assertions: []
"#,
)
.expect("write beta fixture");
let runner = ScenarioRunner::new();
let pattern = dir.join("*.yaml");
let trials = runner.trials(pattern.to_str().expect("valid utf8 path"));
assert_eq!(trials.len(), 3, "one trial per (file, case) pair");
let names: Vec<&str> = trials.iter().map(Trial::name).collect();
assert!(names.contains(&"alpha::case_one"));
assert!(names.contains(&"alpha::case_two"));
assert!(names.contains(&"beta::only_case"));
let filter = Arguments {
filter: Some("case_one".to_string()),
..Default::default()
};
let selected: Vec<&Trial> = trials
.iter()
.filter(|t| !filter.is_filtered_out(t))
.collect();
assert_eq!(selected.len(), 1, "a name filter must select a single case");
assert_eq!(selected[0].name(), "alpha::case_one");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn live_mode_requires_flag_and_env_and_keys() {
unsafe {
std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
}
assert!(matches!(
check_live_mode(false),
Err(LiveModeError::FlagNotSet)
));
assert!(matches!(
check_live_mode(true),
Err(LiveModeError::EnvNotSet)
));
unsafe {
std::env::set_var(PALADIN_EVAL_LIVE_ENV, "1");
}
let compiled_provider =
paladin_llm::provider_factory::LlmProviderFactory::get_default_provider();
match compiled_provider {
None => assert!(matches!(
check_live_mode(true),
Err(LiveModeError::NoProviderKey)
)),
Some(_) => assert!(check_live_mode(true).is_ok()),
}
unsafe {
std::env::remove_var(PALADIN_EVAL_LIVE_ENV);
}
}
fn content_assertion_fixtures() -> Vec<Assertion> {
vec![
Assertion::FinalStateFieldEquals {
field: "status".to_string(),
value: serde_json::json!("done"),
},
Assertion::FinalStateFieldMatches {
field: "status".to_string(),
pattern: "^do".to_string(),
},
Assertion::FieldJsonPathEquals {
path: "/status".to_string(),
value: serde_json::json!("done"),
},
Assertion::FinalStateSnapshot,
]
}
fn structural_assertion_fixtures() -> Vec<Assertion> {
vec![
Assertion::NodeExecuted {
node: "worker".to_string(),
times: Times::Exact(1),
},
Assertion::NodeNotExecuted {
node: "idle".to_string(),
},
Assertion::EdgeFired {
from: "a".to_string(),
to: "b".to_string(),
},
Assertion::RouteTaken(vec!["a".to_string()]),
Assertion::RunStatus(RunStatusValue::Completed),
Assertion::TotalTokensMax(10),
Assertion::SuperstepsMax(10),
Assertion::ParleyRaised {
kind: "approval".to_string(),
node: "worker".to_string(),
},
]
}
#[test]
fn content_assertions_are_skipped_in_live_mode_without_opt_in() {
for assertion in content_assertion_fixtures() {
assert!(
should_skip_for_live(&assertion, true, false),
"{assertion:?} must be skipped in live mode without the scenario's opt-in"
);
}
}
#[test]
fn content_assertions_run_in_live_mode_with_opt_in() {
for assertion in content_assertion_fixtures() {
assert!(
!should_skip_for_live(&assertion, true, true),
"{assertion:?} must run in live mode once the scenario opts in"
);
}
}
#[test]
fn structural_assertions_always_run_in_live_mode() {
for assertion in structural_assertion_fixtures() {
assert!(
!should_skip_for_live(&assertion, true, false),
"{assertion:?} is structural and must always run in live mode"
);
assert!(
!should_skip_for_live(&assertion, true, true),
"{assertion:?} is structural and must always run in live mode, opt-in or not"
);
}
for assertion in content_assertion_fixtures()
.into_iter()
.chain(structural_assertion_fixtures())
{
assert!(!should_skip_for_live(&assertion, false, false));
}
}
#[tokio::test]
async fn default_test_run_never_enters_live_mode() {
let scenario = minimal_scenario(
ScenarioTarget::GraphDoc {
graph_doc: PathBuf::from(
"../paladin-battalion/tests/fixtures/graph_docs/linear.json",
),
},
LlmScript {
global: vec![
ScriptEntry::Text("s1".to_string()),
ScriptEntry::Text("s2".to_string()),
ScriptEntry::Text("s3".to_string()),
],
..Default::default()
},
vec![Assertion::RunStatus(RunStatusValue::Completed)],
);
let runner = ScenarioRunner::new();
let scenario_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("scenario.eval.yaml");
assert!(
!RunOptions::default().live,
"the harness's own default options must never carry live: true"
);
let report = runner
.run_case(
&scenario_path,
&scenario,
&scenario.cases[0],
RunOptions::default(),
)
.await;
assert!(
report.passed(),
"a plain (non-live) run must succeed entirely on scripted mocks: {}",
report.render_failures()
);
}
}