use std::collections::BTreeMap;
use std::fmt;
use std::path::{Path, PathBuf};
use schemars::JsonSchema;
use serde::de::{self, MapAccess, Visitor};
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use thiserror::Error;
use paladin_ports::output::llm_port::LlmError;
pub const EVAL_SCHEMA_VERSION: &str = "1";
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct Scenario {
pub schema_version: String,
pub target: ScenarioTarget,
#[serde(default)]
pub store: StoreKind,
#[serde(default)]
pub llm: LlmScript,
pub cases: Vec<Case>,
#[serde(default)]
pub live: LiveOptions,
#[serde(default)]
pub registries: Option<String>,
}
impl Scenario {
pub fn from_path(path: &Path) -> Result<Self, ScenarioError> {
let contents = std::fs::read_to_string(path).map_err(|source| ScenarioError::Io {
path: path.to_path_buf(),
source,
})?;
let is_yaml = matches!(
path.extension().and_then(|ext| ext.to_str()),
Some("yaml") | Some("yml")
);
let scenario: Scenario = if is_yaml {
serde_yaml::from_str(&contents).map_err(|source| ScenarioError::Parse {
path: path.to_path_buf(),
source: ParseSource::Yaml(source),
})?
} else {
serde_json::from_str(&contents).map_err(|source| ScenarioError::Parse {
path: path.to_path_buf(),
source: ParseSource::Json(source),
})?
};
scenario.validate_schema_version()?;
Ok(scenario)
}
pub fn validate_schema_version(&self) -> Result<(), ScenarioError> {
if self.schema_version != EVAL_SCHEMA_VERSION {
return Err(ScenarioError::UnsupportedSchemaVersion {
found: self.schema_version.clone(),
});
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(untagged)]
pub enum ScenarioTarget {
GraphDoc {
graph_doc: PathBuf,
},
Registered {
registered: String,
},
}
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum StoreKind {
#[default]
InMemory,
SqliteTemp,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct LlmScript {
#[serde(default)]
pub global: Vec<ScriptEntry>,
#[serde(default)]
pub per_node: BTreeMap<String, Vec<ScriptEntry>>,
#[serde(default, rename = "match")]
pub match_rules: Vec<MatchRule>,
}
#[derive(Debug, Clone, JsonSchema, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum ScriptEntry {
Text(String),
ToolCall {
name: String,
arguments: serde_json::Value,
},
Error(LlmErrorKind),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct ToolCallFields {
name: String,
arguments: serde_json::Value,
}
const SCRIPT_ENTRY_VARIANTS: &[&str] = &["text", "tool_call", "error"];
impl Serialize for ScriptEntry {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = serializer.serialize_map(Some(1))?;
match self {
ScriptEntry::Text(text) => map.serialize_entry("text", text)?,
ScriptEntry::ToolCall { name, arguments } => map.serialize_entry(
"tool_call",
&ToolCallFields {
name: name.clone(),
arguments: arguments.clone(),
},
)?,
ScriptEntry::Error(kind) => map.serialize_entry("error", kind)?,
}
map.end()
}
}
impl<'de> Deserialize<'de> for ScriptEntry {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct ScriptEntryVisitor;
impl<'de> Visitor<'de> for ScriptEntryVisitor {
type Value = ScriptEntry;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "a single-key map: `text`, `tool_call` or `error`")
}
fn visit_map<A>(self, mut map: A) -> Result<ScriptEntry, A::Error>
where
A: MapAccess<'de>,
{
let key: String = map.next_key()?.ok_or_else(|| {
de::Error::custom("expected exactly one key (`text`, `tool_call` or `error`)")
})?;
let entry = match key.as_str() {
"text" => ScriptEntry::Text(map.next_value()?),
"tool_call" => {
let fields: ToolCallFields = map.next_value()?;
ScriptEntry::ToolCall {
name: fields.name,
arguments: fields.arguments,
}
}
"error" => ScriptEntry::Error(map.next_value()?),
other => {
return Err(de::Error::unknown_variant(other, SCRIPT_ENTRY_VARIANTS));
}
};
Ok(entry)
}
}
deserializer.deserialize_any(ScriptEntryVisitor)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum LlmErrorKind {
Network,
Authentication,
InvalidPrompt,
RateLimit,
UsageLimit,
ModelNotAvailable,
TokenLimit,
EmptyCompletion,
Processing,
Timeout,
Transient,
}
impl LlmErrorKind {
pub fn to_llm_error(
&self,
node: &str,
call_index: usize,
) -> paladin_ports::output::llm_port::LlmError {
match self {
LlmErrorKind::Network => LlmError::NetworkError(format!(
"scripted network error (node {node}, call {call_index})"
)),
LlmErrorKind::Authentication => LlmError::AuthenticationError(format!(
"scripted authentication error (node {node}, call {call_index})"
)),
LlmErrorKind::InvalidPrompt => LlmError::InvalidPrompt(format!(
"scripted invalid prompt (node {node}, call {call_index})"
)),
LlmErrorKind::RateLimit => LlmError::rate_limited(None),
LlmErrorKind::UsageLimit => LlmError::UsageLimitExceeded {
provider: "scenario".to_string(),
regain_hint: None,
},
LlmErrorKind::ModelNotAvailable => LlmError::ModelNotAvailable(format!(
"scripted model-not-available (node {node}, call {call_index})"
)),
LlmErrorKind::TokenLimit => LlmError::TokenLimitExceeded,
LlmErrorKind::EmptyCompletion => LlmError::empty_completion(format!(
"scripted empty completion (node {node}, call {call_index})"
)),
LlmErrorKind::Processing => LlmError::ProcessingError(format!(
"scripted processing error (node {node}, call {call_index})"
)),
LlmErrorKind::Timeout => {
LlmError::Timeout(format!("scripted timeout (node {node}, call {call_index})"))
}
LlmErrorKind::Transient => LlmError::NetworkError(format!(
"scripted transient error (node {node}, call {call_index})"
)),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct MatchRule {
pub prompt_contains: String,
pub response: ScriptEntry,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct Case {
pub name: String,
#[serde(default)]
pub input: BTreeMap<String, serde_json::Value>,
#[serde(default)]
pub interrupt_after_superstep: Option<u64>,
#[serde(default)]
pub parley_responses: Vec<ParleyScript>,
#[serde(default)]
pub llm: Option<LlmScript>,
pub assertions: Vec<Assertion>,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct ParleyScript {
pub kind: String,
pub value: serde_json::Value,
}
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct LiveOptions {
#[serde(default)]
pub allow_content_assertions: bool,
}
#[derive(Debug, Clone, Copy, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum Times {
Exact(u32),
Min(u32),
Max(u32),
}
const TIMES_VARIANTS: &[&str] = &["exact", "min", "max"];
impl Serialize for Times {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = serializer.serialize_map(Some(1))?;
match self {
Times::Exact(v) => map.serialize_entry("exact", v)?,
Times::Min(v) => map.serialize_entry("min", v)?,
Times::Max(v) => map.serialize_entry("max", v)?,
}
map.end()
}
}
impl<'de> Deserialize<'de> for Times {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct TimesVisitor;
impl<'de> Visitor<'de> for TimesVisitor {
type Value = Times;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "a single-key map: `exact`, `min` or `max`")
}
fn visit_map<A>(self, mut map: A) -> Result<Times, A::Error>
where
A: MapAccess<'de>,
{
let key: String = map.next_key()?.ok_or_else(|| {
de::Error::custom("expected exactly one key (`exact`, `min` or `max`)")
})?;
let times = match key.as_str() {
"exact" => Times::Exact(map.next_value()?),
"min" => Times::Min(map.next_value()?),
"max" => Times::Max(map.next_value()?),
other => return Err(de::Error::unknown_variant(other, TIMES_VARIANTS)),
};
Ok(times)
}
}
deserializer.deserialize_any(TimesVisitor)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum RunStatusValue {
Completed,
Failed,
Halted,
AwaitingInput,
}
#[derive(Debug, Clone, JsonSchema, PartialEq)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum Assertion {
FinalStateFieldEquals {
field: String,
value: serde_json::Value,
},
FinalStateFieldMatches {
field: String,
pattern: String,
},
FieldJsonPathEquals {
path: String,
value: serde_json::Value,
},
NodeExecuted {
node: String,
times: Times,
},
NodeNotExecuted {
node: String,
},
EdgeFired {
from: String,
to: String,
},
RouteTaken(Vec<String>),
RunStatus(RunStatusValue),
TotalTokensMax(u64),
SuperstepsMax(u64),
ParleyRaised {
kind: String,
node: String,
},
FinalStateSnapshot,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct FieldEqualsFields {
field: String,
value: serde_json::Value,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct FieldMatchesFields {
field: String,
pattern: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct JsonPathFields {
path: String,
value: serde_json::Value,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct NodeExecutedFields {
node: String,
times: Times,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct NodeOnlyFields {
node: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct EdgeFields {
from: String,
to: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct ParleyRaisedFields {
kind: String,
node: String,
}
const ASSERTION_VARIANTS: &[&str] = &[
"final_state_field_equals",
"final_state_field_matches",
"field_json_path_equals",
"node_executed",
"node_not_executed",
"edge_fired",
"route_taken",
"run_status",
"total_tokens_max",
"supersteps_max",
"parley_raised",
"final_state_snapshot",
];
impl Serialize for Assertion {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
if let Assertion::FinalStateSnapshot = self {
return serializer.serialize_str("final_state_snapshot");
}
let mut map = serializer.serialize_map(Some(1))?;
match self {
Assertion::FinalStateFieldEquals { field, value } => map.serialize_entry(
"final_state_field_equals",
&FieldEqualsFields {
field: field.clone(),
value: value.clone(),
},
)?,
Assertion::FinalStateFieldMatches { field, pattern } => map.serialize_entry(
"final_state_field_matches",
&FieldMatchesFields {
field: field.clone(),
pattern: pattern.clone(),
},
)?,
Assertion::FieldJsonPathEquals { path, value } => map.serialize_entry(
"field_json_path_equals",
&JsonPathFields {
path: path.clone(),
value: value.clone(),
},
)?,
Assertion::NodeExecuted { node, times } => map.serialize_entry(
"node_executed",
&NodeExecutedFields {
node: node.clone(),
times: *times,
},
)?,
Assertion::NodeNotExecuted { node } => {
map.serialize_entry("node_not_executed", &NodeOnlyFields { node: node.clone() })?
}
Assertion::EdgeFired { from, to } => map.serialize_entry(
"edge_fired",
&EdgeFields {
from: from.clone(),
to: to.clone(),
},
)?,
Assertion::RouteTaken(nodes) => map.serialize_entry("route_taken", nodes)?,
Assertion::RunStatus(status) => map.serialize_entry("run_status", status)?,
Assertion::TotalTokensMax(v) => map.serialize_entry("total_tokens_max", v)?,
Assertion::SuperstepsMax(v) => map.serialize_entry("supersteps_max", v)?,
Assertion::ParleyRaised { kind, node } => map.serialize_entry(
"parley_raised",
&ParleyRaisedFields {
kind: kind.clone(),
node: node.clone(),
},
)?,
Assertion::FinalStateSnapshot => unreachable!("handled by the early return above"),
}
map.end()
}
}
impl<'de> Deserialize<'de> for Assertion {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct AssertionVisitor;
impl<'de> Visitor<'de> for AssertionVisitor {
type Value = Assertion;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"`final_state_snapshot` or a single-key map naming one of the other eleven assertion kinds"
)
}
fn visit_str<E>(self, v: &str) -> Result<Assertion, E>
where
E: de::Error,
{
match v {
"final_state_snapshot" => Ok(Assertion::FinalStateSnapshot),
other => Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
}
}
fn visit_map<A>(self, mut map: A) -> Result<Assertion, A::Error>
where
A: MapAccess<'de>,
{
let key: String = map.next_key()?.ok_or_else(|| {
de::Error::custom("expected exactly one key naming the assertion kind")
})?;
let assertion = match key.as_str() {
"final_state_field_equals" => {
let fields: FieldEqualsFields = map.next_value()?;
Assertion::FinalStateFieldEquals {
field: fields.field,
value: fields.value,
}
}
"final_state_field_matches" => {
let fields: FieldMatchesFields = map.next_value()?;
Assertion::FinalStateFieldMatches {
field: fields.field,
pattern: fields.pattern,
}
}
"field_json_path_equals" => {
let fields: JsonPathFields = map.next_value()?;
Assertion::FieldJsonPathEquals {
path: fields.path,
value: fields.value,
}
}
"node_executed" => {
let fields: NodeExecutedFields = map.next_value()?;
Assertion::NodeExecuted {
node: fields.node,
times: fields.times,
}
}
"node_not_executed" => {
let fields: NodeOnlyFields = map.next_value()?;
Assertion::NodeNotExecuted { node: fields.node }
}
"edge_fired" => {
let fields: EdgeFields = map.next_value()?;
Assertion::EdgeFired {
from: fields.from,
to: fields.to,
}
}
"route_taken" => Assertion::RouteTaken(map.next_value()?),
"run_status" => Assertion::RunStatus(map.next_value()?),
"total_tokens_max" => Assertion::TotalTokensMax(map.next_value()?),
"supersteps_max" => Assertion::SuperstepsMax(map.next_value()?),
"parley_raised" => {
let fields: ParleyRaisedFields = map.next_value()?;
Assertion::ParleyRaised {
kind: fields.kind,
node: fields.node,
}
}
"final_state_snapshot" => {
let _: serde::de::IgnoredAny = map.next_value()?;
Assertion::FinalStateSnapshot
}
other => return Err(de::Error::unknown_variant(other, ASSERTION_VARIANTS)),
};
Ok(assertion)
}
}
deserializer.deserialize_any(AssertionVisitor)
}
}
#[derive(Debug, Error)]
pub enum ParseSource {
#[error(transparent)]
Yaml(#[from] serde_yaml::Error),
#[error(transparent)]
Json(#[from] serde_json::Error),
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum ScenarioError {
#[error(
"unsupported scenario schema_version {found:?}; this build of paladin-eval understands {EVAL_SCHEMA_VERSION:?}"
)]
UnsupportedSchemaVersion {
found: String,
},
#[error("failed to parse scenario file {path}: {source}", path = path.display())]
Parse {
path: PathBuf,
#[source]
source: ParseSource,
},
#[error("failed to read scenario file {path}: {source}", path = path.display())]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
#[cfg(test)]
mod tests {
use super::*;
fn minimal_yaml() -> &'static str {
r#"
schema_version: "1"
target:
graph_doc: "fixtures/linear.json"
cases:
- name: happy_path
input:
foo: "bar"
assertions:
- final_state_field_equals:
field: "status"
value: "done"
"#
}
#[test]
fn minimal_scenario_parses() {
let scenario: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses");
assert_eq!(scenario.schema_version, "1");
assert_eq!(
scenario.target,
ScenarioTarget::GraphDoc {
graph_doc: PathBuf::from("fixtures/linear.json"),
}
);
assert_eq!(scenario.store, StoreKind::InMemory);
assert_eq!(scenario.llm, LlmScript::default());
assert_eq!(scenario.cases.len(), 1);
assert_eq!(scenario.cases[0].name, "happy_path");
assert_eq!(scenario.cases[0].assertions.len(), 1);
}
#[test]
fn json_and_yaml_agree() {
let from_yaml: Scenario = serde_yaml::from_str(minimal_yaml()).expect("parses yaml");
let json = serde_json::json!({
"schema_version": "1",
"target": { "graph_doc": "fixtures/linear.json" },
"cases": [
{
"name": "happy_path",
"input": { "foo": "bar" },
"assertions": [
{ "final_state_field_equals": { "field": "status", "value": "done" } }
]
}
]
});
let from_json: Scenario =
serde_json::from_value(json).expect("parses the equivalent json document");
assert_eq!(from_yaml, from_json);
}
#[test]
fn registered_target_parses() {
let yaml = r#"
schema_version: "1"
target:
registered: "e2e1"
cases: []
"#;
let scenario: Scenario = serde_yaml::from_str(yaml).expect("parses");
assert_eq!(
scenario.target,
ScenarioTarget::Registered {
registered: "e2e1".to_string(),
}
);
}
#[test]
fn unknown_schema_version_is_typed() {
let yaml = r#"
schema_version: "2"
target:
graph_doc: "fixtures/linear.json"
cases: []
"#;
let scenario: Scenario = serde_yaml::from_str(yaml).expect("still deserializes");
let result = scenario.validate_schema_version();
match result {
Err(ScenarioError::UnsupportedSchemaVersion { found }) => {
assert_eq!(found, "2");
}
other => panic!("expected UnsupportedSchemaVersion, got {other:?}"),
}
}
#[test]
fn unknown_assertion_kind_is_an_error() {
let result: Result<Assertion, _> = serde_yaml::from_str("made_up_kind: {}");
let err = result.expect_err("an unrecognised assertion tag must fail to deserialize");
let message = err.to_string();
assert!(
message.contains("made_up_kind"),
"error message {message:?} does not name the unrecognised tag"
);
}
#[test]
fn times_variants_round_trip() {
for (yaml, expected) in [
("exact: 3", Times::Exact(3)),
("min: 1", Times::Min(1)),
("max: 5", Times::Max(5)),
] {
let parsed: Times = serde_yaml::from_str(yaml).expect("parses");
assert_eq!(parsed, expected);
}
}
#[test]
fn script_entry_spellings_round_trip() {
let text: ScriptEntry = serde_yaml::from_str("text: hello").unwrap();
assert_eq!(text, ScriptEntry::Text("hello".to_string()));
let tool_call: ScriptEntry =
serde_yaml::from_str("tool_call:\n name: search\n arguments: {}").unwrap();
assert_eq!(
tool_call,
ScriptEntry::ToolCall {
name: "search".to_string(),
arguments: serde_json::json!({}),
}
);
let error: ScriptEntry = serde_yaml::from_str("error: transient").unwrap();
assert_eq!(error, ScriptEntry::Error(LlmErrorKind::Transient));
}
}