ai_agents_eval/
suite.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3
4use ai_agents_observability::ObservabilityConfig;
5use ai_agents_observability::ObservabilityReport;
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8
9use crate::assertion::{Assertion, AssertionResultDetail};
10use crate::evidence::TurnEvidence;
11use crate::fixtures::FixturesConfig;
12use crate::redaction::RedactedString;
13use crate::reset::ResetOptions;
14use crate::{EvalError, Result};
15
16/// Top-level evaluation suite loaded from YAML or JSONL.
17#[derive(Debug, Clone, Deserialize)]
18#[serde(deny_unknown_fields)]
19pub struct EvalSuite {
20    /// Human-readable name or criterion name.
21    pub name: String,
22    /// Agent YAML path used for this run.
23    #[serde(default)]
24    pub agent: Option<PathBuf>,
25    /// Execution settings for the suite.
26    #[serde(default)]
27    pub settings: EvalSettings,
28    /// Observability assertion, setting, or report value.
29    #[serde(default)]
30    pub observability: Option<ObservabilityConfig>,
31    /// Fixtures applied while building and running agents.
32    #[serde(default)]
33    pub fixtures: FixturesConfig,
34    /// Scenario test cases in this suite.
35    #[serde(default)]
36    pub scenarios: Vec<Scenario>,
37}
38
39impl EvalSuite {
40    pub fn validate(&self, cli_agent: Option<&PathBuf>) -> Result<()> {
41        if self.name.trim().is_empty() {
42            return Err(EvalError::Config(
43                "eval suite name must not be empty".into(),
44            ));
45        }
46        if cli_agent.is_none() && self.agent.is_none() {
47            return Err(EvalError::Config(
48                "agent path is required in suite or CLI".into(),
49            ));
50        }
51        if self.scenarios.is_empty() {
52            return Err(EvalError::Config(
53                "eval suite must contain at least one scenario".into(),
54            ));
55        }
56        self.fixtures.validate()?;
57        if self.settings.max_concurrent == 0 {
58            return Err(EvalError::Config(
59                "settings.max_concurrent must be greater than zero".into(),
60            ));
61        }
62        if self.settings.timeout_per_turn_ms == 0 {
63            return Err(EvalError::Config(
64                "settings.timeout_per_turn_ms must be greater than zero".into(),
65            ));
66        }
67        if matches!(
68            self.settings.isolation,
69            IsolationMode::Suite | IsolationMode::None
70        ) {
71            return Err(EvalError::Config(
72                "settings.isolation currently supports scenario or turn".into(),
73            ));
74        }
75        if self.settings.parallel && self.settings.isolation != IsolationMode::Scenario {
76            return Err(EvalError::Config(
77                "settings.parallel currently requires isolation: scenario".into(),
78            ));
79        }
80        if self.settings.parallel
81            && self
82                .scenarios
83                .iter()
84                .any(|scenario| !scenario.env.is_empty())
85        {
86            return Err(EvalError::Config(
87                "scenario.env cannot be used with parallel execution".into(),
88            ));
89        }
90        if self
91            .scenarios
92            .iter()
93            .any(|scenario| scenario.budget.max_cost_usd.is_some())
94        {
95            let cost = self
96                .observability
97                .as_ref()
98                .map(|observability| &observability.cost)
99                .ok_or_else(|| {
100                    EvalError::Config(
101                        "budget.max_cost_usd requires suite observability.cost pricing".into(),
102                    )
103                })?;
104            if !cost.enabled {
105                return Err(EvalError::Config(
106                    "budget.max_cost_usd requires observability.cost.enabled: true".into(),
107                ));
108            }
109            if cost.pricing.is_empty() && cost.pricing_file.is_none() {
110                return Err(EvalError::Config(
111                    "budget.max_cost_usd requires observability.cost.pricing or pricing_file"
112                        .into(),
113                ));
114            }
115        }
116        let mut ids = std::collections::HashSet::new();
117        for scenario in &self.scenarios {
118            if scenario.id.trim().is_empty() {
119                return Err(EvalError::Config("scenario id must not be empty".into()));
120            }
121            if !ids.insert(scenario.id.clone()) {
122                return Err(EvalError::Config(format!(
123                    "duplicate scenario id: {}",
124                    scenario.id
125                )));
126            }
127            if !scenario.skip.is_skipped() && scenario.turns.is_empty() && scenario.steps.is_empty()
128            {
129                return Err(EvalError::Config(format!(
130                    "scenario '{}' must define turns or steps",
131                    scenario.id
132                )));
133            }
134            scenario.budget.validate(&scenario.id)?;
135            for (turn_index, turn) in scenario.turns.iter().enumerate() {
136                validate_turn_assertion(turn, &scenario.id, &format!("turns[{turn_index}]"))?;
137            }
138            for (step_index, step) in scenario.steps.iter().enumerate() {
139                if let ScenarioStep::Run(run) = step {
140                    for (turn_index, turn) in run.turns.iter().enumerate() {
141                        validate_turn_assertion(
142                            turn,
143                            &scenario.id,
144                            &format!("steps[{step_index}].run.turns[{turn_index}]"),
145                        )?;
146                    }
147                }
148            }
149        }
150        Ok(())
151    }
152}
153
154/// Execution policy for an evaluation suite.
155#[derive(Debug, Clone, Deserialize)]
156#[serde(deny_unknown_fields)]
157pub struct EvalSettings {
158    /// Optional temperature override for eval LLMs.
159    #[serde(default)]
160    pub temperature: Option<f32>,
161    /// Optional provider seed stored in LLM extra config.
162    #[serde(default)]
163    pub seed: Option<u64>,
164    /// Default timeout for one turn in milliseconds.
165    #[serde(default = "default_turn_timeout")]
166    pub timeout_per_turn_ms: u64,
167    /// Timeout for one scenario attempt in milliseconds.
168    #[serde(default)]
169    pub timeout_per_scenario_ms: Option<u64>,
170    /// Optional retry count or suite retry count.
171    #[serde(default)]
172    pub retries: u32,
173    /// Delay between retry attempts in milliseconds.
174    #[serde(default = "default_retry_delay")]
175    pub retry_delay_ms: u64,
176    /// Runtime isolation mode for scenarios or turns.
177    #[serde(default)]
178    pub isolation: IsolationMode,
179    /// Optional scenario concurrency override.
180    #[serde(default)]
181    pub parallel: bool,
182    /// Maximum concurrently running scenarios.
183    #[serde(default = "default_max_concurrent")]
184    pub max_concurrent: usize,
185    /// Stop after the first failed or errored scenario.
186    #[serde(default)]
187    pub fail_fast: bool,
188    /// Whether output artifacts should redact sensitive strings.
189    #[serde(default = "default_true")]
190    pub redact_outputs: bool,
191}
192
193impl Default for EvalSettings {
194    fn default() -> Self {
195        Self {
196            temperature: None,
197            seed: None,
198            timeout_per_turn_ms: default_turn_timeout(),
199            timeout_per_scenario_ms: None,
200            retries: 0,
201            retry_delay_ms: default_retry_delay(),
202            isolation: IsolationMode::Scenario,
203            parallel: false,
204            max_concurrent: default_max_concurrent(),
205            fail_fast: false,
206            redact_outputs: true,
207        }
208    }
209}
210
211/// Runtime isolation mode requested by a suite.
212#[derive(Debug, Clone, Deserialize, Serialize, Default, PartialEq, Eq)]
213#[serde(rename_all = "snake_case")]
214pub enum IsolationMode {
215    Turn,
216    #[default]
217    Scenario,
218    Suite,
219    None,
220}
221
222/// One test case inside an evaluation suite.
223#[derive(Debug, Clone, Deserialize)]
224#[serde(deny_unknown_fields)]
225pub struct Scenario {
226    /// Stable identifier for this item.
227    pub id: String,
228    /// Human-readable name or criterion name.
229    #[serde(default)]
230    pub name: Option<String>,
231    /// Tags used by filters and grouped metrics.
232    #[serde(default)]
233    pub tags: Vec<String>,
234    /// Optional language label for filtering, metrics, and judge context.
235    #[serde(default)]
236    pub language: Option<String>,
237    /// Actor ID used for this scenario, turn, or assertion.
238    #[serde(default)]
239    pub actor: Option<String>,
240    /// Runtime or fixture context value.
241    #[serde(default)]
242    pub context: Value,
243    /// env value for Scenario.
244    #[serde(default)]
245    pub env: HashMap<String, String>,
246    /// skip value for Scenario.
247    #[serde(default)]
248    pub skip: SkipConfig,
249    /// Protective limits shared by all attempts, turns, resets, and LLM aliases.
250    #[serde(default)]
251    pub budget: ScenarioBudget,
252    /// Turns executed by this scenario or step.
253    #[serde(default)]
254    pub turns: Vec<Turn>,
255    /// Advanced steps executed after direct turns.
256    #[serde(default)]
257    pub steps: Vec<ScenarioStep>,
258}
259
260/// Hard provider-usage limits for one scenario, shared across retries and resets.
261#[derive(Debug, Clone, Deserialize, Default)]
262#[serde(deny_unknown_fields)]
263pub struct ScenarioBudget {
264    /// Maximum provider calls that may start for this scenario.
265    #[serde(default)]
266    pub max_llm_calls: Option<u64>,
267    /// Maximum provider-reported or conservatively estimated tokens.
268    #[serde(default)]
269    pub max_total_tokens: Option<u64>,
270    /// Maximum estimated provider cost in US dollars.
271    #[serde(default)]
272    pub max_cost_usd: Option<f64>,
273}
274
275fn validate_turn_assertion(turn: &Turn, scenario_id: &str, location: &str) -> Result<()> {
276    if let Some(assertion) = &turn.assertions {
277        assertion.validate(&format!("scenario '{scenario_id}' {location}.assert"))?;
278    }
279    Ok(())
280}
281
282impl ScenarioBudget {
283    pub(crate) fn is_configured(&self) -> bool {
284        self.max_llm_calls.is_some()
285            || self.max_total_tokens.is_some()
286            || self.max_cost_usd.is_some()
287    }
288
289    fn validate(&self, scenario_id: &str) -> Result<()> {
290        if self.max_llm_calls == Some(0) {
291            return Err(EvalError::Config(format!(
292                "scenario '{scenario_id}' budget.max_llm_calls must be greater than zero"
293            )));
294        }
295        if self.max_total_tokens == Some(0) {
296            return Err(EvalError::Config(format!(
297                "scenario '{scenario_id}' budget.max_total_tokens must be greater than zero"
298            )));
299        }
300        if let Some(max_cost_usd) = self.max_cost_usd
301            && (!max_cost_usd.is_finite() || max_cost_usd <= 0.0)
302        {
303            return Err(EvalError::Config(format!(
304                "scenario '{scenario_id}' budget.max_cost_usd must be finite and greater than zero"
305            )));
306        }
307        Ok(())
308    }
309}
310
311/// One user input and assertion block inside a scenario.
312#[derive(Debug, Clone)]
313pub struct Turn {
314    /// User input sent to the runtime.
315    pub input: String,
316    /// Actor ID used for this scenario, turn, or assertion.
317    pub actor: Option<String>,
318    /// Runtime or fixture context value.
319    pub context: Value,
320    /// Whether to use streaming chat for this turn.
321    pub stream: Option<bool>,
322    /// Optional timeout override for this turn.
323    pub timeout_ms: Option<u64>,
324    /// Assertions evaluated after this turn.
325    pub assertions: Option<Assertion>,
326}
327
328const EXPECT_ERROR_CONTEXT_KEY: &str = "__ai_agents_eval_expect_error";
329
330#[derive(Deserialize)]
331#[serde(deny_unknown_fields)]
332struct TurnDefinition {
333    input: String,
334    #[serde(default)]
335    actor: Option<String>,
336    #[serde(default)]
337    context: Value,
338    #[serde(default)]
339    stream: Option<bool>,
340    #[serde(default)]
341    timeout_ms: Option<u64>,
342    #[serde(default)]
343    expect_error: Option<ExpectedError>,
344    #[serde(default, rename = "assert")]
345    assertions: Option<Assertion>,
346}
347
348impl<'de> Deserialize<'de> for Turn {
349    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
350    where
351        D: serde::Deserializer<'de>,
352    {
353        let definition = TurnDefinition::deserialize(deserializer)?;
354        let mut context = definition.context;
355        if let Some(expect_error) = definition.expect_error {
356            let object = match &mut context {
357                Value::Object(object) => object,
358                _ => {
359                    context = Value::Object(serde_json::Map::new());
360                    context
361                        .as_object_mut()
362                        .expect("context was replaced with an object")
363                }
364            };
365            object.insert(
366                EXPECT_ERROR_CONTEXT_KEY.to_string(),
367                serde_json::to_value(expect_error).map_err(serde::de::Error::custom)?,
368            );
369        }
370        Ok(Self {
371            input: definition.input,
372            actor: definition.actor,
373            context,
374            stream: definition.stream,
375            timeout_ms: definition.timeout_ms,
376            assertions: definition.assertions,
377        })
378    }
379}
380
381pub(crate) fn turn_expected_error(turn: &Turn) -> Option<ExpectedError> {
382    turn.context
383        .get(EXPECT_ERROR_CONTEXT_KEY)
384        .cloned()
385        .and_then(|value| serde_json::from_value(value).ok())
386}
387
388pub(crate) fn turn_runtime_context(turn: &Turn) -> Value {
389    let mut context = turn.context.clone();
390    if let Value::Object(object) = &mut context {
391        object.remove(EXPECT_ERROR_CONTEXT_KEY);
392    }
393    context
394}
395
396/// Runtime error expectation accepting one substring or a list of alternatives.
397#[derive(Debug, Clone, Deserialize, Serialize)]
398#[serde(untagged)]
399pub enum ExpectedError {
400    One(String),
401    Any(Vec<String>),
402}
403
404impl ExpectedError {
405    pub fn matches(&self, error: &str) -> bool {
406        self.items().iter().any(|expected| error.contains(expected))
407    }
408
409    pub fn items(&self) -> Vec<&str> {
410        match self {
411            Self::One(value) => vec![value.as_str()],
412            Self::Any(values) => values.iter().map(String::as_str).collect(),
413        }
414    }
415}
416
417/// Boolean or reason-string skip configuration.
418#[derive(Debug, Clone, Deserialize)]
419#[serde(untagged)]
420pub enum SkipConfig {
421    Bool(bool),
422    Reason(String),
423}
424
425impl Default for SkipConfig {
426    fn default() -> Self {
427        Self::Bool(false)
428    }
429}
430
431impl SkipConfig {
432    pub fn is_skipped(&self) -> bool {
433        match self {
434            Self::Bool(value) => *value,
435            Self::Reason(_) => true,
436        }
437    }
438
439    pub fn reason(&self) -> Option<String> {
440        match self {
441            Self::Bool(_) => None,
442            Self::Reason(reason) => Some(reason.clone()),
443        }
444    }
445}
446
447/// Advanced scenario action used outside direct turn lists.
448#[derive(Debug, Clone, Deserialize)]
449#[serde(rename_all = "snake_case", deny_unknown_fields)]
450pub enum ScenarioStep {
451    Run(RunStep),
452    ResetAgent(ResetStepConfig),
453    SaveSession(String),
454    LoadSession(String),
455    SetContext { values: Value },
456    SetActor { actor: String },
457    CleanupExpired,
458}
459
460/// Advanced step that runs turns and can save a session.
461#[derive(Debug, Clone, Deserialize)]
462#[serde(deny_unknown_fields)]
463pub struct RunStep {
464    /// Turns executed by this scenario or step.
465    #[serde(default)]
466    pub turns: Vec<Turn>,
467    /// Optional session name saved after a run step.
468    #[serde(default)]
469    pub save_session: Option<String>,
470}
471
472/// Boolean or object form for reset-agent steps.
473#[derive(Debug, Clone, Deserialize)]
474#[serde(untagged)]
475pub enum ResetStepConfig {
476    Bool(bool),
477    Options(ResetOptions),
478}
479
480/// Top-level result returned by an eval suite run.
481#[derive(Debug, Clone, Serialize)]
482pub struct EvalResult {
483    /// Machine-readable output schema version.
484    pub schema_version: u32,
485    /// Parsed and validated suite.
486    pub suite: String,
487    /// Agent YAML path used for this run.
488    pub agent: String,
489    /// Total count for this result or group.
490    pub total: usize,
491    /// Passed count or boolean result.
492    pub passed: usize,
493    /// Failed or errored count for this result or group.
494    pub failed: usize,
495    /// Skipped count for this result or group.
496    pub skipped: usize,
497    /// Duration in milliseconds.
498    pub duration_ms: u64,
499    /// Scenario test cases in this suite.
500    pub scenarios: Vec<ScenarioResult>,
501    /// metrics value for EvalResult.
502    pub metrics: crate::metrics::EvalMetrics,
503    /// Observability assertion, setting, or report value.
504    #[serde(skip_serializing_if = "Option::is_none")]
505    pub observability: Option<ObservabilityReport>,
506}
507
508/// Result for one evaluated scenario.
509#[derive(Debug, Clone, Serialize)]
510pub struct ScenarioResult {
511    /// Stable identifier for this item.
512    pub id: String,
513    /// Human-readable name or criterion name.
514    pub name: Option<String>,
515    /// Tags used by filters and grouped metrics.
516    pub tags: Vec<String>,
517    /// Optional language label for filtering, metrics, and judge context.
518    pub language: Option<String>,
519    /// Final or normalized status value.
520    pub status: ScenarioStatus,
521    /// High-level failure category for metrics.
522    pub failure_category: Option<FailureCategory>,
523    /// Number of scenarios that passed after retry.
524    pub flaky: bool,
525    /// Attempt results in execution order.
526    pub attempts: Vec<AttemptResult>,
527    /// Duration in milliseconds.
528    pub duration_ms: u64,
529    /// Number of retries consumed by this scenario.
530    pub retries_used: u32,
531}
532
533/// Result for one scenario attempt.
534#[derive(Debug, Clone, Serialize)]
535pub struct AttemptResult {
536    /// Zero-based attempt index.
537    pub attempt: u32,
538    /// Turns executed by this scenario or step.
539    pub turns: Vec<TurnResult>,
540    /// Final or normalized status value.
541    pub status: ScenarioStatus,
542    /// Duration in milliseconds.
543    pub duration_ms: u64,
544}
545
546/// Final status for a scenario result.
547#[derive(Debug, Clone, Serialize)]
548#[serde(rename_all = "snake_case")]
549pub enum ScenarioStatus {
550    Passed,
551    Failed { reason: String },
552    Skipped { reason: Option<String> },
553    Error { message: String },
554}
555
556impl ScenarioStatus {
557    pub fn is_passed(&self) -> bool {
558        matches!(self, Self::Passed)
559    }
560
561    pub fn is_failed(&self) -> bool {
562        matches!(self, Self::Failed { .. })
563    }
564
565    pub fn is_error(&self) -> bool {
566        matches!(self, Self::Error { .. })
567    }
568}
569
570/// High-level failure category used by metrics and reports.
571#[derive(Debug, Clone, Serialize, PartialEq, Eq, Hash)]
572#[serde(rename_all = "snake_case")]
573pub enum FailureCategory {
574    ConfigError,
575    RuntimeError,
576    AssertionFailed,
577    JudgeError,
578    FlakyPass,
579}
580
581/// Result for one evaluated turn.
582#[derive(Debug, Clone, Serialize)]
583pub struct TurnResult {
584    /// Zero-based turn index within the scenario.
585    pub index: usize,
586    /// User input sent to the runtime.
587    pub input: RedactedString,
588    /// Assistant response text or redacted output value.
589    pub response: RedactedString,
590    /// Whether the runtime produced an assistant response.
591    pub response_present: bool,
592    /// Runtime error emitted while executing this turn.
593    #[serde(skip_serializing_if = "Option::is_none")]
594    pub runtime_error: Option<RedactedString>,
595    /// Current or expected state name.
596    pub state: Option<String>,
597    /// Optional response or tool metadata.
598    #[serde(skip_serializing_if = "Option::is_none")]
599    pub metadata: Option<Value>,
600    /// Full assertion-time evidence for this turn.
601    #[serde(skip_serializing)]
602    pub evidence: TurnEvidence,
603    /// Assertion details produced for this turn.
604    pub assertion_results: Vec<AssertionResultDetail>,
605    /// latency_ms value for TurnResult.
606    pub latency_ms: u64,
607    /// Optional observability span ID.
608    #[serde(skip_serializing_if = "Option::is_none")]
609    pub observability_span_id: Option<String>,
610}
611
612fn default_turn_timeout() -> u64 {
613    30_000
614}
615
616fn default_retry_delay() -> u64 {
617    1_000
618}
619
620fn default_max_concurrent() -> usize {
621    4
622}
623
624fn default_true() -> bool {
625    true
626}
627
628#[cfg(test)]
629mod tests {
630    use super::*;
631
632    fn suite() -> EvalSuite {
633        EvalSuite {
634            name: "suite".to_string(),
635            agent: Some(PathBuf::from("agent.yaml")),
636            settings: EvalSettings::default(),
637            observability: None,
638            fixtures: FixturesConfig::default(),
639            scenarios: vec![Scenario {
640                id: "scenario-1".to_string(),
641                name: None,
642                tags: vec!["smoke".to_string()],
643                language: Some("en".to_string()),
644                actor: None,
645                context: Value::Null,
646                env: HashMap::new(),
647                skip: SkipConfig::default(),
648                budget: ScenarioBudget::default(),
649                turns: vec![Turn {
650                    input: "hello".to_string(),
651                    actor: None,
652                    context: Value::Null,
653                    stream: None,
654                    timeout_ms: None,
655                    assertions: None,
656                }],
657                steps: Vec::new(),
658            }],
659        }
660    }
661
662    #[test]
663    fn validation_accepts_minimal_suite() {
664        assert!(suite().validate(None).is_ok());
665    }
666
667    #[test]
668    fn validation_rejects_empty_suite() {
669        let mut suite = suite();
670        suite.scenarios.clear();
671        let error = suite.validate(None).unwrap_err().to_string();
672        assert!(error.contains("at least one scenario"));
673    }
674
675    #[test]
676    fn validation_rejects_invalid_deferred_mock_routes() {
677        let mut suite = suite();
678        suite.fixtures.mock_server = Some(crate::fixtures::MockServerConfig {
679            enabled: true,
680            port: None,
681            routes: vec![serde_json::json!({
682                "method": "GET",
683                "path": "/ok",
684                "statuz": 200
685            })],
686        });
687        let error = suite.validate(None).unwrap_err().to_string();
688        assert!(error.contains("fixtures.mock_server.routes[0]"));
689        assert!(error.contains("statuz"));
690    }
691
692    #[test]
693    fn validation_rejects_duplicate_ids() {
694        let mut suite = suite();
695        suite.scenarios.push(suite.scenarios[0].clone());
696        let error = suite.validate(None).unwrap_err().to_string();
697        assert!(error.contains("duplicate scenario id"));
698    }
699
700    #[test]
701    fn yaml_rejects_unknown_suite_and_assertion_fields() {
702        let suite_error = serde_yaml::from_str::<EvalSuite>(
703            "name: strict\nagent: agent.yaml\nunknown_setting: true\nscenarios: []\n",
704        )
705        .unwrap_err()
706        .to_string();
707        assert!(suite_error.contains("unknown field `unknown_setting`"));
708
709        let assertion_error = serde_yaml::from_str::<EvalSuite>(
710            r#"
711name: strict
712agent: agent.yaml
713scenarios:
714  - id: strict
715    turns:
716      - input: hello
717        assert:
718          response_contians: hello
719"#,
720        )
721        .unwrap_err()
722        .to_string();
723        assert!(assertion_error.contains("response_contians"));
724    }
725
726    #[test]
727    fn validation_rejects_empty_assertion_trees() {
728        let mut suite = suite();
729        suite.scenarios[0].turns[0].assertions = Some(Assertion::default());
730        let error = suite.validate(None).unwrap_err().to_string();
731        assert!(error.contains("assert must not be empty"));
732
733        suite.scenarios[0].turns[0].assertions = Some(Assertion {
734            all: Some(Vec::new()),
735            ..Default::default()
736        });
737        let error = suite.validate(None).unwrap_err().to_string();
738        assert!(error.contains("assert.all must contain at least one assertion"));
739
740        suite.scenarios[0].turns[0].assertions = Some(Assertion {
741            any: Some(vec![Assertion {
742                all: Some(vec![Assertion::default()]),
743                ..Default::default()
744            }]),
745            ..Default::default()
746        });
747        let error = suite.validate(None).unwrap_err().to_string();
748        assert!(error.contains("assert.any[0].all[0] must not be empty"));
749    }
750
751    #[test]
752    fn validation_rejects_empty_nested_assertion_collections() {
753        use crate::assertion::{
754            ApprovalAssertion, ApprovalAssertionObject, LlmRequestAssertion,
755            OrchestrationAssertion, PathAssertion, StringList, ToolCalledAssertion,
756            ToolCalledObject,
757        };
758        use crate::judge::JudgeAssertion;
759
760        let cases = [
761            (
762                Assertion {
763                    response_contains: Some(StringList::Many(Vec::new())),
764                    ..Default::default()
765                },
766                "response_contains",
767            ),
768            (
769                Assertion {
770                    state_in: Some(Vec::new()),
771                    ..Default::default()
772                },
773                "state_in",
774            ),
775            (
776                Assertion {
777                    llm_request: Some(LlmRequestAssertion {
778                        system_contains: Some(StringList::Many(Vec::new())),
779                        ..Default::default()
780                    }),
781                    ..Default::default()
782                },
783                "llm_request.system_contains",
784            ),
785            (
786                Assertion {
787                    approval_requested: Some(ApprovalAssertion::Object(ApprovalAssertionObject {
788                        message_contains: Some(StringList::Many(Vec::new())),
789                        ..Default::default()
790                    })),
791                    ..Default::default()
792                },
793                "approval_requested.message_contains",
794            ),
795            (
796                Assertion {
797                    tool_called: Some(ToolCalledAssertion::Object(ToolCalledObject {
798                        source_in: Some(Vec::new()),
799                        ..Default::default()
800                    })),
801                    ..Default::default()
802                },
803                "tool_called.source_in",
804            ),
805            (
806                Assertion {
807                    metadata_path: Some(PathAssertion {
808                        path: "value".to_string(),
809                        in_values: Some(Vec::new()),
810                        ..Default::default()
811                    }),
812                    ..Default::default()
813                },
814                "metadata_path.in",
815            ),
816            (
817                Assertion {
818                    orchestration: Some(OrchestrationAssertion {
819                        agents_include: Some(Vec::new()),
820                        ..Default::default()
821                    }),
822                    ..Default::default()
823                },
824                "orchestration.agents_include",
825            ),
826            (
827                Assertion {
828                    metadata_contains: Some(HashMap::new()),
829                    ..Default::default()
830                },
831                "metadata_contains",
832            ),
833            (
834                Assertion {
835                    judge: Some(JudgeAssertion {
836                        llm: None,
837                        pass_threshold: 0.75,
838                        criteria: Vec::new(),
839                    }),
840                    ..Default::default()
841                },
842                "judge.criteria",
843            ),
844        ];
845
846        for (assertion, expected) in cases {
847            let mut suite = suite();
848            suite.scenarios[0].turns[0].assertions = Some(assertion);
849            let error = suite.validate(None).unwrap_err().to_string();
850            assert!(error.contains(expected), "{error}");
851        }
852    }
853
854    #[test]
855    fn yaml_rejects_explicit_empty_observability_collections() {
856        let error = serde_yaml::from_str::<EvalSuite>(
857            r#"
858name: strict
859agent: agent.yaml
860scenarios:
861  - id: strict
862    turns:
863      - input: hello
864        assert:
865          observability:
866            dimension_counts: []
867"#,
868        )
869        .unwrap_err()
870        .to_string();
871        assert!(error.contains("assertion collection must contain at least one value"));
872    }
873
874    #[test]
875    fn expected_error_accepts_string_and_list() {
876        let one: ExpectedError = serde_yaml::from_str("timeout").unwrap();
877        let many: ExpectedError = serde_yaml::from_str("[timeout, unavailable]").unwrap();
878        assert!(one.matches("turn timeout"));
879        assert!(many.matches("service unavailable"));
880        assert!(!many.matches("permission denied"));
881
882        let turn: Turn =
883            serde_yaml::from_str("input: hello\nexpect_error: [timeout, unavailable]").unwrap();
884        assert!(turn_expected_error(&turn).unwrap().matches("turn timeout"));
885        assert_eq!(
886            turn_runtime_context(&turn),
887            Value::Object(serde_json::Map::new())
888        );
889    }
890
891    #[test]
892    fn validation_rejects_parallel_env() {
893        let mut suite = suite();
894        suite.settings.parallel = true;
895        suite.scenarios[0]
896            .env
897            .insert("TOKEN".to_string(), "secret".to_string());
898        let error = suite.validate(None).unwrap_err().to_string();
899        assert!(error.contains("scenario.env"));
900    }
901
902    #[test]
903    fn validation_rejects_invalid_scenario_budgets() {
904        let mut suite = suite();
905        suite.scenarios[0].budget.max_llm_calls = Some(0);
906        let error = suite.validate(None).unwrap_err().to_string();
907        assert!(error.contains("budget.max_llm_calls"));
908
909        suite.scenarios[0].budget.max_llm_calls = Some(1);
910        suite.scenarios[0].budget.max_total_tokens = Some(0);
911        let error = suite.validate(None).unwrap_err().to_string();
912        assert!(error.contains("budget.max_total_tokens"));
913
914        suite.scenarios[0].budget.max_total_tokens = Some(1);
915        suite.scenarios[0].budget.max_cost_usd = Some(f64::NAN);
916        let error = suite.validate(None).unwrap_err().to_string();
917        assert!(error.contains("budget.max_cost_usd"));
918    }
919
920    #[test]
921    fn validation_requires_pricing_for_cost_budgets() {
922        let mut suite = suite();
923        suite.scenarios[0].budget.max_cost_usd = Some(0.01);
924        let error = suite.validate(None).unwrap_err().to_string();
925        assert!(error.contains("observability.cost pricing"));
926
927        suite.observability = Some(ObservabilityConfig::default());
928        let error = suite.validate(None).unwrap_err().to_string();
929        assert!(error.contains("pricing or pricing_file"));
930
931        suite.observability.as_mut().unwrap().cost.pricing_file = Some("pricing.yaml".to_string());
932        assert!(suite.validate(None).is_ok());
933    }
934}
ai_agents_eval/suite.rs

ai_agents_eval/
suite.rs
