ai_agents_eval/
lib.rs

1//! Declarative evaluation runner for YAML-defined agents.
2
3pub mod assertion;
4mod budget;
5pub mod compatibility;
6pub mod evidence;
7pub mod fixtures;
8pub mod judge;
9pub mod metrics;
10pub mod output;
11pub mod redaction;
12pub mod reset;
13pub mod runner;
14pub mod suite;
15
16pub use assertion::{AssertionOutcome, AssertionResultDetail, evaluate_assertion};
17pub use evidence::{DisambiguationStatus, ToolExecutionRecord, ToolExecutionSource, TurnEvidence};
18pub use fixtures::{FixturesConfig, LlmFixtureMode, RecordingToolLog};
19pub use judge::{JudgeConfig, JudgeInput, JudgeResolver, JudgeResult, LLMJudge};
20pub use output::write_outputs;
21pub use redaction::RedactedString;
22pub use reset::{ResetOptions, ResetProfile};
23pub use runner::{EvalRunner, EvalRunnerOptions};
24pub use suite::{
25    EvalResult, EvalSettings, EvalSuite, ScenarioBudget, ScenarioResult, ScenarioStatus,
26};
27
28pub type Result<T> = std::result::Result<T, EvalError>;
29
30/// Error type used by the evaluation crate.
31#[derive(Debug, thiserror::Error)]
32pub enum EvalError {
33    #[error("configuration error: {0}")]
34    Config(String),
35    #[error("runtime error: {0}")]
36    Runtime(String),
37    #[error("assertion failed: {0}")]
38    Assertion(String),
39    #[error("judge error: {0}")]
40    Judge(String),
41    #[error("io error: {0}")]
42    Io(#[from] std::io::Error),
43    #[error("yaml error: {0}")]
44    Yaml(#[from] serde_yaml::Error),
45    #[error("json error: {0}")]
46    Json(#[from] serde_json::Error),
47}
48
49impl From<ai_agents_core::AgentError> for EvalError {
50    fn from(error: ai_agents_core::AgentError) -> Self {
51        Self::Runtime(error.to_string())
52    }
53}
54
55impl From<ai_agents_core::LLMError> for EvalError {
56    fn from(error: ai_agents_core::LLMError) -> Self {
57        Self::Runtime(error.to_string())
58    }
59}
ai_agents_eval/lib.rs

ai_agents_eval/
lib.rs
