ai_agents_eval/
lib.rs

1//! Declarative evaluation runner for YAML-defined agents.
2
3pub mod assertion;
4pub mod compatibility;
5pub mod evidence;
6pub mod fixtures;
7pub mod judge;
8pub mod metrics;
9pub mod output;
10pub mod redaction;
11pub mod reset;
12pub mod runner;
13pub mod suite;
14
15pub use assertion::{AssertionOutcome, AssertionResultDetail, evaluate_assertion};
16pub use evidence::{DisambiguationStatus, ToolExecutionRecord, ToolExecutionSource, TurnEvidence};
17pub use fixtures::{FixturesConfig, LlmFixtureMode, RecordingToolLog};
18pub use judge::{JudgeConfig, JudgeInput, JudgeResolver, JudgeResult, LLMJudge};
19pub use output::write_outputs;
20pub use redaction::RedactedString;
21pub use reset::{ResetOptions, ResetProfile};
22pub use runner::{EvalRunner, EvalRunnerOptions};
23pub use suite::{EvalResult, EvalSettings, EvalSuite, ScenarioResult, ScenarioStatus};
24
25pub type Result<T> = std::result::Result<T, EvalError>;
26
27/// Error type used by the evaluation crate.
28#[derive(Debug, thiserror::Error)]
29pub enum EvalError {
30    #[error("configuration error: {0}")]
31    Config(String),
32    #[error("runtime error: {0}")]
33    Runtime(String),
34    #[error("assertion failed: {0}")]
35    Assertion(String),
36    #[error("judge error: {0}")]
37    Judge(String),
38    #[error("io error: {0}")]
39    Io(#[from] std::io::Error),
40    #[error("yaml error: {0}")]
41    Yaml(#[from] serde_yaml::Error),
42    #[error("json error: {0}")]
43    Json(#[from] serde_json::Error),
44}
45
46impl From<ai_agents_core::AgentError> for EvalError {
47    fn from(error: ai_agents_core::AgentError) -> Self {
48        Self::Runtime(error.to_string())
49    }
50}
51
52impl From<ai_agents_core::LLMError> for EvalError {
53    fn from(error: ai_agents_core::LLMError) -> Self {
54        Self::Runtime(error.to_string())
55    }
56}
ai_agents_eval/lib.rs

ai_agents_eval/
lib.rs
