ai-agents-eval 1.0.0-rc.15

Evaluation runner for YAML-defined AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! Declarative evaluation runner for YAML-defined agents.

pub mod assertion;
pub mod compatibility;
pub mod evidence;
pub mod fixtures;
pub mod judge;
pub mod metrics;
pub mod output;
pub mod redaction;
pub mod reset;
pub mod runner;
pub mod suite;

pub use assertion::{AssertionOutcome, AssertionResultDetail, evaluate_assertion};
pub use evidence::{DisambiguationStatus, ToolExecutionRecord, ToolExecutionSource, TurnEvidence};
pub use fixtures::{FixturesConfig, LlmFixtureMode, RecordingToolLog};
pub use judge::{JudgeConfig, JudgeInput, JudgeResolver, JudgeResult, LLMJudge};
pub use output::write_outputs;
pub use redaction::RedactedString;
pub use reset::{ResetOptions, ResetProfile};
pub use runner::{EvalRunner, EvalRunnerOptions};
pub use suite::{EvalResult, EvalSettings, EvalSuite, ScenarioResult, ScenarioStatus};

pub type Result<T> = std::result::Result<T, EvalError>;

/// Error type used by the evaluation crate.
#[derive(Debug, thiserror::Error)]
pub enum EvalError {
    #[error("configuration error: {0}")]
    Config(String),
    #[error("runtime error: {0}")]
    Runtime(String),
    #[error("assertion failed: {0}")]
    Assertion(String),
    #[error("judge error: {0}")]
    Judge(String),
    #[error("io error: {0}")]
    Io(#[from] std::io::Error),
    #[error("yaml error: {0}")]
    Yaml(#[from] serde_yaml::Error),
    #[error("json error: {0}")]
    Json(#[from] serde_json::Error),
}

impl From<ai_agents_core::AgentError> for EvalError {
    fn from(error: ai_agents_core::AgentError) -> Self {
        Self::Runtime(error.to_string())
    }
}

impl From<ai_agents_core::LLMError> for EvalError {
    fn from(error: ai_agents_core::LLMError) -> Self {
        Self::Runtime(error.to_string())
    }
}