adk-eval 1.0.0

Agent evaluation framework for ADK-Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

//! Error types for the evaluation framework

use thiserror::Error;

/// Result type alias for evaluation operations
pub type Result<T> = std::result::Result<T, EvalError>;

/// Errors that can occur during evaluation
#[derive(Error, Debug)]
pub enum EvalError {
    /// Failed to load test file
    #[error("Failed to load test file: {0}")]
    LoadError(String),

    /// Failed to parse test file
    #[error("Failed to parse test file: {0}")]
    ParseError(String),

    /// Test case execution failed
    #[error("Test case execution failed: {0}")]
    ExecutionError(String),

    /// Agent error during evaluation
    #[error("Agent error: {0}")]
    AgentError(String),

    /// Invalid configuration
    #[error("Invalid configuration: {0}")]
    ConfigError(String),

    /// IO error
    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),

    /// JSON serialization error
    #[error("JSON error: {0}")]
    JsonError(#[from] serde_json::Error),

    /// Scoring error
    #[error("Scoring error: {0}")]
    ScoringError(String),

    /// LLM judge error
    #[error("LLM judge error: {0}")]
    JudgeError(String),

    /// Structured judge parse error
    #[error("Structured judge parse error: {0}")]
    JudgeParseError(String),

    /// Embedding scorer error
    #[error("Embedding scorer error: {0}")]
    EmbeddingError(String),

    /// Baseline I/O error
    #[error("Baseline I/O error: {0}")]
    BaselineError(String),

    /// Annotation format error
    #[error("Annotation format error: {0}")]
    AnnotationError(String),

    /// Test generation error
    #[error("Test generation error: {0}")]
    GenerationError(String),

    /// Statistics computation error
    #[error("Statistics computation error: {0}")]
    StatisticsError(String),
}

impl From<adk_core::AdkError> for EvalError {
    fn from(err: adk_core::AdkError) -> Self {
        EvalError::AgentError(err.to_string())
    }
}

impl From<EvalError> for adk_core::AdkError {
    fn from(err: EvalError) -> Self {
        use adk_core::{ErrorCategory, ErrorComponent};
        let (category, code) = match &err {
            EvalError::LoadError(_) => (ErrorCategory::NotFound, "eval.load"),
            EvalError::ParseError(_) => (ErrorCategory::InvalidInput, "eval.parse"),
            EvalError::ExecutionError(_) => (ErrorCategory::Internal, "eval.execution"),
            EvalError::AgentError(_) => (ErrorCategory::Internal, "eval.agent"),
            EvalError::ConfigError(_) => (ErrorCategory::InvalidInput, "eval.config"),
            EvalError::IoError(_) => (ErrorCategory::Internal, "eval.io"),
            EvalError::JsonError(_) => (ErrorCategory::Internal, "eval.json"),
            EvalError::ScoringError(_) => (ErrorCategory::Internal, "eval.scoring"),
            EvalError::JudgeError(_) => (ErrorCategory::Internal, "eval.judge"),
            EvalError::JudgeParseError(_) => (ErrorCategory::InvalidInput, "eval.judge_parse"),
            EvalError::EmbeddingError(_) => (ErrorCategory::Internal, "eval.embedding"),
            EvalError::BaselineError(_) => (ErrorCategory::Internal, "eval.baseline"),
            EvalError::AnnotationError(_) => (ErrorCategory::InvalidInput, "eval.annotation"),
            EvalError::GenerationError(_) => (ErrorCategory::Internal, "eval.generation"),
            EvalError::StatisticsError(_) => (ErrorCategory::Internal, "eval.statistics"),
        };
        adk_core::AdkError::new(ErrorComponent::Eval, category, code, err.to_string())
            .with_source(err)
    }
}