adk_eval/
lib.rs

1//! # adk-eval
2//!
3//! Agent evaluation framework for ADK-Rust.
4//!
5//! This crate provides comprehensive tools for testing and validating agent behavior,
6//! enabling developers to ensure their agents perform correctly and consistently.
7//!
8//! ## Features
9//!
10//! - **Test Definitions**: Structured format for defining test cases (`.test.json`)
11//! - **Trajectory Evaluation**: Validate tool call sequences
12//! - **Response Quality**: Assess final output quality with multiple metrics
13//! - **Multiple Criteria**: Ground truth, rubric-based, and LLM-judged evaluation
14//! - **Automation**: Run evaluations programmatically or via CLI
15//!
16//! ## Quick Start
17//!
18//! ```rust,ignore
19//! use adk_eval::{Evaluator, EvaluationConfig, EvaluationCriteria};
20//! use std::sync::Arc;
21//!
22//! #[tokio::main]
23//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
24//!     // Create your agent
25//!     let agent = create_my_agent()?;
26//!
27//!     // Configure evaluator
28//!     let config = EvaluationConfig {
29//!         criteria: EvaluationCriteria {
30//!             tool_trajectory_score: Some(1.0),  // Exact tool match
31//!             response_similarity: Some(0.8),    // 80% text similarity
32//!             ..Default::default()
33//!         },
34//!         ..Default::default()
35//!     };
36//!
37//!     let evaluator = Evaluator::new(config);
38//!
39//!     // Run evaluation
40//!     let result = evaluator
41//!         .evaluate_file(agent, "tests/my_agent.test.json")
42//!         .await?;
43//!
44//!     assert!(result.passed, "Evaluation failed: {:?}", result.failures);
45//!     Ok(())
46//! }
47//! ```
48
49pub mod criteria;
50pub mod error;
51pub mod evaluator;
52pub mod llm_judge;
53pub mod report;
54pub mod schema;
55pub mod scoring;
56
57#[cfg(feature = "personas")]
58pub mod personas;
59
60pub mod optimizer;
61
62// New unconditional modules
63pub mod annotation;
64pub mod baseline;
65pub mod conversation_scorer;
66pub mod cost_tracker;
67pub mod pricing;
68pub mod structured_judge;
69pub mod test_generator;
70pub mod trace_analyzer;
71
72// New feature-gated modules
73#[cfg(feature = "embedding")]
74pub mod embedding_scorer;
75
76#[cfg(feature = "ci-helpers")]
77pub mod junit_reporter;
78
79#[cfg(feature = "statistics")]
80pub mod ab_comparator;
81
82#[cfg(feature = "vertex-eval")]
83pub mod vertex;
84
85// Re-exports
86pub use criteria::{
87    EvaluationCriteria, ResponseMatchConfig, Rubric, RubricConfig, ToolTrajectoryConfig,
88};
89pub use error::{EvalError, Result};
90pub use evaluator::{EvaluationConfig, Evaluator};
91pub use llm_judge::{
92    LlmJudge, LlmJudgeConfig, RubricEvaluationResult, RubricScore, SemanticMatchResult,
93};
94pub use report::{EvaluationReport, EvaluationResult, Failure, TestCaseResult};
95pub use schema::{EvalCase, EvalSet, IntermediateData, SessionInput, TestFile, ToolUse, Turn};
96pub use scoring::{ResponseScorer, ToolTrajectoryScorer};
97
98// Optimizer re-exports
99pub use optimizer::{OptimizationResult, OptimizerConfig, PromptOptimizer};
100
101// New module re-exports
102pub use annotation::{AnnotationRecord, AnnotationStore, HumanVerdict};
103pub use baseline::{Baseline, BaselineStore, Regression};
104pub use conversation_scorer::{ConversationMetrics, ConversationScorer, ConversationScorerConfig};
105pub use cost_tracker::{CostMetrics, CostTracker};
106pub use pricing::ModelPricing;
107pub use structured_judge::{
108    JudgeRubric, ScalePoint, StructuredJudge, StructuredJudgeConfig, StructuredVerdict, Verdict,
109};
110pub use test_generator::{EvalCaseMetadata, GeneratorConfig, TestGenerator};
111pub use trace_analyzer::{
112    ToolCallRecord, TraceAnalysis, TraceAnalyzer, TraceDiagnostic, TracePattern,
113};
114
115#[cfg(feature = "embedding")]
116pub use embedding_scorer::EmbeddingScorer;
117
118#[cfg(feature = "ci-helpers")]
119pub use junit_reporter::JunitReporter;
120
121#[cfg(feature = "statistics")]
122pub use ab_comparator::AbComparator;
123
124#[cfg(feature = "vertex-eval")]
125pub use vertex::{
126    AutoraterConfig, PointwiseMetricResult, PointwiseMetricSpec, TrajectoryMetric,
127    VertexEvalClient, VertexEvalConfig, VertexEvalJudge,
128};
129
130/// Prelude for convenient imports
131pub mod prelude {
132    pub use crate::criteria::{
133        EvaluationCriteria, ResponseMatchConfig, Rubric, RubricConfig, ToolTrajectoryConfig,
134    };
135    pub use crate::error::{EvalError, Result};
136    pub use crate::evaluator::{EvaluationConfig, Evaluator};
137    pub use crate::llm_judge::{
138        LlmJudge, LlmJudgeConfig, RubricEvaluationResult, SemanticMatchResult,
139    };
140    pub use crate::report::{EvaluationReport, EvaluationResult, Failure, TestCaseResult};
141    pub use crate::schema::{EvalCase, EvalSet, TestFile, ToolUse, Turn};
142}
adk_eval/lib.rs

adk_eval/
lib.rs
