Struct Evaluator 

pub struct Evaluator { /* private fields */ }

Available on crate feature eval only.

The main evaluator struct

Implementations

impl Evaluator

pub fn new(config: EvaluationConfig) -> Evaluator

Create a new evaluator with default configuration

pub fn with_llm_judge( config: EvaluationConfig, judge_model: Arc<dyn Llm>, ) -> Evaluator

Create an evaluator with an LLM judge for semantic matching and rubric evaluation

pub fn set_llm_judge(&mut self, judge_model: Arc<dyn Llm>)

Set the LLM judge model

pub fn has_llm_judge(&self) -> bool

Check if LLM judge is available

pub async fn evaluate_file( &self, agent: Arc<dyn Agent>, path: impl AsRef<Path>, ) -> Result<EvaluationReport, EvalError>

Evaluate a test file against an agent

pub async fn evaluate_test_file( &self, agent: Arc<dyn Agent>, test_file: &TestFile, ) -> Result<EvaluationReport, EvalError>

Evaluate a TestFile struct

pub async fn evaluate_case( &self, agent: Arc<dyn Agent>, eval_case: &EvalCase, ) -> Result<EvaluationResult, EvalError>

Evaluate a single test case

pub async fn evaluate_cases_parallel( &self, agent: Arc<dyn Agent>, cases: &[EvalCase], concurrency: usize, ) -> Vec<Result<EvaluationResult, EvalError>>

Evaluate multiple test cases in parallel

pub async fn evaluate_directory( &self, agent: Arc<dyn Agent>, dir: impl AsRef<Path>, ) -> Result<Vec<EvaluationReport>, EvalError>

Evaluate a directory of test files

Trait Implementations

impl Default for Evaluator

fn default() -> Evaluator

Returns the “default value” for a type.