Trait Evaluator 

pub trait Evaluator: Module {
    const MAX_CONCURRENCY: usize = 32usize;
    const DISPLAY_PROGRESS: bool = true;

    // Required method
    async fn metric(&self, example: &Example, prediction: &Prediction) -> f32;

    // Provided method
    async fn evaluate(&self, examples: Vec<Example>) -> f32 { ... }
}

Provided Associated Constants

const MAX_CONCURRENCY: usize = 32usize

const DISPLAY_PROGRESS: bool = true

Required Methods

async fn metric(&self, example: &Example, prediction: &Prediction) -> f32

Provided Methods

async fn evaluate(&self, examples: Vec<Example>) -> f32

Examples found in repository

examples/03-evaluate-hotpotqa.rs (line 84)

async fn main() -> anyhow::Result<()> {
    configure(
        LM::builder()
            .model("openai:gpt-4o-mini".to_string())
            .build()
            .await?,
        ChatAdapter {},
    );

    let examples = DataLoader::load_hf(
        "hotpotqa/hotpot_qa",
        vec!["question".to_string()],
        vec!["answer".to_string()],
        "fullwiki",
        "validation",
        true,
    )?[..128]
        .to_vec();

    let evaluator = QARater::builder().build();
    let metric = evaluator.evaluate(examples).await;

    println!("Metric: {metric}");
    Ok(())
}

examples/08-optimize-mipro.rs (line 117)

async fn main() -> Result<()> {
    println!("=== MIPROv2 Optimizer Example ===\n");

    // Configure the LM
    configure(LM::default(), ChatAdapter);

    // Load training data from HuggingFace
    println!("Loading training data from HuggingFace...");
    let train_examples = DataLoader::load_hf(
        "hotpotqa/hotpot_qa",
        vec!["question".to_string()],
        vec!["answer".to_string()],
        "fullwiki",
        "validation",
        true,
    )?;

    // Use a small subset for faster optimization
    let train_subset = train_examples[..15].to_vec();
    println!("Using {} training examples\n", train_subset.len());

    // Create the module
    let mut qa_module = SimpleQA::builder().build();

    // Show initial instruction
    println!("Initial instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test baseline performance
    println!("Evaluating baseline performance...");
    let baseline_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Baseline score: {:.3}\n", baseline_score);

    // Create MIPROv2 optimizer
    let optimizer = MIPROv2::builder()
        .num_candidates(8) // Generate 8 candidate prompts
        .num_trials(15) // Run 15 evaluation trials
        .minibatch_size(10) // Evaluate on 10 examples per candidate
        .temperature(1.0) // Temperature for prompt generation
        .track_stats(true) // Display detailed statistics
        .build();

    // Optimize the module
    println!("Starting MIPROv2 optimization...");
    println!("This will:");
    println!("  1. Generate execution traces");
    println!("  2. Create a program description using LLM");
    println!("  3. Generate {} candidate prompts with best practices", 8);
    println!("  4. Evaluate each candidate");
    println!("  5. Select and apply the best prompt\n");

    optimizer
        .compile(&mut qa_module, train_subset.clone())
        .await?;

    // Show optimized instruction
    println!("\nOptimized instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test optimized performance
    println!("Evaluating optimized performance...");
    let optimized_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Optimized score: {:.3}", optimized_score);

    // Show improvement
    let improvement = ((optimized_score - baseline_score) / baseline_score) * 100.0;
    println!(
        "\n✓ Improvement: {:.1}% ({:.3} -> {:.3})",
        improvement, baseline_score, optimized_score
    );

    // Test on a new example
    println!("\n--- Testing on a new example ---");
    let test_example = example! {
        "question": "input" => "What is the capital of France?",
    };

    let result = qa_module.forward(test_example).await?;
    println!("Question: What is the capital of France?");
    println!("Answer: {}", result.get("answer", None));

    println!("\n=== Example Complete ===");
    Ok(())
}

examples/09-gepa-sentiment.rs (line 175)

async fn main() -> Result<()> {
    println!("GEPA Sentiment Analysis Optimization Example\n");

    // Setup LM
    let lm = LM::builder().temperature(0.7).build().await.unwrap();

    configure(lm.clone(), ChatAdapter);

    // Create training examples with diverse sentiments
    let trainset = vec![
        example! {
            "text": "input" => "This movie was absolutely fantastic! I loved every minute of it.",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "Terrible service, will never come back again.",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "The weather is okay, nothing special.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "Despite some minor issues, I'm quite happy with the purchase.",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "I have mixed feelings about this product.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "This is the worst experience I've ever had!",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "It's fine. Does what it's supposed to do.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "Exceeded all my expectations! Highly recommend!",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "Disappointed and frustrated with the outcome.",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "Standard quality, nothing remarkable.",
            "expected_sentiment": "input" => "neutral"
        },
    ];

    // Create module
    let mut module = SentimentAnalyzer::builder()
        .predictor(Predict::new(SentimentSignature::new()))
        .build();

    // Evaluate baseline performance
    println!("Baseline Performance:");
    let baseline_score = module.evaluate(trainset.clone()).await;
    println!("  Average score: {:.3}\n", baseline_score);

    // Configure GEPA optimizer
    let gepa = GEPA::builder()
        .num_iterations(5)
        .minibatch_size(5)
        .num_trials(3)
        .temperature(0.9)
        .track_stats(true)
        .build();

    // Run optimization
    println!("Starting GEPA optimization...\n");
    let result = gepa
        .compile_with_feedback(&mut module, trainset.clone())
        .await?;

    // Display results
    println!("\nOptimization Results:");
    println!(
        "  Best average score: {:.3}",
        result.best_candidate.average_score()
    );
    println!("  Total rollouts: {}", result.total_rollouts);
    println!("  Total LM calls: {}", result.total_lm_calls);
    println!("  Generations: {}", result.evolution_history.len());

    println!("\nBest Instruction:");
    println!("  {}", result.best_candidate.instruction);

    if !result.evolution_history.is_empty() {
        println!("\nEvolution History:");
        for entry in &result.evolution_history {
            println!("  Generation {}: {:.3}", entry.0, entry.1);
        }
    }

    // Test optimized module on a new example
    println!("\nTesting Optimized Module:");
    let test_example = example! {
        "text": "input" => "This product changed my life! Absolutely amazing!",
        "expected_sentiment": "input" => "positive"
    };

    let test_prediction = module.forward(test_example.clone()).await?;
    let test_feedback = module
        .feedback_metric(&test_example, &test_prediction)
        .await;

    println!(
        "  Test prediction: {}",
        test_prediction.get("sentiment", None)
    );
    println!("  Test score: {:.3}", test_feedback.score);
    println!("  Feedback:\n{}", test_feedback.feedback);

    Ok(())
}

examples/10-gepa-llm-judge.rs (line 267)

async fn main() -> Result<()> {
    println!("GEPA with LLM-as-a-Judge Example\n");
    println!("This example shows how to use an LLM judge to automatically");
    println!("generate rich feedback for optimizing a math solver.\n");

    // Setup: Configure the LLM
    // Main LM for the task
    let task_lm = LM::builder().temperature(0.7).build().await.unwrap();

    // Judge LM (could use a different/cheaper model)
    let judge_lm = LM::builder().temperature(0.3).build().await.unwrap();

    configure(task_lm, ChatAdapter);

    // Create training examples
    let trainset = vec![
        example! {
            "problem": "input" => "Sarah has 12 apples. She gives 3 to her friend and buys 5 more. How many apples does she have now?",
            "expected_answer": "input" => "14"
        },
        example! {
            "problem": "input" => "A train travels 60 miles in 1 hour. How far will it travel in 3.5 hours at the same speed?",
            "expected_answer": "input" => "210"
        },
        example! {
            "problem": "input" => "There are 24 students in a class. If 1/3 of them are absent, how many students are present?",
            "expected_answer": "input" => "16"
        },
        example! {
            "problem": "input" => "A rectangle has length 8 cm and width 5 cm. What is its area?",
            "expected_answer": "input" => "40"
        },
        example! {
            "problem": "input" => "John has $50. He spends $12 on lunch and $8 on a book. How much money does he have left?",
            "expected_answer": "input" => "30"
        },
    ];

    // Create the module
    let mut module = MathSolver::builder()
        .solver(Predict::new(MathWordProblem::new()))
        .judge(Predict::new(MathJudge::new()))
        .judge_lm(Arc::new(judge_lm))
        .build();

    // Evaluate baseline performance
    println!("Step 1: Baseline Performance");
    println!("Testing the solver before optimization...\n");
    let baseline_score = module.evaluate(trainset.clone()).await;
    println!("  Baseline average score: {:.3}\n", baseline_score);

    // Configure GEPA optimizer
    println!("Step 2: Configure GEPA");
    println!("Setting up the optimizer with budget controls...\n");

    let gepa = GEPA::builder()
        .num_iterations(3) // Fewer iterations for demo
        .minibatch_size(3) // Smaller batches
        .temperature(0.9)
        .track_stats(true)
        .maybe_max_lm_calls(Some(100)) // Important: we're using 2x LM calls (task + judge)
        .build();

    // Run GEPA optimization
    println!("Step 3: Run GEPA Optimization");
    println!("The judge will analyze reasoning quality and provide feedback...\n");

    let result = gepa
        .compile_with_feedback(&mut module, trainset.clone())
        .await?;

    // Display results
    println!("\nStep 4: Results");
    println!("===============\n");
    println!("Optimization complete!");
    println!(
        "  Best average score: {:.3}",
        result.best_candidate.average_score()
    );
    println!(
        "  Improvement: {:.3}",
        result.best_candidate.average_score() - baseline_score
    );
    println!("  Total rollouts: {}", result.total_rollouts);
    println!(
        "  Total LM calls: {} (includes judge evaluations)",
        result.total_lm_calls
    );

    println!("\nEvolution over time:");
    for (generation, score) in &result.evolution_history {
        println!("  Generation {}: {:.3}", generation, score);
    }

    println!("\nOptimized instruction:");
    println!("  {}", result.best_candidate.instruction);

    // Test the optimized solver
    println!("\nStep 5: Test Optimized Solver");
    println!("==============================\n");

    let test_problem = example! {
        "problem": "input" => "A store sells pencils for $0.25 each. If you buy 8 pencils, how much will you pay?",
        "expected_answer": "input" => "2"
    };

    let test_prediction = module.forward(test_problem.clone()).await?;
    let test_feedback = module
        .feedback_metric(&test_problem, &test_prediction)
        .await;

    println!(
        "Test problem: A store sells pencils for $0.25 each. If you buy 8 pencils, how much will you pay?"
    );
    println!("\nAnswer: {}", test_prediction.get("answer", None));
    println!("Score: {:.3}\n", test_feedback.score);
    println!("Detailed Feedback from Judge:");
    println!("{}", test_feedback.feedback);

    Ok(())
}

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors