Trait Evaluator 

pub trait Evaluator: Module {
    const MAX_CONCURRENCY: usize = 32usize;
    const DISPLAY_PROGRESS: bool = true;

    // Required method
    async fn metric(&self, example: &Example, prediction: &Prediction) -> f32;

    // Provided method
    async fn evaluate(&self, examples: Vec<Example>) -> f32 { ... }
}

Provided Associated Constants

const MAX_CONCURRENCY: usize = 32usize

const DISPLAY_PROGRESS: bool = true

Required Methods

async fn metric(&self, example: &Example, prediction: &Prediction) -> f32

Provided Methods

async fn evaluate(&self, examples: Vec<Example>) -> f32

Examples found in repository

examples/03-evaluate-hotpotqa.rs (line 80)

async fn main() -> anyhow::Result<()> {
    configure(
        LM::builder()
            .api_key(SecretString::from(std::env::var("OPENAI_API_KEY")?))
            .build(),
        ChatAdapter {},
    );

    let examples = DataLoader::load_hf(
        "hotpotqa/hotpot_qa",
        vec!["question".to_string()],
        vec!["answer".to_string()],
        "fullwiki",
        "validation",
        true,
    )?[..128]
        .to_vec();

    let evaluator = QARater::builder().build();
    let metric = evaluator.evaluate(examples).await;

    println!("Metric: {metric}");
    Ok(())
}

examples/08-optimize-mipro.rs (line 123)

async fn main() -> Result<()> {
    println!("=== MIPROv2 Optimizer Example ===\n");

    // Configure the LM
    configure(
        LM::builder()
            .api_key(SecretString::from(std::env::var("OPENAI_API_KEY")?))
            .build(),
        ChatAdapter {},
    );

    // Load training data from HuggingFace
    println!("Loading training data from HuggingFace...");
    let train_examples = DataLoader::load_hf(
        "hotpotqa/hotpot_qa",
        vec!["question".to_string()],
        vec!["answer".to_string()],
        "fullwiki",
        "validation",
        true,
    )?;

    // Use a small subset for faster optimization
    let train_subset = train_examples[..15].to_vec();
    println!("Using {} training examples\n", train_subset.len());

    // Create the module
    let mut qa_module = SimpleQA::builder().build();

    // Show initial instruction
    println!("Initial instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test baseline performance
    println!("Evaluating baseline performance...");
    let baseline_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Baseline score: {:.3}\n", baseline_score);

    // Create MIPROv2 optimizer
    let optimizer = MIPROv2::builder()
        .num_candidates(8) // Generate 8 candidate prompts
        .num_trials(15) // Run 15 evaluation trials
        .minibatch_size(10) // Evaluate on 10 examples per candidate
        .temperature(1.0) // Temperature for prompt generation
        .track_stats(true) // Display detailed statistics
        .build();

    // Optimize the module
    println!("Starting MIPROv2 optimization...");
    println!("This will:");
    println!("  1. Generate execution traces");
    println!("  2. Create a program description using LLM");
    println!("  3. Generate {} candidate prompts with best practices", 8);
    println!("  4. Evaluate each candidate");
    println!("  5. Select and apply the best prompt\n");

    optimizer.compile(&mut qa_module, train_subset.clone()).await?;

    // Show optimized instruction
    println!("\nOptimized instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test optimized performance
    println!("Evaluating optimized performance...");
    let optimized_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Optimized score: {:.3}", optimized_score);

    // Show improvement
    let improvement = ((optimized_score - baseline_score) / baseline_score) * 100.0;
    println!(
        "\n✓ Improvement: {:.1}% ({:.3} -> {:.3})",
        improvement, baseline_score, optimized_score
    );

    // Test on a new example
    println!("\n--- Testing on a new example ---");
    let test_example = example! {
        "question": "input" => "What is the capital of France?",
    };

    let result = qa_module.forward(test_example).await?;
    println!("Question: What is the capital of France?");
    println!("Answer: {}", result.get("answer", None));

    println!("\n=== Example Complete ===");
    Ok(())
}

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors