Struct Prediction 

pub struct Prediction {
    pub data: HashMap<String, Value>,
    pub lm_usage: LmUsage,
}

Fields

data: HashMap<String, Value>

lm_usage: LmUsage

Implementations

impl Prediction

pub fn new(data: HashMap<String, Value>, lm_usage: LmUsage) -> Self

pub fn get(&self, key: &str, default: Option<&str>) -> Value

Examples found in repository

examples/09-gepa-sentiment.rs (line 53)

    async fn feedback_metric(&self, example: &Example, prediction: &Prediction) -> FeedbackMetric {
        let predicted = prediction
            .get("sentiment", None)
            .as_str()
            .unwrap_or("")
            .to_string()
            .to_lowercase();

        let expected = example
            .get("expected_sentiment", None)
            .as_str()
            .unwrap_or("")
            .to_string()
            .to_lowercase();

        let text = example.get("text", None).as_str().unwrap_or("").to_string();

        let reasoning = prediction
            .get("reasoning", None)
            .as_str()
            .unwrap_or("")
            .to_string();

        // Calculate score
        let correct = predicted == expected;
        let score = if correct { 1.0 } else { 0.0 };

        // Create rich feedback
        let mut feedback = if correct {
            format!("Correct classification: \"{}\"\n", expected)
        } else {
            format!(
                "Incorrect classification\n  Expected: \"{}\"\n  Predicted: \"{}\"\n",
                expected, predicted
            )
        };

        // Add context about the input
        feedback.push_str(&format!("  Input text: \"{}\"\n", text));

        // Add reasoning analysis
        if !reasoning.is_empty() {
            feedback.push_str(&format!("  Reasoning: {}\n", reasoning));

            // Check if reasoning mentions key sentiment words
            let has_reasoning_quality = if correct {
                // For correct answers, check if reasoning is substantive
                reasoning.len() > 20
            } else {
                // For incorrect answers, note what went wrong
                false
            };

            if has_reasoning_quality {
                feedback.push_str("  Reasoning appears detailed\n");
            } else if !correct {
                feedback.push_str("  May have misunderstood the text sentiment\n");
            }
        }

        FeedbackMetric::new(score, feedback)
    }
}

#[tokio::main]
async fn main() -> Result<()> {
    println!("GEPA Sentiment Analysis Optimization Example\n");

    // Setup LM
    let lm = LM::builder().temperature(0.7).build().await.unwrap();

    configure(lm.clone(), ChatAdapter);

    // Create training examples with diverse sentiments
    let trainset = vec![
        example! {
            "text": "input" => "This movie was absolutely fantastic! I loved every minute of it.",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "Terrible service, will never come back again.",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "The weather is okay, nothing special.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "Despite some minor issues, I'm quite happy with the purchase.",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "I have mixed feelings about this product.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "This is the worst experience I've ever had!",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "It's fine. Does what it's supposed to do.",
            "expected_sentiment": "input" => "neutral"
        },
        example! {
            "text": "input" => "Exceeded all my expectations! Highly recommend!",
            "expected_sentiment": "input" => "positive"
        },
        example! {
            "text": "input" => "Disappointed and frustrated with the outcome.",
            "expected_sentiment": "input" => "negative"
        },
        example! {
            "text": "input" => "Standard quality, nothing remarkable.",
            "expected_sentiment": "input" => "neutral"
        },
    ];

    // Create module
    let mut module = SentimentAnalyzer::builder()
        .predictor(Predict::new(SentimentSignature::new()))
        .build();

    // Evaluate baseline performance
    println!("Baseline Performance:");
    let baseline_score = module.evaluate(trainset.clone()).await;
    println!("  Average score: {:.3}\n", baseline_score);

    // Configure GEPA optimizer
    let gepa = GEPA::builder()
        .num_iterations(5)
        .minibatch_size(5)
        .num_trials(3)
        .temperature(0.9)
        .track_stats(true)
        .build();

    // Run optimization
    println!("Starting GEPA optimization...\n");
    let result = gepa
        .compile_with_feedback(&mut module, trainset.clone())
        .await?;

    // Display results
    println!("\nOptimization Results:");
    println!(
        "  Best average score: {:.3}",
        result.best_candidate.average_score()
    );
    println!("  Total rollouts: {}", result.total_rollouts);
    println!("  Total LM calls: {}", result.total_lm_calls);
    println!("  Generations: {}", result.evolution_history.len());

    println!("\nBest Instruction:");
    println!("  {}", result.best_candidate.instruction);

    if !result.evolution_history.is_empty() {
        println!("\nEvolution History:");
        for entry in &result.evolution_history {
            println!("  Generation {}: {:.3}", entry.0, entry.1);
        }
    }

    // Test optimized module on a new example
    println!("\nTesting Optimized Module:");
    let test_example = example! {
        "text": "input" => "This product changed my life! Absolutely amazing!",
        "expected_sentiment": "input" => "positive"
    };

    let test_prediction = module.forward(test_example.clone()).await?;
    let test_feedback = module
        .feedback_metric(&test_example, &test_prediction)
        .await;

    println!(
        "  Test prediction: {}",
        test_prediction.get("sentiment", None)
    );
    println!("  Test score: {:.3}", test_feedback.score);
    println!("  Feedback:\n{}", test_feedback.feedback);

    Ok(())
}

examples/08-optimize-mipro.rs (line 169)

async fn main() -> Result<()> {
    println!("=== MIPROv2 Optimizer Example ===\n");

    // Configure the LM
    configure(LM::default(), ChatAdapter);

    // Load training data from HuggingFace
    println!("Loading training data from HuggingFace...");
    let train_examples = DataLoader::load_hf(
        "hotpotqa/hotpot_qa",
        vec!["question".to_string()],
        vec!["answer".to_string()],
        "fullwiki",
        "validation",
        true,
    )?;

    // Use a small subset for faster optimization
    let train_subset = train_examples[..15].to_vec();
    println!("Using {} training examples\n", train_subset.len());

    // Create the module
    let mut qa_module = SimpleQA::builder().build();

    // Show initial instruction
    println!("Initial instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test baseline performance
    println!("Evaluating baseline performance...");
    let baseline_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Baseline score: {:.3}\n", baseline_score);

    // Create MIPROv2 optimizer
    let optimizer = MIPROv2::builder()
        .num_candidates(8) // Generate 8 candidate prompts
        .num_trials(15) // Run 15 evaluation trials
        .minibatch_size(10) // Evaluate on 10 examples per candidate
        .temperature(1.0) // Temperature for prompt generation
        .track_stats(true) // Display detailed statistics
        .build();

    // Optimize the module
    println!("Starting MIPROv2 optimization...");
    println!("This will:");
    println!("  1. Generate execution traces");
    println!("  2. Create a program description using LLM");
    println!("  3. Generate {} candidate prompts with best practices", 8);
    println!("  4. Evaluate each candidate");
    println!("  5. Select and apply the best prompt\n");

    optimizer
        .compile(&mut qa_module, train_subset.clone())
        .await?;

    // Show optimized instruction
    println!("\nOptimized instruction:");
    println!(
        "  \"{}\"\n",
        qa_module.answerer.get_signature().instruction()
    );

    // Test optimized performance
    println!("Evaluating optimized performance...");
    let optimized_score = qa_module.evaluate(train_subset[..5].to_vec()).await;
    println!("Optimized score: {:.3}", optimized_score);

    // Show improvement
    let improvement = ((optimized_score - baseline_score) / baseline_score) * 100.0;
    println!(
        "\n✓ Improvement: {:.1}% ({:.3} -> {:.3})",
        improvement, baseline_score, optimized_score
    );

    // Test on a new example
    println!("\n--- Testing on a new example ---");
    let test_example = example! {
        "question": "input" => "What is the capital of France?",
    };

    let result = qa_module.forward(test_example).await?;
    println!("Question: What is the capital of France?");
    println!("Answer: {}", result.get("answer", None));

    println!("\n=== Example Complete ===");
    Ok(())
}

examples/10-gepa-llm-judge.rs (line 117)

    async fn feedback_metric(&self, example: &Example, prediction: &Prediction) -> FeedbackMetric {
        // Extract the problem and answers
        let problem = example
            .get("problem", None)
            .as_str()
            .unwrap_or("")
            .to_string();

        let expected = example
            .get("expected_answer", None)
            .as_str()
            .unwrap_or("")
            .to_string();

        let student_answer = prediction
            .get("answer", None)
            .as_str()
            .unwrap_or("")
            .to_string();

        let student_reasoning = prediction
            .get("reasoning", None)
            .as_str()
            .unwrap_or("No reasoning provided")
            .to_string();

        // Quick check: is the answer exactly correct?
        let answer_matches = student_answer.trim() == expected.trim();

        // Use LLM judge to analyze the reasoning quality
        // This is where the magic happens - the judge provides rich feedback
        let judge_input = example! {
            "problem": "input" => &problem,
            "expected_answer": "input" => &expected,
            "student_answer": "input" => &student_answer,
            "student_reasoning": "input" => &student_reasoning
        };

        let judge_output = match self
            .judge
            .forward_with_config(judge_input, Arc::clone(&self.judge_lm))
            .await
        {
            Ok(output) => output,
            Err(_) => {
                // If judge fails, fall back to simple feedback
                let score = if answer_matches { 1.0 } else { 0.0 };
                let simple_feedback = format!(
                    "Problem: {}\nExpected: {}\nPredicted: {}\nAnswer: {}",
                    problem,
                    expected,
                    student_answer,
                    if answer_matches {
                        "CORRECT"
                    } else {
                        "INCORRECT"
                    }
                );
                return FeedbackMetric::new(score, simple_feedback);
            }
        };

        let judge_evaluation = judge_output
            .get("evaluation", None)
            .as_str()
            .unwrap_or("Unable to evaluate")
            .to_string();

        // Calculate score based on answer correctness and reasoning quality
        // The judge's evaluation helps us assign partial credit
        let score = if answer_matches {
            // Correct answer - check if reasoning is also sound
            if judge_evaluation.to_lowercase().contains("sound reasoning")
                || judge_evaluation.to_lowercase().contains("correct approach")
            {
                1.0 // Perfect: right answer, good reasoning
            } else {
                0.7 // Right answer but flawed reasoning (lucky guess?)
            }
        } else {
            // Wrong answer - check if there's any partial credit
            if judge_evaluation.to_lowercase().contains("correct approach")
                || judge_evaluation.to_lowercase().contains("good start")
            {
                0.3 // Wrong answer but some valid steps
            } else {
                0.0 // Completely wrong
            }
        };

        // Construct rich textual feedback
        // This combines factual info with the judge's analysis
        let mut feedback = String::new();

        feedback.push_str(&format!("Problem: {}\n", problem));
        feedback.push_str(&format!("Expected: {}\n", expected));
        feedback.push_str(&format!("Predicted: {}\n", student_answer));

        if answer_matches {
            feedback.push_str("Answer: CORRECT\n\n");
        } else {
            feedback.push_str("Answer: INCORRECT\n\n");
        }

        feedback.push_str("Reasoning Quality Analysis:\n");
        feedback.push_str(&judge_evaluation);

        // Return the feedback metric with score and rich text
        FeedbackMetric::new(score, feedback)
    }
}

// ============================================================================
// Step 6: Main function - Set up and run GEPA optimization
// ============================================================================

#[tokio::main]
async fn main() -> Result<()> {
    println!("GEPA with LLM-as-a-Judge Example\n");
    println!("This example shows how to use an LLM judge to automatically");
    println!("generate rich feedback for optimizing a math solver.\n");

    // Setup: Configure the LLM
    // Main LM for the task
    let task_lm = LM::builder().temperature(0.7).build().await.unwrap();

    // Judge LM (could use a different/cheaper model)
    let judge_lm = LM::builder().temperature(0.3).build().await.unwrap();

    configure(task_lm, ChatAdapter);

    // Create training examples
    let trainset = vec![
        example! {
            "problem": "input" => "Sarah has 12 apples. She gives 3 to her friend and buys 5 more. How many apples does she have now?",
            "expected_answer": "input" => "14"
        },
        example! {
            "problem": "input" => "A train travels 60 miles in 1 hour. How far will it travel in 3.5 hours at the same speed?",
            "expected_answer": "input" => "210"
        },
        example! {
            "problem": "input" => "There are 24 students in a class. If 1/3 of them are absent, how many students are present?",
            "expected_answer": "input" => "16"
        },
        example! {
            "problem": "input" => "A rectangle has length 8 cm and width 5 cm. What is its area?",
            "expected_answer": "input" => "40"
        },
        example! {
            "problem": "input" => "John has $50. He spends $12 on lunch and $8 on a book. How much money does he have left?",
            "expected_answer": "input" => "30"
        },
    ];

    // Create the module
    let mut module = MathSolver::builder()
        .solver(Predict::new(MathWordProblem::new()))
        .judge(Predict::new(MathJudge::new()))
        .judge_lm(Arc::new(judge_lm))
        .build();

    // Evaluate baseline performance
    println!("Step 1: Baseline Performance");
    println!("Testing the solver before optimization...\n");
    let baseline_score = module.evaluate(trainset.clone()).await;
    println!("  Baseline average score: {:.3}\n", baseline_score);

    // Configure GEPA optimizer
    println!("Step 2: Configure GEPA");
    println!("Setting up the optimizer with budget controls...\n");

    let gepa = GEPA::builder()
        .num_iterations(3) // Fewer iterations for demo
        .minibatch_size(3) // Smaller batches
        .temperature(0.9)
        .track_stats(true)
        .maybe_max_lm_calls(Some(100)) // Important: we're using 2x LM calls (task + judge)
        .build();

    // Run GEPA optimization
    println!("Step 3: Run GEPA Optimization");
    println!("The judge will analyze reasoning quality and provide feedback...\n");

    let result = gepa
        .compile_with_feedback(&mut module, trainset.clone())
        .await?;

    // Display results
    println!("\nStep 4: Results");
    println!("===============\n");
    println!("Optimization complete!");
    println!(
        "  Best average score: {:.3}",
        result.best_candidate.average_score()
    );
    println!(
        "  Improvement: {:.3}",
        result.best_candidate.average_score() - baseline_score
    );
    println!("  Total rollouts: {}", result.total_rollouts);
    println!(
        "  Total LM calls: {} (includes judge evaluations)",
        result.total_lm_calls
    );

    println!("\nEvolution over time:");
    for (generation, score) in &result.evolution_history {
        println!("  Generation {}: {:.3}", generation, score);
    }

    println!("\nOptimized instruction:");
    println!("  {}", result.best_candidate.instruction);

    // Test the optimized solver
    println!("\nStep 5: Test Optimized Solver");
    println!("==============================\n");

    let test_problem = example! {
        "problem": "input" => "A store sells pencils for $0.25 each. If you buy 8 pencils, how much will you pay?",
        "expected_answer": "input" => "2"
    };

    let test_prediction = module.forward(test_problem.clone()).await?;
    let test_feedback = module
        .feedback_metric(&test_problem, &test_prediction)
        .await;

    println!(
        "Test problem: A store sells pencils for $0.25 each. If you buy 8 pencils, how much will you pay?"
    );
    println!("\nAnswer: {}", test_prediction.get("answer", None));
    println!("Score: {:.3}\n", test_feedback.score);
    println!("Detailed Feedback from Judge:");
    println!("{}", test_feedback.feedback);

    Ok(())
}

pub fn keys(&self) -> Vec<String>

pub fn values(&self) -> Vec<Value>

pub fn set_lm_usage(&mut self, lm_usage: LmUsage) -> Self

Examples found in repository

examples/01-simple.rs (line 66)

    async fn forward(&self, inputs: Example) -> Result<Prediction> {
        let answerer_prediction = self.answerer.forward(inputs.clone()).await?;

        let question = inputs.data.get("question").unwrap().clone();
        let answer = answerer_prediction.data.get("answer").unwrap().clone();

        let inputs = example! {
            "question": "input" => question.clone(),
            "answer": "output" => answer.clone()
        };

        let rating_prediction = self.rater.forward(inputs).await?;
        Ok(prediction! {
            "answer"=> answer,
            "question"=> question,
            "rating"=> rating_prediction.data.get("rating").unwrap().clone(),
        }
        .set_lm_usage(rating_prediction.lm_usage))
    }

examples/02-module-iteration-and-updation.rs (line 86)

    async fn forward(&self, inputs: Example) -> Result<Prediction> {
        let answerer_prediction = self.answerer.forward(inputs.clone()).await?;

        let question = inputs.data.get("question").unwrap().clone();
        let answer = answerer_prediction.data.get("answer").unwrap().clone();

        let inputs = Example::new(
            hashmap! {
                "answer".to_string() => answer.clone(),
                "question".to_string() => question.clone()
            },
            vec!["answer".to_string(), "question".to_string()],
            vec![],
        );
        let rating_prediction = self.rater.forward(inputs).await?;
        Ok(prediction! {
            "answer"=> answer,
            "question"=> question,
            "rating"=> rating_prediction.data.get("rating").unwrap().clone(),
        }
        .set_lm_usage(rating_prediction.lm_usage))
    }

examples/06-other-providers-batch.rs (line 72)

    async fn forward(&self, inputs: Example) -> Result<Prediction> {
        let answerer_prediction = self.answerer.forward(inputs.clone()).await?;

        let question = inputs.data.get("question").unwrap().clone();
        let answer = answerer_prediction.data.get("answer").unwrap().clone();
        let answer_lm_usage = answerer_prediction.lm_usage;

        let inputs = Example::new(
            hashmap! {
                "answer".to_string() => answer.clone(),
                "question".to_string() => question.clone()
            },
            vec!["answer".to_string(), "question".to_string()],
            vec![],
        );
        let rating_prediction = self.rater.forward(inputs).await?;
        let rating_lm_usage = rating_prediction.lm_usage;

        Ok(prediction! {
            "answer"=> answer,
            "question"=> question,
            "rating"=> rating_prediction.data.get("rating").unwrap().clone(),
        }
        .set_lm_usage(answer_lm_usage + rating_lm_usage))
    }

Trait Implementations

impl Clone for Prediction

fn clone(&self) -> Prediction

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Prediction

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Prediction

fn default() -> Prediction

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for Prediction

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<Vec<(String, Value)>> for Prediction

fn from(value: Vec<(String, Value)>) -> Self

Converts to this type from the input type.

impl Index<String> for Prediction

type Output = Value

The returned type after indexing.

fn index(&self, index: String) -> &Self::Output

Performs the indexing (container[index]) operation.

impl IntoIterator for Prediction

type Item = (String, Value)

The type of the elements being iterated over.

type IntoIter = IntoIter<String, Value>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.

impl Serialize for Prediction

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.