use anyhow::{Context, Result, anyhow};
use bon::Builder;
use serde::{Deserialize, Serialize};
use crate::evaluate::{MetricOutcome, TypedMetric, average_score};
use crate::optimizer::{
Optimizer, evaluate_module_with_metric, predictor_names, with_named_predictor,
};
use crate::predictors::Example;
use crate::{BamlType, BamlValue, Facet, Module, Signature};
use super::pareto::ParetoFrontier;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GEPACandidate {
pub id: usize,
pub instruction: String,
pub module_name: String,
pub example_scores: Vec<f32>,
pub parent_id: Option<usize>,
pub generation: usize,
}
impl GEPACandidate {
pub fn average_score(&self) -> f32 {
if self.example_scores.is_empty() {
return 0.0;
}
self.example_scores.iter().sum::<f32>() / self.example_scores.len() as f32
}
pub fn mutate(&self, new_instruction: String, generation: usize) -> Self {
Self {
id: 0,
instruction: new_instruction,
module_name: self.module_name.clone(),
example_scores: Vec::new(),
parent_id: Some(self.id),
generation,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GEPAResult {
pub best_candidate: GEPACandidate,
pub all_candidates: Vec<GEPACandidate>,
pub total_rollouts: usize,
pub total_lm_calls: usize,
pub evolution_history: Vec<(usize, f32)>,
pub highest_score_achieved_per_val_task: Vec<f32>,
pub best_outputs_valset: Option<Vec<BamlValue>>,
pub frontier_history: Vec<ParetoStatistics>,
}
pub use super::pareto::ParetoStatistics;
#[derive(Builder)]
pub struct GEPA {
#[builder(default = 20)]
pub num_iterations: usize,
#[builder(default = 25)]
pub minibatch_size: usize,
#[builder(default = 10)]
pub num_trials: usize,
#[builder(default = 1.0)]
pub temperature: f32,
#[builder(default = true)]
pub track_stats: bool,
#[builder(default = false)]
pub track_best_outputs: bool,
pub max_rollouts: Option<usize>,
pub max_lm_calls: Option<usize>,
pub prompt_model: Option<crate::LM>,
}
impl GEPA {
fn would_exceed_budget(current: usize, batch_cost: usize, max_budget: Option<usize>) -> bool {
max_budget.is_some_and(|max| current.saturating_add(batch_cost) > max)
}
fn set_instruction<M>(module: &mut M, module_name: &str, instruction: String) -> Result<()>
where
M: for<'a> Facet<'a>,
{
with_named_predictor(module, module_name, |predictor| {
predictor.set_instruction(instruction);
Ok(())
})
}
async fn evaluate_candidate<S, M, MT>(
&self,
module: &mut M,
module_name: &str,
instruction: &str,
examples: &[Example<S>],
metric: &MT,
) -> Result<Vec<MetricOutcome>>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let original_state =
with_named_predictor(module, module_name, |predictor| Ok(predictor.dump_state()))?;
Self::set_instruction(module, module_name, instruction.to_string())?;
let evaluation = evaluate_module_with_metric(&*module, examples, metric).await;
match evaluation {
Ok(outcomes) => {
with_named_predictor(module, module_name, |predictor| {
predictor.load_state(original_state.clone())
})?;
Ok(outcomes)
}
Err(eval_err) => {
if let Err(restore_err) = with_named_predictor(module, module_name, |predictor| {
predictor.load_state(original_state)
}) {
return Err(anyhow!(
"candidate evaluation failed: {eval_err}; failed to restore predictor state: {restore_err}"
));
}
Err(eval_err)
}
}
}
fn require_feedback(
outcomes: &[MetricOutcome],
module_name: &str,
generation: usize,
) -> Result<()> {
if outcomes.iter().any(|o| o.feedback.is_none()) {
return Err(anyhow!(
"GEPA requires feedback for every evaluated example (module=`{module_name}`, generation={generation})"
));
}
Ok(())
}
fn summarize_feedback(outcomes: &[MetricOutcome]) -> String {
let mut lines = Vec::new();
for (idx, outcome) in outcomes.iter().enumerate() {
if let Some(feedback) = &outcome.feedback {
lines.push(format!(
"{}: score={:.3}; {}",
idx + 1,
outcome.score,
feedback.feedback
));
}
}
lines.join("\n")
}
async fn collect_best_outputs<S, M>(
module: &M,
eval_set: &[Example<S>],
) -> Result<Vec<BamlValue>>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input>,
M::Output: BamlType,
{
let mut outputs = Vec::with_capacity(eval_set.len());
for example in eval_set {
let input = example.input.clone();
let predicted = module.call(input).await.map_err(|err| anyhow!("{err}"))?;
outputs.push(predicted.into_inner().to_baml_value());
}
Ok(outputs)
}
pub async fn compile_with_valset<S, M, MT>(
&self,
module: &mut M,
trainset: Vec<Example<S>>,
valset: Option<Vec<Example<S>>>,
metric: &MT,
) -> Result<GEPAResult>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let eval_set = valset.as_deref().unwrap_or(&trainset);
let predictor_names = predictor_names(module)?;
if predictor_names.is_empty() {
return Err(anyhow!("no optimizable predictors found"));
}
let mut frontier = ParetoFrontier::new();
let mut total_lm_calls = 0usize;
let mut total_rollouts = 0usize;
for module_name in &predictor_names {
if Self::would_exceed_budget(total_lm_calls, eval_set.len(), self.max_lm_calls)
|| Self::would_exceed_budget(total_rollouts, eval_set.len(), self.max_rollouts)
{
break;
}
let instruction = {
with_named_predictor(module, module_name, |predictor| Ok(predictor.instruction()))?
};
let outcomes = self
.evaluate_candidate::<S, _, _>(module, module_name, &instruction, eval_set, metric)
.await?;
total_lm_calls = total_lm_calls.saturating_add(outcomes.len());
total_rollouts = total_rollouts.saturating_add(outcomes.len());
Self::require_feedback(&outcomes, module_name, 0)?;
let scores: Vec<f32> = outcomes.iter().map(|o| o.score).collect();
let candidate = GEPACandidate {
id: 0,
instruction,
module_name: module_name.clone(),
example_scores: scores.clone(),
parent_id: None,
generation: 0,
};
frontier.add_candidate(candidate, &scores);
}
let mut all_candidates = Vec::new();
let mut evolution_history = Vec::new();
let mut frontier_history = Vec::new();
for generation in 0..self.num_iterations {
if let Some(max_rollouts) = self.max_rollouts
&& total_rollouts >= max_rollouts
{
break;
}
if let Some(max_lm_calls) = self.max_lm_calls
&& total_lm_calls >= max_lm_calls
{
break;
}
let parent = frontier
.sample_proportional_to_coverage()
.context("failed to sample from frontier")?
.clone();
let minibatch_end = trainset.len().min(self.minibatch_size.max(1));
let minibatch = &trainset[..minibatch_end];
if Self::would_exceed_budget(total_lm_calls, minibatch.len(), self.max_lm_calls)
|| Self::would_exceed_budget(total_rollouts, minibatch.len(), self.max_rollouts)
{
break;
}
let parent_outcomes = self
.evaluate_candidate::<S, _, _>(
module,
&parent.module_name,
&parent.instruction,
minibatch,
metric,
)
.await?;
total_lm_calls = total_lm_calls.saturating_add(parent_outcomes.len());
Self::require_feedback(&parent_outcomes, &parent.module_name, generation)?;
let feedback_summary = Self::summarize_feedback(&parent_outcomes);
let parent_score = average_score(&parent_outcomes);
total_rollouts += parent_outcomes.len();
if Self::would_exceed_budget(total_lm_calls, eval_set.len(), self.max_lm_calls)
|| Self::would_exceed_budget(total_rollouts, eval_set.len(), self.max_rollouts)
{
break;
}
let child_instruction = format!(
"{}\n\n[GEPA gen {}] Improve based on feedback:\n{}\n(Parent score {:.3})",
parent.instruction,
generation + 1,
feedback_summary,
parent_score,
);
let child = parent.mutate(child_instruction, generation + 1);
let child_outcomes = self
.evaluate_candidate::<S, _, _>(
module,
&child.module_name,
&child.instruction,
eval_set,
metric,
)
.await?;
total_lm_calls = total_lm_calls.saturating_add(child_outcomes.len());
Self::require_feedback(&child_outcomes, &child.module_name, generation + 1)?;
let child_scores: Vec<f32> = child_outcomes.iter().map(|o| o.score).collect();
total_rollouts += child_scores.len();
let mut child = child;
child.example_scores = child_scores.clone();
let _added = frontier.add_candidate(child.clone(), &child_scores);
if self.track_stats {
all_candidates.push(child);
let best_avg = frontier
.best_by_average()
.map(|c| c.average_score())
.unwrap_or(0.0);
evolution_history.push((generation, best_avg));
frontier_history.push(frontier.statistics());
}
}
let best_candidate = frontier
.best_by_average()
.cloned()
.context("no candidates available on Pareto frontier")?;
Self::set_instruction(
module,
&best_candidate.module_name,
best_candidate.instruction.clone(),
)?;
let highest_score_achieved_per_val_task = if frontier.is_empty() {
Vec::new()
} else {
let mut highs = vec![f32::MIN; eval_set.len()];
for candidate in frontier.candidates() {
for (idx, score) in candidate.example_scores.iter().enumerate() {
if idx < highs.len() {
highs[idx] = highs[idx].max(*score);
}
}
}
highs
};
let best_outputs_valset = if self.track_best_outputs {
if Self::would_exceed_budget(total_lm_calls, eval_set.len(), self.max_lm_calls)
|| Self::would_exceed_budget(total_rollouts, eval_set.len(), self.max_rollouts)
{
tracing::debug!(
eval_examples = eval_set.len(),
total_lm_calls,
total_rollouts,
max_lm_calls = ?self.max_lm_calls,
max_rollouts = ?self.max_rollouts,
"skipping best output collection because budget would be exceeded"
);
None
} else {
let outputs = Self::collect_best_outputs::<S, _>(module, eval_set).await?;
total_lm_calls = total_lm_calls.saturating_add(eval_set.len());
total_rollouts = total_rollouts.saturating_add(eval_set.len());
Some(outputs)
}
} else {
None
};
Ok(GEPAResult {
best_candidate,
all_candidates,
total_rollouts,
total_lm_calls,
evolution_history,
highest_score_achieved_per_val_task,
best_outputs_valset,
frontier_history,
})
}
}
impl Optimizer for GEPA {
type Report = GEPAResult;
async fn compile<S, M, MT>(
&self,
module: &mut M,
trainset: Vec<Example<S>>,
metric: &MT,
) -> Result<Self::Report>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
self.compile_with_valset::<S, _, _>(module, trainset, None, metric)
.await
}
}
impl GEPA {
pub async fn compile_dyn<M, MT>(
&self,
module: &mut M,
trainset: Vec<crate::data::example::Example>,
metric: &MT,
) -> Result<GEPAResult>
where
M: crate::predictors::DynModule + for<'a> Facet<'a>,
MT: crate::evaluate::DynMetric,
{
use crate::evaluate::{average_score, evaluate_dyn_trainset, input_only};
use crate::optimizer::dyn_compile::ensure_predictors;
use crate::optimizer::pareto::ParetoFrontier;
let names = ensure_predictors(module)?;
let module_name = names[0].clone();
let eval_set = &trainset;
let mut frontier = ParetoFrontier::new();
let mut all_candidates = Vec::new();
let mut evolution_history = Vec::new();
let mut frontier_history = Vec::new();
let mut total_rollouts = 0usize;
let mut total_lm_calls = 0usize;
let base_instruction =
with_named_predictor(module, &module_name, |p| Ok(p.instruction()))?;
Self::set_instruction(module, &module_name, base_instruction.clone())?;
let initial = evaluate_dyn_trainset(module, eval_set, metric).await?;
total_rollouts += initial.len();
total_lm_calls += initial.len();
for outcome in &initial {
if outcome.feedback.is_none() {
return Err(anyhow!(
"GEPA compile_dyn requires MetricOutcome feedback on every example"
));
}
}
let scores: Vec<f32> = initial.iter().map(|o| o.score).collect();
let candidate = GEPACandidate {
id: 0,
instruction: base_instruction,
module_name: module_name.clone(),
example_scores: scores.clone(),
parent_id: None,
generation: 0,
};
frontier.add_candidate(candidate.clone(), &scores);
all_candidates.push(candidate);
let iterations = self.num_iterations.max(self.num_trials.min(self.num_iterations));
for generation in 0..iterations {
if Self::would_exceed_budget(total_rollouts, self.minibatch_size, self.max_rollouts)
|| Self::would_exceed_budget(total_lm_calls, self.minibatch_size, self.max_lm_calls)
{
break;
}
let parent = frontier
.best_by_average()
.cloned()
.context("empty frontier")?;
let feedback_bits: String = initial
.iter()
.filter_map(|o| o.feedback.as_ref().map(|f| f.feedback.clone()))
.take(3)
.collect::<Vec<_>>()
.join(" | ");
let temperature_note = if self.temperature != 1.0 {
format!(" temperature={}", self.temperature)
} else {
String::new()
};
let child_instruction = format!(
"{}\n\nImprove based on feedback:{temperature_note} {}",
parent.instruction, feedback_bits
);
let proposals = crate::optimizer::propose::propose_instructions_with_hint(
&child_instruction,
"improved instruction",
3,
self.prompt_model.as_ref(),
)
.await?;
let child_instruction = proposals
.into_iter()
.nth(1)
.unwrap_or(child_instruction);
Self::set_instruction(module, &module_name, child_instruction.clone())?;
let minibatch_end = eval_set.len().min(self.minibatch_size.max(1));
let outcomes =
evaluate_dyn_trainset(module, &eval_set[..minibatch_end], metric).await?;
total_rollouts += outcomes.len();
total_lm_calls += outcomes.len();
for outcome in &outcomes {
if outcome.feedback.is_none() {
return Err(anyhow!("GEPA compile_dyn requires feedback outcomes"));
}
}
let child_scores: Vec<f32> = outcomes.iter().map(|o| o.score).collect();
let child = GEPACandidate {
id: generation + 1,
instruction: child_instruction,
module_name: module_name.clone(),
example_scores: child_scores.clone(),
parent_id: Some(parent.id),
generation: generation + 1,
};
frontier.add_candidate(child.clone(), &child_scores);
if self.track_stats {
all_candidates.push(child);
evolution_history.push((
generation,
frontier
.best_by_average()
.map(|c| c.average_score())
.unwrap_or(0.0),
));
frontier_history.push(frontier.statistics());
}
}
let best_candidate = frontier
.best_by_average()
.cloned()
.context("no candidates on frontier")?;
Self::set_instruction(
module,
&best_candidate.module_name,
best_candidate.instruction.clone(),
)?;
let best_outputs_valset = if self.track_best_outputs {
let mut outs = Vec::new();
for example in eval_set {
let predicted = module
.call(input_only(example))
.await
.map_err(|err| anyhow!("{err}"))?;
outs.push(crate::BamlValue::Map(
predicted
.into_inner()
.data
.into_iter()
.map(|(k, v)| {
(
k,
crate::BamlValue::try_from(v)
.unwrap_or(crate::BamlValue::Null),
)
})
.collect(),
));
}
Some(outs)
} else {
None
};
let _ = average_score;
Ok(GEPAResult {
best_candidate,
all_candidates: if self.track_stats {
all_candidates
} else {
Vec::new()
},
total_rollouts,
total_lm_calls,
evolution_history,
highest_score_achieved_per_val_task: Vec::new(),
best_outputs_valset,
frontier_history,
})
}
}
#[cfg(test)]
mod tests {
use anyhow::{Result, anyhow};
use super::*;
use crate::evaluate::{MetricOutcome, TypedMetric};
use crate::{CallMetadata, Predict, PredictError, Predicted, Signature};
#[derive(Signature, Clone, Debug)]
struct GepaStateSig {
#[input]
prompt: String,
#[output]
answer: String,
}
#[derive(facet::Facet)]
#[facet(crate = facet)]
struct GepaStateModule {
predictor: Predict<GepaStateSig>,
}
impl Module for GepaStateModule {
type Input = GepaStateSigInput;
type Output = GepaStateSigOutput;
async fn forward(
&self,
input: GepaStateSigInput,
) -> Result<Predicted<GepaStateSigOutput>, PredictError> {
Ok(Predicted::new(
GepaStateSigOutput {
answer: input.prompt,
},
CallMetadata::default(),
))
}
}
struct AlwaysFailMetric;
impl TypedMetric<GepaStateSig, GepaStateModule> for AlwaysFailMetric {
async fn evaluate(
&self,
_example: &Example<GepaStateSig>,
_prediction: &Predicted<GepaStateSigOutput>,
) -> Result<MetricOutcome> {
Err(anyhow!("metric failure"))
}
}
fn eval_set() -> Vec<Example<GepaStateSig>> {
vec![Example::new(
GepaStateSigInput {
prompt: "one".to_string(),
},
GepaStateSigOutput {
answer: "one".to_string(),
},
)]
}
#[tokio::test]
async fn evaluate_candidate_restores_state_when_metric_errors() {
let optimizer = GEPA::builder().num_iterations(1).minibatch_size(1).build();
let mut module = GepaStateModule {
predictor: Predict::<GepaStateSig>::builder()
.instruction("seed-instruction")
.build(),
};
let err = optimizer
.evaluate_candidate::<GepaStateSig, _, _>(
&mut module,
"predictor",
"candidate instruction",
&eval_set(),
&AlwaysFailMetric,
)
.await
.expect_err("candidate evaluation should propagate metric failure");
assert!(err.to_string().contains("metric failure"));
let instruction = with_named_predictor(&mut module, "predictor", |predictor| {
Ok(predictor.instruction())
})
.expect("predictor lookup should succeed");
assert_eq!(instruction, "seed-instruction");
}
}