use anyhow::{Result, anyhow};
use bon::Builder;
use crate::evaluate::{TypedMetric, average_score};
use crate::optimizer::{
Optimizer, evaluate_module_with_metric, predictor_names, with_named_predictor,
};
use crate::predictors::Example;
use crate::{BamlType, BamlValue, Facet, Module, Signature, SignatureSchema};
#[derive(Clone, Debug)]
pub struct Trace<S: Signature> {
pub input: S::Input,
pub outputs: BamlValue,
pub score: Option<f32>,
}
impl<S: Signature> Trace<S> {
pub fn new(input: S::Input, outputs: BamlValue, score: Option<f32>) -> Self {
Self {
input,
outputs,
score,
}
}
pub fn format_for_prompt(&self) -> String {
let mut result = String::new();
result.push_str("Input:\n");
result.push_str(&format!(" {}\n", self.input.to_baml_value()));
result.push_str("Output:\n");
result.push_str(&format!(" {}\n", self.outputs));
if let Some(score) = self.score {
result.push_str(&format!("Score: {:.3}\n", score));
}
result
}
}
#[derive(Clone, Debug)]
pub struct PromptCandidate {
pub instruction: String,
pub score: f32,
}
impl PromptCandidate {
pub fn new(instruction: String) -> Self {
Self {
instruction,
score: 0.0,
}
}
pub fn with_score(mut self, score: f32) -> Self {
self.score = score;
self
}
}
pub struct PromptingTips {
pub tips: Vec<String>,
}
impl PromptingTips {
pub fn default_tips() -> Self {
Self {
tips: vec![
"Use clear and specific language".to_string(),
"Provide context about the task domain".to_string(),
"Specify the desired output format".to_string(),
"Use chain-of-thought reasoning for complex tasks".to_string(),
"Include few-shot examples when helpful".to_string(),
"Break down complex instructions into steps".to_string(),
"Use role-playing (e.g., 'You are an expert...') when appropriate".to_string(),
"Specify constraints and edge cases".to_string(),
"Request explanations or reasoning when needed".to_string(),
"Use structured output formats (JSON, lists, etc.) when applicable".to_string(),
"Consider the model's strengths and limitations".to_string(),
"Be explicit about what to avoid or exclude".to_string(),
"Use positive framing (what to do vs. what not to do)".to_string(),
"Provide examples of both correct and incorrect outputs when useful".to_string(),
"Use delimiters or markers to separate different sections".to_string(),
],
}
}
pub fn format_for_prompt(&self) -> String {
self.tips
.iter()
.enumerate()
.map(|(i, tip)| format!("{}. {}", i + 1, tip))
.collect::<Vec<_>>()
.join("\n")
}
}
#[derive(Builder)]
pub struct MIPROv2 {
#[builder(default = 10)]
pub num_candidates: usize,
#[builder(default = 20)]
pub num_trials: usize,
#[builder(default = 25)]
pub minibatch_size: usize,
}
impl MIPROv2 {
async fn generate_traces<S, M, MT>(
&self,
module: &M,
examples: &[Example<S>],
metric: &MT,
) -> Result<Vec<Trace<S>>>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input>,
MT: TypedMetric<S, M>,
{
let mut traces = Vec::with_capacity(examples.len());
for example in examples {
let input = example.input.clone();
let predicted = module.call(input).await.map_err(|err| anyhow!("{err}"))?;
let outcome = metric.evaluate(example, &predicted).await?;
let (output, _) = predicted.into_parts();
traces.push(Trace::new(
example.input.clone(),
output.to_baml_value(),
Some(outcome.score),
));
}
Ok(traces)
}
pub fn select_best_traces<'a, S: Signature>(
&self,
traces: &'a [Trace<S>],
num_select: usize,
) -> Vec<&'a Trace<S>> {
let mut scored_traces: Vec<_> = traces.iter().filter(|t| t.score.is_some()).collect();
scored_traces.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
scored_traces.into_iter().take(num_select).collect()
}
fn generate_candidate_instructions<S: Signature>(
&self,
program_description: &str,
traces: &[Trace<S>],
num_candidates: usize,
) -> Vec<String> {
let tips = PromptingTips::default_tips();
let score_hint = traces.iter().filter_map(|t| t.score).fold(0.0f32, f32::max);
(0..num_candidates)
.map(|idx| {
let tip = &tips.tips[idx % tips.tips.len()];
format!(
"{program_description}\n\nOptimization candidate {}:\n- {}\n- Target score >= {:.3}",
idx + 1,
tip,
score_hint
)
})
.collect()
}
async fn propose_candidate_instructions(
&self,
program_description: &str,
output_hint: &str,
num_candidates: usize,
) -> Result<Vec<String>> {
crate::optimizer::propose::propose_instructions_with_hint(
program_description,
output_hint,
num_candidates,
None,
)
.await
}
pub fn create_prompt_candidates(&self, instructions: Vec<String>) -> Vec<PromptCandidate> {
instructions.into_iter().map(PromptCandidate::new).collect()
}
async fn evaluate_candidate<S, M, MT>(
&self,
module: &mut M,
candidate: &PromptCandidate,
eval_examples: &[Example<S>],
predictor_name: &str,
metric: &MT,
) -> Result<f32>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let original_state = with_named_predictor(module, predictor_name, |predictor| {
Ok(predictor.dump_state())
})?;
with_named_predictor(module, predictor_name, |predictor| {
predictor.set_instruction(candidate.instruction.clone());
Ok(())
})?;
let minibatch_end = eval_examples.len().min(self.minibatch_size);
let minibatch = &eval_examples[..minibatch_end];
let evaluation = evaluate_module_with_metric(&*module, minibatch, metric).await;
match evaluation {
Ok(outcomes) => {
with_named_predictor(module, predictor_name, |predictor| {
predictor.load_state(original_state.clone())
})?;
Ok(average_score(&outcomes))
}
Err(eval_err) => {
if let Err(restore_err) =
with_named_predictor(module, predictor_name, |predictor| {
predictor.load_state(original_state)
})
{
return Err(anyhow!(
"candidate evaluation failed: {eval_err}; failed to restore predictor state: {restore_err}"
));
}
Err(eval_err)
}
}
}
async fn evaluate_and_select_best<S, M, MT>(
&self,
module: &mut M,
candidates: Vec<PromptCandidate>,
eval_examples: &[Example<S>],
predictor_name: &str,
metric: &MT,
) -> Result<PromptCandidate>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let mut evaluated = Vec::new();
let num_trials = self.num_trials.max(1);
for candidate in candidates.into_iter().take(num_trials) {
let score = self
.evaluate_candidate::<S, _, _>(
module,
&candidate,
eval_examples,
predictor_name,
metric,
)
.await?;
evaluated.push(candidate.with_score(score));
}
evaluated
.into_iter()
.max_by(|a, b| {
a.score
.partial_cmp(&b.score)
.unwrap_or(std::cmp::Ordering::Equal)
})
.ok_or_else(|| anyhow!("no candidates to evaluate"))
}
pub fn format_schema_fields(&self, signature: &SignatureSchema) -> String {
let mut result = String::new();
result.push_str("Input Fields:\n");
for field in signature.input_fields() {
let desc = if field.docs.is_empty() {
"No description"
} else {
field.docs.as_str()
};
result.push_str(&format!(" - {}: {}\n", field.lm_name, desc));
}
result.push_str("\nOutput Fields:\n");
for field in signature.output_fields() {
let desc = if field.docs.is_empty() {
"No description"
} else {
field.docs.as_str()
};
result.push_str(&format!(" - {}: {}\n", field.lm_name, desc));
}
result
}
}
impl Optimizer for MIPROv2 {
type Report = ();
async fn compile<S, M, MT>(
&self,
module: &mut M,
trainset: Vec<Example<S>>,
metric: &MT,
) -> Result<Self::Report>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let predictor_names = predictor_names(module)?;
if predictor_names.is_empty() {
return Err(anyhow!("no optimizable predictors found"));
}
for predictor_name in predictor_names {
let signature_desc = {
with_named_predictor(module, &predictor_name, |predictor| {
Ok(self.format_schema_fields(predictor.schema()))
})?
};
let traces = self
.generate_traces::<S, _, _>(module, &trainset, metric)
.await?;
let output_hint = with_named_predictor(module, &predictor_name, |predictor| {
Ok(predictor
.schema()
.output_fields()
.iter()
.map(|f| f.lm_name.to_string())
.collect::<Vec<_>>()
.join(", "))
})?;
let mut instructions = self
.propose_candidate_instructions(&signature_desc, &output_hint, self.num_candidates)
.await?;
if instructions.len() < self.num_candidates {
let extras = self.generate_candidate_instructions(
&signature_desc,
&traces,
self.num_candidates,
);
for extra in extras {
if instructions.len() >= self.num_candidates {
break;
}
if !instructions.iter().any(|i| i == &extra) {
instructions.push(extra);
}
}
}
let candidates = self.create_prompt_candidates(instructions);
let best_candidate = self
.evaluate_and_select_best::<S, _, _>(
module,
candidates,
&trainset,
&predictor_name,
metric,
)
.await?;
with_named_predictor(module, &predictor_name, |predictor| {
predictor.set_instruction(best_candidate.instruction.clone());
Ok(())
})?;
}
Ok(())
}
}
#[derive(Clone, Debug)]
pub struct DynTrace {
pub input: crate::data::example::Example,
pub output: crate::data::example::Example,
pub score: Option<f32>,
}
impl MIPROv2 {
pub async fn compile_dyn<M, MT>(
&self,
module: &mut M,
trainset: Vec<crate::data::example::Example>,
metric: &MT,
) -> Result<()>
where
M: crate::predictors::DynModule + for<'a> Facet<'a>,
MT: crate::evaluate::DynMetric,
{
use crate::evaluate::input_only;
use crate::optimizer::dyn_compile::{
assign_demos_best_effort, ensure_predictors, score_instruction_dyn,
};
let predictor_names = ensure_predictors(module)?;
for predictor_name in predictor_names {
let signature_desc = with_named_predictor(module, &predictor_name, |predictor| {
Ok(self.format_schema_fields(predictor.schema()))
})?;
let mut traces = Vec::new();
let mut successful_demos = Vec::new();
for example in &trainset {
let predicted = module
.call(input_only(example))
.await
.map_err(|err| anyhow!("{err}"))?;
let outcome = metric.evaluate(example, &predicted).await?;
traces.push(DynTrace {
input: input_only(example),
output: predicted.into_inner(),
score: Some(outcome.score),
});
if outcome.score >= 1.0 || outcome.score >= 0.99 {
successful_demos.push(example.clone());
}
}
let output_hint = with_named_predictor(module, &predictor_name, |predictor| {
Ok(predictor
.schema()
.output_fields()
.iter()
.map(|f| f.lm_name.to_string())
.collect::<Vec<_>>()
.join(", "))
})?;
let mut instructions = self
.propose_candidate_instructions(&signature_desc, &output_hint, self.num_candidates)
.await?;
if instructions.len() < self.num_candidates {
let tips = PromptingTips::default_tips();
let score_hint = traces.iter().filter_map(|t| t.score).fold(0.0f32, f32::max);
for (idx, tip) in tips.tips.iter().enumerate() {
if instructions.len() >= self.num_candidates {
break;
}
instructions.push(format!(
"{signature_desc}\n\nOptimization candidate {}:\n- {tip}\n- Target score >= {score_hint:.3}",
idx + 1
));
}
}
let mut best_instruction = instructions
.first()
.cloned()
.unwrap_or_else(|| "Answer carefully.".into());
let mut best_score = f32::MIN;
for candidate in instructions.into_iter().take(self.num_trials.max(1)) {
let score =
score_instruction_dyn(module, &predictor_name, &candidate, &trainset, metric)
.await?;
if score > best_score {
best_score = score;
best_instruction = candidate;
}
}
with_named_predictor(module, &predictor_name, |predictor| {
predictor.set_instruction(best_instruction);
if !successful_demos.is_empty() {
let demos = successful_demos
.iter()
.take(self.minibatch_size.max(1))
.cloned()
.collect::<Vec<_>>();
let _ = assign_demos_best_effort(predictor, demos);
}
Ok(())
})?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use anyhow::{Result, anyhow};
use super::*;
use crate::evaluate::{MetricOutcome, TypedMetric};
use crate::{CallMetadata, Predict, PredictError, Predicted, Signature};
#[derive(Signature, Clone, Debug)]
struct MiproStateSig {
#[input]
prompt: String,
#[output]
answer: String,
}
#[derive(facet::Facet)]
#[facet(crate = facet)]
struct MiproStateModule {
predictor: Predict<MiproStateSig>,
}
impl Module for MiproStateModule {
type Input = MiproStateSigInput;
type Output = MiproStateSigOutput;
async fn forward(
&self,
input: MiproStateSigInput,
) -> Result<Predicted<MiproStateSigOutput>, PredictError> {
Ok(Predicted::new(
MiproStateSigOutput {
answer: input.prompt,
},
CallMetadata::default(),
))
}
}
struct AlwaysFailMetric;
impl TypedMetric<MiproStateSig, MiproStateModule> for AlwaysFailMetric {
async fn evaluate(
&self,
_example: &Example<MiproStateSig>,
_prediction: &Predicted<MiproStateSigOutput>,
) -> Result<MetricOutcome> {
Err(anyhow!("metric failure"))
}
}
fn trainset() -> Vec<Example<MiproStateSig>> {
vec![Example::new(
MiproStateSigInput {
prompt: "one".to_string(),
},
MiproStateSigOutput {
answer: "one".to_string(),
},
)]
}
#[tokio::test]
async fn evaluate_candidate_restores_state_when_metric_errors() {
let optimizer = MIPROv2::builder()
.num_candidates(2)
.num_trials(1)
.minibatch_size(1)
.build();
let mut module = MiproStateModule {
predictor: Predict::<MiproStateSig>::builder()
.instruction("seed-instruction")
.build(),
};
let candidate = PromptCandidate::new("candidate instruction".to_string());
let err = optimizer
.evaluate_candidate::<MiproStateSig, _, _>(
&mut module,
&candidate,
&trainset(),
"predictor",
&AlwaysFailMetric,
)
.await
.expect_err("candidate evaluation should propagate metric failure");
assert!(err.to_string().contains("metric failure"));
let instruction = with_named_predictor(&mut module, "predictor", |predictor| {
Ok(predictor.instruction())
})
.expect("predictor lookup should succeed");
assert_eq!(instruction, "seed-instruction");
}
}