use anyhow::{Result, anyhow};
use bon::Builder;
use crate::evaluate::{TypedMetric, average_score};
use crate::optimizer::{
Optimizer, evaluate_module_with_metric, predictor_names, with_named_predictor,
};
use crate::predictors::Example;
use crate::{Facet, Module, Signature};
#[derive(Builder)]
pub struct COPRO {
#[builder(default = 10)]
pub breadth: usize,
#[builder(default = 3)]
pub depth: usize,
#[builder(default = 1.4)]
pub init_temperature: f32,
#[builder(default = false)]
pub track_stats: bool,
pub prompt_model: Option<crate::LM>,
}
impl COPRO {
fn current_instruction<M>(module: &mut M, predictor_name: &str) -> Result<String>
where
M: for<'a> Facet<'a>,
{
with_named_predictor(module, predictor_name, |predictor| {
Ok(predictor.instruction())
})
}
fn set_instruction<M>(module: &mut M, predictor_name: &str, instruction: String) -> Result<()>
where
M: for<'a> Facet<'a>,
{
with_named_predictor(module, predictor_name, |predictor| {
predictor.set_instruction(instruction);
Ok(())
})
}
async fn score_candidate<S, M, MT>(
&self,
module: &mut M,
predictor_name: &str,
candidate_instruction: &str,
trainset: &[Example<S>],
metric: &MT,
) -> Result<f32>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
let original_state = with_named_predictor(module, predictor_name, |predictor| {
Ok(predictor.dump_state())
})?;
Self::set_instruction(module, predictor_name, candidate_instruction.to_string())?;
let evaluation = evaluate_module_with_metric(&*module, trainset, metric).await;
match evaluation {
Ok(outcomes) => {
with_named_predictor(module, predictor_name, |predictor| {
predictor.load_state(original_state.clone())
})?;
Ok(average_score(&outcomes))
}
Err(eval_err) => {
if let Err(restore_err) =
with_named_predictor(module, predictor_name, |predictor| {
predictor.load_state(original_state)
})
{
return Err(anyhow!(
"candidate evaluation failed: {eval_err}; failed to restore predictor state: {restore_err}"
));
}
Err(eval_err)
}
}
}
async fn propose_candidates<M>(
&self,
module: &mut M,
predictor_name: &str,
base_instruction: &str,
) -> Result<Vec<String>>
where
M: for<'a> Facet<'a>,
{
let output_hint = with_named_predictor(module, predictor_name, |predictor| {
Ok(predictor
.schema()
.output_fields()
.iter()
.map(|f| f.lm_name.to_string())
.collect::<Vec<_>>()
.join(", "))
})?;
crate::optimizer::propose::propose_instructions_with_hint(
base_instruction,
&output_hint,
self.breadth,
self.prompt_model.as_ref(),
)
.await
}
}
impl Optimizer for COPRO {
type Report = ();
async fn compile<S, M, MT>(
&self,
module: &mut M,
trainset: Vec<Example<S>>,
metric: &MT,
) -> Result<Self::Report>
where
S: Signature,
S::Input: Clone,
M: Module<Input = S::Input> + for<'a> Facet<'a>,
MT: TypedMetric<S, M>,
{
if self.breadth <= 1 {
return Err(anyhow!("breadth must be greater than 1"));
}
let predictor_names = predictor_names(module)?;
if predictor_names.is_empty() {
return Err(anyhow!("no optimizable predictors found"));
}
for _depth in 0..self.depth {
for predictor_name in &predictor_names {
let base_instruction = Self::current_instruction(module, predictor_name)?;
let candidates = self
.propose_candidates(module, predictor_name, &base_instruction)
.await?;
let mut best_instruction = base_instruction.clone();
let mut best_score = f32::MIN;
for candidate in candidates {
let score = self
.score_candidate::<S, _, _>(
module,
predictor_name,
&candidate,
&trainset,
metric,
)
.await?;
if score > best_score {
best_score = score;
best_instruction = candidate;
}
}
Self::set_instruction(module, predictor_name, best_instruction)?;
}
}
Ok(())
}
}
impl COPRO {
pub async fn compile_dyn<M, MT>(
&self,
module: &mut M,
trainset: Vec<crate::data::example::Example>,
metric: &MT,
) -> Result<()>
where
M: crate::predictors::DynModule + for<'a> Facet<'a>,
MT: crate::evaluate::DynMetric,
{
use crate::optimizer::dyn_compile::{ensure_predictors, score_instruction_dyn};
if self.breadth <= 1 {
return Err(anyhow!("breadth must be greater than 1"));
}
let predictor_names = ensure_predictors(module)?;
for _depth in 0..self.depth {
for predictor_name in &predictor_names {
let base_instruction = Self::current_instruction(module, predictor_name)?;
let candidates = self
.propose_candidates(module, predictor_name, &base_instruction)
.await?;
let mut best_instruction = base_instruction;
let mut best_score = f32::MIN;
for candidate in candidates {
let score = score_instruction_dyn(
module,
predictor_name,
&candidate,
&trainset,
metric,
)
.await?;
if score > best_score {
best_score = score;
best_instruction = candidate;
}
}
Self::set_instruction(module, predictor_name, best_instruction)?;
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use anyhow::{Result, anyhow};
use super::*;
use crate::evaluate::{MetricOutcome, TypedMetric};
use crate::{CallMetadata, Predict, PredictError, Predicted, Signature};
#[derive(Signature, Clone, Debug)]
struct CoproStateSig {
#[input]
prompt: String,
#[output]
answer: String,
}
#[derive(facet::Facet)]
#[facet(crate = facet)]
struct CoproStateModule {
predictor: Predict<CoproStateSig>,
}
impl Module for CoproStateModule {
type Input = CoproStateSigInput;
type Output = CoproStateSigOutput;
async fn forward(
&self,
input: CoproStateSigInput,
) -> Result<Predicted<CoproStateSigOutput>, PredictError> {
Ok(Predicted::new(
CoproStateSigOutput {
answer: input.prompt,
},
CallMetadata::default(),
))
}
}
struct AlwaysFailMetric;
impl TypedMetric<CoproStateSig, CoproStateModule> for AlwaysFailMetric {
async fn evaluate(
&self,
_example: &Example<CoproStateSig>,
_prediction: &Predicted<CoproStateSigOutput>,
) -> Result<MetricOutcome> {
Err(anyhow!("metric failure"))
}
}
fn trainset() -> Vec<Example<CoproStateSig>> {
vec![Example::new(
CoproStateSigInput {
prompt: "one".to_string(),
},
CoproStateSigOutput {
answer: "one".to_string(),
},
)]
}
#[tokio::test]
async fn score_candidate_restores_state_when_metric_errors() {
let optimizer = COPRO::builder().breadth(2).depth(1).build();
let mut module = CoproStateModule {
predictor: Predict::<CoproStateSig>::builder()
.instruction("seed-instruction")
.build(),
};
let err = optimizer
.score_candidate::<CoproStateSig, _, _>(
&mut module,
"predictor",
"candidate instruction",
&trainset(),
&AlwaysFailMetric,
)
.await
.expect_err("candidate scoring should propagate metric failure");
assert!(err.to_string().contains("metric failure"));
let instruction = with_named_predictor(&mut module, "predictor", |predictor| {
Ok(predictor.instruction())
})
.expect("predictor lookup should succeed");
assert_eq!(instruction, "seed-instruction");
}
}