use car_eventlog::harness_metrics::HarnessMetrics;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum HarnessComponent {
ToolSchema,
RetrievalPolicy,
PlanningConfig,
RetryConfig,
ContextBudget,
WorkflowTopology,
PermissionRule,
Validator,
Prompt,
}
impl HarnessComponent {
pub fn is_safety_affecting(self) -> bool {
matches!(
self,
HarnessComponent::PermissionRule
| HarnessComponent::Validator
| HarnessComponent::Prompt
| HarnessComponent::WorkflowTopology
)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChangeContract {
pub component: HarnessComponent,
pub target_failure: String,
pub predicted_improvement: String,
pub invariants: Vec<String>,
pub falsifying_eval: String,
pub rollback: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct HarnessMutation {
pub id: String,
pub contract: ChangeContract,
pub rationale: String,
#[serde(default)]
pub patch: Option<HarnessConfigPatch>,
}
impl HarnessMutation {
pub fn requires_human_approval(&self) -> bool {
self.contract.component.is_safety_affecting()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct HarnessConfig {
pub max_retries: u32,
pub retry_backoff_ms: u64,
pub planning_max_replans: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_overlay: Option<String>,
}
impl Default for HarnessConfig {
fn default() -> Self {
Self {
max_retries: 3,
retry_backoff_ms: 0,
planning_max_replans: 2,
prompt_overlay: None,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct HarnessConfigPatch {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_retries: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub retry_backoff_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub planning_max_replans: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_overlay: Option<String>,
}
impl HarnessConfigPatch {
pub fn is_empty(&self) -> bool {
self == &HarnessConfigPatch::default()
}
}
#[derive(Debug, Clone)]
pub enum Governance {
Promoted(PromotionDecision),
HumanApproved,
}
impl HarnessConfig {
pub fn apply(
&mut self,
mutation: &HarnessMutation,
governance: Governance,
) -> Result<HarnessConfigPatch, String> {
let patch = mutation
.patch
.as_ref()
.filter(|p| !p.is_empty())
.ok_or("mutation has no concrete config patch to apply")?;
match governance {
Governance::Promoted(decision) => {
if !decision.is_promote() {
return Err(
"mutation was not promoted by the regression gate; refusing to apply"
.into(),
);
}
if mutation.requires_human_approval() {
return Err(
"safety-affecting mutation cannot be auto-applied; requires human approval"
.into(),
);
}
}
Governance::HumanApproved => {}
}
Ok(self.apply_patch(patch))
}
pub fn apply_patch(&mut self, patch: &HarnessConfigPatch) -> HarnessConfigPatch {
let mut inverse = HarnessConfigPatch::default();
if let Some(v) = patch.max_retries {
inverse.max_retries = Some(self.max_retries);
self.max_retries = v;
}
if let Some(v) = patch.retry_backoff_ms {
inverse.retry_backoff_ms = Some(self.retry_backoff_ms);
self.retry_backoff_ms = v;
}
if let Some(v) = patch.planning_max_replans {
inverse.planning_max_replans = Some(self.planning_max_replans);
self.planning_max_replans = v;
}
if let Some(v) = &patch.prompt_overlay {
inverse.prompt_overlay = Some(self.prompt_overlay.clone().unwrap_or_default());
self.prompt_overlay = if v.trim().is_empty() {
None
} else {
Some(v.clone())
};
}
inverse
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "decision", rename_all = "snake_case")]
pub enum PromotionDecision {
Promote { reason: String },
NeedsApproval { reason: String },
Reject { reason: String },
Incomparable { reason: String },
}
impl PromotionDecision {
pub fn is_promote(&self) -> bool {
matches!(self, PromotionDecision::Promote { .. })
}
pub fn is_incomparable(&self) -> bool {
matches!(self, PromotionDecision::Incomparable { .. })
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct EvolutionConfig {
pub high_retry_ratio: f64,
pub low_success_rate: f64,
pub high_tokens_per_success: f64,
pub regression_tolerance: f64,
pub min_target_improvement: f64,
pub min_failures_for_validator: usize,
}
impl Default for EvolutionConfig {
fn default() -> Self {
Self {
high_retry_ratio: 0.5,
low_success_rate: 0.7,
high_tokens_per_success: 5000.0,
regression_tolerance: 0.02,
min_target_improvement: 0.05,
min_failures_for_validator: 2,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct EvolutionAgent {
pub config: EvolutionConfig,
}
impl EvolutionAgent {
pub fn new() -> Self {
Self::default()
}
pub fn with_config(config: EvolutionConfig) -> Self {
Self { config }
}
pub fn diagnose(&self, m: &HarnessMetrics) -> Vec<HarnessMutation> {
let mut out = Vec::new();
let eff = &m.trajectory_efficiency;
let rec = &m.recovery;
if eff.actions_succeeded > 0 {
let retry_ratio = rec.retries as f64 / eff.actions_succeeded as f64;
if retry_ratio > self.config.high_retry_ratio {
out.push(self.mutation(
HarnessComponent::RetryConfig,
"actions repeatedly fail-then-recover, inflating attempt cost",
"tune retry backoff/limit to cut wasted attempts without lowering recovery",
vec!["overall success rate must not drop".into()],
"replay held-out trajectories; promote only if attempts fall and success holds",
"restore the previous retry config",
format!(
"retries/success = {retry_ratio:.2} exceeds {:.2}",
self.config.high_retry_ratio
),
Some(HarnessConfigPatch {
max_retries: Some(2),
retry_backoff_ms: Some(100),
..Default::default()
}),
));
}
}
if let Some(sr) = eff.success_rate {
if sr < self.config.low_success_rate && rec.replan_exhausted > 0 {
out.push(self.mutation(
HarnessComponent::PlanningConfig,
"plans fail and replanning exhausts without recovering",
"adjust decomposition/replan budget to reach a working trajectory",
vec!["token cost must not increase beyond tolerance".into()],
"replay held-out tasks; promote only if success rises without cost regression",
"restore the previous planning config",
format!(
"success_rate {sr:.2} below {:.2} with {} replan exhaustions",
self.config.low_success_rate, rec.replan_exhausted
),
Some(HarnessConfigPatch {
planning_max_replans: Some(4),
..Default::default()
}),
));
}
}
if eff.actions_succeeded > 0 {
let tps = eff.total_tokens as f64 / eff.actions_succeeded as f64;
if tps > self.config.high_tokens_per_success {
out.push(self.mutation(
HarnessComponent::RetrievalPolicy,
"token spend per successful action is high — retrieval or context is bloated",
"tighten retrieval/context budget to cut tokens without lowering success",
vec![
"success rate must not drop".into(),
"verification strength must hold".into(),
],
"replay held-out trajectories; promote only if tokens fall and success holds",
"restore the previous retrieval policy / context budget",
format!(
"tokens/success = {tps:.0} exceeds {:.0}",
self.config.high_tokens_per_success
),
None,
));
}
}
if eff.failed_attempts >= self.config.min_failures_for_validator
&& m.verification_strength.actions_rejected == 0
{
out.push(self.mutation(
HarnessComponent::Validator,
"actions failed at runtime but verification rejected nothing — weak oracle",
"add/strengthen a pre-execution check to catch this failure class earlier",
vec!["must not raise false-rejection rate on previously-passing tasks".into()],
"replay held-out trajectories incl. known-good ones; promote only if it catches the failure without new false rejections",
"remove the added check",
format!("{} failed attempts with 0 verifier rejections", eff.failed_attempts),
None, ));
}
if m.safety.denials > 0 {
out.push(self.mutation(
HarnessComponent::PermissionRule,
"actions are repeatedly denied by the permission gate",
"re-scope the permission rule so legitimate actions are not blocked",
vec!["must not widen access for genuinely high-risk actions".into()],
"human review of the affected actions plus held-out replay",
"restore the previous permission rule",
format!("{} permission denials observed", m.safety.denials),
None, ));
}
out
}
pub fn evaluate(
&self,
mutation: &HarnessMutation,
baseline: &HarnessMetrics,
candidate: &HarnessMetrics,
) -> PromotionDecision {
if let (Some(base_n), Some(cand_n)) = (
baseline.task_pass_denominator,
candidate.task_pass_denominator,
) {
if base_n != cand_n {
return PromotionDecision::Incomparable {
reason: format!(
"task pass rates are over different task sets — baseline graded \
{base_n} task(s), candidate graded {cand_n}. A rate over a smaller \
denominator is not an improvement over a larger one, and a harness \
that can no longer MEASURE the tasks it would have failed produces \
exactly this shape. Compare the two task sets before reading either \
number (baseline unrunnable: {}, candidate unrunnable: {}).",
baseline
.tasks_unrunnable
.map_or("unreported".to_string(), |n| n.to_string()),
candidate
.tasks_unrunnable
.map_or("unreported".to_string(), |n| n.to_string()),
),
};
}
}
if let (Some(base_tpr), Some(cand_tpr)) =
(baseline.task_pass_rate, candidate.task_pass_rate)
{
if cand_tpr + self.config.regression_tolerance < base_tpr {
return PromotionDecision::Reject {
reason: format!(
"TASK pass rate (end-task success, not tool-attempt success) regressed \
{base_tpr:.3} -> {cand_tpr:.3} (beyond tolerance {:.3})",
self.config.regression_tolerance
),
};
}
}
let base_sr = baseline.trajectory_efficiency.success_rate.unwrap_or(0.0);
let cand_sr = candidate.trajectory_efficiency.success_rate.unwrap_or(0.0);
if cand_sr + self.config.regression_tolerance < base_sr {
return PromotionDecision::Reject {
reason: format!(
"success rate regressed {base_sr:.3} -> {cand_sr:.3} (beyond tolerance {:.3})",
self.config.regression_tolerance
),
};
}
let candidate_did_work = candidate.trajectory_efficiency.actions_succeeded > 0;
let improved = match mutation.contract.component {
HarnessComponent::RetrievalPolicy | HarnessComponent::ContextBudget => {
let base = baseline.trajectory_efficiency.total_tokens as f64;
let cand = candidate.trajectory_efficiency.total_tokens as f64;
candidate_did_work
&& base > 0.0
&& (base - cand) / base >= self.config.min_target_improvement
}
HarnessComponent::RetryConfig => {
let base = baseline.recovery.retries as f64;
let cand = candidate.recovery.retries as f64;
candidate_did_work
&& base > 0.0
&& (base - cand) / base >= self.config.min_target_improvement
}
HarnessComponent::PlanningConfig
| HarnessComponent::ToolSchema
| HarnessComponent::WorkflowTopology
| HarnessComponent::Prompt => cand_sr - base_sr >= self.config.min_target_improvement,
HarnessComponent::Validator => {
candidate.verification_strength.actions_rejected
> baseline.verification_strength.actions_rejected
}
HarnessComponent::PermissionRule => candidate.safety.denials < baseline.safety.denials,
};
if !improved {
return PromotionDecision::Reject {
reason:
"target metric did not improve by the required margin on held-out telemetry"
.into(),
};
}
if mutation.requires_human_approval() {
return PromotionDecision::NeedsApproval {
reason: format!(
"mutation passed the regression gate but targets a safety-affecting component ({:?}); human approval required",
mutation.contract.component
),
};
}
PromotionDecision::Promote {
reason: "target improved and no guarded metric regressed on held-out telemetry".into(),
}
}
#[allow(clippy::too_many_arguments)]
fn mutation(
&self,
component: HarnessComponent,
target_failure: &str,
predicted: &str,
invariants: Vec<String>,
falsifying: &str,
rollback: &str,
rationale: String,
patch: Option<HarnessConfigPatch>,
) -> HarnessMutation {
HarnessMutation {
id: format!(
"mut-{}-{}",
component_slug(component),
mutation_digest(component, patch.as_ref(), target_failure)
),
contract: ChangeContract {
component,
target_failure: target_failure.into(),
predicted_improvement: predicted.into(),
invariants,
falsifying_eval: falsifying.into(),
rollback: rollback.into(),
},
rationale,
patch,
}
}
}
fn mutation_digest(
component: HarnessComponent,
patch: Option<&HarnessConfigPatch>,
target_failure: &str,
) -> String {
let content = match patch {
Some(p) if !p.is_empty() => serde_json::to_string(p).unwrap_or_default(),
_ => target_failure.to_string(),
};
short_hash(&format!("{}|{}", component_slug(component), content))
}
pub fn mutation_fingerprint(m: &HarnessMutation) -> String {
format!(
"harness:{}:{}",
component_slug(m.contract.component),
mutation_digest(
m.contract.component,
m.patch.as_ref(),
&m.contract.target_failure
)
)
}
fn component_slug(c: HarnessComponent) -> &'static str {
match c {
HarnessComponent::ToolSchema => "toolschema",
HarnessComponent::RetrievalPolicy => "retrieval",
HarnessComponent::PlanningConfig => "planning",
HarnessComponent::RetryConfig => "retry",
HarnessComponent::ContextBudget => "context",
HarnessComponent::WorkflowTopology => "topology",
HarnessComponent::PermissionRule => "permission",
HarnessComponent::Validator => "validator",
HarnessComponent::Prompt => "prompt",
}
}
fn short_hash(s: &str) -> String {
let mut h: u64 = 1469598103934665603; for b in s.bytes() {
h ^= b as u64;
h = h.wrapping_mul(1099511628211); }
format!("{:08x}", h & 0xffff_ffff)
}
#[cfg(test)]
mod tests {
use super::*;
use car_eventlog::harness_metrics::HarnessMetrics;
fn metrics() -> HarnessMetrics {
HarnessMetrics::default()
}
#[test]
fn diagnoses_high_retry_into_retry_config() {
let mut m = metrics();
m.trajectory_efficiency.actions_succeeded = 4;
m.recovery.retries = 6; let muts = EvolutionAgent::new().diagnose(&m);
assert!(muts
.iter()
.any(|x| x.contract.component == HarnessComponent::RetryConfig));
}
#[test]
fn diagnoses_weak_validator_as_safety_affecting() {
let mut m = metrics();
m.trajectory_efficiency.failed_attempts = 3;
m.verification_strength.actions_rejected = 0;
let muts = EvolutionAgent::new().diagnose(&m);
let v = muts
.iter()
.find(|x| x.contract.component == HarnessComponent::Validator)
.expect("validator mutation");
assert!(v.requires_human_approval());
}
#[test]
fn clean_telemetry_yields_no_mutations() {
let mut m = metrics();
m.trajectory_efficiency.actions_succeeded = 10;
m.trajectory_efficiency.success_rate = Some(1.0);
assert!(EvolutionAgent::new().diagnose(&m).is_empty());
}
#[test]
fn mutation_id_is_deterministic() {
let mut m = metrics();
m.trajectory_efficiency.actions_succeeded = 4;
m.recovery.retries = 6;
let a = EvolutionAgent::new().diagnose(&m);
let b = EvolutionAgent::new().diagnose(&m);
assert_eq!(a[0].id, b[0].id);
}
#[test]
fn fingerprint_binds_patch_content_not_live_measurements() {
let agent = EvolutionAgent::new();
let mut m1 = metrics();
m1.trajectory_efficiency.actions_succeeded = 4;
m1.recovery.retries = 6; let mut m2 = metrics();
m2.trajectory_efficiency.actions_succeeded = 2;
m2.recovery.retries = 10; let a = &agent.diagnose(&m1)[0];
let b = &agent.diagnose(&m2)[0];
assert_ne!(a.rationale, b.rationale, "rationales differ (live floats)");
assert_eq!(a.id, b.id, "id binds component+patch, not prose");
assert_eq!(mutation_fingerprint(a), mutation_fingerprint(b));
let fp = mutation_fingerprint(a);
assert!(fp.starts_with("harness:retry:"), "{fp}");
}
#[test]
fn fingerprint_differs_across_distinct_changes() {
let retry = retry_mutation();
let mut other_patch = retry_mutation();
other_patch.patch = Some(HarnessConfigPatch {
max_retries: Some(5),
..Default::default()
});
let mut other_component = retry_mutation();
other_component.contract.component = HarnessComponent::PlanningConfig;
assert_ne!(
mutation_fingerprint(&retry),
mutation_fingerprint(&other_patch)
);
assert_ne!(
mutation_fingerprint(&retry),
mutation_fingerprint(&other_component)
);
}
fn retry_mutation() -> HarnessMutation {
HarnessMutation {
id: "m".into(),
rationale: "r".into(),
contract: ChangeContract {
component: HarnessComponent::RetryConfig,
target_failure: "f".into(),
predicted_improvement: "p".into(),
invariants: vec![],
falsifying_eval: "e".into(),
rollback: "rb".into(),
},
patch: Some(HarnessConfigPatch {
max_retries: Some(2),
..Default::default()
}),
}
}
#[test]
fn regression_gate_rejects_success_drop() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.success_rate = Some(0.9);
base.recovery.retries = 10;
let mut cand = metrics();
cand.trajectory_efficiency.success_rate = Some(0.5); cand.recovery.retries = 2;
let d = agent.evaluate(&retry_mutation(), &base, &cand);
assert!(matches!(d, PromotionDecision::Reject { .. }), "{d:?}");
}
#[test]
fn regression_gate_promotes_improvement_without_regression() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(0.9);
base.recovery.retries = 10;
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9; cand.trajectory_efficiency.success_rate = Some(0.9); cand.recovery.retries = 4; let d = agent.evaluate(&retry_mutation(), &base, &cand);
assert!(d.is_promote(), "{d:?}");
}
#[test]
fn task_pass_rate_regression_is_rejected() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(1.0);
base.recovery.retries = 10;
base.task_pass_rate = Some(0.80);
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9;
cand.trajectory_efficiency.success_rate = Some(1.0); cand.recovery.retries = 4; cand.task_pass_rate = Some(0.60);
let d = agent.evaluate(&retry_mutation(), &base, &cand);
match &d {
PromotionDecision::Reject { reason } => assert!(
reason.to_lowercase().contains("task pass rate"),
"the reason must name it as the TASK pass rate so it is not \
confused with the attempt-level guard: {reason}"
),
other => panic!("expected Reject, got {other:?}"),
}
let (mut base2, mut cand2) = (base.clone(), cand.clone());
base2.task_pass_rate = None;
cand2.task_pass_rate = None;
assert!(
agent
.evaluate(&retry_mutation(), &base2, &cand2)
.is_promote(),
"without task_pass_rate this candidate promotes — which is the defect"
);
}
#[test]
fn held_task_pass_rate_leaves_the_decision_unchanged() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(0.9);
base.recovery.retries = 10;
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9;
cand.trajectory_efficiency.success_rate = Some(0.9);
cand.recovery.retries = 4;
let without = agent.evaluate(&retry_mutation(), &base, &cand);
assert!(without.is_promote(), "{without:?}");
let (mut base2, mut cand2) = (base.clone(), cand.clone());
base2.task_pass_rate = Some(0.80);
cand2.task_pass_rate = Some(0.80); assert_eq!(agent.evaluate(&retry_mutation(), &base2, &cand2), without);
cand2.task_pass_rate = Some(0.79);
assert_eq!(agent.evaluate(&retry_mutation(), &base2, &cand2), without);
}
#[test]
fn an_unmeasured_task_pass_rate_changes_nothing() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(0.9);
base.recovery.retries = 10;
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9;
cand.trajectory_efficiency.success_rate = Some(0.9);
cand.recovery.retries = 4;
let baseline_decision = agent.evaluate(&retry_mutation(), &base, &cand);
assert!(baseline_decision.is_promote(), "{baseline_decision:?}");
let (mut b1, mut c1) = (base.clone(), cand.clone());
b1.task_pass_rate = None;
c1.task_pass_rate = Some(0.01);
assert_eq!(
agent.evaluate(&retry_mutation(), &b1, &c1),
baseline_decision
);
let (mut b2, mut c2) = (base.clone(), cand.clone());
b2.task_pass_rate = Some(0.95);
c2.task_pass_rate = None;
assert_eq!(
agent.evaluate(&retry_mutation(), &b2, &c2),
baseline_decision
);
let mut bad = cand.clone();
bad.trajectory_efficiency.success_rate = Some(0.5);
let rejected = agent.evaluate(&retry_mutation(), &base, &bad);
assert!(matches!(rejected, PromotionDecision::Reject { .. }));
let mut bad_none = bad.clone();
bad_none.task_pass_rate = None;
let mut base_measured = base.clone();
base_measured.task_pass_rate = Some(0.95);
assert_eq!(
agent.evaluate(&retry_mutation(), &base_measured, &bad_none),
rejected
);
}
#[test]
fn a_pass_rate_over_a_shrunken_task_set_is_refused_not_promoted() {
let agent = EvolutionAgent::new();
let mut mutation = retry_mutation();
mutation.contract.component = HarnessComponent::ToolSchema;
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(0.90);
base.task_pass_rate = Some(0.75); base.task_pass_denominator = Some(12);
base.tasks_unrunnable = Some(0);
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9;
cand.trajectory_efficiency.success_rate = Some(0.99); cand.task_pass_rate = Some(1.0); cand.task_pass_denominator = Some(8); cand.tasks_unrunnable = Some(4);
let d = agent.evaluate(&mutation, &base, &cand);
match &d {
PromotionDecision::Incomparable { reason } => {
assert!(
reason.contains("12"),
"must name both denominators: {reason}"
);
assert!(
reason.contains('8'),
"must name both denominators: {reason}"
);
}
other => {
panic!("a pass rate over a smaller task set must not be compared; got {other:?}")
}
}
assert!(!d.is_promote());
assert!(d.is_incomparable());
let mut honest = cand.clone();
honest.task_pass_denominator = Some(12);
honest.tasks_unrunnable = Some(0);
assert!(
agent.evaluate(&mutation, &base, &honest).is_promote(),
"with comparable denominators this candidate is a genuine win"
);
}
#[test]
fn an_absent_denominator_does_not_refuse_the_comparison() {
let agent = EvolutionAgent::new();
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 9;
base.trajectory_efficiency.success_rate = Some(0.9);
base.recovery.retries = 10;
base.task_pass_rate = Some(0.8);
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 9;
cand.trajectory_efficiency.success_rate = Some(0.9);
cand.recovery.retries = 4;
cand.task_pass_rate = Some(0.8);
let both_absent = agent.evaluate(&retry_mutation(), &base, &cand);
assert!(both_absent.is_promote(), "{both_absent:?}");
let mut b1 = base.clone();
b1.task_pass_denominator = Some(12);
assert_eq!(agent.evaluate(&retry_mutation(), &b1, &cand), both_absent);
let mut c1 = cand.clone();
c1.task_pass_denominator = Some(12);
assert_eq!(agent.evaluate(&retry_mutation(), &base, &c1), both_absent);
assert_eq!(agent.evaluate(&retry_mutation(), &b1, &c1), both_absent);
}
#[test]
fn a_config_patch_cannot_express_a_toolset_change() {
assert!(!HarnessComponent::ToolSchema.is_safety_affecting());
let full = HarnessConfigPatch {
max_retries: Some(1),
retry_backoff_ms: Some(1),
planning_max_replans: Some(1),
prompt_overlay: Some("x".into()),
};
let mut keys: Vec<String> = serde_json::to_value(&full)
.unwrap()
.as_object()
.unwrap()
.keys()
.cloned()
.collect();
keys.sort();
assert_eq!(
keys,
[
"max_retries",
"planning_max_replans",
"prompt_overlay",
"retry_backoff_ms",
],
"a new field here can change what `apply` is able to mutate — if one \
of them is ever a toolset, the bench's unrunnable-task set becomes \
reachable by an auto-promoted mutation and its pass-rate \
denominator becomes attacker-controlled"
);
let attempted: HarnessConfigPatch = serde_json::from_str(
r#"{"tools":["read_file"],"tool_defs":[],"tool_schema":{"name":"x"}}"#,
)
.unwrap();
assert!(
attempted.is_empty(),
"a patch claiming to change the toolset must decode to nothing: {attempted:?}"
);
}
#[test]
fn cost_win_on_a_candidate_that_does_no_work_is_rejected() {
let agent = EvolutionAgent::new();
let mut mutation = retry_mutation();
mutation.contract.component = HarnessComponent::RetrievalPolicy;
let mut base = metrics();
base.trajectory_efficiency.actions_succeeded = 5;
base.trajectory_efficiency.success_rate = Some(0.8);
base.trajectory_efficiency.total_tokens = 10_000;
let mut cand = metrics();
cand.trajectory_efficiency.actions_succeeded = 0; cand.trajectory_efficiency.total_tokens = 100; let d = agent.evaluate(&mutation, &base, &cand);
assert!(matches!(d, PromotionDecision::Reject { .. }), "{d:?}");
}
#[test]
fn single_failure_does_not_propose_validator() {
let mut m = metrics();
m.trajectory_efficiency.failed_attempts = 1;
m.verification_strength.actions_rejected = 0;
let muts = EvolutionAgent::new().diagnose(&m);
assert!(!muts
.iter()
.any(|x| x.contract.component == HarnessComponent::Validator));
}
#[test]
fn safety_affecting_mutation_needs_approval_even_when_passing() {
let agent = EvolutionAgent::new();
let mut mutation = retry_mutation();
mutation.contract.component = HarnessComponent::Validator;
let mut base = metrics();
base.trajectory_efficiency.success_rate = Some(0.8);
base.verification_strength.actions_rejected = 1;
let mut cand = metrics();
cand.trajectory_efficiency.success_rate = Some(0.8); cand.verification_strength.actions_rejected = 4; let d = agent.evaluate(&mutation, &base, &cand);
assert!(
matches!(d, PromotionDecision::NeedsApproval { .. }),
"{d:?}"
);
}
#[test]
fn apply_promoted_mutation_changes_config_and_returns_rollback() {
let mut cfg = HarnessConfig::default();
assert_eq!(cfg.max_retries, 3);
let m = retry_mutation(); let inverse = cfg
.apply(
&m,
Governance::Promoted(PromotionDecision::Promote {
reason: "ok".into(),
}),
)
.expect("apply");
assert_eq!(cfg.max_retries, 2);
cfg.apply_patch(&inverse);
assert_eq!(cfg.max_retries, 3);
}
#[test]
fn apply_refuses_unpromoted_mutation() {
let mut cfg = HarnessConfig::default();
let m = retry_mutation();
let r = cfg.apply(
&m,
Governance::Promoted(PromotionDecision::Reject {
reason: "no".into(),
}),
);
assert!(r.is_err());
assert_eq!(cfg.max_retries, 3); }
#[test]
fn apply_refuses_safety_mutation_under_auto_promotion() {
let mut cfg = HarnessConfig::default();
let mut m = retry_mutation();
m.contract.component = HarnessComponent::Validator;
m.patch = Some(HarnessConfigPatch {
max_retries: Some(1),
..Default::default()
});
let r = cfg.apply(
&m,
Governance::Promoted(PromotionDecision::Promote { reason: "x".into() }),
);
assert!(r.is_err(), "safety mutation must not auto-apply");
assert_eq!(cfg.max_retries, 3);
}
#[test]
fn human_approved_safety_mutation_applies() {
let mut cfg = HarnessConfig::default();
let mut m = retry_mutation();
m.contract.component = HarnessComponent::WorkflowTopology;
m.patch = Some(HarnessConfigPatch {
planning_max_replans: Some(5),
..Default::default()
});
cfg.apply(&m, Governance::HumanApproved)
.expect("apply approved");
assert_eq!(cfg.planning_max_replans, 5);
}
#[test]
fn apply_refuses_mutation_without_patch() {
let mut cfg = HarnessConfig::default();
let mut m = retry_mutation();
m.patch = None;
let r = cfg.apply(&m, Governance::HumanApproved);
assert!(r.is_err());
}
#[test]
fn diagnosed_tunable_mutation_carries_a_patch() {
let mut mm = metrics();
mm.trajectory_efficiency.actions_succeeded = 4;
mm.recovery.retries = 6;
let muts = EvolutionAgent::new().diagnose(&mm);
let retry = muts
.iter()
.find(|x| x.contract.component == HarnessComponent::RetryConfig)
.unwrap();
assert!(retry.patch.is_some());
}
}