car_server_core/coder/

ab_loop.rs

//! The self-improvement loop: **A/B → attribute → CAR fix → A/B**, until CAR's
//! coder clears the quality bar or no harness lever remains.
//!
//! This is the loop the `/goal` "make a loop of a/b testing, car fixes, and more
//! a/b testing until car quality is high enough" asks for, expressed as a
//! deterministic driver with injected seams so it is unit-testable and its
//! *termination* is provable:
//!
//! 1. Run the paired A/B suite ([`run_ab_suite`]) — CAR's native coder (pinned
//!    on a pinned backbone) vs. the control arm, on the same contracts.
//! 2. [`attribute_round`] the native losses into harness-addressable
//!    interventions vs. backbone-bound (no-lever) losses.
//! 3. Decide against the [`LoopConfig`] quality bar:
//!    - **met** → stop [`LoopStop::QualityMet`];
//!    - **no lever** (remaining losses are all backbone-bound) → stop
//!      [`LoopStop::NoLeverLeft`] (the honest "a better model, not a harness
//!      change, is what's left" outcome — the ALE finding, enforced);
//!    - otherwise hand the interventions to the injected [`AbFixer`] (live: the
//!      Evolution Agent's `evolution.run` harness diagnose→gate→apply, HITL-gated
//!      on the approval ledger). If nothing lands → stop [`LoopStop::FixStalled`].
//! 4. Re-run the A/B — **the applied change's own regression gate**: a fix only
//!    "counts" if the next round's numbers move, and McNemar says whether the
//!    move is real or noise.
//!
//! The fixer and the arm runner are injected (the `car-builder` / `car-verify::cwm`
//! philosophy), so this core carries no live-inference or daemon dependency and
//! its convergence is exercised with scripted doubles. The live wiring (real
//! runner + real `evolution.run` fixer + the `car coder-ab-loop` CLI) sits on
//! top of this exactly as the live A/B runner sits on `run_ab_suite`.

use async_trait::async_trait;
use car_eventlog::harness_adapt::HarnessIntervention;
use serde::{Deserialize, Serialize};

use super::ab::{
    attribute_round, run_ab_suite, AbArmRunner, AbTask, ArmSpec, PairedStats, RoundAttribution,
};

/// Tuning for the improvement loop.
#[derive(Debug, Clone)]
pub struct LoopConfig {
    /// Native pass-rate over the paired intersection required to clear the bar
    /// (e.g. `0.8`).
    pub quality_bar: f64,
    /// Also require that CAR is not *significantly* behind the external arm —
    /// even at a high absolute pass rate, ship only when the competitor isn't
    /// beating us on a statistically real set of tasks (McNemar).
    pub require_not_behind: bool,
    /// Hard cap on rounds — the loop always terminates.
    pub max_rounds: u32,
    /// `min_occurrences` handed to attribution: how many times a pattern must
    /// recur within a run before it's a lever (one-offs are noise).
    pub min_evidence: usize,
}

impl Default for LoopConfig {
    fn default() -> Self {
        Self {
            quality_bar: 0.8,
            require_not_behind: true,
            max_rounds: 5,
            min_evidence: 2,
        }
    }
}

/// Why the loop stopped.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LoopStop {
    /// Native cleared the bar (and, if required, is not significantly behind).
    QualityMet,
    /// No harness-addressable lever remains — the remaining losses are all
    /// backbone-bound. A harness change can't move the number further; a
    /// stronger model is the lever now.
    NoLeverLeft,
    /// A fix round applied nothing (fixer declined, or every intervention is
    /// pending human approval). The loop can't make progress unattended.
    FixStalled,
    /// Exhausted `max_rounds` still below the bar with levers remaining.
    MaxRounds,
}

impl LoopStop {
    /// Whether the loop ended because CAR's quality reached the target. This is
    /// the *goal-satisfied* terminal; the others are honest non-completions.
    pub fn is_success(self) -> bool {
        matches!(self, LoopStop::QualityMet)
    }
}

/// The result of applying a fix round.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FixResult {
    /// A change actually landed (not merely diagnosed or left pending approval).
    pub applied: bool,
    /// Human-readable note (what changed, or why nothing did).
    pub note: String,
}

/// The injected fix seam. The live impl drives the Evolution Agent
/// (`evolution.run`'s harness diagnose→gate→apply) over the interventions,
/// HITL-gated on the shared approval ledger.
#[async_trait]
pub trait AbFixer: Send + Sync {
    async fn apply(&self, interventions: &[HarnessIntervention]) -> FixResult;
}

/// One round of the loop, recorded for the report and the operator trail.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RoundRecord {
    pub round: u32,
    pub stats: PairedStats,
    pub attribution: RoundAttribution,
    /// A fix was attempted this round (i.e. the round did not terminate the loop).
    pub fix_attempted: bool,
    pub fix_applied: bool,
    pub fix_note: String,
}

/// The full loop trace.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImprovementRun {
    pub rounds: Vec<RoundRecord>,
    pub stop: LoopStop,
    /// The exact arms the loop ran, for provenance.
    pub treatment: ArmSpec,
    pub control: ArmSpec,
}

impl ImprovementRun {
    /// The final round's stats (the loop's exit state), if any round ran.
    pub fn final_stats(&self) -> Option<&PairedStats> {
        self.rounds.last().map(|r| &r.stats)
    }

    /// Native pass-rate improvement from the first round to the last — the
    /// headline "did the loop make CAR better" number.
    pub fn pass_rate_gain(&self) -> f64 {
        match (self.rounds.first(), self.rounds.last()) {
            (Some(first), Some(last)) => {
                last.stats.treatment_pass_rate - first.stats.treatment_pass_rate
            }
            _ => 0.0,
        }
    }
}

/// Whether the current round's stats clear the configured quality bar.
fn quality_met(stats: &PairedStats, cfg: &LoopConfig) -> bool {
    if stats.paired_tasks == 0 {
        return false; // nothing scored — never a pass
    }
    if stats.treatment_pass_rate < cfg.quality_bar {
        return false;
    }
    if cfg.require_not_behind {
        // "Significantly behind" = the discordant pairs favor the control arm
        // AND McNemar calls that difference real.
        let behind = stats.mcnemar_significant_05 && stats.control_only > stats.treatment_only;
        if behind {
            return false;
        }
    }
    true
}

/// Run the self-improvement loop to a terminal [`LoopStop`]. Always terminates
/// (bounded by `max_rounds`; each non-terminal round applies a fix or stops).
///
/// `read_events(path)` resolves the treatment arm's transcript path to its event-log
/// JSONL for attribution (live: read the file; tests: an in-memory map).
pub async fn run_improvement_loop(
    corpus: &[AbTask],
    treatment: &ArmSpec,
    control: &ArmSpec,
    runner: &dyn AbArmRunner,
    read_events: impl Fn(&str) -> Option<String>,
    fixer: &dyn AbFixer,
    cfg: &LoopConfig,
) -> ImprovementRun {
    let mut rounds = Vec::new();
    let mut stop = LoopStop::MaxRounds;

    for round in 0..cfg.max_rounds {
        let report = run_ab_suite(corpus, treatment, control, runner).await;
        let attribution = attribute_round(&report, &read_events, cfg.min_evidence);
        let met = quality_met(&report.stats, cfg);

        // Terminal-this-round decisions record the round without a fix attempt.
        if met {
            rounds.push(RoundRecord {
                round,
                stats: report.stats,
                attribution,
                fix_attempted: false,
                fix_applied: false,
                fix_note: String::new(),
            });
            stop = LoopStop::QualityMet;
            break;
        }
        if !attribution.has_lever() {
            rounds.push(RoundRecord {
                round,
                stats: report.stats,
                attribution,
                fix_attempted: false,
                fix_applied: false,
                fix_note: "no harness-addressable lever — remaining losses are backbone-bound"
                    .into(),
            });
            stop = LoopStop::NoLeverLeft;
            break;
        }

        // A lever exists and the bar isn't met → attempt the fix.
        let fix = fixer.apply(&attribution.interventions).await;
        let applied = fix.applied;
        rounds.push(RoundRecord {
            round,
            stats: report.stats,
            attribution,
            fix_attempted: true,
            fix_applied: applied,
            fix_note: fix.note,
        });
        if !applied {
            stop = LoopStop::FixStalled;
            break;
        }
        // else: loop → the next A/B round is this fix's regression gate.
    }

    ImprovementRun {
        rounds,
        stop,
        treatment: treatment.clone(),
        control: control.clone(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::coder::ab::{ArmEngine, ArmOutcome, ArmSpec};

    fn tspec() -> ArmSpec {
        ArmSpec::native(Some("parslee/reasoning".into()))
    }
    fn cspec() -> ArmSpec {
        ArmSpec::external("codex", Some("parslee/reasoning".into()))
    }
    use crate::coder::contract::OutcomeContract;
    use std::collections::HashMap;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::Arc;

    fn empty_contract() -> OutcomeContract {
        OutcomeContract {
            allow_credentials: false,
            description: "tests pass".into(),
            checks: vec![],
        }
    }

    fn tasks(n: usize) -> Vec<AbTask> {
        (0..n)
            .map(|i| AbTask {
                id: format!("t{i}"),
                intent: format!("task {i}"),
                contract: empty_contract(),
                repo: None,
            })
            .collect()
    }

    /// A runner whose native pass-rate *improves* as fixes land: it passes the
    /// first `native_passes(fixes)` tasks, where `fixes` is the shared counter
    /// the fixer bumps. Every native loss carries a transcript that attributes
    /// to a harness lever, so the loop keeps a reason to fix until the bar. The
    /// external arm is fixed at a middling pass-rate.
    struct ImprovingRunner {
        fixes: Arc<AtomicUsize>,
        treatment_passes: fn(usize) -> usize,
        control_passes: usize,
        n: usize,
        loss_transcript: &'static str,
    }

    #[async_trait]
    impl AbArmRunner for ImprovingRunner {
        async fn run_arm(&self, task: &AbTask, arm: &ArmSpec) -> ArmOutcome {
            let idx: usize = task.id.trim_start_matches('t').parse().unwrap();
            match arm.engine {
                ArmEngine::Native => {
                    let passes =
                        (self.treatment_passes)(self.fixes.load(Ordering::SeqCst)).min(self.n);
                    if idx < passes {
                        ArmOutcome {
                            passed: true,
                            iterations: 1,
                            cost_usd: 0.0,
                            wall_ms: 1,
                            error: None,
                            transcript_path: None,
                            infra_failed: false,
                        }
                    } else {
                        ArmOutcome {
                            passed: false,
                            iterations: 3,
                            cost_usd: 0.0,
                            wall_ms: 1,
                            error: None,
                            transcript_path: Some(self.loss_transcript.into()),
                            infra_failed: false,
                        }
                    }
                }
                ArmEngine::External(_) => ArmOutcome {
                    passed: idx < self.control_passes,
                    iterations: 1,
                    cost_usd: 0.0,
                    wall_ms: 1,
                    error: None,
                    transcript_path: None,
                    infra_failed: false,
                },
            }
        }
    }

    /// A fixer that lands a change every call and bumps the shared counter, so
    /// the *next* A/B round improves (simulating a real harness fix landing).
    struct LandingFixer {
        fixes: Arc<AtomicUsize>,
    }
    #[async_trait]
    impl AbFixer for LandingFixer {
        async fn apply(&self, interventions: &[HarnessIntervention]) -> FixResult {
            self.fixes.fetch_add(1, Ordering::SeqCst);
            FixResult {
                applied: true,
                note: format!("applied {} intervention(s)", interventions.len()),
            }
        }
    }

    /// A fixer that never lands anything (everything pending approval).
    struct StalledFixer;
    #[async_trait]
    impl AbFixer for StalledFixer {
        async fn apply(&self, _i: &[HarnessIntervention]) -> FixResult {
            FixResult {
                applied: false,
                note: "all interventions pending human approval".into(),
            }
        }
    }

    /// A trajectory-regulation transcript so every native loss attributes to a
    /// lever (keeps the loop fixing rather than declaring no-lever).
    fn lever_events() -> HashMap<String, String> {
        let jsonl = (0..2)
            .map(|_| {
                r#"{"kind":"action_failed","action_id":"run_command","data":{"error":"runtime boom"}}"#
                    .to_string()
            })
            .collect::<Vec<_>>()
            .join("\n");
        [("loss.jsonl".to_string(), jsonl)].into_iter().collect()
    }

    #[tokio::test]
    async fn loop_converges_to_quality_met() {
        // 5 tasks. native passes = 2 + 2*fixes → round0: 2/5=0.4 (bar .9 unmet,
        // fix), round1: 4/5=0.8 (unmet, fix), round2: 5/5=1.0 → QualityMet.
        // External at 3/5.
        let fixes = Arc::new(AtomicUsize::new(0));
        let runner = ImprovingRunner {
            fixes: fixes.clone(),
            treatment_passes: |f| 2 + 2 * f,
            control_passes: 3,
            n: 5,
            loss_transcript: "loss.jsonl",
        };
        let fixer = LandingFixer {
            fixes: fixes.clone(),
        };
        let events = lever_events();
        let cfg = LoopConfig {
            quality_bar: 0.9,
            require_not_behind: true,
            max_rounds: 5,
            min_evidence: 2,
        };
        let run = run_improvement_loop(
            &tasks(5),
            &tspec(),
            &cspec(),
            &runner,
            |p| events.get(p).cloned(),
            &fixer,
            &cfg,
        )
        .await;

        assert_eq!(run.stop, LoopStop::QualityMet);
        assert!(run.stop.is_success());
        assert_eq!(run.rounds.len(), 3, "two fix rounds then a met round");
        // The first two rounds attempted+applied a fix; the last did not.
        assert!(run.rounds[0].fix_applied && run.rounds[1].fix_applied);
        assert!(!run.rounds[2].fix_attempted);
        // Headline: native pass-rate climbed from 0.4 to 1.0.
        assert!((run.rounds[0].stats.treatment_pass_rate - 0.4).abs() < 1e-9);
        assert!((run.final_stats().unwrap().treatment_pass_rate - 1.0).abs() < 1e-9);
        assert!((run.pass_rate_gain() - 0.6).abs() < 1e-9);
    }

    #[tokio::test]
    async fn loop_stops_when_no_harness_lever_remains() {
        // native stuck at 2/5, but every loss is a CLEAN loss (no lever events)
        // → backbone-bound → NoLeverLeft on round 0, no fix attempted.
        let fixes = Arc::new(AtomicUsize::new(0));
        let runner = ImprovingRunner {
            fixes: fixes.clone(),
            treatment_passes: |_| 2,
            control_passes: 3,
            n: 5,
            loss_transcript: "clean.jsonl", // not in the events map → empty → no lever
        };
        let fixer = LandingFixer { fixes };
        let events: HashMap<String, String> = HashMap::new(); // no transcripts resolve
        let run = run_improvement_loop(
            &tasks(5),
            &tspec(),
            &cspec(),
            &runner,
            |p| events.get(p).cloned(),
            &fixer,
            &LoopConfig::default(),
        )
        .await;
        assert_eq!(run.stop, LoopStop::NoLeverLeft);
        assert!(!run.stop.is_success());
        assert_eq!(run.rounds.len(), 1);
        assert!(!run.rounds[0].fix_attempted);
    }

    #[tokio::test]
    async fn loop_stops_when_fix_stalls() {
        // Levers exist but the fixer lands nothing → FixStalled on round 0.
        let fixes = Arc::new(AtomicUsize::new(0));
        let runner = ImprovingRunner {
            fixes,
            treatment_passes: |_| 2,
            control_passes: 3,
            n: 5,
            loss_transcript: "loss.jsonl",
        };
        let events = lever_events();
        let run = run_improvement_loop(
            &tasks(5),
            &tspec(),
            &cspec(),
            &runner,
            |p| events.get(p).cloned(),
            &StalledFixer,
            &LoopConfig::default(),
        )
        .await;
        assert_eq!(run.stop, LoopStop::FixStalled);
        assert_eq!(run.rounds.len(), 1);
        assert!(run.rounds[0].fix_attempted && !run.rounds[0].fix_applied);
    }

    #[tokio::test]
    async fn loop_respects_max_rounds() {
        // native improves too slowly to hit the bar within the cap → MaxRounds.
        let fixes = Arc::new(AtomicUsize::new(0));
        let runner = ImprovingRunner {
            fixes: fixes.clone(),
            treatment_passes: |_| 2, // never improves despite applied fixes
            control_passes: 1,
            n: 5,
            loss_transcript: "loss.jsonl",
        };
        let fixer = LandingFixer { fixes };
        let events = lever_events();
        let cfg = LoopConfig {
            quality_bar: 0.8,
            require_not_behind: false,
            max_rounds: 3,
            min_evidence: 2,
        };
        let run = run_improvement_loop(
            &tasks(5),
            &tspec(),
            &cspec(),
            &runner,
            |p| events.get(p).cloned(),
            &fixer,
            &cfg,
        )
        .await;
        assert_eq!(run.stop, LoopStop::MaxRounds);
        assert_eq!(run.rounds.len(), 3);
        assert!(run.rounds.iter().all(|r| r.fix_attempted && r.fix_applied));
    }

    #[tokio::test]
    async fn not_behind_gate_blocks_a_high_but_losing_pass_rate() {
        // Give an explicit stats scenario: native 4/5 (≥ .8) but external wins a
        // significant discordant set → require_not_behind blocks QualityMet.
        // Build it via a runner where native passes t0..t4 except t4, external
        // passes exactly the tasks native fails plus enough discordant wins.
        // Simpler: assert quality_met directly on a crafted PairedStats.
        let behind = PairedStats {
            paired_tasks: 20,
            treatment_passes: 16,
            control_passes: 20,
            treatment_pass_rate: 0.8,
            control_pass_rate: 1.0,
            pass_rate_delta: -0.2,
            both_pass: 16,
            treatment_only: 0,
            control_only: 4, // external wins 4 discordant, native 0
            both_fail: 0,
            mcnemar_chi2: 100.0, // clearly significant
            mcnemar_significant_05: true,
            treatment_infra_failures: 0,
            control_infra_failures: 0,
            treatment_mean_cost_usd: 0.0,
            control_mean_cost_usd: 0.0,
            treatment_cost_per_pass: None,
            control_cost_per_pass: None,
        };
        let cfg = LoopConfig {
            quality_bar: 0.8,
            require_not_behind: true,
            ..LoopConfig::default()
        };
        assert!(
            !quality_met(&behind, &cfg),
            "significantly behind → not met"
        );
        // Same numbers, gate off → met (absolute bar cleared).
        let cfg_off = LoopConfig {
            require_not_behind: false,
            ..cfg.clone()
        };
        assert!(quality_met(&behind, &cfg_off));
    }
}