car_server_core/coder/

ab_learnings.rs

//! Turn a coder A/B round into ranked, evidence-backed **durable code-fix
//! proposals** — the "learn what durable fix to make to CAR" step.
//!
//! This is deliberately NOT the runtime-knob path (`ab_fixer`). The valuable
//! output of dogfooding is a *durable source change* — the kind landed by hand
//! this cycle (the coder prompt's set()-order guidance that lifted a corpus
//! 55%→100%, the contract-derivation repair, the MLX device-lock fix). The
//! synthesizer folds the A/B's losses + attribution into concrete proposals that
//! say **which CAR component to change, roughly where, what change, and on what
//! evidence** — ranked so the highest-leverage fix is acted on first. A human
//! (or a coding agent) implements and merges; the next A/B round is the gate.
//!
//! Deterministic by design (like the rest of the `ab` machinery): it maps
//! diagnosed failure mechanisms onto CAR's real coder components via a curated
//! table. It does not invent the *specific* fix — that needs a human reading the
//! evidence (or an injected LLM refiner, a documented extension) — it narrows
//! **where to look and what class of change**, which is the expensive part.

use car_eventlog::harness_adapt::{HarnessIntervention, InterventionLayer};
use serde::{Deserialize, Serialize};

use super::ab::{AbReport, RoundAttribution};

/// Whether the pattern is one the harness can fix (a diagnosable interaction
/// failure) or one that looks like the model reasoning wrong (a clean run, wrong
/// answer). Both can warrant a durable code fix — the biggest win this cycle
/// (the pitfall-guidance prompt change) targeted a *backbone-bound* cluster —
/// but the confidence and the kind of change differ.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProposalKind {
    HarnessAddressable,
    BackboneBound,
}

/// How strongly the evidence points at the proposed change.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Confidence {
    /// A recurring, diagnosable interaction pattern → a specific component.
    Strong,
    /// A real signal, but the specific fix needs a human to read the failures.
    Tentative,
}

/// One durable code-fix proposal targeting CAR's own source.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DurableFixProposal {
    /// Short imperative title.
    pub title: String,
    /// The failure pattern this addresses (human-readable).
    pub pattern: String,
    /// Which CAR component to change.
    pub target_component: String,
    /// A file/area hint into CAR's source (best-effort — verify before editing).
    pub target_hint: String,
    /// The proposed durable change (a class of edit, not a patch).
    pub proposed_change: String,
    /// Task ids that evidence this pattern.
    pub evidence_tasks: Vec<String>,
    /// Recurrence weight (summed evidence / loss count).
    pub evidence_count: usize,
    pub kind: ProposalKind,
    pub confidence: Confidence,
    /// Higher acts first. Derived from evidence + confidence.
    pub priority: u32,
}

/// Map a diagnosed harness intervention onto a CAR coder component + a proposed
/// durable change. Grounded in CAR's actual coder source (`coder/native_loop.rs`
/// system prompt, `coder/contract.rs`, `car-inference` tool rendering).
fn proposal_from_intervention(iv: &HarnessIntervention) -> DurableFixProposal {
    let trig = iv.trigger.to_lowercase();
    let (component, hint, change) = match iv.layer {
        InterventionLayer::EnvironmentContract => (
            "coder tool contract / system prompt",
            "coder/native_loop.rs (system_prompt) or the tool descriptions",
            format!(
                "The model repeatedly proposes a call the runtime rejects for '{}'. Durably clarify \
                 the tool's description/constraints (or the prompt's guidance about it) so the model \
                 stops proposing the disallowed/ill-formed call up front.",
                iv.target
            ),
        ),
        InterventionLayer::ActionRealization => (
            "tool schema / prompt tool-call format",
            "car-inference render_chat_prompt/parse_tool_calls or coder/native_loop.rs prompt",
            format!(
                "The model emits structurally-invalid calls to '{}'. Durably fix the tool-call \
                 rendering/parsing or tighten the prompt's format guidance so valid calls are the \
                 default, rather than relying on retries.",
                iv.target
            ),
        ),
        InterventionLayer::TrajectoryRegulation => {
            if trig.contains("retried") || trig.contains("failed") {
                (
                    "coder repair discipline",
                    "coder/native_loop.rs (system_prompt 'read the actual error before retrying' + failure_feedback)",
                    format!(
                        "'{}' fails/retries repeatedly without recovering. Durably strengthen the \
                         read-the-error-then-fix-the-named-cause discipline (prompt + the failure \
                         feedback the repair loop injects), so the coder converges instead of \
                         thrashing.",
                        iv.target
                    ),
                )
            } else {
                (
                    "coder repair loop / iteration handling",
                    "coder/native_loop.rs (max_iterations, failure_feedback) or the replan path",
                    format!(
                        "Repair for '{}' gives up (replanning exhausted). Durably improve how the \
                         coder carries context across repair rounds (or reconsiders the approach) so \
                         it doesn't burn the budget re-deriving the same dead end.",
                        iv.target
                    ),
                )
            }
        }
        InterventionLayer::ProceduralSkill => (
            "skill distillation",
            "car-memgine skill distillation / coder skill_memory",
            format!(
                "A reusable procedure keeps being re-derived for '{}'. Durably distill it as a coder \
                 skill so future sessions recall it.",
                iv.target
            ),
        ),
    };
    let evidence_count = iv.evidence_count;
    DurableFixProposal {
        title: format!("Fix {component} for recurring failure on '{}'", iv.target),
        pattern: iv.trigger.clone(),
        target_component: component.to_string(),
        target_hint: hint.to_string(),
        proposed_change: change,
        evidence_tasks: Vec::new(), // filled by the caller (attribution loses the per-task link)
        evidence_count,
        kind: ProposalKind::HarnessAddressable,
        confidence: Confidence::Strong,
        // Harness-addressable + strong: base 100 + evidence.
        priority: 100 + evidence_count as u32,
    }
}

/// The backbone-bound cluster: native lost these tasks running *clean* (no
/// diagnosable interaction pattern) — i.e. it reasoned wrong. Often still fixable
/// with a durable prompt-discipline change (the set()-order guidance that lifted
/// 55%→100% targeted exactly this), but the *specific* guidance needs a human to
/// read the failing cases and name the common error class.
fn backbone_bound_proposal(tasks: &[String]) -> DurableFixProposal {
    DurableFixProposal {
        title: format!(
            "Inspect {} clean-but-wrong loss(es) for a common error class",
            tasks.len()
        ),
        pattern: "coder ran cleanly but produced a wrong answer (no diagnosable interaction failure)"
            .to_string(),
        target_component: "coder system prompt (task discipline)".to_string(),
        target_hint: "coder/native_loop.rs (system_prompt)".to_string(),
        proposed_change:
            "Read these failing cases and name the common mistake (e.g. losing collection order, \
             off-by-one, wrong edge case). If they share one, add targeted pitfall guidance to the \
             coder's system prompt — the same move that took a corpus 55%→100% by warning against \
             set() dropping order. If they don't share one, this is a backbone limit, not a harness \
             fix — record it and move on."
                .to_string(),
        evidence_tasks: tasks.to_vec(),
        evidence_count: tasks.len(),
        kind: ProposalKind::BackboneBound,
        confidence: Confidence::Tentative,
        // Below harness-addressable, above nothing: base 40 + count.
        priority: 40 + tasks.len() as u32,
    }
}

/// Synthesize ranked durable-fix proposals from one A/B round. `report` supplies
/// the loss set; `attribution` supplies the diagnosed patterns. Proposals are
/// returned highest-priority first.
pub fn synthesize_proposals(
    _report: &AbReport,
    attribution: &RoundAttribution,
) -> Vec<DurableFixProposal> {
    let mut proposals: Vec<DurableFixProposal> = attribution
        .interventions
        .iter()
        .map(proposal_from_intervention)
        .collect();

    if !attribution.backbone_bound_losses.is_empty() {
        proposals.push(backbone_bound_proposal(&attribution.backbone_bound_losses));
    }

    // Highest priority first; ties broken by evidence then title for stability.
    proposals.sort_by(|a, b| {
        b.priority
            .cmp(&a.priority)
            .then(b.evidence_count.cmp(&a.evidence_count))
            .then(a.title.cmp(&b.title))
    });
    proposals
}

/// Render proposals as a human-facing report (for `car coder-ab` output / a
/// checked-in learnings file).
pub fn render_proposals(proposals: &[DurableFixProposal]) -> String {
    if proposals.is_empty() {
        return "No durable-fix proposals — no attributable coder losses this round.".to_string();
    }
    let mut out = format!("Durable code-fix proposals ({}):\n", proposals.len());
    for (i, p) in proposals.iter().enumerate() {
        out.push_str(&format!(
            "\n{}. [{:?}, {:?}, priority {}] {}\n   pattern: {}\n   target: {} — {}\n   change: {}\n",
            i + 1,
            p.kind,
            p.confidence,
            p.priority,
            p.title,
            p.pattern,
            p.target_component,
            p.target_hint,
            p.proposed_change,
        ));
        if !p.evidence_tasks.is_empty() {
            out.push_str(&format!(
                "   evidence: {} task(s): {}\n",
                p.evidence_count,
                p.evidence_tasks.join(", ")
            ));
        } else {
            out.push_str(&format!("   evidence: recurrence {}\n", p.evidence_count));
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use car_eventlog::harness_adapt::HarnessIntervention;

    fn iv(layer: InterventionLayer, target: &str, trigger: &str, ev: usize) -> HarnessIntervention {
        HarnessIntervention {
            layer,
            target: target.into(),
            trigger: trigger.into(),
            intervention: "x".into(),
            evidence_count: ev,
        }
    }

    fn attribution(
        interventions: Vec<HarnessIntervention>,
        backbone: Vec<&str>,
    ) -> RoundAttribution {
        RoundAttribution {
            harness_addressable_losses: vec![],
            backbone_bound_losses: backbone.into_iter().map(String::from).collect(),
            interventions,
        }
    }

    // A minimal AbReport (unused by the synthesizer beyond presence).
    fn empty_report() -> AbReport {
        AbReport::from_cells(
            vec![],
            crate::coder::ab::ArmSpec::native(Some("parslee/reasoning".into())),
            crate::coder::ab::ArmSpec::external("codex", Some("parslee/reasoning".into())),
        )
    }

    #[test]
    fn trajectory_retry_maps_to_repair_discipline() {
        let attr = attribution(
            vec![iv(
                InterventionLayer::TrajectoryRegulation,
                "run_command",
                "action 'run_command' retried 4×",
                4,
            )],
            vec![],
        );
        let ps = synthesize_proposals(&empty_report(), &attr);
        assert_eq!(ps.len(), 1);
        assert_eq!(ps[0].kind, ProposalKind::HarnessAddressable);
        assert!(ps[0].target_hint.contains("native_loop.rs"));
        assert!(ps[0].proposed_change.contains("read-the-error"));
    }

    #[test]
    fn realization_maps_to_tool_rendering() {
        let attr = attribution(
            vec![iv(
                InterventionLayer::ActionRealization,
                "edit_file",
                "no tool 'edit_file'",
                3,
            )],
            vec![],
        );
        let ps = synthesize_proposals(&empty_report(), &attr);
        assert!(
            ps[0].target_hint.contains("render_chat_prompt")
                || ps[0].target_hint.contains("parse_tool_calls")
        );
    }

    #[test]
    fn backbone_bound_cluster_targets_the_prompt_with_the_55_to_100_precedent() {
        let attr = attribution(vec![], vec!["dedup", "sort_stable"]);
        let ps = synthesize_proposals(&empty_report(), &attr);
        assert_eq!(ps.len(), 1);
        let p = &ps[0];
        assert_eq!(p.kind, ProposalKind::BackboneBound);
        assert_eq!(p.confidence, Confidence::Tentative);
        assert_eq!(p.evidence_tasks, vec!["dedup", "sort_stable"]);
        assert!(p.proposed_change.contains("55%→100%"));
    }

    #[test]
    fn harness_addressable_outranks_backbone_bound() {
        // A weak harness intervention (evidence 2) still outranks a big
        // backbone cluster (5) — a diagnosable fix is more actionable.
        let attr = attribution(
            vec![iv(
                InterventionLayer::TrajectoryRegulation,
                "t",
                "action 't' failed 2×",
                2,
            )],
            vec!["a", "b", "c", "d", "e"],
        );
        let ps = synthesize_proposals(&empty_report(), &attr);
        assert_eq!(ps.len(), 2);
        assert_eq!(ps[0].kind, ProposalKind::HarnessAddressable);
        assert_eq!(ps[1].kind, ProposalKind::BackboneBound);
    }

    #[test]
    fn empty_attribution_yields_no_proposals() {
        let ps = synthesize_proposals(&empty_report(), &attribution(vec![], vec![]));
        assert!(ps.is_empty());
        assert!(render_proposals(&ps).contains("No durable-fix proposals"));
    }

    #[test]
    fn render_is_human_readable() {
        let attr = attribution(
            vec![iv(
                InterventionLayer::EnvironmentContract,
                "shell",
                "action 'shell' rejected 3×",
                3,
            )],
            vec!["x"],
        );
        let text = render_proposals(&synthesize_proposals(&empty_report(), &attr));
        assert!(text.contains("Durable code-fix proposals (2)"));
        assert!(text.contains("target:"));
        assert!(text.contains("change:"));
    }
}