car-eventlog 0.38.0

//! Runtime harness adaptation — diagnose recurring interaction failures into
//! reusable, typed interventions.
//!
//! Applies *Adapting the Interface, Not the Model: Runtime Harness Adaptation
//! for Deterministic LLM Agents* (arXiv 2605.22166, "Life-Harness") to CAR — see
//! `docs/proposals/runtime-harness-adaptation.md`. The paper's thesis is CAR's:
//! many failures in deterministic, rule-governed domains come from the
//! *model–environment interface*, not the weights, and are best fixed by
//! evolving the **runtime harness** rather than retraining. Life-Harness
//! diagnoses recurring interaction failures from trajectories and converts them
//! into reusable interventions across four lifecycle layers.
//!
//! This is the *diagnosis* half: a pure pass over a `car-eventlog` JSONL tail
//! that finds **recurring** failure patterns (one-offs are noise) and proposes a
//! typed [`HarnessIntervention`] for each — complementing
//! `car-memgine::harness_evolution` (which gates and applies mutations to a
//! `HarnessConfig`) and reusing the same telemetry `harness_metrics` reads.

use crate::{Event, EventKind};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// Life-Harness's four lifecycle layers an intervention can target.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum InterventionLayer {
    /// Tool descriptions / interface constraints calibrated *before* interaction
    /// — the model keeps proposing something the contract forbids.
    EnvironmentContract,
    /// Turning intended actions into valid concrete calls — malformed params,
    /// missing tool, schema/type mismatch.
    ActionRealization,
    /// Recovering from degenerate trajectories — repeated runtime failures,
    /// retry thrash, replanning that exhausts.
    TrajectoryRegulation,
    /// Reusable procedures distilled from interaction (deferred to CAR's existing
    /// skill distillation; reserved here for completeness).
    ProceduralSkill,
}

/// One reusable intervention proposed from a recurring failure pattern.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HarnessIntervention {
    pub layer: InterventionLayer,
    /// What the intervention is about — an action id, or `proposal:<id>`.
    pub target: String,
    /// The recurring pattern observed (human/agent-actionable).
    pub trigger: String,
    /// The proposed reusable fix.
    pub intervention: String,
    /// How many times the pattern recurred (≥ `min_occurrences`).
    pub evidence_count: usize,
}

/// The diagnosis result.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AdaptationReport {
    pub interventions: Vec<HarnessIntervention>,
    /// JSONL lines that failed to parse (when diagnosing from a journal tail).
    pub parse_errors: usize,
}

/// Best-effort error text from an event's data — joins the string values of the
/// keys the executor uses to carry failure detail (`error`, `message`,
/// `reason`, `trajectory_persist_error`).
fn error_text(ev: &Event) -> String {
    const KEYS: [&str; 4] = ["error", "message", "reason", "trajectory_persist_error"];
    let mut parts = Vec::new();
    for k in KEYS {
        if let Some(s) = ev.data.get(k).and_then(|v| v.as_str()) {
            parts.push(s.to_string());
        }
    }
    parts.join("; ")
}

/// Does an error string look like an *action-realization* problem (the model
/// produced a structurally-invalid call) rather than a runtime failure?
fn looks_like_realization(err: &str) -> bool {
    let e = err.to_lowercase();
    [
        "no tool",
        "not registered",
        "param",
        "schema",
        "required",
        "type mismatch",
        "invalid argument",
        "unknown tool",
    ]
    .iter()
    .any(|kw| e.contains(kw))
}

/// Diagnose recurring interaction failures into typed interventions. Only
/// patterns recurring at least `min_occurrences` times are emitted (one-offs are
/// noise — the paper converts *recurring* failures into interventions).
pub fn diagnose(events: &[Event], min_occurrences: usize) -> AdaptationReport {
    let min = min_occurrences.max(1);

    // Per-action tallies.
    let mut rejected: HashMap<String, usize> = HashMap::new();
    let mut retried: HashMap<String, usize> = HashMap::new();
    // action_id -> (count, sample_error)
    let mut failed: HashMap<String, (usize, String)> = HashMap::new();
    // proposal_id -> (count, sample_reason)
    let mut replan_exhausted: HashMap<String, (usize, String)> = HashMap::new();
    // GoalEvaluated{met, grounded=false} — the runtime evaluated a completion
    // CLAIM as met but could NOT ground it against tool receipts. This is the
    // false-success signature: the agent reports "done" without a deterministic
    // check confirming it. The verdict is already computed and logged; it was
    // previously dropped here (only failure *kinds* were diagnosed).
    let mut ungrounded_completions: usize = 0;
    // TurnCompleted terminals that signal a problem on the ungrounded default
    // path (which emits no GoalEvaluated): a truncated answer accepted as done
    // (false-success), or a max_turns/stalled terminal (never-finished). A clean
    // empty_tool_calls finish is not counted. Keyed by the problem so each recurs
    // independently.
    let mut turn_problems: HashMap<&'static str, usize> = HashMap::new();

    for ev in events {
        match ev.kind {
            EventKind::ActionRejected => {
                if let Some(id) = &ev.action_id {
                    *rejected.entry(id.clone()).or_insert(0) += 1;
                }
            }
            EventKind::ActionRetrying => {
                if let Some(id) = &ev.action_id {
                    *retried.entry(id.clone()).or_insert(0) += 1;
                }
            }
            EventKind::ActionFailed => {
                if let Some(id) = &ev.action_id {
                    let e = failed.entry(id.clone()).or_insert((0, String::new()));
                    e.0 += 1;
                    if e.1.is_empty() {
                        e.1 = error_text(ev);
                    }
                }
            }
            EventKind::ReplanExhausted => {
                let pid = ev
                    .proposal_id
                    .clone()
                    .unwrap_or_else(|| "unknown".to_string());
                let e = replan_exhausted.entry(pid).or_insert((0, String::new()));
                e.0 += 1;
                if e.1.is_empty() {
                    e.1 = error_text(ev);
                }
            }
            EventKind::GoalEvaluated => {
                // Read the deterministic verdict the goal evaluator already
                // recorded. A met-but-ungrounded completion means the agent
                // claimed done and no receipt-grounded check confirmed it.
                let met = ev
                    .data
                    .get("met")
                    .and_then(|v| v.as_bool())
                    .unwrap_or(false);
                let grounded = ev
                    .data
                    .get("grounded")
                    .and_then(|v| v.as_bool())
                    .unwrap_or(true);
                if met && !grounded {
                    ungrounded_completions += 1;
                }
            }
            EventKind::TurnCompleted => {
                let decision = ev
                    .data
                    .get("decision")
                    .and_then(|v| v.as_str())
                    .unwrap_or("");
                let truncated = ev
                    .data
                    .get("was_truncated")
                    .and_then(|v| v.as_bool())
                    .unwrap_or(false);
                if truncated {
                    *turn_problems.entry("truncated_completion").or_insert(0) += 1;
                } else if decision == "max_turns" {
                    *turn_problems.entry("max_turns").or_insert(0) += 1;
                } else if decision == "stalled" {
                    *turn_problems.entry("stalled").or_insert(0) += 1;
                }
                // a clean empty_tool_calls finish (not truncated) is normal
            }
            _ => {}
        }
    }

    let mut interventions = Vec::new();

    // Pre-execution rejections → environment-contract calibration.
    for (id, count) in &rejected {
        if *count >= min {
            interventions.push(HarnessIntervention {
                layer: InterventionLayer::EnvironmentContract,
                target: id.clone(),
                trigger: format!("action '{id}' rejected before execution {count}×"),
                intervention: format!(
                    "calibrate the tool contract/permission for '{id}' so the model stops proposing a disallowed or ill-formed call (clarify the description/constraints up front)"
                ),
                evidence_count: *count,
            });
        }
    }

    // Repeated runtime failures → realization (if the error is structural) or
    // trajectory regulation (otherwise).
    for (id, (count, err)) in &failed {
        if *count < min {
            continue;
        }
        let (layer, intervention) = if looks_like_realization(err) {
            (
                InterventionLayer::ActionRealization,
                format!(
                    "add an action-realization fixup for '{id}' (normalize params / tool name) — recurring structural error: {}",
                    if err.is_empty() { "<none recorded>" } else { err }
                ),
            )
        } else {
            (
                InterventionLayer::TrajectoryRegulation,
                format!(
                    "add a recovery/circuit-breaker for '{id}' — it fails at runtime repeatedly{}",
                    if err.is_empty() {
                        String::new()
                    } else {
                        format!(": {err}")
                    }
                ),
            )
        };
        interventions.push(HarnessIntervention {
            layer,
            target: id.clone(),
            trigger: format!("action '{id}' failed {count}×"),
            intervention,
            evidence_count: *count,
        });
    }

    // Retry thrash → trajectory regulation.
    for (id, count) in &retried {
        if *count >= min {
            interventions.push(HarnessIntervention {
                layer: InterventionLayer::TrajectoryRegulation,
                target: id.clone(),
                trigger: format!("action '{id}' retried {count}×"),
                intervention: format!(
                    "cap retries for '{id}' and route to replan/alternative instead of thrashing"
                ),
                evidence_count: *count,
            });
        }
    }

    // Replanning exhausted → trajectory regulation at the proposal level.
    for (pid, (count, reason)) in &replan_exhausted {
        if *count >= min {
            interventions.push(HarnessIntervention {
                layer: InterventionLayer::TrajectoryRegulation,
                target: format!("proposal:{pid}"),
                trigger: format!("replanning exhausted {count}× for proposal '{pid}'"),
                intervention: format!(
                    "revisit the goal/contract or seed a procedural skill — replanning repeatedly gives up{}",
                    if reason.is_empty() { String::new() } else { format!(" ({reason})") }
                ),
                evidence_count: *count,
            });
        }
    }

    // Recurring ungrounded completion claims → trajectory regulation. The
    // completion verdict is ground truth the runtime already owns; a repeated
    // met-but-ungrounded outcome is the false-success pattern (the agent
    // reports done without a check passing) that every consumer previously
    // ignored. Turning it into an intervention is the highest-leverage,
    // lowest-cost self-improvement seam.
    if ungrounded_completions >= min {
        interventions.push(HarnessIntervention {
            layer: InterventionLayer::TrajectoryRegulation,
            target: "completion-claim".to_string(),
            trigger: format!(
                "agent reported completion but the runtime could not ground it {ungrounded_completions}×"
            ),
            intervention:
                "recurring ungrounded completion — gate termination on tool-receipt/exit-code ground truth (not the model's summary), or require a deterministic `--until` check so a false-completion cannot end the run"
                    .to_string(),
            evidence_count: ungrounded_completions,
        });
    }

    // Recurring default-path completion problems (the loop that emits no
    // GoalEvaluated). Each turns the newly-captured TurnCompleted decision into a
    // concrete, reusable fix.
    for (kind, count) in &turn_problems {
        if *count < min {
            continue;
        }
        let intervention = match *kind {
            "truncated_completion" => "recurring truncated completion accepted as done — detect stop_reason==length/thinking_truncated and continue the turn (prompt the model to finish / raise per-turn max_tokens) instead of treating a cut-off empty answer as success",
            "max_turns" => "recurring turn-cap exhaustion — set a no_progress_turns bound so a non-progressing run is cut off early, and surface 'capped' distinctly from a real finish",
            _ => "recurring stall on repeated non-progressing actions — strengthen the no-progress detector (args-hash / state digest, not an exact tool-call signature)",
        };
        interventions.push(HarnessIntervention {
            layer: InterventionLayer::TrajectoryRegulation,
            target: format!("turn-completion:{kind}"),
            trigger: format!("assistant loop terminated as '{kind}' {count}×"),
            intervention: intervention.to_string(),
            evidence_count: *count,
        });
    }

    // Deterministic order so the report is stable across runs.
    interventions.sort_by(|a, b| {
        b.evidence_count
            .cmp(&a.evidence_count)
            .then(a.target.cmp(&b.target))
    });

    AdaptationReport {
        interventions,
        parse_errors: 0,
    }
}

/// Diagnose from a JSONL string of events (one per line). Unparseable lines are
/// counted in `parse_errors`, matching `harness_metrics::compute_from_jsonl`.
pub fn diagnose_from_jsonl(jsonl: &str, min_occurrences: usize) -> AdaptationReport {
    let mut parse_errors = 0usize;
    let events: Vec<Event> = jsonl
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|l| match serde_json::from_str::<Event>(l) {
            Ok(ev) => Some(ev),
            Err(_) => {
                parse_errors += 1;
                None
            }
        })
        .collect();
    let mut report = diagnose(&events, min_occurrences);
    report.parse_errors = parse_errors;
    report
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::EventLog;

    fn log_with(events: &[(EventKind, Option<&str>, Option<&str>, Vec<(&str, &str)>)]) -> EventLog {
        let mut log = EventLog::new();
        for (kind, action, proposal, data) in events {
            let map: HashMap<String, serde_json::Value> = data
                .iter()
                .map(|(k, v)| (k.to_string(), serde_json::Value::from(*v)))
                .collect();
            log.append(kind.clone(), *action, *proposal, map);
        }
        log
    }

    /// Append a GoalEvaluated event carrying the deterministic `met`/`grounded`
    /// verdict as real JSON bools (the string-only `log_with` can't).
    fn append_goal_evaluated(log: &mut EventLog, met: bool, grounded: bool) {
        let mut data: HashMap<String, serde_json::Value> = HashMap::new();
        data.insert("met".to_string(), serde_json::Value::Bool(met));
        data.insert("grounded".to_string(), serde_json::Value::Bool(grounded));
        log.append(EventKind::GoalEvaluated, None, None, data);
    }

    #[test]
    fn recurring_ungrounded_completion_is_trajectory_regulation() {
        let mut log = EventLog::new();
        // Two met-but-ungrounded completions = the false-success pattern.
        append_goal_evaluated(&mut log, true, false);
        append_goal_evaluated(&mut log, true, false);
        // A properly grounded completion must NOT be flagged.
        append_goal_evaluated(&mut log, true, true);
        // An in-progress (not-yet-met) evaluation must NOT be flagged.
        append_goal_evaluated(&mut log, false, false);

        let r = diagnose(log.events(), 2);
        let flagged: Vec<_> = r
            .interventions
            .iter()
            .filter(|i| i.target == "completion-claim")
            .collect();
        assert_eq!(
            flagged.len(),
            1,
            "expected exactly one completion-claim intervention"
        );
        assert_eq!(flagged[0].layer, InterventionLayer::TrajectoryRegulation);
        assert_eq!(flagged[0].evidence_count, 2);
        assert!(flagged[0].trigger.contains("could not ground"));
    }

    #[test]
    fn single_ungrounded_completion_is_below_threshold() {
        let mut log = EventLog::new();
        append_goal_evaluated(&mut log, true, false);
        let r = diagnose(log.events(), 2);
        assert!(
            r.interventions
                .iter()
                .all(|i| i.target != "completion-claim"),
            "a one-off ungrounded completion is noise, not an intervention"
        );
    }

    fn append_turn_completed(log: &mut EventLog, decision: &str, was_truncated: bool) {
        let mut data: HashMap<String, serde_json::Value> = HashMap::new();
        data.insert("decision".to_string(), serde_json::Value::from(decision));
        data.insert(
            "was_truncated".to_string(),
            serde_json::Value::Bool(was_truncated),
        );
        log.append(EventKind::TurnCompleted, None, None, data);
    }

    #[test]
    fn recurring_truncated_completion_is_flagged_but_clean_finishes_are_not() {
        let mut log = EventLog::new();
        // Two truncated completions = a mineable false-success pattern.
        append_turn_completed(&mut log, "empty_tool_calls", true);
        append_turn_completed(&mut log, "empty_tool_calls", true);
        // Clean finishes must never be flagged, however many there are.
        for _ in 0..5 {
            append_turn_completed(&mut log, "empty_tool_calls", false);
        }
        let r = diagnose(log.events(), 2);
        let flagged: Vec<_> = r
            .interventions
            .iter()
            .filter(|i| i.target.starts_with("turn-completion:"))
            .collect();
        assert_eq!(flagged.len(), 1);
        assert_eq!(flagged[0].target, "turn-completion:truncated_completion");
        assert_eq!(flagged[0].evidence_count, 2);
    }

    #[test]
    fn recurring_max_turns_is_flagged() {
        let mut log = EventLog::new();
        append_turn_completed(&mut log, "max_turns", false);
        append_turn_completed(&mut log, "max_turns", false);
        let r = diagnose(log.events(), 2);
        assert!(r
            .interventions
            .iter()
            .any(|i| i.target == "turn-completion:max_turns"));
    }

    #[test]
    fn recurring_rejection_is_environment_contract() {
        let log = log_with(&[
            (EventKind::ActionRejected, Some("a1"), Some("p"), vec![]),
            (EventKind::ActionRejected, Some("a1"), Some("p"), vec![]),
        ]);
        let r = diagnose(log.events(), 2);
        assert_eq!(r.interventions.len(), 1);
        assert_eq!(
            r.interventions[0].layer,
            InterventionLayer::EnvironmentContract
        );
        assert_eq!(r.interventions[0].evidence_count, 2);
        assert_eq!(r.interventions[0].target, "a1");
    }

    #[test]
    fn one_off_is_not_emitted() {
        let log = log_with(&[(EventKind::ActionRejected, Some("a1"), Some("p"), vec![])]);
        let r = diagnose(log.events(), 2);
        assert!(
            r.interventions.is_empty(),
            "a single failure is noise, not a pattern"
        );
    }

    #[test]
    fn structural_failure_is_action_realization() {
        let log = log_with(&[
            (
                EventKind::ActionFailed,
                Some("a2"),
                Some("p"),
                vec![("error", "missing required param 'path'")],
            ),
            (
                EventKind::ActionFailed,
                Some("a2"),
                Some("p"),
                vec![("error", "missing required param 'path'")],
            ),
        ]);
        let r = diagnose(log.events(), 2);
        assert_eq!(
            r.interventions[0].layer,
            InterventionLayer::ActionRealization
        );
    }

    #[test]
    fn runtime_failure_is_trajectory_regulation() {
        let log = log_with(&[
            (
                EventKind::ActionFailed,
                Some("a3"),
                Some("p"),
                vec![("error", "connection timed out")],
            ),
            (
                EventKind::ActionFailed,
                Some("a3"),
                Some("p"),
                vec![("error", "connection timed out")],
            ),
        ]);
        let r = diagnose(log.events(), 2);
        assert_eq!(
            r.interventions[0].layer,
            InterventionLayer::TrajectoryRegulation
        );
    }

    #[test]
    fn replan_exhausted_targets_proposal() {
        let log = log_with(&[
            (
                EventKind::ReplanExhausted,
                None,
                Some("p9"),
                vec![("reason", "callback_error")],
            ),
            (
                EventKind::ReplanExhausted,
                None,
                Some("p9"),
                vec![("reason", "callback_error")],
            ),
        ]);
        let r = diagnose(log.events(), 2);
        assert_eq!(
            r.interventions[0].layer,
            InterventionLayer::TrajectoryRegulation
        );
        assert_eq!(r.interventions[0].target, "proposal:p9");
    }

    #[test]
    fn results_sorted_by_evidence_desc() {
        let log = log_with(&[
            (EventKind::ActionRejected, Some("low"), Some("p"), vec![]),
            (EventKind::ActionRejected, Some("low"), Some("p"), vec![]),
            (EventKind::ActionRetrying, Some("high"), Some("p"), vec![]),
            (EventKind::ActionRetrying, Some("high"), Some("p"), vec![]),
            (EventKind::ActionRetrying, Some("high"), Some("p"), vec![]),
        ]);
        let r = diagnose(log.events(), 2);
        assert_eq!(r.interventions[0].target, "high"); // 3 > 2
        assert_eq!(r.interventions[0].evidence_count, 3);
    }

    #[test]
    fn jsonl_counts_parse_errors() {
        let jsonl = [
            "not json",
            r#"{"kind":"action_rejected","action_id":"a1","data":{},"timestamp":"2026-06-28T00:00:00Z"}"#,
            r#"{"kind":"action_rejected","action_id":"a1","data":{},"timestamp":"2026-06-28T00:00:01Z"}"#,
        ]
        .join("\n");
        let r = diagnose_from_jsonl(&jsonl, 2);
        assert_eq!(r.parse_errors, 1);
        assert_eq!(r.interventions.len(), 1);
    }
}