car-eventlog 0.26.0

//! Harness-level evaluation metrics.
//!
//! Survey "Code as Agent Harness" §5.2.1: end-task success conflates the
//! base model, the harness, the tools, and the environment. To evaluate the
//! *operational substrate itself*, success accuracy must be complemented by
//! measurements of "execution reliability, feedback quality, context
//! sustainability, safety, coordination, and reproducibility." This module
//! derives those dimensions from the deep telemetry the event log now
//! records (token/cost/latency metrics, branch decisions, rejected
//! alternatives, permission decisions — see `car_eventlog`).
//!
//! These are descriptive, not normative: they characterize a trajectory so
//! harness variants can be compared, and an Evolution Agent (§3.5.2) can
//! attribute cost/failure to specific harness components.

use crate::{Event, EventKind, MetricsTotals};
use serde::{Deserialize, Serialize};

/// The six §5.2.1 dimensions, computed from a trajectory's event stream.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct HarnessMetrics {
    pub trajectory_efficiency: TrajectoryEfficiency,
    pub verification_strength: VerificationStrength,
    pub recovery: Recovery,
    pub state_consistency: StateConsistency,
    pub safety: Safety,
    pub replayability: Replayability,
    /// JSONL lines that failed to parse when computing from a journal tail
    /// (0 when computed from in-memory events). Nonzero means the metrics
    /// were computed over a partial event set — do not compare across
    /// harnesses without accounting for it (neo review).
    pub parse_errors: usize,
}

/// (i) Trajectory efficiency — how much work was spent reaching the outcome.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct TrajectoryEfficiency {
    /// Successful actions plus *failed attempts* (a retried action
    /// contributes one success and N failed attempts).
    pub attempts_total: usize,
    pub actions_succeeded: usize,
    /// Failed *attempts* (`ActionFailed` events), which include retries —
    /// see `success_rate`.
    pub failed_attempts: usize,
    /// Sum of input + output tokens across inference events.
    pub total_tokens: u64,
    pub total_cost_usd: f64,
    /// Summed wall-clock across metered events (ms).
    pub wall_clock_ms: f64,
    /// **Attempt-level** success: succeeded / (succeeded + failed_attempts).
    /// Counts every `ActionFailed` including retries, so a
    /// retried-then-succeeded action lowers this — a harness with
    /// aggressive retry (good recovery) scores lower here than one that
    /// gives up. Read it alongside `recovery.retries`. `None` when no
    /// attempts ran.
    pub success_rate: Option<f64>,
}

/// (ii) Verification strength — how much the harness checked before
/// accepting. False-acceptance rate needs an external oracle, so it is left
/// `None` here; what the log *can* show is how often verification rejected
/// or policy blocked an action (the verifier doing work).
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct VerificationStrength {
    pub actions_validated: usize,
    pub actions_rejected: usize,
    pub policy_violations: usize,
    /// Rejections / (validated + rejected) — the share caught **by the
    /// validator** before execution (excludes policy blocks, counted
    /// separately). `None` when nothing was validated.
    pub rejection_rate: Option<f64>,
}

/// (iii) Recovery ability — can the harness diagnose and repair failures?
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct Recovery {
    pub replan_attempts: usize,
    pub replan_rejected: usize,
    pub replan_exhausted: usize,
    /// Action-level retries (`ActionRetrying` events) — the within-action
    /// recovery the attempt-level `success_rate` counts against.
    pub retries: usize,
    /// Branch decisions + rejected alternatives — the size of the search
    /// the harness explored when things went wrong.
    pub branch_decisions: usize,
    pub alternatives_rejected: usize,
}

/// (iv) State consistency — how much the shared state churned and how often
/// the harness had to roll back.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct StateConsistency {
    pub state_changes: usize,
    pub snapshots: usize,
    pub rollbacks: usize,
}

/// (v) Safety compliance — permission-gate activity (from the §5.2.5 tier
/// gate). High escalation/denial counts mean the harness governed real
/// risk rather than running unsupervised.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct Safety {
    pub permission_decisions: usize,
    pub escalations: usize,
    pub denials: usize,
    pub approvals_recorded: usize,
}

/// (vi) Replayability — what the log carries for reconstruction and audit.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct Replayability {
    pub total_events: usize,
    /// True when the log carries engine-state delta records (`StateChanged`
    /// / `StateSnapshot`) — i.e. the trajectory's *state effects* can be
    /// replayed. This is distinct from auditability: a pure side-effecting
    /// tool (sends an email, writes an untracked file) succeeds with no
    /// state delta, so this is `false` even though the action is fully
    /// logged. Read it as "captured state effects", not "is auditable"
    /// (neo review — the append-only log is auditable regardless).
    pub state_effects_captured: bool,
}

/// Compute harness-level metrics from a trajectory's events.
pub fn compute_harness_metrics(events: &[Event]) -> HarnessMetrics {
    let mut m = HarnessMetrics::default();
    let totals: MetricsTotals = crate::metrics_totals_of(events);

    let mut succeeded = 0usize;
    let mut failed = 0usize;
    let mut any_state_record = false;

    for ev in events {
        match ev.kind {
            EventKind::ActionSucceeded => succeeded += 1,
            EventKind::ActionFailed => failed += 1,
            EventKind::ActionRetrying => m.recovery.retries += 1,
            EventKind::ActionValidated => m.verification_strength.actions_validated += 1,
            EventKind::ActionRejected => m.verification_strength.actions_rejected += 1,
            EventKind::PolicyViolation => m.verification_strength.policy_violations += 1,
            EventKind::ReplanAttempted => m.recovery.replan_attempts += 1,
            EventKind::ReplanRejected => m.recovery.replan_rejected += 1,
            EventKind::ReplanExhausted => m.recovery.replan_exhausted += 1,
            EventKind::BranchDecision => m.recovery.branch_decisions += 1,
            EventKind::AlternativeRejected => m.recovery.alternatives_rejected += 1,
            EventKind::StateChanged => {
                m.state_consistency.state_changes += 1;
                any_state_record = true;
            }
            EventKind::StateSnapshot => {
                m.state_consistency.snapshots += 1;
                any_state_record = true;
            }
            EventKind::StateRollback => m.state_consistency.rollbacks += 1,
            EventKind::PermissionDecision => {
                m.safety.permission_decisions += 1;
                match ev.data.get("gate_decision").and_then(|v| v.as_str()) {
                    Some("needs_approval") => m.safety.escalations += 1,
                    Some("deny") => m.safety.denials += 1,
                    _ => {}
                }
            }
            EventKind::ApprovalRecorded => m.safety.approvals_recorded += 1,
            _ => {}
        }
    }

    m.trajectory_efficiency = TrajectoryEfficiency {
        attempts_total: succeeded + failed,
        actions_succeeded: succeeded,
        failed_attempts: failed,
        total_tokens: totals.tokens,
        total_cost_usd: totals.cost_usd,
        wall_clock_ms: totals.duration_ms,
        success_rate: ratio(succeeded, succeeded + failed),
    };
    m.verification_strength.rejection_rate = ratio(
        m.verification_strength.actions_rejected,
        m.verification_strength.actions_validated + m.verification_strength.actions_rejected,
    );
    m.replayability = Replayability {
        total_events: events.len(),
        state_effects_captured: any_state_record,
    };
    m
}

fn ratio(num: usize, denom: usize) -> Option<f64> {
    (denom > 0).then(|| num as f64 / denom as f64)
}

/// Compute harness metrics from a JSONL string of events (one `Event` per
/// line) — the form the FFI passes from a journal tail. Unparseable lines
/// are counted in `parse_errors` (not silently dropped), so a half-corrupt
/// tail is distinguishable from a clean one (neo review).
pub fn compute_from_jsonl(jsonl: &str) -> HarnessMetrics {
    let mut parse_errors = 0usize;
    let events: Vec<Event> = jsonl
        .lines()
        .filter(|l| !l.trim().is_empty())
        .filter_map(|l| match serde_json::from_str::<Event>(l) {
            Ok(ev) => Some(ev),
            Err(_) => {
                parse_errors += 1;
                None
            }
        })
        .collect();
    let mut m = compute_harness_metrics(&events);
    m.parse_errors = parse_errors;
    m
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{EventLog, Metrics};

    #[test]
    fn efficiency_and_success_rate() {
        let mut log = EventLog::new();
        log.append_metered(
            EventKind::ActionSucceeded,
            Some("a1"),
            None,
            Default::default(),
            Metrics::latency(50.0),
        );
        log.append_metered(
            EventKind::InferenceMetered,
            None,
            None,
            Default::default(),
            Metrics::inference(100, 40, Some(0.01)),
        );
        log.append(EventKind::ActionFailed, Some("a2"), None, Default::default());

        let m = compute_harness_metrics(log.events());
        assert_eq!(m.trajectory_efficiency.attempts_total, 2);
        assert_eq!(m.trajectory_efficiency.actions_succeeded, 1);
        assert_eq!(m.trajectory_efficiency.failed_attempts, 1);
        assert_eq!(m.trajectory_efficiency.total_tokens, 140);
        assert_eq!(m.trajectory_efficiency.wall_clock_ms, 50.0);
        assert_eq!(m.trajectory_efficiency.success_rate, Some(0.5));
    }

    #[test]
    fn retried_then_succeeded_lowers_attempt_success_but_counts_retries() {
        // One logical action: 2 failed attempts + 2 retries + 1 success.
        let mut log = EventLog::new();
        log.append(EventKind::ActionFailed, Some("a1"), None, Default::default());
        log.append(EventKind::ActionRetrying, Some("a1"), None, Default::default());
        log.append(EventKind::ActionFailed, Some("a1"), None, Default::default());
        log.append(EventKind::ActionRetrying, Some("a1"), None, Default::default());
        log.append(EventKind::ActionSucceeded, Some("a1"), None, Default::default());

        let m = compute_harness_metrics(log.events());
        assert_eq!(m.trajectory_efficiency.failed_attempts, 2);
        assert_eq!(m.recovery.retries, 2);
        // Attempt-level success_rate reflects the retries (1/3), and the
        // retries field explains why — the two are read together.
        assert_eq!(m.trajectory_efficiency.success_rate, Some(1.0 / 3.0));
    }

    #[test]
    fn recovery_and_safety_counts() {
        let mut log = EventLog::new();
        log.append(EventKind::ReplanAttempted, None, None, Default::default());
        log.append(EventKind::BranchDecision, None, None, Default::default());
        log.append(EventKind::AlternativeRejected, None, None, Default::default());
        log.append(
            EventKind::PermissionDecision,
            None,
            None,
            [("gate_decision".to_string(), "needs_approval".into())].into(),
        );
        log.append(
            EventKind::PermissionDecision,
            None,
            None,
            [("gate_decision".to_string(), "deny".into())].into(),
        );
        log.append(EventKind::ApprovalRecorded, None, None, Default::default());

        let m = compute_harness_metrics(log.events());
        assert_eq!(m.recovery.replan_attempts, 1);
        assert_eq!(m.recovery.branch_decisions, 1);
        assert_eq!(m.recovery.alternatives_rejected, 1);
        assert_eq!(m.safety.permission_decisions, 2);
        assert_eq!(m.safety.escalations, 1);
        assert_eq!(m.safety.denials, 1);
        assert_eq!(m.safety.approvals_recorded, 1);
    }

    #[test]
    fn state_effects_captured_tracks_state_records() {
        // A success with no state record → no captured state effects (but
        // still fully logged/auditable — that's the point of the rename).
        let mut log = EventLog::new();
        log.append(EventKind::ActionSucceeded, Some("a1"), None, Default::default());
        assert!(!compute_harness_metrics(log.events()).replayability.state_effects_captured);

        // Add a StateChanged → state effects captured.
        log.append(EventKind::StateChanged, Some("a1"), None, Default::default());
        assert!(compute_harness_metrics(log.events()).replayability.state_effects_captured);
    }

    #[test]
    fn empty_trajectory_has_no_success_rate() {
        let m = compute_harness_metrics(&[]);
        assert_eq!(m.trajectory_efficiency.success_rate, None);
        assert_eq!(m.replayability.total_events, 0);
    }

    #[test]
    fn jsonl_counts_parse_errors() {
        let mut log = EventLog::new();
        log.append(EventKind::ActionSucceeded, Some("a1"), None, Default::default());
        let good = serde_json::to_string(&log.events()[0]).unwrap();
        let jsonl = format!("{good}\nnot json\n{{\"kind\":\"bogus\"}}\n");
        let m = compute_from_jsonl(&jsonl);
        assert_eq!(m.trajectory_efficiency.actions_succeeded, 1);
        // "not json" and the unknown-kind line both fail to parse.
        assert_eq!(m.parse_errors, 2);
    }
}