brainwires_autonomy/

metrics.rs

//! Metrics and reporting for autonomous improvement sessions.

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;

use crate::safety::SafetyStop;

/// Per-strategy task metrics tracking generation, attempt, and success counts.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct StrategyMetrics {
    /// Number of tasks generated by this strategy.
    pub tasks_generated: u32,
    /// Number of tasks attempted.
    pub tasks_attempted: u32,
    /// Number of tasks that succeeded.
    pub tasks_succeeded: u32,
}

/// Comparison result between two execution paths.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComparisonResult {
    /// Whether both paths succeeded.
    pub both_succeeded: bool,
    /// Whether both paths failed.
    pub both_failed: bool,
    /// Whether the diffs match.
    pub diffs_match: bool,
    /// Difference in iteration counts.
    pub iteration_delta: i32,
    /// Errors specific to the bridge path.
    pub bridge_specific_errors: Vec<String>,
}

/// Session-wide metrics tracking aggregate task counts, costs, and per-strategy breakdowns.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SessionMetrics {
    /// Timestamp when the session started.
    pub start_time: DateTime<Utc>,
    /// Total number of tasks attempted.
    pub tasks_attempted: u32,
    /// Total number of tasks that succeeded.
    pub tasks_succeeded: u32,
    /// Total number of tasks that failed.
    pub tasks_failed: u32,
    /// Metrics broken down by strategy name.
    pub per_strategy: HashMap<String, StrategyMetrics>,
    /// Results of dual-path comparisons.
    pub comparisons: Vec<ComparisonResult>,
    /// Estimated total cost in USD.
    pub total_cost: f64,
    /// Total iteration count across all tasks.
    pub total_iterations: u32,
    /// Commit hashes produced during the session.
    pub commits: Vec<String>,
}

impl SessionMetrics {
    /// Create a new session metrics instance starting now.
    pub fn new() -> Self {
        Self {
            start_time: Utc::now(),
            tasks_attempted: 0,
            tasks_succeeded: 0,
            tasks_failed: 0,
            per_strategy: HashMap::new(),
            comparisons: Vec::new(),
            total_cost: 0.0,
            total_iterations: 0,
            commits: Vec::new(),
        }
    }

    /// Record a task attempt for the given strategy.
    pub fn record_attempt(&mut self, strategy: &str) {
        self.tasks_attempted += 1;
        self.per_strategy
            .entry(strategy.to_string())
            .or_default()
            .tasks_attempted += 1;
    }

    /// Record a successful task for the given strategy with its iteration count.
    pub fn record_success(&mut self, strategy: &str, iterations: u32) {
        self.tasks_succeeded += 1;
        self.total_iterations += iterations;
        self.per_strategy
            .entry(strategy.to_string())
            .or_default()
            .tasks_succeeded += 1;
    }

    /// Record a failed task for the given strategy.
    pub fn record_failure(&mut self, strategy: &str) {
        self.tasks_failed += 1;
        self.per_strategy
            .entry(strategy.to_string())
            .or_default()
            .tasks_attempted += 1;
    }

    /// Record the number of tasks generated by a strategy.
    pub fn record_generated(&mut self, strategy: &str, count: u32) {
        self.per_strategy
            .entry(strategy.to_string())
            .or_default()
            .tasks_generated += count;
    }

    /// Record a dual-path comparison result.
    pub fn record_comparison(&mut self, comparison: ComparisonResult) {
        self.comparisons.push(comparison);
    }

    /// Record a commit hash produced during the session.
    pub fn record_commit(&mut self, hash: String) {
        self.commits.push(hash);
    }

    /// Compute the success rate as a fraction between 0.0 and 1.0.
    pub fn success_rate(&self) -> f64 {
        if self.tasks_attempted == 0 {
            0.0
        } else {
            self.tasks_succeeded as f64 / self.tasks_attempted as f64
        }
    }
}

impl Default for SessionMetrics {
    fn default() -> Self {
        Self::new()
    }
}

/// Complete session report combining metrics, duration, and stop reason,
/// with serialization to both JSON and Markdown formats.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SessionReport {
    /// Aggregated session metrics.
    pub metrics: SessionMetrics,
    /// Total wall-clock duration of the session.
    #[serde(with = "duration_serde")]
    pub duration: Duration,
    /// Human-readable reason the session stopped, if a safety stop occurred.
    pub safety_stop_reason: Option<String>,
}

mod duration_serde {
    use serde::{Deserialize, Deserializer, Serialize, Serializer};
    use std::time::Duration;

    pub fn serialize<S>(d: &Duration, s: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        d.as_secs_f64().serialize(s)
    }

    pub fn deserialize<'de, D>(d: D) -> Result<Duration, D::Error>
    where
        D: Deserializer<'de>,
    {
        let secs = f64::deserialize(d)?;
        Ok(Duration::from_secs_f64(secs))
    }
}

impl SessionReport {
    /// Create a new session report from metrics, duration, and optional stop reason.
    pub fn new(
        metrics: SessionMetrics,
        duration: Duration,
        stop_reason: Option<SafetyStop>,
    ) -> Self {
        Self {
            metrics,
            duration,
            safety_stop_reason: stop_reason.map(|r| r.to_string()),
        }
    }

    /// Serialize the report to pretty-printed JSON.
    pub fn to_json(&self) -> Result<String, serde_json::Error> {
        serde_json::to_string_pretty(self)
    }

    /// Render the report as a Markdown document.
    pub fn to_markdown(&self) -> String {
        let mut md = String::new();
        md.push_str("# Self-Improvement Session Report\n\n");
        md.push_str(&format!(
            "**Date**: {}\n",
            self.metrics.start_time.format("%Y-%m-%d %H:%M:%S UTC")
        ));
        md.push_str(&format!(
            "**Duration**: {:.1}s\n",
            self.duration.as_secs_f64()
        ));
        md.push_str(&format!(
            "**Success Rate**: {:.1}%\n\n",
            self.metrics.success_rate() * 100.0
        ));

        md.push_str("## Summary\n\n");
        md.push_str("| Metric | Value |\n|--------|-------|\n");
        md.push_str(&format!(
            "| Tasks Attempted | {} |\n",
            self.metrics.tasks_attempted
        ));
        md.push_str(&format!(
            "| Tasks Succeeded | {} |\n",
            self.metrics.tasks_succeeded
        ));
        md.push_str(&format!(
            "| Tasks Failed | {} |\n",
            self.metrics.tasks_failed
        ));
        md.push_str(&format!(
            "| Total Iterations | {} |\n",
            self.metrics.total_iterations
        ));
        md.push_str(&format!(
            "| Estimated Cost | ${:.4} |\n",
            self.metrics.total_cost
        ));
        md.push_str(&format!("| Commits | {} |\n", self.metrics.commits.len()));

        if !self.metrics.per_strategy.is_empty() {
            md.push_str("\n## Per-Strategy Breakdown\n\n");
            md.push_str("| Strategy | Generated | Attempted | Succeeded |\n");
            md.push_str("|----------|-----------|-----------|----------|\n");
            for (name, stats) in &self.metrics.per_strategy {
                md.push_str(&format!(
                    "| {} | {} | {} | {} |\n",
                    name, stats.tasks_generated, stats.tasks_attempted, stats.tasks_succeeded
                ));
            }
        }

        if !self.metrics.comparisons.is_empty() {
            md.push_str("\n## Dual-Path Comparisons\n\n");
            let both_ok = self
                .metrics
                .comparisons
                .iter()
                .filter(|c| c.both_succeeded)
                .count();
            let diffs_match = self
                .metrics
                .comparisons
                .iter()
                .filter(|c| c.diffs_match)
                .count();
            md.push_str(&format!(
                "- Both paths succeeded: {}/{}\n",
                both_ok,
                self.metrics.comparisons.len()
            ));
            md.push_str(&format!("- Diffs matched: {}/{}\n", diffs_match, both_ok));
        }

        if let Some(ref reason) = self.safety_stop_reason {
            md.push_str(&format!("\n## Stop Reason\n\n{reason}\n"));
        }

        if !self.metrics.commits.is_empty() {
            md.push_str("\n## Commits\n\n");
            for hash in &self.metrics.commits {
                md.push_str(&format!("- `{hash}`\n"));
            }
        }

        md
    }

    /// Save the report to both JSON and Markdown files in the given directory.
    pub fn save(&self, output_dir: &str) -> anyhow::Result<()> {
        std::fs::create_dir_all(output_dir)?;
        let timestamp = self.metrics.start_time.format("%Y%m%d-%H%M%S").to_string();

        let json_path = format!("{output_dir}/session-{timestamp}.json");
        std::fs::write(&json_path, self.to_json()?)?;

        let md_path = format!("{output_dir}/session-{timestamp}.md");
        std::fs::write(&md_path, self.to_markdown())?;

        tracing::info!("Session report saved to {json_path} and {md_path}");
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn session_metrics_new_starts_empty() {
        let m = SessionMetrics::new();
        assert_eq!(m.tasks_attempted, 0);
        assert_eq!(m.tasks_succeeded, 0);
        assert_eq!(m.tasks_failed, 0);
        assert!(m.per_strategy.is_empty());
        assert!(m.commits.is_empty());
    }

    #[test]
    fn record_attempt_increments_counters() {
        let mut m = SessionMetrics::new();
        m.record_attempt("clippy");
        m.record_attempt("clippy");
        m.record_attempt("todo");
        assert_eq!(m.tasks_attempted, 3);
        assert_eq!(m.per_strategy["clippy"].tasks_attempted, 2);
        assert_eq!(m.per_strategy["todo"].tasks_attempted, 1);
    }

    #[test]
    fn record_success_increments_and_tracks_iterations() {
        let mut m = SessionMetrics::new();
        m.record_success("clippy", 5);
        m.record_success("clippy", 10);
        assert_eq!(m.tasks_succeeded, 2);
        assert_eq!(m.total_iterations, 15);
        assert_eq!(m.per_strategy["clippy"].tasks_succeeded, 2);
    }

    #[test]
    fn record_failure_increments_counters() {
        let mut m = SessionMetrics::new();
        m.record_failure("clippy");
        assert_eq!(m.tasks_failed, 1);
        // record_failure also increments tasks_attempted on the strategy
        assert_eq!(m.per_strategy["clippy"].tasks_attempted, 1);
    }

    #[test]
    fn success_rate_zero_when_no_attempts() {
        let m = SessionMetrics::new();
        assert!((m.success_rate() - 0.0).abs() < f64::EPSILON);
    }

    #[test]
    fn success_rate_correct_value() {
        let mut m = SessionMetrics::new();
        m.record_attempt("a");
        m.record_attempt("a");
        m.record_attempt("a");
        m.record_attempt("a");
        m.record_success("a", 1);
        m.record_success("a", 1);
        m.record_success("a", 1);
        // 3 successes out of 4 attempts
        assert!((m.success_rate() - 0.75).abs() < f64::EPSILON);
    }

    #[test]
    fn session_report_to_markdown_produces_valid_output() {
        let mut metrics = SessionMetrics::new();
        metrics.record_attempt("clippy");
        metrics.record_success("clippy", 5);
        metrics.record_commit("abc123".to_string());

        let report = SessionReport::new(metrics, Duration::from_secs(42), None);
        let md = report.to_markdown();

        assert!(md.contains("# Self-Improvement Session Report"));
        assert!(md.contains("Tasks Attempted"));
        assert!(md.contains("Tasks Succeeded"));
        assert!(md.contains("42.0s"));
        assert!(md.contains("`abc123`"));
    }

    #[test]
    fn session_report_to_json_produces_valid_json() {
        let metrics = SessionMetrics::new();
        let report = SessionReport::new(metrics, Duration::from_secs(10), None);
        let json_str = report.to_json().expect("to_json should succeed");
        let parsed: serde_json::Value =
            serde_json::from_str(&json_str).expect("should be valid JSON");
        assert!(parsed.get("metrics").is_some());
        assert!(parsed.get("duration").is_some());
    }
}