use std::collections::{BTreeMap, HashMap};
use super::events::TeamEvent;
#[derive(Debug, Clone, PartialEq, Default)]
pub struct QualityMetrics {
pub narration_ratio: f64,
pub commit_frequency: f64,
pub first_pass_test_rate: f64,
pub retry_rate: f64,
pub time_to_completion_secs: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct CompletionQualityMetrics {
pub backend: String,
pub role: String,
pub task_id: String,
pub narration_ratio: f64,
pub commit_frequency: f64,
pub first_pass_test_rate: f64,
pub retry_rate: f64,
pub time_to_completion_secs: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct BackendQualityStats {
pub backend: String,
pub samples: u32,
pub narration_ratio: f64,
pub commit_frequency: f64,
pub first_pass_test_rate: f64,
pub retry_rate: f64,
pub time_to_completion_secs: f64,
}
pub const BACKEND_COMPARISON_SQL: &str = "SELECT \
json_extract(payload, '$.backend') AS backend, \
ROUND(AVG(json_extract(payload, '$.narration_ratio')), 4) AS narration_ratio, \
ROUND(AVG(json_extract(payload, '$.commit_frequency')), 4) AS commit_frequency, \
ROUND(AVG(json_extract(payload, '$.first_pass_test_rate')), 4) AS first_pass_test_rate, \
ROUND(AVG(json_extract(payload, '$.retry_rate')), 4) AS retry_rate, \
ROUND(AVG(json_extract(payload, '$.time_to_completion_secs')), 2) AS time_to_completion_secs, \
COUNT(*) AS samples \
FROM events \
WHERE event_type='quality_metrics_recorded' \
GROUP BY backend \
ORDER BY backend;";
pub const QUALITY_TRENDS_SQL: &str = "WITH RECURSIVE hours(h) AS ( \
SELECT COALESCE(MIN(timestamp) / 3600, strftime('%s', 'now') / 3600) FROM events WHERE event_type='quality_metrics_recorded' \
UNION ALL \
SELECT h + 1 FROM hours WHERE h < (SELECT COALESCE(MAX(timestamp) / 3600, strftime('%s', 'now') / 3600) FROM events WHERE event_type='quality_metrics_recorded') \
) \
SELECT \
h * 3600 AS time, \
json_extract(e.payload, '$.backend') AS backend, \
ROUND(AVG(json_extract(e.payload, '$.first_pass_test_rate')), 4) AS first_pass_test_rate, \
ROUND(AVG(json_extract(e.payload, '$.retry_rate')), 4) AS retry_rate \
FROM hours \
LEFT JOIN events e \
ON e.event_type='quality_metrics_recorded' \
AND e.timestamp / 3600 = hours.h \
GROUP BY time, backend \
ORDER BY time, backend;";
pub fn calculate_narration_ratio(output: &str) -> f64 {
narration_ratio(output)
}
pub fn narration_ratio(output: &str) -> f64 {
let mut explanation_lines = 0_u32;
let mut code_or_tool_lines = 0_u32;
for line in output.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if is_code_or_tool_line(trimmed) {
code_or_tool_lines += 1;
} else {
explanation_lines += 1;
}
}
ratio(explanation_lines, code_or_tool_lines)
}
pub fn ratio(explanation_lines: u32, code_or_tool_lines: u32) -> f64 {
let total = explanation_lines + code_or_tool_lines;
if total == 0 {
return 0.0;
}
explanation_lines as f64 / total as f64
}
pub fn commit_frequency(commits: u32, time_to_completion_secs: u64) -> f64 {
if commits == 0 || time_to_completion_secs == 0 {
return 0.0;
}
commits as f64 / (time_to_completion_secs as f64 / 3600.0)
}
#[allow(clippy::too_many_arguments)]
pub fn build_completion_quality_metrics(
backend: impl Into<String>,
role: impl Into<String>,
task_id: u32,
output: &str,
commits: u32,
retries_before_success: u32,
started_at: Option<u64>,
completed_at: u64,
) -> CompletionQualityMetrics {
let time_to_completion_secs = started_at
.map(|started| completed_at.saturating_sub(started))
.unwrap_or(0);
CompletionQualityMetrics {
backend: backend.into(),
role: role.into(),
task_id: task_id.to_string(),
narration_ratio: narration_ratio(output),
commit_frequency: commit_frequency(commits, time_to_completion_secs),
first_pass_test_rate: if retries_before_success == 0 {
1.0
} else {
0.0
},
retry_rate: retries_before_success as f64,
time_to_completion_secs,
}
}
pub fn aggregate_by_backend(
metrics: &[(String, QualityMetrics)],
) -> HashMap<String, QualityMetrics> {
let mut grouped: HashMap<String, Vec<&QualityMetrics>> = HashMap::new();
for (backend, quality) in metrics {
grouped.entry(backend.clone()).or_default().push(quality);
}
grouped
.into_iter()
.map(|(backend, samples)| {
let count = samples.len() as f64;
let avg = |f: fn(&QualityMetrics) -> f64| -> f64 {
if samples.is_empty() {
return 0.0;
}
samples.iter().map(|sample| f(sample)).sum::<f64>() / count
};
let avg_u64 = |f: fn(&QualityMetrics) -> u64| -> u64 {
if samples.is_empty() {
return 0;
}
(samples.iter().map(|sample| f(sample) as f64).sum::<f64>() / count).round() as u64
};
(
backend,
QualityMetrics {
narration_ratio: avg(|sample| sample.narration_ratio),
commit_frequency: avg(|sample| sample.commit_frequency),
first_pass_test_rate: avg(|sample| sample.first_pass_test_rate),
retry_rate: avg(|sample| sample.retry_rate),
time_to_completion_secs: avg_u64(|sample| sample.time_to_completion_secs),
},
)
})
.collect()
}
pub fn aggregate_completion_metrics_by_backend(
metrics: &[CompletionQualityMetrics],
) -> BTreeMap<String, BackendQualityStats> {
let mut grouped: BTreeMap<String, Vec<&CompletionQualityMetrics>> = BTreeMap::new();
for metric in metrics {
grouped
.entry(metric.backend.clone())
.or_default()
.push(metric);
}
grouped
.into_iter()
.map(|(backend, samples)| {
let count = samples.len() as u32;
let avg = |f: fn(&CompletionQualityMetrics) -> f64| -> f64 {
if samples.is_empty() {
return 0.0;
}
samples.iter().map(|sample| f(sample)).sum::<f64>() / samples.len() as f64
};
(
backend.clone(),
BackendQualityStats {
backend,
samples: count,
narration_ratio: avg(|sample| sample.narration_ratio),
commit_frequency: avg(|sample| sample.commit_frequency),
first_pass_test_rate: avg(|sample| sample.first_pass_test_rate),
retry_rate: avg(|sample| sample.retry_rate),
time_to_completion_secs: avg(|sample| sample.time_to_completion_secs as f64),
},
)
})
.collect()
}
pub fn assignment_started_at(events: &[TeamEvent], role: &str, task_id: u32) -> Option<u64> {
let task_id = task_id.to_string();
events
.iter()
.filter(|event| event.event == "task_assigned")
.filter(|event| event.role.as_deref() == Some(role))
.filter(|event| event.task.as_deref() == Some(task_id.as_str()))
.map(|event| event.ts)
.min()
}
fn is_code_or_tool_line(line: &str) -> bool {
line.starts_with("```")
|| line.starts_with("$ ")
|| line.starts_with("> ")
|| line.starts_with("Command:")
|| line.starts_with("Output:")
|| line.starts_with("diff --git")
|| line.starts_with("+++ ")
|| line.starts_with("--- ")
|| line.starts_with("@@")
|| line.starts_with('{')
|| line.starts_with('[')
|| line.starts_with("fn ")
|| line.starts_with("let ")
|| line.starts_with("use ")
|| line.starts_with("pub ")
|| line.contains("::")
|| line.contains('{')
|| line.contains('}')
|| (line.contains('(') && line.contains(')') && line.ends_with(';'))
|| line.contains("();")
|| line.starts_with("test ")
|| line.starts_with("running ")
|| line.starts_with("error[")
|| line.starts_with("warning:")
|| line.starts_with("Compiling ")
|| line.starts_with("Finished ")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn narration_ratio_is_zero_for_all_code_lines() {
let output = "```rust\nfn main() {\n println!(\"hi\");\n}\n```\n$ cargo test";
assert_eq!(calculate_narration_ratio(output), 0.0);
}
#[test]
fn narration_ratio_is_one_for_all_text_lines() {
let output =
"I inspected the failure.\nNext I will patch the parser.\nThen I will rerun tests.";
assert_eq!(calculate_narration_ratio(output), 1.0);
}
#[test]
fn narration_ratio_handles_mixed_text_and_code() {
let output = "I found the issue.\nfn main() {}\nI patched the bug.\n$ cargo test";
assert_eq!(calculate_narration_ratio(output), 0.5);
}
#[test]
fn first_pass_rate_aggregation_handles_multiple_completions() {
let stats = aggregate_by_backend(&[
(
"codex".into(),
QualityMetrics {
first_pass_test_rate: 1.0,
retry_rate: 0.0,
..QualityMetrics::default()
},
),
(
"codex".into(),
QualityMetrics {
first_pass_test_rate: 0.0,
retry_rate: 2.0,
..QualityMetrics::default()
},
),
(
"codex".into(),
QualityMetrics {
first_pass_test_rate: 1.0,
retry_rate: 0.0,
..QualityMetrics::default()
},
),
]);
let codex = stats.get("codex").unwrap();
assert!((codex.first_pass_test_rate - (2.0 / 3.0)).abs() < 0.0001);
assert!((codex.retry_rate - (2.0 / 3.0)).abs() < 0.0001);
}
#[test]
fn backend_grouping_produces_correct_per_backend_stats() {
let stats = aggregate_by_backend(&[
(
"claude".into(),
QualityMetrics {
narration_ratio: 0.5,
commit_frequency: 1.0,
first_pass_test_rate: 1.0,
retry_rate: 0.0,
time_to_completion_secs: 3600,
},
),
(
"codex".into(),
QualityMetrics {
narration_ratio: 0.0,
commit_frequency: 4.0,
first_pass_test_rate: 0.0,
retry_rate: 1.0,
time_to_completion_secs: 1800,
},
),
(
"codex".into(),
QualityMetrics {
narration_ratio: 0.5,
commit_frequency: 3.0,
first_pass_test_rate: 1.0,
retry_rate: 0.0,
time_to_completion_secs: 3600,
},
),
]);
assert!((stats.get("claude").unwrap().first_pass_test_rate - 1.0).abs() < 0.0001);
assert!((stats.get("codex").unwrap().first_pass_test_rate - 0.5).abs() < 0.0001);
assert_eq!(stats.get("codex").unwrap().time_to_completion_secs, 2700);
}
#[test]
fn empty_data_is_handled_gracefully() {
assert!(aggregate_by_backend(&[]).is_empty());
assert_eq!(assignment_started_at(&[], "eng-1", 42), None);
assert_eq!(calculate_narration_ratio(""), 0.0);
assert_eq!(commit_frequency(3, 0), 0.0);
}
}