cgp/analysis/
compete.rs

1//! `cgp compete` — Head-to-head competitor comparison.
2//! Spec section 2.5: run two or more commands, measure wall time,
3//! compute TFLOP/s, and produce a comparison table.
4
5use anyhow::Result;
6use serde::Serialize;
7use std::process::Command;
8use std::time::Instant;
9
10/// Result of running one competitor.
11#[derive(Debug, Clone, Serialize)]
12pub struct CompetitorResult {
13    pub label: String,
14    pub command: String,
15    pub wall_time_ms: f64,
16    pub exit_code: i32,
17}
18
19/// Run a command and measure wall time.
20fn run_timed(command: &str) -> Result<CompetitorResult> {
21    // Split command into program + args (basic shell-like splitting)
22    let parts: Vec<&str> = command.split_whitespace().collect();
23    if parts.is_empty() {
24        anyhow::bail!("Empty command");
25    }
26
27    let start = Instant::now();
28    let status = Command::new(parts[0])
29        .args(&parts[1..])
30        .stdout(std::process::Stdio::null())
31        .stderr(std::process::Stdio::null())
32        .status()
33        .map_err(|e| anyhow::anyhow!("Failed to run '{}': {}", parts[0], e))?;
34    let elapsed = start.elapsed();
35
36    Ok(CompetitorResult {
37        label: String::new(),
38        command: command.to_string(),
39        wall_time_ms: elapsed.as_secs_f64() * 1000.0,
40        exit_code: status.code().unwrap_or(-1),
41    })
42}
43
44/// Run the `cgp compete` command.
45pub fn run_compete(
46    workload: &str,
47    ours: &str,
48    theirs: &[String],
49    label: Option<&str>,
50    json: bool,
51) -> Result<()> {
52    let labels: Vec<String> = match label {
53        Some(l) => l.split(',').map(String::from).collect(),
54        None => {
55            let mut v = vec!["ours".to_string()];
56            v.extend((0..theirs.len()).map(|i| format!("theirs-{}", i + 1)));
57            v
58        }
59    };
60
61    println!("\n=== CGP Head-to-Head: {workload} ===\n");
62
63    let mut results: Vec<CompetitorResult> = Vec::new();
64
65    let default_ours = "ours".to_string();
66    let ours_label = labels.first().unwrap_or(&default_ours);
67
68    // Run ours
69    eprint!("  Running: {ours_label} ...");
70    match run_timed(ours) {
71        Ok(mut r) => {
72            r.label = ours_label.clone();
73            eprintln!(" {:.1}ms", r.wall_time_ms);
74            results.push(r);
75        }
76        Err(e) => {
77            eprintln!(" FAILED: {e}");
78            results.push(CompetitorResult {
79                label: ours_label.clone(),
80                command: ours.to_string(),
81                wall_time_ms: f64::INFINITY,
82                exit_code: -1,
83            });
84        }
85    }
86
87    // Run theirs
88    for (i, cmd) in theirs.iter().enumerate() {
89        let default_theirs = format!("theirs-{}", i + 1);
90        let lbl = labels.get(i + 1).unwrap_or(&default_theirs);
91        eprint!("  Running: {lbl} ...");
92        match run_timed(cmd) {
93            Ok(mut r) => {
94                r.label = lbl.to_string();
95                eprintln!(" {:.1}ms", r.wall_time_ms);
96                results.push(r);
97            }
98            Err(e) => {
99                eprintln!(" FAILED: {e}");
100                results.push(CompetitorResult {
101                    label: lbl.to_string(),
102                    command: cmd.clone(),
103                    wall_time_ms: f64::INFINITY,
104                    exit_code: -1,
105                });
106            }
107        }
108    }
109
110    // Find best time
111    let best_time = results
112        .iter()
113        .filter(|r| r.wall_time_ms.is_finite())
114        .map(|r| r.wall_time_ms)
115        .fold(f64::INFINITY, f64::min);
116
117    if json {
118        println!("{}", serde_json::to_string_pretty(&results)?);
119        return Ok(());
120    }
121
122    // Print table
123    println!();
124    println!(
125        "  {:20} {:>12} {:>8} {:>10}",
126        "Competitor", "Time (ms)", "Exit", "vs Best"
127    );
128    println!("  {}", "-".repeat(54));
129
130    for r in &results {
131        let ratio = if best_time > 0.0 && r.wall_time_ms.is_finite() {
132            format!("{:.2}x", r.wall_time_ms / best_time)
133        } else {
134            "N/A".to_string()
135        };
136        let time_str = if r.wall_time_ms.is_finite() {
137            format!("{:.1}", r.wall_time_ms)
138        } else {
139            "FAILED".to_string()
140        };
141        println!(
142            "  {:20} {:>12} {:>8} {:>10}",
143            r.label, time_str, r.exit_code, ratio
144        );
145    }
146
147    // Winner
148    if let Some(winner) = results
149        .iter()
150        .filter(|r| r.wall_time_ms.is_finite())
151        .min_by(|a, b| {
152            a.wall_time_ms
153                .partial_cmp(&b.wall_time_ms)
154                .unwrap_or(std::cmp::Ordering::Equal)
155        })
156    {
157        println!(
158            "\n  Winner: {} ({:.1}ms)",
159            winner.label, winner.wall_time_ms
160        );
161    }
162
163    println!();
164    Ok(())
165}
166
167#[cfg(test)]
168mod tests {
169    use super::*;
170
171    #[test]
172    fn test_run_timed_true() {
173        let result = run_timed("true").unwrap();
174        assert_eq!(result.exit_code, 0);
175        assert!(result.wall_time_ms < 1000.0);
176    }
177
178    #[test]
179    fn test_run_timed_false() {
180        let result = run_timed("false").unwrap();
181        assert_ne!(result.exit_code, 0);
182    }
183
184    #[test]
185    fn test_run_timed_nonexistent() {
186        let result = run_timed("nonexistent_binary_xyz_123");
187        assert!(result.is_err());
188    }
189}
cgp/analysis/compete.rs

cgp/analysis/
compete.rs
