reflect_eval/
runner.rs

1use crate::parsers;
2use reflect_core::error::Result;
3use reflect_core::types::EvalSignal;
4use std::process::Stdio;
5use std::time::Duration;
6use tokio::process::Command;
7
8#[derive(Debug, Clone)]
9pub struct RunnerConfig {
10    pub name: String,
11    pub command: String,
12    pub args: Vec<String>,
13    pub timeout: Duration,
14    pub working_dir: String,
15}
16
17pub async fn run_evaluator(config: &RunnerConfig) -> Result<EvalSignal> {
18    let result = tokio::time::timeout(config.timeout, async {
19        Command::new("sh")
20            .arg("-c")
21            .arg(&config.command)
22            .current_dir(&config.working_dir)
23            .stdout(Stdio::piped())
24            .stderr(Stdio::piped())
25            .output()
26            .await
27    })
28    .await;
29
30    match result {
31        Ok(Ok(output)) => {
32            let stdout = String::from_utf8_lossy(&output.stdout);
33            let stderr = String::from_utf8_lossy(&output.stderr);
34            let combined = format!("{stdout}\n{stderr}");
35            let signal = match config.name.as_str() {
36                "cargo_test" => parsers::parse_cargo_test_output(&combined),
37                "pytest" => parsers::parse_pytest_output(&combined),
38                "eslint" => parsers::parse_eslint_output(&combined),
39                "tsc" => parsers::parse_tsc_output(&combined),
40                _ => EvalSignal {
41                    evaluator: config.name.clone(),
42                    passed: output.status.success(),
43                    summary: if output.status.success() {
44                        "passed".into()
45                    } else {
46                        format!("exit code {}", output.status.code().unwrap_or(-1))
47                    },
48                    errors: vec![],
49                },
50            };
51            Ok(signal)
52        }
53        Ok(Err(e)) => Ok(EvalSignal {
54            evaluator: config.name.clone(),
55            passed: false,
56            summary: format!("failed to execute: {e}"),
57            errors: vec![],
58        }),
59        Err(_) => Ok(EvalSignal {
60            evaluator: config.name.clone(),
61            passed: false,
62            summary: format!("timed out after {}s", config.timeout.as_secs()),
63            errors: vec![],
64        }),
65    }
66}
67
68#[cfg(test)]
69mod tests {
70    use super::*;
71
72    #[tokio::test]
73    async fn run_echo_command() {
74        let config = RunnerConfig {
75            name: "custom".into(),
76            command: "echo hello".into(),
77            args: vec![],
78            timeout: Duration::from_secs(5),
79            working_dir: "/tmp".into(),
80        };
81        let signal = run_evaluator(&config).await.unwrap();
82        assert!(signal.passed);
83    }
84
85    #[tokio::test]
86    async fn run_failing_command() {
87        let config = RunnerConfig {
88            name: "custom".into(),
89            command: "exit 1".into(),
90            args: vec![],
91            timeout: Duration::from_secs(5),
92            working_dir: "/tmp".into(),
93        };
94        let signal = run_evaluator(&config).await.unwrap();
95        assert!(!signal.passed);
96    }
97
98    #[tokio::test]
99    async fn dispatch_pytest_parser() {
100        let config = RunnerConfig {
101            name: "pytest".into(),
102            command: "echo 'FAILED tests/test_a.py::test_x - ValueError: bad\n============================== 1 failed, 0 passed in 0.1s =============================='".into(),
103            args: vec![],
104            timeout: Duration::from_secs(5),
105            working_dir: "/tmp".into(),
106        };
107        let signal = run_evaluator(&config).await.unwrap();
108        assert_eq!(signal.evaluator, "pytest");
109        assert!(!signal.passed);
110    }
111
112    #[tokio::test]
113    async fn dispatch_eslint_parser() {
114        let config = RunnerConfig {
115            name: "eslint".into(),
116            command: "printf '  1:1  error  msg  no-undef\\n\\n✖ 1 problem (1 error, 0 warnings)\\n'".into(),
117            args: vec![],
118            timeout: Duration::from_secs(5),
119            working_dir: "/tmp".into(),
120        };
121        let signal = run_evaluator(&config).await.unwrap();
122        assert_eq!(signal.evaluator, "eslint");
123        assert!(!signal.passed);
124    }
125
126    #[tokio::test]
127    async fn dispatch_tsc_parser() {
128        let config = RunnerConfig {
129            name: "tsc".into(),
130            command: "echo 'src/a.ts(1,1): error TS2304: Cannot find name x.'".into(),
131            args: vec![],
132            timeout: Duration::from_secs(5),
133            working_dir: "/tmp".into(),
134        };
135        let signal = run_evaluator(&config).await.unwrap();
136        assert_eq!(signal.evaluator, "tsc");
137        assert!(!signal.passed);
138    }
139}
reflect_eval/runner.rs

reflect_eval/
runner.rs
