Skip to main content

glass_dev/
testing.rs

1//! Resident test discovery and execution service.
2
3use serde::{Deserialize, Serialize};
4use std::collections::{BTreeMap, BTreeSet};
5use std::fmt;
6use std::io::Read;
7use std::path::{Path, PathBuf};
8use std::process::{Child, Command, ExitStatus, Stdio};
9use std::thread::JoinHandle;
10use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
11
12const MAX_TEST_OUTPUT_BYTES: usize = 2 * 1024 * 1024;
13const MAX_TEST_SUITES: usize = 64;
14const MAX_TEST_RUNS: usize = 256;
15const DEFAULT_TEST_TIMEOUT: Duration = Duration::from_secs(600);
16
17pub type TestResult<T> = Result<T, TestError>;
18
19#[derive(Debug)]
20pub enum TestError {
21    Io(std::io::Error),
22    InvalidInput(String),
23    NotFound(String),
24    Process(String),
25}
26
27impl fmt::Display for TestError {
28    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
29        match self {
30            Self::Io(error) => write!(formatter, "test runtime I/O error: {error}"),
31            Self::InvalidInput(message) => write!(formatter, "invalid test input: {message}"),
32            Self::NotFound(message) => write!(formatter, "test resource not found: {message}"),
33            Self::Process(message) => write!(formatter, "test process error: {message}"),
34        }
35    }
36}
37
38impl std::error::Error for TestError {}
39
40impl From<std::io::Error> for TestError {
41    fn from(error: std::io::Error) -> Self {
42        Self::Io(error)
43    }
44}
45
46#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
47#[serde(rename_all = "camelCase")]
48pub enum TestFramework {
49    Cargo,
50    Pytest,
51    Npm,
52    Pnpm,
53    Yarn,
54    Go,
55    Custom,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
59#[serde(rename_all = "camelCase")]
60pub struct TestSuite {
61    pub id: String,
62    pub name: String,
63    pub framework: TestFramework,
64    pub program: String,
65    pub arguments: Vec<String>,
66    pub source: PathBuf,
67}
68
69#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
70#[serde(rename_all = "camelCase")]
71pub enum TestRunState {
72    Running,
73    Passed,
74    Failed,
75    Cancelled,
76    TimedOut,
77}
78
79#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
80#[serde(rename_all = "camelCase")]
81pub enum TestCaseState {
82    Passed,
83    Failed,
84    Ignored,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
88#[serde(rename_all = "camelCase")]
89pub struct TestCaseResult {
90    pub name: String,
91    pub state: TestCaseState,
92}
93
94#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
95#[serde(rename_all = "camelCase")]
96pub struct TestRun {
97    pub id: String,
98    pub suite_id: String,
99    pub actor_id: String,
100    pub workspace_revision: u64,
101    pub started_at_ms: u64,
102    pub duration_ms: Option<u64>,
103    pub state: TestRunState,
104    pub exit_code: Option<i32>,
105    pub output: String,
106    pub output_truncated: bool,
107    pub cases: Vec<TestCaseResult>,
108}
109
110struct RunningTest {
111    snapshot: TestRun,
112    child: Child,
113    stdout: Option<OutputReader>,
114    stderr: Option<OutputReader>,
115    started: Instant,
116    timeout: Duration,
117    requested_stop: Option<TestRunState>,
118}
119
120type OutputReader = JoinHandle<std::io::Result<(Vec<u8>, bool)>>;
121
122pub struct TestService {
123    root: PathBuf,
124    suites: BTreeMap<String, TestSuite>,
125    running: BTreeMap<String, RunningTest>,
126    completed: Vec<TestRun>,
127    watched: BTreeMap<String, u64>,
128}
129
130impl TestService {
131    pub fn discover(root: impl AsRef<Path>) -> TestResult<Self> {
132        let root = root.as_ref().canonicalize()?;
133        let suites = discover_suites(&root)?
134            .into_iter()
135            .map(|suite| (suite.id.clone(), suite))
136            .collect();
137        Ok(Self {
138            root,
139            suites,
140            running: BTreeMap::new(),
141            completed: Vec::new(),
142            watched: BTreeMap::new(),
143        })
144    }
145
146    pub fn root(&self) -> &Path {
147        &self.root
148    }
149
150    pub fn suites(&self) -> impl Iterator<Item = &TestSuite> {
151        self.suites.values()
152    }
153
154    pub fn register(&mut self, suite: TestSuite) -> TestResult<()> {
155        validate_name(&suite.id, "test suite")?;
156        if suite.program.is_empty() || suite.arguments.len() > 256 {
157            return Err(TestError::InvalidInput(
158                "test suite requires a program and at most 256 arguments".into(),
159            ));
160        }
161        if self.suites.len() == MAX_TEST_SUITES && !self.suites.contains_key(&suite.id) {
162            return Err(TestError::InvalidInput(format!(
163                "test suite limit is {MAX_TEST_SUITES}"
164            )));
165        }
166        self.suites.insert(suite.id.clone(), suite);
167        Ok(())
168    }
169
170    pub fn start(
171        &mut self,
172        run_id: &str,
173        suite_id: &str,
174        actor_id: &str,
175        workspace_revision: u64,
176        timeout: Option<Duration>,
177    ) -> TestResult<TestRun> {
178        validate_name(run_id, "test run")?;
179        validate_name(actor_id, "test actor")?;
180        if self.running.contains_key(run_id)
181            || self.completed.iter().any(|result| result.id == run_id)
182        {
183            return Err(TestError::InvalidInput(format!(
184                "test run {run_id} already exists"
185            )));
186        }
187        if self.running.len() + self.completed.len() >= MAX_TEST_RUNS {
188            return Err(TestError::InvalidInput(format!(
189                "test run history limit is {MAX_TEST_RUNS}"
190            )));
191        }
192        let suite = self
193            .suites
194            .get(suite_id)
195            .ok_or_else(|| TestError::NotFound(format!("test suite {suite_id}")))?;
196        let timeout = timeout.unwrap_or(DEFAULT_TEST_TIMEOUT);
197        if timeout.is_zero() || timeout > Duration::from_secs(3600) {
198            return Err(TestError::InvalidInput(
199                "test timeout must be between 1 ms and 1 hour".into(),
200            ));
201        }
202        let mut child = Command::new(&suite.program)
203            .args(&suite.arguments)
204            .current_dir(&self.root)
205            .stdin(Stdio::null())
206            .stdout(Stdio::piped())
207            .stderr(Stdio::piped())
208            .spawn()?;
209        let stdout = child
210            .stdout
211            .take()
212            .ok_or_else(|| TestError::Process("test stdout was unavailable".into()))?;
213        let stderr = child
214            .stderr
215            .take()
216            .ok_or_else(|| TestError::Process("test stderr was unavailable".into()))?;
217        let snapshot = TestRun {
218            id: run_id.to_string(),
219            suite_id: suite_id.to_string(),
220            actor_id: actor_id.to_string(),
221            workspace_revision,
222            started_at_ms: unix_time_ms(),
223            duration_ms: None,
224            state: TestRunState::Running,
225            exit_code: None,
226            output: String::new(),
227            output_truncated: false,
228            cases: Vec::new(),
229        };
230        self.running.insert(
231            run_id.to_string(),
232            RunningTest {
233                snapshot: snapshot.clone(),
234                child,
235                stdout: Some(read_output(stdout)),
236                stderr: Some(read_output(stderr)),
237                started: Instant::now(),
238                timeout,
239                requested_stop: None,
240            },
241        );
242        Ok(snapshot)
243    }
244
245    pub fn cancel(&mut self, run_id: &str) -> TestResult<()> {
246        let running = self
247            .running
248            .get_mut(run_id)
249            .ok_or_else(|| TestError::NotFound(format!("running test {run_id}")))?;
250        running.requested_stop = Some(TestRunState::Cancelled);
251        if running.child.try_wait()?.is_none() {
252            running.child.kill()?;
253        }
254        Ok(())
255    }
256
257    pub fn poll(&mut self) -> TestResult<Vec<TestRun>> {
258        let ids = self.running.keys().cloned().collect::<Vec<_>>();
259        let mut finished = Vec::new();
260        for id in ids {
261            let running = self.running.get_mut(&id).expect("run ID came from map");
262            if running.started.elapsed() >= running.timeout && running.requested_stop.is_none() {
263                running.requested_stop = Some(TestRunState::TimedOut);
264                if running.child.try_wait()?.is_none() {
265                    running.child.kill()?;
266                }
267            }
268            if let Some(status) = running.child.try_wait()? {
269                let running = self.running.remove(&id).expect("finished run exists");
270                let result = finish_run(running, status)?;
271                self.completed.push(result.clone());
272                finished.push(result);
273            }
274        }
275        Ok(finished)
276    }
277
278    pub fn running(&self) -> impl Iterator<Item = &TestRun> {
279        self.running.values().map(|running| &running.snapshot)
280    }
281
282    pub fn results(&self) -> impl DoubleEndedIterator<Item = &TestRun> {
283        self.completed.iter()
284    }
285
286    pub fn result(&self, run_id: &str) -> Option<&TestRun> {
287        self.completed.iter().find(|result| result.id == run_id)
288    }
289
290    pub fn affected_suites<'a>(
291        &'a self,
292        changed_paths: impl IntoIterator<Item = &'a str>,
293    ) -> Vec<&'a TestSuite> {
294        let extensions = changed_paths
295            .into_iter()
296            .filter_map(|path| Path::new(path).extension().and_then(|value| value.to_str()))
297            .collect::<BTreeSet<_>>();
298        self.suites
299            .values()
300            .filter(|suite| match suite.framework {
301                TestFramework::Cargo => extensions.contains("rs") || extensions.contains("toml"),
302                TestFramework::Pytest => extensions.contains("py"),
303                TestFramework::Npm | TestFramework::Pnpm | TestFramework::Yarn => {
304                    ["js", "jsx", "ts", "tsx", "json"]
305                        .iter()
306                        .any(|extension| extensions.contains(extension))
307                }
308                TestFramework::Go => extensions.contains("go"),
309                TestFramework::Custom => true,
310            })
311            .collect()
312    }
313
314    pub fn watch(&mut self, suite_id: &str, current_revision: u64) -> TestResult<()> {
315        if !self.suites.contains_key(suite_id) {
316            return Err(TestError::NotFound(format!("test suite {suite_id}")));
317        }
318        self.watched.insert(suite_id.to_string(), current_revision);
319        Ok(())
320    }
321
322    pub fn unwatch(&mut self, suite_id: &str) -> bool {
323        self.watched.remove(suite_id).is_some()
324    }
325
326    pub fn changed_watches(&mut self, current_revision: u64) -> Vec<String> {
327        let mut changed = Vec::new();
328        for (suite, revision) in &mut self.watched {
329            if *revision != current_revision {
330                *revision = current_revision;
331                changed.push(suite.clone());
332            }
333        }
334        changed
335    }
336}
337
338impl Drop for TestService {
339    fn drop(&mut self) {
340        for running in self.running.values_mut() {
341            if running.child.try_wait().ok().flatten().is_none() {
342                let _ = running.child.kill();
343                let _ = running.child.wait();
344            }
345            join_reader(running.stdout.take());
346            join_reader(running.stderr.take());
347        }
348    }
349}
350
351fn discover_suites(root: &Path) -> TestResult<Vec<TestSuite>> {
352    let mut suites = Vec::new();
353    if root.join("Cargo.toml").is_file() {
354        suites.push(TestSuite {
355            id: "cargo".into(),
356            name: "Cargo tests".into(),
357            framework: TestFramework::Cargo,
358            program: "cargo".into(),
359            arguments: vec!["test".into()],
360            source: root.join("Cargo.toml"),
361        });
362    }
363    let python_source = ["pyproject.toml", "pytest.ini", "setup.cfg"]
364        .into_iter()
365        .map(|name| root.join(name))
366        .find(|path| path.is_file());
367    if let Some(source) = python_source {
368        suites.push(TestSuite {
369            id: "pytest".into(),
370            name: "Python tests".into(),
371            framework: TestFramework::Pytest,
372            program: if cfg!(windows) { "python" } else { "python3" }.into(),
373            arguments: vec!["-m".into(), "pytest".into()],
374            source,
375        });
376    }
377    let package_json = root.join("package.json");
378    if package_json.is_file() {
379        let value: serde_json::Value = serde_json::from_slice(&std::fs::read(&package_json)?)
380            .map_err(|error| TestError::InvalidInput(format!("invalid package.json: {error}")))?;
381        if value
382            .get("scripts")
383            .and_then(|scripts| scripts.get("test"))
384            .and_then(serde_json::Value::as_str)
385            .is_some()
386        {
387            let (framework, program, arguments) = if root.join("pnpm-lock.yaml").is_file() {
388                (TestFramework::Pnpm, "pnpm", vec!["test".into()])
389            } else if root.join("yarn.lock").is_file() {
390                (TestFramework::Yarn, "yarn", vec!["test".into()])
391            } else {
392                (TestFramework::Npm, "npm", vec!["test".into()])
393            };
394            suites.push(TestSuite {
395                id: "node".into(),
396                name: "Node tests".into(),
397                framework,
398                program: program.into(),
399                arguments,
400                source: package_json,
401            });
402        }
403    }
404    if root.join("go.mod").is_file() {
405        suites.push(TestSuite {
406            id: "go".into(),
407            name: "Go tests".into(),
408            framework: TestFramework::Go,
409            program: "go".into(),
410            arguments: vec!["test".into(), "./...".into()],
411            source: root.join("go.mod"),
412        });
413    }
414    suites.truncate(MAX_TEST_SUITES);
415    Ok(suites)
416}
417
418fn finish_run(mut running: RunningTest, status: ExitStatus) -> TestResult<TestRun> {
419    let stdout = join_output(running.stdout.take())?;
420    let stderr = join_output(running.stderr.take())?;
421    let mut combined = stdout.0;
422    if !combined.is_empty() && !stderr.0.is_empty() {
423        combined.push(b'\n');
424    }
425    let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(combined.len());
426    combined.extend_from_slice(&stderr.0[..stderr.0.len().min(remaining)]);
427    let truncated = stdout.1 || stderr.1 || stderr.0.len() > remaining;
428    let output = String::from_utf8_lossy(&combined).into_owned();
429    running.snapshot.duration_ms = Some(running.started.elapsed().as_millis() as u64);
430    running.snapshot.exit_code = status.code();
431    running.snapshot.state = running.requested_stop.unwrap_or(if status.success() {
432        TestRunState::Passed
433    } else {
434        TestRunState::Failed
435    });
436    running.snapshot.cases = parse_test_cases(&output);
437    running.snapshot.output = output;
438    running.snapshot.output_truncated = truncated;
439    Ok(running.snapshot)
440}
441
442fn read_output(mut stream: impl Read + Send + 'static) -> OutputReader {
443    std::thread::spawn(move || {
444        let mut retained = Vec::new();
445        let mut truncated = false;
446        let mut chunk = [0_u8; 8192];
447        loop {
448            let read = stream.read(&mut chunk)?;
449            if read == 0 {
450                break;
451            }
452            let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(retained.len());
453            retained.extend_from_slice(&chunk[..read.min(remaining)]);
454            truncated |= read > remaining;
455        }
456        Ok((retained, truncated))
457    })
458}
459
460fn join_output(reader: Option<OutputReader>) -> TestResult<(Vec<u8>, bool)> {
461    reader
462        .ok_or_else(|| TestError::Process("test output reader was unavailable".into()))?
463        .join()
464        .map_err(|_| TestError::Process("test output reader panicked".into()))?
465        .map_err(TestError::Io)
466}
467
468fn join_reader(reader: Option<OutputReader>) {
469    if let Some(reader) = reader {
470        let _ = reader.join();
471    }
472}
473
474fn parse_test_cases(output: &str) -> Vec<TestCaseResult> {
475    output
476        .lines()
477        .filter_map(|line| {
478            let rest = line.strip_prefix("test ")?;
479            let (name, state) = if let Some(name) = rest.strip_suffix(" ... ok") {
480                (name, TestCaseState::Passed)
481            } else if let Some(name) = rest.strip_suffix(" ... FAILED") {
482                (name, TestCaseState::Failed)
483            } else {
484                (rest.strip_suffix(" ... ignored")?, TestCaseState::Ignored)
485            };
486            Some(TestCaseResult {
487                name: name.to_string(),
488                state,
489            })
490        })
491        .take(100_000)
492        .collect()
493}
494
495fn validate_name(name: &str, description: &str) -> TestResult<()> {
496    if name.is_empty()
497        || name.len() > 128
498        || !name
499            .chars()
500            .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
501    {
502        return Err(TestError::InvalidInput(format!(
503            "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
504        )));
505    }
506    Ok(())
507}
508
509fn unix_time_ms() -> u64 {
510    SystemTime::now()
511        .duration_since(UNIX_EPOCH)
512        .unwrap_or_default()
513        .as_millis()
514        .try_into()
515        .unwrap_or(u64::MAX)
516}
517
518#[cfg(test)]
519mod tests {
520    use super::*;
521    use std::sync::atomic::{AtomicU64, Ordering};
522
523    static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
524
525    fn root() -> PathBuf {
526        let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
527        let root = std::env::temp_dir().join(format!(
528            "glass-test-service-{}-{sequence}",
529            std::process::id()
530        ));
531        std::fs::create_dir_all(&root).unwrap();
532        root
533    }
534
535    #[test]
536    fn discovers_supported_project_test_suites() {
537        let root = root();
538        std::fs::write(
539            root.join("Cargo.toml"),
540            "[package]\nname='x'\nversion='0.1.0'\n",
541        )
542        .unwrap();
543        std::fs::write(root.join("pyproject.toml"), "[project]\nname='x'\n").unwrap();
544        std::fs::write(
545            root.join("package.json"),
546            r#"{"scripts":{"test":"vitest"}}"#,
547        )
548        .unwrap();
549        std::fs::write(root.join("pnpm-lock.yaml"), "lockfileVersion: 9\n").unwrap();
550        std::fs::write(root.join("go.mod"), "module example.invalid/x\n").unwrap();
551        let service = TestService::discover(&root).unwrap();
552        let frameworks = service
553            .suites()
554            .map(|suite| suite.framework)
555            .collect::<Vec<_>>();
556        assert_eq!(
557            frameworks,
558            vec![
559                TestFramework::Cargo,
560                TestFramework::Go,
561                TestFramework::Pnpm,
562                TestFramework::Pytest
563            ]
564        );
565        std::fs::remove_dir_all(root).unwrap();
566    }
567
568    #[test]
569    fn resident_test_run_records_actor_revision_output_and_result() {
570        let root = root();
571        let mut service = TestService::discover(&root).unwrap();
572        service
573            .register(TestSuite {
574                id: "version".into(),
575                name: "Rust compiler version".into(),
576                framework: TestFramework::Custom,
577                program: "rustc".into(),
578                arguments: vec!["--version".into()],
579                source: root.join("glass.toml"),
580            })
581            .unwrap();
582        service
583            .start(
584                "run-1",
585                "version",
586                "agent-tester",
587                7,
588                Some(Duration::from_secs(10)),
589            )
590            .unwrap();
591        let deadline = Instant::now() + Duration::from_secs(10);
592        let result = loop {
593            if let Some(result) = service.poll().unwrap().into_iter().next() {
594                break result;
595            }
596            assert!(Instant::now() < deadline);
597            std::thread::sleep(Duration::from_millis(10));
598        };
599        assert_eq!(result.actor_id, "agent-tester");
600        assert_eq!(result.workspace_revision, 7);
601        assert_eq!(result.state, TestRunState::Passed);
602        assert!(result.output.contains("rustc"));
603        assert_eq!(service.result("run-1"), Some(&result));
604        std::fs::remove_dir_all(root).unwrap();
605    }
606
607    #[test]
608    fn case_parser_and_watch_revisions_are_structured() {
609        let cases = parse_test_cases(
610            "test alpha ... ok\ntest beta ... FAILED\ntest gamma ... ignored\nnoise\n",
611        );
612        assert_eq!(cases.len(), 3);
613        assert_eq!(cases[1].state, TestCaseState::Failed);
614
615        let root = root();
616        std::fs::write(
617            root.join("Cargo.toml"),
618            "[package]\nname='x'\nversion='0.1.0'\n",
619        )
620        .unwrap();
621        let mut service = TestService::discover(&root).unwrap();
622        service.watch("cargo", 1).unwrap();
623        assert!(service.changed_watches(1).is_empty());
624        assert_eq!(service.changed_watches(2), vec!["cargo"]);
625        assert!(service.changed_watches(2).is_empty());
626        std::fs::remove_dir_all(root).unwrap();
627    }
628}