Skip to main content

glass_dev/
testing.rs

1//! Resident test discovery and execution service.
2
3use serde::{Deserialize, Serialize};
4use std::collections::{BTreeMap, BTreeSet};
5use std::fmt;
6use std::io::Read;
7use std::path::{Path, PathBuf};
8use std::process::{Child, Command, ExitStatus, Stdio};
9use std::thread::JoinHandle;
10use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
11
12const MAX_TEST_OUTPUT_BYTES: usize = 2 * 1024 * 1024;
13const MAX_TEST_SUITES: usize = 64;
14const MAX_TEST_RUNS: usize = 256;
15const DEFAULT_TEST_TIMEOUT: Duration = Duration::from_secs(600);
16
17pub type TestResult<T> = Result<T, TestError>;
18
19#[derive(Debug)]
20pub enum TestError {
21    Io(std::io::Error),
22    InvalidInput(String),
23    NotFound(String),
24    Process(String),
25}
26
27impl fmt::Display for TestError {
28    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
29        match self {
30            Self::Io(error) => write!(formatter, "test runtime I/O error: {error}"),
31            Self::InvalidInput(message) => write!(formatter, "invalid test input: {message}"),
32            Self::NotFound(message) => write!(formatter, "test resource not found: {message}"),
33            Self::Process(message) => write!(formatter, "test process error: {message}"),
34        }
35    }
36}
37
38impl std::error::Error for TestError {}
39
40impl From<std::io::Error> for TestError {
41    fn from(error: std::io::Error) -> Self {
42        Self::Io(error)
43    }
44}
45
46#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
47#[serde(rename_all = "camelCase")]
48pub enum TestFramework {
49    Cargo,
50    Pytest,
51    Npm,
52    Pnpm,
53    Yarn,
54    Go,
55    Custom,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
59#[serde(rename_all = "camelCase")]
60pub struct TestSuite {
61    pub id: String,
62    pub name: String,
63    pub framework: TestFramework,
64    pub program: String,
65    pub arguments: Vec<String>,
66    pub source: PathBuf,
67}
68
69#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
70#[serde(rename_all = "camelCase")]
71pub enum TestRunState {
72    Running,
73    Passed,
74    Failed,
75    Cancelled,
76    TimedOut,
77}
78
79impl TestRunState {
80    /// Human-readable label for terminal and TUI surfaces.
81    pub fn label(&self) -> &'static str {
82        match self {
83            Self::Running => "running",
84            Self::Passed => "passed",
85            Self::Failed => "failed",
86            Self::Cancelled => "cancelled",
87            Self::TimedOut => "timed out",
88        }
89    }
90}
91
92#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
93#[serde(rename_all = "camelCase")]
94pub enum TestCaseState {
95    Passed,
96    Failed,
97    Ignored,
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
101#[serde(rename_all = "camelCase")]
102pub struct TestCaseResult {
103    pub name: String,
104    pub state: TestCaseState,
105}
106
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
108#[serde(rename_all = "camelCase")]
109pub struct TestRun {
110    pub id: String,
111    pub suite_id: String,
112    pub actor_id: String,
113    pub workspace_revision: u64,
114    pub started_at_ms: u64,
115    pub duration_ms: Option<u64>,
116    pub state: TestRunState,
117    pub exit_code: Option<i32>,
118    pub output: String,
119    pub output_truncated: bool,
120    pub cases: Vec<TestCaseResult>,
121}
122
123struct RunningTest {
124    snapshot: TestRun,
125    child: Child,
126    stdout: Option<OutputReader>,
127    stderr: Option<OutputReader>,
128    started: Instant,
129    timeout: Duration,
130    requested_stop: Option<TestRunState>,
131}
132
133type OutputReader = JoinHandle<std::io::Result<(Vec<u8>, bool)>>;
134
135pub struct TestService {
136    root: PathBuf,
137    suites: BTreeMap<String, TestSuite>,
138    running: BTreeMap<String, RunningTest>,
139    completed: Vec<TestRun>,
140    watched: BTreeMap<String, u64>,
141}
142
143impl TestService {
144    pub fn discover(root: impl AsRef<Path>) -> TestResult<Self> {
145        let root = root.as_ref().canonicalize()?;
146        let suites = discover_suites(&root)?
147            .into_iter()
148            .map(|suite| (suite.id.clone(), suite))
149            .collect();
150        Ok(Self {
151            root,
152            suites,
153            running: BTreeMap::new(),
154            completed: Vec::new(),
155            watched: BTreeMap::new(),
156        })
157    }
158
159    pub fn root(&self) -> &Path {
160        &self.root
161    }
162
163    pub fn suites(&self) -> impl Iterator<Item = &TestSuite> {
164        self.suites.values()
165    }
166
167    pub fn register(&mut self, suite: TestSuite) -> TestResult<()> {
168        validate_name(&suite.id, "test suite")?;
169        if suite.program.is_empty() || suite.arguments.len() > 256 {
170            return Err(TestError::InvalidInput(
171                "test suite requires a program and at most 256 arguments".into(),
172            ));
173        }
174        if self.suites.len() == MAX_TEST_SUITES && !self.suites.contains_key(&suite.id) {
175            return Err(TestError::InvalidInput(format!(
176                "test suite limit is {MAX_TEST_SUITES}"
177            )));
178        }
179        self.suites.insert(suite.id.clone(), suite);
180        Ok(())
181    }
182
183    pub fn start(
184        &mut self,
185        run_id: &str,
186        suite_id: &str,
187        actor_id: &str,
188        workspace_revision: u64,
189        timeout: Option<Duration>,
190    ) -> TestResult<TestRun> {
191        validate_name(run_id, "test run")?;
192        validate_name(actor_id, "test actor")?;
193        if self.running.contains_key(run_id)
194            || self.completed.iter().any(|result| result.id == run_id)
195        {
196            return Err(TestError::InvalidInput(format!(
197                "test run {run_id} already exists"
198            )));
199        }
200        if self.running.len() + self.completed.len() >= MAX_TEST_RUNS {
201            return Err(TestError::InvalidInput(format!(
202                "test run history limit is {MAX_TEST_RUNS}"
203            )));
204        }
205        let suite = self
206            .suites
207            .get(suite_id)
208            .ok_or_else(|| TestError::NotFound(format!("test suite {suite_id}")))?;
209        let timeout = timeout.unwrap_or(DEFAULT_TEST_TIMEOUT);
210        if timeout.is_zero() || timeout > Duration::from_secs(3600) {
211            return Err(TestError::InvalidInput(
212                "test timeout must be between 1 ms and 1 hour".into(),
213            ));
214        }
215        let mut child = Command::new(&suite.program)
216            .args(&suite.arguments)
217            .current_dir(&self.root)
218            .stdin(Stdio::null())
219            .stdout(Stdio::piped())
220            .stderr(Stdio::piped())
221            .spawn()?;
222        let stdout = child
223            .stdout
224            .take()
225            .ok_or_else(|| TestError::Process("test stdout was unavailable".into()))?;
226        let stderr = child
227            .stderr
228            .take()
229            .ok_or_else(|| TestError::Process("test stderr was unavailable".into()))?;
230        let snapshot = TestRun {
231            id: run_id.to_string(),
232            suite_id: suite_id.to_string(),
233            actor_id: actor_id.to_string(),
234            workspace_revision,
235            started_at_ms: unix_time_ms(),
236            duration_ms: None,
237            state: TestRunState::Running,
238            exit_code: None,
239            output: String::new(),
240            output_truncated: false,
241            cases: Vec::new(),
242        };
243        self.running.insert(
244            run_id.to_string(),
245            RunningTest {
246                snapshot: snapshot.clone(),
247                child,
248                stdout: Some(read_output(stdout)),
249                stderr: Some(read_output(stderr)),
250                started: Instant::now(),
251                timeout,
252                requested_stop: None,
253            },
254        );
255        Ok(snapshot)
256    }
257
258    pub fn cancel(&mut self, run_id: &str) -> TestResult<()> {
259        let running = self
260            .running
261            .get_mut(run_id)
262            .ok_or_else(|| TestError::NotFound(format!("running test {run_id}")))?;
263        running.requested_stop = Some(TestRunState::Cancelled);
264        if running.child.try_wait()?.is_none() {
265            running.child.kill()?;
266        }
267        Ok(())
268    }
269
270    pub fn poll(&mut self) -> TestResult<Vec<TestRun>> {
271        let ids = self.running.keys().cloned().collect::<Vec<_>>();
272        let mut finished = Vec::new();
273        for id in ids {
274            let running = self.running.get_mut(&id).expect("run ID came from map");
275            if running.started.elapsed() >= running.timeout && running.requested_stop.is_none() {
276                running.requested_stop = Some(TestRunState::TimedOut);
277                if running.child.try_wait()?.is_none() {
278                    running.child.kill()?;
279                }
280            }
281            if let Some(status) = running.child.try_wait()? {
282                let running = self.running.remove(&id).expect("finished run exists");
283                let result = finish_run(running, status)?;
284                self.completed.push(result.clone());
285                finished.push(result);
286            }
287        }
288        Ok(finished)
289    }
290
291    pub fn running(&self) -> impl Iterator<Item = &TestRun> {
292        self.running.values().map(|running| &running.snapshot)
293    }
294
295    pub fn results(&self) -> impl DoubleEndedIterator<Item = &TestRun> {
296        self.completed.iter()
297    }
298
299    pub fn result(&self, run_id: &str) -> Option<&TestRun> {
300        self.completed.iter().find(|result| result.id == run_id)
301    }
302
303    pub fn affected_suites<'a>(
304        &'a self,
305        changed_paths: impl IntoIterator<Item = &'a str>,
306    ) -> Vec<&'a TestSuite> {
307        let extensions = changed_paths
308            .into_iter()
309            .filter_map(|path| Path::new(path).extension().and_then(|value| value.to_str()))
310            .collect::<BTreeSet<_>>();
311        self.suites
312            .values()
313            .filter(|suite| match suite.framework {
314                TestFramework::Cargo => extensions.contains("rs") || extensions.contains("toml"),
315                TestFramework::Pytest => extensions.contains("py"),
316                TestFramework::Npm | TestFramework::Pnpm | TestFramework::Yarn => {
317                    ["js", "jsx", "ts", "tsx", "json"]
318                        .iter()
319                        .any(|extension| extensions.contains(extension))
320                }
321                TestFramework::Go => extensions.contains("go"),
322                TestFramework::Custom => true,
323            })
324            .collect()
325    }
326
327    pub fn watch(&mut self, suite_id: &str, current_revision: u64) -> TestResult<()> {
328        if !self.suites.contains_key(suite_id) {
329            return Err(TestError::NotFound(format!("test suite {suite_id}")));
330        }
331        self.watched.insert(suite_id.to_string(), current_revision);
332        Ok(())
333    }
334
335    pub fn unwatch(&mut self, suite_id: &str) -> bool {
336        self.watched.remove(suite_id).is_some()
337    }
338
339    pub fn changed_watches(&mut self, current_revision: u64) -> Vec<String> {
340        let mut changed = Vec::new();
341        for (suite, revision) in &mut self.watched {
342            if *revision != current_revision {
343                *revision = current_revision;
344                changed.push(suite.clone());
345            }
346        }
347        changed
348    }
349}
350
351impl Drop for TestService {
352    fn drop(&mut self) {
353        for running in self.running.values_mut() {
354            if running.child.try_wait().ok().flatten().is_none() {
355                let _ = running.child.kill();
356                let _ = running.child.wait();
357            }
358            join_reader(running.stdout.take());
359            join_reader(running.stderr.take());
360        }
361    }
362}
363
364fn discover_suites(root: &Path) -> TestResult<Vec<TestSuite>> {
365    let mut suites = Vec::new();
366    if root.join("Cargo.toml").is_file() {
367        suites.push(TestSuite {
368            id: "cargo".into(),
369            name: "Cargo tests".into(),
370            framework: TestFramework::Cargo,
371            program: "cargo".into(),
372            arguments: vec!["test".into()],
373            source: root.join("Cargo.toml"),
374        });
375    }
376    let python_source = ["pyproject.toml", "pytest.ini", "setup.cfg"]
377        .into_iter()
378        .map(|name| root.join(name))
379        .find(|path| path.is_file());
380    if let Some(source) = python_source {
381        suites.push(TestSuite {
382            id: "pytest".into(),
383            name: "Python tests".into(),
384            framework: TestFramework::Pytest,
385            program: if cfg!(windows) { "python" } else { "python3" }.into(),
386            arguments: vec!["-m".into(), "pytest".into()],
387            source,
388        });
389    }
390    let package_json = root.join("package.json");
391    if package_json.is_file() {
392        let value: serde_json::Value = serde_json::from_slice(&std::fs::read(&package_json)?)
393            .map_err(|error| TestError::InvalidInput(format!("invalid package.json: {error}")))?;
394        if value
395            .get("scripts")
396            .and_then(|scripts| scripts.get("test"))
397            .and_then(serde_json::Value::as_str)
398            .is_some()
399        {
400            let (framework, program, arguments) = if root.join("pnpm-lock.yaml").is_file() {
401                (TestFramework::Pnpm, "pnpm", vec!["test".into()])
402            } else if root.join("yarn.lock").is_file() {
403                (TestFramework::Yarn, "yarn", vec!["test".into()])
404            } else {
405                (TestFramework::Npm, "npm", vec!["test".into()])
406            };
407            suites.push(TestSuite {
408                id: "node".into(),
409                name: "Node tests".into(),
410                framework,
411                program: program.into(),
412                arguments,
413                source: package_json,
414            });
415        }
416    }
417    if root.join("go.mod").is_file() {
418        suites.push(TestSuite {
419            id: "go".into(),
420            name: "Go tests".into(),
421            framework: TestFramework::Go,
422            program: "go".into(),
423            arguments: vec!["test".into(), "./...".into()],
424            source: root.join("go.mod"),
425        });
426    }
427    suites.truncate(MAX_TEST_SUITES);
428    Ok(suites)
429}
430
431fn finish_run(mut running: RunningTest, status: ExitStatus) -> TestResult<TestRun> {
432    let stdout = join_output(running.stdout.take())?;
433    let stderr = join_output(running.stderr.take())?;
434    let mut combined = stdout.0;
435    if !combined.is_empty() && !stderr.0.is_empty() {
436        combined.push(b'\n');
437    }
438    let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(combined.len());
439    combined.extend_from_slice(&stderr.0[..stderr.0.len().min(remaining)]);
440    let truncated = stdout.1 || stderr.1 || stderr.0.len() > remaining;
441    let output = String::from_utf8_lossy(&combined).into_owned();
442    running.snapshot.duration_ms = Some(running.started.elapsed().as_millis() as u64);
443    running.snapshot.exit_code = status.code();
444    running.snapshot.state = running.requested_stop.unwrap_or(if status.success() {
445        TestRunState::Passed
446    } else {
447        TestRunState::Failed
448    });
449    running.snapshot.cases = parse_test_cases(&output);
450    running.snapshot.output = output;
451    running.snapshot.output_truncated = truncated;
452    Ok(running.snapshot)
453}
454
455fn read_output(mut stream: impl Read + Send + 'static) -> OutputReader {
456    std::thread::spawn(move || {
457        let mut retained = Vec::new();
458        let mut truncated = false;
459        let mut chunk = [0_u8; 8192];
460        loop {
461            let read = stream.read(&mut chunk)?;
462            if read == 0 {
463                break;
464            }
465            let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(retained.len());
466            retained.extend_from_slice(&chunk[..read.min(remaining)]);
467            truncated |= read > remaining;
468        }
469        Ok((retained, truncated))
470    })
471}
472
473fn join_output(reader: Option<OutputReader>) -> TestResult<(Vec<u8>, bool)> {
474    reader
475        .ok_or_else(|| TestError::Process("test output reader was unavailable".into()))?
476        .join()
477        .map_err(|_| TestError::Process("test output reader panicked".into()))?
478        .map_err(TestError::Io)
479}
480
481fn join_reader(reader: Option<OutputReader>) {
482    if let Some(reader) = reader {
483        let _ = reader.join();
484    }
485}
486
487fn parse_test_cases(output: &str) -> Vec<TestCaseResult> {
488    output
489        .lines()
490        .filter_map(|line| {
491            let rest = line.strip_prefix("test ")?;
492            let (name, state) = if let Some(name) = rest.strip_suffix(" ... ok") {
493                (name, TestCaseState::Passed)
494            } else if let Some(name) = rest.strip_suffix(" ... FAILED") {
495                (name, TestCaseState::Failed)
496            } else {
497                (rest.strip_suffix(" ... ignored")?, TestCaseState::Ignored)
498            };
499            Some(TestCaseResult {
500                name: name.to_string(),
501                state,
502            })
503        })
504        .take(100_000)
505        .collect()
506}
507
508fn validate_name(name: &str, description: &str) -> TestResult<()> {
509    if name.is_empty()
510        || name.len() > 128
511        || !name
512            .chars()
513            .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
514    {
515        return Err(TestError::InvalidInput(format!(
516            "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
517        )));
518    }
519    Ok(())
520}
521
522fn unix_time_ms() -> u64 {
523    SystemTime::now()
524        .duration_since(UNIX_EPOCH)
525        .unwrap_or_default()
526        .as_millis()
527        .try_into()
528        .unwrap_or(u64::MAX)
529}
530
531#[cfg(test)]
532mod tests {
533    use super::*;
534    use std::sync::atomic::{AtomicU64, Ordering};
535
536    static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
537
538    fn root() -> PathBuf {
539        let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
540        let root = std::env::temp_dir().join(format!(
541            "glass-test-service-{}-{sequence}",
542            std::process::id()
543        ));
544        std::fs::create_dir_all(&root).unwrap();
545        root
546    }
547
548    #[test]
549    fn discovers_supported_project_test_suites() {
550        let root = root();
551        std::fs::write(
552            root.join("Cargo.toml"),
553            "[package]\nname='x'\nversion='0.1.0'\n",
554        )
555        .unwrap();
556        std::fs::write(root.join("pyproject.toml"), "[project]\nname='x'\n").unwrap();
557        std::fs::write(
558            root.join("package.json"),
559            r#"{"scripts":{"test":"vitest"}}"#,
560        )
561        .unwrap();
562        std::fs::write(root.join("pnpm-lock.yaml"), "lockfileVersion: 9\n").unwrap();
563        std::fs::write(root.join("go.mod"), "module example.invalid/x\n").unwrap();
564        let service = TestService::discover(&root).unwrap();
565        let frameworks = service
566            .suites()
567            .map(|suite| suite.framework)
568            .collect::<Vec<_>>();
569        assert_eq!(
570            frameworks,
571            vec![
572                TestFramework::Cargo,
573                TestFramework::Go,
574                TestFramework::Pnpm,
575                TestFramework::Pytest
576            ]
577        );
578        std::fs::remove_dir_all(root).unwrap();
579    }
580
581    #[test]
582    fn resident_test_run_records_actor_revision_output_and_result() {
583        let root = root();
584        let mut service = TestService::discover(&root).unwrap();
585        service
586            .register(TestSuite {
587                id: "version".into(),
588                name: "Rust compiler version".into(),
589                framework: TestFramework::Custom,
590                program: "rustc".into(),
591                arguments: vec!["--version".into()],
592                source: root.join("glass.toml"),
593            })
594            .unwrap();
595        service
596            .start(
597                "run-1",
598                "version",
599                "agent-tester",
600                7,
601                Some(Duration::from_secs(10)),
602            )
603            .unwrap();
604        let deadline = Instant::now() + Duration::from_secs(10);
605        let result = loop {
606            if let Some(result) = service.poll().unwrap().into_iter().next() {
607                break result;
608            }
609            assert!(Instant::now() < deadline);
610            std::thread::sleep(Duration::from_millis(10));
611        };
612        assert_eq!(result.actor_id, "agent-tester");
613        assert_eq!(result.workspace_revision, 7);
614        assert_eq!(result.state, TestRunState::Passed);
615        assert!(result.output.contains("rustc"));
616        assert_eq!(service.result("run-1"), Some(&result));
617        std::fs::remove_dir_all(root).unwrap();
618    }
619
620    #[test]
621    fn case_parser_and_watch_revisions_are_structured() {
622        let cases = parse_test_cases(
623            "test alpha ... ok\ntest beta ... FAILED\ntest gamma ... ignored\nnoise\n",
624        );
625        assert_eq!(cases.len(), 3);
626        assert_eq!(cases[1].state, TestCaseState::Failed);
627
628        let root = root();
629        std::fs::write(
630            root.join("Cargo.toml"),
631            "[package]\nname='x'\nversion='0.1.0'\n",
632        )
633        .unwrap();
634        let mut service = TestService::discover(&root).unwrap();
635        service.watch("cargo", 1).unwrap();
636        assert!(service.changed_watches(1).is_empty());
637        assert_eq!(service.changed_watches(2), vec!["cargo"]);
638        assert!(service.changed_watches(2).is_empty());
639        std::fs::remove_dir_all(root).unwrap();
640    }
641}