1use serde::{Deserialize, Serialize};
4use std::collections::{BTreeMap, BTreeSet};
5use std::fmt;
6use std::io::Read;
7use std::path::{Path, PathBuf};
8use std::process::{Child, Command, ExitStatus, Stdio};
9use std::thread::JoinHandle;
10use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
11
12const MAX_TEST_OUTPUT_BYTES: usize = 2 * 1024 * 1024;
13const MAX_TEST_SUITES: usize = 64;
14const MAX_TEST_RUNS: usize = 256;
15const DEFAULT_TEST_TIMEOUT: Duration = Duration::from_secs(600);
16
17pub type TestResult<T> = Result<T, TestError>;
18
19#[derive(Debug)]
20pub enum TestError {
21 Io(std::io::Error),
22 InvalidInput(String),
23 NotFound(String),
24 Process(String),
25}
26
27impl fmt::Display for TestError {
28 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
29 match self {
30 Self::Io(error) => write!(formatter, "test runtime I/O error: {error}"),
31 Self::InvalidInput(message) => write!(formatter, "invalid test input: {message}"),
32 Self::NotFound(message) => write!(formatter, "test resource not found: {message}"),
33 Self::Process(message) => write!(formatter, "test process error: {message}"),
34 }
35 }
36}
37
38impl std::error::Error for TestError {}
39
40impl From<std::io::Error> for TestError {
41 fn from(error: std::io::Error) -> Self {
42 Self::Io(error)
43 }
44}
45
46#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
47#[serde(rename_all = "camelCase")]
48pub enum TestFramework {
49 Cargo,
50 Pytest,
51 Npm,
52 Pnpm,
53 Yarn,
54 Go,
55 Custom,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
59#[serde(rename_all = "camelCase")]
60pub struct TestSuite {
61 pub id: String,
62 pub name: String,
63 pub framework: TestFramework,
64 pub program: String,
65 pub arguments: Vec<String>,
66 pub source: PathBuf,
67}
68
69#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
70#[serde(rename_all = "camelCase")]
71pub enum TestRunState {
72 Running,
73 Passed,
74 Failed,
75 Cancelled,
76 TimedOut,
77}
78
79#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
80#[serde(rename_all = "camelCase")]
81pub enum TestCaseState {
82 Passed,
83 Failed,
84 Ignored,
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
88#[serde(rename_all = "camelCase")]
89pub struct TestCaseResult {
90 pub name: String,
91 pub state: TestCaseState,
92}
93
94#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
95#[serde(rename_all = "camelCase")]
96pub struct TestRun {
97 pub id: String,
98 pub suite_id: String,
99 pub actor_id: String,
100 pub workspace_revision: u64,
101 pub started_at_ms: u64,
102 pub duration_ms: Option<u64>,
103 pub state: TestRunState,
104 pub exit_code: Option<i32>,
105 pub output: String,
106 pub output_truncated: bool,
107 pub cases: Vec<TestCaseResult>,
108}
109
110struct RunningTest {
111 snapshot: TestRun,
112 child: Child,
113 stdout: Option<OutputReader>,
114 stderr: Option<OutputReader>,
115 started: Instant,
116 timeout: Duration,
117 requested_stop: Option<TestRunState>,
118}
119
120type OutputReader = JoinHandle<std::io::Result<(Vec<u8>, bool)>>;
121
122pub struct TestService {
123 root: PathBuf,
124 suites: BTreeMap<String, TestSuite>,
125 running: BTreeMap<String, RunningTest>,
126 completed: Vec<TestRun>,
127 watched: BTreeMap<String, u64>,
128}
129
130impl TestService {
131 pub fn discover(root: impl AsRef<Path>) -> TestResult<Self> {
132 let root = root.as_ref().canonicalize()?;
133 let suites = discover_suites(&root)?
134 .into_iter()
135 .map(|suite| (suite.id.clone(), suite))
136 .collect();
137 Ok(Self {
138 root,
139 suites,
140 running: BTreeMap::new(),
141 completed: Vec::new(),
142 watched: BTreeMap::new(),
143 })
144 }
145
146 pub fn root(&self) -> &Path {
147 &self.root
148 }
149
150 pub fn suites(&self) -> impl Iterator<Item = &TestSuite> {
151 self.suites.values()
152 }
153
154 pub fn register(&mut self, suite: TestSuite) -> TestResult<()> {
155 validate_name(&suite.id, "test suite")?;
156 if suite.program.is_empty() || suite.arguments.len() > 256 {
157 return Err(TestError::InvalidInput(
158 "test suite requires a program and at most 256 arguments".into(),
159 ));
160 }
161 if self.suites.len() == MAX_TEST_SUITES && !self.suites.contains_key(&suite.id) {
162 return Err(TestError::InvalidInput(format!(
163 "test suite limit is {MAX_TEST_SUITES}"
164 )));
165 }
166 self.suites.insert(suite.id.clone(), suite);
167 Ok(())
168 }
169
170 pub fn start(
171 &mut self,
172 run_id: &str,
173 suite_id: &str,
174 actor_id: &str,
175 workspace_revision: u64,
176 timeout: Option<Duration>,
177 ) -> TestResult<TestRun> {
178 validate_name(run_id, "test run")?;
179 validate_name(actor_id, "test actor")?;
180 if self.running.contains_key(run_id)
181 || self.completed.iter().any(|result| result.id == run_id)
182 {
183 return Err(TestError::InvalidInput(format!(
184 "test run {run_id} already exists"
185 )));
186 }
187 if self.running.len() + self.completed.len() >= MAX_TEST_RUNS {
188 return Err(TestError::InvalidInput(format!(
189 "test run history limit is {MAX_TEST_RUNS}"
190 )));
191 }
192 let suite = self
193 .suites
194 .get(suite_id)
195 .ok_or_else(|| TestError::NotFound(format!("test suite {suite_id}")))?;
196 let timeout = timeout.unwrap_or(DEFAULT_TEST_TIMEOUT);
197 if timeout.is_zero() || timeout > Duration::from_secs(3600) {
198 return Err(TestError::InvalidInput(
199 "test timeout must be between 1 ms and 1 hour".into(),
200 ));
201 }
202 let mut child = Command::new(&suite.program)
203 .args(&suite.arguments)
204 .current_dir(&self.root)
205 .stdin(Stdio::null())
206 .stdout(Stdio::piped())
207 .stderr(Stdio::piped())
208 .spawn()?;
209 let stdout = child
210 .stdout
211 .take()
212 .ok_or_else(|| TestError::Process("test stdout was unavailable".into()))?;
213 let stderr = child
214 .stderr
215 .take()
216 .ok_or_else(|| TestError::Process("test stderr was unavailable".into()))?;
217 let snapshot = TestRun {
218 id: run_id.to_string(),
219 suite_id: suite_id.to_string(),
220 actor_id: actor_id.to_string(),
221 workspace_revision,
222 started_at_ms: unix_time_ms(),
223 duration_ms: None,
224 state: TestRunState::Running,
225 exit_code: None,
226 output: String::new(),
227 output_truncated: false,
228 cases: Vec::new(),
229 };
230 self.running.insert(
231 run_id.to_string(),
232 RunningTest {
233 snapshot: snapshot.clone(),
234 child,
235 stdout: Some(read_output(stdout)),
236 stderr: Some(read_output(stderr)),
237 started: Instant::now(),
238 timeout,
239 requested_stop: None,
240 },
241 );
242 Ok(snapshot)
243 }
244
245 pub fn cancel(&mut self, run_id: &str) -> TestResult<()> {
246 let running = self
247 .running
248 .get_mut(run_id)
249 .ok_or_else(|| TestError::NotFound(format!("running test {run_id}")))?;
250 running.requested_stop = Some(TestRunState::Cancelled);
251 if running.child.try_wait()?.is_none() {
252 running.child.kill()?;
253 }
254 Ok(())
255 }
256
257 pub fn poll(&mut self) -> TestResult<Vec<TestRun>> {
258 let ids = self.running.keys().cloned().collect::<Vec<_>>();
259 let mut finished = Vec::new();
260 for id in ids {
261 let running = self.running.get_mut(&id).expect("run ID came from map");
262 if running.started.elapsed() >= running.timeout && running.requested_stop.is_none() {
263 running.requested_stop = Some(TestRunState::TimedOut);
264 if running.child.try_wait()?.is_none() {
265 running.child.kill()?;
266 }
267 }
268 if let Some(status) = running.child.try_wait()? {
269 let running = self.running.remove(&id).expect("finished run exists");
270 let result = finish_run(running, status)?;
271 self.completed.push(result.clone());
272 finished.push(result);
273 }
274 }
275 Ok(finished)
276 }
277
278 pub fn running(&self) -> impl Iterator<Item = &TestRun> {
279 self.running.values().map(|running| &running.snapshot)
280 }
281
282 pub fn results(&self) -> impl DoubleEndedIterator<Item = &TestRun> {
283 self.completed.iter()
284 }
285
286 pub fn result(&self, run_id: &str) -> Option<&TestRun> {
287 self.completed.iter().find(|result| result.id == run_id)
288 }
289
290 pub fn affected_suites<'a>(
291 &'a self,
292 changed_paths: impl IntoIterator<Item = &'a str>,
293 ) -> Vec<&'a TestSuite> {
294 let extensions = changed_paths
295 .into_iter()
296 .filter_map(|path| Path::new(path).extension().and_then(|value| value.to_str()))
297 .collect::<BTreeSet<_>>();
298 self.suites
299 .values()
300 .filter(|suite| match suite.framework {
301 TestFramework::Cargo => extensions.contains("rs") || extensions.contains("toml"),
302 TestFramework::Pytest => extensions.contains("py"),
303 TestFramework::Npm | TestFramework::Pnpm | TestFramework::Yarn => {
304 ["js", "jsx", "ts", "tsx", "json"]
305 .iter()
306 .any(|extension| extensions.contains(extension))
307 }
308 TestFramework::Go => extensions.contains("go"),
309 TestFramework::Custom => true,
310 })
311 .collect()
312 }
313
314 pub fn watch(&mut self, suite_id: &str, current_revision: u64) -> TestResult<()> {
315 if !self.suites.contains_key(suite_id) {
316 return Err(TestError::NotFound(format!("test suite {suite_id}")));
317 }
318 self.watched.insert(suite_id.to_string(), current_revision);
319 Ok(())
320 }
321
322 pub fn unwatch(&mut self, suite_id: &str) -> bool {
323 self.watched.remove(suite_id).is_some()
324 }
325
326 pub fn changed_watches(&mut self, current_revision: u64) -> Vec<String> {
327 let mut changed = Vec::new();
328 for (suite, revision) in &mut self.watched {
329 if *revision != current_revision {
330 *revision = current_revision;
331 changed.push(suite.clone());
332 }
333 }
334 changed
335 }
336}
337
338impl Drop for TestService {
339 fn drop(&mut self) {
340 for running in self.running.values_mut() {
341 if running.child.try_wait().ok().flatten().is_none() {
342 let _ = running.child.kill();
343 let _ = running.child.wait();
344 }
345 join_reader(running.stdout.take());
346 join_reader(running.stderr.take());
347 }
348 }
349}
350
351fn discover_suites(root: &Path) -> TestResult<Vec<TestSuite>> {
352 let mut suites = Vec::new();
353 if root.join("Cargo.toml").is_file() {
354 suites.push(TestSuite {
355 id: "cargo".into(),
356 name: "Cargo tests".into(),
357 framework: TestFramework::Cargo,
358 program: "cargo".into(),
359 arguments: vec!["test".into()],
360 source: root.join("Cargo.toml"),
361 });
362 }
363 let python_source = ["pyproject.toml", "pytest.ini", "setup.cfg"]
364 .into_iter()
365 .map(|name| root.join(name))
366 .find(|path| path.is_file());
367 if let Some(source) = python_source {
368 suites.push(TestSuite {
369 id: "pytest".into(),
370 name: "Python tests".into(),
371 framework: TestFramework::Pytest,
372 program: if cfg!(windows) { "python" } else { "python3" }.into(),
373 arguments: vec!["-m".into(), "pytest".into()],
374 source,
375 });
376 }
377 let package_json = root.join("package.json");
378 if package_json.is_file() {
379 let value: serde_json::Value = serde_json::from_slice(&std::fs::read(&package_json)?)
380 .map_err(|error| TestError::InvalidInput(format!("invalid package.json: {error}")))?;
381 if value
382 .get("scripts")
383 .and_then(|scripts| scripts.get("test"))
384 .and_then(serde_json::Value::as_str)
385 .is_some()
386 {
387 let (framework, program, arguments) = if root.join("pnpm-lock.yaml").is_file() {
388 (TestFramework::Pnpm, "pnpm", vec!["test".into()])
389 } else if root.join("yarn.lock").is_file() {
390 (TestFramework::Yarn, "yarn", vec!["test".into()])
391 } else {
392 (TestFramework::Npm, "npm", vec!["test".into()])
393 };
394 suites.push(TestSuite {
395 id: "node".into(),
396 name: "Node tests".into(),
397 framework,
398 program: program.into(),
399 arguments,
400 source: package_json,
401 });
402 }
403 }
404 if root.join("go.mod").is_file() {
405 suites.push(TestSuite {
406 id: "go".into(),
407 name: "Go tests".into(),
408 framework: TestFramework::Go,
409 program: "go".into(),
410 arguments: vec!["test".into(), "./...".into()],
411 source: root.join("go.mod"),
412 });
413 }
414 suites.truncate(MAX_TEST_SUITES);
415 Ok(suites)
416}
417
418fn finish_run(mut running: RunningTest, status: ExitStatus) -> TestResult<TestRun> {
419 let stdout = join_output(running.stdout.take())?;
420 let stderr = join_output(running.stderr.take())?;
421 let mut combined = stdout.0;
422 if !combined.is_empty() && !stderr.0.is_empty() {
423 combined.push(b'\n');
424 }
425 let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(combined.len());
426 combined.extend_from_slice(&stderr.0[..stderr.0.len().min(remaining)]);
427 let truncated = stdout.1 || stderr.1 || stderr.0.len() > remaining;
428 let output = String::from_utf8_lossy(&combined).into_owned();
429 running.snapshot.duration_ms = Some(running.started.elapsed().as_millis() as u64);
430 running.snapshot.exit_code = status.code();
431 running.snapshot.state = running.requested_stop.unwrap_or(if status.success() {
432 TestRunState::Passed
433 } else {
434 TestRunState::Failed
435 });
436 running.snapshot.cases = parse_test_cases(&output);
437 running.snapshot.output = output;
438 running.snapshot.output_truncated = truncated;
439 Ok(running.snapshot)
440}
441
442fn read_output(mut stream: impl Read + Send + 'static) -> OutputReader {
443 std::thread::spawn(move || {
444 let mut retained = Vec::new();
445 let mut truncated = false;
446 let mut chunk = [0_u8; 8192];
447 loop {
448 let read = stream.read(&mut chunk)?;
449 if read == 0 {
450 break;
451 }
452 let remaining = MAX_TEST_OUTPUT_BYTES.saturating_sub(retained.len());
453 retained.extend_from_slice(&chunk[..read.min(remaining)]);
454 truncated |= read > remaining;
455 }
456 Ok((retained, truncated))
457 })
458}
459
460fn join_output(reader: Option<OutputReader>) -> TestResult<(Vec<u8>, bool)> {
461 reader
462 .ok_or_else(|| TestError::Process("test output reader was unavailable".into()))?
463 .join()
464 .map_err(|_| TestError::Process("test output reader panicked".into()))?
465 .map_err(TestError::Io)
466}
467
468fn join_reader(reader: Option<OutputReader>) {
469 if let Some(reader) = reader {
470 let _ = reader.join();
471 }
472}
473
474fn parse_test_cases(output: &str) -> Vec<TestCaseResult> {
475 output
476 .lines()
477 .filter_map(|line| {
478 let rest = line.strip_prefix("test ")?;
479 let (name, state) = if let Some(name) = rest.strip_suffix(" ... ok") {
480 (name, TestCaseState::Passed)
481 } else if let Some(name) = rest.strip_suffix(" ... FAILED") {
482 (name, TestCaseState::Failed)
483 } else {
484 (rest.strip_suffix(" ... ignored")?, TestCaseState::Ignored)
485 };
486 Some(TestCaseResult {
487 name: name.to_string(),
488 state,
489 })
490 })
491 .take(100_000)
492 .collect()
493}
494
495fn validate_name(name: &str, description: &str) -> TestResult<()> {
496 if name.is_empty()
497 || name.len() > 128
498 || !name
499 .chars()
500 .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
501 {
502 return Err(TestError::InvalidInput(format!(
503 "{description} must be 1..=128 ASCII letters, digits, '-', '_' or '.'"
504 )));
505 }
506 Ok(())
507}
508
509fn unix_time_ms() -> u64 {
510 SystemTime::now()
511 .duration_since(UNIX_EPOCH)
512 .unwrap_or_default()
513 .as_millis()
514 .try_into()
515 .unwrap_or(u64::MAX)
516}
517
518#[cfg(test)]
519mod tests {
520 use super::*;
521 use std::sync::atomic::{AtomicU64, Ordering};
522
523 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
524
525 fn root() -> PathBuf {
526 let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
527 let root = std::env::temp_dir().join(format!(
528 "glass-test-service-{}-{sequence}",
529 std::process::id()
530 ));
531 std::fs::create_dir_all(&root).unwrap();
532 root
533 }
534
535 #[test]
536 fn discovers_supported_project_test_suites() {
537 let root = root();
538 std::fs::write(
539 root.join("Cargo.toml"),
540 "[package]\nname='x'\nversion='0.1.0'\n",
541 )
542 .unwrap();
543 std::fs::write(root.join("pyproject.toml"), "[project]\nname='x'\n").unwrap();
544 std::fs::write(
545 root.join("package.json"),
546 r#"{"scripts":{"test":"vitest"}}"#,
547 )
548 .unwrap();
549 std::fs::write(root.join("pnpm-lock.yaml"), "lockfileVersion: 9\n").unwrap();
550 std::fs::write(root.join("go.mod"), "module example.invalid/x\n").unwrap();
551 let service = TestService::discover(&root).unwrap();
552 let frameworks = service
553 .suites()
554 .map(|suite| suite.framework)
555 .collect::<Vec<_>>();
556 assert_eq!(
557 frameworks,
558 vec![
559 TestFramework::Cargo,
560 TestFramework::Go,
561 TestFramework::Pnpm,
562 TestFramework::Pytest
563 ]
564 );
565 std::fs::remove_dir_all(root).unwrap();
566 }
567
568 #[test]
569 fn resident_test_run_records_actor_revision_output_and_result() {
570 let root = root();
571 let mut service = TestService::discover(&root).unwrap();
572 service
573 .register(TestSuite {
574 id: "version".into(),
575 name: "Rust compiler version".into(),
576 framework: TestFramework::Custom,
577 program: "rustc".into(),
578 arguments: vec!["--version".into()],
579 source: root.join("glass.toml"),
580 })
581 .unwrap();
582 service
583 .start(
584 "run-1",
585 "version",
586 "agent-tester",
587 7,
588 Some(Duration::from_secs(10)),
589 )
590 .unwrap();
591 let deadline = Instant::now() + Duration::from_secs(10);
592 let result = loop {
593 if let Some(result) = service.poll().unwrap().into_iter().next() {
594 break result;
595 }
596 assert!(Instant::now() < deadline);
597 std::thread::sleep(Duration::from_millis(10));
598 };
599 assert_eq!(result.actor_id, "agent-tester");
600 assert_eq!(result.workspace_revision, 7);
601 assert_eq!(result.state, TestRunState::Passed);
602 assert!(result.output.contains("rustc"));
603 assert_eq!(service.result("run-1"), Some(&result));
604 std::fs::remove_dir_all(root).unwrap();
605 }
606
607 #[test]
608 fn case_parser_and_watch_revisions_are_structured() {
609 let cases = parse_test_cases(
610 "test alpha ... ok\ntest beta ... FAILED\ntest gamma ... ignored\nnoise\n",
611 );
612 assert_eq!(cases.len(), 3);
613 assert_eq!(cases[1].state, TestCaseState::Failed);
614
615 let root = root();
616 std::fs::write(
617 root.join("Cargo.toml"),
618 "[package]\nname='x'\nversion='0.1.0'\n",
619 )
620 .unwrap();
621 let mut service = TestService::discover(&root).unwrap();
622 service.watch("cargo", 1).unwrap();
623 assert!(service.changed_watches(1).is_empty());
624 assert_eq!(service.changed_watches(2), vec!["cargo"]);
625 assert!(service.changed_watches(2).is_empty());
626 std::fs::remove_dir_all(root).unwrap();
627 }
628}