use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::config::LitConfig;
use crate::filecheck::FileCheckCommand;
use crate::integrated::parse_integrated_test_script;
use crate::shell::{execute_script_node, ShParser, ShellEnv};
use crate::substitution::{apply_substitutions, get_default_substitutions};
use crate::test::{ResultCode, Test, TestResult};
pub(crate) struct RunContext {
config: LitConfig,
}
enum Preparation {
Commands(Vec<String>),
Result(TestResult),
}
impl RunContext {
pub(crate) fn new(config: LitConfig) -> Self {
Self { config }
}
pub(crate) fn run_with_retries(&self, test: &mut Test) -> TestResult {
test.config = self.config.to_test_config();
test.allowed_retries = test.config.test_retry_attempts;
let started = Instant::now();
let commands = match self.prepare(test) {
Preparation::Commands(commands) => commands,
Preparation::Result(result) => return finish_result(test, result, started.elapsed()),
};
if self.config.no_execute {
let mut output = String::from("Validated commands:\n");
for command in commands {
output.push_str("$ ");
output.push_str(&command);
output.push('\n');
}
return finish_result(
test,
TestResult::new(ResultCode::PASS, output),
started.elapsed(),
);
}
let max_attempts = test.allowed_retries.saturating_add(1);
let mut attempt_outputs = Vec::with_capacity(max_attempts);
let mut final_result = TestResult::new(ResultCode::UNRESOLVED, "test did not run");
let mut saw_failed_attempt = false;
for _ in 0..max_attempts {
let result = self.execute_attempt(test, &commands);
attempt_outputs.push(result.output.clone());
if result.code == ResultCode::FAIL {
saw_failed_attempt = true;
final_result = result;
continue;
}
final_result = result;
break;
}
if final_result.code == ResultCode::PASS && saw_failed_attempt {
final_result.code = ResultCode::FLAKYPASS;
}
final_result.attempt_outputs = attempt_outputs;
finish_result(test, final_result, started.elapsed())
}
fn prepare(&self, test: &mut Test) -> Preparation {
let parsed = match parse_integrated_test_script(&test.source_path()) {
Ok(parsed) => parsed,
Err(error) => {
return Preparation::Result(TestResult::new(
ResultCode::UNRESOLVED,
error.to_string(),
));
}
};
test.xfails = parsed.xfails;
test.requires = parsed.requires;
test.unsupported = parsed.unsupported;
if let Some(allowed_retries) = parsed.allow_retries {
test.allowed_retries = allowed_retries;
}
match enforce_preconditions(test) {
Ok(Some(result)) => return Preparation::Result(result),
Ok(None) => {}
Err(error) => {
return Preparation::Result(TestResult::new(ResultCode::UNRESOLVED, error));
}
}
let (temp_dir, temp_base) = get_temp_paths(test);
let mut substitutions = test.config.substitutions.clone();
substitutions.extend(test.config.default_substitutions.clone());
let mut substitutions = get_default_substitutions(
&test.source_path(),
&temp_dir,
&temp_base,
self.config.normalize_slashes,
&substitutions,
);
match apply_substitutions(
&parsed.directives,
&mut substitutions,
&test.config.available_features,
test.config.recursive_expansion_limit,
) {
Ok(commands) => Preparation::Commands(commands),
Err(error) => {
Preparation::Result(TestResult::new(ResultCode::UNRESOLVED, error.to_string()))
}
}
}
fn execute_attempt(&self, test: &Test, commands: &[String]) -> TestResult {
let exec_dir = test
.exec_path()
.parent()
.map_or_else(|| PathBuf::from("."), Path::to_path_buf);
let (temp_dir, _) = get_temp_paths(test);
if let Err(error) =
fs::create_dir_all(&exec_dir).and_then(|()| fs::create_dir_all(&temp_dir))
{
return TestResult::new(
ResultCode::UNRESOLVED,
format!(
"cannot create execution directory '{}': {error}",
exec_dir.display()
),
);
}
let mut environment = test.config.environment.clone();
environment.insert("LIT_CURRENT_TESTCASE".to_string(), test.get_full_name());
let mut shell =
ShellEnv::from_environment(exec_dir, environment, test.config.normalize_slashes);
for (name, path) in &test.config.tools {
shell.register_tool(name.clone(), path.clone());
}
shell.register_command("FileCheck", Arc::new(FileCheckCommand));
let deadline = test
.config
.max_individual_test_time
.and_then(|duration| Instant::now().checked_add(duration));
let mut final_exit_code = 0;
let mut timed_out = false;
let mut output = String::new();
for line in commands.iter().filter(|line| !line.trim().is_empty()) {
let remaining = match deadline {
Some(deadline) => match deadline.checked_duration_since(Instant::now()) {
Some(remaining) if !remaining.is_zero() => Some(remaining),
_ => {
timed_out = true;
break;
}
},
None => None,
};
let mut parser = ShParser::new(line, cfg!(windows), test.config.pipefail);
let script = match parser.parse() {
Ok(script) => script,
Err(error) => {
return TestResult::new(
ResultCode::UNRESOLVED,
format!("command parse failed in {}: {error}", test.get_full_name()),
);
}
};
let result = match execute_script_node(&script, &mut shell, remaining) {
Ok(result) => result,
Err(error) => {
final_exit_code = 127;
output.push_str("$ ");
output.push_str(line);
output.push('\n');
output.push_str(&error);
output.push('\n');
break;
}
};
final_exit_code = result.exit_code;
timed_out = result.timed_out;
output.push_str("$ ");
output.push_str(line);
output.push('\n');
output.push_str(&result.stdout);
output.push_str(&result.stderr);
if timed_out || final_exit_code != 0 {
break;
}
}
if timed_out {
output.insert_str(0, "Timeout.\n");
}
let code = if timed_out {
ResultCode::TIMEOUT
} else if final_exit_code == 0 {
ResultCode::PASS
} else {
ResultCode::FAIL
};
TestResult::new(code, format!("Exit Code: {final_exit_code}\n{output}"))
}
}
fn finish_result(test: &mut Test, mut result: TestResult, elapsed: Duration) -> TestResult {
result.elapsed = Some(elapsed);
result.max_allowed_attempts = test.allowed_retries.saturating_add(1);
if result.attempt_outputs.is_empty() {
result.attempt_outputs.push(result.output.clone());
}
result.attempts = result.attempt_outputs.len();
if let Err(error) = test.set_result(result) {
let mut unresolved = TestResult::new(ResultCode::UNRESOLVED, error);
unresolved.elapsed = Some(elapsed);
test.result = Some(unresolved.clone());
return unresolved;
}
test.result
.clone()
.expect("set_result stores the supplied result")
}
fn enforce_preconditions(test: &Test) -> Result<Option<TestResult>, String> {
let missing = test.missing_required_features()?;
if !missing.is_empty() {
return Ok(Some(TestResult::new(
ResultCode::UNSUPPORTED,
format!("Missing required features: {}", missing.join(", ")),
)));
}
let unsupported = test.unsupported_features()?;
if !unsupported.is_empty() {
return Ok(Some(TestResult::new(
ResultCode::UNSUPPORTED,
format!("Unsupported features: {}", unsupported.join(", ")),
)));
}
if !test.is_within_feature_limits()? {
return Ok(Some(TestResult::new(
ResultCode::UNSUPPORTED,
"Outside feature limits",
)));
}
Ok(None)
}
pub(crate) fn get_temp_paths(test: &Test) -> (PathBuf, PathBuf) {
let exec_path = test.exec_path();
let exec_dir = exec_path
.parent()
.map_or_else(|| PathBuf::from("."), Path::to_path_buf);
let temp_dir = exec_dir.join("Output");
let temp_base = temp_dir.join(exec_path.file_name().unwrap_or_default());
(temp_dir, temp_base)
}
#[cfg(test)]
mod tests {
use std::fs;
use tempfile::tempdir;
use crate::config::LitConfig;
use crate::discovery;
use crate::test::ResultCode;
#[test]
fn runs_allow_retries_and_no_execute() {
let dir = tempdir().unwrap();
std::fs::write(
dir.path().join("a.mlir"),
"// RUN: true\n// ALLOW_RETRIES: 2\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir").no_execute(true);
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::PASS);
assert_eq!(result.attempts, 1);
}
#[test]
fn internal_shell_respects_command_failure_and_stops() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("fail.mlir"),
"// RUN: echo hi\n// RUN: false\n// RUN: echo should-not-run\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::FAIL);
assert!(result.output.contains("$ echo hi"));
assert!(result.output.contains("hi"));
assert!(!result.output.contains("should-not-run"));
}
#[test]
fn command_timeout_is_reported() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("timeout.mlir"),
"// RUN: /bin/sh -c 'sleep 2'\n",
)
.unwrap();
let config = LitConfig::new(dir.path())
.suffix(".mlir")
.max_individual_test_time(1);
let tests = discovery::discover_tests(dir.path(), &config);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::TIMEOUT);
}
#[test]
fn external_shell_respects_command_failure_and_stops() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("fail.mlir"),
"// RUN: printf '%s\\n' hi\n// RUN: false\n// RUN: echo should-not-run\n",
)
.unwrap();
let config = LitConfig::new(dir.path())
.suffix(".mlir")
.use_external_shell(true);
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::FAIL);
assert!(result.output.contains("$ printf"));
assert!(result.output.contains("hi"));
assert!(!result.output.contains("should-not-run"));
}
#[test]
fn xfail_is_mapped_to_xfail() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("xfail.mlir"),
"// RUN: false\n// XFAIL: *\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::XFAIL);
}
#[test]
fn xpass_is_mapped_to_xpass() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("xpass.mlir"), "// RUN: true\n// XFAIL: *\n").unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::XPASS);
}
}