use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use crate::config::LitConfig;
use crate::integrated::parse_integrated_test_script;
use crate::shell::executor::wait_for_process_timeout;
use crate::shell::{execute_script_node, ShParser, ShellEnv};
use crate::substitution::{apply_substitutions, get_default_substitutions};
use crate::test::{ResultCode, Test, TestResult};
use std::process::Command as SysCommand;
use std::time::Duration;
pub struct RunContext {
pub cfg: LitConfig,
}
static SHELL_SCRIPT_COUNTER: AtomicU64 = AtomicU64::new(0);
impl RunContext {
pub fn new(cfg: LitConfig) -> Self {
Self { cfg }
}
pub fn run_once(&self, test: &mut Test) -> TestResult {
let parsed = match parse_integrated_test_script(&test.source_path()) {
Ok(parsed) => parsed,
Err(message) => {
return TestResult::new(ResultCode::UNRESOLVED, message.to_string());
}
};
test.xfails = parsed.xfails;
test.requires = parsed.requires;
test.unsupported = parsed.unsupported;
test.allowed_retries = parsed.allow_retries;
if let Some(result) = enforce_preconditions(test) {
return result;
}
let (tmp_dir, tmp_base) = get_temp_paths(test);
let mut substitutions = test.config.substitutions.clone();
substitutions.extend(test.config.default_substitutions.clone());
let mut merged_substitutions = get_default_substitutions(
&test.source_path(),
&tmp_dir,
&tmp_base,
self.cfg.normalize_slashes,
&substitutions,
);
let features: HashSet<String> = test.config.available_features.iter().cloned().collect();
let commands = match apply_substitutions(
&parsed.directives,
&mut merged_substitutions,
&features,
test.config.recursive_expansion_limit,
) {
Ok(commands) => commands,
Err(message) => {
return TestResult::new(ResultCode::UNRESOLVED, message.to_string());
}
};
let execdir = test
.exec_path()
.parent()
.map_or_else(|| PathBuf::from("."), PathBuf::from);
let mut env = test.config.environment.clone();
env.insert("LIT_CURRENT_TESTCASE".to_string(), test.get_full_name());
let mut shell_env = ShellEnv::new(execdir, env, test.config.normalize_slashes);
let mut final_exit_code = 0;
let mut timed_out = false;
let mut output = if test.config.run_with_external_shell {
let (exit_code, timed_out_result, script_output) = match execute_external_lit_script(
&commands,
&shell_env,
test.config.max_individual_test_time,
test.config.pipefail,
) {
Ok(values) => values,
Err(err) => {
return TestResult::new(
ResultCode::UNRESOLVED,
format!(
"command execution failed in {}: {err}",
test.get_full_name()
),
);
}
};
final_exit_code = exit_code;
timed_out = timed_out_result;
script_output
} else {
let mut external_output = String::new();
for line in commands {
if line.trim().is_empty() {
continue;
}
let mut parser = ShParser::new(&line, false, test.config.pipefail);
let ast = match parser.parse() {
Ok(ast) => ast,
Err(message) => {
return TestResult::new(
ResultCode::UNRESOLVED,
format!(
"command parse failed in {}: {message}",
test.get_full_name()
),
);
}
};
let result = match execute_script_node(
&ast,
&mut shell_env,
test.config.max_individual_test_time,
) {
Ok(script) => script,
Err(message) => {
return TestResult::new(
ResultCode::UNRESOLVED,
format!(
"command execution failed in {}: {message}",
test.get_full_name()
),
);
}
};
final_exit_code = result.exit_code;
timed_out = result.timed_out;
external_output.push_str(&format!("$ {line}\n"));
if !result.stdout.trim().is_empty() {
external_output.push_str(&result.stdout);
}
if !result.stderr.trim().is_empty() {
external_output.push_str(&result.stderr);
}
if timed_out || result.exit_code != 0 {
break;
}
}
external_output
};
let status = if timed_out {
ResultCode::TIMEOUT
} else if final_exit_code == 0 {
ResultCode::PASS
} else {
ResultCode::FAIL
};
if timed_out {
output = format!("Timeout.\n{output}");
}
TestResult {
code: status,
output: format!("Exit Code: {final_exit_code}\n{output}"),
elapsed: None,
attempts: 1,
max_allowed_attempts: Some(1),
test_updater_outputs: Vec::new(),
metrics: HashMap::new(),
micro_results: HashMap::new(),
}
}
pub fn run_with_retries(&self, test: &mut Test) -> TestResult {
if self.cfg.no_execute {
let mut result = TestResult::new(ResultCode::PASS, "No execute requested\n");
result.max_allowed_attempts = Some(test.allowed_retries + 1);
test.set_result(result.clone());
return result;
}
let attempts = test.allowed_retries + 1;
let mut last = TestResult::new(ResultCode::PASS, String::new());
let mut test_updater_outputs = Vec::new();
let mut seen_failure = false;
for attempt in 0..attempts {
let result = self.run_once(test);
test_updater_outputs.push(result.output.clone());
if result.code != ResultCode::FAIL {
last = result.clone();
break;
}
seen_failure = true;
last = result.clone();
if attempt + 1 == attempts {
break;
}
}
let expects_to_fail = test.is_expected_to_fail();
if test.allowed_retries > 0
&& !expects_to_fail
&& last.code == ResultCode::PASS
&& seen_failure
{
last.code = if test_updater_outputs.iter().any(|out| !out.is_empty()) {
ResultCode::FIXED
} else {
ResultCode::FLAKYPASS
};
}
last.max_allowed_attempts = Some(attempts);
if test.allowed_retries > 0 {
if seen_failure && last.code != ResultCode::PASS {
last.attempts = attempts;
} else {
last.attempts = test_updater_outputs.len();
}
} else {
last.attempts = test_updater_outputs.len();
}
last.test_updater_outputs = test_updater_outputs;
test.set_result(last.clone());
if let Some(mapped) = test.result.clone() {
last = mapped;
}
last
}
}
fn execute_external_lit_script(
commands: &[String],
shell_env: &ShellEnv,
timeout: Option<Duration>,
pipefail: bool,
) -> Result<(i32, bool, String), String> {
if commands.is_empty() {
return Ok((0, false, String::new()));
}
let script = build_external_script(commands, pipefail);
let pid = std::process::id();
let serial = SHELL_SCRIPT_COUNTER.fetch_add(1, Ordering::SeqCst);
let mut stdout_path = Path::new(&std::env::temp_dir()).to_path_buf();
let mut stderr_path = Path::new(&std::env::temp_dir()).to_path_buf();
stdout_path.push(format!("litnik-ext-shell-{}-{}.out", pid, serial));
stderr_path.push(format!("litnik-ext-shell-{}-{}.err", pid, serial));
let stdout = File::create(&stdout_path)
.map_err(|error| format!("failed to create external shell stdout file: {error}"))?;
let stderr = File::create(&stderr_path)
.map_err(|error| format!("failed to create external shell stderr file: {error}"))?;
let mut command = SysCommand::new("/bin/sh");
command
.current_dir(&shell_env.cwd)
.env_clear()
.envs(&shell_env.env)
.arg("-c")
.arg(script)
.stdout(std::process::Stdio::from(stdout))
.stderr(std::process::Stdio::from(stderr));
#[cfg(unix)]
{
use nix::unistd::{setpgid, Pid};
use std::os::unix::process::CommandExt;
unsafe {
command.pre_exec(move || {
setpgid(Pid::from_raw(0), Pid::from_raw(0)).map_err(io::Error::other)?;
Ok(())
});
}
}
let mut process = command
.spawn()
.map_err(|error| format!("failed to spawn external shell: {error}"))?;
let (code, timed_out) = wait_for_process_timeout(&mut process, timeout)
.map_err(|error| format!("failed while waiting for external shell: {error}"))?;
let mut stdout = fs::read_to_string(&stdout_path)
.map_err(|error| format!("failed to read external shell stdout: {error}"))?;
let stderr = fs::read_to_string(&stderr_path)
.map_err(|error| format!("failed to read external shell stderr: {error}"))?;
let _ = fs::remove_file(&stdout_path);
let _ = fs::remove_file(&stderr_path);
if !stderr.is_empty() {
stdout.push_str(&stderr);
}
Ok((code, timed_out, stdout))
}
fn build_external_script(commands: &[String], pipefail: bool) -> String {
let mut script = String::new();
if pipefail {
script.push_str("set -o pipefail;\n");
}
let mut active_count = 0usize;
for line in commands {
if line.trim().is_empty() {
continue;
}
active_count += 1;
}
if active_count == 0 {
return "exit 0\n".to_string();
}
for (idx, line) in commands.iter().enumerate() {
if line.trim().is_empty() {
continue;
}
script.push_str("printf '%s\\n' '$ ");
script.push_str(&shell_escape_single_quotes(line));
script.push_str("'\n");
script.push_str(line);
script.push('\n');
script.push_str("lit_status=$?\n");
script.push_str("if [ \"${lit_status}\" -ne 0 ]; then exit ${lit_status}; fi\n");
if idx + 1 != commands.len() {
script.push('\n');
}
}
script.push_str("exit ${lit_status}\n");
script
}
fn shell_escape_single_quotes(value: &str) -> String {
value.replace('\'', "'\\''")
}
fn enforce_preconditions(test: &mut Test) -> Option<TestResult> {
let missing = test.missing_required_features();
if !missing.is_empty() {
let message = format!("Missing required features: {}", missing.join(", "));
return Some(TestResult::new(ResultCode::UNSUPPORTED, message));
}
let unsupported = test.unsupported_features();
if !unsupported.is_empty() {
let message = format!("Unsupported features: {}", unsupported.join(", "));
return Some(TestResult::new(ResultCode::UNSUPPORTED, message));
}
if !test.is_within_feature_limits() {
return Some(TestResult::new(
ResultCode::UNSUPPORTED,
"Outside feature limits".to_string(),
));
}
None
}
pub fn get_temp_paths(test: &Test) -> (PathBuf, PathBuf) {
let exec_path = test.exec_path();
let execdir = exec_path
.parent()
.map_or_else(|| PathBuf::from("."), PathBuf::from);
let tmp_dir = execdir.join("Output");
let tmp_base = tmp_dir.join(
exec_path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string()
+ ".tmp",
);
(tmp_dir, tmp_base)
}
pub fn run_discovered_tests(root: &std::path::Path, config: &LitConfig) -> Vec<Test> {
crate::discovery::discover_tests(root, config)
}
#[cfg(test)]
mod tests {
use std::fs;
use tempfile::tempdir;
use crate::config::LitConfig;
use crate::discovery;
use crate::test::ResultCode;
#[test]
fn runs_allow_retries_and_no_execute() {
let dir = tempdir().unwrap();
std::fs::write(
dir.path().join("a.mlir"),
"// RUN: true\n// ALLOW_RETRIES: 2\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir").no_execute(true);
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::PASS);
assert_eq!(result.attempts, 1);
}
#[test]
fn internal_shell_respects_command_failure_and_stops() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("fail.mlir"),
"// RUN: echo hi\n// RUN: false\n// RUN: echo should-not-run\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::FAIL);
assert!(result.output.contains("$ echo hi"));
assert!(result.output.contains("hi"));
assert!(!result.output.contains("should-not-run"));
}
#[test]
fn command_timeout_is_reported() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("timeout.mlir"),
"// RUN: /bin/sh -c 'sleep 2'\n",
)
.unwrap();
let config = LitConfig::new(dir.path())
.suffix(".mlir")
.max_individual_test_time(1);
let tests = discovery::discover_tests(dir.path(), &config);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, crate::test::ResultCode::TIMEOUT);
}
#[test]
fn external_shell_respects_command_failure_and_stops() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("fail.mlir"),
"// RUN: printf '%s\\n' hi\n// RUN: false\n// RUN: echo should-not-run\n",
)
.unwrap();
let config = LitConfig::new(dir.path())
.suffix(".mlir")
.use_external_shell(true);
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::FAIL);
assert!(result.output.contains("$ printf"));
assert!(result.output.contains("hi"));
assert!(!result.output.contains("should-not-run"));
}
#[test]
fn xfail_is_mapped_to_xfail() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("xfail.mlir"),
"// RUN: false\n// XFAIL: *\n",
)
.unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::XFAIL);
}
#[test]
fn xpass_is_mapped_to_xpass() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("xpass.mlir"), "// RUN: true\n// XFAIL: *\n").unwrap();
let config = LitConfig::new(dir.path()).suffix(".mlir");
let tests = discovery::discover_tests(dir.path(), &config);
assert_eq!(tests.len(), 1);
let runner = super::RunContext::new(config);
let mut test = tests[0].clone();
let result = runner.run_with_retries(&mut test);
assert_eq!(result.code, ResultCode::XPASS);
}
}