litnik 0.2.1

An embedded Lit and FileCheck runner for compiler test suites.
Documentation
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};

use crate::config::LitConfig;
use crate::filecheck::FileCheckCommand;
use crate::integrated::parse_integrated_test_script;
use crate::shell::{execute_script_node, ShParser, ShellEnv};
use crate::substitution::{apply_substitutions, get_default_substitutions};
use crate::test::{ResultCode, Test, TestResult};

pub(crate) struct RunContext {
    config: LitConfig,
}

enum Preparation {
    Commands(Vec<String>),
    Result(TestResult),
}

impl RunContext {
    pub(crate) fn new(config: LitConfig) -> Self {
        Self { config }
    }

    pub(crate) fn run_with_retries(&self, test: &mut Test) -> TestResult {
        test.config = self.config.to_test_config();
        test.allowed_retries = test.config.test_retry_attempts;
        let started = Instant::now();
        let commands = match self.prepare(test) {
            Preparation::Commands(commands) => commands,
            Preparation::Result(result) => return finish_result(test, result, started.elapsed()),
        };

        if self.config.no_execute {
            let mut output = String::from("Validated commands:\n");
            for command in commands {
                output.push_str("$ ");
                output.push_str(&command);
                output.push('\n');
            }
            return finish_result(
                test,
                TestResult::new(ResultCode::PASS, output),
                started.elapsed(),
            );
        }

        let max_attempts = test.allowed_retries.saturating_add(1);
        let mut attempt_outputs = Vec::with_capacity(max_attempts);
        let mut final_result = TestResult::new(ResultCode::UNRESOLVED, "test did not run");
        let mut saw_failed_attempt = false;

        for _ in 0..max_attempts {
            let result = self.execute_attempt(test, &commands);
            attempt_outputs.push(result.output.clone());
            if result.code == ResultCode::FAIL {
                saw_failed_attempt = true;
                final_result = result;
                continue;
            }
            final_result = result;
            break;
        }

        if final_result.code == ResultCode::PASS && saw_failed_attempt {
            final_result.code = ResultCode::FLAKYPASS;
        }
        final_result.attempt_outputs = attempt_outputs;
        finish_result(test, final_result, started.elapsed())
    }

    fn prepare(&self, test: &mut Test) -> Preparation {
        let parsed = match parse_integrated_test_script(&test.source_path()) {
            Ok(parsed) => parsed,
            Err(error) => {
                return Preparation::Result(TestResult::new(
                    ResultCode::UNRESOLVED,
                    error.to_string(),
                ));
            }
        };

        test.xfails = parsed.xfails;
        test.requires = parsed.requires;
        test.unsupported = parsed.unsupported;
        if let Some(allowed_retries) = parsed.allow_retries {
            test.allowed_retries = allowed_retries;
        }

        match enforce_preconditions(test) {
            Ok(Some(result)) => return Preparation::Result(result),
            Ok(None) => {}
            Err(error) => {
                return Preparation::Result(TestResult::new(ResultCode::UNRESOLVED, error));
            }
        }

        let (temp_dir, temp_base) = get_temp_paths(test);
        let mut substitutions = test.config.substitutions.clone();
        substitutions.extend(test.config.default_substitutions.clone());
        let mut substitutions = get_default_substitutions(
            &test.source_path(),
            &temp_dir,
            &temp_base,
            self.config.normalize_slashes,
            &substitutions,
        );
        match apply_substitutions(
            &parsed.directives,
            &mut substitutions,
            &test.config.available_features,
            test.config.recursive_expansion_limit,
        ) {
            Ok(commands) => Preparation::Commands(commands),
            Err(error) => {
                Preparation::Result(TestResult::new(ResultCode::UNRESOLVED, error.to_string()))
            }
        }
    }

    fn execute_attempt(&self, test: &Test, commands: &[String]) -> TestResult {
        let exec_dir = test
            .exec_path()
            .parent()
            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
        let (temp_dir, _) = get_temp_paths(test);
        if let Err(error) =
            fs::create_dir_all(&exec_dir).and_then(|()| fs::create_dir_all(&temp_dir))
        {
            return TestResult::new(
                ResultCode::UNRESOLVED,
                format!(
                    "cannot create execution directory '{}': {error}",
                    exec_dir.display()
                ),
            );
        }

        let mut environment = test.config.environment.clone();
        environment.insert("LIT_CURRENT_TESTCASE".to_string(), test.get_full_name());
        let mut shell =
            ShellEnv::from_environment(exec_dir, environment, test.config.normalize_slashes);
        for (name, path) in &test.config.tools {
            shell.register_tool(name.clone(), path.clone());
        }
        shell.register_command("FileCheck", Arc::new(FileCheckCommand));

        let deadline = test
            .config
            .max_individual_test_time
            .and_then(|duration| Instant::now().checked_add(duration));
        let mut final_exit_code = 0;
        let mut timed_out = false;
        let mut output = String::new();

        for line in commands.iter().filter(|line| !line.trim().is_empty()) {
            let remaining = match deadline {
                Some(deadline) => match deadline.checked_duration_since(Instant::now()) {
                    Some(remaining) if !remaining.is_zero() => Some(remaining),
                    _ => {
                        timed_out = true;
                        break;
                    }
                },
                None => None,
            };
            let mut parser = ShParser::new(line, cfg!(windows), test.config.pipefail);
            let script = match parser.parse() {
                Ok(script) => script,
                Err(error) => {
                    return TestResult::new(
                        ResultCode::UNRESOLVED,
                        format!("command parse failed in {}: {error}", test.get_full_name()),
                    );
                }
            };
            let result = match execute_script_node(&script, &mut shell, remaining) {
                Ok(result) => result,
                Err(error) => {
                    final_exit_code = 127;
                    output.push_str("$ ");
                    output.push_str(line);
                    output.push('\n');
                    output.push_str(&error);
                    output.push('\n');
                    break;
                }
            };

            final_exit_code = result.exit_code;
            timed_out = result.timed_out;
            output.push_str("$ ");
            output.push_str(line);
            output.push('\n');
            output.push_str(&result.stdout);
            output.push_str(&result.stderr);
            if timed_out || final_exit_code != 0 {
                break;
            }
        }

        if timed_out {
            output.insert_str(0, "Timeout.\n");
        }
        let code = if timed_out {
            ResultCode::TIMEOUT
        } else if final_exit_code == 0 {
            ResultCode::PASS
        } else {
            ResultCode::FAIL
        };
        TestResult::new(code, format!("Exit Code: {final_exit_code}\n{output}"))
    }
}

fn finish_result(test: &mut Test, mut result: TestResult, elapsed: Duration) -> TestResult {
    result.elapsed = Some(elapsed);
    result.max_allowed_attempts = test.allowed_retries.saturating_add(1);
    if result.attempt_outputs.is_empty() {
        result.attempt_outputs.push(result.output.clone());
    }
    result.attempts = result.attempt_outputs.len();
    if let Err(error) = test.set_result(result) {
        let mut unresolved = TestResult::new(ResultCode::UNRESOLVED, error);
        unresolved.elapsed = Some(elapsed);
        test.result = Some(unresolved.clone());
        return unresolved;
    }
    test.result
        .clone()
        .expect("set_result stores the supplied result")
}

fn enforce_preconditions(test: &Test) -> Result<Option<TestResult>, String> {
    let missing = test.missing_required_features()?;
    if !missing.is_empty() {
        return Ok(Some(TestResult::new(
            ResultCode::UNSUPPORTED,
            format!("Missing required features: {}", missing.join(", ")),
        )));
    }
    let unsupported = test.unsupported_features()?;
    if !unsupported.is_empty() {
        return Ok(Some(TestResult::new(
            ResultCode::UNSUPPORTED,
            format!("Unsupported features: {}", unsupported.join(", ")),
        )));
    }
    if !test.is_within_feature_limits()? {
        return Ok(Some(TestResult::new(
            ResultCode::UNSUPPORTED,
            "Outside feature limits",
        )));
    }
    Ok(None)
}

pub(crate) fn get_temp_paths(test: &Test) -> (PathBuf, PathBuf) {
    let exec_path = test.exec_path();
    let exec_dir = exec_path
        .parent()
        .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
    let temp_dir = exec_dir.join("Output");
    let temp_base = temp_dir.join(exec_path.file_name().unwrap_or_default());
    (temp_dir, temp_base)
}

#[cfg(test)]
mod tests {
    use std::fs;

    use tempfile::tempdir;

    use crate::config::LitConfig;
    use crate::discovery;
    use crate::test::ResultCode;

    #[test]
    fn runs_allow_retries_and_no_execute() {
        let dir = tempdir().unwrap();
        std::fs::write(
            dir.path().join("a.mlir"),
            "// RUN: true\n// ALLOW_RETRIES: 2\n",
        )
        .unwrap();
        let config = LitConfig::new(dir.path()).suffix(".mlir").no_execute(true);
        let tests = discovery::discover_tests(dir.path(), &config);
        assert_eq!(tests.len(), 1);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);
        assert_eq!(result.code, crate::test::ResultCode::PASS);
        assert_eq!(result.attempts, 1);
    }

    #[test]
    fn internal_shell_respects_command_failure_and_stops() {
        let dir = tempdir().unwrap();
        fs::write(
            dir.path().join("fail.mlir"),
            "// RUN: echo hi\n// RUN: false\n// RUN: echo should-not-run\n",
        )
        .unwrap();

        let config = LitConfig::new(dir.path()).suffix(".mlir");
        let tests = discovery::discover_tests(dir.path(), &config);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);

        assert_eq!(result.code, crate::test::ResultCode::FAIL);
        assert!(result.output.contains("$ echo hi"));
        assert!(result.output.contains("hi"));
        assert!(!result.output.contains("should-not-run"));
    }

    #[test]
    fn command_timeout_is_reported() {
        let dir = tempdir().unwrap();
        fs::write(
            dir.path().join("timeout.mlir"),
            "// RUN: /bin/sh -c 'sleep 2'\n",
        )
        .unwrap();

        let config = LitConfig::new(dir.path())
            .suffix(".mlir")
            .max_individual_test_time(1);
        let tests = discovery::discover_tests(dir.path(), &config);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);

        assert_eq!(result.code, crate::test::ResultCode::TIMEOUT);
    }

    #[test]
    fn external_shell_respects_command_failure_and_stops() {
        let dir = tempdir().unwrap();
        fs::write(
            dir.path().join("fail.mlir"),
            "// RUN: printf '%s\\n' hi\n// RUN: false\n// RUN: echo should-not-run\n",
        )
        .unwrap();

        let config = LitConfig::new(dir.path())
            .suffix(".mlir")
            .use_external_shell(true);
        let tests = discovery::discover_tests(dir.path(), &config);
        assert_eq!(tests.len(), 1);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);

        assert_eq!(result.code, ResultCode::FAIL);
        assert!(result.output.contains("$ printf"));
        assert!(result.output.contains("hi"));
        assert!(!result.output.contains("should-not-run"));
    }

    #[test]
    fn xfail_is_mapped_to_xfail() {
        let dir = tempdir().unwrap();
        fs::write(
            dir.path().join("xfail.mlir"),
            "// RUN: false\n// XFAIL: *\n",
        )
        .unwrap();

        let config = LitConfig::new(dir.path()).suffix(".mlir");
        let tests = discovery::discover_tests(dir.path(), &config);
        assert_eq!(tests.len(), 1);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);

        assert_eq!(result.code, ResultCode::XFAIL);
    }

    #[test]
    fn xpass_is_mapped_to_xpass() {
        let dir = tempdir().unwrap();
        fs::write(dir.path().join("xpass.mlir"), "// RUN: true\n// XFAIL: *\n").unwrap();

        let config = LitConfig::new(dir.path()).suffix(".mlir");
        let tests = discovery::discover_tests(dir.path(), &config);
        assert_eq!(tests.len(), 1);
        let runner = super::RunContext::new(config);
        let mut test = tests[0].clone();
        let result = runner.run_with_retries(&mut test);

        assert_eq!(result.code, ResultCode::XPASS);
    }
}