litnik 0.2.0

An embedded Lit and FileCheck runner for compiler test suites.
Documentation
//! Test identities, statuses, and results.

use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::time::Duration;

/// A Lit result category.
///
/// Use the associated constants to compare results. [`Self::is_failure()`]
/// reports whether the category makes the test harness fail.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ResultCode {
    name: &'static str,
    label: &'static str,
    is_failure: bool,
}

impl ResultCode {
    /// The suite configuration excluded the test.
    pub const EXCLUDED: Self = Self::new("EXCLUDED", "Excluded", false);
    /// The runner skipped the test before execution.
    pub const SKIPPED: Self = Self::new("SKIPPED", "Skipped", false);
    /// The current features do not support the test.
    pub const UNSUPPORTED: Self = Self::new("UNSUPPORTED", "Unsupported", false);
    /// The test completed successfully.
    pub const PASS: Self = Self::new("PASS", "Passed", false);
    /// The test passed after at least one failed attempt.
    pub const FLAKYPASS: Self = Self::new("FLAKYPASS", "Passed With Retry", false);
    /// A test update converted an earlier failure into a pass.
    pub const FIXED: Self = Self::new("FIXED", "Passed After Update", false);
    /// The test failed as specified by an active `XFAIL:` directive.
    pub const XFAIL: Self = Self::new("XFAIL", "Expectedly Failed", false);
    /// Litnik could not parse or prepare the test.
    pub const UNRESOLVED: Self = Self::new("UNRESOLVED", "Unresolved", true);
    /// The test exceeded its configured time limit.
    pub const TIMEOUT: Self = Self::new("TIMEOUT", "Timed Out", true);
    /// A command in the test failed.
    pub const FAIL: Self = Self::new("FAIL", "Failed", true);
    /// The test passed despite an active `XFAIL:` directive.
    pub const XPASS: Self = Self::new("XPASS", "Unexpectedly Passed", true);

    /// All result categories in stable display order.
    pub const ALL: [Self; 11] = [
        Self::EXCLUDED,
        Self::SKIPPED,
        Self::UNSUPPORTED,
        Self::PASS,
        Self::FLAKYPASS,
        Self::FIXED,
        Self::XFAIL,
        Self::UNRESOLVED,
        Self::TIMEOUT,
        Self::FAIL,
        Self::XPASS,
    ];

    const fn new(name: &'static str, label: &'static str, is_failure: bool) -> Self {
        Self {
            name,
            label,
            is_failure,
        }
    }

    /// Returns the uppercase Lit status name.
    pub const fn name(self) -> &'static str {
        self.name
    }

    /// Returns the short label used in harness output.
    pub const fn label(self) -> &'static str {
        self.label
    }

    /// Reports whether this status makes the suite fail.
    pub const fn is_failure(self) -> bool {
        self.is_failure
    }
}

/// The observable result of one Lit test.
#[derive(Debug, Clone)]
pub struct TestResult {
    /// The final Lit status.
    pub code: ResultCode,
    /// Command traces, process output, or a preparation error.
    pub output: String,
    pub(crate) elapsed: Option<Duration>,
    pub(crate) attempts: usize,
    pub(crate) max_allowed_attempts: usize,
    pub(crate) attempt_outputs: Vec<String>,
}

impl TestResult {
    pub(crate) fn new(code: ResultCode, output: impl Into<String>) -> Self {
        Self {
            code,
            output: output.into(),
            elapsed: None,
            attempts: 1,
            max_allowed_attempts: 1,
            attempt_outputs: Vec::new(),
        }
    }

    /// Reports whether the final status makes the suite fail.
    pub fn is_failure(&self) -> bool {
        self.code.is_failure()
    }

    /// Returns total runner time, including preparation, when it was recorded.
    pub fn elapsed(&self) -> Option<Duration> {
        self.elapsed
    }

    /// Returns the number of recorded attempts.
    ///
    /// A result that stops during preparation records one attempt.
    pub fn attempts(&self) -> usize {
        self.attempts
    }

    /// Returns the maximum attempts allowed for this test.
    pub fn max_allowed_attempts(&self) -> usize {
        self.max_allowed_attempts
    }

    /// Returns the captured output for each attempt in execution order.
    ///
    /// A result that stops during preparation contains its final diagnostic.
    pub fn attempt_outputs(&self) -> &[String] {
        &self.attempt_outputs
    }
}

/// One discovered test file and its suite-relative identity.
#[derive(Debug, Clone)]
pub struct Test {
    pub(crate) suite_name: String,
    pub(crate) source_root: PathBuf,
    pub(crate) exec_root: PathBuf,
    pub(crate) relative_path: PathBuf,
    pub(crate) config: TestConfig,
    pub(crate) xfails: Vec<String>,
    pub(crate) requires: Vec<String>,
    pub(crate) unsupported: Vec<String>,
    pub(crate) allowed_retries: usize,
    pub(crate) result: Option<TestResult>,
}

impl Test {
    pub(crate) fn new(
        suite_name: impl Into<String>,
        source_root: PathBuf,
        exec_root: PathBuf,
        relative_path: impl Into<PathBuf>,
        config: TestConfig,
    ) -> Self {
        let allowed_retries = config.test_retry_attempts;
        Self {
            suite_name: suite_name.into(),
            source_root,
            exec_root,
            relative_path: relative_path.into(),
            config,
            xfails: Vec::new(),
            requires: Vec::new(),
            unsupported: Vec::new(),
            allowed_retries,
            result: None,
        }
    }

    /// Returns the path to the source file.
    pub fn source_path(&self) -> PathBuf {
        self.source_root.join(&self.relative_path)
    }

    /// Returns the path reserved for this test below the execution root.
    pub fn exec_path(&self) -> PathBuf {
        self.exec_root.join(&self.relative_path)
    }

    /// Returns the path relative to the suite source root.
    pub fn relative_path(&self) -> &Path {
        &self.relative_path
    }

    /// Returns the harness name in `suite :: path` form.
    pub fn get_full_name(&self) -> String {
        format!("{} :: {}", self.suite_name, self.relative_path.display())
    }

    /// Returns the result after this test has run.
    pub fn result(&self) -> Option<&TestResult> {
        self.result.as_ref()
    }

    pub(crate) fn set_result(&mut self, mut result: TestResult) -> Result<(), String> {
        if self.is_expected_to_fail()? {
            if result.code == ResultCode::PASS {
                result.code = ResultCode::XPASS;
            } else if result.code == ResultCode::FAIL {
                result.code = ResultCode::XFAIL;
            }
        }
        self.result = Some(result);
        Ok(())
    }

    pub(crate) fn is_expected_to_fail(&self) -> Result<bool, String> {
        if self.xfails.iter().any(|expr| expr == "*") {
            return Ok(true);
        }
        for expression in &self.xfails {
            if self.matching_bool_expr(expression)? {
                return Ok(true);
            }
        }
        Ok(false)
    }

    pub(crate) fn missing_required_features(&self) -> Result<Vec<String>, String> {
        let mut missing = Vec::new();
        for expression in &self.requires {
            if expression != "*" && !self.matching_bool_expr(expression)? {
                missing.push(expression.clone());
            }
        }
        Ok(missing)
    }

    pub(crate) fn unsupported_features(&self) -> Result<Vec<String>, String> {
        let mut unsupported = Vec::new();
        for expression in &self.unsupported {
            if expression == "*" || self.matching_bool_expr(expression)? {
                unsupported.push(expression.clone());
            }
        }
        Ok(unsupported)
    }

    pub(crate) fn is_within_feature_limits(&self) -> Result<bool, String> {
        if self.config.limit_to_features.is_empty() {
            return Ok(true);
        }
        if !self.missing_required_features()?.is_empty() {
            return Ok(false);
        }

        let without_limits: HashSet<String> = self
            .config
            .available_features
            .iter()
            .filter(|feature| !self.config.limit_to_features.contains(feature.as_str()))
            .cloned()
            .collect();
        for expression in &self.requires {
            if expression != "*"
                && !crate::boolean_expression::evaluate(expression, &without_limits)
                    .map_err(|error| error.to_string())?
            {
                return Ok(true);
            }
        }
        Ok(false)
    }

    fn matching_bool_expr(&self, expression: &str) -> Result<bool, String> {
        crate::boolean_expression::evaluate(expression, &self.config.available_features)
            .map_err(|error| error.to_string())
    }
}

#[derive(Debug, Clone, Default)]
pub(crate) struct TestConfig {
    pub(crate) name: String,
    pub(crate) suffixes: HashSet<String>,
    pub(crate) exclusions: HashSet<String>,
    pub(crate) substitutions: Vec<(String, String)>,
    pub(crate) default_substitutions: Vec<(String, String)>,
    pub(crate) normalize_slashes: bool,
    pub(crate) available_features: HashSet<String>,
    pub(crate) pipefail: bool,
    pub(crate) recursive_expansion_limit: Option<usize>,
    pub(crate) limit_to_features: HashSet<String>,
    pub(crate) max_individual_test_time: Option<Duration>,
    pub(crate) environment: HashMap<String, String>,
    pub(crate) test_retry_attempts: usize,
    pub(crate) tools: HashMap<String, PathBuf>,
}

impl TestConfig {
    pub(crate) fn is_excluded_name(&self, name: &str) -> bool {
        self.exclusions.contains(name) || name == "Inputs" || name == ".git" || name == ".svn"
    }
}