litnik 0.2.1

An embedded Lit and FileCheck runner for compiler test suites.
Documentation
//! Discovered test identities and exact Lit results.
//!
//! [`crate::Suite`] maps these results onto the smaller set of outcomes exposed
//! by `libtest-mimic`. Use this module with [`crate::formats`] when a harness
//! needs to distinguish statuses such as [`ResultCode::XFAIL`] and
//! [`ResultCode::FLAKYPASS`].

use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::time::Duration;

/// One status in Lit's result taxonomy.
///
/// The integrated shell runner emits `UNSUPPORTED`, `PASS`, `FLAKYPASS`,
/// `XFAIL`, `UNRESOLVED`, `TIMEOUT`, `FAIL`, and `XPASS`. The remaining
/// constants preserve the wider Lit taxonomy for result handling.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ResultCode {
    name: &'static str,
    label: &'static str,
    is_failure: bool,
}

impl ResultCode {
    /// A suite configuration excluded the test before execution.
    pub const EXCLUDED: Self = Self::new("EXCLUDED", "Excluded", false);
    /// A harness skipped the test before execution.
    pub const SKIPPED: Self = Self::new("SKIPPED", "Skipped", false);
    /// A `REQUIRES:`, `UNSUPPORTED:`, or feature-limit condition skipped the test.
    pub const UNSUPPORTED: Self = Self::new("UNSUPPORTED", "Unsupported", false);
    /// The test completed successfully.
    pub const PASS: Self = Self::new("PASS", "Passed", false);
    /// The test passed after at least one failed attempt.
    pub const FLAKYPASS: Self = Self::new("FLAKYPASS", "Passed With Retry", false);
    /// An update operation converted an earlier failure into a pass.
    ///
    /// The integrated shell runner does not emit this status.
    pub const FIXED: Self = Self::new("FIXED", "Passed After Update", false);
    /// The test failed as specified by an active `XFAIL:` directive.
    pub const XFAIL: Self = Self::new("XFAIL", "Expectedly Failed", false);
    /// Litnik could not read, parse, or prepare the test.
    pub const UNRESOLVED: Self = Self::new("UNRESOLVED", "Unresolved", true);
    /// The test exceeded its configured time limit.
    pub const TIMEOUT: Self = Self::new("TIMEOUT", "Timed Out", true);
    /// A `RUN:` command returned a nonzero status.
    pub const FAIL: Self = Self::new("FAIL", "Failed", true);
    /// The test passed despite an active `XFAIL:` directive.
    pub const XPASS: Self = Self::new("XPASS", "Unexpectedly Passed", true);

    /// Every result category in stable display order.
    pub const ALL: [Self; 11] = [
        Self::EXCLUDED,
        Self::SKIPPED,
        Self::UNSUPPORTED,
        Self::PASS,
        Self::FLAKYPASS,
        Self::FIXED,
        Self::XFAIL,
        Self::UNRESOLVED,
        Self::TIMEOUT,
        Self::FAIL,
        Self::XPASS,
    ];

    const fn new(name: &'static str, label: &'static str, is_failure: bool) -> Self {
        Self {
            name,
            label,
            is_failure,
        }
    }

    /// Returns the uppercase name used in Lit-compatible reports.
    pub const fn name(self) -> &'static str {
        self.name
    }

    /// Returns the human-readable label used in harness output.
    pub const fn label(self) -> &'static str {
        self.label
    }

    /// Returns whether this status makes the suite fail.
    pub const fn is_failure(self) -> bool {
        self.is_failure
    }
}

/// The final status, diagnostics, timing, and attempts for one Lit test.
#[derive(Debug, Clone)]
pub struct TestResult {
    /// The final Lit status.
    pub code: ResultCode,
    /// Rendered commands and their output, or a preparation diagnostic.
    pub output: String,
    pub(crate) elapsed: Option<Duration>,
    pub(crate) attempts: usize,
    pub(crate) max_allowed_attempts: usize,
    pub(crate) attempt_outputs: Vec<String>,
}

impl TestResult {
    pub(crate) fn new(code: ResultCode, output: impl Into<String>) -> Self {
        Self {
            code,
            output: output.into(),
            elapsed: None,
            attempts: 1,
            max_allowed_attempts: 1,
            attempt_outputs: Vec::new(),
        }
    }

    /// Returns whether the final status makes the suite fail.
    pub fn is_failure(&self) -> bool {
        self.code.is_failure()
    }

    /// Returns recorded runner time, including preparation and retries.
    pub fn elapsed(&self) -> Option<Duration> {
        self.elapsed
    }

    /// Returns the attempt count recorded for the result.
    ///
    /// A result that stops during preparation records one attempt.
    pub fn attempts(&self) -> usize {
        self.attempts
    }

    /// Returns one plus the configured retry count for this test.
    ///
    /// The actual attempt count can be smaller when an attempt passes or returns
    /// a status other than `FAIL`.
    pub fn max_allowed_attempts(&self) -> usize {
        self.max_allowed_attempts
    }

    /// Returns captured output in attempt order.
    ///
    /// A result that stops during preparation contains its final diagnostic.
    pub fn attempt_outputs(&self) -> &[String] {
        &self.attempt_outputs
    }
}

/// One discovered test file with source, execution, and result state.
///
/// Discovery creates this value with a snapshot of the suite configuration.
/// [`crate::formats::run_shtest()`] replaces that snapshot with the supplied
/// configuration and stores the returned result in the test.
#[derive(Debug, Clone)]
pub struct Test {
    pub(crate) suite_name: String,
    pub(crate) source_root: PathBuf,
    pub(crate) exec_root: PathBuf,
    pub(crate) relative_path: PathBuf,
    pub(crate) config: TestConfig,
    pub(crate) xfails: Vec<String>,
    pub(crate) requires: Vec<String>,
    pub(crate) unsupported: Vec<String>,
    pub(crate) allowed_retries: usize,
    pub(crate) result: Option<TestResult>,
}

impl Test {
    pub(crate) fn new(
        suite_name: impl Into<String>,
        source_root: PathBuf,
        exec_root: PathBuf,
        relative_path: impl Into<PathBuf>,
        config: TestConfig,
    ) -> Self {
        let allowed_retries = config.test_retry_attempts;
        Self {
            suite_name: suite_name.into(),
            source_root,
            exec_root,
            relative_path: relative_path.into(),
            config,
            xfails: Vec::new(),
            requires: Vec::new(),
            unsupported: Vec::new(),
            allowed_retries,
            result: None,
        }
    }

    /// Returns the source root joined with the suite-relative path.
    pub fn source_path(&self) -> PathBuf {
        self.source_root.join(&self.relative_path)
    }

    /// Returns the logical test path below the execution root.
    ///
    /// The runner uses this path to derive the working directory and the
    /// per-test path below `Output`; it does not create a file at this path.
    pub fn exec_path(&self) -> PathBuf {
        self.exec_root.join(&self.relative_path)
    }

    /// Returns the path that identifies this test below the source root.
    pub fn relative_path(&self) -> &Path {
        &self.relative_path
    }

    /// Returns the harness name in `suite :: path` form.
    pub fn get_full_name(&self) -> String {
        format!("{} :: {}", self.suite_name, self.relative_path.display())
    }

    /// Returns the stored result, or `None` before the test runs.
    pub fn result(&self) -> Option<&TestResult> {
        self.result.as_ref()
    }

    pub(crate) fn set_result(&mut self, mut result: TestResult) -> Result<(), String> {
        if self.is_expected_to_fail()? {
            if result.code == ResultCode::PASS {
                result.code = ResultCode::XPASS;
            } else if result.code == ResultCode::FAIL {
                result.code = ResultCode::XFAIL;
            }
        }
        self.result = Some(result);
        Ok(())
    }

    pub(crate) fn is_expected_to_fail(&self) -> Result<bool, String> {
        if self.xfails.iter().any(|expr| expr == "*") {
            return Ok(true);
        }
        for expression in &self.xfails {
            if self.matching_bool_expr(expression)? {
                return Ok(true);
            }
        }
        Ok(false)
    }

    pub(crate) fn missing_required_features(&self) -> Result<Vec<String>, String> {
        let mut missing = Vec::new();
        for expression in &self.requires {
            if expression != "*" && !self.matching_bool_expr(expression)? {
                missing.push(expression.clone());
            }
        }
        Ok(missing)
    }

    pub(crate) fn unsupported_features(&self) -> Result<Vec<String>, String> {
        let mut unsupported = Vec::new();
        for expression in &self.unsupported {
            if expression == "*" || self.matching_bool_expr(expression)? {
                unsupported.push(expression.clone());
            }
        }
        Ok(unsupported)
    }

    pub(crate) fn is_within_feature_limits(&self) -> Result<bool, String> {
        if self.config.limit_to_features.is_empty() {
            return Ok(true);
        }
        if !self.missing_required_features()?.is_empty() {
            return Ok(false);
        }

        let without_limits: HashSet<String> = self
            .config
            .available_features
            .iter()
            .filter(|feature| !self.config.limit_to_features.contains(feature.as_str()))
            .cloned()
            .collect();
        for expression in &self.requires {
            if expression != "*"
                && !crate::boolean_expression::evaluate(expression, &without_limits)
                    .map_err(|error| error.to_string())?
            {
                return Ok(true);
            }
        }
        Ok(false)
    }

    fn matching_bool_expr(&self, expression: &str) -> Result<bool, String> {
        crate::boolean_expression::evaluate(expression, &self.config.available_features)
            .map_err(|error| error.to_string())
    }
}

#[derive(Debug, Clone, Default)]
pub(crate) struct TestConfig {
    pub(crate) name: String,
    pub(crate) suffixes: HashSet<String>,
    pub(crate) exclusions: HashSet<String>,
    pub(crate) substitutions: Vec<(String, String)>,
    pub(crate) default_substitutions: Vec<(String, String)>,
    pub(crate) normalize_slashes: bool,
    pub(crate) available_features: HashSet<String>,
    pub(crate) pipefail: bool,
    pub(crate) recursive_expansion_limit: Option<usize>,
    pub(crate) limit_to_features: HashSet<String>,
    pub(crate) max_individual_test_time: Option<Duration>,
    pub(crate) environment: HashMap<String, String>,
    pub(crate) test_retry_attempts: usize,
    pub(crate) tools: HashMap<String, PathBuf>,
}

impl TestConfig {
    pub(crate) fn is_excluded_name(&self, name: &str) -> bool {
        self.exclusions.contains(name) || name == "Inputs" || name == ".git" || name == ".svn"
    }
}