litnik 0.2.0

An embedded Lit and FileCheck runner for compiler test suites.
Documentation
//! An embedded Lit and FileCheck runner for compiler test suites.
//!
//! [`Suite`] discovers files, reads Lit directives, and reports each file to
//! Cargo's test harness. The default internal shell does not require LLVM Lit,
//! FileCheck, or a host shell. Register compiler executables with
//! [`Suite::tool()`]; Litnik resolves those command names without changing
//! `PATH`.
//!
//! # Test target
//!
//! Add one harness-free test target to `Cargo.toml`:
//!
//! ```text
//! [[test]]
//! name = "lit"
//! path = "tests/lit.rs"
//! harness = false
//! ```
//!
//! The complete `tests/lit.rs` can stay small:
//!
//! ```no_run
//! litnik::Suite::new("tests/lit")
//!     .suffix("mlir")
//!     .tool("example-opt", "target/debug/example-opt")
//!     .run_from_args();
//! ```
//!
//! A `RUN:` line can invoke `example-opt` by name. It can invoke `FileCheck`
//! without a separate tool registration.
//!
//! # Compatibility boundary
//!
//! Litnik reads Lit's integrated shell-test syntax directly. It does not read
//! Python `lit.cfg` files. Configure the corresponding suite values through
//! [`Suite`] or [`LitConfig`].
//!
//! Embedded FileCheck covers the common LLVM directives and command-line
//! options. It requires UTF-8 text, uses Rust regular expressions, and does not
//! expand response files. The crate README documents the current compatibility
//! boundary.
//!
//! # Platform and Rust versions
//!
//! Litnik supports Rust 1.92 and later. The runner supports Unix and Windows
//! process execution.

#![warn(missing_docs)]

mod boolean_expression;
pub mod config;
pub mod discovery;
mod filecheck;
pub mod formats;
mod integrated;
mod runner;
mod shell;
mod substitution;
pub mod test;

pub use config::LitConfig;
pub use libtest_mimic::{Arguments, Conclusion};
pub use test::{ResultCode, Test, TestResult};

use std::path::PathBuf;

/// A compiler test suite configured from one Rust test entry point.
///
/// Builder methods consume and return the suite. Collection methods add
/// values; scalar methods replace the prior value. Call [`Self::run_from_args()`]
/// in a harness-free Cargo test target for the shortest integration.
#[derive(Debug, Default)]
pub struct Suite {
    config: LitConfig,
}

impl Suite {
    /// Creates a suite that discovers tests below `root`.
    ///
    /// Add at least one suffix with [`Self::suffix()`]. A suite with no matching
    /// tests fails instead of reporting an empty success. A relative root is
    /// anchored when this method is called.
    ///
    /// # Panics
    ///
    /// Panics if `root` is relative and the process working directory is not
    /// available.
    pub fn new(root: impl Into<PathBuf>) -> Self {
        Self {
            config: LitConfig::new(root),
        }
    }

    /// Replaces the discovery root and clears a prior source-root override.
    ///
    /// A relative root is anchored when this method is called.
    ///
    /// # Panics
    ///
    /// Panics if `root` is relative and the process working directory is not
    /// available.
    pub fn with_root(mut self, root: impl Into<PathBuf>) -> Self {
        self.config.root = config::anchor_path(root.into());
        self.config.test_source_root = None;
        self
    }

    /// Registers an executable for a command name used in `RUN:` directives.
    ///
    /// The registered path does not need to appear on `PATH`. A relative path
    /// is anchored when this method is called.
    ///
    /// # Panics
    ///
    /// Panics if `path` is relative and the process working directory is not
    /// available.
    pub fn tool(mut self, name: impl Into<String>, path: impl Into<PathBuf>) -> Self {
        self.config = self.config.tool(name, path);
        self
    }

    /// Adds one test-file suffix.
    pub fn suffix(mut self, suffix: impl Into<String>) -> Self {
        self.config = self.config.suffix(suffix);
        self
    }

    /// Adds test-file suffixes in iteration order.
    pub fn suffixes<I, S>(mut self, values: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.config = self.config.suffixes(values);
        self
    }

    /// Excludes a file or directory base name from discovery.
    pub fn exclude(mut self, value: impl Into<String>) -> Self {
        self.config = self.config.exclude(value);
        self
    }

    /// Adds excluded file or directory base names.
    pub fn excludes<I, S>(mut self, values: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.config = self.config.excludes(values);
        self
    }

    /// Adds a regular-expression substitution for `RUN:` directives.
    pub fn substitution(mut self, pattern: impl Into<String>, value: impl Into<String>) -> Self {
        self.config = self.config.substitution(pattern, value);
        self
    }

    /// Makes one feature available to boolean test directives.
    pub fn feature(mut self, value: impl Into<String>) -> Self {
        self.config = self.config.feature(value);
        self
    }

    /// Makes each supplied feature available to boolean test directives.
    pub fn features<I, S>(mut self, values: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.config = self.config.features(values);
        self
    }

    /// Replaces one variable in the inherited test environment.
    pub fn env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
        self.config = self.config.env(key, value);
        self
    }

    /// Replaces the source root used for discovery and `%s` paths.
    ///
    /// A relative root is anchored when this method is called.
    ///
    /// # Panics
    ///
    /// Panics if the root is relative and the process working directory is not
    /// available.
    pub fn source_root(mut self, root: impl Into<PathBuf>) -> Self {
        self.config = self.config.source_root(root);
        self
    }

    /// Replaces the root used for working directories and temporary files.
    ///
    /// A relative root is anchored when this method is called.
    ///
    /// # Panics
    ///
    /// Panics if the root is relative and the process working directory is not
    /// available.
    pub fn exec_root(mut self, root: impl Into<PathBuf>) -> Self {
        self.config = self.config.exec_root(root);
        self
    }

    /// Sets the maximum number of recursive substitution passes.
    pub fn recursive_expansion_limit(mut self, value: usize) -> Self {
        self.config = self.config.recursive_expansion_limit(Some(value));
        self
    }

    /// Sets the time limit for one test attempt, in seconds.
    ///
    /// A value of zero disables the limit. The runner stops child processes at
    /// the limit. It cannot preempt an embedded command while it runs.
    pub fn max_individual_test_time(mut self, seconds: u64) -> Self {
        self.config = self.config.max_individual_test_time(seconds);
        self
    }

    /// Selects validation without command execution.
    pub fn no_execute(mut self, value: bool) -> Self {
        self.config = self.config.no_execute(value);
        self
    }

    /// Controls whether a failed pipeline stage fails the complete pipeline.
    pub fn with_pipefail(mut self, value: bool) -> Self {
        self.config = self.config.with_pipefail(value);
        self
    }

    /// Controls slash normalization in path substitutions.
    pub fn normalize_slashes(mut self, value: bool) -> Self {
        self.config = self.config.normalize_slashes(value);
        self
    }

    /// Restricts execution to tests that require at least one listed feature.
    pub fn limit_to_features<I, S>(mut self, values: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.config = self.config.limit_to_features(values);
        self
    }

    /// Sets the number of retries after an initial failed attempt.
    pub fn test_retry_attempts(mut self, attempts: usize) -> Self {
        self.config = self.config.test_retry_attempts(attempts);
        self
    }

    /// Discovers and runs the suite through `libtest-mimic`.
    ///
    /// Discovery failures and zero discovered tests appear as failing harness
    /// entries. The returned conclusion does not terminate the process.
    /// `XFAIL` and `FLAKYPASS` appear as successful harness entries; inspect
    /// [`TestResult::code`] through the low-level API when that distinction is
    /// required.
    ///
    /// # Blocking
    ///
    /// This method performs synchronous filesystem access and waits for child
    /// processes. Independent tests can run in parallel through the harness.
    pub fn run(self, args: Arguments) -> Conclusion {
        let tests = match discovery::try_discover_tests(&self.config.root, &self.config) {
            Ok(tests) if !tests.is_empty() => tests,
            Ok(_) => {
                let root = self
                    .config
                    .test_source_root
                    .as_deref()
                    .unwrap_or(&self.config.root)
                    .display()
                    .to_string();
                let trial = libtest_mimic::Trial::test("litnik :: discovery", move || {
                    Err(format!("no tests discovered below '{root}'").into())
                });
                return libtest_mimic::run(&args, vec![trial]);
            }
            Err(error) => {
                let message = error.to_string();
                let trial = libtest_mimic::Trial::test("litnik :: discovery", move || {
                    Err(message.clone().into())
                });
                return libtest_mimic::run(&args, vec![trial]);
            }
        };

        let mut trials = Vec::with_capacity(tests.len());
        for mut test in tests {
            let config = self.config.clone();
            let name = test.get_full_name();
            trials.push(libtest_mimic::Trial::ignorable_test(name, move || {
                let result = formats::run_shtest(&mut test, &config);
                match result.code {
                    ResultCode::EXCLUDED | ResultCode::SKIPPED | ResultCode::UNSUPPORTED => {
                        Ok(libtest_mimic::Completion::Ignored {
                            reason: Some(result.code.label().to_string()),
                        })
                    }
                    code if code.is_failure() => {
                        let message =
                            format!("{} ({})\n{}", code.name(), code.label(), result.output);
                        Err(message.into())
                    }
                    _ => Ok(libtest_mimic::Completion::Completed),
                }
            }));
        }
        libtest_mimic::run(&args, trials)
    }

    /// Parses the current process arguments, runs the suite, and exits.
    ///
    /// This is the terminal call for a harness-free Cargo test target.
    pub fn run_from_args(self) -> ! {
        self.run(Arguments::from_args()).exit()
    }
}

#[cfg(test)]
mod tests {
    use libtest_mimic::Arguments;

    #[test]
    fn config_builder_is_chainable() {
        let suite = crate::Suite::new("/tmp")
            .tool("butcher-opt", "/bin/true")
            .suffix(".mlir")
            .feature("linux")
            .excludes(vec!["Ignore"])
            .test_retry_attempts(2)
            .limit_to_features(vec!["linux"]);

        let cfg = suite.config;
        assert!(cfg.tools.contains_key("butcher-opt"));
        assert!(cfg.suffixes.contains(".mlir"));
        assert!(cfg.available_features.contains("linux"));
    }

    #[test]
    fn suite_runs_with_empty_tests() {
        let suite = super::Suite::new(".").suffix(".does-not-exist");
        let args = Arguments::from_args();
        let _ = suite.run(args);
    }
}