git_checks/

formatting.rs

// Copyright Kitware, Inc.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::ffi::OsStr;
use std::fmt;
use std::io::{self, BufReader, Read};
use std::iter;
use std::os::unix::process::ExitStatusExt;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;

use derive_builder::Builder;
use git_checks_core::impl_prelude::*;
use git_checks_core::AttributeError;
use git_workarea::{GitContext, GitWorkArea};
use itertools::Itertools;
use log::{debug, info, warn};
use rayon::prelude::*;
use thiserror::Error;
use wait_timeout::ChildExt;

#[derive(Debug, Clone, Copy)]
enum FormattingExecStage {
    Run,
    Wait,
    Kill,
    TimeoutWait,
}

impl fmt::Display for FormattingExecStage {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let what = match self {
            FormattingExecStage::Run => "execute",
            FormattingExecStage::Wait => "wait on",
            FormattingExecStage::Kill => "kill (timed out)",
            FormattingExecStage::TimeoutWait => "wait on (timed out)",
        };

        write!(f, "{}", what)
    }
}

#[derive(Debug, Clone, Copy)]
enum ListFilesReason {
    Modified,
    Untracked,
}

impl fmt::Display for ListFilesReason {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let what = match self {
            ListFilesReason::Modified => "modified",
            ListFilesReason::Untracked => "untracked",
        };

        write!(f, "{}", what)
    }
}

#[derive(Debug, Error)]
enum FormattingError {
    #[error("failed to {} the {} formatter: {}", stage, command.display(), source)]
    ExecFormatter {
        command: PathBuf,
        stage: FormattingExecStage,
        #[source]
        source: io::Error,
    },
    #[error("failed to collect stderr from the {} formatter: {}", command.display(), source)]
    CollectStderr {
        command: PathBuf,
        #[source]
        source: io::Error,
    },
    #[error("failed to list {} file in the work area: {}", reason, output)]
    ListFiles {
        reason: ListFilesReason,
        output: String,
    },
    #[error("attribute extraction error: {}", source)]
    Attribute {
        #[from]
        source: AttributeError,
    },
}

impl FormattingError {
    fn exec_formatter(command: PathBuf, stage: FormattingExecStage, source: io::Error) -> Self {
        FormattingError::ExecFormatter {
            command,
            stage,
            source,
        }
    }

    fn collect_stderr(command: PathBuf, source: io::Error) -> Self {
        FormattingError::CollectStderr {
            command,
            source,
        }
    }

    fn list_files(reason: ListFilesReason, output: &[u8]) -> Self {
        FormattingError::ListFiles {
            reason,
            output: String::from_utf8_lossy(output).into(),
        }
    }
}

/// Run a formatter in the repository to check commits for formatting.
///
/// The formatter is passed the following arguments:
///
/// - the path to the file which should be checked.
/// - if provided, the value of the controlling attribute.
///
/// At runtime, the following configuration keys may be used to amend the check's behavior:
///
/// - `formatter.{kind}.path`: the path to the formatter executable
/// - `formatter.env_key`: an environment variable to set to the value `check` so that the
///   formatter knows it is being run for a check.
///
/// The formatter is expected to exit with success whether the path passed to it has a valid format
/// in it or not. A failure exit status is considered a failure of the formatter itself. If any
/// changes (including untracked files) are left inside of the worktree, it is considered to have
/// failed the checks.
///
/// The formatter is run with its current working directory being the top-level of the work tree,
/// but not the proper `GIT_` context. This is because the setup for the workarea is not completely
/// isolated and `git` commands may not behave as expected. The worktree it is working from is only
/// guaranteed to have the files which have changed in the commit being checked on disk, so
/// additional files which should be available for the command to run must be specified with
/// `Formatting::add_config_files`.
#[derive(Builder, Debug, Clone)]
#[builder(field(private))]
pub struct Formatting {
    /// The "name" of the formatter.
    ///
    /// This is used to refer to the formatter in use in error messages.
    ///
    /// Configuration: Optional
    /// Default: the `kind` is used
    #[builder(setter(into, strip_option), default)]
    name: Option<String>,
    /// The "kind" of formatting being performed.
    ///
    /// This is used in the name of the attribute which uses this check.
    ///
    /// Configuration: Required
    #[builder(setter(into))]
    kind: String,
    /// The path to the formatter.
    ///
    /// This may be a command that exists in `PATH` if absolute paths are not wanted.
    ///
    /// Configuration: Required
    #[builder(setter(into))]
    formatter: PathBuf,
    #[builder(private)]
    #[builder(setter(name = "_config_files"))]
    #[builder(default)]
    config_files: Vec<String>,
    /// A message to add when failures occur.
    ///
    /// Projects which check formatting may have a way to fix it automatically. This is here so
    /// that those projects can mention their specific instructions.
    ///
    /// Configuration: Optional
    /// Default: Unused if not provided.
    #[builder(setter(into, strip_option), default)]
    fix_message: Option<String>,
    /// A timeout for running the formatter.
    ///
    /// If the formatter exceeds this timeout, it is considered to have failed.
    ///
    /// Configuration: Optional
    /// Default: No timeout
    #[builder(setter(into, strip_option), default)]
    timeout: Option<Duration>,
}

/// This is the maximum number of files to list in the error message. Beyond this, the number of
/// other files with formatting issues in them are handled by a "and so many other files" note.
const MAX_EXPLICIT_FILE_LIST: usize = 5;
/// How long to wait for a timed-out formatter to respond to `SIGKILL` before leaving it as a
/// zombie process.
const ZOMBIE_TIMEOUT: Duration = Duration::from_secs(1);

impl FormattingBuilder {
    /// Configuration files within the repository the formatter
    ///
    /// Configuration: Optional
    /// Default: `Vec::new()`
    pub fn config_files<I, F>(&mut self, files: I) -> &mut Self
    where
        I: IntoIterator<Item = F>,
        F: Into<String>,
    {
        self.config_files = Some(files.into_iter().map(Into::into).collect());
        self
    }
}

impl Formatting {
    /// Create a new builder.
    pub fn builder() -> FormattingBuilder {
        Default::default()
    }

    /// Check a path using the formatter.
    fn check_path<'a>(
        &self,
        ctx: &GitWorkArea,
        formatter: &Path,
        env_key: Option<&OsStr>,
        path: &'a FileName,
        attr_value: Option<String>,
    ) -> Result<Option<&'a FileName>, FormattingError> {
        let mut cmd = Command::new(formatter);
        ctx.cd_to_work_tree(&mut cmd);
        cmd.arg(path.as_path());
        if let Some(attr_value) = attr_value {
            cmd.arg(attr_value);
        }
        if let Some(env_key) = env_key {
            cmd.env(env_key, "check");
        }

        let (success, output) = if let Some(timeout) = self.timeout {
            let mut child = cmd
                // Formatters should not read anything.
                .stdin(Stdio::null())
                // The output goes nowhere.
                .stdout(Stdio::null())
                // But we want any error messages from them (for logging purposes). If this pipe
                // fills up buffers, it will deadlock and the timeout will "save" us. Any process
                // outputting this much error messages probably is very unhappy anyways.
                .stderr(Stdio::piped())
                .spawn()
                .map_err(|err| {
                    FormattingError::exec_formatter(
                        formatter.to_path_buf(),
                        FormattingExecStage::Run,
                        err,
                    )
                })?;
            let check = child.wait_timeout(timeout).map_err(|err| {
                FormattingError::exec_formatter(
                    formatter.to_path_buf(),
                    FormattingExecStage::Wait,
                    err,
                )
            })?;

            if let Some(status) = check {
                let stderr = child.stderr.expect("spawned with stderr");
                let stderr = BufReader::new(stderr);
                let bytes_output = stderr
                    .bytes()
                    .collect::<Result<Vec<u8>, _>>()
                    .map_err(|err| FormattingError::collect_stderr(formatter.to_path_buf(), err))?;
                (
                    status.success(),
                    format!(
                        "failed with exit code {:?}, signal {:?}, output: {:?}",
                        status.code(),
                        status.signal(),
                        String::from_utf8_lossy(&bytes_output),
                    ),
                )
            } else {
                child.kill().map_err(|err| {
                    FormattingError::exec_formatter(
                        formatter.to_path_buf(),
                        FormattingExecStage::Kill,
                        err,
                    )
                })?;
                let timed_out_status = child.wait_timeout(ZOMBIE_TIMEOUT).map_err(|err| {
                    FormattingError::exec_formatter(
                        formatter.to_path_buf(),
                        FormattingExecStage::TimeoutWait,
                        err,
                    )
                })?;
                if timed_out_status.is_none() {
                    warn!(
                        target: "git-checks/formatting",
                        "leaving a zombie '{}' process; it did not respond to kill",
                        self.kind,
                    );
                }
                (false, "timeout reached".into())
            }
        } else {
            let check = cmd.output().map_err(|err| {
                FormattingError::exec_formatter(
                    formatter.to_path_buf(),
                    FormattingExecStage::Run,
                    err,
                )
            })?;
            (
                check.status.success(),
                String::from_utf8_lossy(&check.stderr).into_owned(),
            )
        };

        Ok(if success {
            debug!(
                target: "git-checks/formatting",
                "succeeded at running the {} formatting command: {output}",
                self.kind,
            );
            None
        } else {
            info!(
                target: "git-checks/formatting",
                "failed to run the {} formatting command: {}",
                self.kind,
                output,
            );
            Some(path)
        })
    }

    /// Create a message for the given paths.
    #[allow(clippy::needless_collect)]
    fn message_for_paths<P>(
        &self,
        results: &mut CheckResult,
        content: &dyn Content,
        paths: Vec<P>,
        description: &str,
    ) where
        P: fmt::Display,
    {
        if !paths.is_empty() {
            let mut all_paths = paths.into_iter();
            // List at least a certain number of files by name.
            let explicit_paths = all_paths
                .by_ref()
                .take(MAX_EXPLICIT_FILE_LIST)
                .map(|path| format!("`{}`", path))
                .collect::<Vec<_>>();
            // Eline the remaining files...
            let next_path = all_paths.next();
            let tail_paths = if let Some(next_path) = next_path {
                let remaining_paths = all_paths.count();
                if remaining_paths == 0 {
                    // ...but avoid saying `and 1 others`.
                    iter::once(format!("`{}`", next_path)).collect::<Vec<_>>()
                } else {
                    iter::once(format!("and {} others", remaining_paths + 1)).collect::<Vec<_>>()
                }
            } else {
                iter::empty().collect::<Vec<_>>()
            }
            .into_iter();
            let paths = explicit_paths.into_iter().chain(tail_paths).join(", ");
            let fix = self
                .fix_message
                .as_ref()
                .map_or_else(String::new, |fix_message| format!(" {}", fix_message));
            results.add_error(format!(
                "{}the following files {} the '{}' check: {}.{}",
                commit_prefix_str(content, "is not allowed because"),
                description,
                self.name.as_ref().unwrap_or(&self.kind),
                paths,
                fix,
            ));
        }
    }
}

impl ContentCheck for Formatting {
    fn name(&self) -> &str {
        "formatting"
    }

    fn check(
        &self,
        ctx: &CheckGitContext,
        content: &dyn Content,
    ) -> Result<CheckResult, Box<dyn Error>> {
        let changed_paths = content.modified_files();

        let gitctx = GitContext::new(ctx.gitdir());
        let mut workarea = content.workarea(&gitctx)?;

        // Create the files necessary on the disk.
        let files_to_checkout = changed_paths
            .iter()
            .map(|path| path.as_path())
            .chain(self.config_files.iter().map(AsRef::as_ref))
            .collect::<Vec<_>>();
        workarea.checkout(&files_to_checkout)?;

        let formatter = ctx
            .configuration(&format!("formatter.{}.path", self.kind))
            .map(Path::new)
            .unwrap_or(&self.formatter);
        let env_key = ctx.configuration("formatter.env_key").map(OsStr::new);

        let attr = format!("format.{}", self.kind);
        let failed_paths = changed_paths
            .par_iter()
            .map(|path| {
                match ctx.check_attr(&attr, path.as_path())? {
                    AttributeState::Set => {
                        self.check_path(&workarea, formatter, env_key, path, None)
                    },
                    AttributeState::Value(v) => {
                        self.check_path(&workarea, formatter, env_key, path, Some(v))
                    },
                    _ => Ok(None),
                }
            })
            .collect::<Vec<Result<_, _>>>()
            .into_iter()
            .collect::<Result<Vec<_>, _>>()?
            .into_iter()
            .flatten()
            .collect::<Vec<_>>();

        let ls_files_m = workarea
            .git()
            .arg("ls-files")
            .arg("-m")
            .output()
            .map_err(|err| GitError::subcommand("ls-files -m", err))?;
        if !ls_files_m.status.success() {
            return Err(
                FormattingError::list_files(ListFilesReason::Modified, &ls_files_m.stderr).into(),
            );
        }
        let modified_paths = String::from_utf8_lossy(&ls_files_m.stdout);

        // It seems that the `HEAD` ref is used rather than the index for `ls-files -m`, so
        // basically every file is considered `deleted` and therefore listed here. Not sure if this
        // is a bug in Git or not.
        let modified_paths_in_commit = modified_paths
            .lines()
            .filter(|&path| {
                changed_paths
                    .iter()
                    .any(|diff_path| diff_path.as_str() == path)
            })
            .collect();

        let ls_files_o = workarea
            .git()
            .arg("ls-files")
            .arg("-o")
            .output()
            .map_err(|err| GitError::subcommand("ls-files -o", err))?;
        if !ls_files_o.status.success() {
            return Err(FormattingError::list_files(
                ListFilesReason::Untracked,
                &ls_files_o.stderr,
            )
            .into());
        }
        let untracked_paths = String::from_utf8_lossy(&ls_files_o.stdout);

        let mut results = CheckResult::new();

        self.message_for_paths(
            &mut results,
            content,
            failed_paths,
            "could not be formatted by",
        );
        self.message_for_paths(
            &mut results,
            content,
            modified_paths_in_commit,
            "are not formatted according to",
        );
        self.message_for_paths(
            &mut results,
            content,
            untracked_paths.lines().collect(),
            "were created by",
        );

        Ok(results)
    }
}

#[cfg(feature = "config")]
pub(crate) mod config {
    #[cfg(test)]
    use std::path::Path;
    use std::time::Duration;

    use git_checks_config::{register_checks, CommitCheckConfig, IntoCheck, TopicCheckConfig};
    use serde::Deserialize;
    #[cfg(test)]
    use serde_json::json;

    #[cfg(test)]
    use crate::test;
    use crate::Formatting;

    /// Configuration for the `Formatting` check.
    ///
    /// The `kind` key is required and is a string. This is used to construct the name of the Git
    /// attribute to look for to find files which are handled by this formatter. The `name` key is
    /// optional, but is a string and defaults to the given `kind`. The `formatter` key is a string
    /// containing the path to the formatter on the system running the checks. Some formatters may
    /// work with configuration files committed to the repository. These will also be checked out
    /// when using this formatter. These may be valid Git path specifications with globs. If
    /// problems are found, the optional `fix_message` key (a string) will be added to the message.
    /// This should describe how to fix the issues found by the formatter. The `timeout` key is an
    /// optional positive integer. If given, formatters not completing within the specified time
    /// are considered failures. Without a timeout, formatters which do not exit will cause the
    /// formatting check to wait forever.
    ///
    /// This check is registered as a commit check with the name `"formatting"` and a topic check
    /// with the name `"formatting/topic"`.
    ///
    /// # Example
    ///
    /// ```json
    /// {
    ///     "name": "formatter name",
    ///     "kind": "kind",
    ///     "formatter": "/path/to/formatter",
    ///     "config_files": [
    ///         "path/to/config/file"
    ///     ],
    ///     "fix_message": "instructions for fixing",
    ///     "timeout": 10,
    /// }
    /// ```
    #[derive(Deserialize, Debug)]
    pub struct FormattingConfig {
        #[serde(default)]
        name: Option<String>,
        kind: String,
        formatter: String,
        #[serde(default)]
        config_files: Option<Vec<String>>,
        #[serde(default)]
        fix_message: Option<String>,
        #[serde(default)]
        timeout: Option<u64>,
    }

    impl IntoCheck for FormattingConfig {
        type Check = Formatting;

        fn into_check(self) -> Self::Check {
            let mut builder = Formatting::builder();

            builder.kind(self.kind).formatter(self.formatter);

            if let Some(name) = self.name {
                builder.name(name);
            }

            if let Some(config_files) = self.config_files {
                builder.config_files(config_files);
            }

            if let Some(fix_message) = self.fix_message {
                builder.fix_message(fix_message);
            }

            if let Some(timeout) = self.timeout {
                builder.timeout(Duration::from_secs(timeout));
            }

            builder
                .build()
                .expect("configuration mismatch for `Formatting`")
        }
    }

    register_checks! {
        FormattingConfig {
            "formatting" => CommitCheckConfig,
            "formatting/topic" => TopicCheckConfig,
        },
    }

    #[test]
    fn test_formatting_config_empty() {
        let json = json!({});
        let err = serde_json::from_value::<FormattingConfig>(json).unwrap_err();
        test::check_missing_json_field(err, "kind");
    }

    #[test]
    fn test_formatting_config_kind_is_required() {
        let exp_formatter = "/path/to/formatter";
        let json = json!({
            "formatter": exp_formatter,
        });
        let err = serde_json::from_value::<FormattingConfig>(json).unwrap_err();
        test::check_missing_json_field(err, "kind");
    }

    #[test]
    fn test_formatting_config_formatter_is_required() {
        let exp_kind = "kind";
        let json = json!({
            "kind": exp_kind,
        });
        let err = serde_json::from_value::<FormattingConfig>(json).unwrap_err();
        test::check_missing_json_field(err, "formatter");
    }

    #[test]
    fn test_formatting_config_minimum_fields() {
        let exp_kind = "kind";
        let exp_formatter = "/path/to/formatter";
        let json = json!({
            "kind": exp_kind,
            "formatter": exp_formatter,
        });
        let check: FormattingConfig = serde_json::from_value(json).unwrap();

        assert_eq!(check.name, None);
        assert_eq!(check.kind, exp_kind);
        assert_eq!(check.formatter, exp_formatter);
        assert_eq!(check.config_files, None);
        assert_eq!(check.fix_message, None);
        assert_eq!(check.timeout, None);

        let check = check.into_check();

        assert_eq!(check.name, None);
        assert_eq!(check.kind, exp_kind);
        assert_eq!(check.formatter, Path::new(exp_formatter));
        itertools::assert_equal(&check.config_files, &[] as &[&str]);
        assert_eq!(check.fix_message, None);
        assert_eq!(check.timeout, None);
    }

    #[test]
    fn test_formatting_config_all_fields() {
        let exp_name: String = "formatter name".into();
        let exp_kind = "kind";
        let exp_formatter = "/path/to/formatter";
        let exp_config: String = "path/to/config/file".into();
        let exp_fix_message: String = "instructions for fixing".into();
        let exp_timeout = 10;
        let json = json!({
            "name": exp_name,
            "kind": exp_kind,
            "formatter": exp_formatter,
            "config_files": [exp_config],
            "fix_message": exp_fix_message,
            "timeout": exp_timeout,
        });
        let check: FormattingConfig = serde_json::from_value(json).unwrap();

        assert_eq!(check.name, Some(exp_name.clone()));
        assert_eq!(check.kind, exp_kind);
        assert_eq!(check.formatter, exp_formatter);
        itertools::assert_equal(
            check.config_files.as_ref().unwrap(),
            std::slice::from_ref(&exp_config),
        );
        assert_eq!(check.fix_message, Some(exp_fix_message.clone()));
        assert_eq!(check.timeout, Some(exp_timeout));

        let check = check.into_check();

        assert_eq!(check.name, Some(exp_name));
        assert_eq!(check.kind, exp_kind);
        assert_eq!(check.formatter, Path::new(exp_formatter));
        itertools::assert_equal(&check.config_files, &[exp_config]);
        assert_eq!(check.fix_message, Some(exp_fix_message));
        assert_eq!(check.timeout, Some(Duration::from_secs(exp_timeout)));
    }
}

#[cfg(test)]
mod tests {
    use std::time::Duration;

    use git_checks_core::{Check, TopicCheck};

    use crate::builders::FormattingBuilder;
    use crate::test::*;
    use crate::Formatting;

    const MISSING_CONFIG_COMMIT: &str = "220efbb4d0380fe932b70444fe15e787506080b0";
    const ADD_CONFIG_COMMIT: &str = "e08e9ac1c5b6a0a67e0b2715cb0dbf99935d9cbf";
    const BAD_FORMAT_COMMIT: &str = "e9a08d956553f94e9c8a0a02b11ca60f62de3c2b";
    const FIX_BAD_FORMAT_COMMIT: &str = "7fe590bdb883e195812cae7602ce9115cbd269ee";
    const OK_FORMAT_COMMIT: &str = "b77d2a5d63cd6afa599d0896dafff95f1ace50b6";
    const ENV_FORMAT_COMMIT: &str = "11107007b2875ac2f98a89085ff8f4f631b369ae";
    const ENV_BAD_FORMAT_COMMIT: &str = "8358d940dbe6cdd6ee75ad229dbaf210a604870a";
    const IGNORE_UNTRACKED_COMMIT: &str = "c0154d1087906d50c5551ff8f60e544e9a492a48";
    const DELETE_FORMAT_COMMIT: &str = "31446c81184df35498814d6aa3c7f933dddf91c2";
    const MANY_BAD_FORMAT_COMMIT: &str = "f0d10d9385ef697175c48fa72324c33d5e973f4b";
    const MANY_MORE_BAD_FORMAT_COMMIT: &str = "0e80ff6dd2495571b7d255e39fb3e7cc9f487fb7";
    const TIMEOUT_CONFIG_COMMIT: &str = "62f5eac20c5021cf323c757a4d24234c81c9c7ad";
    const WITH_ARG_COMMIT: &str = "dfa4c021e96f1e94634ad1787f31c2abdbeaff99";
    const NOEXEC_CONFIG_COMMIT: &str = "1b5be48f8ce45b8fec155a1787cabb6995194ce5";

    #[test]
    fn test_exec_stage_display() {
        assert_eq!(format!("{}", super::FormattingExecStage::Run), "execute");
        assert_eq!(format!("{}", super::FormattingExecStage::Wait), "wait on");
        assert_eq!(
            format!("{}", super::FormattingExecStage::Kill),
            "kill (timed out)"
        );
        assert_eq!(
            format!("{}", super::FormattingExecStage::TimeoutWait),
            "wait on (timed out)"
        );
    }

    #[test]
    fn test_list_files_reason_display() {
        assert_eq!(format!("{}", super::ListFilesReason::Modified), "modified");
        assert_eq!(
            format!("{}", super::ListFilesReason::Untracked),
            "untracked"
        );
    }

    #[test]
    fn test_formatting_builder_default() {
        assert!(Formatting::builder().build().is_err());
    }

    #[test]
    fn test_formatting_builder_kind_is_required() {
        assert!(Formatting::builder()
            .formatter("path/to/formatter")
            .build()
            .is_err());
    }

    #[test]
    fn test_formatting_builder_formatter_is_required() {
        assert!(Formatting::builder().kind("kind").build().is_err());
    }

    #[test]
    fn test_formatting_builder_minimum_fields() {
        assert!(Formatting::builder()
            .formatter("path/to/formatter")
            .kind("kind")
            .build()
            .is_ok());
    }

    #[test]
    fn test_formatting_name_commit() {
        let check = Formatting::builder()
            .formatter("path/to/formatter")
            .kind("kind")
            .build()
            .unwrap();
        assert_eq!(Check::name(&check), "formatting");
    }

    #[test]
    fn test_formatting_name_topic() {
        let check = Formatting::builder()
            .formatter("path/to/formatter")
            .kind("kind")
            .build()
            .unwrap();
        assert_eq!(TopicCheck::name(&check), "formatting");
    }

    fn formatting_check(kind: &str) -> FormattingBuilder {
        let formatter = format!("{}/test/format.{}", env!("CARGO_MANIFEST_DIR"), kind);
        let mut builder = Formatting::builder();
        builder
            .kind(kind)
            .formatter(formatter)
            .config_files(["format-config"].iter().cloned());
        builder
    }

    #[test]
    fn test_formatting_pass() {
        let check = formatting_check("simple").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_pass",
            OK_FORMAT_COMMIT,
            BAD_FORMAT_COMMIT,
            &conf,
        );
        test_result_ok(result);
    }

    #[test]
    fn test_formatting_env() {
        let check = formatting_check("env").build().unwrap();
        let conf = {
            let mut conf = make_check_conf(&check);
            conf.add_configuration("formatter.env_key", "GIT_CHECKS_TEST_FORMATTING_ENV");
            conf
        };

        let result = test_check_base(
            "test_formatting_env",
            ENV_FORMAT_COMMIT,
            ENV_BAD_FORMAT_COMMIT,
            &conf,
        );
        test_result_ok(result);
    }

    #[test]
    fn test_formatting_env_missing() {
        let check = formatting_check("env").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_env_missing",
            ENV_FORMAT_COMMIT,
            ENV_BAD_FORMAT_COMMIT,
            &conf,
        );
        test_result_errors(result, &[
            "commit 11107007b2875ac2f98a89085ff8f4f631b369ae is not allowed because the following \
             files could not be formatted by the 'env' check: `ok.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_pass_with_arg() {
        let check = formatting_check("with_arg").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check("test_formatting_pass_with_arg", WITH_ARG_COMMIT, &conf);
        test_result_errors(result, &[
            "commit dfa4c021e96f1e94634ad1787f31c2abdbeaff99 is not allowed because the following \
             files are not formatted according to the 'with_arg' check: `with-arg.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_fail() {
        let check = formatting_check("simple").build().unwrap();
        let result = run_check(
            "test_formatting_formatter_fail",
            MISSING_CONFIG_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 220efbb4d0380fe932b70444fe15e787506080b0 is not allowed because the following \
             files could not be formatted by the 'simple' check: `empty.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_fail_named() {
        let check = formatting_check("simple").name("renamed").build().unwrap();
        let result = run_check(
            "test_formatting_formatter_fail_named",
            MISSING_CONFIG_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 220efbb4d0380fe932b70444fe15e787506080b0 is not allowed because the following \
             files could not be formatted by the 'renamed' check: `empty.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_fail_fix_message() {
        let check = formatting_check("simple")
            .fix_message("These may be fixed by magic.")
            .build()
            .unwrap();
        let result = run_check(
            "test_formatting_formatter_fail_fix_message",
            MISSING_CONFIG_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 220efbb4d0380fe932b70444fe15e787506080b0 is not allowed because the following \
             files could not be formatted by the 'simple' check: `empty.txt`. These may be fixed \
             by magic.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_untracked_files() {
        let check = formatting_check("untracked").build().unwrap();
        let result = run_check(
            "test_formatting_formatter_untracked_files",
            MISSING_CONFIG_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 220efbb4d0380fe932b70444fe15e787506080b0 is not allowed because the following \
             files were created by the 'untracked' check: `untracked`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_timeout() {
        let check = formatting_check("timeout")
            .timeout(Duration::from_secs(1))
            .build()
            .unwrap();
        let result = run_check(
            "test_formatting_formatter_timeout",
            TIMEOUT_CONFIG_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 62f5eac20c5021cf323c757a4d24234c81c9c7ad is not allowed because the following \
             files could not be formatted by the 'timeout' check: `empty.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_untracked_files_ignored() {
        let check = formatting_check("untracked").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_formatter_untracked_files_ignored",
            IGNORE_UNTRACKED_COMMIT,
            OK_FORMAT_COMMIT,
            &conf,
        );
        test_result_ok(result);
    }

    #[test]
    fn test_formatting_formatter_modified_files() {
        let check = formatting_check("simple").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_formatter_modified_files",
            BAD_FORMAT_COMMIT,
            ADD_CONFIG_COMMIT,
            &conf,
        );
        test_result_errors(result, &[
            "commit e9a08d956553f94e9c8a0a02b11ca60f62de3c2b is not allowed because the following \
             files are not formatted according to the 'simple' check: `bad.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_modified_files_topic() {
        let check = formatting_check("simple").build().unwrap();
        let conf = make_topic_check_conf(&check);

        let result = test_check_base(
            "test_formatting_formatter_modified_files_topic",
            BAD_FORMAT_COMMIT,
            ADD_CONFIG_COMMIT,
            &conf,
        );
        test_result_errors(
            result,
            &["the following files are not formatted according to the 'simple' check: `bad.txt`."],
        );
    }

    #[test]
    fn test_formatting_formatter_modified_files_topic_fixed() {
        let check = formatting_check("simple").build().unwrap();
        run_topic_check_ok(
            "test_formatting_formatter_modified_files_topic_fixed",
            FIX_BAD_FORMAT_COMMIT,
            check,
        );
    }

    #[test]
    fn test_formatting_formatter_many_modified_files() {
        let check = formatting_check("simple").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_formatter_many_modified_files",
            MANY_BAD_FORMAT_COMMIT,
            ADD_CONFIG_COMMIT,
            &conf,
        );
        test_result_errors(result, &[
            "commit f0d10d9385ef697175c48fa72324c33d5e973f4b is not allowed because the following \
             files are not formatted according to the 'simple' check: `1.bad.txt`, `2.bad.txt`, \
             `3.bad.txt`, `4.bad.txt`, `5.bad.txt`, `6.bad.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_many_more_modified_files() {
        let check = formatting_check("simple").build().unwrap();
        let conf = make_check_conf(&check);

        let result = test_check_base(
            "test_formatting_formatter_many_more_modified_files",
            MANY_MORE_BAD_FORMAT_COMMIT,
            ADD_CONFIG_COMMIT,
            &conf,
        );
        test_result_errors(result, &[
            "commit 0e80ff6dd2495571b7d255e39fb3e7cc9f487fb7 is not allowed because the following \
             files are not formatted according to the 'simple' check: `1.bad.txt`, `2.bad.txt`, \
             `3.bad.txt`, `4.bad.txt`, `5.bad.txt`, and 2 others.",
        ]);
    }

    #[test]
    fn test_formatting_script_deleted_files() {
        let check = formatting_check("delete").build().unwrap();
        let result = run_check(
            "test_formatting_script_deleted_files",
            DELETE_FORMAT_COMMIT,
            check,
        );
        test_result_errors(result, &[
            "commit 31446c81184df35498814d6aa3c7f933dddf91c2 is not allowed because the following \
            files are not formatted according to the 'delete' check: `remove.txt`.",
        ]);
    }

    #[test]
    fn test_formatting_formatter_noexec() {
        let check = formatting_check("noexec").build().unwrap();
        let result = run_check(
            "test_formatting_formatter_noexec",
            NOEXEC_CONFIG_COMMIT,
            check,
        );

        assert_eq!(result.warnings().len(), 0);
        assert_eq!(result.alerts().len(), 1);
        assert_eq!(
            result.alerts()[0],
            "failed to run the formatting check on commit 1b5be48f8ce45b8fec155a1787cabb6995194ce5",
        );
        assert_eq!(result.errors().len(), 0);
        assert!(!result.temporary());
        assert!(!result.allowed());
        assert!(!result.pass());
    }

    #[test]
    fn test_formatting_replace_path() {
        let check = formatting_check("simple").build().unwrap();
        let conf = {
            let mut conf = make_check_conf(&check);
            conf.add_configuration(
                "formatter.simple.path",
                format!("{}/test/noexist/format.simple", env!("CARGO_MANIFEST_DIR")),
            );
            conf
        };

        let result = test_check_base(
            "test_formatting_replace_path",
            OK_FORMAT_COMMIT,
            BAD_FORMAT_COMMIT,
            &conf,
        );

        assert_eq!(result.warnings().len(), 0);
        assert_eq!(result.alerts().len(), 1);
        assert_eq!(
            result.alerts()[0],
            "failed to run the formatting check on commit b77d2a5d63cd6afa599d0896dafff95f1ace50b6",
        );
        assert_eq!(result.errors().len(), 0);
        assert!(!result.temporary());
        assert!(!result.allowed());
        assert!(!result.pass());
    }
}