use crate::utils::{prompt, run_in_alternate_screen, DiffStyles};
use ansi_term::{Color, Style};
use anyhow::Result;
use serde_yaml::to_string;
use std::io::Write;
use super::{CaseResult, CaseStatus, SnapshotAction, TestCase};
#[derive(Debug)]
pub enum TestResult {
Success { message: String },
RuleFail { message: String },
MismatchSnapshotOnly { message: String },
}
#[derive(Default, Clone)]
pub(crate) struct TestReportStyle {
ok: Style,
summary_skip: Style,
summary_pass: Style,
summary_fail: Style,
case_id: Style,
result_label: Style,
section: Style,
diff_styles: DiffStyles,
}
impl TestReportStyle {
pub(crate) fn colored() -> Self {
let summary = Style::new().fg(Color::White).bold();
Self {
ok: Color::Green.normal(),
summary_skip: summary.on(Color::Yellow),
summary_pass: summary.on(Color::Green),
summary_fail: summary.on(Color::Red),
case_id: Style::new().bold(),
result_label: Style::new().underline(),
section: Style::new().italic(),
diff_styles: DiffStyles::colored(),
}
}
}
pub(super) trait Reporter {
type Output: Write;
fn get_output(&mut self) -> &mut Self::Output;
fn style(&self) -> &TestReportStyle;
fn before_report(&mut self, test_cases: &[TestCase]) -> Result<()> {
report_case_number(self.get_output(), test_cases)
}
fn after_report(&mut self, results: &[CaseResult]) -> Result<TestResult> {
let mut passed = 0;
let mut failed = 0;
for result in results {
if result.passed() {
passed += 1;
} else {
failed += 1;
}
}
let message = format!("{passed} passed; {failed} failed;");
if failed > 0 {
let all_snapshot_mismatches = results
.iter()
.filter(|r| !r.passed())
.all(|r| r.is_snapshot_mismatch_only_failure());
if all_snapshot_mismatches {
Ok(TestResult::MismatchSnapshotOnly {
message: format!("test failed. {message}"),
})
} else {
Ok(TestResult::RuleFail {
message: format!("test failed. {message}"),
})
}
} else {
let result = self.style().ok.paint("ok").to_string();
Ok(TestResult::Success {
message: format!("test result: {result}. {message}"),
})
}
}
fn report_failed_cases(&mut self, results: &mut [CaseResult]) -> Result<()> {
let output = self.get_output();
writeln!(output)?;
writeln!(output, "----------- Case Details -----------")?;
for result in results {
if result.passed() {
continue;
}
for status in &mut result.cases {
if !self.report_case_detail(result.id, status)? {
return Ok(());
}
}
}
Ok(())
}
fn report_summaries(&mut self, results: &[CaseResult]) -> Result<()> {
for result in results {
self.report_case_summary(result.id, &result.cases)?;
}
let output = self.get_output();
writeln!(output)?;
Ok(())
}
fn report_case_summary(&mut self, case_id: &str, summary: &[CaseStatus]) -> Result<()> {
let passed = summary.iter().all(CaseStatus::is_pass);
let style = self.style();
let case_status = if summary.is_empty() {
style.summary_skip.paint("SKIP").to_string()
} else if passed {
style.summary_pass.paint("PASS").to_string()
} else {
style.summary_fail.paint("FAIL").to_string()
};
let summary = report_summary(summary);
writeln!(self.get_output(), "{case_status} {case_id} {summary}")?;
Ok(())
}
fn report_case_detail(&mut self, case_id: &str, result: &mut CaseStatus) -> Result<bool>;
fn collect_snapshot_action(&self) -> SnapshotAction;
}
fn report_case_number(output: &mut impl Write, test_cases: &[TestCase]) -> Result<()> {
writeln!(output, "Running {} tests", test_cases.len())?;
Ok(())
}
fn report_summary(summary: &[CaseStatus]) -> String {
if summary.len() > 40 {
let mut pass = 0;
let mut updated = 0;
let mut wrong = 0;
let mut missing = 0;
let mut noisy = 0;
let mut error = 0;
for s in summary {
match s {
CaseStatus::Validated | CaseStatus::Reported => pass += 1,
CaseStatus::Updated { .. } => updated += 1,
CaseStatus::Wrong { .. } => wrong += 1,
CaseStatus::Missing(_) => missing += 1,
CaseStatus::Noisy(_) => noisy += 1,
CaseStatus::Error => error += 1,
}
}
let stats = vec![
("Pass", pass),
("Updated", updated),
("Wrong", wrong),
("Missing", missing),
("Noisy", noisy),
("Error", error),
];
let result: Vec<_> = stats
.into_iter()
.filter_map(|(label, count)| {
if count > 0 {
Some(format!("{label} × {count}"))
} else {
None
}
})
.collect();
let result = result.join(", ");
format!("{result:.^50}")
} else {
summary
.iter()
.map(|s| match s {
CaseStatus::Validated | CaseStatus::Reported => '.',
CaseStatus::Wrong { .. } => 'W',
CaseStatus::Updated { .. } => 'U',
CaseStatus::Missing(_) => 'M',
CaseStatus::Noisy(_) => 'N',
CaseStatus::Error => 'E',
})
.collect()
}
}
fn indented_write<W: Write>(output: &mut W, code: &str) -> Result<()> {
for line in code.lines() {
writeln!(output, " {line}")?;
}
Ok(())
}
fn report_case_detail_impl<W: Write>(
output: &mut W,
case_id: &str,
result: &CaseStatus,
style: &TestReportStyle,
) -> Result<bool> {
let case_id = style.case_id.paint(case_id);
let noisy_label = style.result_label.paint("Noisy");
let missing_label = style.result_label.paint("Missing");
let wrong_label = style.result_label.paint("Wrong");
let error_label = style.result_label.paint("Error");
let updated_label = style.result_label.paint("Updated");
let diff_label = style.section.paint("Diff:");
let snapshot_label = style.section.paint("Generated Snapshot:");
let for_code_label = style.section.paint("For Code:");
let diff_styles = &style.diff_styles;
match result {
CaseStatus::Validated | CaseStatus::Reported => (),
CaseStatus::Updated { source, .. } => {
writeln!(
output,
"[{updated_label}] Rule {case_id}'s snapshot baseline has been updated."
)?;
writeln!(output)?;
indented_write(output, source)?;
writeln!(output)?;
}
CaseStatus::Wrong {
source,
actual,
expected,
} => {
if let Some(expected) = expected {
writeln!(
output,
"[{wrong_label}] {case_id} snapshot is different from baseline."
)?;
let actual_str = to_string(&actual)?;
let expected_str = to_string(&expected)?;
writeln!(output, "{diff_label}")?;
diff_styles.print_diff(&expected_str, &actual_str, output, 3)?;
} else {
writeln!(output, "[{wrong_label}] No {case_id} baseline found.")?;
writeln!(output, "{snapshot_label}")?;
indented_write(output, &to_string(&actual)?)?;
}
writeln!(output, "{for_code_label}")?;
indented_write(output, source)?;
writeln!(output)?;
}
CaseStatus::Missing(s) => {
writeln!(
output,
"[{missing_label}] Expect rule {case_id} to report issues, but none found in:"
)?;
writeln!(output)?;
indented_write(output, s)?;
writeln!(output)?;
}
CaseStatus::Noisy(s) => {
writeln!(
output,
"[{noisy_label}] Expect {case_id} to report no issue, but some issues found in:"
)?;
writeln!(output)?;
indented_write(output, s)?;
writeln!(output)?;
}
CaseStatus::Error => {
writeln!(output, "[{error_label}] Fail to apply fix to {case_id}")?;
}
}
Ok(true)
}
pub struct DefaultReporter<Output: Write> {
pub output: Output,
pub update_all: bool,
pub style: TestReportStyle,
}
impl<O: Write> Reporter for DefaultReporter<O> {
type Output = O;
fn get_output(&mut self) -> &mut Self::Output {
&mut self.output
}
fn style(&self) -> &TestReportStyle {
&self.style
}
fn report_case_detail(&mut self, case_id: &str, result: &mut CaseStatus) -> Result<bool> {
if self.update_all {
result.accept();
}
report_case_detail_impl(&mut self.output, case_id, result, &self.style)
}
fn collect_snapshot_action(&self) -> SnapshotAction {
if self.update_all {
SnapshotAction::NeedUpdate
} else {
SnapshotAction::AcceptNone
}
}
}
pub struct InteractiveReporter<Output: Write> {
pub output: Output,
pub should_accept_all: bool,
pub style: TestReportStyle,
}
const PROMPT: &str = "Accept new snapshot? (Yes[y], No[n], Accept All[a], Quit[q])";
impl<O: Write> Reporter for InteractiveReporter<O> {
type Output = O;
fn get_output(&mut self) -> &mut Self::Output {
&mut self.output
}
fn style(&self) -> &TestReportStyle {
&self.style
}
fn collect_snapshot_action(&self) -> SnapshotAction {
SnapshotAction::NeedUpdate
}
fn report_case_detail(&mut self, case_id: &str, status: &mut CaseStatus) -> Result<bool> {
if matches!(status, CaseStatus::Validated | CaseStatus::Reported) {
return Ok(true);
}
run_in_alternate_screen(|| {
let style = &self.style;
report_case_detail_impl(&mut self.output, case_id, status, style)?;
if !matches!(status, CaseStatus::Wrong { .. }) {
let response = prompt("Next[enter], Quit[q]", "q", Some('\n'))?;
return Ok(response != 'q');
}
if self.should_accept_all {
return self.accept_new_snapshot(status);
}
let response = prompt(PROMPT, "ynaq", Some('n'))?;
match response {
'y' => self.accept_new_snapshot(status),
'n' => Ok(true),
'a' => {
self.should_accept_all = true;
self.accept_new_snapshot(status)
}
'q' => Ok(false),
_ => unreachable!(),
}
})
}
}
impl<O: Write> InteractiveReporter<O> {
fn accept_new_snapshot(&mut self, status: &mut CaseStatus) -> Result<bool> {
let accepted = status.accept();
debug_assert!(accepted, "status should be updated");
Ok(true)
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::verify::snapshot::TestSnapshot;
use crate::verify::test::TEST_RULE;
const MOCK: &str = "hello";
fn mock_case_status() -> Vec<CaseStatus<'static>> {
vec![
CaseStatus::Reported,
CaseStatus::Missing(MOCK),
CaseStatus::Noisy(MOCK),
CaseStatus::Wrong {
source: MOCK,
actual: TestSnapshot {
fixed: None,
labels: vec![],
},
expected: None,
},
CaseStatus::Error,
]
}
#[test]
fn test_report_summary() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
reporter.report_case_summary(TEST_RULE, &mock_case_status())?;
let s = String::from_utf8(reporter.output)?;
assert!(s.contains(".MNWE"));
Ok(())
}
#[test]
fn test_many_cases() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
use std::iter::repeat_with;
let cases: Vec<_> = repeat_with(mock_case_status).flatten().take(50).collect();
reporter.report_case_summary(TEST_RULE, &cases)?;
let s = String::from_utf8(reporter.output)?;
assert!(!s.contains(".MNWE"));
assert!(s.contains("Pass × 10, Wrong × 10, Missing × 10, Noisy × 10, Error × 10"));
Ok(())
}
#[test]
fn test_valid_case_detail() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
reporter.report_case_detail(TEST_RULE, &mut CaseStatus::Reported)?;
reporter.report_case_detail(TEST_RULE, &mut CaseStatus::Validated)?;
let s = String::from_utf8(reporter.output)?;
assert_eq!(s, "");
Ok(())
}
#[test]
fn test_invalid_case_detail() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
reporter.report_case_detail(TEST_RULE, &mut CaseStatus::Missing(MOCK))?;
reporter.report_case_detail(TEST_RULE, &mut CaseStatus::Noisy(MOCK))?;
let s = String::from_utf8(reporter.output)?;
assert!(s.contains("Missing"));
assert!(s.contains("Noisy"));
assert!(!s.contains("Error"));
assert!(!s.contains("Wrong"));
assert!(s.contains(MOCK));
assert!(s.contains(TEST_RULE));
Ok(())
}
#[test]
fn test_after_report_snapshot_mismatch_only() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
let results = vec![CaseResult {
id: TEST_RULE,
cases: vec![
CaseStatus::Wrong {
source: MOCK,
actual: TestSnapshot {
fixed: None,
labels: vec![],
},
expected: None,
},
CaseStatus::Wrong {
source: MOCK,
actual: TestSnapshot {
fixed: None,
labels: vec![],
},
expected: None,
},
],
}];
let test_result = reporter.after_report(&results)?;
assert!(matches!(
test_result,
TestResult::MismatchSnapshotOnly { message: _ }
));
Ok(())
}
#[test]
fn test_after_report_mixed_failures() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
let results = vec![CaseResult {
id: TEST_RULE,
cases: vec![
CaseStatus::Wrong {
source: MOCK,
actual: TestSnapshot {
fixed: None,
labels: vec![],
},
expected: None,
},
CaseStatus::Missing(MOCK), ],
}];
let test_result = reporter.after_report(&results)?;
assert!(matches!(test_result, TestResult::RuleFail { message: _ }));
Ok(())
}
#[test]
fn test_after_report_success() -> Result<()> {
let output = vec![];
let mut reporter = DefaultReporter {
output,
update_all: false,
style: TestReportStyle::default(),
};
let results = vec![CaseResult {
id: TEST_RULE,
cases: vec![CaseStatus::Validated, CaseStatus::Reported],
}];
let test_result = reporter.after_report(&results)?;
assert!(matches!(test_result, TestResult::Success { .. }));
Ok(())
}
}