use std::{
collections::{hash_map::HashMap, HashSet},
convert::TryFrom,
env,
fs::{canonicalize, read_to_string},
io::{self, Read, Write},
os::{
raw::c_int,
unix::{io::AsRawFd, process::ExitStatusExt},
},
path::{Path, PathBuf, MAIN_SEPARATOR},
process::{self, Command, ExitStatus},
str,
sync::{
atomic::{AtomicUsize, Ordering},
Arc, Mutex,
},
thread::sleep,
time::{Duration, Instant},
};
use fm::{FMBuilder, FMatchError};
use getopts::Options;
use libc::{
close, fcntl, poll, pollfd, F_GETFL, F_SETFL, O_NONBLOCK, POLLERR, POLLHUP, POLLIN, POLLOUT,
};
use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
use threadpool::ThreadPool;
use walkdir::WalkDir;
use crate::{fatal, parser::parse_tests};
const READBUF: usize = 1024 * 4;
const TIMEOUT: u64 = 60;
const INITIAL_WAIT_TIMEOUT: u64 = 10000;
const MAX_WAIT_TIMEOUT: u64 = 250_000_000;
pub struct LangTester {
use_cmdline_args: bool,
test_file_filter: Option<Box<dyn Fn(&Path) -> bool>>,
cmdline_filters: Option<Vec<String>>,
inner: Arc<LangTesterPooler>,
}
struct LangTesterPooler {
test_dir: Option<PathBuf>,
test_threads: usize,
ignored: bool,
nocapture: bool,
test_extract: Option<Box<dyn Fn(&str) -> Option<String> + Send + Sync>>,
fm_options: Option<
Box<dyn for<'a> Fn(&'a Path, TestStream, FMBuilder<'a>) -> FMBuilder<'a> + Send + Sync>,
>,
test_cmds: Option<Box<dyn Fn(&Path) -> Vec<(&str, Command)> + Send + Sync>>,
}
#[derive(Clone, Copy)]
pub enum TestStream {
Stderr,
Stdout,
}
impl LangTester {
pub fn new() -> Self {
LangTester {
test_file_filter: None,
use_cmdline_args: true,
cmdline_filters: None,
inner: Arc::new(LangTesterPooler {
test_dir: None,
ignored: false,
nocapture: false,
test_threads: num_cpus::get(),
fm_options: None,
test_extract: None,
test_cmds: None,
}),
}
}
pub fn test_dir(&mut self, test_dir: &str) -> &mut Self {
let inner = Arc::get_mut(&mut self.inner).unwrap();
inner.test_dir = Some(canonicalize(test_dir).unwrap());
self
}
pub fn test_threads(&mut self, test_threads: usize) -> &mut Self {
let inner = Arc::get_mut(&mut self.inner).unwrap();
inner.test_threads = test_threads;
self
}
pub fn test_file_filter<F>(&mut self, test_file_filter: F) -> &mut Self
where
F: 'static + Fn(&Path) -> bool,
{
self.test_file_filter = Some(Box::new(test_file_filter));
self
}
pub fn test_extract<F>(&mut self, test_extract: F) -> &mut Self
where
F: 'static + Fn(&str) -> Option<String> + Send + Sync,
{
Arc::get_mut(&mut self.inner).unwrap().test_extract = Some(Box::new(test_extract));
self
}
pub fn fm_options<F>(&mut self, fm_options: F) -> &mut Self
where
F: 'static + for<'a> Fn(&'a Path, TestStream, FMBuilder<'a>) -> FMBuilder<'a> + Send + Sync,
{
Arc::get_mut(&mut self.inner).unwrap().fm_options = Some(Box::new(fm_options));
self
}
pub fn test_cmds<F>(&mut self, test_cmds: F) -> &mut Self
where
F: 'static + Fn(&Path) -> Vec<(&str, Command)> + Send + Sync,
{
Arc::get_mut(&mut self.inner).unwrap().test_cmds = Some(Box::new(test_cmds));
self
}
pub fn use_cmdline_args(&mut self, use_cmdline_args: bool) -> &mut Self {
self.use_cmdline_args = use_cmdline_args;
self
}
fn validate(&self) {
if self.inner.test_dir.is_none() {
fatal("test_dir must be specified.");
}
if self.inner.test_extract.is_none() {
fatal("test_extract must be specified.");
}
if self.inner.test_cmds.is_none() {
fatal("test_cmds must be specified.");
}
}
fn test_files(&self) -> (Vec<PathBuf>, usize) {
let mut num_filtered = 0;
let paths = WalkDir::new(self.inner.test_dir.as_ref().unwrap())
.into_iter()
.filter_map(|x| x.ok())
.filter(|x| x.file_type().is_file())
.map(|x| canonicalize(x.into_path()).unwrap())
.filter(|x| match self.test_file_filter.as_ref() {
Some(f) => f(x),
None => true,
})
.filter(|x| {
let x_path = x.to_str().unwrap();
match self.cmdline_filters.as_ref() {
Some(fs) => {
debug_assert!(self.use_cmdline_args);
for f in fs {
if x_path.contains(f) {
return true;
}
}
num_filtered += 1;
false
}
None => true,
}
})
.collect();
(paths, num_filtered)
}
pub fn run(&mut self) {
self.validate();
if self.use_cmdline_args {
let args: Vec<String> = env::args().collect();
let matches = Options::new()
.optflag("h", "help", "")
.optflag("", "ignored", "Run only ignored tests")
.optflag(
"",
"nocapture",
"Pass command stderr/stdout through to the terminal",
)
.optopt(
"",
"test-threads",
"Number of threads used for running tests in parallel",
"n_threads",
)
.parse(&args[1..])
.unwrap_or_else(|_| usage());
if matches.opt_present("h") {
usage();
}
if matches.opt_present("ignored") {
Arc::get_mut(&mut self.inner).unwrap().ignored = true;
}
if matches.opt_present("nocapture") {
Arc::get_mut(&mut self.inner).unwrap().nocapture = true;
}
if let Some(s) = matches.opt_str("test-threads") {
let test_threads = s.parse::<usize>().unwrap_or_else(|_| usage());
if test_threads == 0 {
fatal("Must specify more than 0 threads.");
}
Arc::get_mut(&mut self.inner).unwrap().test_threads = test_threads;
}
if !matches.free.is_empty() {
self.cmdline_filters = Some(matches.free);
}
}
let (test_files, num_filtered) = self.test_files();
eprint!("\nrunning {} tests", test_files.len());
let test_files_len = test_files.len();
let (failures, num_ignored) = test_file(test_files, Arc::clone(&self.inner));
self.pp_failures(&failures, test_files_len, num_ignored, num_filtered);
if !failures.is_empty() {
process::exit(1);
}
}
fn pp_failures(
&self,
failures: &[(String, TestFailure)],
test_files_len: usize,
num_ignored: usize,
num_filtered: usize,
) {
if !failures.is_empty() {
eprintln!("\n\nfailures:");
for (test_fname, test) in failures {
if let Some(ref status) = test.status {
eprintln!("\n---- lang_tests::{} status ----\n{}", test_fname, status);
}
if test.stdin_remaining != 0 {
eprintln!(
"\n---- lang_tests::{} stdin ----\n{} bytes of stdin were not consumed",
test_fname, test.stdin_remaining
);
}
if let Some(ref stderr) = test.stderr {
eprintln!("\n---- lang_tests::{} stderr ----\n", test_fname);
if let Some(ref stderr_match) = test.stderr_match {
eprint!("{}", stderr_match);
} else {
eprintln!("{}", stderr);
}
}
if let Some(ref stdout) = test.stdout {
eprintln!("\n---- lang_tests::{} stdout ----\n", test_fname);
if let Some(ref stdout_match) = test.stdout_match {
eprint!("{}", stdout_match);
} else {
eprintln!("{}", stdout);
}
}
}
eprintln!("\nfailures:");
for (test_fname, _) in failures {
eprint!(" lang_tests::{}", test_fname);
}
}
eprint!("\n\ntest result: ");
if failures.is_empty() {
write_with_colour("ok", Color::Green);
} else {
write_with_colour("FAILED", Color::Red);
}
eprintln!(
". {} passed; {} failed; {} ignored; 0 measured; {} filtered out\n",
test_files_len - failures.len(),
failures.len(),
num_ignored,
num_filtered
);
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Status {
Success,
Error,
Signal,
Int(i32),
}
#[derive(Clone, Debug)]
pub(crate) struct TestCmd<'a> {
pub status: Status,
pub stdin: Option<String>,
pub stderr: Vec<&'a str>,
pub stdout: Vec<&'a str>,
pub args: Vec<String>,
}
impl<'a> TestCmd<'a> {
pub fn default() -> Self {
Self {
status: Status::Success,
stdin: None,
stderr: vec!["..."],
stdout: vec!["..."],
args: Vec::new(),
}
}
}
pub(crate) struct Tests<'a> {
pub ignore: bool,
pub tests: HashMap<String, TestCmd<'a>>,
}
#[derive(Debug, PartialEq)]
struct TestFailure {
status: Option<String>,
stdin_remaining: usize,
stderr: Option<String>,
stderr_match: Option<FMatchError>,
stdout: Option<String>,
stdout_match: Option<FMatchError>,
}
fn write_with_colour(s: &str, colour: Color) {
let mut stderr = StandardStream::stderr(ColorChoice::Always);
stderr.set_color(ColorSpec::new().set_fg(Some(colour))).ok();
io::stderr().write_all(s.as_bytes()).ok();
stderr.reset().ok();
}
fn write_ignored(test_name: &str, message: &str, inner: Arc<LangTesterPooler>) {
let stderr = StandardStream::stderr(ColorChoice::Always);
let mut handle = stderr.lock();
if inner.test_threads > 1 {
handle
.write_all(&format!("\ntest lang_tests::{} ... ", test_name).as_bytes())
.ok();
}
handle
.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))
.ok();
handle.write_all(b"ignored").ok();
handle.reset().ok();
if !message.is_empty() {
handle.write_all(format!(" ({})", message).as_bytes()).ok();
}
}
fn usage() -> ! {
eprintln!("Usage: [--ignored] [--nocapture] [--test-threads=<n>] [<filter1>] [... <filtern>]");
process::exit(1);
}
fn check_names<'a>(cmd_pairs: &[(String, Command)], tests: &HashMap<String, TestCmd<'a>>) {
let cmd_names = cmd_pairs.iter().map(|x| &x.0).collect::<HashSet<_>>();
let test_names = tests.keys().map(|x| x).collect::<HashSet<_>>();
let diff = test_names
.difference(&cmd_names)
.map(|x| x.as_str())
.collect::<Vec<_>>();
if !diff.is_empty() {
fatal(&format!(
"Command name(s) '{}' in tests are not found in the actual commands.",
diff.join(", ")
));
}
}
fn test_file(
test_files: Vec<PathBuf>,
inner: Arc<LangTesterPooler>,
) -> (Vec<(String, TestFailure)>, usize) {
let failures = Arc::new(Mutex::new(Vec::new()));
let num_ignored = Arc::new(AtomicUsize::new(0));
let pool = ThreadPool::new(inner.test_threads);
for p in test_files {
let test_fname = test_fname(inner.test_dir.as_ref().unwrap(), &p);
let num_ignored = num_ignored.clone();
let failures = failures.clone();
let inner = inner.clone();
pool.execute(move || {
if inner.test_threads == 1 {
eprint!("\ntest lang_test::{} ... ", test_fname);
}
let all_str = read_to_string(p.as_path())
.unwrap_or_else(|_| fatal(&format!("Couldn't read {}", test_fname)));
let test_str = inner.test_extract.as_ref().unwrap()(&all_str).unwrap_or_else(|| {
fatal(&format!("Couldn't extract test string from {}", test_fname))
});
if test_str.is_empty() {
write_ignored(test_fname.as_str(), "test string is empty", inner);
num_ignored.fetch_add(1, Ordering::Relaxed);
return;
}
let tests = parse_tests(&test_str);
if (inner.ignored && !tests.ignore) || (!inner.ignored && tests.ignore) {
write_ignored(test_fname.as_str(), "", inner);
num_ignored.fetch_add(1, Ordering::Relaxed);
return;
}
if run_tests(Arc::clone(&inner), tests.tests, p, test_fname, failures) {
num_ignored.fetch_add(1, Ordering::Relaxed);
}
});
}
pool.join();
let failures = Mutex::into_inner(Arc::try_unwrap(failures).unwrap()).unwrap();
(failures, Arc::try_unwrap(num_ignored).unwrap().into_inner())
}
fn test_fname(test_dir_path: &Path, test_fpath: &Path) -> String {
if let Some(test_fpath) = test_fpath.as_os_str().to_str() {
if let Some(testdir_path) = test_dir_path.as_os_str().to_str() {
if test_fpath.starts_with(testdir_path) {
return test_fpath[testdir_path.len() + MAIN_SEPARATOR.len_utf8()..]
.to_owned()
.replace(MAIN_SEPARATOR, "::");
}
}
}
test_fpath.file_stem().unwrap().to_str().unwrap().to_owned()
}
fn run_tests<'a>(
inner: Arc<LangTesterPooler>,
tests: HashMap<String, TestCmd<'a>>,
path: PathBuf,
test_fname: String,
failures: Arc<Mutex<Vec<(String, TestFailure)>>>,
) -> bool {
if !cfg!(unix) && tests.values().any(|t| t.status == Status::Signal) {
write_ignored(
test_fname.as_str(),
"signal termination not supported on this platform",
inner,
);
return true;
}
let cmd_pairs = inner.test_cmds.as_ref().unwrap()(path.as_path())
.into_iter()
.map(|(test_name, cmd)| (test_name.to_lowercase(), cmd))
.collect::<Vec<_>>();
check_names(&cmd_pairs, &tests);
let mut failure = TestFailure {
status: None,
stdin_remaining: 0,
stderr: None,
stderr_match: None,
stdout: None,
stdout_match: None,
};
for (cmd_name, mut cmd) in cmd_pairs {
let default_test = TestCmd::default();
let test = tests.get(&cmd_name).unwrap_or(&default_test);
cmd.args(&test.args);
let (status, stdin_remaining, stderr, stdout) =
run_cmd(inner.clone(), &test_fname, cmd, &test);
let mut meant_to_error = false;
let stderr_str = test.stderr.join("\n");
let mut stderr_fmb = FMBuilder::new(&stderr_str).unwrap();
let stdout_str = test.stdout.join("\n");
let mut stdout_fmb = FMBuilder::new(&stdout_str).unwrap();
if let Some(ref fm_options) = inner.fm_options {
stderr_fmb = fm_options(path.as_path(), TestStream::Stderr, stderr_fmb);
stdout_fmb = fm_options(path.as_path(), TestStream::Stdout, stdout_fmb);
}
let match_stderr = stderr_fmb.build().unwrap().matches(&stderr);
let match_stdout = stdout_fmb.build().unwrap().matches(&stdout);
let pass_status = match test.status {
Status::Success => status.success(),
Status::Error => {
meant_to_error = true;
!status.success()
}
Status::Signal => status.signal().is_some(),
Status::Int(i) => status.code() == Some(i),
};
if !(pass_status && stdin_remaining == 0 && match_stderr.is_ok() && match_stdout.is_ok()) {
if !pass_status || failure.status.is_none() {
match test.status {
Status::Success | Status::Error => {
if status.success() {
failure.status = Some("Success".to_owned());
} else if status.code().is_none() {
failure.status = Some(format!(
"Exited due to signal: {}",
status.signal().unwrap()
));
} else {
failure.status = Some("Error".to_owned());
}
}
Status::Signal => {
failure.status = Some("Exit was not due to signal".to_owned());
}
Status::Int(_) => {
failure.status =
Some(status.code().map(|x| x.to_string()).unwrap_or_else(|| {
format!("Exited due to signal: {}", status.signal().unwrap())
}))
}
}
}
if match_stderr.is_err() || failure.stderr.is_none() {
failure.stderr = Some(stderr);
}
if let Err(e) = match_stderr {
failure.stderr_match = Some(e);
}
if match_stdout.is_err() || failure.stdout.is_none() {
failure.stdout = Some(stdout);
}
if let Err(e) = match_stdout {
failure.stdout_match = Some(e);
}
failure.stdin_remaining = stdin_remaining;
break;
}
if !status.success() && meant_to_error {
break;
}
}
{
let stderr = StandardStream::stderr(ColorChoice::Always);
let mut handle = stderr.lock();
if inner.test_threads > 1 {
handle
.write_all(&format!("\ntest lang_tests::{} ... ", test_fname).as_bytes())
.ok();
}
if failure
!= (TestFailure {
status: None,
stdin_remaining: 0,
stderr: None,
stderr_match: None,
stdout: None,
stdout_match: None,
})
{
let mut failures = failures.lock().unwrap();
failures.push((test_fname, failure));
handle
.set_color(ColorSpec::new().set_fg(Some(Color::Red)))
.ok();
handle.write_all(b"FAILED").ok();
handle.reset().ok();
} else {
handle
.set_color(ColorSpec::new().set_fg(Some(Color::Green)))
.ok();
handle.write_all(b"ok").ok();
handle.reset().ok();
}
}
false
}
fn run_cmd(
inner: Arc<LangTesterPooler>,
test_fname: &str,
mut cmd: Command,
test: &TestCmd,
) -> (ExitStatus, usize, String, String) {
let mut child = cmd
.stdin(process::Stdio::piped())
.stderr(process::Stdio::piped())
.stdout(process::Stdio::piped())
.spawn()
.unwrap_or_else(|_| fatal(&format!("Couldn't run command {:?}.", cmd)));
let stdin = child.stdin.as_mut().unwrap();
let stderr = child.stderr.as_mut().unwrap();
let stdout = child.stdout.as_mut().unwrap();
let stdin_fd = stdin.as_raw_fd();
let stderr_fd = stderr.as_raw_fd();
let stdout_fd = stdout.as_raw_fd();
if set_nonblock(stdin_fd)
.and_then(|_| set_nonblock(stderr_fd))
.and_then(|_| set_nonblock(stdout_fd))
.is_err()
{
fatal("Couldn't set stdin and/or stderr and/or stdout to be non-blocking");
}
let mut cap_stderr = String::new();
let mut cap_stdout = String::new();
let mut pollfds = [
pollfd {
fd: stdin_fd,
events: 0,
revents: 0,
},
pollfd {
fd: stderr_fd,
events: POLLERR | POLLIN | POLLHUP,
revents: 0,
},
pollfd {
fd: stdout_fd,
events: POLLERR | POLLIN | POLLHUP,
revents: 0,
},
];
let mut stdin_off = 0;
let mut stdin_finished;
if test.stdin.is_none() {
stdin_finished = true;
} else {
stdin_finished = false;
pollfds[0].events = POLLERR | POLLOUT | POLLHUP;
}
let mut buf = [0; READBUF];
let start = Instant::now();
let mut last_warning = Instant::now();
let mut next_warning = last_warning
.checked_add(Duration::from_secs(TIMEOUT))
.unwrap();
loop {
let timeout = i32::try_from(
next_warning
.checked_duration_since(Instant::now())
.map(|d| d.as_millis())
.unwrap_or(1000),
)
.unwrap_or(1000);
if unsafe { poll((&mut pollfds) as *mut _ as *mut pollfd, 3, timeout) } != -1 {
if pollfds[0].revents & POLLOUT == POLLOUT {
let stdin_str = test.stdin.as_ref().unwrap();
if let Ok(i) = stdin.write(&stdin_str.as_bytes()[stdin_off..]) {
stdin_off += i;
}
if stdin_off == stdin_str.len() {
stdin_finished = true;
unsafe {
close(stdin_fd);
}
pollfds[0].events = POLLERR | POLLHUP;
}
}
if pollfds[1].revents & POLLIN == POLLIN {
if let Ok(i) = stderr.read(&mut buf) {
if i > 0 {
let utf8 = str::from_utf8(&buf[..i]).unwrap_or_else(|_| {
fatal(&format!("Can't convert stderr from '{:?}' into UTF-8", cmd))
});
cap_stderr.push_str(&utf8);
if inner.nocapture {
eprint!("{}", utf8);
}
}
}
}
if pollfds[2].revents & POLLIN == POLLIN {
if let Ok(i) = stdout.read(&mut buf) {
if i > 0 {
let utf8 = str::from_utf8(&buf[..i]).unwrap_or_else(|_| {
fatal(&format!("Can't convert stdout from '{:?}' into UTF-8", cmd))
});
cap_stdout.push_str(&utf8);
if inner.nocapture {
print!("{}", utf8);
}
}
}
}
if (stdin_finished || pollfds[0].revents & POLLHUP == POLLHUP)
&& pollfds[1].revents & POLLHUP == POLLHUP
&& pollfds[2].revents & POLLHUP == POLLHUP
{
break;
}
}
if Instant::now() >= next_warning {
let running_for = ((Instant::now() - start).as_secs() / TIMEOUT) * TIMEOUT;
if inner.test_threads == 1 {
eprint!("running for over {} seconds... ", running_for);
} else {
eprintln!(
"\nlang_tests::{} ... has been running for over {} seconds",
test_fname, running_for
);
}
last_warning = next_warning;
next_warning = last_warning
.checked_add(Duration::from_secs(TIMEOUT))
.unwrap();
}
}
let status = {
let mut wait_timeout = INITIAL_WAIT_TIMEOUT;
loop {
match child.try_wait() {
Ok(Some(s)) => break s,
Ok(None) => (),
Err(e) => fatal(&format!("{:?} did not exit correctly: {:?}", cmd, e)),
}
if Instant::now() >= next_warning {
let running_for = ((Instant::now() - start).as_secs() / TIMEOUT) * TIMEOUT;
if inner.test_threads == 1 {
eprint!("running for over {} seconds... ", running_for);
} else {
eprintln!(
"\nlang_tests::{} ... has been running for over {} seconds",
test_fname, running_for
);
}
last_warning = next_warning;
next_warning = last_warning
.checked_add(Duration::from_secs(TIMEOUT))
.unwrap();
}
sleep(Duration::from_nanos(wait_timeout));
wait_timeout *= 2;
if wait_timeout > MAX_WAIT_TIMEOUT {
wait_timeout = MAX_WAIT_TIMEOUT;
}
}
};
let stdin_remaining = if stdin_finished {
0
} else {
let stdin_str = test.stdin.as_ref().unwrap();
stdin_str.len() - stdin_off
};
(status, stdin_remaining, cap_stderr, cap_stdout)
}
fn set_nonblock(fd: c_int) -> Result<(), io::Error> {
let flags = unsafe { fcntl(fd, F_GETFL) };
if flags == -1 || unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } == -1 {
return Err(io::Error::last_os_error());
}
Ok(())
}