use eyre::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode};
const SHIM: &str = "mbx-test-runner";
const MIN_CPU_SAMPLE: std::time::Duration = std::time::Duration::from_secs(1);
const RUNNERS: &str = "MBX_TEST_RUNNERS";
#[derive(Debug, Default, Serialize, Deserialize)]
struct Overlay {
runners: BTreeMap<String, Option<Runner>>,
restore: BTreeMap<String, Option<String>>,
#[serde(default)]
project: Option<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
struct Runner {
path: PathBuf,
args: Vec<OsString>,
}
pub(super) struct TestRunner {
keys: Vec<(String, String)>,
runners: BTreeMap<String, Option<Runner>>,
project: Option<String>,
shim: PathBuf,
}
impl TestRunner {
pub(super) fn prepare(
config: &crate::config::Config,
arguments: &[String],
workspace_root: &Path,
directory: &Path,
) -> Result<Option<Self>> {
if !config.scheduler.enabled || !config.scheduler.tests {
return Ok(None);
}
let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into());
let expanded = super::cargo_invocation::expanded_arguments(&cargo, arguments);
let arguments = expanded.as_deref().unwrap_or(arguments);
if !wraps(arguments) {
return Ok(None);
}
let resolved = (|| -> Result<_> {
let cargo = super::launch::load_cargo_config()?;
let inherited = inherited_overlay().runners;
let mut keys = Vec::new();
let mut runners = BTreeMap::new();
let mut targets = super::launch::requested_targets(&cargo, arguments)?;
let host = cargo.host_triple()?.to_owned();
if !targets
.iter()
.any(|target| target.triple() == host.as_str())
{
targets.extend(cargo.build_target_for_config([host])?);
}
for target in targets {
let triple = target.triple().to_owned();
let runner = cargo.runner(&target)?.map(|runner| Runner {
path: runner.path,
args: runner.args,
});
let runner = match (runner, inherited.get(&triple)) {
(Some(runner), Some(wrapped)) if is_shim(&runner.path) => wrapped.clone(),
(runner, _) => runner,
};
keys.push((super::launch::runner_key(&target), triple.clone()));
runners.insert(triple, runner);
}
Ok((keys, runners))
})();
let (keys, runners) = match resolved {
Ok(resolved) => resolved,
Err(error) => {
log::debug!("running tests without machine-wide permits: {error:#}");
return Ok(None);
}
};
let shim = crate::session::install_shim_named(
&std::env::current_exe()?,
directory,
SHIM,
crate::session::ShimLink::Tracking,
)?;
Ok(Some(Self {
keys,
runners,
project: repository_identity(workspace_root),
shim,
}))
}
pub(super) fn environment(&self, environment: &mut BTreeMap<String, String>) -> Result<()> {
let path = environment
.get("PATH")
.map(OsString::from)
.or_else(|| std::env::var_os("PATH"))
.unwrap_or_default();
let path = std::env::join_paths(
std::iter::once(self.shim.parent().unwrap().to_path_buf())
.chain(std::env::split_paths(&path)),
)?;
environment.insert("PATH".into(), path.to_string_lossy().into_owned());
let mut restore = BTreeMap::new();
for (key, triple) in &self.keys {
restore.insert(key.clone(), std::env::var(key).ok());
environment.insert(key.clone(), format!("{SHIM} {triple}"));
}
let overlay = Overlay {
runners: self.runners.clone(),
restore,
project: self.project.clone(),
};
environment.insert(RUNNERS.into(), serde_json::to_string(&overlay)?);
Ok(())
}
}
fn wraps(arguments: &[String]) -> bool {
let args: Vec<_> = arguments
.iter()
.take_while(|arg| arg.as_str() != "--")
.collect();
matches!(
super::launch::cargo_subcommand(arguments),
Some("test" | "t" | "nextest")
) && !args.iter().any(|arg| {
arg.starts_with('+')
|| arg.starts_with("-C")
|| matches!(
arg.as_str(),
"--config" | "--directory" | "--help" | "-h" | "--no-run"
)
|| arg.starts_with("--config=")
|| arg.starts_with("--directory=")
})
}
fn is_shim(path: &Path) -> bool {
path.file_stem() == Some(OsStr::new(SHIM))
}
fn inherited_overlay() -> Overlay {
std::env::var(RUNNERS)
.ok()
.and_then(|value| serde_json::from_str(&value).ok())
.unwrap_or_default()
}
pub fn dispatch() -> Option<ExitCode> {
let mut arguments = std::env::args_os();
if arguments.next().is_none_or(|arg| !is_shim(Path::new(&arg))) {
return None;
}
let result = (|| -> Result<ExitCode> {
let triple = arguments
.next()
.ok_or_else(|| eyre::eyre!("missing target triple"))?;
let executable = arguments
.next()
.ok_or_else(|| eyre::eyre!("Cargo supplied no test binary"))?;
let rest: Vec<OsString> = arguments.collect();
let mut overlay = inherited_overlay();
let runner = overlay.runners.remove(&*triple.to_string_lossy()).flatten();
let mut command = match &runner {
Some(runner) => {
let mut command = Command::new(&runner.path);
command.args(&runner.args).arg(&executable);
command
}
None => Command::new(&executable),
};
command
.args(&rest)
.env(crate::scheduler::SCHED_DIR_ENV, "")
.env("MBX_SCHEDULER", "0")
.env("MBX_CONTROL_CHILD", "1")
.env_remove(RUNNERS);
for (key, value) in &overlay.restore {
match value {
Some(value) => command.env(key, value),
None => command.env_remove(key),
};
}
let package = std::env::var("CARGO_PKG_NAME").ok();
let nextest_phase = std::env::var("NEXTEST_TEST_PHASE").ok();
let nextest_identity = nextest_test_identity(NextestIdentityInput {
phase: nextest_phase.as_deref(),
execution_mode: std::env::var("NEXTEST_EXECUTION_MODE").ok().as_deref(),
binary_id: std::env::var("NEXTEST_BINARY_ID").ok().as_deref(),
test_name: std::env::var("NEXTEST_TEST_NAME").ok().as_deref(),
package: package.as_deref(),
project: overlay.project.as_deref(),
executable: Path::new(&executable),
arguments: &rest,
});
let suite_demand = test_binary_name(Path::new(&executable)).map(|name| {
crate::scheduler::Demand::test(
&ledger_name(overlay.project.as_deref(), package.as_deref(), &name),
test_threads(&rest, std::env::var("RUST_TEST_THREADS").ok().as_deref()),
)
});
let demand = nextest_identity
.as_ref()
.map_or(suite_demand, |(binary, test)| {
Some(crate::scheduler::Demand::nextest(binary, test))
});
let permit = demand
.as_ref()
.filter(|_| !lists_tests(&rest) && nextest_phase.as_deref() != Some("list"))
.and_then(|demand| {
let pool = crate::scheduler::pool()?;
let permit = if nextest_identity.is_some() {
admit_nextest(pool, demand)
} else {
pool.admit(demand)
}?;
Some((demand, permit))
});
let started = std::time::Instant::now();
let status = command
.status()
.wrap_err_with(|| format!("failed to run {}", Path::new(&executable).display()))?;
let wall = started.elapsed();
if let Some((demand, permit)) = permit {
let cores = (status.code().is_some()
&& (nextest_identity.is_some() || !narrows_suite(&rest)))
.then(crate::scheduler::child_cpu_time)
.flatten()
.map(|cpu| {
if wall < MIN_CPU_SAMPLE {
1
} else {
crate::scheduler::average_cores(cpu, wall)
}
});
crate::scheduler::record_test_run(demand, &status, cores);
drop(permit);
}
Ok(super::cargo::exit_code(status))
})();
Some(result.unwrap_or_else(|error| {
eprintln!("mbx[error]: failed to run test binary: {error:#}");
ExitCode::FAILURE
}))
}
fn test_binary_name(executable: &Path) -> Option<String> {
let stem = executable.file_stem()?.to_str()?;
let (name, hash) = stem.rsplit_once('-')?;
(!name.is_empty() && hash.len() == 16 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()))
.then(|| name.to_owned())
}
fn ledger_name(project: Option<&str>, package: Option<&str>, binary: &str) -> String {
[project, package, Some(binary)]
.into_iter()
.flatten()
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join("/")
}
fn repository_identity(workspace_root: &Path) -> Option<String> {
use sha2::Digest as _;
let checkout = workspace_root
.ancestors()
.find(|directory| directory.join(".git").exists())?;
let dot_git = checkout.join(".git");
let common = if dot_git.is_dir() {
dot_git
} else {
let pointer = std::fs::read_to_string(&dot_git).ok()?;
let git_dir = checkout.join(pointer.strip_prefix("gitdir:")?.trim());
match std::fs::read_to_string(git_dir.join("commondir")) {
Ok(common) => git_dir.join(common.trim()),
Err(_) => git_dir,
}
};
let common = common.canonicalize().ok()?;
let digest = sha2::Sha256::digest(common.as_os_str().as_encoded_bytes());
Some(hex::encode(&digest[..6]))
}
fn test_threads(arguments: &[OsString], environment: Option<&str>) -> Option<u64> {
let mut arguments = arguments.iter().filter_map(|arg| arg.to_str());
let mut stated = None;
while let Some(argument) = arguments.next() {
if argument == "--" {
break;
}
if argument == "--test-threads" {
stated = arguments.next().and_then(|value| value.parse().ok());
} else if let Some(value) = argument.strip_prefix("--test-threads=") {
stated = value.parse().ok();
}
}
stated.or_else(|| environment?.parse().ok())
}
fn narrows_suite(arguments: &[OsString]) -> bool {
let mut arguments = arguments.iter().map(|arg| arg.to_string_lossy());
while let Some(argument) = arguments.next() {
match argument.as_ref() {
"--" => return arguments.next().is_some(),
"--skip" | "--ignored" => return true,
argument if argument.starts_with("--skip=") => return true,
"--test-threads" | "--format" | "--logfile" | "--color" | "--shuffle-seed" | "-Z" => {
arguments.next();
}
argument if !argument.starts_with('-') => return true,
_ => {}
}
}
false
}
fn lists_tests(arguments: &[OsString]) -> bool {
arguments
.iter()
.take_while(|arg| arg.as_os_str() != "--")
.any(|arg| arg == "--list")
}
struct NextestIdentityInput<'a> {
phase: Option<&'a str>,
execution_mode: Option<&'a str>,
binary_id: Option<&'a str>,
test_name: Option<&'a str>,
package: Option<&'a str>,
project: Option<&'a str>,
executable: &'a Path,
arguments: &'a [OsString],
}
fn nextest_test_identity(input: NextestIdentityInput<'_>) -> Option<(String, String)> {
if input.phase == Some("list") || lists_tests(input.arguments) {
return None;
}
let running = input.phase == Some("run")
|| (input.phase.is_none() && input.execution_mode == Some("process-per-test"));
if !running {
return None;
}
let test = input
.test_name
.filter(|name| !name.is_empty())
.map(str::to_owned)
.or_else(|| exact_test_name(input.arguments))?;
let binary = input
.binary_id
.filter(|id| !id.is_empty())
.map(|id| ledger_name(input.project, None, id))
.or_else(|| {
let name = test_binary_name(input.executable)?;
Some(ledger_name(input.project, input.package, &name))
})?;
Some((binary, test))
}
fn exact_test_name(arguments: &[OsString]) -> Option<String> {
let mut arguments = arguments.iter().filter_map(|argument| argument.to_str());
while let Some(argument) = arguments.next() {
if argument == "--" {
return None;
}
if argument == "--exact" {
return arguments.next().map(str::to_owned);
}
if let Some(name) = argument.strip_prefix("--exact=") {
return (!name.is_empty()).then(|| name.to_owned());
}
}
None
}
const WAIT_NOTICE: std::time::Duration = std::time::Duration::from_secs(1);
fn admit_nextest(
pool: &crate::scheduler::Pool,
demand: &crate::scheduler::Demand,
) -> Option<crate::scheduler::Permit> {
let (admitted, wait) = std::sync::mpsc::channel::<()>();
let notice = std::thread::spawn(move || {
if let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = wait.recv_timeout(WAIT_NOTICE) {
eprintln!(
"mbx: waiting for machine-wide scheduler capacity before starting this test; nextest counts this wait toward its slow-timeout"
);
}
});
let started = std::time::Instant::now();
let permit = pool.admit(demand);
let waited = started.elapsed();
drop(admitted);
let _ = notice.join();
if permit.is_some() && waited >= WAIT_NOTICE {
eprintln!(
"mbx: test started after waiting {:.1}s for scheduler capacity",
waited.as_secs_f64()
);
}
permit
}
#[cfg(test)]
mod tests {
use super::*;
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| value.to_string()).collect()
}
fn os(values: &[&str]) -> Vec<OsString> {
values.iter().map(OsString::from).collect()
}
#[test]
fn cargo_test_and_nextest_commands_are_wrapped() {
assert!(wraps(&strings(&["test"])));
assert!(wraps(&strings(&["t", "--workspace"])));
assert!(wraps(&strings(&["nextest", "run"])));
assert!(wraps(&strings(&["nextest", "r"])));
assert!(wraps(&strings(&["nextest", "list"])));
assert!(wraps(&strings(&["--color", "always", "test"])));
assert!(wraps(&strings(&["test", "--", "--config"])));
assert!(!wraps(&strings(&["build"])));
assert!(!wraps(&strings(&["run", "--", "test"])));
assert!(!wraps(&strings(&["test", "--no-run"])));
assert!(!wraps(&strings(&["nextest", "run", "--no-run"])));
assert!(!wraps(&strings(&["+nightly", "test"])));
assert!(!wraps(&strings(&["test", "--config", "a.toml"])));
assert!(!wraps(&strings(&["test", "--config=a.toml"])));
assert!(!wraps(&strings(&["-C", "other", "test"])));
assert!(!wraps(&strings(&["-Cother", "test"])));
assert!(!wraps(&strings(&["--directory", "other", "test"])));
assert!(!wraps(&strings(&["--directory=other", "test"])));
}
#[test]
fn nextest_identity_uses_phase_environment_and_exact_filter() {
let executable = Path::new("target/debug/deps/mbx-0123456789abcdef");
let args = os(&["--exact", "suite::case", "--nocapture"]);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: Some("run"),
execution_mode: Some("process-per-test"),
binary_id: Some("mbx::integration"),
test_name: Some("suite::case"),
package: Some("mbx"),
project: Some("ab12"),
executable,
arguments: &args,
}),
Some(("ab12/mbx::integration".into(), "suite::case".into()))
);
let args = os(&["--exact=suite::case", "--nocapture"]);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: None,
execution_mode: Some("process-per-test"),
binary_id: None,
test_name: None,
package: Some("mbx"),
project: Some("ab12"),
executable,
arguments: &args,
}),
Some(("ab12/mbx/mbx".into(), "suite::case".into()))
);
let args = os(&["--exact", "suite::case"]);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: Some("run"),
execution_mode: None,
binary_id: Some("mbx"),
test_name: None,
package: None,
project: None,
executable,
arguments: &args,
}),
Some(("mbx".into(), "suite::case".into()))
);
}
#[test]
fn nextest_listing_and_unknown_runs_have_no_test_identity() {
let executable = Path::new("target/debug/deps/mbx-0123456789abcdef");
let args = os(&["--exact", "suite::case"]);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: Some("list"),
execution_mode: Some("process-per-test"),
binary_id: Some("mbx"),
test_name: Some("suite::case"),
package: None,
project: None,
executable,
arguments: &args,
}),
None
);
let list_args = os(&["--list"]);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: Some("run"),
execution_mode: None,
binary_id: Some("mbx"),
test_name: Some("suite::case"),
package: None,
project: None,
executable,
arguments: &list_args,
}),
None
);
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: None,
execution_mode: None,
binary_id: Some("mbx"),
test_name: Some("suite::case"),
package: None,
project: None,
executable,
arguments: &args,
}),
None
);
let doctest = Path::new("doctest-temp");
assert_eq!(
nextest_test_identity(NextestIdentityInput {
phase: Some("run"),
execution_mode: None,
binary_id: None,
test_name: None,
package: None,
project: None,
executable: doctest,
arguments: &args,
}),
None
);
}
#[test]
fn only_hashed_cargo_binaries_are_test_binaries() {
assert_eq!(
test_binary_name(Path::new("target/debug/deps/mbx-0123456789abcdef")).as_deref(),
Some("mbx")
);
assert_eq!(
test_binary_name(Path::new("deps/cache_tests-fedcba9876543210.exe")).as_deref(),
Some("cache_tests")
);
assert_eq!(test_binary_name(Path::new("deps/my-tool")), None);
assert_eq!(test_binary_name(Path::new("deps/-0123456789abcdef")), None);
assert_eq!(
test_binary_name(Path::new("/tmp/rustdoctestAbC/rust_out")),
None
);
}
#[test]
fn history_is_kept_per_repository_and_package() {
assert_eq!(
ledger_name(Some("ab12"), Some("parser"), "it"),
"ab12/parser/it"
);
assert_ne!(
ledger_name(Some("ab12"), Some("server"), "it"),
ledger_name(Some("cd34"), Some("server"), "it")
);
assert_eq!(ledger_name(None, Some("parser"), "it"), "parser/it");
assert_eq!(ledger_name(None, None, "it"), "it");
assert_eq!(ledger_name(Some(""), Some(""), "it"), "it");
}
#[test]
fn worktrees_share_a_repository_identity_and_other_repositories_do_not() {
let root = tempfile::tempdir().unwrap();
let main = root.path().join("main");
std::fs::create_dir_all(main.join(".git/worktrees/feature")).unwrap();
std::fs::create_dir_all(main.join("crates/app")).unwrap();
std::fs::write(main.join(".git/worktrees/feature/commondir"), "../..\n").unwrap();
let feature = root.path().join("feature");
std::fs::create_dir_all(&feature).unwrap();
std::fs::write(
feature.join(".git"),
"gitdir: ../main/.git/worktrees/feature\n",
)
.unwrap();
let other_feature = root.path().join("other-feature");
std::fs::create_dir_all(&other_feature).unwrap();
std::fs::write(
other_feature.join(".git"),
format!(
"gitdir: {}\n",
main.join(".git/worktrees/feature").display()
),
)
.unwrap();
let unrelated = root.path().join("unrelated");
std::fs::create_dir_all(unrelated.join(".git")).unwrap();
let identity = repository_identity(&main).expect("a repository has an identity");
assert_eq!(identity.len(), 12);
assert_eq!(
repository_identity(&main.join("crates/app")),
Some(identity.clone())
);
assert_eq!(repository_identity(&feature), Some(identity.clone()));
assert_eq!(repository_identity(&other_feature), Some(identity.clone()));
assert_ne!(repository_identity(&unrelated), Some(identity));
}
#[test]
fn thread_counts_come_from_arguments_before_the_environment() {
assert_eq!(test_threads(&os(&[]), None), None);
assert_eq!(test_threads(&os(&[]), Some("3")), Some(3));
assert_eq!(
test_threads(&os(&["--test-threads", "2"]), Some("3")),
Some(2)
);
assert_eq!(test_threads(&os(&["--test-threads=5"]), None), Some(5));
assert_eq!(test_threads(&os(&["--", "--test-threads=5"]), None), None);
assert_eq!(test_threads(&os(&[]), Some("many")), None);
}
#[test]
fn narrowed_runs_are_recognised() {
assert!(!narrows_suite(&os(&[])));
assert!(!narrows_suite(&os(&["--test-threads", "4", "--nocapture"])));
assert!(!narrows_suite(&os(&[
"--format",
"json",
"--include-ignored"
])));
assert!(narrows_suite(&os(&["parser"])));
assert!(narrows_suite(&os(&["--exact", "parser::tokens"])));
assert!(narrows_suite(&os(&["--skip", "slow"])));
assert!(narrows_suite(&os(&["--skip=slow"])));
assert!(narrows_suite(&os(&["--ignored"])));
assert!(narrows_suite(&os(&["--", "parser"])));
}
#[test]
fn listing_is_recognised() {
assert!(lists_tests(&os(&["--list", "--format", "terse"])));
assert!(!lists_tests(&os(&["--exact", "list"])));
}
#[test]
fn nested_shims_are_recognised_by_name() {
assert!(is_shim(Path::new("/tmp/mbx-session-x/mbx-test-runner")));
assert!(is_shim(Path::new("mbx-test-runner.exe")));
assert!(!is_shim(Path::new("/usr/bin/qemu-aarch64")));
}
}