use std::cmp;
use std::fs::File;
use std::path::Path;
use std::path::PathBuf;
use anyhow::bail;
use colored::Colorize;
use detcore::preemptions::PreemptionReader;
use detcore::preemptions::PreemptionRecord;
use detcore::types::SchedEvent;
use detcore::util::truncated;
use hermit::Error;
use rand::RngExt as _;
use rand::SeedableRng;
use rand_pcg::Pcg64Mcg;
use reverie::PrettyBacktrace;
use reverie::process::ExitStatus;
use reverie::process::Output;
use crate::analyze::consts::*;
use crate::analyze::rundata::RunData;
use crate::analyze::types::AnalyzeOpts;
use crate::analyze::types::ExitStatusConstraint;
use crate::analyze::types::Report;
use crate::analyze::types::ReportCriticalEvent;
use crate::global_opts::GlobalOpts;
use crate::logdiff::LogDiffCLIOpts;
use crate::run::RunOpts;
use crate::schedule_search::Config;
use crate::schedule_search::CriticalSchedule;
use crate::schedule_search::TrialOutcome;
use crate::schedule_search::search_for_critical_schedule;
fn preempt_files_equal(path1: &Path, path2: &Path) -> bool {
let mut pr1 = PreemptionReader::new(path1).load_all();
let mut pr2 = PreemptionReader::new(path2).load_all();
pr1.preemptions_only();
pr2.preemptions_only();
pr1 == pr2
}
fn yellow_msg(msg: &str) {
eprintln!(":: {}", msg.yellow().bold());
}
impl AnalyzeOpts {
pub fn get_tmp(&self) -> anyhow::Result<&Path> {
if let Some(pb) = &self.tmp_dir {
Ok(pb.as_path())
} else {
bail!("Expected tmp_dir to be set at this point!")
}
}
fn launch_search(&self, round: u64, sched_seed: u64) -> Result<Option<RunData>, Error> {
yellow_msg(&format!(
"Searching (round {}) for a target execution, chaos --sched-seed={} ",
round, sched_seed
));
let runname = format!("search_round_{:0wide$}", round, wide = 3);
let mut rundat = RunData::new_baseline(self, runname)?.with_preemption_recording();
rundat.runopts.det_opts.det_config.sched_seed = Some(sched_seed);
if self.imprecise_search {
rundat.runopts.det_opts.det_config.imprecise_timers = true; }
rundat.launch()?;
if rundat.is_a_match() {
eprintln!(
":: {}:\n {}",
"Target run established by --search. Reproducer"
.green()
.bold(),
rundat.to_repro(),
);
Ok(Some(rundat))
} else {
Ok(None)
}
}
pub(super) fn launch_from_preempts_to_sched(
&self,
runname: &str,
preempts_path: &Path,
record_sched_path: Option<&Path>,
) -> anyhow::Result<(bool, RunOpts)> {
let mut run = RunData::new_baseline(self, runname.to_string())?
.with_preempts_path_in(preempts_path.to_path_buf());
if let Some(path) = record_sched_path {
run = run.with_schedule_recording_to(path.to_path_buf());
}
run.launch()?;
let is_match = run.is_a_match();
Ok((is_match, run.into_runopts()))
}
fn launch_for_stacktraces(
&self,
runname: &str,
schedule_path: &Path,
critical_event_index: u64,
) -> anyhow::Result<(RunData, PathBuf, PathBuf)> {
let mut run = RunData::new_baseline(self, runname.to_string())?
.with_schedule_replay_from(schedule_path.to_path_buf());
let root_path = run.root_path();
let stack1_path = root_path.with_extension("stack1");
let stack2_path = root_path.with_extension("stack2");
run.runopts.det_opts.det_config.stacktrace_event = [
(critical_event_index - 1, Some(stack1_path.clone())),
(critical_event_index, Some(stack2_path.clone())),
]
.to_vec();
run.launch()?;
Ok((run, stack1_path, stack2_path))
}
fn has_filters(&self) -> bool {
self.target_stdout.is_some()
|| self.target_stderr.is_some()
|| self.target_exit_code != ExitStatusConstraint::Any
}
fn is_vanilla_criteria(&self) -> bool {
self.target_stdout.is_none()
&& self.target_stderr.is_none()
&& self.target_exit_code == ExitStatusConstraint::NonZero
}
pub fn write_final_schedules(
&self,
failing: Vec<SchedEvent>,
failing_path: &Path,
passing: Vec<SchedEvent>,
passing_path: &Path,
) {
self.write_trial_schedule(failing, failing_path).unwrap();
eprintln!(
"Wrote final on-target ({}) schedule to {}",
self.display_criteria(),
failing_path.display()
);
self.write_trial_schedule(passing, passing_path).unwrap();
eprintln!(
"Wrote final baseline (off-target) schedule to {}",
passing_path.display()
);
}
fn display_criteria(&self) -> String {
let mut strs: Vec<String> = Vec::new();
match &self.target_exit_code {
ExitStatusConstraint::Exact(c) => {
strs.push(format!("exit code={}", c));
}
ExitStatusConstraint::NonZero => {
strs.push("nonzero exit".to_string());
}
ExitStatusConstraint::Any => {}
}
if self.target_stdout.is_some() {
strs.push("matching stdout".to_string());
}
if self.target_stderr.is_some() {
strs.push("matching stderr".to_string());
}
strs.join(", ")
}
fn phase0_initialize(&mut self) -> anyhow::Result<()> {
let dir = tempfile::Builder::new()
.prefix("hermit_analyze")
.tempdir()?;
let tmpdir_path = dir.keep(); eprintln!(":: Temp workspace: {}", tmpdir_path.display());
self.tmp_dir = Some(tmpdir_path);
Ok(())
}
fn phase1_establish_target_run(&self) -> Result<RunData, Error> {
let mut run1data = RunData::new_run1_target(self, "run1_target".to_string())?;
eprintln!(
":: {} {}",
"Studying target execution: ".yellow().bold(),
run1data.to_repro()
);
if let Some(p) = &self.run1_preemptions {
let preempts_path = run1data.preempts_path_out();
std::fs::copy(p, preempts_path).expect("copy file to succeed");
return Ok(run1data);
}
let mut run1data = run1data.with_preemption_recording();
run1data.launch()?;
if !run1data.is_a_match() {
if self.search {
eprintln!(
":: {}",
"First run did not match target criteria; now searching for a matching run..."
.red()
.bold()
);
self.do_search()
} else {
bail!("FAILED. The run did not match the target criteria. Try --search.");
}
} else if self.has_filters() {
eprintln!(
":: {}",
format!(
"First run matched target criteria ({}).",
self.display_criteria()
)
.green()
.bold(),
);
Ok(run1data)
} else {
eprintln!(":: {}", "WARNING: run without any --filter arguments, so accepting ALL runs. This is probably not what you wanted.".red().bold());
Ok(run1data)
}
}
fn phase2_minimize(
&self,
global: &GlobalOpts,
mut last_run: RunData,
) -> anyhow::Result<RunData> {
if self.minimize {
let preempts_path = last_run.preempts_path_out();
let (min_pr, min_pr_path, min_log_path) = self.minimize(preempts_path, global)?;
eprintln!(
":: {}\n {}",
"Successfully minimized to these critical interventions:"
.green()
.bold(),
truncated(1000, serde_json::to_string(&min_pr).unwrap())
);
let runname = "minimized".to_string();
let min_run = RunData::from_minimize_output(
self,
runname,
last_run.runopts.clone(),
min_pr,
min_pr_path,
min_log_path,
);
Ok(min_run)
} else {
Ok(last_run)
}
}
fn log_diff_preemption_replay(
&self,
global: &GlobalOpts,
run1_log_path: &Path,
run2_log_path: &Path,
) -> ExitStatus {
if self.verbose {
yellow_msg("[comparing] with log-diff command:");
eprintln!(
" hermit log-diff --record-envelope=all-records-v1 \
--ignore-lines=CHAOSRAND {} {}",
run1_log_path.display(),
run2_log_path.display(),
);
}
let mut ldopts = LogDiffCLIOpts::new(run1_log_path, run2_log_path);
ldopts.more.ignore_lines = vec!["CHAOSRAND".to_string(), "SCHEDRAND".to_string()];
ldopts.main(global)
}
pub fn phase3_strict_preempt_replay_check(
&self,
global: &GlobalOpts,
min_run: &mut RunData,
) -> Result<(), Error> {
let min_log_path = &min_run.log_path().unwrap().to_path_buf();
let min_preempts_path = min_run.preempts_path_out().to_path_buf();
if self.selfcheck {
yellow_msg("[selfcheck] Verifying target run preserved under preemption-replay");
let runname = "run1b_selfcheck";
let mut run1b = RunData::new_run1_replay(self, runname.to_string(), &min_run.runopts)?
.with_preempts_path_in(min_preempts_path.clone())
.with_preemption_recording();
eprintln!(" {}", run1b.to_repro());
run1b.launch()?;
yellow_msg("[selfcheck] Comparing output from additional run (run1 vs run1b)");
let run1b_log_path = run1b.log_path().unwrap();
let status = self.log_diff_preemption_replay(global, min_log_path, run1b_log_path);
if !run1b.is_a_match() {
bail!("First run matched criteria but second run did not.");
}
if !status.success() {
bail!(
"Log differences found, aborting because --selfcheck requires perfect reproducibility of the target run!"
)
}
let run1b_preempts_path = run1b.preempts_path_out();
if !preempt_files_equal(&min_preempts_path, run1b_preempts_path) {
bail!(
"The preemptions recorded by the additional run did not match the preemptions replayed (no fixed point): {} vs {}",
min_preempts_path.display(),
run1b_preempts_path.display(),
);
}
eprintln!(
":: {}",
"Identical executions confirmed between target run and its preemption-based replay."
.green()
.bold()
);
}
Ok(())
}
pub fn phase4_choose_baseline_sched_events(
&self,
matching_pr: PreemptionRecord,
) -> anyhow::Result<RunData> {
if self.minimize {
assert!(self.run2_seed.is_none());
assert!(self.run2_preemptions.is_none());
assert!(self.run2_schedule.is_none());
let mut pr = matching_pr;
let mut ix = 1;
loop {
let runname = format!("run2_baseline_try{:03}", ix);
let (still_matching, mut newrun) =
self.save_nearby_non_matching_sched_events(&runname, &pr)?;
if still_matching {
pr = newrun.preempts_out().clone();
ix += 1;
} else {
return Ok(newrun);
}
}
} else {
let runname = "run2_baseline".to_string();
let mut newrun = RunData::new_baseline(self, runname)?.with_preemption_recording();
let ro = &mut newrun.runopts;
let from_where;
if let Some(seed) = self.run2_seed {
ro.det_opts.det_config.seed = seed;
from_where = "--run2-seed";
} else if let Some(path) = &self.run2_preemptions {
ro.det_opts.det_config.replay_preemptions_from = Some(path.clone());
from_where = "--run2-preemptions";
} else if let Some(path) = &self.run2_schedule {
if self.selfcheck {
}
let runname = "from_existing_run2_schedule".to_string();
let fakerun = RunData::from_schedule_trace(self, runname, ro.clone(), path.clone());
return Ok(fakerun);
} else {
from_where = "non-chaos (0 extra preemptions) run";
ro.det_opts.det_config.chaos = false;
}
newrun.launch()?;
eprintln!(
":: Recorded schedule from {} as baseline run ({})",
from_where,
newrun
.sched_path_out()
.file_name()
.unwrap()
.to_string_lossy(),
);
if newrun.is_a_match() {
bail!(
"Expectations not met... baseline run matched target criteria when it should not."
)
} else {
eprintln!("Good: baseline run does not match criteria (e.g. pass not fail).");
Ok(newrun)
}
}
}
pub fn phase5_bisect_traces(
&self,
target: &[SchedEvent],
baseline: &[SchedEvent],
) -> anyhow::Result<CriticalSchedule> {
let mut i = 0;
let test_fn = |sched: &[SchedEvent]| -> TrialOutcome {
i += 1;
let runname = format!("bisect_round_{:0wide$}", i, wide = 3);
let mut newrun = RunData::new_baseline(self, runname)?
.with_schedule_replay() .with_schedule_recording(); self.write_trial_schedule(sched.to_owned(), newrun.sched_path_in())?;
if self.verbose {
eprintln!(
":: {}, repro command:\n {}",
format!("Testing execution during search (#{})", i)
.yellow()
.bold(),
newrun.to_repro(),
);
}
newrun.launch()?;
let is_match = newrun.is_a_match();
if is_match {
eprintln!(" => Target condition ({})", self.display_criteria());
} else {
eprintln!(" => Baseline condition (usually absence of crash)");
}
let sched_out = PreemptionReader::new(newrun.sched_path_out()).load_all();
Ok((!is_match, sched_out.into_global()))
};
let target = target.to_vec(); let baseline = baseline.to_vec();
let mut cfg = Config {
verbose: self.verbose,
needleman_search: self.run_needleman,
..Default::default()
};
if let Some(dist) = self.jitter_dist {
cfg.max_jitter_editdist = dist;
cfg.max_jitter_swapdist = dist * dist;
}
let crit = search_for_critical_schedule(test_fn, baseline, target, &cfg)?;
eprintln!(
"Critical event of final on-target schedule is {}",
crit.critical_event_index
);
Ok(crit)
}
pub fn bisect_schedule_pair(
&mut self,
good: Vec<SchedEvent>,
bad: Vec<SchedEvent>,
) -> anyhow::Result<ExitStatus> {
self.phase0_initialize()?;
eprintln!(
":: Beginning schedule bisection using good/bad endpoints ({} / {} events)",
good.len(),
bad.len(),
);
let critical = self.phase5_bisect_traces(&bad, &good)?;
let index = critical.critical_event_index;
if index == 0 || index >= critical.failing_schedule.len() {
bail!("schedule search returned an invalid critical event index {index}");
}
println!("\n---------------- hermit bisect result ----------------");
println!("Schedule divergence localized at bad event #{index}:");
println!(
" preceding event #{}: {}",
index - 1,
critical.failing_schedule[index - 1]
);
println!(
" critical event #{index}: {}",
critical.failing_schedule[index]
);
println!("Swapping these adjacent events reproduces the good outcome.");
let report = self.phase6_record_outputs(critical)?;
if let Some(path) = &self.report_file {
let text = serde_json::to_string_pretty(&report)?;
std::fs::write(path, text)?;
eprintln!(":: Final bisection report written to {}", path.display());
}
Ok(ExitStatus::SUCCESS)
}
fn generate_context(
sched: &[SchedEvent],
critical_event_index: usize,
window_size: usize,
) -> String {
let mut additional_context = "Execution context for the critical events (*):\n".to_string();
{
let start = critical_event_index - 1;
let start = start - cmp::min(window_size, start);
let end = cmp::min(critical_event_index + window_size + 1, sched.len());
for (ix, item) in sched.iter().enumerate().take(end).skip(start) {
additional_context.push_str(&format!(
" {} {}\n",
if ix == critical_event_index || ix == critical_event_index - 1 {
"*"
} else {
" "
},
item
));
}
}
additional_context
}
pub fn phase6_record_outputs(&self, crit: CriticalSchedule) -> Result<Report, Error> {
let tmp_dir = self.get_tmp()?;
let CriticalSchedule {
failing_schedule,
passing_schedule,
critical_event_index,
} = crit;
let runname1 = "final_target_for_stacktraces";
let runname2 = "final_baseline_for_stacktraces";
let final_failing_path = tmp_dir.join(runname1).with_extension(SCHED_EXT);
let final_passing_path = tmp_dir.join(runname2).with_extension(SCHED_EXT);
let passing_context = Self::generate_context(
&passing_schedule,
critical_event_index,
self.execution_context,
);
let failing_context = Self::generate_context(
&failing_schedule,
critical_event_index,
self.execution_context,
);
self.write_final_schedules(
failing_schedule,
&final_failing_path,
passing_schedule,
&final_passing_path,
);
{
let mut header = String::new();
header.push_str(
"These two operations, on different threads, are RACING with eachother.\n",
);
header.push_str(&format!(
"The order of events (#{} and #{} in the schedule) determines the program outcome.\n",
critical_event_index - 1,
critical_event_index,
));
header.push_str(
"You must add synchronization to prevent these operations from racing, or give them a different order.\n",
);
if self.is_vanilla_criteria() {
header.push_str(
"Attached are the stacktraces from a PASSING run, but flipping the order of the two events makes the program FAIL (nonzero exit).\n",
);
} else {
header.push_str(&format!(
"Attached are the stacktraces from a baseline run, while flipping the order makes the program exhibit the target criteria ({}).\n",
self.display_criteria()
));
}
let (stack1, stack2) = self.run_and_retrieve_stacks(
runname2,
final_passing_path,
critical_event_index,
false,
)?;
println!(
"\n------------------------------ hermit analyze report ------------------------------"
);
println!("{}", header);
Self::print_stack("stack1", &stack1);
Self::print_stack("stack2", &stack2);
println!("\n{}", passing_context);
let report = Report {
header,
additional_context: passing_context,
baseline_run: true,
critical_event1: ReportCriticalEvent {
event_index: critical_event_index - 1,
stack: stack1,
},
critical_event2: ReportCriticalEvent {
event_index: critical_event_index,
stack: stack2,
},
};
if self.verbose {
eprintln!(
"\n:: {}",
"[verbose] additional TARGET (failing) run to print those stack traces as well.."
.yellow()
.bold()
);
let (stack1, stack2) = self.run_and_retrieve_stacks(
runname1,
final_failing_path,
critical_event_index,
true,
)?;
println!(
"\n----------------------- stacks from final on-target run -----------------------"
);
Self::print_stack("stack1", &stack1);
Self::print_stack("stack2", &stack2);
println!("\n{}", failing_context);
}
Ok(report)
}
}
fn print_stack(which: &str, m_stack: &Option<PrettyBacktrace>) {
if let Some(s) = m_stack {
println!("{}", s);
} else {
println!(" ({} not available) ", which);
}
}
fn run_and_retrieve_stacks(
&self,
runname: &str,
final_path: PathBuf,
critical_event_index: usize,
expected_match: bool,
) -> Result<(Option<PrettyBacktrace>, Option<PrettyBacktrace>), Error> {
let (rundata, stack1_path, stack2_path) =
self.launch_for_stacktraces(runname, &final_path, critical_event_index as u64)?;
let res = rundata.is_a_match();
eprintln!(
"\n:: {}",
"Final run to print stack traces completed. Repro command:"
.green()
.bold()
);
eprintln!("{}", rundata.to_repro());
let stack1 = File::open(stack1_path)
.map(|file| serde_json::from_reader(file).unwrap())
.ok();
let stack2 = File::open(stack2_path)
.map(|file| serde_json::from_reader(file).unwrap())
.ok();
if res == expected_match {
eprintln!(":: {}", "Completed analysis successfully.".green().bold());
Ok((stack1, stack2))
} else {
bail!(
"Internal error! Final run expected match={}, but observed the opposite!",
expected_match,
)
}
}
fn prepare(&mut self, global: &GlobalOpts) -> anyhow::Result<()> {
self.apply_global(global);
self.refuse_unsupervised_log_cap()?;
self.refuse_options_trials_do_not_apply()?;
self.refuse_outputs_trials_overwrite()?;
self.refuse_unqualified_trial_timeout()?;
self.install_trial_pmu_config()?;
self.refuse_strict_with_inexact_branch_counter()?;
let unimplemented: Vec<&str> = [
(self.run1_schedule.is_some(), "--run1-schedule"),
(self.run2_schedule.is_some(), "--run2-schedule"),
]
.into_iter()
.filter_map(|(given, option)| given.then_some(option))
.collect();
if !unimplemented.is_empty() {
return Err(
anyhow::Error::new(crate::container::PolicyRefusal).context(format!(
"hermit analyze does not implement {}. Identify a run by a preemption record \
(--run1-preemptions or --run2-preemptions, written by `hermit run \
--record-preemptions-to`) or by a seed (--run1-seed or --run2-seed).",
unimplemented.join(" or ")
)),
);
}
let run1 = self.run1_preemptions.clone();
let run2 = self.run2_preemptions.clone();
let recordings: Vec<(&str, &Path)> = [
("--run1-preemptions", run1.as_deref()),
("--run2-preemptions", run2.as_deref()),
]
.into_iter()
.filter_map(|(option, path)| Some((option, path?)))
.collect();
self.adopt_recorded_epoch(&recordings)?;
self.refuse_recordings_under_another_pipe_wake_latency(&recordings)
}
pub fn apply_global(&mut self, global: &GlobalOpts) {
self.backend = global.backend;
self.max_log_bytes = global.max_log_bytes;
self.log_budget = global.log_budget();
}
pub fn main(&mut self, global: &GlobalOpts) -> anyhow::Result<ExitStatus> {
self.prepare(global)?;
self.phase0_initialize()?; {
let dummy = RunData::new_baseline(self, "dummy".to_string())?;
if !dummy.runopts.det_opts.det_config.chaos {
eprintln!(
":: {} You may want to turn it on explicitly, along with a --max-timeslice that works well for this program.",
"WARNING: implicitly activating --chaos.".yellow().bold()
);
}
}
let run1data = self.phase1_establish_target_run()?;
let mut min_run = self.phase2_minimize(global, run1data)?;
self.phase3_strict_preempt_replay_check(global, &mut min_run)?;
min_run.normalize_preempts_out();
let normalized_preempts = min_run.preempts_out().clone();
eprintln!(
":: {}\n {}",
format!(
"Normalized, that preemption record ({}) becomes:",
min_run
.preempts_path_out()
.file_name()
.unwrap()
.to_string_lossy()
)
.green()
.bold(),
truncated(
1000,
serde_json::to_string_pretty(&normalized_preempts.clone_preemptions_only())
.unwrap()
)
);
let mut target_endpoint_run = self.save_final_target_sched_events(min_run)?;
let mut baseline_endpoint_run =
self.phase4_choose_baseline_sched_events(normalized_preempts)?;
let baseline_sched_name = baseline_endpoint_run.sched_out_file_name();
let target_sched_name = target_endpoint_run.sched_out_file_name();
let target = target_endpoint_run.sched_out();
let baseline = baseline_endpoint_run.sched_out();
eprintln!(
":: {} (event lengths {} / {}): {} {}",
"Beginning bisection using endpoints".yellow().bold(),
baseline.len(),
target.len(),
baseline_sched_name,
target_sched_name,
);
let crit_sched = self.phase5_bisect_traces(target, baseline)?;
let report = self.phase6_record_outputs(crit_sched)?;
if let Some(path) = &self.report_file {
let txt = serde_json::to_string(&report).unwrap();
std::fs::write(path, txt).expect("Unable to write report file");
eprintln!(
":: {}\n {}",
"Final analysis report written to:".green().bold(),
path.display()
);
}
self.success_exit_code
.map_or(Ok(ExitStatus::SUCCESS), |exit_code| {
Ok(ExitStatus::Exited(exit_code))
})
}
fn save_final_target_sched_events(&self, mut last_run: RunData) -> anyhow::Result<RunData> {
let pr = last_run.preempts_out();
if pr.contains_schedevents() {
let path = last_run.preempts_path_out();
yellow_msg(&format!(
"Using the last run as the target endpoint ({} contains sched events)",
path.file_name().unwrap().to_string_lossy()
));
Ok(last_run)
} else {
yellow_msg(
"Verify target endpoint preemption record causes criteria to hold and record sched events",
);
let runname = "verify_target_endpoint";
let mut newrun = RunData::new(self, runname.to_string(), last_run.runopts.clone())
.with_preempts_path_in(last_run.preempts_path_out().to_path_buf())
.with_schedule_recording();
newrun.launch()?;
eprintln!(" {}", newrun.to_repro());
if !newrun.is_a_match() {
bail!("Final preemption record still does not match target criteria");
}
Ok(newrun)
}
}
fn save_nearby_non_matching_sched_events(
&self,
runname: &str,
matching_preempts: &PreemptionRecord,
) -> anyhow::Result<(bool, RunData)> {
let non_matching_preempts = matching_preempts.with_latest_preempt_removed();
if let Err(e) = non_matching_preempts.validate() {
bail!(
"Hermit analyzer produced corrupt nearby non-matching preemption record, cannot proceed.\n\
Error: {}\n\n\
Corrupt record: {}",
e,
non_matching_preempts,
);
}
let mut newrun = RunData::new_baseline(self, runname.to_string())?
.with_preempts_in(non_matching_preempts)
.with_schedule_recording();
yellow_msg(
"Verify preemption record *without* latest critical preempt causes criteria non-match. Also record sched events.",
);
newrun.launch()?;
eprintln!(" {}", newrun.to_repro());
if newrun.is_a_match() {
eprintln!(
"{}",
":: New preemption record still matches criteria! Attempting further knockouts.."
.red()
.bold(),
);
Ok((true, newrun))
} else {
Ok((false, newrun))
}
}
fn do_search(&self) -> Result<RunData, Error> {
self.search_rounds(|round, sched_seed| self.launch_search(round, sched_seed))
}
fn search_rounds(
&self,
mut launch: impl FnMut(u64, u64) -> Result<Option<RunData>, Error>,
) -> Result<RunData, Error> {
let search_seed = self.analyze_seed.unwrap_or_else(|| {
let seed: u64 = rand::random();
yellow_msg(&format!("WARNING: performing --search with system randomness, use --analyze-seed={} to repro.", seed));
seed
});
yellow_msg(&format!("Failure search using RNG seed {}", search_seed));
let mut rng = Pcg64Mcg::seed_from_u64(search_seed);
let mut round = 0;
loop {
let sched_seed = rng.random();
if let Some(mut rundat) = launch(round, sched_seed)? {
if self.verbose {
let preempts = rundat.preempts_path_out();
let init_schedule: PreemptionRecord =
PreemptionReader::new(preempts).load_all();
eprintln!(
":: {}:\nSchedule:\n {}",
"Search successfully found a failing run with schedule:"
.green()
.bold(),
truncated(
1000,
serde_json::to_string(&init_schedule.clone_preemptions_only()).unwrap()
),
);
}
return Ok(rundat);
}
round += 1;
}
}
pub fn output_matches(&self, out: &Output) -> bool {
let mut answer = true;
if let Some(pat) = &self.target_stdout {
let str = String::from_utf8_lossy(&out.stdout);
if !pat.is_match(&str) {
if self.verbose {
eprintln!("Mismatch for stdout pattern {}", pat);
eprintln!("Stdout:\n{}", str);
}
answer = false;
}
}
if let Some(pat) = &self.target_stderr {
let str = String::from_utf8_lossy(&out.stderr);
if self.verbose {
eprintln!("Mismatch for stderr pattern {}", pat);
}
if !pat.is_match(&str) {
answer = false;
}
}
if !self.target_exit_code.is_match(out.status) {
if self.verbose {
eprintln!(
" Exit code {} is not what we're looking for.",
out.status.into_raw()
);
}
answer = false;
}
answer
}
}
#[cfg(test)]
mod tests {
use clap::Parser as _;
use super::*;
use crate::container::LogCapExceeded;
use crate::container::PolicyRefusal;
#[test]
fn a_failing_search_round_ends_the_search_with_its_error() {
let options =
AnalyzeOpts::try_parse_from(["analyze", "--analyze-seed=7", "--", "/bin/true"])
.unwrap();
for failing_round in 1..=4 {
let mut rounds = 0;
let result = options.search_rounds(|round, _sched_seed| {
assert_eq!(round, rounds, "rounds are numbered from 0");
rounds += 1;
if rounds == failing_round {
return Err(Error::new(LogCapExceeded));
}
Ok(None)
});
let Err(error) = result else {
panic!("round {failing_round} failed, yet the search returned a run")
};
assert!(
error.downcast_ref::<LogCapExceeded>().is_some(),
"round {failing_round}: the error lost its type: {error:#}"
);
assert_eq!(
rounds, failing_round,
"the search ran more rounds after round {failing_round} failed"
);
}
}
const RECORDED: &str = "2000-12-31T23:59:59.123456789Z";
const RECORDED_RFC3339: &str = "2000-12-31T23:59:59.123456789+00:00";
const OTHER: &str = "2026-01-01T00:00:00Z";
fn record_with_epoch(dir: &Path, name: &str, epoch: Option<&str>) -> PathBuf {
let source = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("test-resources/flaky_cas_sequence_schedules-passing.json");
let mut record: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(source).unwrap()).unwrap();
if let Some(epoch) = epoch {
record["epoch"] = serde_json::Value::from(epoch);
}
let path = dir.join(format!("{name}.json"));
std::fs::write(&path, record.to_string()).unwrap();
path
}
fn prepared_analysis(
dir: &Path,
run1: Option<&str>,
run2: Option<&str>,
run_args: &[&str],
) -> anyhow::Result<AnalyzeOpts> {
let run1 = format!(
"--run1-preemptions={}",
record_with_epoch(dir, "run1", run1).display()
);
let run2 = run2.map(|epoch| {
format!(
"--run2-preemptions={}",
record_with_epoch(dir, "run2", Some(epoch)).display()
)
});
let mut argv = vec!["hermit", "analyze", run1.as_str()];
argv.extend(run2.as_deref());
argv.push("--");
argv.extend(run_args);
argv.push("/bin/true");
let args = crate::Args::try_parse_from(&argv).unwrap();
let crate::Subcommand::Analyze(mut options) = args.command else {
panic!("{argv:?} is not analyze")
};
options.prepare(&args.global)?;
options.tmp_dir = Some(dir.to_path_buf());
Ok(*options)
}
#[test]
fn analyze_refuses_records_under_another_pipe_wake_latency() {
let dir = tempfile::tempdir().unwrap();
let record = |name: &str, latency: Option<u64>| {
let path = record_with_epoch(dir.path(), name, None);
let mut value: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
if let Some(latency) = latency {
value["pipe_wake_latency"] = serde_json::Value::from(latency);
}
std::fs::write(&path, value.to_string()).unwrap();
path
};
let prepare = |run1: &Path, run_args: &[&str]| {
let run1 = format!("--run1-preemptions={}", run1.display());
let mut argv = vec!["hermit", "analyze", run1.as_str(), "--"];
argv.extend(run_args);
argv.push("/bin/true");
let args = crate::Args::try_parse_from(&argv).unwrap();
let crate::Subcommand::Analyze(mut options) = args.command else {
panic!("{argv:?} is not analyze")
};
options.prepare(&args.global).map(|_| options)
};
let off = record("off", None);
let zero = record("zero", Some(0));
let options = prepare(&zero, &["--pipe-wake-latency=0"]).unwrap();
assert_eq!(options.trial_pipe_wake_latency(), Some(0));
let written = |options: &AnalyzeOpts| {
let (failing, passing, trial) = (
dir.path().join("final-failing.json"),
dir.path().join("final-passing.json"),
dir.path().join("trial.json"),
);
options.write_final_schedules(Vec::new(), &failing, Vec::new(), &passing);
options.write_trial_schedule(Vec::new(), &trial).unwrap();
[failing, passing, trial]
.map(|path| detcore::preemptions::read_recorded_pipe_wake_latency(&path).unwrap())
};
assert_eq!(written(&options), [Some(0); 3]);
assert_eq!(
options
.stamp_trial_record(PreemptionRecord::from_vecs(&Default::default()))
.pipe_wake_latency(),
Some(0)
);
let options_off = prepare(&off, &[]).unwrap();
assert_eq!(written(&options_off), [None; 3]);
let replay_flag = format!("--replay-preemptions-from={}", zero.display());
for (run1, run_args, named) in [
(&zero, vec![], "--run1-preemptions"),
(&off, vec!["--pipe-wake-latency=0"], "--run1-preemptions"),
(
&off,
vec![replay_flag.as_str()],
"--replay-preemptions-from",
),
] {
let error = prepare(run1, &run_args).unwrap_err();
assert!(
error
.downcast_ref::<crate::container::PolicyRefusal>()
.is_some(),
"{error:#}"
);
let message = format!("{error:#}");
assert!(
message.contains(named) && message.contains("cannot reproduce the run"),
"{run_args:?}: {message}"
);
}
}
#[test]
fn analyze_stamps_and_checks_the_signal_state_copy_setting() {
let dir = tempfile::tempdir().unwrap();
let record = |name: &str, copy: Option<bool>| {
let path = record_with_epoch(dir.path(), name, None);
let mut value: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
if let Some(copy) = copy {
value["signal_state_copy"] = serde_json::Value::from(copy);
}
std::fs::write(&path, value.to_string()).unwrap();
path
};
let prepare = |run1: &Path, run_args: &[&str]| {
let run1 = format!("--run1-preemptions={}", run1.display());
let mut argv = vec!["hermit", "analyze", run1.as_str(), "--"];
argv.extend(run_args);
argv.push("/bin/true");
let args = crate::Args::try_parse_from(&argv).unwrap();
let crate::Subcommand::Analyze(mut options) = args.command else {
panic!("{argv:?} is not analyze")
};
options.prepare(&args.global).map(|_| options)
};
let unsaid = record("unsaid", None);
let on = record("on", Some(true));
let off = record("off", Some(false));
let written = |options: &AnalyzeOpts| {
let trial = dir.path().join("copy-trial.json");
options.write_trial_schedule(Vec::new(), &trial).unwrap();
detcore::preemptions::read_recorded_signal_state_copy(&trial).unwrap()
};
let default = prepare(&on, &[]).unwrap();
assert!(default.trial_signal_state_copy());
assert_eq!(written(&default), Some(true));
let copy_off = prepare(&off, &["--no-signal-state-copy"]).unwrap();
assert_eq!(written(©_off), Some(false));
assert!(prepare(&unsaid, &[]).is_ok());
assert!(prepare(&unsaid, &["--no-signal-state-copy"]).is_ok());
for (run1, run_args) in [(&off, vec![]), (&on, vec!["--no-signal-state-copy"])] {
let error = prepare(run1, &run_args).unwrap_err();
assert!(
error
.downcast_ref::<crate::container::PolicyRefusal>()
.is_some(),
"{error:#}"
);
assert!(
format!("{error:#}").contains("signal-state copy"),
"{error:#}"
);
}
}
fn epoch_environment_arm(test: &str) -> Option<String> {
const CHILD: &str = "HERMIT_ANALYZE_EPOCH_TEST_CHILD";
if let Some(arm) = std::env::var_os(CHILD) {
return Some(arm.into_string().unwrap());
}
let name = format!("{}::{test}", module_path!().split_once("::").unwrap().1);
for arm in ["unset", "set"] {
let mut child = std::process::Command::new(std::env::current_exe().unwrap());
child
.args(["--exact", &name, "--nocapture"])
.env(CHILD, arm);
if arm == "set" {
child.env("HERMIT_EPOCH", OTHER);
} else {
child.env_remove("HERMIT_EPOCH");
}
let output = child.output().unwrap();
assert!(output.status.success(), "{arm}: {output:?}");
assert!(
String::from_utf8_lossy(&output.stdout).contains("1 passed"),
"{arm}: the child ran no test: {output:?}"
);
}
None
}
fn refused(error: anyhow::Error, expected: &str) {
assert!(error.downcast_ref::<PolicyRefusal>().is_some(), "{error:#}");
assert!(error.to_string().contains(expected), "{error:#}");
}
#[test]
fn analyze_replays_a_record_from_the_epoch_it_was_recorded_under() {
let Some(arm) =
epoch_environment_arm("analyze_replays_a_record_from_the_epoch_it_was_recorded_under")
else {
return;
};
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
if arm == "set" {
refused(
prepared_analysis(dir, Some(RECORDED), None, &[]).unwrap_err(),
&format!(
"the explicit virtual-time epoch 2026-01-01T00:00:00+00:00 (from --epoch or \
HERMIT_EPOCH) differs from the epoch {RECORDED_RFC3339} that \
--run1-preemptions"
),
);
return;
}
let analysis = prepared_analysis(dir, None, None, &[]).unwrap();
assert!(analysis.run_arg.is_empty(), "{:?}", analysis.run_arg);
assert_eq!(analysis.trial_epoch_for_test(), "2026-01-01T00:00:00+00:00");
let analysis = prepared_analysis(dir, Some(RECORDED), None, &[]).unwrap();
assert_eq!(analysis.trial_epoch_for_test(), RECORDED_RFC3339);
let analysis = prepared_analysis(dir, None, Some(RECORDED), &[]).unwrap();
assert_eq!(analysis.trial_epoch_for_test(), RECORDED_RFC3339);
let analysis =
prepared_analysis(dir, Some(RECORDED), None, &["--epoch", RECORDED]).unwrap();
assert!(analysis.run_arg.is_empty(), "{:?}", analysis.run_arg);
assert_eq!(analysis.trial_epoch_for_test(), RECORDED_RFC3339);
refused(
prepared_analysis(dir, Some(RECORDED), None, &["--epoch", OTHER]).unwrap_err(),
&format!("pass --epoch={RECORDED_RFC3339}"),
);
refused(
prepared_analysis(dir, Some(RECORDED), Some(OTHER), &[]).unwrap_err(),
&format!(
"was recorded under the virtual-time epoch {RECORDED_RFC3339} and \
--run2-preemptions"
),
);
}
#[test]
fn analyze_replays_a_network_trace_from_the_epoch_it_was_recorded_under() {
let Some(arm) = epoch_environment_arm(
"analyze_replays_a_network_trace_from_the_epoch_it_was_recorded_under",
) else {
return;
};
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
let trace = |name: &str, epoch: &str| {
let path = dir.join(format!("{name}.trace"));
detcore_model::network_engine::NetworkEngine::new_record(epoch.parse().unwrap())
.finish()
.unwrap()
.write_framed(std::fs::File::create(&path).unwrap())
.unwrap();
format!("--replay-networking={}", path.display())
};
let recorded = trace("recorded", RECORDED);
if arm == "set" {
refused(
prepared_analysis(dir, None, None, &[&recorded]).unwrap_err(),
&format!(
"the explicit virtual-time epoch 2026-01-01T00:00:00+00:00 (from --epoch or \
HERMIT_EPOCH) differs from the epoch {RECORDED_RFC3339} that \
--replay-networking"
),
);
return;
}
let analysis = prepared_analysis(dir, None, None, &[&recorded]).unwrap();
assert_eq!(analysis.trial_epoch_for_test(), RECORDED_RFC3339);
let analysis = prepared_analysis(dir, Some(RECORDED), None, &[&recorded]).unwrap();
assert_eq!(analysis.trial_epoch_for_test(), RECORDED_RFC3339);
let other = trace("other", OTHER);
refused(
prepared_analysis(dir, Some(RECORDED), None, &[&other]).unwrap_err(),
&format!(
"was recorded under the virtual-time epoch {RECORDED_RFC3339} and \
--replay-networking"
),
);
refused(
prepared_analysis(dir, None, None, &[&recorded, "--epoch", OTHER]).unwrap_err(),
&format!("pass --epoch={RECORDED_RFC3339}"),
);
}
#[test]
fn analyze_refuses_the_unimplemented_schedule_inputs_before_any_trial() {
for (options, named) in [
(
&["--run1-schedule=/nonexistent/run1.json"][..],
"--run1-schedule",
),
(
&["--run2-schedule=/nonexistent/run2.json"][..],
"--run2-schedule",
),
(
&[
"--run1-schedule=/nonexistent/run1.json",
"--run2-schedule=/nonexistent/run2.json",
][..],
"--run1-schedule or --run2-schedule",
),
] {
let mut argv = vec!["hermit", "analyze"];
argv.extend(options);
argv.extend(["--", "/bin/true"]);
let args = crate::Args::try_parse_from(&argv).unwrap();
let crate::Subcommand::Analyze(mut analyze) = args.command else {
panic!("{argv:?} is not analyze")
};
let error = analyze.main(&args.global).unwrap_err();
refused(
error,
&format!(
"hermit analyze does not implement {named}. Identify a run by a preemption \
record (--run1-preemptions or --run2-preemptions"
),
);
assert!(
analyze.tmp_dir.is_none(),
"{options:?}: a workspace was created before the refusal"
);
}
}
}