use std::collections::BTreeMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::OnceLock;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::time::Duration;
pub const RACE_PROBE_ENV: &str = "HERMIT_RACE_PROBE";
pub const RACE_PROBE_SEED_ENV: &str = "HERMIT_RACE_PROBE_SEED";
pub const RACE_PROBE_HOLD_DIR_ENV: &str = "HERMIT_RACE_PROBE_HOLD_DIR";
const HOLD_POLL: Duration = Duration::from_millis(10);
pub(crate) const BG_PRE_CALL: &str = "bg_pre_call";
pub(crate) const BG_POST_CALL: &str = "bg_post_call";
pub(crate) const BG_HARVEST_SKIP: &str = "bg_harvest_skip";
pub(crate) const FIFO_STAGING_MAPPED: &str = "fifo_staging_mapped";
pub(crate) const FIFO_STAGING_RECLAIM: &str = "fifo_staging_reclaim";
pub(crate) const PROCFS_SPELLING_RESOLVED: &str = "procfs_spelling_resolved";
const POINTS: [&str; 6] = [
BG_PRE_CALL,
BG_POST_CALL,
BG_HARVEST_SKIP,
FIFO_STAGING_MAPPED,
FIFO_STAGING_RECLAIM,
PROCFS_SPELLING_RESOLVED,
];
#[derive(Clone, Copy, Debug, PartialEq)]
struct Spec {
lo_ms: u64,
hi_ms: u64,
probability: f64,
hold: bool,
}
#[derive(Debug)]
struct Probes {
seed: u64,
counter: AtomicU64,
points: BTreeMap<String, Spec>,
hold_dir: Option<PathBuf>,
}
#[derive(Clone, Debug, PartialEq)]
enum Perturbation {
Wait(Duration),
Hold(PathBuf),
}
fn parse_item(item: &str) -> Result<(String, Spec), String> {
let (name, rest) = item
.split_once('=')
.ok_or_else(|| format!("{item:?} is not point=LO[-HI][@P]"))?;
if !POINTS.contains(&name) {
return Err(format!("unknown point {name:?}; points are {POINTS:?}"));
}
if rest == "hold" {
return Ok((
name.to_owned(),
Spec {
lo_ms: 0,
hi_ms: 0,
probability: 1.0,
hold: true,
},
));
}
let (range, probability) = match rest.split_once('@') {
Some((range, p)) => (
range,
p.parse::<f64>()
.ok()
.filter(|p| (0.0..=1.0).contains(p))
.ok_or_else(|| format!("{item:?}: probability must be in [0, 1]"))?,
),
None => (rest, 1.0),
};
let number = |text: &str| {
text.parse::<u64>()
.map_err(|_| format!("{item:?}: {text:?} is not a number of milliseconds"))
};
let (lo_ms, hi_ms) = match range.split_once('-') {
Some((lo, hi)) => (number(lo)?, number(hi)?),
None => {
let ms = number(range)?;
(ms, ms)
}
};
if hi_ms < lo_ms {
return Err(format!("{item:?}: the range is empty"));
}
Ok((
name.to_owned(),
Spec {
lo_ms,
hi_ms,
probability,
hold: false,
},
))
}
impl Probes {
fn from_settings(
config: Option<&str>,
seed: Option<&str>,
hold_dir: Option<&str>,
) -> Option<Self> {
let config = config?;
let hold_dir = hold_dir.filter(|dir| !dir.is_empty()).map(PathBuf::from);
let mut points = BTreeMap::new();
for item in config.split(',').map(str::trim).filter(|s| !s.is_empty()) {
match parse_item(item) {
Ok((_, spec)) if spec.hold && hold_dir.is_none() => {
eprintln!(
"race_probe: ignoring {item:?}: {RACE_PROBE_HOLD_DIR_ENV} is not set"
);
}
Ok((name, spec)) => {
points.insert(name, spec);
}
Err(error) => eprintln!("race_probe: ignoring {error}"),
}
}
if points.is_empty() {
return None;
}
let seed = match seed.map(str::parse::<u64>) {
Some(Ok(seed)) => seed,
_ => {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos() as u64)
^ u64::from(std::process::id())
}
};
Some(Probes {
seed,
counter: AtomicU64::new(0),
points,
hold_dir,
})
}
fn choose(&self, point: &str) -> Option<Perturbation> {
let spec = self.points.get(point)?;
if spec.hold {
return Some(Perturbation::Hold(self.hold_dir.as_ref()?.join(point)));
}
let visit = self.counter.fetch_add(1, Ordering::Relaxed);
let r = splitmix(self.seed ^ splitmix(visit));
let unit = (r >> 11) as f64 / (1u64 << 53) as f64;
if unit >= spec.probability {
return None;
}
let span = spec.hi_ms - spec.lo_ms;
let ms = spec.lo_ms
+ if span == 0 {
0
} else {
splitmix(r) % (span + 1)
};
Some(Perturbation::Wait(Duration::from_millis(ms)))
}
}
fn probes() -> Option<&'static Probes> {
static PROBES: OnceLock<Option<Probes>> = OnceLock::new();
PROBES
.get_or_init(|| {
let config = std::env::var(RACE_PROBE_ENV).ok()?;
let seed = std::env::var(RACE_PROBE_SEED_ENV).ok();
let hold_dir = std::env::var(RACE_PROBE_HOLD_DIR_ENV).ok();
let probes =
Probes::from_settings(Some(&config), seed.as_deref(), hold_dir.as_deref())?;
eprintln!(
"race_probe: {RACE_PROBE_ENV}={config} {RACE_PROBE_SEED_ENV}={}",
probes.seed
);
Some(probes)
})
.as_ref()
}
fn splitmix(mut z: u64) -> u64 {
z = z.wrapping_add(0x9e37_79b9_7f4a_7c15);
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
z ^ (z >> 31)
}
fn choose(point: &str) -> Option<Perturbation> {
let perturbation = probes()?.choose(point)?;
tracing::debug!(target: "race_probe", "race_probe: {point} perturbed ({perturbation:?})");
Some(perturbation)
}
pub(crate) async fn delay(point: &str) {
match choose(point) {
Some(Perturbation::Wait(wait)) if !wait.is_zero() => tokio::time::sleep(wait).await,
Some(Perturbation::Hold(release)) => hold(point, &release).await,
_ => {}
}
}
async fn hold(point: &str, release: &Path) {
if release.exists() {
return;
}
eprintln!(
"race_probe: {point} holding until {} exists",
release.display()
);
while !release.exists() {
tokio::time::sleep(HOLD_POLL).await;
}
}
pub(crate) fn perturb(point: &str) -> bool {
choose(point).is_some()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn race_probes_are_off_unless_configured() {
assert!(Probes::from_settings(None, None, None).is_none());
assert!(Probes::from_settings(None, Some("7"), None).is_none());
assert!(Probes::from_settings(Some(""), Some("7"), None).is_none());
assert!(Probes::from_settings(Some(" , "), None, None).is_none());
assert!(Probes::from_settings(Some("no_such_point=5"), None, None).is_none());
assert!(std::env::var_os(RACE_PROBE_ENV).is_none());
assert!(probes().is_none());
for point in POINTS {
assert!(!perturb(point));
assert_eq!(choose(point), None);
}
}
#[test]
fn race_probe_settings_parse_as_documented() {
assert_eq!(
parse_item("bg_pre_call=300"),
Ok((
BG_PRE_CALL.to_owned(),
Spec {
lo_ms: 300,
hi_ms: 300,
probability: 1.0,
hold: false,
}
))
);
assert_eq!(
parse_item("bg_post_call=0-50@0.25"),
Ok((
BG_POST_CALL.to_owned(),
Spec {
lo_ms: 0,
hi_ms: 50,
probability: 0.25,
hold: false,
}
))
);
for bad in [
"bg_pre_call",
"bg_pre_call=x",
"bg_pre_call=5-2",
"bg_pre_call=1@2",
"bg_harvest_skip=0@-0.1",
"unknown=1",
] {
assert!(parse_item(bad).is_err(), "{bad} parsed");
}
let probes = Probes::from_settings(
Some("bg_pre_call=x, bg_harvest_skip=0@0.5"),
Some("1"),
None,
)
.unwrap();
assert_eq!(probes.points.len(), 1);
assert!(probes.points.contains_key(BG_HARVEST_SKIP));
let probes = Probes::from_settings(
Some("bg_pre_call=10@0,bg_pre_call=20@1,bg_pre_call=x,bg_harvest_skip=0@0.5"),
Some("7"),
None,
)
.unwrap();
assert_eq!(probes.points.len(), 2);
assert_eq!(
probes.points.get(BG_PRE_CALL),
Some(&Spec {
lo_ms: 20,
hi_ms: 20,
probability: 1.0,
hold: false,
})
);
assert_eq!(
probes.choose(BG_PRE_CALL),
Some(Perturbation::Wait(Duration::from_millis(20)))
);
}
#[test]
fn race_probe_choices_follow_probability_range_and_seed() {
let draw = |config: &str, seed: &str, point: &str| {
let probes = Probes::from_settings(Some(config), Some(seed), None).unwrap();
(0..1000).map(|_| probes.choose(point)).collect::<Vec<_>>()
};
assert!(
draw("bg_harvest_skip=0@0", "1", BG_HARVEST_SKIP)
.iter()
.all(Option::is_none)
);
assert!(
draw("bg_harvest_skip=0@1", "1", BG_HARVEST_SKIP)
.iter()
.all(|wait| *wait == Some(Perturbation::Wait(Duration::ZERO)))
);
assert!(
draw("bg_harvest_skip=0@1", "1", BG_PRE_CALL)
.iter()
.all(Option::is_none)
);
let half = draw("bg_pre_call=10-20@0.5", "3", BG_PRE_CALL);
let taken = half.iter().flatten().count();
assert!((400..600).contains(&taken), "{taken} of 1000 at 0.5");
assert!(half.iter().flatten().all(|wait| {
matches!(wait, Perturbation::Wait(wait)
if (Duration::from_millis(10)..=Duration::from_millis(20)).contains(wait))
}));
assert_eq!(half, draw("bg_pre_call=10-20@0.5", "3", BG_PRE_CALL));
assert_ne!(half, draw("bg_pre_call=10-20@0.5", "4", BG_PRE_CALL));
let config = Some("bg_pre_call=10-20@0.5,bg_harvest_skip=0@0.5");
let with_missing = Probes::from_settings(config, Some("3"), None).unwrap();
let direct = Probes::from_settings(config, Some("3"), None).unwrap();
for _ in 0..1000 {
assert_eq!(with_missing.choose("no_such_point"), None);
assert_eq!(with_missing.choose(BG_POST_CALL), None);
assert_eq!(with_missing.choose(BG_PRE_CALL), direct.choose(BG_PRE_CALL));
assert_eq!(
with_missing.choose(BG_HARVEST_SKIP),
direct.choose(BG_HARVEST_SKIP)
);
}
}
#[test]
fn a_hold_needs_a_hold_directory_and_names_its_release_file() {
assert_eq!(
parse_item("fifo_staging_mapped=hold"),
Ok((
FIFO_STAGING_MAPPED.to_owned(),
Spec {
lo_ms: 0,
hi_ms: 0,
probability: 1.0,
hold: true,
}
))
);
for bad in ["fifo_staging_mapped=hold@0.5", "fifo_staging_mapped=holds"] {
assert!(parse_item(bad).is_err(), "{bad} parsed");
}
let probes = Probes::from_settings(
Some("fifo_staging_mapped=hold,bg_harvest_skip=0"),
Some("1"),
None,
)
.unwrap();
assert!(!probes.points.contains_key(FIFO_STAGING_MAPPED));
assert!(Probes::from_settings(Some("fifo_staging_mapped=hold"), None, Some("")).is_none());
let config = Some("fifo_staging_mapped=hold,fifo_staging_reclaim=hold,bg_pre_call=1-9@0.5");
let holding = Probes::from_settings(config, Some("5"), Some("/hold")).unwrap();
let direct = Probes::from_settings(Some("bg_pre_call=1-9@0.5"), Some("5"), None).unwrap();
for _ in 0..100 {
assert_eq!(
holding.choose(FIFO_STAGING_MAPPED),
Some(Perturbation::Hold(PathBuf::from(
"/hold/fifo_staging_mapped"
)))
);
assert_eq!(
holding.choose(FIFO_STAGING_RECLAIM),
Some(Perturbation::Hold(PathBuf::from(
"/hold/fifo_staging_reclaim"
)))
);
assert_eq!(holding.choose(BG_PRE_CALL), direct.choose(BG_PRE_CALL));
}
}
#[tokio::test(start_paused = true)]
async fn a_held_visit_waits_for_its_release_file() {
let dir = tempfile::tempdir().unwrap();
let release = dir.path().join(FIFO_STAGING_MAPPED);
let mut held = Box::pin(hold(FIFO_STAGING_MAPPED, &release));
tokio::select! {
_ = &mut held => panic!("released before its file existed"),
_ = tokio::time::sleep(Duration::from_secs(60)) => {}
}
std::fs::write(&release, b"").unwrap();
tokio::time::timeout(Duration::from_secs(1), held)
.await
.expect("released once its file exists");
tokio::time::timeout(Duration::ZERO, hold(FIFO_STAGING_MAPPED, &release))
.await
.expect("an existing file releases at once");
}
}