use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
#[cfg(test)]
mod tests;
use serde::{Deserialize, Serialize};
use crate::{Monitor, Notification, Receiver, Session, Watch, WatchOptions};
pub struct Rng(u64);
impl Rng {
pub fn new(seed: u64) -> Self {
Self(seed ^ 0x9E37_79B9_7F4A_7C15)
}
pub fn next_u64(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
pub fn range(&mut self, low: u64, high: u64) -> u64 {
assert!(low <= high, "empty range: {low}..={high}");
let span = (high - low).wrapping_add(1);
if span == 0 {
return self.next_u64();
}
let limit = u64::MAX - (u64::MAX % span);
loop {
let draw = self.next_u64();
if draw < limit {
return low + draw % span;
}
}
}
pub fn duration_range(&mut self, low: Duration, high: Duration) -> Duration {
let lo = u64::try_from(low.as_micros()).unwrap_or(u64::MAX);
let hi = u64::try_from(high.as_micros()).unwrap_or(u64::MAX);
Duration::from_micros(self.range(lo, hi))
}
}
pub const DEFAULT_SEED: u64 = 0x5EED_F17E_1234_5678;
pub fn seed() -> u64 {
std::env::var("WINDOWS_FILE_WATCHER_STRESS_SEED")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(DEFAULT_SEED)
}
pub fn env_u64(name: &str, default: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(default)
}
mod millis {
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::time::Duration;
pub(super) fn serialize<S: Serializer>(
duration: &Duration,
serializer: S,
) -> Result<S::Ok, S::Error> {
u64::try_from(duration.as_millis())
.map_err(|_| {
serde::ser::Error::custom(
"duration exceeds u64::MAX milliseconds and cannot be persisted as JSON",
)
})?
.serialize(serializer)
}
pub(super) fn deserialize<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<Duration, D::Error> {
Ok(Duration::from_millis(u64::deserialize(deserializer)?))
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum Operation {
CreateFile {
path: PathBuf,
},
CreateDir {
path: PathBuf,
},
RemoveFile {
path: PathBuf,
},
RemoveDir {
path: PathBuf,
},
Rename {
from: PathBuf,
to: PathBuf,
},
Wait {
#[serde(with = "millis")]
duration: Duration,
},
WaitRandom {
#[serde(with = "millis")]
low: Duration,
#[serde(with = "millis")]
high: Duration,
},
Repeat {
count: u64,
pattern: Vec<Operation>,
},
OpenSession {
name: String,
},
CloseSession {
name: String,
},
Subscribe {
session: String,
watch: String,
path: PathBuf,
subtree: bool,
},
CancelWatch {
watch: String,
},
OpenSessionBounded {
name: String,
bound: usize,
},
HoldOpen {
path: PathBuf,
#[serde(with = "millis")]
duration: Duration,
#[serde(default)]
ready_barrier: Option<String>,
},
Barrier {
name: String,
},
Concurrent {
branches: Vec<Vec<Operation>>,
},
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Scenario {
pub label: String,
pub operations: Vec<Operation>,
}
impl Scenario {
pub fn new(label: impl Into<String>) -> Self {
Self {
label: label.into(),
operations: Vec::new(),
}
}
#[must_use]
pub fn then(mut self, operation: Operation) -> Self {
self.operations.push(operation);
self
}
#[must_use]
pub fn then_repeated(mut self, count: u64, pattern: Vec<Operation>) -> Self {
self.operations.push(Operation::Repeat { count, pattern });
self
}
pub fn operation_count(&self) -> u64 {
fn count(operations: &[Operation]) -> u64 {
operations
.iter()
.map(|operation| match operation {
Operation::Repeat { count: n, pattern } => n.saturating_mul(count(pattern)),
Operation::Concurrent { branches } => branches
.iter()
.map(|branch| count(branch))
.fold(0u64, |total, branch_count| {
total.saturating_add(branch_count)
}),
_ => 1,
})
.fold(0u64, |total, one| total.saturating_add(one))
}
count(&self.operations)
}
}
pub struct TempDir {
path: PathBuf,
}
impl TempDir {
pub fn new(label: &str) -> Self {
let sanitized: String = label
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
c
} else {
'_'
}
})
.collect();
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_nanos();
let path = std::env::temp_dir().join(format!(
"windows-file-watcher-scenario-{sanitized}-{}-{nonce}",
std::process::id()
));
std::fs::create_dir(&path).expect("create temp dir");
Self { path }
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn cleanup(self) {
let _ = std::fs::remove_dir_all(&self.path);
}
}
pub struct HarnessParams {
pub timeout: Duration,
pub quiet_period: Duration,
}
impl Default for HarnessParams {
fn default() -> Self {
Self {
timeout: Duration::from_secs(120),
quiet_period: Duration::from_millis(300),
}
}
}
impl HarnessParams {
pub fn for_operation_count(operation_count: u64) -> Self {
Self {
timeout: Duration::from_secs(operation_count / 500 + 60).max(Self::default().timeout),
..Self::default()
}
}
}
struct DeadlineBarrier {
state: Mutex<DeadlineBarrierState>,
ready: std::sync::Condvar,
}
struct DeadlineBarrierState {
arrived: usize,
generation: u64,
}
impl DeadlineBarrier {
fn new() -> Self {
Self {
state: Mutex::new(DeadlineBarrierState {
arrived: 0,
generation: 0,
}),
ready: std::sync::Condvar::new(),
}
}
fn wait(&self, deadline: Instant, label: &str) {
let mut state = self
.state
.lock()
.unwrap_or_else(|poison| poison.into_inner());
let my_generation = state.generation;
state.arrived += 1;
if state.arrived >= 2 {
state.arrived = 0;
state.generation = state.generation.wrapping_add(1);
self.ready.notify_all();
return;
}
loop {
let now = Instant::now();
assert!(
now < deadline,
"a Barrier operation named '{label}' wedged: no second participant \
arrived before the harness's overall deadline -- check that the name \
is used by exactly two operations on two genuinely concurrent branches"
);
let (guard, timeout) = self
.ready
.wait_timeout(state, deadline - now)
.unwrap_or_else(|poison| poison.into_inner());
state = guard;
if state.generation != my_generation {
return;
}
if timeout.timed_out() {
continue; }
}
}
}
pub const INITIAL_SESSION: &str = "__initial_session__";
pub const INITIAL_WATCH: &str = "__initial_watch__";
pub struct Fleet<'m> {
monitor: &'m Monitor,
sessions: HashMap<String, (Session, Receiver)>,
watches: HashMap<String, (String, Watch)>,
barriers: HashMap<String, Arc<DeadlineBarrier>>,
}
impl<'m> Fleet<'m> {
pub fn new(monitor: &'m Monitor) -> Self {
Self {
monitor,
sessions: HashMap::new(),
watches: HashMap::new(),
barriers: HashMap::new(),
}
}
fn barrier(&mut self, name: &str) -> Arc<DeadlineBarrier> {
Arc::clone(
self.barriers
.entry(name.to_string())
.or_insert_with(|| Arc::new(DeadlineBarrier::new())),
)
}
pub fn open_session(&mut self, name: &str) {
assert!(
!self.sessions.contains_key(name),
"scenario bug: session '{name}' is already open"
);
let (session, receiver) = self.monitor.session();
self.sessions.insert(name.to_string(), (session, receiver));
}
pub fn open_session_bounded(&mut self, name: &str, bound: usize) {
assert!(
!self.sessions.contains_key(name),
"scenario bug: session '{name}' is already open"
);
let bound = std::num::NonZeroUsize::new(bound)
.unwrap_or_else(|| panic!("scenario bug: bound must be nonzero"));
let (session, receiver) = self.monitor.session_with_bound(bound);
self.sessions.insert(name.to_string(), (session, receiver));
}
pub fn close_session(&mut self, name: &str) {
let orphaned: Vec<String> = self
.watches
.iter()
.filter(|(_, (owner, _))| owner == name)
.map(|(watch_name, _)| watch_name.clone())
.collect();
for watch_name in orphaned {
self.watches.remove(&watch_name); }
self.sessions
.remove(name)
.unwrap_or_else(|| panic!("scenario bug: session '{name}' is not open"));
}
pub fn subscribe(&mut self, session_name: &str, watch_name: &str, path: &Path, subtree: bool) {
assert!(
!self.watches.contains_key(watch_name),
"scenario bug: watch '{watch_name}' already exists"
);
let (session, _) = self
.sessions
.get(session_name)
.unwrap_or_else(|| panic!("scenario bug: session '{session_name}' is not open"));
let watch = session
.subscribe(path, WatchOptions::new().subtree(subtree))
.expect("subscribe");
self.watches
.insert(watch_name.to_string(), (session_name.to_string(), watch));
}
pub fn cancel_watch(&mut self, watch_name: &str) {
let removed = self
.watches
.remove(watch_name)
.unwrap_or_else(|| panic!("scenario bug: watch '{watch_name}' does not exist"));
drop(removed); }
pub fn drain_available(&self, outcome: &mut HarnessOutcome) {
for (_, receiver) in self.sessions.values() {
while let Some(notification) = receiver.try_recv() {
outcome.record(¬ification);
}
}
}
}
#[derive(Debug, Default)]
pub struct HarnessOutcome {
pub batches: u64,
pub changes: u64,
pub desyncs: u64,
pub suspensions: u64,
pub resumptions: u64,
pub establishments: u64,
pub completions: u64,
pub retry_questions: u64,
pub volume_changes: u64,
}
impl HarnessOutcome {
pub fn record(&mut self, notification: &Notification) {
match notification {
Notification::Batch { changes, .. } => {
self.batches += 1;
self.changes += changes.len() as u64;
}
Notification::Desync { .. } => self.desyncs += 1,
Notification::Suspended { .. } => self.suspensions += 1,
Notification::Resumed { .. } => self.resumptions += 1,
Notification::Established { .. } => self.establishments += 1,
Notification::Completion { .. } => self.completions += 1,
Notification::RetryQuestion { .. } => self.retry_questions += 1,
Notification::VolumeChanged { .. } => self.volume_changes += 1,
}
}
pub fn total(&self) -> u64 {
self.batches
+ self.changes
+ self.desyncs
+ self.suspensions
+ self.resumptions
+ self.establishments
+ self.completions
+ self.retry_questions
+ self.volume_changes
}
}
mod share_mode {
pub(super) const READ_WRITE_NO_DELETE: u32 = 0x0000_0001 | 0x0000_0002;
}
pub fn apply_operation(
root: &Path,
fleet: &Mutex<Fleet<'_>>,
operation: &Operation,
rng: &mut Rng,
deadline: Instant,
) {
fn check_bounded_sleep(duration: Duration, deadline: Instant, label: &str) {
assert!(
Instant::now() + duration <= deadline,
"a {label} operation's duration would exceed the harness's overall deadline"
);
}
match operation {
Operation::CreateFile { path } => {
let target = root.join(path);
if let Some(parent) = target.parent() {
std::fs::create_dir_all(parent).expect("create parent directory");
}
std::fs::write(&target, b"x").expect("create file");
}
Operation::CreateDir { path } => {
std::fs::create_dir_all(root.join(path)).expect("create directory");
}
Operation::RemoveFile { path } => {
let _ = std::fs::remove_file(root.join(path));
}
Operation::RemoveDir { path } => {
let _ = std::fs::remove_dir_all(root.join(path));
}
Operation::Rename { from, to } => {
let _ = std::fs::rename(root.join(from), root.join(to));
}
Operation::Wait { duration } => {
check_bounded_sleep(*duration, deadline, "Wait");
std::thread::sleep(*duration);
}
Operation::WaitRandom { low, high } => {
let duration = rng.duration_range(*low, *high);
check_bounded_sleep(duration, deadline, "WaitRandom");
std::thread::sleep(duration);
}
Operation::Repeat { count, pattern } => {
for _ in 0..*count {
assert!(
Instant::now() < deadline,
"a Repeat operation wedged applying its pattern"
);
for step in pattern {
apply_operation(root, fleet, step, rng, deadline);
}
}
}
Operation::OpenSession { name } => fleet.lock().unwrap().open_session(name),
Operation::OpenSessionBounded { name, bound } => {
fleet.lock().unwrap().open_session_bounded(name, *bound)
}
Operation::CloseSession { name } => fleet.lock().unwrap().close_session(name),
Operation::Subscribe {
session,
watch,
path,
subtree,
} => fleet
.lock()
.unwrap()
.subscribe(session, watch, &root.join(path), *subtree),
Operation::CancelWatch { watch } => fleet.lock().unwrap().cancel_watch(watch),
Operation::HoldOpen {
path,
duration,
ready_barrier,
} => {
use std::os::windows::fs::OpenOptionsExt;
let file = std::fs::OpenOptions::new()
.read(true)
.share_mode(share_mode::READ_WRITE_NO_DELETE)
.open(root.join(path))
.expect("open file to hold");
if let Some(name) = ready_barrier {
let barrier = fleet.lock().unwrap().barrier(name);
barrier.wait(deadline, name);
}
check_bounded_sleep(*duration, deadline, "HoldOpen");
std::thread::sleep(*duration);
drop(file);
}
Operation::Barrier { name } => {
let barrier = fleet.lock().unwrap().barrier(name);
barrier.wait(deadline, name);
}
Operation::Concurrent { branches } => {
let branch_seeds: Vec<u64> = branches.iter().map(|_| rng.next_u64()).collect();
std::thread::scope(|scope| {
for (branch, branch_seed) in branches.iter().zip(branch_seeds) {
scope.spawn(move || {
let mut branch_rng = Rng::new(branch_seed);
for step in branch {
apply_operation(root, fleet, step, &mut branch_rng, deadline);
}
});
}
});
}
}
}
fn validate_confined(path: &Path) -> Result<(), String> {
for component in path.components() {
match component {
std::path::Component::Prefix(_) | std::path::Component::RootDir => {
return Err(format!(
"path {path:?} is not relative to the scenario root"
));
}
std::path::Component::ParentDir => {
return Err(format!("path {path:?} escapes the scenario root via '..'"));
}
std::path::Component::CurDir | std::path::Component::Normal(_) => {}
}
}
Ok(())
}
fn validate_paths(operations: &[Operation]) -> Result<(), String> {
for operation in operations {
match operation {
Operation::CreateFile { path }
| Operation::CreateDir { path }
| Operation::RemoveFile { path }
| Operation::RemoveDir { path } => validate_confined(path)?,
Operation::Rename { from, to } => {
validate_confined(from)?;
validate_confined(to)?;
}
Operation::Subscribe { path, .. } => validate_confined(path)?,
Operation::HoldOpen { path, .. } => validate_confined(path)?,
Operation::Wait { .. }
| Operation::WaitRandom { .. }
| Operation::OpenSession { .. }
| Operation::OpenSessionBounded { .. }
| Operation::CloseSession { .. }
| Operation::Barrier { .. }
| Operation::CancelWatch { .. } => {}
Operation::Repeat { pattern, .. } => validate_paths(pattern)?,
Operation::Concurrent { branches } => {
for branch in branches {
validate_paths(branch)?;
}
}
}
}
Ok(())
}
fn count_barrier_uses(operations: &[Operation], counts: &mut HashMap<String, u64>) {
for operation in operations {
match operation {
Operation::Barrier { name } => *counts.entry(name.clone()).or_insert(0) += 1,
Operation::HoldOpen {
ready_barrier: Some(name),
..
} => *counts.entry(name.clone()).or_insert(0) += 1,
Operation::Repeat { count, pattern } => {
let mut nested = HashMap::new();
count_barrier_uses(pattern, &mut nested);
for (name, uses) in nested {
*counts.entry(name).or_insert(0) += uses.saturating_mul(*count);
}
}
Operation::Concurrent { branches } => {
for branch in branches {
count_barrier_uses(branch, counts);
}
}
_ => {}
}
}
}
fn validate_barriers(operations: &[Operation]) -> Result<(), String> {
let mut counts = HashMap::new();
count_barrier_uses(operations, &mut counts);
for (name, count) in counts {
if count % 2 != 0 {
return Err(format!(
"barrier '{name}' is used {count} time(s), but each rendezvous round needs \
exactly 2 uses; an odd count always leaves at least one \
Barrier/HoldOpen.ready_barrier use with no partner, which would wait forever"
));
}
}
Ok(())
}
pub fn run_scenario(scenario: &Scenario, seed: u64, params: &HarnessParams) -> HarnessOutcome {
let (outcome, dir) = run_scenario_keep_dir(scenario, seed, params);
dir.cleanup();
outcome
}
pub fn run_scenario_keep_dir(
scenario: &Scenario,
seed: u64,
params: &HarnessParams,
) -> (HarnessOutcome, TempDir) {
if let Err(reason) = validate_paths(&scenario.operations) {
panic!("scenario '{}' is unsafe to run: {reason}", scenario.label);
}
if let Err(reason) = validate_barriers(&scenario.operations) {
panic!("scenario '{}' is unsafe to run: {reason}", scenario.label);
}
let dir = TempDir::new(&scenario.label);
let monitor = Monitor::new().expect("create the monitor");
let fleet = Mutex::new(Fleet::new(&monitor));
fleet.lock().unwrap().open_session(INITIAL_SESSION);
fleet
.lock()
.unwrap()
.subscribe(INITIAL_SESSION, INITIAL_WATCH, dir.path(), true);
let deadline = Instant::now() + params.timeout;
let mut rng = Rng::new(seed);
let mut outcome = HarnessOutcome::default();
for operation in &scenario.operations {
apply_operation(dir.path(), &fleet, operation, &mut rng, deadline);
fleet.lock().unwrap().drain_available(&mut outcome);
assert!(
Instant::now() < deadline,
"scenario '{}' wedged applying its operations",
scenario.label
);
}
let poll_interval = Duration::from_millis(10).min(params.quiet_period);
let mut last_activity = Instant::now();
while last_activity.elapsed() < params.quiet_period {
let before = outcome.total();
fleet.lock().unwrap().drain_available(&mut outcome);
if outcome.total() != before {
last_activity = Instant::now();
} else {
std::thread::sleep(poll_interval);
}
assert!(
Instant::now() < deadline,
"scenario '{}' wedged draining its notifications",
scenario.label
);
}
drop(fleet);
drop(monitor);
(outcome, dir)
}