use std::num::NonZeroUsize;
use std::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle};
use std::sync::Arc;
use std::time::Duration;
use windows_sys::Win32::Foundation::{WAIT_FAILED, WAIT_OBJECT_0};
use windows_sys::Win32::System::Threading::WaitForSingleObject;
use windows_threadpool_sys::wait::{ThreadpoolWait, WaitableHandle};
use super::{Delivery, Notification, Receiver, WatchId, channel};
use crate::directory::{FailureCode, FaultDetail, OpenFailure};
use crate::notify::{Change, ChangeKind, DesyncCause, RelativeName};
fn deliver(sender: &super::Sender, notification: Notification) {
assert_eq!(
sender.send(notification),
Delivery::Queued,
"the queue was expected to have room"
);
}
fn change(name: &str) -> Change {
Change {
kind: ChangeKind::Added,
name: RelativeName::from_units(name.encode_utf16().collect::<Vec<u16>>()),
}
}
fn batch(watch: WatchId, names: &[&str]) -> Notification {
Notification::Batch {
watch,
changes: names.iter().map(|name| change(name)).collect(),
}
}
fn test_detail() -> FaultDetail {
FaultDetail {
failure: OpenFailure::Retryable,
code: FailureCode::Win32(0),
}
}
fn names(notification: &Notification) -> Vec<String> {
match notification {
Notification::Batch { changes, .. } => changes
.iter()
.map(|c| c.name.to_os_string().to_string_lossy().into_owned())
.collect(),
Notification::Desync { .. }
| Notification::Completion { .. }
| Notification::Suspended { .. }
| Notification::Resumed { .. }
| Notification::Established { .. }
| Notification::RetryQuestion { .. }
| Notification::VolumeChanged { .. } => Vec::new(),
}
}
#[test]
fn an_empty_queue_yields_nothing() {
let (_sender, receiver) = channel();
assert!(receiver.try_recv().is_none());
assert!(receiver.is_empty());
assert_eq!(receiver.len(), 0);
assert!(!receiver.is_disconnected());
}
#[test]
fn a_sent_notification_is_received() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
let received = receiver.try_recv().expect("a notification");
assert_eq!(names(&received), vec!["a.txt"]);
assert_eq!(received.watch(), WatchId::from_raw(1));
}
#[test]
fn notifications_are_received_in_send_order() {
let (sender, receiver) = channel();
let watch = WatchId::from_raw(7);
for index in 0..16 {
deliver(&sender, batch(watch, &[&format!("file-{index}.txt")]));
}
for index in 0..16 {
let received = next(&receiver, "a notification");
assert_eq!(names(&received), vec![format!("file-{index}.txt")]);
}
}
#[test]
fn changes_and_desyncs_share_one_ordered_stream() {
let (sender, receiver) = channel();
let watch = WatchId::from_raw(2);
deliver(&sender, batch(watch, &["before.txt"]));
deliver(
&sender,
Notification::Desync {
watch,
cause: DesyncCause::Overflow,
},
);
deliver(&sender, batch(watch, &["after.txt"]));
assert_eq!(names(&next(&receiver, "first")), vec!["before.txt"]);
assert!(matches!(
next(&receiver, "second"),
Notification::Desync {
cause: DesyncCause::Overflow,
..
}
));
assert_eq!(names(&next(&receiver, "third")), vec!["after.txt"]);
}
#[test]
fn every_notification_carries_its_subscription() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(10), &["a.txt"]));
deliver(
&sender,
Notification::Desync {
watch: WatchId::from_raw(20),
cause: DesyncCause::Coarse,
},
);
assert_eq!(next(&receiver, "a").watch(), WatchId::from_raw(10));
assert_eq!(next(&receiver, "b").watch(), WatchId::from_raw(20));
}
#[test]
fn several_subscriptions_can_share_one_queue() {
let (sender, receiver) = channel();
for id in 0..8_u64 {
deliver(&sender, batch(WatchId::from_raw(id), &["shared.txt"]));
}
let mut seen: Vec<u64> = Vec::new();
while let Some(item) = receiver.try_recv() {
seen.push(item.watch().get());
}
assert_eq!(seen, (0..8).collect::<Vec<u64>>());
}
#[test]
fn the_queue_length_tracks_what_is_pending() {
let (sender, receiver) = channel();
assert_eq!(receiver.len(), 0);
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
deliver(&sender, batch(WatchId::from_raw(1), &["b.txt"]));
assert_eq!(receiver.len(), 2);
assert!(!receiver.is_empty());
let _ = receiver.try_recv();
assert_eq!(receiver.len(), 1);
let _ = receiver.try_recv();
assert!(receiver.is_empty());
}
#[test]
fn many_senders_can_enqueue_concurrently() {
let (sender, receiver) = channel();
const SENDERS: u64 = 8;
const EACH: usize = 64;
let handles: Vec<_> = (0..SENDERS)
.map(|id| {
let sender = sender.clone();
std::thread::spawn(move || {
for index in 0..EACH {
deliver(
&sender,
batch(WatchId::from_raw(id), &[&format!("f-{index}")]),
);
}
})
})
.collect();
for handle in handles {
handle.join().expect("sender thread");
}
drop(sender);
let mut per_watch = std::collections::HashMap::<u64, usize>::new();
for _ in 0..(SENDERS as usize * EACH) {
let item = next(&receiver, "a notification from one of the senders");
*per_watch.entry(item.watch().get()).or_default() += 1;
}
assert_stream_ended(&receiver, "after every sender finished and dropped");
assert_eq!(per_watch.len(), SENDERS as usize);
for id in 0..SENDERS {
assert_eq!(per_watch[&id], EACH, "watch {id} lost notifications");
}
}
#[test]
fn order_is_preserved_per_sender_under_concurrency() {
let (sender, receiver) = channel();
const EACH: usize = 128;
let second = sender.clone();
let a = std::thread::spawn(move || {
for index in 0..EACH {
deliver(
&sender,
batch(WatchId::from_raw(1), &[&format!("a-{index}")]),
);
}
});
let b = std::thread::spawn(move || {
for index in 0..EACH {
deliver(
&second,
batch(WatchId::from_raw(2), &[&format!("b-{index}")]),
);
}
});
a.join().expect("a");
b.join().expect("b");
let mut seen_a = Vec::new();
let mut seen_b = Vec::new();
for _ in 0..(2 * EACH) {
let item = next(&receiver, "a notification from either producer");
let name = names(&item).remove(0);
if item.watch() == WatchId::from_raw(1) {
seen_a.push(name);
} else {
seen_b.push(name);
}
}
assert_stream_ended(&receiver, "after both producers finished and dropped");
let expected_a: Vec<String> = (0..EACH).map(|i| format!("a-{i}")).collect();
let expected_b: Vec<String> = (0..EACH).map(|i| format!("b-{i}")).collect();
assert_eq!(seen_a, expected_a);
assert_eq!(seen_b, expected_b);
}
#[test]
fn recv_blocks_until_something_arrives() {
let (sender, receiver) = channel();
let handle = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
deliver(&sender, batch(WatchId::from_raw(1), &["late.txt"]));
});
let received = next(&receiver, "the late notification");
assert_eq!(names(&received), vec!["late.txt"]);
handle.join().expect("sender thread");
}
#[test]
fn recv_returns_none_once_every_sender_is_gone() {
let (sender, receiver) = channel();
drop(sender);
assert_stream_ended(&receiver, "every sender is gone");
assert!(receiver.is_disconnected());
}
#[test]
fn a_blocked_receiver_is_woken_by_the_last_sender_dropping() {
let (sender, receiver) = channel();
let (tx, rx) = std::sync::mpsc::channel();
let waiter = std::thread::spawn(move || {
let outcome = receiver.recv();
let _ = tx.send(outcome.is_none());
});
let dropper = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
drop(sender);
});
match rx.recv_timeout(Duration::from_secs(5)) {
Ok(disconnected) => assert!(
disconnected,
"a woken receiver must observe disconnection, not a notification"
),
Err(_) => panic!(
"dropping the last sender did not wake a blocked receiver within 5s \
-- the wake in `Drop for Sender` is the rule this asserts"
),
}
dropper.join().expect("dropper thread");
drop(waiter);
}
#[test]
fn queued_items_are_drained_before_disconnection_is_reported() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
deliver(&sender, batch(WatchId::from_raw(1), &["b.txt"]));
drop(sender);
assert_eq!(names(&next(&receiver, "a")), vec!["a.txt"]);
assert_eq!(names(&next(&receiver, "b")), vec!["b.txt"]);
assert_stream_ended(&receiver, "the stream should be finished");
}
#[test]
fn disconnection_needs_every_sender_gone_not_just_one() {
let (sender, receiver) = channel();
let clone = sender.clone();
drop(sender);
assert!(
!receiver.is_disconnected(),
"one surviving sender keeps the queue open"
);
drop(clone);
assert!(receiver.is_disconnected());
}
#[test]
fn recv_timeout_gives_up_and_can_be_retried() {
let (sender, receiver) = channel();
let start = std::time::Instant::now();
assert!(receiver.recv_timeout(Duration::from_millis(50)).is_none());
assert!(
start.elapsed() >= Duration::from_millis(40),
"recv_timeout returned early"
);
deliver(&sender, batch(WatchId::from_raw(1), &["eventual.txt"]));
let received = receiver
.recv_timeout(Duration::from_millis(500))
.expect("the notification");
assert_eq!(names(&received), vec!["eventual.txt"]);
}
#[test]
fn recv_timeout_returns_immediately_when_something_is_queued() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["ready.txt"]));
let start = std::time::Instant::now();
assert!(receiver.recv_timeout(Duration::from_secs(30)).is_some());
assert!(
start.elapsed() < Duration::from_secs(1),
"a queued item must not wait for the timeout"
);
}
#[test]
fn a_sender_survives_the_receiver_being_dropped() {
let (sender, receiver) = channel();
drop(receiver);
deliver(&sender, batch(WatchId::from_raw(1), &["orphan.txt"]));
deliver(&sender, batch(WatchId::from_raw(1), &["orphan2.txt"]));
}
#[test]
fn senders_can_be_cloned_across_threads() {
let (sender, receiver) = channel();
let shared = Arc::new(sender);
let handles: Vec<_> = (0..4)
.map(|id| {
let sender = Arc::clone(&shared);
std::thread::spawn(move || deliver(&sender, batch(WatchId::from_raw(id), &["x"])))
})
.collect();
for handle in handles {
handle.join().expect("thread");
}
assert_eq!(receiver.len(), 4);
}
#[test]
fn a_watch_id_round_trips_its_raw_value() {
for raw in [0_u64, 1, 42, u64::MAX] {
assert_eq!(WatchId::from_raw(raw).get(), raw);
}
}
#[test]
fn watch_ids_compare_and_hash_by_value() {
use std::collections::HashSet;
let mut set = HashSet::new();
assert!(set.insert(WatchId::from_raw(1)));
assert!(!set.insert(WatchId::from_raw(1)), "equal ids are one entry");
assert!(set.insert(WatchId::from_raw(2)));
assert_ne!(WatchId::from_raw(1), WatchId::from_raw(2));
}
fn bounded(bound: usize) -> (super::Sender, super::Receiver) {
super::channel_with_bound(NonZeroUsize::new(bound).expect("a non-zero bound"))
}
fn fill(sender: &super::Sender, watch: WatchId, count: usize) {
for index in 0..count {
deliver(sender, batch(watch, &[&format!("fill-{index}")]));
}
}
#[test]
fn the_default_bound_is_reported() {
let (_sender, receiver) = channel();
assert_eq!(receiver.capacity(), super::DEFAULT_BOUND.get());
}
#[test]
fn a_chosen_bound_is_honoured() {
let (sender, receiver) = bounded(4);
assert_eq!(receiver.capacity(), 4);
fill(&sender, WatchId::from_raw(1), 4);
assert_eq!(receiver.len(), 4);
assert_eq!(
sender.send(batch(WatchId::from_raw(1), &["overflow.txt"])),
Delivery::Latched
);
assert_eq!(receiver.len(), 4, "the queue must not exceed its bound");
}
#[test]
fn has_room_accounts_for_a_pending_latch() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(1);
fill(&sender, watch, 1);
assert_eq!(
sender.send(batch(watch, &["lost.txt"])),
Delivery::Latched,
"the one slot is already full"
);
assert!(!sender.has_room(), "the queue is full, no room at all");
let _ = next(&receiver, "the queued entry");
assert!(
!sender.has_room(),
"the freed slot is already earmarked for the pending latch flush"
);
assert_eq!(
sender.send(batch(watch, &["new.txt"])),
Delivery::Latched,
"the freed slot went to the owed desync flush, not this notification"
);
}
#[test]
fn a_dropped_notification_is_reported_as_a_desync_not_lost_silently() {
let (sender, receiver) = bounded(2);
let watch = WatchId::from_raw(9);
fill(&sender, watch, 2);
assert_eq!(
sender.send(batch(watch, &["dropped.txt"])),
Delivery::Latched
);
assert_eq!(names(&next(&receiver, "first")), vec!["fill-0"]);
assert_eq!(names(&next(&receiver, "second")), vec!["fill-1"]);
let reported = next(&receiver, "the loss report");
assert!(matches!(
reported,
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert_eq!(reported.watch(), watch);
}
#[test]
fn a_latched_loss_is_reported_after_everything_that_preceded_it() {
let (sender, receiver) = bounded(3);
let watch = WatchId::from_raw(1);
deliver(&sender, batch(watch, &["first.txt"]));
deliver(&sender, batch(watch, &["second.txt"]));
deliver(&sender, batch(watch, &["third.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(names(&next(&receiver, "first")), vec!["first.txt"]);
assert_eq!(names(&next(&receiver, "second")), vec!["second.txt"]);
deliver(&sender, batch(watch, &["fourth.txt"]));
assert_eq!(names(&next(&receiver, "third")), vec!["third.txt"]);
assert!(
matches!(
next(&receiver, "the desync"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
),
"the loss belongs after the changes that preceded it and before the one that followed"
);
assert_eq!(names(&next(&receiver, "fourth")), vec!["fourth.txt"]);
}
#[test]
fn a_latched_loss_reaches_a_receiver_even_if_nothing_further_is_sent() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(3);
deliver(&sender, batch(watch, &["only.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(names(&next(&receiver, "the queued one")), vec!["only.txt"]);
assert!(matches!(
next(&receiver, "the latched one"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert!(receiver.try_recv().is_none());
}
#[track_caller]
fn next(receiver: &Receiver, expected: &str) -> Notification {
receiver
.recv_timeout(Duration::from_secs(5))
.unwrap_or_else(|| panic!("expected {expected} within 5s, but nothing arrived"))
}
#[track_caller]
fn assert_stream_ended(receiver: &Receiver, what: &str) {
assert!(
receiver.recv_timeout(Duration::from_secs(5)).is_none(),
"{what}: a notification arrived where the stream should have ended, so \
more was delivered than this test accounted for"
);
assert!(
receiver.is_disconnected(),
"{what}: nothing arrived within 5s, but the queue still reports senders, \
so the stream has not ended -- either a sender outlived the point this \
test expected it to drop, or nothing was delivered in time"
);
}
#[test]
fn a_latched_loss_is_delivered_even_after_every_sender_is_gone() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(4);
deliver(&sender, batch(watch, &["only.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
drop(sender);
assert_eq!(names(&next(&receiver, "the queued one")), vec!["only.txt"]);
assert!(matches!(
next(&receiver, "the latched one"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert_stream_ended(&receiver, "and only then is it finished");
}
#[test]
fn repeated_losses_for_one_subscription_coalesce() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(5);
deliver(&sender, batch(watch, &["only.txt"]));
for index in 0..100 {
assert_eq!(
sender.send(batch(watch, &[&format!("lost-{index}")])),
Delivery::Latched
);
}
assert_eq!(
receiver.latched(),
1,
"one subscription, one pending desync"
);
}
#[test]
fn losses_are_latched_per_subscription() {
let (sender, receiver) = bounded(1);
deliver(&sender, batch(WatchId::from_raw(0), &["only.txt"]));
for watch in 1..5_u64 {
assert_eq!(
sender.send(batch(WatchId::from_raw(watch), &["lost.txt"])),
Delivery::Latched
);
}
assert_eq!(receiver.latched(), 4, "each subscription is owed its own");
let _ = next(&receiver, "the queued one");
let mut reported: Vec<u64> = Vec::new();
while let Some(item) = receiver.try_recv() {
reported.push(item.watch().get());
}
reported.sort_unstable();
assert_eq!(reported, vec![1, 2, 3, 4]);
}
#[test]
fn a_reserved_slot_cannot_be_taken_by_the_best_effort_path() {
let (sender, receiver) = bounded(2);
let reservation = sender.reserve().expect("a slot");
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
assert_eq!(
sender.send(batch(WatchId::from_raw(1), &["b.txt"])),
Delivery::Latched,
"observation must not consume the reserved slot"
);
reservation.send(Notification::Desync {
watch: WatchId::from_raw(1),
cause: DesyncCause::Reestablished,
});
assert_eq!(receiver.len(), 2);
}
#[test]
fn a_reservation_is_refused_when_there_is_no_room() {
let (sender, _receiver) = bounded(2);
let first = sender.reserve().expect("a slot");
let second = sender.reserve().expect("a slot");
assert!(
sender.reserve().is_none(),
"the bound applies to reservations"
);
drop(first);
assert!(sender.reserve().is_some(), "releasing one frees it");
drop(second);
}
#[test]
fn a_freed_slot_reports_the_owed_loss_before_it_carries_new_changes() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(1);
let reservation = sender.reserve().expect("a slot");
assert_eq!(
sender.send(batch(watch, &["blocked.txt"])),
Delivery::Latched
);
drop(reservation);
assert_eq!(sender.send(batch(watch, &["next.txt"])), Delivery::Latched);
assert!(matches!(
next(&receiver, "the flushed desync"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert!(matches!(
next(&receiver, "the second desync"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert_eq!(receiver.latched(), 0, "nothing further is owed");
deliver(&sender, batch(watch, &["now.txt"]));
assert_eq!(names(&next(&receiver, "now")), vec!["now.txt"]);
}
#[test]
fn a_reserved_send_never_fails_however_full_the_queue_is() {
let (sender, receiver) = bounded(4);
let reservation = sender.reserve().expect("a slot");
let watch = WatchId::from_raw(1);
for index in 0..100 {
let _ = sender.send(batch(watch, &[&format!("noise-{index}")]));
}
assert_eq!(receiver.len(), 3, "the reserved slot stayed empty");
reservation.send(Notification::Desync {
watch,
cause: DesyncCause::Reestablished,
});
assert_eq!(receiver.len(), 4);
let mut causes = Vec::new();
while let Some(item) = receiver.try_recv() {
if let Notification::Desync { cause, .. } = item {
causes.push(cause);
}
}
assert!(
causes.contains(&DesyncCause::Reestablished),
"the reserved notification must be delivered, saw {causes:?}"
);
}
#[test]
fn a_reservation_keeps_the_queue_connected() {
let (sender, receiver) = bounded(2);
let reservation = sender.reserve().expect("a slot");
drop(sender);
assert!(
!receiver.is_disconnected(),
"an outstanding reservation is a pending delivery"
);
reservation.send(Notification::Desync {
watch: WatchId::from_raw(1),
cause: DesyncCause::Reestablished,
});
assert!(receiver.recv_timeout(Duration::from_secs(5)).is_some());
assert!(receiver.is_disconnected());
}
#[test]
fn reservations_can_be_taken_from_several_threads() {
let (sender, receiver) = bounded(64);
let handles: Vec<_> = (0..8)
.map(|id| {
let sender = sender.clone();
std::thread::spawn(move || {
for _ in 0..8 {
let reservation = sender.reserve().expect("a slot");
reservation.send(Notification::Desync {
watch: WatchId::from_raw(id),
cause: DesyncCause::Reestablished,
});
}
})
})
.collect();
for handle in handles {
handle.join().expect("thread");
}
assert_eq!(receiver.len(), 64, "every reserved send was delivered");
}
#[test]
fn a_standing_slot_can_send_repeatedly_without_ever_failing() {
let (sender, receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
for _ in 0..5 {
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert!(receiver.try_recv().is_some(), "each send is delivered");
}
}
#[test]
fn a_standing_send_does_not_overflow_capacity_while_the_queue_is_otherwise_full() {
let (sender, receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
deliver(&sender, batch(watch, &["a.txt"]));
assert_eq!(
sender.send(batch(watch, &["b.txt"])),
Delivery::Latched,
"the queue has no free capacity left"
);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert_eq!(receiver.len(), 2, "the standing send still landed");
}
#[test]
fn dropping_a_standing_slot_while_its_message_is_still_queued_releases_capacity_once() {
let (sender, receiver) = bounded(1);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
drop(slot);
assert!(
sender.reserve().is_none(),
"the message still occupies the queue's only slot"
);
assert!(
receiver.try_recv().is_some(),
"the queued message still arrives"
);
assert!(
sender.reserve().is_some(),
"capacity is returned to the pool once the slot's last message drains"
);
}
#[test]
fn dropping_an_unused_standing_slot_returns_its_capacity() {
let (sender, _receiver) = bounded(1);
let slot = sender.reserve_standing().expect("a slot");
assert!(
sender.reserve().is_none(),
"the standing slot's carve-out is the only capacity"
);
drop(slot);
assert!(
sender.reserve().is_some(),
"an unused standing slot returns its capacity on drop"
);
}
#[test]
#[should_panic(expected = "must release the reservation under the `items` lock")]
fn a_hold_that_outlives_its_entry_while_the_queue_is_alive_trips_the_tripwire() {
let (sender, _receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let hold = super::StandingHold {
shared: Arc::downgrade(&sender.shared),
state: Arc::clone(&slot.state),
resolved: false,
};
drop(hold);
}
#[test]
fn a_second_standing_send_while_the_first_is_still_queued_coalesces_in_place() {
let (sender, receiver) = bounded(1);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Arm,
detail: test_detail(),
});
assert_eq!(receiver.len(), 1, "the two sends coalesced into one entry");
let Some(Notification::RetryQuestion { operation, .. }) = receiver.try_recv() else {
panic!("expected the coalesced RetryQuestion");
};
assert_eq!(
operation,
crate::retry::FaultOperation::Arm,
"the coalesced entry carries the later send's content"
);
assert!(
sender.reserve().is_none(),
"the slot's one carved-out unit is still its own, not double-spent"
);
}
#[test]
fn draining_a_standing_send_frees_no_extra_capacity_for_a_racing_producer() {
let (sender, receiver) = bounded(1);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
assert!(
sender.reserve().is_none(),
"before the send: the slot's carve-out is the only capacity"
);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert!(
sender.reserve().is_none(),
"while queued: the reservation lives in the queue slot instead"
);
assert!(receiver.try_recv().is_some(), "the message drains");
assert!(
sender.reserve().is_none(),
"after drain: the reservation is restored, not freed to the general pool"
);
}
#[test]
fn a_bound_of_one_still_reports_its_own_saturation() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(1);
deliver(&sender, batch(watch, &["a.txt"]));
assert_eq!(sender.send(batch(watch, &["b.txt"])), Delivery::Latched);
assert_eq!(sender.send(batch(watch, &["c.txt"])), Delivery::Latched);
assert_eq!(names(&next(&receiver, "a")), vec!["a.txt"]);
assert!(matches!(
next(&receiver, "the desync"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert!(receiver.try_recv().is_none());
}
#[test]
fn a_blocked_receiver_is_woken_by_a_latched_loss() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(1);
deliver(&sender, batch(watch, &["a.txt"]));
assert_eq!(names(&next(&receiver, "a")), vec!["a.txt"]);
let (delivered_tx, delivered_rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
deliver(&sender, batch(watch, &["b.txt"]));
delivered_tx.send(sender).expect("signal delivery");
});
let sender = delivered_rx
.recv()
.expect("the background thread delivered");
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(names(&next(&receiver, "b")), vec!["b.txt"]);
assert!(matches!(
next(&receiver, "the desync"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
handle.join().expect("sender thread");
}
fn is_signalled(handle: BorrowedHandle<'_>) -> bool {
unsafe { WaitForSingleObject(handle.as_raw_handle(), 0) == WAIT_OBJECT_0 }
}
fn await_signal(handle: BorrowedHandle<'_>) -> bool {
unsafe { WaitForSingleObject(handle.as_raw_handle(), 5_000) == WAIT_OBJECT_0 }
}
#[test]
fn a_receiver_that_never_asks_allocates_no_doorbell() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
let _ = next(&receiver, "a notification");
assert!(
receiver.shared.doorbell.get().is_none(),
"the event must not exist until it is asked for"
);
}
#[test]
fn the_doorbell_is_created_to_match_the_queue_it_reports_on() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
let doorbell = receiver.doorbell().expect("create the doorbell");
assert!(is_signalled(doorbell));
}
#[test]
fn an_idle_queue_has_an_unsignalled_doorbell() {
let (_sender, receiver) = channel();
let doorbell = receiver.doorbell().expect("create the doorbell");
assert!(!is_signalled(doorbell));
}
#[test]
fn sending_signals_the_doorbell_and_draining_resets_it() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell().expect("create the doorbell");
assert!(!is_signalled(doorbell));
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
assert!(is_signalled(doorbell));
let _ = receiver.try_recv().expect("a notification");
assert!(
!is_signalled(doorbell),
"an emptied queue must stop claiming there is something to take"
);
}
#[test]
fn a_partial_drain_leaves_the_doorbell_signalled() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell().expect("create the doorbell");
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
deliver(&sender, batch(WatchId::from_raw(1), &["b.txt"]));
let _ = receiver.try_recv().expect("the first");
assert!(is_signalled(doorbell));
let _ = receiver.try_recv().expect("the second");
assert!(!is_signalled(doorbell));
}
#[test]
fn an_owed_loss_signals_the_doorbell_even_with_an_empty_queue() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(1);
let doorbell = receiver.doorbell().expect("create the doorbell");
deliver(&sender, batch(watch, &["a.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
let _ = receiver.try_recv().expect("the queued one");
assert!(receiver.is_empty());
assert!(
is_signalled(doorbell),
"the queue is empty but a loss is still owed"
);
let _ = receiver.try_recv().expect("the latched one");
assert!(!is_signalled(doorbell));
}
#[test]
fn disconnection_signals_the_doorbell() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell().expect("create the doorbell");
assert!(!is_signalled(doorbell));
drop(sender);
assert!(is_signalled(doorbell));
assert_stream_ended(&receiver, "disconnection");
assert!(
is_signalled(doorbell),
"the end of the stream is permanent, so it stays signalled"
);
}
#[test]
fn the_doorbell_is_created_once_and_reused() {
let (_sender, receiver) = channel();
let first = receiver.doorbell().expect("create").as_raw_handle();
let second = receiver.doorbell().expect("reuse").as_raw_handle();
assert_eq!(first, second, "the doorbell is created once, not per call");
}
#[test]
fn an_owned_doorbell_refers_to_the_same_event() {
let (sender, receiver) = channel();
let owned = receiver.doorbell_owned().expect("duplicate the doorbell");
assert_ne!(
owned.as_raw_handle(),
receiver.doorbell().expect("borrow").as_raw_handle(),
"a duplicate is a distinct handle"
);
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
assert!(
is_signalled(owned.as_handle()),
"but it refers to the same event, so signalling reaches both"
);
drop(owned);
assert!(is_signalled(receiver.doorbell().expect("still there")));
}
#[test]
fn a_shared_doorbell_is_the_queues_own_handle_and_not_a_duplicate() {
let (_sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("the shared doorbell");
assert_eq!(
shared.as_raw_handle(),
receiver.doorbell().expect("borrow").as_raw_handle(),
"a shared holder is one more owner of the queue's handle, so no new handle exists"
);
let owned = receiver.doorbell_owned().expect("a duplicate");
assert_ne!(
shared.as_raw_handle(),
owned.as_raw_handle(),
"the owned form is a duplicate, which is what the shared one is not"
);
}
#[test]
fn every_shared_doorbell_is_a_holder_of_one_handle() {
let (_sender, receiver) = channel();
let first = receiver.doorbell_shared().expect("first");
let second = receiver.doorbell_shared().expect("second");
let cloned = first.clone();
assert!(first.same_handle(&second), "two asks share one handle");
assert!(first.same_handle(&cloned), "and so does a clone");
assert_eq!(first.as_raw_handle(), second.as_raw_handle());
}
#[test]
fn asking_for_the_shared_doorbell_first_creates_it_for_every_other_form() {
let (_sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("create");
assert_eq!(
receiver.doorbell().expect("reuse").as_raw_handle(),
shared.as_raw_handle(),
"the doorbell is created once, whichever method asks first"
);
}
#[test]
fn a_shared_doorbell_follows_the_queue_in_both_directions() {
let (sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("the shared doorbell");
assert!(!is_signalled(shared.as_handle()), "an empty queue is quiet");
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
assert!(is_signalled(shared.as_handle()), "a send reaches it");
assert!(receiver.try_recv().is_some());
assert!(
!is_signalled(shared.as_handle()),
"draining the queue resets it"
);
}
#[test]
fn a_shared_doorbell_asked_for_late_finds_the_queues_state() {
let (sender, receiver) = channel();
deliver(&sender, batch(WatchId::from_raw(1), &["early.txt"]));
let shared = receiver.doorbell_shared().expect("the shared doorbell");
assert!(
is_signalled(shared.as_handle()),
"notifications that arrived before the first ask are already signalled"
);
}
#[test]
fn dropping_a_shared_holder_leaves_the_queues_doorbell_working() {
let (sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("the shared doorbell");
let other = shared.clone();
drop(shared);
drop(other);
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
assert!(
is_signalled(receiver.doorbell().expect("still there")),
"the queue's own holder keeps the handle open"
);
}
#[test]
fn a_shared_doorbell_outlives_the_queue_as_a_valid_handle() {
let (sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("the shared doorbell");
deliver(&sender, batch(WatchId::from_raw(1), &["a.txt"]));
drop(sender);
drop(receiver);
let waited = unsafe { WaitForSingleObject(shared.as_raw_handle(), 0) };
assert_ne!(waited, WAIT_FAILED, "the handle is still a live event");
assert!(shared.try_into_owned().is_ok(), "it is the last holder now");
}
#[test]
fn a_waiting_client_on_a_shared_doorbell_is_woken_by_a_send() {
let (sender, receiver) = channel();
let shared = receiver.doorbell_shared().expect("the shared doorbell");
let handle = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
deliver(&sender, batch(WatchId::from_raw(1), &["late.txt"]));
});
let waited = unsafe { WaitForSingleObject(shared.as_raw_handle(), 10_000) };
assert_eq!(waited, WAIT_OBJECT_0, "the send woke the waiter");
handle.join().expect("the sender thread");
assert!(receiver.try_recv().is_some());
}
#[test]
fn a_waiting_client_is_woken_by_a_send() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell_owned().expect("the doorbell");
let handle = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
deliver(&sender, batch(WatchId::from_raw(1), &["late.txt"]));
});
assert!(
await_signal(doorbell.as_handle()),
"the doorbell must ring for a notification that arrives while waiting"
);
assert_eq!(names(&receiver.try_recv().expect("it")), vec!["late.txt"]);
handle.join().expect("sender thread");
}
#[test]
fn no_wakeup_is_lost_under_a_concurrent_burst() {
const TOTAL: usize = 2_000;
let (sender, receiver) = bounded(16);
let doorbell = receiver.doorbell_owned().expect("the doorbell");
let producer = std::thread::spawn(move || {
for index in 0..TOTAL {
let _ = sender.send(batch(WatchId::from_raw(1), &[&format!("f-{index}")]));
}
drop(sender);
});
let mut seen = 0_usize;
let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop {
assert!(
std::time::Instant::now() < deadline,
"the stream never ended: saw {seen} of {TOTAL}, disconnected={}, \
empty={}, latched={}",
receiver.is_disconnected(),
receiver.is_empty(),
receiver.latched()
);
assert!(
await_signal(doorbell.as_handle()),
"the doorbell stopped ringing after {seen} notifications"
);
while receiver.try_recv().is_some() {
seen += 1;
}
if receiver.is_disconnected() && receiver.is_empty() && receiver.latched() == 0 {
break;
}
}
producer.join().expect("producer");
assert!(seen > 0, "nothing was delivered at all");
}
#[test]
fn a_client_can_drain_from_its_own_threadpool_wait() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell_owned().expect("the doorbell");
let receiver = Arc::new(receiver);
let drained = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
let consumer = Arc::clone(&receiver);
let sink = Arc::clone(&drained);
let signal = Arc::clone(&finished);
let waitable = unsafe { WaitableHandle::assume_waitable(doorbell) };
let wait = ThreadpoolWait::new(
waitable,
move |activation| {
while let Some(item) = consumer.try_recv() {
let mut names = names(&item);
if !names.is_empty() {
sink.lock().expect("record").push(names.remove(0));
}
}
if consumer.is_disconnected() {
signal.store(true, std::sync::atomic::Ordering::SeqCst);
} else {
activation.rearm(None);
}
},
None,
)
.expect("create the wait");
wait.arm(None);
for index in 0..64 {
deliver(
&sender,
batch(WatchId::from_raw(1), &[&format!("f-{index}")]),
);
}
drop(sender);
let deadline = std::time::Instant::now() + Duration::from_secs(30);
while !finished.load(std::sync::atomic::Ordering::SeqCst) {
assert!(
std::time::Instant::now() < deadline,
"the client's own pool callback never drained the queue"
);
std::thread::sleep(Duration::from_millis(1));
}
wait.stop_and_drain();
let seen = drained.lock().expect("read").clone();
assert_eq!(seen.len(), 64, "every notification reached the client");
let expected: Vec<String> = (0..64).map(|index| format!("f-{index}")).collect();
assert_eq!(seen, expected);
}
#[test]
fn a_client_can_arm_its_own_threadpool_wait_on_the_shared_doorbell() {
let (sender, receiver) = channel();
let doorbell = receiver.doorbell_shared().expect("the doorbell");
let receiver = Arc::new(receiver);
let drained = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
let consumer = Arc::clone(&receiver);
let count = Arc::clone(&drained);
let signal = Arc::clone(&finished);
let waitable = unsafe { WaitableHandle::assume_waitable_shared(doorbell) };
let wait = ThreadpoolWait::new(
waitable,
move |activation| {
while consumer.try_recv().is_some() {
count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
if consumer.is_disconnected() {
signal.store(true, std::sync::atomic::Ordering::SeqCst);
} else {
activation.rearm(None);
}
},
None,
)
.expect("create the wait");
wait.arm(None);
for index in 0..32 {
deliver(
&sender,
batch(WatchId::from_raw(1), &[&format!("f-{index}")]),
);
}
drop(sender);
let deadline = std::time::Instant::now() + Duration::from_secs(30);
while !finished.load(std::sync::atomic::Ordering::SeqCst) {
assert!(
std::time::Instant::now() < deadline,
"the pool callback never drained the queue"
);
std::thread::sleep(Duration::from_millis(1));
}
wait.stop_and_drain();
assert_eq!(drained.load(std::sync::atomic::Ordering::SeqCst), 32);
}
#[test]
fn a_drained_queue_with_a_loss_owed_is_empty_but_still_has_something_to_take() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(77);
deliver(&sender, batch(watch, &["queued.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(
names(&next(&receiver, "the queued batch")),
vec!["queued.txt"]
);
assert!(receiver.is_empty(), "nothing is *queued*");
assert_eq!(receiver.len(), 0);
assert_eq!(receiver.latched(), 1, "but a loss is owed");
assert!(
receiver.has_pending(),
"so there is still something to take -- what `is_empty` cannot say"
);
assert!(matches!(
next(&receiver, "the synthesised loss report"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert!(!receiver.has_pending(), "now genuinely drained");
}
#[test]
fn has_pending_tracks_the_doorbell_exactly_through_a_latched_loss() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(78);
let doorbell = receiver.doorbell().expect("a doorbell");
assert_eq!(receiver.has_pending(), is_signalled(doorbell));
assert!(!receiver.has_pending());
deliver(&sender, batch(watch, &["queued.txt"]));
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(receiver.has_pending(), is_signalled(doorbell));
next(&receiver, "the queued batch");
assert_eq!(
receiver.has_pending(),
is_signalled(doorbell),
"still agreed while only a latched loss remains"
);
assert!(is_signalled(doorbell), "the doorbell is still ringing");
assert!(receiver.is_empty(), "yet the queue reports itself empty");
next(&receiver, "the loss report");
assert_eq!(receiver.has_pending(), is_signalled(doorbell));
assert!(!is_signalled(doorbell), "and only now does it stop");
}
#[test]
fn a_disconnected_empty_queue_still_has_something_to_take() {
let (sender, receiver) = channel();
assert!(!receiver.has_pending());
drop(sender);
assert!(receiver.is_empty());
assert_eq!(receiver.latched(), 0);
assert!(
receiver.has_pending(),
"disconnection is collectable: recv returns None rather than blocking"
);
assert_stream_ended(&receiver, "the stream should be finished");
}
#[derive(Default)]
struct CountingResumer {
prods: std::sync::atomic::AtomicUsize,
}
impl CountingResumer {
fn count(&self) -> usize {
self.prods.load(std::sync::atomic::Ordering::SeqCst)
}
}
impl super::Resume for CountingResumer {
fn resume(&self) {
self.prods.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[test]
fn a_parked_producer_is_prodded_at_the_slot_has_room_actually_becomes_true() {
let (sender, receiver) = bounded(4);
let watch = WatchId::from_raw(101);
let producer = Arc::new(CountingResumer::default());
sender.register_resume(&producer);
fill(&sender, watch, 4);
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(receiver.latched(), 1, "a loss is owed");
assert!(!sender.has_room(), "saturated");
assert_eq!(producer.count(), 0, "nothing freed yet");
next(&receiver, "first");
assert!(
!sender.has_room(),
"the freed slot is owed to the latch flush, not to a new notification"
);
assert_eq!(
producer.count(),
0,
"prodding while has_room is still false burns the one wake the \
producer was going to get"
);
next(&receiver, "second");
assert!(sender.has_room(), "room genuinely exists now");
assert_eq!(
producer.count(),
1,
"the producer must have been prodded at the transition, or it stays \
parked forever with room available"
);
}
#[test]
fn draining_only_latched_reports_still_reaches_the_resume_edge() {
let (sender, receiver) = bounded(2);
let producer = Arc::new(CountingResumer::default());
sender.register_resume(&producer);
fill(&sender, WatchId::from_raw(1), 2);
for id in [2, 3] {
assert_eq!(
sender.send(batch(WatchId::from_raw(id), &["lost.txt"])),
Delivery::Latched
);
}
assert_eq!(receiver.latched(), 2);
next(&receiver, "first queued");
next(&receiver, "second queued");
assert!(!sender.has_room());
assert_eq!(producer.count(), 0);
assert!(matches!(
next(&receiver, "a synthesised loss report"),
Notification::Desync {
cause: DesyncCause::QueueFull,
..
}
));
assert!(sender.has_room());
assert_eq!(
producer.count(),
1,
"the edge must be reachable by draining latched reports alone"
);
}
#[test]
fn a_take_that_took_nothing_is_not_a_crossing_and_does_not_prod() {
let (sender, receiver) = bounded(1);
let watch = WatchId::from_raw(202);
let producer = Arc::new(CountingResumer::default());
sender.register_resume(&producer);
assert!(receiver.try_recv().is_none(), "nothing to take");
assert_eq!(
producer.count(),
0,
"a poll that took nothing crossed no edge"
);
fill(&sender, watch, 1);
assert!(!sender.has_room(), "saturated");
assert!(receiver.try_recv().is_some(), "the queued item");
assert_eq!(producer.count(), 1, "taking the item crossed the edge");
assert!(receiver.try_recv().is_none(), "drained");
assert_eq!(
producer.count(),
1,
"an empty poll must not re-fire an edge it did not cross"
);
}
#[test]
fn draining_a_standing_send_returns_the_carve_out_to_the_slot_not_to_the_pool() {
let (sender, receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
let held = sender.reserve().expect("one unreserved unit exists");
assert!(
sender.reserve().is_none(),
"the standing slot's carve-out is not available to the best-effort path"
);
drop(held);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert!(receiver.try_recv().is_some(), "the standing send arrives");
let held = sender.reserve().expect("the one best-effort unit is back");
assert!(
sender.reserve().is_none(),
"the carve-out must return to the slot, not to the best-effort pool -- \
a second unit here means the reservation was released twice"
);
drop(held);
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert!(
receiver.try_recv().is_some(),
"the slot can send again, which is what the carve-out is for"
);
}
#[test]
fn a_standing_slot_keeps_its_carve_out_across_many_send_drain_cycles() {
let (sender, receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let watch = WatchId::from_raw(1);
for cycle in 0..8 {
slot.send(Notification::RetryQuestion {
watch,
operation: crate::retry::FaultOperation::Open,
detail: test_detail(),
});
assert!(
receiver.try_recv().is_some(),
"cycle {cycle}: the standing send arrives"
);
let held = sender.reserve().expect("one best-effort unit, every cycle");
assert!(
sender.reserve().is_none(),
"cycle {cycle}: the carve-out must still be carved out"
);
drop(held);
}
}
#[test]
fn the_opaque_handles_name_themselves_when_formatted() {
let (sender, _receiver) = bounded(2);
let slot = sender.reserve_standing().expect("a slot");
let reservation = sender.reserve().expect("a reservation");
let rendered = format!("{slot:?}");
assert!(
rendered.contains("StandingSlot"),
"a standing slot must name itself when formatted, got {rendered:?}"
);
let rendered = format!("{reservation:?}");
assert!(
rendered.contains("Reservation"),
"a reservation must name itself when formatted, got {rendered:?}"
);
let rendered = format!("{sender:?}");
assert!(
rendered.contains("Sender"),
"a sender must name itself when formatted, got {rendered:?}"
);
}
#[test]
fn a_formatted_receiver_reports_the_state_a_wedge_is_diagnosed_from() {
let (sender, receiver) = bounded(2);
let watch = WatchId::from_raw(303);
fill(&sender, watch, 2);
assert_eq!(sender.send(batch(watch, &["lost.txt"])), Delivery::Latched);
assert_eq!(
format!("{receiver:?}"),
"Receiver { queued: 2, capacity: 2, latched: 1, disconnected: false, .. }",
"a live receiver must report its occupancy and that it is still connected"
);
drop(sender);
assert_eq!(
format!("{receiver:?}"),
"Receiver { queued: 2, capacity: 2, latched: 1, disconnected: true, .. }",
"once every sender is gone the receiver must say so"
);
}