use std::{
collections::HashMap,
ffi::OsString,
num::NonZeroUsize,
path::Path,
sync::{Arc, Mutex},
time::Duration,
};
use agnostic_lite::tokio::TokioRuntime;
use tributary_proto::{Epoch, Location};
use super::{CONTROL_CAPACITY, Demux, Lane};
use crate::{
driver::Tributaries,
error::WatchError,
event::{Event, EventKind},
options::{TributariesOptions, WatchOptions},
source::{Armed, Source, SourceEvent},
subscription::Subscription,
};
fn key(path: &str) -> Vec<OsString> {
Path::new(path)
.components()
.map(|c| c.as_os_str().to_os_string())
.collect()
}
const DEADLINE: Duration = Duration::from_secs(10);
const STARVED: Duration = Duration::from_millis(300);
const QUIESCE: Duration = Duration::from_millis(500);
type Roots = Arc<Mutex<HashMap<Vec<OsString>, u32>>>;
struct StreamSource {
next_handle: u32,
live: HashMap<u32, Vec<OsString>>,
roots: Roots,
events: async_channel::Receiver<SourceEvent<OsString, u32>>,
}
impl Source<OsString> for StreamSource {
type Handle = u32;
fn canonicalize_key(&self, k: &[OsString]) -> Result<Vec<OsString>, WatchError> {
Ok(k.to_vec())
}
async fn arm(&mut self, key: &[OsString]) -> Result<Armed<OsString, u32>, WatchError> {
self.next_handle += 1;
let handle = self.next_handle;
self.live.insert(handle, key.to_vec());
self
.roots
.lock()
.expect("roots registry")
.insert(key.to_vec(), handle);
Ok(Armed::new(handle, key.to_vec()))
}
fn disarm(&mut self, handle: u32) {
if let Some(key) = self.live.remove(&handle) {
self.roots.lock().expect("roots registry").remove(&key);
}
}
async fn next(&mut self) -> Option<SourceEvent<OsString, u32>> {
self.events.recv().await.ok()
}
fn root_key(&self, handle: u32) -> Option<Vec<OsString>> {
self.live.get(&handle).cloned()
}
}
struct Feed {
tx: async_channel::Sender<SourceEvent<OsString, u32>>,
roots: Roots,
epochs: HashMap<u32, u64>,
}
impl Feed {
async fn modified(&mut self, root: &str, path: &str) {
let handle = *self
.roots
.lock()
.expect("roots registry")
.get(&key(root))
.expect("the root is armed");
let epoch = self.epochs.entry(handle).or_insert(0);
*epoch += 1;
let event = SourceEvent::new(
handle,
key(path),
EventKind::Modified,
Location::new(),
Epoch::new(*epoch),
None,
);
self.tx.send(event).await.expect("the source is live");
}
}
fn rig(capacity: usize) -> (Tributaries<OsString, (), TokioRuntime, u32>, Feed) {
let (tx, rx) = async_channel::unbounded();
let roots: Roots = Arc::default();
let source = StreamSource {
next_handle: 0,
live: HashMap::new(),
roots: roots.clone(),
events: rx,
};
let options = TributariesOptions::new()
.with_event_capacity(NonZeroUsize::new(capacity).expect("nonzero capacity"));
let watcher = Tributaries::with_source(source, options);
(
watcher,
Feed {
tx,
roots,
epochs: HashMap::new(),
},
)
}
async fn watch(w: &Tributaries<OsString, (), TokioRuntime, u32>, path: &str) -> Subscription {
w.watch(key(path), (), WatchOptions::new())
.await
.expect("watch")
}
async fn recv(lane: &Lane<OsString, ()>) -> Event<OsString, ()> {
tokio::time::timeout(DEADLINE, lane.recv())
.await
.expect("a lane delivery within the deadline")
.expect("the stream is still open")
}
#[tokio::test]
async fn events_route_to_their_own_lanes_in_per_sub_order() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_b = watch(&w, "/b").await;
let (demux, _rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 16).await;
let lane_b = demux.lane(sub_b, 16).await;
for i in 0..3 {
feed.modified("/a", &format!("/a/f{i}")).await;
feed.modified("/b", &format!("/b/f{i}")).await;
}
for i in 0..3 {
let a = recv(&lane_a).await;
assert_eq!(a.subscription(), sub_a, "lane A carries only sub A");
assert_eq!(
a.key(),
key(&format!("/a/f{i}")).as_slice(),
"lane A preserves per-sub delivery order"
);
let b = recv(&lane_b).await;
assert_eq!(b.subscription(), sub_b, "lane B carries only sub B");
assert_eq!(
b.key(),
key(&format!("/b/f{i}")).as_slice(),
"lane B preserves per-sub delivery order"
);
}
}
#[tokio::test]
async fn unregistered_subscription_lands_on_rest_until_registered() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_b = watch(&w, "/b").await;
let (demux, rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 16).await;
feed.modified("/b", "/b/one").await;
feed.modified("/a", "/a/one").await;
let unclaimed = recv(&rest).await;
assert_eq!(
unclaimed.subscription(),
sub_b,
"the unregistered sub's event lands on the rest lane"
);
assert_eq!(unclaimed.key(), key("/b/one").as_slice());
let claimed = recv(&lane_a).await;
assert_eq!(
claimed.subscription(),
sub_a,
"the registered sub's event bypasses rest"
);
let lane_b = demux.lane(sub_b, 16).await;
feed.modified("/b", "/b/two").await;
let routed = recv(&lane_b).await;
assert_eq!(
routed.key(),
key("/b/two").as_slice(),
"from registration onward the sub's events go to its lane"
);
}
#[tokio::test]
async fn stalled_lane_backs_up_shared_channel_and_actor_sheds_to_rescans() {
let (w, mut feed) = rig(1);
let sub_a = watch(&w, "/a").await;
let sub_b = watch(&w, "/b").await;
let (demux, _rest) = Demux::spawn(w.clone(), 4);
let lane_a = demux.lane(sub_a, 1).await;
let lane_b = demux.lane(sub_b, 4).await;
for i in 0..6 {
feed.modified("/a", &format!("/a/f{i}")).await;
}
tokio::time::sleep(QUIESCE).await;
for i in 0..3 {
feed.modified("/b", &format!("/b/g{i}")).await;
}
assert!(
tokio::time::timeout(STARVED, lane_b.recv()).await.is_err(),
"lane B starves while the demux is stalled on the full lane A (latency, not loss)"
);
let mut last_ordinary: Option<Epoch> = None;
let rescan_a = loop {
let event = recv(&lane_a).await;
assert_eq!(event.subscription(), sub_a, "lane A carries only sub A");
if event.is_rescan() {
break event;
}
if let Some(prior) = last_ordinary {
assert!(
event.epoch() > prior,
"lane A's ordinary deltas arrive in epoch order"
);
}
last_ordinary = Some(event.epoch());
};
assert_eq!(
rescan_a.key(),
key("/a").as_slice(),
"the shed Rescan names A's covered key to re-enumerate"
);
if let Some(max) = last_ordinary {
assert!(
rescan_a.epoch() > max,
"the actor-minted Rescan dominates every ordinary delivery before it"
);
}
let mut b_ordinary: Option<Epoch> = None;
let rescan_b = loop {
let event = recv(&lane_b).await;
assert_eq!(event.subscription(), sub_b, "lane B carries only sub B");
if event.is_rescan() {
break event;
}
b_ordinary = Some(event.epoch());
};
assert_eq!(
rescan_b.key(),
key("/b").as_slice(),
"B was shed by the ACTOR: a genuine dominating Rescan naming B's covered key"
);
if let Some(max) = b_ordinary {
assert!(
rescan_b.epoch() > max,
"B's Rescan dominates any delta delivered before it"
);
}
}
#[tokio::test]
async fn dropped_lane_releases_its_sub_to_rest_without_stalling_the_demux() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_b = watch(&w, "/b").await;
let (demux, rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 1).await;
let lane_b = demux.lane(sub_b, 16).await;
feed.modified("/a", "/a/before").await;
let before = recv(&lane_a).await;
assert_eq!(
before.subscription(),
sub_a,
"lane A was live before the drop"
);
drop(lane_a);
for i in 0..4 {
feed.modified("/a", &format!("/a/after{i}")).await;
}
feed.modified("/b", "/b/alive").await;
let alive = recv(&lane_b).await;
assert_eq!(
alive.subscription(),
sub_b,
"lane B keeps flowing past the dead lane (no stall on it)"
);
assert_eq!(alive.key(), key("/b/alive").as_slice());
drop(feed);
let mut rest_a = 0;
loop {
let drained = tokio::time::timeout(DEADLINE, rest.recv())
.await
.expect("rest drains within the deadline");
match drained {
Some(event) => {
if event.subscription() == sub_a {
rest_a += 1;
}
}
None => break,
}
}
assert_eq!(
rest_a, 4,
"all post-release /a traffic flows to rest — the sub is unclaimed again, nothing lost"
);
assert_eq!(
demux.tracked_lanes(),
1,
"the released sub's slot was reclaimed; only lane B remains tracked"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn registration_backpressures_while_the_demux_is_stalled() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, _rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 1).await;
feed.modified("/a", "/a/one").await;
feed.modified("/a", "/a/two").await;
let demux = std::sync::Arc::new(demux);
let registrar = {
let demux = std::sync::Arc::clone(&demux);
let w = w.clone();
tokio::spawn(async move {
for i in 0..=CONTROL_CAPACITY {
let sub = watch(&w, &format!("/reg{i}")).await;
demux.lane(sub, 1).await;
}
})
};
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
assert!(
!registrar.is_finished(),
"the registration past the control bound parks while the demux is stalled — \
registration cannot outgrow the bounded queue"
);
assert_eq!(recv(&lane_a).await.key(), key("/a/one").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/two").as_slice());
tokio::time::timeout(DEADLINE, registrar)
.await
.expect("the registrar unparks once the stall clears")
.expect("registrar task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn demux_parts_routes_when_caller_spawned() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, rest, driver) = Demux::parts(w.clone(), 16);
let routing = tokio::spawn(driver);
let lane_a = demux.lane(sub_a, 16).await;
feed.modified("/a", "/a/one").await;
assert_eq!(recv(&lane_a).await.subscription(), sub_a);
drop(feed);
assert!(
tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("lane settles")
.is_none(),
"lane ends after end-of-stream fan-in"
);
assert!(
tokio::time::timeout(DEADLINE, rest.recv())
.await
.expect("rest settles")
.is_none(),
"rest ends too"
);
tokio::time::timeout(DEADLINE, routing)
.await
.expect("the routing future completes at end-of-stream")
.expect("routing task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_stops_and_drains_losslessly_even_while_stalled() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, rest, driver) = Demux::parts(w.clone(), 16);
let routing = tokio::spawn(driver);
let lane_a = demux.lane(sub_a, 1).await;
feed.modified("/a", "/a/one").await;
assert_eq!(recv(&lane_a).await.key(), key("/a/one").as_slice());
feed.modified("/a", "/a/two").await;
feed.modified("/a", "/a/three").await;
feed.modified("/a", "/a/four").await;
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
demux.shutdown().await;
assert_eq!(recv(&lane_a).await.key(), key("/a/two").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/three").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/four").as_slice());
assert!(
tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("lane settles")
.is_none(),
"the router exited after completing its in-flight delivery"
);
assert!(
tokio::time::timeout(DEADLINE, rest.recv())
.await
.expect("rest settles")
.is_none(),
"rest ends with the router"
);
tokio::time::timeout(DEADLINE, routing)
.await
.expect("the routing future resolves after shutdown")
.expect("routing task");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn aborting_the_routing_future_loses_at_most_the_in_flight_delivery() {
let (mut w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, _rest, driver) = Demux::parts(w.clone(), 16);
let routing = tokio::spawn(driver);
let lane_a = demux.lane(sub_a, 1).await;
feed.modified("/a", "/a/one").await;
feed.modified("/a", "/a/two").await;
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
routing.abort();
let _ = routing.await;
assert_eq!(recv(&lane_a).await.key(), key("/a/one").as_slice());
assert!(
tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("lane settles")
.is_none(),
"after an abort the lane ends after its buffered tail (at most one in-flight loss)"
);
feed.modified("/a", "/a/after-abort").await;
let later = tokio::time::timeout(DEADLINE, w.next())
.await
.expect("next settles")
.expect("the stream is still live for surviving clones");
assert_eq!(later.key(), key("/a/after-abort").as_slice());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_barrier_is_snapshot_bounded_under_a_live_producer() {
let (mut w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, _rest, driver) = Demux::parts(w.clone(), 16);
let routing = tokio::spawn(driver);
let lane_a = demux.lane(sub_a, 1).await;
feed.modified("/a", "/a/e1").await;
assert_eq!(recv(&lane_a).await.key(), key("/a/e1").as_slice());
feed.modified("/a", "/a/e2").await;
feed.modified("/a", "/a/e3").await;
feed.modified("/a", "/a/e4").await;
feed.modified("/a", "/a/e5").await;
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
demux.shutdown().await;
assert_eq!(recv(&lane_a).await.key(), key("/a/e2").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/e3").as_slice());
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
feed.modified("/a", "/a/e6").await;
feed.modified("/a", "/a/e7").await;
assert_eq!(recv(&lane_a).await.key(), key("/a/e4").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/e5").as_slice());
assert!(
tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("lane settles")
.is_none(),
"the barrier stopped at the snapshot — no post-stop absorption"
);
tokio::time::timeout(DEADLINE, routing)
.await
.expect("the routing future resolves under a live producer")
.expect("routing task");
for expected in ["/a/e6", "/a/e7"] {
let event = tokio::time::timeout(DEADLINE, w.next())
.await
.expect("next settles")
.expect("stream live for survivors");
assert_eq!(event.key(), key(expected).as_slice());
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn lost_releases_on_a_full_control_queue_cannot_leak_the_table() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let (demux, _rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 1).await;
feed.modified("/a", "/a/one").await;
feed.modified("/a", "/a/two").await;
let mut doomed = Vec::new();
for i in 0..CONTROL_CAPACITY {
let sub = watch(&w, &format!("/doomed{i}")).await;
doomed.push(demux.lane(sub, 1).await);
}
drop(doomed);
assert_eq!(recv(&lane_a).await.key(), key("/a/one").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/two").as_slice());
let probe_sub = watch(&w, "/probe").await;
let probe = demux.lane(probe_sub, 4).await;
feed.modified("/probe", "/probe/touch").await;
assert_eq!(recv(&probe).await.subscription(), probe_sub);
assert_eq!(
demux.tracked_lanes(),
2,
"sixteen lost releases leaked nothing — the table is bounded by live lanes \
(register-time sweep + dead registrations never installed)"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn dropped_replacement_still_displaces_the_predecessor_lane() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_x = watch(&w, "/x").await;
let (demux, rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 1).await;
let lane_x_old = demux.lane(sub_x, 4).await;
feed.modified("/x", "/x/before").await;
assert_eq!(recv(&lane_x_old).await.subscription(), sub_x);
feed.modified("/a", "/a/one").await;
feed.modified("/a", "/a/two").await;
let lane_x_new = demux.lane(sub_x, 4).await;
drop(lane_x_new);
assert_eq!(recv(&lane_a).await.key(), key("/a/one").as_slice());
assert_eq!(recv(&lane_a).await.key(), key("/a/two").as_slice());
assert!(
tokio::time::timeout(DEADLINE, lane_x_old.recv())
.await
.expect("predecessor settles within the deadline")
.is_none(),
"the superseded lane ends — it must not keep receiving"
);
feed.modified("/x", "/x/after").await;
loop {
let event = tokio::time::timeout(DEADLINE, rest.recv())
.await
.expect("rest receives within the deadline")
.expect("stream still live");
if event.subscription() == sub_x {
assert_eq!(event.key(), key("/x/after").as_slice());
break;
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn lane_churn_leaves_no_tracked_residue() {
let (w, mut feed) = rig(1024);
let (demux, _rest) = Demux::spawn(w.clone(), 16);
for i in 0..32 {
let path = format!("/churn{i}");
let sub = watch(&w, &path).await;
let lane = demux.lane(sub, 4).await;
feed.modified(&path, &format!("{path}/touch")).await;
assert_eq!(recv(&lane).await.subscription(), sub);
w.unwatch(sub).await.expect("unwatch churn sub");
drop(lane);
}
let probe_sub = watch(&w, "/probe").await;
let probe = demux.lane(probe_sub, 4).await;
feed.modified("/probe", "/probe/touch").await;
assert_eq!(recv(&probe).await.subscription(), probe_sub);
assert_eq!(
demux.tracked_lanes(),
1,
"32 churn cycles left zero residue — only the live probe lane is tracked \
(bounded by concurrent lanes, not lifetime registrations)"
);
}
#[tokio::test]
async fn closed_watcher_ends_every_lane_after_draining_buffers() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_m = watch(&w, "/m").await;
let (demux, rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 16).await;
feed.modified("/a", "/a/one").await;
let one = recv(&lane_a).await;
assert_eq!(one.key(), key("/a/one").as_slice());
drop(demux);
feed.modified("/a", "/a/two").await;
let two = recv(&lane_a).await;
assert_eq!(
two.key(),
key("/a/two").as_slice(),
"routing survives the dropped Demux handle"
);
feed.modified("/a", "/a/three").await;
feed.modified("/m", "/m/marker").await;
let marker = recv(&rest).await;
assert_eq!(
marker.subscription(),
sub_m,
"the marker proves /a/three is buffered"
);
w.close().await.expect("close");
let three = tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("the buffered tail drains");
assert_eq!(
three
.expect("the buffered event precedes end-of-stream")
.key(),
key("/a/three").as_slice(),
"the lane drains its buffer before ending"
);
assert!(
tokio::time::timeout(DEADLINE, lane_a.recv())
.await
.expect("the lane ends")
.is_none(),
"a drained lane yields None once the watcher is closed"
);
assert!(
tokio::time::timeout(DEADLINE, rest.recv())
.await
.expect("rest ends")
.is_none(),
"the rest lane ends with the stream too"
);
}
#[tokio::test]
async fn unwatch_stragglers_arrive_on_the_kept_lane() {
let (w, mut feed) = rig(1024);
let sub_a = watch(&w, "/a").await;
let sub_m = watch(&w, "/m").await;
let (demux, rest) = Demux::spawn(w.clone(), 16);
let lane_a = demux.lane(sub_a, 16).await;
for i in 0..3 {
feed.modified("/a", &format!("/a/s{i}")).await;
}
feed.modified("/m", "/m/marker").await;
let marker = recv(&rest).await;
assert_eq!(
marker.subscription(),
sub_m,
"the stragglers are buffered in lane A"
);
w.unwatch(sub_a).await.expect("unwatch");
for i in 0..3 {
let straggler = recv(&lane_a).await;
assert_eq!(
straggler.subscription(),
sub_a,
"a straggler still carries the retired subscription"
);
assert_eq!(straggler.key(), key(&format!("/a/s{i}")).as_slice());
}
drop(lane_a);
feed.modified("/m", "/m/after").await;
let after = recv(&rest).await;
assert_eq!(
after.key(),
key("/m/after").as_slice(),
"the demux keeps routing after the retired sub's lane is dropped"
);
}