1use crate::{broadcast, cache, stats, track};
2use kio::Pollable;
3use std::{
4 cmp::Reverse,
5 collections::{BTreeMap, HashMap, HashSet},
6 fmt,
7 sync::Arc,
8 sync::atomic::{AtomicU64, Ordering},
9 task::{Poll, ready},
10 time::Duration,
11};
12
13use rand::RngExt;
14use web_async::Lock;
15
16use super::{Requests, WeakCache};
17use crate::{
18 AsPath, Error, Path, PathOwned, PathPrefixes,
19 coding::{BoundsExceeded, Decode, DecodeError, Encode, EncodeError},
20};
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
28pub struct Origin {
29 id: u64,
31}
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35#[non_exhaustive]
36pub struct InvalidOrigin;
37
38impl fmt::Display for InvalidOrigin {
39 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40 write!(f, "local origin id must be non-zero and below 2^62")
41 }
42}
43
44impl std::error::Error for InvalidOrigin {}
45
46impl Origin {
47 pub(crate) const UNKNOWN: Self = Self { id: 0 };
50
51 pub fn new(id: u64) -> Result<Self, InvalidOrigin> {
57 if id == 0 || id >= 1u64 << 62 {
58 return Err(InvalidOrigin);
59 }
60 Ok(Self { id })
61 }
62
63 pub fn random() -> Self {
72 let mut rng = rand::rng();
73 let id = rng.random_range(1..(1u64 << 53));
74 Self { id }
75 }
76
77 pub fn id(self) -> u64 {
79 self.id
80 }
81
82 pub fn produce(self) -> Producer {
85 Info::new(self).produce()
86 }
87}
88
89#[derive(Clone, Debug)]
99#[non_exhaustive]
100pub struct Info {
101 pub id: Origin,
104
105 pub pool: cache::Pool,
111
112 pub cache_duration: Duration,
119
120 pub latency_default: Duration,
130
131 pub linger: Duration,
140}
141
142impl Default for Info {
143 fn default() -> Self {
146 Self {
147 id: Origin::UNKNOWN,
148 pool: cache::Pool::default(),
149 cache_duration: Duration::MAX,
150 latency_default: track::DEFAULT_LATENCY_MAX,
151 linger: Duration::ZERO,
152 }
153 }
154}
155
156impl Info {
157 pub fn new(id: Origin) -> Self {
159 Self { id, ..Self::default() }
160 }
161
162 pub fn with_pool(mut self, pool: cache::Pool) -> Self {
164 self.pool = pool;
165 self
166 }
167
168 pub fn with_cache_duration(mut self, cache_duration: Duration) -> Self {
171 self.cache_duration = cache_duration;
172 self
173 }
174
175 pub fn with_latency_default(mut self, latency_default: Duration) -> Self {
178 self.latency_default = latency_default;
179 self
180 }
181
182 pub fn with_linger(mut self, linger: Duration) -> Self {
185 self.linger = linger;
186 self
187 }
188
189 pub fn produce(self) -> Producer {
191 Producer::new(self)
192 }
193}
194
195impl TryFrom<u64> for Origin {
196 type Error = InvalidOrigin;
197
198 fn try_from(id: u64) -> Result<Self, Self::Error> {
199 Self::new(id)
200 }
201}
202
203impl fmt::Display for Origin {
204 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
205 self.id.fmt(f)
206 }
207}
208
209impl<V: Copy> Encode<V> for Origin
210where
211 u64: Encode<V>,
212{
213 fn encode<W: bytes::BufMut>(&self, w: &mut W, version: V) -> Result<(), EncodeError> {
214 self.id.encode(w, version)
215 }
216}
217
218impl<V: Copy> Decode<V> for Origin
219where
220 u64: Decode<V>,
221{
222 fn decode<R: bytes::Buf>(r: &mut R, version: V) -> Result<Self, DecodeError> {
223 let id = u64::decode(r, version)?;
224 if id >= 1u64 << 62 {
225 return Err(DecodeError::InvalidValue);
226 }
227 Ok(Self { id })
228 }
229}
230
231pub(crate) const MAX_HOPS: usize = 32;
237
238#[derive(Debug, Clone, Default, PartialEq, Eq)]
243pub struct OriginList(Vec<Origin>);
244
245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247#[non_exhaustive]
248pub struct TooManyOrigins;
249
250impl fmt::Display for TooManyOrigins {
251 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
252 write!(f, "too many origins (max {MAX_HOPS})")
253 }
254}
255
256impl std::error::Error for TooManyOrigins {}
257
258impl From<TooManyOrigins> for DecodeError {
259 fn from(_: TooManyOrigins) -> Self {
260 DecodeError::BoundsExceeded
261 }
262}
263
264impl OriginList {
265 pub fn new() -> Self {
267 Self(Vec::new())
268 }
269
270 pub fn push(&mut self, origin: Origin) -> Result<(), TooManyOrigins> {
272 if self.0.len() >= MAX_HOPS {
273 return Err(TooManyOrigins);
274 }
275 self.0.push(origin);
276 Ok(())
277 }
278
279 pub fn replace_first(&mut self, target: Origin, replacement: Origin) -> bool {
282 for entry in &mut self.0 {
283 if *entry == target {
284 *entry = replacement;
285 return true;
286 }
287 }
288 false
289 }
290
291 pub fn contains(&self, origin: &Origin) -> bool {
293 self.0.contains(origin)
294 }
295
296 pub fn len(&self) -> usize {
298 self.0.len()
299 }
300
301 pub fn is_empty(&self) -> bool {
303 self.0.is_empty()
304 }
305
306 pub fn iter(&self) -> std::slice::Iter<'_, Origin> {
308 self.0.iter()
309 }
310
311 pub fn as_slice(&self) -> &[Origin] {
313 &self.0
314 }
315}
316
317impl TryFrom<Vec<Origin>> for OriginList {
318 type Error = TooManyOrigins;
319
320 fn try_from(v: Vec<Origin>) -> Result<Self, Self::Error> {
321 if v.len() > MAX_HOPS {
322 return Err(TooManyOrigins);
323 }
324 Ok(Self(v))
325 }
326}
327
328impl<'a> IntoIterator for &'a OriginList {
329 type Item = &'a Origin;
330 type IntoIter = std::slice::Iter<'a, Origin>;
331
332 fn into_iter(self) -> Self::IntoIter {
333 self.iter()
334 }
335}
336
337impl<V: Copy> Encode<V> for OriginList
338where
339 u64: Encode<V>,
340 Origin: Encode<V>,
341{
342 fn encode<W: bytes::BufMut>(&self, w: &mut W, version: V) -> Result<(), EncodeError> {
343 (self.0.len() as u64).encode(w, version)?;
344 for origin in &self.0 {
345 origin.encode(w, version)?;
346 }
347 Ok(())
348 }
349}
350
351impl<V: Copy> Decode<V> for OriginList
352where
353 u64: Decode<V>,
354 Origin: Decode<V>,
355{
356 fn decode<R: bytes::Buf>(r: &mut R, version: V) -> Result<Self, DecodeError> {
357 let count = u64::decode(r, version)? as usize;
358 if count > MAX_HOPS {
359 return Err(DecodeError::BoundsExceeded);
360 }
361 let mut list = Vec::with_capacity(count);
362 for _ in 0..count {
363 list.push(Origin::decode(r, version)?);
364 }
365 Ok(Self(list))
366 }
367}
368
369static NEXT_CONSUMER_ID: AtomicU64 = AtomicU64::new(0);
370
371#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
372struct ConsumerId(u64);
373
374impl ConsumerId {
375 fn new() -> Self {
376 Self(NEXT_CONSUMER_ID.fetch_add(1, Ordering::Relaxed))
377 }
378}
379
380struct OriginBroadcast {
386 path: PathOwned,
387 broadcast: broadcast::Producer,
389 state: kio::Producer<FrontState>,
392 announced: bool,
393}
394
395fn route_key(name: &Path, hops: &OriginList) -> (usize, u64) {
404 (hops.len(), fnv_key(name, hops.iter().copied()))
405}
406
407fn fnv_key(name: &Path, origins: impl IntoIterator<Item = Origin>) -> u64 {
420 const SEED: u64 = 0x420C0DECB00B; const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
422
423 let mut hash = SEED;
424 for &byte in name.as_str().as_bytes() {
425 hash = (hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME);
426 }
427 for origin in origins {
428 for &byte in &origin.id().to_le_bytes() {
429 hash = (hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME);
430 }
431 }
432
433 hash
434}
435
436fn route_order(name: &Path, route: &FrontRoute) -> (bool, u64, usize, u64, Reverse<u64>) {
455 let (len, hash) = route_key(name, &route.route.hops);
456 (!route.route.announce, route.route.cost, len, hash, Reverse(route.id))
457}
458
459enum PendingUpdate {
468 Announce(broadcast::Consumer),
469 Unannounce,
470 UnannounceAnnounce(broadcast::Consumer),
471}
472
473#[derive(Default)]
478struct OriginConsumerState {
479 pending: BTreeMap<PathOwned, PendingUpdate>,
480}
481
482impl OriginConsumerState {
483 fn apply_announce(&mut self, path: PathOwned, broadcast: broadcast::Consumer) {
484 let new = match self.pending.remove(&path) {
485 None | Some(PendingUpdate::Announce(_)) => PendingUpdate::Announce(broadcast),
487 Some(PendingUpdate::Unannounce | PendingUpdate::UnannounceAnnounce(_)) => {
489 PendingUpdate::UnannounceAnnounce(broadcast)
490 }
491 };
492 self.pending.insert(path, new);
493 }
494
495 fn apply_unannounce(&mut self, path: PathOwned) {
496 match self.pending.remove(&path) {
497 Some(PendingUpdate::Announce(_)) => {}
499 None | Some(PendingUpdate::Unannounce) => {
500 self.pending.insert(path, PendingUpdate::Unannounce);
501 }
502 Some(PendingUpdate::UnannounceAnnounce(_)) => {
505 self.pending.insert(path, PendingUpdate::Unannounce);
506 }
507 }
508 }
509
510 fn take(&mut self) -> Option<OriginAnnounce> {
512 let path = self.pending.keys().next()?.clone();
513 let broadcast = match self.pending.remove(&path).unwrap() {
514 PendingUpdate::Announce(broadcast) => Some(broadcast),
515 PendingUpdate::Unannounce => None,
516 PendingUpdate::UnannounceAnnounce(broadcast) => {
517 self.pending.insert(path.clone(), PendingUpdate::Announce(broadcast));
520 None
521 }
522 };
523 Some(OriginAnnounce { path, broadcast })
524 }
525}
526
527#[derive(Clone)]
528struct AnnounceConsumerNotify {
529 root: PathOwned,
530 state: kio::Producer<OriginConsumerState>,
531 exclude: Option<Origin>,
536}
537
538impl AnnounceConsumerNotify {
539 fn announce(&self, path: impl AsPath, broadcast: broadcast::Consumer, front: &kio::Producer<FrontState>) {
540 let path = path.as_path().strip_prefix(&self.root).unwrap().to_owned();
541
542 let broadcast = match self.exclude.and_then(|peer| ExclusionGuard::new(front, peer)) {
547 Some(guard) => broadcast.with_exclusion(guard),
548 None => broadcast,
549 };
550
551 self.state
552 .write()
553 .ok()
554 .expect("consumer closed")
555 .apply_announce(path, broadcast);
556 }
557
558 fn unannounce(&self, path: impl AsPath) {
559 let path = path.as_path().strip_prefix(&self.root).unwrap().to_owned();
560 self.state.write().ok().expect("consumer closed").apply_unannounce(path);
561 }
562}
563
564struct NotifyNode {
565 parent: Option<Lock<NotifyNode>>,
566
567 consumers: HashMap<ConsumerId, AnnounceConsumerNotify>,
570}
571
572impl NotifyNode {
573 fn new(parent: Option<Lock<NotifyNode>>) -> Self {
574 Self {
575 parent,
576 consumers: HashMap::new(),
577 }
578 }
579
580 fn announce(&mut self, path: impl AsPath, broadcast: &broadcast::Consumer, state: &kio::Producer<FrontState>) {
584 for consumer in self.consumers.values() {
585 consumer.announce(path.as_path(), broadcast.clone(), state);
586 }
587
588 if let Some(parent) = &self.parent {
589 parent.lock().announce(path, broadcast, state);
590 }
591 }
592
593 fn unannounce(&mut self, path: impl AsPath) {
594 for consumer in self.consumers.values() {
595 consumer.unannounce(path.as_path());
596 }
597
598 if let Some(parent) = &self.parent {
599 parent.lock().unannounce(path);
600 }
601 }
602}
603
604pub(crate) struct ExclusionGuard {
611 state: kio::Producer<FrontState>,
612 peer: Origin,
613}
614
615impl ExclusionGuard {
616 fn new(state: &kio::Producer<FrontState>, peer: Origin) -> Option<Arc<Self>> {
619 let mut s = state.write().ok()?;
620 if s.closed {
621 return None;
622 }
623 *s.excluded.entry(peer).or_default() += 1;
624 drop(s);
625 Some(Arc::new(Self {
626 state: state.clone(),
627 peer,
628 }))
629 }
630}
631
632impl Drop for ExclusionGuard {
633 fn drop(&mut self) {
634 let Ok(mut state) = self.state.write() else { return };
635 if let std::collections::hash_map::Entry::Occupied(mut entry) = state.excluded.entry(self.peer) {
636 match entry.get() {
637 1 => drop(entry.remove()),
638 n => *entry.get_mut() = n - 1,
639 }
640 }
641 }
642}
643
644enum Resolved {
651 Found(broadcast::Consumer),
654 Excluded,
656 OutOfScope,
660 Missing,
662}
663
664struct OriginNode {
665 broadcast: Option<OriginBroadcast>,
668
669 sources: usize,
674
675 nested: HashMap<String, Lock<OriginNode>>,
677
678 notify: Lock<NotifyNode>,
680}
681
682impl OriginNode {
683 fn new(parent: Option<Lock<NotifyNode>>) -> Self {
684 Self {
685 broadcast: None,
686 sources: 0,
687 nested: HashMap::new(),
688 notify: Lock::new(NotifyNode::new(parent)),
689 }
690 }
691
692 fn leaf(&mut self, path: &Path) -> Lock<OriginNode> {
693 let (dir, rest) = path.next_part().expect("leaf called with empty path");
694
695 let next = self.entry(dir);
696 if rest.is_empty() { next } else { next.lock().leaf(&rest) }
697 }
698
699 fn entry(&mut self, dir: &str) -> Lock<OriginNode> {
700 match self.nested.get(dir) {
701 Some(next) => next.clone(),
702 None => {
703 let next = Lock::new(OriginNode::new(Some(self.notify.clone())));
704 self.nested.insert(dir.to_string(), next.clone());
705 next
706 }
707 }
708 }
709
710 fn set_announced(&mut self, expect: &kio::Producer<FrontState>, announce: bool) {
714 let Some(existing) = &mut self.broadcast else { return };
715 if !existing.state.same_channel(expect) || existing.announced == announce {
716 return;
717 }
718 existing.announced = announce;
719 let path = existing.path.clone();
720 let consumer = existing.broadcast.consume();
721 let state = existing.state.clone();
722 let mut notify = self.notify.lock();
723 if announce {
724 notify.announce(&path, &consumer, &state);
725 } else {
726 notify.unannounce(&path);
727 }
728 }
729
730 fn consume_at(&mut self, id: ConsumerId, notify: AnnounceConsumerNotify, relative: impl AsPath) {
737 let relative = relative.as_path();
738
739 let Some((dir, relative)) = relative.next_part() else {
740 return self.consume(id, notify);
741 };
742
743 let nested = self.entry(dir);
744 nested.lock().consume_at(id, notify, &relative);
745 }
746
747 fn consume(&mut self, id: ConsumerId, mut notify: AnnounceConsumerNotify) {
748 self.consume_initial(&mut notify);
749 self.notify.lock().consumers.insert(id, notify);
750 }
751
752 fn consume_initial(&mut self, notify: &mut AnnounceConsumerNotify) {
753 if let Some(broadcast) = &self.broadcast
756 && broadcast.announced
757 {
758 notify.announce(&broadcast.path, broadcast.broadcast.consume(), &broadcast.state);
759 }
760
761 for nested in self.nested.values() {
763 nested.lock().consume_initial(notify);
764 }
765 }
766
767 fn resolve_broadcast(&self, rest: impl AsPath, exclude: Option<Origin>) -> Resolved {
768 let rest = rest.as_path();
769
770 if let Some((dir, rest)) = rest.next_part() {
771 let Some(node) = self.nested.get(dir) else {
772 return Resolved::Missing;
773 };
774 let node = node.lock();
775 return node.resolve_broadcast(&rest, exclude);
776 }
777
778 let Some(broadcast) = self.broadcast.as_ref() else {
779 return Resolved::Missing;
780 };
781 let Some(origin) = exclude else {
782 return Resolved::Found(broadcast.broadcast.consume());
783 };
784
785 let state = broadcast.state.read();
801 if !state.routes.iter().any(|r| r.route.hops.contains(&origin)) {
802 drop(state);
803 let shared = broadcast.broadcast.consume();
804 return match ExclusionGuard::new(&broadcast.state, origin) {
805 Some(guard) => Resolved::Found(shared.with_exclusion(guard)),
806 None => Resolved::Found(shared),
809 };
810 }
811 match state
812 .dispatch(Some(origin))
813 .and_then(|clean| state.routes.iter().find(|r| r.id == clean))
814 {
815 Some(route) => Resolved::Found(route.source.clone()),
816 None => Resolved::Excluded,
817 }
818 }
819
820 fn detach(&mut self, claim: Claim, relative: impl AsPath) {
828 let relative = relative.as_path();
829
830 let Some((dir, relative)) = relative.next_part() else {
831 match claim {
832 Claim::Consumer(id) => {
833 self.notify.lock().consumers.remove(&id).expect("consumer not found");
834 }
835 Claim::Source => self.sources -= 1,
836 }
837 return;
838 };
839
840 let nested = self.nested.get(dir).expect("claimed node missing").clone();
843 let mut locked = nested.lock();
844 locked.detach(claim, &relative);
845
846 if locked.is_empty() {
847 drop(locked);
848 self.nested.remove(dir);
849 }
850 }
851
852 fn reserve(&mut self, relative: impl AsPath) {
855 let relative = relative.as_path();
856
857 let Some((dir, relative)) = relative.next_part() else {
858 self.sources += 1;
859 return;
860 };
861
862 let nested = self.entry(dir);
863 nested.lock().reserve(&relative);
864 }
865
866 fn remove(&mut self, expect: &kio::Producer<FrontState>, relative: impl AsPath) {
870 let relative = relative.as_path();
871
872 if let Some((dir, relative)) = relative.next_part() {
873 let Some(nested) = self.nested.get(dir) else { return };
874 let nested = nested.clone();
875 let mut locked = nested.lock();
876 locked.remove(expect, &relative);
877
878 if locked.is_empty() {
879 drop(locked);
880 self.nested.remove(dir);
881 }
882 } else if let Some(existing) = &self.broadcast
883 && existing.state.same_channel(expect)
884 {
885 let existing = self.broadcast.take().expect("checked above");
886 if existing.announced {
887 self.notify.lock().unannounce(&existing.path);
888 }
889 }
890 }
891
892 fn is_empty(&self) -> bool {
893 self.broadcast.is_none()
894 && self.sources == 0
895 && self.nested.is_empty()
896 && self.notify.lock().consumers.is_empty()
897 }
898
899 #[cfg(test)]
902 fn count(&self) -> usize {
903 1 + self.nested.values().map(|nested| nested.lock().count()).sum::<usize>()
904 }
905}
906
907#[derive(Clone, Copy)]
910enum Claim {
911 Consumer(ConsumerId),
913 Source,
915}
916
917struct SourceReservation {
929 tree: Lock<OriginNode>,
930 path: PathOwned,
931}
932
933impl SourceReservation {
934 fn new(tree: Lock<OriginNode>, path: PathOwned) -> Self {
935 tree.lock().reserve(&path);
936 Self { tree, path }
937 }
938}
939
940impl Drop for SourceReservation {
941 fn drop(&mut self) {
942 self.tree.lock().detach(Claim::Source, &self.path);
943 }
944}
945
946#[derive(Clone)]
954struct OriginNodes {
955 tree: Lock<OriginNode>,
958
959 nodes: Vec<(PathOwned, PathOwned)>,
962}
963
964impl OriginNodes {
965 fn empty() -> Self {
968 Self {
969 tree: Lock::new(OriginNode::new(None)),
970 nodes: Vec::new(),
971 }
972 }
973
974 pub fn select(&self, prefixes: &PathPrefixes) -> Option<Self> {
977 let mut roots = Vec::new();
978
979 for (root, absolute) in &self.nodes {
980 for prefix in prefixes {
981 if root.has_prefix(prefix) {
982 roots.push((root.to_owned(), absolute.clone()));
984 continue;
985 }
986
987 if let Some(suffix) = prefix.strip_prefix(root) {
988 roots.push((prefix.to_owned(), absolute.join(&suffix)));
990 }
991 }
992 }
993
994 if roots.is_empty() {
995 None
996 } else {
997 Some(self.with_nodes(roots))
998 }
999 }
1000
1001 pub fn root(&self, new_root: impl AsPath) -> Option<Self> {
1002 let new_root = new_root.as_path();
1003 let mut roots = Vec::new();
1004
1005 if new_root.is_empty() {
1006 return Some(self.clone());
1007 }
1008
1009 for (root, absolute) in &self.nodes {
1010 if let Some(suffix) = root.strip_prefix(&new_root) {
1011 roots.push((suffix.to_owned(), absolute.clone()));
1013 } else if let Some(suffix) = new_root.strip_prefix(root) {
1014 roots.push(("".into(), absolute.join(&suffix)));
1017 }
1018 }
1019
1020 if roots.is_empty() {
1021 None
1022 } else {
1023 Some(self.with_nodes(roots))
1024 }
1025 }
1026
1027 fn with_nodes(&self, nodes: Vec<(PathOwned, PathOwned)>) -> Self {
1028 Self {
1029 tree: self.tree.clone(),
1030 nodes,
1031 }
1032 }
1033
1034 pub fn get(&self, path: impl AsPath) -> Option<PathOwned> {
1036 let path = path.as_path();
1037
1038 for (root, absolute) in &self.nodes {
1039 if let Some(suffix) = path.strip_prefix(root) {
1040 return Some(absolute.join(&suffix));
1041 }
1042 }
1043
1044 None
1045 }
1046}
1047
1048impl Default for OriginNodes {
1049 fn default() -> Self {
1050 Self {
1051 tree: Lock::new(OriginNode::new(None)),
1052 nodes: vec![("".into(), "".into())],
1053 }
1054 }
1055}
1056
1057#[derive(Clone)]
1059pub struct OriginAnnounce {
1060 pub path: PathOwned,
1062 pub broadcast: Option<broadcast::Consumer>,
1068}
1069
1070#[derive(Clone)]
1072pub struct Producer {
1073 info: Origin,
1077
1078 nodes: OriginNodes,
1081
1082 root: PathOwned,
1084
1085 dynamic: kio::Shared<OriginDynamicState>,
1089
1090 pool: cache::Pool,
1093
1094 cache_duration: Duration,
1097
1098 latency_default: Duration,
1101
1102 linger: Duration,
1105
1106 stats: stats::Session,
1110}
1111
1112impl std::ops::Deref for Producer {
1113 type Target = Origin;
1114
1115 fn deref(&self) -> &Self::Target {
1116 &self.info
1117 }
1118}
1119
1120impl Producer {
1121 pub fn new(info: Info) -> Self {
1125 Self {
1126 info: info.id,
1127 nodes: OriginNodes::default(),
1128 root: PathOwned::default(),
1129 dynamic: kio::Shared::default(),
1130 pool: info.pool,
1131 cache_duration: info.cache_duration,
1132 latency_default: info.latency_default,
1133 linger: info.linger,
1134 stats: stats::Session::default(),
1135 }
1136 }
1137
1138 pub fn with_stats(mut self, session: stats::Session) -> Self {
1142 self.stats = session;
1143 self
1144 }
1145
1146 pub fn with_linger(mut self, linger: Duration) -> Self {
1155 self.linger = linger;
1156 self
1157 }
1158
1159 pub fn info(&self) -> Info {
1162 Info {
1163 id: self.info,
1164 pool: self.pool.clone(),
1165 cache_duration: self.cache_duration,
1166 latency_default: self.latency_default,
1167 linger: self.linger,
1168 }
1169 }
1170
1171 pub(crate) fn latency_default(&self) -> Duration {
1174 self.latency_default
1175 }
1176
1177 pub(crate) fn empty(info: Origin) -> Self {
1182 Self {
1183 info,
1184 nodes: OriginNodes::empty(),
1185 root: PathOwned::default(),
1186 dynamic: kio::Shared::default(),
1187 pool: cache::Pool::default(),
1188 cache_duration: Duration::MAX,
1189 latency_default: track::DEFAULT_LATENCY_MAX,
1190 linger: Duration::ZERO,
1191 stats: stats::Session::default(),
1192 }
1193 }
1194
1195 pub fn create_broadcast(&self, path: impl AsPath, route: broadcast::Route) -> Result<broadcast::Producer, Error> {
1246 let path = path.as_path();
1247
1248 debug_assert!(
1249 !route.hops.contains(&self.info),
1250 "create_broadcast called with a looping hop chain",
1251 );
1252
1253 let full = self.nodes.get(&path).ok_or(Error::Unauthorized)?;
1258 let tree = self.nodes.tree.clone();
1259
1260 if full.parts().count() > Path::MAX_PARTS {
1264 return Err(BoundsExceeded.into());
1265 }
1266
1267 let ingress = self.stats.ingress(&full);
1271
1272 let mut source = broadcast::Info { origin: self.info() }
1273 .produce()
1274 .with_stats(ingress.clone());
1275 source.set_route(route).expect("fresh producer");
1276
1277 let reservation = SourceReservation::new(tree.clone(), full.clone());
1280 web_async::spawn(run_source(
1281 self.info(),
1282 tree,
1283 full,
1284 source.consume(),
1285 ingress,
1286 reservation,
1287 ));
1288
1289 Ok(source)
1290 }
1291
1292 pub fn scope(&self, prefixes: &[Path]) -> Option<Producer> {
1298 let prefixes = PathPrefixes::new(prefixes);
1299 Some(Producer {
1300 info: self.info,
1301 nodes: self.nodes.select(&prefixes)?,
1302 root: self.root.clone(),
1303 dynamic: self.dynamic.clone(),
1304 pool: self.pool.clone(),
1305 cache_duration: self.cache_duration,
1306 latency_default: self.latency_default,
1307 linger: self.linger,
1308 stats: self.stats.clone(),
1309 })
1310 }
1311
1312 pub fn dynamic(&self) -> Dynamic {
1321 Dynamic::new(self.info, self.root.clone(), self.dynamic.clone())
1322 }
1323
1324 pub fn consume(&self) -> Consumer {
1329 Consumer::new(
1332 self.info,
1333 self.root.clone(),
1334 self.nodes.clone(),
1335 self.dynamic.clone(),
1336 stats::Session::default(),
1337 )
1338 }
1339
1340 pub fn announces(&self) -> AnnounceProducer {
1346 AnnounceProducer::new(self.root.clone(), self.nodes.clone())
1347 }
1348
1349 pub fn with_root(&self, prefix: impl AsPath) -> Option<Self> {
1354 let prefix = prefix.as_path();
1355
1356 Some(Self {
1357 info: self.info,
1358 root: self.root.join(&prefix).to_owned(),
1359 nodes: self.nodes.root(&prefix)?,
1360 dynamic: self.dynamic.clone(),
1361 pool: self.pool.clone(),
1362 cache_duration: self.cache_duration,
1363 latency_default: self.latency_default,
1364 linger: self.linger,
1365 stats: self.stats.clone(),
1366 })
1367 }
1368
1369 pub fn root(&self) -> &Path<'_> {
1371 &self.root
1372 }
1373
1374 pub fn allowed(&self) -> impl Iterator<Item = &Path<'_>> {
1377 self.nodes.nodes.iter().map(|(root, _)| root)
1378 }
1379
1380 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
1382 self.root.join(path)
1383 }
1384
1385 #[cfg(test)]
1387 pub(crate) fn node_count(&self) -> usize {
1388 self.nodes.tree.lock().count()
1389 }
1390}
1391
1392const TRACK_IDLE_LINGER: Duration = Duration::from_secs(30);
1405
1406struct FrontRoute {
1408 id: u64,
1409 route: broadcast::Route,
1412 source: broadcast::Consumer,
1414}
1415
1416struct FrontState {
1418 path: PathOwned,
1420 self_origin: Origin,
1422 publisher: Option<Origin>,
1430 next_route: u64,
1433 routes: Vec<FrontRoute>,
1434 track_info: HashMap<Arc<str>, track::Info>,
1437 excluded: HashMap<Origin, usize>,
1450 active: Option<u64>,
1452 linger: Duration,
1455 closed: bool,
1460}
1461
1462impl FrontState {
1463 fn accept_track_info(&mut self, name: &Arc<str>, info: track::Info) -> Result<(), Error> {
1465 if self.closed {
1466 return Err(Error::Closed);
1467 }
1468 if let Some(expected) = self.track_info.get(name) {
1469 let track::Info {
1470 timescale,
1471 latency_max,
1472 priority,
1473 ordered,
1474 } = info;
1475 if timescale != expected.timescale
1476 || latency_max != expected.latency_max
1477 || priority != expected.priority
1478 || ordered != expected.ordered
1479 {
1480 return Err(Error::Unsupported);
1481 }
1482 } else {
1483 self.track_info.insert(name.clone(), info);
1484 }
1485 Ok(())
1486 }
1487
1488 fn pick(&self, keep: impl Fn(&FrontRoute) -> bool, untainted: bool) -> Option<u64> {
1495 let candidates: Vec<&FrontRoute> = self.routes.iter().filter(|r| keep(r)).collect();
1496 let candidates = match untainted {
1497 true => self.prefer_untainted(&candidates),
1498 false => candidates,
1499 };
1500 candidates
1501 .into_iter()
1502 .min_by_key(|r| route_order(&self.path.as_path(), r))
1503 .map(|r| r.id)
1504 }
1505
1506 fn best_route(&self) -> Option<u64> {
1511 self.pick(|_| true, true)
1512 }
1513
1514 fn dispatch(&self, exclude: Option<Origin>) -> Option<u64> {
1521 self.pick(|r| exclude.is_none_or(|origin| !r.route.hops.contains(&origin)), false)
1522 }
1523
1524 fn taints_a_reader(&self, route: &broadcast::Route) -> bool {
1531 route.hops.iter().any(|hop| self.excluded.contains_key(hop))
1532 }
1533
1534 fn prefer_untainted<'a>(&self, candidates: &[&'a FrontRoute]) -> Vec<&'a FrontRoute> {
1544 if self.excluded.is_empty() {
1545 return candidates.to_vec();
1546 }
1547 let clean: Vec<&FrontRoute> = candidates
1548 .iter()
1549 .copied()
1550 .filter(|r| !self.taints_a_reader(&r.route))
1551 .collect();
1552 match clean.is_empty() {
1553 true => candidates.to_vec(),
1554 false => clean,
1555 }
1556 }
1557
1558 fn serve_route(&self, skip: impl Fn(u64) -> bool) -> Option<u64> {
1567 if let Some(active) = self.active
1568 && !skip(active)
1569 && let Some(route) = self.routes.iter().find(|r| r.id == active)
1570 && !self.taints_a_reader(&route.route)
1571 {
1572 return Some(active);
1573 }
1574 self.pick(|r| !skip(r.id), true)
1575 }
1576
1577 fn reselect(&mut self, carrying: bool) {
1595 let best = self.best_route();
1596 if carrying
1597 && let (Some(best_id), Some(cur_id)) = (best, self.active)
1598 && best_id != cur_id
1599 && let Some(candidate) = self.routes.iter().find(|r| r.id == best_id)
1600 && let Some(incumbent) = self.routes.iter().find(|r| r.id == cur_id)
1601 && incumbent.route.announce
1602 && candidate.route.cost < incumbent.route.cost
1603 && candidate.route.advertised == 0
1604 && candidate.route.hops.len() >= 2
1605 && !self.handover_allowed(&candidate.route)
1606 {
1607 return;
1609 }
1610 self.active = best;
1611 }
1612
1613 fn handover_allowed(&self, route: &broadcast::Route) -> bool {
1625 let name = self.path.as_path();
1626 match route.hops.iter().last() {
1627 Some(peer) => fnv_key(&name, [*peer]) < fnv_key(&name, [self.self_origin]),
1628 None => true,
1629 }
1630 }
1631
1632 fn routes_snapshot(&self) -> Vec<broadcast::Route> {
1638 let mut routes: Vec<&FrontRoute> = self.routes.iter().collect();
1639 routes.sort_by_key(|r| route_order(&self.path.as_path(), r));
1640 routes.sort_by_key(|r| Some(r.id) != self.active);
1641 routes.into_iter().map(|r| r.route.clone()).collect()
1642 }
1643}
1644
1645fn sync_front(state: &kio::Producer<FrontState>, broadcast: &broadcast::Producer, leaf: &Lock<OriginNode>) {
1655 let mut leaf_guard = leaf.lock();
1660 let routes = state.read().routes_snapshot();
1661 if let Some(advert) = routes.first() {
1662 let announce = advert.announce;
1663 broadcast.clone().set_routes(routes);
1664 leaf_guard.set_announced(state, announce);
1665 }
1666}
1667
1668fn detach_source(
1681 state: &kio::Producer<FrontState>,
1682 broadcast: &broadcast::Producer,
1683 leaf: &Lock<OriginNode>,
1684 id: u64,
1685 graceful: bool,
1686) {
1687 let close = {
1688 let carrying = broadcast.demand().is_used();
1692 let Ok(mut s) = state.write() else { return };
1693 let Some(pos) = s.routes.iter().position(|r| r.id == id) else {
1694 return;
1695 };
1696 s.routes.remove(pos);
1697 s.reselect(carrying);
1698 if s.routes.is_empty() && !s.closed && (graceful || s.linger.is_zero()) {
1699 s.closed = true;
1702 true
1703 } else {
1704 false
1705 }
1706 };
1707 if close {
1708 broadcast.abort_spliced(Error::Dropped);
1709 }
1710 sync_front(state, broadcast, leaf);
1711}
1712
1713fn sync_announce(guard: &mut Option<stats::Announce>, announced: bool, ingress: &stats::Scope) {
1716 match (announced, guard.is_some()) {
1717 (true, false) => *guard = Some(ingress.announce()),
1718 (false, true) => *guard = None,
1719 _ => {}
1720 }
1721}
1722
1723async fn run_source(
1733 origin: Info,
1734 tree: Lock<OriginNode>,
1735 full: PathOwned,
1736 mut source: broadcast::Consumer,
1737 ingress: stats::Scope,
1738 _reservation: SourceReservation,
1740) {
1741 let ctx = AttachContext {
1742 origin: &origin,
1743 tree: &tree,
1744 full: &full,
1745 };
1746
1747 let Ok(mut route) = source.route_changed().await else {
1751 return;
1753 };
1754
1755 let mut announce = route.announce.then(|| ingress.announce());
1760 let mut may_take_over = true;
1766
1767 let leaf = if full.is_empty() {
1771 tree.clone()
1772 } else {
1773 tree.lock().leaf(&full)
1774 };
1775
1776 'attach: loop {
1777 let (state, broadcast, id) = match attach_source(&ctx, &leaf, &source, route.clone(), may_take_over) {
1778 Attach::Ready(state, broadcast, id) => (state, broadcast, id),
1779 Attach::Parked(incumbent) => {
1780 tracing::debug!(
1781 broadcast = %full,
1782 "path already live with a different publisher; parking this source until it ends",
1783 );
1784 let update = kio::wait(|waiter| {
1788 if let Poll::Ready(update) = source.poll_route_changed(waiter) {
1789 return Poll::Ready(Some(update));
1790 }
1791 match incumbent.poll(waiter, |s| if s.closed { Poll::Ready(()) } else { Poll::Pending }) {
1794 Poll::Ready(_) => Poll::Ready(None),
1795 Poll::Pending => Poll::Pending,
1796 }
1797 })
1798 .await;
1799 match update {
1800 Some(Ok(update)) => {
1802 sync_announce(&mut announce, update.announce, &ingress);
1803 if update.hops.iter().next().copied() != route.hops.iter().next().copied() {
1808 may_take_over = true;
1809 }
1810 route = update;
1811 }
1812 Some(Err(_)) => return,
1814 None => {}
1816 }
1817 continue 'attach;
1818 }
1819 };
1820 let publisher = route.hops.iter().next().copied();
1821
1822 loop {
1823 let update = kio::wait(|waiter| {
1824 if state
1830 .poll_ref(waiter, |s| if s.closed { Poll::Ready(()) } else { Poll::Pending })
1831 .is_ready()
1832 {
1833 return Poll::Ready(None);
1834 }
1835 source.poll_route_changed(waiter).map(Some)
1836 })
1837 .await;
1838 match update {
1839 None => {
1840 may_take_over = false;
1843 continue 'attach;
1844 }
1845 Some(Ok(update)) => {
1846 let announced = update.announce;
1847 if update.hops.iter().next().copied() != publisher {
1858 detach_source(&state, &broadcast, &leaf, id, true);
1859 sync_announce(&mut announce, announced, &ingress);
1860 may_take_over = true;
1864 route = update;
1865 continue 'attach;
1866 }
1867 {
1868 let carrying = broadcast.demand().is_used();
1869 let Ok(mut s) = state.write() else { return };
1870 let Some(entry) = s.routes.iter_mut().find(|r| r.id == id) else {
1871 return;
1872 };
1873 if entry.route == update {
1874 continue;
1875 }
1876 entry.route = update;
1877 s.reselect(carrying);
1878 }
1879 sync_announce(&mut announce, announced, &ingress);
1881 sync_front(&state, &broadcast, &leaf);
1882 }
1883 Some(Err(_)) => {
1884 detach_source(&state, &broadcast, &leaf, id, source.is_finished());
1887 return;
1888 }
1889 }
1890 }
1891 }
1892}
1893
1894enum Attach {
1896 Ready(kio::Producer<FrontState>, broadcast::Producer, u64),
1899 Parked(kio::Producer<FrontState>),
1906}
1907
1908struct AttachContext<'a> {
1910 origin: &'a Info,
1911 tree: &'a Lock<OriginNode>,
1914 full: &'a PathOwned,
1917}
1918
1919fn same_publisher(a: Option<Origin>, b: Option<Origin>) -> bool {
1928 if a == Some(Origin::UNKNOWN) || b == Some(Origin::UNKNOWN) {
1929 return false;
1930 }
1931 a == b
1932}
1933
1934fn attach_source(
1960 ctx: &AttachContext,
1961 leaf: &Lock<OriginNode>,
1962 source: &broadcast::Consumer,
1963 route: broadcast::Route,
1964 may_take_over: bool,
1965) -> Attach {
1966 let publisher = route.hops.iter().next().copied();
1967 let mut leaf_guard = leaf.lock();
1968
1969 if let Some(existing) = &leaf_guard.broadcast {
1972 let mut joined = None;
1973 let carrying = existing.broadcast.demand().is_used();
1974 if let Ok(mut s) = existing.state.write()
1975 && !s.closed
1976 {
1977 if same_publisher(s.publisher, publisher) {
1978 let id = s.next_route;
1979 s.next_route += 1;
1980 s.routes.push(FrontRoute {
1981 id,
1982 route: route.clone(),
1983 source: source.clone(),
1984 });
1985 s.reselect(carrying);
1986 joined = Some(id);
1987 } else if !may_take_over || !route.announce || s.taints_a_reader(&route) {
1988 return Attach::Parked(existing.state.clone());
1989 } else {
1990 s.closed = true;
1998 tracing::warn!(broadcast = %ctx.full, "replacing a live broadcast from a different publisher");
1999 }
2000 }
2001 if let Some(id) = joined {
2002 let state = existing.state.clone();
2003 let broadcast = existing.broadcast.clone();
2004 drop(leaf_guard);
2005 sync_front(&state, &broadcast, leaf);
2006 return Attach::Ready(state, broadcast, id);
2007 }
2008 }
2009
2010 let announce = route.announce;
2012 let broadcast = broadcast::Producer::new_spliced(broadcast::Info {
2013 origin: ctx.origin.clone(),
2014 });
2015 let _ = broadcast.clone().set_route(route.clone());
2016 let state = kio::Producer::new(FrontState {
2017 path: ctx.full.clone(),
2018 self_origin: ctx.origin.id,
2019 publisher,
2020 next_route: 1,
2021 excluded: HashMap::new(),
2022 track_info: HashMap::new(),
2023 routes: vec![FrontRoute {
2024 id: 0,
2025 route,
2026 source: source.clone(),
2027 }],
2028 active: Some(0),
2029 linger: ctx.origin.linger,
2030 closed: false,
2031 });
2032
2033 if let Some(stale) = leaf_guard.broadcast.take()
2037 && stale.announced
2038 {
2039 leaf_guard.notify.lock().unannounce(&stale.path);
2040 }
2041 let entry = OriginBroadcast {
2042 path: ctx.full.clone(),
2043 broadcast: broadcast.clone(),
2044 state: state.clone(),
2045 announced: announce,
2046 };
2047 if entry.announced {
2048 leaf_guard
2049 .notify
2050 .lock()
2051 .announce(ctx.full, &broadcast.consume(), &state);
2052 }
2053 leaf_guard.broadcast = Some(entry);
2054 drop(leaf_guard);
2055
2056 web_async::spawn(run_front(
2057 state.clone(),
2058 broadcast.clone(),
2059 ctx.tree.clone(),
2060 ctx.full.clone(),
2061 ));
2062
2063 Attach::Ready(state, broadcast, 0)
2064}
2065
2066async fn run_front(
2069 state: kio::Producer<FrontState>,
2070 mut broadcast: broadcast::Producer,
2071 tree: Lock<OriginNode>,
2072 full: PathOwned,
2073) {
2074 enum Step {
2075 Serve(Arc<str>, super::resume::Producer),
2076 Changed,
2078 Expired,
2080 Closed,
2081 }
2082
2083 let linger = state.read().linger;
2084 let mut deadline = kio::time::Deadline::new();
2089
2090 loop {
2091 let empty = {
2092 let s = state.read();
2093 !s.closed && s.routes.is_empty()
2094 };
2095 deadline.set(match (empty, deadline.deadline()) {
2096 (true, None) => web_async::time::Instant::now().checked_add(linger),
2099 (true, at) => at,
2100 (false, _) => None,
2101 });
2102
2103 let step = {
2104 kio::wait(|waiter| {
2105 if let Poll::Ready((name, resume)) = broadcast.poll_spliced_assigned(waiter) {
2106 return Poll::Ready(Step::Serve(name, resume));
2107 }
2108 match state.poll(waiter, |s| {
2111 if s.closed || s.routes.is_empty() != empty {
2112 Poll::Ready(())
2113 } else {
2114 Poll::Pending
2115 }
2116 }) {
2117 Poll::Ready(Ok(guard)) => {
2118 return Poll::Ready(if guard.closed { Step::Closed } else { Step::Changed });
2119 }
2120 Poll::Ready(Err(_)) => return Poll::Ready(Step::Closed),
2121 Poll::Pending => {}
2122 }
2123 deadline.poll(waiter).map(|_| Step::Expired)
2124 })
2125 .await
2126 };
2127
2128 match step {
2129 Step::Serve(name, resume) => {
2130 web_async::spawn(serve_track(state.clone(), name, resume));
2133 }
2134 Step::Changed => {}
2135 Step::Expired => {
2136 let close = {
2140 let Ok(mut s) = state.write() else { break };
2141 if !s.closed && s.routes.is_empty() {
2142 s.closed = true;
2143 true
2144 } else {
2145 false
2146 }
2147 };
2148 if close {
2149 break;
2150 }
2151 }
2152 Step::Closed => break,
2153 }
2154 }
2155
2156 broadcast.abort_spliced(Error::Dropped);
2158
2159 broadcast.finish();
2161
2162 tree.lock().remove(&state, &full);
2165}
2166
2167async fn serve_track(state: kio::Producer<FrontState>, name: Arc<str>, mut resume: super::resume::Producer) {
2181 enum Step {
2182 Closed,
2183 Splice(u64, broadcast::Consumer),
2184 Complete,
2185 Failed(Error),
2186 NoRoute,
2191 Idle,
2193 Demand,
2195 }
2196
2197 let mut serving: Option<(u64, track::Consumer)> = None;
2199 let mut spliced_edge: Option<u64> = None;
2205 let mut refused: HashSet<u64> = HashSet::new();
2210 let mut refusal: Option<Error> = None;
2211 let mut dead: HashSet<u64> = HashSet::new();
2216 let mut idle_since: Option<web_async::time::Instant> = None;
2218 let mut deadline = kio::time::Deadline::new();
2219
2220 loop {
2221 let serving_id = serving.as_ref().map(|(id, _)| *id);
2222
2223 {
2230 let s = state.read();
2231 refused.retain(|id| s.routes.iter().any(|r| r.id == *id));
2232 dead.retain(|id| s.routes.iter().any(|r| r.id == *id));
2233 let exhausted = !s.routes.is_empty()
2234 && s.serve_route(|id| refused.contains(&id) || dead.contains(&id))
2235 .is_none();
2236 if exhausted && dead.is_empty() {
2237 drop(s);
2238 let err = refusal.take().unwrap_or(Error::NotFound);
2239 tracing::debug!(name = %name, %err, "every source refused track; aborting");
2240 let _ = resume.abort(err);
2241 return;
2242 }
2243 }
2244
2245 let used = resume.is_used();
2257 idle_since = match (resume.is_spliced(), used) {
2258 (true, false) => idle_since.or_else(|| Some(web_async::time::Instant::now())),
2259 _ => None,
2260 };
2261 deadline.set(idle_since.and_then(|at| at.checked_add(TRACK_IDLE_LINGER)));
2262
2263 let step = {
2264 let skip = |id: u64| refused.contains(&id) || dead.contains(&id);
2265 kio::wait(|waiter| {
2266 match state.poll(waiter, |s| {
2272 let gone = serving_id.is_some_and(|id| !s.routes.iter().any(|r| r.id == id));
2273 if s.closed
2274 || (used && (gone || matches!(s.serve_route(skip), Some(next) if Some(next) != serving_id)))
2275 {
2276 Poll::Ready(())
2277 } else {
2278 Poll::Pending
2279 }
2280 }) {
2281 Poll::Ready(Ok(guard)) => {
2282 if guard.closed {
2283 return Poll::Ready(Step::Closed);
2284 }
2285 let Some(next) = guard.serve_route(skip) else {
2286 return Poll::Ready(Step::NoRoute);
2287 };
2288 let source = guard
2289 .routes
2290 .iter()
2291 .find(|r| r.id == next)
2292 .expect("servable source in table")
2293 .source
2294 .clone();
2295 return Poll::Ready(Step::Splice(next, source));
2296 }
2297 Poll::Ready(Err(_)) => return Poll::Ready(Step::Closed),
2298 Poll::Pending => {}
2299 }
2300
2301 let edge = match used {
2306 true => resume.poll_unused(waiter),
2307 false => resume.poll_used(waiter),
2308 };
2309 if edge.is_ready() {
2310 return Poll::Ready(Step::Demand);
2311 }
2312
2313 if let Some((_, track)) = &serving
2316 && let Poll::Ready(result) = track.poll_complete(waiter)
2317 {
2318 return Poll::Ready(match result {
2319 Ok(()) => Step::Complete,
2320 Err(err) => Step::Failed(err),
2321 });
2322 }
2323
2324 deadline.poll(waiter).map(|_| Step::Idle)
2325 })
2326 .await
2327 };
2328
2329 match step {
2330 Step::Closed => return,
2332 Step::Complete => {
2333 let _ = resume.finish();
2334 return;
2335 }
2336 Step::Failed(err) => {
2337 if resume.latest() == spliced_edge
2345 && let Some(id) = serving_id
2346 {
2347 let closing = state
2348 .read()
2349 .routes
2350 .iter()
2351 .find(|r| r.id == id)
2352 .is_some_and(|r| r.source.is_closing());
2353 if closing {
2354 dead.insert(id);
2355 } else {
2356 refused.insert(id);
2357 refusal = Some(err);
2358 }
2359 }
2360 serving = None;
2361 }
2362 Step::Demand => {}
2364 Step::NoRoute => serving = None,
2370 Step::Idle => {
2371 if resume.release().is_err() {
2376 return;
2378 }
2379 serving = None;
2380 }
2381 Step::Splice(id, source) => {
2382 let attempt = match source.track(&name) {
2386 Ok(track) => {
2387 let query = track.info().into_inner();
2390 let skip = |id: u64| refused.contains(&id) || dead.contains(&id);
2391 let info = kio::wait(|waiter| {
2392 if let Poll::Ready(result) = query.poll(waiter) {
2393 return Poll::Ready(Some(result));
2394 }
2395 match state.poll(waiter, |s| {
2396 if s.closed || s.serve_route(skip) != Some(id) {
2397 Poll::Ready(())
2398 } else {
2399 Poll::Pending
2400 }
2401 }) {
2402 Poll::Ready(_) => Poll::Ready(None),
2403 Poll::Pending => Poll::Pending,
2404 }
2405 })
2406 .await;
2407 match info {
2408 None => continue,
2410 Some(Ok(info)) => match track.poll_complete(&kio::Waiter::noop()) {
2413 Poll::Ready(Err(err)) => Err(err),
2414 _ => match state.write() {
2415 Ok(mut state) => state.accept_track_info(&name, info).map(|()| track),
2416 Err(_) => Err(Error::Dropped),
2417 },
2418 },
2419 Some(Err(err)) => Err(err),
2420 }
2421 }
2422 Err(err) => Err(err),
2423 };
2424
2425 match attempt {
2426 Ok(track) => {
2427 if let Err(err) = resume.takeover(&track) {
2428 let _ = resume.abort(err);
2433 return;
2434 }
2435 spliced_edge = resume.latest();
2447 serving = Some((id, track));
2448 }
2449 Err(_) if source.is_closing() => {
2453 dead.insert(id);
2454 serving = None;
2455 }
2456 Err(err) => {
2462 tracing::debug!(name = %name, source = id, %err, "source refused track");
2463 refused.insert(id);
2464 refusal = Some(err);
2465 serving = None;
2466 }
2467 }
2468 }
2469 }
2470 }
2471}
2472
2473#[derive(Default)]
2479struct OriginDynamicState {
2480 requests: Requests<PathOwned, kio::Producer<PendingBroadcast>>,
2483
2484 served: WeakCache<PathOwned, broadcast::WeakConsumer>,
2490}
2491
2492#[derive(Default)]
2499struct PendingBroadcast {
2500 resolved: Option<Result<broadcast::Consumer, Error>>,
2501}
2502
2503pub struct Dynamic {
2514 info: Origin,
2515 root: PathOwned,
2516 state: kio::Shared<OriginDynamicState>,
2517}
2518
2519impl Clone for Dynamic {
2520 fn clone(&self) -> Self {
2521 self.state.lock().requests.add_handler();
2525
2526 Self {
2527 info: self.info,
2528 root: self.root.clone(),
2529 state: self.state.clone(),
2530 }
2531 }
2532}
2533
2534impl Dynamic {
2535 fn new(info: Origin, root: PathOwned, state: kio::Shared<OriginDynamicState>) -> Self {
2536 state.lock().requests.add_handler();
2537
2538 Self { info, root, state }
2539 }
2540
2541 pub fn info(&self) -> &Origin {
2543 &self.info
2544 }
2545
2546 pub fn poll_requested_broadcast(&mut self, waiter: &kio::Waiter) -> Poll<Result<Request, Error>> {
2548 let mut state = ready!(self.state.poll(waiter, |state| {
2549 if state.requests.has_queued() {
2550 Poll::Ready(())
2551 } else {
2552 Poll::Pending
2553 }
2554 }));
2555
2556 let path = state.requests.pop().expect("predicate guaranteed a request");
2557 let producer = state.requests.get(&path).expect("popped key must be pending").clone();
2563 Poll::Ready(Ok(Request {
2564 path,
2565 producer,
2566 state: self.state.clone(),
2567 }))
2568 }
2569
2570 pub async fn requested_broadcast(&mut self) -> Result<Request, Error> {
2573 kio::wait(|waiter| self.poll_requested_broadcast(waiter)).await
2574 }
2575
2576 pub fn root(&self) -> &Path<'_> {
2578 &self.root
2579 }
2580}
2581
2582impl Drop for Dynamic {
2583 fn drop(&mut self) {
2584 let mut state = self.state.lock();
2587 if state.requests.remove_handler() {
2588 state.requests.drain_queued();
2592 }
2593 }
2594}
2595
2596pub struct Request {
2603 path: PathOwned,
2605
2606 producer: kio::Producer<PendingBroadcast>,
2609
2610 state: kio::Shared<OriginDynamicState>,
2612}
2613
2614impl Request {
2615 pub fn path(&self) -> &Path<'_> {
2617 &self.path
2618 }
2619
2620 pub fn accept(self, broadcast: impl Consume<broadcast::Consumer>) {
2626 let broadcast = broadcast.consume();
2627
2628 let resolved = {
2634 let mut state = self.state.lock();
2635 let existing = state.served.insert(self.path.clone(), broadcast.weak());
2636 state
2637 .requests
2638 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2639 existing.map(|weak| weak.consume()).unwrap_or(broadcast)
2640 };
2641
2642 if let Ok(mut pending) = self.producer.write() {
2643 pending.resolved = Some(Ok(resolved));
2644 }
2645 }
2647
2648 pub fn reject(self, err: Error) {
2650 self.state
2651 .lock()
2652 .requests
2653 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2654 if let Ok(mut state) = self.producer.write() {
2655 state.resolved = Some(Err(err));
2656 }
2657 }
2658}
2659
2660impl Drop for Request {
2661 fn drop(&mut self) {
2662 self.state
2670 .lock()
2671 .requests
2672 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2673 }
2674}
2675
2676pub struct Requesting {
2683 inner: RequestState,
2684 stats: stats::Scope,
2687}
2688
2689enum RequestState {
2690 Ready(broadcast::Consumer),
2692 Failed(Error),
2695 Pending(kio::Consumer<PendingBroadcast>),
2697}
2698
2699impl Requesting {
2700 fn ready(broadcast: broadcast::Consumer) -> Self {
2701 Self {
2702 inner: RequestState::Ready(broadcast),
2703 stats: stats::Scope::default(),
2704 }
2705 }
2706
2707 fn failed(error: Error) -> Self {
2708 Self {
2709 inner: RequestState::Failed(error),
2710 stats: stats::Scope::default(),
2711 }
2712 }
2713
2714 fn pending(consumer: kio::Consumer<PendingBroadcast>) -> Self {
2715 Self {
2716 inner: RequestState::Pending(consumer),
2717 stats: stats::Scope::default(),
2718 }
2719 }
2720
2721 fn with_stats(mut self, scope: stats::Scope) -> Self {
2722 self.stats = scope;
2723 self
2724 }
2725
2726 pub fn poll_ok(&self, waiter: &kio::Waiter) -> Poll<Result<broadcast::Consumer, Error>> {
2728 match &self.inner {
2729 RequestState::Ready(broadcast) => Poll::Ready(Ok(broadcast.clone().with_stats(self.stats.clone()))),
2730 RequestState::Failed(error) => Poll::Ready(Err(error.clone())),
2731 RequestState::Pending(consumer) => Poll::Ready(
2732 match ready!(consumer.poll(waiter, |state| match &state.resolved {
2733 Some(result) => Poll::Ready(result.clone()),
2734 None => Poll::Pending,
2735 })) {
2736 Ok(result) => result.map(|broadcast| broadcast.with_stats(self.stats.clone())),
2737 Err(_closed) => Err(Error::Unroutable),
2739 },
2740 ),
2741 }
2742 }
2743}
2744
2745impl kio::Pollable for Requesting {
2746 type Output = Result<broadcast::Consumer, Error>;
2747
2748 fn poll(&self, waiter: &kio::Waiter) -> Poll<Self::Output> {
2749 self.poll_ok(waiter)
2750 }
2751}
2752
2753pub trait Consume<T> {
2761 fn consume(&self) -> T;
2763}
2764
2765impl<T, U: Consume<T>> Consume<T> for &U {
2766 fn consume(&self) -> T {
2767 (**self).consume()
2768 }
2769}
2770
2771impl Consume<Consumer> for Producer {
2772 fn consume(&self) -> Consumer {
2773 Consumer::new(
2777 self.info,
2778 self.root.clone(),
2779 self.nodes.clone(),
2780 self.dynamic.clone(),
2781 stats::Session::default(),
2782 )
2783 }
2784}
2785
2786impl Consume<Consumer> for Consumer {
2787 fn consume(&self) -> Consumer {
2788 self.clone()
2789 }
2790}
2791
2792impl Consume<broadcast::Consumer> for broadcast::Producer {
2793 fn consume(&self) -> broadcast::Consumer {
2794 self.consume()
2796 }
2797}
2798
2799impl Consume<broadcast::Consumer> for broadcast::Consumer {
2800 fn consume(&self) -> broadcast::Consumer {
2801 self.clone()
2802 }
2803}
2804
2805impl Consume<track::Consumer> for track::Producer {
2806 fn consume(&self) -> track::Consumer {
2807 self.consume()
2808 }
2809}
2810
2811impl Consume<track::Consumer> for track::Consumer {
2812 fn consume(&self) -> track::Consumer {
2813 self.clone()
2814 }
2815}
2816
2817#[derive(Clone)]
2823pub struct Consumer {
2824 info: Origin,
2826 nodes: OriginNodes,
2827
2828 root: PathOwned,
2830
2831 dynamic: kio::Shared<OriginDynamicState>,
2834
2835 stats: stats::Session,
2839
2840 exclude: Option<Origin>,
2844}
2845
2846impl std::ops::Deref for Consumer {
2847 type Target = Origin;
2848
2849 fn deref(&self) -> &Self::Target {
2850 &self.info
2851 }
2852}
2853
2854impl Consumer {
2855 fn new(
2856 info: Origin,
2857 root: PathOwned,
2858 nodes: OriginNodes,
2859 dynamic: kio::Shared<OriginDynamicState>,
2860 stats: stats::Session,
2861 ) -> Self {
2862 Self {
2863 info,
2864 nodes,
2865 root,
2866 dynamic,
2867 stats,
2868 exclude: None,
2869 }
2870 }
2871
2872 pub(crate) fn excluding(mut self, peer: Origin) -> Self {
2877 self.exclude = Some(peer);
2878 self
2879 }
2880
2881 pub fn with_stats(mut self, session: stats::Session) -> Self {
2885 self.stats = session;
2886 self
2887 }
2888
2889 fn untagged(&self) -> Self {
2893 Self {
2894 stats: stats::Session::default(),
2895 ..self.clone()
2896 }
2897 }
2898
2899 pub(crate) fn empty(&self) -> Self {
2904 Self {
2905 info: self.info,
2906 nodes: OriginNodes::empty(),
2907 root: self.root.clone(),
2908 dynamic: self.dynamic.clone(),
2909 stats: self.stats.clone(),
2910 exclude: self.exclude,
2911 }
2912 }
2913
2914 pub fn announced(&self) -> AnnounceConsumer {
2921 AnnounceConsumer::new(self.root.clone(), self.nodes.clone(), self.stats.clone(), self.exclude)
2922 }
2923
2924 pub fn consume(&self) -> Self {
2926 self.clone()
2927 }
2928
2929 fn resolve(&self, path: impl AsPath) -> Resolved {
2938 let path = path.as_path();
2939 let Some(rest) = self.nodes.get(&path) else {
2940 return Resolved::OutOfScope;
2941 };
2942 let state = self.nodes.tree.lock();
2943 state.resolve_broadcast(&rest, self.exclude)
2944 }
2945
2946 #[cfg(test)]
2948 pub(crate) fn get_broadcast(&self, path: impl AsPath) -> Option<broadcast::Consumer> {
2949 match self.resolve(path) {
2950 Resolved::Found(broadcast) => Some(broadcast),
2951 Resolved::Excluded | Resolved::OutOfScope | Resolved::Missing => None,
2952 }
2953 }
2954
2955 pub async fn announced_broadcast(&self, path: impl AsPath) -> Option<broadcast::Consumer> {
2966 let path = path.as_path();
2967
2968 let consumer = self.scope(std::slice::from_ref(&path))?;
2970
2971 if !consumer.allowed().any(|allowed| path.has_prefix(allowed)) {
2975 return None;
2976 }
2977
2978 let mut announced = consumer.untagged().announced();
2982 let scope = self.stats.egress(self.root.join(&path).to_owned());
2983 loop {
2984 let OriginAnnounce {
2985 path: announced_path,
2986 broadcast,
2987 } = announced.next().await?;
2988 if announced_path.as_path() == path
2990 && let Some(broadcast) = broadcast
2991 {
2992 return Some(broadcast.with_stats(scope));
2993 }
2994 }
2995 }
2996
2997 pub fn scope(&self, prefixes: &[Path]) -> Option<Consumer> {
3003 let prefixes = PathPrefixes::new(prefixes);
3004 Some(Consumer {
3005 info: self.info,
3006 root: self.root.clone(),
3007 nodes: self.nodes.select(&prefixes)?,
3008 dynamic: self.dynamic.clone(),
3009 stats: self.stats.clone(),
3010 exclude: self.exclude,
3011 })
3012 }
3013
3014 pub fn request_broadcast(&self, path: impl AsPath) -> kio::Pending<Requesting> {
3034 let path = path.as_path();
3035
3036 let absolute = self.root.join(&path).to_owned();
3040 let scope = self.stats.egress(&absolute);
3041
3042 match self.resolve(&path) {
3057 Resolved::Found(broadcast) => return kio::Pending::new(Requesting::ready(broadcast).with_stats(scope)),
3058 Resolved::Excluded => return kio::Pending::new(Requesting::failed(Error::Unroutable)),
3059 Resolved::OutOfScope => return kio::Pending::new(Requesting::failed(Error::Unauthorized)),
3060 Resolved::Missing => {}
3061 }
3062
3063 let mut state = self.dynamic.lock();
3064
3065 if let Some(weak) = state.served.get(&absolute) {
3069 return kio::Pending::new(Requesting::ready(weak.consume()).with_stats(scope));
3070 }
3071
3072 let consumer = if let Some(producer) = state.requests.join(&absolute) {
3075 producer.consume()
3076 } else {
3077 let producer = kio::Producer::<PendingBroadcast>::default();
3078 let consumer = producer.consume();
3079 if state.requests.insert(absolute, producer).is_err() {
3080 return kio::Pending::new(Requesting::failed(Error::Unroutable));
3081 }
3082 consumer
3083 };
3084
3085 kio::Pending::new(Requesting::pending(consumer).with_stats(scope))
3086 }
3087
3088 pub fn with_root(&self, prefix: impl AsPath) -> Option<Self> {
3093 let prefix = prefix.as_path();
3094
3095 Some(Self {
3096 info: self.info,
3097 root: self.root.join(&prefix).to_owned(),
3098 nodes: self.nodes.root(&prefix)?,
3099 dynamic: self.dynamic.clone(),
3100 stats: self.stats.clone(),
3101 exclude: self.exclude,
3102 })
3103 }
3104
3105 pub fn root(&self) -> &Path<'_> {
3107 &self.root
3108 }
3109
3110 pub fn allowed(&self) -> impl Iterator<Item = &Path<'_>> {
3113 self.nodes.nodes.iter().map(|(root, _)| root)
3114 }
3115
3116 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
3118 self.root.join(path)
3119 }
3120}
3121
3122#[derive(Clone)]
3127pub struct AnnounceProducer {
3128 nodes: OriginNodes,
3129 root: PathOwned,
3130}
3131
3132impl AnnounceProducer {
3133 fn new(root: PathOwned, nodes: OriginNodes) -> Self {
3134 Self { nodes, root }
3135 }
3136
3137 pub fn consume(&self) -> AnnounceConsumer {
3143 AnnounceConsumer::new(self.root.clone(), self.nodes.clone(), stats::Session::default(), None)
3146 }
3147
3148 pub fn root(&self) -> &Path<'_> {
3150 &self.root
3151 }
3152}
3153
3154pub struct AnnounceConsumer {
3159 id: ConsumerId,
3160 nodes: OriginNodes,
3161 root: PathOwned,
3162
3163 state: kio::Producer<OriginConsumerState>,
3166
3167 stats: stats::Session,
3170
3171 guards: HashMap<PathOwned, stats::Announce>,
3175}
3176
3177impl AnnounceConsumer {
3178 fn new(root: PathOwned, nodes: OriginNodes, stats: stats::Session, exclude: Option<Origin>) -> Self {
3179 let state = kio::Producer::<OriginConsumerState>::default();
3180 let id = ConsumerId::new();
3181
3182 for (_, absolute) in &nodes.nodes {
3183 let notify = AnnounceConsumerNotify {
3184 root: root.clone(),
3185 state: state.clone(),
3186 exclude,
3187 };
3188 nodes.tree.lock().consume_at(id, notify, absolute);
3189 }
3190
3191 Self {
3192 id,
3193 nodes,
3194 root,
3195 state,
3196 stats,
3197 guards: HashMap::new(),
3198 }
3199 }
3200
3201 fn attribute(&mut self, update: OriginAnnounce) -> OriginAnnounce {
3207 let OriginAnnounce { path, broadcast } = update;
3208 let absolute = self.root.join(&path).to_owned();
3209 match broadcast {
3210 Some(broadcast) => {
3211 let scope = self.stats.egress(&absolute);
3212 self.guards.entry(absolute).or_insert_with(|| scope.announce());
3213 OriginAnnounce {
3214 path,
3215 broadcast: Some(broadcast.with_stats(scope)),
3216 }
3217 }
3218 None => {
3219 self.guards.remove(&absolute);
3220 OriginAnnounce { path, broadcast: None }
3221 }
3222 }
3223 }
3224
3225 pub async fn next(&mut self) -> Option<OriginAnnounce> {
3232 kio::wait(|waiter| self.poll_next(waiter)).await
3233 }
3234
3235 pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll<Option<OriginAnnounce>> {
3241 let update = {
3242 let mut state = match ready!(self.state.poll(waiter, |state| {
3243 if state.pending.is_empty() {
3244 Poll::Pending
3245 } else {
3246 Poll::Ready(())
3247 }
3248 })) {
3249 Ok(state) => state,
3250 Err(_) => return Poll::Ready(None),
3252 };
3253 state.take().expect("predicate guaranteed an update")
3254 };
3255 Poll::Ready(Some(self.attribute(update)))
3256 }
3257
3258 pub fn try_next(&mut self) -> Option<OriginAnnounce> {
3263 let update = self.state.write().ok()?.take()?;
3264 Some(self.attribute(update))
3265 }
3266
3267 pub fn is_closed(&self) -> bool {
3269 self.state.write().is_err()
3270 }
3271
3272 pub fn root(&self) -> &Path<'_> {
3274 &self.root
3275 }
3276
3277 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
3279 self.root.join(path)
3280 }
3281}
3282
3283impl Drop for AnnounceConsumer {
3284 fn drop(&mut self) {
3285 for (_, absolute) in &self.nodes.nodes {
3286 self.nodes.tree.lock().detach(Claim::Consumer(self.id), absolute);
3287 }
3288 }
3289}
3290
3291#[cfg(test)]
3292use futures::FutureExt;
3293
3294#[cfg(test)]
3295#[allow(missing_docs)] impl AnnounceConsumer {
3297 pub fn assert_next(&mut self, expected: impl AsPath, broadcast: &broadcast::Consumer) {
3298 let expected = expected.as_path();
3299 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3300 assert_eq!(announce.path, expected, "wrong path");
3301 let announced = announce.broadcast.expect("should be an active announce");
3302 assert!(announced.is_clone(broadcast), "should be the same broadcast");
3303 }
3304
3305 pub fn assert_next_some(&mut self, expected: impl AsPath) -> broadcast::Consumer {
3309 let expected = expected.as_path();
3310 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3311 assert_eq!(announce.path, expected, "wrong path");
3312 announce.broadcast.expect("should be an active announce")
3313 }
3314
3315 pub fn assert_try_next(&mut self, expected: impl AsPath, broadcast: &broadcast::Consumer) {
3316 let expected = expected.as_path();
3317 let announce = self.try_next().expect("no next");
3318 assert_eq!(announce.path, expected, "wrong path");
3319 let announced = announce.broadcast.expect("should be an active announce");
3320 assert!(announced.is_clone(broadcast), "should be the same broadcast");
3321 }
3322
3323 pub fn assert_try_next_some(&mut self, expected: impl AsPath) -> broadcast::Consumer {
3325 let expected = expected.as_path();
3326 let announce = self.try_next().expect("no next");
3327 assert_eq!(announce.path, expected, "wrong path");
3328 announce.broadcast.expect("should be an active announce")
3329 }
3330
3331 pub fn assert_next_none(&mut self, expected: impl AsPath) {
3332 let expected = expected.as_path();
3333 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3334 assert_eq!(announce.path, expected, "wrong path");
3335 assert!(announce.broadcast.is_none(), "should be unannounced");
3336 }
3337
3338 pub fn assert_next_wait(&mut self) {
3339 if let Some(res) = self.next().now_or_never() {
3340 panic!("next should block: got {:?}", res.map(|a| a.path));
3341 }
3342 }
3343
3344 }
3353
3354#[cfg(test)]
3355mod tests {
3356 use crate::Timescale;
3357 use crate::coding::Decode;
3358 use crate::group;
3359
3360 use super::*;
3361
3362 fn announce() -> broadcast::Route {
3364 broadcast::Route::new().with_announce(true)
3365 }
3366
3367 fn origin_keyed(name: &str, peer: Origin, above: bool) -> Origin {
3373 let name = Path::new(name);
3374 let peer_key = fnv_key(&name, [peer]);
3375 (100u64..)
3376 .map(|id| Origin::new(id).unwrap())
3377 .find(|origin| (fnv_key(&name, [*origin]) > peer_key) == above)
3378 .unwrap()
3379 }
3380
3381 fn front_state(self_origin: Origin, routes: Vec<broadcast::Route>) -> FrontState {
3384 let source = broadcast::Info::new().produce().consume();
3385 FrontState {
3386 path: Path::new("test").to_owned(),
3387 self_origin,
3388 publisher: routes.first().and_then(|r| r.hops.iter().next().copied()),
3389 next_route: routes.len() as u64,
3390 excluded: HashMap::new(),
3391 track_info: HashMap::new(),
3392 routes: routes
3393 .into_iter()
3394 .enumerate()
3395 .map(|(id, route)| FrontRoute {
3396 id: id as u64,
3397 route,
3398 source: source.clone(),
3399 })
3400 .collect(),
3401 active: Some(0),
3402 linger: Duration::ZERO,
3403 closed: false,
3404 }
3405 }
3406
3407 fn sibling_route(peer: Origin) -> broadcast::Route {
3410 let hops = OriginList::try_from(vec![Origin::new(90).unwrap(), peer]).unwrap();
3411 announce().with_hops(hops)
3412 }
3413
3414 fn upstream_route(cost: u64) -> broadcast::Route {
3416 let hops = OriginList::try_from(vec![Origin::new(90).unwrap()]).unwrap();
3417 announce().with_hops(hops).with_cost(cost)
3418 }
3419
3420 #[test]
3424 fn test_carrying_gate_keys() {
3425 let peer = Origin::new(3).unwrap();
3426
3427 let mut lost = front_state(
3429 origin_keyed("test", peer, false),
3430 vec![upstream_route(10), sibling_route(peer)],
3431 );
3432 lost.reselect(true);
3433 assert_eq!(
3434 lost.active,
3435 Some(0),
3436 "carrying front re-parented onto a higher-keyed peer"
3437 );
3438 lost.reselect(false);
3439 assert_eq!(lost.active, Some(1), "idle front must take the cheaper route");
3440
3441 let mut won = front_state(
3443 origin_keyed("test", peer, true),
3444 vec![upstream_route(10), sibling_route(peer)],
3445 );
3446 won.reselect(true);
3447 assert_eq!(won.active, Some(1), "carrying front must follow a lower-keyed peer");
3448 }
3449
3450 #[test]
3455 fn test_carrying_gate_symmetric_race() {
3456 let a = Origin::new(1).unwrap();
3457 let b = Origin::new(2).unwrap();
3458
3459 let mut a_view = front_state(a, vec![upstream_route(10), sibling_route(b)]);
3460 let mut b_view = front_state(b, vec![upstream_route(10), sibling_route(a)]);
3461 a_view.reselect(true);
3462 b_view.reselect(true);
3463
3464 let a_moved = a_view.active == Some(1);
3465 let b_moved = b_view.active == Some(1);
3466 assert!(
3467 a_moved != b_moved,
3468 "exactly one side must re-parent (a: {a_moved}, b: {b_moved})"
3469 );
3470 }
3471
3472 #[test]
3477 fn test_carrying_switches_to_benign_routes() {
3478 let peer = Origin::new(3).unwrap();
3479 let lost = origin_keyed("test", peer, false);
3480
3481 let mut forwarder = sibling_route(peer).with_cost(4);
3483 forwarder.advertised = 4;
3484 let mut state = front_state(lost, vec![upstream_route(10), forwarder]);
3485 state.reselect(true);
3486 assert_eq!(
3487 state.active,
3488 Some(1),
3489 "a cheaper forwarder path must win while carrying"
3490 );
3491
3492 let direct = announce().with_hops(OriginList::try_from(vec![peer]).unwrap());
3494 let mut state = front_state(lost, vec![upstream_route(10), direct]);
3495 state.reselect(true);
3496 assert_eq!(
3497 state.active,
3498 Some(1),
3499 "a direct publisher route must win while carrying"
3500 );
3501
3502 let mut state = front_state(lost, vec![sibling_route(peer), sibling_route(peer)]);
3507 state.reselect(true);
3508 assert_eq!(
3509 state.active,
3510 Some(1),
3511 "a reconnect on an identical chain must win while carrying"
3512 );
3513 }
3514
3515 #[test]
3518 fn test_carrying_gate_ignores_unannounced_incumbent() {
3519 let peer = Origin::new(3).unwrap();
3520 let unannounced = upstream_route(10).with_announce(false);
3521 let mut state = front_state(
3522 origin_keyed("test", peer, false),
3523 vec![unannounced, sibling_route(peer)],
3524 );
3525 state.reselect(true);
3526 assert_eq!(
3527 state.active,
3528 Some(1),
3529 "an unannounced incumbent must always be displaced"
3530 );
3531 }
3532
3533 #[test]
3538 fn test_reflection_through_an_exposed_peer_cannot_take_over() {
3539 let peer = Origin::new(42).unwrap();
3540 let upstream = OriginList::try_from(vec![Origin::new(7).unwrap()]).unwrap();
3541 let reflected = announce().with_hops(OriginList::try_from(vec![peer]).unwrap());
3542
3543 let mut state = front_state(Origin::new(1).unwrap(), vec![announce().with_hops(upstream)]);
3544
3545 assert!(!state.taints_a_reader(&reflected));
3547
3548 *state.excluded.entry(peer).or_default() += 1;
3551 assert!(state.taints_a_reader(&reflected));
3552 }
3553
3554 #[test]
3557 fn test_rival_publisher_through_another_peer_still_takes_over() {
3558 let peer = Origin::new(42).unwrap();
3559 let elsewhere = Origin::new(43).unwrap();
3560 let upstream = OriginList::try_from(vec![Origin::new(7).unwrap()]).unwrap();
3561 let rival = announce().with_hops(OriginList::try_from(vec![Origin::UNKNOWN, elsewhere]).unwrap());
3562
3563 let mut state = front_state(Origin::new(1).unwrap(), vec![announce().with_hops(upstream)]);
3564 *state.excluded.entry(peer).or_default() += 1;
3565
3566 assert!(!state.taints_a_reader(&rival), "only the peer we feed is a reflection");
3567 }
3568
3569 #[test]
3574 fn test_opaque_peer_understates_its_depth() {
3575 let us = Origin::new(1).unwrap();
3576
3577 let direct = || {
3579 announce()
3580 .with_hops(OriginList::try_from(vec![Origin::new(7).unwrap(), Origin::new(8).unwrap()]).unwrap())
3581 .with_cost(2)
3582 };
3583 let opaque = |cost| {
3586 announce()
3587 .with_hops(OriginList::try_from(vec![Origin::new(42).unwrap()]).unwrap())
3588 .with_cost(cost)
3589 };
3590
3591 let mut state = front_state(us, vec![direct(), opaque(1)]);
3593 state.reselect(false);
3594 assert_eq!(
3595 state.active,
3596 Some(1),
3597 "an unpriced opaque link out-ranks a shorter real path"
3598 );
3599
3600 let mut state = front_state(us, vec![direct(), opaque(16)]);
3602 state.reselect(false);
3603 assert_eq!(
3604 state.active,
3605 Some(0),
3606 "pricing the opaque link restores the intended order"
3607 );
3608 }
3609
3610 async fn settle() {
3613 tokio::time::sleep(tokio::time::Duration::from_millis(1)).await;
3614 }
3615
3616 fn origin_scoped(prefixes: &[Path]) -> Producer {
3618 Origin::random().produce().scope(prefixes).expect("in scope")
3619 }
3620
3621 async fn accept_track(dynamic: &mut broadcast::Dynamic, name: &str) -> track::Producer {
3624 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic.requested_track())
3625 .await
3626 .expect("timed out waiting for a track request")
3627 .expect("source closed");
3628 assert_eq!(request.name(), name, "unexpected track dispatched");
3629 request.accept(None)
3630 }
3631
3632 async fn accept_tracks(dynamic: &mut broadcast::Dynamic, count: usize) -> HashMap<String, track::Producer> {
3636 let mut accepted = HashMap::new();
3637 for _ in 0..count {
3638 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic.requested_track())
3639 .await
3640 .expect("timed out waiting for a track request")
3641 .expect("source closed");
3642 let name = request.name().to_string();
3643 accepted.insert(name, request.accept(None));
3644 }
3645 accepted
3646 }
3647
3648 #[tokio::test]
3652 async fn test_stats_tagged_end_to_end() {
3653 use crate::Timestamp;
3654 use crate::stats::{Config, Registry, Tier};
3655 use bytes::Bytes;
3656
3657 tokio::time::pause();
3658
3659 let registry = Registry::new(Config::new());
3660 let ctx = registry.tier(Tier::default()).session("acme");
3661
3662 let origin = Origin::random().produce();
3663 let ingress = origin.clone().with_stats(ctx.clone());
3664 let egress = origin.consume().with_stats(ctx.clone());
3665
3666 let mut announced = egress.announced();
3669
3670 let source = ingress.create_broadcast("demo", announce()).unwrap();
3672 let mut dynamic = source.dynamic();
3673 settle().await;
3674 settle().await;
3675
3676 let update = announced.next().await.unwrap();
3678 assert_eq!(update.path.as_str(), "demo");
3679 let broadcast = update.broadcast.unwrap();
3680
3681 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3683 let mut producer = accept_track(&mut dynamic, "video").await;
3684 settle().await;
3685 let mut sub = subscribing.await.unwrap();
3686
3687 let mut group = producer.append_group().unwrap();
3689 group
3690 .write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
3691 .unwrap();
3692 group
3693 .write_frame(Timestamp::ZERO, Bytes::from_static(b"world"))
3694 .unwrap();
3695 group.finish().unwrap();
3696
3697 let mut group_c = sub.recv_group().await.unwrap().unwrap();
3699 let mut frames = 0;
3700 while let Some(frame) = group_c.read_frame().await.unwrap() {
3701 assert_eq!(frame.payload.len(), 5);
3702 frames += 1;
3703 }
3704 assert_eq!(frames, 2);
3705 settle().await;
3706
3707 let report = registry.report();
3708 let entry = report
3709 .traffic
3710 .iter()
3711 .find(|e| e.path.as_str() == "demo")
3712 .expect("demo tracked");
3713 let path_len = "demo".len() as u64;
3714
3715 let egress = &entry.publisher;
3717 assert_eq!(egress.announced, 1, "one egress announce");
3718 assert_eq!(egress.announced_bytes, path_len);
3719 assert_eq!(egress.subscriptions, 1, "one egress subscription");
3720 assert_eq!(egress.broadcasts, 1, "one viewer");
3721 assert_eq!(egress.groups, 1);
3722 assert_eq!(egress.frames, 2);
3723 assert_eq!(egress.bytes, 10);
3724 assert_eq!(egress.fetches, 0);
3725
3726 let ingress = &entry.subscriber;
3728 assert_eq!(ingress.announced, 1, "one ingress announce");
3729 assert_eq!(ingress.announced_bytes, path_len);
3730 assert_eq!(ingress.subscriptions, 1, "one ingress track");
3731 assert_eq!(ingress.broadcasts, 0, "ingress has no viewer refcount");
3732 assert_eq!(ingress.groups, 1);
3733 assert_eq!(ingress.frames, 2);
3734 assert_eq!(ingress.bytes, 10);
3735
3736 let fetched = broadcast.track("video").unwrap().fetch_group(0, None).await.unwrap();
3738 let _ = fetched;
3739 settle().await;
3740 let report = registry.report();
3741 let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap();
3742 assert_eq!(entry.publisher.fetches, 1, "one fetch");
3743 assert_eq!(entry.publisher.subscriptions, 1, "fetch does not bump subscriptions");
3744 assert_eq!(entry.publisher.broadcasts, 1, "fetch does not bump the viewer refcount");
3745 assert_eq!(entry.subscriber.fetches, 0, "ingress cannot fetch");
3749 }
3750
3751 #[tokio::test]
3756 async fn test_stats_read_frame_counts_once() {
3757 use crate::Timestamp;
3758 use crate::stats::{Config, Registry, Tier};
3759 use bytes::Bytes;
3760
3761 tokio::time::pause();
3762
3763 let registry = Registry::new(Config::new());
3764 let ctx = registry.tier(Tier::default()).session("acme");
3765
3766 let origin = Origin::random().produce();
3767 let ingress = origin.clone().with_stats(ctx.clone());
3768 let egress = origin.consume().with_stats(ctx.clone());
3769
3770 let mut announced = egress.announced();
3771 let source = ingress.create_broadcast("demo", announce()).unwrap();
3772 let mut dynamic = source.dynamic();
3773 settle().await;
3774 settle().await;
3775
3776 let broadcast = announced.next().await.unwrap().broadcast.unwrap();
3777 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3778 let mut producer = accept_track(&mut dynamic, "video").await;
3779 settle().await;
3780 let mut sub = subscribing.await.unwrap();
3781
3782 producer
3784 .write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
3785 .unwrap();
3786
3787 let frame = sub.read_frame().await.unwrap().expect("frame");
3788 assert_eq!(frame.payload.len(), 5);
3789 settle().await;
3790
3791 let report = registry.report();
3792 let entry = report
3793 .traffic
3794 .iter()
3795 .find(|e| e.path.as_str() == "demo")
3796 .expect("demo tracked");
3797 assert_eq!(entry.publisher.groups, 1, "one group, counted once");
3798 assert_eq!(entry.publisher.frames, 1, "one frame, counted once");
3799 assert_eq!(
3800 entry.publisher.bytes, 5,
3801 "payload counted once, not zero and not doubled"
3802 );
3803 }
3804
3805 #[tokio::test]
3809 async fn test_stats_datagrams_counted_both_sides() {
3810 use crate::Timestamp;
3811 use crate::stats::{Config, Registry, Tier};
3812
3813 tokio::time::pause();
3814
3815 let registry = Registry::new(Config::new());
3816 let ctx = registry.tier(Tier::default()).session("acme");
3817
3818 let origin = Origin::random().produce();
3819 let ingress = origin.clone().with_stats(ctx.clone());
3820 let egress = origin.consume().with_stats(ctx.clone());
3821
3822 let mut announced = egress.announced();
3823 let source = ingress.create_broadcast("demo", announce()).unwrap();
3824 let mut dynamic = source.dynamic();
3825 settle().await;
3826 settle().await;
3827
3828 let broadcast = announced.next().await.unwrap().broadcast.unwrap();
3829 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3830 let mut producer = accept_track(&mut dynamic, "video").await;
3831 settle().await;
3832 let mut sub = subscribing.await.unwrap();
3833
3834 producer.append_datagram(Timestamp::ZERO, &b"hello"[..]).unwrap();
3835 let datagram = sub.recv_datagram().await.unwrap().expect("datagram");
3836 assert_eq!(&datagram.payload[..], b"hello");
3837 settle().await;
3838
3839 let report = registry.report();
3840 let entry = report
3841 .traffic
3842 .iter()
3843 .find(|e| e.path.as_str() == "demo")
3844 .expect("demo tracked");
3845
3846 for (side, traffic) in [("egress", &entry.publisher), ("ingress", &entry.subscriber)] {
3847 assert_eq!(traffic.datagrams, 1, "{side}: one datagram");
3848 assert_eq!(traffic.groups, 1, "{side}: counted as its single-frame group");
3849 assert_eq!(traffic.frames, 1, "{side}: one frame");
3850 assert_eq!(traffic.bytes, 5, "{side}: payload counted once");
3851 }
3852 }
3853
3854 #[test]
3855 fn origin_rejects_reserved_ids() {
3856 assert!(Origin::new(0).is_err());
3857 assert!(Origin::new(1u64 << 62).is_err());
3858 assert_eq!(Origin::new(1).unwrap().id(), 1);
3859
3860 let mut zero = [0u8].as_slice();
3861 assert_eq!(
3862 Origin::decode(&mut zero, crate::lite::Version::Lite05).unwrap(),
3863 Origin::UNKNOWN
3864 );
3865 }
3866
3867 #[test]
3868 fn origin_list_push_fails_at_limit() {
3869 let mut list = OriginList::new();
3870 for _ in 0..MAX_HOPS {
3871 list.push(Origin::random()).unwrap();
3872 }
3873 assert_eq!(list.len(), MAX_HOPS);
3874 assert_eq!(list.push(Origin::random()), Err(TooManyOrigins));
3875 }
3876
3877 #[test]
3878 fn origin_list_replace_first() {
3879 let mut list = OriginList::new();
3880 for _ in 0..3 {
3881 list.push(Origin::UNKNOWN).unwrap();
3882 }
3883
3884 assert!(list.replace_first(Origin::UNKNOWN, Origin::new(7).unwrap()));
3886 assert_eq!(
3887 list.as_slice(),
3888 &[Origin::new(7).unwrap(), Origin::UNKNOWN, Origin::UNKNOWN]
3889 );
3890
3891 assert!(!list.replace_first(Origin::new(99).unwrap(), Origin::new(8).unwrap()));
3893 assert_eq!(list.len(), 3);
3894 }
3895
3896 #[test]
3897 fn origin_list_try_from_vec_enforces_limit() {
3898 let under: Vec<Origin> = (0..MAX_HOPS).map(|_| Origin::random()).collect();
3899 assert!(OriginList::try_from(under).is_ok());
3900
3901 let over: Vec<Origin> = (0..MAX_HOPS + 1).map(|_| Origin::random()).collect();
3902 assert_eq!(OriginList::try_from(over), Err(TooManyOrigins));
3903 }
3904
3905 #[tokio::test]
3909 async fn test_idle_announce_prunes_its_nodes() {
3910 tokio::time::pause();
3911
3912 let origin = Origin::random().produce();
3913 let consumer = origin.consume();
3914 let start = origin.node_count();
3915
3916 for i in 0..32 {
3917 let path = format!("channel{i}/chat");
3918 let scoped = consumer.scope(&[Path::new(path.as_str())]).expect("in scope");
3919
3920 let mut announced = scoped.announced();
3921 announced.assert_next_wait();
3922 assert_eq!(origin.node_count(), start + 2, "the subtree exists while subscribed");
3923
3924 drop(announced);
3925 drop(scoped);
3926 assert_eq!(origin.node_count(), start, "cycle {i} left a node behind");
3927 }
3928 }
3929
3930 #[tokio::test]
3933 async fn test_prune_spares_live_nodes() {
3934 tokio::time::pause();
3935
3936 let origin = Origin::random().produce();
3937 let consumer = origin.consume();
3938 let bare = origin.node_count();
3939
3940 let mut broadcast = origin.create_broadcast("channel/chat", announce()).unwrap();
3941 settle().await;
3942 let live = origin.node_count();
3943 assert_eq!(live, bare + 2, "the broadcast should have created its subtree");
3944
3945 let mut announced = consumer.announced();
3947 announced.assert_next_some("channel/chat");
3948 drop(announced);
3949 assert_eq!(origin.node_count(), live, "an announced path was pruned");
3950 assert!(consumer.get_broadcast("channel/chat").is_some());
3951
3952 let idle = consumer.scope(&[Path::new("idle")]).expect("in scope");
3954 let first = idle.announced();
3955 let second = idle.announced();
3956 assert_eq!(origin.node_count(), live + 1);
3957 drop(first);
3958 assert_eq!(origin.node_count(), live + 1, "a node with a consumer was pruned");
3959 drop(second);
3960 assert_eq!(origin.node_count(), live, "the last cursor left the node behind");
3961
3962 broadcast.finish();
3964 settle().await;
3965 assert_eq!(origin.node_count(), bare);
3966 }
3967
3968 #[tokio::test]
3972 async fn test_multi_prefix_cursor_prunes_both_branches() {
3973 tokio::time::pause();
3974
3975 let origin = Origin::random().produce();
3976 let consumer = origin.consume();
3977 let bare = origin.node_count();
3978
3979 let scoped = consumer
3980 .scope(&[Path::new("room/a"), Path::new("room/b")])
3981 .expect("in scope");
3982 assert_eq!(origin.node_count(), bare, "scoping should not create nodes");
3983 assert!(consumer.with_root("room/c").is_some());
3984 assert_eq!(origin.node_count(), bare, "rooting should not create nodes");
3985
3986 let announced = scoped.announced();
3987 assert_eq!(origin.node_count(), bare + 3, "room, room/a and room/b");
3988
3989 drop(announced);
3990 assert_eq!(origin.node_count(), bare, "a two-branch cursor left nodes behind");
3991 }
3992
3993 #[tokio::test]
3996 async fn test_prune_stops_at_a_live_ancestor() {
3997 tokio::time::pause();
3998
3999 let origin = Origin::random().produce();
4000 let consumer = origin.consume();
4001 let bare = origin.node_count();
4002
4003 let outer = consumer.scope(&[Path::new("room/a")]).expect("in scope").announced();
4004 let inner = consumer
4005 .scope(&[Path::new("room/a/deep/leaf")])
4006 .expect("in scope")
4007 .announced();
4008 assert_eq!(origin.node_count(), bare + 4, "room, a, deep and leaf");
4009
4010 drop(inner);
4011 assert_eq!(origin.node_count(), bare + 2, "pruning ran past the cursor above it");
4012
4013 drop(outer);
4014 assert_eq!(origin.node_count(), bare);
4015 }
4016
4017 #[tokio::test]
4022 async fn test_pending_source_holds_its_node() {
4023 tokio::time::pause();
4024
4025 let origin = Origin::random().produce();
4026 let consumer = origin.consume();
4027 let bare = origin.node_count();
4028
4029 let waiting = consumer
4031 .scope(&[Path::new("channel/chat")])
4032 .expect("in scope")
4033 .announced();
4034 assert_eq!(origin.node_count(), bare + 2);
4035
4036 let _broadcast = origin.create_broadcast("channel/chat", announce()).unwrap();
4037
4038 drop(waiting);
4040 assert_eq!(origin.node_count(), bare + 2, "a pending source's node was pruned");
4041
4042 settle().await;
4043 assert!(
4044 consumer.get_broadcast("channel/chat").is_some(),
4045 "the source attached into an orphan"
4046 );
4047 }
4048
4049 #[tokio::test]
4053 async fn test_scoped_handles_survive_a_prune() {
4054 tokio::time::pause();
4055
4056 let scope = [Path::new("channel")];
4057 let producer = origin_scoped(&scope);
4058 let consumer = producer.consume().scope(&scope).expect("in scope");
4059
4060 drop(consumer.announced());
4062
4063 let _broadcast = producer.create_broadcast("channel/chat", announce()).unwrap();
4064 settle().await;
4065
4066 let mut announced = consumer.announced();
4067 announced.assert_next_some("channel/chat");
4068 assert!(consumer.get_broadcast("channel/chat").is_some());
4069 }
4070
4071 #[tokio::test]
4072 async fn test_announce() {
4073 tokio::time::pause();
4074
4075 let origin = Origin::random().produce();
4076
4077 let mut consumer1 = origin.consume().announced();
4078 consumer1.assert_next_wait();
4079
4080 let mut broadcast1 = origin.create_broadcast("test1", announce()).unwrap();
4082 settle().await;
4083
4084 consumer1.assert_next_some("test1");
4085 consumer1.assert_next_wait();
4086
4087 let mut consumer2 = origin.consume().announced();
4090
4091 let mut broadcast2 = origin.create_broadcast("test2", announce()).unwrap();
4093 settle().await;
4094
4095 consumer1.assert_next_some("test2");
4096 consumer1.assert_next_wait();
4097
4098 consumer2.assert_next_some("test1");
4099 consumer2.assert_next_some("test2");
4100 consumer2.assert_next_wait();
4101
4102 broadcast1.finish();
4104 settle().await;
4105
4106 consumer1.assert_next_none("test1");
4108 consumer2.assert_next_none("test1");
4109 consumer1.assert_next_wait();
4110 consumer2.assert_next_wait();
4111
4112 let mut consumer3 = origin.consume().announced();
4114 consumer3.assert_next_some("test2");
4115 consumer3.assert_next_wait();
4116
4117 broadcast2.finish();
4118 settle().await;
4119
4120 consumer1.assert_next_none("test2");
4121 consumer2.assert_next_none("test2");
4122 consumer3.assert_next_none("test2");
4123 }
4124
4125 #[tokio::test]
4129 async fn test_duplicate() {
4130 tokio::time::pause();
4131
4132 let origin = Origin::random().produce();
4133 let consumer = origin.consume();
4134 let mut announced = consumer.announced();
4135
4136 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
4137 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
4138 let mut broadcast3 = origin.create_broadcast("test", announce()).unwrap();
4139 settle().await;
4140 assert!(consumer.get_broadcast("test").is_some());
4141
4142 announced.assert_next_some("test");
4143 announced.assert_next_wait();
4144
4145 broadcast2.finish();
4147 settle().await;
4148 assert!(consumer.get_broadcast("test").is_some());
4149 announced.assert_next_wait();
4150
4151 broadcast1.finish();
4153 settle().await;
4154 assert!(consumer.get_broadcast("test").is_some());
4155 announced.assert_next_wait();
4156
4157 broadcast3.finish();
4159 settle().await;
4160 assert!(consumer.get_broadcast("test").is_none());
4161
4162 announced.assert_next_none("test");
4163 announced.assert_next_wait();
4164 }
4165
4166 #[tokio::test]
4169 async fn test_route_failover() {
4170 tokio::time::pause();
4171
4172 let origin = Origin::random().produce();
4173 let consumer = origin.consume();
4174 let mut announced = consumer.announced();
4175
4176 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4179 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4180
4181 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4183 let mut dynamic_a = source_a.dynamic();
4184 settle().await;
4185 settle().await;
4186 let broadcast = consumer.request_broadcast("test").await.unwrap();
4187 announced.assert_next_some("test");
4188
4189 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4191 let mut dynamic_b = source_b.dynamic();
4192 settle().await;
4193 settle().await;
4194 announced.assert_next_wait();
4195
4196 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4198 let mut producer = accept_track(&mut dynamic_a, "video").await;
4199 settle().await;
4200 dynamic_b.assert_no_request();
4201
4202 let mut sub = subscribing.await.unwrap();
4203 sub.assert_no_group();
4206 assert_eq!(producer.subscription().unwrap().group_start, None);
4207
4208 producer.append_group().unwrap();
4209 producer.append_group().unwrap();
4210 assert_eq!(sub.assert_group().sequence, 0);
4211 assert_eq!(sub.assert_group().sequence, 1);
4212
4213 producer.abort(Error::Dropped).unwrap();
4217 source_a.abort(Error::Dropped).unwrap();
4218 drop(dynamic_a);
4219 settle().await;
4220 announced.assert_next_wait();
4221
4222 let mut producer = accept_track(&mut dynamic_b, "video").await;
4226 settle().await;
4227 sub.assert_no_group();
4228 assert_eq!(producer.subscription().unwrap().group_start, None);
4229 producer.create_group(group::Info { sequence: 1 }).unwrap();
4230 producer.create_group(group::Info { sequence: 2 }).unwrap();
4231 assert_eq!(sub.assert_group().sequence, 2, "groups below the boundary are filtered");
4232 sub.assert_not_closed();
4233 }
4234
4235 #[tokio::test]
4237 async fn test_failover_rejects_different_track_info() {
4238 tokio::time::pause();
4239
4240 for replacement in [
4241 track::Info::default().with_timescale(Timescale::MICRO),
4242 track::Info::default().with_priority(7),
4243 track::Info::default().with_ordered(true),
4244 track::Info::default().with_latency_max(Duration::from_secs(7)),
4245 ] {
4246 let origin = Origin::random().produce();
4247 let consumer = origin.consume();
4248
4249 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4251 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4252
4253 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4254 let mut dynamic_a = source_a.dynamic();
4255 settle().await;
4256 settle().await;
4257 let broadcast = consumer.request_broadcast("test").await.unwrap();
4258
4259 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4260 let mut dynamic_b = source_b.dynamic();
4261 settle().await;
4262 settle().await;
4263
4264 let track = broadcast.track("video").unwrap();
4267
4268 let querying = track.info();
4270 let mut producer = accept_track(&mut dynamic_a, "video").await;
4271 let info = tokio::time::timeout(std::time::Duration::from_secs(1), querying)
4272 .await
4273 .expect("timed out resolving the first source's info")
4274 .unwrap();
4275 assert_eq!(info.timescale, Timescale::default());
4276
4277 let mut reader = track.subscribe(None).await.unwrap();
4278 producer
4279 .create_group(group::Info { sequence: 0 })
4280 .unwrap()
4281 .finish()
4282 .unwrap();
4283 assert_eq!(reader.recv_group().await.unwrap().unwrap().sequence, 0);
4284
4285 let mut joining = track.subscribe(None).await.unwrap();
4287
4288 source_a.abort(Error::Dropped).unwrap();
4290 drop(dynamic_a);
4291 settle().await;
4292
4293 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_b.requested_track())
4294 .await
4295 .expect("timed out waiting for a track request")
4296 .expect("source closed");
4297 let mut successor = request.accept(replacement);
4298 successor
4299 .create_group(group::Info { sequence: 1 })
4300 .unwrap()
4301 .finish()
4302 .unwrap();
4303 settle().await;
4304
4305 assert!(matches!(track.info().await, Err(Error::Unsupported)));
4306 let cached = joining.recv_group().await.unwrap().unwrap();
4307 assert_eq!(cached.sequence, 0);
4308 assert_eq!(cached.timescale(), info.timescale);
4309 assert!(matches!(joining.recv_group().await, Err(Error::Unsupported)));
4310 assert!(matches!(track.subscribe(None).await, Err(Error::Unsupported)));
4311 assert!(matches!(reader.recv_group().await, Err(Error::Unsupported)));
4312 assert!(matches!(track.fetch_group(1, None).await, Err(Error::Unsupported)));
4313
4314 let reopened = broadcast.track("video").unwrap();
4316 assert!(matches!(reopened.info().await, Err(Error::Unsupported)));
4317 }
4318 }
4319
4320 #[tokio::test]
4327 async fn test_route_failover_restores_every_track() {
4328 tokio::time::pause();
4329
4330 let origin = Origin::random().produce();
4331 let consumer = origin.consume();
4332
4333 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4335 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4336
4337 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4338 let mut dynamic_a = source_a.dynamic();
4339 settle().await;
4340 settle().await;
4341 let broadcast = consumer.request_broadcast("test").await.unwrap();
4342
4343 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4345 let mut dynamic_b = source_b.dynamic();
4346 settle().await;
4347 settle().await;
4348
4349 const TRACKS: [(&str, u64); 3] = [("video", 3), ("audio", 1), ("data", 2)];
4353
4354 let subscribing: Vec<_> = TRACKS
4355 .iter()
4356 .map(|(name, _)| broadcast.track(name).unwrap().subscribe(None))
4357 .collect();
4358 let mut producers_a = accept_tracks(&mut dynamic_a, TRACKS.len()).await;
4359 settle().await;
4360
4361 let mut subs = Vec::new();
4362 for ((name, groups), subscribing) in TRACKS.iter().zip(subscribing) {
4363 let mut sub = subscribing.await.unwrap();
4364 let producer = producers_a
4365 .get_mut(*name)
4366 .unwrap_or_else(|| panic!("{name} was never dispatched"));
4367 for expected in 0..*groups {
4368 producer.append_group().unwrap();
4369 assert_eq!(sub.assert_group().sequence, expected, "{name} did not start");
4370 }
4371 subs.push((*name, *groups, sub));
4372 }
4373
4374 for (_, producer) in producers_a.drain() {
4376 producer.abort(Error::Dropped).unwrap();
4377 }
4378 source_a.abort(Error::Dropped).unwrap();
4379 drop(dynamic_a);
4380 settle().await;
4381
4382 let mut producers_b = accept_tracks(&mut dynamic_b, TRACKS.len()).await;
4385 settle().await;
4386
4387 for (_, _, sub) in subs.iter_mut() {
4390 sub.assert_no_group();
4391 }
4392 settle().await;
4393
4394 for (name, groups, sub) in subs.iter_mut() {
4395 let producer = producers_b
4396 .get_mut(*name)
4397 .unwrap_or_else(|| panic!("{name} was never re-dispatched to the standby"));
4398 assert_eq!(
4401 producer
4402 .subscription()
4403 .unwrap_or_else(|| panic!("{name} resumed without a subscription"))
4404 .group_start,
4405 None,
4406 "{name} must keep the subscriber's live-edge demand"
4407 );
4408 let boundary = *groups;
4409
4410 producer.create_group(group::Info { sequence: boundary - 1 }).unwrap();
4412 producer.create_group(group::Info { sequence: boundary }).unwrap();
4413 assert_eq!(sub.assert_group().sequence, boundary, "{name} did not resume");
4414 sub.assert_not_closed();
4415 }
4416 }
4417
4418 #[tokio::test]
4421 async fn test_broadcast_route_watch() {
4422 let mut producer = broadcast::Info::new().produce();
4423 let mut consumer = producer.consume();
4424
4425 assert_eq!(consumer.route_changed().await.unwrap(), broadcast::Route::default());
4427
4428 producer.set_route(broadcast::Route::default()).unwrap();
4430 assert!(consumer.route_changed().now_or_never().is_none());
4431
4432 let mut hops = OriginList::new();
4433 hops.push(Origin::new(7).unwrap()).unwrap();
4434 let route = broadcast::Route::new().with_hops(hops).with_cost(3);
4435 producer.set_route(route.clone()).unwrap();
4436 assert_eq!(consumer.route_changed().await.unwrap(), route);
4437
4438 let mut fresh = producer.consume();
4440 assert_eq!(fresh.route_changed().await.unwrap(), route);
4441
4442 drop(producer);
4443 assert!(matches!(consumer.route_changed().await.unwrap_err(), Error::Dropped));
4444 }
4445
4446 #[tokio::test]
4450 async fn test_route_cost_update() {
4451 tokio::time::pause();
4452
4453 let origin = Info::new(origin_keyed("test", Origin::new(3).unwrap(), true)).produce();
4457 let consumer = origin.consume();
4458 let mut announced = consumer.announced();
4459
4460 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4463 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4464
4465 let mut source_a = origin
4467 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4468 .unwrap();
4469 let mut dynamic_a = source_a.dynamic();
4470 settle().await;
4471 let broadcast = consumer.request_broadcast("test").await.unwrap();
4472 announced.assert_next_some("test");
4473
4474 let mut watch = broadcast.clone();
4475 assert_eq!(watch.route_changed().await.unwrap().hops, hops_a);
4476
4477 let mut source_b = origin
4478 .create_broadcast("test", announce().with_hops(hops_b.clone()))
4479 .unwrap();
4480 let mut dynamic_b = source_b.dynamic();
4481 settle().await;
4482 assert!(
4483 watch.route_changed().now_or_never().is_none(),
4484 "a losing standby must not change the advertised route"
4485 );
4486
4487 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4489 let mut producer = accept_track(&mut dynamic_a, "video").await;
4490 settle().await;
4491 let mut sub = subscribing.await.unwrap();
4492 producer.append_group().unwrap();
4493 assert_eq!(sub.assert_group().sequence, 0);
4494
4495 source_a
4498 .set_route(announce().with_hops(hops_a.clone()).with_cost(10))
4499 .unwrap();
4500 settle().await;
4501 assert_eq!(watch.route_changed().await.unwrap().hops, hops_b);
4502 announced.assert_next_wait();
4503
4504 let mut producer_b = accept_track(&mut dynamic_b, "video").await;
4505 settle().await;
4506 sub.assert_no_group();
4509 assert_eq!(producer_b.subscription().unwrap().group_start, None);
4510 producer_b.create_group(group::Info { sequence: 1 }).unwrap();
4511 assert_eq!(sub.assert_group().sequence, 1);
4512 sub.assert_not_closed();
4513
4514 source_b
4516 .set_route(announce().with_hops(hops_b.clone()).with_cost(5))
4517 .unwrap();
4518 settle().await;
4519 let advertised = watch.route_changed().await.unwrap();
4520 assert_eq!(advertised.hops, hops_b);
4521 assert_eq!(advertised.cost, 5);
4522 announced.assert_next_wait();
4523 }
4524
4525 #[tokio::test]
4528 async fn test_completed_track_survives_route_churn() {
4529 tokio::time::pause();
4530
4531 let origin = Origin::random().produce();
4532 let consumer = origin.consume();
4533
4534 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4536 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4537
4538 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4539 let mut dynamic_a = source_a.dynamic();
4540 settle().await;
4541 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4542 let mut dynamic_b = source_b.dynamic();
4543 settle().await;
4544 settle().await;
4545 let broadcast = consumer.request_broadcast("test").await.unwrap();
4546
4547 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4549 let mut producer = accept_track(&mut dynamic_a, "video").await;
4550 settle().await;
4551 let mut sub = subscribing.await.unwrap();
4552 producer.append_group().unwrap();
4553 assert_eq!(sub.assert_group().sequence, 0);
4554 producer.finish().unwrap();
4555 drop(producer);
4556 settle().await;
4557 sub.assert_closed();
4558
4559 source_a.abort(Error::Dropped).unwrap();
4561 drop(dynamic_a);
4562 settle().await;
4563 dynamic_b.assert_no_request();
4564
4565 let mut late = broadcast.track("video").unwrap().subscribe(None).await.unwrap();
4567 late.assert_closed();
4568 }
4569
4570 #[tokio::test]
4574 async fn test_refused_track_aborts_instantly() {
4575 tokio::time::pause();
4576
4577 let origin = Origin::random().produce();
4578 let consumer = origin.consume();
4579
4580 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4581 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4582 let mut dynamic = source.dynamic();
4583 settle().await;
4584 settle().await;
4585 let broadcast = consumer.request_broadcast("test").await.unwrap();
4586
4587 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4588 let request = dynamic.requested_track().await.unwrap();
4589 request.reject(Error::NotFound);
4590 settle().await;
4591
4592 assert!(matches!(subscribing.await, Err(Error::NotFound)));
4594 dynamic.assert_no_request();
4595 }
4596
4597 #[tokio::test]
4602 async fn test_stale_rejection_does_not_abort_a_handover() {
4603 tokio::time::pause();
4604
4605 let origin = Origin::random().produce();
4606 let consumer = origin.consume();
4607
4608 let publisher = Origin::new(1).unwrap();
4609 let peer = Origin::new(5).unwrap();
4610 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
4611 let local = OriginList::try_from(vec![publisher]).unwrap();
4612
4613 let source_remote = origin
4615 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
4616 .unwrap();
4617 let mut dynamic_remote = source_remote.dynamic();
4618 settle().await;
4619 settle().await;
4620 let broadcast = consumer.request_broadcast("test").await.unwrap();
4621 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4622 let request_remote = dynamic_remote.requested_track().await.unwrap();
4623
4624 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
4627 let mut dynamic_local = source_local.dynamic();
4628 request_remote.reject(Error::NotFound);
4629 settle().await;
4630
4631 let mut producer_local = accept_track(&mut dynamic_local, "video").await;
4633 settle().await;
4634 let mut sub = subscribing
4635 .await
4636 .expect("the handover must win over the stale rejection");
4637 producer_local.append_group().unwrap();
4638 assert_eq!(sub.assert_group().sequence, 0);
4639 sub.assert_not_closed();
4640 }
4641
4642 #[tokio::test]
4646 async fn test_route_handover() {
4647 tokio::time::pause();
4648
4649 let origin = Origin::random().produce();
4650 let consumer = origin.consume();
4651 let mut announced = consumer.announced();
4652
4653 let hops_long = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4655 let hops_short = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4656
4657 let source_a = origin
4658 .create_broadcast("test", announce().with_hops(hops_long))
4659 .unwrap();
4660 let mut dynamic_a = source_a.dynamic();
4661 settle().await;
4662 settle().await;
4663 let broadcast = consumer.request_broadcast("test").await.unwrap();
4664 announced.assert_next_some("test");
4665
4666 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4667 let mut producer_a = accept_track(&mut dynamic_a, "video").await;
4668 settle().await;
4669 let mut sub = subscribing.await.unwrap();
4670 producer_a.append_group().unwrap();
4671 producer_a.append_group().unwrap();
4672 assert_eq!(sub.assert_group().sequence, 0);
4673 assert_eq!(sub.assert_group().sequence, 1);
4674
4675 let source_b = origin
4678 .create_broadcast("test", announce().with_hops(hops_short))
4679 .unwrap();
4680 let mut dynamic_b = source_b.dynamic();
4681 settle().await;
4682 settle().await;
4683 announced.assert_next_wait();
4684
4685 let mut producer_b = accept_track(&mut dynamic_b, "video").await;
4686 settle().await;
4687
4688 sub.assert_no_group();
4692 assert_eq!(producer_a.subscription().unwrap().group_end, Some(1));
4693 assert_eq!(producer_b.subscription().unwrap().group_start, None);
4694
4695 producer_a.create_group(group::Info { sequence: 2 }).unwrap();
4697 producer_b.create_group(group::Info { sequence: 2 }).unwrap();
4698 producer_b.create_group(group::Info { sequence: 3 }).unwrap();
4699 assert_eq!(sub.assert_group().sequence, 2);
4700 assert_eq!(sub.assert_group().sequence, 3);
4701 sub.assert_no_group();
4702 sub.assert_not_closed();
4703 }
4704
4705 #[tokio::test(start_paused = true)]
4708 async fn test_route_unannounce_immediate() {
4709 let origin = Origin::random().produce();
4710 let consumer = origin.consume();
4711 let mut announced = consumer.announced();
4712
4713 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4714 let mut source = origin
4715 .create_broadcast("test", announce().with_hops(hops.clone()))
4716 .unwrap();
4717 settle().await;
4718 let broadcast = consumer.request_broadcast("test").await.unwrap();
4719 announced.assert_next_some("test");
4720
4721 source.finish();
4724 settle().await;
4725 announced.assert_next_none("test");
4726
4727 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4729 settle().await;
4730 let fresh = consumer.request_broadcast("test").await.unwrap();
4731 announced.assert_next_some("test");
4732 assert!(
4733 !fresh.is_clone(&broadcast),
4734 "re-create must not splice the old broadcast"
4735 );
4736 }
4737
4738 #[tokio::test(start_paused = true)]
4743 async fn test_route_detach_immediate() {
4744 let origin = Origin::random().produce();
4745 let consumer = origin.consume();
4746 let mut announced = consumer.announced();
4747
4748 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4749 let source = origin
4750 .create_broadcast("test", announce().with_hops(hops.clone()))
4751 .unwrap();
4752 let mut dynamic = source.dynamic();
4753 settle().await;
4754 settle().await;
4755 let broadcast = consumer.request_broadcast("test").await.unwrap();
4756 announced.assert_next_some("test");
4757
4758 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4759 let producer = accept_track(&mut dynamic, "video").await;
4760 settle().await;
4761 let mut sub = subscribing.await.unwrap();
4762
4763 drop(producer);
4765 source.abort(Error::Dropped).unwrap();
4766 drop(dynamic);
4767
4768 settle().await;
4769 announced.assert_next_none("test");
4770 sub.assert_error();
4771
4772 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4775 settle().await;
4776 settle().await;
4777 let fresh = consumer.request_broadcast("test").await.unwrap();
4778 announced.assert_next_some("test");
4779 assert!(
4780 !fresh.is_clone(&broadcast),
4781 "re-create must not splice the old broadcast"
4782 );
4783 }
4784
4785 #[tokio::test(start_paused = true)]
4790 async fn test_idle_track_releases_without_respinning() {
4791 for incompatible in [false, true] {
4792 let origin = Info::new(Origin::random()).produce();
4793 let consumer = origin.consume();
4794
4795 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4796 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4797 let mut dynamic = source.dynamic();
4798 settle().await;
4799 let broadcast = consumer.request_broadcast("test").await.unwrap();
4800
4801 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4802 let producer = accept_track(&mut dynamic, "video").await;
4803 settle().await;
4804 let sub = subscribing.await.unwrap();
4805
4806 drop(sub);
4809 tokio::time::sleep(TRACK_IDLE_LINGER / 2).await;
4810 settle().await;
4811 assert!(
4812 producer.poll_unused(&kio::Waiter::noop()).is_pending(),
4813 "the copy must stay spliced inside the linger",
4814 );
4815
4816 tokio::time::sleep(TRACK_IDLE_LINGER).await;
4819 settle().await;
4820 assert!(
4821 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4822 "an idle copy must be released after the linger",
4823 );
4824
4825 for _ in 0..3 {
4829 tokio::time::sleep(TRACK_IDLE_LINGER).await;
4830 settle().await;
4831 assert!(
4832 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4833 "an unread copy must stay released, not be re-spliced",
4834 );
4835 }
4836 assert!(
4837 dynamic.requested_track().now_or_never().is_none(),
4838 "an unread track must not be re-requested",
4839 );
4840 drop(producer);
4841
4842 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4844 let request = dynamic.requested_track().await.unwrap();
4845 let info = track::Info::default().with_timescale(if incompatible {
4846 Timescale::MICRO
4847 } else {
4848 Timescale::MILLI
4849 });
4850 let mut producer = request.accept(info);
4851 if incompatible {
4852 assert!(matches!(subscribing.await, Err(Error::Unsupported)));
4853 continue;
4854 }
4855
4856 settle().await;
4857 let mut sub = subscribing.await.unwrap();
4858 producer.append_group().unwrap();
4859 assert_eq!(sub.assert_group().sequence, 0);
4860 }
4861 }
4862
4863 #[tokio::test(start_paused = true)]
4867 async fn test_back_to_back_fetches_reuse_the_track() {
4868 let origin = Info::new(Origin::random()).produce();
4869 let consumer = origin.consume();
4870
4871 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4872 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4873 let mut dynamic = source.dynamic();
4874 settle().await;
4875 let broadcast = consumer.request_broadcast("test").await.unwrap();
4876
4877 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4879 let mut producer = accept_track(&mut dynamic, "video").await;
4880 producer.append_group().unwrap().finish().unwrap();
4881 settle().await;
4882 let first = fetching.await.expect("first fetch");
4883 drop(first);
4884
4885 settle().await;
4887 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4888 settle().await;
4889 assert!(
4890 dynamic.requested_track().now_or_never().is_none(),
4891 "a fetch inside the linger must reuse the track, not re-request it",
4892 );
4893 drop(fetching.await.expect("second fetch"));
4894
4895 tokio::time::sleep(TRACK_IDLE_LINGER * 2).await;
4897 settle().await;
4898 assert!(
4899 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4900 "the copy must be released once the fetches stop",
4901 );
4902 drop(producer);
4903
4904 settle().await;
4906 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4907 let mut producer = accept_track(&mut dynamic, "video").await;
4908 producer.append_group().unwrap().finish().unwrap();
4909 settle().await;
4910 fetching.await.expect("fetch after the linger");
4911 }
4912
4913 #[tokio::test(start_paused = true)]
4917 async fn test_linger_reconnect_splices() {
4918 let origin = Info::new(Origin::random())
4919 .with_linger(Duration::from_secs(5))
4920 .produce();
4921 let consumer = origin.consume();
4922 let mut announced = consumer.announced();
4923
4924 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4925 let source = origin
4926 .create_broadcast("test", announce().with_hops(hops.clone()))
4927 .unwrap();
4928 let mut dynamic = source.dynamic();
4929 settle().await;
4930 settle().await;
4931 let broadcast = consumer.request_broadcast("test").await.unwrap();
4932 announced.assert_next_some("test");
4933
4934 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4935 let mut producer = accept_track(&mut dynamic, "video").await;
4936 settle().await;
4937 let mut sub = subscribing.await.unwrap();
4938
4939 producer.append_group().unwrap();
4940 producer.append_group().unwrap();
4941 assert_eq!(sub.assert_group().sequence, 0);
4942 assert_eq!(sub.assert_group().sequence, 1);
4943
4944 drop(producer);
4947 source.abort(Error::Dropped).unwrap();
4948 drop(dynamic);
4949 settle().await;
4950
4951 announced.assert_next_wait();
4953 sub.assert_no_group();
4954 sub.assert_not_closed();
4955
4956 let during = consumer.request_broadcast("test").await.unwrap();
4958 assert!(during.is_clone(&broadcast), "the lingering broadcast still resolves");
4959
4960 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4963 let mut dynamic = source.dynamic();
4964 settle().await;
4965 settle().await;
4966 announced.assert_next_wait();
4967 let again = consumer.request_broadcast("test").await.unwrap();
4968 assert!(again.is_clone(&broadcast), "the reconnect must splice, not replace");
4969
4970 let mut producer = accept_track(&mut dynamic, "video").await;
4974 settle().await;
4975 sub.assert_no_group();
4976 assert_eq!(producer.subscription().unwrap().group_start, None);
4977 producer.create_group(group::Info { sequence: 2 }).unwrap();
4978 assert_eq!(sub.assert_group().sequence, 2);
4979 sub.assert_not_closed();
4980 }
4981
4982 #[tokio::test(start_paused = true)]
4985 async fn test_linger_expiry_closes() {
4986 let origin = Info::new(Origin::random())
4987 .with_linger(Duration::from_secs(5))
4988 .produce();
4989 let consumer = origin.consume();
4990 let mut announced = consumer.announced();
4991
4992 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4993 let source = origin
4994 .create_broadcast("test", announce().with_hops(hops.clone()))
4995 .unwrap();
4996 let mut dynamic = source.dynamic();
4997 settle().await;
4998 settle().await;
4999 let broadcast = consumer.request_broadcast("test").await.unwrap();
5000 announced.assert_next_some("test");
5001
5002 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5003 let producer = accept_track(&mut dynamic, "video").await;
5004 settle().await;
5005 let mut sub = subscribing.await.unwrap();
5006
5007 drop(producer);
5008 source.abort(Error::Dropped).unwrap();
5009 drop(dynamic);
5010 settle().await;
5011 announced.assert_next_wait();
5012
5013 tokio::time::sleep(std::time::Duration::from_secs(6)).await;
5015 settle().await;
5016 announced.assert_next_none("test");
5017 sub.assert_error();
5018
5019 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5021 settle().await;
5022 settle().await;
5023 let fresh = consumer.request_broadcast("test").await.unwrap();
5024 announced.assert_next_some("test");
5025 assert!(
5026 !fresh.is_clone(&broadcast),
5027 "a late re-create must not splice the expired broadcast"
5028 );
5029 }
5030
5031 #[tokio::test(start_paused = true)]
5035 async fn test_linger_forever() {
5036 let origin = Info::new(Origin::random()).with_linger(Duration::MAX).produce();
5037 let consumer = origin.consume();
5038 let mut announced = consumer.announced();
5039
5040 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5041 let source = origin
5042 .create_broadcast("test", announce().with_hops(hops.clone()))
5043 .unwrap();
5044 settle().await;
5045 let broadcast = consumer.request_broadcast("test").await.unwrap();
5046 announced.assert_next_some("test");
5047
5048 source.abort(Error::Dropped).unwrap();
5049 settle().await;
5050
5051 tokio::time::sleep(std::time::Duration::from_secs(60 * 60 * 24 * 3)).await;
5053 announced.assert_next_wait();
5054 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5055 settle().await;
5056 settle().await;
5057 let again = consumer.request_broadcast("test").await.unwrap();
5058 assert!(again.is_clone(&broadcast), "the reconnect must splice, not replace");
5059 drop(source);
5060 }
5061
5062 #[tokio::test(start_paused = true)]
5076 async fn test_linger_parks_a_live_subscription() {
5077 let origin = Info::new(Origin::random())
5078 .with_linger(Duration::from_secs(5))
5079 .produce();
5080 let consumer = origin.consume();
5081
5082 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5083 let source = origin
5084 .create_broadcast("test", announce().with_hops(hops.clone()))
5085 .unwrap();
5086 let mut dynamic = source.dynamic();
5087 settle().await;
5088 settle().await;
5089 let broadcast = consumer.request_broadcast("test").await.unwrap();
5090
5091 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
5094 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
5095 settle().await;
5096 let mut sub = subscribing.await.unwrap();
5097 producer.append_group().unwrap();
5098 assert_eq!(sub.assert_group().sequence, 0);
5099
5100 source.abort(Error::Dropped).unwrap();
5105 settle().await;
5106 settle().await;
5107 sub.assert_not_closed();
5108
5109 tokio::time::sleep(Duration::from_secs(4)).await;
5112 settle().await;
5113 sub.assert_not_closed();
5114
5115 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5117 let mut dynamic = source.dynamic();
5118 settle().await;
5119 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
5120 settle().await;
5121 producer.create_group(group::Info { sequence: 1 }).unwrap();
5122 assert_eq!(sub.assert_group().sequence, 1);
5123 sub.assert_not_closed();
5124 }
5125
5126 #[tokio::test(start_paused = true)]
5137 async fn test_idle_release_survives_the_route_leaving() {
5138 let origin = Info::new(Origin::random())
5139 .with_linger(Duration::from_secs(600))
5140 .produce();
5141 let consumer = origin.consume();
5142
5143 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5144 let source = origin
5145 .create_broadcast("test", announce().with_hops(hops.clone()))
5146 .unwrap();
5147 let mut dynamic = source.dynamic();
5148 settle().await;
5149 settle().await;
5150 let broadcast = consumer.request_broadcast("test").await.unwrap();
5151
5152 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
5153 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
5154 settle().await;
5155 let mut sub = subscribing.await.unwrap();
5156 producer.append_group().unwrap();
5157 assert_eq!(sub.assert_group().sequence, 0);
5158
5159 source.abort(Error::Dropped).unwrap();
5162 settle().await;
5163 settle().await;
5164 drop(sub);
5165 drop(producer);
5166 drop(dynamic);
5167 settle().await;
5168 tokio::time::sleep(TRACK_IDLE_LINGER + Duration::from_secs(1)).await;
5169 settle().await;
5170
5171 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5174 let mut dynamic = source.dynamic();
5175 settle().await;
5176 settle().await;
5177 let broadcast = consumer.request_broadcast("test").await.unwrap();
5178 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
5179 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
5180 settle().await;
5181 let mut sub = subscribing.await.unwrap();
5182 producer.append_group().unwrap();
5183 assert_eq!(
5184 sub.assert_group().sequence,
5185 0,
5186 "the reconnect's first group must not be filtered by a stale boundary"
5187 );
5188 }
5189
5190 #[tokio::test(start_paused = true)]
5193 async fn test_linger_skipped_on_finish() {
5194 let origin = Info::new(Origin::random())
5195 .with_linger(Duration::from_secs(5))
5196 .produce();
5197 let consumer = origin.consume();
5198 let mut announced = consumer.announced();
5199
5200 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5201 let mut source = origin
5202 .create_broadcast("test", announce().with_hops(hops.clone()))
5203 .unwrap();
5204 settle().await;
5205 let broadcast = consumer.request_broadcast("test").await.unwrap();
5206 announced.assert_next_some("test");
5207
5208 source.finish();
5211 settle().await;
5212 announced.assert_next_none("test");
5213
5214 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5216 settle().await;
5217 let fresh = consumer.request_broadcast("test").await.unwrap();
5218 announced.assert_next_some("test");
5219 assert!(
5220 !fresh.is_clone(&broadcast),
5221 "a finish must not leave a lingering broadcast to splice into"
5222 );
5223 }
5224
5225 #[tokio::test]
5228 async fn test_announce_toggle() {
5229 tokio::time::pause();
5230
5231 let origin = Origin::random().produce();
5232 let consumer = origin.consume();
5233 let mut announced = consumer.announced();
5234
5235 let mut source = origin.create_broadcast("test", broadcast::Route::new()).unwrap();
5236 settle().await;
5237
5238 announced.assert_next_wait();
5240 let broadcast = consumer
5241 .get_broadcast("test")
5242 .expect("offline broadcast is still routable");
5243 assert!(!broadcast.route().announce);
5244
5245 let requested = consumer.request_broadcast("test").await.unwrap();
5247 assert!(requested.is_clone(&broadcast));
5248
5249 source.set_route(announce()).unwrap();
5251 settle().await;
5252 let face = announced.assert_next_some("test");
5253 assert!(face.is_clone(&broadcast));
5254
5255 let mut fresh = origin.consume().announced();
5257 fresh.assert_next_some("test");
5258 fresh.assert_next_wait();
5259
5260 source.set_route(broadcast::Route::new()).unwrap();
5262 settle().await;
5263 announced.assert_next_none("test");
5264 assert!(consumer.get_broadcast("test").is_some());
5265 let mut fresh = origin.consume().announced();
5266 fresh.assert_next_wait();
5267
5268 source.finish();
5269 settle().await;
5270 assert!(consumer.get_broadcast("test").is_none());
5271 }
5272
5273 #[tokio::test]
5276 async fn test_announce_beats_offline() {
5277 tokio::time::pause();
5278
5279 let origin = Origin::random().produce();
5280 let consumer = origin.consume();
5281 let mut announced = consumer.announced();
5282
5283 let _offline = origin.create_broadcast("test", broadcast::Route::new()).unwrap();
5285 settle().await;
5286 announced.assert_next_wait();
5287
5288 let mut announced_source = origin.create_broadcast("test", announce().with_cost(10)).unwrap();
5291 settle().await;
5292 announced.assert_next_some("test");
5293 let face = consumer.get_broadcast("test").unwrap();
5294 assert!(face.route().announce);
5295 assert_eq!(face.route().cost, 10);
5296
5297 announced_source.finish();
5300 settle().await;
5301 announced.assert_next_none("test");
5302 assert!(consumer.get_broadcast("test").is_some());
5303 }
5304
5305 #[tokio::test]
5308 async fn test_better_source_no_churn() {
5309 tokio::time::pause();
5310
5311 let origin = Origin::random().produce();
5312 let mut announced = origin.consume().announced();
5313
5314 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
5317 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5318 let _a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
5319 settle().await;
5320 let face = announced.assert_next_some("test");
5321
5322 let _b = origin
5323 .create_broadcast("test", announce().with_hops(hops_b.clone()))
5324 .unwrap();
5325 settle().await;
5326 announced.assert_next_wait();
5327 let current = origin.consume().get_broadcast("test").unwrap();
5328 assert!(current.is_clone(&face), "the broadcast identity must not change");
5329 assert_eq!(current.route().hops, hops_b);
5331 }
5332
5333 #[tokio::test]
5339 async fn test_publisher_mismatch_replaces() {
5340 tokio::time::pause();
5341
5342 let origin = Origin::random().produce();
5343 let consumer = origin.consume();
5344 let mut announced = consumer.announced();
5345
5346 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5347 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5348
5349 let mut source_a = origin
5350 .create_broadcast("test", announce().with_hops(hops_a.clone()))
5351 .unwrap();
5352 settle().await;
5353 let face_a = announced.assert_next_some("test");
5354
5355 let _source_b = origin
5358 .create_broadcast("test", announce().with_hops(hops_b.clone()))
5359 .unwrap();
5360 settle().await;
5361 settle().await;
5362 announced.assert_next_none("test");
5363 let face_b = announced.assert_next_some("test");
5364 assert!(!face_b.is_clone(&face_a), "a replacement, never a splice");
5365 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_b);
5366 assert!(face_a.is_closed(), "the displaced front must close");
5370
5371 source_a.finish();
5373 settle().await;
5374 settle().await;
5375 announced.assert_next_wait();
5376 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_b);
5377 }
5378
5379 #[tokio::test]
5382 async fn test_displaced_publisher_reclaims_path_when_replacement_leaves() {
5383 tokio::time::pause();
5384
5385 let origin = Origin::random().produce();
5386 let consumer = origin.consume();
5387 let mut announced = consumer.announced();
5388
5389 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5390 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5391
5392 let mut source_a = origin
5394 .create_broadcast("test", announce().with_hops(hops_a.clone()))
5395 .unwrap();
5396 settle().await;
5397 announced.assert_next_some("test");
5398
5399 let mut source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
5402 settle().await;
5403 settle().await;
5404 announced.assert_next_none("test");
5405 announced.assert_next_some("test");
5406 announced.assert_next_wait();
5409
5410 source_b.finish();
5412 settle().await;
5413 settle().await;
5414
5415 assert!(
5416 consumer.get_broadcast("test").is_some(),
5417 "the still-live publisher A should reclaim the path once its replacement leaves"
5418 );
5419 let recovered = consumer.request_broadcast("test").await.unwrap();
5420 assert_eq!(recovered.route().hops, hops_a);
5421
5422 announced.assert_next_none("test");
5423 announced.assert_next_some("test");
5424 announced.assert_next_wait();
5425
5426 source_a.finish();
5427 }
5428
5429 #[tokio::test]
5433 async fn test_reclaimed_publisher_can_still_replace_itself() {
5434 tokio::time::pause();
5435
5436 let origin = Origin::random().produce();
5437 let consumer = origin.consume();
5438
5439 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5440 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5441 let hops_c = OriginList::try_from(vec![Origin::new(3).unwrap()]).unwrap();
5442
5443 let mut source_a1 = origin
5445 .create_broadcast("test", announce().with_hops(hops_a.clone()))
5446 .unwrap();
5447 let mut source_a2 = origin
5448 .create_broadcast("test", announce().with_hops(hops_a.clone()))
5449 .unwrap();
5450 settle().await;
5451 settle().await;
5452 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_a);
5453
5454 let mut source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
5456 settle().await;
5457 settle().await;
5458
5459 source_b.finish();
5461 settle().await;
5462 settle().await;
5463 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_a);
5464
5465 source_a1.set_route(announce().with_hops(hops_c.clone())).unwrap();
5469 settle().await;
5470 settle().await;
5471 assert_eq!(
5472 consumer.get_broadcast("test").unwrap().route().hops,
5473 hops_c,
5474 "the new publisher must take the path over, not stand by behind the old front"
5475 );
5476
5477 source_a1.finish();
5478 source_a2.finish();
5479 }
5480
5481 #[tokio::test]
5484 async fn test_repricing_does_not_earn_a_takeover() {
5485 tokio::time::pause();
5486
5487 let origin = Origin::random().produce();
5488 let consumer = origin.consume();
5489 let mut announced = consumer.announced();
5490
5491 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5492 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5493
5494 let mut source_a = origin
5495 .create_broadcast("test", announce().with_hops(hops_a.clone()).with_cost(5))
5496 .unwrap();
5497 settle().await;
5498 announced.assert_next_some("test");
5499
5500 let mut source_b = origin
5502 .create_broadcast("test", announce().with_hops(hops_b.clone()))
5503 .unwrap();
5504 settle().await;
5505 settle().await;
5506 announced.assert_next_none("test");
5507 announced.assert_next_some("test");
5508 announced.assert_next_wait();
5509
5510 source_a
5513 .set_route(announce().with_hops(hops_a.clone()).with_cost(9))
5514 .unwrap();
5515 settle().await;
5516 settle().await;
5517 assert_eq!(
5518 consumer.get_broadcast("test").unwrap().route().hops,
5519 hops_b,
5520 "a repricing must not take the path back from the live front"
5521 );
5522 announced.assert_next_wait();
5523
5524 source_a.finish();
5525 source_b.finish();
5526 }
5527
5528 #[tokio::test]
5533 async fn test_reconnect_wins_over_stale_route() {
5534 tokio::time::pause();
5535
5536 let origin = Origin::random().produce();
5537 let consumer = origin.consume();
5538
5539 let publisher = Origin::new(1).unwrap();
5540 let hops = OriginList::try_from(vec![publisher]).unwrap();
5541
5542 let stale = origin
5545 .create_broadcast("test", announce().with_hops(hops.clone()))
5546 .unwrap();
5547 let mut stale_dynamic = stale.dynamic();
5548 settle().await;
5549
5550 let fresh = origin
5552 .create_broadcast("test", announce().with_hops(hops.clone()))
5553 .unwrap();
5554 let mut fresh_dynamic = fresh.dynamic();
5555 settle().await;
5556 settle().await;
5557
5558 let broadcast = consumer.request_broadcast("test").await.unwrap();
5560 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5561 settle().await;
5562 let _producer = accept_track(&mut fresh_dynamic, "video").await;
5563 settle().await;
5564 subscribing.await.unwrap();
5565 stale_dynamic.assert_no_request();
5566 }
5567
5568 #[tokio::test]
5574 async fn test_carrying_reconnect_switches_immediately() {
5575 tokio::time::pause();
5576
5577 let origin = Origin::random().produce();
5578 let consumer = origin.consume();
5579
5580 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5581
5582 let stale = origin
5583 .create_broadcast("test", announce().with_hops(hops.clone()))
5584 .unwrap();
5585 let mut stale_dynamic = stale.dynamic();
5586 settle().await;
5587
5588 let broadcast = consumer.request_broadcast("test").await.unwrap();
5590 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5591 settle().await;
5592 let _stale_producer = accept_track(&mut stale_dynamic, "video").await;
5593 settle().await;
5594 let _subscription = subscribing.await.unwrap();
5596
5597 let fresh = origin
5599 .create_broadcast("test", announce().with_hops(hops.clone()))
5600 .unwrap();
5601 let mut fresh_dynamic = fresh.dynamic();
5602 settle().await;
5603 settle().await;
5604
5605 let _fresh_producer = accept_track(&mut fresh_dynamic, "video").await;
5608 }
5609
5610 #[tokio::test]
5620 async fn test_offline_mismatch_never_evicts_a_live_front() {
5621 tokio::time::pause();
5622
5623 let origin = Origin::random().produce();
5624 let consumer = origin.consume();
5625 let mut announced = consumer.announced();
5626
5627 let hops_live = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5628 let hops_cache = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5629
5630 let mut live = origin
5631 .create_broadcast("test", announce().with_hops(hops_live.clone()))
5632 .unwrap();
5633 let mut live_dynamic = live.dynamic();
5634 settle().await;
5635 let face = announced.assert_next_some("test");
5636
5637 let broadcast = consumer.request_broadcast("test").await.unwrap();
5639 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5640 settle().await;
5641 let _producer = accept_track(&mut live_dynamic, "video").await;
5642 settle().await;
5643 subscribing.await.unwrap();
5644
5645 let cache = origin
5648 .create_broadcast("test", broadcast::Route::new().with_hops(hops_cache.clone()))
5649 .unwrap();
5650 settle().await;
5651 settle().await;
5652 announced.assert_next_wait();
5653 assert!(!face.is_closed(), "the live front must survive");
5654 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_live);
5655
5656 live.finish();
5659 settle().await;
5660 settle().await;
5661 announced.assert_next_none("test");
5662 let taken = consumer
5663 .get_broadcast("test")
5664 .expect("the parked source must take over");
5665 assert_eq!(taken.route().hops, hops_cache);
5666 announced.assert_next_wait();
5668 drop(cache);
5669 }
5670
5671 #[tokio::test]
5677 async fn test_dispatch_excludes_requester() {
5678 tokio::time::pause();
5679
5680 let origin = Origin::random().produce();
5681 let consumer = origin.consume();
5682
5683 let peer = Origin::new(5).unwrap();
5684 let publisher = Origin::new(1).unwrap();
5685 let tainted = OriginList::try_from(vec![publisher, peer]).unwrap();
5687 let clean = OriginList::try_from(vec![publisher]).unwrap();
5688
5689 let source_a = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
5690 let mut dynamic_a = source_a.dynamic();
5691 settle().await;
5692 let source_b = origin
5693 .create_broadcast("test", announce().with_hops(clean).with_cost(5))
5694 .unwrap();
5695 let mut dynamic_b = source_b.dynamic();
5696 settle().await;
5697 settle().await;
5698
5699 let shared = consumer.request_broadcast("test").await.unwrap();
5702 let subscribing = shared.track("video").unwrap().subscribe(None);
5703 let _producer_a = accept_track(&mut dynamic_a, "video").await;
5704 settle().await;
5705 subscribing.await.unwrap();
5706
5707 let scoped = consumer.clone().excluding(peer);
5712 let pinned = scoped.request_broadcast("test").await.unwrap();
5713 let subscribing = pinned.track("video").unwrap().subscribe(None);
5714 let _producer_b = accept_track(&mut dynamic_b, "video").await;
5715 settle().await;
5716 subscribing.await.unwrap();
5717 dynamic_a.assert_no_request();
5718 }
5719
5720 #[tokio::test]
5725 async fn test_unknown_publishers_do_not_splice() {
5726 tokio::time::pause();
5727
5728 let origin = Origin::random().produce();
5729 let consumer = origin.consume();
5730 let mut announced = consumer.announced();
5731
5732 let unknown_a = OriginList::try_from(vec![Origin::UNKNOWN]).unwrap();
5733 let unknown_b = OriginList::try_from(vec![Origin::UNKNOWN]).unwrap();
5734
5735 let mut source_a = origin
5736 .create_broadcast("test", announce().with_hops(unknown_a.clone()))
5737 .unwrap();
5738 settle().await;
5739 settle().await;
5740 announced.assert_next_some("test");
5741
5742 let source_b = origin
5746 .create_broadcast("test", announce().with_hops(unknown_b))
5747 .unwrap();
5748 settle().await;
5749 settle().await;
5750 announced.assert_next_none("test");
5751 let live = announced.assert_next_some("test");
5752
5753 source_a
5756 .set_route(announce().with_hops(unknown_a).with_cost(9))
5757 .unwrap();
5758 settle().await;
5759 settle().await;
5760 assert!(
5761 consumer.get_broadcast("test").unwrap().is_clone(&live),
5762 "UNKNOWN-to-UNKNOWN repricing must not replace the live front"
5763 );
5764 announced.assert_next_wait();
5765
5766 drop(source_a);
5767 drop(source_b);
5768 }
5769
5770 #[tokio::test]
5773 async fn test_known_publishers_still_splice() {
5774 tokio::time::pause();
5775
5776 let origin = Origin::random().produce();
5777 let consumer = origin.consume();
5778 let mut announced = consumer.announced();
5779
5780 let publisher = Origin::new(1).unwrap();
5781 let hops_a = OriginList::try_from(vec![publisher]).unwrap();
5782 let hops_b = OriginList::try_from(vec![publisher, Origin::new(3).unwrap()]).unwrap();
5783
5784 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
5785 settle().await;
5786 settle().await;
5787 announced.assert_next_some("test");
5788
5789 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
5791 settle().await;
5792 settle().await;
5793 announced.assert_next_wait();
5794
5795 drop(source_a);
5796 drop(source_b);
5797 }
5798
5799 #[tokio::test]
5805 async fn test_standby_join_splices_live_subscriber() {
5806 tokio::time::pause();
5807
5808 let origin = Origin::random().produce();
5809 let consumer = origin.consume();
5810
5811 let publisher = Origin::new(1).unwrap();
5812 let peer = Origin::new(5).unwrap();
5813 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5814 let local = OriginList::try_from(vec![publisher]).unwrap();
5815
5816 let source_remote = origin
5818 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5819 .unwrap();
5820 let mut dynamic_remote = source_remote.dynamic();
5821 settle().await;
5822 settle().await;
5823 let broadcast = consumer.request_broadcast("test").await.unwrap();
5824 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5825 let mut producer_remote = accept_track(&mut dynamic_remote, "video").await;
5826 settle().await;
5827 let mut sub = subscribing.await.unwrap();
5828 producer_remote.append_group().unwrap();
5829 assert_eq!(sub.assert_group().sequence, 0);
5830
5831 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5835 let mut dynamic_local = source_local.dynamic();
5836 settle().await;
5837 let mut producer_local = accept_track(&mut dynamic_local, "video").await;
5838 settle().await;
5839 sub.assert_no_group();
5840 assert_eq!(producer_local.subscription().unwrap().group_start, None);
5841 producer_local.create_group(group::Info { sequence: 1 }).unwrap();
5842 assert_eq!(sub.assert_group().sequence, 1);
5843 sub.assert_not_closed();
5844 }
5845
5846 #[tokio::test]
5853 async fn test_standby_with_a_partial_track_list_splits_per_track() {
5854 tokio::time::pause();
5855
5856 let origin = Origin::random().produce();
5857 let consumer = origin.consume();
5858
5859 let publisher = Origin::new(1).unwrap();
5860 let peer = Origin::new(5).unwrap();
5861 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5862 let local = OriginList::try_from(vec![publisher]).unwrap();
5863
5864 let source_remote = origin
5866 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5867 .unwrap();
5868 let mut dynamic_remote = source_remote.dynamic();
5869 settle().await;
5870 settle().await;
5871 let broadcast = consumer.request_broadcast("test").await.unwrap();
5872
5873 let subscribing_video = broadcast.track("video").unwrap().subscribe(None);
5874 let subscribing_audio = broadcast.track("audio").unwrap().subscribe(None);
5875 let mut producers_remote = accept_tracks(&mut dynamic_remote, 2).await;
5876 settle().await;
5877
5878 let mut sub_video = subscribing_video.await.unwrap();
5879 let mut sub_audio = subscribing_audio.await.unwrap();
5880 for name in ["video", "audio"] {
5881 producers_remote.get_mut(name).unwrap().append_group().unwrap();
5882 }
5883 assert_eq!(sub_video.assert_group().sequence, 0);
5884 assert_eq!(sub_audio.assert_group().sequence, 0);
5885
5886 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5888 let mut dynamic_local = source_local.dynamic();
5889 settle().await;
5890
5891 let mut producer_local = None;
5892 for _ in 0..2 {
5893 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_local.requested_track())
5894 .await
5895 .expect("timed out waiting for a track request")
5896 .expect("source closed");
5897 match request.name() {
5898 "video" => producer_local = Some(request.accept(None)),
5899 "audio" => request.reject(Error::NotFound),
5900 other => panic!("unexpected track dispatched: {other}"),
5901 }
5902 }
5903 settle().await;
5904 let mut producer_local = producer_local.expect("the standby was never asked for video");
5905
5906 sub_video.assert_no_group();
5909 assert_eq!(producer_local.subscription().unwrap().group_start, None);
5910 producer_local.create_group(group::Info { sequence: 1 }).unwrap();
5911 assert_eq!(
5912 sub_video.assert_group().sequence,
5913 1,
5914 "video did not move to the standby"
5915 );
5916
5917 producers_remote.get_mut("audio").unwrap().append_group().unwrap();
5919 assert_eq!(
5920 sub_audio.assert_group().sequence,
5921 1,
5922 "audio did not stay on the incumbent"
5923 );
5924 sub_video.assert_not_closed();
5925 sub_audio.assert_not_closed();
5926 }
5927
5928 #[tokio::test]
5935 async fn test_standby_refusal_keeps_incumbent() {
5936 tokio::time::pause();
5937 for incompatible in [false, true] {
5938 let origin = Origin::random().produce();
5939 let consumer = origin.consume();
5940
5941 let publisher = Origin::new(1).unwrap();
5942 let peer = Origin::new(5).unwrap();
5943 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5944 let local = OriginList::try_from(vec![publisher]).unwrap();
5945
5946 let source_remote = origin
5948 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5949 .unwrap();
5950 let mut dynamic_remote = source_remote.dynamic();
5951 settle().await;
5952 settle().await;
5953 let broadcast = consumer.request_broadcast("test").await.unwrap();
5954 let subscribing = broadcast.track("audio").unwrap().subscribe(None);
5955 let mut producer_remote = accept_track(&mut dynamic_remote, "audio").await;
5956 settle().await;
5957 let mut sub = subscribing.await.unwrap();
5958 producer_remote.append_group().unwrap();
5959 assert_eq!(sub.assert_group().sequence, 0);
5960
5961 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5964 let mut dynamic_local = source_local.dynamic();
5965 settle().await;
5966 let request = dynamic_local.requested_track().await.unwrap();
5967 assert_eq!(request.name(), "audio");
5968 let incompatible_track = if incompatible {
5969 Some(request.accept(track::Info::default().with_timescale(Timescale::MICRO)))
5970 } else {
5971 request.reject(Error::NotFound);
5972 None
5973 };
5974 settle().await;
5975
5976 producer_remote.append_group().unwrap();
5978 assert_eq!(sub.assert_group().sequence, 1);
5979 sub.assert_not_closed();
5980
5981 source_remote.abort(Error::Dropped).unwrap();
5984 settle().await;
5985 settle().await;
5986 sub.assert_closed();
5987 dynamic_local.assert_no_request();
5988
5989 drop(incompatible_track);
5990 let retry = broadcast.track("audio").unwrap().subscribe(None);
5992 let mut producer_local = accept_track(&mut dynamic_local, "audio").await;
5993 settle().await;
5994 let mut sub = retry.await.expect("a fresh request must reach the standby");
5995 producer_local.create_group(group::Info { sequence: 2 }).unwrap();
5996 assert_eq!(sub.assert_group().sequence, 2);
5997 }
5998 }
5999
6000 #[tokio::test]
6005 async fn test_unservable_track_retried_by_a_later_request() {
6006 tokio::time::pause();
6007
6008 let origin = Origin::random().produce();
6009 let consumer = origin.consume();
6010
6011 let source = origin.create_broadcast("test", announce()).unwrap();
6012 let mut dynamic = source.dynamic();
6013 settle().await;
6014 settle().await;
6015 let broadcast = consumer.request_broadcast("test").await.unwrap();
6016
6017 let subscribing = broadcast.track("audio").unwrap().subscribe(None);
6019 let request = dynamic.requested_track().await.unwrap();
6020 request.reject(Error::NotFound);
6021 settle().await;
6022 assert!(matches!(subscribing.await, Err(Error::NotFound)));
6023
6024 let retry = broadcast.track("audio").unwrap().subscribe(None);
6026 let mut producer = accept_track(&mut dynamic, "audio").await;
6027 settle().await;
6028 let mut sub = retry.await.expect("a fresh request must reach the source");
6029 producer.append_group().unwrap();
6030 assert_eq!(sub.assert_group().sequence, 0);
6031 }
6032
6033 #[tokio::test]
6039 async fn test_track_dying_without_progress_aborts() {
6040 tokio::time::pause();
6041
6042 let origin = Origin::random().produce();
6043 let consumer = origin.consume();
6044
6045 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
6046 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
6047 let mut dynamic = source.dynamic();
6048 settle().await;
6049 settle().await;
6050 let broadcast = consumer.request_broadcast("test").await.unwrap();
6051
6052 let subscribing = broadcast.track("video").unwrap().subscribe(None);
6053 let producer = accept_track(&mut dynamic, "video").await;
6054 settle().await;
6055 let mut sub = subscribing.await.unwrap();
6056
6057 drop(producer);
6060 settle().await;
6061 sub.assert_closed();
6062 dynamic.assert_no_request();
6063
6064 let retry = broadcast.track("video").unwrap().subscribe(None);
6066 let mut producer = accept_track(&mut dynamic, "video").await;
6067 settle().await;
6068 let mut sub = retry.await.expect("a fresh request must reach the source");
6069 producer.append_group().unwrap();
6070 assert_eq!(sub.assert_group().sequence, 0);
6071 }
6072
6073 #[tokio::test]
6078 async fn test_delivered_copy_death_survives_unrelated_wakes() {
6079 tokio::time::pause();
6080
6081 let origin = Origin::random().produce();
6082 let consumer = origin.consume();
6083
6084 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
6085 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
6086 let mut dynamic = source.dynamic();
6087 settle().await;
6088 settle().await;
6089 let broadcast = consumer.request_broadcast("test").await.unwrap();
6090
6091 let subscribing = broadcast.track("video").unwrap().subscribe(None);
6092 let mut producer = accept_track(&mut dynamic, "video").await;
6093 settle().await;
6094 let mut sub = subscribing.await.unwrap();
6095 producer.append_group().unwrap();
6096 assert_eq!(sub.assert_group().sequence, 0);
6097
6098 drop(sub);
6101 settle().await;
6102 let resubscribing = broadcast.track("video").unwrap().subscribe(None);
6103 settle().await;
6104 let mut sub = resubscribing.await.unwrap();
6105 assert_eq!(sub.assert_group().sequence, 0, "cached group re-served");
6106
6107 drop(producer);
6110 let mut producer = accept_track(&mut dynamic, "video").await;
6111 settle().await;
6112 producer.create_group(group::Info { sequence: 1 }).unwrap();
6113 assert_eq!(sub.assert_group().sequence, 1);
6114 sub.assert_not_closed();
6115 }
6116
6117 #[tokio::test]
6123 async fn test_per_track_fallback_respects_exclusion() {
6124 tokio::time::pause();
6125
6126 let origin = Origin::random().produce();
6127 let consumer = origin.consume();
6128
6129 let publisher = Origin::new(1).unwrap();
6130 let peer = Origin::new(5).unwrap();
6131 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
6132 let local = OriginList::try_from(vec![publisher]).unwrap();
6133
6134 let source_tainted = origin
6136 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
6137 .unwrap();
6138 let mut dynamic_tainted = source_tainted.dynamic();
6139 settle().await;
6140 settle().await;
6141
6142 let source_clean = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
6145 let mut dynamic_clean = source_clean.dynamic();
6146 settle().await;
6147
6148 let scoped = consumer.clone().excluding(peer);
6149 let broadcast = scoped.request_broadcast("test").await.unwrap();
6150 let _subscribing = broadcast.track("video").unwrap().subscribe(None);
6151 settle().await;
6152
6153 let request = dynamic_clean.requested_track().await.unwrap();
6157 request.reject(Error::NotFound);
6158 settle().await;
6159 dynamic_tainted.assert_no_request();
6160 }
6161
6162 #[tokio::test]
6166 async fn test_exclusion_survives_failover_onto_a_tainted_route() {
6167 tokio::time::pause();
6168
6169 let origin = Origin::random().produce();
6170 let consumer = origin.consume();
6171
6172 let publisher = Origin::new(1).unwrap();
6173 let peer = Origin::new(5).unwrap();
6174 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
6175 let local = OriginList::try_from(vec![publisher]).unwrap();
6176
6177 let source_tainted = origin
6178 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
6179 .unwrap();
6180 let mut dynamic_tainted = source_tainted.dynamic();
6181 settle().await;
6182 settle().await;
6183 let source_clean = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
6184 let mut dynamic_clean = source_clean.dynamic();
6185 settle().await;
6186
6187 let scoped = consumer.clone().excluding(peer);
6188 let broadcast = scoped.request_broadcast("test").await.unwrap();
6189 let _subscribing = broadcast.track("video").unwrap().subscribe(None);
6190 let _clean = accept_track(&mut dynamic_clean, "video").await;
6191 settle().await;
6192
6193 source_clean.abort(Error::Dropped).unwrap();
6195 settle().await;
6196 settle().await;
6197 dynamic_tainted.assert_no_request();
6198
6199 assert!(matches!(scoped.request_broadcast("test").await, Err(Error::Unroutable)));
6202 }
6203
6204 #[tokio::test]
6209 async fn test_exclusion_holds_when_a_tainted_route_attaches_later() {
6210 tokio::time::pause();
6211
6212 let origin = Origin::random().produce();
6213 let consumer = origin.consume();
6214
6215 let publisher = Origin::new(1).unwrap();
6216 let peer = Origin::new(5).unwrap();
6217 let local = OriginList::try_from(vec![publisher]).unwrap();
6218 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
6219
6220 let source_clean = origin
6224 .create_broadcast("test", announce().with_hops(local).with_cost(5))
6225 .unwrap();
6226 let mut dynamic_clean = source_clean.dynamic();
6227 settle().await;
6228 settle().await;
6229 let scoped = consumer.clone().excluding(peer);
6230 let broadcast = scoped.request_broadcast("test").await.unwrap();
6231 let subscribing = broadcast.track("video").unwrap().subscribe(None);
6232 let mut producer_clean = accept_track(&mut dynamic_clean, "video").await;
6233 settle().await;
6234 let mut sub = subscribing.await.unwrap();
6235 producer_clean.append_group().unwrap();
6236 assert_eq!(sub.assert_group().sequence, 0);
6237
6238 let mut tainted = announce().with_hops(via_peer.clone()).with_cost(0);
6245 tainted.advertised = 1;
6246 let mut source_tainted = origin.create_broadcast("test", tainted).unwrap();
6247 let mut dynamic_tainted = source_tainted.dynamic();
6248 settle().await;
6249 settle().await;
6250 dynamic_tainted.assert_no_request();
6251 producer_clean.append_group().unwrap();
6252 assert_eq!(sub.assert_group().sequence, 1);
6253 sub.assert_not_closed();
6254
6255 drop(sub);
6258 drop(broadcast);
6259 drop(scoped);
6260 settle().await;
6261 let mut bumped = announce().with_hops(via_peer).with_cost(1);
6262 bumped.advertised = 1;
6263 source_tainted.set_route(bumped).unwrap();
6264 settle().await;
6265 let plain = consumer.request_broadcast("test").await.unwrap();
6266 let _plain_track = plain.track("video").unwrap().subscribe(None);
6267 settle().await;
6268 settle().await;
6269 assert!(
6270 dynamic_tainted.requested_track().now_or_never().is_some(),
6271 "the front must be free to use the route again once the peer is gone"
6272 );
6273 }
6274
6275 #[tokio::test]
6279 async fn test_excluded_path_never_reaches_the_dynamic_handler() {
6280 tokio::time::pause();
6281
6282 let origin = Origin::random().produce();
6283 let consumer = origin.consume();
6284 let mut dynamic = origin.dynamic();
6285
6286 let peer = Origin::new(5).unwrap();
6287 let tainted = OriginList::try_from(vec![Origin::new(1).unwrap(), peer]).unwrap();
6288 let _source = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
6289 settle().await;
6290 settle().await;
6291
6292 let scoped = consumer.clone().excluding(peer);
6293 assert!(matches!(scoped.request_broadcast("test").await, Err(Error::Unroutable)));
6294 assert!(
6295 dynamic.requested_broadcast().now_or_never().is_none(),
6296 "the dynamic handler was asked to route around the exclusion"
6297 );
6298
6299 let _pending = scoped.request_broadcast("other");
6301 settle().await;
6302 assert!(
6303 dynamic.requested_broadcast().now_or_never().is_some(),
6304 "a genuinely missing path must still fall back"
6305 );
6306 }
6307
6308 #[tokio::test]
6316 async fn test_out_of_scope_path_never_reaches_the_dynamic_handler() {
6317 let origin = Origin::random().produce();
6318 let mut dynamic = origin.dynamic();
6319
6320 let scoped = origin.consume().scope(&["tenant-a".into()]).unwrap();
6321
6322 for path in ["tenant-b/live", "tenant-a-other/live"] {
6325 let refused = scoped
6329 .request_broadcast(path)
6330 .now_or_never()
6331 .expect("an out-of-scope request must be refused synchronously, not queued");
6332 assert!(matches!(refused, Err(Error::Unauthorized)));
6333 assert!(
6334 dynamic.requested_broadcast().now_or_never().is_none(),
6335 "the dynamic handler was asked to create a broadcast the requester may not read"
6336 );
6337 }
6338 }
6339
6340 #[tokio::test]
6343 async fn test_out_of_scope_path_is_unauthorized_without_a_handler() {
6344 let origin = Origin::random().produce();
6345 let scoped = origin.consume().scope(&["tenant-a".into()]).unwrap();
6346
6347 let refused = scoped
6348 .request_broadcast("tenant-b/live")
6349 .now_or_never()
6350 .expect("an out-of-scope request must be refused synchronously, not queued");
6351 assert!(matches!(refused, Err(Error::Unauthorized)));
6352 }
6353
6354 #[tokio::test]
6357 async fn test_in_scope_path_still_reaches_the_dynamic_handler() {
6358 let origin = Origin::random().produce();
6359 let mut dynamic = origin.dynamic();
6360
6361 let scoped = origin.consume().scope(&["tenant-a".into()]).unwrap();
6362 let requested = scoped.request_broadcast("tenant-a/live");
6363
6364 let served = broadcast::Info::new().produce();
6368 let request = dynamic
6369 .requested_broadcast()
6370 .now_or_never()
6371 .expect("an in-scope request must reach the handler")
6372 .unwrap();
6373 assert_eq!(request.path(), &Path::new("tenant-a/live"));
6374 request.accept(&served);
6375
6376 let broadcast = requested.now_or_never().expect("served synchronously").unwrap();
6377 assert!(broadcast.is_clone(&served.consume()));
6378 }
6379
6380 #[tokio::test]
6387 async fn test_whole_tree_scope_still_reaches_the_dynamic_handler() {
6388 let origin = Origin::random().produce();
6389 let mut dynamic = origin.dynamic();
6390
6391 for (consumer, path) in [
6392 (origin.consume(), "tenant-a/live"),
6393 (origin.consume().scope(&["".into()]).unwrap(), "tenant-b/live"),
6394 ] {
6395 let requested = consumer.request_broadcast(path);
6396
6397 let served = broadcast::Info::new().produce();
6398 let request = dynamic
6399 .requested_broadcast()
6400 .now_or_never()
6401 .expect("a whole-tree request must reach the handler")
6402 .unwrap();
6403 assert_eq!(request.path(), &Path::new(path));
6404 request.accept(&served);
6405
6406 let broadcast = requested.now_or_never().expect("served synchronously").unwrap();
6407 assert!(broadcast.is_clone(&served.consume()));
6408 }
6409 }
6410
6411 #[tokio::test]
6414 async fn test_dispatch_all_tainted_unroutable() {
6415 tokio::time::pause();
6416
6417 let origin = Origin::random().produce();
6418 let consumer = origin.consume();
6419
6420 let peer = Origin::new(5).unwrap();
6421 let tainted = OriginList::try_from(vec![Origin::new(1).unwrap(), peer]).unwrap();
6422 let _source = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
6423 settle().await;
6424 settle().await;
6425
6426 let scoped = consumer.clone().excluding(peer);
6427 match scoped.request_broadcast("test").await {
6428 Err(Error::Unroutable) => {}
6429 Err(err) => panic!("expected Unroutable, got {err:?}"),
6430 Ok(_) => panic!("expected Unroutable, got a broadcast"),
6431 }
6432
6433 consumer.request_broadcast("test").await.unwrap();
6435 }
6436
6437 #[tokio::test]
6438 async fn test_duplicate_reverse() {
6439 tokio::time::pause();
6440
6441 let origin = Origin::random().produce();
6442
6443 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
6444 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
6445 settle().await;
6446 assert!(origin.consume().get_broadcast("test").is_some());
6447
6448 broadcast2.finish();
6450 settle().await;
6451 assert!(origin.consume().get_broadcast("test").is_some());
6452
6453 broadcast1.finish();
6454 settle().await;
6455 assert!(origin.consume().get_broadcast("test").is_none());
6456 }
6457
6458 #[tokio::test]
6459 async fn test_deterministic_tiebreak() {
6460 tokio::time::pause();
6461
6462 fn hops(ids: &[u64]) -> OriginList {
6463 OriginList::try_from(
6464 ids.iter()
6465 .copied()
6466 .map(|id| Origin::new(id).unwrap())
6467 .collect::<Vec<_>>(),
6468 )
6469 .unwrap()
6470 }
6471
6472 async fn winner(first: &[u64], second: &[u64]) -> OriginList {
6475 let origin = Origin::random().produce();
6476 let _a = origin
6477 .create_broadcast("test", announce().with_hops(hops(first)))
6478 .unwrap();
6479 let _b = origin
6480 .create_broadcast("test", announce().with_hops(hops(second)))
6481 .unwrap();
6482 settle().await;
6483 origin.consume().get_broadcast("test").unwrap().route().hops
6484 }
6485
6486 let forward = winner(&[5, 20], &[5, 40]).await;
6490 let reverse = winner(&[5, 40], &[5, 20]).await;
6491 assert_eq!(forward, reverse, "tie-break must not depend on publish order");
6492
6493 assert_eq!(winner(&[5, 20], &[5]).await.len(), 1);
6495 assert_eq!(winner(&[5], &[5, 20]).await.len(), 1);
6496 }
6497
6498 #[tokio::test]
6503 async fn test_many_announces() {
6504 let origin = Origin::random().produce();
6505
6506 let mut consumer = origin.consume().announced();
6507 let mut broadcasts = Vec::new();
6509 for i in 0..256 {
6510 broadcasts.push(origin.create_broadcast(format!("test{i:03}"), announce()).unwrap());
6511 settle().await;
6512 }
6513
6514 for i in 0..256 {
6515 consumer.assert_next_some(format!("test{i:03}"));
6516 }
6517 consumer.assert_next_wait();
6518 }
6519
6520 #[tokio::test]
6521 async fn test_many_announces_try() {
6522 let origin = Origin::random().produce();
6523
6524 let mut consumer = origin.consume().announced();
6525 let mut broadcasts = Vec::new();
6527 for i in 0..256 {
6528 broadcasts.push(origin.create_broadcast(format!("test{i:03}"), announce()).unwrap());
6529 settle().await;
6530 }
6531
6532 for i in 0..256 {
6533 consumer.assert_try_next_some(format!("test{i:03}"));
6534 }
6535 }
6536
6537 #[tokio::test]
6538 async fn test_with_root_basic() {
6539 let origin = Origin::random().produce();
6540
6541 let foo_producer = origin.with_root("foo").expect("should create root");
6543 assert_eq!(foo_producer.root().as_str(), "foo");
6544
6545 let mut consumer = origin.consume().announced();
6546
6547 let _broadcast = foo_producer
6549 .create_broadcast("bar/baz", announce())
6550 .expect("publish allowed");
6551 settle().await;
6552 consumer.assert_next_some("foo/bar/baz");
6554
6555 let mut foo_consumer = foo_producer.consume().announced();
6557 foo_consumer.assert_next_some("bar/baz");
6558 }
6559
6560 #[tokio::test]
6561 async fn test_with_root_nested() {
6562 let origin = Origin::random().produce();
6563
6564 let foo_producer = origin.with_root("foo").expect("should create foo root");
6566 let foo_bar_producer = foo_producer.with_root("bar").expect("should create bar root");
6567 assert_eq!(foo_bar_producer.root().as_str(), "foo/bar");
6568
6569 let mut consumer = origin.consume().announced();
6570
6571 let _broadcast = foo_bar_producer
6573 .create_broadcast("baz", announce())
6574 .expect("publish allowed");
6575 settle().await;
6576 consumer.assert_next_some("foo/bar/baz");
6578
6579 let mut foo_bar_consumer = foo_bar_producer.consume().announced();
6581 foo_bar_consumer.assert_next_some("baz");
6582 }
6583
6584 #[tokio::test]
6585 async fn test_publish_scope_allows() {
6586 let origin = Origin::random().produce();
6587
6588 let limited_producer = origin
6590 .scope(&["allowed/path1".into(), "allowed/path2".into()])
6591 .expect("should create limited producer");
6592
6593 let _broadcast = limited_producer
6595 .create_broadcast("allowed/path1", announce())
6596 .expect("publish allowed");
6597 let _keep2 = limited_producer
6598 .create_broadcast("allowed/path1/nested", announce())
6599 .expect("publish allowed");
6600 let _keep3 = limited_producer
6601 .create_broadcast("allowed/path2", announce())
6602 .expect("publish allowed");
6603 settle().await;
6604
6605 assert!(limited_producer.create_broadcast("notallowed", announce()).is_err());
6607 assert!(limited_producer.create_broadcast("allowed", announce()).is_err()); assert!(limited_producer.create_broadcast("other/path", announce()).is_err());
6609 }
6610
6611 #[tokio::test]
6612 async fn test_publish_max_parts() {
6613 let origin = Origin::random().produce();
6614
6615 let at_limit = (0..Path::MAX_PARTS)
6616 .map(|i| i.to_string())
6617 .collect::<Vec<_>>()
6618 .join("/");
6619 let _broadcast = origin
6620 .create_broadcast(at_limit.as_str(), announce())
6621 .expect("publish allowed");
6622 settle().await;
6623
6624 let too_deep = format!("{at_limit}/extra");
6625 assert!(origin.create_broadcast(too_deep.as_str(), announce()).is_err());
6626
6627 let rooted = origin.with_root("root").expect("wildcard allows any root");
6629 assert!(rooted.create_broadcast(at_limit.as_str(), announce()).is_err());
6630 }
6631
6632 #[tokio::test]
6633 async fn test_publish_scope_empty() {
6634 let origin = Origin::random().produce();
6635
6636 assert!(origin.scope(&[]).is_none());
6638 }
6639
6640 #[tokio::test]
6641 async fn test_consume_scope_filters() {
6642 let origin = Origin::random().produce();
6643
6644 let mut consumer = origin.consume().announced();
6645
6646 let _broadcast1 = origin.create_broadcast("allowed", announce()).unwrap();
6648 let _broadcast2 = origin.create_broadcast("allowed/nested", announce()).unwrap();
6649 let _broadcast3 = origin.create_broadcast("notallowed", announce()).unwrap();
6650 settle().await;
6651
6652 let mut limited_consumer = origin
6654 .consume()
6655 .scope(&["allowed".into()])
6656 .expect("should create limited consumer")
6657 .announced();
6658
6659 limited_consumer.assert_next_some("allowed");
6661 limited_consumer.assert_next_some("allowed/nested");
6662 limited_consumer.assert_next_wait(); consumer.assert_next_some("allowed");
6666 consumer.assert_next_some("allowed/nested");
6667 consumer.assert_next_some("notallowed");
6668 }
6669
6670 #[tokio::test]
6671 async fn test_consume_scope_multiple_prefixes() {
6672 let origin = Origin::random().produce();
6673
6674 let _broadcast1 = origin.create_broadcast("foo/test", announce()).unwrap();
6675 let _broadcast2 = origin.create_broadcast("bar/test", announce()).unwrap();
6676 let _broadcast3 = origin.create_broadcast("baz/test", announce()).unwrap();
6677 settle().await;
6678
6679 let mut limited_consumer = origin
6681 .consume()
6682 .scope(&["foo".into(), "bar".into()])
6683 .expect("should create limited consumer")
6684 .announced();
6685
6686 limited_consumer.assert_next_some("bar/test");
6688 limited_consumer.assert_next_some("foo/test");
6689 limited_consumer.assert_next_wait(); }
6691
6692 #[tokio::test]
6693 async fn test_with_root_and_publish_scope() {
6694 let origin = Origin::random().produce();
6695
6696 let foo_producer = origin.with_root("foo").expect("should create foo root");
6698
6699 let limited_producer = foo_producer
6701 .scope(&["bar".into(), "goop/pee".into()])
6702 .expect("should create limited producer");
6703
6704 let mut consumer = origin.consume().announced();
6705
6706 let _broadcast = limited_producer
6708 .create_broadcast("bar", announce())
6709 .expect("publish allowed");
6710 let _keep2 = limited_producer
6711 .create_broadcast("bar/nested", announce())
6712 .expect("publish allowed");
6713 let _keep3 = limited_producer
6714 .create_broadcast("goop/pee", announce())
6715 .expect("publish allowed");
6716 let _keep4 = limited_producer
6717 .create_broadcast("goop/pee/nested", announce())
6718 .expect("publish allowed");
6719 settle().await;
6720
6721 assert!(limited_producer.create_broadcast("baz", announce()).is_err());
6723 assert!(limited_producer.create_broadcast("goop", announce()).is_err()); assert!(limited_producer.create_broadcast("goop/other", announce()).is_err());
6725
6726 consumer.assert_next_some("foo/bar");
6728 consumer.assert_next_some("foo/bar/nested");
6729 consumer.assert_next_some("foo/goop/pee");
6730 consumer.assert_next_some("foo/goop/pee/nested");
6731 }
6732
6733 #[tokio::test]
6734 async fn test_with_root_and_consume_scope() {
6735 let origin = Origin::random().produce();
6736
6737 let _broadcast1 = origin.create_broadcast("foo/bar/test", announce()).unwrap();
6739 let _broadcast2 = origin.create_broadcast("foo/goop/pee/test", announce()).unwrap();
6740 let _broadcast3 = origin.create_broadcast("foo/other/test", announce()).unwrap();
6741 settle().await;
6742
6743 let foo_producer = origin.with_root("foo").expect("should create foo root");
6745
6746 let mut limited_consumer = foo_producer
6748 .consume()
6749 .scope(&["bar".into(), "goop/pee".into()])
6750 .expect("should create limited consumer")
6751 .announced();
6752
6753 limited_consumer.assert_next_some("bar/test");
6755 limited_consumer.assert_next_some("goop/pee/test");
6756 limited_consumer.assert_next_wait(); }
6758
6759 #[tokio::test]
6760 async fn test_with_root_unauthorized() {
6761 let origin = Origin::random().produce();
6762
6763 let limited_producer = origin
6765 .scope(&["allowed".into()])
6766 .expect("should create limited producer");
6767
6768 assert!(limited_producer.with_root("notallowed").is_none());
6770
6771 let allowed_root = limited_producer
6773 .with_root("allowed")
6774 .expect("should create allowed root");
6775 assert_eq!(allowed_root.root().as_str(), "allowed");
6776 }
6777
6778 #[tokio::test]
6779 async fn test_wildcard_permission() {
6780 let origin = Origin::random().produce();
6781
6782 let root_producer = origin.clone();
6784
6785 let _broadcast = root_producer
6787 .create_broadcast("any/path", announce())
6788 .expect("publish allowed");
6789 let _keep2 = root_producer
6790 .create_broadcast("other/path", announce())
6791 .expect("publish allowed");
6792 settle().await;
6793
6794 let foo_producer = root_producer.with_root("foo").expect("should create any root");
6796 assert_eq!(foo_producer.root().as_str(), "foo");
6797 }
6798
6799 #[tokio::test]
6800 async fn test_consume_broadcast_with_permissions() {
6801 let origin = Origin::random().produce();
6802
6803 let _broadcast1 = origin.create_broadcast("allowed/test", announce()).unwrap();
6804 let _broadcast2 = origin.create_broadcast("notallowed/test", announce()).unwrap();
6805 settle().await;
6806
6807 let limited_consumer = origin
6809 .consume()
6810 .scope(&["allowed".into()])
6811 .expect("should create limited consumer");
6812
6813 let result = limited_consumer.get_broadcast("allowed/test");
6815 assert!(result.is_some());
6816 assert!(
6817 result
6818 .unwrap()
6819 .is_clone(&origin.consume().get_broadcast("allowed/test").unwrap())
6820 );
6821
6822 assert!(limited_consumer.get_broadcast("notallowed/test").is_none());
6824
6825 let consumer = origin.consume();
6827 assert!(consumer.get_broadcast("allowed/test").is_some());
6828 assert!(consumer.get_broadcast("notallowed/test").is_some());
6829 }
6830
6831 #[tokio::test]
6832 async fn test_nested_paths_with_permissions() {
6833 let origin = Origin::random().produce();
6834
6835 let limited_producer = origin.scope(&["a/b/c".into()]).expect("should create limited producer");
6837
6838 let _broadcast = limited_producer
6840 .create_broadcast("a/b/c", announce())
6841 .expect("publish allowed");
6842 let _keep2 = limited_producer
6843 .create_broadcast("a/b/c/d", announce())
6844 .expect("publish allowed");
6845 let _keep3 = limited_producer
6846 .create_broadcast("a/b/c/d/e", announce())
6847 .expect("publish allowed");
6848 settle().await;
6849
6850 assert!(limited_producer.create_broadcast("a", announce()).is_err());
6852 assert!(limited_producer.create_broadcast("a/b", announce()).is_err());
6853 assert!(limited_producer.create_broadcast("a/b/other", announce()).is_err());
6854 }
6855
6856 #[tokio::test]
6857 async fn test_multiple_consumers_with_different_permissions() {
6858 let origin = Origin::random().produce();
6859
6860 let _broadcast1 = origin.create_broadcast("foo/test", announce()).unwrap();
6862 let _broadcast2 = origin.create_broadcast("bar/test", announce()).unwrap();
6863 let _broadcast3 = origin.create_broadcast("baz/test", announce()).unwrap();
6864 settle().await;
6865
6866 let mut foo_consumer = origin
6868 .consume()
6869 .scope(&["foo".into()])
6870 .expect("should create foo consumer")
6871 .announced();
6872
6873 let mut bar_consumer = origin
6874 .consume()
6875 .scope(&["bar".into()])
6876 .expect("should create bar consumer")
6877 .announced();
6878
6879 let mut foobar_consumer = origin
6880 .consume()
6881 .scope(&["foo".into(), "bar".into()])
6882 .expect("should create foobar consumer")
6883 .announced();
6884
6885 foo_consumer.assert_next_some("foo/test");
6887 foo_consumer.assert_next_wait();
6888
6889 bar_consumer.assert_next_some("bar/test");
6890 bar_consumer.assert_next_wait();
6891
6892 foobar_consumer.assert_next_some("bar/test");
6893 foobar_consumer.assert_next_some("foo/test");
6894 foobar_consumer.assert_next_wait();
6895 }
6896
6897 #[tokio::test]
6898 async fn test_select_with_empty_prefix() {
6899 let origin = Origin::random().produce();
6900
6901 let demo_producer = origin.with_root("demo").expect("should create demo root");
6903 let limited_producer = demo_producer
6904 .scope(&["worm-node".into(), "foobar".into()])
6905 .expect("should create limited producer");
6906
6907 let _broadcast1 = limited_producer
6909 .create_broadcast("worm-node/test", announce())
6910 .expect("publish allowed");
6911 let _broadcast2 = limited_producer
6912 .create_broadcast("foobar/test", announce())
6913 .expect("publish allowed");
6914 settle().await;
6915
6916 let mut consumer = limited_producer
6918 .consume()
6919 .scope(&["".into()])
6920 .expect("should create consumer with empty prefix")
6921 .announced();
6922
6923 let a1 = consumer.try_next().expect("expected first announcement");
6925 let a2 = consumer.try_next().expect("expected second announcement");
6926 consumer.assert_next_wait();
6927
6928 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
6929 paths.sort();
6930 assert_eq!(paths, ["foobar/test", "worm-node/test"]);
6931 }
6932
6933 #[tokio::test]
6934 async fn test_select_narrowing_scope() {
6935 let origin = Origin::random().produce();
6936
6937 let demo_producer = origin.with_root("demo").expect("should create demo root");
6939 let limited_producer = demo_producer
6940 .scope(&["worm-node".into(), "foobar".into()])
6941 .expect("should create limited producer");
6942
6943 let _broadcast1 = limited_producer
6945 .create_broadcast("worm-node", announce())
6946 .expect("publish allowed");
6947 let _broadcast2 = limited_producer
6948 .create_broadcast("worm-node/foo", announce())
6949 .expect("publish allowed");
6950 let _broadcast3 = limited_producer
6951 .create_broadcast("foobar/bar", announce())
6952 .expect("publish allowed");
6953 settle().await;
6954
6955 let mut worm_consumer = limited_producer
6957 .consume()
6958 .scope(&["worm-node".into()])
6959 .expect("should create worm-node consumer")
6960 .announced();
6961
6962 worm_consumer.assert_next_some("worm-node");
6964 worm_consumer.assert_next_some("worm-node/foo");
6965 worm_consumer.assert_next_wait(); let mut foo_consumer = limited_producer
6969 .consume()
6970 .scope(&["worm-node/foo".into()])
6971 .expect("should create worm-node/foo consumer")
6972 .announced();
6973
6974 foo_consumer.assert_next_some("worm-node/foo");
6975 foo_consumer.assert_next_wait(); }
6977
6978 #[tokio::test]
6979 async fn test_select_multiple_roots_with_empty_prefix() {
6980 let origin = Origin::random().produce();
6981
6982 let limited_producer = origin
6984 .scope(&["app1".into(), "app2".into(), "shared".into()])
6985 .expect("should create limited producer");
6986
6987 let _broadcast1 = limited_producer
6989 .create_broadcast("app1/data", announce())
6990 .expect("publish allowed");
6991 let _broadcast2 = limited_producer
6992 .create_broadcast("app2/config", announce())
6993 .expect("publish allowed");
6994 let _broadcast3 = limited_producer
6995 .create_broadcast("shared/resource", announce())
6996 .expect("publish allowed");
6997 settle().await;
6998
6999 let mut consumer = limited_producer
7001 .consume()
7002 .scope(&["".into()])
7003 .expect("should create consumer with empty prefix")
7004 .announced();
7005
7006 consumer.assert_next_some("app1/data");
7008 consumer.assert_next_some("app2/config");
7009 consumer.assert_next_some("shared/resource");
7010 consumer.assert_next_wait();
7011 }
7012
7013 #[tokio::test]
7014 async fn test_publish_scope_with_empty_prefix() {
7015 let origin = Origin::random().produce();
7016
7017 let limited_producer = origin
7019 .scope(&["services/api".into(), "services/web".into()])
7020 .expect("should create limited producer");
7021
7022 let same_producer = limited_producer
7024 .scope(&["".into()])
7025 .expect("should create producer with empty prefix");
7026
7027 let _broadcast = same_producer
7029 .create_broadcast("services/api", announce())
7030 .expect("publish allowed");
7031 let _keep2 = same_producer
7032 .create_broadcast("services/web", announce())
7033 .expect("publish allowed");
7034 assert!(same_producer.create_broadcast("services/db", announce()).is_err());
7035 assert!(same_producer.create_broadcast("other", announce()).is_err());
7036 }
7037
7038 #[tokio::test]
7039 async fn test_select_narrowing_to_deeper_path() {
7040 let origin = Origin::random().produce();
7041
7042 let limited_producer = origin.scope(&["org".into()]).expect("should create limited producer");
7044
7045 let _broadcast1 = limited_producer
7047 .create_broadcast("org/team1/project1", announce())
7048 .expect("publish allowed");
7049 let _broadcast2 = limited_producer
7050 .create_broadcast("org/team1/project2", announce())
7051 .expect("publish allowed");
7052 let _broadcast3 = limited_producer
7053 .create_broadcast("org/team2/project1", announce())
7054 .expect("publish allowed");
7055 settle().await;
7056
7057 let mut team2_consumer = limited_producer
7059 .consume()
7060 .scope(&["org/team2".into()])
7061 .expect("should create team2 consumer")
7062 .announced();
7063
7064 team2_consumer.assert_next_some("org/team2/project1");
7065 team2_consumer.assert_next_wait(); let mut project1_consumer = limited_producer
7069 .consume()
7070 .scope(&["org/team1/project1".into()])
7071 .expect("should create project1 consumer")
7072 .announced();
7073
7074 project1_consumer.assert_next_some("org/team1/project1");
7076 project1_consumer.assert_next_wait();
7077 }
7078
7079 #[tokio::test]
7080 async fn test_select_with_non_matching_prefix() {
7081 let origin = Origin::random().produce();
7082
7083 let limited_producer = origin
7085 .scope(&["allowed/path".into()])
7086 .expect("should create limited producer");
7087
7088 assert!(limited_producer.consume().scope(&["different/path".into()]).is_none());
7090
7091 assert!(limited_producer.scope(&["other/path".into()]).is_none());
7093 }
7094
7095 #[tokio::test]
7098 async fn test_with_root_trailing_slash_consumer() {
7099 let origin = Origin::random().produce();
7100
7101 let prefix = "some_prefix/".to_string();
7103 let mut consumer = origin.consume().with_root(prefix).unwrap().announced();
7104
7105 let _b = origin.create_broadcast("some_prefix/test", announce()).unwrap();
7106 settle().await;
7107 consumer.assert_next_some("test");
7108 }
7109
7110 #[tokio::test]
7112 async fn test_with_root_trailing_slash_producer() {
7113 let origin = Origin::random().produce();
7114
7115 let prefix = "some_prefix/".to_string();
7117 let rooted = origin.with_root(prefix).unwrap();
7118
7119 let _b = rooted.create_broadcast("test", announce()).unwrap();
7120 settle().await;
7121
7122 let mut consumer = rooted.consume().announced();
7123 consumer.assert_next_some("test");
7124 }
7125
7126 #[tokio::test]
7128 async fn test_with_root_trailing_slash_unannounce() {
7129 tokio::time::pause();
7130
7131 let origin = Origin::random().produce();
7132
7133 let prefix = "some_prefix/".to_string();
7134 let mut consumer = origin.consume().with_root(prefix).unwrap().announced();
7135
7136 let mut b = origin.create_broadcast("some_prefix/test", announce()).unwrap();
7137 settle().await;
7138 consumer.assert_next_some("test");
7139
7140 b.finish();
7142 settle().await;
7143
7144 consumer.assert_next_none("test");
7146 }
7147
7148 #[tokio::test]
7149 async fn test_select_maintains_access_with_wider_prefix() {
7150 let origin = Origin::random().produce();
7151
7152 let demo_producer = origin.with_root("demo").expect("should create demo root");
7154 let user_producer = demo_producer
7155 .scope(&["worm-node".into(), "foobar".into()])
7156 .expect("should create user producer");
7157
7158 let _broadcast1 = user_producer
7160 .create_broadcast("worm-node/data", announce())
7161 .expect("publish allowed");
7162 let _broadcast2 = user_producer
7163 .create_broadcast("foobar", announce())
7164 .expect("publish allowed");
7165 settle().await;
7166
7167 let mut consumer = user_producer
7169 .consume()
7170 .scope(&["".into()])
7171 .expect("scope with empty prefix should not fail when user has specific permissions")
7172 .announced();
7173
7174 let a1 = consumer.try_next().expect("expected first announcement");
7176 let a2 = consumer.try_next().expect("expected second announcement");
7177 consumer.assert_next_wait();
7178
7179 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
7180 paths.sort();
7181 assert_eq!(paths, ["foobar", "worm-node/data"]);
7182
7183 let mut narrow_consumer = user_producer
7185 .consume()
7186 .scope(&["worm-node".into()])
7187 .expect("should be able to narrow scope to worm-node")
7188 .announced();
7189
7190 narrow_consumer.assert_next_some("worm-node/data");
7191 narrow_consumer.assert_next_wait(); }
7193
7194 #[tokio::test]
7195 async fn test_duplicate_prefixes_deduped() {
7196 let origin = Origin::random().produce();
7197
7198 let producer = origin
7200 .scope(&["demo".into(), "demo".into()])
7201 .expect("should create producer");
7202
7203 let _broadcast = producer
7204 .create_broadcast("demo/stream", announce())
7205 .expect("publish allowed");
7206 settle().await;
7207
7208 let mut consumer = producer.consume().announced();
7209 consumer.assert_next_some("demo/stream");
7210 consumer.assert_next_wait();
7211 }
7212
7213 #[tokio::test]
7214 async fn test_overlapping_prefixes_deduped() {
7215 let origin = Origin::random().produce();
7216
7217 let producer = origin
7219 .scope(&["demo".into(), "demo/foo".into()])
7220 .expect("should create producer");
7221
7222 let _broadcast = producer
7224 .create_broadcast("demo/bar/stream", announce())
7225 .expect("publish allowed");
7226 settle().await;
7227
7228 let mut consumer = producer.consume().announced();
7229 consumer.assert_next_some("demo/bar/stream");
7230 consumer.assert_next_wait();
7231 }
7232
7233 #[tokio::test]
7234 async fn test_overlapping_prefixes_no_duplicate_announcements() {
7235 let origin = Origin::random().produce();
7236
7237 let producer = origin
7239 .scope(&["demo".into(), "demo/foo".into()])
7240 .expect("should create producer");
7241
7242 let _broadcast = producer
7243 .create_broadcast("demo/foo/stream", announce())
7244 .expect("publish allowed");
7245 settle().await;
7246
7247 let mut consumer = producer.consume().announced();
7248 consumer.assert_next_some("demo/foo/stream");
7250 consumer.assert_next_wait();
7251 }
7252
7253 #[tokio::test]
7254 async fn test_allowed_returns_deduped_prefixes() {
7255 let origin = Origin::random().produce();
7256
7257 let producer = origin
7258 .scope(&["demo".into(), "demo/foo".into(), "anon".into()])
7259 .expect("should create producer");
7260
7261 let allowed: Vec<_> = producer.allowed().collect();
7262 assert_eq!(allowed.len(), 2, "demo/foo should be subsumed by demo");
7263 }
7264
7265 #[tokio::test]
7266 async fn test_announced_broadcast_already_announced() {
7267 let origin = Origin::random().produce();
7268
7269 let _broadcast = origin.create_broadcast("test", announce()).unwrap();
7270 settle().await;
7271
7272 let consumer = origin.consume();
7273 let result = consumer.announced_broadcast("test").await.expect("should find it");
7274 assert!(result.is_clone(&consumer.get_broadcast("test").unwrap()));
7275 }
7276
7277 #[tokio::test]
7278 async fn test_announced_broadcast_delayed() {
7279 tokio::time::pause();
7280
7281 let origin = Origin::random().produce();
7282
7283 let consumer = origin.consume();
7284
7285 let wait = tokio::spawn({
7287 let consumer = consumer.clone();
7288 async move { consumer.announced_broadcast("test").await }
7289 });
7290
7291 tokio::task::yield_now().await;
7293
7294 let _broadcast = origin.create_broadcast("test", announce()).unwrap();
7295 settle().await;
7296
7297 let result = wait.await.unwrap().expect("should find it");
7298 assert!(result.is_clone(&consumer.get_broadcast("test").unwrap()));
7299 }
7300
7301 #[tokio::test]
7302 async fn test_announced_broadcast_ignores_unrelated_paths() {
7303 tokio::time::pause();
7304
7305 let origin = Origin::random().produce();
7306
7307 let consumer = origin.consume();
7308
7309 let wait = tokio::spawn({
7310 let consumer = consumer.clone();
7311 async move { consumer.announced_broadcast("target").await }
7312 });
7313
7314 tokio::task::yield_now().await;
7315
7316 let _other = origin.create_broadcast("other", announce()).unwrap();
7318 settle().await;
7319 tokio::task::yield_now().await;
7320 assert!(!wait.is_finished(), "must not resolve on unrelated path");
7321
7322 let _target = origin.create_broadcast("target", announce()).unwrap();
7323 settle().await;
7324 let result = wait.await.unwrap().expect("should find target");
7325 assert!(result.is_clone(&consumer.get_broadcast("target").unwrap()));
7326 }
7327
7328 #[tokio::test]
7329 async fn test_announced_broadcast_skips_nested_paths() {
7330 tokio::time::pause();
7331
7332 let origin = Origin::random().produce();
7333
7334 let consumer = origin.consume();
7335
7336 let wait = tokio::spawn({
7337 let consumer = consumer.clone();
7338 async move { consumer.announced_broadcast("foo").await }
7339 });
7340
7341 tokio::task::yield_now().await;
7342
7343 let _nested = origin.create_broadcast("foo/bar", announce()).unwrap();
7345 settle().await;
7346 tokio::task::yield_now().await;
7347 assert!(!wait.is_finished(), "must not resolve on a nested path");
7348
7349 let _exact = origin.create_broadcast("foo", announce()).unwrap();
7350 settle().await;
7351 let result = wait.await.unwrap().expect("should find foo exactly");
7352 assert!(result.is_clone(&consumer.get_broadcast("foo").unwrap()));
7353 }
7354
7355 #[tokio::test]
7356 async fn test_announced_broadcast_disallowed() {
7357 let origin = Origin::random().produce();
7358 let limited = origin
7359 .consume()
7360 .scope(&["allowed".into()])
7361 .expect("should create limited");
7362
7363 assert!(limited.announced_broadcast("notallowed").await.is_none());
7365 }
7366
7367 #[tokio::test]
7368 async fn test_announced_broadcast_scope_too_narrow() {
7369 let origin = Origin::random().produce();
7372 let limited = origin
7373 .consume()
7374 .scope(&["foo/specific".into()])
7375 .expect("should create limited");
7376
7377 let result = limited
7379 .announced_broadcast("foo")
7380 .now_or_never()
7381 .expect("must not block");
7382 assert!(result.is_none());
7383 }
7384
7385 #[tokio::test]
7389 async fn test_coalesce_announce_then_unannounce() {
7390 tokio::time::pause();
7392
7393 let origin = Origin::random().produce();
7394 let mut announced = origin.consume().announced();
7395
7396 let mut broadcast = origin.create_broadcast("test", announce()).unwrap();
7397 settle().await;
7398 broadcast.finish();
7399
7400 settle().await;
7401
7402 announced.assert_next_wait();
7403 }
7404
7405 #[tokio::test]
7406 async fn test_coalesce_announce_unannounce_announce() {
7407 tokio::time::pause();
7410
7411 let origin = Origin::random().produce();
7412 let mut announced = origin.consume().announced();
7413
7414 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
7415 settle().await;
7416 broadcast1.finish();
7417 settle().await;
7418 let _broadcast2 = origin.create_broadcast("test", announce()).unwrap();
7419 settle().await;
7420
7421 announced.assert_next_some("test");
7422 announced.assert_next_wait();
7423 }
7424
7425 #[tokio::test]
7426 async fn test_coalesce_unannounce_announce_preserved() {
7427 tokio::time::pause();
7430
7431 let origin = Origin::random().produce();
7432 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
7433 settle().await;
7434
7435 let mut announced = origin.consume().announced();
7436 announced.assert_next_some("test");
7437
7438 broadcast1.finish();
7440 settle().await;
7441
7442 let _broadcast2 = origin.create_broadcast("test", announce()).unwrap();
7443 settle().await;
7444
7445 announced.assert_next_none("test");
7447 announced.assert_next_some("test");
7448 announced.assert_next_wait();
7449 }
7450
7451 #[tokio::test]
7452 async fn test_coalesce_unannounce_announce_unannounce() {
7453 tokio::time::pause();
7456
7457 let origin = Origin::random().produce();
7458 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
7459 settle().await;
7460
7461 let mut announced = origin.consume().announced();
7462 announced.assert_next_some("test");
7463
7464 broadcast1.finish();
7465 settle().await;
7466
7467 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
7468 settle().await;
7469 broadcast2.finish();
7470 settle().await;
7471
7472 announced.assert_next_none("test");
7473 announced.assert_next_wait();
7474 }
7475
7476 #[tokio::test]
7477 async fn test_coalesce_churn_bounded() {
7478 tokio::time::pause();
7483
7484 let origin = Origin::random().produce();
7485 let mut announced = origin.consume().announced();
7486
7487 for _ in 0..1000 {
7488 let mut broadcast = origin.create_broadcast("test", announce()).unwrap();
7489 settle().await;
7490 broadcast.finish();
7491 }
7492 settle().await;
7493
7494 let mut collected = Vec::new();
7495 while let Some(update) = announced.try_next() {
7496 collected.push(update);
7497 }
7498 assert!(
7499 collected.len() <= 1,
7500 "expected at most one pending update, got {}",
7501 collected.len()
7502 );
7503 assert!(
7504 collected.iter().all(|a| a.path == Path::new("test")),
7505 "unexpected path in pending updates",
7506 );
7507 }
7508
7509 #[tokio::test]
7513 async fn test_consumer_clone_is_side_effect_free() {
7514 let origin = Origin::random().produce();
7515
7516 let _broadcast1 = origin.create_broadcast("test1", announce()).unwrap();
7517 let _broadcast2 = origin.create_broadcast("test2", announce()).unwrap();
7518 settle().await;
7519
7520 let consumer = origin.consume();
7521 let mut announced = consumer.announced();
7522
7523 for _ in 0..16 {
7526 let cloned = consumer.clone();
7527 assert!(cloned.get_broadcast("test1").is_some());
7528 assert!(cloned.get_broadcast("test2").is_some());
7529 }
7530
7531 let a1 = announced.try_next().expect("first announcement");
7534 let a2 = announced.try_next().expect("second announcement");
7535 announced.assert_next_wait();
7536
7537 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
7538 paths.sort();
7539 assert_eq!(paths, ["test1", "test2"]);
7540
7541 let mut fresh = consumer.announced();
7543 let b1 = fresh.try_next().expect("backlog: first");
7544 let b2 = fresh.try_next().expect("backlog: second");
7545 fresh.assert_next_wait();
7546
7547 let mut paths: Vec<_> = [&b1, &b2].iter().map(|a| a.path.to_string()).collect();
7548 paths.sort();
7549 assert_eq!(paths, ["test1", "test2"]);
7550 }
7551
7552 #[tokio::test]
7554 async fn dynamic_request_unroutable_without_handler() {
7555 let origin = Origin::random().produce();
7556 let consumer = origin.consume();
7557 assert!(matches!(
7558 consumer.request_broadcast("missing").await,
7559 Err(Error::Unroutable)
7560 ));
7561 }
7562
7563 #[tokio::test(start_paused = true)]
7566 async fn dynamic_request_served_not_announced() {
7567 let origin = Origin::random().produce();
7568 let mut dynamic = origin.dynamic();
7569 let consumer = origin.consume();
7570
7571 let mut announced = origin.consume().announced();
7573 announced.assert_next_wait();
7574
7575 let served = broadcast::Info::new().produce();
7576 let request_fut = consumer.request_broadcast("fallback");
7579
7580 let mut served_dynamic = served.dynamic();
7582
7583 let request = dynamic.requested_broadcast().await.unwrap();
7584 assert_eq!(request.path(), &Path::new("fallback"));
7585 request.accept(&served);
7586
7587 let broadcast = request_fut.await.unwrap();
7588 assert!(broadcast.is_clone(&served.consume()));
7589
7590 let track_fut = broadcast.track("video").unwrap().subscribe(None);
7592 let mut producer = served_dynamic.requested_track().await.unwrap().accept(None);
7593 let mut track = track_fut.await.unwrap();
7594 producer.append_group().unwrap();
7595 track.assert_group();
7596
7597 announced.assert_next_wait();
7599 }
7600
7601 #[tokio::test(start_paused = true)]
7603 async fn dynamic_request_coalesces() {
7604 let origin = Origin::random().produce();
7605 let mut dynamic = origin.dynamic();
7606 let consumer = origin.consume();
7607
7608 let f1 = consumer.request_broadcast("dup");
7610 let f2 = consumer.request_broadcast("dup");
7611
7612 let request = dynamic.requested_broadcast().await.unwrap();
7614 assert_eq!(request.path(), &Path::new("dup"));
7615 assert!(
7616 dynamic.requested_broadcast().now_or_never().is_none(),
7617 "a coalesced request must not be served twice"
7618 );
7619
7620 let served = broadcast::Info::new().produce();
7622 request.accept(&served);
7623 assert!(f1.await.unwrap().is_clone(&served.consume()));
7624 assert!(f2.await.unwrap().is_clone(&served.consume()));
7625 }
7626
7627 #[tokio::test(start_paused = true)]
7630 async fn dynamic_request_dedups_served() {
7631 let origin = Origin::random().produce();
7632 let mut dynamic = origin.dynamic();
7633 let consumer = origin.consume();
7634
7635 let request_fut = consumer.request_broadcast("fallback");
7636 let request = dynamic.requested_broadcast().await.unwrap();
7637 let served = broadcast::Info::new().produce();
7638 request.accept(&served);
7639 let first = request_fut.await.unwrap();
7640 assert!(first.is_clone(&served.consume()));
7641
7642 let second = consumer.request_broadcast("fallback").await.unwrap();
7644 assert!(second.is_clone(&served.consume()));
7645
7646 assert!(
7648 dynamic.requested_broadcast().now_or_never().is_none(),
7649 "a still-live served broadcast must not be re-requested from the handler"
7650 );
7651 }
7652
7653 #[tokio::test(start_paused = true)]
7655 async fn dynamic_request_reserves_after_close() {
7656 let origin = Origin::random().produce();
7657 let mut dynamic = origin.dynamic();
7658 let consumer = origin.consume();
7659
7660 let request_fut = consumer.request_broadcast("fallback");
7661 let request = dynamic.requested_broadcast().await.unwrap();
7662 let served = broadcast::Info::new().produce();
7663 request.accept(&served);
7664 request_fut.await.unwrap();
7665
7666 drop(served);
7668
7669 let request_fut = consumer.request_broadcast("fallback");
7671 let request = dynamic.requested_broadcast().await.unwrap();
7672 assert_eq!(request.path(), &Path::new("fallback"));
7673 let served = broadcast::Info::new().produce();
7674 request.accept(&served);
7675 assert!(request_fut.await.unwrap().is_clone(&served.consume()));
7676 }
7677
7678 #[tokio::test(start_paused = true)]
7681 async fn dynamic_request_served_cache_bounded() {
7682 let origin = Origin::random().produce();
7683 let mut dynamic = origin.dynamic();
7684 let consumer = origin.consume();
7685
7686 for i in 0..100 {
7687 let path = format!("one-shot/{i}");
7688 let request_fut = consumer.request_broadcast(&path);
7689 let request = dynamic.requested_broadcast().await.unwrap();
7690 let served = broadcast::Info::new().produce();
7691 request.accept(&served);
7692 request_fut.await.unwrap();
7693 drop(served);
7695 }
7696
7697 assert!(
7700 origin.dynamic.read().served.len() <= 4,
7701 "stale served entries must be reclaimed, not accumulate per distinct path: {}",
7702 origin.dynamic.read().served.len()
7703 );
7704 }
7705
7706 #[tokio::test(start_paused = true)]
7709 async fn dynamic_request_coalesces_after_handoff() {
7710 let origin = Origin::random().produce();
7711 let mut dynamic = origin.dynamic();
7712 let consumer = origin.consume();
7713
7714 let f1 = consumer.request_broadcast("fallback");
7715 let request = dynamic.requested_broadcast().await.unwrap();
7717
7718 let f2 = consumer.request_broadcast("fallback");
7720 assert!(
7721 dynamic.requested_broadcast().now_or_never().is_none(),
7722 "a repeat request during hand-off must coalesce, not re-queue"
7723 );
7724
7725 let served = broadcast::Info::new().produce();
7727 request.accept(&served);
7728 assert!(f1.await.unwrap().is_clone(&served.consume()));
7729 assert!(f2.await.unwrap().is_clone(&served.consume()));
7730 }
7731
7732 #[tokio::test(start_paused = true)]
7734 async fn dynamic_request_dropped_after_handoff() {
7735 let origin = Origin::random().produce();
7736 let mut dynamic = origin.dynamic();
7737 let consumer = origin.consume();
7738
7739 let f1 = consumer.request_broadcast("fallback");
7740 let request = dynamic.requested_broadcast().await.unwrap();
7741 let f2 = consumer.request_broadcast("fallback");
7742
7743 drop(request);
7745 assert!(matches!(f1.await, Err(Error::Unroutable)));
7746 assert!(matches!(f2.await, Err(Error::Unroutable)));
7747 }
7748
7749 #[tokio::test(start_paused = true)]
7751 async fn dynamic_request_rejected() {
7752 let origin = Origin::random().produce();
7753 let mut dynamic = origin.dynamic();
7754 let consumer = origin.consume();
7755
7756 let request_fut = consumer.request_broadcast("fallback");
7757
7758 let request = dynamic.requested_broadcast().await.unwrap();
7759 request.reject(Error::Cancel);
7760
7761 assert!(matches!(request_fut.await, Err(Error::Cancel)));
7762 }
7763
7764 #[tokio::test(start_paused = true)]
7768 async fn dynamic_request_rerequest_after_reject() {
7769 let origin = Origin::random().produce();
7770 let mut dynamic = origin.dynamic();
7771 let consumer = origin.consume();
7772
7773 let f1 = consumer.request_broadcast("fallback");
7774 dynamic.requested_broadcast().await.unwrap().reject(Error::Unroutable);
7775 assert!(matches!(f1.await, Err(Error::Unroutable)));
7776
7777 let served = broadcast::Info::new().produce();
7778 let f2 = consumer.request_broadcast("fallback");
7780 let request = dynamic.requested_broadcast().await.unwrap();
7781 assert_eq!(request.path(), &Path::new("fallback"));
7782 request.accept(&served);
7783 assert!(f2.await.unwrap().is_clone(&served.consume()));
7784 }
7785
7786 #[tokio::test(start_paused = true)]
7789 async fn dynamic_request_handler_dropped() {
7790 let origin = Origin::random().produce();
7791 let dynamic = origin.dynamic();
7792 let consumer = origin.consume();
7793
7794 let request_fut = consumer.request_broadcast("fallback");
7795 drop(dynamic);
7796 assert!(matches!(request_fut.await, Err(Error::Unroutable)));
7797
7798 assert!(matches!(
7800 consumer.request_broadcast("again").await,
7801 Err(Error::Unroutable)
7802 ));
7803 }
7804
7805 #[tokio::test(start_paused = true)]
7809 async fn dynamic_request_accept_after_handler_dropped() {
7810 let origin = Origin::random().produce();
7811 let mut dynamic = origin.dynamic();
7812 let consumer = origin.consume();
7813
7814 let request_fut = consumer.request_broadcast("fallback");
7815
7816 let request = dynamic.requested_broadcast().await.unwrap();
7818 drop(dynamic);
7819
7820 let served = broadcast::Info::new().produce();
7821 request.accept(&served);
7823 assert!(request_fut.await.unwrap().is_clone(&served.consume()));
7824 }
7825
7826 #[tokio::test(start_paused = true)]
7828 async fn dynamic_request_prefers_announced() {
7829 let origin = Origin::random().produce();
7830 let mut dynamic = origin.dynamic();
7831 let consumer = origin.consume();
7832
7833 let _broadcast = origin.create_broadcast("live", announce()).unwrap();
7834 settle().await;
7835
7836 let got = consumer.request_broadcast("live").await.unwrap();
7837 assert!(
7838 got.is_clone(&consumer.get_broadcast("live").unwrap()),
7839 "should return the published broadcast"
7840 );
7841 assert!(
7842 dynamic.requested_broadcast().now_or_never().is_none(),
7843 "a published path must not queue a fallback request"
7844 );
7845 }
7846
7847 #[tokio::test(start_paused = true)]
7849 async fn dynamic_clone_keeps_alive() {
7850 let origin = Origin::random().produce();
7851 let dynamic = origin.dynamic();
7852 let consumer = origin.consume();
7853
7854 drop(dynamic.clone());
7855
7856 let request_fut = consumer.request_broadcast("fallback");
7859 assert!(
7860 request_fut.now_or_never().is_none(),
7861 "request should stay pending until served"
7862 );
7863 }
7864
7865 fn wedge_watchdog<F>(name: &str, secs: u64, scenario: F)
7871 where
7872 F: std::future::Future<Output = ()> + Send + 'static,
7873 {
7874 let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
7875 let handle = std::thread::spawn(move || {
7876 let rt = ::tokio::runtime::Builder::new_current_thread()
7877 .enable_time()
7878 .build()
7879 .unwrap();
7880 rt.block_on(scenario);
7881 let _ = done_tx.send(());
7882 });
7883 match done_rx.recv_timeout(std::time::Duration::from_secs(secs)) {
7884 Ok(()) => {
7885 let _ = handle.join();
7886 }
7887 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
7888 let err = handle.join().unwrap_err();
7890 std::panic::resume_unwind(err);
7891 }
7892 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
7893 panic!("{name}: scenario wedged; a task is spinning inside a single poll")
7894 }
7895 }
7896 }
7897
7898 #[test]
7911 fn test_active_corpse_does_not_livelock_takeover() {
7912 wedge_watchdog("active-corpse", 20, async {
7913 let origin = Origin::random().produce();
7914 let consumer = origin.consume();
7915 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
7916
7917 let source_a = origin
7920 .create_broadcast("test", announce().with_hops(hops.clone()))
7921 .unwrap();
7922 let mut dynamic_a = source_a.dynamic();
7923 settle().await;
7924 settle().await;
7925
7926 let broadcast = consumer.request_broadcast("test").await.unwrap();
7927 let subscribing = broadcast.track("video").unwrap().subscribe(None);
7928 let mut producer_a = accept_track(&mut dynamic_a, "video").await;
7929 settle().await;
7930 let mut sub = subscribing.await.unwrap();
7931 producer_a.append_group().unwrap();
7932 sub.assert_group();
7933
7934 let source_b = origin
7938 .create_broadcast("test", announce().with_hops(hops.clone()))
7939 .unwrap();
7940 let dynamic_b = source_b.dynamic();
7941 settle().await;
7942
7943 drop(dynamic_b);
7951 source_b.abort(Error::Dropped).unwrap();
7952 settle().await;
7953
7954 producer_a.append_group().unwrap();
7957 sub.assert_group();
7958 sub.assert_not_closed();
7959 });
7960 }
7961
7962 #[test]
7970 fn test_route_churn_never_wedges() {
7971 for seed in 1..=8u64 {
7972 wedge_watchdog(&format!("churn seed {seed}"), 30, churn_scenario(seed));
7973 }
7974 }
7975
7976 async fn churn_scenario(seed: u64) {
7977 let mut rng = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
7979 let mut next = move || {
7980 rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
7981 rng >> 33
7982 };
7983
7984 let origin = Origin::random().produce();
7985 let consumer = origin.consume();
7986 let names: Vec<Arc<str>> = (0..8).map(|i| Arc::from(format!("t{i}"))).collect();
7987
7988 let mut subs: Vec<track::Subscriber> = Vec::new();
7989 let mut pending_subs: Vec<kio::Pending<track::Subscribing>> = Vec::new();
7990
7991 struct Source {
7992 producer: Option<broadcast::Producer>,
7993 server: ::tokio::task::JoinHandle<()>,
7994 }
7995 let mut sources: Vec<Source> = Vec::new();
7996
7997 for step in 0..400u64 {
7998 match next() % 10 {
7999 0 | 1 => {
8002 if sources.len() >= 3 {
8003 continue;
8004 }
8005 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
8006 let route = announce().with_hops(hops).with_cost(next() % 4);
8007 let Ok(source) = origin.create_broadcast("test", route) else {
8008 continue;
8009 };
8010 let mut dynamic = source.dynamic();
8011 let behavior = next();
8012 let server = ::tokio::spawn(async move {
8013 let mut round = 0u64;
8014 let mut kept: Vec<track::Producer> = Vec::new();
8015 while let Ok(request) = dynamic.requested_track().await {
8016 round += 1;
8017 match (behavior >> (round % 16)) % 4 {
8018 0 => drop(request), 1 => {
8020 let mut producer = request.accept(None);
8021 let _ = producer.create_group(group::Info { sequence: round });
8022 let _ = producer.abort(Error::Dropped);
8023 }
8024 2 => {
8025 let mut producer = request.accept(None);
8026 let _ = producer.create_group(group::Info { sequence: round });
8027 kept.push(producer);
8028 }
8029 _ => {
8030 let mut producer = request.accept(None);
8031 let _ = producer.finish();
8032 }
8033 }
8034 }
8035 });
8036 sources.push(Source {
8037 producer: Some(source),
8038 server,
8039 });
8040 }
8041 2 | 3 => {
8043 if sources.is_empty() {
8044 continue;
8045 }
8046 let i = (next() as usize) % sources.len();
8047 let mut source = sources.swap_remove(i);
8048 if next() % 2 == 0
8049 && let Some(producer) = source.producer.take()
8050 {
8051 let _ = producer.abort(Error::Dropped);
8052 }
8053 source.server.abort();
8054 }
8055 4..=6 => {
8057 if subs.len() + pending_subs.len() >= 24 {
8058 continue;
8059 }
8060 let Some(broadcast) = consumer.get_broadcast("test") else {
8061 continue;
8062 };
8063 let name = &names[(next() as usize) % names.len()];
8064 if let Ok(track) = broadcast.track(name.as_ref()) {
8065 pending_subs.push(track.subscribe(None));
8066 }
8067 }
8068 7 => {
8070 if subs.is_empty() {
8071 continue;
8072 }
8073 let i = (next() as usize) % subs.len();
8074 subs.swap_remove(i);
8075 }
8076 _ => {
8078 for sub in pending_subs.drain(..) {
8079 match ::tokio::time::timeout(std::time::Duration::from_millis(5), sub).await {
8080 Ok(Ok(sub)) => subs.push(sub),
8081 Ok(Err(_)) => {}
8082 Err(_) => {}
8084 }
8085 }
8086 for sub in subs.iter_mut() {
8087 while let Some(Ok(Some(_))) = sub.recv_group().now_or_never() {}
8088 }
8089 }
8090 }
8091 if step % 16 == 0 {
8092 settle().await;
8093 }
8094 for _ in 0..(next() % 3) {
8096 ::tokio::task::yield_now().await;
8097 }
8098 }
8099
8100 for source in sources {
8101 source.server.abort();
8102 }
8103 settle().await;
8104 }
8105}