1use crate::{broadcast, cache, stats, track};
2use kio::Pollable;
3use std::{
4 cmp::Reverse,
5 collections::{BTreeMap, HashMap, HashSet},
6 fmt,
7 sync::Arc,
8 sync::atomic::{AtomicU64, Ordering},
9 task::{Poll, ready},
10 time::Duration,
11};
12
13use rand::RngExt;
14use web_async::Lock;
15
16use super::{Requests, WeakCache};
17use crate::{
18 AsPath, Error, Path, PathOwned, PathPrefixes,
19 coding::{BoundsExceeded, Decode, DecodeError, Encode, EncodeError},
20};
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
28pub struct Origin {
29 id: u64,
31}
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35#[non_exhaustive]
36pub struct InvalidOrigin;
37
38impl fmt::Display for InvalidOrigin {
39 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40 write!(f, "local origin id must be non-zero and below 2^62")
41 }
42}
43
44impl std::error::Error for InvalidOrigin {}
45
46impl Origin {
47 pub(crate) const UNKNOWN: Self = Self { id: 0 };
50
51 pub fn new(id: u64) -> Result<Self, InvalidOrigin> {
57 if id == 0 || id >= 1u64 << 62 {
58 return Err(InvalidOrigin);
59 }
60 Ok(Self { id })
61 }
62
63 pub fn random() -> Self {
72 let mut rng = rand::rng();
73 let id = rng.random_range(1..(1u64 << 53));
74 Self { id }
75 }
76
77 pub fn id(self) -> u64 {
79 self.id
80 }
81
82 pub fn produce(self) -> Producer {
85 Info::new(self).produce()
86 }
87}
88
89#[derive(Clone, Debug)]
99#[non_exhaustive]
100pub struct Info {
101 pub id: Origin,
104
105 pub pool: cache::Pool,
111
112 pub cache_duration: Duration,
119
120 pub latency_default: Duration,
130
131 pub linger: Duration,
140}
141
142impl Default for Info {
143 fn default() -> Self {
146 Self {
147 id: Origin::UNKNOWN,
148 pool: cache::Pool::default(),
149 cache_duration: Duration::MAX,
150 latency_default: track::DEFAULT_LATENCY_MAX,
151 linger: Duration::ZERO,
152 }
153 }
154}
155
156impl Info {
157 pub fn new(id: Origin) -> Self {
159 Self { id, ..Self::default() }
160 }
161
162 pub fn with_pool(mut self, pool: cache::Pool) -> Self {
164 self.pool = pool;
165 self
166 }
167
168 pub fn with_cache_duration(mut self, cache_duration: Duration) -> Self {
171 self.cache_duration = cache_duration;
172 self
173 }
174
175 pub fn with_latency_default(mut self, latency_default: Duration) -> Self {
178 self.latency_default = latency_default;
179 self
180 }
181
182 pub fn with_linger(mut self, linger: Duration) -> Self {
185 self.linger = linger;
186 self
187 }
188
189 pub fn produce(self) -> Producer {
191 Producer::new(self)
192 }
193}
194
195impl TryFrom<u64> for Origin {
196 type Error = InvalidOrigin;
197
198 fn try_from(id: u64) -> Result<Self, Self::Error> {
199 Self::new(id)
200 }
201}
202
203impl fmt::Display for Origin {
204 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
205 self.id.fmt(f)
206 }
207}
208
209impl<V: Copy> Encode<V> for Origin
210where
211 u64: Encode<V>,
212{
213 fn encode<W: bytes::BufMut>(&self, w: &mut W, version: V) -> Result<(), EncodeError> {
214 self.id.encode(w, version)
215 }
216}
217
218impl<V: Copy> Decode<V> for Origin
219where
220 u64: Decode<V>,
221{
222 fn decode<R: bytes::Buf>(r: &mut R, version: V) -> Result<Self, DecodeError> {
223 let id = u64::decode(r, version)?;
224 if id >= 1u64 << 62 {
225 return Err(DecodeError::InvalidValue);
226 }
227 Ok(Self { id })
228 }
229}
230
231pub(crate) const MAX_HOPS: usize = 32;
237
238#[derive(Debug, Clone, Default, PartialEq, Eq)]
243pub struct OriginList(Vec<Origin>);
244
245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247#[non_exhaustive]
248pub struct TooManyOrigins;
249
250impl fmt::Display for TooManyOrigins {
251 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
252 write!(f, "too many origins (max {MAX_HOPS})")
253 }
254}
255
256impl std::error::Error for TooManyOrigins {}
257
258impl From<TooManyOrigins> for DecodeError {
259 fn from(_: TooManyOrigins) -> Self {
260 DecodeError::BoundsExceeded
261 }
262}
263
264impl OriginList {
265 pub fn new() -> Self {
267 Self(Vec::new())
268 }
269
270 pub fn push(&mut self, origin: Origin) -> Result<(), TooManyOrigins> {
272 if self.0.len() >= MAX_HOPS {
273 return Err(TooManyOrigins);
274 }
275 self.0.push(origin);
276 Ok(())
277 }
278
279 pub fn replace_first(&mut self, target: Origin, replacement: Origin) -> bool {
282 for entry in &mut self.0 {
283 if *entry == target {
284 *entry = replacement;
285 return true;
286 }
287 }
288 false
289 }
290
291 pub fn contains(&self, origin: &Origin) -> bool {
293 self.0.contains(origin)
294 }
295
296 pub fn len(&self) -> usize {
298 self.0.len()
299 }
300
301 pub fn is_empty(&self) -> bool {
303 self.0.is_empty()
304 }
305
306 pub fn iter(&self) -> std::slice::Iter<'_, Origin> {
308 self.0.iter()
309 }
310
311 pub fn as_slice(&self) -> &[Origin] {
313 &self.0
314 }
315}
316
317impl TryFrom<Vec<Origin>> for OriginList {
318 type Error = TooManyOrigins;
319
320 fn try_from(v: Vec<Origin>) -> Result<Self, Self::Error> {
321 if v.len() > MAX_HOPS {
322 return Err(TooManyOrigins);
323 }
324 Ok(Self(v))
325 }
326}
327
328impl<'a> IntoIterator for &'a OriginList {
329 type Item = &'a Origin;
330 type IntoIter = std::slice::Iter<'a, Origin>;
331
332 fn into_iter(self) -> Self::IntoIter {
333 self.iter()
334 }
335}
336
337impl<V: Copy> Encode<V> for OriginList
338where
339 u64: Encode<V>,
340 Origin: Encode<V>,
341{
342 fn encode<W: bytes::BufMut>(&self, w: &mut W, version: V) -> Result<(), EncodeError> {
343 (self.0.len() as u64).encode(w, version)?;
344 for origin in &self.0 {
345 origin.encode(w, version)?;
346 }
347 Ok(())
348 }
349}
350
351impl<V: Copy> Decode<V> for OriginList
352where
353 u64: Decode<V>,
354 Origin: Decode<V>,
355{
356 fn decode<R: bytes::Buf>(r: &mut R, version: V) -> Result<Self, DecodeError> {
357 let count = u64::decode(r, version)? as usize;
358 if count > MAX_HOPS {
359 return Err(DecodeError::BoundsExceeded);
360 }
361 let mut list = Vec::with_capacity(count);
362 for _ in 0..count {
363 list.push(Origin::decode(r, version)?);
364 }
365 Ok(Self(list))
366 }
367}
368
369static NEXT_CONSUMER_ID: AtomicU64 = AtomicU64::new(0);
370
371#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
372struct ConsumerId(u64);
373
374impl ConsumerId {
375 fn new() -> Self {
376 Self(NEXT_CONSUMER_ID.fetch_add(1, Ordering::Relaxed))
377 }
378}
379
380struct OriginBroadcast {
386 path: PathOwned,
387 broadcast: broadcast::Producer,
389 state: kio::Producer<FrontState>,
392 announced: bool,
393}
394
395fn route_key(name: &Path, hops: &OriginList) -> (usize, u64) {
404 (hops.len(), fnv_key(name, hops.iter().copied()))
405}
406
407fn fnv_key(name: &Path, origins: impl IntoIterator<Item = Origin>) -> u64 {
420 const SEED: u64 = 0x420C0DECB00B; const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
422
423 let mut hash = SEED;
424 for &byte in name.as_str().as_bytes() {
425 hash = (hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME);
426 }
427 for origin in origins {
428 for &byte in &origin.id().to_le_bytes() {
429 hash = (hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME);
430 }
431 }
432
433 hash
434}
435
436fn route_order(name: &Path, route: &FrontRoute) -> (bool, u64, usize, u64, Reverse<u64>) {
455 let (len, hash) = route_key(name, &route.route.hops);
456 (!route.route.announce, route.route.cost, len, hash, Reverse(route.id))
457}
458
459enum PendingUpdate {
468 Announce(broadcast::Consumer),
469 Unannounce,
470 UnannounceAnnounce(broadcast::Consumer),
471}
472
473#[derive(Default)]
478struct OriginConsumerState {
479 pending: BTreeMap<PathOwned, PendingUpdate>,
480}
481
482impl OriginConsumerState {
483 fn apply_announce(&mut self, path: PathOwned, broadcast: broadcast::Consumer) {
484 let new = match self.pending.remove(&path) {
485 None | Some(PendingUpdate::Announce(_)) => PendingUpdate::Announce(broadcast),
487 Some(PendingUpdate::Unannounce | PendingUpdate::UnannounceAnnounce(_)) => {
489 PendingUpdate::UnannounceAnnounce(broadcast)
490 }
491 };
492 self.pending.insert(path, new);
493 }
494
495 fn apply_unannounce(&mut self, path: PathOwned) {
496 match self.pending.remove(&path) {
497 Some(PendingUpdate::Announce(_)) => {}
499 None | Some(PendingUpdate::Unannounce) => {
500 self.pending.insert(path, PendingUpdate::Unannounce);
501 }
502 Some(PendingUpdate::UnannounceAnnounce(_)) => {
505 self.pending.insert(path, PendingUpdate::Unannounce);
506 }
507 }
508 }
509
510 fn take(&mut self) -> Option<OriginAnnounce> {
512 let path = self.pending.keys().next()?.clone();
513 let broadcast = match self.pending.remove(&path).unwrap() {
514 PendingUpdate::Announce(broadcast) => Some(broadcast),
515 PendingUpdate::Unannounce => None,
516 PendingUpdate::UnannounceAnnounce(broadcast) => {
517 self.pending.insert(path.clone(), PendingUpdate::Announce(broadcast));
520 None
521 }
522 };
523 Some(OriginAnnounce { path, broadcast })
524 }
525}
526
527#[derive(Clone)]
528struct AnnounceConsumerNotify {
529 root: PathOwned,
530 state: kio::Producer<OriginConsumerState>,
531 exclude: Option<Origin>,
536}
537
538impl AnnounceConsumerNotify {
539 fn announce(&self, path: impl AsPath, broadcast: broadcast::Consumer, front: &kio::Producer<FrontState>) {
540 let path = path.as_path().strip_prefix(&self.root).unwrap().to_owned();
541
542 let broadcast = match self.exclude.and_then(|peer| ExclusionGuard::new(front, peer)) {
547 Some(guard) => broadcast.with_exclusion(guard),
548 None => broadcast,
549 };
550
551 self.state
552 .write()
553 .ok()
554 .expect("consumer closed")
555 .apply_announce(path, broadcast);
556 }
557
558 fn unannounce(&self, path: impl AsPath) {
559 let path = path.as_path().strip_prefix(&self.root).unwrap().to_owned();
560 self.state.write().ok().expect("consumer closed").apply_unannounce(path);
561 }
562}
563
564struct NotifyNode {
565 parent: Option<Lock<NotifyNode>>,
566
567 consumers: HashMap<ConsumerId, AnnounceConsumerNotify>,
570}
571
572impl NotifyNode {
573 fn new(parent: Option<Lock<NotifyNode>>) -> Self {
574 Self {
575 parent,
576 consumers: HashMap::new(),
577 }
578 }
579
580 fn announce(&mut self, path: impl AsPath, broadcast: &broadcast::Consumer, state: &kio::Producer<FrontState>) {
584 for consumer in self.consumers.values() {
585 consumer.announce(path.as_path(), broadcast.clone(), state);
586 }
587
588 if let Some(parent) = &self.parent {
589 parent.lock().announce(path, broadcast, state);
590 }
591 }
592
593 fn unannounce(&mut self, path: impl AsPath) {
594 for consumer in self.consumers.values() {
595 consumer.unannounce(path.as_path());
596 }
597
598 if let Some(parent) = &self.parent {
599 parent.lock().unannounce(path);
600 }
601 }
602}
603
604pub(crate) struct ExclusionGuard {
611 state: kio::Producer<FrontState>,
612 peer: Origin,
613}
614
615impl ExclusionGuard {
616 fn new(state: &kio::Producer<FrontState>, peer: Origin) -> Option<Arc<Self>> {
619 let mut s = state.write().ok()?;
620 if s.closed {
621 return None;
622 }
623 *s.excluded.entry(peer).or_default() += 1;
624 drop(s);
625 Some(Arc::new(Self {
626 state: state.clone(),
627 peer,
628 }))
629 }
630}
631
632impl Drop for ExclusionGuard {
633 fn drop(&mut self) {
634 let Ok(mut state) = self.state.write() else { return };
635 if let std::collections::hash_map::Entry::Occupied(mut entry) = state.excluded.entry(self.peer) {
636 match entry.get() {
637 1 => drop(entry.remove()),
638 n => *entry.get_mut() = n - 1,
639 }
640 }
641 }
642}
643
644enum Resolved {
651 Found(broadcast::Consumer),
654 Excluded,
656 Missing,
658}
659
660struct OriginNode {
661 broadcast: Option<OriginBroadcast>,
664
665 nested: HashMap<String, Lock<OriginNode>>,
667
668 notify: Lock<NotifyNode>,
670}
671
672impl OriginNode {
673 fn new(parent: Option<Lock<NotifyNode>>) -> Self {
674 Self {
675 broadcast: None,
676 nested: HashMap::new(),
677 notify: Lock::new(NotifyNode::new(parent)),
678 }
679 }
680
681 fn leaf(&mut self, path: &Path) -> Lock<OriginNode> {
682 let (dir, rest) = path.next_part().expect("leaf called with empty path");
683
684 let next = self.entry(dir);
685 if rest.is_empty() { next } else { next.lock().leaf(&rest) }
686 }
687
688 fn entry(&mut self, dir: &str) -> Lock<OriginNode> {
689 match self.nested.get(dir) {
690 Some(next) => next.clone(),
691 None => {
692 let next = Lock::new(OriginNode::new(Some(self.notify.clone())));
693 self.nested.insert(dir.to_string(), next.clone());
694 next
695 }
696 }
697 }
698
699 fn set_announced(&mut self, expect: &kio::Producer<FrontState>, announce: bool) {
703 let Some(existing) = &mut self.broadcast else { return };
704 if !existing.state.same_channel(expect) || existing.announced == announce {
705 return;
706 }
707 existing.announced = announce;
708 let path = existing.path.clone();
709 let consumer = existing.broadcast.consume();
710 let state = existing.state.clone();
711 let mut notify = self.notify.lock();
712 if announce {
713 notify.announce(&path, &consumer, &state);
714 } else {
715 notify.unannounce(&path);
716 }
717 }
718
719 fn consume(&mut self, id: ConsumerId, mut notify: AnnounceConsumerNotify) {
720 self.consume_initial(&mut notify);
721 self.notify.lock().consumers.insert(id, notify);
722 }
723
724 fn consume_initial(&mut self, notify: &mut AnnounceConsumerNotify) {
725 if let Some(broadcast) = &self.broadcast
728 && broadcast.announced
729 {
730 notify.announce(&broadcast.path, broadcast.broadcast.consume(), &broadcast.state);
731 }
732
733 for nested in self.nested.values() {
735 nested.lock().consume_initial(notify);
736 }
737 }
738
739 fn resolve_broadcast(&self, rest: impl AsPath, exclude: Option<Origin>) -> Resolved {
740 let rest = rest.as_path();
741
742 if let Some((dir, rest)) = rest.next_part() {
743 let Some(node) = self.nested.get(dir) else {
744 return Resolved::Missing;
745 };
746 let node = node.lock();
747 return node.resolve_broadcast(&rest, exclude);
748 }
749
750 let Some(broadcast) = self.broadcast.as_ref() else {
751 return Resolved::Missing;
752 };
753 let Some(origin) = exclude else {
754 return Resolved::Found(broadcast.broadcast.consume());
755 };
756
757 let state = broadcast.state.read();
773 if !state.routes.iter().any(|r| r.route.hops.contains(&origin)) {
774 drop(state);
775 let shared = broadcast.broadcast.consume();
776 return match ExclusionGuard::new(&broadcast.state, origin) {
777 Some(guard) => Resolved::Found(shared.with_exclusion(guard)),
778 None => Resolved::Found(shared),
781 };
782 }
783 match state
784 .dispatch(Some(origin))
785 .and_then(|clean| state.routes.iter().find(|r| r.id == clean))
786 {
787 Some(route) => Resolved::Found(route.source.clone()),
788 None => Resolved::Excluded,
789 }
790 }
791
792 fn unconsume(&mut self, id: ConsumerId) {
793 self.notify.lock().consumers.remove(&id).expect("consumer not found");
794 if self.is_empty() {
795 }
798 }
799
800 fn remove(&mut self, expect: &kio::Producer<FrontState>, relative: impl AsPath) {
804 let relative = relative.as_path();
805
806 if let Some((dir, relative)) = relative.next_part() {
807 let Some(nested) = self.nested.get(dir) else { return };
808 let nested = nested.clone();
809 let mut locked = nested.lock();
810 locked.remove(expect, &relative);
811
812 if locked.is_empty() {
813 drop(locked);
814 self.nested.remove(dir);
815 }
816 } else if let Some(existing) = &self.broadcast
817 && existing.state.same_channel(expect)
818 {
819 let existing = self.broadcast.take().expect("checked above");
820 if existing.announced {
821 self.notify.lock().unannounce(&existing.path);
822 }
823 }
824 }
825
826 fn is_empty(&self) -> bool {
827 self.broadcast.is_none() && self.nested.is_empty() && self.notify.lock().consumers.is_empty()
828 }
829}
830
831#[derive(Clone)]
832struct OriginNodes {
833 nodes: Vec<(PathOwned, Lock<OriginNode>)>,
834}
835
836impl OriginNodes {
837 pub fn select(&self, prefixes: &PathPrefixes) -> Option<Self> {
840 let mut roots = Vec::new();
841
842 for (root, state) in &self.nodes {
843 for prefix in prefixes {
844 if root.has_prefix(prefix) {
845 roots.push((root.to_owned(), state.clone()));
847 continue;
848 }
849
850 if let Some(suffix) = prefix.strip_prefix(root) {
851 let nested = state.lock().leaf(&suffix);
853 roots.push((prefix.to_owned(), nested));
854 }
855 }
856 }
857
858 if roots.is_empty() {
859 None
860 } else {
861 Some(Self { nodes: roots })
862 }
863 }
864
865 pub fn root(&self, new_root: impl AsPath) -> Option<Self> {
866 let new_root = new_root.as_path();
867 let mut roots = Vec::new();
868
869 if new_root.is_empty() {
870 return Some(self.clone());
871 }
872
873 for (root, state) in &self.nodes {
874 if let Some(suffix) = root.strip_prefix(&new_root) {
875 roots.push((suffix.to_owned(), state.clone()));
877 } else if let Some(suffix) = new_root.strip_prefix(root) {
878 let nested = state.lock().leaf(&suffix);
881 roots.push(("".into(), nested));
882 }
883 }
884
885 if roots.is_empty() {
886 None
887 } else {
888 Some(Self { nodes: roots })
889 }
890 }
891
892 pub fn get(&self, path: impl AsPath) -> Option<(Lock<OriginNode>, PathOwned)> {
894 let path = path.as_path();
895
896 for (root, state) in &self.nodes {
897 if let Some(suffix) = path.strip_prefix(root) {
898 return Some((state.clone(), suffix.to_owned()));
899 }
900 }
901
902 None
903 }
904}
905
906impl Default for OriginNodes {
907 fn default() -> Self {
908 Self {
909 nodes: vec![("".into(), Lock::new(OriginNode::new(None)))],
910 }
911 }
912}
913
914#[derive(Clone)]
916pub struct OriginAnnounce {
917 pub path: PathOwned,
919 pub broadcast: Option<broadcast::Consumer>,
925}
926
927#[derive(Clone)]
929pub struct Producer {
930 info: Origin,
934
935 nodes: OriginNodes,
938
939 root: PathOwned,
941
942 dynamic: kio::Shared<OriginDynamicState>,
946
947 pool: cache::Pool,
950
951 cache_duration: Duration,
954
955 latency_default: Duration,
958
959 linger: Duration,
962
963 stats: stats::Session,
967}
968
969impl std::ops::Deref for Producer {
970 type Target = Origin;
971
972 fn deref(&self) -> &Self::Target {
973 &self.info
974 }
975}
976
977impl Producer {
978 pub fn new(info: Info) -> Self {
982 Self {
983 info: info.id,
984 nodes: OriginNodes::default(),
985 root: PathOwned::default(),
986 dynamic: kio::Shared::default(),
987 pool: info.pool,
988 cache_duration: info.cache_duration,
989 latency_default: info.latency_default,
990 linger: info.linger,
991 stats: stats::Session::default(),
992 }
993 }
994
995 pub fn with_stats(mut self, session: stats::Session) -> Self {
999 self.stats = session;
1000 self
1001 }
1002
1003 pub fn with_linger(mut self, linger: Duration) -> Self {
1012 self.linger = linger;
1013 self
1014 }
1015
1016 pub fn info(&self) -> Info {
1019 Info {
1020 id: self.info,
1021 pool: self.pool.clone(),
1022 cache_duration: self.cache_duration,
1023 latency_default: self.latency_default,
1024 linger: self.linger,
1025 }
1026 }
1027
1028 pub(crate) fn latency_default(&self) -> Duration {
1031 self.latency_default
1032 }
1033
1034 pub(crate) fn empty(info: Origin) -> Self {
1039 Self {
1040 info,
1041 nodes: OriginNodes { nodes: Vec::new() },
1042 root: PathOwned::default(),
1043 dynamic: kio::Shared::default(),
1044 pool: cache::Pool::default(),
1045 cache_duration: Duration::MAX,
1046 latency_default: track::DEFAULT_LATENCY_MAX,
1047 linger: Duration::ZERO,
1048 stats: stats::Session::default(),
1049 }
1050 }
1051
1052 pub fn create_broadcast(&self, path: impl AsPath, route: broadcast::Route) -> Result<broadcast::Producer, Error> {
1103 let path = path.as_path();
1104
1105 debug_assert!(
1106 !route.hops.contains(&self.info),
1107 "create_broadcast called with a looping hop chain",
1108 );
1109
1110 let (node, rest) = self.nodes.get(&path).ok_or(Error::Unauthorized)?;
1111 let full = self.root.join(&path).to_owned();
1112
1113 if full.parts().count() > Path::MAX_PARTS {
1117 return Err(BoundsExceeded.into());
1118 }
1119
1120 let ingress = self.stats.ingress(&full);
1124
1125 let mut source = broadcast::Info { origin: self.info() }
1126 .produce()
1127 .with_stats(ingress.clone());
1128 source.set_route(route).expect("fresh producer");
1129
1130 web_async::spawn(run_source(self.info(), node, full, rest, source.consume(), ingress));
1131
1132 Ok(source)
1133 }
1134
1135 pub fn scope(&self, prefixes: &[Path]) -> Option<Producer> {
1141 let prefixes = PathPrefixes::new(prefixes);
1142 Some(Producer {
1143 info: self.info,
1144 nodes: self.nodes.select(&prefixes)?,
1145 root: self.root.clone(),
1146 dynamic: self.dynamic.clone(),
1147 pool: self.pool.clone(),
1148 cache_duration: self.cache_duration,
1149 latency_default: self.latency_default,
1150 linger: self.linger,
1151 stats: self.stats.clone(),
1152 })
1153 }
1154
1155 pub fn dynamic(&self) -> Dynamic {
1164 Dynamic::new(self.info, self.root.clone(), self.dynamic.clone())
1165 }
1166
1167 pub fn consume(&self) -> Consumer {
1172 Consumer::new(
1175 self.info,
1176 self.root.clone(),
1177 self.nodes.clone(),
1178 self.dynamic.clone(),
1179 stats::Session::default(),
1180 )
1181 }
1182
1183 pub fn announces(&self) -> AnnounceProducer {
1189 AnnounceProducer::new(self.root.clone(), self.nodes.clone())
1190 }
1191
1192 pub fn with_root(&self, prefix: impl AsPath) -> Option<Self> {
1197 let prefix = prefix.as_path();
1198
1199 Some(Self {
1200 info: self.info,
1201 root: self.root.join(&prefix).to_owned(),
1202 nodes: self.nodes.root(&prefix)?,
1203 dynamic: self.dynamic.clone(),
1204 pool: self.pool.clone(),
1205 cache_duration: self.cache_duration,
1206 latency_default: self.latency_default,
1207 linger: self.linger,
1208 stats: self.stats.clone(),
1209 })
1210 }
1211
1212 pub fn root(&self) -> &Path<'_> {
1214 &self.root
1215 }
1216
1217 pub fn allowed(&self) -> impl Iterator<Item = &Path<'_>> {
1220 self.nodes.nodes.iter().map(|(root, _)| root)
1221 }
1222
1223 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
1225 self.root.join(path)
1226 }
1227}
1228
1229const TRACK_IDLE_LINGER: Duration = Duration::from_secs(30);
1242
1243struct FrontRoute {
1245 id: u64,
1246 route: broadcast::Route,
1249 source: broadcast::Consumer,
1251}
1252
1253struct FrontState {
1255 path: PathOwned,
1257 self_origin: Origin,
1259 publisher: Option<Origin>,
1267 next_route: u64,
1270 routes: Vec<FrontRoute>,
1271 excluded: HashMap<Origin, usize>,
1284 active: Option<u64>,
1286 linger: Duration,
1289 closed: bool,
1294}
1295
1296impl FrontState {
1297 fn pick(&self, keep: impl Fn(&FrontRoute) -> bool, untainted: bool) -> Option<u64> {
1304 let candidates: Vec<&FrontRoute> = self.routes.iter().filter(|r| keep(r)).collect();
1305 let candidates = match untainted {
1306 true => self.prefer_untainted(&candidates),
1307 false => candidates,
1308 };
1309 candidates
1310 .into_iter()
1311 .min_by_key(|r| route_order(&self.path.as_path(), r))
1312 .map(|r| r.id)
1313 }
1314
1315 fn best_route(&self) -> Option<u64> {
1320 self.pick(|_| true, true)
1321 }
1322
1323 fn dispatch(&self, exclude: Option<Origin>) -> Option<u64> {
1330 self.pick(|r| exclude.is_none_or(|origin| !r.route.hops.contains(&origin)), false)
1331 }
1332
1333 fn taints_a_reader(&self, route: &broadcast::Route) -> bool {
1340 route.hops.iter().any(|hop| self.excluded.contains_key(hop))
1341 }
1342
1343 fn prefer_untainted<'a>(&self, candidates: &[&'a FrontRoute]) -> Vec<&'a FrontRoute> {
1353 if self.excluded.is_empty() {
1354 return candidates.to_vec();
1355 }
1356 let clean: Vec<&FrontRoute> = candidates
1357 .iter()
1358 .copied()
1359 .filter(|r| !self.taints_a_reader(&r.route))
1360 .collect();
1361 match clean.is_empty() {
1362 true => candidates.to_vec(),
1363 false => clean,
1364 }
1365 }
1366
1367 fn serve_route(&self, skip: impl Fn(u64) -> bool) -> Option<u64> {
1376 if let Some(active) = self.active
1377 && !skip(active)
1378 && let Some(route) = self.routes.iter().find(|r| r.id == active)
1379 && !self.taints_a_reader(&route.route)
1380 {
1381 return Some(active);
1382 }
1383 self.pick(|r| !skip(r.id), true)
1384 }
1385
1386 fn reselect(&mut self, carrying: bool) {
1404 let best = self.best_route();
1405 if carrying
1406 && let (Some(best_id), Some(cur_id)) = (best, self.active)
1407 && best_id != cur_id
1408 && let Some(candidate) = self.routes.iter().find(|r| r.id == best_id)
1409 && let Some(incumbent) = self.routes.iter().find(|r| r.id == cur_id)
1410 && incumbent.route.announce
1411 && candidate.route.cost < incumbent.route.cost
1412 && candidate.route.advertised == 0
1413 && candidate.route.hops.len() >= 2
1414 && !self.handover_allowed(&candidate.route)
1415 {
1416 return;
1418 }
1419 self.active = best;
1420 }
1421
1422 fn handover_allowed(&self, route: &broadcast::Route) -> bool {
1434 let name = self.path.as_path();
1435 match route.hops.iter().last() {
1436 Some(peer) => fnv_key(&name, [*peer]) < fnv_key(&name, [self.self_origin]),
1437 None => true,
1438 }
1439 }
1440
1441 fn routes_snapshot(&self) -> Vec<broadcast::Route> {
1447 let mut routes: Vec<&FrontRoute> = self.routes.iter().collect();
1448 routes.sort_by_key(|r| route_order(&self.path.as_path(), r));
1449 routes.sort_by_key(|r| Some(r.id) != self.active);
1450 routes.into_iter().map(|r| r.route.clone()).collect()
1451 }
1452}
1453
1454fn sync_front(state: &kio::Producer<FrontState>, broadcast: &broadcast::Producer, leaf: &Lock<OriginNode>) {
1464 let mut leaf_guard = leaf.lock();
1469 let routes = state.read().routes_snapshot();
1470 if let Some(advert) = routes.first() {
1471 let announce = advert.announce;
1472 broadcast.clone().set_routes(routes);
1473 leaf_guard.set_announced(state, announce);
1474 }
1475}
1476
1477fn detach_source(
1490 state: &kio::Producer<FrontState>,
1491 broadcast: &broadcast::Producer,
1492 leaf: &Lock<OriginNode>,
1493 id: u64,
1494 graceful: bool,
1495) {
1496 let close = {
1497 let carrying = broadcast.demand().is_used();
1501 let Ok(mut s) = state.write() else { return };
1502 let Some(pos) = s.routes.iter().position(|r| r.id == id) else {
1503 return;
1504 };
1505 s.routes.remove(pos);
1506 s.reselect(carrying);
1507 if s.routes.is_empty() && !s.closed && (graceful || s.linger.is_zero()) {
1508 s.closed = true;
1511 true
1512 } else {
1513 false
1514 }
1515 };
1516 if close {
1517 broadcast.abort_spliced(Error::Dropped);
1518 }
1519 sync_front(state, broadcast, leaf);
1520}
1521
1522fn sync_announce(guard: &mut Option<stats::Announce>, announced: bool, ingress: &stats::Scope) {
1525 match (announced, guard.is_some()) {
1526 (true, false) => *guard = Some(ingress.announce()),
1527 (false, true) => *guard = None,
1528 _ => {}
1529 }
1530}
1531
1532async fn run_source(
1542 origin: Info,
1543 node: Lock<OriginNode>,
1544 full: PathOwned,
1545 rest: PathOwned,
1546 mut source: broadcast::Consumer,
1547 ingress: stats::Scope,
1548) {
1549 let ctx = AttachContext {
1550 origin: &origin,
1551 node: &node,
1552 full: &full,
1553 rest: &rest,
1554 };
1555
1556 let Ok(mut route) = source.route_changed().await else {
1560 return;
1562 };
1563
1564 let mut announce = route.announce.then(|| ingress.announce());
1569 let mut may_take_over = true;
1575
1576 'attach: loop {
1577 let leaf = if rest.is_empty() {
1582 node.clone()
1583 } else {
1584 node.lock().leaf(&rest)
1585 };
1586
1587 let (state, broadcast, id) = match attach_source(&ctx, &leaf, &source, route.clone(), may_take_over) {
1588 Attach::Ready(state, broadcast, id) => (state, broadcast, id),
1589 Attach::Parked(incumbent) => {
1590 tracing::debug!(
1591 broadcast = %full,
1592 "path already live with a different publisher; parking this source until it ends",
1593 );
1594 let update = kio::wait(|waiter| {
1598 if let Poll::Ready(update) = source.poll_route_changed(waiter) {
1599 return Poll::Ready(Some(update));
1600 }
1601 match incumbent.poll(waiter, |s| if s.closed { Poll::Ready(()) } else { Poll::Pending }) {
1604 Poll::Ready(_) => Poll::Ready(None),
1605 Poll::Pending => Poll::Pending,
1606 }
1607 })
1608 .await;
1609 match update {
1610 Some(Ok(update)) => {
1612 sync_announce(&mut announce, update.announce, &ingress);
1613 if update.hops.iter().next().copied() != route.hops.iter().next().copied() {
1618 may_take_over = true;
1619 }
1620 route = update;
1621 }
1622 Some(Err(_)) => return,
1624 None => {}
1626 }
1627 continue 'attach;
1628 }
1629 };
1630 let publisher = route.hops.iter().next().copied();
1631
1632 loop {
1633 let update = kio::wait(|waiter| {
1634 if state
1640 .poll_ref(waiter, |s| if s.closed { Poll::Ready(()) } else { Poll::Pending })
1641 .is_ready()
1642 {
1643 return Poll::Ready(None);
1644 }
1645 source.poll_route_changed(waiter).map(Some)
1646 })
1647 .await;
1648 match update {
1649 None => {
1650 may_take_over = false;
1653 continue 'attach;
1654 }
1655 Some(Ok(update)) => {
1656 let announced = update.announce;
1657 if update.hops.iter().next().copied() != publisher {
1668 detach_source(&state, &broadcast, &leaf, id, true);
1669 sync_announce(&mut announce, announced, &ingress);
1670 may_take_over = true;
1674 route = update;
1675 continue 'attach;
1676 }
1677 {
1678 let carrying = broadcast.demand().is_used();
1679 let Ok(mut s) = state.write() else { return };
1680 let Some(entry) = s.routes.iter_mut().find(|r| r.id == id) else {
1681 return;
1682 };
1683 if entry.route == update {
1684 continue;
1685 }
1686 entry.route = update;
1687 s.reselect(carrying);
1688 }
1689 sync_announce(&mut announce, announced, &ingress);
1691 sync_front(&state, &broadcast, &leaf);
1692 }
1693 Some(Err(_)) => {
1694 detach_source(&state, &broadcast, &leaf, id, source.is_finished());
1697 return;
1698 }
1699 }
1700 }
1701 }
1702}
1703
1704enum Attach {
1706 Ready(kio::Producer<FrontState>, broadcast::Producer, u64),
1709 Parked(kio::Producer<FrontState>),
1716}
1717
1718struct AttachContext<'a> {
1720 origin: &'a Info,
1721 node: &'a Lock<OriginNode>,
1722 full: &'a PathOwned,
1724 rest: &'a PathOwned,
1726}
1727
1728fn same_publisher(a: Option<Origin>, b: Option<Origin>) -> bool {
1737 if a == Some(Origin::UNKNOWN) || b == Some(Origin::UNKNOWN) {
1738 return false;
1739 }
1740 a == b
1741}
1742
1743fn attach_source(
1769 ctx: &AttachContext,
1770 leaf: &Lock<OriginNode>,
1771 source: &broadcast::Consumer,
1772 route: broadcast::Route,
1773 may_take_over: bool,
1774) -> Attach {
1775 let publisher = route.hops.iter().next().copied();
1776 let mut leaf_guard = leaf.lock();
1777
1778 if let Some(existing) = &leaf_guard.broadcast {
1781 let mut joined = None;
1782 let carrying = existing.broadcast.demand().is_used();
1783 if let Ok(mut s) = existing.state.write()
1784 && !s.closed
1785 {
1786 if same_publisher(s.publisher, publisher) {
1787 let id = s.next_route;
1788 s.next_route += 1;
1789 s.routes.push(FrontRoute {
1790 id,
1791 route: route.clone(),
1792 source: source.clone(),
1793 });
1794 s.reselect(carrying);
1795 joined = Some(id);
1796 } else if !may_take_over || !route.announce || s.taints_a_reader(&route) {
1797 return Attach::Parked(existing.state.clone());
1798 } else {
1799 s.closed = true;
1807 tracing::warn!(broadcast = %ctx.full, "replacing a live broadcast from a different publisher");
1808 }
1809 }
1810 if let Some(id) = joined {
1811 let state = existing.state.clone();
1812 let broadcast = existing.broadcast.clone();
1813 drop(leaf_guard);
1814 sync_front(&state, &broadcast, leaf);
1815 return Attach::Ready(state, broadcast, id);
1816 }
1817 }
1818
1819 let announce = route.announce;
1821 let broadcast = broadcast::Producer::new_spliced(broadcast::Info {
1822 origin: ctx.origin.clone(),
1823 });
1824 let _ = broadcast.clone().set_route(route.clone());
1825 let state = kio::Producer::new(FrontState {
1826 path: ctx.full.clone(),
1827 self_origin: ctx.origin.id,
1828 publisher,
1829 next_route: 1,
1830 excluded: HashMap::new(),
1831 routes: vec![FrontRoute {
1832 id: 0,
1833 route,
1834 source: source.clone(),
1835 }],
1836 active: Some(0),
1837 linger: ctx.origin.linger,
1838 closed: false,
1839 });
1840
1841 if let Some(stale) = leaf_guard.broadcast.take()
1845 && stale.announced
1846 {
1847 leaf_guard.notify.lock().unannounce(&stale.path);
1848 }
1849 let entry = OriginBroadcast {
1850 path: ctx.full.clone(),
1851 broadcast: broadcast.clone(),
1852 state: state.clone(),
1853 announced: announce,
1854 };
1855 if entry.announced {
1856 leaf_guard
1857 .notify
1858 .lock()
1859 .announce(ctx.full, &broadcast.consume(), &state);
1860 }
1861 leaf_guard.broadcast = Some(entry);
1862 drop(leaf_guard);
1863
1864 web_async::spawn(run_front(
1865 state.clone(),
1866 broadcast.clone(),
1867 ctx.node.clone(),
1868 ctx.rest.clone(),
1869 ));
1870
1871 Attach::Ready(state, broadcast, 0)
1872}
1873
1874async fn run_front(
1877 state: kio::Producer<FrontState>,
1878 mut broadcast: broadcast::Producer,
1879 node: Lock<OriginNode>,
1880 rest: PathOwned,
1881) {
1882 enum Step {
1883 Serve(Arc<str>, super::resume::Producer),
1884 Changed,
1886 Expired,
1888 Closed,
1889 }
1890
1891 let linger = state.read().linger;
1892 let mut deadline = kio::time::Deadline::new();
1897
1898 loop {
1899 let empty = {
1900 let s = state.read();
1901 !s.closed && s.routes.is_empty()
1902 };
1903 deadline.set(match (empty, deadline.deadline()) {
1904 (true, None) => web_async::time::Instant::now().checked_add(linger),
1907 (true, at) => at,
1908 (false, _) => None,
1909 });
1910
1911 let step = {
1912 kio::wait(|waiter| {
1913 if let Poll::Ready((name, resume)) = broadcast.poll_spliced_assigned(waiter) {
1914 return Poll::Ready(Step::Serve(name, resume));
1915 }
1916 match state.poll(waiter, |s| {
1919 if s.closed || s.routes.is_empty() != empty {
1920 Poll::Ready(())
1921 } else {
1922 Poll::Pending
1923 }
1924 }) {
1925 Poll::Ready(Ok(guard)) => {
1926 return Poll::Ready(if guard.closed { Step::Closed } else { Step::Changed });
1927 }
1928 Poll::Ready(Err(_)) => return Poll::Ready(Step::Closed),
1929 Poll::Pending => {}
1930 }
1931 deadline.poll(waiter).map(|_| Step::Expired)
1932 })
1933 .await
1934 };
1935
1936 match step {
1937 Step::Serve(name, resume) => {
1938 web_async::spawn(serve_track(state.clone(), name, resume));
1941 }
1942 Step::Changed => {}
1943 Step::Expired => {
1944 let close = {
1948 let Ok(mut s) = state.write() else { break };
1949 if !s.closed && s.routes.is_empty() {
1950 s.closed = true;
1951 true
1952 } else {
1953 false
1954 }
1955 };
1956 if close {
1957 break;
1958 }
1959 }
1960 Step::Closed => break,
1961 }
1962 }
1963
1964 broadcast.abort_spliced(Error::Dropped);
1966
1967 broadcast.finish();
1969
1970 node.lock().remove(&state, &rest);
1973}
1974
1975async fn serve_track(state: kio::Producer<FrontState>, name: Arc<str>, mut resume: super::resume::Producer) {
1988 enum Step {
1989 Closed,
1990 Splice(u64, broadcast::Consumer),
1991 Complete,
1992 Failed(Error),
1993 NoRoute,
1998 Idle,
2000 Demand,
2002 }
2003
2004 let mut serving: Option<(u64, track::Consumer)> = None;
2006 let mut spliced_edge: Option<u64> = None;
2012 let mut refused: HashSet<u64> = HashSet::new();
2017 let mut refusal: Option<Error> = None;
2018 let mut dead: HashSet<u64> = HashSet::new();
2023 let mut idle_since: Option<web_async::time::Instant> = None;
2025 let mut deadline = kio::time::Deadline::new();
2026
2027 loop {
2028 let serving_id = serving.as_ref().map(|(id, _)| *id);
2029
2030 {
2037 let s = state.read();
2038 refused.retain(|id| s.routes.iter().any(|r| r.id == *id));
2039 dead.retain(|id| s.routes.iter().any(|r| r.id == *id));
2040 let exhausted = !s.routes.is_empty()
2041 && s.serve_route(|id| refused.contains(&id) || dead.contains(&id))
2042 .is_none();
2043 if exhausted && dead.is_empty() {
2044 drop(s);
2045 let err = refusal.take().unwrap_or(Error::NotFound);
2046 tracing::debug!(name = %name, %err, "every source refused track; aborting");
2047 let _ = resume.abort(err);
2048 return;
2049 }
2050 }
2051
2052 let used = resume.is_used();
2064 idle_since = match (resume.is_spliced(), used) {
2065 (true, false) => idle_since.or_else(|| Some(web_async::time::Instant::now())),
2066 _ => None,
2067 };
2068 deadline.set(idle_since.and_then(|at| at.checked_add(TRACK_IDLE_LINGER)));
2069
2070 let step = {
2071 let skip = |id: u64| refused.contains(&id) || dead.contains(&id);
2072 kio::wait(|waiter| {
2073 match state.poll(waiter, |s| {
2079 let gone = serving_id.is_some_and(|id| !s.routes.iter().any(|r| r.id == id));
2080 if s.closed
2081 || (used && (gone || matches!(s.serve_route(skip), Some(next) if Some(next) != serving_id)))
2082 {
2083 Poll::Ready(())
2084 } else {
2085 Poll::Pending
2086 }
2087 }) {
2088 Poll::Ready(Ok(guard)) => {
2089 if guard.closed {
2090 return Poll::Ready(Step::Closed);
2091 }
2092 let Some(next) = guard.serve_route(skip) else {
2093 return Poll::Ready(Step::NoRoute);
2094 };
2095 let source = guard
2096 .routes
2097 .iter()
2098 .find(|r| r.id == next)
2099 .expect("servable source in table")
2100 .source
2101 .clone();
2102 return Poll::Ready(Step::Splice(next, source));
2103 }
2104 Poll::Ready(Err(_)) => return Poll::Ready(Step::Closed),
2105 Poll::Pending => {}
2106 }
2107
2108 let edge = match used {
2113 true => resume.poll_unused(waiter),
2114 false => resume.poll_used(waiter),
2115 };
2116 if edge.is_ready() {
2117 return Poll::Ready(Step::Demand);
2118 }
2119
2120 if let Some((_, track)) = &serving
2123 && let Poll::Ready(result) = track.poll_complete(waiter)
2124 {
2125 return Poll::Ready(match result {
2126 Ok(()) => Step::Complete,
2127 Err(err) => Step::Failed(err),
2128 });
2129 }
2130
2131 deadline.poll(waiter).map(|_| Step::Idle)
2132 })
2133 .await
2134 };
2135
2136 match step {
2137 Step::Closed => return,
2139 Step::Complete => {
2140 let _ = resume.finish();
2141 return;
2142 }
2143 Step::Failed(err) => {
2144 if resume.latest() == spliced_edge
2152 && let Some(id) = serving_id
2153 {
2154 let closing = state
2155 .read()
2156 .routes
2157 .iter()
2158 .find(|r| r.id == id)
2159 .is_some_and(|r| r.source.is_closing());
2160 if closing {
2161 dead.insert(id);
2162 } else {
2163 refused.insert(id);
2164 refusal = Some(err);
2165 }
2166 }
2167 serving = None;
2168 }
2169 Step::Demand => {}
2171 Step::NoRoute => serving = None,
2177 Step::Idle => {
2178 if resume.release().is_err() {
2183 return;
2185 }
2186 serving = None;
2187 }
2188 Step::Splice(id, source) => {
2189 let attempt = match source.track(&name) {
2193 Ok(track) => {
2194 let query = track.info().into_inner();
2197 let skip = |id: u64| refused.contains(&id) || dead.contains(&id);
2198 let info = kio::wait(|waiter| {
2199 if let Poll::Ready(result) = query.poll(waiter) {
2200 return Poll::Ready(Some(result));
2201 }
2202 match state.poll(waiter, |s| {
2203 if s.closed || s.serve_route(skip) != Some(id) {
2204 Poll::Ready(())
2205 } else {
2206 Poll::Pending
2207 }
2208 }) {
2209 Poll::Ready(_) => Poll::Ready(None),
2210 Poll::Pending => Poll::Pending,
2211 }
2212 })
2213 .await;
2214 match info {
2215 None => continue,
2217 Some(Ok(_)) => match track.poll_complete(&kio::Waiter::noop()) {
2220 Poll::Ready(Err(err)) => Err(err),
2221 _ => Ok(track),
2222 },
2223 Some(Err(err)) => Err(err),
2224 }
2225 }
2226 Err(err) => Err(err),
2227 };
2228
2229 match attempt {
2230 Ok(track) => {
2231 if let Err(err) = resume.takeover(&track) {
2232 let _ = resume.abort(err);
2237 return;
2238 }
2239 spliced_edge = resume.latest();
2251 serving = Some((id, track));
2252 }
2253 Err(_) if source.is_closing() => {
2257 dead.insert(id);
2258 serving = None;
2259 }
2260 Err(err) => {
2266 tracing::debug!(name = %name, source = id, %err, "source refused track");
2267 refused.insert(id);
2268 refusal = Some(err);
2269 serving = None;
2270 }
2271 }
2272 }
2273 }
2274 }
2275}
2276
2277#[derive(Default)]
2283struct OriginDynamicState {
2284 requests: Requests<PathOwned, kio::Producer<PendingBroadcast>>,
2287
2288 served: WeakCache<PathOwned, broadcast::WeakConsumer>,
2294}
2295
2296#[derive(Default)]
2303struct PendingBroadcast {
2304 resolved: Option<Result<broadcast::Consumer, Error>>,
2305}
2306
2307pub struct Dynamic {
2318 info: Origin,
2319 root: PathOwned,
2320 state: kio::Shared<OriginDynamicState>,
2321}
2322
2323impl Clone for Dynamic {
2324 fn clone(&self) -> Self {
2325 self.state.lock().requests.add_handler();
2329
2330 Self {
2331 info: self.info,
2332 root: self.root.clone(),
2333 state: self.state.clone(),
2334 }
2335 }
2336}
2337
2338impl Dynamic {
2339 fn new(info: Origin, root: PathOwned, state: kio::Shared<OriginDynamicState>) -> Self {
2340 state.lock().requests.add_handler();
2341
2342 Self { info, root, state }
2343 }
2344
2345 pub fn info(&self) -> &Origin {
2347 &self.info
2348 }
2349
2350 pub fn poll_requested_broadcast(&mut self, waiter: &kio::Waiter) -> Poll<Result<Request, Error>> {
2352 let mut state = ready!(self.state.poll(waiter, |state| {
2353 if state.requests.has_queued() {
2354 Poll::Ready(())
2355 } else {
2356 Poll::Pending
2357 }
2358 }));
2359
2360 let path = state.requests.pop().expect("predicate guaranteed a request");
2361 let producer = state.requests.get(&path).expect("popped key must be pending").clone();
2367 Poll::Ready(Ok(Request {
2368 path,
2369 producer,
2370 state: self.state.clone(),
2371 }))
2372 }
2373
2374 pub async fn requested_broadcast(&mut self) -> Result<Request, Error> {
2377 kio::wait(|waiter| self.poll_requested_broadcast(waiter)).await
2378 }
2379
2380 pub fn root(&self) -> &Path<'_> {
2382 &self.root
2383 }
2384}
2385
2386impl Drop for Dynamic {
2387 fn drop(&mut self) {
2388 let mut state = self.state.lock();
2391 if state.requests.remove_handler() {
2392 state.requests.drain_queued();
2396 }
2397 }
2398}
2399
2400pub struct Request {
2407 path: PathOwned,
2409
2410 producer: kio::Producer<PendingBroadcast>,
2413
2414 state: kio::Shared<OriginDynamicState>,
2416}
2417
2418impl Request {
2419 pub fn path(&self) -> &Path<'_> {
2421 &self.path
2422 }
2423
2424 pub fn accept(self, broadcast: impl Consume<broadcast::Consumer>) {
2430 let broadcast = broadcast.consume();
2431
2432 let resolved = {
2438 let mut state = self.state.lock();
2439 let existing = state.served.insert(self.path.clone(), broadcast.weak());
2440 state
2441 .requests
2442 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2443 existing.map(|weak| weak.consume()).unwrap_or(broadcast)
2444 };
2445
2446 if let Ok(mut pending) = self.producer.write() {
2447 pending.resolved = Some(Ok(resolved));
2448 }
2449 }
2451
2452 pub fn reject(self, err: Error) {
2454 self.state
2455 .lock()
2456 .requests
2457 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2458 if let Ok(mut state) = self.producer.write() {
2459 state.resolved = Some(Err(err));
2460 }
2461 }
2462}
2463
2464impl Drop for Request {
2465 fn drop(&mut self) {
2466 self.state
2474 .lock()
2475 .requests
2476 .remove_if(&self.path, |producer| producer.same_channel(&self.producer));
2477 }
2478}
2479
2480pub struct Requesting {
2487 inner: RequestState,
2488 stats: stats::Scope,
2491}
2492
2493enum RequestState {
2494 Ready(broadcast::Consumer),
2496 Failed(Error),
2499 Pending(kio::Consumer<PendingBroadcast>),
2501}
2502
2503impl Requesting {
2504 fn ready(broadcast: broadcast::Consumer) -> Self {
2505 Self {
2506 inner: RequestState::Ready(broadcast),
2507 stats: stats::Scope::default(),
2508 }
2509 }
2510
2511 fn failed(error: Error) -> Self {
2512 Self {
2513 inner: RequestState::Failed(error),
2514 stats: stats::Scope::default(),
2515 }
2516 }
2517
2518 fn pending(consumer: kio::Consumer<PendingBroadcast>) -> Self {
2519 Self {
2520 inner: RequestState::Pending(consumer),
2521 stats: stats::Scope::default(),
2522 }
2523 }
2524
2525 fn with_stats(mut self, scope: stats::Scope) -> Self {
2526 self.stats = scope;
2527 self
2528 }
2529
2530 pub fn poll_ok(&self, waiter: &kio::Waiter) -> Poll<Result<broadcast::Consumer, Error>> {
2532 match &self.inner {
2533 RequestState::Ready(broadcast) => Poll::Ready(Ok(broadcast.clone().with_stats(self.stats.clone()))),
2534 RequestState::Failed(error) => Poll::Ready(Err(error.clone())),
2535 RequestState::Pending(consumer) => Poll::Ready(
2536 match ready!(consumer.poll(waiter, |state| match &state.resolved {
2537 Some(result) => Poll::Ready(result.clone()),
2538 None => Poll::Pending,
2539 })) {
2540 Ok(result) => result.map(|broadcast| broadcast.with_stats(self.stats.clone())),
2541 Err(_closed) => Err(Error::Unroutable),
2543 },
2544 ),
2545 }
2546 }
2547}
2548
2549impl kio::Pollable for Requesting {
2550 type Output = Result<broadcast::Consumer, Error>;
2551
2552 fn poll(&self, waiter: &kio::Waiter) -> Poll<Self::Output> {
2553 self.poll_ok(waiter)
2554 }
2555}
2556
2557pub trait Consume<T> {
2565 fn consume(&self) -> T;
2567}
2568
2569impl<T, U: Consume<T>> Consume<T> for &U {
2570 fn consume(&self) -> T {
2571 (**self).consume()
2572 }
2573}
2574
2575impl Consume<Consumer> for Producer {
2576 fn consume(&self) -> Consumer {
2577 Consumer::new(
2581 self.info,
2582 self.root.clone(),
2583 self.nodes.clone(),
2584 self.dynamic.clone(),
2585 stats::Session::default(),
2586 )
2587 }
2588}
2589
2590impl Consume<Consumer> for Consumer {
2591 fn consume(&self) -> Consumer {
2592 self.clone()
2593 }
2594}
2595
2596impl Consume<broadcast::Consumer> for broadcast::Producer {
2597 fn consume(&self) -> broadcast::Consumer {
2598 self.consume()
2600 }
2601}
2602
2603impl Consume<broadcast::Consumer> for broadcast::Consumer {
2604 fn consume(&self) -> broadcast::Consumer {
2605 self.clone()
2606 }
2607}
2608
2609impl Consume<track::Consumer> for track::Producer {
2610 fn consume(&self) -> track::Consumer {
2611 self.consume()
2612 }
2613}
2614
2615impl Consume<track::Consumer> for track::Consumer {
2616 fn consume(&self) -> track::Consumer {
2617 self.clone()
2618 }
2619}
2620
2621#[derive(Clone)]
2627pub struct Consumer {
2628 info: Origin,
2630 nodes: OriginNodes,
2631
2632 root: PathOwned,
2634
2635 dynamic: kio::Shared<OriginDynamicState>,
2638
2639 stats: stats::Session,
2643
2644 exclude: Option<Origin>,
2648}
2649
2650impl std::ops::Deref for Consumer {
2651 type Target = Origin;
2652
2653 fn deref(&self) -> &Self::Target {
2654 &self.info
2655 }
2656}
2657
2658impl Consumer {
2659 fn new(
2660 info: Origin,
2661 root: PathOwned,
2662 nodes: OriginNodes,
2663 dynamic: kio::Shared<OriginDynamicState>,
2664 stats: stats::Session,
2665 ) -> Self {
2666 Self {
2667 info,
2668 nodes,
2669 root,
2670 dynamic,
2671 stats,
2672 exclude: None,
2673 }
2674 }
2675
2676 pub(crate) fn excluding(mut self, peer: Origin) -> Self {
2681 self.exclude = Some(peer);
2682 self
2683 }
2684
2685 pub fn with_stats(mut self, session: stats::Session) -> Self {
2689 self.stats = session;
2690 self
2691 }
2692
2693 fn untagged(&self) -> Self {
2697 Self {
2698 stats: stats::Session::default(),
2699 ..self.clone()
2700 }
2701 }
2702
2703 pub(crate) fn empty(&self) -> Self {
2708 Self {
2709 info: self.info,
2710 nodes: OriginNodes { nodes: Vec::new() },
2711 root: self.root.clone(),
2712 dynamic: self.dynamic.clone(),
2713 stats: self.stats.clone(),
2714 exclude: self.exclude,
2715 }
2716 }
2717
2718 pub fn announced(&self) -> AnnounceConsumer {
2725 AnnounceConsumer::new(self.root.clone(), self.nodes.clone(), self.stats.clone(), self.exclude)
2726 }
2727
2728 pub fn consume(&self) -> Self {
2730 self.clone()
2731 }
2732
2733 fn resolve(&self, path: impl AsPath) -> Resolved {
2742 let path = path.as_path();
2743 let Some((root, rest)) = self.nodes.get(&path) else {
2744 return Resolved::Missing;
2745 };
2746 let state = root.lock();
2747 state.resolve_broadcast(&rest, self.exclude)
2748 }
2749
2750 #[cfg(test)]
2752 pub(crate) fn get_broadcast(&self, path: impl AsPath) -> Option<broadcast::Consumer> {
2753 match self.resolve(path) {
2754 Resolved::Found(broadcast) => Some(broadcast),
2755 Resolved::Excluded | Resolved::Missing => None,
2756 }
2757 }
2758
2759 pub async fn announced_broadcast(&self, path: impl AsPath) -> Option<broadcast::Consumer> {
2770 let path = path.as_path();
2771
2772 let consumer = self.scope(std::slice::from_ref(&path))?;
2774
2775 if !consumer.allowed().any(|allowed| path.has_prefix(allowed)) {
2779 return None;
2780 }
2781
2782 let mut announced = consumer.untagged().announced();
2786 let scope = self.stats.egress(self.root.join(&path).to_owned());
2787 loop {
2788 let OriginAnnounce {
2789 path: announced_path,
2790 broadcast,
2791 } = announced.next().await?;
2792 if announced_path.as_path() == path
2794 && let Some(broadcast) = broadcast
2795 {
2796 return Some(broadcast.with_stats(scope));
2797 }
2798 }
2799 }
2800
2801 pub fn scope(&self, prefixes: &[Path]) -> Option<Consumer> {
2807 let prefixes = PathPrefixes::new(prefixes);
2808 Some(Consumer {
2809 info: self.info,
2810 root: self.root.clone(),
2811 nodes: self.nodes.select(&prefixes)?,
2812 dynamic: self.dynamic.clone(),
2813 stats: self.stats.clone(),
2814 exclude: self.exclude,
2815 })
2816 }
2817
2818 pub fn request_broadcast(&self, path: impl AsPath) -> kio::Pending<Requesting> {
2837 let path = path.as_path();
2838
2839 let absolute = self.root.join(&path).to_owned();
2843 let scope = self.stats.egress(&absolute);
2844
2845 match self.resolve(&path) {
2851 Resolved::Found(broadcast) => return kio::Pending::new(Requesting::ready(broadcast).with_stats(scope)),
2852 Resolved::Excluded => return kio::Pending::new(Requesting::failed(Error::Unroutable)),
2853 Resolved::Missing => {}
2854 }
2855
2856 let mut state = self.dynamic.lock();
2857
2858 if let Some(weak) = state.served.get(&absolute) {
2862 return kio::Pending::new(Requesting::ready(weak.consume()).with_stats(scope));
2863 }
2864
2865 let consumer = if let Some(producer) = state.requests.join(&absolute) {
2868 producer.consume()
2869 } else {
2870 let producer = kio::Producer::<PendingBroadcast>::default();
2871 let consumer = producer.consume();
2872 if state.requests.insert(absolute, producer).is_err() {
2873 return kio::Pending::new(Requesting::failed(Error::Unroutable));
2874 }
2875 consumer
2876 };
2877
2878 kio::Pending::new(Requesting::pending(consumer).with_stats(scope))
2879 }
2880
2881 pub fn with_root(&self, prefix: impl AsPath) -> Option<Self> {
2886 let prefix = prefix.as_path();
2887
2888 Some(Self {
2889 info: self.info,
2890 root: self.root.join(&prefix).to_owned(),
2891 nodes: self.nodes.root(&prefix)?,
2892 dynamic: self.dynamic.clone(),
2893 stats: self.stats.clone(),
2894 exclude: self.exclude,
2895 })
2896 }
2897
2898 pub fn root(&self) -> &Path<'_> {
2900 &self.root
2901 }
2902
2903 pub fn allowed(&self) -> impl Iterator<Item = &Path<'_>> {
2906 self.nodes.nodes.iter().map(|(root, _)| root)
2907 }
2908
2909 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
2911 self.root.join(path)
2912 }
2913}
2914
2915#[derive(Clone)]
2920pub struct AnnounceProducer {
2921 nodes: OriginNodes,
2922 root: PathOwned,
2923}
2924
2925impl AnnounceProducer {
2926 fn new(root: PathOwned, nodes: OriginNodes) -> Self {
2927 Self { nodes, root }
2928 }
2929
2930 pub fn consume(&self) -> AnnounceConsumer {
2936 AnnounceConsumer::new(self.root.clone(), self.nodes.clone(), stats::Session::default(), None)
2939 }
2940
2941 pub fn root(&self) -> &Path<'_> {
2943 &self.root
2944 }
2945}
2946
2947pub struct AnnounceConsumer {
2952 id: ConsumerId,
2953 nodes: OriginNodes,
2954 root: PathOwned,
2955
2956 state: kio::Producer<OriginConsumerState>,
2959
2960 stats: stats::Session,
2963
2964 guards: HashMap<PathOwned, stats::Announce>,
2968}
2969
2970impl AnnounceConsumer {
2971 fn new(root: PathOwned, nodes: OriginNodes, stats: stats::Session, exclude: Option<Origin>) -> Self {
2972 let state = kio::Producer::<OriginConsumerState>::default();
2973 let id = ConsumerId::new();
2974
2975 for (_, node) in &nodes.nodes {
2976 let notify = AnnounceConsumerNotify {
2977 root: root.clone(),
2978 state: state.clone(),
2979 exclude,
2980 };
2981 node.lock().consume(id, notify);
2982 }
2983
2984 Self {
2985 id,
2986 nodes,
2987 root,
2988 state,
2989 stats,
2990 guards: HashMap::new(),
2991 }
2992 }
2993
2994 fn attribute(&mut self, update: OriginAnnounce) -> OriginAnnounce {
3000 let OriginAnnounce { path, broadcast } = update;
3001 let absolute = self.root.join(&path).to_owned();
3002 match broadcast {
3003 Some(broadcast) => {
3004 let scope = self.stats.egress(&absolute);
3005 self.guards.entry(absolute).or_insert_with(|| scope.announce());
3006 OriginAnnounce {
3007 path,
3008 broadcast: Some(broadcast.with_stats(scope)),
3009 }
3010 }
3011 None => {
3012 self.guards.remove(&absolute);
3013 OriginAnnounce { path, broadcast: None }
3014 }
3015 }
3016 }
3017
3018 pub async fn next(&mut self) -> Option<OriginAnnounce> {
3025 kio::wait(|waiter| self.poll_next(waiter)).await
3026 }
3027
3028 pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll<Option<OriginAnnounce>> {
3034 let update = {
3035 let mut state = match ready!(self.state.poll(waiter, |state| {
3036 if state.pending.is_empty() {
3037 Poll::Pending
3038 } else {
3039 Poll::Ready(())
3040 }
3041 })) {
3042 Ok(state) => state,
3043 Err(_) => return Poll::Ready(None),
3045 };
3046 state.take().expect("predicate guaranteed an update")
3047 };
3048 Poll::Ready(Some(self.attribute(update)))
3049 }
3050
3051 pub fn try_next(&mut self) -> Option<OriginAnnounce> {
3056 let update = self.state.write().ok()?.take()?;
3057 Some(self.attribute(update))
3058 }
3059
3060 pub fn is_closed(&self) -> bool {
3062 self.state.write().is_err()
3063 }
3064
3065 pub fn root(&self) -> &Path<'_> {
3067 &self.root
3068 }
3069
3070 pub fn absolute(&self, path: impl AsPath) -> Path<'_> {
3072 self.root.join(path)
3073 }
3074}
3075
3076impl Drop for AnnounceConsumer {
3077 fn drop(&mut self) {
3078 for (_, root) in &self.nodes.nodes {
3079 root.lock().unconsume(self.id);
3080 }
3081 }
3082}
3083
3084#[cfg(test)]
3085use futures::FutureExt;
3086
3087#[cfg(test)]
3088#[allow(missing_docs)] impl AnnounceConsumer {
3090 pub fn assert_next(&mut self, expected: impl AsPath, broadcast: &broadcast::Consumer) {
3091 let expected = expected.as_path();
3092 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3093 assert_eq!(announce.path, expected, "wrong path");
3094 let announced = announce.broadcast.expect("should be an active announce");
3095 assert!(announced.is_clone(broadcast), "should be the same broadcast");
3096 }
3097
3098 pub fn assert_next_some(&mut self, expected: impl AsPath) -> broadcast::Consumer {
3102 let expected = expected.as_path();
3103 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3104 assert_eq!(announce.path, expected, "wrong path");
3105 announce.broadcast.expect("should be an active announce")
3106 }
3107
3108 pub fn assert_try_next(&mut self, expected: impl AsPath, broadcast: &broadcast::Consumer) {
3109 let expected = expected.as_path();
3110 let announce = self.try_next().expect("no next");
3111 assert_eq!(announce.path, expected, "wrong path");
3112 let announced = announce.broadcast.expect("should be an active announce");
3113 assert!(announced.is_clone(broadcast), "should be the same broadcast");
3114 }
3115
3116 pub fn assert_try_next_some(&mut self, expected: impl AsPath) -> broadcast::Consumer {
3118 let expected = expected.as_path();
3119 let announce = self.try_next().expect("no next");
3120 assert_eq!(announce.path, expected, "wrong path");
3121 announce.broadcast.expect("should be an active announce")
3122 }
3123
3124 pub fn assert_next_none(&mut self, expected: impl AsPath) {
3125 let expected = expected.as_path();
3126 let announce = self.next().now_or_never().expect("next blocked").expect("no next");
3127 assert_eq!(announce.path, expected, "wrong path");
3128 assert!(announce.broadcast.is_none(), "should be unannounced");
3129 }
3130
3131 pub fn assert_next_wait(&mut self) {
3132 if let Some(res) = self.next().now_or_never() {
3133 panic!("next should block: got {:?}", res.map(|a| a.path));
3134 }
3135 }
3136
3137 }
3146
3147#[cfg(test)]
3148mod tests {
3149 use crate::coding::Decode;
3150 use crate::group;
3151
3152 use super::*;
3153
3154 fn announce() -> broadcast::Route {
3156 broadcast::Route::new().with_announce(true)
3157 }
3158
3159 fn origin_keyed(name: &str, peer: Origin, above: bool) -> Origin {
3165 let name = Path::new(name);
3166 let peer_key = fnv_key(&name, [peer]);
3167 (100u64..)
3168 .map(|id| Origin::new(id).unwrap())
3169 .find(|origin| (fnv_key(&name, [*origin]) > peer_key) == above)
3170 .unwrap()
3171 }
3172
3173 fn front_state(self_origin: Origin, routes: Vec<broadcast::Route>) -> FrontState {
3176 let source = broadcast::Info::new().produce().consume();
3177 FrontState {
3178 path: Path::new("test").to_owned(),
3179 self_origin,
3180 publisher: routes.first().and_then(|r| r.hops.iter().next().copied()),
3181 next_route: routes.len() as u64,
3182 excluded: HashMap::new(),
3183 routes: routes
3184 .into_iter()
3185 .enumerate()
3186 .map(|(id, route)| FrontRoute {
3187 id: id as u64,
3188 route,
3189 source: source.clone(),
3190 })
3191 .collect(),
3192 active: Some(0),
3193 linger: Duration::ZERO,
3194 closed: false,
3195 }
3196 }
3197
3198 fn sibling_route(peer: Origin) -> broadcast::Route {
3201 let hops = OriginList::try_from(vec![Origin::new(90).unwrap(), peer]).unwrap();
3202 announce().with_hops(hops)
3203 }
3204
3205 fn upstream_route(cost: u64) -> broadcast::Route {
3207 let hops = OriginList::try_from(vec![Origin::new(90).unwrap()]).unwrap();
3208 announce().with_hops(hops).with_cost(cost)
3209 }
3210
3211 #[test]
3215 fn test_carrying_gate_keys() {
3216 let peer = Origin::new(3).unwrap();
3217
3218 let mut lost = front_state(
3220 origin_keyed("test", peer, false),
3221 vec![upstream_route(10), sibling_route(peer)],
3222 );
3223 lost.reselect(true);
3224 assert_eq!(
3225 lost.active,
3226 Some(0),
3227 "carrying front re-parented onto a higher-keyed peer"
3228 );
3229 lost.reselect(false);
3230 assert_eq!(lost.active, Some(1), "idle front must take the cheaper route");
3231
3232 let mut won = front_state(
3234 origin_keyed("test", peer, true),
3235 vec![upstream_route(10), sibling_route(peer)],
3236 );
3237 won.reselect(true);
3238 assert_eq!(won.active, Some(1), "carrying front must follow a lower-keyed peer");
3239 }
3240
3241 #[test]
3246 fn test_carrying_gate_symmetric_race() {
3247 let a = Origin::new(1).unwrap();
3248 let b = Origin::new(2).unwrap();
3249
3250 let mut a_view = front_state(a, vec![upstream_route(10), sibling_route(b)]);
3251 let mut b_view = front_state(b, vec![upstream_route(10), sibling_route(a)]);
3252 a_view.reselect(true);
3253 b_view.reselect(true);
3254
3255 let a_moved = a_view.active == Some(1);
3256 let b_moved = b_view.active == Some(1);
3257 assert!(
3258 a_moved != b_moved,
3259 "exactly one side must re-parent (a: {a_moved}, b: {b_moved})"
3260 );
3261 }
3262
3263 #[test]
3268 fn test_carrying_switches_to_benign_routes() {
3269 let peer = Origin::new(3).unwrap();
3270 let lost = origin_keyed("test", peer, false);
3271
3272 let mut forwarder = sibling_route(peer).with_cost(4);
3274 forwarder.advertised = 4;
3275 let mut state = front_state(lost, vec![upstream_route(10), forwarder]);
3276 state.reselect(true);
3277 assert_eq!(
3278 state.active,
3279 Some(1),
3280 "a cheaper forwarder path must win while carrying"
3281 );
3282
3283 let direct = announce().with_hops(OriginList::try_from(vec![peer]).unwrap());
3285 let mut state = front_state(lost, vec![upstream_route(10), direct]);
3286 state.reselect(true);
3287 assert_eq!(
3288 state.active,
3289 Some(1),
3290 "a direct publisher route must win while carrying"
3291 );
3292
3293 let mut state = front_state(lost, vec![sibling_route(peer), sibling_route(peer)]);
3298 state.reselect(true);
3299 assert_eq!(
3300 state.active,
3301 Some(1),
3302 "a reconnect on an identical chain must win while carrying"
3303 );
3304 }
3305
3306 #[test]
3309 fn test_carrying_gate_ignores_unannounced_incumbent() {
3310 let peer = Origin::new(3).unwrap();
3311 let unannounced = upstream_route(10).with_announce(false);
3312 let mut state = front_state(
3313 origin_keyed("test", peer, false),
3314 vec![unannounced, sibling_route(peer)],
3315 );
3316 state.reselect(true);
3317 assert_eq!(
3318 state.active,
3319 Some(1),
3320 "an unannounced incumbent must always be displaced"
3321 );
3322 }
3323
3324 #[test]
3329 fn test_reflection_through_an_exposed_peer_cannot_take_over() {
3330 let peer = Origin::new(42).unwrap();
3331 let upstream = OriginList::try_from(vec![Origin::new(7).unwrap()]).unwrap();
3332 let reflected = announce().with_hops(OriginList::try_from(vec![peer]).unwrap());
3333
3334 let mut state = front_state(Origin::new(1).unwrap(), vec![announce().with_hops(upstream)]);
3335
3336 assert!(!state.taints_a_reader(&reflected));
3338
3339 *state.excluded.entry(peer).or_default() += 1;
3342 assert!(state.taints_a_reader(&reflected));
3343 }
3344
3345 #[test]
3348 fn test_rival_publisher_through_another_peer_still_takes_over() {
3349 let peer = Origin::new(42).unwrap();
3350 let elsewhere = Origin::new(43).unwrap();
3351 let upstream = OriginList::try_from(vec![Origin::new(7).unwrap()]).unwrap();
3352 let rival = announce().with_hops(OriginList::try_from(vec![Origin::UNKNOWN, elsewhere]).unwrap());
3353
3354 let mut state = front_state(Origin::new(1).unwrap(), vec![announce().with_hops(upstream)]);
3355 *state.excluded.entry(peer).or_default() += 1;
3356
3357 assert!(!state.taints_a_reader(&rival), "only the peer we feed is a reflection");
3358 }
3359
3360 #[test]
3365 fn test_opaque_peer_understates_its_depth() {
3366 let us = Origin::new(1).unwrap();
3367
3368 let direct = || {
3370 announce()
3371 .with_hops(OriginList::try_from(vec![Origin::new(7).unwrap(), Origin::new(8).unwrap()]).unwrap())
3372 .with_cost(2)
3373 };
3374 let opaque = |cost| {
3377 announce()
3378 .with_hops(OriginList::try_from(vec![Origin::new(42).unwrap()]).unwrap())
3379 .with_cost(cost)
3380 };
3381
3382 let mut state = front_state(us, vec![direct(), opaque(1)]);
3384 state.reselect(false);
3385 assert_eq!(
3386 state.active,
3387 Some(1),
3388 "an unpriced opaque link out-ranks a shorter real path"
3389 );
3390
3391 let mut state = front_state(us, vec![direct(), opaque(16)]);
3393 state.reselect(false);
3394 assert_eq!(
3395 state.active,
3396 Some(0),
3397 "pricing the opaque link restores the intended order"
3398 );
3399 }
3400
3401 async fn settle() {
3404 tokio::time::sleep(tokio::time::Duration::from_millis(1)).await;
3405 }
3406
3407 async fn accept_track(dynamic: &mut broadcast::Dynamic, name: &str) -> track::Producer {
3410 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic.requested_track())
3411 .await
3412 .expect("timed out waiting for a track request")
3413 .expect("source closed");
3414 assert_eq!(request.name(), name, "unexpected track dispatched");
3415 request.accept(None)
3416 }
3417
3418 async fn accept_tracks(dynamic: &mut broadcast::Dynamic, count: usize) -> HashMap<String, track::Producer> {
3422 let mut accepted = HashMap::new();
3423 for _ in 0..count {
3424 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic.requested_track())
3425 .await
3426 .expect("timed out waiting for a track request")
3427 .expect("source closed");
3428 let name = request.name().to_string();
3429 accepted.insert(name, request.accept(None));
3430 }
3431 accepted
3432 }
3433
3434 #[tokio::test]
3438 async fn test_stats_tagged_end_to_end() {
3439 use crate::Timestamp;
3440 use crate::stats::{Config, Registry, Tier};
3441 use bytes::Bytes;
3442
3443 tokio::time::pause();
3444
3445 let registry = Registry::new(Config::new());
3446 let ctx = registry.tier(Tier::default()).session("acme");
3447
3448 let origin = Origin::random().produce();
3449 let ingress = origin.clone().with_stats(ctx.clone());
3450 let egress = origin.consume().with_stats(ctx.clone());
3451
3452 let mut announced = egress.announced();
3455
3456 let source = ingress.create_broadcast("demo", announce()).unwrap();
3458 let mut dynamic = source.dynamic();
3459 settle().await;
3460 settle().await;
3461
3462 let update = announced.next().await.unwrap();
3464 assert_eq!(update.path.as_str(), "demo");
3465 let broadcast = update.broadcast.unwrap();
3466
3467 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3469 let mut producer = accept_track(&mut dynamic, "video").await;
3470 settle().await;
3471 let mut sub = subscribing.await.unwrap();
3472
3473 let mut group = producer.append_group().unwrap();
3475 group
3476 .write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
3477 .unwrap();
3478 group
3479 .write_frame(Timestamp::ZERO, Bytes::from_static(b"world"))
3480 .unwrap();
3481 group.finish().unwrap();
3482
3483 let mut group_c = sub.recv_group().await.unwrap().unwrap();
3485 let mut frames = 0;
3486 while let Some(frame) = group_c.read_frame().await.unwrap() {
3487 assert_eq!(frame.payload.len(), 5);
3488 frames += 1;
3489 }
3490 assert_eq!(frames, 2);
3491 settle().await;
3492
3493 let report = registry.report();
3494 let entry = report
3495 .traffic
3496 .iter()
3497 .find(|e| e.path.as_str() == "demo")
3498 .expect("demo tracked");
3499 let path_len = "demo".len() as u64;
3500
3501 let egress = &entry.publisher;
3503 assert_eq!(egress.announced, 1, "one egress announce");
3504 assert_eq!(egress.announced_bytes, path_len);
3505 assert_eq!(egress.subscriptions, 1, "one egress subscription");
3506 assert_eq!(egress.broadcasts, 1, "one viewer");
3507 assert_eq!(egress.groups, 1);
3508 assert_eq!(egress.frames, 2);
3509 assert_eq!(egress.bytes, 10);
3510 assert_eq!(egress.fetches, 0);
3511
3512 let ingress = &entry.subscriber;
3514 assert_eq!(ingress.announced, 1, "one ingress announce");
3515 assert_eq!(ingress.announced_bytes, path_len);
3516 assert_eq!(ingress.subscriptions, 1, "one ingress track");
3517 assert_eq!(ingress.broadcasts, 0, "ingress has no viewer refcount");
3518 assert_eq!(ingress.groups, 1);
3519 assert_eq!(ingress.frames, 2);
3520 assert_eq!(ingress.bytes, 10);
3521
3522 let fetched = broadcast.track("video").unwrap().fetch_group(0, None).await.unwrap();
3524 let _ = fetched;
3525 settle().await;
3526 let report = registry.report();
3527 let entry = report.traffic.iter().find(|e| e.path.as_str() == "demo").unwrap();
3528 assert_eq!(entry.publisher.fetches, 1, "one fetch");
3529 assert_eq!(entry.publisher.subscriptions, 1, "fetch does not bump subscriptions");
3530 assert_eq!(entry.publisher.broadcasts, 1, "fetch does not bump the viewer refcount");
3531 assert_eq!(entry.subscriber.fetches, 0, "ingress cannot fetch");
3535 }
3536
3537 #[tokio::test]
3542 async fn test_stats_read_frame_counts_once() {
3543 use crate::Timestamp;
3544 use crate::stats::{Config, Registry, Tier};
3545 use bytes::Bytes;
3546
3547 tokio::time::pause();
3548
3549 let registry = Registry::new(Config::new());
3550 let ctx = registry.tier(Tier::default()).session("acme");
3551
3552 let origin = Origin::random().produce();
3553 let ingress = origin.clone().with_stats(ctx.clone());
3554 let egress = origin.consume().with_stats(ctx.clone());
3555
3556 let mut announced = egress.announced();
3557 let source = ingress.create_broadcast("demo", announce()).unwrap();
3558 let mut dynamic = source.dynamic();
3559 settle().await;
3560 settle().await;
3561
3562 let broadcast = announced.next().await.unwrap().broadcast.unwrap();
3563 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3564 let mut producer = accept_track(&mut dynamic, "video").await;
3565 settle().await;
3566 let mut sub = subscribing.await.unwrap();
3567
3568 producer
3570 .write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
3571 .unwrap();
3572
3573 let frame = sub.read_frame().await.unwrap().expect("frame");
3574 assert_eq!(frame.payload.len(), 5);
3575 settle().await;
3576
3577 let report = registry.report();
3578 let entry = report
3579 .traffic
3580 .iter()
3581 .find(|e| e.path.as_str() == "demo")
3582 .expect("demo tracked");
3583 assert_eq!(entry.publisher.groups, 1, "one group, counted once");
3584 assert_eq!(entry.publisher.frames, 1, "one frame, counted once");
3585 assert_eq!(
3586 entry.publisher.bytes, 5,
3587 "payload counted once, not zero and not doubled"
3588 );
3589 }
3590
3591 #[tokio::test]
3595 async fn test_stats_datagrams_counted_both_sides() {
3596 use crate::Timestamp;
3597 use crate::stats::{Config, Registry, Tier};
3598
3599 tokio::time::pause();
3600
3601 let registry = Registry::new(Config::new());
3602 let ctx = registry.tier(Tier::default()).session("acme");
3603
3604 let origin = Origin::random().produce();
3605 let ingress = origin.clone().with_stats(ctx.clone());
3606 let egress = origin.consume().with_stats(ctx.clone());
3607
3608 let mut announced = egress.announced();
3609 let source = ingress.create_broadcast("demo", announce()).unwrap();
3610 let mut dynamic = source.dynamic();
3611 settle().await;
3612 settle().await;
3613
3614 let broadcast = announced.next().await.unwrap().broadcast.unwrap();
3615 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3616 let mut producer = accept_track(&mut dynamic, "video").await;
3617 settle().await;
3618 let mut sub = subscribing.await.unwrap();
3619
3620 producer.append_datagram(Timestamp::ZERO, &b"hello"[..]).unwrap();
3621 let datagram = sub.recv_datagram().await.unwrap().expect("datagram");
3622 assert_eq!(&datagram.payload[..], b"hello");
3623 settle().await;
3624
3625 let report = registry.report();
3626 let entry = report
3627 .traffic
3628 .iter()
3629 .find(|e| e.path.as_str() == "demo")
3630 .expect("demo tracked");
3631
3632 for (side, traffic) in [("egress", &entry.publisher), ("ingress", &entry.subscriber)] {
3633 assert_eq!(traffic.datagrams, 1, "{side}: one datagram");
3634 assert_eq!(traffic.groups, 1, "{side}: counted as its single-frame group");
3635 assert_eq!(traffic.frames, 1, "{side}: one frame");
3636 assert_eq!(traffic.bytes, 5, "{side}: payload counted once");
3637 }
3638 }
3639
3640 #[test]
3641 fn origin_rejects_reserved_ids() {
3642 assert!(Origin::new(0).is_err());
3643 assert!(Origin::new(1u64 << 62).is_err());
3644 assert_eq!(Origin::new(1).unwrap().id(), 1);
3645
3646 let mut zero = [0u8].as_slice();
3647 assert_eq!(
3648 Origin::decode(&mut zero, crate::lite::Version::Lite05).unwrap(),
3649 Origin::UNKNOWN
3650 );
3651 }
3652
3653 #[test]
3654 fn origin_list_push_fails_at_limit() {
3655 let mut list = OriginList::new();
3656 for _ in 0..MAX_HOPS {
3657 list.push(Origin::random()).unwrap();
3658 }
3659 assert_eq!(list.len(), MAX_HOPS);
3660 assert_eq!(list.push(Origin::random()), Err(TooManyOrigins));
3661 }
3662
3663 #[test]
3664 fn origin_list_replace_first() {
3665 let mut list = OriginList::new();
3666 for _ in 0..3 {
3667 list.push(Origin::UNKNOWN).unwrap();
3668 }
3669
3670 assert!(list.replace_first(Origin::UNKNOWN, Origin::new(7).unwrap()));
3672 assert_eq!(
3673 list.as_slice(),
3674 &[Origin::new(7).unwrap(), Origin::UNKNOWN, Origin::UNKNOWN]
3675 );
3676
3677 assert!(!list.replace_first(Origin::new(99).unwrap(), Origin::new(8).unwrap()));
3679 assert_eq!(list.len(), 3);
3680 }
3681
3682 #[test]
3683 fn origin_list_try_from_vec_enforces_limit() {
3684 let under: Vec<Origin> = (0..MAX_HOPS).map(|_| Origin::random()).collect();
3685 assert!(OriginList::try_from(under).is_ok());
3686
3687 let over: Vec<Origin> = (0..MAX_HOPS + 1).map(|_| Origin::random()).collect();
3688 assert_eq!(OriginList::try_from(over), Err(TooManyOrigins));
3689 }
3690
3691 #[tokio::test]
3692 async fn test_announce() {
3693 tokio::time::pause();
3694
3695 let origin = Origin::random().produce();
3696
3697 let mut consumer1 = origin.consume().announced();
3698 consumer1.assert_next_wait();
3699
3700 let mut broadcast1 = origin.create_broadcast("test1", announce()).unwrap();
3702 settle().await;
3703
3704 consumer1.assert_next_some("test1");
3705 consumer1.assert_next_wait();
3706
3707 let mut consumer2 = origin.consume().announced();
3710
3711 let mut broadcast2 = origin.create_broadcast("test2", announce()).unwrap();
3713 settle().await;
3714
3715 consumer1.assert_next_some("test2");
3716 consumer1.assert_next_wait();
3717
3718 consumer2.assert_next_some("test1");
3719 consumer2.assert_next_some("test2");
3720 consumer2.assert_next_wait();
3721
3722 broadcast1.finish();
3724 settle().await;
3725
3726 consumer1.assert_next_none("test1");
3728 consumer2.assert_next_none("test1");
3729 consumer1.assert_next_wait();
3730 consumer2.assert_next_wait();
3731
3732 let mut consumer3 = origin.consume().announced();
3734 consumer3.assert_next_some("test2");
3735 consumer3.assert_next_wait();
3736
3737 broadcast2.finish();
3738 settle().await;
3739
3740 consumer1.assert_next_none("test2");
3741 consumer2.assert_next_none("test2");
3742 consumer3.assert_next_none("test2");
3743 }
3744
3745 #[tokio::test]
3749 async fn test_duplicate() {
3750 tokio::time::pause();
3751
3752 let origin = Origin::random().produce();
3753 let consumer = origin.consume();
3754 let mut announced = consumer.announced();
3755
3756 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
3757 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
3758 let mut broadcast3 = origin.create_broadcast("test", announce()).unwrap();
3759 settle().await;
3760 assert!(consumer.get_broadcast("test").is_some());
3761
3762 announced.assert_next_some("test");
3763 announced.assert_next_wait();
3764
3765 broadcast2.finish();
3767 settle().await;
3768 assert!(consumer.get_broadcast("test").is_some());
3769 announced.assert_next_wait();
3770
3771 broadcast1.finish();
3773 settle().await;
3774 assert!(consumer.get_broadcast("test").is_some());
3775 announced.assert_next_wait();
3776
3777 broadcast3.finish();
3779 settle().await;
3780 assert!(consumer.get_broadcast("test").is_none());
3781
3782 announced.assert_next_none("test");
3783 announced.assert_next_wait();
3784 }
3785
3786 #[tokio::test]
3789 async fn test_route_failover() {
3790 tokio::time::pause();
3791
3792 let origin = Origin::random().produce();
3793 let consumer = origin.consume();
3794 let mut announced = consumer.announced();
3795
3796 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
3799 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
3800
3801 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
3803 let mut dynamic_a = source_a.dynamic();
3804 settle().await;
3805 settle().await;
3806 let broadcast = consumer.request_broadcast("test").await.unwrap();
3807 announced.assert_next_some("test");
3808
3809 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
3811 let mut dynamic_b = source_b.dynamic();
3812 settle().await;
3813 settle().await;
3814 announced.assert_next_wait();
3815
3816 let subscribing = broadcast.track("video").unwrap().subscribe(None);
3818 let mut producer = accept_track(&mut dynamic_a, "video").await;
3819 settle().await;
3820 dynamic_b.assert_no_request();
3821
3822 let mut sub = subscribing.await.unwrap();
3823 sub.assert_no_group();
3826 assert_eq!(producer.subscription().unwrap().group_start, None);
3827
3828 producer.append_group().unwrap();
3829 producer.append_group().unwrap();
3830 assert_eq!(sub.assert_group().sequence, 0);
3831 assert_eq!(sub.assert_group().sequence, 1);
3832
3833 producer.abort(Error::Dropped).unwrap();
3837 source_a.abort(Error::Dropped).unwrap();
3838 drop(dynamic_a);
3839 settle().await;
3840 announced.assert_next_wait();
3841
3842 let mut producer = accept_track(&mut dynamic_b, "video").await;
3846 settle().await;
3847 sub.assert_no_group();
3848 assert_eq!(producer.subscription().unwrap().group_start, None);
3849 producer.create_group(group::Info { sequence: 1 }).unwrap();
3850 producer.create_group(group::Info { sequence: 2 }).unwrap();
3851 assert_eq!(sub.assert_group().sequence, 2, "groups below the boundary are filtered");
3852 sub.assert_not_closed();
3853 }
3854
3855 #[tokio::test]
3862 async fn test_route_failover_restores_every_track() {
3863 tokio::time::pause();
3864
3865 let origin = Origin::random().produce();
3866 let consumer = origin.consume();
3867
3868 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
3870 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
3871
3872 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
3873 let mut dynamic_a = source_a.dynamic();
3874 settle().await;
3875 settle().await;
3876 let broadcast = consumer.request_broadcast("test").await.unwrap();
3877
3878 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
3880 let mut dynamic_b = source_b.dynamic();
3881 settle().await;
3882 settle().await;
3883
3884 const TRACKS: [(&str, u64); 3] = [("video", 3), ("audio", 1), ("data", 2)];
3888
3889 let subscribing: Vec<_> = TRACKS
3890 .iter()
3891 .map(|(name, _)| broadcast.track(name).unwrap().subscribe(None))
3892 .collect();
3893 let mut producers_a = accept_tracks(&mut dynamic_a, TRACKS.len()).await;
3894 settle().await;
3895
3896 let mut subs = Vec::new();
3897 for ((name, groups), subscribing) in TRACKS.iter().zip(subscribing) {
3898 let mut sub = subscribing.await.unwrap();
3899 let producer = producers_a
3900 .get_mut(*name)
3901 .unwrap_or_else(|| panic!("{name} was never dispatched"));
3902 for expected in 0..*groups {
3903 producer.append_group().unwrap();
3904 assert_eq!(sub.assert_group().sequence, expected, "{name} did not start");
3905 }
3906 subs.push((*name, *groups, sub));
3907 }
3908
3909 for (_, producer) in producers_a.drain() {
3911 producer.abort(Error::Dropped).unwrap();
3912 }
3913 source_a.abort(Error::Dropped).unwrap();
3914 drop(dynamic_a);
3915 settle().await;
3916
3917 let mut producers_b = accept_tracks(&mut dynamic_b, TRACKS.len()).await;
3920 settle().await;
3921
3922 for (_, _, sub) in subs.iter_mut() {
3925 sub.assert_no_group();
3926 }
3927 settle().await;
3928
3929 for (name, groups, sub) in subs.iter_mut() {
3930 let producer = producers_b
3931 .get_mut(*name)
3932 .unwrap_or_else(|| panic!("{name} was never re-dispatched to the standby"));
3933 assert_eq!(
3936 producer
3937 .subscription()
3938 .unwrap_or_else(|| panic!("{name} resumed without a subscription"))
3939 .group_start,
3940 None,
3941 "{name} must keep the subscriber's live-edge demand"
3942 );
3943 let boundary = *groups;
3944
3945 producer.create_group(group::Info { sequence: boundary - 1 }).unwrap();
3947 producer.create_group(group::Info { sequence: boundary }).unwrap();
3948 assert_eq!(sub.assert_group().sequence, boundary, "{name} did not resume");
3949 sub.assert_not_closed();
3950 }
3951 }
3952
3953 #[tokio::test]
3956 async fn test_broadcast_route_watch() {
3957 let mut producer = broadcast::Info::new().produce();
3958 let mut consumer = producer.consume();
3959
3960 assert_eq!(consumer.route_changed().await.unwrap(), broadcast::Route::default());
3962
3963 producer.set_route(broadcast::Route::default()).unwrap();
3965 assert!(consumer.route_changed().now_or_never().is_none());
3966
3967 let mut hops = OriginList::new();
3968 hops.push(Origin::new(7).unwrap()).unwrap();
3969 let route = broadcast::Route::new().with_hops(hops).with_cost(3);
3970 producer.set_route(route.clone()).unwrap();
3971 assert_eq!(consumer.route_changed().await.unwrap(), route);
3972
3973 let mut fresh = producer.consume();
3975 assert_eq!(fresh.route_changed().await.unwrap(), route);
3976
3977 drop(producer);
3978 assert!(matches!(consumer.route_changed().await.unwrap_err(), Error::Dropped));
3979 }
3980
3981 #[tokio::test]
3985 async fn test_route_cost_update() {
3986 tokio::time::pause();
3987
3988 let origin = Info::new(origin_keyed("test", Origin::new(3).unwrap(), true)).produce();
3992 let consumer = origin.consume();
3993 let mut announced = consumer.announced();
3994
3995 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
3998 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
3999
4000 let mut source_a = origin
4002 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4003 .unwrap();
4004 let mut dynamic_a = source_a.dynamic();
4005 settle().await;
4006 let broadcast = consumer.request_broadcast("test").await.unwrap();
4007 announced.assert_next_some("test");
4008
4009 let mut watch = broadcast.clone();
4010 assert_eq!(watch.route_changed().await.unwrap().hops, hops_a);
4011
4012 let mut source_b = origin
4013 .create_broadcast("test", announce().with_hops(hops_b.clone()))
4014 .unwrap();
4015 let mut dynamic_b = source_b.dynamic();
4016 settle().await;
4017 assert!(
4018 watch.route_changed().now_or_never().is_none(),
4019 "a losing standby must not change the advertised route"
4020 );
4021
4022 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4024 let mut producer = accept_track(&mut dynamic_a, "video").await;
4025 settle().await;
4026 let mut sub = subscribing.await.unwrap();
4027 producer.append_group().unwrap();
4028 assert_eq!(sub.assert_group().sequence, 0);
4029
4030 source_a
4033 .set_route(announce().with_hops(hops_a.clone()).with_cost(10))
4034 .unwrap();
4035 settle().await;
4036 assert_eq!(watch.route_changed().await.unwrap().hops, hops_b);
4037 announced.assert_next_wait();
4038
4039 let mut producer_b = accept_track(&mut dynamic_b, "video").await;
4040 settle().await;
4041 sub.assert_no_group();
4044 assert_eq!(producer_b.subscription().unwrap().group_start, None);
4045 producer_b.create_group(group::Info { sequence: 1 }).unwrap();
4046 assert_eq!(sub.assert_group().sequence, 1);
4047 sub.assert_not_closed();
4048
4049 source_b
4051 .set_route(announce().with_hops(hops_b.clone()).with_cost(5))
4052 .unwrap();
4053 settle().await;
4054 let advertised = watch.route_changed().await.unwrap();
4055 assert_eq!(advertised.hops, hops_b);
4056 assert_eq!(advertised.cost, 5);
4057 announced.assert_next_wait();
4058 }
4059
4060 #[tokio::test]
4063 async fn test_completed_track_survives_route_churn() {
4064 tokio::time::pause();
4065
4066 let origin = Origin::random().produce();
4067 let consumer = origin.consume();
4068
4069 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4071 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4072
4073 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4074 let mut dynamic_a = source_a.dynamic();
4075 settle().await;
4076 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4077 let mut dynamic_b = source_b.dynamic();
4078 settle().await;
4079 settle().await;
4080 let broadcast = consumer.request_broadcast("test").await.unwrap();
4081
4082 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4084 let mut producer = accept_track(&mut dynamic_a, "video").await;
4085 settle().await;
4086 let mut sub = subscribing.await.unwrap();
4087 producer.append_group().unwrap();
4088 assert_eq!(sub.assert_group().sequence, 0);
4089 producer.finish().unwrap();
4090 drop(producer);
4091 settle().await;
4092 sub.assert_closed();
4093
4094 source_a.abort(Error::Dropped).unwrap();
4096 drop(dynamic_a);
4097 settle().await;
4098 dynamic_b.assert_no_request();
4099
4100 let mut late = broadcast.track("video").unwrap().subscribe(None).await.unwrap();
4102 late.assert_closed();
4103 }
4104
4105 #[tokio::test]
4109 async fn test_refused_track_aborts_instantly() {
4110 tokio::time::pause();
4111
4112 let origin = Origin::random().produce();
4113 let consumer = origin.consume();
4114
4115 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4116 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4117 let mut dynamic = source.dynamic();
4118 settle().await;
4119 settle().await;
4120 let broadcast = consumer.request_broadcast("test").await.unwrap();
4121
4122 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4123 let request = dynamic.requested_track().await.unwrap();
4124 request.reject(Error::NotFound);
4125 settle().await;
4126
4127 assert!(matches!(subscribing.await, Err(Error::NotFound)));
4129 dynamic.assert_no_request();
4130 }
4131
4132 #[tokio::test]
4137 async fn test_stale_rejection_does_not_abort_a_handover() {
4138 tokio::time::pause();
4139
4140 let origin = Origin::random().produce();
4141 let consumer = origin.consume();
4142
4143 let publisher = Origin::new(1).unwrap();
4144 let peer = Origin::new(5).unwrap();
4145 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
4146 let local = OriginList::try_from(vec![publisher]).unwrap();
4147
4148 let source_remote = origin
4150 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
4151 .unwrap();
4152 let mut dynamic_remote = source_remote.dynamic();
4153 settle().await;
4154 settle().await;
4155 let broadcast = consumer.request_broadcast("test").await.unwrap();
4156 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4157 let request_remote = dynamic_remote.requested_track().await.unwrap();
4158
4159 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
4162 let mut dynamic_local = source_local.dynamic();
4163 request_remote.reject(Error::NotFound);
4164 settle().await;
4165
4166 let mut producer_local = accept_track(&mut dynamic_local, "video").await;
4168 settle().await;
4169 let mut sub = subscribing
4170 .await
4171 .expect("the handover must win over the stale rejection");
4172 producer_local.append_group().unwrap();
4173 assert_eq!(sub.assert_group().sequence, 0);
4174 sub.assert_not_closed();
4175 }
4176
4177 #[tokio::test]
4181 async fn test_route_handover() {
4182 tokio::time::pause();
4183
4184 let origin = Origin::random().produce();
4185 let consumer = origin.consume();
4186 let mut announced = consumer.announced();
4187
4188 let hops_long = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4190 let hops_short = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4191
4192 let source_a = origin
4193 .create_broadcast("test", announce().with_hops(hops_long))
4194 .unwrap();
4195 let mut dynamic_a = source_a.dynamic();
4196 settle().await;
4197 settle().await;
4198 let broadcast = consumer.request_broadcast("test").await.unwrap();
4199 announced.assert_next_some("test");
4200
4201 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4202 let mut producer_a = accept_track(&mut dynamic_a, "video").await;
4203 settle().await;
4204 let mut sub = subscribing.await.unwrap();
4205 producer_a.append_group().unwrap();
4206 producer_a.append_group().unwrap();
4207 assert_eq!(sub.assert_group().sequence, 0);
4208 assert_eq!(sub.assert_group().sequence, 1);
4209
4210 let source_b = origin
4213 .create_broadcast("test", announce().with_hops(hops_short))
4214 .unwrap();
4215 let mut dynamic_b = source_b.dynamic();
4216 settle().await;
4217 settle().await;
4218 announced.assert_next_wait();
4219
4220 let mut producer_b = accept_track(&mut dynamic_b, "video").await;
4221 settle().await;
4222
4223 sub.assert_no_group();
4227 assert_eq!(producer_a.subscription().unwrap().group_end, Some(1));
4228 assert_eq!(producer_b.subscription().unwrap().group_start, None);
4229
4230 producer_a.create_group(group::Info { sequence: 2 }).unwrap();
4232 producer_b.create_group(group::Info { sequence: 2 }).unwrap();
4233 producer_b.create_group(group::Info { sequence: 3 }).unwrap();
4234 assert_eq!(sub.assert_group().sequence, 2);
4235 assert_eq!(sub.assert_group().sequence, 3);
4236 sub.assert_no_group();
4237 sub.assert_not_closed();
4238 }
4239
4240 #[tokio::test(start_paused = true)]
4243 async fn test_route_unannounce_immediate() {
4244 let origin = Origin::random().produce();
4245 let consumer = origin.consume();
4246 let mut announced = consumer.announced();
4247
4248 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4249 let mut source = origin
4250 .create_broadcast("test", announce().with_hops(hops.clone()))
4251 .unwrap();
4252 settle().await;
4253 let broadcast = consumer.request_broadcast("test").await.unwrap();
4254 announced.assert_next_some("test");
4255
4256 source.finish();
4259 settle().await;
4260 announced.assert_next_none("test");
4261
4262 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4264 settle().await;
4265 let fresh = consumer.request_broadcast("test").await.unwrap();
4266 announced.assert_next_some("test");
4267 assert!(
4268 !fresh.is_clone(&broadcast),
4269 "re-create must not splice the old broadcast"
4270 );
4271 }
4272
4273 #[tokio::test(start_paused = true)]
4278 async fn test_route_detach_immediate() {
4279 let origin = Origin::random().produce();
4280 let consumer = origin.consume();
4281 let mut announced = consumer.announced();
4282
4283 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4284 let source = origin
4285 .create_broadcast("test", announce().with_hops(hops.clone()))
4286 .unwrap();
4287 let mut dynamic = source.dynamic();
4288 settle().await;
4289 settle().await;
4290 let broadcast = consumer.request_broadcast("test").await.unwrap();
4291 announced.assert_next_some("test");
4292
4293 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4294 let producer = accept_track(&mut dynamic, "video").await;
4295 settle().await;
4296 let mut sub = subscribing.await.unwrap();
4297
4298 drop(producer);
4300 source.abort(Error::Dropped).unwrap();
4301 drop(dynamic);
4302
4303 settle().await;
4304 announced.assert_next_none("test");
4305 sub.assert_error();
4306
4307 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4310 settle().await;
4311 settle().await;
4312 let fresh = consumer.request_broadcast("test").await.unwrap();
4313 announced.assert_next_some("test");
4314 assert!(
4315 !fresh.is_clone(&broadcast),
4316 "re-create must not splice the old broadcast"
4317 );
4318 }
4319
4320 #[tokio::test(start_paused = true)]
4325 async fn test_idle_track_releases_without_respinning() {
4326 let origin = Info::new(Origin::random()).produce();
4327 let consumer = origin.consume();
4328
4329 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4330 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4331 let mut dynamic = source.dynamic();
4332 settle().await;
4333 let broadcast = consumer.request_broadcast("test").await.unwrap();
4334
4335 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4336 let producer = accept_track(&mut dynamic, "video").await;
4337 settle().await;
4338 let sub = subscribing.await.unwrap();
4339
4340 drop(sub);
4343 tokio::time::sleep(TRACK_IDLE_LINGER / 2).await;
4344 settle().await;
4345 assert!(
4346 producer.poll_unused(&kio::Waiter::noop()).is_pending(),
4347 "the copy must stay spliced inside the linger",
4348 );
4349
4350 tokio::time::sleep(TRACK_IDLE_LINGER).await;
4353 settle().await;
4354 assert!(
4355 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4356 "an idle copy must be released after the linger",
4357 );
4358
4359 for _ in 0..3 {
4363 tokio::time::sleep(TRACK_IDLE_LINGER).await;
4364 settle().await;
4365 assert!(
4366 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4367 "an unread copy must stay released, not be re-spliced",
4368 );
4369 }
4370 assert!(
4371 dynamic.requested_track().now_or_never().is_none(),
4372 "an unread track must not be re-requested",
4373 );
4374 drop(producer);
4375
4376 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4378 let mut producer = accept_track(&mut dynamic, "video").await;
4379 settle().await;
4380 let mut sub = subscribing.await.unwrap();
4381 producer.append_group().unwrap();
4382 assert_eq!(sub.assert_group().sequence, 0);
4383 }
4384
4385 #[tokio::test(start_paused = true)]
4389 async fn test_back_to_back_fetches_reuse_the_track() {
4390 let origin = Info::new(Origin::random()).produce();
4391 let consumer = origin.consume();
4392
4393 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4394 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4395 let mut dynamic = source.dynamic();
4396 settle().await;
4397 let broadcast = consumer.request_broadcast("test").await.unwrap();
4398
4399 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4401 let mut producer = accept_track(&mut dynamic, "video").await;
4402 producer.append_group().unwrap().finish().unwrap();
4403 settle().await;
4404 let first = fetching.await.expect("first fetch");
4405 drop(first);
4406
4407 settle().await;
4409 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4410 settle().await;
4411 assert!(
4412 dynamic.requested_track().now_or_never().is_none(),
4413 "a fetch inside the linger must reuse the track, not re-request it",
4414 );
4415 drop(fetching.await.expect("second fetch"));
4416
4417 tokio::time::sleep(TRACK_IDLE_LINGER * 2).await;
4419 settle().await;
4420 assert!(
4421 producer.poll_unused(&kio::Waiter::noop()).is_ready(),
4422 "the copy must be released once the fetches stop",
4423 );
4424 drop(producer);
4425
4426 settle().await;
4428 let fetching = broadcast.track("video").unwrap().fetch_group(0, None);
4429 let mut producer = accept_track(&mut dynamic, "video").await;
4430 producer.append_group().unwrap().finish().unwrap();
4431 settle().await;
4432 fetching.await.expect("fetch after the linger");
4433 }
4434
4435 #[tokio::test(start_paused = true)]
4439 async fn test_linger_reconnect_splices() {
4440 let origin = Info::new(Origin::random())
4441 .with_linger(Duration::from_secs(5))
4442 .produce();
4443 let consumer = origin.consume();
4444 let mut announced = consumer.announced();
4445
4446 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4447 let source = origin
4448 .create_broadcast("test", announce().with_hops(hops.clone()))
4449 .unwrap();
4450 let mut dynamic = source.dynamic();
4451 settle().await;
4452 settle().await;
4453 let broadcast = consumer.request_broadcast("test").await.unwrap();
4454 announced.assert_next_some("test");
4455
4456 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4457 let mut producer = accept_track(&mut dynamic, "video").await;
4458 settle().await;
4459 let mut sub = subscribing.await.unwrap();
4460
4461 producer.append_group().unwrap();
4462 producer.append_group().unwrap();
4463 assert_eq!(sub.assert_group().sequence, 0);
4464 assert_eq!(sub.assert_group().sequence, 1);
4465
4466 drop(producer);
4469 source.abort(Error::Dropped).unwrap();
4470 drop(dynamic);
4471 settle().await;
4472
4473 announced.assert_next_wait();
4475 sub.assert_no_group();
4476 sub.assert_not_closed();
4477
4478 let during = consumer.request_broadcast("test").await.unwrap();
4480 assert!(during.is_clone(&broadcast), "the lingering broadcast still resolves");
4481
4482 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4485 let mut dynamic = source.dynamic();
4486 settle().await;
4487 settle().await;
4488 announced.assert_next_wait();
4489 let again = consumer.request_broadcast("test").await.unwrap();
4490 assert!(again.is_clone(&broadcast), "the reconnect must splice, not replace");
4491
4492 let mut producer = accept_track(&mut dynamic, "video").await;
4496 settle().await;
4497 sub.assert_no_group();
4498 assert_eq!(producer.subscription().unwrap().group_start, None);
4499 producer.create_group(group::Info { sequence: 2 }).unwrap();
4500 assert_eq!(sub.assert_group().sequence, 2);
4501 sub.assert_not_closed();
4502 }
4503
4504 #[tokio::test(start_paused = true)]
4507 async fn test_linger_expiry_closes() {
4508 let origin = Info::new(Origin::random())
4509 .with_linger(Duration::from_secs(5))
4510 .produce();
4511 let consumer = origin.consume();
4512 let mut announced = consumer.announced();
4513
4514 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4515 let source = origin
4516 .create_broadcast("test", announce().with_hops(hops.clone()))
4517 .unwrap();
4518 let mut dynamic = source.dynamic();
4519 settle().await;
4520 settle().await;
4521 let broadcast = consumer.request_broadcast("test").await.unwrap();
4522 announced.assert_next_some("test");
4523
4524 let subscribing = broadcast.track("video").unwrap().subscribe(None);
4525 let producer = accept_track(&mut dynamic, "video").await;
4526 settle().await;
4527 let mut sub = subscribing.await.unwrap();
4528
4529 drop(producer);
4530 source.abort(Error::Dropped).unwrap();
4531 drop(dynamic);
4532 settle().await;
4533 announced.assert_next_wait();
4534
4535 tokio::time::sleep(std::time::Duration::from_secs(6)).await;
4537 settle().await;
4538 announced.assert_next_none("test");
4539 sub.assert_error();
4540
4541 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4543 settle().await;
4544 settle().await;
4545 let fresh = consumer.request_broadcast("test").await.unwrap();
4546 announced.assert_next_some("test");
4547 assert!(
4548 !fresh.is_clone(&broadcast),
4549 "a late re-create must not splice the expired broadcast"
4550 );
4551 }
4552
4553 #[tokio::test(start_paused = true)]
4557 async fn test_linger_forever() {
4558 let origin = Info::new(Origin::random()).with_linger(Duration::MAX).produce();
4559 let consumer = origin.consume();
4560 let mut announced = consumer.announced();
4561
4562 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4563 let source = origin
4564 .create_broadcast("test", announce().with_hops(hops.clone()))
4565 .unwrap();
4566 settle().await;
4567 let broadcast = consumer.request_broadcast("test").await.unwrap();
4568 announced.assert_next_some("test");
4569
4570 source.abort(Error::Dropped).unwrap();
4571 settle().await;
4572
4573 tokio::time::sleep(std::time::Duration::from_secs(60 * 60 * 24 * 3)).await;
4575 announced.assert_next_wait();
4576 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4577 settle().await;
4578 settle().await;
4579 let again = consumer.request_broadcast("test").await.unwrap();
4580 assert!(again.is_clone(&broadcast), "the reconnect must splice, not replace");
4581 drop(source);
4582 }
4583
4584 #[tokio::test(start_paused = true)]
4598 async fn test_linger_parks_a_live_subscription() {
4599 let origin = Info::new(Origin::random())
4600 .with_linger(Duration::from_secs(5))
4601 .produce();
4602 let consumer = origin.consume();
4603
4604 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4605 let source = origin
4606 .create_broadcast("test", announce().with_hops(hops.clone()))
4607 .unwrap();
4608 let mut dynamic = source.dynamic();
4609 settle().await;
4610 settle().await;
4611 let broadcast = consumer.request_broadcast("test").await.unwrap();
4612
4613 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
4616 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
4617 settle().await;
4618 let mut sub = subscribing.await.unwrap();
4619 producer.append_group().unwrap();
4620 assert_eq!(sub.assert_group().sequence, 0);
4621
4622 source.abort(Error::Dropped).unwrap();
4627 settle().await;
4628 settle().await;
4629 sub.assert_not_closed();
4630
4631 tokio::time::sleep(Duration::from_secs(4)).await;
4634 settle().await;
4635 sub.assert_not_closed();
4636
4637 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4639 let mut dynamic = source.dynamic();
4640 settle().await;
4641 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
4642 settle().await;
4643 producer.create_group(group::Info { sequence: 1 }).unwrap();
4644 assert_eq!(sub.assert_group().sequence, 1);
4645 sub.assert_not_closed();
4646 }
4647
4648 #[tokio::test(start_paused = true)]
4659 async fn test_idle_release_survives_the_route_leaving() {
4660 let origin = Info::new(Origin::random())
4661 .with_linger(Duration::from_secs(600))
4662 .produce();
4663 let consumer = origin.consume();
4664
4665 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4666 let source = origin
4667 .create_broadcast("test", announce().with_hops(hops.clone()))
4668 .unwrap();
4669 let mut dynamic = source.dynamic();
4670 settle().await;
4671 settle().await;
4672 let broadcast = consumer.request_broadcast("test").await.unwrap();
4673
4674 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
4675 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
4676 settle().await;
4677 let mut sub = subscribing.await.unwrap();
4678 producer.append_group().unwrap();
4679 assert_eq!(sub.assert_group().sequence, 0);
4680
4681 source.abort(Error::Dropped).unwrap();
4684 settle().await;
4685 settle().await;
4686 drop(sub);
4687 drop(producer);
4688 drop(dynamic);
4689 settle().await;
4690 tokio::time::sleep(TRACK_IDLE_LINGER + Duration::from_secs(1)).await;
4691 settle().await;
4692
4693 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4696 let mut dynamic = source.dynamic();
4697 settle().await;
4698 settle().await;
4699 let broadcast = consumer.request_broadcast("test").await.unwrap();
4700 let subscribing = broadcast.track("catalog.json").unwrap().subscribe(None);
4701 let mut producer = accept_track(&mut dynamic, "catalog.json").await;
4702 settle().await;
4703 let mut sub = subscribing.await.unwrap();
4704 producer.append_group().unwrap();
4705 assert_eq!(
4706 sub.assert_group().sequence,
4707 0,
4708 "the reconnect's first group must not be filtered by a stale boundary"
4709 );
4710 }
4711
4712 #[tokio::test(start_paused = true)]
4715 async fn test_linger_skipped_on_finish() {
4716 let origin = Info::new(Origin::random())
4717 .with_linger(Duration::from_secs(5))
4718 .produce();
4719 let consumer = origin.consume();
4720 let mut announced = consumer.announced();
4721
4722 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4723 let mut source = origin
4724 .create_broadcast("test", announce().with_hops(hops.clone()))
4725 .unwrap();
4726 settle().await;
4727 let broadcast = consumer.request_broadcast("test").await.unwrap();
4728 announced.assert_next_some("test");
4729
4730 source.finish();
4733 settle().await;
4734 announced.assert_next_none("test");
4735
4736 let _source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
4738 settle().await;
4739 let fresh = consumer.request_broadcast("test").await.unwrap();
4740 announced.assert_next_some("test");
4741 assert!(
4742 !fresh.is_clone(&broadcast),
4743 "a finish must not leave a lingering broadcast to splice into"
4744 );
4745 }
4746
4747 #[tokio::test]
4750 async fn test_announce_toggle() {
4751 tokio::time::pause();
4752
4753 let origin = Origin::random().produce();
4754 let consumer = origin.consume();
4755 let mut announced = consumer.announced();
4756
4757 let mut source = origin.create_broadcast("test", broadcast::Route::new()).unwrap();
4758 settle().await;
4759
4760 announced.assert_next_wait();
4762 let broadcast = consumer
4763 .get_broadcast("test")
4764 .expect("offline broadcast is still routable");
4765 assert!(!broadcast.route().announce);
4766
4767 let requested = consumer.request_broadcast("test").await.unwrap();
4769 assert!(requested.is_clone(&broadcast));
4770
4771 source.set_route(announce()).unwrap();
4773 settle().await;
4774 let face = announced.assert_next_some("test");
4775 assert!(face.is_clone(&broadcast));
4776
4777 let mut fresh = origin.consume().announced();
4779 fresh.assert_next_some("test");
4780 fresh.assert_next_wait();
4781
4782 source.set_route(broadcast::Route::new()).unwrap();
4784 settle().await;
4785 announced.assert_next_none("test");
4786 assert!(consumer.get_broadcast("test").is_some());
4787 let mut fresh = origin.consume().announced();
4788 fresh.assert_next_wait();
4789
4790 source.finish();
4791 settle().await;
4792 assert!(consumer.get_broadcast("test").is_none());
4793 }
4794
4795 #[tokio::test]
4798 async fn test_announce_beats_offline() {
4799 tokio::time::pause();
4800
4801 let origin = Origin::random().produce();
4802 let consumer = origin.consume();
4803 let mut announced = consumer.announced();
4804
4805 let _offline = origin.create_broadcast("test", broadcast::Route::new()).unwrap();
4807 settle().await;
4808 announced.assert_next_wait();
4809
4810 let mut announced_source = origin.create_broadcast("test", announce().with_cost(10)).unwrap();
4813 settle().await;
4814 announced.assert_next_some("test");
4815 let face = consumer.get_broadcast("test").unwrap();
4816 assert!(face.route().announce);
4817 assert_eq!(face.route().cost, 10);
4818
4819 announced_source.finish();
4822 settle().await;
4823 announced.assert_next_none("test");
4824 assert!(consumer.get_broadcast("test").is_some());
4825 }
4826
4827 #[tokio::test]
4830 async fn test_better_source_no_churn() {
4831 tokio::time::pause();
4832
4833 let origin = Origin::random().produce();
4834 let mut announced = origin.consume().announced();
4835
4836 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap(), Origin::new(3).unwrap()]).unwrap();
4839 let hops_b = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4840 let _a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
4841 settle().await;
4842 let face = announced.assert_next_some("test");
4843
4844 let _b = origin
4845 .create_broadcast("test", announce().with_hops(hops_b.clone()))
4846 .unwrap();
4847 settle().await;
4848 announced.assert_next_wait();
4849 let current = origin.consume().get_broadcast("test").unwrap();
4850 assert!(current.is_clone(&face), "the broadcast identity must not change");
4851 assert_eq!(current.route().hops, hops_b);
4853 }
4854
4855 #[tokio::test]
4861 async fn test_publisher_mismatch_replaces() {
4862 tokio::time::pause();
4863
4864 let origin = Origin::random().produce();
4865 let consumer = origin.consume();
4866 let mut announced = consumer.announced();
4867
4868 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4869 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
4870
4871 let mut source_a = origin
4872 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4873 .unwrap();
4874 settle().await;
4875 let face_a = announced.assert_next_some("test");
4876
4877 let _source_b = origin
4880 .create_broadcast("test", announce().with_hops(hops_b.clone()))
4881 .unwrap();
4882 settle().await;
4883 settle().await;
4884 announced.assert_next_none("test");
4885 let face_b = announced.assert_next_some("test");
4886 assert!(!face_b.is_clone(&face_a), "a replacement, never a splice");
4887 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_b);
4888 assert!(face_a.is_closed(), "the displaced front must close");
4892
4893 source_a.finish();
4895 settle().await;
4896 settle().await;
4897 announced.assert_next_wait();
4898 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_b);
4899 }
4900
4901 #[tokio::test]
4904 async fn test_displaced_publisher_reclaims_path_when_replacement_leaves() {
4905 tokio::time::pause();
4906
4907 let origin = Origin::random().produce();
4908 let consumer = origin.consume();
4909 let mut announced = consumer.announced();
4910
4911 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4912 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
4913
4914 let mut source_a = origin
4916 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4917 .unwrap();
4918 settle().await;
4919 announced.assert_next_some("test");
4920
4921 let mut source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4924 settle().await;
4925 settle().await;
4926 announced.assert_next_none("test");
4927 announced.assert_next_some("test");
4928 announced.assert_next_wait();
4931
4932 source_b.finish();
4934 settle().await;
4935 settle().await;
4936
4937 assert!(
4938 consumer.get_broadcast("test").is_some(),
4939 "the still-live publisher A should reclaim the path once its replacement leaves"
4940 );
4941 let recovered = consumer.request_broadcast("test").await.unwrap();
4942 assert_eq!(recovered.route().hops, hops_a);
4943
4944 announced.assert_next_none("test");
4945 announced.assert_next_some("test");
4946 announced.assert_next_wait();
4947
4948 source_a.finish();
4949 }
4950
4951 #[tokio::test]
4955 async fn test_reclaimed_publisher_can_still_replace_itself() {
4956 tokio::time::pause();
4957
4958 let origin = Origin::random().produce();
4959 let consumer = origin.consume();
4960
4961 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
4962 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
4963 let hops_c = OriginList::try_from(vec![Origin::new(3).unwrap()]).unwrap();
4964
4965 let mut source_a1 = origin
4967 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4968 .unwrap();
4969 let mut source_a2 = origin
4970 .create_broadcast("test", announce().with_hops(hops_a.clone()))
4971 .unwrap();
4972 settle().await;
4973 settle().await;
4974 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_a);
4975
4976 let mut source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
4978 settle().await;
4979 settle().await;
4980
4981 source_b.finish();
4983 settle().await;
4984 settle().await;
4985 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_a);
4986
4987 source_a1.set_route(announce().with_hops(hops_c.clone())).unwrap();
4991 settle().await;
4992 settle().await;
4993 assert_eq!(
4994 consumer.get_broadcast("test").unwrap().route().hops,
4995 hops_c,
4996 "the new publisher must take the path over, not stand by behind the old front"
4997 );
4998
4999 source_a1.finish();
5000 source_a2.finish();
5001 }
5002
5003 #[tokio::test]
5006 async fn test_repricing_does_not_earn_a_takeover() {
5007 tokio::time::pause();
5008
5009 let origin = Origin::random().produce();
5010 let consumer = origin.consume();
5011 let mut announced = consumer.announced();
5012
5013 let hops_a = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5014 let hops_b = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5015
5016 let mut source_a = origin
5017 .create_broadcast("test", announce().with_hops(hops_a.clone()).with_cost(5))
5018 .unwrap();
5019 settle().await;
5020 announced.assert_next_some("test");
5021
5022 let mut source_b = origin
5024 .create_broadcast("test", announce().with_hops(hops_b.clone()))
5025 .unwrap();
5026 settle().await;
5027 settle().await;
5028 announced.assert_next_none("test");
5029 announced.assert_next_some("test");
5030 announced.assert_next_wait();
5031
5032 source_a
5035 .set_route(announce().with_hops(hops_a.clone()).with_cost(9))
5036 .unwrap();
5037 settle().await;
5038 settle().await;
5039 assert_eq!(
5040 consumer.get_broadcast("test").unwrap().route().hops,
5041 hops_b,
5042 "a repricing must not take the path back from the live front"
5043 );
5044 announced.assert_next_wait();
5045
5046 source_a.finish();
5047 source_b.finish();
5048 }
5049
5050 #[tokio::test]
5055 async fn test_reconnect_wins_over_stale_route() {
5056 tokio::time::pause();
5057
5058 let origin = Origin::random().produce();
5059 let consumer = origin.consume();
5060
5061 let publisher = Origin::new(1).unwrap();
5062 let hops = OriginList::try_from(vec![publisher]).unwrap();
5063
5064 let stale = origin
5067 .create_broadcast("test", announce().with_hops(hops.clone()))
5068 .unwrap();
5069 let mut stale_dynamic = stale.dynamic();
5070 settle().await;
5071
5072 let fresh = origin
5074 .create_broadcast("test", announce().with_hops(hops.clone()))
5075 .unwrap();
5076 let mut fresh_dynamic = fresh.dynamic();
5077 settle().await;
5078 settle().await;
5079
5080 let broadcast = consumer.request_broadcast("test").await.unwrap();
5082 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5083 settle().await;
5084 let _producer = accept_track(&mut fresh_dynamic, "video").await;
5085 settle().await;
5086 subscribing.await.unwrap();
5087 stale_dynamic.assert_no_request();
5088 }
5089
5090 #[tokio::test]
5096 async fn test_carrying_reconnect_switches_immediately() {
5097 tokio::time::pause();
5098
5099 let origin = Origin::random().produce();
5100 let consumer = origin.consume();
5101
5102 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5103
5104 let stale = origin
5105 .create_broadcast("test", announce().with_hops(hops.clone()))
5106 .unwrap();
5107 let mut stale_dynamic = stale.dynamic();
5108 settle().await;
5109
5110 let broadcast = consumer.request_broadcast("test").await.unwrap();
5112 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5113 settle().await;
5114 let _stale_producer = accept_track(&mut stale_dynamic, "video").await;
5115 settle().await;
5116 let _subscription = subscribing.await.unwrap();
5118
5119 let fresh = origin
5121 .create_broadcast("test", announce().with_hops(hops.clone()))
5122 .unwrap();
5123 let mut fresh_dynamic = fresh.dynamic();
5124 settle().await;
5125 settle().await;
5126
5127 let _fresh_producer = accept_track(&mut fresh_dynamic, "video").await;
5130 }
5131
5132 #[tokio::test]
5142 async fn test_offline_mismatch_never_evicts_a_live_front() {
5143 tokio::time::pause();
5144
5145 let origin = Origin::random().produce();
5146 let consumer = origin.consume();
5147 let mut announced = consumer.announced();
5148
5149 let hops_live = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5150 let hops_cache = OriginList::try_from(vec![Origin::new(2).unwrap()]).unwrap();
5151
5152 let mut live = origin
5153 .create_broadcast("test", announce().with_hops(hops_live.clone()))
5154 .unwrap();
5155 let mut live_dynamic = live.dynamic();
5156 settle().await;
5157 let face = announced.assert_next_some("test");
5158
5159 let broadcast = consumer.request_broadcast("test").await.unwrap();
5161 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5162 settle().await;
5163 let _producer = accept_track(&mut live_dynamic, "video").await;
5164 settle().await;
5165 subscribing.await.unwrap();
5166
5167 let cache = origin
5170 .create_broadcast("test", broadcast::Route::new().with_hops(hops_cache.clone()))
5171 .unwrap();
5172 settle().await;
5173 settle().await;
5174 announced.assert_next_wait();
5175 assert!(!face.is_closed(), "the live front must survive");
5176 assert_eq!(consumer.get_broadcast("test").unwrap().route().hops, hops_live);
5177
5178 live.finish();
5181 settle().await;
5182 settle().await;
5183 announced.assert_next_none("test");
5184 let taken = consumer
5185 .get_broadcast("test")
5186 .expect("the parked source must take over");
5187 assert_eq!(taken.route().hops, hops_cache);
5188 announced.assert_next_wait();
5190 drop(cache);
5191 }
5192
5193 #[tokio::test]
5199 async fn test_dispatch_excludes_requester() {
5200 tokio::time::pause();
5201
5202 let origin = Origin::random().produce();
5203 let consumer = origin.consume();
5204
5205 let peer = Origin::new(5).unwrap();
5206 let publisher = Origin::new(1).unwrap();
5207 let tainted = OriginList::try_from(vec![publisher, peer]).unwrap();
5209 let clean = OriginList::try_from(vec![publisher]).unwrap();
5210
5211 let source_a = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
5212 let mut dynamic_a = source_a.dynamic();
5213 settle().await;
5214 let source_b = origin
5215 .create_broadcast("test", announce().with_hops(clean).with_cost(5))
5216 .unwrap();
5217 let mut dynamic_b = source_b.dynamic();
5218 settle().await;
5219 settle().await;
5220
5221 let shared = consumer.request_broadcast("test").await.unwrap();
5224 let subscribing = shared.track("video").unwrap().subscribe(None);
5225 let _producer_a = accept_track(&mut dynamic_a, "video").await;
5226 settle().await;
5227 subscribing.await.unwrap();
5228
5229 let scoped = consumer.clone().excluding(peer);
5234 let pinned = scoped.request_broadcast("test").await.unwrap();
5235 let subscribing = pinned.track("video").unwrap().subscribe(None);
5236 let _producer_b = accept_track(&mut dynamic_b, "video").await;
5237 settle().await;
5238 subscribing.await.unwrap();
5239 dynamic_a.assert_no_request();
5240 }
5241
5242 #[tokio::test]
5247 async fn test_unknown_publishers_do_not_splice() {
5248 tokio::time::pause();
5249
5250 let origin = Origin::random().produce();
5251 let consumer = origin.consume();
5252 let mut announced = consumer.announced();
5253
5254 let unknown_a = OriginList::try_from(vec![Origin::UNKNOWN]).unwrap();
5255 let unknown_b = OriginList::try_from(vec![Origin::UNKNOWN]).unwrap();
5256
5257 let mut source_a = origin
5258 .create_broadcast("test", announce().with_hops(unknown_a.clone()))
5259 .unwrap();
5260 settle().await;
5261 settle().await;
5262 announced.assert_next_some("test");
5263
5264 let source_b = origin
5268 .create_broadcast("test", announce().with_hops(unknown_b))
5269 .unwrap();
5270 settle().await;
5271 settle().await;
5272 announced.assert_next_none("test");
5273 let live = announced.assert_next_some("test");
5274
5275 source_a
5278 .set_route(announce().with_hops(unknown_a).with_cost(9))
5279 .unwrap();
5280 settle().await;
5281 settle().await;
5282 assert!(
5283 consumer.get_broadcast("test").unwrap().is_clone(&live),
5284 "UNKNOWN-to-UNKNOWN repricing must not replace the live front"
5285 );
5286 announced.assert_next_wait();
5287
5288 drop(source_a);
5289 drop(source_b);
5290 }
5291
5292 #[tokio::test]
5295 async fn test_known_publishers_still_splice() {
5296 tokio::time::pause();
5297
5298 let origin = Origin::random().produce();
5299 let consumer = origin.consume();
5300 let mut announced = consumer.announced();
5301
5302 let publisher = Origin::new(1).unwrap();
5303 let hops_a = OriginList::try_from(vec![publisher]).unwrap();
5304 let hops_b = OriginList::try_from(vec![publisher, Origin::new(3).unwrap()]).unwrap();
5305
5306 let source_a = origin.create_broadcast("test", announce().with_hops(hops_a)).unwrap();
5307 settle().await;
5308 settle().await;
5309 announced.assert_next_some("test");
5310
5311 let source_b = origin.create_broadcast("test", announce().with_hops(hops_b)).unwrap();
5313 settle().await;
5314 settle().await;
5315 announced.assert_next_wait();
5316
5317 drop(source_a);
5318 drop(source_b);
5319 }
5320
5321 #[tokio::test]
5327 async fn test_standby_join_splices_live_subscriber() {
5328 tokio::time::pause();
5329
5330 let origin = Origin::random().produce();
5331 let consumer = origin.consume();
5332
5333 let publisher = Origin::new(1).unwrap();
5334 let peer = Origin::new(5).unwrap();
5335 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5336 let local = OriginList::try_from(vec![publisher]).unwrap();
5337
5338 let source_remote = origin
5340 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5341 .unwrap();
5342 let mut dynamic_remote = source_remote.dynamic();
5343 settle().await;
5344 settle().await;
5345 let broadcast = consumer.request_broadcast("test").await.unwrap();
5346 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5347 let mut producer_remote = accept_track(&mut dynamic_remote, "video").await;
5348 settle().await;
5349 let mut sub = subscribing.await.unwrap();
5350 producer_remote.append_group().unwrap();
5351 assert_eq!(sub.assert_group().sequence, 0);
5352
5353 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5357 let mut dynamic_local = source_local.dynamic();
5358 settle().await;
5359 let mut producer_local = accept_track(&mut dynamic_local, "video").await;
5360 settle().await;
5361 sub.assert_no_group();
5362 assert_eq!(producer_local.subscription().unwrap().group_start, None);
5363 producer_local.create_group(group::Info { sequence: 1 }).unwrap();
5364 assert_eq!(sub.assert_group().sequence, 1);
5365 sub.assert_not_closed();
5366 }
5367
5368 #[tokio::test]
5375 async fn test_standby_with_a_partial_track_list_splits_per_track() {
5376 tokio::time::pause();
5377
5378 let origin = Origin::random().produce();
5379 let consumer = origin.consume();
5380
5381 let publisher = Origin::new(1).unwrap();
5382 let peer = Origin::new(5).unwrap();
5383 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5384 let local = OriginList::try_from(vec![publisher]).unwrap();
5385
5386 let source_remote = origin
5388 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5389 .unwrap();
5390 let mut dynamic_remote = source_remote.dynamic();
5391 settle().await;
5392 settle().await;
5393 let broadcast = consumer.request_broadcast("test").await.unwrap();
5394
5395 let subscribing_video = broadcast.track("video").unwrap().subscribe(None);
5396 let subscribing_audio = broadcast.track("audio").unwrap().subscribe(None);
5397 let mut producers_remote = accept_tracks(&mut dynamic_remote, 2).await;
5398 settle().await;
5399
5400 let mut sub_video = subscribing_video.await.unwrap();
5401 let mut sub_audio = subscribing_audio.await.unwrap();
5402 for name in ["video", "audio"] {
5403 producers_remote.get_mut(name).unwrap().append_group().unwrap();
5404 }
5405 assert_eq!(sub_video.assert_group().sequence, 0);
5406 assert_eq!(sub_audio.assert_group().sequence, 0);
5407
5408 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5410 let mut dynamic_local = source_local.dynamic();
5411 settle().await;
5412
5413 let mut producer_local = None;
5414 for _ in 0..2 {
5415 let request = tokio::time::timeout(std::time::Duration::from_secs(1), dynamic_local.requested_track())
5416 .await
5417 .expect("timed out waiting for a track request")
5418 .expect("source closed");
5419 match request.name() {
5420 "video" => producer_local = Some(request.accept(None)),
5421 "audio" => request.reject(Error::NotFound),
5422 other => panic!("unexpected track dispatched: {other}"),
5423 }
5424 }
5425 settle().await;
5426 let mut producer_local = producer_local.expect("the standby was never asked for video");
5427
5428 sub_video.assert_no_group();
5431 assert_eq!(producer_local.subscription().unwrap().group_start, None);
5432 producer_local.create_group(group::Info { sequence: 1 }).unwrap();
5433 assert_eq!(
5434 sub_video.assert_group().sequence,
5435 1,
5436 "video did not move to the standby"
5437 );
5438
5439 producers_remote.get_mut("audio").unwrap().append_group().unwrap();
5441 assert_eq!(
5442 sub_audio.assert_group().sequence,
5443 1,
5444 "audio did not stay on the incumbent"
5445 );
5446 sub_video.assert_not_closed();
5447 sub_audio.assert_not_closed();
5448 }
5449
5450 #[tokio::test]
5457 async fn test_standby_missing_track_keeps_incumbent() {
5458 tokio::time::pause();
5459
5460 let origin = Origin::random().produce();
5461 let consumer = origin.consume();
5462
5463 let publisher = Origin::new(1).unwrap();
5464 let peer = Origin::new(5).unwrap();
5465 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5466 let local = OriginList::try_from(vec![publisher]).unwrap();
5467
5468 let source_remote = origin
5470 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5471 .unwrap();
5472 let mut dynamic_remote = source_remote.dynamic();
5473 settle().await;
5474 settle().await;
5475 let broadcast = consumer.request_broadcast("test").await.unwrap();
5476 let subscribing = broadcast.track("audio").unwrap().subscribe(None);
5477 let mut producer_remote = accept_track(&mut dynamic_remote, "audio").await;
5478 settle().await;
5479 let mut sub = subscribing.await.unwrap();
5480 producer_remote.append_group().unwrap();
5481 assert_eq!(sub.assert_group().sequence, 0);
5482
5483 let source_local = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5486 let mut dynamic_local = source_local.dynamic();
5487 settle().await;
5488 let request = dynamic_local.requested_track().await.unwrap();
5489 assert_eq!(request.name(), "audio");
5490 request.reject(Error::NotFound);
5491 settle().await;
5492
5493 producer_remote.append_group().unwrap();
5495 assert_eq!(sub.assert_group().sequence, 1);
5496 sub.assert_not_closed();
5497
5498 source_remote.abort(Error::Dropped).unwrap();
5501 settle().await;
5502 settle().await;
5503 sub.assert_closed();
5504 dynamic_local.assert_no_request();
5505
5506 let retry = broadcast.track("audio").unwrap().subscribe(None);
5508 let mut producer_local = accept_track(&mut dynamic_local, "audio").await;
5509 settle().await;
5510 let mut sub = retry.await.expect("a fresh request must reach the standby");
5511 producer_local.create_group(group::Info { sequence: 2 }).unwrap();
5512 assert_eq!(sub.assert_group().sequence, 2);
5513 }
5514
5515 #[tokio::test]
5520 async fn test_unservable_track_retried_by_a_later_request() {
5521 tokio::time::pause();
5522
5523 let origin = Origin::random().produce();
5524 let consumer = origin.consume();
5525
5526 let source = origin.create_broadcast("test", announce()).unwrap();
5527 let mut dynamic = source.dynamic();
5528 settle().await;
5529 settle().await;
5530 let broadcast = consumer.request_broadcast("test").await.unwrap();
5531
5532 let subscribing = broadcast.track("audio").unwrap().subscribe(None);
5534 let request = dynamic.requested_track().await.unwrap();
5535 request.reject(Error::NotFound);
5536 settle().await;
5537 assert!(matches!(subscribing.await, Err(Error::NotFound)));
5538
5539 let retry = broadcast.track("audio").unwrap().subscribe(None);
5541 let mut producer = accept_track(&mut dynamic, "audio").await;
5542 settle().await;
5543 let mut sub = retry.await.expect("a fresh request must reach the source");
5544 producer.append_group().unwrap();
5545 assert_eq!(sub.assert_group().sequence, 0);
5546 }
5547
5548 #[tokio::test]
5554 async fn test_track_dying_without_progress_aborts() {
5555 tokio::time::pause();
5556
5557 let origin = Origin::random().produce();
5558 let consumer = origin.consume();
5559
5560 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5561 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5562 let mut dynamic = source.dynamic();
5563 settle().await;
5564 settle().await;
5565 let broadcast = consumer.request_broadcast("test").await.unwrap();
5566
5567 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5568 let producer = accept_track(&mut dynamic, "video").await;
5569 settle().await;
5570 let mut sub = subscribing.await.unwrap();
5571
5572 drop(producer);
5575 settle().await;
5576 sub.assert_closed();
5577 dynamic.assert_no_request();
5578
5579 let retry = broadcast.track("video").unwrap().subscribe(None);
5581 let mut producer = accept_track(&mut dynamic, "video").await;
5582 settle().await;
5583 let mut sub = retry.await.expect("a fresh request must reach the source");
5584 producer.append_group().unwrap();
5585 assert_eq!(sub.assert_group().sequence, 0);
5586 }
5587
5588 #[tokio::test]
5593 async fn test_delivered_copy_death_survives_unrelated_wakes() {
5594 tokio::time::pause();
5595
5596 let origin = Origin::random().produce();
5597 let consumer = origin.consume();
5598
5599 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
5600 let source = origin.create_broadcast("test", announce().with_hops(hops)).unwrap();
5601 let mut dynamic = source.dynamic();
5602 settle().await;
5603 settle().await;
5604 let broadcast = consumer.request_broadcast("test").await.unwrap();
5605
5606 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5607 let mut producer = accept_track(&mut dynamic, "video").await;
5608 settle().await;
5609 let mut sub = subscribing.await.unwrap();
5610 producer.append_group().unwrap();
5611 assert_eq!(sub.assert_group().sequence, 0);
5612
5613 drop(sub);
5616 settle().await;
5617 let resubscribing = broadcast.track("video").unwrap().subscribe(None);
5618 settle().await;
5619 let mut sub = resubscribing.await.unwrap();
5620 assert_eq!(sub.assert_group().sequence, 0, "cached group re-served");
5621
5622 drop(producer);
5625 let mut producer = accept_track(&mut dynamic, "video").await;
5626 settle().await;
5627 producer.create_group(group::Info { sequence: 1 }).unwrap();
5628 assert_eq!(sub.assert_group().sequence, 1);
5629 sub.assert_not_closed();
5630 }
5631
5632 #[tokio::test]
5638 async fn test_per_track_fallback_respects_exclusion() {
5639 tokio::time::pause();
5640
5641 let origin = Origin::random().produce();
5642 let consumer = origin.consume();
5643
5644 let publisher = Origin::new(1).unwrap();
5645 let peer = Origin::new(5).unwrap();
5646 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5647 let local = OriginList::try_from(vec![publisher]).unwrap();
5648
5649 let source_tainted = origin
5651 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5652 .unwrap();
5653 let mut dynamic_tainted = source_tainted.dynamic();
5654 settle().await;
5655 settle().await;
5656
5657 let source_clean = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5660 let mut dynamic_clean = source_clean.dynamic();
5661 settle().await;
5662
5663 let scoped = consumer.clone().excluding(peer);
5664 let broadcast = scoped.request_broadcast("test").await.unwrap();
5665 let _subscribing = broadcast.track("video").unwrap().subscribe(None);
5666 settle().await;
5667
5668 let request = dynamic_clean.requested_track().await.unwrap();
5672 request.reject(Error::NotFound);
5673 settle().await;
5674 dynamic_tainted.assert_no_request();
5675 }
5676
5677 #[tokio::test]
5681 async fn test_exclusion_survives_failover_onto_a_tainted_route() {
5682 tokio::time::pause();
5683
5684 let origin = Origin::random().produce();
5685 let consumer = origin.consume();
5686
5687 let publisher = Origin::new(1).unwrap();
5688 let peer = Origin::new(5).unwrap();
5689 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5690 let local = OriginList::try_from(vec![publisher]).unwrap();
5691
5692 let source_tainted = origin
5693 .create_broadcast("test", announce().with_hops(via_peer).with_cost(2))
5694 .unwrap();
5695 let mut dynamic_tainted = source_tainted.dynamic();
5696 settle().await;
5697 settle().await;
5698 let source_clean = origin.create_broadcast("test", announce().with_hops(local)).unwrap();
5699 let mut dynamic_clean = source_clean.dynamic();
5700 settle().await;
5701
5702 let scoped = consumer.clone().excluding(peer);
5703 let broadcast = scoped.request_broadcast("test").await.unwrap();
5704 let _subscribing = broadcast.track("video").unwrap().subscribe(None);
5705 let _clean = accept_track(&mut dynamic_clean, "video").await;
5706 settle().await;
5707
5708 source_clean.abort(Error::Dropped).unwrap();
5710 settle().await;
5711 settle().await;
5712 dynamic_tainted.assert_no_request();
5713
5714 assert!(matches!(scoped.request_broadcast("test").await, Err(Error::Unroutable)));
5717 }
5718
5719 #[tokio::test]
5724 async fn test_exclusion_holds_when_a_tainted_route_attaches_later() {
5725 tokio::time::pause();
5726
5727 let origin = Origin::random().produce();
5728 let consumer = origin.consume();
5729
5730 let publisher = Origin::new(1).unwrap();
5731 let peer = Origin::new(5).unwrap();
5732 let local = OriginList::try_from(vec![publisher]).unwrap();
5733 let via_peer = OriginList::try_from(vec![publisher, peer]).unwrap();
5734
5735 let source_clean = origin
5739 .create_broadcast("test", announce().with_hops(local).with_cost(5))
5740 .unwrap();
5741 let mut dynamic_clean = source_clean.dynamic();
5742 settle().await;
5743 settle().await;
5744 let scoped = consumer.clone().excluding(peer);
5745 let broadcast = scoped.request_broadcast("test").await.unwrap();
5746 let subscribing = broadcast.track("video").unwrap().subscribe(None);
5747 let mut producer_clean = accept_track(&mut dynamic_clean, "video").await;
5748 settle().await;
5749 let mut sub = subscribing.await.unwrap();
5750 producer_clean.append_group().unwrap();
5751 assert_eq!(sub.assert_group().sequence, 0);
5752
5753 let mut tainted = announce().with_hops(via_peer.clone()).with_cost(0);
5760 tainted.advertised = 1;
5761 let mut source_tainted = origin.create_broadcast("test", tainted).unwrap();
5762 let mut dynamic_tainted = source_tainted.dynamic();
5763 settle().await;
5764 settle().await;
5765 dynamic_tainted.assert_no_request();
5766 producer_clean.append_group().unwrap();
5767 assert_eq!(sub.assert_group().sequence, 1);
5768 sub.assert_not_closed();
5769
5770 drop(sub);
5773 drop(broadcast);
5774 drop(scoped);
5775 settle().await;
5776 let mut bumped = announce().with_hops(via_peer).with_cost(1);
5777 bumped.advertised = 1;
5778 source_tainted.set_route(bumped).unwrap();
5779 settle().await;
5780 let plain = consumer.request_broadcast("test").await.unwrap();
5781 let _plain_track = plain.track("video").unwrap().subscribe(None);
5782 settle().await;
5783 settle().await;
5784 assert!(
5785 dynamic_tainted.requested_track().now_or_never().is_some(),
5786 "the front must be free to use the route again once the peer is gone"
5787 );
5788 }
5789
5790 #[tokio::test]
5794 async fn test_excluded_path_never_reaches_the_dynamic_handler() {
5795 tokio::time::pause();
5796
5797 let origin = Origin::random().produce();
5798 let consumer = origin.consume();
5799 let mut dynamic = origin.dynamic();
5800
5801 let peer = Origin::new(5).unwrap();
5802 let tainted = OriginList::try_from(vec![Origin::new(1).unwrap(), peer]).unwrap();
5803 let _source = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
5804 settle().await;
5805 settle().await;
5806
5807 let scoped = consumer.clone().excluding(peer);
5808 assert!(matches!(scoped.request_broadcast("test").await, Err(Error::Unroutable)));
5809 assert!(
5810 dynamic.requested_broadcast().now_or_never().is_none(),
5811 "the dynamic handler was asked to route around the exclusion"
5812 );
5813
5814 let _pending = scoped.request_broadcast("other");
5816 settle().await;
5817 assert!(
5818 dynamic.requested_broadcast().now_or_never().is_some(),
5819 "a genuinely missing path must still fall back"
5820 );
5821 }
5822
5823 #[tokio::test]
5826 async fn test_dispatch_all_tainted_unroutable() {
5827 tokio::time::pause();
5828
5829 let origin = Origin::random().produce();
5830 let consumer = origin.consume();
5831
5832 let peer = Origin::new(5).unwrap();
5833 let tainted = OriginList::try_from(vec![Origin::new(1).unwrap(), peer]).unwrap();
5834 let _source = origin.create_broadcast("test", announce().with_hops(tainted)).unwrap();
5835 settle().await;
5836 settle().await;
5837
5838 let scoped = consumer.clone().excluding(peer);
5839 match scoped.request_broadcast("test").await {
5840 Err(Error::Unroutable) => {}
5841 Err(err) => panic!("expected Unroutable, got {err:?}"),
5842 Ok(_) => panic!("expected Unroutable, got a broadcast"),
5843 }
5844
5845 consumer.request_broadcast("test").await.unwrap();
5847 }
5848
5849 #[tokio::test]
5850 async fn test_duplicate_reverse() {
5851 tokio::time::pause();
5852
5853 let origin = Origin::random().produce();
5854
5855 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
5856 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
5857 settle().await;
5858 assert!(origin.consume().get_broadcast("test").is_some());
5859
5860 broadcast2.finish();
5862 settle().await;
5863 assert!(origin.consume().get_broadcast("test").is_some());
5864
5865 broadcast1.finish();
5866 settle().await;
5867 assert!(origin.consume().get_broadcast("test").is_none());
5868 }
5869
5870 #[tokio::test]
5871 async fn test_deterministic_tiebreak() {
5872 tokio::time::pause();
5873
5874 fn hops(ids: &[u64]) -> OriginList {
5875 OriginList::try_from(
5876 ids.iter()
5877 .copied()
5878 .map(|id| Origin::new(id).unwrap())
5879 .collect::<Vec<_>>(),
5880 )
5881 .unwrap()
5882 }
5883
5884 async fn winner(first: &[u64], second: &[u64]) -> OriginList {
5887 let origin = Origin::random().produce();
5888 let _a = origin
5889 .create_broadcast("test", announce().with_hops(hops(first)))
5890 .unwrap();
5891 let _b = origin
5892 .create_broadcast("test", announce().with_hops(hops(second)))
5893 .unwrap();
5894 settle().await;
5895 origin.consume().get_broadcast("test").unwrap().route().hops
5896 }
5897
5898 let forward = winner(&[5, 20], &[5, 40]).await;
5902 let reverse = winner(&[5, 40], &[5, 20]).await;
5903 assert_eq!(forward, reverse, "tie-break must not depend on publish order");
5904
5905 assert_eq!(winner(&[5, 20], &[5]).await.len(), 1);
5907 assert_eq!(winner(&[5], &[5, 20]).await.len(), 1);
5908 }
5909
5910 #[tokio::test]
5915 async fn test_many_announces() {
5916 let origin = Origin::random().produce();
5917
5918 let mut consumer = origin.consume().announced();
5919 let mut broadcasts = Vec::new();
5921 for i in 0..256 {
5922 broadcasts.push(origin.create_broadcast(format!("test{i:03}"), announce()).unwrap());
5923 settle().await;
5924 }
5925
5926 for i in 0..256 {
5927 consumer.assert_next_some(format!("test{i:03}"));
5928 }
5929 consumer.assert_next_wait();
5930 }
5931
5932 #[tokio::test]
5933 async fn test_many_announces_try() {
5934 let origin = Origin::random().produce();
5935
5936 let mut consumer = origin.consume().announced();
5937 let mut broadcasts = Vec::new();
5939 for i in 0..256 {
5940 broadcasts.push(origin.create_broadcast(format!("test{i:03}"), announce()).unwrap());
5941 settle().await;
5942 }
5943
5944 for i in 0..256 {
5945 consumer.assert_try_next_some(format!("test{i:03}"));
5946 }
5947 }
5948
5949 #[tokio::test]
5950 async fn test_with_root_basic() {
5951 let origin = Origin::random().produce();
5952
5953 let foo_producer = origin.with_root("foo").expect("should create root");
5955 assert_eq!(foo_producer.root().as_str(), "foo");
5956
5957 let mut consumer = origin.consume().announced();
5958
5959 let _broadcast = foo_producer
5961 .create_broadcast("bar/baz", announce())
5962 .expect("publish allowed");
5963 settle().await;
5964 consumer.assert_next_some("foo/bar/baz");
5966
5967 let mut foo_consumer = foo_producer.consume().announced();
5969 foo_consumer.assert_next_some("bar/baz");
5970 }
5971
5972 #[tokio::test]
5973 async fn test_with_root_nested() {
5974 let origin = Origin::random().produce();
5975
5976 let foo_producer = origin.with_root("foo").expect("should create foo root");
5978 let foo_bar_producer = foo_producer.with_root("bar").expect("should create bar root");
5979 assert_eq!(foo_bar_producer.root().as_str(), "foo/bar");
5980
5981 let mut consumer = origin.consume().announced();
5982
5983 let _broadcast = foo_bar_producer
5985 .create_broadcast("baz", announce())
5986 .expect("publish allowed");
5987 settle().await;
5988 consumer.assert_next_some("foo/bar/baz");
5990
5991 let mut foo_bar_consumer = foo_bar_producer.consume().announced();
5993 foo_bar_consumer.assert_next_some("baz");
5994 }
5995
5996 #[tokio::test]
5997 async fn test_publish_scope_allows() {
5998 let origin = Origin::random().produce();
5999
6000 let limited_producer = origin
6002 .scope(&["allowed/path1".into(), "allowed/path2".into()])
6003 .expect("should create limited producer");
6004
6005 let _broadcast = limited_producer
6007 .create_broadcast("allowed/path1", announce())
6008 .expect("publish allowed");
6009 let _keep2 = limited_producer
6010 .create_broadcast("allowed/path1/nested", announce())
6011 .expect("publish allowed");
6012 let _keep3 = limited_producer
6013 .create_broadcast("allowed/path2", announce())
6014 .expect("publish allowed");
6015 settle().await;
6016
6017 assert!(limited_producer.create_broadcast("notallowed", announce()).is_err());
6019 assert!(limited_producer.create_broadcast("allowed", announce()).is_err()); assert!(limited_producer.create_broadcast("other/path", announce()).is_err());
6021 }
6022
6023 #[tokio::test]
6024 async fn test_publish_max_parts() {
6025 let origin = Origin::random().produce();
6026
6027 let at_limit = (0..Path::MAX_PARTS)
6028 .map(|i| i.to_string())
6029 .collect::<Vec<_>>()
6030 .join("/");
6031 let _broadcast = origin
6032 .create_broadcast(at_limit.as_str(), announce())
6033 .expect("publish allowed");
6034 settle().await;
6035
6036 let too_deep = format!("{at_limit}/extra");
6037 assert!(origin.create_broadcast(too_deep.as_str(), announce()).is_err());
6038
6039 let rooted = origin.with_root("root").expect("wildcard allows any root");
6041 assert!(rooted.create_broadcast(at_limit.as_str(), announce()).is_err());
6042 }
6043
6044 #[tokio::test]
6045 async fn test_publish_scope_empty() {
6046 let origin = Origin::random().produce();
6047
6048 assert!(origin.scope(&[]).is_none());
6050 }
6051
6052 #[tokio::test]
6053 async fn test_consume_scope_filters() {
6054 let origin = Origin::random().produce();
6055
6056 let mut consumer = origin.consume().announced();
6057
6058 let _broadcast1 = origin.create_broadcast("allowed", announce()).unwrap();
6060 let _broadcast2 = origin.create_broadcast("allowed/nested", announce()).unwrap();
6061 let _broadcast3 = origin.create_broadcast("notallowed", announce()).unwrap();
6062 settle().await;
6063
6064 let mut limited_consumer = origin
6066 .consume()
6067 .scope(&["allowed".into()])
6068 .expect("should create limited consumer")
6069 .announced();
6070
6071 limited_consumer.assert_next_some("allowed");
6073 limited_consumer.assert_next_some("allowed/nested");
6074 limited_consumer.assert_next_wait(); consumer.assert_next_some("allowed");
6078 consumer.assert_next_some("allowed/nested");
6079 consumer.assert_next_some("notallowed");
6080 }
6081
6082 #[tokio::test]
6083 async fn test_consume_scope_multiple_prefixes() {
6084 let origin = Origin::random().produce();
6085
6086 let _broadcast1 = origin.create_broadcast("foo/test", announce()).unwrap();
6087 let _broadcast2 = origin.create_broadcast("bar/test", announce()).unwrap();
6088 let _broadcast3 = origin.create_broadcast("baz/test", announce()).unwrap();
6089 settle().await;
6090
6091 let mut limited_consumer = origin
6093 .consume()
6094 .scope(&["foo".into(), "bar".into()])
6095 .expect("should create limited consumer")
6096 .announced();
6097
6098 limited_consumer.assert_next_some("bar/test");
6100 limited_consumer.assert_next_some("foo/test");
6101 limited_consumer.assert_next_wait(); }
6103
6104 #[tokio::test]
6105 async fn test_with_root_and_publish_scope() {
6106 let origin = Origin::random().produce();
6107
6108 let foo_producer = origin.with_root("foo").expect("should create foo root");
6110
6111 let limited_producer = foo_producer
6113 .scope(&["bar".into(), "goop/pee".into()])
6114 .expect("should create limited producer");
6115
6116 let mut consumer = origin.consume().announced();
6117
6118 let _broadcast = limited_producer
6120 .create_broadcast("bar", announce())
6121 .expect("publish allowed");
6122 let _keep2 = limited_producer
6123 .create_broadcast("bar/nested", announce())
6124 .expect("publish allowed");
6125 let _keep3 = limited_producer
6126 .create_broadcast("goop/pee", announce())
6127 .expect("publish allowed");
6128 let _keep4 = limited_producer
6129 .create_broadcast("goop/pee/nested", announce())
6130 .expect("publish allowed");
6131 settle().await;
6132
6133 assert!(limited_producer.create_broadcast("baz", announce()).is_err());
6135 assert!(limited_producer.create_broadcast("goop", announce()).is_err()); assert!(limited_producer.create_broadcast("goop/other", announce()).is_err());
6137
6138 consumer.assert_next_some("foo/bar");
6140 consumer.assert_next_some("foo/bar/nested");
6141 consumer.assert_next_some("foo/goop/pee");
6142 consumer.assert_next_some("foo/goop/pee/nested");
6143 }
6144
6145 #[tokio::test]
6146 async fn test_with_root_and_consume_scope() {
6147 let origin = Origin::random().produce();
6148
6149 let _broadcast1 = origin.create_broadcast("foo/bar/test", announce()).unwrap();
6151 let _broadcast2 = origin.create_broadcast("foo/goop/pee/test", announce()).unwrap();
6152 let _broadcast3 = origin.create_broadcast("foo/other/test", announce()).unwrap();
6153 settle().await;
6154
6155 let foo_producer = origin.with_root("foo").expect("should create foo root");
6157
6158 let mut limited_consumer = foo_producer
6160 .consume()
6161 .scope(&["bar".into(), "goop/pee".into()])
6162 .expect("should create limited consumer")
6163 .announced();
6164
6165 limited_consumer.assert_next_some("bar/test");
6167 limited_consumer.assert_next_some("goop/pee/test");
6168 limited_consumer.assert_next_wait(); }
6170
6171 #[tokio::test]
6172 async fn test_with_root_unauthorized() {
6173 let origin = Origin::random().produce();
6174
6175 let limited_producer = origin
6177 .scope(&["allowed".into()])
6178 .expect("should create limited producer");
6179
6180 assert!(limited_producer.with_root("notallowed").is_none());
6182
6183 let allowed_root = limited_producer
6185 .with_root("allowed")
6186 .expect("should create allowed root");
6187 assert_eq!(allowed_root.root().as_str(), "allowed");
6188 }
6189
6190 #[tokio::test]
6191 async fn test_wildcard_permission() {
6192 let origin = Origin::random().produce();
6193
6194 let root_producer = origin.clone();
6196
6197 let _broadcast = root_producer
6199 .create_broadcast("any/path", announce())
6200 .expect("publish allowed");
6201 let _keep2 = root_producer
6202 .create_broadcast("other/path", announce())
6203 .expect("publish allowed");
6204 settle().await;
6205
6206 let foo_producer = root_producer.with_root("foo").expect("should create any root");
6208 assert_eq!(foo_producer.root().as_str(), "foo");
6209 }
6210
6211 #[tokio::test]
6212 async fn test_consume_broadcast_with_permissions() {
6213 let origin = Origin::random().produce();
6214
6215 let _broadcast1 = origin.create_broadcast("allowed/test", announce()).unwrap();
6216 let _broadcast2 = origin.create_broadcast("notallowed/test", announce()).unwrap();
6217 settle().await;
6218
6219 let limited_consumer = origin
6221 .consume()
6222 .scope(&["allowed".into()])
6223 .expect("should create limited consumer");
6224
6225 let result = limited_consumer.get_broadcast("allowed/test");
6227 assert!(result.is_some());
6228 assert!(
6229 result
6230 .unwrap()
6231 .is_clone(&origin.consume().get_broadcast("allowed/test").unwrap())
6232 );
6233
6234 assert!(limited_consumer.get_broadcast("notallowed/test").is_none());
6236
6237 let consumer = origin.consume();
6239 assert!(consumer.get_broadcast("allowed/test").is_some());
6240 assert!(consumer.get_broadcast("notallowed/test").is_some());
6241 }
6242
6243 #[tokio::test]
6244 async fn test_nested_paths_with_permissions() {
6245 let origin = Origin::random().produce();
6246
6247 let limited_producer = origin.scope(&["a/b/c".into()]).expect("should create limited producer");
6249
6250 let _broadcast = limited_producer
6252 .create_broadcast("a/b/c", announce())
6253 .expect("publish allowed");
6254 let _keep2 = limited_producer
6255 .create_broadcast("a/b/c/d", announce())
6256 .expect("publish allowed");
6257 let _keep3 = limited_producer
6258 .create_broadcast("a/b/c/d/e", announce())
6259 .expect("publish allowed");
6260 settle().await;
6261
6262 assert!(limited_producer.create_broadcast("a", announce()).is_err());
6264 assert!(limited_producer.create_broadcast("a/b", announce()).is_err());
6265 assert!(limited_producer.create_broadcast("a/b/other", announce()).is_err());
6266 }
6267
6268 #[tokio::test]
6269 async fn test_multiple_consumers_with_different_permissions() {
6270 let origin = Origin::random().produce();
6271
6272 let _broadcast1 = origin.create_broadcast("foo/test", announce()).unwrap();
6274 let _broadcast2 = origin.create_broadcast("bar/test", announce()).unwrap();
6275 let _broadcast3 = origin.create_broadcast("baz/test", announce()).unwrap();
6276 settle().await;
6277
6278 let mut foo_consumer = origin
6280 .consume()
6281 .scope(&["foo".into()])
6282 .expect("should create foo consumer")
6283 .announced();
6284
6285 let mut bar_consumer = origin
6286 .consume()
6287 .scope(&["bar".into()])
6288 .expect("should create bar consumer")
6289 .announced();
6290
6291 let mut foobar_consumer = origin
6292 .consume()
6293 .scope(&["foo".into(), "bar".into()])
6294 .expect("should create foobar consumer")
6295 .announced();
6296
6297 foo_consumer.assert_next_some("foo/test");
6299 foo_consumer.assert_next_wait();
6300
6301 bar_consumer.assert_next_some("bar/test");
6302 bar_consumer.assert_next_wait();
6303
6304 foobar_consumer.assert_next_some("bar/test");
6305 foobar_consumer.assert_next_some("foo/test");
6306 foobar_consumer.assert_next_wait();
6307 }
6308
6309 #[tokio::test]
6310 async fn test_select_with_empty_prefix() {
6311 let origin = Origin::random().produce();
6312
6313 let demo_producer = origin.with_root("demo").expect("should create demo root");
6315 let limited_producer = demo_producer
6316 .scope(&["worm-node".into(), "foobar".into()])
6317 .expect("should create limited producer");
6318
6319 let _broadcast1 = limited_producer
6321 .create_broadcast("worm-node/test", announce())
6322 .expect("publish allowed");
6323 let _broadcast2 = limited_producer
6324 .create_broadcast("foobar/test", announce())
6325 .expect("publish allowed");
6326 settle().await;
6327
6328 let mut consumer = limited_producer
6330 .consume()
6331 .scope(&["".into()])
6332 .expect("should create consumer with empty prefix")
6333 .announced();
6334
6335 let a1 = consumer.try_next().expect("expected first announcement");
6337 let a2 = consumer.try_next().expect("expected second announcement");
6338 consumer.assert_next_wait();
6339
6340 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
6341 paths.sort();
6342 assert_eq!(paths, ["foobar/test", "worm-node/test"]);
6343 }
6344
6345 #[tokio::test]
6346 async fn test_select_narrowing_scope() {
6347 let origin = Origin::random().produce();
6348
6349 let demo_producer = origin.with_root("demo").expect("should create demo root");
6351 let limited_producer = demo_producer
6352 .scope(&["worm-node".into(), "foobar".into()])
6353 .expect("should create limited producer");
6354
6355 let _broadcast1 = limited_producer
6357 .create_broadcast("worm-node", announce())
6358 .expect("publish allowed");
6359 let _broadcast2 = limited_producer
6360 .create_broadcast("worm-node/foo", announce())
6361 .expect("publish allowed");
6362 let _broadcast3 = limited_producer
6363 .create_broadcast("foobar/bar", announce())
6364 .expect("publish allowed");
6365 settle().await;
6366
6367 let mut worm_consumer = limited_producer
6369 .consume()
6370 .scope(&["worm-node".into()])
6371 .expect("should create worm-node consumer")
6372 .announced();
6373
6374 worm_consumer.assert_next_some("worm-node");
6376 worm_consumer.assert_next_some("worm-node/foo");
6377 worm_consumer.assert_next_wait(); let mut foo_consumer = limited_producer
6381 .consume()
6382 .scope(&["worm-node/foo".into()])
6383 .expect("should create worm-node/foo consumer")
6384 .announced();
6385
6386 foo_consumer.assert_next_some("worm-node/foo");
6387 foo_consumer.assert_next_wait(); }
6389
6390 #[tokio::test]
6391 async fn test_select_multiple_roots_with_empty_prefix() {
6392 let origin = Origin::random().produce();
6393
6394 let limited_producer = origin
6396 .scope(&["app1".into(), "app2".into(), "shared".into()])
6397 .expect("should create limited producer");
6398
6399 let _broadcast1 = limited_producer
6401 .create_broadcast("app1/data", announce())
6402 .expect("publish allowed");
6403 let _broadcast2 = limited_producer
6404 .create_broadcast("app2/config", announce())
6405 .expect("publish allowed");
6406 let _broadcast3 = limited_producer
6407 .create_broadcast("shared/resource", announce())
6408 .expect("publish allowed");
6409 settle().await;
6410
6411 let mut consumer = limited_producer
6413 .consume()
6414 .scope(&["".into()])
6415 .expect("should create consumer with empty prefix")
6416 .announced();
6417
6418 consumer.assert_next_some("app1/data");
6420 consumer.assert_next_some("app2/config");
6421 consumer.assert_next_some("shared/resource");
6422 consumer.assert_next_wait();
6423 }
6424
6425 #[tokio::test]
6426 async fn test_publish_scope_with_empty_prefix() {
6427 let origin = Origin::random().produce();
6428
6429 let limited_producer = origin
6431 .scope(&["services/api".into(), "services/web".into()])
6432 .expect("should create limited producer");
6433
6434 let same_producer = limited_producer
6436 .scope(&["".into()])
6437 .expect("should create producer with empty prefix");
6438
6439 let _broadcast = same_producer
6441 .create_broadcast("services/api", announce())
6442 .expect("publish allowed");
6443 let _keep2 = same_producer
6444 .create_broadcast("services/web", announce())
6445 .expect("publish allowed");
6446 assert!(same_producer.create_broadcast("services/db", announce()).is_err());
6447 assert!(same_producer.create_broadcast("other", announce()).is_err());
6448 }
6449
6450 #[tokio::test]
6451 async fn test_select_narrowing_to_deeper_path() {
6452 let origin = Origin::random().produce();
6453
6454 let limited_producer = origin.scope(&["org".into()]).expect("should create limited producer");
6456
6457 let _broadcast1 = limited_producer
6459 .create_broadcast("org/team1/project1", announce())
6460 .expect("publish allowed");
6461 let _broadcast2 = limited_producer
6462 .create_broadcast("org/team1/project2", announce())
6463 .expect("publish allowed");
6464 let _broadcast3 = limited_producer
6465 .create_broadcast("org/team2/project1", announce())
6466 .expect("publish allowed");
6467 settle().await;
6468
6469 let mut team2_consumer = limited_producer
6471 .consume()
6472 .scope(&["org/team2".into()])
6473 .expect("should create team2 consumer")
6474 .announced();
6475
6476 team2_consumer.assert_next_some("org/team2/project1");
6477 team2_consumer.assert_next_wait(); let mut project1_consumer = limited_producer
6481 .consume()
6482 .scope(&["org/team1/project1".into()])
6483 .expect("should create project1 consumer")
6484 .announced();
6485
6486 project1_consumer.assert_next_some("org/team1/project1");
6488 project1_consumer.assert_next_wait();
6489 }
6490
6491 #[tokio::test]
6492 async fn test_select_with_non_matching_prefix() {
6493 let origin = Origin::random().produce();
6494
6495 let limited_producer = origin
6497 .scope(&["allowed/path".into()])
6498 .expect("should create limited producer");
6499
6500 assert!(limited_producer.consume().scope(&["different/path".into()]).is_none());
6502
6503 assert!(limited_producer.scope(&["other/path".into()]).is_none());
6505 }
6506
6507 #[tokio::test]
6510 async fn test_with_root_trailing_slash_consumer() {
6511 let origin = Origin::random().produce();
6512
6513 let prefix = "some_prefix/".to_string();
6515 let mut consumer = origin.consume().with_root(prefix).unwrap().announced();
6516
6517 let _b = origin.create_broadcast("some_prefix/test", announce()).unwrap();
6518 settle().await;
6519 consumer.assert_next_some("test");
6520 }
6521
6522 #[tokio::test]
6524 async fn test_with_root_trailing_slash_producer() {
6525 let origin = Origin::random().produce();
6526
6527 let prefix = "some_prefix/".to_string();
6529 let rooted = origin.with_root(prefix).unwrap();
6530
6531 let _b = rooted.create_broadcast("test", announce()).unwrap();
6532 settle().await;
6533
6534 let mut consumer = rooted.consume().announced();
6535 consumer.assert_next_some("test");
6536 }
6537
6538 #[tokio::test]
6540 async fn test_with_root_trailing_slash_unannounce() {
6541 tokio::time::pause();
6542
6543 let origin = Origin::random().produce();
6544
6545 let prefix = "some_prefix/".to_string();
6546 let mut consumer = origin.consume().with_root(prefix).unwrap().announced();
6547
6548 let mut b = origin.create_broadcast("some_prefix/test", announce()).unwrap();
6549 settle().await;
6550 consumer.assert_next_some("test");
6551
6552 b.finish();
6554 settle().await;
6555
6556 consumer.assert_next_none("test");
6558 }
6559
6560 #[tokio::test]
6561 async fn test_select_maintains_access_with_wider_prefix() {
6562 let origin = Origin::random().produce();
6563
6564 let demo_producer = origin.with_root("demo").expect("should create demo root");
6566 let user_producer = demo_producer
6567 .scope(&["worm-node".into(), "foobar".into()])
6568 .expect("should create user producer");
6569
6570 let _broadcast1 = user_producer
6572 .create_broadcast("worm-node/data", announce())
6573 .expect("publish allowed");
6574 let _broadcast2 = user_producer
6575 .create_broadcast("foobar", announce())
6576 .expect("publish allowed");
6577 settle().await;
6578
6579 let mut consumer = user_producer
6581 .consume()
6582 .scope(&["".into()])
6583 .expect("scope with empty prefix should not fail when user has specific permissions")
6584 .announced();
6585
6586 let a1 = consumer.try_next().expect("expected first announcement");
6588 let a2 = consumer.try_next().expect("expected second announcement");
6589 consumer.assert_next_wait();
6590
6591 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
6592 paths.sort();
6593 assert_eq!(paths, ["foobar", "worm-node/data"]);
6594
6595 let mut narrow_consumer = user_producer
6597 .consume()
6598 .scope(&["worm-node".into()])
6599 .expect("should be able to narrow scope to worm-node")
6600 .announced();
6601
6602 narrow_consumer.assert_next_some("worm-node/data");
6603 narrow_consumer.assert_next_wait(); }
6605
6606 #[tokio::test]
6607 async fn test_duplicate_prefixes_deduped() {
6608 let origin = Origin::random().produce();
6609
6610 let producer = origin
6612 .scope(&["demo".into(), "demo".into()])
6613 .expect("should create producer");
6614
6615 let _broadcast = producer
6616 .create_broadcast("demo/stream", announce())
6617 .expect("publish allowed");
6618 settle().await;
6619
6620 let mut consumer = producer.consume().announced();
6621 consumer.assert_next_some("demo/stream");
6622 consumer.assert_next_wait();
6623 }
6624
6625 #[tokio::test]
6626 async fn test_overlapping_prefixes_deduped() {
6627 let origin = Origin::random().produce();
6628
6629 let producer = origin
6631 .scope(&["demo".into(), "demo/foo".into()])
6632 .expect("should create producer");
6633
6634 let _broadcast = producer
6636 .create_broadcast("demo/bar/stream", announce())
6637 .expect("publish allowed");
6638 settle().await;
6639
6640 let mut consumer = producer.consume().announced();
6641 consumer.assert_next_some("demo/bar/stream");
6642 consumer.assert_next_wait();
6643 }
6644
6645 #[tokio::test]
6646 async fn test_overlapping_prefixes_no_duplicate_announcements() {
6647 let origin = Origin::random().produce();
6648
6649 let producer = origin
6651 .scope(&["demo".into(), "demo/foo".into()])
6652 .expect("should create producer");
6653
6654 let _broadcast = producer
6655 .create_broadcast("demo/foo/stream", announce())
6656 .expect("publish allowed");
6657 settle().await;
6658
6659 let mut consumer = producer.consume().announced();
6660 consumer.assert_next_some("demo/foo/stream");
6662 consumer.assert_next_wait();
6663 }
6664
6665 #[tokio::test]
6666 async fn test_allowed_returns_deduped_prefixes() {
6667 let origin = Origin::random().produce();
6668
6669 let producer = origin
6670 .scope(&["demo".into(), "demo/foo".into(), "anon".into()])
6671 .expect("should create producer");
6672
6673 let allowed: Vec<_> = producer.allowed().collect();
6674 assert_eq!(allowed.len(), 2, "demo/foo should be subsumed by demo");
6675 }
6676
6677 #[tokio::test]
6678 async fn test_announced_broadcast_already_announced() {
6679 let origin = Origin::random().produce();
6680
6681 let _broadcast = origin.create_broadcast("test", announce()).unwrap();
6682 settle().await;
6683
6684 let consumer = origin.consume();
6685 let result = consumer.announced_broadcast("test").await.expect("should find it");
6686 assert!(result.is_clone(&consumer.get_broadcast("test").unwrap()));
6687 }
6688
6689 #[tokio::test]
6690 async fn test_announced_broadcast_delayed() {
6691 tokio::time::pause();
6692
6693 let origin = Origin::random().produce();
6694
6695 let consumer = origin.consume();
6696
6697 let wait = tokio::spawn({
6699 let consumer = consumer.clone();
6700 async move { consumer.announced_broadcast("test").await }
6701 });
6702
6703 tokio::task::yield_now().await;
6705
6706 let _broadcast = origin.create_broadcast("test", announce()).unwrap();
6707 settle().await;
6708
6709 let result = wait.await.unwrap().expect("should find it");
6710 assert!(result.is_clone(&consumer.get_broadcast("test").unwrap()));
6711 }
6712
6713 #[tokio::test]
6714 async fn test_announced_broadcast_ignores_unrelated_paths() {
6715 tokio::time::pause();
6716
6717 let origin = Origin::random().produce();
6718
6719 let consumer = origin.consume();
6720
6721 let wait = tokio::spawn({
6722 let consumer = consumer.clone();
6723 async move { consumer.announced_broadcast("target").await }
6724 });
6725
6726 tokio::task::yield_now().await;
6727
6728 let _other = origin.create_broadcast("other", announce()).unwrap();
6730 settle().await;
6731 tokio::task::yield_now().await;
6732 assert!(!wait.is_finished(), "must not resolve on unrelated path");
6733
6734 let _target = origin.create_broadcast("target", announce()).unwrap();
6735 settle().await;
6736 let result = wait.await.unwrap().expect("should find target");
6737 assert!(result.is_clone(&consumer.get_broadcast("target").unwrap()));
6738 }
6739
6740 #[tokio::test]
6741 async fn test_announced_broadcast_skips_nested_paths() {
6742 tokio::time::pause();
6743
6744 let origin = Origin::random().produce();
6745
6746 let consumer = origin.consume();
6747
6748 let wait = tokio::spawn({
6749 let consumer = consumer.clone();
6750 async move { consumer.announced_broadcast("foo").await }
6751 });
6752
6753 tokio::task::yield_now().await;
6754
6755 let _nested = origin.create_broadcast("foo/bar", announce()).unwrap();
6757 settle().await;
6758 tokio::task::yield_now().await;
6759 assert!(!wait.is_finished(), "must not resolve on a nested path");
6760
6761 let _exact = origin.create_broadcast("foo", announce()).unwrap();
6762 settle().await;
6763 let result = wait.await.unwrap().expect("should find foo exactly");
6764 assert!(result.is_clone(&consumer.get_broadcast("foo").unwrap()));
6765 }
6766
6767 #[tokio::test]
6768 async fn test_announced_broadcast_disallowed() {
6769 let origin = Origin::random().produce();
6770 let limited = origin
6771 .consume()
6772 .scope(&["allowed".into()])
6773 .expect("should create limited");
6774
6775 assert!(limited.announced_broadcast("notallowed").await.is_none());
6777 }
6778
6779 #[tokio::test]
6780 async fn test_announced_broadcast_scope_too_narrow() {
6781 let origin = Origin::random().produce();
6784 let limited = origin
6785 .consume()
6786 .scope(&["foo/specific".into()])
6787 .expect("should create limited");
6788
6789 let result = limited
6791 .announced_broadcast("foo")
6792 .now_or_never()
6793 .expect("must not block");
6794 assert!(result.is_none());
6795 }
6796
6797 #[tokio::test]
6801 async fn test_coalesce_announce_then_unannounce() {
6802 tokio::time::pause();
6804
6805 let origin = Origin::random().produce();
6806 let mut announced = origin.consume().announced();
6807
6808 let mut broadcast = origin.create_broadcast("test", announce()).unwrap();
6809 settle().await;
6810 broadcast.finish();
6811
6812 settle().await;
6813
6814 announced.assert_next_wait();
6815 }
6816
6817 #[tokio::test]
6818 async fn test_coalesce_announce_unannounce_announce() {
6819 tokio::time::pause();
6822
6823 let origin = Origin::random().produce();
6824 let mut announced = origin.consume().announced();
6825
6826 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
6827 settle().await;
6828 broadcast1.finish();
6829 settle().await;
6830 let _broadcast2 = origin.create_broadcast("test", announce()).unwrap();
6831 settle().await;
6832
6833 announced.assert_next_some("test");
6834 announced.assert_next_wait();
6835 }
6836
6837 #[tokio::test]
6838 async fn test_coalesce_unannounce_announce_preserved() {
6839 tokio::time::pause();
6842
6843 let origin = Origin::random().produce();
6844 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
6845 settle().await;
6846
6847 let mut announced = origin.consume().announced();
6848 announced.assert_next_some("test");
6849
6850 broadcast1.finish();
6852 settle().await;
6853
6854 let _broadcast2 = origin.create_broadcast("test", announce()).unwrap();
6855 settle().await;
6856
6857 announced.assert_next_none("test");
6859 announced.assert_next_some("test");
6860 announced.assert_next_wait();
6861 }
6862
6863 #[tokio::test]
6864 async fn test_coalesce_unannounce_announce_unannounce() {
6865 tokio::time::pause();
6868
6869 let origin = Origin::random().produce();
6870 let mut broadcast1 = origin.create_broadcast("test", announce()).unwrap();
6871 settle().await;
6872
6873 let mut announced = origin.consume().announced();
6874 announced.assert_next_some("test");
6875
6876 broadcast1.finish();
6877 settle().await;
6878
6879 let mut broadcast2 = origin.create_broadcast("test", announce()).unwrap();
6880 settle().await;
6881 broadcast2.finish();
6882 settle().await;
6883
6884 announced.assert_next_none("test");
6885 announced.assert_next_wait();
6886 }
6887
6888 #[tokio::test]
6889 async fn test_coalesce_churn_bounded() {
6890 tokio::time::pause();
6895
6896 let origin = Origin::random().produce();
6897 let mut announced = origin.consume().announced();
6898
6899 for _ in 0..1000 {
6900 let mut broadcast = origin.create_broadcast("test", announce()).unwrap();
6901 settle().await;
6902 broadcast.finish();
6903 }
6904 settle().await;
6905
6906 let mut collected = Vec::new();
6907 while let Some(update) = announced.try_next() {
6908 collected.push(update);
6909 }
6910 assert!(
6911 collected.len() <= 1,
6912 "expected at most one pending update, got {}",
6913 collected.len()
6914 );
6915 assert!(
6916 collected.iter().all(|a| a.path == Path::new("test")),
6917 "unexpected path in pending updates",
6918 );
6919 }
6920
6921 #[tokio::test]
6925 async fn test_consumer_clone_is_side_effect_free() {
6926 let origin = Origin::random().produce();
6927
6928 let _broadcast1 = origin.create_broadcast("test1", announce()).unwrap();
6929 let _broadcast2 = origin.create_broadcast("test2", announce()).unwrap();
6930 settle().await;
6931
6932 let consumer = origin.consume();
6933 let mut announced = consumer.announced();
6934
6935 for _ in 0..16 {
6938 let cloned = consumer.clone();
6939 assert!(cloned.get_broadcast("test1").is_some());
6940 assert!(cloned.get_broadcast("test2").is_some());
6941 }
6942
6943 let a1 = announced.try_next().expect("first announcement");
6946 let a2 = announced.try_next().expect("second announcement");
6947 announced.assert_next_wait();
6948
6949 let mut paths: Vec<_> = [&a1, &a2].iter().map(|a| a.path.to_string()).collect();
6950 paths.sort();
6951 assert_eq!(paths, ["test1", "test2"]);
6952
6953 let mut fresh = consumer.announced();
6955 let b1 = fresh.try_next().expect("backlog: first");
6956 let b2 = fresh.try_next().expect("backlog: second");
6957 fresh.assert_next_wait();
6958
6959 let mut paths: Vec<_> = [&b1, &b2].iter().map(|a| a.path.to_string()).collect();
6960 paths.sort();
6961 assert_eq!(paths, ["test1", "test2"]);
6962 }
6963
6964 #[tokio::test]
6966 async fn dynamic_request_unroutable_without_handler() {
6967 let origin = Origin::random().produce();
6968 let consumer = origin.consume();
6969 assert!(matches!(
6970 consumer.request_broadcast("missing").await,
6971 Err(Error::Unroutable)
6972 ));
6973 }
6974
6975 #[tokio::test(start_paused = true)]
6978 async fn dynamic_request_served_not_announced() {
6979 let origin = Origin::random().produce();
6980 let mut dynamic = origin.dynamic();
6981 let consumer = origin.consume();
6982
6983 let mut announced = origin.consume().announced();
6985 announced.assert_next_wait();
6986
6987 let served = broadcast::Info::new().produce();
6988 let request_fut = consumer.request_broadcast("fallback");
6991
6992 let mut served_dynamic = served.dynamic();
6994
6995 let request = dynamic.requested_broadcast().await.unwrap();
6996 assert_eq!(request.path(), &Path::new("fallback"));
6997 request.accept(&served);
6998
6999 let broadcast = request_fut.await.unwrap();
7000 assert!(broadcast.is_clone(&served.consume()));
7001
7002 let track_fut = broadcast.track("video").unwrap().subscribe(None);
7004 let mut producer = served_dynamic.requested_track().await.unwrap().accept(None);
7005 let mut track = track_fut.await.unwrap();
7006 producer.append_group().unwrap();
7007 track.assert_group();
7008
7009 announced.assert_next_wait();
7011 }
7012
7013 #[tokio::test(start_paused = true)]
7015 async fn dynamic_request_coalesces() {
7016 let origin = Origin::random().produce();
7017 let mut dynamic = origin.dynamic();
7018 let consumer = origin.consume();
7019
7020 let f1 = consumer.request_broadcast("dup");
7022 let f2 = consumer.request_broadcast("dup");
7023
7024 let request = dynamic.requested_broadcast().await.unwrap();
7026 assert_eq!(request.path(), &Path::new("dup"));
7027 assert!(
7028 dynamic.requested_broadcast().now_or_never().is_none(),
7029 "a coalesced request must not be served twice"
7030 );
7031
7032 let served = broadcast::Info::new().produce();
7034 request.accept(&served);
7035 assert!(f1.await.unwrap().is_clone(&served.consume()));
7036 assert!(f2.await.unwrap().is_clone(&served.consume()));
7037 }
7038
7039 #[tokio::test(start_paused = true)]
7042 async fn dynamic_request_dedups_served() {
7043 let origin = Origin::random().produce();
7044 let mut dynamic = origin.dynamic();
7045 let consumer = origin.consume();
7046
7047 let request_fut = consumer.request_broadcast("fallback");
7048 let request = dynamic.requested_broadcast().await.unwrap();
7049 let served = broadcast::Info::new().produce();
7050 request.accept(&served);
7051 let first = request_fut.await.unwrap();
7052 assert!(first.is_clone(&served.consume()));
7053
7054 let second = consumer.request_broadcast("fallback").await.unwrap();
7056 assert!(second.is_clone(&served.consume()));
7057
7058 assert!(
7060 dynamic.requested_broadcast().now_or_never().is_none(),
7061 "a still-live served broadcast must not be re-requested from the handler"
7062 );
7063 }
7064
7065 #[tokio::test(start_paused = true)]
7067 async fn dynamic_request_reserves_after_close() {
7068 let origin = Origin::random().produce();
7069 let mut dynamic = origin.dynamic();
7070 let consumer = origin.consume();
7071
7072 let request_fut = consumer.request_broadcast("fallback");
7073 let request = dynamic.requested_broadcast().await.unwrap();
7074 let served = broadcast::Info::new().produce();
7075 request.accept(&served);
7076 request_fut.await.unwrap();
7077
7078 drop(served);
7080
7081 let request_fut = consumer.request_broadcast("fallback");
7083 let request = dynamic.requested_broadcast().await.unwrap();
7084 assert_eq!(request.path(), &Path::new("fallback"));
7085 let served = broadcast::Info::new().produce();
7086 request.accept(&served);
7087 assert!(request_fut.await.unwrap().is_clone(&served.consume()));
7088 }
7089
7090 #[tokio::test(start_paused = true)]
7093 async fn dynamic_request_served_cache_bounded() {
7094 let origin = Origin::random().produce();
7095 let mut dynamic = origin.dynamic();
7096 let consumer = origin.consume();
7097
7098 for i in 0..100 {
7099 let path = format!("one-shot/{i}");
7100 let request_fut = consumer.request_broadcast(&path);
7101 let request = dynamic.requested_broadcast().await.unwrap();
7102 let served = broadcast::Info::new().produce();
7103 request.accept(&served);
7104 request_fut.await.unwrap();
7105 drop(served);
7107 }
7108
7109 assert!(
7112 origin.dynamic.read().served.len() <= 4,
7113 "stale served entries must be reclaimed, not accumulate per distinct path: {}",
7114 origin.dynamic.read().served.len()
7115 );
7116 }
7117
7118 #[tokio::test(start_paused = true)]
7121 async fn dynamic_request_coalesces_after_handoff() {
7122 let origin = Origin::random().produce();
7123 let mut dynamic = origin.dynamic();
7124 let consumer = origin.consume();
7125
7126 let f1 = consumer.request_broadcast("fallback");
7127 let request = dynamic.requested_broadcast().await.unwrap();
7129
7130 let f2 = consumer.request_broadcast("fallback");
7132 assert!(
7133 dynamic.requested_broadcast().now_or_never().is_none(),
7134 "a repeat request during hand-off must coalesce, not re-queue"
7135 );
7136
7137 let served = broadcast::Info::new().produce();
7139 request.accept(&served);
7140 assert!(f1.await.unwrap().is_clone(&served.consume()));
7141 assert!(f2.await.unwrap().is_clone(&served.consume()));
7142 }
7143
7144 #[tokio::test(start_paused = true)]
7146 async fn dynamic_request_dropped_after_handoff() {
7147 let origin = Origin::random().produce();
7148 let mut dynamic = origin.dynamic();
7149 let consumer = origin.consume();
7150
7151 let f1 = consumer.request_broadcast("fallback");
7152 let request = dynamic.requested_broadcast().await.unwrap();
7153 let f2 = consumer.request_broadcast("fallback");
7154
7155 drop(request);
7157 assert!(matches!(f1.await, Err(Error::Unroutable)));
7158 assert!(matches!(f2.await, Err(Error::Unroutable)));
7159 }
7160
7161 #[tokio::test(start_paused = true)]
7163 async fn dynamic_request_rejected() {
7164 let origin = Origin::random().produce();
7165 let mut dynamic = origin.dynamic();
7166 let consumer = origin.consume();
7167
7168 let request_fut = consumer.request_broadcast("fallback");
7169
7170 let request = dynamic.requested_broadcast().await.unwrap();
7171 request.reject(Error::Cancel);
7172
7173 assert!(matches!(request_fut.await, Err(Error::Cancel)));
7174 }
7175
7176 #[tokio::test(start_paused = true)]
7180 async fn dynamic_request_rerequest_after_reject() {
7181 let origin = Origin::random().produce();
7182 let mut dynamic = origin.dynamic();
7183 let consumer = origin.consume();
7184
7185 let f1 = consumer.request_broadcast("fallback");
7186 dynamic.requested_broadcast().await.unwrap().reject(Error::Unroutable);
7187 assert!(matches!(f1.await, Err(Error::Unroutable)));
7188
7189 let served = broadcast::Info::new().produce();
7190 let f2 = consumer.request_broadcast("fallback");
7192 let request = dynamic.requested_broadcast().await.unwrap();
7193 assert_eq!(request.path(), &Path::new("fallback"));
7194 request.accept(&served);
7195 assert!(f2.await.unwrap().is_clone(&served.consume()));
7196 }
7197
7198 #[tokio::test(start_paused = true)]
7201 async fn dynamic_request_handler_dropped() {
7202 let origin = Origin::random().produce();
7203 let dynamic = origin.dynamic();
7204 let consumer = origin.consume();
7205
7206 let request_fut = consumer.request_broadcast("fallback");
7207 drop(dynamic);
7208 assert!(matches!(request_fut.await, Err(Error::Unroutable)));
7209
7210 assert!(matches!(
7212 consumer.request_broadcast("again").await,
7213 Err(Error::Unroutable)
7214 ));
7215 }
7216
7217 #[tokio::test(start_paused = true)]
7221 async fn dynamic_request_accept_after_handler_dropped() {
7222 let origin = Origin::random().produce();
7223 let mut dynamic = origin.dynamic();
7224 let consumer = origin.consume();
7225
7226 let request_fut = consumer.request_broadcast("fallback");
7227
7228 let request = dynamic.requested_broadcast().await.unwrap();
7230 drop(dynamic);
7231
7232 let served = broadcast::Info::new().produce();
7233 request.accept(&served);
7235 assert!(request_fut.await.unwrap().is_clone(&served.consume()));
7236 }
7237
7238 #[tokio::test(start_paused = true)]
7240 async fn dynamic_request_prefers_announced() {
7241 let origin = Origin::random().produce();
7242 let mut dynamic = origin.dynamic();
7243 let consumer = origin.consume();
7244
7245 let _broadcast = origin.create_broadcast("live", announce()).unwrap();
7246 settle().await;
7247
7248 let got = consumer.request_broadcast("live").await.unwrap();
7249 assert!(
7250 got.is_clone(&consumer.get_broadcast("live").unwrap()),
7251 "should return the published broadcast"
7252 );
7253 assert!(
7254 dynamic.requested_broadcast().now_or_never().is_none(),
7255 "a published path must not queue a fallback request"
7256 );
7257 }
7258
7259 #[tokio::test(start_paused = true)]
7261 async fn dynamic_clone_keeps_alive() {
7262 let origin = Origin::random().produce();
7263 let dynamic = origin.dynamic();
7264 let consumer = origin.consume();
7265
7266 drop(dynamic.clone());
7267
7268 let request_fut = consumer.request_broadcast("fallback");
7271 assert!(
7272 request_fut.now_or_never().is_none(),
7273 "request should stay pending until served"
7274 );
7275 }
7276
7277 fn wedge_watchdog<F>(name: &str, secs: u64, scenario: F)
7283 where
7284 F: std::future::Future<Output = ()> + Send + 'static,
7285 {
7286 let (done_tx, done_rx) = std::sync::mpsc::channel::<()>();
7287 let handle = std::thread::spawn(move || {
7288 let rt = ::tokio::runtime::Builder::new_current_thread()
7289 .enable_time()
7290 .build()
7291 .unwrap();
7292 rt.block_on(scenario);
7293 let _ = done_tx.send(());
7294 });
7295 match done_rx.recv_timeout(std::time::Duration::from_secs(secs)) {
7296 Ok(()) => {
7297 let _ = handle.join();
7298 }
7299 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
7300 let err = handle.join().unwrap_err();
7302 std::panic::resume_unwind(err);
7303 }
7304 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
7305 panic!("{name}: scenario wedged; a task is spinning inside a single poll")
7306 }
7307 }
7308 }
7309
7310 #[test]
7323 fn test_active_corpse_does_not_livelock_takeover() {
7324 wedge_watchdog("active-corpse", 20, async {
7325 let origin = Origin::random().produce();
7326 let consumer = origin.consume();
7327 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
7328
7329 let source_a = origin
7332 .create_broadcast("test", announce().with_hops(hops.clone()))
7333 .unwrap();
7334 let mut dynamic_a = source_a.dynamic();
7335 settle().await;
7336 settle().await;
7337
7338 let broadcast = consumer.request_broadcast("test").await.unwrap();
7339 let subscribing = broadcast.track("video").unwrap().subscribe(None);
7340 let mut producer_a = accept_track(&mut dynamic_a, "video").await;
7341 settle().await;
7342 let mut sub = subscribing.await.unwrap();
7343 producer_a.append_group().unwrap();
7344 sub.assert_group();
7345
7346 let source_b = origin
7350 .create_broadcast("test", announce().with_hops(hops.clone()))
7351 .unwrap();
7352 let dynamic_b = source_b.dynamic();
7353 settle().await;
7354
7355 drop(dynamic_b);
7363 source_b.abort(Error::Dropped).unwrap();
7364 settle().await;
7365
7366 producer_a.append_group().unwrap();
7369 sub.assert_group();
7370 sub.assert_not_closed();
7371 });
7372 }
7373
7374 #[test]
7382 fn test_route_churn_never_wedges() {
7383 for seed in 1..=8u64 {
7384 wedge_watchdog(&format!("churn seed {seed}"), 30, churn_scenario(seed));
7385 }
7386 }
7387
7388 async fn churn_scenario(seed: u64) {
7389 let mut rng = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
7391 let mut next = move || {
7392 rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
7393 rng >> 33
7394 };
7395
7396 let origin = Origin::random().produce();
7397 let consumer = origin.consume();
7398 let names: Vec<Arc<str>> = (0..8).map(|i| Arc::from(format!("t{i}"))).collect();
7399
7400 let mut subs: Vec<track::Subscriber> = Vec::new();
7401 let mut pending_subs: Vec<kio::Pending<track::Subscribing>> = Vec::new();
7402
7403 struct Source {
7404 producer: Option<broadcast::Producer>,
7405 server: ::tokio::task::JoinHandle<()>,
7406 }
7407 let mut sources: Vec<Source> = Vec::new();
7408
7409 for step in 0..400u64 {
7410 match next() % 10 {
7411 0 | 1 => {
7414 if sources.len() >= 3 {
7415 continue;
7416 }
7417 let hops = OriginList::try_from(vec![Origin::new(1).unwrap()]).unwrap();
7418 let route = announce().with_hops(hops).with_cost(next() % 4);
7419 let Ok(source) = origin.create_broadcast("test", route) else {
7420 continue;
7421 };
7422 let mut dynamic = source.dynamic();
7423 let behavior = next();
7424 let server = ::tokio::spawn(async move {
7425 let mut round = 0u64;
7426 let mut kept: Vec<track::Producer> = Vec::new();
7427 while let Ok(request) = dynamic.requested_track().await {
7428 round += 1;
7429 match (behavior >> (round % 16)) % 4 {
7430 0 => drop(request), 1 => {
7432 let mut producer = request.accept(None);
7433 let _ = producer.create_group(group::Info { sequence: round });
7434 let _ = producer.abort(Error::Dropped);
7435 }
7436 2 => {
7437 let mut producer = request.accept(None);
7438 let _ = producer.create_group(group::Info { sequence: round });
7439 kept.push(producer);
7440 }
7441 _ => {
7442 let mut producer = request.accept(None);
7443 let _ = producer.finish();
7444 }
7445 }
7446 }
7447 });
7448 sources.push(Source {
7449 producer: Some(source),
7450 server,
7451 });
7452 }
7453 2 | 3 => {
7455 if sources.is_empty() {
7456 continue;
7457 }
7458 let i = (next() as usize) % sources.len();
7459 let mut source = sources.swap_remove(i);
7460 if next() % 2 == 0
7461 && let Some(producer) = source.producer.take()
7462 {
7463 let _ = producer.abort(Error::Dropped);
7464 }
7465 source.server.abort();
7466 }
7467 4..=6 => {
7469 if subs.len() + pending_subs.len() >= 24 {
7470 continue;
7471 }
7472 let Some(broadcast) = consumer.get_broadcast("test") else {
7473 continue;
7474 };
7475 let name = &names[(next() as usize) % names.len()];
7476 if let Ok(track) = broadcast.track(name.as_ref()) {
7477 pending_subs.push(track.subscribe(None));
7478 }
7479 }
7480 7 => {
7482 if subs.is_empty() {
7483 continue;
7484 }
7485 let i = (next() as usize) % subs.len();
7486 subs.swap_remove(i);
7487 }
7488 _ => {
7490 for sub in pending_subs.drain(..) {
7491 match ::tokio::time::timeout(std::time::Duration::from_millis(5), sub).await {
7492 Ok(Ok(sub)) => subs.push(sub),
7493 Ok(Err(_)) => {}
7494 Err(_) => {}
7496 }
7497 }
7498 for sub in subs.iter_mut() {
7499 while let Some(Ok(Some(_))) = sub.recv_group().now_or_never() {}
7500 }
7501 }
7502 }
7503 if step % 16 == 0 {
7504 settle().await;
7505 }
7506 for _ in 0..(next() % 3) {
7508 ::tokio::task::yield_now().await;
7509 }
7510 }
7511
7512 for source in sources {
7513 source.server.abort();
7514 }
7515 settle().await;
7516 }
7517}