1#[cfg(all(test, feature = "std", feature = "discovery"))]
12#[path = "tests/radio_trace_decoder.rs"]
13mod radio_trace_decoder;
14
15#[cfg(all(test, feature = "std", feature = "discovery"))]
16#[path = "tests/accepted_route_ownership.rs"]
17mod accepted_route_ownership;
18
19#[cfg(all(test, feature = "std"))]
20#[path = "tests/independent_side_tx_router.rs"]
21mod independent_side_tx;
22#[cfg(all(test, feature = "std", feature = "discovery"))]
23#[path = "tests/ingress_target_filter.rs"]
24mod ingress_target_filter_tests;
25#[cfg(all(test, feature = "std", feature = "discovery", feature = "compression"))]
26#[path = "tests/payload_codec_negotiation.rs"]
27mod payload_codec_negotiation;
28#[cfg(all(test, feature = "std", feature = "discovery"))]
29#[path = "tests/reliable_tx_window_backpressure.rs"]
30mod reliable_tx_window_backpressure_tests;
31#[cfg(all(test, feature = "std", feature = "discovery"))]
32#[path = "tests/shared_chunk_budget_router.rs"]
33mod shared_chunk_budget_tests;
34#[cfg(test)]
36#[allow(clippy::duplicate_mod)]
37#[path = "tests/side_template_index.rs"]
38mod side_template_index_tests;
39#[cfg(feature = "discovery")]
40use crate::config::runtime_reliable_max_end_to_end_pending;
41#[cfg(all(test, feature = "std", feature = "discovery"))]
42#[path = "tests/replay_ownership_router.rs"]
43mod replay_ownership_tests;
44use crate::side_transport::{
45 SIDE_TRANSPORT_FLAG_PACKET_NONCE, SIDE_TRANSPORT_KIND_CHUNK, SIDE_TRANSPORT_KIND_COMPACT,
46 SIDE_TRANSPORT_KIND_COMPACT_DELTA, SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP,
47 SIDE_TRANSPORT_KIND_FULL, SideChunkAssembly, SideCompactTimestampMode, SideHeaderTemplate,
48 SideTransportFrames, extract_side_header_template, extract_side_header_template_view,
49 parse_side_transport_wrapper, read_uleb128_local, reconstruct_side_compact_frame,
50 wrap_side_transport_frame, write_uleb128_local,
51};
52#[cfg(all(test, feature = "discovery"))]
53#[path = "tests/ack_return_identity.rs"]
54mod ack_return_identity;
55#[cfg(all(test, feature = "discovery"))]
56#[path = "tests/ack_split_horizon.rs"]
57mod ack_split_horizon;
58
59#[cfg(all(test, feature = "discovery"))]
60#[path = "tests/discovery_liveness.rs"]
61mod discovery_liveness;
62
63#[cfg(all(test, feature = "discovery"))]
64mod compact_summary_tests {
65 use super::*;
66
67 #[test]
68 fn ordered_pending_window_is_bounded_and_exhaustion_cancels_the_tail() {
69 crate::tests::ensure_common_test_schema();
70 let ep = DataEndpoint::named("RADIO");
71 let ty = crate::config::register_data_type_with_description(
72 "ORDERED_WINDOW_REGRESSION",
73 "bounded ordered window",
74 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
75 &[ep],
76 crate::ReliableMode::Ordered,
77 200,
78 )
79 .unwrap();
80 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
81 let clock = now.clone();
82 let router = Router::new_with_clock(
83 RouterConfig::new([]),
84 Box::new(move || clock.load(Ordering::Relaxed)),
85 );
86 let capacity = runtime_reliable_max_end_to_end_pending().max(1);
87 for index in 0..capacity {
88 let item = RouterItem::Packet(
89 Packet::new(
90 ty,
91 &[ep],
92 "SOURCE",
93 index as u64 + 1,
94 Arc::<[u8]>::from([index as u8]),
95 )
96 .unwrap(),
97 );
98 assert_eq!(
99 router.register_end_to_end_reliable_tx(&item, None).unwrap(),
100 index == 0
101 );
102 }
103 let rejected = RouterItem::Packet(
104 Packet::new(
105 ty,
106 &[ep],
107 "SOURCE",
108 capacity as u64 + 1,
109 Arc::<[u8]>::from([0]),
110 )
111 .unwrap(),
112 );
113 assert!(
114 router
115 .register_end_to_end_reliable_tx(&rejected, None)
116 .is_err()
117 );
118 {
119 let mut state = router.state.lock();
120 assert_eq!(state.end_to_end_reliable_tx.len(), capacity);
121 let first = state.end_to_end_reliable_tx_order[0];
122 state
123 .end_to_end_reliable_tx
124 .get_mut(&first)
125 .unwrap()
126 .retries = runtime_reliable_max_retries();
127 }
128 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
129 assert!(router.process_end_to_end_reliable_timeouts().is_err());
130 assert!(router.state.lock().end_to_end_reliable_tx.is_empty());
131 }
132 #[test]
133 fn compact_reachability_survives_incomplete_detailed_topology() {
134 crate::tests::ensure_common_test_schema();
135 let router = Router::new(RouterConfig::new([]).with_sender("GB"));
136 let endpoint = DataEndpoint::try_named("GROUND_STATION")
137 .or_else(|| DataEndpoint::try_named("RADIO"))
138 .unwrap();
139 router.compact_summary_fixture(endpoint);
140 let state = router.state.lock();
141 let now = router.clock.now_ms();
142 let advertised =
143 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(1));
144 assert!(
145 advertised.contains(&endpoint),
146 "compact subscriber route was lost at the bridge"
147 );
148 let reflected =
149 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(0));
150 assert!(
151 !reflected.contains(&endpoint),
152 "split horizon must still exclude ingress"
153 );
154 assert_eq!(
155 router.advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(1)),
156 vec!["clock-behind-bridge".to_string()]
157 );
158 assert!(
159 router
160 .advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(0))
161 .is_empty()
162 );
163 assert!(
164 !router
165 .advertised_discovery_endpoints_for_link_locked(
166 &state,
167 now + DISCOVERY_ROUTE_TTL_MS + 1,
168 true,
169 Some(1)
170 )
171 .contains(&endpoint)
172 );
173 assert!(
174 router
175 .advertised_discovery_timesync_sources_for_link_locked(
176 &state,
177 now + DISCOVERY_ROUTE_TTL_MS + 1,
178 Some(1)
179 )
180 .is_empty()
181 );
182 }
183}
184
185use crate::config::RuntimeMemoryConfig;
186use crate::diagnostics::{
187 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
188 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
189 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
190};
191#[cfg(feature = "discovery")]
192use crate::discovery::{
193 self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
194 DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState,
195 TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS, TopologyAnnouncerRoute, TopologyBoardNode,
196 TopologySideRoute, TopologySnapshot,
197};
198use crate::packet::{hash_bytes_u64, sender_address_u32};
199use crate::queue::{BoundedDeque, ByteCost};
200#[cfg(all(not(feature = "std"), target_os = "none"))]
201use crate::seds_error_msg;
202#[cfg(feature = "timesync")]
203use crate::timesync::{
204 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
205 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
206 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
207 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
208 decode_timesync_request, decode_timesync_response,
209};
210use crate::{
211 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
212 config::{
213 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
214 runtime_reliable_max_pending, runtime_reliable_max_retries,
215 runtime_reliable_max_return_routes, runtime_reliable_retransmit_ms,
216 },
217 get_needed_message_size, impl_letype_num, is_reliable_type,
218 lock::{ReentryGate, RouterMutex},
219 message_e2e_encryption_policy, message_meta, message_priority,
220 packet::Packet,
221 reliable_mode, wire_format,
222};
223use alloc::string::{String, ToString};
224
225#[cfg(any(test, all(feature = "embedded", not(feature = "std"))))]
226use crate::small_map::SmallMap as RouterMap;
227#[cfg(not(any(test, all(feature = "embedded", not(feature = "std")))))]
228use alloc::collections::BTreeMap as RouterMap;
229
230use alloc::collections::BTreeMap;
231
232use alloc::{
233 borrow::ToOwned,
234 boxed::Box,
235 collections::{BTreeSet, VecDeque},
236 format,
237 sync::Arc,
238 vec,
239 vec::Vec,
240};
241use core::cell::UnsafeCell;
242use core::fmt;
243use core::fmt::{Debug, Formatter};
244use core::mem::size_of;
245use core::ops::{Deref, DerefMut};
246use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
247#[cfg(feature = "std")]
248use std::time::Instant;
249
250pub type RouterSideId = usize;
252
253const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
254pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
255pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
256pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
257static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
258
259#[derive(Clone, Copy, Debug, PartialEq, Eq)]
260pub enum SideTransportProfile {
261 Canonical,
262 Template,
263 Ipv6Like,
264 Ipv4Like,
265}
266
267impl SideTransportProfile {
268 #[inline]
269 pub const fn as_str(self) -> &'static str {
270 match self {
271 Self::Canonical => "canonical",
272 Self::Template => "template",
273 Self::Ipv6Like => "ipv6_like",
274 Self::Ipv4Like => "ipv4_like",
275 }
276 }
277
278 #[cfg(feature = "discovery")]
279 #[inline]
280 pub const fn discovery_code(self) -> u8 {
281 match self {
282 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
283 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
284 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
285 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
286 }
287 }
288}
289
290#[cfg(test)]
291mod chunk_assembly_memory_tests {
292 use super::*;
293 #[test]
294 fn full_chunk_budget_reclaims_other_partial_transfers_before_active_schema() {
295 let mut state = SideTransportState::default();
296 for id in 1..=4 {
297 let mut assembly = SideChunkAssembly {
298 total: 28,
299 last_seen_ms: id as u64,
300 ..Default::default()
301 };
302 for index in 0..23 {
303 assembly.insert(
304 index,
305 crate::shared_bytes::convert(Arc::<[u8]>::from([id as u8; 112])),
306 );
307 }
308 state.rx_chunks.insert(id, assembly);
309 }
310 state.rx_chunks.get_mut(&1).unwrap().insert(
311 23,
312 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 56])),
313 );
314 state.refresh_rx_chunk_bytes();
316 SideChunkAssembly::make_room(
317 &mut state.rx_chunks,
318 &mut state.rx_chunk_bytes,
319 4,
320 23,
321 28,
322 112,
323 16384,
324 )
325 .unwrap();
326 assert!(!state.rx_chunks.contains_key(&1));
327 assert_eq!(state.rx_chunks[&4].received_len(), 23);
328 for index in 23..28 {
329 state.refresh_rx_chunk_bytes();
330 SideChunkAssembly::make_room(
331 &mut state.rx_chunks,
332 &mut state.rx_chunk_bytes,
333 4,
334 index,
335 28,
336 112,
337 16384,
338 )
339 .unwrap();
340 state.rx_chunks.get_mut(&4).unwrap().insert(
341 index,
342 crate::shared_bytes::convert(Arc::<[u8]>::from([4u8; 112])),
343 );
344 }
345 assert_eq!(
346 state.rx_chunks.remove(&4).unwrap().assemble().unwrap(),
347 vec![4u8; 28 * 112]
348 );
349 assert!(
350 state
351 .rx_chunks
352 .values()
353 .flat_map(|a| a.received.iter().flatten())
354 .map(|p| p.len() + 64)
355 .sum::<usize>()
356 <= 16384
357 );
358 }
359
360 #[test]
361 fn duplicate_chunk_keeps_partial_transfers_but_oversize_transfer_is_rejected() {
362 let mut state = SideTransportState::default();
363 let mut assembly = SideChunkAssembly {
364 total: 2,
365 ..Default::default()
366 };
367 assembly.insert(
368 0,
369 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 112])),
370 );
371 state.rx_chunks.insert(1, assembly.clone());
372 state.rx_chunks.insert(2, assembly);
373 state.refresh_rx_chunk_bytes();
374 SideChunkAssembly::make_room(
375 &mut state.rx_chunks,
376 &mut state.rx_chunk_bytes,
377 1,
378 0,
379 2,
380 112,
381 352,
382 )
383 .unwrap();
384 assert_eq!(state.rx_chunks.len(), 2);
385 state.refresh_rx_chunk_bytes();
386 assert!(
387 SideChunkAssembly::make_room(
388 &mut state.rx_chunks,
389 &mut state.rx_chunk_bytes,
390 1,
391 1,
392 2,
393 400,
394 352
395 )
396 .is_err()
397 );
398 assert!(!state.rx_chunks.contains_key(&1));
399 assert!(state.rx_chunks.contains_key(&2));
400 }
401
402 #[test]
403 fn schema_sized_reassembly_allocates_exactly_and_preserves_order() {
404 let payload: Vec<u8> = (0..3652).map(|i| (i % 251) as u8).collect();
405 let mut assembly = SideChunkAssembly {
406 total: payload.len().div_ceil(56) as u16,
407 ..Default::default()
408 };
409 for (index, chunk) in payload.chunks(56).enumerate().rev() {
410 assembly.insert(
411 index as u16,
412 crate::shared_bytes::convert(Arc::<[u8]>::from(chunk)),
413 );
414 }
415 assert_eq!(assembly.received_len(), usize::from(assembly.total));
416 let out = assembly.assemble().unwrap();
417 assert_eq!(out, payload);
418 assert_eq!(
419 out.capacity(),
420 payload.len(),
421 "do not double discovery reassembly capacity"
422 );
423 }
424 #[test]
425 fn incomplete_or_noncontiguous_chunks_are_rejected() {
426 let mut assembly = SideChunkAssembly {
427 total: 2,
428 ..Default::default()
429 };
430 assembly.insert(0, crate::shared_bytes::convert(Arc::<[u8]>::from([1u8])));
431 assert!(assembly.clone().assemble().is_err());
432 assert!(
433 assembly
434 .insert_with(2, 0, || Arc::<[u8]>::from([2u8]))
435 .is_err()
436 );
437 assert!(assembly.assemble().is_err());
438 }
439}
440
441#[derive(Clone, Debug, Default)]
442struct SideTransportState {
443 #[cfg(feature = "std")]
444 tx_callback_gate: Arc<ReentryGate>,
445 tx_chunk_progress: Option<Box<crate::side_transport::SideTxProgress>>,
446 #[cfg(all(feature = "compression", feature = "discovery"))]
447 payload_codec: crate::payload_codec::Ledger,
448 #[cfg(all(feature = "compression", feature = "discovery"))]
449 payload_codec_choice: u64,
450 tx_template_ids: RouterMap<u64, u32>,
451 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
452 tx_last_timestamps: RouterMap<u32, u64>,
453 tx_compact_uses: RouterMap<u32, u8>,
454 rx_templates: RouterMap<u64, u32>,
455 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
456 rx_last_timestamps: RouterMap<u32, u64>,
457 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
458 rx_chunk_reclaims: u32,
459 rx_chunk_bytes: usize,
460}
461
462impl SideTransportState {
463 #[inline(never)]
464 fn take_chunk(&mut self, id: u32, reclaimed: bool) -> Option<SideChunkAssembly> {
465 let value = SideChunkAssembly::remove(&mut self.rx_chunks, &mut self.rx_chunk_bytes, id)?;
466 if reclaimed {
467 self.rx_chunk_reclaims = self.rx_chunk_reclaims.saturating_add(1);
468 }
469 Some(value)
470 }
471 #[cfg(test)]
473 #[inline(never)]
474 fn refresh_rx_chunk_bytes(&mut self) {
475 self.rx_chunk_bytes = self
476 .rx_chunks
477 .values()
478 .map(SideChunkAssembly::byte_cost)
479 .sum();
480 }
481 fn refresh_transmit_header_templates(&mut self) {
482 for uses in self.tx_compact_uses.values_mut() {
486 *uses = u8::MAX;
487 }
488 }
491
492 fn tx_template_count(&self) -> usize {
493 self.tx_template_ids.len()
494 }
495
496 fn rx_template_count(&self) -> usize {
497 self.rx_templates_by_id.len()
498 }
499
500 fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
501 let mut removed = 0usize;
502 while self.tx_template_ids.len() > limit {
503 let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
504 break;
505 };
506 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
507 self.tx_last_timestamps.remove(&old_id);
508 self.tx_compact_uses.remove(&old_id);
509 }
510 self.tx_templates.remove(&old_hash);
511 removed += 1;
512 }
513 removed
514 }
515
516 fn insert_tx_template(
517 &mut self,
518 template: SideHeaderTemplate,
519 template_id: u32,
520 max_templates: usize,
521 ) -> bool {
522 if max_templates == 0 {
523 return false;
524 }
525 let evicted = !self.tx_template_ids.contains_key(&template.hash)
526 && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
527 self.tx_template_ids.insert(template.hash, template_id);
528 self.tx_templates.insert(template.hash, template);
529 self.tx_compact_uses.insert(template_id, 0);
530 evicted
531 }
532
533 fn insert_rx_template(
534 &mut self,
535 template_id: u32,
536 template: SideHeaderTemplate,
537 max_templates: usize,
538 ) -> bool {
539 if max_templates == 0 {
540 return false;
541 }
542 if let Some(old_id) = self.rx_templates.get(&template.hash).copied()
545 && old_id != template_id
546 {
547 self.rx_templates_by_id.remove(&old_id);
548 self.rx_last_timestamps.remove(&old_id);
549 self.rx_templates.remove(&template.hash);
550 }
551 if let Some(previous) = self.rx_templates_by_id.get(&template_id)
552 && previous.hash != template.hash
553 {
554 self.rx_templates.remove(&previous.hash);
555 self.rx_last_timestamps.remove(&template_id);
556 }
557 let mut evicted = false;
558 if self.rx_templates_by_id.len() >= max_templates
559 && !self.rx_templates_by_id.contains_key(&template_id)
560 && let Some((&old_hash, &old_id)) = self.rx_templates.first_key_value()
563 {
564 self.rx_templates.remove(&old_hash);
565 self.rx_templates_by_id.remove(&old_id);
566 self.rx_last_timestamps.remove(&old_id);
567 evicted = true;
568 }
569 self.rx_templates.insert(template.hash, template_id);
570 self.rx_templates_by_id.insert(template_id, template);
571 evicted
572 }
573}
574
575#[derive(Clone, Debug, PartialEq, Eq)]
576pub struct CompactTimestampOmissionPolicy {
577 all: bool,
578 types: Option<Arc<[DataType]>>,
579}
580impl CompactTimestampOmissionPolicy {
581 pub const fn none() -> Self {
582 Self {
583 all: false,
584 types: None,
585 }
586 }
587 pub const fn all() -> Self {
588 Self {
589 all: true,
590 types: None,
591 }
592 }
593 pub fn with_data_type(mut self, ty: DataType) -> Self {
594 self.insert(ty);
595 self
596 }
597 pub fn insert(&mut self, ty: DataType) {
598 if self.contains(ty) {
599 return;
600 }
601 let current = self.types.as_deref().unwrap_or(&[]);
602 let mut types = Vec::with_capacity(current.len() + 1);
603 types.extend_from_slice(current);
604 let index = types.binary_search(&ty).unwrap_err();
605 types.insert(index, ty);
606 self.types = Some(Arc::from(types));
607 }
608 pub fn contains(&self, ty: DataType) -> bool {
609 self.all
610 || self
611 .types
612 .as_deref()
613 .is_some_and(|types| types.binary_search(&ty).is_ok())
614 }
615 pub fn is_empty(&self) -> bool {
616 !self.all && self.types.is_none()
617 }
618}
619
620impl Default for CompactTimestampOmissionPolicy {
621 #[inline]
622 fn default() -> Self {
623 Self::none()
624 }
625}
626
627#[derive(Clone, Debug, PartialEq, Eq)]
628pub enum RouterItem {
629 Packet(Packet),
630 Packed(crate::SharedBytes),
631}
632
633#[cfg(feature = "compact-packet-store")]
634impl RouterItem {
635 fn park_for_queue(&mut self) -> TelemetryResult<()> {
636 match self {
637 Self::Packet(packet) => packet.park_for_queue()?,
638 Self::Packed(bytes) => bytes.park_for_queue()?,
639 }
640 Ok(())
641 }
642}
643
644#[derive(Clone, Debug, PartialEq, Eq)]
645struct RouterRxItem {
646 src: Option<RouterSideId>,
647 data: RouterItem,
648 priority: u8,
649}
650
651#[derive(Clone, Debug, PartialEq, Eq)]
652enum RouterTxItem {
653 Broadcast(RouterItem),
654 EndToEndReplay {
655 packet_id: u64,
656 },
657 ToSide {
658 src: Option<RouterSideId>,
659 dst: RouterSideId,
660 data: RouterItem,
661 },
662 ReliableReplay {
663 dst: RouterSideId,
664 ty: DataType,
665 seq: u32,
666 bytes: crate::SharedBytes,
667 },
668}
669
670impl ByteCost for RouterRxItem {
671 #[cfg(feature = "compact-packet-store")]
672 fn park_for_queue(&mut self) -> TelemetryResult<()> {
673 self.data.park_for_queue()
674 }
675 #[inline]
676 fn byte_cost(&self) -> usize {
677 match &self.data {
678 RouterItem::Packet(pkt) => pkt.byte_cost(),
679 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
680 }
681 }
682}
683
684impl ByteCost for RouterTxItem {
685 #[cfg(feature = "compact-packet-store")]
686 fn park_for_queue(&mut self) -> TelemetryResult<()> {
687 match self {
688 Self::Broadcast(item) | Self::ToSide { data: item, .. } => item.park_for_queue(),
689 Self::ReliableReplay { bytes, .. } => bytes.park_for_queue(),
690 _ => Ok(()),
691 }
692 }
693 #[inline]
694 fn byte_cost(&self) -> usize {
695 match self {
696 RouterTxItem::Broadcast(data) => match data {
697 RouterItem::Packet(pkt) => pkt.byte_cost(),
698 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
699 },
700 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
701 RouterTxItem::ToSide { data, .. } => match data {
702 RouterItem::Packet(pkt) => pkt.byte_cost(),
703 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
704 },
705 RouterTxItem::ReliableReplay { bytes, .. } => {
706 size_of::<crate::SharedBytes>() + bytes.len()
707 }
708 }
709 }
710}
711
712#[derive(Clone, Copy)]
713enum ReplayMarkerKey {
714 EndToEnd(u64),
715 Hop(RouterSideId, DataType, u32),
716}
717impl RouterTxItem {
718 fn replay_marker_key(&self) -> Option<ReplayMarkerKey> {
719 match self {
720 Self::EndToEndReplay { packet_id } => Some(ReplayMarkerKey::EndToEnd(*packet_id)),
721 Self::ReliableReplay { dst, ty, seq, .. } => {
722 Some(ReplayMarkerKey::Hop(*dst, *ty, *seq))
723 }
724 _ => None,
725 }
726 }
727}
728
729#[repr(u8)]
730#[derive(Clone, Copy, Debug, PartialEq, Eq)]
731enum QueuedDelivery {
732 Untracked,
733 OrderedTail,
734 Ordered,
735 DeferredOrdered,
736 DeferredUnordered,
737 Unordered,
738 ClaimedOrdered,
739 ClaimedUnordered,
740}
741
742#[derive(Clone, Debug, PartialEq, Eq)]
746struct TxQueued {
747 disposable: bool,
749 item: RouterTxItem,
750 ignore_local: bool,
751 priority: u8,
752 delivery_state: QueuedDelivery,
753 accepted_type: u32,
754 deferred_since_ms: u64,
755 retry_after_ms: u64,
756}
757
758#[derive(Clone, Copy)]
759struct TxClaim {
760 token: u32,
761 ordered: bool,
762}
763#[derive(Default)]
764struct DispatchContext {
765 claim: Option<TxClaim>,
766 since: Option<u64>,
767 terminal: bool,
768 owner_transferred: bool,
769}
770struct TxWork {
771 queued: TxQueued,
772 claim: Option<TxClaim>,
773}
774
775const MAX_PRIORITY_BURST: u8 = 8;
780
781const _: () = {
787 assert!(size_of::<TxQueued>()
788 == size_of::<(RouterTxItem, bool, u8, QueuedDelivery, u32, u64, u64)>());
789 let metadata_bytes = 2 * size_of::<u64>() + size_of::<u32>()
790 + size_of::<QueuedDelivery>() + 2 * size_of::<bool>() + size_of::<u8>();
791 let alignment = core::mem::align_of::<TxQueued>();
792 let expected = (size_of::<RouterTxItem>() + metadata_bytes)
793 .div_ceil(alignment) * alignment;
794 assert!(size_of::<TxQueued>() == expected);
795};
796
797impl ByteCost for TxQueued {
798 #[cfg(feature = "compact-packet-store")]
799 fn park_for_queue(&mut self) -> TelemetryResult<()> {
800 self.item.park_for_queue()
801 }
802 #[inline]
804 fn byte_cost(&self) -> usize {
805 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>() + 2 * size_of::<u64>()
806 }
807}
808
809impl ByteCost for u64 {
811 #[inline]
813 fn byte_cost(&self) -> usize {
814 size_of::<u64>()
815 }
816}
817
818#[derive(Debug, Clone)]
821struct ReliableTxState {
822 next_seq: u32,
823 sent_order: VecDeque<u32>,
824 sent: BTreeMap<u32, ReliableSent>,
825}
826
827#[derive(Debug, Clone)]
828struct ReliableSent {
829 bytes: crate::SharedBytes,
830 last_send_ms: u64,
831 retries: u32,
832 queued: bool,
833 partial_acked: bool,
834}
835
836#[derive(Debug, Clone)]
837struct EndToEndReliableSent {
838 ordered_type: Option<u32>,
839 waiting_for_predecessor: bool,
840 data: RouterItem,
841 pending_destinations: BTreeMap<u64, RouterSideId>,
842 tracked_destinations: bool,
843 last_send_ms: u64,
844 retries: u32,
845 queued: bool,
846}
847
848#[derive(Debug, Clone)]
849struct ReliableRxState {
850 expected_seq: u32,
851 buffered: RouterMap<u32, crate::SharedBytes>,
852}
853
854#[derive(Debug, Clone)]
855struct ReliableReturnRouteState {
856 side: RouterSideId,
857 locally_delivered: bool,
860 last_replay_ms: u64,
861 upstream_contract: Option<(u64, u64)>,
865}
866
867#[cfg(feature = "discovery")]
868#[derive(Debug, Clone, Default, PartialEq, Eq)]
869struct DiscoverySenderState {
870 advertised_address: Option<u32>,
873 reachable: Vec<DataEndpoint>,
874 advertised_reachable: Vec<DataEndpoint>,
875 reachable_network_variables: Vec<DataType>,
876 reachable_timesync_sources: Vec<String>,
877 advertised_reachable_timesync_sources: Vec<String>,
878 link_capabilities: Option<discovery::LinkCapabilities>,
879 topology_boards: Arc<Vec<TopologyBoardNode>>,
880 has_full_topology: bool,
881 has_schema: bool,
882 last_seen_ms: u64,
883}
884
885#[cfg(feature = "discovery")]
886#[derive(Debug, Clone, Default, PartialEq, Eq)]
887struct DiscoverySideState {
888 reachable: Vec<DataEndpoint>,
889 reachable_network_variables: Vec<DataType>,
890 requested_network_variables: Vec<(DataType, u64)>,
891 reachable_timesync_sources: Vec<String>,
892 last_seen_ms: u64,
893 announcers: BTreeMap<String, DiscoverySenderState>,
894}
895
896#[cfg(feature = "discovery")]
897#[derive(Debug, Clone, Default)]
898struct DiscoverySideThrottleState {
899 next_topology_request_ms: u64,
900 next_schema_request_ms: u64,
901 next_ping_ms: u64,
902 last_liveness_tx_ms: u64,
904 pending_incremental: bool,
905 incremental_retries_remaining: u8,
906 has_sent_full: bool,
907 last_topology: Vec<TopologyBoardNode>,
908}
909
910#[cfg(feature = "discovery")]
911impl DiscoverySideThrottleState {
912 fn liveness_due(&self, now_ms: u64) -> bool {
913 now_ms >= self.next_ping_ms
914 && now_ms.saturating_sub(self.last_liveness_tx_ms)
915 >= DISCOVERY_SLOW_LINK_PING_INTERVAL_MS
916 }
917}
918
919#[cfg(all(feature = "discovery", feature = "timesync"))]
920#[derive(Debug, Clone, Default)]
921struct TimeSyncSideThrottleState {
922 next_allowed_ms: u64,
923}
924
925#[cfg(feature = "discovery")]
926#[derive(Debug, Clone, Copy, PartialEq, Eq)]
927enum DiscoveryAdvertiseLevel {
928 MinimalPing,
929 Incremental,
930 Full,
931}
932
933#[derive(Debug, Clone, Default)]
934struct AdaptiveRouteStats {
935 estimated_bandwidth_bps: u64,
936 peak_bandwidth_bps: u64,
937 last_observed_ms: u64,
938 last_slow_observed_ms: u64,
939 sample_count: u64,
940 window_started_ms: u64,
941 window_bytes: u64,
942 peak_usage_bps: u64,
943}
944
945impl AdaptiveRouteStats {
946 #[inline]
947 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
948 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
949 sample_bps
950 } else if sample_bps >= self.estimated_bandwidth_bps {
951 self.estimated_bandwidth_bps
952 .saturating_mul(3)
953 .saturating_add(sample_bps.saturating_mul(5))
954 / 8
955 } else {
956 self.estimated_bandwidth_bps
957 .saturating_mul(7)
958 .saturating_add(sample_bps)
959 / 8
960 };
961 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
962 self.last_observed_ms = now_ms;
963 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
964 self.last_slow_observed_ms = now_ms;
965 }
966 self.sample_count = self.sample_count.saturating_add(1);
967 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
968 self.window_started_ms = now_ms;
969 self.window_bytes = 0;
970 }
971 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
972 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
973 }
974
975 #[inline]
976 fn current_usage_bps(&self, now_ms: u64) -> u64 {
977 if self.window_started_ms == 0 {
978 return 0;
979 }
980 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
981 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
982 .min(u128::from(u64::MAX)) as u64
983 }
984
985 #[inline]
986 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
987 let capacity = self
988 .estimated_bandwidth_bps
989 .max(self.peak_bandwidth_bps)
990 .max(1);
991 capacity.saturating_sub(self.current_usage_bps(now_ms))
992 }
993
994 #[inline]
995 fn weight(&self, now_ms: u64) -> u64 {
996 self.available_headroom_bps(now_ms).max(1)
997 }
998
999 #[inline]
1000 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
1001 let current_usage_bps = self.current_usage_bps(now_ms);
1002 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
1003 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
1004 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
1005 AdaptiveLinkStats {
1006 auto_balancing_enabled,
1007 estimated_capacity_bps,
1008 peak_capacity_bps,
1009 current_usage_bps,
1010 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
1011 available_headroom_bps,
1012 effective_weight: available_headroom_bps.max(1),
1013 last_observed_ms: self.last_observed_ms,
1014 sample_count: self.sample_count,
1015 }
1016 }
1017}
1018
1019use crate::type_stats::TypeRuntimeStatsInner;
1020
1021#[derive(Debug, Clone, Default)]
1022struct SideRuntimeStatsInner {
1023 tx_packets: u64,
1024 tx_bytes: u64,
1025 rx_packets: u64,
1026 rx_bytes: u64,
1027 relayed_tx_packets: u64,
1028 relayed_tx_bytes: u64,
1029 relayed_rx_packets: u64,
1030 relayed_rx_bytes: u64,
1031 local_delivery_packets: u64,
1032 tx_retries: u64,
1033 tx_handler_failures: u64,
1034 local_handler_failures: u64,
1035 total_handler_retries: u64,
1036 side_transport_full_frames: u64,
1037 side_transport_compact_frames: u64,
1038 side_transport_compact_delta_frames: u64,
1039 side_transport_compact_omitted_timestamp_frames: u64,
1040 side_transport_chunk_frames: u64,
1041 side_transport_raw_bytes: u64,
1042 side_transport_wire_bytes: u64,
1043 side_transport_bytes_saved: u64,
1044 side_transport_min_compact_overhead_bytes: Option<usize>,
1045 side_transport_max_compact_overhead_bytes: Option<usize>,
1046 side_transport_compact_target_misses: u64,
1047 side_transport_template_evictions: u64,
1048 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
1049}
1050
1051impl SideRuntimeStatsInner {
1052 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
1053 self.data_types.entry(ty.as_u32()).or_default()
1054 }
1055
1056 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
1057 self.tx_packets = self.tx_packets.saturating_add(1);
1058 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
1059 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1060 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1061 if relayed {
1062 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
1063 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
1064 }
1065 self.type_stats_mut(ty)
1066 .tx(bytes as u64, relayed, retries as u64);
1067 }
1068
1069 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
1070 self.rx_packets = self.rx_packets.saturating_add(1);
1071 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
1072 if relayed {
1073 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
1074 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
1075 }
1076 self.type_stats_mut(ty).rx(bytes as u64, relayed);
1077 }
1078
1079 fn note_local_delivery(&mut self, ty: DataType) {
1080 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
1081 self.type_stats_mut(ty).rx(0, false);
1082 }
1083
1084 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
1085 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
1086 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1087 self.type_stats_mut(ty).failure(0);
1088 }
1089
1090 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
1091 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
1092 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1093 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1094 self.type_stats_mut(ty).failure(retries as u64);
1095 }
1096
1097 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
1098 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
1099 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1100 }
1101
1102 fn note_side_transport_compact(
1103 &mut self,
1104 raw_bytes: usize,
1105 wire_bytes: usize,
1106 compact_overhead_bytes: usize,
1107 used_timestamp_delta: bool,
1108 omitted_timestamp: bool,
1109 ) {
1110 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
1111 if used_timestamp_delta {
1112 self.side_transport_compact_delta_frames =
1113 self.side_transport_compact_delta_frames.saturating_add(1);
1114 }
1115 if omitted_timestamp {
1116 self.side_transport_compact_omitted_timestamp_frames = self
1117 .side_transport_compact_omitted_timestamp_frames
1118 .saturating_add(1);
1119 }
1120 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1121 self.side_transport_min_compact_overhead_bytes = Some(
1122 self.side_transport_min_compact_overhead_bytes
1123 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
1124 );
1125 self.side_transport_max_compact_overhead_bytes = Some(
1126 self.side_transport_max_compact_overhead_bytes
1127 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
1128 );
1129 }
1130
1131 fn note_side_transport_chunks(&mut self, chunks: usize) {
1132 self.side_transport_chunk_frames = self
1133 .side_transport_chunk_frames
1134 .saturating_add(chunks as u64);
1135 }
1136
1137 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
1138 self.side_transport_raw_bytes = self
1139 .side_transport_raw_bytes
1140 .saturating_add(raw_bytes as u64);
1141 self.side_transport_wire_bytes = self
1142 .side_transport_wire_bytes
1143 .saturating_add(wire_bytes as u64);
1144 if raw_bytes > wire_bytes {
1145 self.side_transport_bytes_saved = self
1146 .side_transport_bytes_saved
1147 .saturating_add((raw_bytes - wire_bytes) as u64);
1148 }
1149 }
1150
1151 fn note_side_transport_compact_target_miss(&mut self) {
1152 self.side_transport_compact_target_misses =
1153 self.side_transport_compact_target_misses.saturating_add(1);
1154 }
1155
1156 fn note_side_transport_template_eviction(&mut self) {
1157 self.side_transport_template_evictions =
1158 self.side_transport_template_evictions.saturating_add(1);
1159 }
1160}
1161
1162#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1163enum RouteSelectionOrigin {
1164 Flood,
1165 Discovered,
1166}
1167
1168type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1171
1172type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
1174type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
1175
1176#[derive(Clone)]
1182pub enum EndpointHandlerFn {
1183 Packet(Arc<PacketHandlerFn>),
1184 Packed(Arc<PackedHandlerFn>),
1185}
1186
1187pub struct EndpointHandler {
1189 endpoint: DataEndpoint,
1190 handler: EndpointHandlerFn,
1191}
1192
1193impl Debug for EndpointHandlerFn {
1195 #[inline]
1196 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1197 match self {
1198 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
1199 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
1200 }
1201 }
1202}
1203
1204#[derive(Clone)]
1206pub enum RouterTxHandlerFn {
1207 Packed(Arc<PackedHandlerFn>),
1208 PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
1209 Packet(Arc<PacketHandlerFn>),
1210}
1211
1212impl Debug for RouterTxHandlerFn {
1213 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1214 match self {
1215 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
1216 RouterTxHandlerFn::PackedWithPriority(_) => f
1217 .debug_tuple("PackedWithPriority")
1218 .field(&"<handler>")
1219 .finish(),
1220 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
1221 }
1222 }
1223}
1224
1225#[derive(Clone, Debug)]
1226pub struct RouterSideOptions {
1227 pub reliable_enabled: bool,
1238 pub link_local_enabled: bool,
1240 pub header_template_enabled: bool,
1247 pub max_frame_bytes: usize,
1253 pub compact_header_target_bytes: usize,
1260 pub max_side_transport_templates: usize,
1266 pub omit_unchanged_compact_timestamps: bool,
1272 pub compact_timestamp_omission_types: CompactTimestampOmissionPolicy,
1277 pub side_transport_profile: SideTransportProfile,
1279 pub ingress_enabled: bool,
1281 pub egress_enabled: bool,
1283}
1284
1285impl Default for RouterSideOptions {
1286 fn default() -> Self {
1287 Self {
1288 reliable_enabled: false,
1289 link_local_enabled: false,
1290 header_template_enabled: false,
1291 max_frame_bytes: 0,
1292 compact_header_target_bytes: 0,
1293 max_side_transport_templates: DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT,
1294 omit_unchanged_compact_timestamps: false,
1295 compact_timestamp_omission_types: CompactTimestampOmissionPolicy::none(),
1296 side_transport_profile: SideTransportProfile::Canonical,
1297 ingress_enabled: true,
1298 egress_enabled: true,
1299 }
1300 }
1301}
1302
1303impl RouterSideOptions {
1304 #[inline]
1309 pub fn with_small_packet_transport(mut self, max_frame_bytes: usize) -> Self {
1310 self.header_template_enabled = true;
1311 self.max_frame_bytes = max_frame_bytes;
1312 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
1313 self.side_transport_profile = SideTransportProfile::Ipv6Like;
1314 self
1315 }
1316
1317 #[inline]
1318 pub fn with_ipv4_like_compact_header_target(mut self) -> Self {
1319 self.header_template_enabled = true;
1320 self.compact_header_target_bytes = IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES;
1321 self.omit_unchanged_compact_timestamps = true;
1322 self.side_transport_profile = SideTransportProfile::Ipv4Like;
1323 self
1324 }
1325
1326 #[inline]
1327 pub fn with_ipv6_like_compact_header_target(mut self) -> Self {
1328 self.header_template_enabled = true;
1329 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
1330 self.side_transport_profile = SideTransportProfile::Ipv6Like;
1331 self
1332 }
1333
1334 #[inline]
1335 pub fn with_template_transport(mut self) -> Self {
1336 self.header_template_enabled = true;
1337 self.side_transport_profile = SideTransportProfile::Template;
1338 self
1339 }
1340
1341 #[inline]
1342 pub fn with_omitted_unchanged_compact_timestamps(mut self) -> Self {
1343 self.header_template_enabled = true;
1344 self.omit_unchanged_compact_timestamps = true;
1345 self
1346 }
1347
1348 #[inline]
1349 pub fn with_omitted_unchanged_compact_timestamps_for_type(mut self, ty: DataType) -> Self {
1350 self.header_template_enabled = true;
1351 self.compact_timestamp_omission_types.insert(ty);
1352 self
1353 }
1354
1355 #[inline]
1356 pub fn effective_transport_profile(&self) -> SideTransportProfile {
1357 if !self.header_template_enabled && self.max_frame_bytes == 0 {
1358 SideTransportProfile::Canonical
1359 } else if self.side_transport_profile == SideTransportProfile::Canonical {
1360 SideTransportProfile::Template
1361 } else {
1362 self.side_transport_profile
1363 }
1364 }
1365
1366 #[cfg(feature = "discovery")]
1367 #[inline]
1368 pub fn link_capabilities(&self) -> discovery::LinkCapabilities {
1369 let mut flags = discovery::LINK_CAPABILITY_END_TO_END_RELIABILITY;
1370 if self.header_template_enabled {
1371 flags |= discovery::LINK_CAPABILITY_HEADER_TEMPLATES;
1372 }
1373 if self.max_frame_bytes != 0 {
1374 flags |= discovery::LINK_CAPABILITY_CHUNKING;
1375 }
1376 if self.reliable_enabled {
1377 flags |= discovery::LINK_CAPABILITY_RELIABILITY;
1378 }
1379 if self.omit_unchanged_compact_timestamps
1380 || !self.compact_timestamp_omission_types.is_empty()
1381 {
1382 flags |= discovery::LINK_CAPABILITY_OMIT_UNCHANGED_TIMESTAMPS;
1383 }
1384 #[cfg(feature = "cryptography")]
1385 {
1386 flags |= discovery::LINK_CAPABILITY_CRYPTO;
1387 }
1388 #[cfg(feature = "compression")]
1389 {
1390 flags |= discovery::LINK_CAPABILITY_PAYLOAD_COMPRESSION;
1391 }
1392 discovery::LinkCapabilities {
1393 version: 1,
1394 flags,
1395 profile: self.effective_transport_profile().discovery_code(),
1396 max_frame_bytes: self.max_frame_bytes.min(u32::MAX as usize) as u32,
1397 compact_header_target_bytes: self.compact_header_target_bytes.min(u32::MAX as usize)
1398 as u32,
1399 max_side_transport_templates: self.max_side_transport_templates.min(u32::MAX as usize)
1400 as u32,
1401 }
1402 }
1403}
1404
1405#[derive(Clone, Debug)]
1407pub struct RouterSide {
1408 pub name: Arc<str>,
1409 pub tx_handler: RouterTxHandlerFn,
1410 pub opts: RouterSideOptions,
1411}
1412
1413impl Debug for EndpointHandler {
1415 #[inline]
1416 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1417 f.debug_struct("EndpointHandler")
1418 .field("endpoint", &self.endpoint)
1419 .field("handler", &self.handler)
1420 .finish()
1421 }
1422}
1423
1424#[inline]
1425pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1426 #[cfg(feature = "timesync")]
1427 if matches!(endpoint, DataEndpoint::TimeSync) {
1428 return true;
1429 }
1430 #[cfg(feature = "discovery")]
1431 {
1432 discovery::is_discovery_endpoint(endpoint)
1433 }
1434 #[cfg(not(feature = "discovery"))]
1435 {
1436 let _ = endpoint;
1437 false
1438 }
1439}
1440
1441impl EndpointHandler {
1442 #[inline]
1446 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1447 where
1448 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1449 {
1450 assert!(
1451 !endpoint_is_router_internal(endpoint),
1452 "reserved internal endpoint handlers must not be user-registered"
1453 );
1454 #[cfg(feature = "std")]
1455 crate::config::ensure_endpoint_id(endpoint, false)
1456 .expect("endpoint handler endpoint registration failed");
1457 Self {
1458 endpoint,
1459 handler: EndpointHandlerFn::Packet(Arc::new(f)),
1460 }
1461 }
1462
1463 #[inline]
1465 #[cfg(feature = "std")]
1466 pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1467 where
1468 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1469 {
1470 Self::new_packet_handler(endpoint.id, f)
1471 }
1472
1473 #[cfg(not(feature = "std"))]
1475 #[inline]
1476 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1477 where
1478 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1479 {
1480 Self::new_packet_handler(endpoint.id, f)
1481 }
1482
1483 #[cfg(feature = "std")]
1485 #[inline]
1486 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1487 where
1488 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1489 {
1490 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1491 .ok_or(TelemetryError::BadArg)?;
1492 Ok(Self::new_packet_handler(endpoint.id, f))
1493 }
1494
1495 #[inline]
1499 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1500 where
1501 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1502 {
1503 assert!(
1504 !endpoint_is_router_internal(endpoint),
1505 "reserved internal endpoint handlers must not be user-registered"
1506 );
1507 #[cfg(feature = "std")]
1508 crate::config::ensure_endpoint_id(endpoint, false)
1509 .expect("endpoint handler endpoint registration failed");
1510 Self {
1511 endpoint,
1512 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1513 }
1514 }
1515
1516 #[inline]
1518 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1519 where
1520 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1521 {
1522 Self::new_packed_handler(endpoint.id, f)
1523 }
1524
1525 #[cfg(feature = "std")]
1527 #[inline]
1528 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1529 where
1530 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1531 {
1532 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1533 .ok_or(TelemetryError::BadArg)?;
1534 Ok(Self::new_packed_handler(endpoint.id, f))
1535 }
1536
1537 #[inline]
1539 pub fn get_endpoint(&self) -> DataEndpoint {
1540 self.endpoint
1541 }
1542
1543 #[inline]
1545 pub fn get_handler(&self) -> &EndpointHandlerFn {
1546 &self.handler
1547 }
1548}
1549
1550pub trait Clock {
1551 fn now_ms(&self) -> u64;
1553
1554 fn now_ns(&self) -> u64 {
1558 self.now_ms().saturating_mul(1_000_000)
1559 }
1560}
1561
1562impl<T: Fn() -> u64> Clock for T {
1563 #[inline]
1564 fn now_ms(&self) -> u64 {
1565 self()
1566 }
1567}
1568
1569#[cfg(feature = "std")]
1570#[derive(Debug)]
1571struct StdMonotonicClock {
1572 start: Instant,
1573}
1574
1575#[cfg(feature = "std")]
1576impl Default for StdMonotonicClock {
1577 fn default() -> Self {
1578 Self {
1579 start: Instant::now(),
1580 }
1581 }
1582}
1583
1584#[cfg(feature = "std")]
1585impl Clock for StdMonotonicClock {
1586 fn now_ms(&self) -> u64 {
1587 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1588 }
1589
1590 fn now_ns(&self) -> u64 {
1591 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1592 }
1593}
1594
1595#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1597pub enum RouterE2eEncryptionMode {
1598 Disabled,
1600 RequiredOnly,
1602 Preferred,
1604 ForceAll,
1606}
1607
1608pub type NodeAddress = u32;
1609pub type P2pPort = u16;
1610pub type P2pStreamId = u32;
1611
1612const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1613const P2P_STREAM_VERSION: u8 = 1;
1614const P2P_STREAM_SYN: u8 = 0x01;
1615const P2P_STREAM_ACK: u8 = 0x02;
1616const P2P_STREAM_FIN: u8 = 0x04;
1617const P2P_STREAM_RST: u8 = 0x08;
1618const P2P_STREAM_DATA: u8 = 0x10;
1619
1620#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1621pub enum AddressAssignmentMode {
1622 Dynamic,
1623 Requested(NodeAddress),
1624 Static(NodeAddress),
1625}
1626
1627impl AddressAssignmentMode {
1628 #[inline]
1629 #[cfg(feature = "discovery")]
1630 fn mode_code(self) -> u8 {
1631 match self {
1632 Self::Dynamic => 0,
1633 Self::Requested(_) => 1,
1634 Self::Static(_) => 2,
1635 }
1636 }
1637
1638 #[inline]
1639 fn requested_address(self) -> NodeAddress {
1640 match self {
1641 Self::Dynamic => 0,
1642 Self::Requested(addr) | Self::Static(addr) => addr,
1643 }
1644 }
1645}
1646
1647#[derive(Debug, Clone, PartialEq, Eq)]
1648pub enum AddressChangeReason {
1649 Configured,
1650 DynamicConflict,
1651 RequestedConflict,
1652 StaticConflict,
1653 HostnameConflict,
1654}
1655
1656#[derive(Debug, Clone, PartialEq, Eq)]
1657pub struct AddressChange {
1658 pub old_hostname: Arc<str>,
1659 pub new_hostname: Arc<str>,
1660 pub old_address: NodeAddress,
1661 pub new_address: NodeAddress,
1662 pub reason: AddressChangeReason,
1663}
1664
1665#[derive(Debug, Clone, PartialEq, Eq)]
1666pub struct AddressBookEntry {
1667 pub hostname: Arc<str>,
1668 pub address: NodeAddress,
1669 pub requested_address: NodeAddress,
1670 pub mode: AddressAssignmentMode,
1671 pub birth_ms: u64,
1672 pub owner_hash: u64,
1673 pub last_seen_ms: u64,
1674}
1675
1676pub struct P2pMessage<'a> {
1677 pub source_hostname: &'a str,
1678 pub source_address: NodeAddress,
1679 pub source_port: P2pPort,
1680 pub destination_port: P2pPort,
1681 pub payload: &'a [u8],
1682}
1683
1684#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1685pub enum P2pStreamEventKind {
1686 Accepted,
1687 Connected,
1688 Data,
1689 Closed,
1690 Reset,
1691}
1692
1693pub struct P2pStreamEvent<'a> {
1694 pub kind: P2pStreamEventKind,
1695 pub stream_id: P2pStreamId,
1696 pub peer_stream_id: P2pStreamId,
1697 pub sequence: u32,
1698 pub peer_hostname: &'a str,
1699 pub peer_address: NodeAddress,
1700 pub local_port: P2pPort,
1701 pub peer_port: P2pPort,
1702 pub payload: &'a [u8],
1703}
1704
1705struct P2pDecoded<'a> {
1706 source_hostname: &'a str,
1707 source_address: NodeAddress,
1708 source_port: P2pPort,
1709 destination_port: P2pPort,
1710 payload: &'a [u8],
1711}
1712
1713#[derive(Debug, Clone)]
1714struct P2pStreamDecoded<'a> {
1715 flags: u8,
1716 source_stream_id: P2pStreamId,
1717 destination_stream_id: P2pStreamId,
1718 sequence: u32,
1719 payload: &'a [u8],
1720}
1721
1722type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1723
1724#[derive(Clone)]
1725struct AddressChangeHandler {
1726 handler: Arc<AddressChangeFn>,
1727}
1728
1729impl Debug for AddressChangeHandler {
1730 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1731 f.write_str("AddressChangeHandler(<handler>)")
1732 }
1733}
1734
1735type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1736
1737#[derive(Clone)]
1738struct P2pPortHandler {
1739 handler: Arc<P2pPortHandlerFn>,
1740}
1741
1742impl Debug for P2pPortHandler {
1743 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1744 f.write_str("P2pPortHandler(<handler>)")
1745 }
1746}
1747
1748type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1749
1750#[derive(Clone)]
1751struct P2pStreamHandler {
1752 handler: Arc<P2pStreamHandlerFn>,
1753}
1754
1755impl Debug for P2pStreamHandler {
1756 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1757 f.write_str("P2pStreamHandler(<handler>)")
1758 }
1759}
1760
1761#[derive(Debug, Clone)]
1762struct P2pStreamSession {
1763 peer_hostname: Arc<str>,
1764 peer_address: NodeAddress,
1765 local_port: P2pPort,
1766 peer_port: P2pPort,
1767 peer_stream_id: P2pStreamId,
1768 next_sequence: u32,
1769 connected: bool,
1770}
1771
1772#[derive(Debug, Clone)]
1773struct PendingP2pStreamEvent {
1774 handlers: Vec<P2pStreamHandler>,
1775 kind: P2pStreamEventKind,
1776 stream_id: P2pStreamId,
1777 peer_stream_id: P2pStreamId,
1778 peer_hostname: Arc<str>,
1779 peer_address: NodeAddress,
1780 local_port: P2pPort,
1781 peer_port: P2pPort,
1782}
1783
1784#[derive(Debug, Clone)]
1785pub struct RouterConfig {
1786 handlers: Arc<[EndpointHandler]>,
1788 reliable_enabled: bool,
1790 sender: Option<Arc<str>>,
1792 address_mode: AddressAssignmentMode,
1794 address_change_handlers: Arc<[AddressChangeHandler]>,
1796 e2e_encryption: RouterE2eEncryptionMode,
1798 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
1800 e2e_key_id: u32,
1801 memory: RuntimeMemoryConfig,
1802 #[cfg(feature = "discovery")]
1803 preferred_discovery_master: Option<Arc<str>>,
1804 #[cfg(feature = "timesync")]
1805 timesync: Option<TimeSyncConfig>,
1806}
1807
1808impl RouterConfig {
1809 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
1814 #[cfg(feature = "cryptography")]
1815 {
1816 RouterE2eEncryptionMode::Preferred
1817 }
1818 #[cfg(not(feature = "cryptography"))]
1819 {
1820 RouterE2eEncryptionMode::Disabled
1821 }
1822 }
1823
1824 pub fn new<H>(handlers: H) -> Self
1826 where
1827 H: Into<Arc<[EndpointHandler]>>,
1828 {
1829 let handlers: Arc<[EndpointHandler]> = handlers.into();
1830 assert!(
1831 handlers
1832 .iter()
1833 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
1834 "reserved internal endpoint handlers must not be user-registered"
1835 );
1836 Self {
1837 handlers,
1838 reliable_enabled: true,
1839 sender: None,
1840 address_mode: AddressAssignmentMode::Dynamic,
1841 address_change_handlers: Arc::from([]),
1842 e2e_encryption: Self::default_e2e_encryption_mode(),
1843 e2e_key_id: 0,
1844 memory: RuntimeMemoryConfig::default(),
1845 #[cfg(feature = "discovery")]
1846 preferred_discovery_master: None,
1847 #[cfg(feature = "timesync")]
1848 timesync: None,
1849 }
1850 }
1851
1852 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
1854 self.reliable_enabled = enabled;
1855 self
1856 }
1857
1858 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
1860 self.sender = Some(Arc::from(sender.as_ref()));
1861 self
1862 }
1863
1864 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
1866 self.with_sender(hostname)
1867 }
1868
1869 pub fn with_dynamic_address(mut self) -> Self {
1871 self.address_mode = AddressAssignmentMode::Dynamic;
1872 self
1873 }
1874
1875 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
1877 self.address_mode = AddressAssignmentMode::Requested(address);
1878 self
1879 }
1880
1881 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
1883 self.address_mode = AddressAssignmentMode::Static(address);
1884 self
1885 }
1886
1887 pub fn on_address_change<F>(mut self, f: F) -> Self
1889 where
1890 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
1891 {
1892 let mut handlers = self.address_change_handlers.to_vec();
1893 handlers.push(AddressChangeHandler {
1894 handler: Arc::new(f),
1895 });
1896 self.address_change_handlers = Arc::from(handlers);
1897 self
1898 }
1899
1900 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
1902 self.e2e_encryption = mode;
1903 self
1904 }
1905
1906 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
1908 self.e2e_key_id = key_id;
1909 self
1910 }
1911
1912 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
1914 memory.validate()?;
1915 self.memory = memory;
1916 Ok(self)
1917 }
1918
1919 #[cfg(feature = "discovery")]
1925 pub fn with_preferred_discovery_master<S: AsRef<str>>(mut self, hostname: S) -> Self {
1926 let hostname = hostname.as_ref();
1927 self.preferred_discovery_master = if hostname.is_empty() {
1928 None
1929 } else {
1930 Some(Arc::from(hostname))
1931 };
1932 self
1933 }
1934
1935 #[cfg(feature = "timesync")]
1936 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
1938 self.timesync = Some(cfg);
1939 self
1940 }
1941
1942 #[inline]
1943 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1945 if endpoint_is_router_internal(ep) {
1946 return false;
1947 }
1948 self.handlers.iter().any(|h| h.endpoint == ep)
1949 }
1950
1951 #[inline]
1952 fn reliable_enabled(&self) -> bool {
1953 self.reliable_enabled
1954 }
1955
1956 #[inline]
1957 fn sender(&self) -> String {
1958 self.sender
1959 .as_deref()
1960 .map(ToString::to_string)
1961 .unwrap_or_else(runtime_device_identifier)
1962 }
1963
1964 #[inline]
1965 fn address_mode(&self) -> AddressAssignmentMode {
1966 self.address_mode
1967 }
1968
1969 #[inline]
1970 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1971 self.e2e_encryption
1972 }
1973
1974 #[cfg(feature = "cryptography")]
1975 #[inline]
1976 fn e2e_key_id(&self) -> u32 {
1977 self.e2e_key_id
1978 }
1979
1980 #[cfg(feature = "timesync")]
1981 #[inline]
1982 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1983 self.timesync
1984 }
1985
1986 #[inline]
1987 fn memory_config(&self) -> RuntimeMemoryConfig {
1988 self.memory
1989 }
1990}
1991
1992impl Default for RouterConfig {
1993 fn default() -> Self {
1994 Self {
1995 handlers: Arc::from([]),
1996 reliable_enabled: true,
1997 sender: None,
1998 address_mode: AddressAssignmentMode::Dynamic,
1999 address_change_handlers: Arc::from([]),
2000 e2e_encryption: Self::default_e2e_encryption_mode(),
2001 e2e_key_id: 0,
2002 memory: RuntimeMemoryConfig::default(),
2003 #[cfg(feature = "discovery")]
2004 preferred_discovery_master: None,
2005 #[cfg(feature = "timesync")]
2006 timesync: None,
2007 }
2008 }
2009}
2010
2011pub trait LeBytes: Copy + Sized {
2014 const WIDTH: usize;
2015 fn write_le(self, out: &mut [u8]);
2016 fn from_le_slice(bytes: &[u8]) -> Self;
2017}
2018
2019impl_letype_num!(u8, 1);
2020impl_letype_num!(u16, 2);
2021impl_letype_num!(u32, 4);
2022impl_letype_num!(u64, 8);
2023impl_letype_num!(u128, 16);
2024impl_letype_num!(i8, 1);
2025impl_letype_num!(i16, 2);
2026impl_letype_num!(i32, 4);
2027impl_letype_num!(i64, 8);
2028impl_letype_num!(i128, 16);
2029impl_letype_num!(f32, 4);
2030impl_letype_num!(f64, 8);
2031
2032pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
2034 let total = data.len() * T::WIDTH;
2035 let mut buf = vec![0u8; total];
2036
2037 for (i, v) in data.iter().copied().enumerate() {
2038 let start = i * T::WIDTH;
2039 v.write_le(&mut buf[start..start + T::WIDTH]);
2040 }
2041
2042 Arc::from(buf)
2043}
2044
2045fn make_error_payload(msg: &str) -> crate::SharedBytes {
2048 let meta = message_meta(DataType::TelemetryError);
2049 match meta.element {
2050 MessageElement::Static(_, _, _) => {
2051 let max = get_needed_message_size(DataType::TelemetryError);
2052 let bytes = msg.as_bytes();
2053 let n = core::cmp::min(max, bytes.len());
2054 let mut buf = vec![0u8; max];
2055 if n > 0 {
2056 buf[..n].copy_from_slice(&bytes[..n]);
2057 }
2058 crate::shared_bytes::convert(Arc::<[u8]>::from(buf))
2059 }
2060 MessageElement::Dynamic(_, _) => {
2061 crate::shared_bytes::convert(Arc::<[u8]>::from(msg.as_bytes()))
2062 }
2063 }
2064}
2065
2066fn log_raw<T, F>(
2070 sender: Arc<str>,
2071 ty: DataType,
2072 data: &[T],
2073 timestamp: u64,
2074 mut tx_function: F,
2075) -> TelemetryResult<()>
2076where
2077 T: LeBytes,
2078 F: FnMut(Packet) -> TelemetryResult<()>,
2079{
2080 let meta = message_meta(ty);
2081 let got = data
2082 .len()
2083 .checked_mul(T::WIDTH)
2084 .ok_or(TelemetryError::PacketTooLarge(
2085 "typed payload size overflow",
2086 ))?;
2087 if T::WIDTH == 0 {
2088 return Err(TelemetryError::BadArg);
2089 }
2090
2091 match meta.element {
2092 MessageElement::Static(_, _, _) => {
2093 let expected = crate::data_type_size(meta.element.data_type()) * meta.element;
2094 if got != expected {
2095 return Err(TelemetryError::SizeMismatch { expected, got });
2096 }
2097 }
2098 MessageElement::Dynamic(_, _) => {
2099 }
2102 }
2103
2104 const INLINE_ENCODE_BYTES: usize = if crate::config::STACK_PAYLOAD_SIZE < u8::MAX as usize {
2105 crate::config::STACK_PAYLOAD_SIZE
2106 } else {
2107 u8::MAX as usize
2108 };
2109 let payload = if got <= INLINE_ENCODE_BYTES {
2110 let mut bytes = [0u8; INLINE_ENCODE_BYTES];
2113 for (value, destination) in data
2114 .iter()
2115 .copied()
2116 .zip(bytes[..got].chunks_exact_mut(T::WIDTH))
2117 {
2118 value.write_le(destination);
2119 }
2120 crate::config::StandardSmallPayload::new(&bytes[..got])
2121 } else {
2122 crate::config::StandardSmallPayload::from_arc(encode_slice_le(data))
2123 };
2124 let pkt = Packet::new_local(ty, meta.endpoints_owned(), sender, timestamp, payload)?;
2125 tx_function(pkt)
2126}
2127
2128#[cfg(feature = "std")]
2130fn reliable_trace_enabled() -> bool {
2131 static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
2132 *ENABLED.get_or_init(|| {
2133 matches!(
2134 std::env::var("SEDSNET_TRACE_RELIABLE").as_deref(),
2135 Ok("1" | "true")
2136 )
2137 })
2138}
2139
2140fn fallback_stdout(_msg: &str) {
2144 #[cfg(feature = "std")]
2145 {
2146 eprintln!("{}", _msg);
2147 }
2148 #[cfg(all(not(feature = "std"), target_os = "none"))]
2149 {
2150 unsafe {
2153 seds_error_msg(_msg.as_ptr(), _msg.len());
2154 }
2155 }
2156}
2157
2158#[derive(Debug, Clone)]
2163struct RouterInner {
2164 #[cfg(feature = "compression")]
2165 codec_reserved_bytes: usize,
2166 memory: RuntimeMemoryConfig,
2167 sides: Vec<Option<RouterSide>>,
2168 route_overrides: RouterMap<(Option<RouterSideId>, RouterSideId), bool>,
2169 typed_route_overrides: RouterMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
2170 route_weights: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2171 route_priorities: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2172 source_route_modes: RouterMap<Option<RouterSideId>, RouteSelectionMode>,
2173 route_selection_cursors: RouterMap<Option<RouterSideId>, u64>,
2174 adaptive_route_stats: RouterMap<RouterSideId, AdaptiveRouteStats>,
2175 side_runtime_stats: RouterMap<RouterSideId, Box<SideRuntimeStatsInner>>,
2178 side_transport: RouterMap<RouterSideId, Box<SideTransportState>>,
2179 managed_variable_types: BTreeSet<u32>,
2180 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
2181 managed_variable_latest: BTreeMap<u32, Box<ManagedVariableCacheEntry>>,
2182 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
2183 local_address: AddressBookEntry,
2184 address_book: BTreeMap<String, AddressBookEntry>,
2185 address_by_value: BTreeMap<NodeAddress, String>,
2186 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
2187 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
2188 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
2189 next_p2p_stream_id: P2pStreamId,
2190 received_queue: BoundedDeque<RouterRxItem>,
2191 transmit_queue: BoundedDeque<TxQueued>,
2192 tx_priority_burst: u8,
2193 has_ordered_retry: bool,
2194 ordered_claim_seq: u32,
2195 rx_priority_burst: u8,
2196 recent_rx: BoundedDeque<u64>,
2197 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
2198 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
2199 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
2200 reliable_return_route_order: VecDeque<u64>,
2201 end_to_end_reliable_tx: BTreeMap<u64, EndToEndReliableSent>,
2202 end_to_end_reliable_tx_order: VecDeque<u64>,
2203 total_handler_failures: u64,
2204 total_handler_retries: u64,
2205 #[cfg(feature = "discovery")]
2206 discovery_routes: RouterMap<RouterSideId, DiscoverySideState>,
2207 #[cfg(feature = "discovery")]
2208 discovery_cadence: DiscoveryCadenceState,
2209 #[cfg(feature = "discovery")]
2210 discovery_side_throttle: RouterMap<RouterSideId, DiscoverySideThrottleState>,
2211 #[cfg(feature = "discovery")]
2212 preferred_discovery_master: Option<Arc<str>>,
2213 #[cfg(all(feature = "discovery", feature = "timesync"))]
2214 timesync_side_throttle: RouterMap<RouterSideId, TimeSyncSideThrottleState>,
2215}
2216
2217#[derive(Debug, Clone)]
2218struct ManagedVariableCacheEntry {
2219 packet: Packet,
2220 sender_address: u32,
2221 #[cfg(feature = "discovery")]
2222 cached_at_ms: u64,
2223}
2224
2225type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
2226
2227#[derive(Clone)]
2228struct NetworkVariableUpdateHandler {
2229 handler: Arc<NetworkVariableUpdateFn>,
2230}
2231
2232impl Debug for NetworkVariableUpdateHandler {
2233 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2234 f.write_str("NetworkVariableUpdateHandler(<handler>)")
2235 }
2236}
2237
2238#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2240pub struct NetworkVariablePermissions {
2241 pub read: bool,
2242 pub write: bool,
2243}
2244
2245impl NetworkVariablePermissions {
2246 pub const NONE: Self = Self {
2247 read: false,
2248 write: false,
2249 };
2250 pub const READ_ONLY: Self = Self {
2251 read: true,
2252 write: false,
2253 };
2254 pub const WRITE_ONLY: Self = Self {
2255 read: false,
2256 write: true,
2257 };
2258 pub const READ_WRITE: Self = Self {
2259 read: true,
2260 write: true,
2261 };
2262}
2263
2264#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2265enum RouterQueueKind {
2266 Received,
2267 Transmit,
2268}
2269
2270#[cfg(feature = "compression")]
2271impl crate::payload_codec::QueueBudget for RouterInner {
2272 fn codec_reserved(&mut self) -> &mut usize {
2273 &mut self.codec_reserved_bytes
2274 }
2275 fn codec_used(&self) -> usize {
2276 self.shared_queue_bytes_used()
2277 }
2278 fn codec_limit(&self) -> usize {
2279 self.memory.max_queue_budget
2280 }
2281}
2282
2283impl RouterInner {
2284 #[cfg(all(feature = "compression", feature = "discovery"))]
2285 fn reclaim_codec_metadata(&mut self) -> bool {
2286 self.side_transport.values_mut().any(|transport| {
2287 transport
2288 .payload_codec
2289 .evict_for_budget(DISCOVERY_ROUTE_TTL_MS)
2290 })
2291 }
2292
2293 #[cfg(feature = "compression")]
2298 fn prepare_codec_queue_owner(&mut self, side: Option<usize>, data: &mut RouterItem) {
2299 let RouterItem::Packed(bytes) = data else {
2300 return;
2301 };
2302 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
2303 return;
2304 }
2305 let capable = {
2306 #[cfg(feature = "discovery")]
2307 {
2308 side.and_then(|side| self.side_transport.get(&side))
2309 .is_some_and(|transport| {
2310 transport.payload_codec.retention_can_stay_compressed()
2311 })
2312 }
2313 #[cfg(not(feature = "discovery"))]
2314 {
2315 let _ = side;
2316 false
2317 }
2318 };
2319 if capable {
2320 return;
2321 }
2322 let Ok(required) = wire_format::decoded_frame_upper_bound(bytes.as_ref()) else {
2323 return;
2324 };
2325 if self
2326 .shared_queue_bytes_used()
2327 .checked_add(required)
2328 .and_then(|used| {
2329 used.checked_add(
2330 size_of::<crate::SharedBytes>()
2331 + size_of::<bool>()
2332 + size_of::<u8>()
2333 + size_of::<u64>(),
2334 )
2335 })
2336 .is_none_or(|used| used > self.memory.max_queue_budget)
2337 {
2338 return;
2339 }
2340 if let Ok(Some(raw)) = wire_format::payload_encoding_for_retention(
2341 bytes.as_ref(),
2342 self.memory.max_queue_budget,
2343 ) {
2344 *bytes = crate::shared_bytes::convert(raw);
2345 }
2346 }
2347
2348 fn retain_transmit<F>(&mut self, keep: F)
2349 where
2350 F: FnMut(&TxQueued) -> bool,
2351 {
2352 let hop = &mut self.reliable_tx;
2353 let end = &mut self.end_to_end_reliable_tx;
2354 self.transmit_queue.retain_observed(keep, |evicted| {
2355 if let Some(key) = evicted.item.replay_marker_key() {
2356 Self::set_replay_queued(hop, end, key, false);
2357 }
2358 });
2359 }
2360
2361 #[inline(never)]
2362 fn claim_index(&self, token: u32) -> Option<usize> {
2363 self.transmit_queue.iter().position(|item| {
2364 matches!(
2365 item.delivery_state,
2366 QueuedDelivery::ClaimedOrdered | QueuedDelivery::ClaimedUnordered
2367 ) && item.deferred_since_ms == u64::from(token)
2368 })
2369 }
2370
2371 #[inline(never)]
2372 fn refresh_retry_hint(&mut self) {
2373 self.has_ordered_retry = self.transmit_queue.iter().any(|item| {
2374 matches!(
2375 item.delivery_state,
2376 QueuedDelivery::ClaimedOrdered
2377 | QueuedDelivery::ClaimedUnordered
2378 | QueuedDelivery::DeferredOrdered
2379 )
2380 });
2381 }
2382
2383 fn cancel_accepted_type(&mut self, ty: u32) {
2384 self.retain_transmit(|item| {
2385 !matches!(&item.item,
2386 RouterTxItem::Broadcast(_) if item.accepted_type == ty)
2387 });
2388 }
2389
2390 fn replay_is_queued(&self, key: ReplayMarkerKey) -> Option<bool> {
2391 match key {
2392 ReplayMarkerKey::EndToEnd(id) => self.end_to_end_reliable_tx.get(&id).map(|s| s.queued),
2393 ReplayMarkerKey::Hop(side, ty, seq) => self
2394 .reliable_tx
2395 .get(&(side, ty.as_u32()))
2396 .and_then(|tx| tx.sent.get(&seq))
2397 .map(|s| s.queued),
2398 }
2399 }
2400 fn set_replay_queued(
2401 hop: &mut BTreeMap<(RouterSideId, u32), ReliableTxState>,
2402 end: &mut BTreeMap<u64, EndToEndReliableSent>,
2403 key: ReplayMarkerKey,
2404 queued: bool,
2405 ) {
2406 let flag = match key {
2407 ReplayMarkerKey::EndToEnd(id) => end.get_mut(&id).map(|s| &mut s.queued),
2408 ReplayMarkerKey::Hop(side, ty, seq) => hop
2409 .get_mut(&(side, ty.as_u32()))
2410 .and_then(|tx| tx.sent.get_mut(&seq))
2411 .map(|s| &mut s.queued),
2412 };
2413 if let Some(flag) = flag {
2414 *flag = queued;
2415 }
2416 }
2417
2418 #[cfg(any(feature = "std", feature = "fair-admission"))]
2419 fn fair_bulk_owner(data: &RouterItem, priority: u8) -> Option<u32> {
2420 match data {
2421 RouterItem::Packet(packet) => crate::admission_fairness::packet_owner(packet, priority),
2422 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2423 crate::admission_fairness::packed_owner(view, priority)
2424 })
2425 .ok()
2426 .flatten(),
2427 }
2428 }
2429 #[cfg(any(feature = "std", feature = "fair-admission"))]
2430 fn tx_bulk_owner(item: &TxQueued) -> Option<u32> {
2431 match &item.item {
2432 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2433 Self::fair_bulk_owner(data, item.priority)
2434 }
2435 _ => None,
2436 }
2437 }
2438 #[cfg(any(feature = "std", feature = "fair-admission"))]
2439 fn bulk_stream(data: &RouterItem) -> Option<u32> {
2440 match data {
2441 RouterItem::Packet(packet) => Some(packet.data_type().as_u32()),
2442 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2443 wire_format::scheduling_type(view).map(|ty| ty.as_u32())
2444 })
2445 .ok()
2446 .flatten(),
2447 }
2448 }
2449 #[cfg(any(feature = "std", feature = "fair-admission"))]
2450 fn tx_bulk_stream(item: &TxQueued) -> Option<u32> {
2451 match &item.item {
2452 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2453 Self::bulk_stream(data)
2454 }
2455 _ => None,
2456 }
2457 }
2458 #[cfg(any(feature = "std", feature = "fair-admission"))]
2459 fn bulk_loads(&self, priority: u8) -> crate::admission_fairness::Loads {
2460 let mut loads = crate::admission_fairness::Loads::new();
2461 for item in self.received_queue.iter() {
2462 if item.priority <= priority
2463 && let Some(owner) = Self::fair_bulk_owner(&item.data, item.priority)
2464 {
2465 loads.record(owner, item.byte_cost());
2466 }
2467 }
2468 for item in self.transmit_queue.iter() {
2469 if item.priority <= priority
2470 && let Some(owner) = Self::tx_bulk_owner(item)
2471 {
2472 loads.record(owner, item.byte_cost());
2473 }
2474 }
2475 loads
2476 }
2477 #[cfg(any(feature = "std", feature = "fair-admission"))]
2478 fn competing_bulk_owner(&self, incoming: u32, priority: u8) -> Option<u32> {
2479 self.bulk_loads(priority).dominant_other(incoming)
2480 }
2481 #[cfg(any(feature = "std", feature = "fair-admission"))]
2482 fn reclaim_bulk_from(
2483 &mut self,
2484 owner: u32,
2485 priority: u8,
2486 incoming_stream: Option<u32>,
2487 ) -> bool {
2488 let mut streams = crate::admission_fairness::Loads::new();
2489 for item in self.received_queue.iter() {
2490 if item.priority <= priority
2491 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2492 && let Some(ty) = Self::bulk_stream(&item.data)
2493 {
2494 streams.record(ty, item.byte_cost());
2495 }
2496 }
2497 for item in self.transmit_queue.iter() {
2498 if item.priority <= priority
2499 && Self::tx_bulk_owner(item) == Some(owner)
2500 && let Some(ty) = Self::tx_bulk_stream(item)
2501 {
2502 streams.record(ty, item.byte_cost());
2503 }
2504 }
2505 let stream = streams.queue_victim(incoming_stream);
2506 let rx = self
2507 .received_queue
2508 .iter()
2509 .enumerate()
2510 .filter(|(_, item)| {
2511 item.priority <= priority
2512 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2513 && stream.is_none_or(|ty| Self::bulk_stream(&item.data) == Some(ty))
2514 })
2515 .min_by_key(|(index, item)| (item.priority, *index))
2516 .map(|(index, item)| (index, item.priority));
2517 let tx = self
2518 .transmit_queue
2519 .iter()
2520 .enumerate()
2521 .filter(|(_, item)| {
2522 item.priority <= priority
2523 && Self::tx_bulk_owner(item) == Some(owner)
2524 && stream.is_none_or(|ty| Self::tx_bulk_stream(item) == Some(ty))
2525 })
2526 .min_by_key(|(index, item)| (item.priority, *index))
2527 .map(|(index, item)| (index, item.priority));
2528 match (rx, tx) {
2529 (Some((index, rp)), Some((_, tp))) if rp <= tp => {
2530 self.received_queue.remove_pos(index).is_some()
2531 }
2532 (Some((index, _)), None) => self.received_queue.remove_pos(index).is_some(),
2533 (_, Some((index, _))) => self.transmit_queue.remove_pos(index).is_some(),
2534 _ => false,
2535 }
2536 }
2537
2538 #[cfg(feature = "discovery")]
2539 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
2540 board
2541 .sender_id
2542 .len()
2543 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
2544 .saturating_add(
2545 board
2546 .reachable_timesync_sources
2547 .iter()
2548 .map(|s| s.len())
2549 .sum::<usize>(),
2550 )
2551 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
2552 }
2553
2554 #[cfg(feature = "discovery")]
2555 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
2556 sender
2557 .len()
2558 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
2559 .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
2560 .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
2561 .saturating_add(
2562 state
2563 .reachable_timesync_sources
2564 .iter()
2565 .map(|s| s.len())
2566 .sum::<usize>(),
2567 )
2568 .saturating_add(
2569 state
2570 .advertised_reachable_timesync_sources
2571 .iter()
2572 .map(|s| s.len())
2573 .sum::<usize>(),
2574 )
2575 .saturating_add(
2576 state
2577 .topology_boards
2578 .iter()
2579 .map(Self::topology_board_byte_cost)
2580 .sum::<usize>(),
2581 )
2582 .saturating_add(size_of::<DiscoverySenderState>())
2583 }
2584
2585 #[cfg(feature = "discovery")]
2586 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
2587 size_of::<RouterSideId>()
2588 .saturating_add(size_of::<DiscoverySideState>())
2589 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
2590 .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
2591 .saturating_add(
2592 route.requested_network_variables.len()
2593 * (size_of::<DataType>() + size_of::<u64>()),
2594 )
2595 .saturating_add(
2596 route
2597 .reachable_timesync_sources
2598 .iter()
2599 .map(|s| s.len())
2600 .sum::<usize>(),
2601 )
2602 .saturating_add(
2603 route
2604 .announcers
2605 .iter()
2606 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
2607 .sum::<usize>(),
2608 )
2609 .saturating_add(side.saturating_sub(side))
2610 }
2611
2612 #[cfg(feature = "discovery")]
2613 fn discovery_bytes_used(&self) -> usize {
2614 self.discovery_routes
2615 .iter()
2616 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
2617 .sum()
2618 }
2619
2620 #[inline]
2621 fn reliable_rx_buffered_bytes(&self) -> usize {
2622 self.reliable_rx
2623 .values()
2624 .flat_map(|state| state.buffered.values())
2625 .map(|bytes| size_of::<crate::SharedBytes>() + bytes.len())
2626 .sum()
2627 }
2628
2629 fn reliable_rx_buffer_len(&self) -> usize {
2630 self.reliable_rx
2631 .values()
2632 .map(|state| state.buffered.len())
2633 .sum()
2634 }
2635
2636 #[inline(never)]
2637 fn side_chunk_bytes_used(&self) -> usize {
2638 self.side_transport
2639 .values()
2640 .map(|side| side.rx_chunk_bytes)
2641 .sum()
2642 }
2643 fn discard_side_chunk(&mut self, side: RouterSideId, id: u32) -> bool {
2644 self.side_transport
2645 .get_mut(&side)
2646 .is_some_and(|state| state.take_chunk(id, true).is_some())
2647 }
2648 fn reclaim_oldest_side_chunk(&mut self, keep: Option<(RouterSideId, u32)>) -> bool {
2649 let mut victim = None;
2650 let mut time = u64::MAX;
2651 for (&side, state) in &self.side_transport {
2652 let preserved = match keep {
2653 Some((owner, id)) if owner == side => Some(id),
2654 _ => None,
2655 };
2656 if let Some((id, seen)) = SideChunkAssembly::oldest(&state.rx_chunks, preserved)
2657 && (victim.is_none() || seen < time)
2658 {
2659 time = seen;
2660 victim = Some((side, id));
2661 }
2662 }
2663 match victim {
2664 Some((side, id)) => self.discard_side_chunk(side, id),
2665 None => false,
2666 }
2667 }
2668 fn make_side_chunk_room(
2669 &mut self,
2670 side: RouterSideId,
2671 id: u32,
2672 index: u16,
2673 total: u16,
2674 payload_len: usize,
2675 ) -> TelemetryResult<()> {
2676 let budget = self
2677 .memory
2678 .max_queue_budget
2679 .saturating_sub(self.protected_metadata_bytes_used());
2680 let state = self
2681 .side_transport
2682 .get_mut(&side)
2683 .ok_or(TelemetryError::BadArg)?;
2684 let existing = state.rx_chunks.get(&id).map(|a| a.total);
2685 if existing.is_some_and(|old| old != 0 && old != total) {
2686 state.take_chunk(id, true);
2687 return Err(TelemetryError::Unpack("side chunk total mismatch"));
2688 }
2689 let before = state.rx_chunks.len();
2690 let room = SideChunkAssembly::make_room(
2691 &mut state.rx_chunks,
2692 &mut state.rx_chunk_bytes,
2693 id,
2694 index,
2695 total,
2696 payload_len,
2697 budget,
2698 );
2699 if before != state.rx_chunks.len() {
2700 state.rx_chunk_reclaims = state
2701 .rx_chunk_reclaims
2702 .saturating_add((before - state.rx_chunks.len()) as u32);
2703 }
2704 let cost = room?;
2705 if cost == 0 {
2706 return Ok(());
2707 }
2708 if existing.is_none()
2709 && state.rx_chunks.len() >= 4
2710 && let Some((oldest, _)) = SideChunkAssembly::oldest(&state.rx_chunks, None)
2711 {
2712 state.take_chunk(oldest, true);
2713 }
2714 while self.shared_queue_bytes_used().saturating_add(cost) > self.memory.max_queue_budget {
2715 #[cfg(any(feature = "std", feature = "fair-admission"))]
2716 if let Some(owner) = self.bulk_loads(199).queue_victim(None)
2717 && self.reclaim_bulk_from(owner, 199, None)
2718 {
2719 continue;
2720 }
2721 if self.reclaim_oldest_side_chunk(Some((side, id))) {
2722 continue;
2723 }
2724 return Err(TelemetryError::Io("memory pressure"));
2727 }
2728 Ok(())
2729 }
2730
2731 #[inline(never)]
2732 fn protected_metadata_bytes_used(&self) -> usize {
2733 let codec_bytes = {
2734 #[cfg(all(feature = "compression", feature = "discovery"))]
2735 {
2736 self.side_transport
2737 .values()
2738 .map(|side| side.payload_codec.heap_bytes())
2739 .sum::<usize>()
2740 }
2741 #[cfg(not(all(feature = "compression", feature = "discovery")))]
2742 {
2743 0usize
2744 }
2745 };
2746 codec_bytes.saturating_add(
2747 self.recent_rx
2748 .max_bytes()
2749 .saturating_add(self.reliable_rx_buffered_bytes())
2750 .saturating_add(crate::config::schema_bytes_used())
2751 .saturating_add({
2752 #[cfg(feature = "discovery")]
2753 {
2754 self.discovery_bytes_used()
2755 }
2756 #[cfg(not(feature = "discovery"))]
2757 {
2758 0
2759 }
2760 }),
2761 )
2762 }
2763 #[inline]
2764 fn shared_queue_bytes_used(&self) -> usize {
2765 self.received_queue
2766 .bytes_used()
2767 .saturating_add(self.transmit_queue.bytes_used())
2768 .saturating_add(self.side_chunk_bytes_used())
2769 .saturating_add(self.protected_metadata_bytes_used())
2770 .saturating_add({
2771 #[cfg(feature = "compression")]
2772 {
2773 self.codec_reserved_bytes
2774 }
2775 #[cfg(not(feature = "compression"))]
2776 {
2777 0
2778 }
2779 })
2780 }
2781
2782 fn classify_transmit_disposable(item: &TxQueued) -> bool {
2786 let data = match &item.item {
2787 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => data,
2788 _ => return true,
2789 };
2790 match data {
2791 RouterItem::Packet(pkt) => {
2792 is_internal_control_type(pkt.data_type())
2793 || (!is_reliable_type(pkt.data_type()) && pkt.wire_target_senders().is_empty())
2794 }
2795 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2796 wire_format::peek_routing_frame_info(view).map_or(true, |frame| {
2797 is_internal_control_type(frame.envelope.ty)
2798 || (!is_reliable_type(frame.envelope.ty)
2799 && frame.reliable.is_none()
2800 && frame.envelope.target_senders().len() == 0)
2801 })
2802 })
2803 .unwrap_or(false),
2804 }
2805 }
2806
2807 fn transmit_eviction_candidate(queue: &BoundedDeque<TxQueued>) -> Option<usize> {
2808 queue
2809 .iter()
2810 .enumerate()
2811 .filter(|(_, queued)| queued.disposable)
2812 .min_by_key(|(index, queued)| (queued.priority, core::cmp::Reverse(*index)))
2813 .map(|(index, _)| index)
2814 }
2815
2816 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
2817 match preferred {
2818 RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
2823 RouterQueueKind::Transmit => {
2824 let Some(index) = Self::transmit_eviction_candidate(&self.transmit_queue) else {
2825 return false;
2826 };
2827 let Some(item) = self.transmit_queue.remove_pos(index) else {
2828 return false;
2829 };
2830 if let Some(key) = item.item.replay_marker_key() {
2831 Self::set_replay_queued(
2832 &mut self.reliable_tx,
2833 &mut self.end_to_end_reliable_tx,
2834 key,
2835 false,
2836 );
2837 }
2838 true
2839 }
2840 }
2841 }
2842
2843 fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
2844 let received = self
2845 .received_queue
2846 .lowest_priority(|queued| queued.priority)
2847 .map(|priority| (RouterQueueKind::Received, priority));
2848 let transmit = self
2849 .transmit_queue
2850 .iter()
2851 .filter(|queued| queued.disposable)
2852 .map(|queued| queued.priority)
2853 .min()
2854 .map(|priority| (RouterQueueKind::Transmit, priority));
2855 [received, transmit]
2856 .into_iter()
2857 .flatten()
2858 .filter(|(_, priority)| {
2859 *priority < incoming_priority
2860 || (incoming_priority != 254 && *priority == incoming_priority)
2861 })
2862 .min_by_key(|(kind, priority)| {
2863 (
2864 *priority,
2865 if *kind == RouterQueueKind::Received {
2866 0
2867 } else {
2868 1
2869 },
2870 )
2871 })
2872 .map(|(kind, _)| kind)
2873 }
2874
2875 fn make_shared_queue_room(
2876 &mut self,
2877 incoming_cost: usize,
2878 incoming_priority: u8,
2879 _incoming: Option<&RouterItem>,
2880 ) -> TelemetryResult<()> {
2881 if incoming_cost > self.memory.max_queue_budget {
2882 return Err(TelemetryError::PacketTooLarge(
2883 "Item exceeds maximum shared queue budget",
2884 ));
2885 }
2886
2887 #[cfg(any(feature = "std", feature = "fair-admission"))]
2888 let mut fair_victim = None;
2889 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
2890 > self.memory.max_queue_budget
2891 {
2892 #[cfg(all(feature = "compression", feature = "discovery"))]
2893 if self.reclaim_codec_metadata() {
2894 continue;
2895 }
2896
2897 #[cfg(any(feature = "std", feature = "fair-admission"))]
2898 {
2899 let incoming =
2900 _incoming.and_then(|data| Self::fair_bulk_owner(data, incoming_priority));
2901 let victim = *fair_victim.get_or_insert_with(|| {
2902 self.bulk_loads(incoming_priority.min(199))
2903 .queue_victim(incoming)
2904 });
2905 if let Some(victim) = victim
2906 && self.reclaim_bulk_from(
2907 victim,
2908 incoming_priority.min(199),
2909 (incoming == Some(victim))
2910 .then(|| _incoming.and_then(Self::bulk_stream))
2911 .flatten(),
2912 )
2913 {
2914 continue;
2915 }
2916 if incoming.is_some() {
2917 return Err(TelemetryError::Io("shared fair queue saturated"));
2918 }
2919 }
2920 let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
2921 if incoming_priority >= 200 && self.reclaim_oldest_side_chunk(None) {
2922 continue;
2923 }
2924 return Err(TelemetryError::Io("shared priority queue saturated"));
2925 };
2926 if !self.pop_shared_queue_item(victim) {
2927 return Err(TelemetryError::PacketTooLarge(
2928 "Item exceeds maximum shared queue budget",
2929 ));
2930 }
2931 }
2932
2933 Ok(())
2934 }
2935
2936 #[cfg(feature = "discovery")]
2937 fn schema_growth_budget(&self) -> usize {
2938 let protected = self
2941 .recent_rx
2942 .max_bytes()
2943 .saturating_add(self.discovery_bytes_used())
2944 .saturating_add(self.reliable_rx_buffered_bytes())
2945 .saturating_add({
2946 #[cfg(feature = "compression")]
2947 {
2948 self.codec_reserved_bytes
2949 }
2950 #[cfg(not(feature = "compression"))]
2951 {
2952 0
2953 }
2954 });
2955 self.memory.max_queue_budget.saturating_sub(protected)
2956 }
2957
2958 #[cfg(feature = "discovery")]
2959 fn commit_discovery_route(
2960 &mut self,
2961 side: RouterSideId,
2962 route: DiscoverySideState,
2963 ) -> TelemetryResult<()> {
2964 let previous = self
2965 .discovery_routes
2966 .get(&side)
2967 .map_or(0, |route| Self::discovery_route_byte_cost(side, route));
2968 let incoming = Self::discovery_route_byte_cost(side, &route);
2969 let growth = incoming.saturating_sub(previous);
2970 self.make_shared_queue_room(growth, 255, None)?;
2971 self.discovery_routes.insert(side, route);
2972 Ok(())
2973 }
2974
2975 #[cfg(feature = "discovery")]
2976 fn fit_discovery_budget(&mut self) {
2977 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
2978 #[cfg(all(feature = "compression", feature = "discovery"))]
2979 if self.reclaim_codec_metadata() {
2980 continue;
2981 }
2982
2983 if let Some(victim) = self.lower_priority_shared_queue(255)
2986 && self.pop_shared_queue_item(victim)
2987 {
2988 continue;
2989 }
2990 break;
2993 }
2994 }
2995
2996 fn reliable_original_type(item: &TxQueued) -> Option<DataType> {
2997 let RouterTxItem::Broadcast(data) = &item.item else {
2998 return None;
2999 };
3000 let _ = data;
3001 let ty = DataType(item.accepted_type);
3002 (item.delivery_state != QueuedDelivery::Untracked).then_some(ty)
3003 }
3004
3005 fn ordered_original_type(item: &TxQueued) -> Option<DataType> {
3006 let RouterTxItem::Broadcast(data) = &item.item else {
3007 return None;
3008 };
3009 let _ = data;
3010 let ty = DataType(item.accepted_type);
3011 matches!(
3012 item.delivery_state,
3013 QueuedDelivery::Ordered
3014 | QueuedDelivery::OrderedTail
3015 | QueuedDelivery::DeferredOrdered
3016 | QueuedDelivery::ClaimedOrdered
3017 )
3018 .then_some(ty)
3019 }
3020
3021 fn pop_received_fair(&mut self) -> Option<RouterRxItem> {
3022 self.received_queue.pop_front_bulk_fair(
3023 &mut self.rx_priority_burst,
3024 MAX_PRIORITY_BURST,
3025 |item| item.priority,
3026 )
3027 }
3028
3029 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
3030 self.make_shared_queue_room(item.byte_cost(), item.priority, Some(&item.data))?;
3031 #[cfg(any(feature = "std", feature = "fair-admission"))]
3032 {
3033 #[cfg(any(feature = "std", feature = "fair-admission"))]
3034 let mut fair_victim = None;
3035 self.received_queue.push_back_prioritized_select(
3036 item,
3037 |queued| queued.priority,
3038 |queue, _incoming| {
3039 #[cfg(any(feature = "std", feature = "fair-admission"))]
3040 {
3041 crate::admission_fairness::queue_victim(
3042 queue,
3043 _incoming,
3044 |item| Self::fair_bulk_owner(&item.data, item.priority),
3045 |item| item.priority,
3046 |item| Self::bulk_stream(&item.data),
3047 &mut fair_victim,
3048 )
3049 }
3050 },
3051 core::mem::drop,
3052 )
3053 }
3054 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3055 {
3056 self.received_queue
3057 .push_back_prioritized(item, |queued| queued.priority)
3058 }
3059 }
3060
3061 fn push_transmit(&mut self, mut item: TxQueued) -> TelemetryResult<()> {
3062 #[cfg(feature = "compression")]
3063 match &mut item.item {
3064 RouterTxItem::Broadcast(data) => self.prepare_codec_queue_owner(None, data),
3065 RouterTxItem::ToSide { dst, data, .. } => {
3066 self.prepare_codec_queue_owner(Some(*dst), data)
3067 }
3068 _ => {}
3069 }
3070 if let RouterTxItem::Broadcast(data) = &item.item {
3071 item.accepted_type = match data {
3072 RouterItem::Packet(packet) => packet.data_type().as_u32(),
3073 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3074 wire_format::scheduling_type(view)
3075 })
3076 .ok()
3077 .flatten()
3078 .map_or(0, |ty| ty.as_u32()),
3079 };
3080 }
3081 if item.delivery_state == QueuedDelivery::Untracked
3082 && matches!(&item.item, RouterTxItem::Broadcast(_))
3083 {
3084 item.delivery_state = match reliable_mode(DataType(item.accepted_type)) {
3085 crate::ReliableMode::Ordered => QueuedDelivery::Ordered,
3086 crate::ReliableMode::Unordered => QueuedDelivery::Unordered,
3087 crate::ReliableMode::None => QueuedDelivery::Untracked,
3088 };
3089 }
3090 if matches!(
3093 item.delivery_state,
3094 QueuedDelivery::Ordered | QueuedDelivery::DeferredOrdered
3095 ) && let Some(priority) = self.transmit_queue.iter().find_map(|queued| {
3096 (Self::ordered_original_type(queued).is_some()
3097 && queued.accepted_type == item.accepted_type)
3098 .then_some(queued.priority)
3099 }) {
3100 item.priority = priority;
3101 item.delivery_state = QueuedDelivery::OrderedTail;
3102 }
3103 item.disposable = if item.delivery_state != QueuedDelivery::Untracked {
3104 false
3105 } else {
3106 Self::classify_transmit_disposable(&item)
3107 };
3108 let ordered_retry = Self::ordered_original_type(&item).is_some()
3109 && item.delivery_state == QueuedDelivery::DeferredOrdered;
3110 let replay = item.item.replay_marker_key();
3111 if replay.is_some_and(|key| self.replay_is_queued(key).is_none_or(|queued| queued)) {
3114 return Ok(());
3115 }
3116 #[cfg(feature = "discovery")]
3117 if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
3118 if let RouterItem::Packet(incoming) = data
3121 && self.transmit_queue.iter().any(|queued| {
3122 matches!(&queued.item,
3123 RouterTxItem::ToSide { dst: queued_dst, data: RouterItem::Packet(existing), .. }
3124 if queued_dst == dst && discovery::same_schema_snapshot(existing, incoming))
3125 })
3126 {
3127 return Ok(());
3128 }
3129 let incoming = match data {
3130 RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
3131 Some((pkt.data_type(), pkt.sender().to_owned()))
3132 }
3133 RouterItem::Packet(_) | RouterItem::Packed(_) => None,
3134 };
3135 if let Some((incoming_ty, incoming_sender)) = incoming {
3136 self.transmit_queue.retain(|queued| {
3137 let RouterTxItem::ToSide {
3138 dst: queued_dst,
3139 data: queued_data,
3140 ..
3141 } = &queued.item
3142 else {
3143 return true;
3144 };
3145 let same_snapshot = match queued_data {
3146 RouterItem::Packet(pkt) => {
3147 pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
3148 }
3149 RouterItem::Packed(_) => false,
3150 };
3151 *queued_dst != *dst || !same_snapshot
3152 });
3153 }
3154 }
3155 self.make_shared_queue_room(
3156 item.byte_cost(),
3157 item.priority,
3158 match &item.item {
3159 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => Some(data),
3160 _ => None,
3161 },
3162 )?;
3163 let reliable_tx = &mut self.reliable_tx;
3164 let end_to_end = &mut self.end_to_end_reliable_tx;
3165 let result = {
3166 #[cfg(any(feature = "std", feature = "fair-admission"))]
3167 {
3168 #[cfg(any(feature = "std", feature = "fair-admission"))]
3169 let mut fair_victim = None;
3170 self.transmit_queue.push_back_prioritized_select(
3171 item,
3172 |queued| queued.priority,
3173 |queue, _incoming| {
3174 #[cfg(any(feature = "std", feature = "fair-admission"))]
3175 {
3176 if Self::tx_bulk_owner(_incoming).is_some() {
3177 crate::admission_fairness::queue_victim(
3178 queue,
3179 _incoming,
3180 Self::tx_bulk_owner,
3181 |item| item.priority,
3182 Self::tx_bulk_stream,
3183 &mut fair_victim,
3184 )
3185 } else {
3186 Self::transmit_eviction_candidate(queue)
3187 }
3188 }
3189 },
3190 |evicted: TxQueued| {
3191 if let Some(key) = evicted.item.replay_marker_key() {
3192 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3193 }
3194 },
3195 )
3196 }
3197 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3198 {
3199 self.transmit_queue.push_back_prioritized_select(
3200 item,
3201 |queued| queued.priority,
3202 |queue, _| Self::transmit_eviction_candidate(queue),
3203 |evicted: TxQueued| {
3204 if let Some(key) = evicted.item.replay_marker_key() {
3205 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3206 }
3207 },
3208 )
3209 }
3210 };
3211 if result.is_ok() && ordered_retry {
3212 self.has_ordered_retry = true;
3213 }
3214 if result.is_ok()
3215 && let Some(key) = replay
3216 {
3217 Self::set_replay_queued(reliable_tx, end_to_end, key, true);
3218 }
3219 result
3220 }
3221
3222 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
3223 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
3224 let _ = self.recent_rx.pop_front();
3225 }
3226 self.make_shared_queue_room(0, 0, None)?;
3227 self.recent_rx.push_back(id)
3228 }
3229
3230 #[allow(unused_mut)]
3231 fn buffer_reliable_rx(
3232 &mut self,
3233 side: RouterSideId,
3234 ty: DataType,
3235 seq: u32,
3236 mut bytes: crate::SharedBytes,
3237 ) -> TelemetryResult<()> {
3238 let key = Router::reliable_key(side, ty);
3239 if self
3240 .reliable_rx
3241 .get(&key)
3242 .is_some_and(|state| state.buffered.contains_key(&seq))
3243 {
3244 return Ok(());
3245 }
3246 let cost = size_of::<crate::SharedBytes>() + bytes.len();
3247 self.make_shared_queue_room(cost, crate::transport_priority(ty), None)?;
3248 #[cfg(feature = "compact-packet-store")]
3249 bytes.park_for_queue()?;
3250 let rx_state = self
3251 .reliable_rx
3252 .entry(key)
3253 .or_insert_with(|| ReliableRxState {
3254 expected_seq: 1,
3255 buffered: Default::default(),
3256 });
3257 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
3258 let _ = rx_state.buffered.pop_first();
3259 }
3260 rx_state.buffered.insert(seq, bytes);
3261 Ok(())
3262 }
3263}
3264
3265struct IsrRxQueue {
3270 busy: AtomicBool,
3271 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
3272}
3273
3274unsafe impl Send for IsrRxQueue {}
3275unsafe impl Sync for IsrRxQueue {}
3276
3277struct IsrRxQueueGuard<'a> {
3278 owner: &'a IsrRxQueue,
3279}
3280
3281impl Deref for IsrRxQueueGuard<'_> {
3282 type Target = BoundedDeque<RouterRxItem>;
3283
3284 #[inline]
3285 fn deref(&self) -> &Self::Target {
3286 unsafe { &*self.owner.q.get() }
3287 }
3288}
3289
3290impl DerefMut for IsrRxQueueGuard<'_> {
3291 #[inline]
3292 fn deref_mut(&mut self) -> &mut Self::Target {
3293 unsafe { &mut *self.owner.q.get() }
3294 }
3295}
3296
3297impl Drop for IsrRxQueueGuard<'_> {
3298 #[inline]
3299 fn drop(&mut self) {
3300 self.owner.busy.store(false, Ordering::Release);
3301 }
3302}
3303
3304impl IsrRxQueue {
3305 #[inline]
3306 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
3307 Self {
3308 busy: AtomicBool::new(false),
3309 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
3310 }
3311 }
3312
3313 #[inline]
3314 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
3315 match self
3316 .busy
3317 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
3318 {
3319 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
3320 Err(_) => Err(TelemetryError::Io("rx queue busy")),
3321 }
3322 }
3323
3324 #[allow(dead_code)]
3325 #[inline]
3326 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
3327 let mut g = self.try_lock()?;
3328 g.push_back(item)
3329 }
3330
3331 #[inline]
3332 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
3333 let mut g = self.try_lock()?;
3334 g.push_back_prioritized(item, |queued| queued.priority)
3335 }
3336
3337 #[inline]
3338 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
3339 let mut g = self.try_lock()?;
3340 Ok(g.pop_front())
3341 }
3342
3343 #[inline]
3344 fn clear(&self) -> TelemetryResult<()> {
3345 let mut g = self.try_lock()?;
3346 g.clear();
3347 Ok(())
3348 }
3349
3350 #[inline]
3351 fn snapshot(&self) -> Option<(usize, usize)> {
3352 let g = self.try_lock().ok()?;
3353 Some((g.len(), g.bytes_used()))
3354 }
3355}
3356
3357pub struct Router {
3361 sender: RouterMutex<Arc<str>>,
3362 cfg: RouterConfig,
3363 state: RouterMutex<RouterInner>,
3364 isr_rx_queue: IsrRxQueue,
3365 #[cfg(not(feature = "std"))]
3366 side_tx_gate: ReentryGate,
3367 #[cfg(feature = "std")]
3368 side_tx_activity: crate::lock::SideTxActivity,
3369 #[cfg(feature = "std")]
3370 active_queued_tx: core::sync::atomic::AtomicUsize,
3371 clock: Box<dyn Clock + Send + Sync>,
3372 #[cfg(feature = "timesync")]
3373 timesync: RouterMutex<TimeSyncRuntime>,
3374}
3375
3376#[cfg(feature = "std")]
3377struct QueuedTxActivityGuard<'a>(&'a core::sync::atomic::AtomicUsize);
3378#[cfg(feature = "std")]
3379impl Drop for QueuedTxActivityGuard<'_> {
3380 fn drop(&mut self) {
3381 self.0.fetch_sub(1, core::sync::atomic::Ordering::Release);
3382 }
3383}
3384
3385#[cfg(feature = "timesync")]
3386#[derive(Debug, Clone)]
3387struct PendingTimeSyncRequest {
3388 seq: u64,
3389 t1_mono_ms: u64,
3390 source: String,
3391}
3392
3393#[cfg(feature = "timesync")]
3394#[derive(Debug, Clone)]
3395struct RemoteTimeSyncSource {
3396 priority: u64,
3397 last_sample_mono_ms: u64,
3398 sample_unix_ms: u64,
3399}
3400
3401#[cfg(feature = "timesync")]
3402#[derive(Debug, Clone)]
3403struct TimeSyncRuntime {
3404 cfg: Option<TimeSyncConfig>,
3405 tracker: Option<TimeSyncTracker>,
3406 clock: NetworkClock,
3407 disciplined_clock: SlewedNetworkClock,
3408 remote_sources: BTreeMap<String, RemoteTimeSyncSource>,
3409 next_seq: u64,
3410 next_announce_mono_ms: u64,
3411 next_request_mono_ms: u64,
3412 pending_request: Option<PendingTimeSyncRequest>,
3413}
3414
3415#[cfg(feature = "timesync")]
3416impl TimeSyncRuntime {
3417 fn new(cfg: Option<TimeSyncConfig>) -> Self {
3418 Self {
3419 tracker: cfg.map(TimeSyncTracker::new),
3420 cfg,
3421 clock: NetworkClock::default(),
3422 disciplined_clock: SlewedNetworkClock::new(
3423 cfg.map(|c| c.max_slew_ppm)
3424 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
3425 ),
3426 remote_sources: Default::default(),
3427 next_seq: 1,
3428 next_announce_mono_ms: 0,
3429 next_request_mono_ms: 0,
3430 pending_request: None,
3431 }
3432 }
3433}
3434
3435enum RemoteSidePlan {
3436 Target(Vec<RouterSideId>),
3437}
3438
3439#[cfg(feature = "discovery")]
3440#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
3441struct DiscoveryCandidateMatch {
3442 side: RouterSideId,
3443 overlap: usize,
3444}
3445
3446#[cfg(feature = "discovery")]
3447#[inline]
3448const fn discovery_is_hop_local_advertisement(ty: DataType) -> bool {
3449 matches!(
3450 ty,
3451 DataType::DiscoveryAnnounce
3452 | DataType::DiscoveryTimeSyncSources
3453 | DataType::DiscoveryTopology
3454 | DataType::DiscoveryAddress
3455 | DataType::DiscoveryLinkCapabilities
3456 )
3457}
3458
3459#[cfg(not(feature = "discovery"))]
3460const fn discovery_is_hop_local_advertisement(_ty: DataType) -> bool {
3461 false
3462}
3463
3464impl Debug for Router {
3465 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
3466 let sender = self.sender();
3467 f.debug_struct("Router")
3468 .field("sender", &sender)
3469 .field("cfg", &self.cfg)
3470 .field("state", &"<mutex>")
3471 .field("clock", &"Clock")
3472 .finish()
3473 }
3474}
3475
3476#[inline]
3479fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &RouterConfig) -> bool {
3480 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
3481}
3482
3483#[inline]
3484fn force_remote_for_type(ty: DataType) -> bool {
3485 matches!(
3486 ty,
3487 DataType::ReliableAck
3488 | DataType::ReliablePartialAck
3489 | DataType::ReliablePacketRequest
3490 | DataType::P2pMessage
3491 ) || {
3492 #[cfg(feature = "timesync")]
3493 {
3494 matches!(
3495 ty,
3496 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
3497 )
3498 }
3499 #[cfg(not(feature = "timesync"))]
3500 {
3501 false
3502 }
3503 }
3504}
3505
3506#[inline]
3507fn is_internal_control_type(ty: DataType) -> bool {
3508 if matches!(
3509 ty,
3510 DataType::ReliableAck
3511 | DataType::ReliablePartialAck
3512 | DataType::ReliablePacketRequest
3513 | DataType::P2pMessage
3514 ) {
3515 return true;
3516 }
3517
3518 #[cfg(feature = "timesync")]
3519 if matches!(
3520 ty,
3521 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
3522 ) {
3523 return true;
3524 }
3525
3526 #[cfg(feature = "discovery")]
3527 if discovery::is_discovery_type(ty) {
3528 return true;
3529 }
3530
3531 let _ = ty;
3532 false
3533}
3534
3535fn with_retries<F>(
3537 this: &Router,
3538 dest: DataEndpoint,
3539 data: &RouterItem,
3540 pkt_for_ctx: Option<&Packet>,
3541 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
3542 called_from_queue: bool,
3543 run: F,
3544) -> TelemetryResult<()>
3545where
3546 F: Fn() -> TelemetryResult<()>,
3547{
3548 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
3549 Ok(((), attempts)) => {
3550 if attempts > 1 {
3551 let mut st = this.state.lock();
3552 st.total_handler_retries = st
3553 .total_handler_retries
3554 .saturating_add(attempts.saturating_sub(1) as u64);
3555 }
3556 Ok(())
3557 }
3558 Err((e, attempts)) => {
3559 {
3560 let mut st = this.state.lock();
3561 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3562 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3563 }
3564
3565 this.remove_pkt_id(data);
3567
3568 if let Some(pkt) = pkt_for_ctx {
3570 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
3571 } else if let Some(env) = env_for_ctx {
3572 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
3573 }
3574
3575 Err(TelemetryError::HandlerError("local handler failed"))
3576 }
3577 }
3578}
3579impl Router {
3581 #[cfg(feature = "discovery")]
3582 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
3583 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
3584
3585 #[inline]
3586 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
3587 st.sides
3588 .get(side)
3589 .and_then(|side| side.as_ref())
3590 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
3591 }
3592
3593 fn note_side_tx_success(
3594 &self,
3595 side: RouterSideId,
3596 ty: DataType,
3597 bytes: usize,
3598 relayed: bool,
3599 attempts: usize,
3600 ) {
3601 let mut st = self.state.lock();
3602 #[cfg(feature = "discovery")]
3603 if ty == crate::DataType::DiscoveryAnnounce
3604 && !relayed
3605 && let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
3606 {
3607 throttle.last_liveness_tx_ms = self.clock.now_ms();
3608 }
3609 #[cfg(all(feature = "discovery", feature = "timesync"))]
3610 if Self::is_timesync_type(ty) {
3611 let now = self.clock.now_ms();
3612 if Self::side_is_slow_control_link_locked(&st, side, now) {
3613 st.timesync_side_throttle
3614 .entry(side)
3615 .or_default()
3616 .next_allowed_ms = now.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
3617 } else {
3618 st.timesync_side_throttle.remove(&side);
3619 }
3620 }
3621 let entry = st.side_runtime_stats.entry(side).or_default();
3622 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
3623 }
3624
3625 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
3626 let mut st = self.state.lock();
3627 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3628 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3629 let entry = st.side_runtime_stats.entry(side).or_default();
3630 entry.note_tx_failure(ty, attempts);
3631 }
3632
3633 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
3634 let mut st = self.state.lock();
3635 let entry = st.side_runtime_stats.entry(side).or_default();
3636 entry.note_rx(ty, bytes, relayed);
3637 }
3638
3639 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
3640 let mut st = self.state.lock();
3641 let entry = st.side_runtime_stats.entry(side).or_default();
3642 entry.note_local_delivery(ty);
3643 }
3644
3645 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
3646 let mut st = self.state.lock();
3647 let entry = st.side_runtime_stats.entry(side).or_default();
3648 entry.note_local_handler_failure(ty, retries);
3649 }
3650
3651 fn cache_managed_variable_packet(
3652 &self,
3653 pkt: &Packet,
3654 notify_handlers: bool,
3655 ) -> TelemetryResult<()> {
3656 let sender_address = pkt
3657 .sender()
3658 .strip_prefix("@addr:")
3659 .and_then(|value| value.parse::<u32>().ok())
3660 .unwrap_or_else(|| sender_address_u32(pkt.sender()));
3661 let handlers = {
3662 let mut st = self.state.lock();
3663 if !st
3664 .managed_variable_types
3665 .contains(&pkt.data_type().as_u32())
3666 {
3667 return Ok(());
3668 }
3669 let existing = st.managed_variable_latest.get(&pkt.data_type().as_u32());
3670 if existing.is_some_and(|entry| {
3677 let current = &entry.packet;
3678 sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
3679 }) {
3680 return Ok(());
3681 }
3682 let changed = existing.is_none_or(|entry| entry.packet != *pkt);
3683 let updated = ManagedVariableCacheEntry {
3687 packet: pkt.clone(),
3688 sender_address,
3689 #[cfg(feature = "discovery")]
3690 cached_at_ms: self.clock.now_ms(),
3691 };
3692 if let Some(entry) = st
3693 .managed_variable_latest
3694 .get_mut(&pkt.data_type().as_u32())
3695 {
3696 **entry = updated;
3697 } else {
3698 st.managed_variable_latest
3699 .insert(pkt.data_type().as_u32(), Box::new(updated));
3700 }
3701 if notify_handlers && changed {
3702 st.network_variable_update_handlers
3703 .get(&pkt.data_type().as_u32())
3704 .cloned()
3705 .unwrap_or_default()
3706 } else {
3707 Vec::new()
3708 }
3709 };
3710 for handler in handlers {
3711 (handler.handler)(pkt)?;
3712 }
3713 Ok(())
3714 }
3715
3716 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
3717 self.cache_managed_variable_packet(pkt, true)
3718 }
3719
3720 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
3721 let st = self.state.lock();
3722 st.managed_variable_latest
3723 .get(&ty.as_u32())
3724 .map(|entry| entry.packet.clone())
3725 }
3726
3727 #[cfg(feature = "discovery")]
3728 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
3729 let now_ms = self.clock.now_ms();
3730 let st = self.state.lock();
3731 st.managed_variable_latest.get(&ty.as_u32()).map(|entry| {
3732 (
3733 entry.packet.clone(),
3734 now_ms.saturating_sub(entry.cached_at_ms),
3735 )
3736 })
3737 }
3738
3739 fn is_managed_variable_type(&self, ty: DataType) -> bool {
3740 let st = self.state.lock();
3741 st.managed_variable_types.contains(&ty.as_u32())
3742 }
3743
3744 fn managed_variable_permissions_locked(
3745 st: &RouterInner,
3746 ty: DataType,
3747 ) -> NetworkVariablePermissions {
3748 st.managed_variable_permissions
3749 .get(&ty.as_u32())
3750 .copied()
3751 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
3752 }
3753
3754 fn can_read_managed_variable(&self, ty: DataType) -> bool {
3755 let st = self.state.lock();
3756 Self::managed_variable_permissions_locked(&st, ty).read
3757 }
3758
3759 #[cfg(feature = "discovery")]
3760 fn can_write_managed_variable(&self, ty: DataType) -> bool {
3761 let st = self.state.lock();
3762 Self::managed_variable_permissions_locked(&st, ty).write
3763 }
3764
3765 #[inline]
3766 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
3767 let st = self.state.lock();
3768 let side_ref = Self::side_ref(&st, side)?;
3769 if side_ref.opts.ingress_enabled {
3770 Ok(())
3771 } else {
3772 Err(TelemetryError::HandlerError(
3773 "router: ingress disabled for side id",
3774 ))
3775 }
3776 }
3777
3778 #[inline]
3779 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
3780 src != Some(dst)
3781 }
3782
3783 #[inline]
3784 fn route_allowed_locked(
3785 &self,
3786 st: &RouterInner,
3787 src: Option<RouterSideId>,
3788 ty: Option<DataType>,
3789 dst: RouterSideId,
3790 ) -> bool {
3791 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
3792 return false;
3793 };
3794 if !dst_side.opts.egress_enabled {
3795 return false;
3796 }
3797 if let Some(src_id) = src {
3798 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
3799 return false;
3800 };
3801 if !src_side.opts.ingress_enabled {
3802 return false;
3803 }
3804 }
3805 let base_allowed = st
3806 .route_overrides
3807 .get(&(src, dst))
3808 .copied()
3809 .unwrap_or_else(|| self.default_route_enabled(src, dst));
3810 if !base_allowed {
3811 return false;
3812 }
3813
3814 let Some(ty) = ty else {
3815 return true;
3816 };
3817 if st
3818 .typed_route_overrides
3819 .keys()
3820 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3821 {
3822 return st
3823 .typed_route_overrides
3824 .get(&(src, ty.as_u32(), dst))
3825 .copied()
3826 .unwrap_or(false);
3827 }
3828 true
3829 }
3830
3831 #[cfg(feature = "discovery")]
3832 fn has_typed_route_overrides_locked(
3833 st: &RouterInner,
3834 src: Option<RouterSideId>,
3835 ty: DataType,
3836 ) -> bool {
3837 st.typed_route_overrides
3838 .keys()
3839 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3840 }
3841
3842 #[cfg(feature = "discovery")]
3843 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
3844 where
3845 I: IntoIterator<Item = DataEndpoint>,
3846 {
3847 let mut overlap = 0usize;
3848 for ep in reachable {
3849 if eps.contains(&ep) {
3850 overlap = overlap.saturating_add(1);
3851 }
3852 }
3853 overlap
3854 }
3855
3856 #[inline]
3857 #[cfg(feature = "discovery")]
3858 fn preferred_scoring_endpoints<'a>(
3859 &self,
3860 eps: &'a [DataEndpoint],
3861 prefer_nonlocal: bool,
3862 ) -> alloc::borrow::Cow<'a, [DataEndpoint]> {
3863 if !prefer_nonlocal || eps.iter().all(|&ep| !self.cfg.is_local_endpoint(ep)) {
3864 return alloc::borrow::Cow::Borrowed(eps);
3865 }
3866 let nonlocal: Vec<DataEndpoint> = eps
3867 .iter()
3868 .copied()
3869 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
3870 .collect();
3871 if nonlocal.is_empty() {
3872 alloc::borrow::Cow::Borrowed(eps)
3873 } else {
3874 alloc::borrow::Cow::Owned(nonlocal)
3875 }
3876 }
3877
3878 fn pop_next_queued_control_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
3879 {
3880 let mut isr_rx = self.isr_rx_queue.try_lock()?;
3881 let idx = isr_rx.iter().position(Self::queued_rx_item_is_control);
3882 if let Some(idx) = idx {
3883 return Ok(isr_rx.remove_pos(idx));
3884 }
3885 }
3886
3887 let mut st = self.state.lock();
3888 let idx = st
3889 .received_queue
3890 .iter()
3891 .position(Self::queued_rx_item_is_control);
3892 if let Some(idx) = idx {
3893 return Ok(st.received_queue.remove_pos(idx));
3894 }
3895 Ok(None)
3896 }
3897
3898 fn queued_rx_item_is_control(item: &RouterRxItem) -> bool {
3899 if item.priority < crate::transport_priority(DataType::ManagedVariableValue) {
3903 return false;
3904 }
3905 if item.priority == crate::transport_priority(DataType::ManagedVariableValue) {
3908 return true;
3909 }
3910 let ty = match &item.data {
3911 RouterItem::Packet(pkt) => Some(pkt.data_type()),
3912 RouterItem::Packed(bytes) => {
3913 if bytes.len() <= 128
3914 && wire_format::peek_routing_frame_info(bytes)
3915 .is_ok_and(|frame| frame.ack_only())
3916 {
3917 return true;
3918 }
3919 wire_format::scheduling_type(bytes.as_ref())
3920 }
3921 };
3922 match ty {
3923 Some(
3924 DataType::ReliableAck
3925 | DataType::ReliablePartialAck
3926 | DataType::ReliablePacketRequest,
3927 ) => true,
3928 #[cfg(feature = "discovery")]
3929 Some(ty) => discovery::is_discovery_type(ty),
3930 _ => false,
3931 }
3932 }
3933
3934 fn drain_queued_control_rx_before_tx(
3935 &self,
3936 start: u64,
3937 timeout_ms: u32,
3938 ) -> TelemetryResult<bool> {
3939 let mut did_any = false;
3940 while let Some(item) = self.pop_next_queued_control_rx_item()? {
3941 self.process_rx_queue_item(item)?;
3942 did_any = true;
3943 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms) {
3946 break;
3947 }
3948 }
3949 Ok(did_any)
3950 }
3951
3952 fn eligible_side_ids_locked(
3953 &self,
3954 st: &RouterInner,
3955 src: Option<RouterSideId>,
3956 ty: Option<DataType>,
3957 restrict_link_local: bool,
3958 ) -> Vec<RouterSideId> {
3959 st.sides
3960 .iter()
3961 .enumerate()
3962 .filter_map(|(side_id, side)| {
3963 let side = side.as_ref()?;
3964 if restrict_link_local && !side.opts.link_local_enabled {
3965 return None;
3966 }
3967 if self.route_allowed_locked(st, src, ty, side_id) {
3968 Some(side_id)
3969 } else {
3970 None
3971 }
3972 })
3973 .collect()
3974 }
3975
3976 fn apply_route_selection_locked(
3977 &self,
3978 st: &mut RouterInner,
3979 src: Option<RouterSideId>,
3980 mut sides: Vec<RouterSideId>,
3981 origin: RouteSelectionOrigin,
3982 ) -> Vec<RouterSideId> {
3983 if sides.len() <= 1 {
3984 return sides;
3985 }
3986
3987 let selection_mode = st.source_route_modes.get(&src).copied();
3988 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
3989 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
3990 }
3991
3992 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
3993 RouteSelectionMode::Fanout => sides,
3994 RouteSelectionMode::Weighted => {
3995 crate::small_sort::sort(&mut sides);
3996 let total_weight = sides.iter().fold(0_u64, |acc, side| {
3997 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
3998 });
3999 if total_weight == 0 {
4000 return Vec::new();
4001 }
4002 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4003 let pick = *cursor % total_weight;
4004 *cursor = cursor.wrapping_add(1);
4005 let mut remaining = pick;
4006 for side in sides {
4007 let weight =
4008 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
4009 if remaining < weight {
4010 return vec![side];
4011 }
4012 remaining -= weight;
4013 }
4014 Vec::new()
4015 }
4016 RouteSelectionMode::Failover => {
4017 sides.sort_by_key(|side| {
4018 (
4019 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
4020 *side,
4021 )
4022 });
4023 sides.truncate(1);
4024 sides
4025 }
4026 }
4027 }
4028
4029 fn apply_adaptive_discovery_selection_locked(
4030 &self,
4031 st: &mut RouterInner,
4032 src: Option<RouterSideId>,
4033 mut sides: Vec<RouterSideId>,
4034 ) -> Vec<RouterSideId> {
4035 crate::small_sort::sort(&mut sides);
4036 let mut unmeasured: Vec<_> = sides
4037 .iter()
4038 .copied()
4039 .filter(|side| !st.adaptive_route_stats.contains_key(side))
4040 .collect();
4041 if !unmeasured.is_empty() {
4042 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4043 let pick = (*cursor as usize) % unmeasured.len();
4044 *cursor = cursor.wrapping_add(1);
4045 return vec![unmeasured.swap_remove(pick)];
4046 }
4047
4048 let now_ms = self.clock.now_ms();
4049 let total_weight = sides.iter().fold(0_u64, |acc, side| {
4050 acc + st
4051 .adaptive_route_stats
4052 .get(side)
4053 .map(|stats| stats.weight(now_ms))
4054 .unwrap_or(1)
4055 });
4056 if total_weight == 0 {
4057 sides.truncate(1);
4058 return sides;
4059 }
4060
4061 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4062 let pick = *cursor % total_weight;
4063 *cursor = cursor.wrapping_add(1);
4064 let mut remaining = pick;
4065 for side in sides {
4066 let weight = st
4067 .adaptive_route_stats
4068 .get(&side)
4069 .map(|stats| stats.weight(now_ms))
4070 .unwrap_or(1);
4071 if remaining < weight {
4072 return vec![side];
4073 }
4074 remaining -= weight;
4075 }
4076 Vec::new()
4077 }
4078
4079 fn record_side_tx_sample(
4080 &self,
4081 side: RouterSideId,
4082 bytes: usize,
4083 started_ms: u64,
4084 ended_ms: u64,
4085 ) {
4086 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
4087 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
4088 .min(u128::from(u64::MAX)) as u64;
4089 let mut st = self.state.lock();
4090 st.adaptive_route_stats
4091 .entry(side)
4092 .or_default()
4093 .observe(bytes, sample_bps, ended_ms);
4094 }
4095
4096 pub fn note_side_link_probe_sample(
4101 &self,
4102 side: RouterSideId,
4103 bytes: usize,
4104 duration_ms: u64,
4105 ) -> TelemetryResult<()> {
4106 {
4107 let st = self.state.lock();
4108 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
4109 }
4110 let ended_ms = self.clock.now_ms();
4111 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
4112 Ok(())
4113 }
4114
4115 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
4116 match data {
4117 RouterItem::Packet(pkt) => Ok(wire_format::packet_wire_size(pkt)),
4118 RouterItem::Packed(bytes) => Ok(bytes.len()),
4119 }
4120 }
4121
4122 #[inline]
4132 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
4133 match data {
4134 RouterItem::Packet(pkt) => {
4135 if !Self::is_end_to_end_ack_packet(pkt) {
4136 return Ok(None);
4137 }
4138 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4139 }
4140 RouterItem::Packed(bytes) => {
4141 if wire_format::peek_routing_frame_info(bytes.as_ref())
4142 .ok()
4143 .is_some_and(|frame| {
4144 frame.envelope.ty != DataType::ReliableAck || frame.ack_only()
4145 })
4146 {
4147 return Ok(None);
4148 }
4149 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4150 if !Self::is_end_to_end_ack_packet(&pkt) {
4151 return Ok(None);
4152 }
4153 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4154 }
4155 }
4156 }
4157
4158 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
4159 if payload.len() != 8 {
4160 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
4161 }
4162 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
4163 }
4164
4165 #[inline]
4166 fn sender_hash(sender: &str) -> u64 {
4167 if let Some(address) = sender
4168 .strip_prefix("@addr:")
4169 .and_then(|v| v.parse::<u32>().ok())
4170 {
4171 return u64::from(address);
4172 }
4173 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
4174 }
4175
4176 #[inline]
4177 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
4178 let hash = Self::sender_hash(hostname);
4179 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
4180 if address == 0 {
4181 address = 1;
4182 }
4183 address
4184 }
4185
4186 fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
4191 let Some(address) = sender
4192 .strip_prefix("@addr:")
4193 .and_then(|value| value.parse::<u32>().ok())
4194 else {
4195 return sender.to_string();
4196 };
4197 Self::hostname_for_address_locked(st, address).unwrap_or_else(|| sender.to_string())
4198 }
4199
4200 fn hostname_for_address_locked(st: &RouterInner, address: NodeAddress) -> Option<String> {
4207 Self::hostname_for_address_borrowed_locked(st, address).map(str::to_string)
4208 }
4209
4210 fn hostname_for_address_borrowed_locked<'a>(
4211 st: &'a RouterInner,
4212 address: NodeAddress,
4213 ) -> Option<&'a str> {
4214 if let Some(hostname) = st.address_by_value.get(&address) {
4215 return Some(hostname.as_str());
4216 }
4217
4218 let mut found: Option<&'a str> = None;
4219 let mut collision = false;
4220 let mut consider = |candidate: &'a str| {
4221 if sender_address_u32(candidate) != address {
4222 return;
4223 }
4224 match found {
4225 None => found = Some(candidate),
4226 Some(existing) if existing == candidate => {}
4227 Some(_) => collision = true,
4231 }
4232 };
4233
4234 for entry in st.address_book.values() {
4235 consider(entry.hostname.as_ref());
4236 }
4237 #[cfg(feature = "discovery")]
4238 for route in st.discovery_routes.values() {
4239 for (announcer, state) in &route.announcers {
4240 consider(announcer);
4241 for board in state.topology_boards.iter() {
4242 consider(&board.sender_id);
4243 for peer in &board.connections {
4249 consider(peer);
4250 }
4251 }
4252 }
4253 }
4254 if collision { None } else { found }
4255 }
4256
4257 #[cfg(feature = "discovery")]
4258 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
4259 match mode {
4260 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
4261 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
4262 _ => AddressAssignmentMode::Dynamic,
4263 }
4264 }
4265
4266 #[cfg(feature = "discovery")]
4267 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
4268 let rank = match entry.mode {
4269 AddressAssignmentMode::Static(_) => 2,
4270 AddressAssignmentMode::Requested(_) => 1,
4271 AddressAssignmentMode::Dynamic => 0,
4272 };
4273 (
4274 rank,
4275 core::cmp::Reverse(entry.birth_ms),
4276 u64::MAX - entry.owner_hash,
4277 )
4278 }
4279
4280 #[cfg(feature = "discovery")]
4281 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
4282 (
4283 core::cmp::Reverse(entry.birth_ms),
4284 u64::MAX - entry.owner_hash,
4285 )
4286 }
4287
4288 #[cfg(feature = "discovery")]
4289 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
4290 let mut candidate = if seed == 0 { 1 } else { seed };
4291 for _ in 0..u32::MAX {
4292 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
4293 return candidate;
4294 }
4295 candidate = candidate.wrapping_add(1);
4296 if candidate == 0 {
4297 candidate = 1;
4298 }
4299 }
4300 1
4301 }
4302
4303 #[cfg(feature = "discovery")]
4304 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
4305 if !st.address_book.contains_key(base) {
4306 return base.to_string();
4307 }
4308 let stem = if base.is_empty() { "node" } else { base };
4309 let mut candidate = format!("{stem}-{owner_hash:08x}");
4310 let mut suffix = 1u32;
4311 while st.address_book.contains_key(&candidate) {
4312 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
4313 suffix = suffix.saturating_add(1);
4314 }
4315 candidate
4316 }
4317
4318 fn update_local_identity_locked(
4319 &self,
4320 st: &mut RouterInner,
4321 mut local: AddressBookEntry,
4322 reason: AddressChangeReason,
4323 ) -> AddressChange {
4324 let old = st.local_address.clone();
4325 st.address_book.remove(old.hostname.as_ref());
4326 st.address_by_value.remove(&old.address);
4327 local.last_seen_ms = self.clock.now_ms();
4328 st.address_by_value
4329 .insert(local.address, local.hostname.to_string());
4330 st.address_book
4331 .insert(local.hostname.to_string(), local.clone());
4332 st.local_address = local.clone();
4333 *self.sender.lock() = local.hostname.clone();
4334 #[cfg(feature = "discovery")]
4335 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
4336 AddressChange {
4337 old_hostname: old.hostname,
4338 new_hostname: local.hostname,
4339 old_address: old.address,
4340 new_address: local.address,
4341 reason,
4342 }
4343 }
4344
4345 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
4346 for handler in self.cfg.address_change_handlers.iter() {
4347 (handler.handler)(change.clone())?;
4348 }
4349 Ok(())
4350 }
4351
4352 #[cfg(feature = "discovery")]
4353 fn local_address_advertisement(
4354 &self,
4355 reachable_endpoints: Vec<DataEndpoint>,
4356 reachable_network_variables: Vec<DataType>,
4357 reachable_timesync_sources: Vec<String>,
4358 link_capabilities: discovery::LinkCapabilities,
4359 state: u8,
4360 ) -> discovery::AddressAdvertisement {
4361 let st = self.state.lock();
4362 discovery::AddressAdvertisement {
4363 hostname: st.local_address.hostname.to_string(),
4364 address: st.local_address.address,
4365 requested_address: st.local_address.requested_address,
4366 mode: st.local_address.mode.mode_code(),
4367 state,
4368 birth_ms: st.local_address.birth_ms,
4369 owner_hash: st.local_address.owner_hash,
4370 reachable_endpoints,
4371 reachable_network_variables,
4372 reachable_timesync_sources,
4373 link_capabilities,
4374 }
4375 }
4376
4377 #[cfg(feature = "discovery")]
4378 fn ingest_address_advertisement(
4379 &self,
4380 ad: discovery::AddressAdvertisement,
4381 ) -> TelemetryResult<bool> {
4382 let now_ms = self.clock.now_ms();
4383 let mut remote = AddressBookEntry {
4384 hostname: Arc::from(ad.hostname.as_str()),
4385 address: ad.address,
4386 requested_address: ad.requested_address,
4387 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
4388 birth_ms: ad.birth_ms,
4389 owner_hash: ad.owner_hash,
4390 last_seen_ms: now_ms,
4391 };
4392 let mut change = None;
4393 let mut changed = false;
4394 {
4395 let mut st = self.state.lock();
4396 let local = st.local_address.clone();
4397 let hostname_conflict =
4398 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
4399 let address_conflict =
4400 remote.address == local.address && remote.owner_hash != local.owner_hash;
4401
4402 if hostname_conflict
4403 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
4404 {
4405 let mut next = local.clone();
4406 next.hostname = Arc::from(Self::unique_hostname_locked(
4407 &st,
4408 local.hostname.as_ref(),
4409 local.owner_hash,
4410 ));
4411 change = Some(self.update_local_identity_locked(
4412 &mut st,
4413 next,
4414 AddressChangeReason::HostnameConflict,
4415 ));
4416 changed = true;
4417 } else if address_conflict
4418 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
4419 {
4420 let mut next = local.clone();
4421 let seed = next
4422 .requested_address
4423 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
4424 next.address = Self::allocate_free_address_locked(&st, seed);
4425 let reason = match next.mode {
4426 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
4427 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
4428 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
4429 };
4430 change = Some(self.update_local_identity_locked(&mut st, next, reason));
4431 changed = true;
4432 } else if hostname_conflict {
4433 remote.hostname = Arc::from(Self::unique_hostname_locked(
4434 &st,
4435 remote.hostname.as_ref(),
4436 remote.owner_hash,
4437 ));
4438 changed = true;
4439 } else if address_conflict {
4440 let seed = remote
4441 .requested_address
4442 .max(Self::fallback_address_for_hostname(
4443 remote.hostname.as_ref(),
4444 ));
4445 remote.address = Self::allocate_free_address_locked(&st, seed);
4446 changed = true;
4447 }
4448
4449 st.address_book.remove(remote.hostname.as_ref());
4450 st.address_by_value.remove(&remote.address);
4451 st.address_by_value
4452 .insert(remote.address, remote.hostname.to_string());
4453 st.address_book
4454 .insert(remote.hostname.to_string(), remote.clone());
4455 }
4456 if let Some(change) = change {
4457 self.notify_address_change(change)?;
4458 }
4459 Ok(changed)
4460 }
4461
4462 #[inline]
4463 #[cfg(feature = "discovery")]
4464 fn is_end_to_end_ack_sender(sender: &str) -> bool {
4465 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
4466 }
4467
4468 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
4469 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
4470 && !ack_sender.is_empty()
4471 {
4472 return Some(Self::sender_hash(ack_sender));
4473 }
4474 None
4475 }
4476
4477 fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
4483 if pkt.data_type() != DataType::ReliableAck {
4484 return None;
4485 }
4486 pkt.wire_target_senders()
4487 .first()
4488 .copied()
4489 .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
4490 }
4491
4492 fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
4493 Self::end_to_end_ack_sender_hash(pkt).is_some()
4494 && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
4495 }
4496
4497 fn encode_end_to_end_ack_sender(&self) -> String {
4498 let sender = self.sender_arc();
4499 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
4500 }
4501
4502 #[cfg(feature = "discovery")]
4503 fn is_end_to_end_destination_sender(sender: &str) -> bool {
4504 sender != "RELAY" && !Self::is_end_to_end_ack_sender(sender)
4505 }
4506
4507 fn encode_p2p_payload(
4508 src_hostname: &str,
4509 src_address: NodeAddress,
4510 src_port: P2pPort,
4511 dst_port: P2pPort,
4512 payload: &[u8],
4513 ) -> TelemetryResult<crate::SharedBytes> {
4514 let host_len = u16::try_from(src_hostname.len())
4515 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
4516 let payload_len = u32::try_from(payload.len())
4517 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
4518 let mut out = Vec::with_capacity(
4519 15usize
4520 .saturating_add(src_hostname.len())
4521 .saturating_add(payload.len()),
4522 );
4523 out.push(1);
4524 out.extend_from_slice(&dst_port.to_le_bytes());
4525 out.extend_from_slice(&src_port.to_le_bytes());
4526 out.extend_from_slice(&src_address.to_le_bytes());
4527 out.extend_from_slice(&host_len.to_le_bytes());
4528 out.extend_from_slice(&payload_len.to_le_bytes());
4529 out.extend_from_slice(src_hostname.as_bytes());
4530 out.extend_from_slice(payload);
4531 Ok(out.into())
4532 }
4533
4534 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
4535 if payload.len() < 15 {
4536 return Err(TelemetryError::Unpack("p2p frame short"));
4537 }
4538 if payload[0] != 1 {
4539 return Err(TelemetryError::Unpack("p2p frame version"));
4540 }
4541 let destination_port =
4542 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
4543 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
4544 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
4545 let host_len =
4546 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
4547 let body_len =
4548 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
4549 let host_start = 15usize;
4550 let host_end = host_start.saturating_add(host_len);
4551 let body_end = host_end.saturating_add(body_len);
4552 if host_end > payload.len() || body_end != payload.len() {
4553 return Err(TelemetryError::Unpack("p2p frame length"));
4554 }
4555 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
4556 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
4557 Ok(P2pDecoded {
4558 source_hostname,
4559 source_address,
4560 source_port,
4561 destination_port,
4562 payload: &payload[host_end..body_end],
4563 })
4564 }
4565
4566 fn encode_p2p_stream_payload(
4567 flags: u8,
4568 source_stream_id: P2pStreamId,
4569 destination_stream_id: P2pStreamId,
4570 sequence: u32,
4571 payload: &[u8],
4572 ) -> TelemetryResult<crate::SharedBytes> {
4573 let payload_len = u32::try_from(payload.len())
4574 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
4575 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
4576 out.extend_from_slice(&P2P_STREAM_MAGIC);
4577 out.push(P2P_STREAM_VERSION);
4578 out.push(flags);
4579 out.extend_from_slice(&source_stream_id.to_le_bytes());
4580 out.extend_from_slice(&destination_stream_id.to_le_bytes());
4581 out.extend_from_slice(&sequence.to_le_bytes());
4582 out.extend_from_slice(&payload_len.to_le_bytes());
4583 out.extend_from_slice(payload);
4584 Ok(out.into())
4585 }
4586
4587 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
4588 if !payload.starts_with(&P2P_STREAM_MAGIC) {
4589 return Ok(None);
4590 }
4591 if payload.len() < 22 {
4592 return Err(TelemetryError::Unpack("p2p stream frame short"));
4593 }
4594 if payload[4] != P2P_STREAM_VERSION {
4595 return Err(TelemetryError::Unpack("p2p stream frame version"));
4596 }
4597 let flags = payload[5];
4598 let source_stream_id =
4599 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
4600 let destination_stream_id =
4601 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
4602 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
4603 let body_len =
4604 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
4605 let body_end = 22usize.saturating_add(body_len);
4606 if body_end != payload.len() {
4607 return Err(TelemetryError::Unpack("p2p stream frame length"));
4608 }
4609 Ok(Some(P2pStreamDecoded {
4610 flags,
4611 source_stream_id,
4612 destination_stream_id,
4613 sequence,
4614 payload: &payload[22..],
4615 }))
4616 }
4617
4618 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
4619 for _ in 0..u32::MAX {
4620 let id = st.next_p2p_stream_id.max(1);
4621 st.next_p2p_stream_id = st.next_p2p_stream_id.wrapping_add(1).max(1);
4622 if !st.p2p_stream_sessions.contains_key(&id) {
4623 return id;
4624 }
4625 }
4626 0
4627 }
4628
4629 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
4630 if pkt.data_type() != DataType::P2pMessage {
4631 return Ok(());
4632 }
4633 let decoded = Self::decode_p2p_payload(pkt.payload())?;
4634 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
4635 return self.dispatch_p2p_stream_frame(&decoded, &stream);
4636 }
4637 let handlers = {
4638 let st = self.state.lock();
4639 st.p2p_port_handlers
4640 .get(&decoded.destination_port)
4641 .cloned()
4642 .unwrap_or_default()
4643 };
4644 for handler in handlers {
4645 (handler.handler)(P2pMessage {
4646 source_hostname: decoded.source_hostname,
4647 source_address: decoded.source_address,
4648 source_port: decoded.source_port,
4649 destination_port: decoded.destination_port,
4650 payload: decoded.payload,
4651 })?;
4652 }
4653 Ok(())
4654 }
4655
4656 fn dispatch_p2p_stream_frame(
4657 &self,
4658 msg: &P2pDecoded<'_>,
4659 frame: &P2pStreamDecoded<'_>,
4660 ) -> TelemetryResult<()> {
4661 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
4662 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, crate::SharedBytes)> = None;
4663 {
4664 let mut st = self.state.lock();
4665 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
4666 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
4667 let existing_id = st.p2p_stream_sessions.iter().find_map(|(id, session)| {
4668 (session.peer_stream_id == frame.source_stream_id
4669 && session.local_port == msg.destination_port
4670 && session.peer_port == msg.source_port
4671 && session.peer_address == msg.source_address
4672 && session.peer_hostname.as_ref() == msg.source_hostname)
4673 .then_some(*id)
4674 });
4675 let local_id = if let Some(local_id) = existing_id {
4676 local_id
4677 } else {
4678 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
4679 if local_id == 0 {
4680 return Err(TelemetryError::Io("p2p stream id exhausted"));
4681 }
4682 st.p2p_stream_sessions.insert(
4683 local_id,
4684 P2pStreamSession {
4685 peer_hostname: peer_hostname.clone(),
4686 peer_address: msg.source_address,
4687 local_port: msg.destination_port,
4688 peer_port: msg.source_port,
4689 peer_stream_id: frame.source_stream_id,
4690 next_sequence: 1,
4691 connected: true,
4692 },
4693 );
4694 let handlers = st
4695 .p2p_stream_handlers
4696 .get(&msg.destination_port)
4697 .cloned()
4698 .unwrap_or_default();
4699 events.push(PendingP2pStreamEvent {
4700 handlers,
4701 kind: P2pStreamEventKind::Accepted,
4702 stream_id: local_id,
4703 peer_stream_id: frame.source_stream_id,
4704 peer_hostname: peer_hostname.clone(),
4705 peer_address: msg.source_address,
4706 local_port: msg.destination_port,
4707 peer_port: msg.source_port,
4708 });
4709 local_id
4710 };
4711 let dst =
4712 st.address_book
4713 .get(msg.source_hostname)
4714 .cloned()
4715 .unwrap_or(AddressBookEntry {
4716 hostname: peer_hostname,
4717 address: msg.source_address,
4718 requested_address: msg.source_address,
4719 mode: AddressAssignmentMode::Dynamic,
4720 birth_ms: self.clock.now_ms(),
4721 owner_hash: Self::sender_hash(msg.source_hostname),
4722 last_seen_ms: self.clock.now_ms(),
4723 });
4724 let payload = Self::encode_p2p_stream_payload(
4725 P2P_STREAM_SYN | P2P_STREAM_ACK,
4726 local_id,
4727 frame.source_stream_id,
4728 0,
4729 &[],
4730 )?;
4731 reply = Some((dst, msg.source_port, msg.destination_port, payload));
4732 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
4733 if let Some(session) = st.p2p_stream_sessions.get_mut(&frame.destination_stream_id)
4734 {
4735 session.peer_stream_id = frame.source_stream_id;
4736 session.connected = true;
4737 let peer_hostname = session.peer_hostname.clone();
4738 let peer_address = session.peer_address;
4739 let local_port = session.local_port;
4740 let peer_port = session.peer_port;
4741 let handlers = st
4742 .p2p_stream_handlers
4743 .get(&local_port)
4744 .cloned()
4745 .unwrap_or_default();
4746 events.push(PendingP2pStreamEvent {
4747 handlers,
4748 kind: P2pStreamEventKind::Connected,
4749 stream_id: frame.destination_stream_id,
4750 peer_stream_id: frame.source_stream_id,
4751 peer_hostname,
4752 peer_address,
4753 local_port,
4754 peer_port,
4755 });
4756 }
4757 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
4758 let kind = if frame.flags & P2P_STREAM_RST != 0 {
4759 P2pStreamEventKind::Reset
4760 } else if frame.flags & P2P_STREAM_FIN != 0 {
4761 P2pStreamEventKind::Closed
4762 } else {
4763 P2pStreamEventKind::Data
4764 };
4765 let session_id = if frame.destination_stream_id != 0 {
4766 Some(frame.destination_stream_id)
4767 } else {
4768 st.p2p_stream_sessions.iter().find_map(|(id, session)| {
4769 (session.peer_stream_id == frame.source_stream_id
4770 && session.local_port == msg.destination_port
4771 && session.peer_port == msg.source_port
4772 && session.peer_address == msg.source_address
4773 && session.peer_hostname.as_ref() == msg.source_hostname)
4774 .then_some(*id)
4775 })
4776 };
4777 if let Some(session_id) = session_id
4778 && let Some(session) = st.p2p_stream_sessions.get(&session_id)
4779 {
4780 let handlers = st
4781 .p2p_stream_handlers
4782 .get(&session.local_port)
4783 .cloned()
4784 .unwrap_or_default();
4785 events.push(PendingP2pStreamEvent {
4786 handlers,
4787 kind,
4788 stream_id: session_id,
4789 peer_stream_id: frame.source_stream_id,
4790 peer_hostname: session.peer_hostname.clone(),
4791 peer_address: session.peer_address,
4792 local_port: session.local_port,
4793 peer_port: session.peer_port,
4794 });
4795 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
4796 st.p2p_stream_sessions.remove(&session_id);
4797 }
4798 }
4799 }
4800 }
4801 if let Some((dst, dst_port, src_port, payload)) = reply {
4802 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
4803 }
4804 for pending in events {
4805 for handler in pending.handlers {
4806 (handler.handler)(P2pStreamEvent {
4807 kind: pending.kind,
4808 stream_id: pending.stream_id,
4809 peer_stream_id: pending.peer_stream_id,
4810 sequence: frame.sequence,
4811 peer_hostname: pending.peer_hostname.as_ref(),
4812 peer_address: pending.peer_address,
4813 local_port: pending.local_port,
4814 peer_port: pending.peer_port,
4815 payload: frame.payload,
4816 })?;
4817 }
4818 }
4819 Ok(())
4820 }
4821
4822 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) -> bool {
4828 let mut st = self.state.lock();
4829 if st.end_to_end_reliable_tx.contains_key(&packet_id)
4833 && st
4834 .reliable_return_routes
4835 .get(&packet_id)
4836 .is_some_and(|route| route.upstream_contract.is_some())
4837 {
4838 return false;
4839 }
4840 let now = self.clock.now_ms();
4844 let retry_ms = runtime_reliable_retransmit_ms();
4845 let duplicate = st.recent_rx.contains(&packet_id)
4846 || st
4847 .reliable_return_routes
4848 .get(&packet_id)
4849 .is_some_and(|route| route.locally_delivered);
4850 let last_replay_ms = if duplicate {
4851 now.saturating_sub(retry_ms)
4852 } else {
4853 now
4854 };
4855 let replace_side = !duplicate
4861 || st
4862 .reliable_return_routes
4863 .get(&packet_id)
4864 .is_none_or(|route| {
4865 now.saturating_sub(route.last_replay_ms) >= retry_ms
4866 || !Self::side_ref(&st, route.side)
4867 .is_ok_and(|side| side.opts.egress_enabled)
4868 });
4869 if !Self::remember_reliable_return_route_locked(&mut st, packet_id) {
4870 return true;
4871 }
4872 st.reliable_return_routes
4873 .entry(packet_id)
4874 .and_modify(|route| {
4875 if replace_side {
4876 route.side = side;
4877 }
4878 })
4879 .or_insert(ReliableReturnRouteState {
4880 side,
4881 locally_delivered: false,
4882 last_replay_ms,
4883 upstream_contract: None,
4884 });
4885 true
4886 }
4887
4888 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) -> bool {
4894 let cap = runtime_reliable_max_return_routes().max(1);
4895 st.reliable_return_route_order
4896 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
4897 while st.reliable_return_route_order.len() >= cap {
4898 let Some(index) = st
4899 .reliable_return_route_order
4900 .iter()
4901 .position(|id| !st.end_to_end_reliable_tx.contains_key(id))
4902 else {
4903 return false;
4904 };
4905 if let Some(oldest) = st.reliable_return_route_order.remove(index) {
4906 st.reliable_return_routes.remove(&oldest);
4907 }
4908 }
4909 st.reliable_return_route_order.push_back(packet_id);
4910 true
4911 }
4912
4913 #[cfg(feature = "discovery")]
4914 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
4915 let cap = runtime_reliable_max_end_to_end_pending().max(1);
4916 st.end_to_end_reliable_tx_order
4917 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
4918 while st.end_to_end_reliable_tx_order.len() >= cap {
4919 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
4920 st.end_to_end_reliable_tx.remove(&oldest);
4921 } else {
4922 break;
4923 }
4924 }
4925 st.end_to_end_reliable_tx_order.push_back(packet_id);
4926 }
4927
4928 #[cfg(feature = "discovery")]
4929 fn expected_end_to_end_destinations_locked(
4930 &self,
4931 st: &RouterInner,
4932 data: &RouterItem,
4933 exclude: Option<RouterSideId>,
4934 managed: bool,
4935 resolve_proxy_segment: bool,
4936 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
4937 let (eps, ty) = self.item_route_info(data)?;
4938 let frozen_targets = if resolve_proxy_segment {
4941 wire_format::TargetSenderSet::Hashes(&[])
4942 } else {
4943 match data {
4944 RouterItem::Packet(packet) => {
4945 wire_format::TargetSenderSet::Hashes(packet.wire_target_senders())
4946 }
4947 RouterItem::Packed(bytes) => {
4948 wire_format::peek_routing_envelope(bytes)?.target_sender_set()
4949 }
4950 }
4951 };
4952 let now_ms = self.clock.now_ms();
4953 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
4954 if managed || st.managed_variable_types.contains(&ty.as_u32()) {
4963 let mut out = BTreeMap::new();
4964 for (&side, route) in st.discovery_routes.iter() {
4965 if Some(side) == exclude {
4966 continue;
4967 }
4968 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4969 continue;
4970 }
4971 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
4972 continue;
4973 };
4974 if restrict_link_local && !side_ref.opts.link_local_enabled {
4975 continue;
4976 }
4977 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
4978 continue;
4979 }
4980 for (sender_id, sender_state) in route.announcers.iter() {
4981 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
4982 || !sender_state.reachable_network_variables.contains(&ty)
4983 || !Self::is_end_to_end_destination_sender(sender_id)
4984 {
4985 continue;
4986 }
4987 let owner = Self::sender_hash(sender_id);
4988 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
4989 continue;
4990 }
4991 out.insert(owner, side);
4992 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
4993 return Ok(out);
4994 }
4995 }
4996 }
4997 return Ok(out);
4998 }
4999 let prefer_best_overlap =
5000 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
5001 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
5002 let mut best_overlap = 0usize;
5003 let limit = runtime_reliable_max_end_to_end_pending().max(1);
5004 let mut out = BTreeMap::new();
5005 for (&side, route) in st.discovery_routes.iter() {
5006 if Some(side) == exclude {
5007 continue;
5008 }
5009 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5010 continue;
5011 }
5012 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5013 continue;
5014 };
5015 if restrict_link_local && !side_ref.opts.link_local_enabled {
5016 continue;
5017 }
5018 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5019 continue;
5020 }
5021 for sender_state in route.announcers.values() {
5022 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5023 continue;
5024 }
5025 for board in sender_state.topology_boards.iter() {
5026 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
5027 continue;
5028 }
5029 let owner = Self::sender_hash(&board.sender_id);
5030 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5031 continue;
5032 }
5033 let overlap = Self::endpoint_overlap_count(
5034 board.reachable_endpoints.iter().copied(),
5035 &scoring_eps,
5036 );
5037 if overlap > 0 {
5038 if prefer_best_overlap {
5039 if overlap < best_overlap {
5040 continue;
5041 }
5042 if overlap > best_overlap {
5043 out.clear();
5044 best_overlap = overlap;
5045 }
5046 if out.len() >= limit {
5050 continue;
5051 }
5052 }
5053 out.insert(owner, side);
5054 if !prefer_best_overlap && out.len() >= limit {
5055 return Ok(out);
5056 }
5057 }
5058 }
5059 }
5060 }
5061
5062 Ok(out)
5063 }
5064
5065 #[cfg(feature = "discovery")]
5066 #[allow(clippy::too_many_arguments)]
5067 fn discovered_route_candidates_locked(
5068 &self,
5069 st: &RouterInner,
5070 exclude: Option<RouterSideId>,
5071 ty: DataType,
5072 eps: &[DataEndpoint],
5073 target_senders: wire_format::TargetSenderSet<'_>,
5074 prefer_nonlocal: bool,
5075 preferred_timesync_source: Option<&str>,
5076 ) -> Vec<DiscoveryCandidateMatch> {
5077 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
5078 let now_ms = self.clock.now_ms();
5079 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
5080 let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
5081 exclude != Some(side)
5082 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5083 && route.reachable_network_variables.contains(&ty)
5084 });
5085 let mut out = Vec::new();
5086 let mut unique_endpoint_fallbacks = Vec::new();
5087
5088 for (&side, route) in st.discovery_routes.iter() {
5089 if exclude == Some(side)
5090 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
5091 {
5092 continue;
5093 }
5094 if restrict_link_local
5095 && st
5096 .sides
5097 .get(side)
5098 .and_then(|s| s.as_ref())
5099 .map(|s| !s.opts.link_local_enabled)
5100 .unwrap_or(true)
5101 {
5102 continue;
5103 }
5104 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5105 continue;
5106 }
5107 if !target_senders.is_empty()
5108 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
5109 {
5110 if scoring_eps
5111 .iter()
5112 .copied()
5113 .any(|ep| route.reachable.contains(&ep))
5114 {
5115 unique_endpoint_fallbacks.push(side);
5116 }
5117 continue;
5118 }
5119 if !target_senders.is_empty() {
5120 out.push(DiscoveryCandidateMatch {
5121 side,
5122 overlap: usize::MAX,
5123 });
5124 continue;
5125 }
5126 if has_variable_owner {
5127 if route.reachable_network_variables.contains(&ty) {
5128 out.push(DiscoveryCandidateMatch {
5129 side,
5130 overlap: usize::MAX,
5131 });
5132 }
5133 continue;
5134 }
5135 if preferred_timesync_source
5136 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
5137 {
5138 out.push(DiscoveryCandidateMatch {
5139 side,
5140 overlap: usize::MAX,
5141 });
5142 continue;
5143 }
5144 let overlap =
5145 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
5146 if overlap > 0 {
5147 out.push(DiscoveryCandidateMatch { side, overlap });
5148 }
5149 }
5150 if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
5151 out.push(DiscoveryCandidateMatch {
5152 side: unique_endpoint_fallbacks[0],
5153 overlap: usize::MAX,
5154 });
5155 }
5156 out
5157 }
5158
5159 #[cfg(feature = "discovery")]
5165 fn retain_shortest_discovery_candidates_locked(
5166 st: &RouterInner,
5167 matches: &mut Vec<DiscoveryCandidateMatch>,
5168 endpoints: &[DataEndpoint],
5169 target_senders: wire_format::TargetSenderSet<'_>,
5170 now_ms: u64,
5171 ) {
5172 if matches.len() <= 1 || endpoints.is_empty() {
5173 return;
5174 }
5175
5176 let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
5177 let route = st.discovery_routes.get(&side)?;
5178 route
5179 .announcers
5180 .iter()
5181 .filter(|(_, state)| {
5182 now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5183 })
5184 .filter_map(|(announcer, state)| {
5185 let is_target = |board: &TopologyBoardNode| {
5186 target_sender
5187 .map(|target| Self::sender_hash(&board.sender_id) == target)
5188 .unwrap_or_else(|| {
5189 endpoints
5190 .iter()
5191 .any(|endpoint| board.reachable_endpoints.contains(endpoint))
5192 })
5193 };
5194 if !state.topology_boards.iter().any(is_target) {
5197 return if let Some(target) = target_sender {
5198 state
5199 .topology_boards
5200 .iter()
5201 .any(|board| Self::sender_hash(&board.sender_id) == target)
5202 .then_some(usize::MAX / 2)
5203 } else {
5204 state
5205 .reachable
5206 .iter()
5207 .any(|endpoint| endpoints.contains(endpoint))
5208 .then_some(usize::MAX / 2)
5209 };
5210 }
5211
5212 let mut visited: BTreeSet<&str> = BTreeSet::new();
5213 let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
5214 while let Some((sender, distance)) = pending.pop_front() {
5215 if !visited.insert(sender) {
5216 continue;
5217 }
5218 for board in state.topology_boards.iter() {
5219 if board.sender_id == sender {
5220 if is_target(board) {
5221 return Some(distance);
5222 }
5223 for peer in &board.connections {
5224 if !visited.contains(peer.as_str()) {
5225 pending.push_back((peer.as_str(), distance + 1));
5226 }
5227 }
5228 } else if board.connections.iter().any(|peer| peer == sender)
5229 && !visited.contains(board.sender_id.as_str())
5230 {
5231 pending.push_back((board.sender_id.as_str(), distance + 1));
5232 }
5233 }
5234 }
5235 None
5236 })
5237 .min()
5238 };
5239
5240 let route_targets = target_senders
5241 .iter()
5242 .map(Some)
5243 .chain(core::iter::once(None).filter(|_| target_senders.is_empty()));
5244 let mut selected = BTreeSet::new();
5245 for target in route_targets {
5246 let distances: Vec<(RouterSideId, usize)> = matches
5247 .iter()
5248 .filter_map(|candidate| {
5249 route_distance(candidate.side, target)
5250 .map(|distance| (candidate.side, distance))
5251 })
5252 .collect();
5253 let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
5254 continue;
5255 };
5256 selected.extend(
5257 distances
5258 .into_iter()
5259 .filter(|(_, distance)| *distance == best)
5260 .map(|(side, _)| side),
5261 );
5262 }
5263 if !selected.is_empty() {
5264 matches.retain(|candidate| selected.contains(&candidate.side));
5265 }
5266 }
5267
5268 #[cfg(feature = "discovery")]
5272 fn select_endpoint_role_sides_locked(
5273 &self,
5274 st: &mut RouterInner,
5275 exclude: Option<RouterSideId>,
5276 endpoints: &[DataEndpoint],
5277 matches: &[DiscoveryCandidateMatch],
5278 now_ms: u64,
5279 ) -> Vec<RouterSideId> {
5280 let mut selected = Vec::new();
5281 for endpoint in endpoints {
5282 let mut candidates: Vec<_> = matches
5283 .iter()
5284 .copied()
5285 .filter(|candidate| {
5286 st.discovery_routes
5287 .get(&candidate.side)
5288 .is_some_and(|route| route.reachable.contains(endpoint))
5289 })
5290 .collect();
5291 Self::retain_shortest_discovery_candidates_locked(
5292 st,
5293 &mut candidates,
5294 core::slice::from_ref(endpoint),
5295 wire_format::TargetSenderSet::Hashes(&[]),
5296 now_ms,
5297 );
5298 if candidates
5299 .iter()
5300 .any(|candidate| selected.contains(&candidate.side))
5301 {
5302 continue;
5303 }
5304 let sides = self.apply_route_selection_locked(
5305 st,
5306 exclude,
5307 candidates
5308 .into_iter()
5309 .map(|candidate| candidate.side)
5310 .collect(),
5311 RouteSelectionOrigin::Discovered,
5312 );
5313 for side in sides {
5314 if !selected.contains(&side) {
5315 selected.push(side);
5316 }
5317 }
5318 }
5319 selected
5320 }
5321
5322 #[cfg(feature = "discovery")]
5323 fn select_discovered_candidate_sides_locked(
5324 &self,
5325 st: &mut RouterInner,
5326 exclude: Option<RouterSideId>,
5327 ty: DataType,
5328 target_senders: wire_format::TargetSenderSet<'_>,
5329 prefer_best_overlap: bool,
5330 matches: Vec<DiscoveryCandidateMatch>,
5331 ) -> Vec<RouterSideId> {
5332 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
5333 || !target_senders.is_empty()
5334 {
5335 RouteSelectionOrigin::Flood
5336 } else {
5337 RouteSelectionOrigin::Discovered
5338 };
5339
5340 let mut matches = matches;
5341 if matches.iter().any(|m| m.overlap == usize::MAX) {
5342 matches.retain(|m| m.overlap == usize::MAX);
5343 }
5344
5345 let selected: Vec<RouterSideId> = if prefer_best_overlap {
5346 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
5347 matches
5348 .into_iter()
5349 .filter(|m| m.overlap == best_overlap)
5350 .map(|m| m.side)
5351 .collect()
5352 } else {
5353 matches.into_iter().map(|m| m.side).collect()
5354 };
5355
5356 let has_distributed_variable_owners = selected.iter().any(|side| {
5363 st.discovery_routes
5364 .get(side)
5365 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
5366 });
5367 if DataType::try_named("HEARTBEAT") == Some(ty) || has_distributed_variable_owners {
5368 selected
5369 } else {
5370 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
5371 }
5372 }
5373
5374 #[cfg(feature = "discovery")]
5375 fn register_end_to_end_reliable_tx(
5376 &self,
5377 data: &RouterItem,
5378 destinations: Option<BTreeMap<u64, RouterSideId>>,
5379 ) -> TelemetryResult<bool> {
5380 let packet_id = Self::get_hash(data);
5381 let now_ms = self.clock.now_ms();
5382 let ty = match data {
5383 RouterItem::Packet(pkt) => pkt.data_type(),
5384 RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
5385 };
5386 let mut st = self.state.lock();
5387 #[cfg(feature = "discovery")]
5388 let mut pending_destinations = match destinations {
5389 Some(destinations) => destinations,
5390 None => self.expected_end_to_end_destinations_locked(&st, data, None, false, false)?,
5391 };
5392 #[cfg(not(feature = "discovery"))]
5393 let mut pending_destinations = destinations.unwrap_or_default();
5394 let frozen = match data {
5398 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
5399 RouterItem::Packed(bytes) => {
5400 wire_format::peek_routing_envelope(bytes)?
5401 .target_senders()
5402 .len()
5403 != 0
5404 }
5405 };
5406 self.filter_trackable_end_to_end_destinations_locked(
5407 &st,
5408 ty,
5409 &mut pending_destinations,
5410 frozen,
5411 );
5412 let tracked_destinations = !pending_destinations.is_empty();
5413 if let Some(existing) = st.end_to_end_reliable_tx.get(&packet_id) {
5414 return Ok(!existing.waiting_for_predecessor);
5415 }
5416 if st.end_to_end_reliable_tx.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5418 return Err(TelemetryError::Io("reliable window full"));
5419 }
5420 let ordered_type =
5421 matches!(reliable_mode(ty), crate::ReliableMode::Ordered).then_some(ty.as_u32());
5422 let waiting_for_predecessor = ordered_type.is_some()
5423 && st
5424 .end_to_end_reliable_tx
5425 .values()
5426 .any(|sent| sent.ordered_type == ordered_type);
5427 #[cfg(feature = "std")]
5428 if reliable_trace_enabled() {
5429 eprintln!(
5430 "[sedsnet-reliable] register time={now_ms} type={} id={packet_id:#x} waiting={waiting_for_predecessor} targets={pending_destinations:?}",
5431 ty.as_u32()
5432 );
5433 }
5434 #[cfg(feature = "compression")]
5435 let retained = match data {
5436 RouterItem::Packed(bytes) => RouterItem::Packed(
5437 wire_format::payload_encoding_for_retention(
5438 bytes.as_ref(),
5439 st.memory.max_queue_budget,
5440 )?
5441 .map(crate::shared_bytes::convert)
5442 .unwrap_or_else(|| bytes.clone()),
5443 ),
5444 packet => packet.clone(),
5445 };
5446 #[cfg(not(feature = "compression"))]
5447 let retained = data.clone();
5448 #[cfg(feature = "compact-packet-store")]
5449 let mut retained = retained;
5450 #[cfg(feature = "compact-packet-store")]
5451 retained.park_for_queue()?;
5452 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
5453 st.end_to_end_reliable_tx.insert(
5454 packet_id,
5455 EndToEndReliableSent {
5456 ordered_type,
5457 waiting_for_predecessor,
5458 data: retained,
5459 pending_destinations,
5460 tracked_destinations,
5461 last_send_ms: now_ms,
5462 retries: 0,
5463 queued: false,
5464 },
5465 );
5466 Ok(!waiting_for_predecessor)
5467 }
5468
5469 fn release_ordered_successors(&self) -> TelemetryResult<()> {
5472 let ready = {
5473 let mut st = self.state.lock();
5474 let RouterInner {
5475 end_to_end_reliable_tx_order: order,
5476 end_to_end_reliable_tx: pending,
5477 ..
5478 } = &mut *st;
5479 let mut ready = Vec::new();
5480 for (index, id) in order.iter().copied().enumerate() {
5481 let Some(sent) = pending.get(&id) else {
5482 continue;
5483 };
5484 if !sent.waiting_for_predecessor {
5485 continue;
5486 }
5487 let Some(ty) = sent.ordered_type else {
5488 continue;
5489 };
5490 if order.iter().take(index).any(|earlier| {
5494 pending
5495 .get(earlier)
5496 .is_some_and(|sent| sent.ordered_type == Some(ty))
5497 }) {
5498 continue;
5499 }
5500 let sent = pending.get_mut(&id).expect("retained ordered entry");
5501 sent.waiting_for_predecessor = false;
5502 sent.last_send_ms = self.clock.now_ms();
5503 ready.push(id);
5504 }
5505 ready
5506 };
5507 for id in ready {
5508 self.queue_end_to_end_reliable_retransmit(id)?;
5509 }
5510 Ok(())
5511 }
5512
5513 #[cfg(feature = "discovery")]
5514 fn reconcile_end_to_end_reliable_destinations_locked(
5515 &self,
5516 st: &mut RouterInner,
5517 ) -> TelemetryResult<()> {
5518 if st.end_to_end_reliable_tx.is_empty() {
5519 return Ok(());
5520 }
5521 let active_destinations = self.active_end_to_end_destinations_locked(st);
5522 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
5523
5524 for packet_id in packet_ids {
5525 let Some(data) = st
5526 .end_to_end_reliable_tx
5527 .get(&packet_id)
5528 .map(|sent| sent.data.clone())
5529 else {
5530 continue;
5531 };
5532 let excluded = st
5533 .reliable_return_routes
5534 .get(&packet_id)
5535 .filter(|route| route.upstream_contract.is_some())
5536 .map(|route| route.side);
5537 let expected =
5538 self.expected_end_to_end_destinations_locked(st, &data, excluded, false, false)?;
5539 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
5540 continue;
5541 };
5542 if !sent.tracked_destinations {
5543 continue;
5544 }
5545 for (sender, side) in &mut sent.pending_destinations {
5549 if let Some(next_side) = expected.get(sender).or_else(|| {
5550 active_destinations
5551 .get(sender)
5552 .filter(|side| Some(**side) != excluded)
5553 }) {
5554 *side = *next_side;
5555 }
5556 }
5557 }
5558
5559 Ok(())
5560 }
5561
5562 #[cfg(feature = "discovery")]
5563 fn active_end_to_end_destinations_locked(
5564 &self,
5565 st: &RouterInner,
5566 ) -> BTreeMap<u64, RouterSideId> {
5567 let now_ms = self.clock.now_ms();
5568 let mut out = BTreeMap::new();
5569 for (&side, route) in st.discovery_routes.iter() {
5570 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5571 continue;
5572 }
5573 for (sender_name, sender_state) in &route.announcers {
5574 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5575 continue;
5576 }
5577 if Self::is_end_to_end_destination_sender(sender_name) {
5578 out.insert(Self::sender_hash(sender_name), side);
5579 }
5580 for board in sender_state.topology_boards.iter() {
5581 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
5582 continue;
5583 }
5584 out.insert(Self::sender_hash(&board.sender_id), side);
5585 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5586 return out;
5587 }
5588 }
5589 }
5590 }
5591 out
5592 }
5593
5594 #[cfg(feature = "discovery")]
5595 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
5596 matches!(
5597 st.sides
5598 .get(side)
5599 .and_then(Option::as_ref)
5600 .map(|side| &side.tx_handler),
5601 Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
5602 )
5603 }
5604
5605 #[cfg(feature = "discovery")]
5606 fn filter_trackable_end_to_end_destinations_locked(
5607 &self,
5608 st: &RouterInner,
5609 ty: DataType,
5610 pending: &mut BTreeMap<u64, RouterSideId>,
5611 frozen: bool,
5612 ) {
5613 if DataType::try_named("HEARTBEAT") == Some(ty) {
5619 pending.clear();
5620 return;
5621 }
5622 if frozen {
5623 return;
5624 }
5625 let now_ms = self.clock.now_ms();
5626 pending.retain(|_, side| {
5627 Self::side_supports_end_to_end_tracking_locked(st, *side)
5628 && (st.managed_variable_types.contains(&ty.as_u32())
5629 || is_reliable_type(ty)
5630 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
5631 });
5632 }
5633
5634 #[cfg(feature = "discovery")]
5635 fn side_has_multiple_announcers_locked(
5636 &self,
5637 st: &RouterInner,
5638 side: RouterSideId,
5639 now_ms: u64,
5640 ) -> bool {
5641 st.discovery_routes
5642 .get(&side)
5643 .map(|route| {
5644 route
5645 .announcers
5646 .values()
5647 .filter(|sender| {
5648 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5649 })
5650 .take(2)
5651 .count()
5652 > 1
5653 })
5654 .unwrap_or(false)
5655 }
5656
5657 #[cfg(not(feature = "discovery"))]
5658 fn side_has_multiple_announcers_locked(
5659 &self,
5660 _st: &RouterInner,
5661 _side: RouterSideId,
5662 _now_ms: u64,
5663 ) -> bool {
5664 false
5665 }
5666
5667 fn queue_end_to_end_reliable_ack(
5668 &self,
5669 pkt: &Packet,
5670 called_from_queue: bool,
5671 ) -> TelemetryResult<()> {
5672 self.queue_end_to_end_reliable_ack_for_packet_id(
5673 pkt.packet_id(),
5674 pkt.sender(),
5675 called_from_queue,
5676 )
5677 }
5678
5679 fn queue_end_to_end_reliable_ack_for_packet_id(
5680 &self,
5681 packet_id: u64,
5682 original_sender: &str,
5683 called_from_queue: bool,
5684 ) -> TelemetryResult<()> {
5685 if !self.cfg.reliable_enabled() {
5686 return Ok(());
5687 }
5688 let original_sender_hash = {
5689 let mut st = self.state.lock();
5690 if let Some(route) = st.reliable_return_routes.get_mut(&packet_id) {
5695 route.locally_delivered = true;
5696 }
5697 Self::sender_hash(&Self::canonical_sender_locked(&st, original_sender))
5698 };
5699 self.queue_end_to_end_reliable_ack_for_sender_hash(
5700 packet_id,
5701 original_sender_hash,
5702 called_from_queue,
5703 )
5704 }
5705
5706 fn queue_end_to_end_reliable_ack_for_sender_hash(
5707 &self,
5708 packet_id: u64,
5709 original_sender_hash: u64,
5710 called_from_queue: bool,
5711 ) -> TelemetryResult<()> {
5712 if !self.cfg.reliable_enabled() {
5713 return Ok(());
5714 }
5715 let ack_sender = self.encode_end_to_end_ack_sender();
5716 let ack = Packet::new_local(
5717 DataType::ReliableAck,
5718 message_meta(DataType::ReliableAck).endpoints_owned(),
5719 Arc::from(ack_sender),
5720 self.packet_timestamp_ms(),
5721 crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
5722 )?;
5723 let local_sender = self.sender_arc();
5724 let ack = self.attach_wire_contract_to_item(
5728 RouterItem::Packet(ack),
5729 &[
5730 Self::sender_hash(local_sender.as_ref()),
5731 original_sender_hash,
5732 ],
5733 )?;
5734 let return_side = {
5738 let st = self.state.lock();
5739 st.reliable_return_routes
5740 .get(&packet_id)
5741 .map(|route| route.side)
5742 };
5743 let item = match return_side {
5744 Some(dst) => RouterTxItem::ToSide {
5745 src: None,
5746 dst,
5747 data: ack,
5748 },
5749 None => RouterTxItem::Broadcast(ack),
5750 };
5751 self.emit_internal_tx(item, true, called_from_queue)
5752 }
5753
5754 fn emit_internal_tx(
5755 &self,
5756 item: RouterTxItem,
5757 ignore_local: bool,
5758 called_from_queue: bool,
5759 ) -> TelemetryResult<()> {
5760 if called_from_queue {
5761 self.tx_queue_item_with_flags(item, ignore_local)
5762 } else {
5763 self.tx_item_impl(item, ignore_local, false)
5764 }
5765 }
5766
5767 fn emit_internal_tx_with_priority(
5768 &self,
5769 item: RouterTxItem,
5770 ignore_local: bool,
5771 priority: u8,
5772 called_from_queue: bool,
5773 ) -> TelemetryResult<()> {
5774 if called_from_queue {
5775 self.tx_queue_item_with_priority(item, ignore_local, priority)
5776 } else {
5777 self.tx_item_impl(item, ignore_local, false)
5778 }
5779 }
5780
5781 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
5782 let now = self.clock.now_ms();
5783 let result = self.tx_queue_item_with_priority(
5784 RouterTxItem::EndToEndReplay { packet_id },
5785 true,
5786 Self::router_item_priority_bumped(DataType::ReliableAck),
5787 );
5788 if result.is_ok()
5792 && let Some(sent) = self.state.lock().end_to_end_reliable_tx.get_mut(&packet_id)
5793 {
5794 sent.last_send_ms = sent.last_send_ms.max(now);
5795 }
5796 result
5797 }
5798
5799 fn end_to_end_retransmit_sides(
5800 &self,
5801 packet_id: u64,
5802 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
5803 let mut st = self.state.lock();
5804 let (data, tracked_destinations, mut sides) = {
5805 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
5806 #[cfg(feature = "std")]
5807 if reliable_trace_enabled() {
5808 eprintln!(
5809 "[sedsnet-reliable] replay time={} id={packet_id:#x} retries={} targets={:?}",
5810 self.clock.now_ms(),
5811 sent.retries,
5812 sent.pending_destinations
5813 );
5814 }
5815 sent.queued = false;
5816 sent.last_send_ms = self.clock.now_ms();
5817 let data = sent.data.clone();
5818 let tracked_destinations = sent.tracked_destinations;
5819 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
5820 (data, tracked_destinations, sides)
5821 };
5822 if tracked_destinations && sides.is_empty() {
5823 st.end_to_end_reliable_tx.remove(&packet_id);
5824 return None;
5825 }
5826 crate::small_sort::sort(&mut sides);
5827 sides.dedup();
5828 Some((data, sides))
5829 }
5830
5831 fn item_type_priority(data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
5832 let (ty, control) = match data {
5833 RouterItem::Packet(pkt) => (
5834 pkt.data_type(),
5835 pkt.payload().len() <= 16
5836 && pkt.endpoints().len() <= 3
5837 && crate::small_frozen_control(
5838 pkt.data_type(),
5839 pkt.wire_target_senders().len(),
5840 pkt.wire_shape(),
5841 ),
5842 ),
5843 RouterItem::Packed(bytes) => {
5844 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
5845 let env = &frame.envelope;
5846 if bytes.len() <= 128 && frame.ack_only() {
5849 return Ok((
5850 env.ty,
5851 crate::transport_priority(crate::DataType::ReliableAck),
5852 ));
5853 }
5854 let control = bytes.len() <= 128
5855 && bytes.first().is_some_and(|flags| flags & 0x11 == 0)
5856 && env.endpoints().take(4).count() <= 3
5857 && crate::small_frozen_control(
5858 env.ty,
5859 env.target_senders().len(),
5860 env.wire_shape,
5861 );
5862 (env.ty, control)
5863 }
5864 };
5865 let priority = crate::scheduler_priority(ty);
5866 Ok((ty, if control { priority.max(254) } else { priority }))
5867 }
5868
5869 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
5870 Self::item_type_priority(data).map(|(_, priority)| priority)
5871 }
5872
5873 fn item_service_type_priority(&self, data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
5874 let (ty, priority) = Self::item_type_priority(data)?;
5875 let priority = if self.is_managed_variable_type(ty) {
5876 crate::transport_priority(DataType::ManagedVariableValue)
5877 } else {
5878 priority
5879 };
5880 Ok((ty, priority))
5881 }
5882
5883 fn item_service_priority(&self, data: &RouterItem) -> TelemetryResult<u8> {
5884 self.item_service_type_priority(data)
5885 .map(|(_, priority)| priority)
5886 }
5887
5888 #[inline]
5889 fn router_item_priority_bumped(ty: DataType) -> u8 {
5890 crate::scheduler_priority(ty).saturating_add(16)
5891 }
5892
5893 #[inline]
5894 fn is_transient_tx_error(err: &TelemetryError) -> bool {
5895 matches!(
5896 err,
5897 TelemetryError::Io("side tx busy" | "memory pressure" | "reliable window full")
5898 )
5899 }
5900
5901 #[cfg(feature = "timesync")]
5902 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
5903 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
5904 || st
5905 .clock
5906 .current_time(now_mono_ns)
5907 .and_then(|reading| reading.unix_time_ms)
5908 .is_some()
5909 }
5910
5911 #[cfg(feature = "timesync")]
5912 fn reconcile_pending_timesync_request_locked(
5913 st: &mut TimeSyncRuntime,
5914 leader: &Option<TimeSyncLeader>,
5915 now_ms: u64,
5916 ) {
5917 let active_remote = match leader {
5918 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
5919 _ => None,
5920 };
5921 let should_clear = st
5922 .pending_request
5923 .as_ref()
5924 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
5925 if should_clear {
5926 st.pending_request = None;
5927 st.next_request_mono_ms = now_ms;
5928 }
5929 }
5930
5931 #[inline]
5933 fn enqueue_to_sides(
5934 &self,
5935 data: RouterItem,
5936 exclude: Option<RouterSideId>,
5937 ignore_local: bool,
5938 ) -> TelemetryResult<()> {
5939 let Some(data) = self.clear_intermediate_hop_targets(data, true)? else {
5940 return Ok(());
5941 };
5942 let plan = self.remote_side_plan(&data, exclude)?;
5943 let mut st = self.state.lock();
5944 let ty = Self::item_data_type(&data)?;
5945 let priority = if st.managed_variable_types.contains(&ty.as_u32()) {
5946 crate::transport_priority(DataType::ManagedVariableValue)
5947 } else {
5948 Self::router_item_priority(&data)?
5949 };
5950
5951 let RemoteSidePlan::Target(sides) = plan;
5952 for idx in sides {
5953 st.push_transmit(TxQueued {
5954 disposable: true,
5955 item: RouterTxItem::ToSide {
5956 src: exclude,
5957 dst: idx,
5958 data: data.clone(),
5959 },
5960 ignore_local,
5961 priority,
5962 delivery_state: QueuedDelivery::Untracked,
5963 accepted_type: 0,
5964 deferred_since_ms: 0,
5965 retry_after_ms: 0,
5966 })?;
5967 }
5968
5969 Ok(())
5970 }
5971
5972 fn relay_send(
5973 &self,
5974 data: RouterItem,
5975 src: Option<RouterSideId>,
5976 called_from_queue: bool,
5977 ) -> TelemetryResult<()> {
5978 if called_from_queue {
5979 return self.enqueue_to_sides(data, src, true);
5980 }
5981
5982 let Some(data) = self.clear_intermediate_hop_targets(data, false)? else {
5983 return Ok(());
5984 };
5985 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
5986 for side in sides {
5987 self.tx_item_impl(
5988 RouterTxItem::ToSide {
5989 src,
5990 dst: side,
5991 data: data.clone(),
5992 },
5993 true,
5994 false,
5995 )?;
5996 }
5997
5998 Ok(())
5999 }
6000
6001 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
6002 match data {
6003 RouterItem::Packet(pkt) => {
6004 pkt.validate()?;
6005 let mut eps = pkt.endpoints().to_vec();
6006 crate::small_sort::sort(&mut eps);
6007 eps.dedup();
6008 Ok((eps, pkt.data_type()))
6009 }
6010 RouterItem::Packed(bytes) => {
6011 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6012 let mut eps: Vec<DataEndpoint> = env.endpoints().collect();
6013 crate::small_sort::sort(&mut eps);
6014 eps.dedup();
6015 Ok((eps, env.ty))
6016 }
6017 }
6018 }
6019
6020 fn unpack_packet_for_role(&self, bytes: &[u8]) -> TelemetryResult<Packet> {
6021 let managed = crate::memory_admission::supports_work_kinds()
6022 && wire_format::scheduling_type(bytes).is_some_and(|ty| {
6023 !is_internal_control_type(ty) && self.is_managed_variable_type(ty)
6024 });
6025 wire_format::unpack_packet_with_role(bytes, managed)
6026 }
6027
6028 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
6029 match data {
6030 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
6031 RouterItem::Packed(bytes) => Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.ty),
6032 }
6033 }
6034
6035 fn e2e_crypto_supported(&self) -> bool {
6036 #[cfg(feature = "cryptography")]
6037 {
6038 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
6039 && crate::crypto::registered_crypto_available()
6040 }
6041 #[cfg(not(feature = "cryptography"))]
6042 {
6043 false
6044 }
6045 }
6046
6047 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
6048 if is_internal_control_type(ty) {
6049 return false;
6050 }
6051 match self.cfg.e2e_encryption() {
6052 RouterE2eEncryptionMode::Disabled => {
6053 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
6054 }
6055 RouterE2eEncryptionMode::RequiredOnly => {
6056 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
6057 }
6058 RouterE2eEncryptionMode::Preferred => matches!(
6059 message_e2e_encryption_policy(ty),
6060 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
6061 ),
6062 RouterE2eEncryptionMode::ForceAll => true,
6063 }
6064 }
6065
6066 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
6067 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
6068 return Err(TelemetryError::BadArg);
6069 }
6070 Ok(())
6071 }
6072
6073 #[cfg(feature = "cryptography")]
6074 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
6075 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
6076 Some(wire_format::E2eSealConfig {
6077 key_id: self.cfg.e2e_key_id(),
6078 })
6079 } else {
6080 None
6081 }
6082 }
6083
6084 #[inline]
6085 fn pack_packet_for_router(
6086 &self,
6087 pkt: &Packet,
6088 reliable: Option<wire_format::ReliableHeader>,
6089 ) -> TelemetryResult<crate::SharedBytes> {
6090 #[cfg(feature = "compression")]
6091 {
6092 #[cfg(feature = "cryptography")]
6093 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
6094 #[cfg(not(feature = "cryptography"))]
6095 let e2e = None;
6096 let reliable = reliable.or_else(|| {
6097 is_reliable_type(pkt.data_type()).then_some(wire_format::ReliableHeader {
6098 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
6099 seq: 0,
6100 ack: 0,
6101 })
6102 });
6103 wire_format::pack_routed_packet(
6104 pkt,
6105 reliable,
6106 pkt.wire_shape(),
6107 pkt.wire_target_senders(),
6108 e2e,
6109 )
6110 .map(crate::shared_bytes::convert)
6111 }
6112 #[cfg(not(feature = "compression"))]
6113 {
6114 #[cfg(feature = "cryptography")]
6115 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
6116 return (wire_format::pack_packet_with_wire_contract_e2e(
6117 pkt,
6118 reliable,
6119 pkt.wire_shape(),
6120 pkt.wire_target_senders(),
6121 e2e,
6122 ))
6123 .map(crate::shared_bytes::convert);
6124 }
6125 Ok(match reliable {
6126 Some(hdr) => {
6127 crate::shared_bytes::convert(wire_format::pack_packet_with_reliable(pkt, hdr))
6128 }
6129 None => crate::shared_bytes::convert(wire_format::pack_packet(pkt)),
6130 })
6131 }
6132 }
6133
6134 #[inline]
6135 fn pack_packet_for_contract(
6136 &self,
6137 pkt: &Packet,
6138 reliable: Option<wire_format::ReliableHeader>,
6139 shape: Option<MessageElement>,
6140 target_senders: &[u64],
6141 ) -> TelemetryResult<crate::SharedBytes> {
6142 #[cfg(feature = "compression")]
6143 {
6144 #[cfg(feature = "cryptography")]
6145 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
6146 #[cfg(not(feature = "cryptography"))]
6147 let e2e = None;
6148 wire_format::pack_routed_packet(pkt, reliable, shape, target_senders, e2e)
6149 .map(crate::shared_bytes::convert)
6150 }
6151 #[cfg(not(feature = "compression"))]
6152 {
6153 #[cfg(feature = "cryptography")]
6154 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
6155 return (wire_format::pack_packet_with_wire_contract_e2e(
6156 pkt,
6157 reliable,
6158 shape,
6159 target_senders,
6160 e2e,
6161 ))
6162 .map(crate::shared_bytes::convert);
6163 }
6164 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
6165 .map(crate::shared_bytes::convert)
6166 }
6167 }
6168
6169 #[cfg(feature = "cryptography")]
6170 #[inline]
6171 fn prepare_packed_for_remote(
6172 &self,
6173 bytes: crate::SharedBytes,
6174 reliable_override: Option<Option<wire_format::ReliableHeader>>,
6175 ) -> TelemetryResult<crate::SharedBytes> {
6176 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
6177 if frame.ack_only() || self.e2e_seal_config_for_type(frame.envelope.ty).is_none() {
6178 return Ok(bytes);
6179 }
6180 let reliable = reliable_override.unwrap_or(frame.reliable);
6181 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
6182 self.pack_packet_for_contract(
6183 &pkt,
6184 reliable,
6185 frame.envelope.wire_shape,
6186 &frame.envelope.target_senders,
6187 )
6188 }
6189
6190 fn item_requires_end_to_end_tracking(
6194 &self,
6195 data: &RouterItem,
6196 ty: DataType,
6197 ) -> TelemetryResult<bool> {
6198 if is_reliable_type(ty) || self.is_managed_variable_type(ty) {
6199 return Ok(true);
6200 }
6201 match data {
6202 RouterItem::Packet(packet) => Ok(!packet.wire_target_senders().is_empty()),
6203 RouterItem::Packed(bytes) => {
6204 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
6205 Ok(frame.reliable.is_some() || frame.envelope.target_senders().len() != 0)
6206 }
6207 }
6208 }
6209
6210 #[cfg(feature = "discovery")]
6211 fn item_target_senders<'a>(
6212 &self,
6213 data: &'a RouterItem,
6214 ) -> TelemetryResult<wire_format::TargetSenderSet<'a>> {
6215 match data {
6216 RouterItem::Packet(pkt) => Ok(wire_format::TargetSenderSet::Hashes(
6217 pkt.wire_target_senders(),
6218 )),
6219 RouterItem::Packed(bytes) => {
6220 Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.target_sender_set())
6221 }
6222 }
6223 }
6224
6225 fn item_targets_another_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
6226 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
6227 match data {
6228 RouterItem::Packet(pkt) => {
6229 Ok(pkt.wire_target_senders().iter().any(|v| *v != local_hash))
6230 }
6231 RouterItem::Packed(bytes) => {
6232 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6233 Ok(env.target_senders().any(|target| target != local_hash))
6234 }
6235 }
6236 }
6237
6238 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
6239 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
6240 match data {
6241 RouterItem::Packet(pkt) => Ok(pkt.wire_target_senders().is_empty()
6242 || pkt.wire_target_senders().contains(&local_hash)),
6243 RouterItem::Packed(bytes) => {
6244 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6245 Ok(env.target_senders().len() == 0
6246 || env.target_senders().any(|target| target == local_hash))
6247 }
6248 }
6249 }
6250
6251 fn item_targets_router_as_intermediate_hop(
6252 &self,
6253 data: &RouterItem,
6254 endpoints: &[DataEndpoint],
6255 ) -> TelemetryResult<bool> {
6256 let local_target = match data {
6259 RouterItem::Packet(pkt) => {
6260 if pkt.wire_target_senders().is_empty() || is_internal_control_type(pkt.data_type())
6261 {
6262 return Ok(false);
6263 }
6264 pkt.wire_target_senders()
6265 .contains(&Self::sender_hash(self.sender_arc().as_ref()))
6266 }
6267 RouterItem::Packed(bytes) => {
6268 let env = wire_format::peek_routing_envelope(bytes)?;
6269 if env.target_senders().len() == 0 || is_internal_control_type(env.ty) {
6270 return Ok(false);
6271 }
6272 let local = Self::sender_hash(self.sender_arc().as_ref());
6273 env.target_senders().any(|owner| owner == local)
6274 }
6275 };
6276 Ok(local_target
6277 && !endpoints.iter().copied().any(|endpoint| {
6278 self.endpoint_has_packet_handler(endpoint)
6279 || self.endpoint_has_packed_handler(endpoint)
6280 }))
6281 }
6282
6283 fn clear_intermediate_hop_targets(
6284 &self,
6285 data: RouterItem,
6286 called_from_queue: bool,
6287 ) -> TelemetryResult<Option<RouterItem>> {
6288 let has_targets = match &data {
6292 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
6293 RouterItem::Packed(bytes) => {
6294 wire_format::peek_routing_envelope(bytes)?
6295 .target_senders()
6296 .len()
6297 != 0
6298 }
6299 };
6300 if !has_targets {
6301 return Ok(Some(data));
6302 }
6303 let (endpoints, _) = self.item_route_info(&data)?;
6304 if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
6305 return Ok(Some(data));
6306 }
6307 let pkt = match &data {
6308 RouterItem::Packet(pkt) => pkt.clone(),
6309 RouterItem::Packed(bytes) => self.unpack_packet_for_role(bytes.as_ref())?,
6310 };
6311 let reliable = match &data {
6312 RouterItem::Packet(_) => None,
6313 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
6314 };
6315 let shape = match &data {
6316 RouterItem::Packet(pkt) => pkt.wire_shape(),
6317 RouterItem::Packed(bytes) => {
6318 wire_format::peek_routing_envelope(bytes.as_ref())?.wire_shape
6319 }
6320 };
6321 let upstream_id = Self::get_hash(&data);
6322 #[cfg(feature = "discovery")]
6323 let destinations = {
6324 let st = self.state.lock();
6325 let upstream = st
6326 .reliable_return_routes
6327 .get(&upstream_id)
6328 .map(|route| route.side);
6329 let mut pending =
6330 self.expected_end_to_end_destinations_locked(&st, &data, upstream, true, true)?;
6331 if pending.is_empty() {
6332 pending = self
6333 .expected_end_to_end_destinations_locked(&st, &data, upstream, false, true)?;
6334 }
6335 pending
6336 };
6337 #[cfg(feature = "discovery")]
6338 let targets: Vec<u64> = destinations.keys().copied().collect();
6339 #[cfg(not(feature = "discovery"))]
6340 let targets: Vec<u64> = Vec::new();
6341 #[cfg(feature = "discovery")]
6342 if targets.is_empty() {
6343 if self.packet_has_local_delivery(&pkt) {
6344 self.queue_end_to_end_reliable_ack_for_packet_id(
6345 upstream_id,
6346 pkt.sender(),
6347 called_from_queue,
6348 )?;
6349 return Ok(Some(data));
6350 }
6351 if !is_reliable_type(pkt.data_type()) && reliable.is_none() {
6352 return Err(TelemetryError::Io("no discovered route"));
6353 }
6354 }
6355 let forwarded = self.pack_packet_for_contract(&pkt, reliable, shape, &targets)?;
6359 let downstream_id = wire_format::packet_id_from_wire(forwarded.as_ref())?;
6360 let forwarded = RouterItem::Packed(forwarded);
6361 #[cfg(feature = "discovery")]
6362 let already_pending = self
6363 .state
6364 .lock()
6365 .end_to_end_reliable_tx
6366 .contains_key(&downstream_id);
6367 #[cfg(feature = "discovery")]
6368 let ready = self.register_end_to_end_reliable_tx(&forwarded, Some(destinations))?;
6369 #[cfg(not(feature = "discovery"))]
6370 let (ready, already_pending) = {
6371 let _ = called_from_queue;
6372 (true, false)
6373 };
6374 #[cfg(feature = "std")]
6375 if reliable_trace_enabled() {
6376 eprintln!(
6377 "[sedsnet-reliable] proxy retain upstream={upstream_id:#x} downstream={downstream_id:#x} existing={already_pending} ready={ready}"
6378 );
6379 }
6380 {
6381 let mut st = self.state.lock();
6382 let Some(mut route) = st.reliable_return_routes.get(&upstream_id).cloned() else {
6383 st.end_to_end_reliable_tx.remove(&downstream_id);
6384 return Err(TelemetryError::Io("no reliable return route"));
6385 };
6386 route.upstream_contract = Some((
6387 upstream_id,
6388 Self::sender_hash(&Self::canonical_sender_locked(&st, pkt.sender())),
6389 ));
6390 if !Self::remember_reliable_return_route_locked(&mut st, downstream_id) {
6391 st.end_to_end_reliable_tx.remove(&downstream_id);
6392 return Err(TelemetryError::Io("reliable return route window full"));
6393 }
6394 st.reliable_return_routes.insert(downstream_id, route);
6395 }
6396 Ok((ready && !already_pending).then_some(forwarded))
6397 }
6398
6399 #[cfg(feature = "discovery")]
6400 fn side_matches_target_senders_locked(
6401 st: &RouterInner,
6402 side: RouterSideId,
6403 target_senders: wire_format::TargetSenderSet<'_>,
6404 now_ms: u64,
6405 ) -> bool {
6406 st.discovery_routes
6407 .get(&side)
6408 .map(|route| {
6409 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6410 return false;
6411 }
6412 route.announcers.iter().any(|(announcer, sender_state)| {
6413 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6414 return false;
6415 }
6416 target_senders.contains(Self::sender_hash(announcer))
6417 || sender_state
6418 .advertised_address
6419 .is_some_and(|address| target_senders.contains(u64::from(address)))
6420 || sender_state.topology_boards.iter().any(|board| {
6421 target_senders.contains(Self::sender_hash(&board.sender_id))
6422 })
6423 })
6424 })
6425 .unwrap_or(false)
6426 }
6427
6428 fn attach_wire_contract_to_item(
6429 &self,
6430 data: RouterItem,
6431 target_senders: &[u64],
6432 ) -> TelemetryResult<RouterItem> {
6433 let pkt = match data {
6434 RouterItem::Packet(packet) => packet,
6435 RouterItem::Packed(bytes) => {
6436 if wire_format::peek_routing_envelope(&bytes)?
6439 .target_senders()
6440 .len()
6441 != 0
6442 {
6443 return Ok(RouterItem::Packed(bytes));
6444 }
6445 self.unpack_packet_for_role(&bytes)?
6446 }
6447 };
6448 let reliable = if is_reliable_type(pkt.data_type()) {
6449 Some(wire_format::ReliableHeader {
6450 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
6451 seq: 0,
6452 ack: 0,
6453 })
6454 } else {
6455 None
6456 };
6457 let targets = if pkt.wire_target_senders().is_empty() {
6460 target_senders
6461 } else {
6462 pkt.wire_target_senders()
6463 };
6464 let bytes = self.pack_packet_for_contract(
6465 &pkt,
6466 reliable,
6467 pkt.wire_shape()
6468 .or_else(|| Some(message_meta(pkt.data_type()).element)),
6469 targets,
6470 )?;
6471 Ok(RouterItem::Packed(bytes))
6472 }
6473
6474 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
6475 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
6476 }
6477
6478 fn should_route_remote(
6479 &self,
6480 data: &RouterItem,
6481 exclude: Option<RouterSideId>,
6482 ) -> TelemetryResult<bool> {
6483 #[cfg(feature = "discovery")]
6484 {
6485 let RemoteSidePlan::Target(sides) =
6486 self.remote_side_plan_with_selection(data, exclude, false)?;
6487 Ok(!sides.is_empty())
6488 }
6489
6490 #[cfg(not(feature = "discovery"))]
6491 {
6492 let (eps, ty) = self.item_route_info(data)?;
6493 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
6494 return Ok(false);
6495 }
6496 let st = self.state.lock();
6497 Ok(!self
6498 .eligible_side_ids_locked(
6499 &st,
6500 exclude,
6501 Some(ty),
6502 Self::endpoints_are_link_local_only(&eps),
6503 )
6504 .is_empty())
6505 }
6506 }
6507
6508 #[cfg(feature = "discovery")]
6509 fn has_explicit_route_policy_locked(
6510 st: &RouterInner,
6511 src: Option<RouterSideId>,
6512 ty: DataType,
6513 ) -> bool {
6514 st.route_overrides
6515 .keys()
6516 .any(|(route_src, _)| *route_src == src)
6517 || Self::has_typed_route_overrides_locked(st, src, ty)
6518 }
6519
6520 fn remote_side_plan(
6521 &self,
6522 data: &RouterItem,
6523 exclude: Option<RouterSideId>,
6524 ) -> TelemetryResult<RemoteSidePlan> {
6525 self.remote_side_plan_with_selection(data, exclude, true)
6526 }
6527
6528 #[inline(never)]
6531 fn remote_side_plan_with_selection(
6532 &self,
6533 data: &RouterItem,
6534 exclude: Option<RouterSideId>,
6535 select_routes: bool,
6536 ) -> TelemetryResult<RemoteSidePlan> {
6537 let select_sides = |state: &mut RouterInner, sides: Vec<RouterSideId>, origin| {
6538 if select_routes {
6539 self.apply_route_selection_locked(state, exclude, sides, origin)
6540 } else {
6541 sides
6542 }
6543 };
6544 #[cfg(feature = "discovery")]
6545 {
6546 let (eps, ty) = self.item_route_info(data)?;
6547 let target_senders = self.item_target_senders(data)?;
6548 let resolve_beyond_local_hop =
6549 self.item_targets_router_as_intermediate_hop(data, &eps)?;
6550 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
6551 let routing_target_senders = if preferred_packet_id.is_some() {
6552 target_senders.skip(1)
6555 } else if resolve_beyond_local_hop {
6556 wire_format::TargetSenderSet::Hashes(&[])
6557 } else {
6558 target_senders
6559 };
6560 if discovery::is_discovery_type(ty) {
6561 let mut st = self.state.lock();
6562 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6563 return Ok(RemoteSidePlan::Target(select_sides(
6564 &mut st,
6565 sides,
6566 RouteSelectionOrigin::Flood,
6567 )));
6568 }
6569 #[cfg(feature = "timesync")]
6570 if ty == DataType::TimeSyncAnnounce {
6571 let mut st = self.state.lock();
6578 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6579 let sides = if select_routes {
6580 Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
6581 } else {
6582 sides
6583 };
6584 return Ok(RemoteSidePlan::Target(select_sides(
6585 &mut st,
6586 sides,
6587 RouteSelectionOrigin::Flood,
6588 )));
6589 }
6590 let mut st = self.state.lock();
6591 let now_ms = self.clock.now_ms();
6596 let has_remote_variable_owner = st.managed_variable_types.contains(&ty.as_u32())
6597 && st.discovery_routes.iter().any(|(&side, route)| {
6598 exclude != Some(side)
6599 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6600 && route.reachable_network_variables.contains(&ty)
6601 });
6602 if !(has_nonlocal_endpoint(&eps, &self.cfg)
6603 || force_remote_for_type(ty)
6604 || !target_senders.is_empty()
6605 || has_remote_variable_owner)
6606 {
6607 return Ok(RemoteSidePlan::Target(Vec::new()));
6608 }
6609
6610 #[cfg(feature = "timesync")]
6611 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
6612 #[cfg(not(feature = "timesync"))]
6613 let preferred_timesync_source: Option<String> = None;
6614
6615 if let Some(packet_id) = preferred_packet_id {
6616 let target_side = st
6617 .reliable_return_routes
6618 .get(&packet_id)
6619 .map(|route| route.side);
6620 if let Some(side) = target_side
6621 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
6622 {
6623 #[cfg(feature = "timesync")]
6624 if select_routes
6625 && !Self::timesync_allowed_for_side_locked(
6626 &st,
6627 side,
6628 ty,
6629 self.clock.now_ms(),
6630 )
6631 {
6632 return Ok(RemoteSidePlan::Target(Vec::new()));
6633 }
6634 return Ok(RemoteSidePlan::Target(vec![side]));
6635 }
6636 let eligible = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6639 let exact: Vec<_> = eligible
6640 .iter()
6641 .copied()
6642 .filter(|&side| {
6643 !routing_target_senders.is_empty()
6644 && Self::side_matches_target_senders_locked(
6645 &st,
6646 side,
6647 routing_target_senders,
6648 now_ms,
6649 )
6650 })
6651 .collect();
6652 if !exact.is_empty() {
6653 return Ok(RemoteSidePlan::Target(select_sides(
6654 &mut st,
6655 exact,
6656 RouteSelectionOrigin::Discovered,
6657 )));
6658 }
6659 let publisher_known = !routing_target_senders.is_empty()
6664 && st.discovery_routes.keys().any(|&side| {
6665 Self::side_matches_target_senders_locked(
6666 &st,
6667 side,
6668 routing_target_senders,
6669 now_ms,
6670 )
6671 });
6672 return Ok(RemoteSidePlan::Target(
6673 if !publisher_known && exclude.is_some() && eligible.len() == 1 {
6674 eligible
6675 } else {
6676 Vec::new()
6677 },
6678 ));
6679 }
6680 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
6681 let prefer_best_overlap = is_reliable_type(ty)
6682 && routing_target_senders.is_empty()
6683 && preferred_packet_id.is_none();
6684 if st.discovery_routes.is_empty() {
6685 let mut fallback =
6686 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
6687 #[cfg(feature = "timesync")]
6688 {
6689 if select_routes {
6690 fallback = Self::filter_timesync_sides_locked(
6691 &st,
6692 ty,
6693 self.clock.now_ms(),
6694 fallback,
6695 );
6696 }
6697 }
6698 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
6699 fallback
6700 } else {
6701 Vec::new()
6702 }));
6703 }
6704 let mut matches = self.discovered_route_candidates_locked(
6705 &st,
6706 exclude,
6707 ty,
6708 &eps,
6709 routing_target_senders,
6710 prefer_best_overlap,
6711 preferred_timesync_source.as_deref(),
6712 );
6713 #[cfg(feature = "timesync")]
6714 {
6715 if select_routes {
6716 matches =
6717 Self::filter_timesync_matches_locked(&st, ty, self.clock.now_ms(), matches);
6718 }
6719 }
6720 let has_distributed_variable_owners = matches.iter().any(|candidate| {
6726 st.discovery_routes
6727 .get(&candidate.side)
6728 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
6729 });
6730 if DataType::try_named("HEARTBEAT") != Some(ty) && !has_distributed_variable_owners {
6731 if eps.len() > 1
6735 && matches.len() > 1
6736 && !is_reliable_type(ty)
6737 && !st.managed_variable_types.contains(&ty.as_u32())
6738 && routing_target_senders.is_empty()
6739 && preferred_packet_id.is_none()
6740 && preferred_timesync_source.is_none()
6741 && !Self::has_explicit_route_policy_locked(&st, exclude, ty)
6742 && !st.source_route_modes.contains_key(&exclude)
6743 {
6744 if !select_routes {
6745 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
6746 }
6747 let sides = self.select_endpoint_role_sides_locked(
6748 &mut st,
6749 exclude,
6750 &eps,
6751 &matches,
6752 self.clock.now_ms(),
6753 );
6754 return Ok(RemoteSidePlan::Target(sides));
6755 }
6756 Self::retain_shortest_discovery_candidates_locked(
6757 &st,
6758 &mut matches,
6759 &eps,
6760 routing_target_senders,
6761 self.clock.now_ms(),
6762 );
6763 }
6764
6765 if !matches.is_empty() {
6766 if !select_routes {
6767 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
6770 }
6771 Ok(RemoteSidePlan::Target(
6772 self.select_discovered_candidate_sides_locked(
6773 &mut st,
6774 exclude,
6775 ty,
6776 routing_target_senders,
6777 prefer_best_overlap,
6778 matches,
6779 ),
6780 ))
6781 } else if prefer_best_overlap {
6782 Ok(RemoteSidePlan::Target(Vec::new()))
6783 } else {
6784 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
6785 let mut sides =
6786 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
6787 #[cfg(feature = "timesync")]
6788 {
6789 if select_routes {
6790 sides = Self::filter_timesync_sides_locked(
6791 &st,
6792 ty,
6793 self.clock.now_ms(),
6794 sides,
6795 );
6796 }
6797 }
6798 Ok(RemoteSidePlan::Target(select_sides(
6799 &mut st,
6800 sides,
6801 RouteSelectionOrigin::Flood,
6802 )))
6803 } else {
6804 Ok(RemoteSidePlan::Target(Vec::new()))
6805 }
6806 }
6807 }
6808 #[cfg(not(feature = "discovery"))]
6809 {
6810 let ty = Self::item_data_type(data)?;
6811 let mut st = self.state.lock();
6812 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
6813 let target_side = st
6814 .reliable_return_routes
6815 .get(&packet_id)
6816 .map(|route| route.side);
6817 if let Some(side) = target_side
6818 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
6819 {
6820 return Ok(RemoteSidePlan::Target(vec![side]));
6821 }
6822 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6823 return Ok(RemoteSidePlan::Target(
6824 if exclude.is_some() && sides.len() == 1 {
6825 sides
6826 } else {
6827 Vec::new()
6828 },
6829 ));
6830 }
6831 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6832 Ok(RemoteSidePlan::Target(select_sides(
6833 &mut st,
6834 sides,
6835 RouteSelectionOrigin::Flood,
6836 )))
6837 }
6838 }
6839
6840 #[cfg(feature = "discovery")]
6841 fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
6842 let mut eps: Vec<DataEndpoint> = self.cfg.handlers.iter().map(|h| h.endpoint).collect();
6843 let _ = st;
6849 #[cfg(feature = "timesync")]
6850 if self.cfg.timesync_config().is_some() {
6851 eps.push(DataEndpoint::TimeSync);
6852 }
6853 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
6854 crate::small_sort::sort(&mut eps);
6855 eps.dedup();
6856 eps
6857 }
6858
6859 #[cfg(feature = "discovery")]
6860 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
6861 #[cfg(feature = "timesync")]
6862 {
6863 let st = self.timesync.lock();
6864 if let Some(tracker) = st.tracker.as_ref()
6865 && tracker.should_serve(
6866 now_ms,
6867 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
6868 )
6869 {
6870 return vec![self.sender_arc().to_string()];
6871 }
6872 }
6873 Vec::new()
6874 }
6875
6876 #[cfg(all(feature = "discovery", feature = "timesync"))]
6877 fn preferred_timesync_route_source(
6878 &self,
6879 data: &RouterItem,
6880 ty: DataType,
6881 ) -> TelemetryResult<Option<String>> {
6882 if !matches!(
6883 ty,
6884 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
6885 ) {
6886 return Ok(None);
6887 }
6888
6889 match data {
6890 RouterItem::Packet(pkt) => match ty {
6891 DataType::TimeSyncRequest => {
6892 let local_sender = self.sender_arc();
6893 if pkt.sender() == local_sender.as_ref() {
6894 Ok(self.timesync.lock().tracker.as_ref().and_then(|tracker| {
6895 tracker.current_source().map(|src| src.sender.clone())
6896 }))
6897 } else {
6898 Ok(None)
6899 }
6900 }
6901 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
6902 Ok(Some(pkt.sender().to_owned()))
6903 }
6904 _ => Ok(None),
6905 },
6906 RouterItem::Packed(bytes) => {
6907 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
6908 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
6909 }
6910 }
6911 }
6912
6913 #[cfg(feature = "discovery")]
6914 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
6915 st.discovery_cadence.on_topology_change(now_ms);
6916 for side_id in st
6921 .sides
6922 .iter()
6923 .enumerate()
6924 .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
6925 {
6926 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
6927 throttle.pending_incremental = true;
6928 throttle.incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
6929 }
6930 for transport in st.side_transport.values_mut() {
6934 transport.refresh_transmit_header_templates();
6935 }
6936 }
6937
6938 #[cfg(feature = "discovery")]
6939 fn sender_topology_board_mut<'a>(
6940 sender_state: &'a mut DiscoverySenderState,
6941 sender_id: &str,
6942 ) -> &'a mut TopologyBoardNode {
6943 let boards = Arc::make_mut(&mut sender_state.topology_boards);
6944 if let Some(idx) = boards.iter().position(|board| board.sender_id == sender_id) {
6945 return &mut boards[idx];
6946 }
6947 boards.push(TopologyBoardNode {
6948 sender_id: sender_id.to_string(),
6949 reachable_endpoints: Vec::new(),
6950 reachable_timesync_sources: Vec::new(),
6951 connections: Vec::new(),
6952 });
6953 boards.last_mut().expect("inserted above")
6954 }
6955
6956 #[cfg(feature = "discovery")]
6957 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
6958 if let Some(boards) = Arc::get_mut(&mut sender_state.topology_boards) {
6959 discovery::normalize_topology_boards(boards);
6960 }
6961 let (mut reachable, mut reachable_timesync_sources) =
6962 discovery::summarize_topology_boards(&sender_state.topology_boards);
6963 reachable.extend(sender_state.advertised_reachable.iter().copied());
6964 crate::small_sort::sort(&mut reachable);
6965 reachable.dedup();
6966 reachable_timesync_sources.extend(
6967 sender_state
6968 .advertised_reachable_timesync_sources
6969 .iter()
6970 .cloned(),
6971 );
6972 reachable_timesync_sources.sort_unstable();
6973 reachable_timesync_sources.dedup();
6974 sender_state.reachable = reachable;
6975 sender_state.reachable_timesync_sources = reachable_timesync_sources;
6976 }
6977
6978 #[cfg(feature = "discovery")]
6979 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
6980 let mut reachable = Vec::new();
6981 let mut reachable_network_variables = Vec::new();
6982 let mut reachable_timesync_sources = Vec::new();
6983 let mut last_seen_ms = 0u64;
6984 for sender in route.announcers.values() {
6985 reachable.extend(sender.reachable.iter().copied());
6986 reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
6987 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
6988 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
6989 }
6990 reachable_network_variables
6991 .extend(route.requested_network_variables.iter().map(|(ty, _)| *ty));
6992 last_seen_ms = last_seen_ms.max(
6993 route
6994 .requested_network_variables
6995 .iter()
6996 .map(|(_, last_seen_ms)| *last_seen_ms)
6997 .max()
6998 .unwrap_or(0),
6999 );
7000 crate::small_sort::sort(&mut reachable);
7001 reachable.dedup();
7002 crate::small_sort::sort(&mut reachable_network_variables);
7003 reachable_network_variables.dedup();
7004 reachable_timesync_sources.sort_unstable();
7005 reachable_timesync_sources.dedup();
7006 route.reachable = reachable;
7007 route.reachable_network_variables = reachable_network_variables;
7008 route.reachable_timesync_sources = reachable_timesync_sources;
7009 route.last_seen_ms = last_seen_ms;
7010 }
7011
7012 #[cfg(feature = "discovery")]
7013 fn local_discovery_topology_board(
7014 &self,
7015 st: &RouterInner,
7016 now_ms: u64,
7017 link_local_enabled: bool,
7018 ) -> TopologyBoardNode {
7019 let mut reachable_endpoints = self.local_discovery_endpoints(st);
7020 if !link_local_enabled {
7021 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
7022 }
7023 let mut connections = Vec::new();
7024 for route in st.discovery_routes.values() {
7025 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7026 continue;
7027 }
7028 for (sender, sender_state) in route.announcers.iter() {
7029 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7030 connections.push(sender.clone());
7031 }
7032 }
7033 }
7034 connections.sort_unstable();
7035 connections.dedup();
7036 let sender = self.sender_arc();
7037 TopologyBoardNode {
7038 sender_id: sender.to_string(),
7039 reachable_endpoints,
7040 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
7041 connections,
7042 }
7043 }
7044
7045 #[cfg(feature = "discovery")]
7046 fn advertised_discovery_topology_for_link_locked(
7047 &self,
7048 st: &RouterInner,
7049 now_ms: u64,
7050 link_local_enabled: bool,
7051 exclude_side: Option<RouterSideId>,
7052 ) -> Vec<TopologyBoardNode> {
7053 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
7054 for (&route_side, route) in st.discovery_routes.iter() {
7055 if exclude_side == Some(route_side) {
7056 continue;
7057 }
7058 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7059 continue;
7060 }
7061 for (announcer, sender_state) in route.announcers.iter() {
7062 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7063 continue;
7064 }
7065 if sender_state.topology_boards.is_empty() {
7066 let sender = self.sender_arc();
7067 let fallback = TopologyBoardNode {
7068 sender_id: announcer.clone(),
7069 reachable_endpoints: sender_state.reachable.clone(),
7070 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
7071 connections: vec![sender.to_string()],
7072 };
7073 discovery::merge_topology_boards(&mut boards, core::slice::from_ref(&fallback));
7074 } else {
7075 discovery::merge_topology_boards(&mut boards, &sender_state.topology_boards);
7078 if let Some(board) = boards
7079 .iter_mut()
7080 .find(|board| board.sender_id == *announcer)
7081 {
7082 let sender = self.sender_arc();
7083 if !board.connections.iter().any(|peer| peer == sender.as_ref()) {
7084 board.connections.push(sender.to_string());
7085 }
7086 }
7087 }
7088 }
7089 }
7090 if !link_local_enabled {
7091 for board in &mut boards {
7092 board
7093 .reachable_endpoints
7094 .retain(|ep| !ep.is_link_local_only());
7095 }
7096 }
7097 discovery::normalize_topology_boards(&mut boards);
7098 boards
7099 }
7100
7101 #[cfg(feature = "discovery")]
7102 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
7103 let mut changed = false;
7106 st.discovery_routes.retain(|_, route| {
7107 let before_peers = route.announcers.len();
7108 let before_requests = route.requested_network_variables.len();
7109 route.announcers.retain(|_, sender| {
7110 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7111 });
7112 route
7113 .requested_network_variables
7114 .retain(|(_, last_seen_ms)| {
7115 now_ms.saturating_sub(*last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7116 });
7117 if before_peers != route.announcers.len()
7118 || before_requests != route.requested_network_variables.len()
7119 {
7120 Self::recompute_discovery_side_state(route);
7121 changed = true;
7122 }
7123 let keep =
7124 !route.announcers.is_empty() || !route.requested_network_variables.is_empty();
7125 changed |= !keep;
7126 keep
7127 });
7128 changed
7129 }
7130
7131 #[cfg(feature = "discovery")]
7132 fn advertised_summaries_from_topology_locked(
7133 &self,
7134 st: &RouterInner,
7135 now_ms: u64,
7136 link_local_enabled: bool,
7137 exclude_side: Option<RouterSideId>,
7138 topology: &[TopologyBoardNode],
7139 ) -> (Vec<crate::DataEndpoint>, Vec<String>) {
7140 let (mut endpoints, mut sources) = discovery::summarize_topology_boards(topology);
7141 for (&side, route) in &st.discovery_routes {
7142 if exclude_side == Some(side)
7143 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7144 {
7145 continue;
7146 }
7147 for peer in route.announcers.values() {
7148 if now_ms.saturating_sub(peer.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7149 continue;
7150 }
7151 for endpoint in &peer.advertised_reachable {
7152 if !discovery::is_discovery_endpoint(*endpoint)
7153 && (link_local_enabled || !endpoint.is_link_local_only())
7154 && !endpoints.contains(endpoint)
7155 {
7156 endpoints.push(*endpoint);
7157 }
7158 }
7159 for source in &peer.advertised_reachable_timesync_sources {
7160 if !sources.contains(source) {
7161 sources.push(source.clone());
7162 }
7163 }
7164 }
7165 }
7166 endpoints.retain(|endpoint| {
7167 !discovery::is_discovery_endpoint(*endpoint)
7168 && (link_local_enabled || !endpoint.is_link_local_only())
7169 });
7170 crate::small_sort::sort(&mut endpoints);
7171 sources.sort_unstable();
7172 (endpoints, sources)
7173 }
7174
7175 #[cfg(feature = "discovery")]
7176 fn advertised_discovery_endpoints_for_link_locked(
7177 &self,
7178 st: &RouterInner,
7179 now_ms: u64,
7180 link_local_enabled: bool,
7181 exclude_side: Option<RouterSideId>,
7182 ) -> Vec<DataEndpoint> {
7183 let (mut reachable_endpoints, _) = discovery::summarize_topology_boards(
7184 &self.advertised_discovery_topology_for_link_locked(
7185 st,
7186 now_ms,
7187 link_local_enabled,
7188 exclude_side,
7189 ),
7190 );
7191 for (&side, route) in &st.discovery_routes {
7194 if exclude_side == Some(side)
7195 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7196 {
7197 continue;
7198 }
7199 for sender in route.announcers.values() {
7200 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7201 reachable_endpoints.extend(sender.advertised_reachable.iter().copied());
7202 }
7203 }
7204 }
7205 crate::small_sort::sort(&mut reachable_endpoints);
7206 reachable_endpoints.dedup();
7207 reachable_endpoints
7208 .into_iter()
7209 .filter(|ep| {
7210 !discovery::is_discovery_endpoint(*ep)
7211 && (link_local_enabled || !ep.is_link_local_only())
7212 })
7213 .collect()
7214 }
7215
7216 #[cfg(all(test, feature = "discovery"))]
7217 fn compact_summary_fixture(&self, endpoint: DataEndpoint) {
7218 let mut sender = DiscoverySenderState {
7219 advertised_reachable: vec![endpoint],
7220 advertised_reachable_timesync_sources: vec!["clock-behind-bridge".to_string()],
7221 last_seen_ms: self.clock.now_ms(),
7222 topology_boards: Arc::new(vec![TopologyBoardNode {
7223 sender_id: "bridge".to_string(),
7224 reachable_endpoints: Vec::new(),
7225 reachable_timesync_sources: Vec::new(),
7226 connections: Vec::new(),
7227 }]),
7228 ..Default::default()
7229 };
7230 Self::refresh_sender_topology_state(&mut sender);
7231 let mut route = DiscoverySideState::default();
7232 route.announcers.insert("bridge".to_string(), sender);
7233 Self::recompute_discovery_side_state(&mut route);
7234 self.state.lock().discovery_routes.insert(0, route);
7235 }
7236
7237 #[cfg(feature = "discovery")]
7238 fn advertised_discovery_timesync_sources_for_link_locked(
7239 &self,
7240 st: &RouterInner,
7241 now_ms: u64,
7242 exclude_side: Option<RouterSideId>,
7243 ) -> Vec<String> {
7244 let (_, mut sources) = discovery::summarize_topology_boards(
7245 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
7246 );
7247 for (&side, route) in &st.discovery_routes {
7248 if exclude_side == Some(side)
7249 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7250 {
7251 continue;
7252 }
7253 for sender in route.announcers.values() {
7254 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7255 sources.extend(sender.advertised_reachable_timesync_sources.iter().cloned());
7256 }
7257 }
7258 }
7259 sources.sort_unstable();
7260 sources.dedup();
7261 sources
7262 }
7263
7264 #[cfg(feature = "discovery")]
7265 fn advertised_network_variables_for_link_locked(
7266 &self,
7267 st: &RouterInner,
7268 now_ms: u64,
7269 exclude_side: Option<RouterSideId>,
7270 ) -> Vec<DataType> {
7271 let mut types: Vec<DataType> = st
7272 .managed_variable_types
7273 .iter()
7274 .copied()
7275 .map(DataType)
7276 .collect();
7277 for (&route_side, route) in st.discovery_routes.iter() {
7278 if exclude_side == Some(route_side) {
7279 continue;
7280 }
7281 if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7282 types.extend(route.reachable_network_variables.iter().copied());
7283 }
7284 }
7285 crate::small_sort::sort(&mut types);
7286 types.dedup();
7287 types
7288 }
7289
7290 #[cfg(feature = "discovery")]
7291 fn preferred_discovery_master_present_locked<'a>(
7292 &self,
7293 st: &'a RouterInner,
7294 now_ms: u64,
7295 ) -> Option<&'a str> {
7296 if let Some(preferred) = st.preferred_discovery_master.as_deref()
7297 && (preferred == self.sender_arc().as_ref()
7298 || st
7299 .discovery_routes
7300 .values()
7301 .filter(|route| {
7302 now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7303 })
7304 .any(|route| {
7305 route
7306 .announcers
7307 .iter()
7308 .filter(|(_, peer)| {
7309 now_ms.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7310 })
7311 .any(|(name, peer)| {
7312 name == preferred
7313 || peer.topology_boards.iter().any(|board| {
7314 board.sender_id == preferred
7315 || board
7316 .connections
7317 .iter()
7318 .any(|name| name == preferred)
7319 })
7320 })
7321 }))
7322 {
7323 return Some(preferred);
7326 }
7327 None
7328 }
7329
7330 #[cfg(feature = "discovery")]
7331 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
7332 if let Some(preferred) = self.preferred_discovery_master_present_locked(st, now_ms) {
7333 return preferred.to_string();
7334 }
7335 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
7336 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
7337 }
7338
7339 #[cfg(all(feature = "discovery", feature = "timesync"))]
7340 #[inline]
7341 fn side_is_slow_control_link_locked(
7342 st: &RouterInner,
7343 side_id: RouterSideId,
7344 now_ms: u64,
7345 ) -> bool {
7346 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
7347 let recent_slow = stats.last_slow_observed_ms > 0
7348 && now_ms.saturating_sub(stats.last_slow_observed_ms)
7349 <= discovery::DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
7350 stats.sample_count > 0
7351 && ((stats.estimated_bandwidth_bps > 0
7352 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
7353 || recent_slow)
7354 })
7355 }
7356
7357 #[cfg(feature = "discovery")]
7358 fn discovery_level_for_side_locked(
7359 st: &mut RouterInner,
7360 side_id: RouterSideId,
7361 now_ms: u64,
7362 ) -> Option<DiscoveryAdvertiseLevel> {
7363 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
7364 if !throttle.has_sent_full {
7365 throttle.has_sent_full = true;
7366 throttle.pending_incremental = false;
7367 throttle.incremental_retries_remaining = 0;
7368 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7369 return Some(DiscoveryAdvertiseLevel::Full);
7370 }
7371 if throttle.pending_incremental {
7372 throttle.incremental_retries_remaining =
7373 throttle.incremental_retries_remaining.saturating_sub(1);
7374 throttle.pending_incremental = throttle.incremental_retries_remaining > 0;
7375 return Some(DiscoveryAdvertiseLevel::Incremental);
7376 }
7377 if throttle.liveness_due(now_ms) {
7380 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7381 return Some(DiscoveryAdvertiseLevel::MinimalPing);
7382 }
7383 None
7384 }
7385
7386 #[cfg(all(feature = "discovery", feature = "timesync"))]
7387 #[inline]
7388 fn is_timesync_type(ty: DataType) -> bool {
7389 matches!(
7390 ty,
7391 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
7392 )
7393 }
7394
7395 #[cfg(all(feature = "discovery", feature = "timesync"))]
7396 #[inline(never)]
7397 fn timesync_allowed_for_side_locked(
7398 st: &RouterInner,
7399 side_id: RouterSideId,
7400 ty: DataType,
7401 now_ms: u64,
7402 ) -> bool {
7403 if !Self::is_timesync_type(ty) {
7404 return true;
7405 }
7406 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
7407 return true;
7408 }
7409 st.timesync_side_throttle
7412 .get(&side_id)
7413 .is_none_or(|throttle| now_ms >= throttle.next_allowed_ms)
7414 }
7415
7416 #[cfg(all(feature = "discovery", feature = "timesync"))]
7417 fn filter_timesync_sides_locked(
7418 st: &RouterInner,
7419 ty: DataType,
7420 now_ms: u64,
7421 sides: Vec<RouterSideId>,
7422 ) -> Vec<RouterSideId> {
7423 sides
7424 .into_iter()
7425 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
7426 .collect()
7427 }
7428
7429 #[cfg(all(feature = "discovery", feature = "timesync"))]
7430 fn filter_timesync_matches_locked(
7431 st: &RouterInner,
7432 ty: DataType,
7433 now_ms: u64,
7434 matches: Vec<DiscoveryCandidateMatch>,
7435 ) -> Vec<DiscoveryCandidateMatch> {
7436 matches
7437 .into_iter()
7438 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
7439 .collect()
7440 }
7441
7442 #[cfg(feature = "discovery")]
7443 fn refresh_known_discovery_liveness(
7444 &self,
7445 pkt: &Packet,
7446 side: RouterSideId,
7447 ) -> TelemetryResult<bool> {
7448 if pkt.data_type() == crate::DataType::DiscoveryAddress {
7449 return self.refresh_known_address_payload(pkt.payload(), side);
7450 }
7451 if pkt.data_type() == crate::DataType::DiscoveryTopology {
7452 crate::memory_admission::check(512, 128)?;
7456 let now = self.clock.now_ms();
7457 let mut st = self.state.lock();
7458 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
7459 let Some(route) = st.discovery_routes.get_mut(&side) else {
7460 return Ok(false);
7461 };
7462 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
7463 return Ok(false);
7464 };
7465 let Some(full) =
7466 discovery::topology_payload_matches_retained(pkt.payload(), &peer.topology_boards)
7467 else {
7468 return Ok(false);
7469 };
7470 peer.has_full_topology |= full;
7471 peer.last_seen_ms = now;
7472 route.last_seen_ms = route.last_seen_ms.max(now);
7473 return Ok(true);
7474 }
7475 if pkt.data_type() != crate::DataType::DiscoveryAnnounce || !pkt.payload().is_empty() {
7476 return Ok(false);
7477 }
7478 crate::memory_admission::check(512, 128)?;
7479 let now = self.clock.now_ms();
7480 let mut st = self.state.lock();
7481 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
7482 let Some(route) = st.discovery_routes.get_mut(&side) else {
7483 return Ok(false);
7484 };
7485 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
7486 return Ok(false);
7487 };
7488 peer.last_seen_ms = now;
7489 route.last_seen_ms = route.last_seen_ms.max(now);
7490 Ok(true)
7491 }
7492
7493 #[cfg(feature = "discovery")]
7494 fn refresh_known_address_payload(
7495 &self,
7496 payload: &[u8],
7497 side: RouterSideId,
7498 ) -> TelemetryResult<bool> {
7499 let ad = discovery::decode_address_view(payload)?;
7500 let now = self.clock.now_ms();
7501 let mut st = self.state.lock();
7502 let link_local = st
7503 .sides
7504 .get(side)
7505 .and_then(|entry| entry.as_ref())
7506 .is_some_and(|entry| entry.opts.link_local_enabled);
7507 {
7508 let Some(entry) = st.address_book.get(ad.hostname) else {
7509 return Ok(false);
7510 };
7511 if entry.address != ad.address
7512 || entry.requested_address != ad.requested_address
7513 || entry.mode != Self::address_mode_from_code(ad.mode, ad.requested_address)
7514 || entry.owner_hash != ad.owner_hash
7515 {
7516 return Ok(false);
7517 }
7518 if entry.birth_ms != ad.birth_ms
7519 && st.address_book.iter().any(|(hostname, other)| {
7520 hostname.as_str() != ad.hostname
7521 && [ad.address, ad.requested_address].iter().any(|candidate| {
7522 *candidate != 0
7523 && (other.address == *candidate
7524 || other.requested_address == *candidate)
7525 })
7526 })
7527 {
7528 return Ok(false);
7532 }
7533 }
7534 let sender = ad.hostname;
7535 let Some(route) = st.discovery_routes.get_mut(&side) else {
7536 return Ok(false);
7537 };
7538 let Some(peer) = route.announcers.get_mut(sender) else {
7539 return Ok(false);
7540 };
7541 if peer.advertised_address != Some(ad.address)
7542 || !ad
7543 .endpoints()
7544 .filter(|ep| link_local || !ep.is_link_local_only())
7545 .eq(peer.advertised_reachable.iter().copied())
7546 || !ad.sources().eq(peer
7547 .advertised_reachable_timesync_sources
7548 .iter()
7549 .map(String::as_str))
7550 || !ad
7551 .variables()
7552 .eq(peer.reachable_network_variables.iter().copied())
7553 || peer.link_capabilities != Some(ad.link_capabilities)
7554 {
7555 return Ok(false);
7556 }
7557 peer.last_seen_ms = now;
7560 route.last_seen_ms = route.last_seen_ms.max(now);
7561 if let Some(entry) = st.address_book.get_mut(ad.hostname) {
7562 entry.birth_ms = ad.birth_ms;
7565 entry.last_seen_ms = now;
7566 }
7567 #[cfg(feature = "compression")]
7568 {
7569 drop(st);
7570 self.note_payload_codec_address(side, payload, true);
7571 }
7572 Ok(true)
7573 }
7574
7575 #[cfg(feature = "discovery")]
7576 fn queue_pressure_keepalives(
7577 &self,
7578 requested_side: Option<RouterSideId>,
7579 ) -> TelemetryResult<()> {
7580 let now = self.clock.now_ms();
7581 let sender = self.sender_arc();
7582 crate::memory_admission::check(
7583 256 + sender.len().saturating_mul(4),
7584 sender.len().max(256),
7585 )?;
7586 let side_count = self.state.lock().sides.len();
7587 for side in 0..side_count {
7588 let due = {
7589 let st = self.state.lock();
7590 requested_side.is_none_or(|wanted| wanted == side)
7591 && st.sides[side].is_some()
7592 && self.route_allowed_locked(
7593 &st,
7594 None,
7595 Some(crate::DataType::DiscoveryAnnounce),
7596 side,
7597 )
7598 && st
7599 .discovery_side_throttle
7600 .get(&side)
7601 .is_none_or(|throttle| now >= throttle.next_ping_ms)
7602 };
7603 if !due {
7604 continue;
7605 }
7606 let summary = {
7609 let st = self.state.lock();
7610 let extra = st
7611 .discovery_routes
7612 .iter()
7613 .filter(|(id, _)| **id != side)
7614 .fold(0usize, |total, (_, route)| {
7615 total
7616 .saturating_add(route.reachable.len().saturating_mul(32))
7617 .saturating_add(
7618 route.reachable_network_variables.len().saturating_mul(32),
7619 )
7620 .saturating_add(
7621 route
7622 .reachable_timesync_sources
7623 .iter()
7624 .map(|source| source.len().saturating_add(32).saturating_mul(4))
7625 .sum::<usize>(),
7626 )
7627 });
7628 if crate::memory_admission::check(4096usize.saturating_add(extra), 2048).is_ok() {
7629 let opts = st.sides[side].as_ref().expect("checked above").opts.clone();
7630 let mut endpoints = self.local_discovery_endpoints(&st);
7631 let mut variables =
7632 self.advertised_network_variables_for_link_locked(&st, now, Some(side));
7633 let mut sources = self.local_discovery_timesync_sources(now);
7634 for (&route_side, route) in &st.discovery_routes {
7635 if route_side == side {
7636 continue;
7637 }
7638 for peer in route.announcers.values() {
7639 if now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7640 endpoints.extend(peer.reachable.iter().copied());
7641 sources.extend(peer.reachable_timesync_sources.iter().cloned());
7642 }
7643 }
7644 }
7645 endpoints.retain(|ep| {
7646 !discovery::is_discovery_endpoint(*ep)
7647 && (opts.link_local_enabled || !ep.is_link_local_only())
7648 });
7649 crate::small_sort::sort(&mut endpoints);
7650 endpoints.dedup();
7651 crate::small_sort::sort(&mut variables);
7652 variables.dedup();
7653 sources.sort_unstable();
7654 sources.dedup();
7655 Some((endpoints, variables, sources, opts.link_capabilities()))
7656 } else {
7657 None
7658 }
7659 };
7660 let pkt = if let Some((endpoints, variables, sources, capabilities)) = summary {
7661 let address = self.local_address_advertisement(
7662 endpoints,
7663 variables,
7664 sources,
7665 capabilities,
7666 discovery::ADDRESS_STATE_REQUEST,
7667 );
7668 discovery::build_discovery_address(sender.as_ref(), now, &address)?
7669 } else {
7670 discovery::build_discovery_announce(sender.as_ref(), now, &[])?
7671 };
7672 self.emit_internal_tx(
7673 RouterTxItem::ToSide {
7674 src: None,
7675 dst: side,
7676 data: RouterItem::Packet(pkt),
7677 },
7678 true,
7679 true,
7680 )?;
7681 self.state
7682 .lock()
7683 .discovery_side_throttle
7684 .entry(side)
7685 .or_default()
7686 .next_ping_ms = now.saturating_add(discovery::DISCOVERY_SLOW_INTERVAL_MS);
7687 }
7688 Ok(())
7689 }
7690
7691 #[cfg(feature = "discovery")]
7692 fn emit_discovery_snapshot(
7693 &self,
7694 called_from_queue: bool,
7695 include_schema: bool,
7696 include_topology: bool,
7697 skip_in_flight_sides: bool,
7698 ) -> TelemetryResult<()> {
7699 self.emit_discovery_snapshot_on_side(
7700 called_from_queue,
7701 include_schema,
7702 include_topology,
7703 skip_in_flight_sides,
7704 None,
7705 )
7706 }
7707
7708 #[cfg(feature = "discovery")]
7709 fn emit_discovery_snapshot_on_side(
7710 &self,
7711 called_from_queue: bool,
7712 include_schema: bool,
7713 include_topology: bool,
7714 skip_in_flight_sides: bool,
7715 requested_side: Option<RouterSideId>,
7716 ) -> TelemetryResult<()> {
7717 if requested_side.is_none() {
7718 let count = self.state.lock().sides.len();
7719 for side in 0..count {
7720 if self.state.lock().sides[side].is_some() {
7721 self.emit_discovery_snapshot_on_side(
7722 called_from_queue,
7723 include_schema,
7724 include_topology,
7725 skip_in_flight_sides,
7726 Some(side),
7727 )?;
7728 }
7729 }
7730 return Ok(());
7731 }
7732 let (scratch, largest) = {
7733 let now = self.clock.now_ms();
7734 let sender = self.sender_arc();
7735 let st = self.state.lock();
7736 let election_required = self
7737 .preferred_discovery_master_present_locked(&st, now)
7738 .is_none();
7739 let snapshot = discovery::topology_snapshot_scratch(
7740 sender.as_ref(),
7741 self.cfg.handlers.len(),
7742 st.managed_variable_types.len().saturating_add(
7743 st.discovery_routes
7744 .iter()
7745 .filter(|(side, _)| Some(**side) != requested_side)
7746 .map(|(_, r)| r.reachable_network_variables.len())
7747 .sum::<usize>(),
7748 ),
7749 4,
7750 true,
7751 st.discovery_routes
7752 .iter()
7753 .filter(|(side, route)| {
7754 (Some(**side) != requested_side)
7755 && now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7756 })
7757 .flat_map(|(_, route)| route.announcers.iter())
7758 .filter(|(_, peer)| {
7759 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7760 })
7761 .map(|(name, peer)| {
7762 (
7763 name.as_str(),
7764 peer.reachable.as_slice(),
7765 peer.reachable_timesync_sources.as_slice(),
7766 peer.topology_boards.as_slice(),
7767 )
7768 }),
7769 );
7770 if !election_required {
7771 snapshot
7772 } else {
7773 let election = discovery::topology_snapshot_scratch(
7774 sender.as_ref(),
7775 self.cfg.handlers.len(),
7776 0,
7777 2,
7778 false,
7779 st.discovery_routes
7780 .values()
7781 .filter(|route| {
7782 now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7783 })
7784 .flat_map(|route| route.announcers.iter())
7785 .filter(|(_, peer)| {
7786 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7787 })
7788 .map(|(name, peer)| {
7789 (
7790 name.as_str(),
7791 peer.reachable.as_slice(),
7792 peer.reachable_timesync_sources.as_slice(),
7793 peer.topology_boards.as_slice(),
7794 )
7795 }),
7796 );
7797 (snapshot.0.max(election.0), snapshot.1.max(election.1))
7798 }
7799 };
7800 let (scratch, largest) = if include_schema {
7803 (scratch.max(8192), largest.max(2048))
7804 } else {
7805 (scratch, largest)
7806 };
7807 while let Err(err) = crate::memory_admission::check(scratch, largest) {
7812 if matches!(err, TelemetryError::Io("memory pressure"))
7813 && crate::memory_admission::check(512, 256).is_ok()
7814 {
7815 let mut st = self.state.lock();
7816 if let Some(victim) = st.lower_priority_shared_queue(199)
7817 && st.pop_shared_queue_item(victim)
7818 {
7819 continue;
7820 }
7821 }
7822 if !include_schema && matches!(err, TelemetryError::Io("memory pressure")) {
7823 return self.queue_pressure_keepalives(requested_side);
7824 }
7825 return Err(err);
7826 }
7827 let now_ms = self.clock.now_ms();
7828 let per_side = {
7829 let mut st = self.state.lock();
7830 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
7831 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7832 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7833 }
7834 st.fit_discovery_budget();
7835 let local_is_master =
7836 self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
7837 let mut per_side = None;
7839 for side_id in 0..st.sides.len() {
7840 let Some(side) = st.sides[side_id].as_ref() else {
7841 continue;
7842 };
7843 let opts = side.opts.clone();
7844 let link_local_enabled = opts.link_local_enabled;
7845 if requested_side.is_some_and(|requested| requested != side_id) {
7846 continue;
7847 }
7848 let discovery_in_flight = skip_in_flight_sides
7849 && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
7850 *pending_side == side_id
7851 && !tx_state.sent.is_empty()
7852 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
7853 });
7854 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
7855 {
7856 continue;
7857 }
7858 let level = if discovery_in_flight {
7862 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
7863 if !throttle.liveness_due(now_ms) {
7864 None
7865 } else {
7866 throttle.next_ping_ms =
7867 now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7868 Some(DiscoveryAdvertiseLevel::MinimalPing)
7869 }
7870 } else {
7871 Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
7872 };
7873 let Some(level) = level else {
7874 continue;
7875 };
7876 let include_side_topology = include_topology
7877 && self.route_allowed_locked(
7878 &st,
7879 None,
7880 Some(DataType::DiscoveryTopology),
7881 side_id,
7882 );
7883 let capabilities = opts.link_capabilities();
7884 if level == DiscoveryAdvertiseLevel::MinimalPing {
7885 per_side = Some((
7886 side_id,
7887 level,
7888 Vec::new(),
7889 Vec::new(),
7890 Vec::new(),
7891 Vec::new(),
7892 Vec::new(),
7893 capabilities,
7894 local_is_master,
7895 ));
7896 continue;
7897 }
7898 let current_topology = if include_side_topology {
7899 self.advertised_discovery_topology_for_link_locked(
7900 &st,
7901 now_ms,
7902 link_local_enabled,
7903 Some(side_id),
7904 )
7905 } else {
7906 Vec::new()
7907 };
7908 let (endpoints, timesync_sources) = if include_side_topology {
7909 self.advertised_summaries_from_topology_locked(
7910 &st,
7911 now_ms,
7912 link_local_enabled,
7913 Some(side_id),
7914 ¤t_topology,
7915 )
7916 } else {
7917 (
7918 self.advertised_discovery_endpoints_for_link_locked(
7919 &st,
7920 now_ms,
7921 link_local_enabled,
7922 Some(side_id),
7923 ),
7924 self.advertised_discovery_timesync_sources_for_link_locked(
7925 &st,
7926 now_ms,
7927 Some(side_id),
7928 ),
7929 )
7930 };
7931 let (topology, removed_topology) = match level {
7932 DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
7933 DiscoveryAdvertiseLevel::Incremental => {
7934 let previous = &st
7935 .discovery_side_throttle
7936 .get(&side_id)
7937 .expect("discovery throttle exists")
7938 .last_topology;
7939 discovery::topology_delta(previous, ¤t_topology)
7940 }
7941 DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
7942 };
7943 let throttle = st
7944 .discovery_side_throttle
7945 .get_mut(&side_id)
7946 .expect("discovery throttle exists");
7947 if level == DiscoveryAdvertiseLevel::Full
7948 || (level == DiscoveryAdvertiseLevel::Incremental
7949 && throttle.incremental_retries_remaining == 0)
7950 {
7951 throttle.last_topology = current_topology;
7952 }
7953 per_side = Some((
7954 side_id,
7955 level,
7956 endpoints,
7957 self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
7958 timesync_sources,
7959 topology,
7960 removed_topology,
7961 capabilities,
7962 local_is_master,
7963 ));
7964 }
7965 per_side
7966 };
7967 if let Some((
7968 side_id,
7969 level,
7970 endpoints,
7971 network_variables,
7972 timesync_sources,
7973 topology,
7974 removed_topology,
7975 capabilities,
7976 local_is_master,
7977 )) = per_side
7978 {
7979 let sender = self.sender_arc();
7980 if matches!(
7981 level,
7982 DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
7983 ) {
7984 let address = self.local_address_advertisement(
7985 endpoints.clone(),
7986 network_variables.clone(),
7987 timesync_sources.clone(),
7988 capabilities,
7989 if local_is_master {
7990 discovery::ADDRESS_STATE_APPROVED
7991 } else {
7992 discovery::ADDRESS_STATE_REQUEST
7993 },
7994 );
7995 let pkt = discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?;
7996 self.emit_internal_tx(
7997 RouterTxItem::ToSide {
7998 src: None,
7999 dst: side_id,
8000 data: RouterItem::Packet(pkt),
8001 },
8002 true,
8003 called_from_queue,
8004 )?;
8005 }
8006 if level == DiscoveryAdvertiseLevel::MinimalPing {
8007 let pkt = discovery::build_discovery_announce(
8008 sender.as_ref(),
8009 now_ms,
8010 endpoints.as_slice(),
8011 )?;
8012 self.emit_internal_tx(
8013 RouterTxItem::ToSide {
8014 src: None,
8015 dst: side_id,
8016 data: RouterItem::Packet(pkt),
8017 },
8018 true,
8019 called_from_queue,
8020 )?;
8021 }
8022 if include_topology
8023 && (!topology.is_empty() || !removed_topology.is_empty())
8024 && level != DiscoveryAdvertiseLevel::MinimalPing
8025 {
8026 let pkt = if level == DiscoveryAdvertiseLevel::Full {
8027 discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
8028 } else {
8029 discovery::build_discovery_topology_delta(
8030 sender.as_ref(),
8031 now_ms,
8032 &topology,
8033 &removed_topology,
8034 )?
8035 };
8036 self.emit_internal_tx(
8037 RouterTxItem::ToSide {
8038 src: None,
8039 dst: side_id,
8040 data: RouterItem::Packet(pkt),
8041 },
8042 true,
8043 called_from_queue,
8044 )?;
8045 }
8046 drop((
8049 endpoints,
8050 network_variables,
8051 timesync_sources,
8052 topology,
8053 removed_topology,
8054 ));
8055 if include_schema && level == DiscoveryAdvertiseLevel::Full {
8060 let pkt = discovery::build_discovery_schema(sender.as_ref(), now_ms)?;
8061 self.emit_internal_tx(
8062 RouterTxItem::ToSide {
8063 src: None,
8064 dst: side_id,
8065 data: RouterItem::Packet(pkt),
8066 },
8067 true,
8068 called_from_queue,
8069 )?;
8070 }
8071 }
8072 Ok(())
8073 }
8074
8075 #[cfg(feature = "discovery")]
8076 fn queue_discovery_announce(
8077 &self,
8078 include_schema: bool,
8079 skip_in_flight_sides: bool,
8080 ) -> TelemetryResult<()> {
8081 let now_ms = self.clock.now_ms();
8082 {
8083 let mut st = self.state.lock();
8084 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
8085 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8086 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8087 }
8088 st.fit_discovery_budget();
8089 if st.sides.iter().all(|side| side.is_none()) {
8090 return Ok(());
8091 }
8092 if include_schema {
8095 for throttle in st.discovery_side_throttle.values_mut() {
8096 throttle.has_sent_full = false;
8097 }
8098 }
8099 st.discovery_cadence.on_announce_sent(now_ms);
8100 }
8101 self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
8102 }
8103
8104 #[cfg(feature = "discovery")]
8105 fn note_schema_received(&self, side: RouterSideId, sender: &str) {
8106 let mut st = self.state.lock();
8107 let address = sender
8111 .strip_prefix("@addr:")
8112 .and_then(|value| value.parse::<u32>().ok());
8113 if let Some(route) = st.discovery_routes.get_mut(&side) {
8114 for (name, peer) in &mut route.announcers {
8115 if name == sender
8116 || address.is_some_and(|address| {
8117 peer.advertised_address == Some(address)
8118 || sender_address_u32(name) == address
8119 })
8120 {
8121 peer.has_schema = true;
8122 break;
8123 }
8124 }
8125 }
8126 }
8127
8128 #[cfg(feature = "discovery")]
8129 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
8130 let now_ms = self.clock.now_ms();
8131 let recovery_sides = {
8135 let mut st = self.state.lock();
8136 let mut due = Vec::new();
8137 for side in 0..st.sides.len() {
8138 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
8139 route.announcers.iter().any(|(name, peer)| {
8140 !peer.has_full_topology
8141 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
8142 && !route.announcers.values().any(|bridge| {
8143 bridge.has_full_topology
8144 && now_ms.saturating_sub(bridge.last_seen_ms)
8145 < DISCOVERY_ROUTE_TTL_MS
8146 && bridge
8147 .topology_boards
8148 .iter()
8149 .any(|board| board.sender_id == *name)
8150 })
8151 })
8152 });
8153 if !missing {
8154 continue;
8155 }
8156 let throttle = st.discovery_side_throttle.entry(side).or_default();
8157 if throttle.next_topology_request_ms == 0 {
8158 throttle.next_topology_request_ms =
8159 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
8160 } else if now_ms >= throttle.next_topology_request_ms {
8161 throttle.next_topology_request_ms =
8162 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
8163 due.push(side);
8164 }
8165 }
8166 due
8167 };
8168 let recovering = !recovery_sides.is_empty();
8169 for side in recovery_sides {
8170 let pkt =
8171 discovery::build_discovery_topology_request(self.sender_arc().as_ref(), now_ms)?;
8172 self.emit_internal_tx(
8173 RouterTxItem::ToSide {
8174 src: None,
8175 dst: side,
8176 data: RouterItem::Packet(pkt),
8177 },
8178 true,
8179 true,
8180 )?;
8181 }
8182 let schema_recovery_sides = {
8183 let mut st = self.state.lock();
8184 let mut due = Vec::new();
8185 for side in 0..st.sides.len() {
8186 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
8187 route.announcers.values().any(|peer| {
8188 peer.has_full_topology
8189 && !peer.has_schema
8190 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
8191 })
8192 });
8193 if !missing {
8194 continue;
8195 }
8196 let throttle = st.discovery_side_throttle.entry(side).or_default();
8197 if throttle.next_schema_request_ms == 0 {
8198 throttle.next_schema_request_ms =
8199 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
8200 } else if now_ms >= throttle.next_schema_request_ms {
8201 throttle.next_schema_request_ms =
8202 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
8203 due.push(side);
8204 }
8205 }
8206 due
8207 };
8208 let recovering = recovering || !schema_recovery_sides.is_empty();
8209 for side in schema_recovery_sides {
8210 let pkt =
8211 discovery::build_discovery_schema_request(self.sender_arc().as_ref(), now_ms)?;
8212 self.emit_internal_tx(
8213 RouterTxItem::ToSide {
8214 src: None,
8215 dst: side,
8216 data: RouterItem::Packet(pkt),
8217 },
8218 true,
8219 true,
8220 )?;
8221 }
8222 let due = {
8223 let mut st = self.state.lock();
8224 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
8225 if removed {
8226 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8227 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8228 }
8229 st.fit_discovery_budget();
8230 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
8231 let Some(side) = side.as_ref() else {
8232 return false;
8233 };
8234 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
8235 {
8236 return false;
8237 }
8238 let _ = side;
8239 let in_flight = st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
8240 *pending_side == side_id
8241 && !tx_state.sent.is_empty()
8242 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
8243 });
8244 st.discovery_side_throttle
8245 .get(&side_id)
8246 .is_none_or(|throttle| {
8247 (!in_flight && (!throttle.has_sent_full || throttle.pending_incremental))
8248 || throttle.liveness_due(now_ms)
8249 })
8250 });
8251 if st.sides.is_empty() || !has_any {
8252 return Ok(recovering);
8253 }
8254 st.discovery_cadence.due(now_ms)
8255 };
8256 if !due {
8257 return Ok(recovering);
8258 }
8259 self.queue_discovery_announce(false, true)?;
8268 Ok(true)
8269 }
8270
8271 #[cfg(feature = "discovery")]
8272 fn learn_discovery_packet(
8273 &self,
8274 pkt: &Packet,
8275 src: Option<RouterSideId>,
8276 called_from_queue: bool,
8277 ) -> TelemetryResult<bool> {
8278 if !discovery::is_discovery_type(pkt.data_type()) {
8279 return Ok(false);
8280 }
8281 let Some(side) = src else {
8282 return Ok(true);
8283 };
8284 #[cfg(feature = "compression")]
8285 self.note_payload_codec_packet(side, pkt, false);
8286 if self.refresh_known_discovery_liveness(pkt, side)? {
8287 return Ok(true);
8288 }
8289 #[cfg(not(feature = "std"))]
8290 if pkt.data_type() == crate::DataType::DiscoverySchema {
8291 self.merge_schema_changes(pkt.payload())?;
8292 self.note_schema_received(side, pkt.sender());
8293 return Ok(true);
8294 }
8295 if matches!(
8299 pkt.data_type(),
8300 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
8301 ) {
8302 crate::memory_admission::check(1024, 256)?;
8303 if let Some(transport) = self.state.lock().side_transport.get_mut(&side) {
8308 transport.refresh_transmit_header_templates();
8309 }
8310 }
8311 if matches!(
8312 pkt.data_type(),
8313 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
8314 ) {
8315 let now_ms = self.clock.now_ms();
8316 {
8317 let mut st = self.state.lock();
8318 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
8319 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8320 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8321 }
8322 let throttle = st.discovery_side_throttle.entry(side).or_default();
8323 throttle.has_sent_full = false;
8324 throttle.last_topology = Vec::new();
8328 }
8329 self.emit_discovery_snapshot_on_side(
8334 called_from_queue,
8335 pkt.data_type() == DataType::DiscoverySchemaRequest,
8336 true,
8337 false,
8338 Some(side),
8339 )?;
8340 return Ok(true);
8341 }
8342 let small_topology_scratch =
8343 if pkt.data_type() == DataType::DiscoveryTopology && pkt.payload().len() <= 128 {
8344 let st = self.state.lock();
8345 let sender = pkt
8346 .sender()
8347 .strip_prefix("@addr:")
8348 .and_then(|value| value.parse::<u32>().ok())
8349 .and_then(|address| Self::hostname_for_address_borrowed_locked(&st, address))
8350 .unwrap_or(pkt.sender());
8351 st.discovery_routes.get(&side).and_then(|route| {
8352 let peer = route.announcers.get(sender)?;
8353 let mut shallow = discovery::CloneScratch::default();
8354 shallow.buffer(route.reachable.len().saturating_mul(4));
8355 shallow.buffer(route.reachable_network_variables.len().saturating_mul(4));
8356 shallow.buffer(
8357 route
8358 .requested_network_variables
8359 .len()
8360 .saturating_mul(core::mem::size_of::<(DataType, u64)>()),
8361 );
8362 shallow.strings(&route.reachable_timesync_sources);
8363 for (name, value) in &route.announcers {
8364 shallow.buffer(name.len());
8365 shallow.buffer(value.reachable.len().saturating_mul(4));
8366 shallow.buffer(value.advertised_reachable.len().saturating_mul(4));
8367 shallow.strings(&value.reachable_timesync_sources);
8368 shallow.strings(&value.advertised_reachable_timesync_sources);
8369 shallow.buffer(value.reachable_network_variables.len().saturating_mul(4));
8370 }
8371 let entry = core::mem::size_of::<String>()
8372 + core::mem::size_of::<DiscoverySenderState>();
8373 shallow.bytes = shallow
8374 .bytes
8375 .saturating_add(route.announcers.len().saturating_mul(entry));
8376 shallow.largest = shallow
8377 .largest
8378 .max(entry.saturating_mul(11).saturating_add(128));
8379 let mut deep = discovery::CloneScratch::default();
8380 if discovery::topology_payload_is_incremental(pkt.payload()) {
8381 deep.graph(&peer.topology_boards, pkt.payload().len() / 16);
8382 }
8383 Some((
8384 2048usize
8385 .saturating_add(shallow.bytes.saturating_mul(3))
8386 .saturating_add(deep.bytes)
8387 .saturating_add(pkt.payload().len().saturating_mul(4)),
8388 shallow.largest.max(deep.largest).max(2048),
8389 ))
8390 })
8391 } else {
8392 None
8393 };
8394 let retained_summary = if small_topology_scratch.is_none()
8395 && matches!(
8396 pkt.data_type(),
8397 DataType::DiscoveryAddress
8398 | DataType::DiscoveryTopology
8399 | DataType::DiscoveryAnnounce
8400 | DataType::DiscoveryTimeSyncSources
8401 | DataType::ManagedVariableRequest
8402 ) {
8403 self.state
8404 .lock()
8405 .discovery_routes
8406 .get(&side)
8407 .map_or(0, |route| {
8408 RouterInner::discovery_route_byte_cost(side, route)
8409 })
8410 } else {
8411 0
8412 };
8413 let (decode_bytes, largest_block) = if let Some(scratch) = small_topology_scratch {
8414 scratch
8415 } else if pkt.data_type() == DataType::ManagedVariableRequest {
8416 let ty = discovery::decode_managed_variable_request(pkt)?;
8417 let existing = self
8418 .state
8419 .lock()
8420 .discovery_routes
8421 .get(&side)
8422 .is_some_and(|route| {
8423 route
8424 .requested_network_variables
8425 .iter()
8426 .any(|(id, _)| *id == ty)
8427 });
8428 if existing {
8429 (1024, 256)
8430 } else {
8431 (
8434 2048usize.saturating_add(retained_summary.saturating_mul(2)),
8435 retained_summary
8436 .checked_next_power_of_two()
8437 .unwrap_or(usize::MAX)
8438 .max(1024),
8439 )
8440 }
8441 } else {
8442 let scratch = if pkt.data_type() == DataType::DiscoveryAddress {
8443 4096usize.saturating_add(retained_summary)
8444 } else {
8445 8192usize.saturating_add(retained_summary)
8446 };
8447 let decode_bytes = pkt
8448 .payload()
8449 .len()
8450 .saturating_mul(4)
8451 .saturating_add(scratch);
8452 let largest_block = pkt.payload().len().saturating_mul(2).max(2048);
8453 (decode_bytes, largest_block)
8454 };
8455 while let Err(err) = crate::memory_admission::check(decode_bytes, largest_block) {
8456 if !matches!(err, TelemetryError::Io("memory pressure")) {
8457 return Err(err);
8458 }
8459 let mut st = self.state.lock();
8460 #[cfg(feature = "compression")]
8461 if st.reclaim_codec_metadata() {
8462 continue;
8463 }
8464
8465 let Some(victim) = st.lower_priority_shared_queue(255) else {
8466 return Err(err);
8467 };
8468 if !st.pop_shared_queue_item(victim) {
8469 return Err(err);
8470 }
8471 }
8472 if pkt.data_type() == DataType::DiscoveryAddress {
8473 let mut ad = discovery::decode_discovery_address(pkt)?;
8474 let identity = ad.clone();
8475 let mut changed = false;
8476 let sender_id = if ad.hostname.is_empty() {
8482 pkt.sender()
8483 } else {
8484 ad.hostname.as_str()
8485 };
8486 let now_ms = self.clock.now_ms();
8487 let mut st = self.state.lock();
8488 let side_link_local_enabled = st
8489 .sides
8490 .get(side)
8491 .and_then(|entry| entry.as_ref())
8492 .map(|side_ref| side_ref.opts.link_local_enabled)
8493 .unwrap_or(false);
8494 if !side_link_local_enabled {
8495 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
8496 }
8497 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
8498 let previously_reachable_network_variables = route.reachable_network_variables.clone();
8499 if pkt.sender() != sender_id {
8500 route.announcers.remove(pkt.sender());
8501 }
8502 let mut sender_state =
8503 core::mem::take(route.announcers.entry(sender_id.to_string()).or_default());
8504 if sender_state.advertised_address != Some(ad.address) {
8510 sender_state.advertised_address = Some(ad.address);
8511 changed = true;
8512 }
8513 if sender_state.advertised_reachable != ad.reachable_endpoints {
8514 sender_state.advertised_reachable = ad.reachable_endpoints;
8515 changed = true;
8516 }
8517 if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
8518 sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
8519 changed = true;
8520 }
8521 if sender_state.reachable_network_variables != ad.reachable_network_variables {
8522 sender_state.reachable_network_variables = ad.reachable_network_variables;
8523 changed = true;
8524 }
8525 if sender_state.link_capabilities != Some(ad.link_capabilities) {
8526 sender_state.link_capabilities = Some(ad.link_capabilities);
8527 changed = true;
8528 }
8529 Self::refresh_sender_topology_state(&mut sender_state);
8530 sender_state.last_seen_ms = now_ms;
8531 route.announcers.insert(sender_id.to_string(), sender_state);
8532 Self::recompute_discovery_side_state(&mut route);
8533 let newly_reachable_network_variables: Vec<DataType> = route
8534 .reachable_network_variables
8535 .iter()
8536 .copied()
8537 .filter(|ty| !previously_reachable_network_variables.contains(ty))
8538 .collect();
8539 st.commit_discovery_route(side, route)?;
8540 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8541 if changed {
8542 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8543 }
8544 let replay: Vec<Packet> = newly_reachable_network_variables
8550 .iter()
8551 .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
8552 .filter_map(|ty| {
8553 st.managed_variable_latest
8554 .get(&ty.as_u32())
8555 .map(|entry| entry.packet.clone())
8556 })
8557 .collect();
8558 drop(st);
8559 self.ingest_address_advertisement(identity)?;
8560 #[cfg(feature = "compression")]
8561 self.note_payload_codec_packet(side, pkt, true);
8562 for value in replay {
8563 self.emit_internal_tx_with_priority(
8564 RouterTxItem::ToSide {
8565 src: None,
8566 dst: side,
8567 data: RouterItem::Packet(value),
8568 },
8569 true,
8570 crate::transport_priority(DataType::ManagedVariableValue),
8571 called_from_queue,
8572 )?;
8573 }
8574 return Ok(true);
8575 }
8576 let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
8577 Some(discovery::decode_discovery_topology_update(pkt)?)
8578 } else {
8579 None
8580 };
8581 let packet_sender = {
8582 let st = self.state.lock();
8583 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
8584 if canonical != pkt.sender() {
8585 canonical
8586 } else if let Some(address) = pkt
8587 .sender()
8588 .strip_prefix("@addr:")
8589 .and_then(|value| value.parse::<u32>().ok())
8590 {
8591 decoded_topology
8597 .as_ref()
8598 .and_then(|update| {
8599 update
8600 .boards
8601 .iter()
8602 .find(|board| sender_address_u32(&board.sender_id) == address)
8603 })
8604 .map(|board| board.sender_id.clone())
8605 .unwrap_or(canonical)
8606 } else {
8607 canonical
8608 }
8609 };
8610 let local_sender = self.sender_arc();
8611 if packet_sender == local_sender.as_ref() {
8612 return Ok(true);
8613 }
8614
8615 if pkt.data_type() == DataType::ManagedVariableRequest {
8616 let ty = discovery::decode_managed_variable_request(pkt)?;
8617 {
8623 let now_ms = self.clock.now_ms();
8624 let mut st = self.state.lock();
8625 let existing = st.discovery_routes.get_mut(&side).and_then(|route| {
8629 let (_, last_seen_ms) = route
8630 .requested_network_variables
8631 .iter_mut()
8632 .find(|(requested, _)| *requested == ty)?;
8633 *last_seen_ms = now_ms;
8634 route.last_seen_ms = route.last_seen_ms.max(now_ms);
8635 Some(())
8636 });
8637 let newly_reachable = existing.is_none();
8638 if newly_reachable {
8639 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
8640 route.requested_network_variables.push((ty, now_ms));
8641 Self::recompute_discovery_side_state(&mut route);
8642 st.commit_discovery_route(side, route)?;
8643 }
8644 if newly_reachable {
8645 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8646 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8647 }
8648 }
8649 if !self.can_write_managed_variable(ty) {
8650 return Ok(false);
8655 }
8656 if let Some(value) = self.managed_variable_latest(ty) {
8657 self.emit_internal_tx(
8658 RouterTxItem::ToSide {
8659 src: None,
8660 dst: side,
8661 data: RouterItem::Packet(value),
8662 },
8663 true,
8664 called_from_queue,
8665 )?;
8666 return Ok(true);
8667 }
8668 return Ok(false);
8671 }
8672 #[cfg(feature = "std")]
8673 if pkt.data_type() == DataType::DiscoverySchema {
8674 #[cfg(feature = "compression")]
8675 let snapshot =
8676 self.decode_required_schema(|| discovery::decode_discovery_schema(pkt))?;
8677 #[cfg(not(feature = "compression"))]
8678 let snapshot = discovery::decode_discovery_schema(pkt)?;
8679 let mut st = self.state.lock();
8688 let budget = st.schema_growth_budget();
8689 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
8690 if report.changed() {
8691 st.fit_discovery_budget();
8692 }
8693 drop(st);
8694 self.note_schema_received(side, &packet_sender);
8695 return Ok(true);
8696 }
8697 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
8698 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
8699 #[cfg(feature = "compression")]
8700 self.note_payload_codec_packet(side, pkt, true);
8701 return Ok(true);
8702 }
8703 let mut announce = if pkt.data_type() == DataType::DiscoveryAnnounce {
8704 Some(discovery::decode_discovery_announce(pkt)?)
8705 } else {
8706 None
8707 };
8708 let mut time_sources = if pkt.data_type() == DataType::DiscoveryTimeSyncSources {
8709 Some(discovery::decode_discovery_timesync_sources(pkt)?)
8710 } else {
8711 None
8712 };
8713 let mut st = self.state.lock();
8714 let now_ms = self.clock.now_ms();
8715 if pkt.data_type() == DataType::DiscoveryLeave {
8716 let leaving = packet_sender.as_str();
8717 let before = st.discovery_routes.clone();
8718 for route in st.discovery_routes.values_mut() {
8719 route.announcers.remove(leaving);
8720 for sender_state in route.announcers.values_mut() {
8721 Arc::make_mut(&mut sender_state.topology_boards)
8722 .retain(|board| board.sender_id != leaving);
8723 for board in Arc::make_mut(&mut sender_state.topology_boards).iter_mut() {
8724 board.connections.retain(|peer| peer != leaving);
8725 }
8726 Self::refresh_sender_topology_state(sender_state);
8727 }
8728 Self::recompute_discovery_side_state(route);
8729 }
8730 st.discovery_routes
8731 .retain(|_, route| !route.announcers.is_empty());
8732 if st.discovery_routes != before {
8733 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8734 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8735 }
8736 #[cfg(feature = "compression")]
8737 {
8738 drop(st);
8739 self.note_payload_codec_packet(side, pkt, true);
8740 }
8741 return Ok(true);
8742 }
8743 if let Some(update) = decoded_topology.as_mut() {
8745 for board in update.boards.iter_mut() {
8746 board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
8747 for peer in board.connections.iter_mut() {
8748 *peer = Self::canonical_sender_locked(&st, peer);
8749 }
8750 }
8751 }
8752 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
8753 let alias_state = if packet_sender != pkt.sender() {
8754 route.announcers.remove(pkt.sender())
8755 } else {
8756 None
8757 };
8758 let side_link_local_enabled = st
8759 .sides
8760 .get(side)
8761 .and_then(|entry| entry.as_ref())
8762 .map(|side_ref| side_ref.opts.link_local_enabled)
8763 .unwrap_or(false);
8764 let mut sender_state = route
8765 .announcers
8766 .get_mut(&packet_sender)
8767 .map(core::mem::take)
8768 .or(alias_state)
8769 .unwrap_or_default();
8770 let changed = match pkt.data_type() {
8771 DataType::DiscoveryAnnounce => {
8772 let mut reachable = announce.take().expect("decoded above");
8773 if reachable.is_empty() {
8778 false
8779 } else {
8780 if !side_link_local_enabled {
8781 reachable.retain(|ep| !ep.is_link_local_only());
8782 }
8783 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
8784 let changed = board.reachable_endpoints != reachable;
8785 board.reachable_endpoints = reachable;
8786 Self::refresh_sender_topology_state(&mut sender_state);
8787 changed
8788 }
8789 }
8790 DataType::DiscoveryTimeSyncSources => {
8791 let sources = time_sources.take().expect("decoded above");
8792 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
8793 let changed = board.reachable_timesync_sources != sources;
8794 board.reachable_timesync_sources = sources;
8795 Self::refresh_sender_topology_state(&mut sender_state);
8796 changed
8797 }
8798 DataType::DiscoveryTopology => {
8799 let mut update = decoded_topology
8800 .take()
8801 .expect("topology packet was decoded before route selection");
8802 if !side_link_local_enabled {
8803 for board in update.boards.iter_mut() {
8804 board
8805 .reachable_endpoints
8806 .retain(|ep| !ep.is_link_local_only());
8807 }
8808 }
8809 if !update.incremental {
8810 sender_state.has_full_topology = true;
8811 }
8812 let changed = discovery::apply_shared_topology_update(
8813 &mut sender_state.topology_boards,
8814 update,
8815 );
8816 Self::refresh_sender_topology_state(&mut sender_state);
8817 changed
8818 }
8819 DataType::DiscoverySchema => false,
8820 _ => false,
8821 };
8822 sender_state.last_seen_ms = now_ms;
8823 route.announcers.insert(packet_sender, sender_state);
8824 Self::recompute_discovery_side_state(&mut route);
8825 st.commit_discovery_route(side, route)?;
8826 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8827 if changed {
8828 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8829 }
8830 Ok(true)
8831 }
8832
8833 #[cfg(not(feature = "discovery"))]
8834 fn learn_discovery_packet(
8835 &self,
8836 _pkt: &Packet,
8837 _src: Option<RouterSideId>,
8838 _called_from_queue: bool,
8839 ) -> TelemetryResult<bool> {
8840 Ok(false)
8841 }
8842
8843 #[inline]
8844 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
8845 (side, ty.as_u32())
8846 }
8847
8848 fn reliable_tx_state_mut<'a>(
8849 &'a self,
8850 st: &'a mut RouterInner,
8851 side: RouterSideId,
8852 ty: DataType,
8853 ) -> &'a mut ReliableTxState {
8854 let key = Self::reliable_key(side, ty);
8855 st.reliable_tx
8856 .entry(key)
8857 .or_insert_with(|| ReliableTxState {
8858 next_seq: 1,
8859 sent_order: VecDeque::new(),
8860 sent: Default::default(),
8861 })
8862 }
8863
8864 fn reliable_rx_state_mut<'a>(
8865 &'a self,
8866 st: &'a mut RouterInner,
8867 side: RouterSideId,
8868 ty: DataType,
8869 ) -> &'a mut ReliableRxState {
8870 let key = Self::reliable_key(side, ty);
8871 st.reliable_rx
8872 .entry(key)
8873 .or_insert_with(|| ReliableRxState {
8874 expected_seq: 1,
8875 buffered: Default::default(),
8876 })
8877 }
8878
8879 fn reliable_control_packet(
8880 &self,
8881 control_ty: DataType,
8882 ty: DataType,
8883 seq: u32,
8884 ) -> TelemetryResult<Packet> {
8885 let mut bytes = [0u8; 8];
8886 bytes[..4].copy_from_slice(&ty.as_u32().to_le_bytes());
8887 bytes[4..].copy_from_slice(&seq.to_le_bytes());
8888 Packet::new_local(
8889 control_ty,
8890 message_meta(control_ty).endpoints_owned(),
8891 self.sender_arc(),
8892 self.packet_timestamp_ms(),
8893 crate::config::StandardSmallPayload::new(&bytes),
8894 )
8895 }
8896
8897 fn queue_reliable_ack(
8898 &self,
8899 side: RouterSideId,
8900 ty: DataType,
8901 seq: u32,
8902 called_from_queue: bool,
8903 ) -> TelemetryResult<()> {
8904 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
8905 self.emit_internal_tx_with_priority(
8906 RouterTxItem::ToSide {
8907 src: None,
8908 dst: side,
8909 data: RouterItem::Packet(pkt),
8910 },
8911 true,
8912 message_priority(DataType::ReliableAck),
8913 called_from_queue,
8914 )
8915 }
8916
8917 fn queue_reliable_packet_request(
8918 &self,
8919 side: RouterSideId,
8920 ty: DataType,
8921 seq: u32,
8922 called_from_queue: bool,
8923 ) -> TelemetryResult<()> {
8924 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
8925 self.emit_internal_tx_with_priority(
8926 RouterTxItem::ToSide {
8927 src: None,
8928 dst: side,
8929 data: RouterItem::Packet(pkt),
8930 },
8931 true,
8932 message_priority(DataType::ReliablePacketRequest),
8933 called_from_queue,
8934 )
8935 }
8936
8937 fn queue_reliable_partial_ack(
8938 &self,
8939 side: RouterSideId,
8940 ty: DataType,
8941 seq: u32,
8942 called_from_queue: bool,
8943 ) -> TelemetryResult<()> {
8944 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
8945 self.emit_internal_tx_with_priority(
8946 RouterTxItem::ToSide {
8947 src: None,
8948 dst: side,
8949 data: RouterItem::Packet(pkt),
8950 },
8951 true,
8952 message_priority(DataType::ReliablePartialAck),
8953 called_from_queue,
8954 )
8955 }
8956
8957 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
8958 let mut st = self.state.lock();
8959 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8960 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
8961 tx_state.sent.remove(&ack);
8962 tx_state.sent_order.retain(|seq| *seq != ack);
8963 return;
8964 }
8965
8966 while let Some(seq) = tx_state.sent_order.front().copied() {
8967 if seq > ack {
8968 break;
8969 }
8970 tx_state.sent_order.pop_front();
8971 tx_state.sent.remove(&seq);
8972 }
8973 }
8974
8975 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
8976 let mut st = self.state.lock();
8977 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8978 if let Some(sent) = tx_state.sent.get_mut(&seq) {
8979 sent.partial_acked = true;
8980 }
8981 }
8982
8983 fn queue_reliable_retransmit(
8984 &self,
8985 side: RouterSideId,
8986 ty: DataType,
8987 seq: u32,
8988 called_from_queue: bool,
8989 ) -> TelemetryResult<()> {
8990 let mut queued = None;
8991 {
8992 let mut st = self.state.lock();
8993 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8994 if let Some(sent) = tx_state.sent.get_mut(&seq)
8995 && !sent.queued
8996 {
8997 sent.partial_acked = false;
8998 queued = Some(sent.bytes.clone());
8999 }
9000 }
9001
9002 if let Some(bytes) = queued {
9003 let now = self.clock.now_ms();
9004 if called_from_queue {
9005 self.tx_queue_item_with_priority(
9006 RouterTxItem::ReliableReplay {
9007 dst: side,
9008 ty,
9009 seq,
9010 bytes,
9011 },
9012 true,
9013 Self::router_item_priority_bumped(ty),
9014 )?;
9015 } else {
9016 self.tx_item_impl(
9017 RouterTxItem::ReliableReplay {
9018 dst: side,
9019 ty,
9020 seq,
9021 bytes,
9022 },
9023 true,
9024 false,
9025 )?;
9026 }
9027 let mut st = self.state.lock();
9028 if let Some(sent) = st
9029 .reliable_tx
9030 .get_mut(&Self::reliable_key(side, ty))
9031 .and_then(|tx| tx.sent.get_mut(&seq))
9032 {
9033 sent.last_send_ms = sent.last_send_ms.max(now);
9034 }
9035 }
9036
9037 Ok(())
9038 }
9039
9040 fn send_reliable_raw_to_side(
9041 &self,
9042 side: RouterSideId,
9043 bytes: crate::SharedBytes,
9044 relayed: bool,
9045 ) -> TelemetryResult<()> {
9046 let handler = {
9047 let st = self.state.lock();
9048 let side_ref = Self::side_ref(&st, side)?;
9049 if !side_ref.opts.egress_enabled {
9050 return Ok(());
9051 }
9052 (side_ref.tx_handler.clone(), side_ref.opts.clone())
9053 };
9054
9055 let (handler, opts) = handler;
9056
9057 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
9058 return Err(TelemetryError::Io("side tx busy"));
9059 };
9060 let started_ms = self.clock.now_ms();
9061 let (ty, priority) = Self::item_type_priority(&RouterItem::Packed(bytes.clone()))?;
9062 let priority = priority.max(self.effective_transport_priority(ty));
9063 let result = match handler {
9064 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
9065 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
9066 let mut attempts_total = 0usize;
9067 let mut sent_bytes = 0usize;
9068 self.send_side_frames_resumable(side, frames, |frame| {
9069 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9070 Self::packed_handler_call(&packed, frame, priority)
9071 }) {
9072 Ok((_, attempts)) => {
9073 attempts_total = attempts_total.saturating_add(attempts);
9074 sent_bytes = sent_bytes.saturating_add(frame.len());
9075 }
9076 Err((err, attempts)) => {
9077 self.note_side_tx_failure(
9078 side,
9079 ty,
9080 attempts_total.saturating_add(attempts),
9081 );
9082 return Err(err);
9083 }
9084 }
9085
9086 Ok(())
9087 })?;
9088 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9089 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9090 return Ok(());
9091 }
9092 RouterTxHandlerFn::Packet(f) => {
9093 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
9094 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
9095 }
9096 };
9097 match result {
9098 Ok((_, attempts)) => {
9099 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
9100 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
9101 Ok(())
9102 }
9103 Err((err, attempts)) => {
9104 self.note_side_tx_failure(side, ty, attempts);
9105 Err(err)
9106 }
9107 }
9108 }
9109
9110 #[allow(unused_mut)]
9111 #[cfg(feature = "compression")]
9112 fn send_queued_to_side(
9113 &self,
9114 side: RouterSideId,
9115 data: &mut RouterItem,
9116 relayed: bool,
9117 ) -> (
9118 TelemetryResult<()>,
9119 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
9120 ) {
9121 let reservation = match self.prepare_queued_payload(side, data) {
9122 Ok(reservation) => reservation,
9123 Err(error) => return (Err(error), None),
9124 };
9125 (
9126 self.send_reliable_to_side(side, data.clone(), relayed),
9127 reservation,
9128 )
9129 }
9130
9131 fn send_reliable_to_side(
9132 &self,
9133 side: RouterSideId,
9134 data: RouterItem,
9135 relayed: bool,
9136 ) -> TelemetryResult<()> {
9137 let (ty, addressed) = match &data {
9140 RouterItem::Packet(pkt) => (pkt.data_type(), !pkt.wire_target_senders().is_empty()),
9141 RouterItem::Packed(bytes) => {
9142 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
9143 (env.ty, env.target_senders().len() != 0)
9144 }
9145 };
9146 let end_to_end_ordered =
9147 addressed && matches!(reliable_mode(ty), crate::ReliableMode::Ordered);
9148 let (handler, opts, hop_reliable_enabled) = {
9149 let st = self.state.lock();
9150 let side_ref = Self::side_ref(&st, side)?;
9151 let opts = side_ref.opts.clone();
9152 let hop_reliable_enabled = opts.reliable_enabled
9153 && !end_to_end_ordered
9154 && self.cfg.reliable_enabled()
9155 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
9156 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
9157 };
9158
9159 if !matches!(
9160 &handler,
9161 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
9162 ) {
9163 return self.call_side_tx_handler(side, &handler, &data, relayed);
9164 }
9165
9166 if !hop_reliable_enabled {
9167 let mut adjusted_opts = opts;
9168 adjusted_opts.reliable_enabled = false;
9169 let preserve_end_to_end_ack = true;
9174 if let Some(adjusted) =
9175 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
9176 {
9177 return self.call_side_tx_handler(side, &handler, &adjusted, relayed);
9178 }
9179 return Ok(());
9180 }
9181
9182 if let RouterItem::Packed(bytes) = &data {
9183 wire_format::peek_routing_frame_info(bytes.as_ref())?;
9184 }
9185 let priority = self.item_service_priority(&data)?;
9186
9187 if !is_reliable_type(ty) {
9188 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
9189 self.call_side_tx_handler(side, &handler, &adjusted, relayed)?;
9190 }
9191 return Ok(());
9192 }
9193
9194 #[cfg(feature = "compression")]
9195 let data = match data {
9196 RouterItem::Packed(bytes) => {
9197 let budget = self.state.lock().memory.max_queue_budget;
9198 let normalized =
9199 wire_format::payload_encoding_for_retention(bytes.as_ref(), budget)?;
9200 RouterItem::Packed(
9201 normalized
9202 .map(crate::shared_bytes::convert)
9203 .unwrap_or(bytes),
9204 )
9205 }
9206 packet => packet,
9207 };
9208 let (seq, flags) = {
9209 let mut st = self.state.lock();
9210 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
9211 if tx_state.sent.len() >= runtime_reliable_max_pending() {
9212 return Err(TelemetryError::Io("reliable window full"));
9213 }
9214 let seq = tx_state.next_seq;
9215 let next = tx_state.next_seq.wrapping_add(1);
9216 tx_state.next_seq = if next == 0 { 1 } else { next };
9217 let flags = match reliable_mode(ty) {
9218 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
9219 _ => 0,
9220 };
9221 (seq, flags)
9222 };
9223
9224 let bytes: crate::SharedBytes = match data {
9225 RouterItem::Packet(pkt) => self.pack_packet_for_router(
9226 &pkt,
9227 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
9228 )?,
9229 RouterItem::Packed(bytes) => {
9230 #[cfg(feature = "cryptography")]
9231 if self.e2e_seal_config_for_type(ty).is_some() {
9232 self.prepare_packed_for_remote(
9233 bytes,
9234 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
9235 )?
9236 } else {
9237 let Some(rewritten) =
9238 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
9239 else {
9240 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
9241 return Err(TelemetryError::Io("side tx busy"));
9242 };
9243 let started_ms = self.clock.now_ms();
9244 let frames =
9245 self.encode_side_transport_frames(side, opts, bytes.clone())?;
9246 let mut attempts_total = 0usize;
9247 let mut sent_bytes = 0usize;
9248 self.send_side_frames_resumable(side, frames, |frame| {
9249 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9250 Self::packed_handler_call(&handler, frame, priority)
9251 }) {
9252 Ok((_, attempts)) => {
9253 attempts_total = attempts_total.saturating_add(attempts);
9254 sent_bytes = sent_bytes.saturating_add(frame.len());
9255 }
9256 Err((err, attempts)) => {
9257 self.note_side_tx_failure(
9258 side,
9259 ty,
9260 attempts_total.saturating_add(attempts),
9261 );
9262 return Err(err);
9263 }
9264 }
9265
9266 Ok(())
9267 })?;
9268 self.record_side_tx_sample(
9269 side,
9270 sent_bytes,
9271 started_ms,
9272 self.clock.now_ms(),
9273 );
9274 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9275 return Ok(());
9276 };
9277 crate::shared_bytes::convert(rewritten)
9278 }
9279 #[cfg(not(feature = "cryptography"))]
9280 {
9281 let Some(rewritten) =
9282 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
9283 else {
9284 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
9285 return Err(TelemetryError::Io("side tx busy"));
9286 };
9287 let started_ms = self.clock.now_ms();
9288 let frames =
9289 self.encode_side_transport_frames(side, opts, bytes.clone())?;
9290 let mut attempts_total = 0usize;
9291 let mut sent_bytes = 0usize;
9292 self.send_side_frames_resumable(side, frames, |frame| {
9293 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9294 Self::packed_handler_call(&handler, frame, priority)
9295 }) {
9296 Ok((_, attempts)) => {
9297 attempts_total = attempts_total.saturating_add(attempts);
9298 sent_bytes = sent_bytes.saturating_add(frame.len());
9299 }
9300 Err((err, attempts)) => {
9301 self.note_side_tx_failure(
9302 side,
9303 ty,
9304 attempts_total.saturating_add(attempts),
9305 );
9306 return Err(err);
9307 }
9308 }
9309
9310 Ok(())
9311 })?;
9312 self.record_side_tx_sample(
9313 side,
9314 sent_bytes,
9315 started_ms,
9316 self.clock.now_ms(),
9317 );
9318 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9319 return Ok(());
9320 };
9321 crate::shared_bytes::convert(rewritten)
9322 }
9323 }
9324 };
9325
9326 #[cfg(feature = "compact-packet-store")]
9327 let mut bytes = bytes;
9328
9329 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
9330 return Err(TelemetryError::Io("side tx busy"));
9331 };
9332 let started_ms = self.clock.now_ms();
9333 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
9334 let mut attempts_total = 0usize;
9335 let mut sent_bytes = 0usize;
9336 self.send_side_frames_resumable(side, frames, |frame| {
9337 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9338 Self::packed_handler_call(&handler, frame, priority)
9339 }) {
9340 Ok((_, attempts)) => {
9341 attempts_total = attempts_total.saturating_add(attempts);
9342 sent_bytes = sent_bytes.saturating_add(frame.len());
9343 }
9344 Err((err, attempts)) => {
9345 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
9346 return Err(err);
9347 }
9348 }
9349
9350 Ok(())
9351 })?;
9352 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9353 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9354
9355 #[cfg(feature = "compact-packet-store")]
9356 {
9357 let _ = bytes.park_for_queue();
9358 }
9359
9360 {
9361 let mut st = self.state.lock();
9362 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
9363 tx_state.sent_order.push_back(seq);
9364 tx_state.sent.insert(
9365 seq,
9366 ReliableSent {
9367 bytes: bytes.clone(),
9368 last_send_ms: self.clock.now_ms(),
9369 retries: 0,
9370 queued: false,
9371 partial_acked: false,
9372 },
9373 );
9374 }
9375
9376 Ok(())
9377 }
9378
9379 fn run_side_chunk_transfer<T>(
9380 &self,
9381 side: RouterSideId,
9382 key: u64,
9383 send: impl FnOnce(&mut usize) -> TelemetryResult<T>,
9384 ) -> TelemetryResult<T> {
9385 let mut next = {
9386 let mut st = self.state.lock();
9387 let transport = st
9388 .side_transport
9389 .get_mut(&side)
9390 .ok_or(TelemetryError::BadArg)?;
9391 if transport.tx_chunk_progress.is_none() {
9392 crate::memory_admission::check(
9393 512,
9394 core::mem::size_of::<crate::side_transport::SideTxProgress>(),
9395 )?;
9396 transport.tx_chunk_progress =
9397 Some(Box::new(crate::side_transport::SideTxProgress::default()));
9398 }
9399 transport
9400 .tx_chunk_progress
9401 .as_mut()
9402 .expect("initialized")
9403 .begin(key, self.clock.now_ms())?
9404 };
9405 let result = send(&mut next);
9406 if let Some(progress) = self
9407 .state
9408 .lock()
9409 .side_transport
9410 .get_mut(&side)
9411 .and_then(|transport| transport.tx_chunk_progress.as_mut())
9412 {
9413 progress.finish(key, next, self.clock.now_ms(), &result);
9414 }
9415 result
9416 }
9417
9418 fn send_side_frames_resumable(
9419 &self,
9420 side: RouterSideId,
9421 mut frames: SideTransportFrames,
9422 mut send: impl FnMut(&[u8]) -> TelemetryResult<()>,
9423 ) -> TelemetryResult<()> {
9424 let bootstraps_template = frames.bootstraps_template();
9425 let result = if frames.len() == 1 {
9426 frames.send_resuming(&mut 0, send)
9427 } else {
9428 let key = frames.progress_key();
9429 self.run_side_chunk_transfer(side, key, |next| frames.send_resuming(next, &mut send))
9430 };
9431 if result.is_err() && bootstraps_template {
9432 if let Some(state) = self.state.lock().side_transport.get_mut(&side) {
9437 state.refresh_transmit_header_templates();
9438 }
9439 }
9440 result
9441 }
9442
9443 fn split_side_transport_frame(
9444 &self,
9445 _side: RouterSideId,
9446 frame: crate::SharedBytes,
9447 max_frame_bytes: usize,
9448 ) -> TelemetryResult<SideTransportFrames> {
9449 SideTransportFrames::split(frame, max_frame_bytes, self.sender_arc().as_bytes())
9450 }
9451
9452 #[cfg(all(feature = "compression", feature = "discovery"))]
9453 fn note_payload_codec_packet(&self, side: RouterSideId, pkt: &Packet, accepted: bool) {
9454 let now = self.clock.now_ms();
9455 let mut st = self.state.lock();
9456 if !st
9457 .sides
9458 .get(side)
9459 .and_then(|entry| entry.as_ref())
9460 .is_some_and(|entry| {
9461 matches!(
9462 entry.tx_handler,
9463 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
9464 )
9465 })
9466 {
9467 return;
9468 }
9469 if accepted && pkt.data_type() == crate::DataType::DiscoveryLeave {
9470 for transport in st.side_transport.values_mut() {
9471 let _ = transport.payload_codec.discovery_packet(
9472 pkt,
9473 true,
9474 now,
9475 DISCOVERY_ROUTE_TTL_MS,
9476 );
9477 }
9478 } else if let Some(transport) = st.side_transport.get_mut(&side) {
9479 let _ = transport.payload_codec.discovery_packet(
9480 pkt,
9481 accepted,
9482 now,
9483 DISCOVERY_ROUTE_TTL_MS,
9484 );
9485 }
9486 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
9487 && let Some(transport) = st.side_transport.get_mut(&side)
9488 {
9489 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
9490 }
9491 }
9492 #[cfg(all(feature = "compression", feature = "discovery"))]
9493 fn note_payload_codec_address(&self, side: RouterSideId, payload: &[u8], accepted: bool) {
9494 let now = self.clock.now_ms();
9495 let mut st = self.state.lock();
9496 if !st
9497 .sides
9498 .get(side)
9499 .and_then(|entry| entry.as_ref())
9500 .is_some_and(|entry| {
9501 matches!(
9502 entry.tx_handler,
9503 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
9504 )
9505 })
9506 {
9507 return;
9508 }
9509 if let Some(transport) = st.side_transport.get_mut(&side) {
9510 let _ = transport.payload_codec.address_payload(
9511 payload,
9512 accepted,
9513 now,
9514 DISCOVERY_ROUTE_TTL_MS,
9515 );
9516 }
9517 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
9518 && let Some(transport) = st.side_transport.get_mut(&side)
9519 {
9520 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
9521 }
9522 }
9523
9524 #[cfg(feature = "compression")]
9525 fn payload_for_transport_side(
9526 &self,
9527 side: RouterSideId,
9528 raw: crate::SharedBytes,
9529 ) -> TelemetryResult<crate::SharedBytes> {
9530 #[cfg(not(feature = "discovery"))]
9531 let _ = side;
9532 let (allowed, budget) = {
9533 #[cfg(feature = "discovery")]
9534 let mut st = self.state.lock();
9535 #[cfg(not(feature = "discovery"))]
9536 let st = self.state.lock();
9537 let budget = st.memory.max_queue_budget;
9538 #[cfg(feature = "discovery")]
9539 let allowed = {
9540 let transport = st
9541 .side_transport
9542 .get_mut(&side)
9543 .ok_or(TelemetryError::BadArg)?;
9544 let generation = transport.payload_codec.generation();
9545 if transport.payload_codec_choice != generation {
9546 transport.payload_codec_choice = generation;
9547 transport.tx_chunk_progress = None;
9548 transport.refresh_transmit_header_templates();
9549 }
9550 transport.payload_codec.allowed(self.clock.now_ms())
9551 };
9552 #[cfg(not(feature = "discovery"))]
9553 let allowed = false;
9554 (allowed, budget)
9555 };
9556 Ok(
9557 match wire_format::payload_encoding_for_side(raw.as_ref(), allowed, budget)? {
9558 Some(frame) => crate::shared_bytes::convert(frame),
9559 None => raw,
9560 },
9561 )
9562 }
9563
9564 #[cfg(feature = "compression")]
9568 fn prepare_queued_payload(
9569 &self,
9570 side: RouterSideId,
9571 data: &mut RouterItem,
9572 ) -> TelemetryResult<Option<crate::payload_codec::QueueReservation<'_, RouterInner>>> {
9573 let RouterItem::Packed(bytes) = data else {
9574 return Ok(None);
9575 };
9576 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
9577 return Ok(None);
9578 }
9579 let capable = {
9580 #[cfg(feature = "discovery")]
9581 {
9582 self.state
9583 .lock()
9584 .side_transport
9585 .get(&side)
9586 .is_some_and(|transport| transport.payload_codec.allowed(self.clock.now_ms()))
9587 }
9588 #[cfg(not(feature = "discovery"))]
9589 {
9590 false
9591 }
9592 };
9593 if capable {
9594 return Ok(None);
9595 }
9596 let required = wire_format::decoded_frame_upper_bound(bytes.as_ref())?
9597 .checked_add(
9598 size_of::<crate::SharedBytes>()
9599 + size_of::<bool>()
9600 + size_of::<u8>()
9601 + size_of::<u64>(),
9602 )
9603 .ok_or(TelemetryError::Io("memory pressure"))?;
9604 let reservation = crate::payload_codec::QueueReservation::acquire(&self.state, required)?;
9605 *bytes = self.payload_for_transport_side(side, bytes.clone())?;
9606 Ok(Some(reservation))
9607 }
9608
9609 fn encode_side_transport_frames(
9610 &self,
9611 side: RouterSideId,
9612 opts: RouterSideOptions,
9613 raw: crate::SharedBytes,
9614 ) -> TelemetryResult<SideTransportFrames> {
9615 #[cfg(feature = "compression")]
9616 let raw = self.payload_for_transport_side(side, raw)?;
9617 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
9618 return Ok(SideTransportFrames::single(raw));
9619 }
9620
9621 let raw_len = raw.len();
9622 if opts.max_frame_bytes != 0 && raw_len > opts.max_frame_bytes {
9623 let frames = self.split_side_transport_frame(side, raw, opts.max_frame_bytes)?;
9626 let mut st = self.state.lock();
9627 let stats = st.side_runtime_stats.entry(side).or_default();
9628 stats.note_side_transport_full(raw_len, frames.wire_len());
9629 stats.note_side_transport_chunks(frames.len());
9630 return Ok(frames);
9631 }
9632
9633 if opts.header_template_enabled {
9634 let frame = wire_format::peek_routing_frame_info(raw.as_ref())?;
9638 let without_dictionary = opts.max_side_transport_templates == 0 || {
9639 let st = self.state.lock();
9640 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
9641 };
9642 let reliable = is_reliable_type(frame.envelope.ty)
9643 || frame.reliable.is_some()
9644 || frame.envelope.target_senders().len() != 0
9650 || matches!(
9651 frame.envelope.ty,
9652 crate::DataType::ReliableAck
9653 | crate::DataType::ReliablePartialAck
9654 | crate::DataType::ReliablePacketRequest
9655 );
9656 if without_dictionary || reliable {
9657 let mut st = self.state.lock();
9658 if without_dictionary {
9659 let removed = st
9660 .side_transport
9661 .get_mut(&side)
9662 .ok_or(TelemetryError::BadArg)?
9663 .trim_tx_templates_to(0);
9664 for _ in 0..removed {
9665 st.side_runtime_stats
9666 .entry(side)
9667 .or_default()
9668 .note_side_transport_template_eviction();
9669 }
9670 }
9671 st.side_runtime_stats
9672 .entry(side)
9673 .or_default()
9674 .note_side_transport_full(raw_len, raw_len);
9675 return Ok(SideTransportFrames::single(raw));
9676 }
9677 }
9678
9679 if opts.header_template_enabled && crate::memory_admission::supports_work_kinds() {
9680 let (template, ..) = extract_side_header_template_view(raw.as_ref())?;
9681 let cached = self
9682 .state
9683 .lock()
9684 .side_transport
9685 .get(&side)
9686 .is_some_and(|transport| {
9687 transport.tx_template_ids.contains_key(&template.hash)
9688 && transport.tx_templates.contains_key(&template.hash)
9689 });
9690 if !cached
9691 && matches!(
9692 crate::memory_admission::check_receive(raw.as_ref()),
9693 Err(TelemetryError::Io("memory pressure"))
9694 )
9695 {
9696 self.state
9700 .lock()
9701 .side_runtime_stats
9702 .entry(side)
9703 .or_default()
9704 .note_side_transport_full(raw_len, raw_len);
9705 return Ok(SideTransportFrames::single(raw));
9706 }
9707 }
9708
9709 let mut compact_payload_len = None;
9710 let mut used_compact = false;
9711 let used_timestamp_delta = false;
9712 let omitted_timestamp = false;
9713 let wrapped = if opts.header_template_enabled {
9714 let (template, _ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
9715 extract_side_header_template_view(raw.as_ref())?;
9716 let (template_id, use_compact, _previous_timestamp) = {
9719 let mut st = self.state.lock();
9720 #[cfg(feature = "discovery")]
9725 let mut effective_template_limit = opts.max_side_transport_templates;
9726 #[cfg(not(feature = "discovery"))]
9727 let effective_template_limit = opts.max_side_transport_templates;
9728 #[cfg(feature = "discovery")]
9729 {
9730 if let Some(route) = st.discovery_routes.get(&side) {
9731 let active_peers = route
9732 .announcers
9733 .values()
9734 .filter(|sender| {
9735 self.clock.now_ms().saturating_sub(sender.last_seen_ms)
9736 <= DISCOVERY_ROUTE_TTL_MS
9737 })
9738 .count();
9739 if active_peers > 1 {
9747 effective_template_limit = 0;
9748 } else {
9749 effective_template_limit = route
9750 .announcers
9751 .values()
9752 .filter_map(|sender| sender.link_capabilities)
9753 .filter(|caps| {
9754 caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
9755 && caps.max_side_transport_templates > 0
9756 })
9757 .map(|caps| caps.max_side_transport_templates as usize)
9758 .fold(effective_template_limit, core::cmp::min);
9759 }
9760 }
9761 }
9762 let trimmed = st
9763 .side_transport
9764 .get_mut(&side)
9765 .ok_or(TelemetryError::BadArg)?
9766 .trim_tx_templates_to(effective_template_limit);
9767 for _ in 0..trimmed {
9768 st.side_runtime_stats
9769 .entry(side)
9770 .or_default()
9771 .note_side_transport_template_eviction();
9772 }
9773 let side_state = st
9774 .side_transport
9775 .get_mut(&side)
9776 .ok_or(TelemetryError::BadArg)?;
9777 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
9778 let previous = side_state.tx_last_timestamps.get(&id).copied();
9779 const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
9782 let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
9783 #[cfg(feature = "discovery")]
9785 let discovery_refresh = discovery::is_discovery_type(_ty);
9786 #[cfg(not(feature = "discovery"))]
9787 let discovery_refresh = false;
9788 let use_compact =
9789 *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES && !discovery_refresh;
9790 if use_compact {
9791 *compact_uses = compact_uses.saturating_add(1);
9792 } else {
9793 *compact_uses = 0;
9794 }
9795 (id, use_compact, previous)
9796 } else {
9797 let template_id = ((template.hash >> 32) as u32 ^ template.hash as u32).max(1);
9803 let evicted = side_state.insert_tx_template(
9804 template.into_owned(),
9805 template_id,
9806 effective_template_limit,
9807 );
9808 if evicted {
9809 st.side_runtime_stats
9810 .entry(side)
9811 .or_default()
9812 .note_side_transport_template_eviction();
9813 }
9814 if let Some(side_state) = st.side_transport.get_mut(&side)
9815 && effective_template_limit > 0
9816 {
9817 side_state.tx_last_timestamps.insert(template_id, timestamp);
9818 }
9819 (template_id, false, None)
9820 }
9821 };
9822 if use_compact {
9823 used_compact = true;
9824 compact_payload_len = Some(payload.len());
9825 let timestamp_field = Some(timestamp);
9828 let mut body = Vec::with_capacity(payload.len() + 32);
9829 body.push(flags);
9830 write_uleb128_local(u64::from(template_id), &mut body);
9831 write_uleb128_local(data_size, &mut body);
9832 if let Some(timestamp_field) = timestamp_field {
9833 write_uleb128_local(timestamp_field, &mut body);
9834 }
9835 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
9836 write_uleb128_local(u64::from(nonce), &mut body);
9837 }
9838 if let Some((seq, ack)) = reliable_seq_ack {
9839 write_uleb128_local(u64::from(seq), &mut body);
9840 write_uleb128_local(u64::from(ack), &mut body);
9841 }
9842 body.extend_from_slice(payload);
9843 {
9844 let mut st = self.state.lock();
9845 if let Some(side_state) = st.side_transport.get_mut(&side) {
9846 side_state.tx_last_timestamps.insert(template_id, timestamp);
9847 }
9848 }
9849 let kind = if omitted_timestamp {
9850 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
9851 } else if used_timestamp_delta {
9852 SIDE_TRANSPORT_KIND_COMPACT_DELTA
9853 } else {
9854 SIDE_TRANSPORT_KIND_COMPACT
9855 };
9856 wrap_side_transport_frame(kind, &body)
9857 } else {
9858 let mut body = Vec::with_capacity(raw.len() + 4);
9859 write_uleb128_local(u64::from(template_id), &mut body);
9860 body.extend_from_slice(raw.as_ref());
9861 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
9862 }
9863 } else {
9864 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
9865 };
9866
9867 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
9868 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
9869 } else {
9870 Ok(SideTransportFrames::single(wrapped))
9871 }?;
9872 let wire_len = frames.wire_len();
9873 let mut st = self.state.lock();
9874 let stats = st.side_runtime_stats.entry(side).or_default();
9875 if used_compact {
9876 let overhead = compact_payload_len
9877 .map(|payload_len| wire_len.saturating_sub(payload_len))
9878 .unwrap_or(wire_len);
9879 stats.note_side_transport_compact(
9880 raw_len,
9881 wire_len,
9882 overhead,
9883 used_timestamp_delta,
9884 omitted_timestamp,
9885 );
9886 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
9887 {
9888 stats.note_side_transport_compact_target_miss();
9889 }
9890 } else {
9891 stats.note_side_transport_full(raw_len, wire_len);
9892 }
9893 if frames.len() > 1 {
9894 stats.note_side_transport_chunks(frames.len());
9895 }
9896 Ok(frames)
9897 }
9898
9899 fn expire_side_chunks(&self) {
9900 let now = self.clock.now_ms();
9901 let mut st = self.state.lock();
9902 for side in st.side_transport.values_mut() {
9903 let side = side.as_mut();
9904 #[cfg(all(feature = "compression", feature = "discovery"))]
9905 side.payload_codec.expire(now, DISCOVERY_ROUTE_TTL_MS);
9906 let reclaimed =
9907 SideChunkAssembly::expire(&mut side.rx_chunks, &mut side.rx_chunk_bytes, now);
9908 side.rx_chunk_reclaims = side.rx_chunk_reclaims.saturating_add(reclaimed);
9909 }
9910 }
9911
9912 #[cfg(feature = "discovery")]
9916 fn refresh_ingress_peer_liveness<'a>(
9917 &self,
9918 side: RouterSideId,
9919 bytes: &'a [u8],
9920 ) -> Option<wire_format::RoutingFrameInfo<'a>> {
9921 let frame = wire_format::peek_routing_frame_info(bytes).ok()?;
9922 self.refresh_peer_liveness(side, frame.envelope.source_address);
9923 #[cfg(feature = "compression")]
9924 if frame.envelope.ty == crate::DataType::DiscoveryAddress
9925 && let Ok(Some((_, payload))) =
9926 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryAddress)
9927 {
9928 self.note_payload_codec_address(side, payload, false);
9929 }
9930 Some(frame)
9931 }
9932
9933 #[cfg(feature = "discovery")]
9934 #[cfg_attr(not(feature = "std"), inline(never))]
9935 fn refresh_peer_liveness(&self, side: RouterSideId, address: u32) {
9936 let now = self.clock.now_ms();
9937 let mut st = self.state.lock();
9938 #[cfg(feature = "compression")]
9939 if let Some(transport) = st.side_transport.get_mut(&side) {
9940 transport
9941 .payload_codec
9942 .observe(address, now, DISCOVERY_ROUTE_TTL_MS);
9943 }
9944 let Some(route) = st.discovery_routes.get_mut(&side) else {
9945 return;
9946 };
9947 let mut found = None;
9948 for (name, peer) in &mut route.announcers {
9949 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
9950 if found.is_some() {
9952 return;
9953 }
9954 found = Some(peer);
9955 }
9956 }
9957 let Some(peer) = found else {
9958 return;
9959 };
9960 peer.last_seen_ms = now;
9961 route.last_seen_ms = route.last_seen_ms.max(now);
9962 }
9963
9964 #[cfg(feature = "discovery")]
9968 fn consume_known_packed_topology(
9969 &self,
9970 side: RouterSideId,
9971 bytes: &[u8],
9972 ) -> TelemetryResult<bool> {
9973 let Some((address, payload)) =
9974 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryTopology)?
9975 else {
9976 return Ok(false);
9977 };
9978 let now = self.clock.now_ms();
9979 let mut st = self.state.lock();
9980 let Some(route) = st.discovery_routes.get_mut(&side) else {
9981 return Ok(false);
9982 };
9983 let mut found = None;
9984 for (name, peer) in &mut route.announcers {
9985 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
9986 if found.is_some() {
9987 return Ok(false);
9988 }
9989 found = Some(peer);
9990 }
9991 }
9992 let Some(peer) = found else {
9993 return Ok(false);
9994 };
9995 let Some(full) =
9996 discovery::topology_payload_matches_retained(payload, &peer.topology_boards)
9997 else {
9998 return Ok(false);
9999 };
10000 peer.has_full_topology |= full;
10001 peer.last_seen_ms = now;
10002 route.last_seen_ms = route.last_seen_ms.max(now);
10003 Ok(true)
10004 }
10005
10006 #[cfg(all(feature = "discovery", not(feature = "std")))]
10007 fn consume_known_packed_schema(
10008 &self,
10009 side: RouterSideId,
10010 bytes: &[u8],
10011 ) -> TelemetryResult<bool> {
10012 let Some((address, payload)) =
10013 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoverySchema)?
10014 else {
10015 return Ok(false);
10016 };
10017 self.refresh_peer_liveness(side, address);
10018 self.merge_schema_changes(payload)?;
10019 let mut st = self.state.lock();
10020 if let Some(route) = st.discovery_routes.get_mut(&side) {
10021 for (name, peer) in &mut route.announcers {
10022 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
10023 peer.has_schema = true;
10024 break;
10025 }
10026 }
10027 }
10028 Ok(true)
10029 }
10030
10031 #[cfg(all(feature = "compression", feature = "discovery"))]
10032 fn decode_required_schema<T>(
10033 &self,
10034 mut decode: impl FnMut() -> TelemetryResult<T>,
10035 ) -> TelemetryResult<T> {
10036 loop {
10037 match decode() {
10038 Err(error @ TelemetryError::Io("memory pressure")) => {
10039 if !self.state.lock().reclaim_codec_metadata() {
10040 return Err(error);
10041 }
10042 }
10043 result => return result,
10044 }
10045 }
10046 }
10047
10048 #[cfg(all(feature = "discovery", not(feature = "std")))]
10049 #[inline(never)]
10050 fn merge_schema_changes(&self, payload: &[u8]) -> TelemetryResult<()> {
10051 #[cfg(feature = "compression")]
10054 let snapshot = self.decode_required_schema(|| discovery::decode_schema_changes(payload))?;
10055 #[cfg(not(feature = "compression"))]
10056 let snapshot = discovery::decode_schema_changes(payload)?;
10057 let mut st = self.state.lock();
10058 let budget = st.schema_growth_budget();
10059 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
10060 if report.changed() {
10061 st.fit_discovery_budget();
10062 }
10063 Ok(())
10064 }
10065
10066 #[cfg(all(
10071 feature = "discovery",
10072 any(feature = "std", feature = "ingress-target-filter")
10073 ))]
10074 fn targets_only_ingress(
10075 &self,
10076 side: RouterSideId,
10077 frame: &wire_format::RoutingFrameInfo<'_>,
10078 ) -> bool {
10079 let ty = frame.envelope.ty;
10080 if is_internal_control_type(ty)
10083 || frame.envelope.target_senders().len() == 0
10084 || frame.reliable.is_some_and(|h| {
10085 h.flags != wire_format::RELIABLE_FLAG_UNSEQUENCED || h.seq != 0 || h.ack != 0
10086 })
10087 {
10088 return false;
10089 }
10090 let Some(meta) = crate::config::try_get_message_meta(ty) else {
10091 return false;
10092 };
10093 if frame
10094 .envelope
10095 .wire_shape
10096 .is_some_and(|shape| shape != meta.element)
10097 {
10098 return false;
10099 }
10100 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
10101 let st = self.state.lock();
10102 if st.managed_variable_types.contains(&ty.as_u32())
10103 || Self::has_explicit_route_policy_locked(&st, Some(side), ty)
10104 {
10105 return false;
10106 }
10107 let now = self.clock.now_ms();
10108 frame.envelope.target_senders().all(|target| {
10109 target != local_hash
10110 && target != u64::from(st.local_address.address)
10111 && Self::side_matches_target_senders_locked(
10112 &st,
10113 side,
10114 wire_format::TargetSenderSet::Hashes(&[target]),
10115 now,
10116 )
10117 && !st.sides.iter().enumerate().any(|(other, entry)| {
10118 other != side
10119 && entry.as_ref().is_some_and(|e| e.opts.egress_enabled)
10120 && Self::side_matches_target_senders_locked(
10121 &st,
10122 other,
10123 wire_format::TargetSenderSet::Hashes(&[target]),
10124 now,
10125 )
10126 })
10127 })
10128 }
10129
10130 #[cfg(feature = "discovery")]
10133 fn cached_transit_ready(
10134 &self,
10135 side: RouterSideId,
10136 bytes: &[u8],
10137 view: Option<&wire_format::RoutingFrameInfo<'_>>,
10138 ) -> TelemetryResult<bool> {
10139 if bytes.len() > 128 || bytes.first().is_none_or(|flags| flags & 0x10 != 0) {
10144 return Ok(false);
10145 }
10146 let parsed;
10147 let frame = match view {
10148 Some(frame) => frame,
10149 None => {
10150 parsed = wire_format::peek_routing_frame_info(bytes)?;
10151 &parsed
10152 }
10153 };
10154 let ty = frame.envelope.ty;
10155 if is_internal_control_type(ty)
10156 || frame.envelope.is_reliable
10157 || frame.reliable.is_some()
10158 || frame.envelope.target_senders().len() != 0
10159 || frame.envelope.wire_shape.is_some_and(|shape| {
10163 shape != crate::message_meta(ty).element
10164 })
10165 || !(1..=3).contains(&frame.envelope.endpoints().take(4).count())
10166 {
10167 return Ok(false);
10168 }
10169 if self.is_managed_variable_type(ty)
10170 || frame.envelope.endpoints().any(|ep| {
10171 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
10172 })
10173 {
10174 return Ok(false);
10175 }
10176
10177 let canonical_possible = {
10181 let st = self.state.lock();
10182 st.discovery_routes.get(&side).is_some_and(|route| {
10183 route.announcers.iter().any(|(name, peer)| {
10184 peer.advertised_address == Some(frame.envelope.source_address)
10185 || sender_address_u32(name) == frame.envelope.source_address
10186 })
10187 }) && st.sides.iter().flatten().all(|entry| {
10188 entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes
10189 })
10190 };
10191 let template_hash = if canonical_possible {
10192 None
10193 } else {
10194 Some(extract_side_header_template_view(bytes)?.0.hash)
10195 };
10196 let st = self.state.lock();
10197 let active: usize = st.sides.iter().filter(|entry| entry.is_some()).count();
10198 if active != 2 {
10199 return Ok(false);
10200 }
10201 for (id, entry) in st
10202 .sides
10203 .iter()
10204 .enumerate()
10205 .filter_map(|(id, entry)| entry.as_ref().map(|e| (id, e)))
10206 {
10207 if entry.opts.reliable_enabled
10208 || !matches!(
10209 entry.tx_handler,
10210 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
10211 )
10212 || !st
10213 .side_runtime_stats
10214 .get(&id)
10215 .is_some_and(|stats| stats.data_types.contains_key(&ty.as_u32()))
10216 {
10217 return Ok(false);
10218 }
10219 if id == side {
10220 continue;
10221 }
10222 #[cfg(feature = "discovery")]
10223 if !st.discovery_routes.get(&id).is_some_and(|route| {
10224 self.clock.now_ms().saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
10225 && frame
10226 .envelope
10227 .endpoints()
10228 .any(|ep| route.reachable.contains(&ep))
10229 }) {
10230 return Ok(false);
10231 }
10232 if entry.opts.header_template_enabled
10233 && entry.opts.max_frame_bytes != 0
10234 && entry.opts.max_side_transport_templates != 0
10235 && !self.side_has_multiple_announcers_locked(&st, id, self.clock.now_ms())
10236 && !template_hash.is_some_and(|hash| {
10237 st.side_transport.get(&id).is_some_and(|transport| {
10238 transport.tx_template_ids.contains_key(&hash)
10239 && transport.tx_templates.contains_key(&hash)
10240 })
10241 })
10242 {
10243 if (entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes)
10248 && st.discovery_routes.get(&side).is_some_and(|route| {
10249 route.announcers.iter().any(|(name, peer)| {
10250 peer.advertised_address == Some(frame.envelope.source_address)
10251 || sender_address_u32(name) == frame.envelope.source_address
10252 })
10253 })
10254 {
10255 continue;
10256 }
10257 return Ok(false);
10258 }
10259 }
10260 Ok(true)
10261 }
10262 #[cfg(not(feature = "discovery"))]
10263 fn cached_transit_ready(
10264 &self,
10265 _side: RouterSideId,
10266 _bytes: &[u8],
10267 _view: Option<&wire_format::RoutingFrameInfo<'_>>,
10268 ) -> TelemetryResult<bool> {
10269 Ok(false)
10270 }
10271 #[cfg(any(feature = "std", feature = "fair-admission"))]
10272 fn reclaim_for_receive(&self, bytes: &[u8], victim: &mut Option<(u32, u8)>) -> bool {
10273 let Some((incoming, priority)) = crate::admission_fairness::incoming(bytes) else {
10274 return false;
10275 };
10276 self.reclaim_for_incoming(incoming, priority, victim)
10277 }
10278 #[cfg(any(feature = "std", feature = "fair-admission"))]
10279 fn reclaim_for_incoming(
10280 &self,
10281 incoming: u32,
10282 priority: u8,
10283 victim: &mut Option<(u32, u8)>,
10284 ) -> bool {
10285 let mut st = self.state.lock();
10286 let (owner, priority) = if let Some(selected) = *victim {
10287 selected
10288 } else {
10289 let Some(owner) = st.competing_bulk_owner(incoming, priority) else {
10290 return false;
10291 };
10292 *victim = Some((owner, priority));
10293 (owner, priority)
10294 };
10295 st.reclaim_bulk_from(owner, priority, None)
10296 }
10297 fn admit_received_frame(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<()> {
10298 self.admit_received_frame_with_view(side, bytes, None)
10299 }
10300
10301 fn admit_received_frame_with_view(
10304 &self,
10305 side: RouterSideId,
10306 bytes: &[u8],
10307 view: Option<&wire_format::RoutingFrameInfo<'_>>,
10308 ) -> TelemetryResult<()> {
10309 if !crate::memory_admission::has_probe() {
10310 return Ok(());
10311 }
10312 #[cfg(any(feature = "std", feature = "fair-admission"))]
10313 let mut fair_victim = None;
10314 if crate::memory_admission::supports_work_kinds()
10315 && self.cached_transit_ready(side, bytes, view)?
10316 {
10317 let result = crate::memory_admission::check_cached_transit(bytes.len());
10318 #[cfg(all(feature = "compression", feature = "discovery"))]
10319 let result = {
10320 let mut result = result;
10321 while matches!(result, Err(TelemetryError::Io("memory pressure")))
10322 && self.state.lock().reclaim_codec_metadata()
10323 {
10324 result = crate::memory_admission::check_cached_transit(bytes.len());
10325 }
10326 result
10327 };
10328
10329 #[cfg(any(feature = "std", feature = "fair-admission"))]
10330 let result = {
10331 let mut result = result;
10332 while matches!(result, Err(TelemetryError::Io("memory pressure"))) {
10333 if !self.reclaim_for_receive(bytes, &mut fair_victim) {
10334 break;
10335 }
10336 result = crate::memory_admission::check_cached_transit(bytes.len());
10337 }
10338 result
10339 };
10340 result
10341 } else {
10342 let managed = wire_format::scheduling_type(bytes)
10343 .is_some_and(|ty| self.is_managed_variable_type(ty));
10344 let mut protected = None;
10345 loop {
10346 match crate::memory_admission::check_receive_with_role(bytes, managed) {
10347 Err(TelemetryError::Io("memory pressure")) => {
10348 #[cfg(all(feature = "compression", feature = "discovery"))]
10349 if self.state.lock().reclaim_codec_metadata() {
10350 continue;
10351 }
10352
10353 let protected = *protected.get_or_insert_with(|| {
10354 wire_format::peek_routing_frame_info(bytes)
10355 .ok()
10356 .is_some_and(|frame| {
10357 managed
10358 || message_priority(frame.envelope.ty) >= 200
10359 || crate::small_frozen_control(
10360 frame.envelope.ty,
10361 frame.envelope.target_senders().len(),
10362 frame.envelope.wire_shape,
10363 )
10364 })
10365 });
10366 if !protected {
10367 #[cfg(any(feature = "std", feature = "fair-admission"))]
10368 if self.reclaim_for_receive(bytes, &mut fair_victim) {
10369 continue;
10370 }
10371 return Err(TelemetryError::Io("memory pressure"));
10372 }
10373 let mut st = self.state.lock();
10377 let reclaimed = st
10378 .lower_priority_shared_queue(199)
10379 .is_some_and(|queue| st.pop_shared_queue_item(queue));
10380 if !reclaimed {
10381 return Err(TelemetryError::Io("memory pressure"));
10382 }
10383 }
10384 result => return result,
10385 }
10386 }
10387 }
10388 }
10389
10390 #[cfg(feature = "discovery")]
10391 fn consume_known_packed_discovery(
10392 &self,
10393 side: RouterSideId,
10394 bytes: &[u8],
10395 ) -> TelemetryResult<bool> {
10396 let result = match wire_format::scheduling_type(bytes) {
10399 #[cfg(not(feature = "std"))]
10400 Some(crate::DataType::DiscoverySchema) => self.consume_known_packed_schema(side, bytes),
10401 Some(crate::DataType::DiscoveryTopology) => {
10402 self.consume_known_packed_topology(side, bytes)
10403 }
10404 Some(crate::DataType::DiscoveryAddress) => {
10405 match wire_format::peek_plain_discovery_payload(
10406 bytes,
10407 crate::DataType::DiscoveryAddress,
10408 ) {
10409 Ok(Some((address, payload))) => {
10410 self.refresh_peer_liveness(side, address);
10411 self.refresh_known_address_payload(payload, side)
10412 }
10413 Ok(None) => Ok(false),
10414 Err(error) => Err(error),
10415 }
10416 }
10417 _ => Ok(false),
10418 };
10419 if result.is_err() {
10420 let _ = self.refresh_ingress_peer_liveness(side, bytes);
10424 }
10425 result
10426 }
10427
10428 fn decode_side_transport_frame(
10429 &self,
10430 side: RouterSideId,
10431 bytes: &[u8],
10432 ) -> TelemetryResult<Option<crate::SharedBytes>> {
10433 self.expire_side_chunks();
10435 let Some((kind, body)) = parse_side_transport_wrapper(bytes)? else {
10436 #[cfg(feature = "discovery")]
10437 if self.consume_known_packed_discovery(side, bytes)? {
10438 return Ok(None);
10439 }
10440 #[cfg(feature = "discovery")]
10441 let view = self.refresh_ingress_peer_liveness(side, bytes);
10442 #[cfg(not(feature = "discovery"))]
10443 let view = None;
10444 #[cfg(all(
10445 feature = "discovery",
10446 any(feature = "std", feature = "ingress-target-filter")
10447 ))]
10448 if bytes.first().is_some_and(|flags| flags & 0x10 == 0)
10449 && view
10450 .as_ref()
10451 .is_some_and(|frame| self.targets_only_ingress(side, frame))
10452 {
10453 return Ok(None);
10454 }
10455 self.admit_received_frame_with_view(side, bytes, view.as_ref())?;
10456 return Ok(Some(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes))));
10457 };
10458 match kind {
10459 SIDE_TRANSPORT_KIND_FULL => {
10460 let header_templates_enabled = self
10461 .state
10462 .lock()
10463 .sides
10464 .get(side)
10465 .and_then(|entry| entry.as_ref())
10466 .map(|side| side.opts.header_template_enabled)
10467 .unwrap_or(false);
10468 let mut off = 0usize;
10469 let template_id = if header_templates_enabled {
10470 Some(
10471 u32::try_from(read_uleb128_local(body, &mut off)?)
10472 .map_err(|_| TelemetryError::Unpack("side template id too large"))?,
10473 )
10474 } else {
10475 None
10476 };
10477 #[cfg(feature = "discovery")]
10478 if self.consume_known_packed_discovery(side, &body[off..])? {
10479 return Ok(None);
10480 }
10481 #[cfg(feature = "discovery")]
10482 let view = self.refresh_ingress_peer_liveness(side, &body[off..]);
10483 #[cfg(not(feature = "discovery"))]
10484 let view = None;
10485 self.admit_received_frame_with_view(side, &body[off..], view.as_ref())?;
10486 let raw = Arc::<[u8]>::from(&body[off..]);
10487 if let (Some(template_id), Ok((template, _, _, _, timestamp, _, _, _))) =
10488 (template_id, extract_side_header_template(raw.as_ref()))
10489 {
10490 let mut st = self.state.lock();
10491 let max_templates = st
10492 .sides
10493 .get(side)
10494 .and_then(|side| side.as_ref())
10495 .map(|side| side.opts.max_side_transport_templates)
10496 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
10497 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
10498 let evicted =
10499 side_state.insert_rx_template(template_id, template, max_templates);
10500 if max_templates > 0 {
10501 side_state.rx_last_timestamps.insert(template_id, timestamp);
10502 }
10503 evicted
10504 });
10505 if evicted {
10506 st.side_runtime_stats
10507 .entry(side)
10508 .or_default()
10509 .note_side_transport_template_eviction();
10510 }
10511 }
10512 Ok(Some(crate::shared_bytes::convert(raw)))
10513 }
10514 SIDE_TRANSPORT_KIND_COMPACT
10515 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
10516 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
10517 if body.is_empty() {
10518 return Err(TelemetryError::Unpack("short side compact frame"));
10519 }
10520 let mut off = 1usize;
10521 let template_id = u32::try_from(read_uleb128_local(body, &mut off)?)
10522 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
10523 let (template, timestamp_base) = {
10524 let st = self.state.lock();
10525 let state = st.side_transport.get(&side);
10526 let template = state
10527 .and_then(|state| state.rx_templates_by_id.get(&template_id))
10528 .cloned();
10529 let timestamp_base = if matches!(
10530 kind,
10531 SIDE_TRANSPORT_KIND_COMPACT_DELTA
10532 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
10533 ) {
10534 state
10535 .and_then(|state| state.rx_last_timestamps.get(&template_id))
10536 .copied()
10537 } else {
10538 None
10539 };
10540 (template, timestamp_base)
10541 };
10542 let Some(template) = template else {
10546 return Ok(None);
10547 };
10548 #[cfg(feature = "discovery")]
10551 if let Some(address) = template.source_address() {
10552 self.refresh_peer_liveness(side, address);
10553 }
10554 let reconstructed_bound = body
10555 .len()
10556 .saturating_add(template.prefix().len())
10557 .saturating_add(template.between().len())
10558 .saturating_add(32);
10559 let protected = template.data_type().is_some_and(|ty| {
10563 message_priority(ty) >= 200 || self.is_managed_variable_type(ty)
10564 });
10565 let ack = template.is_ack();
10566 #[cfg(any(feature = "std", feature = "fair-admission"))]
10567 let mut fair_incoming = None;
10568 #[cfg(any(feature = "std", feature = "fair-admission"))]
10569 let mut fair_victim = None;
10570 loop {
10571 match crate::memory_admission::check_compact_reconstruction(
10572 reconstructed_bound,
10573 ack,
10574 ) {
10575 Err(TelemetryError::Io("memory pressure")) if protected => {
10576 let mut st = self.state.lock();
10577 let reclaimed = st
10578 .lower_priority_shared_queue(199)
10579 .is_some_and(|queue| st.pop_shared_queue_item(queue));
10580 if !reclaimed {
10581 return Err(TelemetryError::Io("memory pressure"));
10582 }
10583 }
10584 Err(TelemetryError::Io("memory pressure")) => {
10585 #[cfg(any(feature = "std", feature = "fair-admission"))]
10589 if let Some((incoming, priority)) =
10590 *fair_incoming.get_or_insert_with(|| {
10591 crate::admission_fairness::compact_incoming(&template)
10592 })
10593 && self.reclaim_for_incoming(incoming, priority, &mut fair_victim)
10594 {
10595 continue;
10596 }
10597 return Err(TelemetryError::Io("memory pressure"));
10598 }
10599 result => {
10600 result?;
10601 break;
10602 }
10603 }
10604 }
10605 let timestamp_mode = match kind {
10606 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
10607 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
10608 _ => SideCompactTimestampMode::Absolute,
10609 };
10610 let (frame, timestamp) = reconstruct_side_compact_frame(
10611 &template,
10612 body[0],
10613 &body[off..],
10614 timestamp_mode,
10615 timestamp_base,
10616 )?;
10617 {
10618 let mut st = self.state.lock();
10619 if let Some(side_state) = st.side_transport.get_mut(&side) {
10620 side_state.rx_last_timestamps.insert(template_id, timestamp);
10621 }
10622 }
10623 if crate::memory_admission::supports_work_kinds() {
10627 self.admit_received_frame(side, frame.as_ref())?;
10628 }
10629 Ok(Some(frame))
10630 }
10631 SIDE_TRANSPORT_KIND_CHUNK => {
10632 if body.len() < 8 {
10633 return Err(TelemetryError::Unpack("short side chunk frame"));
10634 }
10635 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
10636 let index = u16::from_le_bytes([body[4], body[5]]);
10637 let total = u16::from_le_bytes([body[6], body[7]]);
10638 if total == 0 || index >= total {
10639 return Err(TelemetryError::Unpack("invalid side chunk index"));
10640 }
10641 let assembled = {
10642 let mut st = self.state.lock();
10643 let now_ms = self.clock.now_ms();
10644 st.make_side_chunk_room(side, transfer_id, index, total, body.len() - 8)?;
10645 if let Some(entry) = st
10649 .side_transport
10650 .get_mut(&side)
10651 .and_then(|state| state.rx_chunks.get_mut(&transfer_id))
10652 && entry.contains_index(index)
10653 {
10654 return Ok(None);
10657 }
10658 let retained = st
10661 .side_transport
10662 .get(&side)
10663 .and_then(|state| state.rx_chunks.get(&transfer_id))
10664 .map(|entry| entry.received_len());
10665 loop {
10666 match crate::memory_admission::check_retained_fragment(bytes, retained) {
10667 Err(TelemetryError::Io("memory pressure")) => {
10668 if st
10669 .lower_priority_shared_queue(199)
10670 .is_some_and(|queue| st.pop_shared_queue_item(queue))
10671 || st.reclaim_oldest_side_chunk(Some((side, transfer_id)))
10672 {
10673 continue;
10674 }
10675 return Err(TelemetryError::Io("memory pressure"));
10676 }
10677 result => {
10678 result?;
10679 break;
10680 }
10681 }
10682 }
10683 let side_state = st
10684 .side_transport
10685 .get_mut(&side)
10686 .ok_or(TelemetryError::BadArg)?;
10687 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
10688 entry.total = total;
10689 let added =
10690 entry.insert_with(index, now_ms, || Arc::<[u8]>::from(&body[8..]))?;
10691 side_state.rx_chunk_bytes = side_state.rx_chunk_bytes.saturating_add(added);
10692 if entry.received_len() == usize::from(total) {
10693 let entry = side_state
10694 .take_chunk(transfer_id, false)
10695 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
10696 Some(entry.assemble().inspect_err(|_| {
10697 side_state.rx_chunk_reclaims =
10698 side_state.rx_chunk_reclaims.saturating_add(1);
10699 })?)
10700 } else {
10701 None
10702 }
10703 };
10704 match assembled {
10705 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
10706 None => Ok(None),
10707 }
10708 }
10709 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
10710 }
10711 }
10712
10713 fn packed_handler_call(
10714 handler: &RouterTxHandlerFn,
10715 bytes: &[u8],
10716 priority: u8,
10717 ) -> TelemetryResult<()> {
10718 match handler {
10719 RouterTxHandlerFn::Packed(handler) => handler(bytes),
10720 RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
10721 RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
10722 }
10723 }
10724
10725 fn effective_transport_priority(&self, ty: DataType) -> u8 {
10726 if self.is_managed_variable_type(ty) {
10727 crate::transport_priority(DataType::ManagedVariableValue)
10728 } else {
10729 crate::transport_priority(ty)
10730 }
10731 }
10732
10733 fn call_side_tx_handler(
10734 &self,
10735 side: RouterSideId,
10736 handler: &RouterTxHandlerFn,
10737 data: &RouterItem,
10738 relayed: bool,
10739 ) -> TelemetryResult<()> {
10740 let opts = {
10741 let st = self.state.lock();
10742 Self::side_ref(&st, side)?.opts.clone()
10743 };
10744 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10745 return Err(TelemetryError::Io("side tx busy"));
10746 };
10747 let started_ms = self.clock.now_ms();
10748 let (ty, priority) = self.item_service_type_priority(data)?;
10750 #[cfg(all(feature = "discovery", feature = "timesync"))]
10751 if !Self::timesync_allowed_for_side_locked(
10752 &self.state.lock(),
10753 side,
10754 ty,
10755 self.clock.now_ms(),
10756 ) {
10757 return Ok(());
10758 }
10759 let result = match (handler, data) {
10760 (
10761 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
10762 RouterItem::Packed(bytes),
10763 ) => {
10764 #[cfg(feature = "cryptography")]
10765 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
10766 #[cfg(not(feature = "cryptography"))]
10767 let send_bytes = bytes.clone();
10768 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
10769 let mut attempts_total = 0usize;
10770 let mut sent_bytes = 0usize;
10771 self.send_side_frames_resumable(side, frames, |frame| {
10772 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10773 Self::packed_handler_call(packed, frame, priority)
10774 }) {
10775 Ok((_, attempts)) => {
10776 attempts_total = attempts_total.saturating_add(attempts);
10777 sent_bytes = sent_bytes.saturating_add(frame.len());
10778 }
10779 Err((err, attempts)) => {
10780 self.note_side_tx_failure(
10781 side,
10782 ty,
10783 attempts_total.saturating_add(attempts),
10784 );
10785 return Err(err);
10786 }
10787 }
10788
10789 Ok(())
10790 })?;
10791 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10792 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10793 return Ok(());
10794 }
10795 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
10796 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
10797 }
10798 (
10799 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
10800 RouterItem::Packet(pkt),
10801 ) => {
10802 if ty == DataType::DiscoverySchema
10806 && opts.max_frame_bytes != 0
10807 && pkt.payload().len() > opts.max_frame_bytes
10808 {
10809 let mut attempts_total = 0usize;
10810 let mut sent_bytes = 0usize;
10811 let key = crate::packet::hash_bytes_u64(
10812 pkt.packet_id(),
10813 &opts.max_frame_bytes.to_le_bytes(),
10814 );
10815 let result = self.run_side_chunk_transfer(side, key, |next| {
10816 crate::side_transport::send_packet_chunks(
10817 pkt,
10818 opts.max_frame_bytes,
10819 self.sender_arc().as_bytes(),
10820 next,
10821 |frame| match self
10822 .retry_with_attempts(runtime_max_handler_retries(), || {
10823 Self::packed_handler_call(packed, frame, priority)
10824 }) {
10825 Ok((_, attempts)) => {
10826 attempts_total += attempts;
10827 sent_bytes += frame.len();
10828 Ok(())
10829 }
10830 Err((err, attempts)) => {
10831 attempts_total += attempts;
10832 Err(err)
10833 }
10834 },
10835 )
10836 });
10837 let (raw_len, wire_len, chunks) = match result {
10838 Ok(stats) => stats,
10839 Err(err) => {
10840 self.note_side_tx_failure(side, ty, attempts_total);
10841 return Err(err);
10842 }
10843 };
10844 {
10845 let mut st = self.state.lock();
10846 let stats = st.side_runtime_stats.entry(side).or_default();
10847 stats.note_side_transport_full(raw_len, wire_len);
10848 if chunks > 1 {
10849 stats.note_side_transport_chunks(chunks);
10850 }
10851 }
10852 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10853 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10854 return Ok(());
10855 }
10856 let owned = self.pack_packet_for_router(pkt, None)?;
10857 let frames = self.encode_side_transport_frames(side, opts, owned)?;
10858 let mut attempts_total = 0usize;
10859 let mut sent_bytes = 0usize;
10860 self.send_side_frames_resumable(side, frames, |frame| {
10861 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10862 Self::packed_handler_call(packed, frame, priority)
10863 }) {
10864 Ok((_, attempts)) => {
10865 attempts_total = attempts_total.saturating_add(attempts);
10866 sent_bytes = sent_bytes.saturating_add(frame.len());
10867 }
10868 Err((err, attempts)) => {
10869 self.note_side_tx_failure(
10870 side,
10871 ty,
10872 attempts_total.saturating_add(attempts),
10873 );
10874 return Err(err);
10875 }
10876 }
10877
10878 Ok(())
10879 })?;
10880 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10881 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10882 return Ok(());
10883 }
10884 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
10885 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
10886 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
10887 }
10888 };
10889 match result {
10890 Ok((_, attempts)) => {
10891 if let Ok(bytes) = Self::router_item_wire_len(data) {
10892 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
10893 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
10894 }
10895 Ok(())
10896 }
10897 Err((err, attempts)) => {
10898 self.note_side_tx_failure(side, ty, attempts);
10899 Err(err)
10900 }
10901 }
10902 }
10903
10904 fn adjust_reliable_for_side(
10905 &self,
10906 opts: RouterSideOptions,
10907 data: RouterItem,
10908 preserve_end_to_end_ack: bool,
10909 ) -> TelemetryResult<Option<RouterItem>> {
10910 if opts.reliable_enabled {
10911 return Ok(Some(data));
10912 }
10913
10914 match data {
10915 RouterItem::Packed(bytes) => {
10916 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
10917 if is_reliable_type(frame.envelope.ty)
10918 && let Some(hdr) = frame.reliable
10919 {
10920 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
10921 return Ok(None);
10922 }
10923 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
10924 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
10925 bytes.as_ref(),
10926 wire_format::RELIABLE_FLAG_UNSEQUENCED,
10927 hdr.seq,
10928 0,
10929 )?
10930 else {
10931 return Ok(Some(RouterItem::Packed(bytes)));
10932 };
10933 return Ok(Some(RouterItem::Packed(crate::shared_bytes::convert(
10934 rewritten,
10935 ))));
10936 }
10937 }
10938 Ok(Some(RouterItem::Packed(bytes)))
10939 }
10940 RouterItem::Packet(pkt) => {
10941 if matches!(
10942 pkt.data_type(),
10943 DataType::ReliableAck
10944 | DataType::ReliablePartialAck
10945 | DataType::ReliablePacketRequest
10946 ) {
10947 if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
10948 return Ok(Some(RouterItem::Packet(pkt)));
10949 }
10950 return Ok(None);
10951 }
10952 Ok(Some(RouterItem::Packet(pkt)))
10953 }
10954 }
10955 }
10956
10957 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
10958 let now = self.clock.now_ms();
10959 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
10960
10961 {
10962 let mut st = self.state.lock();
10963 if st.reliable_tx.is_empty() {
10964 return Ok(());
10965 }
10966
10967 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
10968 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
10969 continue;
10970 };
10971 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
10972 for seq in sent_order {
10973 let Some(sent) = tx_state.sent.get_mut(&seq) else {
10974 continue;
10975 };
10976 if sent.queued
10977 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
10978 {
10979 continue;
10980 }
10981 if sent.partial_acked {
10982 sent.partial_acked = false;
10989 sent.last_send_ms = now;
10990 continue;
10991 }
10992 if sent.retries >= runtime_reliable_max_retries() {
10993 tx_state.sent.remove(&seq);
10994 tx_state.sent_order.retain(|existing| *existing != seq);
10995 continue;
10996 }
10997 sent.retries += 1;
10998 requeue.push((*side, ty, seq));
10999 }
11000 }
11001 }
11002
11003 for (side, ty, seq) in requeue {
11004 self.queue_reliable_retransmit(side, ty, seq, true)?;
11005 }
11006
11007 Ok(())
11008 }
11009
11010 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
11011 let now = self.clock.now_ms();
11012 let mut requeue = Vec::new();
11013 let mut failed_ordered_types = BTreeSet::new();
11014
11015 {
11016 let mut st = self.state.lock();
11017 #[cfg(feature = "discovery")]
11018 {
11019 if Self::prune_discovery_routes_locked(&mut st, now) {
11020 Self::note_discovery_topology_change_locked(&mut st, now);
11021 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
11022 }
11023 }
11028 st.end_to_end_reliable_tx.retain(|&packet_id, sent| {
11031 if sent.waiting_for_predecessor
11032 || sent.queued
11033 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
11034 {
11035 return true;
11036 }
11037 if sent.retries >= runtime_reliable_max_retries() {
11038 if let Some(ty) = sent.ordered_type {
11039 failed_ordered_types.insert(ty);
11040 }
11041 return false;
11042 }
11043 sent.retries += 1;
11044 requeue.push(packet_id);
11045 true
11046 });
11047 st.end_to_end_reliable_tx.retain(|_, sent| {
11049 !sent
11050 .ordered_type
11051 .is_some_and(|ty| failed_ordered_types.contains(&ty))
11052 });
11053 }
11054
11055 for packet_id in requeue {
11056 self.queue_end_to_end_reliable_retransmit(packet_id)?;
11057 }
11058 self.release_ordered_successors()?;
11059 if !failed_ordered_types.is_empty() {
11060 return Err(TelemetryError::HandlerError(
11061 "ordered end-to-end delivery exhausted retries",
11062 ));
11063 }
11064 Ok(())
11065 }
11066
11067 #[cfg(feature = "timesync")]
11068 #[inline]
11069 fn monotonic_now_ns(&self) -> u64 {
11070 self.clock.now_ns()
11071 }
11072
11073 #[cfg(feature = "timesync")]
11074 #[inline]
11075 fn monotonic_now_ms(&self) -> u64 {
11076 self.clock.now_ms()
11077 }
11078
11079 #[cfg(feature = "timesync")]
11080 fn refresh_timesync_state(&self, now_mono_ms: u64) {
11081 let now_mono_ns = self.monotonic_now_ns();
11082 let mut st = self.timesync.lock();
11083 st.clock.prune_expired(now_mono_ms);
11084 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
11085 st.remote_sources
11086 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
11087 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
11088 let leader = if let Some(tracker) = st.tracker.as_mut() {
11089 let _ = tracker.refresh(now_mono_ms);
11090 tracker.leader(now_mono_ms, has_usable_time)
11091 } else {
11092 None
11093 };
11094 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
11095 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
11096 let target_ms = st.remote_sources.get(remote.sender.as_str()).map(|src| {
11099 src.sample_unix_ms
11100 .saturating_add(now_mono_ms.saturating_sub(src.last_sample_mono_ms))
11101 });
11102 if let Some(target_ms) = target_ms {
11103 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
11104 }
11105 }
11106 }
11107
11108 #[cfg(feature = "timesync")]
11109 pub fn update_network_time_source(
11111 &self,
11112 source: &str,
11113 priority: u64,
11114 time: PartialNetworkTime,
11115 ttl_ms: Option<u64>,
11116 ) {
11117 let now_ms = self.monotonic_now_ms();
11118 let now_ns = self.monotonic_now_ns();
11119 let mut st = self.timesync.lock();
11120 st.clock
11121 .update_source(source, priority, time, now_ms, now_ns, ttl_ms);
11122 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
11123 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
11124 }
11125 }
11126
11127 #[cfg(feature = "timesync")]
11128 fn set_network_time_source_impl(
11129 &self,
11130 source: &str,
11131 priority: u64,
11132 time: PartialNetworkTime,
11133 ttl_ms: Option<u64>,
11134 ) {
11135 let observed_mono_ms = self.monotonic_now_ms();
11136 let observed_mono_ns = self.monotonic_now_ns();
11137 let mut st = self.timesync.lock();
11138 let commit_mono_ms = self.monotonic_now_ms();
11139 let commit_mono_ns = self.monotonic_now_ns();
11140 let adjusted = if let Some(base) = time.to_network_time() {
11141 let elapsed_ns = commit_mono_ns.saturating_sub(observed_mono_ns);
11142 advance_network_time(base, elapsed_ns)
11143 .map(PartialNetworkTime::from)
11144 .unwrap_or(time)
11145 } else {
11146 time
11147 };
11148 let adjusted_mono_ms =
11149 observed_mono_ms.saturating_add(commit_mono_ms.saturating_sub(observed_mono_ms));
11150 st.clock.update_source(
11151 source,
11152 priority,
11153 adjusted,
11154 commit_mono_ms.max(adjusted_mono_ms),
11155 commit_mono_ns,
11156 ttl_ms,
11157 );
11158 if let Some(unix_ms) = adjusted.to_network_time().and_then(|t| t.as_unix_ms()) {
11159 st.disciplined_clock.steer_unix_ms(commit_mono_ns, unix_ms);
11160 }
11161 }
11162
11163 #[cfg(feature = "timesync")]
11164 fn local_network_time_priority(&self) -> u64 {
11165 let st = self.timesync.lock();
11166 st.cfg.map(|cfg| cfg.priority).unwrap_or(0)
11167 }
11168
11169 #[cfg(feature = "timesync")]
11170 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
11172 let priority = self.local_network_time_priority();
11173 if time.is_complete_date() && time.is_complete_time() {
11174 self.set_network_time_source_impl(LOCAL_TIMESYNC_FULL_SOURCE_ID, priority, time, None);
11175 let mut st = self.timesync.lock();
11176 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
11177 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
11178 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
11179 return;
11180 }
11181
11182 {
11183 let mut st = self.timesync.lock();
11184 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
11185 }
11186
11187 if time.year.is_some() || time.month.is_some() || time.day.is_some() {
11188 self.set_network_time_source_impl(
11189 LOCAL_TIMESYNC_DATE_SOURCE_ID,
11190 priority,
11191 PartialNetworkTime {
11192 year: time.year,
11193 month: time.month,
11194 day: time.day,
11195 ..Default::default()
11196 },
11197 None,
11198 );
11199 }
11200
11201 if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
11202 self.set_network_time_source_impl(
11203 LOCAL_TIMESYNC_TOD_SOURCE_ID,
11204 priority,
11205 PartialNetworkTime {
11206 hour: time.hour,
11207 minute: time.minute,
11208 second: time.second,
11209 nanosecond: time.nanosecond,
11210 ..Default::default()
11211 },
11212 None,
11213 );
11214 }
11215
11216 if time.nanosecond.is_some() {
11217 self.set_network_time_source_impl(
11218 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
11219 priority,
11220 PartialNetworkTime {
11221 nanosecond: time.nanosecond,
11222 ..Default::default()
11223 },
11224 None,
11225 );
11226 }
11227 }
11228
11229 #[cfg(feature = "timesync")]
11230 pub fn clear_local_network_time(&self) {
11232 let mut st = self.timesync.lock();
11233 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
11234 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
11235 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
11236 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
11237 }
11238
11239 #[cfg(feature = "timesync")]
11240 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
11242 self.set_local_network_time(PartialNetworkTime {
11243 year: Some(year),
11244 month: Some(month),
11245 day: Some(day),
11246 ..Default::default()
11247 });
11248 }
11249
11250 #[cfg(feature = "timesync")]
11251 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
11253 self.set_local_network_time(PartialNetworkTime {
11254 hour: Some(hour),
11255 minute: Some(minute),
11256 ..Default::default()
11257 });
11258 }
11259
11260 #[cfg(feature = "timesync")]
11261 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
11263 self.set_local_network_time(PartialNetworkTime {
11264 hour: Some(hour),
11265 minute: Some(minute),
11266 second: Some(second),
11267 ..Default::default()
11268 });
11269 }
11270
11271 #[cfg(feature = "timesync")]
11272 pub fn set_local_network_time_hms_millis(
11274 &self,
11275 hour: u8,
11276 minute: u8,
11277 second: u8,
11278 millisecond: u16,
11279 ) {
11280 self.set_local_network_time(PartialNetworkTime {
11281 hour: Some(hour),
11282 minute: Some(minute),
11283 second: Some(second),
11284 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
11285 ..Default::default()
11286 });
11287 }
11288
11289 #[cfg(feature = "timesync")]
11290 pub fn set_local_network_time_hms_nanos(
11292 &self,
11293 hour: u8,
11294 minute: u8,
11295 second: u8,
11296 nanosecond: u32,
11297 ) {
11298 self.set_local_network_time(PartialNetworkTime {
11299 hour: Some(hour),
11300 minute: Some(minute),
11301 second: Some(second),
11302 nanosecond: Some(nanosecond),
11303 ..Default::default()
11304 });
11305 }
11306
11307 #[cfg(feature = "timesync")]
11308 pub fn set_local_network_datetime(
11310 &self,
11311 year: i32,
11312 month: u8,
11313 day: u8,
11314 hour: u8,
11315 minute: u8,
11316 second: u8,
11317 ) {
11318 self.set_local_network_time(PartialNetworkTime {
11319 year: Some(year),
11320 month: Some(month),
11321 day: Some(day),
11322 hour: Some(hour),
11323 minute: Some(minute),
11324 second: Some(second),
11325 ..Default::default()
11326 });
11327 }
11328
11329 #[cfg(feature = "timesync")]
11330 #[allow(clippy::too_many_arguments)]
11331 pub fn set_local_network_datetime_millis(
11333 &self,
11334 year: i32,
11335 month: u8,
11336 day: u8,
11337 hour: u8,
11338 minute: u8,
11339 second: u8,
11340 millisecond: u16,
11341 ) {
11342 self.set_local_network_time(PartialNetworkTime {
11343 year: Some(year),
11344 month: Some(month),
11345 day: Some(day),
11346 hour: Some(hour),
11347 minute: Some(minute),
11348 second: Some(second),
11349 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
11350 });
11351 }
11352
11353 #[cfg(feature = "timesync")]
11354 #[allow(clippy::too_many_arguments)]
11355 pub fn set_local_network_datetime_nanos(
11357 &self,
11358 year: i32,
11359 month: u8,
11360 day: u8,
11361 hour: u8,
11362 minute: u8,
11363 second: u8,
11364 nanosecond: u32,
11365 ) {
11366 self.set_local_network_time(PartialNetworkTime {
11367 year: Some(year),
11368 month: Some(month),
11369 day: Some(day),
11370 hour: Some(hour),
11371 minute: Some(minute),
11372 second: Some(second),
11373 nanosecond: Some(nanosecond),
11374 });
11375 }
11376
11377 #[cfg(feature = "timesync")]
11378 pub fn clear_network_time_source(&self, source: &str) {
11380 let mut st = self.timesync.lock();
11381 st.clock.remove_source(source);
11382 }
11383
11384 #[cfg(feature = "timesync")]
11385 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
11387 let mut st = self.timesync.lock();
11388 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
11389 st.cfg = cfg;
11390 st.tracker = cfg.map(TimeSyncTracker::new);
11391 st.disciplined_clock = SlewedNetworkClock::new(
11392 cfg.map(|c| c.max_slew_ppm)
11393 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
11394 );
11395 st.remote_sources.clear();
11396 st.next_seq = 1;
11397 st.next_announce_mono_ms = 0;
11398 st.next_request_mono_ms = 0;
11399 st.pending_request = None;
11400 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
11401 for source in stale_remote_sources {
11402 st.clock.remove_source(&source);
11403 }
11404 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
11405 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
11406 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
11407 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
11408 }
11409
11410 #[cfg(feature = "timesync")]
11411 pub fn network_time(&self) -> Option<NetworkTimeReading> {
11413 let now_ms = self.monotonic_now_ms();
11414 let now_ns = self.monotonic_now_ns();
11415 self.refresh_timesync_state(now_ms);
11416 let st = self.timesync.lock();
11417 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
11418 return Some(NetworkTimeReading {
11419 time: PartialNetworkTime::from_unix_ms(unix_ms),
11420 unix_time_ms: Some(unix_ms),
11421 });
11422 }
11423 st.clock.current_time(now_ns)
11424 }
11425
11426 #[cfg(feature = "timesync")]
11427 pub fn network_time_ms(&self) -> Option<u64> {
11429 self.network_time().and_then(|t| t.unix_time_ms)
11430 }
11431
11432 #[cfg(feature = "timesync")]
11433 fn packet_timestamp_ms(&self) -> u64 {
11434 self.network_time_ms()
11435 .unwrap_or_else(|| self.monotonic_now_ms())
11436 }
11437
11438 #[cfg(not(feature = "timesync"))]
11439 fn packet_timestamp_ms(&self) -> u64 {
11440 self.clock.now_ms()
11441 }
11442
11443 #[cfg(feature = "timesync")]
11444 fn queue_internal_timesync_request(
11445 &self,
11446 seq: u64,
11447 t1_mono_ms: u64,
11448 called_from_queue: bool,
11449 ) -> TelemetryResult<()> {
11450 let pkt_ts = self.packet_timestamp_ms();
11451 if called_from_queue {
11452 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
11453 } else {
11454 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
11455 }
11456 }
11457
11458 #[cfg(feature = "timesync")]
11459 fn queue_internal_timesync_response(
11460 &self,
11461 seq: u64,
11462 t1_mono_ms: u64,
11463 t2_network_ms: u64,
11464 t3_network_ms: u64,
11465 dst: Option<RouterSideId>,
11466 called_from_queue: bool,
11467 ) -> TelemetryResult<()> {
11468 let pkt_ts = self.packet_timestamp_ms();
11469 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
11470 let sender = self.sender_arc();
11471 let pkt = Packet::new(
11472 DataType::TimeSyncResponse,
11473 &[DataEndpoint::TimeSync],
11474 sender.as_ref(),
11475 pkt_ts,
11476 payload,
11477 )?;
11478 match dst {
11479 Some(dst) => self.emit_internal_tx(
11480 RouterTxItem::ToSide {
11481 src: None,
11482 dst,
11483 data: RouterItem::Packet(pkt),
11484 },
11485 true,
11486 called_from_queue,
11487 ),
11488 None => self.emit_internal_tx(
11489 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
11490 true,
11491 called_from_queue,
11492 ),
11493 }
11494 }
11495
11496 #[cfg(feature = "timesync")]
11497 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
11499 let now_ms = self.monotonic_now_ms();
11500 let now_ns = self.monotonic_now_ns();
11501 let mut queued_any = false;
11502 let mut announce_priority = None;
11503 let mut request = None;
11504
11505 {
11506 let mut st = self.timesync.lock();
11507 st.clock.prune_expired(now_ms);
11508 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
11509 st.remote_sources
11510 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
11511 let Some(cfg) = st.cfg else {
11512 return Ok(false);
11513 };
11514
11515 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
11516 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
11517 let _ = tracker.refresh(now_ms);
11518 (
11519 tracker.leader(now_ms, has_usable_time),
11520 tracker.local_announce_priority(now_ms, has_usable_time),
11521 )
11522 } else {
11523 (None, None)
11524 };
11525 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
11526
11527 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
11528 let target_ms = st.remote_sources.get(&remote.sender).map(|src| {
11529 src.sample_unix_ms
11530 .saturating_add(now_ms.saturating_sub(src.last_sample_mono_ms))
11531 });
11532 if let Some(target_ms) = target_ms {
11533 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
11534 }
11535 }
11536
11537 if let Some(priority) = announce_prio
11538 && now_ms >= st.next_announce_mono_ms
11539 {
11540 announce_priority = Some(priority);
11541 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
11542 }
11543
11544 if let Some(TimeSyncLeader::Remote(remote)) = leader
11545 && now_ms >= st.next_request_mono_ms
11546 && st.pending_request.is_none()
11547 {
11548 let seq = st.next_seq;
11549 let next = st.next_seq.wrapping_add(1);
11550 st.next_seq = if next == 0 { 1 } else { next };
11551 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
11552 st.pending_request = Some(PendingTimeSyncRequest {
11553 seq,
11554 t1_mono_ms: now_ms,
11555 source: remote.sender,
11556 });
11557 request = Some((seq, now_ms));
11558 }
11559 }
11560
11561 if let Some(priority) = announce_priority {
11562 let time_ms = self.packet_timestamp_ms();
11563 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
11564 queued_any = true;
11565 }
11566 if let Some((seq, t1_mono_ms)) = request {
11567 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
11568 queued_any = true;
11569 }
11570
11571 Ok(queued_any)
11572 }
11573
11574 #[cfg(feature = "timesync")]
11575 fn handle_internal_timesync_packet(
11576 &self,
11577 pkt: &Packet,
11578 src: Option<RouterSideId>,
11579 called_from_queue: bool,
11580 ) -> TelemetryResult<bool> {
11581 let Some(cfg) = self.cfg.timesync_config() else {
11582 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
11583 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
11584 }
11585 return Ok(true);
11586 };
11587
11588 let now_mono_ms = self.monotonic_now_ms();
11589 let now_mono_ns = self.monotonic_now_ns();
11590 let mut response = None;
11591 let mut poll_after = false;
11592
11593 {
11594 let mut st = self.timesync.lock();
11595 st.clock.prune_expired(now_mono_ms);
11596 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
11597 st.remote_sources
11598 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
11599 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
11600 if st.tracker.is_none() {
11601 return Ok(true);
11602 }
11603
11604 match pkt.data_type() {
11605 DataType::TimeSyncAnnounce => {
11606 let ann = decode_timesync_announce(pkt)?;
11607 let should_steer = {
11608 let tracker = st.tracker.as_mut().expect("tracker checked above");
11609 let _ = tracker.handle_announce(pkt, now_mono_ms)?;
11610 matches!(
11611 tracker.leader(now_mono_ms, has_usable_time),
11612 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender()
11613 )
11614 };
11615 st.remote_sources.insert(
11616 pkt.sender().to_owned(),
11617 RemoteTimeSyncSource {
11618 priority: ann.priority,
11619 last_sample_mono_ms: now_mono_ms,
11620 sample_unix_ms: ann.time_ms,
11621 },
11622 );
11623 st.clock.update_source(
11624 pkt.sender(),
11625 ann.priority,
11626 PartialNetworkTime::from_unix_ms(ann.time_ms),
11627 now_mono_ms,
11628 now_mono_ns,
11629 Some(cfg.source_timeout_ms),
11630 );
11631 if should_steer {
11632 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
11633 }
11634 poll_after = true;
11635 }
11636 DataType::TimeSyncRequest => {
11637 let should_serve = {
11638 let tracker = st.tracker.as_ref().expect("tracker checked above");
11639 tracker.should_serve(now_mono_ms, has_usable_time)
11640 };
11641 if should_serve {
11642 let req = decode_timesync_request(pkt)?;
11643 let network_now = st
11644 .disciplined_clock
11645 .read_unix_ms(now_mono_ns)
11646 .or_else(|| {
11647 st.clock
11648 .current_time(now_mono_ns)
11649 .and_then(|t| t.unix_time_ms)
11650 })
11651 .unwrap_or(now_mono_ms);
11652 let t2 = network_now;
11653 let t3 = network_now;
11654 response = Some((req.seq, req.t1_ms, t2, t3, src));
11655 }
11656 }
11657 DataType::TimeSyncResponse => {
11658 let resp = decode_timesync_response(pkt)?;
11659 let pending = st.pending_request.clone();
11660 if let Some(pending) = pending
11661 && pending.seq == resp.seq
11662 && pending.source == pkt.sender()
11663 {
11664 let source_priority = {
11665 let tracker = st.tracker.as_ref().expect("tracker checked above");
11666 tracker
11667 .best_active_source(now_mono_ms)
11668 .map(|s| s.priority)
11669 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
11670 .unwrap_or(cfg.priority)
11671 };
11672 let (estimate_ms, _delay_ms) = compute_network_time_sample(
11673 pending.t1_mono_ms,
11674 resp.t2_ms,
11675 resp.t3_ms,
11676 now_mono_ms,
11677 );
11678 st.remote_sources.insert(
11679 pkt.sender().to_owned(),
11680 RemoteTimeSyncSource {
11681 priority: source_priority,
11682 last_sample_mono_ms: now_mono_ms,
11683 sample_unix_ms: estimate_ms,
11684 },
11685 );
11686 st.clock.update_source(
11687 pkt.sender(),
11688 source_priority,
11689 PartialNetworkTime::from_unix_ms(estimate_ms),
11690 now_mono_ms,
11691 now_mono_ns,
11692 Some(cfg.source_timeout_ms),
11693 );
11694 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
11695 st.pending_request = None;
11696 }
11697 }
11698 _ => {}
11699 }
11700 }
11701
11702 if let Some((seq, t1, t2, t3, dst)) = response {
11703 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
11704 }
11705 if poll_after {
11706 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
11707 }
11708
11709 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
11710 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
11711 }
11712
11713 Ok(true)
11714 }
11715
11716 #[cfg(feature = "std")]
11718 pub fn new(cfg: RouterConfig) -> Self {
11719 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
11720 }
11721
11722 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
11724 #[cfg(feature = "timesync")]
11725 let timesync_cfg = cfg.timesync_config();
11726 #[cfg(feature = "discovery")]
11727 let preferred_discovery_master = cfg.preferred_discovery_master.clone();
11728 let memory = cfg.memory_config();
11729 let hostname: Arc<str> = Arc::from(cfg.sender());
11730 let address_mode = cfg.address_mode();
11731 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
11732 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
11733 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
11734 let requested = address_mode.requested_address();
11735 let address = if requested == 0 { fallback } else { requested };
11736 let local_address = AddressBookEntry {
11737 hostname: hostname.clone(),
11738 address,
11739 requested_address: requested,
11740 mode: address_mode,
11741 birth_ms: clock
11742 .now_ms()
11743 .saturating_mul(1_000_000)
11744 .saturating_add(instance_seq),
11745 owner_hash,
11746 last_seen_ms: clock.now_ms(),
11747 };
11748 let mut address_book = BTreeMap::new();
11749 address_book.insert(hostname.to_string(), local_address.clone());
11750 let mut address_by_value = BTreeMap::new();
11751 address_by_value.insert(address, hostname.to_string());
11752 Self {
11753 sender: RouterMutex::new(hostname),
11754 cfg,
11755 state: RouterMutex::new(RouterInner {
11756 #[cfg(feature = "compression")]
11757 codec_reserved_bytes: 0,
11758 memory,
11759 sides: Vec::new(),
11760 route_overrides: Default::default(),
11761 typed_route_overrides: Default::default(),
11762 route_weights: Default::default(),
11763 route_priorities: Default::default(),
11764 source_route_modes: Default::default(),
11765 route_selection_cursors: Default::default(),
11766 adaptive_route_stats: Default::default(),
11767 side_runtime_stats: Default::default(),
11768 side_transport: Default::default(),
11769 managed_variable_types: BTreeSet::new(),
11770 managed_variable_permissions: Default::default(),
11771 managed_variable_latest: Default::default(),
11772 network_variable_update_handlers: Default::default(),
11773 local_address,
11774 address_book,
11775 address_by_value,
11776 p2p_port_handlers: Default::default(),
11777 p2p_stream_handlers: Default::default(),
11778 p2p_stream_sessions: Default::default(),
11779 next_p2p_stream_id: 1,
11780 received_queue: BoundedDeque::new(
11781 memory.max_queue_budget,
11782 memory.starting_queue_size,
11783 memory.queue_grow_step,
11784 ),
11785 transmit_queue: BoundedDeque::new(
11786 memory.max_queue_budget,
11787 memory.starting_queue_size,
11788 memory.queue_grow_step,
11789 ),
11790 tx_priority_burst: 0,
11791 has_ordered_retry: false,
11792 ordered_claim_seq: 0,
11793 rx_priority_burst: 0,
11794 recent_rx: BoundedDeque::new(
11795 memory.recent_rx_queue_bytes(),
11796 memory.recent_rx_queue_bytes(),
11797 memory.queue_grow_step,
11798 ),
11799 reliable_tx: Default::default(),
11800 reliable_rx: Default::default(),
11801 reliable_return_routes: Default::default(),
11802 reliable_return_route_order: VecDeque::new(),
11803 end_to_end_reliable_tx: Default::default(),
11804 end_to_end_reliable_tx_order: VecDeque::new(),
11805 total_handler_failures: 0,
11806 total_handler_retries: 0,
11807 #[cfg(feature = "discovery")]
11808 discovery_routes: Default::default(),
11809 #[cfg(feature = "discovery")]
11810 discovery_cadence: DiscoveryCadenceState::default(),
11811 #[cfg(feature = "discovery")]
11812 discovery_side_throttle: Default::default(),
11813 #[cfg(feature = "discovery")]
11814 preferred_discovery_master,
11815 #[cfg(all(feature = "discovery", feature = "timesync"))]
11816 timesync_side_throttle: Default::default(),
11817 }),
11818 isr_rx_queue: IsrRxQueue::new(
11819 memory.max_queue_budget,
11820 memory.starting_queue_size,
11821 memory.queue_grow_step,
11822 ),
11823 #[cfg(not(feature = "std"))]
11824 side_tx_gate: ReentryGate::new(),
11825 #[cfg(feature = "std")]
11826 side_tx_activity: crate::lock::SideTxActivity::default(),
11827 #[cfg(feature = "std")]
11828 active_queued_tx: core::sync::atomic::AtomicUsize::new(0),
11829 clock,
11830 #[cfg(feature = "timesync")]
11831 timesync: RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)),
11832 }
11833 }
11834
11835 #[inline]
11836 fn sender_arc(&self) -> Arc<str> {
11837 self.sender.lock().clone()
11838 }
11839
11840 #[inline]
11841 pub fn sender(&self) -> Arc<str> {
11842 self.sender_arc()
11843 }
11844
11845 pub fn current_address(&self) -> NodeAddress {
11846 self.state.lock().local_address.address
11847 }
11848
11849 pub fn hostname(&self) -> Arc<str> {
11850 self.sender_arc()
11851 }
11852
11853 pub fn set_preferred_discovery_master(&self, hostname: Option<&str>) {
11858 #[cfg(feature = "discovery")]
11859 {
11860 let preferred = hostname
11861 .filter(|value| !value.is_empty())
11862 .map(Arc::<str>::from);
11863 let mut st = self.state.lock();
11864 if st.preferred_discovery_master != preferred {
11865 st.preferred_discovery_master = preferred;
11866 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11867 }
11868 }
11869 #[cfg(not(feature = "discovery"))]
11870 let _ = hostname;
11871 }
11872
11873 pub fn preferred_discovery_master(&self) -> Option<Arc<str>> {
11875 #[cfg(feature = "discovery")]
11876 {
11877 return self.state.lock().preferred_discovery_master.clone();
11878 }
11879 #[cfg(not(feature = "discovery"))]
11880 None
11881 }
11882
11883 pub fn address_book(&self) -> Vec<AddressBookEntry> {
11884 self.state.lock().address_book.values().cloned().collect()
11885 }
11886
11887 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
11888 let now_ms = self.clock.now_ms();
11889 #[allow(unused_mut)]
11890 let mut st = self.state.lock();
11891 #[cfg(feature = "discovery")]
11892 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
11893 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11894 }
11895 if let Some(entry) = st.address_book.get(hostname) {
11896 return Some(entry.clone());
11897 }
11898 let address = sender_address_u32(hostname);
11904 if Self::hostname_for_address_locked(&st, address).as_deref() != Some(hostname) {
11905 return None;
11906 }
11907 Some(AddressBookEntry {
11908 hostname: Arc::from(hostname),
11909 address,
11910 requested_address: 0,
11911 mode: AddressAssignmentMode::Dynamic,
11912 birth_ms: 0,
11913 owner_hash: Self::sender_hash(hostname),
11914 last_seen_ms: now_ms,
11915 })
11916 }
11917
11918 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
11919 let st = self.state.lock();
11920 let direct = st
11921 .address_by_value
11922 .get(&address)
11923 .and_then(|hostname| st.address_book.get(hostname))
11924 .cloned();
11925 if direct.is_some() {
11926 return direct;
11927 }
11928
11929 let hostname = Self::hostname_for_address_locked(&st, address)?;
11930 if let Some(entry) = st.address_book.get(&hostname) {
11931 return Some(entry.clone());
11932 }
11933
11934 let hostname: Arc<str> = Arc::from(hostname);
11939 Some(AddressBookEntry {
11940 hostname: hostname.clone(),
11941 address,
11942 requested_address: 0,
11943 mode: AddressAssignmentMode::Dynamic,
11944 birth_ms: 0,
11945 owner_hash: Self::sender_hash(hostname.as_ref()),
11946 last_seen_ms: self.clock.now_ms(),
11947 })
11948 }
11949
11950 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
11951 where
11952 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
11953 {
11954 if port == 0 {
11955 return Err(TelemetryError::BadArg);
11956 }
11957 let mut st = self.state.lock();
11958 st.p2p_port_handlers
11959 .entry(port)
11960 .or_default()
11961 .push(P2pPortHandler {
11962 handler: Arc::new(f),
11963 });
11964 Ok(())
11965 }
11966
11967 pub fn clear_p2p_port(&self, port: P2pPort) {
11968 self.state.lock().p2p_port_handlers.remove(&port);
11969 }
11970
11971 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
11972 where
11973 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
11974 {
11975 if port == 0 {
11976 return Err(TelemetryError::BadArg);
11977 }
11978 let mut st = self.state.lock();
11979 st.p2p_stream_handlers
11980 .entry(port)
11981 .or_default()
11982 .push(P2pStreamHandler {
11983 handler: Arc::new(f),
11984 });
11985 Ok(())
11986 }
11987
11988 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
11989 self.state.lock().p2p_stream_handlers.remove(&port);
11990 }
11991
11992 fn send_p2p_to_entry(
11993 &self,
11994 dst: AddressBookEntry,
11995 dst_port: P2pPort,
11996 src_port: P2pPort,
11997 payload: &[u8],
11998 ) -> TelemetryResult<()> {
11999 if dst_port == 0 {
12000 return Err(TelemetryError::BadArg);
12001 }
12002 let local = self.state.lock().local_address.clone();
12003 let payload = Self::encode_p2p_payload(
12004 local.hostname.as_ref(),
12005 local.address,
12006 src_port,
12007 dst_port,
12008 payload,
12009 )?;
12010 let pkt = Packet::new(
12011 DataType::P2pMessage,
12012 &[DataEndpoint::Discovery],
12013 local.hostname.as_ref(),
12014 self.packet_timestamp_ms(),
12015 crate::shared_bytes::convert(payload),
12016 )?;
12017 let target = Self::sender_hash(dst.hostname.as_ref());
12018 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
12019 if target == Self::sender_hash(self.sender_arc().as_ref()) {
12020 self.dispatch_p2p_packet(&pkt)?;
12021 return Ok(());
12022 }
12023 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
12024 }
12025
12026 pub fn send_p2p_to_hostname(
12027 &self,
12028 hostname: &str,
12029 dst_port: P2pPort,
12030 src_port: P2pPort,
12031 payload: &[u8],
12032 ) -> TelemetryResult<()> {
12033 let dst = self
12034 .resolve_hostname(hostname)
12035 .ok_or(TelemetryError::BadArg)?;
12036 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
12037 }
12038
12039 pub fn send_p2p_to_address(
12040 &self,
12041 address: NodeAddress,
12042 dst_port: P2pPort,
12043 src_port: P2pPort,
12044 payload: &[u8],
12045 ) -> TelemetryResult<()> {
12046 let dst = self
12047 .resolve_address(address)
12048 .ok_or(TelemetryError::BadArg)?;
12049 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
12050 }
12051
12052 fn open_p2p_stream_to_entry(
12053 &self,
12054 dst: AddressBookEntry,
12055 dst_port: P2pPort,
12056 src_port: P2pPort,
12057 ) -> TelemetryResult<P2pStreamId> {
12058 if dst_port == 0 || src_port == 0 {
12059 return Err(TelemetryError::BadArg);
12060 }
12061 let stream_id = {
12062 let mut st = self.state.lock();
12063 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
12064 if stream_id == 0 {
12065 return Err(TelemetryError::Io("p2p stream id exhausted"));
12066 }
12067 st.p2p_stream_sessions.insert(
12068 stream_id,
12069 P2pStreamSession {
12070 peer_hostname: dst.hostname.clone(),
12071 peer_address: dst.address,
12072 local_port: src_port,
12073 peer_port: dst_port,
12074 peer_stream_id: 0,
12075 next_sequence: 1,
12076 connected: false,
12077 },
12078 );
12079 stream_id
12080 };
12081 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
12082 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
12083 Ok(stream_id)
12084 }
12085
12086 pub fn open_p2p_stream_to_hostname(
12087 &self,
12088 hostname: &str,
12089 dst_port: P2pPort,
12090 src_port: P2pPort,
12091 ) -> TelemetryResult<P2pStreamId> {
12092 let dst = self
12093 .resolve_hostname(hostname)
12094 .ok_or(TelemetryError::BadArg)?;
12095 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
12096 }
12097
12098 pub fn open_p2p_stream_to_address(
12099 &self,
12100 address: NodeAddress,
12101 dst_port: P2pPort,
12102 src_port: P2pPort,
12103 ) -> TelemetryResult<P2pStreamId> {
12104 let dst = self
12105 .resolve_address(address)
12106 .ok_or(TelemetryError::BadArg)?;
12107 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
12108 }
12109
12110 fn send_p2p_stream_control(
12111 &self,
12112 stream_id: P2pStreamId,
12113 flags: u8,
12114 payload: &[u8],
12115 ) -> TelemetryResult<()> {
12116 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
12117 let mut st = self.state.lock();
12118 let Some(session) = st.p2p_stream_sessions.get_mut(&stream_id) else {
12119 return Err(TelemetryError::BadArg);
12120 };
12121 let peer_hostname = session.peer_hostname.clone();
12122 let peer_address = session.peer_address;
12123 let peer_port = session.peer_port;
12124 let local_port = session.local_port;
12125 let peer_stream_id = session.peer_stream_id;
12126 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
12127 let seq = session.next_sequence;
12128 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
12129 let dst = st
12130 .address_book
12131 .get(peer_hostname.as_ref())
12132 .cloned()
12133 .unwrap_or(AddressBookEntry {
12134 hostname: peer_hostname.clone(),
12135 address: peer_address,
12136 requested_address: peer_address,
12137 mode: AddressAssignmentMode::Dynamic,
12138 birth_ms: self.clock.now_ms(),
12139 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
12140 last_seen_ms: self.clock.now_ms(),
12141 });
12142 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
12143 };
12144 if refresh_syn {
12145 let syn_payload =
12146 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
12147 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
12148 }
12149 let stream_payload =
12150 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
12151 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
12152 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
12153 self.state.lock().p2p_stream_sessions.remove(&stream_id);
12154 }
12155 Ok(())
12156 }
12157
12158 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
12159 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
12160 }
12161
12162 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
12163 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
12164 }
12165
12166 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
12167 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
12168 }
12169
12170 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
12171 let hostname: Arc<str> = Arc::from(sender.as_ref());
12172 let mut st = self.state.lock();
12173 let mut local = st.local_address.clone();
12174 local.hostname = hostname.clone();
12175 local.owner_hash = Self::sender_hash(hostname.as_ref());
12176 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
12177 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
12178 local.requested_address = 0;
12179 }
12180 let change =
12181 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
12182 drop(st);
12183 let _ = self.notify_address_change(change);
12184 }
12185
12186 pub fn set_address_assignment(
12187 &self,
12188 mode: AddressAssignmentMode,
12189 ) -> TelemetryResult<AddressChange> {
12190 let mut st = self.state.lock();
12191 let mut local = st.local_address.clone();
12192 local.mode = mode;
12193 local.requested_address = mode.requested_address();
12194 local.address = match mode {
12195 AddressAssignmentMode::Dynamic => {
12196 Self::fallback_address_for_hostname(local.hostname.as_ref())
12197 }
12198 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
12199 if address == 0 {
12200 1
12201 } else {
12202 address
12203 }
12204 }
12205 };
12206 let change =
12207 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
12208 drop(st);
12209 self.notify_address_change(change.clone())?;
12210 Ok(change)
12211 }
12212
12213 pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
12222 where
12223 N: AsRef<str>,
12224 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
12225 {
12226 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
12227 }
12228
12229 pub fn add_side_packed_small_packets<N, F>(
12233 &self,
12234 name: N,
12235 tx: F,
12236 max_frame_bytes: usize,
12237 ) -> RouterSideId
12238 where
12239 N: AsRef<str>,
12240 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
12241 {
12242 self.add_side_packed_with_options(
12243 name,
12244 tx,
12245 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
12246 )
12247 }
12248
12249 pub fn add_side_packed_with_options<N, F>(
12258 &self,
12259 name: N,
12260 tx: F,
12261 opts: RouterSideOptions,
12262 ) -> RouterSideId
12263 where
12264 N: AsRef<str>,
12265 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
12266 {
12267 let mut st = self.state.lock();
12268 let side = Some(RouterSide {
12269 name: Arc::from(name.as_ref()),
12270 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
12271 opts,
12272 });
12273 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
12274 st.sides[id] = side;
12275 id
12276 } else {
12277 let id = st.sides.len();
12278 st.sides.push(side);
12279 id
12280 };
12281 st.side_runtime_stats
12282 .insert(id, Box::<SideRuntimeStatsInner>::default());
12283 st.side_transport
12284 .insert(id, Box::<SideTransportState>::default());
12285 #[cfg(feature = "discovery")]
12286 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12287 id
12288 }
12289
12290 pub fn add_side_packed_with_priority_and_options<N, F>(
12299 &self,
12300 name: N,
12301 tx: F,
12302 opts: RouterSideOptions,
12303 ) -> RouterSideId
12304 where
12305 N: AsRef<str>,
12306 F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
12307 {
12308 let mut st = self.state.lock();
12309 let side = Some(RouterSide {
12310 name: Arc::from(name.as_ref()),
12311 tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
12312 opts,
12313 });
12314 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
12315 st.sides[id] = side;
12316 id
12317 } else {
12318 let id = st.sides.len();
12319 st.sides.push(side);
12320 id
12321 };
12322 st.side_runtime_stats
12323 .insert(id, Box::<SideRuntimeStatsInner>::default());
12324 st.side_transport
12325 .insert(id, Box::<SideTransportState>::default());
12326 #[cfg(feature = "discovery")]
12327 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12328 id
12329 }
12330
12331 pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
12336 where
12337 N: AsRef<str>,
12338 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
12339 {
12340 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
12341 }
12342
12343 pub fn add_side_packet_with_options<N, F>(
12349 &self,
12350 name: N,
12351 tx: F,
12352 opts: RouterSideOptions,
12353 ) -> RouterSideId
12354 where
12355 N: AsRef<str>,
12356 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
12357 {
12358 let mut st = self.state.lock();
12359 let side = Some(RouterSide {
12360 name: Arc::from(name.as_ref()),
12361 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
12362 opts,
12363 });
12364 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
12365 st.sides[id] = side;
12366 id
12367 } else {
12368 let id = st.sides.len();
12369 st.sides.push(side);
12370 id
12371 };
12372 st.side_runtime_stats
12373 .insert(id, Box::<SideRuntimeStatsInner>::default());
12374 st.side_transport
12375 .insert(id, Box::<SideTransportState>::default());
12376 #[cfg(feature = "discovery")]
12377 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12378 id
12379 }
12380
12381 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
12386 #[cfg(feature = "discovery")]
12387 let now_ms = self.clock.now_ms();
12388 {
12389 let mut st = self.state.lock();
12390 #[cfg(feature = "std")]
12391 if st
12392 .side_transport
12393 .get(&side)
12394 .is_some_and(|state| state.tx_callback_gate.is_active())
12395 {
12396 return Err(TelemetryError::Io("side tx busy"));
12397 }
12398 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
12399 if slot.is_none() {
12400 return Err(TelemetryError::BadArg);
12401 }
12402 *slot = None;
12403 while st.sides.last().is_some_and(Option::is_none) {
12404 st.sides.pop();
12405 }
12406 st.route_overrides
12411 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
12412 st.typed_route_overrides
12413 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
12414 st.route_weights
12415 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
12416 st.route_priorities
12417 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
12418 st.source_route_modes.remove(&Some(side));
12419 st.route_selection_cursors.remove(&Some(side));
12420 st.adaptive_route_stats.remove(&side);
12421 #[cfg(feature = "discovery")]
12422 st.discovery_side_throttle.remove(&side);
12423 #[cfg(all(feature = "discovery", feature = "timesync"))]
12424 st.timesync_side_throttle.remove(&side);
12425 st.side_runtime_stats.remove(&side);
12426 st.side_transport.remove(&side);
12427 st.reliable_return_routes
12428 .retain(|_, route| route.side != side);
12429 st.retain_transmit(
12430 |queued| {
12431 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
12432 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
12433 },
12434 );
12435 st.received_queue.retain(|queued| queued.src != Some(side));
12436 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
12437 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
12438 #[cfg(feature = "discovery")]
12439 {
12440 st.discovery_routes.remove(&side);
12441 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12442 }
12443 }
12444 let mut isr_rx = self.isr_rx_queue.try_lock()?;
12445 isr_rx.retain(|queued| queued.src != Some(side));
12446 Ok(())
12447 }
12448
12449 pub fn set_side_ingress_enabled(
12454 &self,
12455 side: RouterSideId,
12456 enabled: bool,
12457 ) -> TelemetryResult<()> {
12458 #[cfg(feature = "discovery")]
12459 let now_ms = self.clock.now_ms();
12460 let mut st = self.state.lock();
12461 let side_ref = st
12462 .sides
12463 .get_mut(side)
12464 .and_then(|side| side.as_mut())
12465 .ok_or(TelemetryError::BadArg)?;
12466 side_ref.opts.ingress_enabled = enabled;
12467 #[cfg(feature = "discovery")]
12468 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12469 Ok(())
12470 }
12471
12472 pub fn set_side_egress_enabled(
12476 &self,
12477 side: RouterSideId,
12478 enabled: bool,
12479 ) -> TelemetryResult<()> {
12480 #[cfg(feature = "discovery")]
12481 let now_ms = self.clock.now_ms();
12482 let mut st = self.state.lock();
12483 let side_ref = st
12484 .sides
12485 .get_mut(side)
12486 .and_then(|side| side.as_mut())
12487 .ok_or(TelemetryError::BadArg)?;
12488 side_ref.opts.egress_enabled = enabled;
12489 if !enabled {
12490 st.retain_transmit(
12491 |queued| {
12492 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
12493 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
12494 },
12495 );
12496 }
12497 #[cfg(feature = "discovery")]
12498 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12499 Ok(())
12500 }
12501
12502 pub fn set_source_route_mode(
12508 &self,
12509 src: Option<RouterSideId>,
12510 mode: RouteSelectionMode,
12511 ) -> TelemetryResult<()> {
12512 #[cfg(feature = "discovery")]
12513 let now_ms = self.clock.now_ms();
12514 let mut st = self.state.lock();
12515 if let Some(src) = src {
12516 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12517 }
12518 st.source_route_modes.insert(src, mode);
12522 st.route_selection_cursors.remove(&src);
12523 #[cfg(feature = "discovery")]
12524 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12525 Ok(())
12526 }
12527
12528 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
12530 #[cfg(feature = "discovery")]
12531 let now_ms = self.clock.now_ms();
12532 let mut st = self.state.lock();
12533 if let Some(src) = src {
12534 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12535 }
12536 st.source_route_modes.remove(&src);
12537 st.route_selection_cursors.remove(&src);
12538 #[cfg(feature = "discovery")]
12539 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12540 Ok(())
12541 }
12542
12543 pub fn set_route_weight(
12548 &self,
12549 src: Option<RouterSideId>,
12550 dst: RouterSideId,
12551 weight: u32,
12552 ) -> TelemetryResult<()> {
12553 #[cfg(feature = "discovery")]
12554 let now_ms = self.clock.now_ms();
12555 let mut st = self.state.lock();
12556 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12557 if let Some(src) = src {
12558 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12559 }
12560 st.route_weights.insert((src, dst), weight);
12561 st.route_selection_cursors.remove(&src);
12562 #[cfg(feature = "discovery")]
12563 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12564 Ok(())
12565 }
12566
12567 pub fn clear_route_weight(
12569 &self,
12570 src: Option<RouterSideId>,
12571 dst: RouterSideId,
12572 ) -> TelemetryResult<()> {
12573 #[cfg(feature = "discovery")]
12574 let now_ms = self.clock.now_ms();
12575 let mut st = self.state.lock();
12576 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12577 if let Some(src) = src {
12578 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12579 }
12580 st.route_weights.remove(&(src, dst));
12581 st.route_selection_cursors.remove(&src);
12582 #[cfg(feature = "discovery")]
12583 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12584 Ok(())
12585 }
12586
12587 pub fn set_route_priority(
12592 &self,
12593 src: Option<RouterSideId>,
12594 dst: RouterSideId,
12595 priority: u32,
12596 ) -> TelemetryResult<()> {
12597 #[cfg(feature = "discovery")]
12598 let now_ms = self.clock.now_ms();
12599 let mut st = self.state.lock();
12600 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12601 if let Some(src) = src {
12602 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12603 }
12604 st.route_priorities.insert((src, dst), priority);
12605 #[cfg(feature = "discovery")]
12606 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12607 Ok(())
12608 }
12609
12610 pub fn clear_route_priority(
12612 &self,
12613 src: Option<RouterSideId>,
12614 dst: RouterSideId,
12615 ) -> TelemetryResult<()> {
12616 #[cfg(feature = "discovery")]
12617 let now_ms = self.clock.now_ms();
12618 let mut st = self.state.lock();
12619 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12620 if let Some(src) = src {
12621 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12622 }
12623 st.route_priorities.remove(&(src, dst));
12624 #[cfg(feature = "discovery")]
12625 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12626 Ok(())
12627 }
12628
12629 pub fn set_route(
12634 &self,
12635 src: Option<RouterSideId>,
12636 dst: RouterSideId,
12637 enabled: bool,
12638 ) -> TelemetryResult<()> {
12639 #[cfg(feature = "discovery")]
12640 let now_ms = self.clock.now_ms();
12641 let mut st = self.state.lock();
12642 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12643 if let Some(src) = src {
12644 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12645 }
12646 st.route_overrides.insert((src, dst), enabled);
12647 #[cfg(feature = "discovery")]
12648 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12649 Ok(())
12650 }
12651
12652 pub fn set_typed_route(
12657 &self,
12658 src: Option<RouterSideId>,
12659 ty: DataType,
12660 dst: RouterSideId,
12661 enabled: bool,
12662 ) -> TelemetryResult<()> {
12663 #[cfg(feature = "discovery")]
12664 let now_ms = self.clock.now_ms();
12665 let mut st = self.state.lock();
12666 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12667 if let Some(src) = src {
12668 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12669 }
12670 st.typed_route_overrides
12671 .insert((src, ty.as_u32(), dst), enabled);
12672 #[cfg(feature = "discovery")]
12673 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12674 Ok(())
12675 }
12676
12677 pub fn clear_typed_route(
12679 &self,
12680 src: Option<RouterSideId>,
12681 ty: DataType,
12682 dst: RouterSideId,
12683 ) -> TelemetryResult<()> {
12684 #[cfg(feature = "discovery")]
12685 let now_ms = self.clock.now_ms();
12686 let mut st = self.state.lock();
12687 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12688 if let Some(src) = src {
12689 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12690 }
12691 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
12692 #[cfg(feature = "discovery")]
12693 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12694 Ok(())
12695 }
12696
12697 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
12699 #[cfg(feature = "discovery")]
12700 let now_ms = self.clock.now_ms();
12701 let mut st = self.state.lock();
12702 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
12703 if let Some(src) = src {
12704 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
12705 }
12706 st.route_overrides.remove(&(src, dst));
12707 #[cfg(feature = "discovery")]
12708 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12709 Ok(())
12710 }
12711
12712 #[cfg(feature = "discovery")]
12714 pub fn announce_discovery(&self) -> TelemetryResult<()> {
12715 self.queue_discovery_announce(true, false)
12716 }
12717
12718 #[cfg(feature = "discovery")]
12720 pub fn announce_leave(&self) -> TelemetryResult<()> {
12721 let sender = self.sender_arc();
12722 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
12723 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
12724 }
12725
12726 #[cfg(feature = "discovery")]
12728 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
12729 self.poll_discovery_announce()
12730 }
12731
12732 #[cfg(feature = "discovery")]
12733 pub fn request_topology(&self) -> TelemetryResult<()> {
12734 let sender = self.sender_arc();
12735 let pkt =
12736 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
12737 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
12738 }
12739
12740 #[cfg(feature = "discovery")]
12741 pub fn request_schema(&self) -> TelemetryResult<()> {
12742 let sender = self.sender_arc();
12743 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
12744 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
12745 }
12746
12747 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
12754 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
12755 }
12756
12757 pub fn enable_network_variable(
12762 &self,
12763 ty: DataType,
12764 permissions: NetworkVariablePermissions,
12765 ) -> TelemetryResult<()> {
12766 if is_internal_control_type(ty) {
12767 return Err(TelemetryError::InvalidType);
12768 }
12769 let mut st = self.state.lock();
12770 #[cfg(feature = "discovery")]
12771 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
12772 #[cfg(not(feature = "discovery"))]
12773 st.managed_variable_types.insert(ty.as_u32());
12774 st.managed_variable_permissions
12775 .insert(ty.as_u32(), permissions);
12776 #[cfg(feature = "discovery")]
12777 if newly_advertised {
12778 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12779 }
12780 Ok(())
12781 }
12782
12783 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
12789 where
12790 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
12791 {
12792 if is_internal_control_type(ty) {
12793 return Err(TelemetryError::InvalidType);
12794 }
12795 let mut st = self.state.lock();
12796 #[cfg(feature = "discovery")]
12797 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
12798 #[cfg(not(feature = "discovery"))]
12799 st.managed_variable_types.insert(ty.as_u32());
12800 st.network_variable_update_handlers
12801 .entry(ty.as_u32())
12802 .or_default()
12803 .push(NetworkVariableUpdateHandler {
12804 handler: Arc::new(f),
12805 });
12806 #[cfg(feature = "discovery")]
12807 if newly_advertised {
12808 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12809 }
12810 Ok(())
12811 }
12812
12813 pub fn disable_managed_variable(&self, ty: DataType) {
12814 let mut st = self.state.lock();
12815 #[cfg(feature = "discovery")]
12816 let was_advertised = st.managed_variable_types.remove(&ty.as_u32());
12817 #[cfg(not(feature = "discovery"))]
12818 st.managed_variable_types.remove(&ty.as_u32());
12819 st.managed_variable_permissions.remove(&ty.as_u32());
12820 st.managed_variable_latest.remove(&ty.as_u32());
12821 st.network_variable_update_handlers.remove(&ty.as_u32());
12822 #[cfg(feature = "discovery")]
12823 if was_advertised {
12824 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12825 }
12826 }
12827
12828 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
12829 if is_internal_control_type(pkt.data_type()) {
12830 return Err(TelemetryError::InvalidType);
12831 }
12832 pkt.validate()?;
12833 {
12834 let mut st = self.state.lock();
12835 #[cfg(feature = "discovery")]
12836 let newly_advertised = st.managed_variable_types.insert(pkt.data_type().as_u32());
12837 #[cfg(not(feature = "discovery"))]
12838 st.managed_variable_types.insert(pkt.data_type().as_u32());
12839 #[cfg(feature = "discovery")]
12840 if newly_advertised {
12841 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12842 }
12843 }
12844 self.cache_managed_variable_packet(&pkt, false)
12845 }
12846
12847 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
12848 self.managed_variable_latest(ty)
12849 }
12850
12851 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
12857 if is_internal_control_type(pkt.data_type()) {
12858 return Err(TelemetryError::InvalidType);
12859 }
12860 pkt.validate()?;
12861 let ty = pkt.data_type();
12862 {
12863 let mut st = self.state.lock();
12864 st.managed_variable_types.insert(ty.as_u32());
12865 let perms = Self::managed_variable_permissions_locked(&st, ty);
12866 if !perms.write {
12867 return Err(TelemetryError::PermissionDenied);
12868 }
12869 }
12870 self.cache_managed_variable_packet(&pkt, false)?;
12871 #[cfg(feature = "discovery")]
12872 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12873 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
12874 if self.side_tx_active() {
12875 return self.tx_queue_item_with_priority(
12876 item,
12877 false,
12878 crate::transport_priority(DataType::ManagedVariableValue),
12879 );
12880 }
12881 self.tx_item(item)
12882 }
12883
12884 #[cfg(feature = "discovery")]
12891 pub fn get_network_variable(
12892 &self,
12893 ty: DataType,
12894 stale_after_ms: Option<u64>,
12895 ) -> TelemetryResult<Option<Packet>> {
12896 if is_internal_control_type(ty) {
12897 return Err(TelemetryError::InvalidType);
12898 }
12899 {
12900 let mut st = self.state.lock();
12901 st.managed_variable_types.insert(ty.as_u32());
12902 let perms = Self::managed_variable_permissions_locked(&st, ty);
12903 if !perms.read {
12904 return Err(TelemetryError::PermissionDenied);
12905 }
12906 }
12907 let cached = self.managed_variable_latest_with_age(ty);
12908 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
12909 (None, _) => true,
12910 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
12911 (Some(_), None) => false,
12912 };
12913 if needs_refresh {
12914 self.request_managed_variable(ty)?;
12915 }
12916 Ok(cached.map(|(pkt, _age_ms)| pkt))
12917 }
12918
12919 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
12921 if is_internal_control_type(ty) {
12922 return Err(TelemetryError::InvalidType);
12923 }
12924 if !self.can_read_managed_variable(ty) {
12925 return Err(TelemetryError::PermissionDenied);
12926 }
12927 Ok(self.managed_variable_latest(ty))
12928 }
12929
12930 #[cfg(feature = "discovery")]
12931 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
12932 if is_internal_control_type(ty) {
12933 return Err(TelemetryError::InvalidType);
12934 }
12935 if !self.can_read_managed_variable(ty) {
12936 return Err(TelemetryError::PermissionDenied);
12937 }
12938 let sender = self.sender_arc();
12939 let pkt =
12940 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
12941 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
12942 }
12943
12944 #[cfg(feature = "discovery")]
12945 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
12946 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
12947 self.request_managed_variable(ty)
12948 }
12949
12950 #[cfg(feature = "discovery")]
12952 pub fn export_topology(&self) -> TopologySnapshot {
12953 let now_ms = self.clock.now_ms();
12954 let mut st = self.state.lock();
12955 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
12956 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12957 }
12958 let routes = st
12959 .discovery_routes
12960 .iter()
12961 .filter_map(|(&side_id, route)| {
12962 let side = st.sides.get(side_id)?.as_ref()?;
12963 let announcers = route
12964 .announcers
12965 .iter()
12966 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
12967 sender_id: sender_id.clone(),
12968 reachable_endpoints: sender_state
12969 .reachable
12970 .iter()
12971 .copied()
12972 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
12973 .collect(),
12974 reachable_network_variables: sender_state
12975 .reachable_network_variables
12976 .clone(),
12977 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
12978 routers: sender_state.topology_boards.as_ref().clone(),
12979 last_seen_ms: sender_state.last_seen_ms,
12980 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
12981 })
12982 .collect();
12983 Some(TopologySideRoute {
12984 side_id,
12985 side_name: side.name.to_string(),
12986 reachable_endpoints: route
12987 .reachable
12988 .iter()
12989 .copied()
12990 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
12991 .collect(),
12992 reachable_network_variables: route.reachable_network_variables.clone(),
12993 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
12994 announcers,
12995 last_seen_ms: route.last_seen_ms,
12996 age_ms: now_ms.saturating_sub(route.last_seen_ms),
12997 })
12998 })
12999 .collect();
13000 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
13001 let advertised_endpoints =
13002 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
13003 let advertised_timesync_sources =
13004 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
13005 let links = discovery::topology_links_from_boards(&routers);
13006 TopologySnapshot {
13007 advertised_endpoints,
13008 advertised_timesync_sources,
13009 routers,
13010 links,
13011 routes,
13012 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
13013 next_announce_ms: st.discovery_cadence.next_announce_ms,
13014 }
13015 }
13016
13017 #[cfg(feature = "discovery")]
13018 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
13019 let now_ms = self.clock.now_ms();
13020 let st = self.state.lock();
13021 let mut side_ids = Vec::new();
13022 let mut side_names = Vec::new();
13023 let mut last_seen_ms = None::<u64>;
13024 let mut reachable_endpoints = Vec::new();
13025 let mut reachable_timesync_sources = Vec::new();
13026 let mut packets_sent = 0u64;
13027 let mut packets_received = 0u64;
13028 let mut bytes_sent = 0u64;
13029 let mut bytes_received = 0u64;
13030
13031 for (side_id, route) in &st.discovery_routes {
13032 let Some(sender_state) = route.announcers.get(sender_id) else {
13033 continue;
13034 };
13035 side_ids.push(*side_id);
13036 if let Some(side_name) = st
13037 .sides
13038 .get(*side_id)
13039 .and_then(|side| side.as_ref())
13040 .map(|side| side.name.clone())
13041 {
13042 side_names.push(side_name.to_string());
13043 }
13044 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
13045 reachable_endpoints.extend(sender_state.reachable.iter().copied());
13046 reachable_timesync_sources
13047 .extend(sender_state.reachable_timesync_sources.iter().cloned());
13048 if let Some(stats) = st.side_runtime_stats.get(side_id) {
13049 packets_sent = packets_sent.saturating_add(stats.tx_packets);
13050 packets_received = packets_received.saturating_add(stats.rx_packets);
13051 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
13052 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
13053 }
13054 }
13055
13056 if side_ids.is_empty() {
13057 return None;
13058 }
13059 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
13060 crate::small_sort::sort(&mut reachable_endpoints);
13061 reachable_endpoints.dedup();
13062 reachable_timesync_sources.sort_unstable();
13063 reachable_timesync_sources.dedup();
13064 crate::small_sort::sort(&mut side_ids);
13065 side_ids.dedup();
13066 side_names.sort_unstable();
13067 side_names.dedup();
13068 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
13069 Some(ClientStatsSnapshot {
13070 sender_id: sender_id.to_string(),
13071 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
13072 side_ids,
13073 side_names,
13074 last_seen_ms,
13075 age_ms,
13076 reachable_endpoints,
13077 reachable_timesync_sources,
13078 packets_sent,
13079 packets_received,
13080 bytes_sent,
13081 bytes_received,
13082 })
13083 }
13084
13085 pub fn copy_reliable_pending_stats(
13089 &self,
13090 first: usize,
13091 rows: &mut [crate::diagnostics::ReliablePendingRow],
13092 ) -> usize {
13093 rows.fill(crate::diagnostics::ReliablePendingRow::default());
13094 let st = self.state.lock();
13095 for (row, (&id, sent)) in rows
13096 .iter_mut()
13097 .zip(st.end_to_end_reliable_tx.iter().skip(first))
13098 {
13099 row.packet_id = id;
13100 row.last_send_ms = sent.last_send_ms;
13101 row.data_type = Self::item_data_type(&sent.data).map_or(0, DataType::as_u32);
13102 row.pending_count = sent.pending_destinations.len().min(u32::MAX as usize) as u32;
13103 row.retries = sent.retries;
13104 row.flags = u32::from(sent.queued)
13105 | (u32::from(sent.waiting_for_predecessor) << 1)
13106 | (u32::from(
13107 st.reliable_return_routes
13108 .get(&id)
13109 .is_some_and(|route| route.upstream_contract.is_some()),
13110 ) << 2)
13111 | (u32::from(sent.tracked_destinations) << 3);
13112 for (out, owner) in row
13113 .pending_owners
13114 .iter_mut()
13115 .zip(sent.pending_destinations.keys())
13116 {
13117 *out = *owner;
13118 }
13119 }
13120 st.end_to_end_reliable_tx.len()
13121 }
13122
13123 pub fn export_reliable_pending_stats(&self) -> Vec<crate::diagnostics::ReliablePendingStats> {
13127 let st = self.state.lock();
13128 st.end_to_end_reliable_tx
13129 .iter()
13130 .filter_map(|(&packet_id, sent)| {
13131 Some(crate::diagnostics::ReliablePendingStats {
13132 packet_id,
13133 data_type: Self::item_data_type(&sent.data).ok()?,
13134 pending_destinations: sent.pending_destinations.keys().copied().collect(),
13135 retries: sent.retries,
13136 queued: sent.queued,
13137 waiting_for_predecessor: sent.waiting_for_predecessor,
13138 last_send_ms: sent.last_send_ms,
13139 })
13140 })
13141 .collect()
13142 }
13143
13144 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
13145 let now_ms = self.clock.now_ms();
13146 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
13147 let st = self.state.lock();
13148
13149 let mut sides = Vec::new();
13150 for (side_id, side) in st.sides.iter().enumerate() {
13151 let Some(side) = side.as_ref() else { continue };
13152 let empty_stats = SideRuntimeStatsInner::default();
13153 let stats = st
13154 .side_runtime_stats
13155 .get(&side_id)
13156 .map(Box::as_ref)
13157 .unwrap_or(&empty_stats);
13158 let adaptive = st
13159 .adaptive_route_stats
13160 .get(&side_id)
13161 .cloned()
13162 .unwrap_or_default()
13163 .snapshot(now_ms, true);
13164 let (tx_template_count, rx_template_count) = st
13165 .side_transport
13166 .get(&side_id)
13167 .map(|state| (state.tx_template_count(), state.rx_template_count()))
13168 .unwrap_or((0, 0));
13169 let mut data_types: Vec<RuntimeTypeStats> = stats
13170 .data_types
13171 .iter()
13172 .map(|(&ty, item)| {
13173 let item = item.counters();
13174 RuntimeTypeStats {
13175 data_type: DataType(ty),
13176 tx_packets: item.tx_packets,
13177 tx_bytes: item.tx_bytes,
13178 rx_packets: item.rx_packets,
13179 rx_bytes: item.rx_bytes,
13180 relayed_tx_packets: item.relayed_tx_packets,
13181 relayed_tx_bytes: item.relayed_tx_bytes,
13182 relayed_rx_packets: item.relayed_rx_packets,
13183 relayed_rx_bytes: item.relayed_rx_bytes,
13184 tx_retries: item.tx_retries,
13185 handler_failures: item.handler_failures,
13186 }
13187 })
13188 .collect();
13189 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
13190 sides.push(RuntimeSideStats {
13191 side_id,
13192 side_name: side.name.to_string(),
13193 reliable_enabled: side.opts.reliable_enabled,
13194 link_local_enabled: side.opts.link_local_enabled,
13195 header_template_enabled: side.opts.header_template_enabled,
13196 max_frame_bytes: side.opts.max_frame_bytes,
13197 compact_header_target_bytes: side.opts.compact_header_target_bytes,
13198 side_transport_profile: side.opts.effective_transport_profile().as_str(),
13199 ingress_enabled: side.opts.ingress_enabled,
13200 egress_enabled: side.opts.egress_enabled,
13201 tx_packets: stats.tx_packets,
13202 tx_bytes: stats.tx_bytes,
13203 rx_packets: stats.rx_packets,
13204 rx_bytes: stats.rx_bytes,
13205 relayed_tx_packets: stats.relayed_tx_packets,
13206 relayed_tx_bytes: stats.relayed_tx_bytes,
13207 relayed_rx_packets: stats.relayed_rx_packets,
13208 relayed_rx_bytes: stats.relayed_rx_bytes,
13209 local_delivery_packets: stats.local_delivery_packets,
13210 tx_retries: stats.tx_retries,
13211 tx_handler_failures: stats.tx_handler_failures,
13212 local_handler_failures: stats.local_handler_failures,
13213 total_handler_retries: stats.total_handler_retries,
13214 side_transport_full_frames: stats.side_transport_full_frames,
13215 side_transport_compact_frames: stats.side_transport_compact_frames,
13216 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
13217 side_transport_compact_omitted_timestamp_frames: stats
13218 .side_transport_compact_omitted_timestamp_frames,
13219 side_transport_chunk_frames: stats.side_transport_chunk_frames,
13220 side_transport_chunk_reclaims: st
13221 .side_transport
13222 .get(&side_id)
13223 .map_or(0, |state| u64::from(state.rx_chunk_reclaims)),
13224 side_transport_raw_bytes: stats.side_transport_raw_bytes,
13225 side_transport_wire_bytes: stats.side_transport_wire_bytes,
13226 side_transport_bytes_saved: stats.side_transport_bytes_saved,
13227 side_transport_min_compact_overhead_bytes: stats
13228 .side_transport_min_compact_overhead_bytes,
13229 side_transport_max_compact_overhead_bytes: stats
13230 .side_transport_max_compact_overhead_bytes,
13231 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
13232 side_transport_template_evictions: stats.side_transport_template_evictions,
13233 side_transport_tx_template_count: tx_template_count,
13234 side_transport_rx_template_count: rx_template_count,
13235 max_side_transport_templates: side.opts.max_side_transport_templates,
13236 adaptive,
13237 data_types,
13238 });
13239 }
13240
13241 let mut route_modes: Vec<RouteModeStats> = st
13242 .route_selection_cursors
13243 .iter()
13244 .map(|(src, cursor)| RouteModeStats {
13245 src_side_id: *src,
13246 selection_mode: st.source_route_modes.get(src).copied(),
13247 cursor: *cursor,
13248 })
13249 .collect();
13250 for src in st.source_route_modes.keys() {
13251 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
13252 route_modes.push(RouteModeStats {
13253 src_side_id: *src,
13254 selection_mode: st.source_route_modes.get(src).copied(),
13255 cursor: 0,
13256 });
13257 }
13258 }
13259 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
13260
13261 let mut route_overrides: Vec<RouteOverrideStats> = st
13262 .route_overrides
13263 .iter()
13264 .map(|((src, dst), enabled)| RouteOverrideStats {
13265 src_side_id: *src,
13266 dst_side_id: *dst,
13267 enabled: *enabled,
13268 })
13269 .collect();
13270 route_overrides.sort_unstable_by_key(|item| {
13271 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
13272 });
13273
13274 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
13275 .typed_route_overrides
13276 .iter()
13277 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
13278 src_side_id: *src,
13279 data_type: DataType(*ty),
13280 dst_side_id: *dst,
13281 enabled: *enabled,
13282 })
13283 .collect();
13284 typed_route_overrides.sort_unstable_by_key(|item| {
13285 (
13286 item.src_side_id.unwrap_or(usize::MAX),
13287 item.data_type.as_u32(),
13288 item.dst_side_id,
13289 )
13290 });
13291
13292 let mut route_weights: Vec<RouteWeightStats> = st
13293 .route_weights
13294 .iter()
13295 .map(|((src, dst), weight)| RouteWeightStats {
13296 src_side_id: *src,
13297 dst_side_id: *dst,
13298 weight: *weight,
13299 })
13300 .collect();
13301 route_weights.sort_unstable_by_key(|item| {
13302 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
13303 });
13304
13305 let mut route_priorities: Vec<RoutePriorityStats> = st
13306 .route_priorities
13307 .iter()
13308 .map(|((src, dst), priority)| RoutePriorityStats {
13309 src_side_id: *src,
13310 dst_side_id: *dst,
13311 priority: *priority,
13312 })
13313 .collect();
13314 route_priorities.sort_unstable_by_key(|item| {
13315 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
13316 });
13317
13318 #[cfg(feature = "discovery")]
13319 let discovery = DiscoveryRuntimeStats {
13320 route_count: st.discovery_routes.len(),
13321 announcer_count: st
13322 .discovery_routes
13323 .values()
13324 .map(|route| route.announcers.len())
13325 .sum(),
13326 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
13327 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
13328 };
13329 #[cfg(not(feature = "discovery"))]
13330 let discovery = DiscoveryRuntimeStats {
13331 route_count: 0,
13332 announcer_count: 0,
13333 current_announce_interval_ms: None,
13334 next_announce_ms: None,
13335 };
13336
13337 RuntimeStatsSnapshot {
13338 sides,
13339 route_modes,
13340 route_overrides,
13341 typed_route_overrides,
13342 route_weights,
13343 route_priorities,
13344 queues: QueueRuntimeStats {
13345 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
13346 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
13347 tx_len: st.transmit_queue.len(),
13348 tx_bytes: st.transmit_queue.bytes_used(),
13349 replay_len: 0,
13350 replay_bytes: 0,
13351 recent_rx_len: st.recent_rx.len(),
13352 recent_rx_bytes: st.recent_rx.bytes_used(),
13353 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
13354 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
13355 shared_queue_bytes_used: st.shared_queue_bytes_used(),
13356 },
13357 reliable: ReliableRuntimeStats {
13358 reliable_return_route_count: st.reliable_return_routes.len(),
13359 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
13360 end_to_end_pending_destination_count: st
13361 .end_to_end_reliable_tx
13362 .values()
13363 .map(|sent| sent.pending_destinations.len())
13364 .sum(),
13365 end_to_end_acked_cache_count: 0,
13366 },
13367 discovery,
13368 total_handler_failures: st.total_handler_failures,
13369 total_handler_retries: st.total_handler_retries,
13370 }
13371 }
13372
13373 #[cfg(feature = "std")]
13382 pub fn has_pending_reliable_tx_type(&self, ty: DataType) -> bool {
13383 let st = self.state.lock();
13384 if self
13385 .active_queued_tx
13386 .load(core::sync::atomic::Ordering::Acquire)
13387 != 0
13388 {
13389 return true;
13390 }
13391 let matches_data = |data: &RouterItem| match data {
13392 RouterItem::Packet(packet) => packet.data_type() == ty,
13393 RouterItem::Packed(_) => true,
13394 };
13395 st.transmit_queue.iter().any(|queued| {
13396 if queued.accepted_type != 0 {
13397 return queued.accepted_type == ty.as_u32();
13398 }
13399 match &queued.item {
13400 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
13401 matches_data(data)
13402 }
13403 RouterTxItem::ReliableReplay { ty: found, .. } => *found == ty,
13404 RouterTxItem::EndToEndReplay { packet_id } => st
13405 .end_to_end_reliable_tx
13406 .get(packet_id)
13407 .is_some_and(|owner| matches_data(&owner.data)),
13408 }
13409 }) || st
13410 .end_to_end_reliable_tx
13411 .values()
13412 .any(|owner| matches_data(&owner.data))
13413 || st
13414 .reliable_tx
13415 .iter()
13416 .any(|((_, found), owner)| *found == ty.as_u32() && !owner.sent.is_empty())
13417 }
13418
13419 #[cfg(feature = "std")]
13422 fn track_queued_tx(&self, work: Option<TxWork>) -> Option<(TxWork, QueuedTxActivityGuard<'_>)> {
13423 work.map(|work| {
13424 self.active_queued_tx
13425 .fetch_add(1, core::sync::atomic::Ordering::AcqRel);
13426 (work, QueuedTxActivityGuard(&self.active_queued_tx))
13427 })
13428 }
13429
13430 pub fn has_queued_work(&self) -> bool {
13437 if self
13438 .isr_rx_queue
13439 .try_lock()
13440 .map_or(true, |queue| !queue.is_empty())
13441 {
13442 return true;
13443 }
13444 let state = self.state.lock();
13445 !state.received_queue.is_empty() || !state.transmit_queue.is_empty()
13446 }
13447
13448 pub fn queue_counters(&self) -> [u32; 16] {
13454 let now = self.clock.now_ms();
13455 let st = self.state.lock();
13456 let ty_of = |item: &RouterTxItem| match item {
13457 RouterTxItem::Broadcast(RouterItem::Packet(packet))
13458 | RouterTxItem::ToSide {
13459 data: RouterItem::Packet(packet),
13460 ..
13461 } => Some(packet.data_type()),
13462 RouterTxItem::Broadcast(RouterItem::Packed(_))
13463 | RouterTxItem::ToSide {
13464 data: RouterItem::Packed(_),
13465 ..
13466 }
13467 | RouterTxItem::ReliableReplay { .. } => None,
13468 RouterTxItem::EndToEndReplay { packet_id } => st
13469 .end_to_end_reliable_tx
13470 .get(packet_id)
13471 .and_then(|pending| match &pending.data {
13472 RouterItem::Packet(packet) => Some(packet.data_type()),
13473 RouterItem::Packed(_) => None,
13474 }),
13475 };
13476 let sum_cost = |total: usize, cost: usize| total.saturating_add(cost);
13477 let front = st.transmit_queue.iter().next();
13478 let ack_count = st
13479 .transmit_queue
13480 .iter()
13481 .filter(|q| {
13482 matches!(
13483 ty_of(&q.item),
13484 Some(
13485 DataType::ReliableAck
13486 | DataType::ReliablePartialAck
13487 | DataType::ReliablePacketRequest
13488 )
13489 )
13490 })
13491 .count();
13492 let schema_count = st
13493 .transmit_queue
13494 .iter()
13495 .filter(|q| ty_of(&q.item) == Some(DataType::DiscoverySchema))
13496 .count();
13497 [
13498 1,
13499 st.received_queue.len() as u32,
13500 st.transmit_queue.len() as u32,
13501 st.received_queue
13502 .iter()
13503 .map(ByteCost::byte_cost)
13504 .fold(0, sum_cost) as u32,
13505 st.transmit_queue
13506 .iter()
13507 .map(ByteCost::byte_cost)
13508 .fold(0, sum_cost) as u32,
13509 front
13510 .and_then(|q| ty_of(&q.item))
13511 .map_or(0, DataType::as_u32),
13512 front.map_or(0, |q| q.priority.into()),
13513 front.map_or(0, |q| {
13514 q.retry_after_ms.saturating_sub(now).min(u32::MAX.into()) as u32
13515 }),
13516 ack_count as u32,
13517 schema_count as u32,
13518 st.transmit_queue
13519 .iter()
13520 .map(|q| u32::from(q.priority))
13521 .min()
13522 .unwrap_or(0),
13523 st.end_to_end_reliable_tx.len() as u32,
13524 st.end_to_end_reliable_tx
13525 .values()
13526 .map(|entry| match &entry.data {
13527 RouterItem::Packet(packet) => packet.byte_cost(),
13528 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
13529 })
13530 .fold(0, sum_cost) as u32,
13531 front.map_or(0, |q| match q.item {
13532 RouterTxItem::Broadcast(_) => 1,
13533 RouterTxItem::ToSide { .. } => 2,
13534 RouterTxItem::ReliableReplay { .. } => 3,
13535 RouterTxItem::EndToEndReplay { .. } => 4,
13536 }),
13537 now as u32,
13538 (now >> 32) as u32,
13539 ]
13540 }
13541
13542 pub fn memory_counters(&self) -> [u32; 16] {
13543 let st = self.state.lock();
13544 #[cfg(feature = "discovery")]
13545 let discovery = st.discovery_bytes_used();
13546 #[cfg(not(feature = "discovery"))]
13547 let discovery = 0;
13548 let templates = st
13549 .side_transport
13550 .values()
13551 .map(|side| side.tx_templates.len() + side.rx_templates_by_id.len())
13552 .sum::<usize>();
13553 let template_bytes = st
13554 .side_transport
13555 .values()
13556 .flat_map(|side| {
13557 side.tx_templates
13558 .values()
13559 .chain(side.rx_templates_by_id.values())
13560 })
13561 .map(|t| size_of::<SideHeaderTemplate>() + t.prefix().len() + t.between().len())
13562 .sum::<usize>();
13563 let chunks = st
13564 .side_transport
13565 .values()
13566 .flat_map(|side| side.rx_chunks.values())
13567 .flat_map(|assembly| assembly.received.iter().flatten())
13568 .map(|bytes| bytes.len())
13569 .sum::<usize>();
13570 let type_stats = st
13571 .side_runtime_stats
13572 .values()
13573 .map(|side| side.data_types.len())
13574 .sum::<usize>();
13575 let managed = st
13576 .managed_variable_latest
13577 .values()
13578 .map(|entry| entry.packet.byte_cost())
13579 .sum::<usize>();
13580 let pending = st
13581 .end_to_end_reliable_tx
13582 .values()
13583 .map(|entry| match &entry.data {
13584 RouterItem::Packet(packet) => packet.byte_cost(),
13585 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
13586 })
13587 .sum::<usize>();
13588 [
13589 1,
13590 st.memory.max_queue_budget,
13591 st.shared_queue_bytes_used(),
13592 st.received_queue.bytes_used(),
13593 st.transmit_queue.bytes_used(),
13594 st.received_queue.capacity() * size_of::<RouterRxItem>()
13595 + st.transmit_queue.capacity() * size_of::<TxQueued>()
13596 + st.recent_rx.capacity() * size_of::<u64>(),
13597 st.reliable_rx_buffered_bytes(),
13598 discovery,
13599 crate::config::schema_bytes_used(),
13600 managed,
13601 pending,
13602 templates,
13603 template_bytes,
13604 chunks,
13605 type_stats,
13606 st.side_runtime_stats
13607 .values()
13608 .flat_map(|side| side.data_types.values())
13609 .map(|stats| size_of::<TypeRuntimeStatsInner>() + stats.heap_bytes())
13610 .sum::<usize>(),
13611 ]
13612 .map(|value| u32::try_from(value).unwrap_or(u32::MAX))
13613 }
13614
13615 pub fn export_memory_layout_json(&self) -> String {
13617 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
13618 let st = self.state.lock();
13619 #[cfg(feature = "discovery")]
13620 let discovery_bytes = st.discovery_bytes_used();
13621 #[cfg(not(feature = "discovery"))]
13622 let discovery_bytes = 0usize;
13623 let schema_bytes = crate::config::schema_bytes_used();
13624 let network_variable_cache_bytes = st
13625 .managed_variable_latest
13626 .values()
13627 .map(|entry| entry.packet.byte_cost())
13628 .sum::<usize>();
13629 let mut out = String::new();
13630 let memory = st.memory;
13631 let _ = fmt::Write::write_fmt(
13632 &mut out,
13633 format_args!(
13634 "{{\"kind\":\"router\",\
13635 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
13636 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
13637 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
13638 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
13639 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
13640 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
13641 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
13642 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
13643 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
13644 st.shared_queue_bytes_used(),
13645 memory.max_queue_budget,
13646 st.received_queue.bytes_used(),
13647 st.received_queue.max_bytes(),
13648 st.received_queue.len(),
13649 isr_rx.1,
13650 memory.max_queue_budget,
13651 isr_rx.0,
13652 st.transmit_queue.bytes_used(),
13653 st.transmit_queue.max_bytes(),
13654 st.transmit_queue.len(),
13655 st.recent_rx.bytes_used(),
13656 st.recent_rx.max_bytes(),
13657 st.recent_rx.len(),
13658 st.reliable_rx_buffered_bytes(),
13659 memory.max_queue_budget,
13660 st.reliable_rx_buffer_len(),
13661 discovery_bytes,
13662 memory.max_queue_budget,
13663 schema_bytes,
13664 memory.max_queue_budget,
13665 network_variable_cache_bytes,
13666 memory.max_queue_budget,
13667 st.managed_variable_latest.len(),
13668 ),
13669 );
13670 out
13671 }
13672
13673 #[cfg(test)]
13674 #[cfg(feature = "discovery")]
13675 pub(crate) fn debug_end_to_end_pending_destination_count(
13676 &self,
13677 packet_id: u64,
13678 ) -> Option<usize> {
13679 let st = self.state.lock();
13680 st.end_to_end_reliable_tx
13681 .get(&packet_id)
13682 .map(|sent| sent.pending_destinations.len())
13683 }
13684
13685 #[cfg(test)]
13686 #[cfg(feature = "discovery")]
13687 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
13688 let st = self.state.lock();
13689 st.end_to_end_reliable_tx.len()
13690 }
13691
13692 #[cfg(test)]
13693 #[cfg(feature = "discovery")]
13694 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
13695 let st = self.state.lock();
13696 st.reliable_return_routes.len()
13697 }
13698
13699 #[cfg(test)]
13700 pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
13701 let st = self.state.lock();
13702 (st.sides.len(), st.sides.capacity())
13703 }
13704
13705 #[cfg(test)]
13706 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
13707 let st = self.state.lock();
13708 (
13709 st.received_queue.len(),
13710 st.transmit_queue.len(),
13711 st.recent_rx.len(),
13712 )
13713 }
13714
13715 #[cfg(test)]
13716 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
13717 let st = self.state.lock();
13718 st.shared_queue_bytes_used()
13719 }
13720
13721 #[cfg(test)]
13722 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
13723 let st = self.state.lock();
13724 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
13725 }
13726
13727 fn get_hash(item: &RouterItem) -> u64 {
13731 match item {
13732 RouterItem::Packet(pkt) => pkt.packet_id(),
13733 RouterItem::Packed(bytes) => {
13734 match wire_format::packet_id_from_wire(bytes.as_ref()) {
13735 Ok(id) => id,
13736 Err(_e) => {
13737 let h: u64 = 0x9E37_79B9_7F4A_7C15;
13740 hash_bytes_u64(h, bytes.as_ref())
13741 }
13742 }
13743 }
13744 }
13745 }
13746
13747 fn remove_pkt_id(&self, item: &RouterItem) {
13749 let hash = Self::get_hash(item);
13750 let mut st = self.state.lock();
13751 st.recent_rx.remove_value(&hash);
13752 }
13753
13754 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
13757 let id = Self::get_hash(item);
13758 let mut st = self.state.lock();
13759 if st.recent_rx.contains(&id)
13760 || st
13761 .reliable_return_routes
13762 .get(&id)
13763 .is_some_and(|route| route.locally_delivered)
13764 {
13765 Ok(true)
13766 } else {
13767 st.push_recent_rx(id)?;
13768 Ok(false)
13769 }
13770 }
13771
13772 fn handle_callback_error(
13777 &self,
13778 pkt: &Packet,
13779 dest: Option<DataEndpoint>,
13780 e: TelemetryError,
13781 called_from_queue: bool,
13782 ) -> TelemetryResult<()> {
13783 if matches!(&e, TelemetryError::Io("memory pressure")) {
13786 return Err(e);
13787 }
13788
13789 let device = self.sender_arc();
13790 let error_msg = match dest {
13791 Some(failed_local) => format!(
13792 "Handler for endpoint {:?} failed on device {:?}: {:?}",
13793 failed_local, device, e
13794 ),
13795 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
13796 };
13797
13798 let mut recipients: Vec<DataEndpoint> = pkt
13799 .endpoints()
13800 .iter()
13801 .copied()
13802 .filter(|&ep| self.cfg.is_local_endpoint(ep))
13803 .collect();
13804 crate::small_sort::sort(&mut recipients);
13805 recipients.dedup();
13806
13807 if let Some(failed_local) = dest {
13808 recipients.retain(|&ep| ep != failed_local);
13809 }
13810
13811 if recipients.is_empty() && dest.is_none() {
13815 fallback_stdout(&error_msg);
13816 return Ok(());
13817 }
13818
13819 if recipients.is_empty() {
13822 recipients = pkt.endpoints().to_vec();
13823 crate::small_sort::sort(&mut recipients);
13824 recipients.dedup();
13825 if let Some(failed_local) = dest {
13826 recipients.retain(|&ep| ep != failed_local);
13827 }
13828 }
13829
13830 if recipients.is_empty() {
13831 fallback_stdout(&error_msg);
13832 return Ok(());
13833 }
13834
13835 let payload = make_error_payload(&error_msg);
13836
13837 let sender = self.sender_arc();
13838 let error_pkt = Packet::new(
13839 DataType::TelemetryError,
13840 &recipients,
13841 sender.as_ref(),
13842 self.packet_timestamp_ms(),
13843 crate::shared_bytes::convert(payload),
13844 )?;
13845
13846 self.emit_internal_tx(
13847 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
13848 false,
13849 called_from_queue,
13850 )
13851 }
13852
13853 #[inline]
13857 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
13858 self.process_tx_queue_with_timeout(0)
13859 }
13860
13861 #[inline]
13863 pub fn process_all_queues(&self) -> TelemetryResult<()> {
13864 self.process_all_queues_with_timeout(0)
13865 }
13866
13867 #[inline]
13869 pub fn clear_queues(&self) {
13870 let mut st = self.state.lock();
13871 st.retain_transmit(|_| false);
13872 st.has_ordered_retry = false;
13873 st.received_queue.clear();
13874 st.rx_priority_burst = 0;
13875 drop(st);
13876 let _ = self.isr_rx_queue.clear();
13877 }
13878
13879 #[inline]
13881 pub fn clear_rx_queue(&self) {
13882 let mut st = self.state.lock();
13883 st.received_queue.clear();
13884 st.rx_priority_burst = 0;
13885 drop(st);
13886 let _ = self.isr_rx_queue.clear();
13887 }
13888
13889 #[inline]
13891 pub fn clear_tx_queue(&self) {
13892 let mut st = self.state.lock();
13893 st.retain_transmit(|_| false);
13894 st.tx_priority_burst = 0;
13895 st.has_ordered_retry = false;
13896 }
13897
13898 #[inline(never)]
13899 fn pop_transmit_queue_locked(
13900 st: &mut RouterInner,
13901 now_ms: u64,
13902 control_only: bool,
13903 ) -> Option<TxWork> {
13904 let slow = st.has_ordered_retry;
13905 let mut next_token = st.ordered_claim_seq.wrapping_add(1).max(1);
13906 if slow {
13907 while st.claim_index(next_token).is_some() {
13908 next_token = next_token.wrapping_add(1).max(1);
13909 }
13910 }
13911 let selected = st.transmit_queue.select_front_fair_with_context(
13912 &mut st.tx_priority_burst,
13913 MAX_PRIORITY_BURST,
13914 |item| item.priority,
13915 |_queue, item| {
13916 !matches!(item.delivery_state,
13917 QueuedDelivery::ClaimedOrdered | QueuedDelivery::ClaimedUnordered
13918 | QueuedDelivery::OrderedTail)
13919 && item.retry_after_ms <= now_ms
13920 && (!control_only || item.priority >= 200)
13921 },
13922 )?;
13923 let current = st.transmit_queue.get(selected)?;
13924 let reliable = RouterInner::reliable_original_type(current).is_some();
13925 let ordered = reliable && RouterInner::ordered_original_type(current).is_some();
13926 let (queued, claim) = if reliable {
13927 let queued = current.clone();
13928 st.transmit_queue.update_metadata(selected, |item| {
13929 item.delivery_state = if ordered {
13930 QueuedDelivery::ClaimedOrdered
13931 } else {
13932 QueuedDelivery::ClaimedUnordered
13933 };
13934 item.deferred_since_ms = u64::from(next_token);
13935 });
13936 st.ordered_claim_seq = next_token;
13937 (
13938 queued,
13939 Some(TxClaim {
13940 token: next_token,
13941 ordered,
13942 }),
13943 )
13944 } else {
13945 (st.transmit_queue.remove_pos(selected)?, None)
13946 };
13947 if claim.is_some() {
13948 st.has_ordered_retry = true;
13949 } else if slow {
13950 st.refresh_retry_hint();
13951 }
13952 Some(TxWork { queued, claim })
13953 }
13954
13955 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
13958 self.expire_side_chunks();
13959 let start = self.clock.now_ms();
13960 loop {
13961 let received_control = crate::memory_admission::allow_drain(
13962 self.drain_queued_control_rx_before_tx(start, timeout_ms),
13963 )?;
13964 let control_budget_used = received_control
13965 && timeout_ms != 0
13966 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
13967 if !control_budget_used {
13970 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
13971 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
13972 }
13973
13974 let pkt_opt = {
13975 let mut st = self.state.lock();
13976 let work = Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false);
13977 #[cfg(feature = "std")]
13978 let work = self.track_queued_tx(work);
13979 work
13980 };
13981 let Some(pkt) = pkt_opt else { break };
13982 #[cfg(feature = "std")]
13983 let (pkt, _activity) = pkt;
13984 self.dispatch_queued_item(pkt)?;
13985 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
13986 break;
13987 }
13988 }
13989 Ok(())
13990 }
13991
13992 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
13995 #[cfg(feature = "timesync")]
13996 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
13997 #[cfg(feature = "discovery")]
13998 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
13999 self.process_tx_queue_with_timeout_impl(timeout_ms)
14000 }
14001
14002 pub fn dispatch_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
14008 self.process_tx_queue_with_timeout_impl(timeout_ms)
14009 }
14010
14011 #[inline]
14013 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
14014 self.rx_item(&item, true)
14015 }
14016
14017 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
14020 self.expire_side_chunks();
14021 let start = self.clock.now_ms();
14022 loop {
14023 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
14024 let mut st = self.state.lock();
14025 st.pop_received_fair()
14026 });
14027 let Some(item) = item_opt else { break };
14028 self.process_rx_queue_item(item)?;
14029 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
14030 break;
14031 }
14032 }
14033 Ok(())
14034 }
14035
14036 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
14039 #[cfg(feature = "timesync")]
14040 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
14041 #[cfg(feature = "discovery")]
14042 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14043 self.process_rx_queue_with_timeout_impl(timeout_ms)
14044 }
14045
14046 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
14049 self.expire_side_chunks();
14050 let start = self.clock.now_ms();
14055 loop {
14056 let received_control = crate::memory_admission::allow_drain(
14057 self.drain_queued_control_rx_before_tx(start, timeout_ms),
14058 )?;
14059 let mut did_any = received_control;
14060 let control_budget_used = received_control
14061 && timeout_ms != 0
14062 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
14063 if !control_budget_used {
14064 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
14065 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
14066 }
14067
14068 if let Some(pkt) = {
14069 let mut st = self.state.lock();
14070 let now = self.clock.now_ms();
14071 let work = Self::pop_transmit_queue_locked(&mut st, now, false);
14074 #[cfg(feature = "std")]
14075 let work = self.track_queued_tx(work);
14076 work
14077 } {
14078 #[cfg(feature = "std")]
14079 let (pkt, _activity) = pkt;
14080 self.dispatch_queued_item(pkt)?;
14081 did_any = true;
14082 }
14083 if control_budget_used {
14085 return Ok(());
14086 }
14087 if let Some(item) = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
14088 let mut st = self.state.lock();
14089 st.pop_received_fair()
14090 }) {
14091 self.process_rx_queue_item(item)?;
14092 did_any = true;
14093 }
14094 if !did_any
14095 || (timeout_ms != 0
14096 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms))
14097 {
14098 break;
14099 }
14100 }
14101
14102 Ok(())
14103 }
14104
14105 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
14108 #[cfg(feature = "timesync")]
14109 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
14110 #[cfg(feature = "discovery")]
14111 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14112 self.process_all_queues_with_timeout_impl(timeout_ms)
14113 }
14114
14115 pub fn drain_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
14120 self.process_all_queues_with_timeout_impl(timeout_ms)
14121 }
14122
14123 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
14128 #[cfg(feature = "timesync")]
14129 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
14130
14131 #[cfg(feature = "discovery")]
14132 {
14133 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14134 }
14135
14136 self.process_all_queues_with_timeout_impl(timeout_ms)
14137 }
14138
14139 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
14144 #[cfg(feature = "discovery")]
14145 {
14146 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14147 }
14148
14149 self.process_all_queues_with_timeout_impl(timeout_ms)
14150 }
14151
14152 #[inline]
14154 fn tx_queue_item_with_flags(
14155 &self,
14156 item: RouterTxItem,
14157 ignore_local: bool,
14158 ) -> TelemetryResult<()> {
14159 self.tx_queue_item_with_flags_at(item, ignore_local, 0)
14160 }
14161
14162 #[cfg(feature = "compression")]
14163 fn retry_codec_owner(
14164 &self,
14165 item: RouterTxItem,
14166 ignore_local: bool,
14167 reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
14168 ) -> TelemetryResult<()> {
14169 let priority = match &item {
14170 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
14171 _ => return Err(TelemetryError::BadArg),
14172 };
14173 let mut state = self.state.lock();
14174 if let Some(reservation) = reservation {
14175 reservation.release_locked(&mut state);
14176 }
14177 state.push_transmit(TxQueued {
14178 disposable: true,
14179 item,
14180 ignore_local,
14181 priority,
14182 delivery_state: QueuedDelivery::Untracked,
14183 accepted_type: 0,
14184 deferred_since_ms: 0,
14185 retry_after_ms: self.clock.now_ms().saturating_add(1),
14186 })
14187 }
14188
14189 fn claim_schema_is_current(
14190 &self,
14191 ty: DataType,
14192 claim: Option<TxClaim>,
14193 ) -> TelemetryResult<bool> {
14194 let Some(claim) = claim else {
14195 return Ok(true);
14196 };
14197 let expected = if claim.ordered {
14198 crate::ReliableMode::Ordered
14199 } else {
14200 crate::ReliableMode::Unordered
14201 };
14202 let current = reliable_mode(ty);
14203 let mut st = self.state.lock();
14204 let owned = st.claim_index(claim.token).is_some();
14205 if !owned {
14206 return Ok(false);
14207 }
14208 if current == expected {
14209 return Ok(true);
14210 }
14211 st.cancel_accepted_type(ty.as_u32());
14212 st.refresh_retry_hint();
14213 Err(TelemetryError::HandlerError(
14214 "accepted reliable schema changed before dispatch",
14215 ))
14216 }
14217
14218 fn mark_claim_local_delivered(&self, claim: Option<TxClaim>) -> bool {
14219 let Some(claim) = claim else {
14220 return true;
14221 };
14222 let mut st = self.state.lock();
14223 let index = st.claim_index(claim.token);
14224 let Some(index) = index else {
14225 return false;
14226 };
14227 st.transmit_queue
14228 .update_metadata(index, |item| item.ignore_local = true);
14229 true
14230 }
14231
14232 fn dispatch_queued_item(&self, work: TxWork) -> TelemetryResult<()> {
14233 let queued = work.queued;
14234 let since = matches!(
14235 queued.delivery_state,
14236 QueuedDelivery::DeferredOrdered | QueuedDelivery::DeferredUnordered
14237 )
14238 .then_some(queued.deferred_since_ms);
14239 if !self.claim_schema_is_current(DataType(queued.accepted_type), work.claim)? {
14240 return Ok(());
14241 }
14242 let mut context = DispatchContext {
14243 claim: work.claim,
14244 since,
14245 terminal: false,
14246 owner_transferred: false,
14247 };
14248 let mut result =
14249 self.tx_item_impl_with_retry(queued.item, queued.ignore_local, true, &mut context);
14250 if let Some(claim) = work.claim {
14251 let mut st = self.state.lock();
14252 if result.is_err() && !context.terminal && !context.owner_transferred {
14253 let Some(index) = st.claim_index(claim.token) else {
14254 return result;
14255 };
14256 let original = st.transmit_queue.get(index).expect("claimed owner");
14257 if !original.ignore_local
14258 && let RouterTxItem::Broadcast(data) = &original.item
14259 {
14260 let id = Self::get_hash(data);
14261 st.recent_rx.remove_value(&id);
14262 }
14263 drop(st);
14264 result = self.defer_accepted_original(&mut context);
14265 if result.is_ok() {
14266 return Ok(());
14267 }
14268 st = self.state.lock();
14269 }
14270 if let Some(index) = st.claim_index(claim.token) {
14271 if context.terminal && claim.ordered {
14272 let ty = st
14273 .transmit_queue
14274 .get(index)
14275 .expect("claimed owner")
14276 .accepted_type;
14277 st.cancel_accepted_type(ty);
14278 } else {
14279 let ty = st.transmit_queue.get(index).expect("claimed owner").accepted_type;
14280 let _ = st.transmit_queue.remove_pos(index);
14281 if claim.ordered {
14282 let next = st.transmit_queue.iter().position(|item|
14285 item.delivery_state == QueuedDelivery::OrderedTail && item.accepted_type == ty);
14286 if let Some(next) = next {
14287 st.transmit_queue.update_metadata(next, |item| item.delivery_state = QueuedDelivery::Ordered);
14288 }
14289 }
14290 }
14291 }
14292 st.refresh_retry_hint();
14293 }
14294 result
14295 }
14296
14297 fn expire_deferred_original(
14298 &self,
14299 ty: DataType,
14300 since: u64,
14301 ordered: bool,
14302 ) -> TelemetryResult<()> {
14303 let lifetime = runtime_reliable_retransmit_ms()
14304 .saturating_mul(u64::from(runtime_reliable_max_retries()) + 1);
14305 let now = self.clock.now_ms();
14306 if now >= since && now - since < lifetime {
14307 return Ok(());
14308 }
14309 if ordered {
14310 let mut st = self.state.lock();
14311 st.cancel_accepted_type(ty.as_u32());
14312 st.refresh_retry_hint();
14313 }
14314 Err(TelemetryError::HandlerError(
14315 "accepted reliable delivery expired before dispatch",
14316 ))
14317 }
14318
14319 fn defer_accepted_original(&self, context: &mut DispatchContext) -> TelemetryResult<()> {
14320 let Some(claim) = context.claim else {
14321 return Err(TelemetryError::Io("reliable owner unavailable"));
14322 };
14323 let now = self.clock.now_ms();
14324 let Some(retry_after_ms) = now.checked_add(1) else {
14325 context.terminal = true;
14326 return Err(TelemetryError::HandlerError(
14327 "accepted reliable clock exhausted",
14328 ));
14329 };
14330 let mut st = self.state.lock();
14331 let Some(index) = st.claim_index(claim.token) else {
14332 return Ok(());
14333 };
14334 st.transmit_queue.update_metadata(index, |item| {
14335 item.delivery_state = if claim.ordered {
14336 QueuedDelivery::DeferredOrdered
14337 } else {
14338 QueuedDelivery::DeferredUnordered
14339 };
14340 item.deferred_since_ms = context.since.unwrap_or(now);
14341 item.retry_after_ms = retry_after_ms;
14342 });
14343 st.has_ordered_retry = true;
14344 Ok(())
14345 }
14346
14347 #[allow(unused_mut)]
14348 fn replace_claim_payload(&self, claim: TxClaim, mut data: RouterItem) -> TelemetryResult<()> {
14349 #[cfg(feature = "compact-packet-store")]
14350 data.park_for_queue()?;
14351 let payload_cost = match &data {
14352 RouterItem::Packet(packet) => packet.byte_cost(),
14353 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
14354 };
14355 let mut st = self.state.lock();
14356 let Some(index) = st.claim_index(claim.token) else {
14357 return Ok(());
14358 };
14359 let original = st.transmit_queue.get(index).expect("claimed owner");
14360 let old_cost = original.byte_cost();
14361 let new_cost = old_cost
14362 .saturating_sub(original.item.byte_cost())
14363 .saturating_add(payload_cost);
14364 let priority = original.priority;
14365 let growth = new_cost.saturating_sub(old_cost);
14366 if growth != 0 {
14367 st.make_shared_queue_room(growth, priority, None)?;
14368 }
14369 let Some(index) = st.claim_index(claim.token) else {
14370 return Ok(());
14371 };
14372 st.transmit_queue.update_admitted(index, new_cost, |item| {
14373 item.item = RouterTxItem::Broadcast(data)
14374 })
14375 }
14376
14377 fn tx_queue_item_for_retry(
14378 &self,
14379 item: RouterTxItem,
14380 ignore_local: bool,
14381 ) -> TelemetryResult<()> {
14382 self.tx_queue_item_with_flags_at(item, ignore_local, self.clock.now_ms().saturating_add(1))
14383 }
14384
14385 fn tx_queue_item_with_flags_at(
14386 &self,
14387 item: RouterTxItem,
14388 ignore_local: bool,
14389 retry_after_ms: u64,
14390 ) -> TelemetryResult<()> {
14391 let priority = match &item {
14392 RouterTxItem::Broadcast(data) => self.item_service_priority(data)?,
14393 RouterTxItem::EndToEndReplay { .. } => {
14394 Self::router_item_priority_bumped(DataType::ReliableAck)
14395 }
14396 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
14397 RouterTxItem::ReliableReplay { ty, .. } => Self::router_item_priority_bumped(*ty),
14398 };
14399 self.tx_queue_item_with_priority_at(item, ignore_local, priority, retry_after_ms)
14400 }
14401
14402 #[inline]
14403 fn tx_queue_item_with_priority(
14404 &self,
14405 item: RouterTxItem,
14406 ignore_local: bool,
14407 priority: u8,
14408 ) -> TelemetryResult<()> {
14409 self.tx_queue_item_with_priority_at(item, ignore_local, priority, 0)
14410 }
14411
14412 fn tx_queue_item_with_priority_at(
14413 &self,
14414 item: RouterTxItem,
14415 ignore_local: bool,
14416 priority: u8,
14417 retry_after_ms: u64,
14418 ) -> TelemetryResult<()> {
14419 let mut st = self.state.lock();
14420 st.push_transmit(TxQueued {
14421 disposable: true,
14422 item,
14423 ignore_local,
14424 priority,
14425 delivery_state: QueuedDelivery::Untracked,
14426 accepted_type: 0,
14427 deferred_since_ms: 0,
14428 retry_after_ms,
14429 })?;
14430 Ok(())
14431 }
14432
14433 #[inline]
14435 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
14436 self.tx_queue_item_with_flags(item, false)
14437 }
14438
14439 #[cfg(feature = "std")]
14440 #[inline]
14441 fn try_enter_side_tx(&self, side: usize) -> Option<crate::lock::ActiveSideTxGuard<'_>> {
14442 let gate = self
14443 .state
14444 .lock()
14445 .side_transport
14446 .get(&side)?
14447 .tx_callback_gate
14448 .clone();
14449 self.side_tx_activity.try_enter(gate)
14450 }
14451 #[cfg(not(feature = "std"))]
14452 #[inline]
14453 fn try_enter_side_tx(&self, _side: usize) -> Option<crate::lock::ReentryGuard<'_>> {
14454 self.side_tx_gate.try_enter()
14455 }
14456 #[inline]
14457 fn side_tx_active(&self) -> bool {
14458 #[cfg(feature = "std")]
14459 {
14460 self.side_tx_activity.is_active()
14461 }
14462 #[cfg(not(feature = "std"))]
14463 {
14464 self.side_tx_gate.is_active()
14465 }
14466 }
14467
14468 #[inline]
14472 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
14473 self.process_rx_queue_with_timeout(0)
14474 }
14475
14476 #[inline]
14478 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
14479 crate::memory_admission::check_receive(bytes)?;
14480 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
14481 let priority = self.item_service_priority(&data)?;
14482 let mut st = self.state.lock();
14483 st.push_received(RouterRxItem {
14484 src: None,
14485 data,
14486 priority,
14487 })?;
14488 Ok(())
14489 }
14490
14491 #[inline]
14496 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
14497 crate::memory_admission::check_receive(bytes)?;
14498 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
14499 let priority = Self::router_item_priority(&data)?;
14500 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
14501 src: None,
14502 data,
14503 priority,
14504 })
14505 }
14506
14507 #[inline]
14509 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
14510 pkt.validate()?;
14511 let data = RouterItem::Packet(pkt);
14512 let priority = self.item_service_priority(&data)?;
14513 let mut st = self.state.lock();
14514 st.push_received(RouterRxItem {
14515 src: None,
14516 data,
14517 priority,
14518 })?;
14519 Ok(())
14520 }
14521
14522 #[inline]
14527 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
14528 pkt.validate()?;
14529 let data = RouterItem::Packet(pkt);
14530 let priority = Self::router_item_priority(&data)?;
14531 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
14532 src: None,
14533 data,
14534 priority,
14535 })
14536 }
14537
14538 #[inline]
14540 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
14541 self.ensure_side_ingress_enabled(side)?;
14542 pkt.validate()?;
14543 let data = RouterItem::Packet(pkt);
14544 let priority = self.item_service_priority(&data)?;
14545 let mut st = self.state.lock();
14546 st.push_received(RouterRxItem {
14547 src: Some(side),
14548 data,
14549 priority,
14550 })?;
14551 Ok(())
14552 }
14553
14554 #[inline]
14559 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
14560 self.ensure_side_ingress_enabled(side)?;
14561 pkt.validate()?;
14562 let data = RouterItem::Packet(pkt);
14563 let priority = Self::router_item_priority(&data)?;
14564 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
14565 src: Some(side),
14566 data,
14567 priority,
14568 })
14569 }
14570
14571 #[inline]
14573 pub fn rx_packed_queue_from_side(
14574 &self,
14575 bytes: &[u8],
14576 side: RouterSideId,
14577 ) -> TelemetryResult<()> {
14578 self.ensure_side_ingress_enabled(side)?;
14579 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
14580 return Ok(());
14581 };
14582 let data = RouterItem::Packed(decoded);
14583 let priority = self.item_service_priority(&data)?;
14584 let mut st = self.state.lock();
14585 st.push_received(RouterRxItem {
14586 src: Some(side),
14587 data,
14588 priority,
14589 })?;
14590 Ok(())
14591 }
14592
14593 #[inline]
14598 pub fn rx_packed_queue_from_side_isr(
14599 &self,
14600 bytes: &[u8],
14601 side: RouterSideId,
14602 ) -> TelemetryResult<()> {
14603 self.ensure_side_ingress_enabled(side)?;
14604 crate::memory_admission::check_receive(bytes)?;
14605 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
14606 let priority = Self::router_item_priority(&data)?;
14607 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
14608 src: Some(side),
14609 data,
14610 priority,
14611 })
14612 }
14613
14614 fn retry_with_attempts<F, T>(
14615 &self,
14616 times: usize,
14617 f: F,
14618 ) -> Result<(T, usize), (TelemetryError, usize)>
14619 where
14620 F: Fn() -> TelemetryResult<T>,
14621 {
14622 let mut last_err = None;
14623 for attempt in 0..times {
14624 match f() {
14625 Ok(v) => return Ok((v, attempt + 1)),
14626 Err(e) if Self::is_transient_tx_error(&e) => return Err((e, attempt + 1)),
14629 Err(e) => last_err = Some((e, attempt + 1)),
14630 }
14631 }
14632 Err(last_err.expect("times > 0"))
14633 }
14634
14635 #[inline]
14637 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
14638 self.cfg
14639 .handlers
14640 .iter()
14641 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
14642 }
14643
14644 #[inline]
14646 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
14647 self.cfg
14648 .handlers
14649 .iter()
14650 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
14651 }
14652
14653 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
14654 pkt.endpoints()
14655 .iter()
14656 .copied()
14657 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
14658 }
14659
14660 fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
14667 self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
14668 }
14669
14670 fn call_handler_with_retries(
14676 &self,
14677 dest: DataEndpoint,
14678 handler: &EndpointHandler,
14679 data: Option<&[u8]>,
14680 pkt_for_ctx: Option<&Packet>,
14681 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
14682 called_from_queue: bool,
14683 ) -> TelemetryResult<()> {
14684 let owned_tmp: Option<RouterItem>;
14685
14686 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
14687 (Some(d), _) => {
14688 owned_tmp = Some(RouterItem::Packed(crate::shared_bytes::convert(
14689 Arc::<[u8]>::from(d),
14690 )));
14691 owned_tmp.as_ref().unwrap()
14692 }
14693 (None, Some(pkt)) => {
14694 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
14695 owned_tmp.as_ref().unwrap()
14696 }
14697 (None, None) => {
14698 debug_assert!(
14699 false,
14700 "call_handler_with_retries called without data or packet context"
14701 );
14702 return Ok(());
14703 }
14704 };
14705
14706 match (&handler.handler, data) {
14707 (EndpointHandlerFn::Packet(f), _) => {
14708 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
14709 with_retries(
14710 self,
14711 dest,
14712 item_for_ctx,
14713 pkt_for_ctx,
14714 env_for_ctx,
14715 called_from_queue,
14716 || f(pkt),
14717 )
14718 }
14719
14720 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
14721 self,
14722 dest,
14723 item_for_ctx,
14724 pkt_for_ctx,
14725 env_for_ctx,
14726 called_from_queue,
14727 || f(bytes),
14728 ),
14729
14730 (EndpointHandlerFn::Packed(_), None) => Ok(()),
14731 }
14732 }
14733
14734 fn handle_callback_error_from_env(
14739 &self,
14740 env: &wire_format::TelemetryEnvelope,
14741 dest: Option<DataEndpoint>,
14742 e: TelemetryError,
14743 called_from_queue: bool,
14744 ) -> TelemetryResult<()> {
14745 if matches!(&e, TelemetryError::Io("memory pressure")) {
14748 return Err(e);
14749 }
14750
14751 let mut recipients: Vec<DataEndpoint> = env
14752 .endpoints
14753 .iter()
14754 .copied()
14755 .filter(|&ep| self.cfg.is_local_endpoint(ep))
14756 .collect();
14757 crate::small_sort::sort(&mut recipients);
14758 recipients.dedup();
14759 if let Some(failed) = dest {
14760 recipients.retain(|&ep| ep != failed);
14761 }
14762
14763 let device = self.sender_arc();
14764 let error_msg = format!(
14765 "Handler for endpoint {:?} failed on device {:?}: {:?}",
14766 dest, device, e
14767 );
14768
14769 if recipients.is_empty() && dest.is_none() {
14770 fallback_stdout(&error_msg);
14771 return Ok(());
14772 }
14773
14774 if recipients.is_empty() {
14775 recipients = env.endpoints.to_vec();
14776 crate::small_sort::sort(&mut recipients);
14777 recipients.dedup();
14778 if let Some(failed) = dest {
14779 recipients.retain(|&ep| ep != failed);
14780 }
14781 }
14782
14783 if recipients.is_empty() {
14784 fallback_stdout(&error_msg);
14785 return Ok(());
14786 }
14787
14788 let payload = make_error_payload(&error_msg);
14789
14790 let error_pkt = Packet::new(
14791 DataType::TelemetryError,
14792 &recipients,
14793 &env.sender.clone(),
14794 env.timestamp_ms,
14795 crate::shared_bytes::convert(payload),
14796 )?;
14797 self.emit_internal_tx(
14798 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
14799 false,
14800 called_from_queue,
14801 )
14802 }
14803
14804 fn handle_internal_reliable_packet(
14805 &self,
14806 pkt: &Packet,
14807 src: Option<RouterSideId>,
14808 called_from_queue: bool,
14809 ) -> TelemetryResult<bool> {
14810 if !matches!(
14811 pkt.data_type(),
14812 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
14813 ) {
14814 return Ok(false);
14815 }
14816
14817 let Some(src) = src else {
14818 return Ok(false);
14819 };
14820
14821 if Self::is_end_to_end_ack_packet(pkt)
14822 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
14823 {
14824 let alias = {
14825 let st = self.state.lock();
14826 st.reliable_return_routes
14827 .get(&packet_id)
14828 .filter(|route| route.upstream_contract.is_some())
14829 .cloned()
14830 };
14831 if let Some(alias) = alias {
14832 let (upstream_id, publisher_hash) = alias.upstream_contract.expect("checked alias");
14833 #[cfg(feature = "std")]
14834 if reliable_trace_enabled() {
14835 eprintln!(
14836 "[sedsnet-reliable] proxy receipt id={packet_id:#x} upstream={upstream_id:#x} owner={:?}",
14837 Self::end_to_end_ack_sender_hash(pkt)
14838 );
14839 }
14840 let completed = {
14841 let mut st = self.state.lock();
14842 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
14843 if let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
14844 sent.pending_destinations.remove(&owner);
14845 }
14846 if !sent.pending_destinations.is_empty() {
14847 return Ok(true);
14848 }
14849 st.end_to_end_reliable_tx.remove(&packet_id);
14850 true
14851 } else {
14852 return Ok(true);
14855 }
14856 };
14857 if completed {
14858 let mut st = self.state.lock();
14859 if Self::remember_reliable_return_route_locked(&mut st, upstream_id) {
14860 st.reliable_return_routes.insert(
14861 upstream_id,
14862 ReliableReturnRouteState {
14863 side: alias.side,
14864 locally_delivered: false,
14865 last_replay_ms: alias.last_replay_ms,
14866 upstream_contract: None,
14867 },
14868 );
14869 }
14870 }
14871 self.queue_end_to_end_reliable_ack_for_sender_hash(
14872 upstream_id,
14873 publisher_hash,
14874 called_from_queue,
14875 )?;
14876 self.release_ordered_successors()?;
14877 return Ok(true);
14878 }
14879 let targets_local_sender = pkt.wire_target_senders().is_empty()
14886 || pkt
14887 .wire_target_senders()
14888 .contains(&Self::sender_hash(self.sender_arc().as_ref()));
14889 let mut st = self.state.lock();
14890 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
14891 if sent.waiting_for_predecessor {
14892 return Ok(targets_local_sender);
14893 }
14894 if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
14895 let _removed = sent.pending_destinations.remove(&sender_hash);
14896 #[cfg(feature = "std")]
14897 if reliable_trace_enabled() {
14898 eprintln!(
14899 "[sedsnet-reliable] receipt time={} id={packet_id:#x} sender={sender_hash:#x} matched={} remaining={:?}",
14900 self.clock.now_ms(),
14901 _removed.is_some(),
14902 sent.pending_destinations
14903 );
14904 }
14905 if sent.pending_destinations.is_empty() {
14906 st.end_to_end_reliable_tx.remove(&packet_id);
14907 }
14908 drop(st);
14909 self.release_ordered_successors()?;
14910 return Ok(targets_local_sender);
14911 }
14912 st.end_to_end_reliable_tx.remove(&packet_id);
14913 drop(st);
14914 self.release_ordered_successors()?;
14915 return Ok(targets_local_sender);
14916 }
14917 return Ok(targets_local_sender);
14920 }
14921
14922 let vals = pkt.data_as_u32()?;
14923 if vals.len() != 2 {
14924 return Err(TelemetryError::Unpack("bad reliable control payload"));
14925 }
14926 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
14927 let seq = vals[1];
14928
14929 match pkt.data_type() {
14930 DataType::ReliableAck => {
14931 self.handle_reliable_ack(src, ty, seq);
14932 Ok(true)
14933 }
14934 DataType::ReliablePartialAck => {
14935 self.handle_reliable_partial_ack(src, ty, seq);
14936 Ok(true)
14937 }
14938 DataType::ReliablePacketRequest => {
14939 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
14940 Ok(true)
14941 }
14942 _ => Ok(false),
14943 }
14944 }
14945
14946 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
14952 let packed_frame = match &item.data {
14956 RouterItem::Packed(bytes) => Some(wire_format::peek_routing_frame_info(bytes.as_ref())),
14957 RouterItem::Packet(_) => None,
14958 };
14959 if crate::memory_admission::supports_work_kinds()
14960 && let (Some(side), RouterItem::Packed(bytes)) = (item.src, &item.data)
14961 {
14962 self.admit_received_frame_with_view(
14963 side,
14964 bytes,
14965 packed_frame.as_ref().and_then(|frame| frame.as_ref().ok()),
14966 )?;
14967 }
14968 let packed_routing = packed_frame
14969 .as_ref()
14970 .and_then(|frame| frame.as_ref().ok())
14971 .map(|frame| &frame.envelope);
14972 if let Some(src) = item.src {
14973 self.ensure_side_ingress_enabled(src)?;
14974 match &item.data {
14975 RouterItem::Packet(pkt) => {
14976 let bytes = wire_format::packet_wire_size(pkt);
14977 self.note_side_rx(src, pkt.data_type(), bytes, true);
14978 }
14979 RouterItem::Packed(bytes) => {
14980 if let Some(env) = packed_routing.as_ref() {
14981 self.note_side_rx(src, env.ty, bytes.len(), true);
14982 }
14983 }
14984 }
14985 match &item.data {
14986 RouterItem::Packet(pkt) => {
14987 if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
14988 && !is_internal_control_type(pkt.data_type())
14989 && !self.note_reliable_return_route(src, pkt.packet_id())
14990 {
14991 return Ok(());
14992 }
14993 }
14994 RouterItem::Packed(bytes) => {
14995 if let Some(env) = packed_routing.as_ref()
14996 && (is_reliable_type(env.ty) || env.target_senders().len() != 0)
14997 && !is_internal_control_type(env.ty)
14998 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
14999 && !self.note_reliable_return_route(src, packet_id)
15000 {
15001 return Ok(());
15002 }
15003 }
15004 }
15005 }
15006 match &item.data {
15007 RouterItem::Packet(pkt) => {
15008 if !is_internal_control_type(pkt.data_type()) {
15009 self.remember_managed_variable_packet(pkt)?;
15010 }
15011 }
15012 RouterItem::Packed(bytes) => {
15013 if let Some(env) = packed_routing.as_ref()
15014 && !is_internal_control_type(env.ty)
15015 && self.is_managed_variable_type(env.ty)
15016 {
15017 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15018 pkt.validate()?;
15019 self.remember_managed_variable_packet(&pkt)?;
15020 }
15021 }
15022 }
15023 let mut released_buffered: Vec<crate::SharedBytes> = Vec::new();
15024 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
15025 let (_opts, handler_is_packed, hop_reliable_enabled) = {
15026 let st = self.state.lock();
15027 let side_ref = Self::side_ref(&st, src)?;
15028 let opts = side_ref.opts.clone();
15029 (
15030 opts.clone(),
15031 matches!(
15032 side_ref.tx_handler,
15033 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
15034 ),
15035 opts.reliable_enabled
15036 && self.cfg.reliable_enabled()
15037 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
15038 )
15039 };
15040
15041 if hop_reliable_enabled && handler_is_packed {
15042 let frame = match packed_frame
15043 .as_ref()
15044 .expect("packed frame")
15045 .as_ref()
15046 .map_err(Clone::clone)
15047 {
15048 Ok(frame) => frame,
15049 Err(e) => {
15050 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
15051 if let Ok(frame) =
15052 wire_format::peek_frame_info_unchecked(bytes.as_ref())
15053 && is_reliable_type(frame.envelope.ty)
15054 && let Some(hdr) = frame.reliable
15055 {
15056 let unordered =
15057 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
15058 let unsequenced =
15059 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
15060
15061 if !unsequenced {
15062 let requested = if unordered {
15063 hdr.seq
15064 } else {
15065 let mut st = self.state.lock();
15066 let rx_state = self.reliable_rx_state_mut(
15067 &mut st,
15068 src,
15069 frame.envelope.ty,
15070 );
15071 rx_state.expected_seq.min(hdr.seq)
15072 };
15073 self.queue_reliable_packet_request(
15074 src,
15075 frame.envelope.ty,
15076 requested,
15077 called_from_queue,
15078 )?;
15079 }
15080 }
15081 return Ok(());
15082 }
15083 return Err(e);
15084 }
15085 };
15086 if is_reliable_type(frame.envelope.ty)
15087 && let Some(hdr) = frame.reliable
15088 {
15089 if frame.ack_only() {
15090 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
15091 return Ok(());
15092 }
15093 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
15094 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
15095
15096 if !unsequenced {
15097 if unordered {
15098 self.queue_reliable_ack(
15099 src,
15100 frame.envelope.ty,
15101 hdr.seq,
15102 called_from_queue,
15103 )?;
15104 } else {
15105 let mut release: Vec<crate::SharedBytes> = Vec::new();
15106 let mut last_delivered = None;
15107 let mut ack_old = None;
15108 let mut request_missing = None;
15109 let mut partial_ack = None;
15110 {
15111 let mut st = self.state.lock();
15112 let rx_state =
15113 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
15114 let expected_seq = rx_state.expected_seq;
15115 if hdr.seq < expected_seq {
15116 ack_old = Some(expected_seq.saturating_sub(1));
15117 } else if hdr.seq > expected_seq {
15118 request_missing = Some(expected_seq);
15119 partial_ack = Some(hdr.seq);
15120 st.buffer_reliable_rx(
15121 src,
15122 frame.envelope.ty,
15123 hdr.seq,
15124 bytes.clone(),
15125 )?;
15126 } else {
15127 release.push(bytes.clone());
15128 last_delivered = Some(hdr.seq);
15129 let mut next_expected = hdr.seq.wrapping_add(1);
15130 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
15131 release.push(buf);
15132 last_delivered = Some(next_expected);
15133 let next = next_expected.wrapping_add(1);
15134 next_expected = if next == 0 { 1 } else { next };
15135 }
15136 rx_state.expected_seq = next_expected;
15137 }
15138 }
15139
15140 if let Some(ack_seq) = ack_old {
15141 self.queue_reliable_ack(
15142 src,
15143 frame.envelope.ty,
15144 ack_seq,
15145 called_from_queue,
15146 )?;
15147 return Ok(());
15148 }
15149 if let Some(request_seq) = request_missing {
15150 if let Some(partial_seq) = partial_ack {
15151 self.queue_reliable_partial_ack(
15152 src,
15153 frame.envelope.ty,
15154 partial_seq,
15155 called_from_queue,
15156 )?;
15157 }
15158 self.queue_reliable_packet_request(
15159 src,
15160 frame.envelope.ty,
15161 request_seq,
15162 called_from_queue,
15163 )?;
15164 return Ok(());
15165 }
15166
15167 if let Some(ack_seq) = last_delivered {
15168 self.queue_reliable_ack(
15169 src,
15170 frame.envelope.ty,
15171 ack_seq,
15172 called_from_queue,
15173 )?;
15174 }
15175
15176 released_buffered.extend(release.into_iter().skip(1));
15177 }
15178 }
15179 }
15180 } else {
15181 match packed_frame
15182 .as_ref()
15183 .expect("packed frame")
15184 .as_ref()
15185 .map_err(Clone::clone)
15186 {
15187 Ok(frame) => {
15188 if frame.ack_only() {
15189 return Ok(());
15190 }
15191 }
15192 Err(e) => {
15193 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
15194 return Ok(());
15195 }
15196 return Err(e);
15197 }
15198 }
15199 }
15200 }
15201
15202 if self.is_duplicate_pkt(&item.data)? {
15203 if (item.src.is_some() && self.item_targets_another_sender(&item.data)?)
15206 || (item.src.is_some() && self.should_route_remote(&item.data, item.src)? && {
15207 let (endpoints, _) = self.item_route_info(&item.data)?;
15208 self.item_targets_router_as_intermediate_hop(&item.data, &endpoints)?
15209 })
15210 {
15211 let id = Self::get_hash(&item.data);
15212 let replay = {
15213 let mut st = self.state.lock();
15214 st.reliable_return_routes.get_mut(&id).is_some_and(|route| {
15215 let now = self.clock.now_ms();
15216 if now.saturating_sub(route.last_replay_ms)
15217 < runtime_reliable_retransmit_ms()
15218 {
15219 return false;
15220 }
15221 route.last_replay_ms = now;
15222 true
15223 })
15224 };
15225 if replay {
15226 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
15227 }
15228 }
15229 if item.src.is_some() {
15230 let local_sender = self.sender_arc();
15231 match &item.data {
15232 RouterItem::Packet(pkt)
15233 if (is_reliable_type(pkt.data_type())
15234 || !pkt.wire_target_senders().is_empty())
15235 && pkt.sender() != local_sender.as_ref()
15236 && self.item_targets_local_sender(&item.data)?
15237 && self.packet_has_local_delivery(pkt)
15238 && !(self.should_route_remote(&item.data, item.src)?
15239 && self.item_targets_router_as_intermediate_hop(
15240 &item.data,
15241 pkt.endpoints(),
15242 )?) =>
15243 {
15244 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
15245 }
15246 RouterItem::Packed(bytes) => {
15247 if let Ok(pkt) = self.unpack_packet_for_role(bytes.as_ref())
15248 && (is_reliable_type(pkt.data_type())
15249 || !pkt.wire_target_senders().is_empty())
15250 && pkt.sender() != local_sender.as_ref()
15251 && self.item_targets_local_sender(&item.data)?
15252 && self.packet_has_local_delivery(&pkt)
15253 && !(self.should_route_remote(&item.data, item.src)?
15254 && self.item_targets_router_as_intermediate_hop(
15255 &item.data,
15256 pkt.endpoints(),
15257 )?)
15258 {
15259 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
15260 .unwrap_or_else(|_| pkt.packet_id());
15261 self.queue_end_to_end_reliable_ack_for_packet_id(
15262 packet_id,
15263 pkt.sender(),
15264 called_from_queue,
15265 )?;
15266 }
15267 }
15268 _ => {}
15269 }
15270 }
15271 return Ok(());
15272 }
15273
15274 self.dispatch_rx_data(item, called_from_queue)?;
15275
15276 for release_bytes in released_buffered {
15277 let release_data = RouterItem::Packed(release_bytes.clone());
15278 if self.is_duplicate_pkt(&release_data)? {
15279 continue;
15280 }
15281 let release_item = RouterRxItem {
15282 src: item.src,
15283 priority: Self::router_item_priority(&release_data)?,
15284 data: release_data,
15285 };
15286 self.dispatch_rx_data(&release_item, called_from_queue)?;
15287 }
15288
15289 Ok(())
15290 }
15291
15292 fn dispatch_rx_data(
15293 &self,
15294 item: &RouterRxItem,
15295 called_from_queue: bool,
15296 ) -> TelemetryResult<()> {
15297 match &item.data {
15298 RouterItem::Packet(pkt) => {
15299 pkt.validate()?;
15300 #[cfg(feature = "discovery")]
15301 if let Some(side) = item.src {
15302 self.refresh_peer_liveness(side, sender_address_u32(pkt.sender()));
15303 }
15304
15305 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
15306 return Ok(());
15307 }
15308
15309 if pkt.data_type() == DataType::P2pMessage {
15310 if self.item_targets_local_sender(&item.data)? {
15311 self.dispatch_p2p_packet(pkt)?;
15312 if item.src.is_some() {
15313 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
15314 }
15315 }
15316 if self.should_route_remote(&item.data, item.src)? {
15317 self.relay_send(
15318 RouterItem::Packet(pkt.to_owned()),
15319 item.src,
15320 called_from_queue,
15321 )?;
15322 }
15323 return Ok(());
15324 }
15325
15326 #[cfg(feature = "timesync")]
15327 if matches!(
15328 pkt.data_type(),
15329 DataType::TimeSyncAnnounce
15330 | DataType::TimeSyncRequest
15331 | DataType::TimeSyncResponse
15332 ) {
15333 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
15334 return Ok(());
15335 }
15336
15337 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
15338 if !discovery_is_hop_local_advertisement(pkt.data_type())
15343 && self.should_route_remote(&item.data, item.src)?
15344 {
15345 self.relay_send(
15346 RouterItem::Packet(pkt.to_owned()),
15347 item.src,
15348 called_from_queue,
15349 )?;
15350 }
15351 return Ok(());
15352 }
15353
15354 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
15355 crate::small_sort::sort(&mut eps);
15356 eps.dedup();
15357 let had_local_handler = eps.iter().copied().any(|ep| {
15358 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
15359 });
15360 let had_local_delivery =
15361 had_local_handler || self.is_managed_variable_type(pkt.data_type());
15362
15363 let has_remote = self.should_route_remote(&item.data, item.src)?;
15364 let targets_local = self.item_targets_local_sender(&item.data)?;
15365
15366 let has_packed_local = eps
15367 .iter()
15368 .copied()
15369 .any(|ep| self.endpoint_has_packed_handler(ep));
15370 let bytes_opt = if has_packed_local {
15371 Some(wire_format::pack_packet(pkt))
15372 } else {
15373 None
15374 };
15375
15376 let mut local_delivery_error = None;
15377 if targets_local {
15378 for dest in eps {
15379 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
15380 let result = match (&h.handler, &bytes_opt) {
15381 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
15382 .call_handler_with_retries(
15383 dest,
15384 h,
15385 Some(bytes.as_ref()),
15386 Some(pkt),
15387 None,
15388 called_from_queue,
15389 ),
15390 (EndpointHandlerFn::Packed(_), None) => {
15391 let bytes = wire_format::pack_packet(pkt);
15392 self.call_handler_with_retries(
15393 dest,
15394 h,
15395 Some(bytes.as_ref()),
15396 Some(pkt),
15397 None,
15398 called_from_queue,
15399 )
15400 }
15401 (EndpointHandlerFn::Packet(_), _) => self
15402 .call_handler_with_retries(
15403 dest,
15404 h,
15405 None,
15406 Some(pkt),
15407 None,
15408 called_from_queue,
15409 ),
15410 };
15411 if let Err(error) = result {
15412 if let Some(src) = item.src {
15413 self.note_side_local_handler_failure(
15414 src,
15415 pkt.data_type(),
15416 runtime_max_handler_retries(),
15417 );
15418 }
15419 local_delivery_error.get_or_insert(error);
15420 }
15421 }
15422 }
15423 }
15424
15425 if let Some(src) = item.src
15426 && had_local_delivery
15427 && local_delivery_error.is_none()
15428 && targets_local
15429 {
15430 self.note_side_local_delivery(src, pkt.data_type());
15431 }
15432
15433 if item.src.is_some()
15434 && had_local_delivery
15435 && local_delivery_error.is_none()
15436 && targets_local
15437 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
15438 && !(has_remote
15439 && self
15440 .item_targets_router_as_intermediate_hop(&item.data, pkt.endpoints())?)
15441 {
15442 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
15443 }
15444
15445 if has_remote {
15446 let relay_item = RouterItem::Packet(pkt.to_owned());
15447 self.relay_send(relay_item, item.src, called_from_queue)?;
15448 }
15449
15450 local_delivery_error.map_or(Ok(()), Err)
15454 }
15455 RouterItem::Packed(bytes) => {
15456 let routing = wire_format::peek_routing_envelope(bytes.as_ref())?;
15460 if !is_internal_control_type(routing.ty)
15461 && !routing.is_reliable
15462 && routing.target_senders().len() == 0
15463 && !self.is_managed_variable_type(routing.ty)
15464 && !routing.endpoints().any(|ep| {
15465 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
15466 })
15467 {
15468 return self.relay_send(item.data.clone(), item.src, called_from_queue);
15472 }
15473
15474 let env = wire_format::peek_envelope(bytes.as_ref())?;
15475
15476 if matches!(
15477 env.ty,
15478 DataType::ReliableAck
15479 | DataType::ReliablePartialAck
15480 | DataType::ReliablePacketRequest
15481 ) {
15482 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15483 pkt.validate()?;
15484 let handled =
15485 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
15486 if !handled && self.should_route_remote(&item.data, item.src)? {
15489 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
15490 }
15491 return Ok(());
15492 }
15493
15494 if env.ty == DataType::P2pMessage {
15495 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15496 pkt.validate()?;
15497 if self.item_targets_local_sender(&item.data)? {
15498 self.dispatch_p2p_packet(&pkt)?;
15499 if item.src.is_some() {
15500 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
15501 .unwrap_or_else(|_| pkt.packet_id());
15502 self.queue_end_to_end_reliable_ack_for_packet_id(
15503 packet_id,
15504 pkt.sender(),
15505 called_from_queue,
15506 )?;
15507 }
15508 }
15509 if self.should_route_remote(&item.data, item.src)? {
15510 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
15511 }
15512 return Ok(());
15513 }
15514
15515 #[cfg(feature = "timesync")]
15516 if matches!(
15517 env.ty,
15518 DataType::TimeSyncAnnounce
15519 | DataType::TimeSyncRequest
15520 | DataType::TimeSyncResponse
15521 ) {
15522 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15523 pkt.validate()?;
15524 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
15525 return Ok(());
15526 }
15527
15528 #[cfg(feature = "discovery")]
15529 if discovery::is_discovery_type(env.ty) {
15530 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15531 pkt.validate()?;
15532 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
15533 if !discovery_is_hop_local_advertisement(env.ty)
15534 && self.should_route_remote(&item.data, item.src)?
15535 {
15536 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
15537 }
15538 return Ok(());
15539 }
15540
15541 let any_packet_needed = env
15542 .endpoints
15543 .iter()
15544 .copied()
15545 .any(|ep| self.endpoint_has_packet_handler(ep));
15546
15547 let mut pkt_opt = if any_packet_needed {
15548 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15549 pkt.validate()?;
15550 Some(pkt)
15551 } else {
15552 None
15553 };
15554
15555 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
15556 crate::small_sort::sort(&mut eps);
15557 eps.dedup();
15558 let had_local_handler = eps.iter().copied().any(|ep| {
15559 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
15560 });
15561 let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
15562
15563 let has_remote = self.should_route_remote(&item.data, item.src)?;
15564 let targets_local = self.item_targets_local_sender(&item.data)?;
15565
15566 let mut local_delivery_error = None;
15567 if targets_local {
15568 for dest in eps {
15569 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
15570 let result = match &h.handler {
15571 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
15572 dest,
15573 h,
15574 Some(bytes.as_ref()),
15575 pkt_opt.as_ref(),
15576 Some(&env),
15577 called_from_queue,
15578 ),
15579 EndpointHandlerFn::Packet(_) => {
15580 if pkt_opt.is_none() {
15581 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15582 pkt.validate()?;
15583 pkt_opt = Some(pkt);
15584 }
15585 let pkt_ref = pkt_opt.as_ref().expect("just set");
15586 self.call_handler_with_retries(
15587 dest,
15588 h,
15589 None,
15590 Some(pkt_ref),
15591 Some(&env),
15592 called_from_queue,
15593 )
15594 }
15595 };
15596 if let Err(error) = result {
15597 if let Some(src) = item.src {
15598 self.note_side_local_handler_failure(
15599 src,
15600 env.ty,
15601 runtime_max_handler_retries(),
15602 );
15603 }
15604 local_delivery_error.get_or_insert(error);
15605 }
15606 }
15607 }
15608 }
15609
15610 if item.src.is_some()
15611 && had_local_delivery
15612 && local_delivery_error.is_none()
15613 && targets_local
15614 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
15615 && !(has_remote
15616 && self
15617 .item_targets_router_as_intermediate_hop(&item.data, &env.endpoints)?)
15618 {
15619 let packet_id =
15620 wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
15621 pkt_opt
15622 .as_ref()
15623 .map(Packet::packet_id)
15624 .ok_or(TelemetryError::Unpack("missing packet id"))
15625 })?;
15626 self.queue_end_to_end_reliable_ack_for_packet_id(
15627 packet_id,
15628 &env.sender,
15629 called_from_queue,
15630 )?;
15631 }
15632
15633 if has_remote {
15634 let relay_item = match pkt_opt {
15635 Some(ref p) => RouterItem::Packet(p.clone()),
15636 None => RouterItem::Packed(bytes.clone()),
15637 };
15638 self.relay_send(relay_item, item.src, called_from_queue)?;
15639 }
15640
15641 local_delivery_error.map_or(Ok(()), Err)
15645 }
15646 }
15647 }
15648
15649 fn dispatch_local_for_item(
15650 &self,
15651 item: &RouterItem,
15652 called_from_queue: bool,
15653 ) -> TelemetryResult<()> {
15654 match item {
15655 RouterItem::Packet(pkt) => {
15656 pkt.validate()?;
15657 if is_internal_control_type(pkt.data_type()) {
15658 return Ok(());
15659 }
15660 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
15661 if !self.item_targets_local_sender(item)? {
15662 return Ok(());
15663 }
15664
15665 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
15666 crate::small_sort::sort(&mut eps);
15667 eps.dedup();
15668
15669 let has_packed_local = eps
15670 .iter()
15671 .copied()
15672 .any(|ep| self.endpoint_has_packed_handler(ep));
15673 let bytes_opt = if has_packed_local {
15674 Some(wire_format::pack_packet(pkt))
15675 } else {
15676 None
15677 };
15678
15679 for dest in eps {
15680 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
15681 match (&h.handler, &bytes_opt) {
15682 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
15683 self.call_handler_with_retries(
15684 dest,
15685 h,
15686 Some(bytes.as_ref()),
15687 Some(pkt),
15688 None,
15689 called_from_queue,
15690 )?;
15691 }
15692 (EndpointHandlerFn::Packed(_), None) => {
15693 let bytes = wire_format::pack_packet(pkt);
15694 self.call_handler_with_retries(
15695 dest,
15696 h,
15697 Some(bytes.as_ref()),
15698 Some(pkt),
15699 None,
15700 called_from_queue,
15701 )?;
15702 }
15703 (EndpointHandlerFn::Packet(_), _) => {
15704 self.call_handler_with_retries(
15705 dest,
15706 h,
15707 None,
15708 Some(pkt),
15709 None,
15710 called_from_queue,
15711 )?;
15712 }
15713 }
15714 }
15715 }
15716 }
15717 RouterItem::Packed(bytes) => {
15718 let env = wire_format::peek_envelope(bytes.as_ref())?;
15719 if is_internal_control_type(env.ty) {
15720 return Ok(());
15721 }
15722 self.ensure_e2e_policy_supported_for_type(env.ty)?;
15723 if !self.item_targets_local_sender(item)? {
15724 return Ok(());
15725 }
15726
15727 let any_packet_needed = env
15728 .endpoints
15729 .iter()
15730 .copied()
15731 .any(|ep| self.endpoint_has_packet_handler(ep));
15732
15733 let mut pkt_opt = if any_packet_needed {
15734 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15735 pkt.validate()?;
15736 Some(pkt)
15737 } else {
15738 None
15739 };
15740
15741 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
15742 crate::small_sort::sort(&mut eps);
15743 eps.dedup();
15744
15745 for dest in eps {
15746 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
15747 match &h.handler {
15748 EndpointHandlerFn::Packed(_) => {
15749 self.call_handler_with_retries(
15750 dest,
15751 h,
15752 Some(bytes.as_ref()),
15753 pkt_opt.as_ref(),
15754 Some(&env),
15755 called_from_queue,
15756 )?;
15757 }
15758 EndpointHandlerFn::Packet(_) => {
15759 if pkt_opt.is_none() {
15760 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
15761 pkt.validate()?;
15762 pkt_opt = Some(pkt);
15763 }
15764 let pkt_ref = pkt_opt.as_ref().expect("just set");
15765 self.call_handler_with_retries(
15766 dest,
15767 h,
15768 None,
15769 Some(pkt_ref),
15770 Some(&env),
15771 called_from_queue,
15772 )?;
15773 }
15774 }
15775 }
15776 }
15777 }
15778 }
15779
15780 Ok(())
15781 }
15782
15783 fn tx_item_impl(
15789 &self,
15790 item: RouterTxItem,
15791 ignore_local: bool,
15792 called_from_queue: bool,
15793 ) -> TelemetryResult<()> {
15794 self.tx_item_impl_with_retry(
15795 item,
15796 ignore_local,
15797 called_from_queue,
15798 &mut DispatchContext::default(),
15799 )
15800 }
15801
15802 fn tx_item_impl_with_retry(
15803 &self,
15804 item: RouterTxItem,
15805 ignore_local: bool,
15806 called_from_queue: bool,
15807 context: &mut DispatchContext,
15808 ) -> TelemetryResult<()> {
15809 match item {
15810 RouterTxItem::Broadcast(data) => {
15811 let claim = context.claim;
15812 let deferred_since = context.since;
15813 let ty = Self::item_data_type(&data)?;
15814
15815 if let Some(since) = deferred_since
15816 && let Err(error) = self.expire_deferred_original(
15817 ty, since, claim.is_some_and(|claim| claim.ordered))
15818 {
15819 context.terminal = true;
15820 return Err(error);
15821 }
15822 if !called_from_queue
15823 && !ignore_local
15824 && matches!(reliable_mode(ty), crate::ReliableMode::Ordered)
15825 && self
15826 .state
15827 .lock()
15828 .transmit_queue
15829 .iter()
15830 .any(|queued| RouterInner::ordered_original_type(queued) == Some(ty))
15831 {
15832 return self.tx_queue_item(RouterTxItem::Broadcast(data));
15833 }
15834
15835 self.ensure_e2e_policy_supported_for_type(ty)?;
15836 if let RouterItem::Packet(pkt) = &data
15837 && !is_internal_control_type(pkt.data_type())
15838 {
15839 self.remember_managed_variable_packet(pkt)?;
15840 }
15841 #[cfg(feature = "discovery")]
15842 let is_discovery = discovery::is_discovery_type(ty);
15843 if !ignore_local {
15844 if self.is_duplicate_pkt(&data)? {
15845 return Ok(());
15846 }
15847 #[cfg(feature = "discovery")]
15848 if !is_discovery && !is_internal_control_type(ty)
15849 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
15850 {
15851 if claim.is_some() {
15853 context.terminal = true;
15854 return Err(TelemetryError::HandlerError("accepted local delivery failed"));
15855 }
15856 return Err(error);
15857 }
15858 #[cfg(not(feature = "discovery"))]
15859 if !is_internal_control_type(ty)
15860 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
15861 {
15862 if claim.is_some() {
15864 context.terminal = true;
15865 return Err(TelemetryError::HandlerError("accepted local delivery failed"));
15866 }
15867 return Err(error);
15868 }
15869 }
15870
15871 if !ignore_local && !self.mark_claim_local_delivered(claim) {
15872 return Ok(());
15874 }
15875
15876 if !self.claim_schema_is_current(ty, claim)? {
15877 return Ok(());
15878 }
15879
15880 match &data {
15881 RouterItem::Packet(packet) => packet.validate()?,
15882 RouterItem::Packed(_) => {}
15884 }
15885 let freezes_contract = !is_internal_control_type(ty)
15888 && self.item_requires_end_to_end_tracking(&data, ty)?;
15889 #[cfg(feature = "discovery")]
15895 let cached_plan = if freezes_contract {
15896 None
15897 } else {
15898 Some(self.remote_side_plan(&data, None)?)
15899 };
15900 #[cfg(feature = "discovery")]
15901 let send_remote = match &cached_plan {
15902 Some(RemoteSidePlan::Target(sides)) => !sides.is_empty(),
15903 None => self.should_route_remote(&data, None)?,
15904 };
15905 #[cfg(not(feature = "discovery"))]
15906 let send_remote = self.should_route_remote(&data, None)?;
15907
15908 if !send_remote {
15909 let (endpoints, ty) = self.item_route_info(&data)?;
15910 if !is_internal_control_type(ty)
15911 && is_reliable_type(ty)
15912 && has_nonlocal_endpoint(&endpoints, &self.cfg)
15913 && !self
15914 .state
15915 .lock()
15916 .managed_variable_types
15917 .contains(&ty.as_u32())
15918 {
15919 self.remove_pkt_id(&data);
15923 if called_from_queue {
15924 return self.defer_accepted_original(context);
15925 }
15926 return Err(TelemetryError::Io("no discovered route"));
15927 }
15928 return Ok(());
15929 }
15930 #[cfg(feature = "discovery")]
15931 let mut data = data;
15932 if freezes_contract {
15933 #[cfg(feature = "discovery")]
15934 {
15935 let pending = {
15936 let st = self.state.lock();
15937 let mut pending = self.expected_end_to_end_destinations_locked(
15938 &st, &data, None, false, false,
15939 )?;
15940 self.filter_trackable_end_to_end_destinations_locked(
15941 &st,
15942 ty,
15943 &mut pending,
15944 false,
15945 );
15946 pending
15947 };
15948 if !pending.is_empty() {
15949 let targets: Vec<u64> = pending.keys().copied().collect();
15951 data = self.attach_wire_contract_to_item(data, &targets)?;
15952 match self.register_end_to_end_reliable_tx(&data, Some(pending)) {
15953 Ok(ready) => {
15954 context.owner_transferred = true;
15955 if !ready {
15956 return Ok(());
15957 }
15958 }
15959 Err(error) => {
15960 self.remove_pkt_id(&data);
15963 if called_from_queue
15964 && matches!(
15965 error,
15966 TelemetryError::Io("reliable window full")
15967 )
15968 {
15969 if let Some(claim) = claim {
15973 self.replace_claim_payload(claim, data)?;
15974 }
15975 self.defer_accepted_original(context)?;
15976 return Ok(());
15977 }
15978 return Err(error);
15979 }
15980 }
15981 }
15982 }
15983 }
15984 #[cfg(feature = "discovery")]
15985 let plan = match cached_plan {
15986 Some(plan) => plan,
15987 None => self.remote_side_plan(&data, None)?,
15988 };
15989 #[cfg(not(feature = "discovery"))]
15990 let plan = self.remote_side_plan(&data, None)?;
15991 let RemoteSidePlan::Target(sides) = plan;
15992 for side in sides.iter().copied() {
15993 #[cfg(feature = "compression")]
15994 let mut side_data = data.clone();
15995 #[cfg(feature = "compression")]
15996 let (send_result, mut reservation) =
15997 self.send_queued_to_side(side, &mut side_data, false);
15998 #[cfg(not(feature = "compression"))]
15999 let send_result = self.send_reliable_to_side(side, data.clone(), false);
16000 if let Err(e) = send_result {
16001 if Self::is_transient_tx_error(&e) {
16002 #[cfg(feature = "compression")]
16003 self.retry_codec_owner(
16004 RouterTxItem::ToSide {
16005 src: None,
16006 dst: side,
16007 data: side_data,
16008 },
16009 true,
16010 &mut reservation,
16011 )?;
16012 #[cfg(not(feature = "compression"))]
16013 self.tx_queue_item_for_retry(
16014 RouterTxItem::ToSide {
16015 src: None,
16016 dst: side,
16017 data: data.clone(),
16018 },
16019 true,
16020 )?;
16021 continue;
16022 }
16023 match &data {
16024 RouterItem::Packet(pkt) => {
16025 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
16026 }
16027 RouterItem::Packed(bytes) => {
16028 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
16029 let _ = self.handle_callback_error_from_env(
16030 &env,
16031 None,
16032 e,
16033 called_from_queue,
16034 );
16035 }
16036 }
16037 }
16038 return Err(TelemetryError::HandlerError("tx handler failed"));
16039 }
16040 }
16041 }
16042 RouterTxItem::ToSide { src, dst, data } => {
16043 #[cfg(feature = "compression")]
16044 let mut data = data;
16045 let ty = Self::item_data_type(&data)?;
16046 self.ensure_e2e_policy_supported_for_type(ty)?;
16047 if let RouterItem::Packet(pkt) = &data
16048 && !is_internal_control_type(pkt.data_type())
16049 {
16050 self.remember_managed_variable_packet(pkt)?;
16051 }
16052 if !ignore_local {
16053 if self.is_duplicate_pkt(&data)? {
16054 return Ok(());
16055 }
16056 let suppress_local = is_internal_control_type(ty);
16057 if !suppress_local {
16058 self.dispatch_local_for_item(&data, called_from_queue)?;
16059 }
16060 }
16061 let allowed = {
16062 let st = self.state.lock();
16063 let ty = Some(ty);
16064 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
16065 #[cfg(all(feature = "discovery", feature = "timesync"))]
16066 let timesync_allowed = ty
16067 .map(|ty| {
16068 Self::timesync_allowed_for_side_locked(
16069 &st,
16070 dst,
16071 ty,
16072 self.clock.now_ms(),
16073 )
16074 })
16075 .unwrap_or(true);
16076 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
16077 let timesync_allowed = true;
16078 route_allowed && timesync_allowed
16079 };
16080 if !allowed {
16081 return Ok(());
16082 }
16083 #[cfg(feature = "compression")]
16084 let (send_result, mut reservation) =
16085 self.send_queued_to_side(dst, &mut data, src.is_some());
16086 #[cfg(not(feature = "compression"))]
16087 let send_result = self.send_reliable_to_side(dst, data.clone(), src.is_some());
16088 if let Err(e) = send_result {
16089 if Self::is_transient_tx_error(&e) {
16090 #[cfg(feature = "compression")]
16091 self.retry_codec_owner(
16092 RouterTxItem::ToSide { src, dst, data },
16093 true,
16094 &mut reservation,
16095 )?;
16096 #[cfg(not(feature = "compression"))]
16097 self.tx_queue_item_for_retry(
16098 RouterTxItem::ToSide { src, dst, data },
16099 true,
16100 )?;
16101 return Ok(());
16102 }
16103 match &data {
16104 RouterItem::Packet(pkt) => {
16105 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
16106 }
16107 RouterItem::Packed(bytes) => {
16108 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
16109 let _ = self.handle_callback_error_from_env(
16110 &env,
16111 None,
16112 e,
16113 called_from_queue,
16114 );
16115 }
16116 }
16117 }
16118 return Err(TelemetryError::HandlerError("tx handler failed"));
16119 }
16120 }
16121 RouterTxItem::EndToEndReplay { packet_id } => {
16122 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
16123 return Ok(());
16124 };
16125 if sides.is_empty() {
16126 let RemoteSidePlan::Target(fallback_sides) =
16127 self.remote_side_plan(&data, None)?;
16128 sides = fallback_sides;
16129 }
16130 for side in sides.iter().copied() {
16131 #[cfg(feature = "compression")]
16132 let mut side_data = data.clone();
16133 #[cfg(feature = "compression")]
16134 let (send_result, mut reservation) =
16135 self.send_queued_to_side(side, &mut side_data, false);
16136 #[cfg(not(feature = "compression"))]
16137 let send_result = self.send_reliable_to_side(side, data.clone(), false);
16138 if let Err(e) = send_result {
16139 if Self::is_transient_tx_error(&e) {
16140 #[cfg(feature = "compression")]
16141 self.retry_codec_owner(
16142 RouterTxItem::ToSide {
16143 src: None,
16144 dst: side,
16145 data: side_data,
16146 },
16147 true,
16148 &mut reservation,
16149 )?;
16150 #[cfg(not(feature = "compression"))]
16151 self.tx_queue_item_for_retry(
16152 RouterTxItem::ToSide {
16153 src: None,
16154 dst: side,
16155 data: data.clone(),
16156 },
16157 true,
16158 )?;
16159 continue;
16160 }
16161 match &data {
16162 RouterItem::Packet(pkt) => {
16163 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
16164 }
16165 RouterItem::Packed(bytes) => {
16166 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
16167 let _ = self.handle_callback_error_from_env(
16168 &env,
16169 None,
16170 e,
16171 called_from_queue,
16172 );
16173 }
16174 }
16175 }
16176 return Err(TelemetryError::HandlerError("tx handler failed"));
16177 }
16178 }
16179 }
16180 RouterTxItem::ReliableReplay {
16181 dst,
16182 ty,
16183 seq,
16184 bytes,
16185 } => {
16186 {
16187 let mut st = self.state.lock();
16188 let Some(sent) = st
16189 .reliable_tx
16190 .get_mut(&Self::reliable_key(dst, ty))
16191 .and_then(|tx| tx.sent.get_mut(&seq))
16192 else {
16193 return Ok(());
16194 };
16195 sent.queued = false;
16196 sent.last_send_ms = self.clock.now_ms();
16197 }
16198 if let Err(e) = self.send_reliable_raw_to_side(dst, bytes.clone(), false) {
16199 if Self::is_transient_tx_error(&e) {
16200 self.tx_queue_item_for_retry(
16201 RouterTxItem::ReliableReplay {
16202 dst,
16203 ty,
16204 seq,
16205 bytes,
16206 },
16207 true,
16208 )?;
16209 return Ok(());
16210 }
16211 return Err(e);
16212 }
16213 let mut st = self.state.lock();
16214 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
16215 if let Some(sent) = tx_state.sent.get_mut(&seq) {
16216 sent.last_send_ms = self.clock.now_ms();
16217 }
16218 }
16219 }
16220
16221 Ok(())
16222 }
16223
16224 #[inline]
16226 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
16227 self.tx_item_impl(item, false, false)
16228 }
16229
16230 #[inline]
16237 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
16238 if self.side_tx_active() {
16239 return self.rx_packed_queue(bytes);
16240 }
16241 crate::memory_admission::check_receive(bytes)?;
16242 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
16243 let item = RouterRxItem {
16244 src: None,
16245 priority: Self::router_item_priority(&data)?,
16246 data,
16247 };
16248 self.rx_item(&item, false)
16249 }
16250
16251 #[inline]
16256 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
16257 if self.side_tx_active() {
16258 return self.rx_queue(pkt.clone());
16259 }
16260 let data = RouterItem::Packet(pkt.clone());
16261 let item = RouterRxItem {
16262 src: None,
16263 priority: Self::router_item_priority(&data)?,
16264 data,
16265 };
16266 self.rx_item(&item, false)
16267 }
16268
16269 #[inline]
16274 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
16275 if self.side_tx_active() {
16276 return self.rx_queue_from_side(pkt.clone(), side);
16277 }
16278 self.ensure_side_ingress_enabled(side)?;
16279 let data = RouterItem::Packet(pkt.clone());
16280 let item = RouterRxItem {
16281 src: Some(side),
16282 priority: Self::router_item_priority(&data)?,
16283 data,
16284 };
16285 self.rx_item(&item, false)
16286 }
16287
16288 #[inline]
16293 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
16294 if self.side_tx_active() {
16295 return self.rx_packed_queue_from_side(bytes, side);
16296 }
16297 self.ensure_side_ingress_enabled(side)?;
16298 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
16299 return Ok(());
16300 };
16301 let data = RouterItem::Packed(decoded);
16302 let item = RouterRxItem {
16303 src: Some(side),
16304 priority: Self::router_item_priority(&data)?,
16305 data,
16306 };
16307 self.rx_item(&item, false)
16308 }
16309
16310 #[inline]
16317 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
16318 #[cfg(feature = "discovery")]
16319 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16320 if self.side_tx_active() {
16321 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
16322 }
16323 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
16324 }
16325
16326 #[inline]
16331 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
16332 #[cfg(feature = "discovery")]
16333 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16334 if self.side_tx_active() {
16335 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
16336 crate::shared_bytes::convert(pkt),
16337 )));
16338 }
16339 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(
16340 crate::shared_bytes::convert(pkt),
16341 )))
16342 }
16343
16344 #[inline]
16350 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
16351 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
16352 }
16353
16354 #[inline]
16356 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
16357 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
16358 crate::shared_bytes::convert(data),
16359 )))
16360 }
16361
16362 #[inline(never)]
16368 fn log_dispatch<T: LeBytes>(
16369 &self,
16370 ty: DataType,
16371 data: &[T],
16372 timestamp: Option<u64>,
16373 mut queued: bool,
16374 ) -> TelemetryResult<()> {
16375 if !queued {
16376 #[cfg(feature = "discovery")]
16377 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16378 queued = self.side_tx_active();
16379 }
16380 let timestamp = timestamp.unwrap_or_else(|| self.packet_timestamp_ms());
16381 log_raw(self.sender_arc(), ty, data, timestamp, |pkt| {
16382 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
16383 if queued {
16384 self.tx_queue_item(item)
16385 } else {
16386 self.tx_item(item)
16387 }
16388 })
16389 }
16390
16391 #[inline]
16394 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
16395 self.log_dispatch(ty, data, None, false)
16396 }
16397
16398 #[inline]
16400 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
16401 self.log_dispatch(ty, data, None, true)
16402 }
16403
16404 #[inline]
16406 pub fn log_ts<T: LeBytes>(
16407 &self,
16408 ty: DataType,
16409 timestamp: u64,
16410 data: &[T],
16411 ) -> TelemetryResult<()> {
16412 self.log_dispatch(ty, data, Some(timestamp), false)
16413 }
16414
16415 #[inline]
16417 pub fn log_queue_ts<T: LeBytes>(
16418 &self,
16419 ty: DataType,
16420 timestamp: u64,
16421 data: &[T],
16422 ) -> TelemetryResult<()> {
16423 self.log_dispatch(ty, data, Some(timestamp), true)
16424 }
16425}
16426
16427#[cfg(all(test, feature = "std", feature = "discovery"))]
16428mod queued_schema_identity_tests {
16429 use super::*;
16430
16431 fn catalog(timestamp: u64, byte: u8) -> Packet {
16432 Packet::new(
16433 crate::DataType::DiscoverySchema,
16434 &[DataEndpoint::Discovery],
16435 "SCHEMA_PEER",
16436 timestamp,
16437 crate::shared_bytes::convert(Arc::<[u8]>::from(vec![byte; 1024])),
16438 )
16439 .unwrap()
16440 }
16441
16442 #[test]
16443 fn inline_and_large_typed_publications_share_headers_and_keep_le_bytes() {
16444 crate::tests::ensure_common_test_schema();
16445 let endpoint = DataEndpoint::named("SD_CARD");
16446 let ty = crate::config::register_data_type_with_description(
16447 "INLINE_TYPED_BOUNDARY",
16448 "",
16449 MessageElement::Dynamic(crate::MessageDataType::UInt32, crate::MessageClass::Data),
16450 &[endpoint],
16451 crate::ReliableMode::None,
16452 16,
16453 )
16454 .unwrap();
16455 let sender: Arc<str> = Arc::from("DAQ_SOURCE");
16456 let endpoints = message_meta(ty);
16457 for count in [
16458 0,
16459 1,
16460 crate::config::STACK_PAYLOAD_SIZE / 4,
16461 crate::config::STACK_PAYLOAD_SIZE / 4 + 1,
16462 64,
16463 ] {
16464 let values = (0..count)
16465 .map(|i| 0x1234_5678u32.wrapping_add(i as u32))
16466 .collect::<Vec<_>>();
16467 log_raw(sender.clone(), ty, &values, 42, |packet| {
16468 let bytes = values
16469 .iter()
16470 .flat_map(|v| v.to_le_bytes())
16471 .collect::<Vec<_>>();
16472 assert_eq!(packet.payload(), bytes);
16473 assert_eq!(packet.sender().as_ptr(), sender.as_ptr());
16474 assert_eq!(
16475 packet.endpoints().as_ptr(),
16476 endpoints.endpoints_ref().as_ptr()
16477 );
16478 packet.validate().unwrap();
16479 Ok(())
16480 })
16481 .unwrap();
16482 }
16483 #[derive(Clone, Copy)]
16484 struct BadWidth;
16485 impl LeBytes for BadWidth {
16486 const WIDTH: usize = 0;
16487 fn write_le(self, _: &mut [u8]) {
16488 unreachable!()
16489 }
16490 fn from_le_slice(_: &[u8]) -> Self {
16491 Self
16492 }
16493 }
16494 assert!(matches!(
16495 log_raw(sender.clone(), ty, &[BadWidth], 42, |_| Ok(())),
16496 Err(TelemetryError::BadArg)
16497 ));
16498 #[derive(Clone, Copy)]
16499 struct HugeWidth;
16500 impl LeBytes for HugeWidth {
16501 const WIDTH: usize = usize::MAX;
16502 fn write_le(self, _: &mut [u8]) {
16503 unreachable!()
16504 }
16505 fn from_le_slice(_: &[u8]) -> Self {
16506 Self
16507 }
16508 }
16509 assert!(matches!(
16510 log_raw(sender, ty, &[HugeWidth, HugeWidth], 42, |_| Ok(())),
16511 Err(TelemetryError::PacketTooLarge(_))
16512 ));
16513 }
16514
16515 #[test]
16516 fn queue_accounting_reports_live_cost_without_draining_or_pinning() {
16517 crate::tests::ensure_common_test_schema();
16518 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 42u64));
16519 let packet = Packet::new(
16520 crate::DataType::named("IMU_DATA"),
16521 &[DataEndpoint::named("SD_CARD")],
16522 "SOURCE",
16523 1,
16524 crate::shared_bytes::convert(Arc::<[u8]>::from([0u8; 24])),
16525 )
16526 .unwrap();
16527 router.tx_queue(packet).unwrap();
16528 assert!(router.has_queued_work());
16529 let expected = router.memory_counters();
16530 let a = router.queue_counters();
16531 assert_eq!(a[0], 1);
16532 assert_eq!(a[1], 0);
16533 assert_eq!(a[2], 1);
16534 assert_eq!(a[3], expected[3]);
16535 assert_eq!(a[4], expected[4]);
16536 assert_eq!(a[5], crate::DataType::named("IMU_DATA").as_u32());
16537 assert_eq!(a[14], 42);
16538 assert_eq!(a[15], 0);
16539 assert_eq!(router.queue_counters(), a);
16540 assert_eq!(router.memory_counters(), expected);
16541 router.drain_queues_with_timeout(0).unwrap();
16542 let empty = router.queue_counters();
16543 assert_eq!(empty[2], 0);
16544 assert_eq!(empty[4], 0);
16545 assert_eq!(router.memory_counters()[4], 0);
16546 assert!(!router.has_queued_work());
16547 }
16548
16549 #[test]
16550 fn repeated_schema_preserves_partial_transfer_but_updates_and_other_sides_do_not_coalesce() {
16551 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
16552 let side = router.add_side_packed("A", |_| Ok(()));
16553 let other_side = router.add_side_packed("B", |_| Ok(()));
16554 let original = catalog(1, 42);
16555 let duplicate = catalog(2, 42);
16556 assert_ne!(original.packet_id(), duplicate.packet_id());
16557 let mut st = router.state.lock();
16558 st.transmit_queue.clear();
16559 let queued = |dst, pkt| TxQueued {
16560 disposable: true,
16561 item: RouterTxItem::ToSide {
16562 src: None,
16563 dst,
16564 data: RouterItem::Packet(pkt),
16565 },
16566 ignore_local: true,
16567 priority: 255,
16568 delivery_state: QueuedDelivery::Untracked,
16569 accepted_type: 0,
16570 deferred_since_ms: 0,
16571 retry_after_ms: 123,
16572 };
16573 st.push_transmit(queued(side, original.clone())).unwrap();
16574 st.push_transmit(queued(side, duplicate.clone())).unwrap();
16575 assert_eq!(st.transmit_queue.len(), 1);
16576 let pending = st.transmit_queue.iter().next().unwrap();
16577 assert_eq!(pending.retry_after_ms, 123);
16578 assert!(
16579 matches!(&pending.item, RouterTxItem::ToSide { data: RouterItem::Packet(pkt), .. }
16580 if pkt.packet_id() == original.packet_id())
16581 );
16582 st.push_transmit(queued(other_side, duplicate)).unwrap();
16583 assert_eq!(st.transmit_queue.len(), 2);
16584 let changed = catalog(3, 43);
16585 st.push_transmit(queued(side, changed.clone())).unwrap();
16586 assert_eq!(st.transmit_queue.len(), 2);
16587 assert!(st.transmit_queue.iter().any(|queued| matches!(&queued.item,
16588 RouterTxItem::ToSide { dst, data: RouterItem::Packet(pkt), .. }
16589 if *dst == side && pkt.packet_id() == changed.packet_id())));
16590 }
16591
16592 #[test]
16593 fn repeated_announcements_complete_one_fragment_per_dispatch() {
16594 use core::sync::atomic::{AtomicUsize, Ordering};
16595 let allowance = Arc::new(AtomicUsize::new(0));
16596 let accepted = Arc::new(std::sync::Mutex::new(Vec::<Arc<[u8]>>::new()));
16597 let available = allowance.clone();
16598 let frames = accepted.clone();
16599 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 1u64));
16600 let side = node.add_side_packed_with_options(
16601 "bounded CAN",
16602 move |bytes| {
16603 if available
16604 .try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
16605 count.checked_sub(1)
16606 })
16607 .is_err()
16608 {
16609 return Err(TelemetryError::Io("side tx busy"));
16610 }
16611 frames.lock().unwrap().push(Arc::<[u8]>::from(bytes));
16612 Ok(())
16613 },
16614 RouterSideOptions::default().with_small_packet_transport(128),
16615 );
16616 let handler = node.state.lock().sides[side]
16617 .as_ref()
16618 .unwrap()
16619 .tx_handler
16620 .clone();
16621 let mut complete = false;
16622 for timestamp in 1..100 {
16623 let mut state = 0x517cc1b7u32;
16625 let payload: Vec<u8> = (0..3600)
16626 .map(|_| {
16627 state ^= state << 13;
16628 state ^= state >> 17;
16629 state ^= state << 5;
16630 state as u8
16631 })
16632 .collect();
16633 let packet = Packet::new(
16634 crate::DataType::DiscoverySchema,
16635 &[crate::DataEndpoint::Discovery],
16636 "SCHEMA_PEER",
16637 timestamp,
16638 crate::shared_bytes::convert(Arc::<[u8]>::from(payload)),
16639 )
16640 .unwrap();
16641 node.state
16642 .lock()
16643 .push_transmit(TxQueued {
16644 disposable: true,
16645 item: RouterTxItem::ToSide {
16646 src: None,
16647 dst: side,
16648 data: RouterItem::Packet(packet),
16649 },
16650 ignore_local: true,
16651 priority: 255,
16652 delivery_state: QueuedDelivery::Untracked,
16653 accepted_type: 0,
16654 deferred_since_ms: 0,
16655 retry_after_ms: 0,
16656 })
16657 .unwrap();
16658 let data = match &node.state.lock().transmit_queue.iter().next().unwrap().item {
16659 RouterTxItem::ToSide { data, .. } => data.clone(),
16660 _ => panic!("unexpected item"),
16661 };
16662 allowance.store(1, Ordering::Relaxed);
16663 match node.call_side_tx_handler(side, &handler, &data, false) {
16664 Ok(()) => {
16665 complete = true;
16666 break;
16667 }
16668 Err(TelemetryError::Io(_)) => {}
16669 other => panic!("unexpected result: {other:?}"),
16670 }
16671 }
16672 assert!(
16673 complete,
16674 "new announcement timestamps must not replay the prefix forever"
16675 );
16676 let received = accepted.lock().unwrap();
16677 assert!(
16680 received.len() > 1,
16681 "fixture must exercise fragmentation, not a compressed single frame"
16682 );
16683 let first = &received[0];
16684 assert_eq!(&first[..4], &[83, 68, 84, 3]);
16685 assert_eq!(
16686 u16::from_le_bytes([first[10], first[11]]) as usize,
16687 received.len()
16688 );
16689 let transfer_id = &first[4..8];
16690 for (index, frame) in received.iter().enumerate() {
16691 assert_eq!(&frame[4..8], transfer_id);
16692 assert_eq!(u16::from_le_bytes([frame[8], frame[9]]) as usize, index);
16693 }
16694 }
16695 #[test]
16696 fn schema_receipt_resolves_assigned_numeric_address_without_adding_peers() {
16697 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
16698 let side = router.add_side_packed("A", |_| Ok(()));
16699 let assigned = 0x1234u32;
16700 assert_ne!(sender_address_u32("SCHEMA_PEER"), assigned);
16701 {
16702 let mut st = router.state.lock();
16703 st.discovery_routes
16704 .entry(side)
16705 .or_default()
16706 .announcers
16707 .insert(
16708 "SCHEMA_PEER".into(),
16709 DiscoverySenderState {
16710 advertised_address: Some(assigned),
16711 ..DiscoverySenderState::default()
16712 },
16713 );
16714 }
16715 router.note_schema_received(side, "@addr:4660");
16716 router.note_schema_received(side, "@addr:4661");
16717 let st = router.state.lock();
16718 let peers = &st.discovery_routes.get(&side).unwrap().announcers;
16719 assert_eq!(peers.len(), 1);
16720 assert!(peers["SCHEMA_PEER"].has_schema);
16721 }
16722 #[test]
16723 fn busy_handler_returns_after_one_attempt_and_other_errors_keep_retry_budget() {
16724 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
16725 let attempts = core::cell::Cell::new(0);
16726 let result: Result<((), usize), _> = router.retry_with_attempts(8, || {
16727 attempts.set(attempts.get() + 1);
16728 Err(TelemetryError::Io("side tx busy"))
16729 });
16730 assert!(matches!(
16731 result,
16732 Err((TelemetryError::Io("side tx busy"), 1))
16733 ));
16734 assert_eq!(attempts.get(), 1);
16735 attempts.set(0);
16736 let result: Result<((), usize), _> = router.retry_with_attempts(3, || {
16737 attempts.set(attempts.get() + 1);
16738 Err(TelemetryError::Io("permanent handler failure"))
16739 });
16740 assert!(matches!(
16741 result,
16742 Err((TelemetryError::Io("permanent handler failure"), 3))
16743 ));
16744 assert_eq!(attempts.get(), 3);
16745 }
16746 #[test]
16747 fn rejected_first_template_frame_is_retried_self_describing() {
16748 crate::tests::ensure_common_test_schema();
16749 let ty = crate::config::register_data_type_with_description(
16750 "ROUTER_BUSY_FIRST_TEMPLATE",
16751 "",
16752 crate::MessageElement::Static(
16753 1,
16754 crate::MessageDataType::UInt8,
16755 crate::MessageClass::Data,
16756 ),
16757 &[crate::DataEndpoint::named("RADIO")],
16758 crate::ReliableMode::None,
16759 1,
16760 )
16761 .unwrap();
16762 let tx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
16763 let rx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
16764 let opts = RouterSideOptions::default().with_small_packet_transport(0);
16765 let a = tx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
16766 let b = rx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
16767 let packet = Packet::new(
16768 ty,
16769 &[crate::DataEndpoint::named("RADIO")],
16770 "BUSY_PEER",
16771 1,
16772 crate::shared_bytes::convert(Arc::<[u8]>::from([42u8])),
16773 )
16774 .unwrap();
16775 let raw = wire_format::pack_packet(&packet);
16776 let first = tx
16778 .encode_side_transport_frames(
16779 a,
16780 opts.clone(),
16781 crate::shared_bytes::convert(raw.clone()),
16782 )
16783 .unwrap();
16784 assert!(
16785 tx.send_side_frames_resumable(a, first, |_| Err(TelemetryError::Io("side tx busy")))
16786 .is_err()
16787 );
16788 let retry = tx
16789 .encode_side_transport_frames(
16790 a,
16791 opts.clone(),
16792 crate::shared_bytes::convert(raw.clone()),
16793 )
16794 .unwrap();
16795 let mut received = None;
16796 tx.send_side_frames_resumable(a, retry, |frame| {
16797 assert_eq!(
16798 frame[3], SIDE_TRANSPORT_KIND_FULL,
16799 "retry requires the unaccepted dictionary bootstrap"
16800 );
16801 received = rx.decode_side_transport_frame(b, frame)?;
16802 Ok(())
16803 })
16804 .unwrap();
16805 assert_eq!(received, Some(crate::shared_bytes::convert(raw.clone())));
16806 let next = tx
16808 .encode_side_transport_frames(a, opts, crate::shared_bytes::convert(raw.clone()))
16809 .unwrap();
16810 tx.send_side_frames_resumable(a, next, |frame| {
16811 assert_eq!(frame[3], SIDE_TRANSPORT_KIND_COMPACT);
16812 assert_eq!(
16813 rx.decode_side_transport_frame(b, frame)?,
16814 Some(crate::shared_bytes::convert(raw.clone()))
16815 );
16816 Ok(())
16817 })
16818 .unwrap();
16819 }
16820 #[test]
16821 fn already_queued_receipt_precedes_ordered_retry_exhaustion() {
16822 crate::tests::ensure_common_test_schema();
16823 let ep = DataEndpoint::named("RADIO");
16824 let ty = crate::config::register_data_type_with_description(
16825 "QUEUED_RECEIPT_DEADLINE",
16826 "",
16827 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
16828 &[ep],
16829 crate::ReliableMode::Ordered,
16830 1,
16831 )
16832 .unwrap();
16833 for isr in [false, true] {
16834 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
16835 let clock = now.clone();
16836 let router = Router::new_with_clock(
16837 RouterConfig::new([]).with_hostname("FC"),
16838 Box::new(move || clock.load(Ordering::Relaxed)),
16839 );
16840 let side = router.add_side_packed("CAN", |_| Ok(()));
16841 let packet = Packet::new(ty, &[ep], "FC", 1, Arc::<[u8]>::from([42u8])).unwrap();
16842 let id = packet.packet_id();
16843 let owner = Router::sender_hash("GS");
16844 let mut destinations = BTreeMap::new();
16845 destinations.insert(owner, side);
16846 router.state.lock().end_to_end_reliable_tx.insert(
16847 id,
16848 EndToEndReliableSent {
16849 ordered_type: Some(ty.as_u32()),
16850 waiting_for_predecessor: false,
16851 data: RouterItem::Packet(packet),
16852 pending_destinations: destinations,
16853 tracked_destinations: true,
16854 last_send_ms: 0,
16855 retries: runtime_reliable_max_retries(),
16856 queued: false,
16857 },
16858 );
16859 let snapshot = router.export_reliable_pending_stats();
16860 assert_eq!(snapshot.len(), 1);
16861 assert_eq!(snapshot[0].packet_id, id);
16862 assert_eq!(snapshot[0].data_type, ty);
16863 assert_eq!(snapshot[0].pending_destinations, vec![owner]);
16864 assert_eq!(snapshot[0].retries, runtime_reliable_max_retries());
16865 assert!(!snapshot[0].queued);
16866 assert!(!snapshot[0].waiting_for_predecessor);
16867 let ack = Packet::new_with_wire_contract(
16868 DataType::ReliableAck,
16869 message_meta(DataType::ReliableAck).endpoints_ref(),
16870 "E2EACK:GS",
16871 1,
16872 1,
16873 Arc::<[u8]>::from(id.to_le_bytes()),
16874 None,
16875 Arc::from([owner, Router::sender_hash("FC")]),
16876 )
16877 .unwrap();
16878 let packed = wire_format::pack_packet(&ack);
16879 if isr {
16880 router.rx_packed_queue_from_side_isr(&packed, side).unwrap();
16881 } else {
16882 router.rx_packed_queue_from_side(&packed, side).unwrap();
16883 }
16884 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
16885 router
16886 .drain_queues_with_timeout(1)
16887 .expect("a received ACK must be processed before cancelling its ordered delivery");
16888 assert!(!router.state.lock().end_to_end_reliable_tx.contains_key(&id));
16889 assert!(router.export_reliable_pending_stats().is_empty());
16890 }
16891 }
16892 #[test]
16893 fn embedded_gateway_receipt_with_numeric_sender_clears_host_pending_owner() {
16894 crate::tests::ensure_common_test_schema();
16895 let bytes = [
16898 0x0c, 0x01, 0x01, 0x08, 0xd4, 0xe5, 0xea, 0xb6, 0xb7, 0x33, 0xff, 0x90, 0x02, 0xb4,
16899 0xe6, 0xdd, 0x94, 0x07, 0x14, 0x03, 0x44, 0x02, 0x02, 0x06, 0x25, 0xd2, 0x59, 0x1b,
16900 0xbb, 0xfc, 0xa7, 0x5f, 0x43, 0x92, 0x9e, 0x12, 0xbb, 0xfc, 0xa7, 0x26, 0xa2, 0xa0,
16901 0xbe, 0x39, 0x65, 0xe2, 0xd8, 0x61, 0x68, 0x22, 0xfd,
16902 ];
16903 let ack = wire_format::unpack_packet(&bytes).unwrap();
16904 let owner = Router::sender_hash("GB");
16905 assert_eq!(Router::end_to_end_ack_sender_hash(&ack), Some(owner));
16906 let id = Router::decode_end_to_end_reliable_ack(ack.payload()).unwrap();
16907 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
16908 let side = router.add_side_packed("Pico", |_| Ok(()));
16909 let data = ack.clone();
16910 router.state.lock().end_to_end_reliable_tx.insert(
16911 id,
16912 EndToEndReliableSent {
16913 ordered_type: None,
16914 waiting_for_predecessor: false,
16915 data: RouterItem::Packet(data),
16916 pending_destinations: BTreeMap::from([(owner, side)]),
16917 tracked_destinations: true,
16918 last_send_ms: 0,
16919 retries: 0,
16920 queued: false,
16921 },
16922 );
16923 router.rx_packed_queue_from_side(&bytes, side).unwrap();
16924 router.drain_queues_with_timeout(1).unwrap();
16925 assert!(router.export_reliable_pending_stats().is_empty());
16926 }
16927
16928 #[test]
16929 fn frozen_delivery_contract_bypasses_dictionary_even_for_best_effort_schema() {
16930 crate::tests::ensure_common_test_schema();
16931 let ep = DataEndpoint::named("RADIO");
16932 let ty = crate::config::register_data_type_with_description(
16933 "MANAGED_CONTROL_WITH_FROZEN_OWNERS",
16934 "",
16935 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
16936 &[ep],
16937 crate::ReliableMode::None,
16938 16,
16939 )
16940 .unwrap();
16941 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
16942 let opts = RouterSideOptions::default().with_template_transport();
16943 let side = router.add_side_packed_with_options("radio", |_| Ok(()), opts.clone());
16944 let packet = Packet::new_with_wire_contract(
16945 ty,
16946 &[ep],
16947 "GS",
16948 1,
16949 1,
16950 Arc::<[u8]>::from([1u8]),
16951 None,
16952 Arc::from([Router::sender_hash("RF")]),
16953 )
16954 .unwrap();
16955 let raw = wire_format::pack_packet(&packet);
16956 assert_eq!(
16957 router
16958 .item_service_priority(&RouterItem::Packet(packet.clone()))
16959 .unwrap(),
16960 254
16961 );
16962 assert_eq!(
16963 router
16964 .item_service_priority(&RouterItem::Packed(crate::shared_bytes::convert(
16965 raw.clone()
16966 )))
16967 .unwrap(),
16968 254
16969 );
16970 assert!(
16971 !wire_format::peek_routing_frame_info(&raw)
16972 .unwrap()
16973 .envelope
16974 .is_reliable
16975 );
16976 for _ in 0..3 {
16977 let frames = router
16978 .encode_side_transport_frames(
16979 side,
16980 opts.clone(),
16981 crate::shared_bytes::convert(raw.clone()),
16982 )
16983 .unwrap();
16984 let mut count = 0;
16985 router
16986 .send_side_frames_resumable(side, frames, |bytes| {
16987 assert_eq!(
16988 bytes,
16989 raw.as_ref(),
16990 "delivery contract must remain independently decodable"
16991 );
16992 count += 1;
16993 Ok(())
16994 })
16995 .unwrap();
16996 assert_eq!(count, 1);
16997 }
16998 }
16999
17000 #[test]
17001 fn managed_receive_queues_keep_service_priority_with_low_schema_priority() {
17002 crate::tests::ensure_common_test_schema();
17003 let ep = DataEndpoint::named("RADIO");
17004 let ty = crate::config::register_data_type_with_description(
17005 "MANAGED_RX_SERVICE_PRIORITY",
17006 "",
17007 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
17008 &[ep],
17009 crate::ReliableMode::None,
17010 16,
17011 )
17012 .unwrap();
17013 let router = Router::new(RouterConfig::new([]).with_hostname("RF"));
17014 router
17015 .enable_network_variable(ty, NetworkVariablePermissions::READ_ONLY)
17016 .unwrap();
17017 let side = router.add_side_packed("radio", |_| Ok(()));
17018 router.clear_queues();
17019 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
17020 let wire = wire_format::pack_packet(&packet);
17021 router.rx_queue(packet.clone()).unwrap();
17022 router.rx_queue_from_side(packet.clone(), side).unwrap();
17023 router.rx_packed_queue(&wire).unwrap();
17024 router.rx_packed_queue_from_side(&wire, side).unwrap();
17025 for _ in 0..4 {
17026 let queued = router
17027 .pop_next_queued_control_rx_item()
17028 .unwrap()
17029 .expect("registered managed controls must precede queued bulk work");
17030 assert_eq!(
17031 queued.priority,
17032 crate::transport_priority(DataType::ManagedVariableValue)
17033 );
17034 }
17035 assert!(router.pop_next_queued_control_rx_item().unwrap().is_none());
17036 }
17037
17038 #[test]
17039 fn managed_value_keeps_service_priority_after_side_backpressure() {
17040 crate::tests::ensure_common_test_schema();
17041 let ep = DataEndpoint::named("RADIO");
17042 let ty = crate::config::register_data_type_with_description(
17043 "MANAGED_RETRY_SERVICE_PRIORITY",
17044 "",
17045 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
17046 &[ep],
17047 crate::ReliableMode::None,
17048 16,
17049 )
17050 .unwrap();
17051 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
17052 router
17053 .enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
17054 .unwrap();
17055 let side = router.add_side_packed("radio", |_| Err(TelemetryError::Io("side tx busy")));
17056 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
17057 let item = RouterItem::Packet(packet);
17058 for queued in [
17059 RouterTxItem::Broadcast(item.clone()),
17060 RouterTxItem::ToSide {
17061 src: None,
17062 dst: side,
17063 data: item.clone(),
17064 },
17065 ] {
17066 router.tx_queue_item_for_retry(queued, true).unwrap();
17067 let queued = router.state.lock().transmit_queue.pop_front().unwrap();
17068 assert_eq!(
17069 queued.priority,
17070 crate::transport_priority(DataType::ManagedVariableValue),
17071 "a busy side must not demote managed controls to bulk application priority"
17072 );
17073 assert!(queued.retry_after_ms > 0);
17074 }
17075 }
17076 #[test]
17077 fn receipt_budget_does_not_starve_transmit() {
17078 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
17079 crate::tests::ensure_common_test_schema();
17080 let now = Arc::new(AtomicU64::new(0));
17081 let clock = now.clone();
17082 let count = Arc::new(AtomicUsize::new(0));
17083 let accepted = count.clone();
17084 let node = Router::new_with_clock(
17085 RouterConfig::default(),
17086 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
17087 );
17088 let side = node.add_side_packed("wire", move |_| {
17089 accepted.fetch_add(1, Ordering::Relaxed);
17090 Ok(())
17091 });
17092 node.clear_queues();
17093 let packet = Packet::new(
17094 crate::DataType::ReliableAck,
17095 &[crate::DataEndpoint::TelemetryError],
17096 "E2EACK:PEER",
17097 1,
17098 Arc::<[u8]>::from([0u8; 8]),
17099 )
17100 .unwrap();
17101 node.state
17102 .lock()
17103 .push_transmit(TxQueued {
17104 disposable: true,
17105 item: RouterTxItem::ToSide {
17106 src: None,
17107 dst: side,
17108 data: RouterItem::Packet(packet.clone()),
17109 },
17110 ignore_local: true,
17111 priority: 250,
17112 delivery_state: QueuedDelivery::Untracked,
17113 accepted_type: 0,
17114 deferred_since_ms: 0,
17115 retry_after_ms: 0,
17116 })
17117 .unwrap();
17118 node.rx_packed_queue_from_side(
17119 &wire_format::pack_packet(
17120 &Packet::new(
17121 crate::DataType::ReliableAck,
17122 &[crate::DataEndpoint::TelemetryError],
17123 "RX_PEER",
17124 2,
17125 Arc::<[u8]>::from([0u8; 8]),
17126 )
17127 .unwrap(),
17128 ),
17129 side,
17130 )
17131 .unwrap();
17132 node.drain_queues_with_timeout(1).unwrap();
17133 assert_eq!(
17134 count.load(Ordering::Relaxed),
17135 1,
17136 "control RX exhausting its cooperative budget must still allow one queued TX"
17137 );
17138 }
17139 #[test]
17140 fn receipt_budget_does_not_starve_bulk_transmit() {
17141 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
17142 crate::tests::ensure_common_test_schema();
17143 let now = Arc::new(AtomicU64::new(0));
17144 let clock = now.clone();
17145 let count = Arc::new(AtomicUsize::new(0));
17146 let accepted = count.clone();
17147 let node = Router::new_with_clock(
17148 RouterConfig::default(),
17149 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
17150 );
17151 let side = node.add_side_packed("wire", move |_| {
17152 accepted.fetch_add(1, Ordering::Relaxed);
17153 Ok(())
17154 });
17155 node.clear_queues();
17156 let packet = Packet::new(
17157 crate::DataType::ReliableAck,
17158 &[crate::DataEndpoint::TelemetryError],
17159 "E2EACK:PEER",
17160 1,
17161 Arc::<[u8]>::from([0u8; 8]),
17162 )
17163 .unwrap();
17164 node.state
17165 .lock()
17166 .push_transmit(TxQueued {
17167 disposable: true,
17168 item: RouterTxItem::ToSide {
17169 src: None,
17170 dst: side,
17171 data: RouterItem::Packet(packet.clone()),
17172 },
17173 ignore_local: true,
17174 priority: 1,
17175 delivery_state: QueuedDelivery::Untracked,
17176 accepted_type: 0,
17177 deferred_since_ms: 0,
17178 retry_after_ms: 0,
17179 })
17180 .unwrap();
17181 node.rx_packed_queue_from_side(
17182 &wire_format::pack_packet(
17183 &Packet::new(
17184 crate::DataType::ReliableAck,
17185 &[crate::DataEndpoint::TelemetryError],
17186 "RX_PEER",
17187 2,
17188 Arc::<[u8]>::from([0u8; 8]),
17189 )
17190 .unwrap(),
17191 ),
17192 side,
17193 )
17194 .unwrap();
17195 node.process_all_queues_with_timeout_impl(1).unwrap();
17196 assert_eq!(
17197 count.load(Ordering::Relaxed),
17198 1,
17199 "control RX exhausting its cooperative budget must still allow one queued bulk TX"
17200 );
17201 }
17202}
17203
17204#[cfg(all(test, feature = "discovery", feature = "std"))]
17205fn untemplated_test_node() -> Router {
17206 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0))
17207}
17208#[cfg(all(test, feature = "discovery", feature = "std"))]
17209fn untemplated_test_options() -> RouterSideOptions {
17210 RouterSideOptions::default().with_small_packet_transport(64)
17211}
17212#[cfg(all(test, feature = "discovery", feature = "std"))]
17213#[allow(clippy::duplicate_mod)]
17214#[path = "tests/untemplated_transport.rs"]
17215mod untemplated_transport_tests;
17216
17217#[cfg(all(test, feature = "std", feature = "discovery"))]
17218mod queued_producer_maintenance_tests {
17219 use super::*;
17220 #[test]
17221 fn queued_publication_defers_discovery_to_worker_maintenance() {
17222 crate::tests::ensure_common_test_schema();
17223 let ty = DataType::named("GPS_DATA");
17224 for variant in 0..4 {
17225 let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
17226 let output = sent.clone();
17227 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
17228 router.add_side_packed("wire", move |bytes| {
17229 output
17230 .lock()
17231 .unwrap()
17232 .push(wire_format::unpack_packet(bytes)?.data_type());
17233 Ok(())
17234 });
17235 router.clear_queues();
17236 let packet =
17237 Packet::from_f32_slice(ty, &[1.0, 2.0, 3.0], message_meta(ty).endpoints_ref(), 42)
17238 .unwrap();
17239 match variant {
17240 0 => router.tx_queue(packet).unwrap(),
17241 1 => router
17242 .tx_packed_queue(wire_format::pack_packet(&packet))
17243 .unwrap(),
17244 2 => router.log_queue(ty, &[1.0f32, 2.0, 3.0]).unwrap(),
17245 _ => router.log_queue_ts(ty, 42, &[1.0f32, 2.0, 3.0]).unwrap(),
17246 }
17247 assert!(
17248 sent.lock().unwrap().is_empty(),
17249 "producer invoked a transport"
17250 );
17251 assert_eq!(
17252 router.state.lock().transmit_queue.len(),
17253 1,
17254 "queued API performed discovery on the producer stack (variant {variant})"
17255 );
17256 router.periodic(0).unwrap();
17257 let sent = sent.lock().unwrap();
17258 assert!(sent.contains(&ty));
17259 assert!(
17260 sent.iter().copied().any(discovery::is_discovery_type),
17261 "worker maintenance must still announce the board"
17262 );
17263 }
17264 }
17265}
17266
17267#[cfg(all(test, feature = "std"))]
17268mod bounded_fair_queue_tests {
17269 use super::*;
17270 #[test]
17271 fn small_canonical_reports_survive_fast_senders_fixed_ring_bursts() {
17272 let ep = crate::config::register_endpoint("ROUTER_SMALL_QUEUE_DEST", false).unwrap();
17273 let ty = crate::config::register_data_type_with_description(
17274 "ROUTER_SMALL_QUEUE_REPORT",
17275 "",
17276 crate::MessageElement::Static(
17277 4,
17278 crate::MessageDataType::UInt8,
17279 crate::MessageClass::Data,
17280 ),
17281 &[ep],
17282 crate::ReliableMode::None,
17283 0,
17284 )
17285 .unwrap();
17286 for tx in [false, true] {
17287 for slow_last in [false, true] {
17288 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
17289 let make = |sender| {
17290 RouterItem::Packed(crate::shared_bytes::convert(wire_format::pack_packet(
17291 &Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 4])).unwrap(),
17292 )))
17293 };
17294 let rx = |data| RouterRxItem {
17295 src: None,
17296 data,
17297 priority: 0,
17298 };
17299 let tx_item = |data| TxQueued {
17300 disposable: true,
17301 item: RouterTxItem::Broadcast(data),
17302 ignore_local: true,
17303 priority: 0,
17304 delivery_state: QueuedDelivery::Untracked,
17305 accepted_type: 0,
17306 deferred_since_ms: 0,
17307 retry_after_ms: 0,
17308 };
17309 let mut st = node.state.lock();
17310 st.memory.max_queue_budget = usize::MAX;
17311 if tx {
17312 st.transmit_queue = BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
17313 } else {
17314 st.received_queue = BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
17315 }
17316 let order = if slow_last {
17317 ["FAST", "FAST", "SLOW"]
17318 } else {
17319 ["SLOW", "FAST", "FAST"]
17320 };
17321 for sender in order.into_iter().chain(core::iter::repeat_n("FAST", 32)) {
17322 if tx {
17323 st.push_transmit(tx_item(make(sender))).unwrap();
17324 } else {
17325 st.push_received(rx(make(sender))).unwrap();
17326 }
17327 }
17328 let slow = wire_format::source_address_for_sender("SLOW");
17329 let retained = if tx {
17330 st.transmit_queue
17331 .iter()
17332 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
17333 } else {
17334 st.received_queue.iter().any(|item| {
17335 RouterInner::fair_bulk_owner(&item.data, item.priority) == Some(slow)
17336 })
17337 };
17338 assert!(
17339 retained,
17340 "small report was displaced: tx={tx}, slow_last={slow_last}"
17341 );
17342 }
17343 }
17344 }
17345
17346 #[test]
17347 fn slow_stream_survives_same_board_fast_stream_bursts() {
17348 let ep = crate::config::register_endpoint("ROUTER_STREAM_QUEUE_DEST", false).unwrap();
17349 let ty = crate::config::register_data_type_with_description(
17350 "ROUTER_STREAM_QUEUE_FAST",
17351 "",
17352 crate::MessageElement::Static(
17353 64,
17354 crate::MessageDataType::UInt8,
17355 crate::MessageClass::Data,
17356 ),
17357 &[ep],
17358 crate::ReliableMode::None,
17359 0,
17360 )
17361 .unwrap();
17362 let slow_ty = crate::config::register_data_type_with_description(
17363 "ROUTER_STREAM_QUEUE_SLOW",
17364 "",
17365 crate::MessageElement::Static(
17366 64,
17367 crate::MessageDataType::UInt8,
17368 crate::MessageClass::Data,
17369 ),
17370 &[ep],
17371 crate::ReliableMode::None,
17372 0,
17373 )
17374 .unwrap();
17375 for mode in 0..3 {
17376 for tx in [false, true] {
17377 for slow_last in [false, true] {
17378 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
17379 let make = |sender| {
17380 let packet = Packet::new(
17381 if sender == "SLOW" { slow_ty } else { ty },
17382 &[ep],
17383 "SAME_BOARD",
17384 1,
17385 Arc::<[u8]>::from([1; 64]),
17386 )
17387 .unwrap();
17388 if mode == 1 {
17389 RouterItem::Packet(packet)
17390 } else {
17391 RouterItem::Packed(crate::shared_bytes::convert(
17392 wire_format::pack_packet(&packet),
17393 ))
17394 }
17395 };
17396 let rx = |data| RouterRxItem {
17397 src: None,
17398 data,
17399 priority: 0,
17400 };
17401 let tx_item = |data| TxQueued {
17402 disposable: true,
17403 item: RouterTxItem::Broadcast(data),
17404 ignore_local: true,
17405 priority: 0,
17406 delivery_state: QueuedDelivery::Untracked,
17407 accepted_type: 0,
17408 deferred_since_ms: 0,
17409 retry_after_ms: 0,
17410 };
17411 let mut st = node.state.lock();
17412 st.memory.max_queue_budget = usize::MAX;
17413 if mode == 0 {
17414 if tx {
17415 st.transmit_queue =
17416 BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
17417 } else {
17418 st.received_queue =
17419 BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
17420 }
17421 } else if mode == 1 {
17422 let cost = if tx {
17423 tx_item(make("SLOW")).byte_cost()
17424 + 2 * tx_item(make("FAST")).byte_cost()
17425 } else {
17426 rx(make("SLOW")).byte_cost() + 2 * rx(make("FAST")).byte_cost()
17427 };
17428 if tx {
17429 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
17430 } else {
17431 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
17432 }
17433 }
17434 let order = if slow_last {
17435 ["FAST", "FAST", "SLOW"]
17436 } else {
17437 ["SLOW", "FAST", "FAST"]
17438 };
17439 for (index, sender) in order
17440 .into_iter()
17441 .chain(core::iter::repeat_n("FAST", 32))
17442 .enumerate()
17443 {
17444 if mode == 2 && index == 3 {
17445 st.memory.max_queue_budget = st.shared_queue_bytes_used();
17446 }
17447 if tx {
17448 st.push_transmit(tx_item(make(sender))).unwrap();
17449 } else {
17450 st.push_received(rx(make(sender))).unwrap();
17451 }
17452 }
17453 let is_slow = |data: &RouterItem| match data {
17454 RouterItem::Packet(p) => p.data_type() == slow_ty,
17455 RouterItem::Packed(b) => {
17456 wire_format::scheduling_type(b.as_ref()) == Some(slow_ty)
17457 }
17458 };
17459 let retained = if tx {
17460 st.transmit_queue.iter().any(|item| {
17461 let data = match &item.item {
17462 RouterTxItem::Broadcast(data)
17463 | RouterTxItem::ToSide { data, .. } => data,
17464 _ => return false,
17465 };
17466 is_slow(data)
17467 })
17468 } else {
17469 st.received_queue.iter().any(|item| is_slow(&item.data))
17470 };
17471 assert!(
17472 retained,
17473 "slow stream displaced: mode={mode}, tx={tx}, slow_last={slow_last}"
17474 );
17475 }
17476 }
17477 }
17478 }
17479
17480 #[test]
17481 fn fast_producer_preserves_slow_reports_at_each_queue_boundary() {
17482 let ep = crate::config::register_endpoint("QUEUE_FAIR_DEST", false).unwrap();
17483 let ty = crate::config::register_data_type_with_description(
17484 "QUEUE_FAIR_REPORT",
17485 "",
17486 MessageElement::Static(64, crate::MessageDataType::UInt8, crate::MessageClass::Data),
17487 &[ep],
17488 crate::ReliableMode::None,
17489 0,
17490 )
17491 .unwrap();
17492 for mode in 0..3 {
17493 for tx in [false, true] {
17494 for slow_last in [false, true] {
17495 for next_sender in ["FAST", "NEW_SLOW"] {
17496 let router =
17497 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
17498 let make = |sender| {
17499 RouterItem::Packet(
17500 Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 64]))
17501 .unwrap(),
17502 )
17503 };
17504 let rx = |data| RouterRxItem {
17505 src: None,
17506 data,
17507 priority: 0,
17508 };
17509 let tx_item = |data| TxQueued {
17510 disposable: true,
17511 item: RouterTxItem::Broadcast(data),
17512 ignore_local: true,
17513 priority: 0,
17514 delivery_state: QueuedDelivery::Untracked,
17515 accepted_type: 0,
17516 deferred_since_ms: 0,
17517 retry_after_ms: 0,
17518 };
17519 let order = if slow_last {
17520 ["FAST", "FAST", "SLOW"]
17521 } else {
17522 ["SLOW", "FAST", "FAST"]
17523 };
17524 let mut st = router.state.lock();
17525 st.memory.max_queue_budget = usize::MAX;
17526 let cost = if tx {
17527 order.iter().map(|s| tx_item(make(s)).byte_cost()).sum()
17528 } else {
17529 order.iter().map(|s| rx(make(s)).byte_cost()).sum()
17530 };
17531 if mode == 0 {
17532 if tx {
17533 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
17534 } else {
17535 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
17536 }
17537 } else if mode == 1 {
17538 if tx {
17539 st.transmit_queue =
17540 BoundedDeque::new(cost * 10, 3 * size_of::<TxQueued>(), 1.0);
17541 } else {
17542 st.received_queue = BoundedDeque::new(
17543 cost * 10,
17544 3 * size_of::<RouterRxItem>(),
17545 1.0,
17546 );
17547 }
17548 }
17549 for sender in order {
17550 if tx {
17551 st.push_transmit(tx_item(make(sender))).unwrap();
17552 } else {
17553 st.push_received(rx(make(sender))).unwrap();
17554 }
17555 }
17556 if mode == 2 {
17557 st.memory.max_queue_budget = st.shared_queue_bytes_used();
17558 }
17559 if tx {
17560 st.push_transmit(tx_item(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
17561 } else {
17562 st.push_received(rx(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
17563 }
17564 let slow = wire_format::source_address_for_sender("SLOW");
17565 let retained = if tx {
17566 st.transmit_queue
17567 .iter()
17568 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
17569 } else {
17570 st.received_queue.iter().any(|item| {
17571 RouterInner::fair_bulk_owner(&item.data, item.priority)
17572 == Some(slow)
17573 })
17574 };
17575 assert!(
17576 retained,
17577 "slow report evicted: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"
17578 );
17579 }
17580 }
17581 }
17582 }
17583 }
17584}
17585
17586#[cfg(all(test, feature = "discovery"))]
17587#[test]
17588fn cow_discovery_rejected_growth_keeps_original_graph_and_routes() {
17589 crate::tests::ensure_common_test_schema();
17590 let node = Router::new_with_clock(RouterConfig::default().with_sender("GB"), Box::new(|| 0));
17591 let side = node.add_side_packed("can", |_| Ok(()));
17592 let baseline = discovery::build_discovery_topology(
17593 "PEER",
17594 0,
17595 &[TopologyBoardNode {
17596 sender_id: "PEER".into(),
17597 reachable_endpoints: Vec::new(),
17598 reachable_timesync_sources: Vec::new(),
17599 connections: Vec::new(),
17600 }],
17601 )
17602 .unwrap();
17603 node.rx_from_side(&baseline, side).unwrap();
17604 node.process_all_queues().unwrap();
17605 node.clear_queues();
17606 let before = node.export_topology();
17607 let original = {
17608 let mut st = node.state.lock();
17609 let graph = st.discovery_routes[&side].announcers["PEER"]
17610 .topology_boards
17611 .clone();
17612 st.memory.max_queue_budget = st.shared_queue_bytes_used();
17613 graph
17614 };
17615 let update = discovery::build_discovery_topology_delta(
17616 "PEER",
17617 1,
17618 &[TopologyBoardNode {
17619 sender_id: "PEER".into(),
17620 reachable_endpoints: Vec::new(),
17621 reachable_timesync_sources: vec!["NEW_CLOCK".into()],
17622 connections: vec!["NEW_NEIGHBOR".into()],
17623 }],
17624 &[],
17625 )
17626 .unwrap();
17627 assert!(
17628 node.rx_from_side(&update, side)
17629 .and_then(|_| node.process_all_queues())
17630 .is_err(),
17631 "growth must fail when no queue budget can be reclaimed"
17632 );
17633 assert_eq!(node.export_topology().routes, before.routes);
17634 let st = node.state.lock();
17635 assert!(Arc::ptr_eq(
17636 &original,
17637 &st.discovery_routes[&side].announcers["PEER"].topology_boards
17638 ));
17639 assert!(original[0].connections.is_empty());
17640 assert!(original[0].reachable_timesync_sources.is_empty());
17641}
17642
17643#[cfg(all(test, feature = "std"))]
17644mod bounded_rx_priority_tests {
17645 use super::*;
17646 #[test]
17647 fn retained_slow_report_cannot_starve_behind_continuous_fast_data() {
17648 let ep =
17649 crate::config::register_endpoint("RouterRxItem_FAIR_EP_RouterInner", false).unwrap();
17650 let ty = crate::config::register_data_type_with_description(
17651 "RouterRxItem_FAIR_DATA_RouterInner",
17652 "",
17653 crate::MessageElement::Static(
17654 1,
17655 crate::MessageDataType::Float32,
17656 crate::MessageClass::Data,
17657 ),
17658 &[ep],
17659 crate::ReliableMode::None,
17660 0,
17661 )
17662 .unwrap();
17663 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
17664 let mut st = node.state.lock();
17665 st.memory.max_queue_budget = 65536;
17666 st.received_queue = BoundedDeque::new(32768, 16 * size_of::<RouterRxItem>(), 1.0);
17667 let make = |sender: &str, stamp, priority| {
17668 let packet = Packet::new(ty, &[ep], sender, stamp, Arc::<[u8]>::from([0; 4])).unwrap();
17669 RouterRxItem {
17670 src: None,
17671 data: RouterItem::Packet(packet),
17672 priority,
17673 }
17674 };
17675 st.push_received(make("VALVE", 0, 0)).unwrap();
17676 let mut delivered = false;
17677 for tick in 1..=32u64 {
17678 for index in 0..3 {
17679 st.push_received(make("DAQ", tick * 3 + index, 16)).unwrap();
17680 }
17681 let item = st.pop_received_fair().unwrap();
17682 let sender = match &item.data {
17683 RouterItem::Packet(packet) => packet.sender(),
17684 _ => panic!("expected packet"),
17685 };
17686 if sender == "VALVE" {
17687 delivered = true;
17688 assert!(tick <= u64::from(MAX_PRIORITY_BURST) + 1);
17689 break;
17690 }
17691 }
17692 assert!(
17693 delivered,
17694 "retention alone does not deliver low-rate reports under a continuous higher-priority source"
17695 );
17696 st.received_queue.clear();
17697 st.rx_priority_burst = MAX_PRIORITY_BURST;
17698 st.push_received(make("VALVE", 100, 0)).unwrap();
17699 st.push_received(make("VALVE", 101, 0)).unwrap();
17700 st.push_received(make("DAQ", 102, 16)).unwrap();
17701 st.push_received(make("COMMAND", 103, 254)).unwrap();
17702 assert_eq!(
17703 st.pop_received_fair().unwrap().priority,
17704 254,
17705 "a fairness turn must never delay protected control"
17706 );
17707 let mut first_slow = None;
17708 for tick in 0..=MAX_PRIORITY_BURST {
17709 st.push_received(make("DAQ", 200 + u64::from(tick), 16))
17710 .unwrap();
17711 let item = st.pop_received_fair().unwrap();
17712 if let RouterItem::Packet(packet) = &item.data
17713 && packet.sender() == "VALVE"
17714 {
17715 first_slow = Some(packet.timestamp());
17716 break;
17717 }
17718 }
17719 assert_eq!(
17720 first_slow,
17721 Some(100),
17722 "fairness preserves same-priority FIFO order"
17723 );
17724 st.rx_priority_burst = MAX_PRIORITY_BURST;
17725 drop(st);
17726 node.clear_rx_queue();
17727 assert_eq!(node.state.lock().rx_priority_burst, 0);
17728 }
17729}
17730
17731#[cfg(all(test, feature = "std", feature = "discovery"))]
17732#[path = "tests/accepted_tx_ownership.rs"]
17733mod accepted_tx_ownership;