1#[cfg(all(test, feature = "std", feature = "discovery"))]
12#[path = "tests/shared_chunk_budget_router.rs"]
13mod shared_chunk_budget_tests;
14#[cfg(test)]
16#[allow(clippy::duplicate_mod)]
17#[path = "tests/side_template_index.rs"]
18mod side_template_index_tests;
19use crate::side_transport::{
20 SIDE_TRANSPORT_FLAG_PACKET_NONCE, SIDE_TRANSPORT_KIND_CHUNK, SIDE_TRANSPORT_KIND_COMPACT,
21 SIDE_TRANSPORT_KIND_COMPACT_DELTA, SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP,
22 SIDE_TRANSPORT_KIND_FULL, SideChunkAssembly, SideCompactTimestampMode, SideHeaderTemplate,
23 SideTransportFrames, extract_side_header_template, extract_side_header_template_view,
24 parse_side_transport_wrapper, read_uleb128_local, reconstruct_side_compact_frame,
25 wrap_side_transport_frame, write_uleb128_local,
26};
27#[cfg(all(test, feature = "discovery"))]
28#[path = "tests/ack_return_identity.rs"]
29mod ack_return_identity;
30
31#[cfg(all(test, feature = "discovery"))]
32#[path = "tests/discovery_liveness.rs"]
33mod discovery_liveness;
34
35#[cfg(all(test, feature = "discovery"))]
36mod compact_summary_tests {
37 use super::*;
38
39 #[test]
40 fn ordered_pending_window_is_bounded_and_exhaustion_cancels_the_tail() {
41 crate::tests::ensure_common_test_schema();
42 let ep = DataEndpoint::named("RADIO");
43 let ty = crate::config::register_data_type_with_description(
44 "ORDERED_WINDOW_REGRESSION",
45 "bounded ordered window",
46 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
47 &[ep],
48 crate::ReliableMode::Ordered,
49 200,
50 )
51 .unwrap();
52 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
53 let clock = now.clone();
54 let router = Router::new_with_clock(
55 RouterConfig::new([]),
56 Box::new(move || clock.load(Ordering::Relaxed)),
57 );
58 let capacity = runtime_reliable_max_end_to_end_pending().max(1);
59 for index in 0..capacity {
60 let item = RouterItem::Packet(
61 Packet::new(
62 ty,
63 &[ep],
64 "SOURCE",
65 index as u64 + 1,
66 Arc::<[u8]>::from([index as u8]),
67 )
68 .unwrap(),
69 );
70 assert_eq!(
71 router.register_end_to_end_reliable_tx(&item, None).unwrap(),
72 index == 0
73 );
74 }
75 let rejected = RouterItem::Packet(
76 Packet::new(
77 ty,
78 &[ep],
79 "SOURCE",
80 capacity as u64 + 1,
81 Arc::<[u8]>::from([0]),
82 )
83 .unwrap(),
84 );
85 assert!(
86 router
87 .register_end_to_end_reliable_tx(&rejected, None)
88 .is_err()
89 );
90 {
91 let mut state = router.state.lock();
92 assert_eq!(state.end_to_end_reliable_tx.len(), capacity);
93 let first = state.end_to_end_reliable_tx_order[0];
94 state
95 .end_to_end_reliable_tx
96 .get_mut(&first)
97 .unwrap()
98 .retries = runtime_reliable_max_retries();
99 }
100 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
101 assert!(router.process_end_to_end_reliable_timeouts().is_err());
102 assert!(router.state.lock().end_to_end_reliable_tx.is_empty());
103 }
104 #[test]
105 fn compact_reachability_survives_incomplete_detailed_topology() {
106 crate::tests::ensure_common_test_schema();
107 let router = Router::new(RouterConfig::new([]).with_sender("GB"));
108 let endpoint = DataEndpoint::try_named("GROUND_STATION")
109 .or_else(|| DataEndpoint::try_named("RADIO"))
110 .unwrap();
111 router.compact_summary_fixture(endpoint);
112 let state = router.state.lock();
113 let now = router.clock.now_ms();
114 let advertised =
115 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(1));
116 assert!(
117 advertised.contains(&endpoint),
118 "compact subscriber route was lost at the bridge"
119 );
120 let reflected =
121 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(0));
122 assert!(
123 !reflected.contains(&endpoint),
124 "split horizon must still exclude ingress"
125 );
126 assert_eq!(
127 router.advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(1)),
128 vec!["clock-behind-bridge".to_string()]
129 );
130 assert!(
131 router
132 .advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(0))
133 .is_empty()
134 );
135 assert!(
136 !router
137 .advertised_discovery_endpoints_for_link_locked(
138 &state,
139 now + DISCOVERY_ROUTE_TTL_MS + 1,
140 true,
141 Some(1)
142 )
143 .contains(&endpoint)
144 );
145 assert!(
146 router
147 .advertised_discovery_timesync_sources_for_link_locked(
148 &state,
149 now + DISCOVERY_ROUTE_TTL_MS + 1,
150 Some(1)
151 )
152 .is_empty()
153 );
154 }
155}
156
157use crate::config::RuntimeMemoryConfig;
158use crate::diagnostics::{
159 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
160 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
161 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
162};
163#[cfg(feature = "discovery")]
164use crate::discovery::{
165 self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
166 DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState,
167 TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS, TopologyAnnouncerRoute, TopologyBoardNode,
168 TopologySideRoute, TopologySnapshot,
169};
170use crate::packet::{hash_bytes_u64, sender_address_u32};
171use crate::queue::{BoundedDeque, ByteCost};
172#[cfg(all(not(feature = "std"), target_os = "none"))]
173use crate::seds_error_msg;
174#[cfg(feature = "timesync")]
175use crate::timesync::{
176 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
177 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
178 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
179 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
180 decode_timesync_request, decode_timesync_response,
181};
182use crate::{
183 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
184 config::{
185 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
186 runtime_reliable_max_end_to_end_pending, runtime_reliable_max_pending,
187 runtime_reliable_max_retries, runtime_reliable_max_return_routes,
188 runtime_reliable_retransmit_ms,
189 },
190 get_needed_message_size, impl_letype_num, is_reliable_type,
191 lock::{ReentryGate, RouterMutex},
192 message_e2e_encryption_policy, message_meta, message_priority,
193 packet::Packet,
194 reliable_mode, wire_format,
195};
196use alloc::string::{String, ToString};
197use alloc::{
198 borrow::ToOwned,
199 boxed::Box,
200 collections::{BTreeMap, BTreeSet, VecDeque},
201 format,
202 sync::Arc,
203 vec,
204 vec::Vec,
205};
206use core::cell::UnsafeCell;
207use core::fmt;
208use core::fmt::{Debug, Formatter};
209use core::mem::size_of;
210use core::ops::{Deref, DerefMut};
211use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
212#[cfg(feature = "std")]
213use std::time::Instant;
214
215pub type RouterSideId = usize;
217
218const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
219pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
220pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
221pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
222static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
223
224#[derive(Clone, Copy, Debug, PartialEq, Eq)]
225pub enum SideTransportProfile {
226 Canonical,
227 Template,
228 Ipv6Like,
229 Ipv4Like,
230}
231
232impl SideTransportProfile {
233 #[inline]
234 pub const fn as_str(self) -> &'static str {
235 match self {
236 Self::Canonical => "canonical",
237 Self::Template => "template",
238 Self::Ipv6Like => "ipv6_like",
239 Self::Ipv4Like => "ipv4_like",
240 }
241 }
242
243 #[cfg(feature = "discovery")]
244 #[inline]
245 pub const fn discovery_code(self) -> u8 {
246 match self {
247 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
248 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
249 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
250 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
251 }
252 }
253}
254
255#[cfg(test)]
256mod chunk_assembly_memory_tests {
257 use super::*;
258 #[test]
259 fn full_chunk_budget_reclaims_other_partial_transfers_before_active_schema() {
260 let mut state = SideTransportState::default();
261 for id in 1..=4 {
262 let mut assembly = SideChunkAssembly {
263 total: 28,
264 last_seen_ms: id as u64,
265 ..Default::default()
266 };
267 for index in 0..23 {
268 assembly.insert(
269 index,
270 crate::shared_bytes::convert(Arc::<[u8]>::from([id as u8; 112])),
271 );
272 }
273 state.rx_chunks.insert(id, assembly);
274 }
275 state.rx_chunks.get_mut(&1).unwrap().insert(
276 23,
277 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 56])),
278 );
279 state.refresh_rx_chunk_bytes();
281 SideChunkAssembly::make_room(
282 &mut state.rx_chunks,
283 &mut state.rx_chunk_bytes,
284 4,
285 23,
286 28,
287 112,
288 16384,
289 )
290 .unwrap();
291 assert!(!state.rx_chunks.contains_key(&1));
292 assert_eq!(state.rx_chunks[&4].received.len(), 23);
293 for index in 23..28 {
294 state.refresh_rx_chunk_bytes();
295 SideChunkAssembly::make_room(
296 &mut state.rx_chunks,
297 &mut state.rx_chunk_bytes,
298 4,
299 index,
300 28,
301 112,
302 16384,
303 )
304 .unwrap();
305 state.rx_chunks.get_mut(&4).unwrap().insert(
306 index,
307 crate::shared_bytes::convert(Arc::<[u8]>::from([4u8; 112])),
308 );
309 }
310 assert_eq!(
311 state.rx_chunks.remove(&4).unwrap().assemble().unwrap(),
312 vec![4u8; 28 * 112]
313 );
314 assert!(
315 state
316 .rx_chunks
317 .values()
318 .flat_map(|a| a.received.values())
319 .map(|p| p.len() + 64)
320 .sum::<usize>()
321 <= 16384
322 );
323 }
324
325 #[test]
326 fn duplicate_chunk_keeps_partial_transfers_but_oversize_transfer_is_rejected() {
327 let mut state = SideTransportState::default();
328 let mut assembly = SideChunkAssembly {
329 total: 2,
330 ..Default::default()
331 };
332 assembly.insert(
333 0,
334 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 112])),
335 );
336 state.rx_chunks.insert(1, assembly.clone());
337 state.rx_chunks.insert(2, assembly);
338 state.refresh_rx_chunk_bytes();
339 SideChunkAssembly::make_room(
340 &mut state.rx_chunks,
341 &mut state.rx_chunk_bytes,
342 1,
343 0,
344 2,
345 112,
346 352,
347 )
348 .unwrap();
349 assert_eq!(state.rx_chunks.len(), 2);
350 state.refresh_rx_chunk_bytes();
351 assert!(
352 SideChunkAssembly::make_room(
353 &mut state.rx_chunks,
354 &mut state.rx_chunk_bytes,
355 1,
356 1,
357 2,
358 400,
359 352
360 )
361 .is_err()
362 );
363 assert!(!state.rx_chunks.contains_key(&1));
364 assert!(state.rx_chunks.contains_key(&2));
365 }
366
367 #[test]
368 fn schema_sized_reassembly_allocates_exactly_and_preserves_order() {
369 let payload: Vec<u8> = (0..3652).map(|i| (i % 251) as u8).collect();
370 let mut assembly = SideChunkAssembly::default();
371 for (index, chunk) in payload.chunks(56).enumerate().rev() {
372 assembly.insert(
373 index as u16,
374 crate::shared_bytes::convert(Arc::<[u8]>::from(chunk)),
375 );
376 }
377 assembly.total = assembly.received.len() as u16;
378 let out = assembly.assemble().unwrap();
379 assert_eq!(out, payload);
380 assert_eq!(
381 out.capacity(),
382 payload.len(),
383 "do not double discovery reassembly capacity"
384 );
385 }
386 #[test]
387 fn incomplete_or_noncontiguous_chunks_are_rejected() {
388 let mut assembly = SideChunkAssembly {
389 total: 2,
390 ..Default::default()
391 };
392 assembly.insert(0, crate::shared_bytes::convert(Arc::<[u8]>::from([1u8])));
393 assert!(assembly.clone().assemble().is_err());
394 assembly.insert(2, crate::shared_bytes::convert(Arc::<[u8]>::from([2u8])));
395 assert!(assembly.assemble().is_err());
396 }
397}
398
399#[derive(Clone, Debug, Default)]
400struct SideTransportState {
401 tx_chunk_progress: Option<Box<crate::side_transport::SideTxProgress>>,
402 tx_template_ids: BTreeMap<u64, u32>,
403 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
404 tx_last_timestamps: BTreeMap<u32, u64>,
405 tx_compact_uses: BTreeMap<u32, u8>,
406 rx_templates: BTreeMap<u64, u32>,
407 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
408 rx_last_timestamps: BTreeMap<u32, u64>,
409 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
410 rx_chunk_reclaims: u32,
411 rx_chunk_bytes: usize,
412}
413
414impl SideTransportState {
415 #[inline(never)]
416 fn take_chunk(&mut self, id: u32, reclaimed: bool) -> Option<SideChunkAssembly> {
417 let value = SideChunkAssembly::remove(&mut self.rx_chunks, &mut self.rx_chunk_bytes, id)?;
418 if reclaimed {
419 self.rx_chunk_reclaims = self.rx_chunk_reclaims.saturating_add(1);
420 }
421 Some(value)
422 }
423 #[cfg(test)]
425 #[inline(never)]
426 fn refresh_rx_chunk_bytes(&mut self) {
427 self.rx_chunk_bytes = self
428 .rx_chunks
429 .values()
430 .map(SideChunkAssembly::byte_cost)
431 .sum();
432 }
433 fn refresh_transmit_header_templates(&mut self) {
434 for uses in self.tx_compact_uses.values_mut() {
438 *uses = u8::MAX;
439 }
440 }
443
444 fn tx_template_count(&self) -> usize {
445 self.tx_template_ids.len()
446 }
447
448 fn rx_template_count(&self) -> usize {
449 self.rx_templates_by_id.len()
450 }
451
452 fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
453 let mut removed = 0usize;
454 while self.tx_template_ids.len() > limit {
455 let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
456 break;
457 };
458 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
459 self.tx_last_timestamps.remove(&old_id);
460 self.tx_compact_uses.remove(&old_id);
461 }
462 self.tx_templates.remove(&old_hash);
463 removed += 1;
464 }
465 removed
466 }
467
468 fn insert_tx_template(
469 &mut self,
470 template: SideHeaderTemplate,
471 template_id: u32,
472 max_templates: usize,
473 ) -> bool {
474 if max_templates == 0 {
475 return false;
476 }
477 let evicted = !self.tx_template_ids.contains_key(&template.hash)
478 && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
479 self.tx_template_ids.insert(template.hash, template_id);
480 self.tx_templates.insert(template.hash, template);
481 self.tx_compact_uses.insert(template_id, 0);
482 evicted
483 }
484
485 fn insert_rx_template(
486 &mut self,
487 template_id: u32,
488 template: SideHeaderTemplate,
489 max_templates: usize,
490 ) -> bool {
491 if max_templates == 0 {
492 return false;
493 }
494 if let Some(old_id) = self.rx_templates.get(&template.hash).copied()
497 && old_id != template_id
498 {
499 self.rx_templates_by_id.remove(&old_id);
500 self.rx_last_timestamps.remove(&old_id);
501 self.rx_templates.remove(&template.hash);
502 }
503 if let Some(previous) = self.rx_templates_by_id.get(&template_id)
504 && previous.hash != template.hash
505 {
506 self.rx_templates.remove(&previous.hash);
507 self.rx_last_timestamps.remove(&template_id);
508 }
509 let mut evicted = false;
510 if self.rx_templates_by_id.len() >= max_templates
511 && !self.rx_templates_by_id.contains_key(&template_id)
512 && let Some((&old_hash, &old_id)) = self.rx_templates.first_key_value()
515 {
516 self.rx_templates.remove(&old_hash);
517 self.rx_templates_by_id.remove(&old_id);
518 self.rx_last_timestamps.remove(&old_id);
519 evicted = true;
520 }
521 self.rx_templates.insert(template.hash, template_id);
522 self.rx_templates_by_id.insert(template_id, template);
523 evicted
524 }
525}
526
527#[derive(Clone, Debug, PartialEq, Eq)]
528pub struct CompactTimestampOmissionPolicy {
529 all: bool,
530 types: Option<Arc<[DataType]>>,
531}
532impl CompactTimestampOmissionPolicy {
533 pub const fn none() -> Self {
534 Self {
535 all: false,
536 types: None,
537 }
538 }
539 pub const fn all() -> Self {
540 Self {
541 all: true,
542 types: None,
543 }
544 }
545 pub fn with_data_type(mut self, ty: DataType) -> Self {
546 self.insert(ty);
547 self
548 }
549 pub fn insert(&mut self, ty: DataType) {
550 if self.contains(ty) {
551 return;
552 }
553 let current = self.types.as_deref().unwrap_or(&[]);
554 let mut types = Vec::with_capacity(current.len() + 1);
555 types.extend_from_slice(current);
556 let index = types.binary_search(&ty).unwrap_err();
557 types.insert(index, ty);
558 self.types = Some(Arc::from(types));
559 }
560 pub fn contains(&self, ty: DataType) -> bool {
561 self.all
562 || self
563 .types
564 .as_deref()
565 .is_some_and(|types| types.binary_search(&ty).is_ok())
566 }
567 pub fn is_empty(&self) -> bool {
568 !self.all && self.types.is_none()
569 }
570}
571
572impl Default for CompactTimestampOmissionPolicy {
573 #[inline]
574 fn default() -> Self {
575 Self::none()
576 }
577}
578
579#[derive(Clone, Debug, PartialEq, Eq)]
580pub enum RouterItem {
581 Packet(Packet),
582 Packed(crate::SharedBytes),
583}
584
585#[cfg(feature = "compact-packet-store")]
586impl RouterItem {
587 fn park_for_queue(&mut self) -> TelemetryResult<()> {
588 match self {
589 Self::Packet(packet) => packet.park_for_queue()?,
590 Self::Packed(bytes) => bytes.park_for_queue()?,
591 }
592 Ok(())
593 }
594}
595
596#[derive(Clone, Debug, PartialEq, Eq)]
597struct RouterRxItem {
598 src: Option<RouterSideId>,
599 data: RouterItem,
600 priority: u8,
601}
602
603#[derive(Clone, Debug, PartialEq, Eq)]
604enum RouterTxItem {
605 Broadcast(RouterItem),
606 EndToEndReplay {
607 packet_id: u64,
608 },
609 ToSide {
610 src: Option<RouterSideId>,
611 dst: RouterSideId,
612 data: RouterItem,
613 },
614 ReliableReplay {
615 dst: RouterSideId,
616 bytes: crate::SharedBytes,
617 },
618}
619
620impl ByteCost for RouterRxItem {
621 #[cfg(feature = "compact-packet-store")]
622 fn park_for_queue(&mut self) -> TelemetryResult<()> {
623 self.data.park_for_queue()
624 }
625 #[inline]
626 fn byte_cost(&self) -> usize {
627 match &self.data {
628 RouterItem::Packet(pkt) => pkt.byte_cost(),
629 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
630 }
631 }
632}
633
634impl ByteCost for RouterTxItem {
635 #[cfg(feature = "compact-packet-store")]
636 fn park_for_queue(&mut self) -> TelemetryResult<()> {
637 match self {
638 Self::Broadcast(item) | Self::ToSide { data: item, .. } => item.park_for_queue(),
639 Self::ReliableReplay { bytes, .. } => bytes.park_for_queue(),
640 _ => Ok(()),
641 }
642 }
643 #[inline]
644 fn byte_cost(&self) -> usize {
645 match self {
646 RouterTxItem::Broadcast(data) => match data {
647 RouterItem::Packet(pkt) => pkt.byte_cost(),
648 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
649 },
650 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
651 RouterTxItem::ToSide { data, .. } => match data {
652 RouterItem::Packet(pkt) => pkt.byte_cost(),
653 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
654 },
655 RouterTxItem::ReliableReplay { bytes, .. } => {
656 size_of::<crate::SharedBytes>() + bytes.len()
657 }
658 }
659 }
660}
661
662#[derive(Clone, Debug, PartialEq, Eq)]
666struct TxQueued {
667 item: RouterTxItem,
668 ignore_local: bool,
669 priority: u8,
670 retry_after_ms: u64,
671}
672
673const MAX_PRIORITY_BURST: u8 = 8;
678
679impl ByteCost for TxQueued {
681 #[cfg(feature = "compact-packet-store")]
682 fn park_for_queue(&mut self) -> TelemetryResult<()> {
683 self.item.park_for_queue()
684 }
685 #[inline]
687 fn byte_cost(&self) -> usize {
688 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>() + size_of::<u64>()
689 }
690}
691
692impl ByteCost for u64 {
694 #[inline]
696 fn byte_cost(&self) -> usize {
697 size_of::<u64>()
698 }
699}
700
701#[derive(Debug, Clone)]
704struct ReliableTxState {
705 next_seq: u32,
706 sent_order: VecDeque<u32>,
707 sent: BTreeMap<u32, ReliableSent>,
708}
709
710#[derive(Debug, Clone)]
711struct ReliableSent {
712 bytes: crate::SharedBytes,
713 last_send_ms: u64,
714 retries: u32,
715 queued: bool,
716 partial_acked: bool,
717}
718
719#[derive(Debug, Clone)]
720struct EndToEndReliableSent {
721 ordered_type: Option<u32>,
722 waiting_for_predecessor: bool,
723 data: RouterItem,
724 pending_destinations: BTreeMap<u64, RouterSideId>,
725 tracked_destinations: bool,
726 last_send_ms: u64,
727 retries: u32,
728 queued: bool,
729}
730
731#[derive(Debug, Clone)]
732struct ReliableRxState {
733 expected_seq: u32,
734 buffered: BTreeMap<u32, crate::SharedBytes>,
735}
736
737#[derive(Debug, Clone)]
738struct ReliableReturnRouteState {
739 side: RouterSideId,
740 last_replay_ms: u64,
741 upstream_contract: Option<(u64, u64)>,
745}
746
747#[cfg(feature = "discovery")]
748#[derive(Debug, Clone, Default, PartialEq, Eq)]
749struct DiscoverySenderState {
750 advertised_address: Option<u32>,
753 reachable: Vec<DataEndpoint>,
754 advertised_reachable: Vec<DataEndpoint>,
755 reachable_network_variables: Vec<DataType>,
756 reachable_timesync_sources: Vec<String>,
757 advertised_reachable_timesync_sources: Vec<String>,
758 link_capabilities: Option<discovery::LinkCapabilities>,
759 topology_boards: Arc<Vec<TopologyBoardNode>>,
760 has_full_topology: bool,
761 has_schema: bool,
762 last_seen_ms: u64,
763}
764
765#[cfg(feature = "discovery")]
766#[derive(Debug, Clone, Default, PartialEq, Eq)]
767struct DiscoverySideState {
768 reachable: Vec<DataEndpoint>,
769 reachable_network_variables: Vec<DataType>,
770 requested_network_variables: Vec<(DataType, u64)>,
771 reachable_timesync_sources: Vec<String>,
772 last_seen_ms: u64,
773 announcers: BTreeMap<String, DiscoverySenderState>,
774}
775
776#[cfg(feature = "discovery")]
777#[derive(Debug, Clone, Default)]
778struct DiscoverySideThrottleState {
779 next_topology_request_ms: u64,
780 next_schema_request_ms: u64,
781 next_ping_ms: u64,
782 last_liveness_tx_ms: u64,
784 pending_incremental: bool,
785 incremental_retries_remaining: u8,
786 has_sent_full: bool,
787 last_topology: Vec<TopologyBoardNode>,
788}
789
790#[cfg(feature = "discovery")]
791impl DiscoverySideThrottleState {
792 fn liveness_due(&self, now_ms: u64) -> bool {
793 now_ms >= self.next_ping_ms
794 && now_ms.saturating_sub(self.last_liveness_tx_ms)
795 >= DISCOVERY_SLOW_LINK_PING_INTERVAL_MS
796 }
797}
798
799#[cfg(all(feature = "discovery", feature = "timesync"))]
800#[derive(Debug, Clone, Default)]
801struct TimeSyncSideThrottleState {
802 next_allowed_ms: u64,
803}
804
805#[cfg(feature = "discovery")]
806#[derive(Debug, Clone, Copy, PartialEq, Eq)]
807enum DiscoveryAdvertiseLevel {
808 MinimalPing,
809 Incremental,
810 Full,
811}
812
813#[derive(Debug, Clone, Default)]
814struct AdaptiveRouteStats {
815 estimated_bandwidth_bps: u64,
816 peak_bandwidth_bps: u64,
817 last_observed_ms: u64,
818 last_slow_observed_ms: u64,
819 sample_count: u64,
820 window_started_ms: u64,
821 window_bytes: u64,
822 peak_usage_bps: u64,
823}
824
825impl AdaptiveRouteStats {
826 #[inline]
827 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
828 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
829 sample_bps
830 } else if sample_bps >= self.estimated_bandwidth_bps {
831 self.estimated_bandwidth_bps
832 .saturating_mul(3)
833 .saturating_add(sample_bps.saturating_mul(5))
834 / 8
835 } else {
836 self.estimated_bandwidth_bps
837 .saturating_mul(7)
838 .saturating_add(sample_bps)
839 / 8
840 };
841 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
842 self.last_observed_ms = now_ms;
843 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
844 self.last_slow_observed_ms = now_ms;
845 }
846 self.sample_count = self.sample_count.saturating_add(1);
847 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
848 self.window_started_ms = now_ms;
849 self.window_bytes = 0;
850 }
851 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
852 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
853 }
854
855 #[inline]
856 fn current_usage_bps(&self, now_ms: u64) -> u64 {
857 if self.window_started_ms == 0 {
858 return 0;
859 }
860 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
861 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
862 .min(u128::from(u64::MAX)) as u64
863 }
864
865 #[inline]
866 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
867 let capacity = self
868 .estimated_bandwidth_bps
869 .max(self.peak_bandwidth_bps)
870 .max(1);
871 capacity.saturating_sub(self.current_usage_bps(now_ms))
872 }
873
874 #[inline]
875 fn weight(&self, now_ms: u64) -> u64 {
876 self.available_headroom_bps(now_ms).max(1)
877 }
878
879 #[inline]
880 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
881 let current_usage_bps = self.current_usage_bps(now_ms);
882 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
883 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
884 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
885 AdaptiveLinkStats {
886 auto_balancing_enabled,
887 estimated_capacity_bps,
888 peak_capacity_bps,
889 current_usage_bps,
890 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
891 available_headroom_bps,
892 effective_weight: available_headroom_bps.max(1),
893 last_observed_ms: self.last_observed_ms,
894 sample_count: self.sample_count,
895 }
896 }
897}
898
899use crate::type_stats::TypeRuntimeStatsInner;
900
901#[derive(Debug, Clone, Default)]
902struct SideRuntimeStatsInner {
903 tx_packets: u64,
904 tx_bytes: u64,
905 rx_packets: u64,
906 rx_bytes: u64,
907 relayed_tx_packets: u64,
908 relayed_tx_bytes: u64,
909 relayed_rx_packets: u64,
910 relayed_rx_bytes: u64,
911 local_delivery_packets: u64,
912 tx_retries: u64,
913 tx_handler_failures: u64,
914 local_handler_failures: u64,
915 total_handler_retries: u64,
916 side_transport_full_frames: u64,
917 side_transport_compact_frames: u64,
918 side_transport_compact_delta_frames: u64,
919 side_transport_compact_omitted_timestamp_frames: u64,
920 side_transport_chunk_frames: u64,
921 side_transport_raw_bytes: u64,
922 side_transport_wire_bytes: u64,
923 side_transport_bytes_saved: u64,
924 side_transport_min_compact_overhead_bytes: Option<usize>,
925 side_transport_max_compact_overhead_bytes: Option<usize>,
926 side_transport_compact_target_misses: u64,
927 side_transport_template_evictions: u64,
928 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
929}
930
931impl SideRuntimeStatsInner {
932 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
933 self.data_types.entry(ty.as_u32()).or_default()
934 }
935
936 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
937 self.tx_packets = self.tx_packets.saturating_add(1);
938 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
939 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
940 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
941 if relayed {
942 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
943 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
944 }
945 self.type_stats_mut(ty)
946 .tx(bytes as u64, relayed, retries as u64);
947 }
948
949 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
950 self.rx_packets = self.rx_packets.saturating_add(1);
951 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
952 if relayed {
953 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
954 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
955 }
956 self.type_stats_mut(ty).rx(bytes as u64, relayed);
957 }
958
959 fn note_local_delivery(&mut self, ty: DataType) {
960 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
961 self.type_stats_mut(ty).rx(0, false);
962 }
963
964 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
965 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
966 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
967 self.type_stats_mut(ty).failure(0);
968 }
969
970 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
971 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
972 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
973 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
974 self.type_stats_mut(ty).failure(retries as u64);
975 }
976
977 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
978 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
979 self.note_side_transport_bytes(raw_bytes, wire_bytes);
980 }
981
982 fn note_side_transport_compact(
983 &mut self,
984 raw_bytes: usize,
985 wire_bytes: usize,
986 compact_overhead_bytes: usize,
987 used_timestamp_delta: bool,
988 omitted_timestamp: bool,
989 ) {
990 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
991 if used_timestamp_delta {
992 self.side_transport_compact_delta_frames =
993 self.side_transport_compact_delta_frames.saturating_add(1);
994 }
995 if omitted_timestamp {
996 self.side_transport_compact_omitted_timestamp_frames = self
997 .side_transport_compact_omitted_timestamp_frames
998 .saturating_add(1);
999 }
1000 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1001 self.side_transport_min_compact_overhead_bytes = Some(
1002 self.side_transport_min_compact_overhead_bytes
1003 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
1004 );
1005 self.side_transport_max_compact_overhead_bytes = Some(
1006 self.side_transport_max_compact_overhead_bytes
1007 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
1008 );
1009 }
1010
1011 fn note_side_transport_chunks(&mut self, chunks: usize) {
1012 self.side_transport_chunk_frames = self
1013 .side_transport_chunk_frames
1014 .saturating_add(chunks as u64);
1015 }
1016
1017 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
1018 self.side_transport_raw_bytes = self
1019 .side_transport_raw_bytes
1020 .saturating_add(raw_bytes as u64);
1021 self.side_transport_wire_bytes = self
1022 .side_transport_wire_bytes
1023 .saturating_add(wire_bytes as u64);
1024 if raw_bytes > wire_bytes {
1025 self.side_transport_bytes_saved = self
1026 .side_transport_bytes_saved
1027 .saturating_add((raw_bytes - wire_bytes) as u64);
1028 }
1029 }
1030
1031 fn note_side_transport_compact_target_miss(&mut self) {
1032 self.side_transport_compact_target_misses =
1033 self.side_transport_compact_target_misses.saturating_add(1);
1034 }
1035
1036 fn note_side_transport_template_eviction(&mut self) {
1037 self.side_transport_template_evictions =
1038 self.side_transport_template_evictions.saturating_add(1);
1039 }
1040}
1041
1042#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1043enum RouteSelectionOrigin {
1044 Flood,
1045 Discovered,
1046}
1047
1048type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1051
1052type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
1054type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
1055
1056#[derive(Clone)]
1062pub enum EndpointHandlerFn {
1063 Packet(Arc<PacketHandlerFn>),
1064 Packed(Arc<PackedHandlerFn>),
1065}
1066
1067pub struct EndpointHandler {
1069 endpoint: DataEndpoint,
1070 handler: EndpointHandlerFn,
1071}
1072
1073impl Debug for EndpointHandlerFn {
1075 #[inline]
1076 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1077 match self {
1078 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
1079 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
1080 }
1081 }
1082}
1083
1084#[derive(Clone)]
1086pub enum RouterTxHandlerFn {
1087 Packed(Arc<PackedHandlerFn>),
1088 PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
1089 Packet(Arc<PacketHandlerFn>),
1090}
1091
1092impl Debug for RouterTxHandlerFn {
1093 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1094 match self {
1095 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
1096 RouterTxHandlerFn::PackedWithPriority(_) => f
1097 .debug_tuple("PackedWithPriority")
1098 .field(&"<handler>")
1099 .finish(),
1100 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
1101 }
1102 }
1103}
1104
1105#[derive(Clone, Debug)]
1106pub struct RouterSideOptions {
1107 pub reliable_enabled: bool,
1118 pub link_local_enabled: bool,
1120 pub header_template_enabled: bool,
1127 pub max_frame_bytes: usize,
1133 pub compact_header_target_bytes: usize,
1140 pub max_side_transport_templates: usize,
1146 pub omit_unchanged_compact_timestamps: bool,
1152 pub compact_timestamp_omission_types: CompactTimestampOmissionPolicy,
1157 pub side_transport_profile: SideTransportProfile,
1159 pub ingress_enabled: bool,
1161 pub egress_enabled: bool,
1163}
1164
1165impl Default for RouterSideOptions {
1166 fn default() -> Self {
1167 Self {
1168 reliable_enabled: false,
1169 link_local_enabled: false,
1170 header_template_enabled: false,
1171 max_frame_bytes: 0,
1172 compact_header_target_bytes: 0,
1173 max_side_transport_templates: DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT,
1174 omit_unchanged_compact_timestamps: false,
1175 compact_timestamp_omission_types: CompactTimestampOmissionPolicy::none(),
1176 side_transport_profile: SideTransportProfile::Canonical,
1177 ingress_enabled: true,
1178 egress_enabled: true,
1179 }
1180 }
1181}
1182
1183impl RouterSideOptions {
1184 #[inline]
1189 pub fn with_small_packet_transport(mut self, max_frame_bytes: usize) -> Self {
1190 self.header_template_enabled = true;
1191 self.max_frame_bytes = max_frame_bytes;
1192 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
1193 self.side_transport_profile = SideTransportProfile::Ipv6Like;
1194 self
1195 }
1196
1197 #[inline]
1198 pub fn with_ipv4_like_compact_header_target(mut self) -> Self {
1199 self.header_template_enabled = true;
1200 self.compact_header_target_bytes = IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES;
1201 self.omit_unchanged_compact_timestamps = true;
1202 self.side_transport_profile = SideTransportProfile::Ipv4Like;
1203 self
1204 }
1205
1206 #[inline]
1207 pub fn with_ipv6_like_compact_header_target(mut self) -> Self {
1208 self.header_template_enabled = true;
1209 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
1210 self.side_transport_profile = SideTransportProfile::Ipv6Like;
1211 self
1212 }
1213
1214 #[inline]
1215 pub fn with_template_transport(mut self) -> Self {
1216 self.header_template_enabled = true;
1217 self.side_transport_profile = SideTransportProfile::Template;
1218 self
1219 }
1220
1221 #[inline]
1222 pub fn with_omitted_unchanged_compact_timestamps(mut self) -> Self {
1223 self.header_template_enabled = true;
1224 self.omit_unchanged_compact_timestamps = true;
1225 self
1226 }
1227
1228 #[inline]
1229 pub fn with_omitted_unchanged_compact_timestamps_for_type(mut self, ty: DataType) -> Self {
1230 self.header_template_enabled = true;
1231 self.compact_timestamp_omission_types.insert(ty);
1232 self
1233 }
1234
1235 #[inline]
1236 pub fn effective_transport_profile(&self) -> SideTransportProfile {
1237 if !self.header_template_enabled && self.max_frame_bytes == 0 {
1238 SideTransportProfile::Canonical
1239 } else if self.side_transport_profile == SideTransportProfile::Canonical {
1240 SideTransportProfile::Template
1241 } else {
1242 self.side_transport_profile
1243 }
1244 }
1245
1246 #[cfg(feature = "discovery")]
1247 #[inline]
1248 pub fn link_capabilities(&self) -> discovery::LinkCapabilities {
1249 let mut flags = discovery::LINK_CAPABILITY_END_TO_END_RELIABILITY;
1250 if self.header_template_enabled {
1251 flags |= discovery::LINK_CAPABILITY_HEADER_TEMPLATES;
1252 }
1253 if self.max_frame_bytes != 0 {
1254 flags |= discovery::LINK_CAPABILITY_CHUNKING;
1255 }
1256 if self.reliable_enabled {
1257 flags |= discovery::LINK_CAPABILITY_RELIABILITY;
1258 }
1259 if self.omit_unchanged_compact_timestamps
1260 || !self.compact_timestamp_omission_types.is_empty()
1261 {
1262 flags |= discovery::LINK_CAPABILITY_OMIT_UNCHANGED_TIMESTAMPS;
1263 }
1264 #[cfg(feature = "cryptography")]
1265 {
1266 flags |= discovery::LINK_CAPABILITY_CRYPTO;
1267 }
1268 discovery::LinkCapabilities {
1269 version: 1,
1270 flags,
1271 profile: self.effective_transport_profile().discovery_code(),
1272 max_frame_bytes: self.max_frame_bytes.min(u32::MAX as usize) as u32,
1273 compact_header_target_bytes: self.compact_header_target_bytes.min(u32::MAX as usize)
1274 as u32,
1275 max_side_transport_templates: self.max_side_transport_templates.min(u32::MAX as usize)
1276 as u32,
1277 }
1278 }
1279}
1280
1281#[derive(Clone, Debug)]
1283pub struct RouterSide {
1284 pub name: Arc<str>,
1285 pub tx_handler: RouterTxHandlerFn,
1286 pub opts: RouterSideOptions,
1287}
1288
1289impl Debug for EndpointHandler {
1291 #[inline]
1292 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1293 f.debug_struct("EndpointHandler")
1294 .field("endpoint", &self.endpoint)
1295 .field("handler", &self.handler)
1296 .finish()
1297 }
1298}
1299
1300#[inline]
1301pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1302 #[cfg(feature = "timesync")]
1303 if matches!(endpoint, DataEndpoint::TimeSync) {
1304 return true;
1305 }
1306 #[cfg(feature = "discovery")]
1307 {
1308 discovery::is_discovery_endpoint(endpoint)
1309 }
1310 #[cfg(not(feature = "discovery"))]
1311 {
1312 let _ = endpoint;
1313 false
1314 }
1315}
1316
1317impl EndpointHandler {
1318 #[inline]
1322 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1323 where
1324 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1325 {
1326 assert!(
1327 !endpoint_is_router_internal(endpoint),
1328 "reserved internal endpoint handlers must not be user-registered"
1329 );
1330 #[cfg(feature = "std")]
1331 crate::config::ensure_endpoint_id(endpoint, false)
1332 .expect("endpoint handler endpoint registration failed");
1333 Self {
1334 endpoint,
1335 handler: EndpointHandlerFn::Packet(Arc::new(f)),
1336 }
1337 }
1338
1339 #[inline]
1341 #[cfg(feature = "std")]
1342 pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1343 where
1344 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1345 {
1346 Self::new_packet_handler(endpoint.id, f)
1347 }
1348
1349 #[cfg(not(feature = "std"))]
1351 #[inline]
1352 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1353 where
1354 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1355 {
1356 Self::new_packet_handler(endpoint.id, f)
1357 }
1358
1359 #[cfg(feature = "std")]
1361 #[inline]
1362 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1363 where
1364 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1365 {
1366 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1367 .ok_or(TelemetryError::BadArg)?;
1368 Ok(Self::new_packet_handler(endpoint.id, f))
1369 }
1370
1371 #[inline]
1375 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1376 where
1377 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1378 {
1379 assert!(
1380 !endpoint_is_router_internal(endpoint),
1381 "reserved internal endpoint handlers must not be user-registered"
1382 );
1383 #[cfg(feature = "std")]
1384 crate::config::ensure_endpoint_id(endpoint, false)
1385 .expect("endpoint handler endpoint registration failed");
1386 Self {
1387 endpoint,
1388 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1389 }
1390 }
1391
1392 #[inline]
1394 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1395 where
1396 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1397 {
1398 Self::new_packed_handler(endpoint.id, f)
1399 }
1400
1401 #[cfg(feature = "std")]
1403 #[inline]
1404 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1405 where
1406 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1407 {
1408 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1409 .ok_or(TelemetryError::BadArg)?;
1410 Ok(Self::new_packed_handler(endpoint.id, f))
1411 }
1412
1413 #[inline]
1415 pub fn get_endpoint(&self) -> DataEndpoint {
1416 self.endpoint
1417 }
1418
1419 #[inline]
1421 pub fn get_handler(&self) -> &EndpointHandlerFn {
1422 &self.handler
1423 }
1424}
1425
1426pub trait Clock {
1427 fn now_ms(&self) -> u64;
1429
1430 fn now_ns(&self) -> u64 {
1434 self.now_ms().saturating_mul(1_000_000)
1435 }
1436}
1437
1438impl<T: Fn() -> u64> Clock for T {
1439 #[inline]
1440 fn now_ms(&self) -> u64 {
1441 self()
1442 }
1443}
1444
1445#[cfg(feature = "std")]
1446#[derive(Debug)]
1447struct StdMonotonicClock {
1448 start: Instant,
1449}
1450
1451#[cfg(feature = "std")]
1452impl Default for StdMonotonicClock {
1453 fn default() -> Self {
1454 Self {
1455 start: Instant::now(),
1456 }
1457 }
1458}
1459
1460#[cfg(feature = "std")]
1461impl Clock for StdMonotonicClock {
1462 fn now_ms(&self) -> u64 {
1463 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1464 }
1465
1466 fn now_ns(&self) -> u64 {
1467 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1468 }
1469}
1470
1471#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1473pub enum RouterE2eEncryptionMode {
1474 Disabled,
1476 RequiredOnly,
1478 Preferred,
1480 ForceAll,
1482}
1483
1484pub type NodeAddress = u32;
1485pub type P2pPort = u16;
1486pub type P2pStreamId = u32;
1487
1488const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1489const P2P_STREAM_VERSION: u8 = 1;
1490const P2P_STREAM_SYN: u8 = 0x01;
1491const P2P_STREAM_ACK: u8 = 0x02;
1492const P2P_STREAM_FIN: u8 = 0x04;
1493const P2P_STREAM_RST: u8 = 0x08;
1494const P2P_STREAM_DATA: u8 = 0x10;
1495
1496#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1497pub enum AddressAssignmentMode {
1498 Dynamic,
1499 Requested(NodeAddress),
1500 Static(NodeAddress),
1501}
1502
1503impl AddressAssignmentMode {
1504 #[inline]
1505 fn mode_code(self) -> u8 {
1506 match self {
1507 Self::Dynamic => 0,
1508 Self::Requested(_) => 1,
1509 Self::Static(_) => 2,
1510 }
1511 }
1512
1513 #[inline]
1514 fn requested_address(self) -> NodeAddress {
1515 match self {
1516 Self::Dynamic => 0,
1517 Self::Requested(addr) | Self::Static(addr) => addr,
1518 }
1519 }
1520}
1521
1522#[derive(Debug, Clone, PartialEq, Eq)]
1523pub enum AddressChangeReason {
1524 Configured,
1525 DynamicConflict,
1526 RequestedConflict,
1527 StaticConflict,
1528 HostnameConflict,
1529}
1530
1531#[derive(Debug, Clone, PartialEq, Eq)]
1532pub struct AddressChange {
1533 pub old_hostname: Arc<str>,
1534 pub new_hostname: Arc<str>,
1535 pub old_address: NodeAddress,
1536 pub new_address: NodeAddress,
1537 pub reason: AddressChangeReason,
1538}
1539
1540#[derive(Debug, Clone, PartialEq, Eq)]
1541pub struct AddressBookEntry {
1542 pub hostname: Arc<str>,
1543 pub address: NodeAddress,
1544 pub requested_address: NodeAddress,
1545 pub mode: AddressAssignmentMode,
1546 pub birth_ms: u64,
1547 pub owner_hash: u64,
1548 pub last_seen_ms: u64,
1549}
1550
1551pub struct P2pMessage<'a> {
1552 pub source_hostname: &'a str,
1553 pub source_address: NodeAddress,
1554 pub source_port: P2pPort,
1555 pub destination_port: P2pPort,
1556 pub payload: &'a [u8],
1557}
1558
1559#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1560pub enum P2pStreamEventKind {
1561 Accepted,
1562 Connected,
1563 Data,
1564 Closed,
1565 Reset,
1566}
1567
1568pub struct P2pStreamEvent<'a> {
1569 pub kind: P2pStreamEventKind,
1570 pub stream_id: P2pStreamId,
1571 pub peer_stream_id: P2pStreamId,
1572 pub sequence: u32,
1573 pub peer_hostname: &'a str,
1574 pub peer_address: NodeAddress,
1575 pub local_port: P2pPort,
1576 pub peer_port: P2pPort,
1577 pub payload: &'a [u8],
1578}
1579
1580struct P2pDecoded<'a> {
1581 source_hostname: &'a str,
1582 source_address: NodeAddress,
1583 source_port: P2pPort,
1584 destination_port: P2pPort,
1585 payload: &'a [u8],
1586}
1587
1588#[derive(Debug, Clone)]
1589struct P2pStreamDecoded<'a> {
1590 flags: u8,
1591 source_stream_id: P2pStreamId,
1592 destination_stream_id: P2pStreamId,
1593 sequence: u32,
1594 payload: &'a [u8],
1595}
1596
1597type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1598
1599#[derive(Clone)]
1600struct AddressChangeHandler {
1601 handler: Arc<AddressChangeFn>,
1602}
1603
1604impl Debug for AddressChangeHandler {
1605 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1606 f.write_str("AddressChangeHandler(<handler>)")
1607 }
1608}
1609
1610type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1611
1612#[derive(Clone)]
1613struct P2pPortHandler {
1614 handler: Arc<P2pPortHandlerFn>,
1615}
1616
1617impl Debug for P2pPortHandler {
1618 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1619 f.write_str("P2pPortHandler(<handler>)")
1620 }
1621}
1622
1623type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1624
1625#[derive(Clone)]
1626struct P2pStreamHandler {
1627 handler: Arc<P2pStreamHandlerFn>,
1628}
1629
1630impl Debug for P2pStreamHandler {
1631 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1632 f.write_str("P2pStreamHandler(<handler>)")
1633 }
1634}
1635
1636#[derive(Debug, Clone)]
1637struct P2pStreamSession {
1638 peer_hostname: Arc<str>,
1639 peer_address: NodeAddress,
1640 local_port: P2pPort,
1641 peer_port: P2pPort,
1642 peer_stream_id: P2pStreamId,
1643 next_sequence: u32,
1644 connected: bool,
1645}
1646
1647#[derive(Debug, Clone)]
1648struct PendingP2pStreamEvent {
1649 handlers: Vec<P2pStreamHandler>,
1650 kind: P2pStreamEventKind,
1651 stream_id: P2pStreamId,
1652 peer_stream_id: P2pStreamId,
1653 peer_hostname: Arc<str>,
1654 peer_address: NodeAddress,
1655 local_port: P2pPort,
1656 peer_port: P2pPort,
1657}
1658
1659#[derive(Debug, Clone)]
1660pub struct RouterConfig {
1661 handlers: Arc<[EndpointHandler]>,
1663 reliable_enabled: bool,
1665 sender: Option<Arc<str>>,
1667 address_mode: AddressAssignmentMode,
1669 address_change_handlers: Arc<[AddressChangeHandler]>,
1671 e2e_encryption: RouterE2eEncryptionMode,
1673 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
1675 e2e_key_id: u32,
1676 memory: RuntimeMemoryConfig,
1677 #[cfg(feature = "discovery")]
1678 preferred_discovery_master: Option<Arc<str>>,
1679 #[cfg(feature = "timesync")]
1680 timesync: Option<TimeSyncConfig>,
1681}
1682
1683impl RouterConfig {
1684 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
1689 #[cfg(feature = "cryptography")]
1690 {
1691 RouterE2eEncryptionMode::Preferred
1692 }
1693 #[cfg(not(feature = "cryptography"))]
1694 {
1695 RouterE2eEncryptionMode::Disabled
1696 }
1697 }
1698
1699 pub fn new<H>(handlers: H) -> Self
1701 where
1702 H: Into<Arc<[EndpointHandler]>>,
1703 {
1704 let handlers: Arc<[EndpointHandler]> = handlers.into();
1705 assert!(
1706 handlers
1707 .iter()
1708 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
1709 "reserved internal endpoint handlers must not be user-registered"
1710 );
1711 Self {
1712 handlers,
1713 reliable_enabled: true,
1714 sender: None,
1715 address_mode: AddressAssignmentMode::Dynamic,
1716 address_change_handlers: Arc::from([]),
1717 e2e_encryption: Self::default_e2e_encryption_mode(),
1718 e2e_key_id: 0,
1719 memory: RuntimeMemoryConfig::default(),
1720 #[cfg(feature = "discovery")]
1721 preferred_discovery_master: None,
1722 #[cfg(feature = "timesync")]
1723 timesync: None,
1724 }
1725 }
1726
1727 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
1729 self.reliable_enabled = enabled;
1730 self
1731 }
1732
1733 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
1735 self.sender = Some(Arc::from(sender.as_ref()));
1736 self
1737 }
1738
1739 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
1741 self.with_sender(hostname)
1742 }
1743
1744 pub fn with_dynamic_address(mut self) -> Self {
1746 self.address_mode = AddressAssignmentMode::Dynamic;
1747 self
1748 }
1749
1750 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
1752 self.address_mode = AddressAssignmentMode::Requested(address);
1753 self
1754 }
1755
1756 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
1758 self.address_mode = AddressAssignmentMode::Static(address);
1759 self
1760 }
1761
1762 pub fn on_address_change<F>(mut self, f: F) -> Self
1764 where
1765 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
1766 {
1767 let mut handlers = self.address_change_handlers.to_vec();
1768 handlers.push(AddressChangeHandler {
1769 handler: Arc::new(f),
1770 });
1771 self.address_change_handlers = Arc::from(handlers);
1772 self
1773 }
1774
1775 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
1777 self.e2e_encryption = mode;
1778 self
1779 }
1780
1781 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
1783 self.e2e_key_id = key_id;
1784 self
1785 }
1786
1787 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
1789 memory.validate()?;
1790 self.memory = memory;
1791 Ok(self)
1792 }
1793
1794 #[cfg(feature = "discovery")]
1800 pub fn with_preferred_discovery_master<S: AsRef<str>>(mut self, hostname: S) -> Self {
1801 let hostname = hostname.as_ref();
1802 self.preferred_discovery_master = if hostname.is_empty() {
1803 None
1804 } else {
1805 Some(Arc::from(hostname))
1806 };
1807 self
1808 }
1809
1810 #[cfg(feature = "timesync")]
1811 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
1813 self.timesync = Some(cfg);
1814 self
1815 }
1816
1817 #[inline]
1818 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1820 if endpoint_is_router_internal(ep) {
1821 return false;
1822 }
1823 self.handlers.iter().any(|h| h.endpoint == ep)
1824 }
1825
1826 #[inline]
1827 fn reliable_enabled(&self) -> bool {
1828 self.reliable_enabled
1829 }
1830
1831 #[inline]
1832 fn sender(&self) -> String {
1833 self.sender
1834 .as_deref()
1835 .map(ToString::to_string)
1836 .unwrap_or_else(runtime_device_identifier)
1837 }
1838
1839 #[inline]
1840 fn address_mode(&self) -> AddressAssignmentMode {
1841 self.address_mode
1842 }
1843
1844 #[inline]
1845 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1846 self.e2e_encryption
1847 }
1848
1849 #[cfg(feature = "cryptography")]
1850 #[inline]
1851 fn e2e_key_id(&self) -> u32 {
1852 self.e2e_key_id
1853 }
1854
1855 #[cfg(feature = "timesync")]
1856 #[inline]
1857 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1858 self.timesync
1859 }
1860
1861 #[inline]
1862 fn memory_config(&self) -> RuntimeMemoryConfig {
1863 self.memory
1864 }
1865}
1866
1867impl Default for RouterConfig {
1868 fn default() -> Self {
1869 Self {
1870 handlers: Arc::from([]),
1871 reliable_enabled: true,
1872 sender: None,
1873 address_mode: AddressAssignmentMode::Dynamic,
1874 address_change_handlers: Arc::from([]),
1875 e2e_encryption: Self::default_e2e_encryption_mode(),
1876 e2e_key_id: 0,
1877 memory: RuntimeMemoryConfig::default(),
1878 #[cfg(feature = "discovery")]
1879 preferred_discovery_master: None,
1880 #[cfg(feature = "timesync")]
1881 timesync: None,
1882 }
1883 }
1884}
1885
1886pub trait LeBytes: Copy + Sized {
1889 const WIDTH: usize;
1890 fn write_le(self, out: &mut [u8]);
1891 fn from_le_slice(bytes: &[u8]) -> Self;
1892}
1893
1894impl_letype_num!(u8, 1);
1895impl_letype_num!(u16, 2);
1896impl_letype_num!(u32, 4);
1897impl_letype_num!(u64, 8);
1898impl_letype_num!(u128, 16);
1899impl_letype_num!(i8, 1);
1900impl_letype_num!(i16, 2);
1901impl_letype_num!(i32, 4);
1902impl_letype_num!(i64, 8);
1903impl_letype_num!(i128, 16);
1904impl_letype_num!(f32, 4);
1905impl_letype_num!(f64, 8);
1906
1907pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
1909 let total = data.len() * T::WIDTH;
1910 let mut buf = vec![0u8; total];
1911
1912 for (i, v) in data.iter().copied().enumerate() {
1913 let start = i * T::WIDTH;
1914 v.write_le(&mut buf[start..start + T::WIDTH]);
1915 }
1916
1917 Arc::from(buf)
1918}
1919
1920fn make_error_payload(msg: &str) -> crate::SharedBytes {
1923 let meta = message_meta(DataType::TelemetryError);
1924 match meta.element {
1925 MessageElement::Static(_, _, _) => {
1926 let max = get_needed_message_size(DataType::TelemetryError);
1927 let bytes = msg.as_bytes();
1928 let n = core::cmp::min(max, bytes.len());
1929 let mut buf = vec![0u8; max];
1930 if n > 0 {
1931 buf[..n].copy_from_slice(&bytes[..n]);
1932 }
1933 crate::shared_bytes::convert(Arc::<[u8]>::from(buf))
1934 }
1935 MessageElement::Dynamic(_, _) => {
1936 crate::shared_bytes::convert(Arc::<[u8]>::from(msg.as_bytes()))
1937 }
1938 }
1939}
1940
1941fn log_raw<T, F>(
1945 sender: &str,
1946 ty: DataType,
1947 data: &[T],
1948 timestamp: u64,
1949 mut tx_function: F,
1950) -> TelemetryResult<()>
1951where
1952 T: LeBytes,
1953 F: FnMut(Packet) -> TelemetryResult<()>,
1954{
1955 let meta = message_meta(ty);
1956 let got = data.len() * T::WIDTH;
1957
1958 match meta.element {
1959 MessageElement::Static(_, _, _) => {
1960 if got != get_needed_message_size(ty) {
1961 return Err(TelemetryError::SizeMismatch {
1962 expected: get_needed_message_size(ty),
1963 got,
1964 });
1965 }
1966 }
1967 MessageElement::Dynamic(_, _) => {
1968 if !got.is_multiple_of(T::WIDTH) {
1970 return Err(TelemetryError::SizeMismatch {
1971 expected: T::WIDTH,
1972 got,
1973 });
1974 }
1975 }
1976 }
1977
1978 let payload = encode_slice_le(data);
1979 let pkt = Packet::new(ty, meta.endpoints_ref(), sender, timestamp, payload)?;
1980 tx_function(pkt)
1981}
1982
1983#[cfg(feature = "std")]
1985fn reliable_trace_enabled() -> bool {
1986 static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1987 *ENABLED.get_or_init(|| {
1988 matches!(
1989 std::env::var("SEDSNET_TRACE_RELIABLE").as_deref(),
1990 Ok("1" | "true")
1991 )
1992 })
1993}
1994
1995fn fallback_stdout(_msg: &str) {
1999 #[cfg(feature = "std")]
2000 {
2001 eprintln!("{}", _msg);
2002 }
2003 #[cfg(all(not(feature = "std"), target_os = "none"))]
2004 {
2005 let message = format!("{}\n", _msg);
2006 unsafe {
2007 seds_error_msg(message.as_ptr(), message.len());
2008 }
2009 }
2010}
2011
2012#[derive(Debug, Clone)]
2017struct RouterInner {
2018 memory: RuntimeMemoryConfig,
2019 sides: Vec<Option<RouterSide>>,
2020 route_overrides: BTreeMap<(Option<RouterSideId>, RouterSideId), bool>,
2021 typed_route_overrides: BTreeMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
2022 route_weights: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
2023 route_priorities: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
2024 source_route_modes: BTreeMap<Option<RouterSideId>, RouteSelectionMode>,
2025 route_selection_cursors: BTreeMap<Option<RouterSideId>, u64>,
2026 adaptive_route_stats: BTreeMap<RouterSideId, AdaptiveRouteStats>,
2027 side_runtime_stats: BTreeMap<RouterSideId, Box<SideRuntimeStatsInner>>,
2030 side_transport: BTreeMap<RouterSideId, Box<SideTransportState>>,
2031 managed_variable_types: BTreeSet<u32>,
2032 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
2033 managed_variable_latest: BTreeMap<u32, Box<ManagedVariableCacheEntry>>,
2034 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
2035 local_address: AddressBookEntry,
2036 address_book: BTreeMap<String, AddressBookEntry>,
2037 address_by_value: BTreeMap<NodeAddress, String>,
2038 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
2039 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
2040 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
2041 next_p2p_stream_id: P2pStreamId,
2042 received_queue: BoundedDeque<RouterRxItem>,
2043 transmit_queue: BoundedDeque<TxQueued>,
2044 tx_priority_burst: u8,
2045 recent_rx: BoundedDeque<u64>,
2046 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
2047 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
2048 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
2049 reliable_return_route_order: VecDeque<u64>,
2050 end_to_end_reliable_tx: BTreeMap<u64, EndToEndReliableSent>,
2051 end_to_end_reliable_tx_order: VecDeque<u64>,
2052 total_handler_failures: u64,
2053 total_handler_retries: u64,
2054 #[cfg(feature = "discovery")]
2055 discovery_routes: BTreeMap<RouterSideId, DiscoverySideState>,
2056 #[cfg(feature = "discovery")]
2057 discovery_cadence: DiscoveryCadenceState,
2058 #[cfg(feature = "discovery")]
2059 discovery_side_throttle: BTreeMap<RouterSideId, DiscoverySideThrottleState>,
2060 #[cfg(feature = "discovery")]
2061 preferred_discovery_master: Option<Arc<str>>,
2062 #[cfg(all(feature = "discovery", feature = "timesync"))]
2063 timesync_side_throttle: BTreeMap<RouterSideId, TimeSyncSideThrottleState>,
2064}
2065
2066#[derive(Debug, Clone)]
2067struct ManagedVariableCacheEntry {
2068 packet: Packet,
2069 sender_address: u32,
2070 cached_at_ms: u64,
2071}
2072
2073type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
2074
2075#[derive(Clone)]
2076struct NetworkVariableUpdateHandler {
2077 handler: Arc<NetworkVariableUpdateFn>,
2078}
2079
2080impl Debug for NetworkVariableUpdateHandler {
2081 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2082 f.write_str("NetworkVariableUpdateHandler(<handler>)")
2083 }
2084}
2085
2086#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2088pub struct NetworkVariablePermissions {
2089 pub read: bool,
2090 pub write: bool,
2091}
2092
2093impl NetworkVariablePermissions {
2094 pub const NONE: Self = Self {
2095 read: false,
2096 write: false,
2097 };
2098 pub const READ_ONLY: Self = Self {
2099 read: true,
2100 write: false,
2101 };
2102 pub const WRITE_ONLY: Self = Self {
2103 read: false,
2104 write: true,
2105 };
2106 pub const READ_WRITE: Self = Self {
2107 read: true,
2108 write: true,
2109 };
2110}
2111
2112#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2113enum RouterQueueKind {
2114 Received,
2115 Transmit,
2116}
2117
2118impl RouterInner {
2119 #[cfg(any(feature = "std", feature = "fair-admission"))]
2120 fn fair_bulk_owner(data: &RouterItem, priority: u8) -> Option<u32> {
2121 match data {
2122 RouterItem::Packet(packet) => crate::admission_fairness::packet_owner(packet, priority),
2123 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2124 crate::admission_fairness::packed_owner(view, priority)
2125 })
2126 .ok()
2127 .flatten(),
2128 }
2129 }
2130 #[cfg(any(feature = "std", feature = "fair-admission"))]
2131 fn tx_bulk_owner(item: &TxQueued) -> Option<u32> {
2132 match &item.item {
2133 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2134 Self::fair_bulk_owner(data, item.priority)
2135 }
2136 _ => None,
2137 }
2138 }
2139 #[cfg(any(feature = "std", feature = "fair-admission"))]
2140 fn bulk_stream(data: &RouterItem) -> Option<u32> {
2141 match data {
2142 RouterItem::Packet(packet) => Some(packet.data_type().as_u32()),
2143 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2144 wire_format::scheduling_type(view).map(|ty| ty.as_u32())
2145 })
2146 .ok()
2147 .flatten(),
2148 }
2149 }
2150 #[cfg(any(feature = "std", feature = "fair-admission"))]
2151 fn tx_bulk_stream(item: &TxQueued) -> Option<u32> {
2152 match &item.item {
2153 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2154 Self::bulk_stream(data)
2155 }
2156 _ => None,
2157 }
2158 }
2159 #[cfg(any(feature = "std", feature = "fair-admission"))]
2160 fn bulk_loads(&self, priority: u8) -> crate::admission_fairness::Loads {
2161 let mut loads = crate::admission_fairness::Loads::new();
2162 for item in self.received_queue.iter() {
2163 if item.priority <= priority
2164 && let Some(owner) = Self::fair_bulk_owner(&item.data, item.priority)
2165 {
2166 loads.record(owner, item.byte_cost());
2167 }
2168 }
2169 for item in self.transmit_queue.iter() {
2170 if item.priority <= priority
2171 && let Some(owner) = Self::tx_bulk_owner(item)
2172 {
2173 loads.record(owner, item.byte_cost());
2174 }
2175 }
2176 loads
2177 }
2178 #[cfg(any(feature = "std", feature = "fair-admission"))]
2179 fn competing_bulk_owner(&self, incoming: u32, priority: u8) -> Option<u32> {
2180 self.bulk_loads(priority).dominant_other(incoming)
2181 }
2182 #[cfg(any(feature = "std", feature = "fair-admission"))]
2183 fn reclaim_bulk_from(
2184 &mut self,
2185 owner: u32,
2186 priority: u8,
2187 incoming_stream: Option<u32>,
2188 ) -> bool {
2189 let mut streams = crate::admission_fairness::Loads::new();
2190 for item in self.received_queue.iter() {
2191 if item.priority <= priority
2192 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2193 && let Some(ty) = Self::bulk_stream(&item.data)
2194 {
2195 streams.record(ty, item.byte_cost());
2196 }
2197 }
2198 for item in self.transmit_queue.iter() {
2199 if item.priority <= priority
2200 && Self::tx_bulk_owner(item) == Some(owner)
2201 && let Some(ty) = Self::tx_bulk_stream(item)
2202 {
2203 streams.record(ty, item.byte_cost());
2204 }
2205 }
2206 let stream = streams.queue_victim(incoming_stream);
2207 let rx = self
2208 .received_queue
2209 .iter()
2210 .enumerate()
2211 .filter(|(_, item)| {
2212 item.priority <= priority
2213 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2214 && stream.is_none_or(|ty| Self::bulk_stream(&item.data) == Some(ty))
2215 })
2216 .min_by_key(|(index, item)| (item.priority, *index))
2217 .map(|(index, item)| (index, item.priority));
2218 let tx = self
2219 .transmit_queue
2220 .iter()
2221 .enumerate()
2222 .filter(|(_, item)| {
2223 item.priority <= priority
2224 && Self::tx_bulk_owner(item) == Some(owner)
2225 && stream.is_none_or(|ty| Self::tx_bulk_stream(item) == Some(ty))
2226 })
2227 .min_by_key(|(index, item)| (item.priority, *index))
2228 .map(|(index, item)| (index, item.priority));
2229 match (rx, tx) {
2230 (Some((index, rp)), Some((_, tp))) if rp <= tp => {
2231 self.received_queue.remove_pos(index).is_some()
2232 }
2233 (Some((index, _)), None) => self.received_queue.remove_pos(index).is_some(),
2234 (_, Some((index, _))) => self.transmit_queue.remove_pos(index).is_some(),
2235 _ => false,
2236 }
2237 }
2238
2239 #[cfg(feature = "discovery")]
2240 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
2241 board
2242 .sender_id
2243 .len()
2244 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
2245 .saturating_add(
2246 board
2247 .reachable_timesync_sources
2248 .iter()
2249 .map(|s| s.len())
2250 .sum::<usize>(),
2251 )
2252 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
2253 }
2254
2255 #[cfg(feature = "discovery")]
2256 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
2257 sender
2258 .len()
2259 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
2260 .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
2261 .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
2262 .saturating_add(
2263 state
2264 .reachable_timesync_sources
2265 .iter()
2266 .map(|s| s.len())
2267 .sum::<usize>(),
2268 )
2269 .saturating_add(
2270 state
2271 .advertised_reachable_timesync_sources
2272 .iter()
2273 .map(|s| s.len())
2274 .sum::<usize>(),
2275 )
2276 .saturating_add(
2277 state
2278 .topology_boards
2279 .iter()
2280 .map(Self::topology_board_byte_cost)
2281 .sum::<usize>(),
2282 )
2283 .saturating_add(size_of::<DiscoverySenderState>())
2284 }
2285
2286 #[cfg(feature = "discovery")]
2287 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
2288 size_of::<RouterSideId>()
2289 .saturating_add(size_of::<DiscoverySideState>())
2290 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
2291 .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
2292 .saturating_add(
2293 route.requested_network_variables.len()
2294 * (size_of::<DataType>() + size_of::<u64>()),
2295 )
2296 .saturating_add(
2297 route
2298 .reachable_timesync_sources
2299 .iter()
2300 .map(|s| s.len())
2301 .sum::<usize>(),
2302 )
2303 .saturating_add(
2304 route
2305 .announcers
2306 .iter()
2307 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
2308 .sum::<usize>(),
2309 )
2310 .saturating_add(side.saturating_sub(side))
2311 }
2312
2313 #[cfg(feature = "discovery")]
2314 fn discovery_bytes_used(&self) -> usize {
2315 self.discovery_routes
2316 .iter()
2317 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
2318 .sum()
2319 }
2320
2321 #[inline]
2322 fn reliable_rx_buffered_bytes(&self) -> usize {
2323 self.reliable_rx
2324 .values()
2325 .flat_map(|state| state.buffered.values())
2326 .map(|bytes| size_of::<crate::SharedBytes>() + bytes.len())
2327 .sum()
2328 }
2329
2330 fn reliable_rx_buffer_len(&self) -> usize {
2331 self.reliable_rx
2332 .values()
2333 .map(|state| state.buffered.len())
2334 .sum()
2335 }
2336
2337 #[inline(never)]
2338 fn side_chunk_bytes_used(&self) -> usize {
2339 self.side_transport
2340 .values()
2341 .map(|side| side.rx_chunk_bytes)
2342 .sum()
2343 }
2344 fn discard_side_chunk(&mut self, side: RouterSideId, id: u32) -> bool {
2345 self.side_transport
2346 .get_mut(&side)
2347 .is_some_and(|state| state.take_chunk(id, true).is_some())
2348 }
2349 fn reclaim_oldest_side_chunk(&mut self, keep: Option<(RouterSideId, u32)>) -> bool {
2350 let mut victim = None;
2351 let mut time = u64::MAX;
2352 for (&side, state) in &self.side_transport {
2353 let preserved = match keep {
2354 Some((owner, id)) if owner == side => Some(id),
2355 _ => None,
2356 };
2357 if let Some((id, seen)) = SideChunkAssembly::oldest(&state.rx_chunks, preserved)
2358 && (victim.is_none() || seen < time)
2359 {
2360 time = seen;
2361 victim = Some((side, id));
2362 }
2363 }
2364 match victim {
2365 Some((side, id)) => self.discard_side_chunk(side, id),
2366 None => false,
2367 }
2368 }
2369 fn make_side_chunk_room(
2370 &mut self,
2371 side: RouterSideId,
2372 id: u32,
2373 index: u16,
2374 total: u16,
2375 payload_len: usize,
2376 ) -> TelemetryResult<()> {
2377 let budget = self
2378 .memory
2379 .max_queue_budget
2380 .saturating_sub(self.protected_metadata_bytes_used());
2381 let state = self
2382 .side_transport
2383 .get_mut(&side)
2384 .ok_or(TelemetryError::BadArg)?;
2385 let existing = state.rx_chunks.get(&id).map(|a| a.total);
2386 if existing.is_some_and(|old| old != 0 && old != total) {
2387 state.take_chunk(id, true);
2388 return Err(TelemetryError::Unpack("side chunk total mismatch"));
2389 }
2390 let before = state.rx_chunks.len();
2391 let room = SideChunkAssembly::make_room(
2392 &mut state.rx_chunks,
2393 &mut state.rx_chunk_bytes,
2394 id,
2395 index,
2396 total,
2397 payload_len,
2398 budget,
2399 );
2400 if before != state.rx_chunks.len() {
2401 state.rx_chunk_reclaims = state
2402 .rx_chunk_reclaims
2403 .saturating_add((before - state.rx_chunks.len()) as u32);
2404 }
2405 let cost = room?;
2406 if cost == 0 {
2407 return Ok(());
2408 }
2409 if existing.is_none()
2410 && state.rx_chunks.len() >= 4
2411 && let Some((oldest, _)) = SideChunkAssembly::oldest(&state.rx_chunks, None)
2412 {
2413 state.take_chunk(oldest, true);
2414 }
2415 while self.shared_queue_bytes_used().saturating_add(cost) > self.memory.max_queue_budget {
2416 #[cfg(any(feature = "std", feature = "fair-admission"))]
2417 if let Some(owner) = self.bulk_loads(199).queue_victim(None)
2418 && self.reclaim_bulk_from(owner, 199, None)
2419 {
2420 continue;
2421 }
2422 if self.reclaim_oldest_side_chunk(Some((side, id))) {
2423 continue;
2424 }
2425 return Err(TelemetryError::Io("memory pressure"));
2428 }
2429 Ok(())
2430 }
2431
2432 #[inline(never)]
2433 fn protected_metadata_bytes_used(&self) -> usize {
2434 self.recent_rx
2435 .max_bytes()
2436 .saturating_add(self.reliable_rx_buffered_bytes())
2437 .saturating_add(crate::config::schema_bytes_used())
2438 .saturating_add({
2439 #[cfg(feature = "discovery")]
2440 {
2441 self.discovery_bytes_used()
2442 }
2443 #[cfg(not(feature = "discovery"))]
2444 {
2445 0
2446 }
2447 })
2448 }
2449 #[inline]
2450 fn shared_queue_bytes_used(&self) -> usize {
2451 self.received_queue
2452 .bytes_used()
2453 .saturating_add(self.transmit_queue.bytes_used())
2454 .saturating_add(self.side_chunk_bytes_used())
2455 .saturating_add(self.protected_metadata_bytes_used())
2456 }
2457
2458 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
2459 match preferred {
2460 RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
2465 RouterQueueKind::Transmit => self.transmit_queue.pop_lowest_priority().is_some(),
2466 }
2467 }
2468
2469 fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
2470 let received = self
2471 .received_queue
2472 .lowest_priority(|queued| queued.priority)
2473 .map(|priority| (RouterQueueKind::Received, priority));
2474 let transmit = self
2475 .transmit_queue
2476 .lowest_priority(|queued| queued.priority)
2477 .map(|priority| (RouterQueueKind::Transmit, priority));
2478 [received, transmit]
2479 .into_iter()
2480 .flatten()
2481 .filter(|(_, priority)| {
2482 *priority < incoming_priority
2483 || (incoming_priority != 254 && *priority == incoming_priority)
2484 })
2485 .min_by_key(|(kind, priority)| {
2486 (
2487 *priority,
2488 if *kind == RouterQueueKind::Received {
2489 0
2490 } else {
2491 1
2492 },
2493 )
2494 })
2495 .map(|(kind, _)| kind)
2496 }
2497
2498 fn make_shared_queue_room(
2499 &mut self,
2500 incoming_cost: usize,
2501 incoming_priority: u8,
2502 _incoming: Option<&RouterItem>,
2503 ) -> TelemetryResult<()> {
2504 if incoming_cost > self.memory.max_queue_budget {
2505 return Err(TelemetryError::PacketTooLarge(
2506 "Item exceeds maximum shared queue budget",
2507 ));
2508 }
2509
2510 #[cfg(any(feature = "std", feature = "fair-admission"))]
2511 let mut fair_victim = None;
2512 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
2513 > self.memory.max_queue_budget
2514 {
2515 #[cfg(any(feature = "std", feature = "fair-admission"))]
2516 {
2517 let incoming =
2518 _incoming.and_then(|data| Self::fair_bulk_owner(data, incoming_priority));
2519 let victim = *fair_victim.get_or_insert_with(|| {
2520 self.bulk_loads(incoming_priority.min(199))
2521 .queue_victim(incoming)
2522 });
2523 if let Some(victim) = victim
2524 && self.reclaim_bulk_from(
2525 victim,
2526 incoming_priority.min(199),
2527 (incoming == Some(victim))
2528 .then(|| _incoming.and_then(Self::bulk_stream))
2529 .flatten(),
2530 )
2531 {
2532 continue;
2533 }
2534 if incoming.is_some() {
2535 return Err(TelemetryError::Io("shared fair queue saturated"));
2536 }
2537 }
2538 let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
2539 if incoming_priority >= 200 && self.reclaim_oldest_side_chunk(None) {
2540 continue;
2541 }
2542 return Err(TelemetryError::Io("shared priority queue saturated"));
2543 };
2544 if !self.pop_shared_queue_item(victim) {
2545 return Err(TelemetryError::PacketTooLarge(
2546 "Item exceeds maximum shared queue budget",
2547 ));
2548 }
2549 }
2550
2551 Ok(())
2552 }
2553
2554 #[cfg(feature = "discovery")]
2555 fn schema_growth_budget(&self) -> usize {
2556 let protected = self
2559 .recent_rx
2560 .max_bytes()
2561 .saturating_add(self.discovery_bytes_used())
2562 .saturating_add(self.reliable_rx_buffered_bytes());
2563 self.memory.max_queue_budget.saturating_sub(protected)
2564 }
2565
2566 #[cfg(feature = "discovery")]
2567 fn commit_discovery_route(
2568 &mut self,
2569 side: RouterSideId,
2570 route: DiscoverySideState,
2571 ) -> TelemetryResult<()> {
2572 let previous = self
2573 .discovery_routes
2574 .get(&side)
2575 .map_or(0, |route| Self::discovery_route_byte_cost(side, route));
2576 let incoming = Self::discovery_route_byte_cost(side, &route);
2577 let growth = incoming.saturating_sub(previous);
2578 self.make_shared_queue_room(growth, 255, None)?;
2579 self.discovery_routes.insert(side, route);
2580 Ok(())
2581 }
2582
2583 #[cfg(feature = "discovery")]
2584 fn fit_discovery_budget(&mut self) {
2585 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
2586 if let Some(victim) = self.lower_priority_shared_queue(255)
2589 && self.pop_shared_queue_item(victim)
2590 {
2591 continue;
2592 }
2593 break;
2596 }
2597 }
2598
2599 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
2600 self.make_shared_queue_room(item.byte_cost(), item.priority, Some(&item.data))?;
2601 #[cfg(any(feature = "std", feature = "fair-admission"))]
2602 {
2603 #[cfg(any(feature = "std", feature = "fair-admission"))]
2604 let mut fair_victim = None;
2605 self.received_queue.push_back_prioritized_select(
2606 item,
2607 |queued| queued.priority,
2608 |queue, _incoming| {
2609 #[cfg(any(feature = "std", feature = "fair-admission"))]
2610 {
2611 crate::admission_fairness::queue_victim(
2612 queue,
2613 _incoming,
2614 |item| Self::fair_bulk_owner(&item.data, item.priority),
2615 |item| item.priority,
2616 |item| Self::bulk_stream(&item.data),
2617 &mut fair_victim,
2618 )
2619 }
2620 },
2621 )
2622 }
2623 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
2624 {
2625 self.received_queue
2626 .push_back_prioritized(item, |queued| queued.priority)
2627 }
2628 }
2629
2630 fn push_transmit(&mut self, item: TxQueued) -> TelemetryResult<()> {
2631 #[cfg(feature = "discovery")]
2632 if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
2633 if let RouterItem::Packet(incoming) = data
2636 && self.transmit_queue.iter().any(|queued| {
2637 matches!(&queued.item,
2638 RouterTxItem::ToSide { dst: queued_dst, data: RouterItem::Packet(existing), .. }
2639 if queued_dst == dst && discovery::same_schema_snapshot(existing, incoming))
2640 })
2641 {
2642 return Ok(());
2643 }
2644 let incoming = match data {
2645 RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
2646 Some((pkt.data_type(), pkt.sender().to_owned()))
2647 }
2648 RouterItem::Packet(_) | RouterItem::Packed(_) => None,
2649 };
2650 if let Some((incoming_ty, incoming_sender)) = incoming {
2651 self.transmit_queue.retain(|queued| {
2652 let RouterTxItem::ToSide {
2653 dst: queued_dst,
2654 data: queued_data,
2655 ..
2656 } = &queued.item
2657 else {
2658 return true;
2659 };
2660 let same_snapshot = match queued_data {
2661 RouterItem::Packet(pkt) => {
2662 pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
2663 }
2664 RouterItem::Packed(_) => false,
2665 };
2666 *queued_dst != *dst || !same_snapshot
2667 });
2668 }
2669 }
2670 self.make_shared_queue_room(
2671 item.byte_cost(),
2672 item.priority,
2673 match &item.item {
2674 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => Some(data),
2675 _ => None,
2676 },
2677 )?;
2678 #[cfg(any(feature = "std", feature = "fair-admission"))]
2679 {
2680 #[cfg(any(feature = "std", feature = "fair-admission"))]
2681 let mut fair_victim = None;
2682 self.transmit_queue.push_back_prioritized_select(
2683 item,
2684 |queued| queued.priority,
2685 |queue, _incoming| {
2686 #[cfg(any(feature = "std", feature = "fair-admission"))]
2687 {
2688 crate::admission_fairness::queue_victim(
2689 queue,
2690 _incoming,
2691 Self::tx_bulk_owner,
2692 |item| item.priority,
2693 Self::tx_bulk_stream,
2694 &mut fair_victim,
2695 )
2696 }
2697 },
2698 )
2699 }
2700 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
2701 {
2702 self.transmit_queue
2703 .push_back_prioritized(item, |queued| queued.priority)
2704 }
2705 }
2706
2707 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
2708 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
2709 let _ = self.recent_rx.pop_front();
2710 }
2711 self.make_shared_queue_room(0, 0, None)?;
2712 self.recent_rx.push_back(id)
2713 }
2714
2715 #[allow(unused_mut)]
2716 fn buffer_reliable_rx(
2717 &mut self,
2718 side: RouterSideId,
2719 ty: DataType,
2720 seq: u32,
2721 mut bytes: crate::SharedBytes,
2722 ) -> TelemetryResult<()> {
2723 let key = Router::reliable_key(side, ty);
2724 if self
2725 .reliable_rx
2726 .get(&key)
2727 .is_some_and(|state| state.buffered.contains_key(&seq))
2728 {
2729 return Ok(());
2730 }
2731 let cost = size_of::<crate::SharedBytes>() + bytes.len();
2732 self.make_shared_queue_room(cost, crate::transport_priority(ty), None)?;
2733 #[cfg(feature = "compact-packet-store")]
2734 bytes.park_for_queue()?;
2735 let rx_state = self
2736 .reliable_rx
2737 .entry(key)
2738 .or_insert_with(|| ReliableRxState {
2739 expected_seq: 1,
2740 buffered: BTreeMap::new(),
2741 });
2742 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
2743 let _ = rx_state.buffered.pop_first();
2744 }
2745 rx_state.buffered.insert(seq, bytes);
2746 Ok(())
2747 }
2748}
2749
2750struct IsrRxQueue {
2755 busy: AtomicBool,
2756 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
2757}
2758
2759unsafe impl Send for IsrRxQueue {}
2760unsafe impl Sync for IsrRxQueue {}
2761
2762struct IsrRxQueueGuard<'a> {
2763 owner: &'a IsrRxQueue,
2764}
2765
2766impl Deref for IsrRxQueueGuard<'_> {
2767 type Target = BoundedDeque<RouterRxItem>;
2768
2769 #[inline]
2770 fn deref(&self) -> &Self::Target {
2771 unsafe { &*self.owner.q.get() }
2772 }
2773}
2774
2775impl DerefMut for IsrRxQueueGuard<'_> {
2776 #[inline]
2777 fn deref_mut(&mut self) -> &mut Self::Target {
2778 unsafe { &mut *self.owner.q.get() }
2779 }
2780}
2781
2782impl Drop for IsrRxQueueGuard<'_> {
2783 #[inline]
2784 fn drop(&mut self) {
2785 self.owner.busy.store(false, Ordering::Release);
2786 }
2787}
2788
2789impl IsrRxQueue {
2790 #[inline]
2791 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
2792 Self {
2793 busy: AtomicBool::new(false),
2794 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
2795 }
2796 }
2797
2798 #[inline]
2799 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
2800 match self
2801 .busy
2802 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
2803 {
2804 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
2805 Err(_) => Err(TelemetryError::Io("rx queue busy")),
2806 }
2807 }
2808
2809 #[allow(dead_code)]
2810 #[inline]
2811 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
2812 let mut g = self.try_lock()?;
2813 g.push_back(item)
2814 }
2815
2816 #[inline]
2817 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
2818 let mut g = self.try_lock()?;
2819 g.push_back_prioritized(item, |queued| queued.priority)
2820 }
2821
2822 #[inline]
2823 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
2824 let mut g = self.try_lock()?;
2825 Ok(g.pop_front())
2826 }
2827
2828 #[inline]
2829 fn clear(&self) -> TelemetryResult<()> {
2830 let mut g = self.try_lock()?;
2831 g.clear();
2832 Ok(())
2833 }
2834
2835 #[inline]
2836 fn snapshot(&self) -> Option<(usize, usize)> {
2837 let g = self.try_lock().ok()?;
2838 Some((g.len(), g.bytes_used()))
2839 }
2840}
2841
2842pub struct Router {
2846 sender: RouterMutex<Arc<str>>,
2847 cfg: RouterConfig,
2848 state: RouterMutex<RouterInner>,
2849 isr_rx_queue: IsrRxQueue,
2850 side_tx_gate: ReentryGate,
2851 clock: Box<dyn Clock + Send + Sync>,
2852 #[cfg(feature = "timesync")]
2853 timesync: RouterMutex<TimeSyncRuntime>,
2854}
2855
2856#[cfg(feature = "timesync")]
2857#[derive(Debug, Clone)]
2858struct PendingTimeSyncRequest {
2859 seq: u64,
2860 t1_mono_ms: u64,
2861 source: String,
2862}
2863
2864#[cfg(feature = "timesync")]
2865#[derive(Debug, Clone)]
2866struct RemoteTimeSyncSource {
2867 priority: u64,
2868 last_sample_mono_ms: u64,
2869 sample_unix_ms: u64,
2870}
2871
2872#[cfg(feature = "timesync")]
2873#[derive(Debug, Clone)]
2874struct TimeSyncRuntime {
2875 cfg: Option<TimeSyncConfig>,
2876 tracker: Option<TimeSyncTracker>,
2877 clock: NetworkClock,
2878 disciplined_clock: SlewedNetworkClock,
2879 remote_sources: BTreeMap<String, RemoteTimeSyncSource>,
2880 next_seq: u64,
2881 next_announce_mono_ms: u64,
2882 next_request_mono_ms: u64,
2883 pending_request: Option<PendingTimeSyncRequest>,
2884}
2885
2886#[cfg(feature = "timesync")]
2887impl TimeSyncRuntime {
2888 fn new(cfg: Option<TimeSyncConfig>) -> Self {
2889 Self {
2890 tracker: cfg.map(TimeSyncTracker::new),
2891 cfg,
2892 clock: NetworkClock::default(),
2893 disciplined_clock: SlewedNetworkClock::new(
2894 cfg.map(|c| c.max_slew_ppm)
2895 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
2896 ),
2897 remote_sources: BTreeMap::new(),
2898 next_seq: 1,
2899 next_announce_mono_ms: 0,
2900 next_request_mono_ms: 0,
2901 pending_request: None,
2902 }
2903 }
2904}
2905
2906enum RemoteSidePlan {
2907 Target(Vec<RouterSideId>),
2908}
2909
2910#[cfg(feature = "discovery")]
2911#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
2912struct DiscoveryCandidateMatch {
2913 side: RouterSideId,
2914 overlap: usize,
2915}
2916
2917#[cfg(feature = "discovery")]
2918#[inline]
2919const fn discovery_is_hop_local_advertisement(ty: DataType) -> bool {
2920 matches!(
2921 ty,
2922 DataType::DiscoveryAnnounce
2923 | DataType::DiscoveryTimeSyncSources
2924 | DataType::DiscoveryTopology
2925 | DataType::DiscoveryAddress
2926 | DataType::DiscoveryLinkCapabilities
2927 )
2928}
2929
2930#[cfg(not(feature = "discovery"))]
2931const fn discovery_is_hop_local_advertisement(_ty: DataType) -> bool {
2932 false
2933}
2934
2935impl Debug for Router {
2936 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2937 let sender = self.sender();
2938 f.debug_struct("Router")
2939 .field("sender", &sender)
2940 .field("cfg", &self.cfg)
2941 .field("state", &"<mutex>")
2942 .field("clock", &"Clock")
2943 .finish()
2944 }
2945}
2946
2947#[inline]
2950fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &RouterConfig) -> bool {
2951 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
2952}
2953
2954#[inline]
2955fn force_remote_for_type(ty: DataType) -> bool {
2956 matches!(
2957 ty,
2958 DataType::ReliableAck
2959 | DataType::ReliablePartialAck
2960 | DataType::ReliablePacketRequest
2961 | DataType::P2pMessage
2962 ) || {
2963 #[cfg(feature = "timesync")]
2964 {
2965 matches!(
2966 ty,
2967 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2968 )
2969 }
2970 #[cfg(not(feature = "timesync"))]
2971 {
2972 false
2973 }
2974 }
2975}
2976
2977#[inline]
2978fn is_internal_control_type(ty: DataType) -> bool {
2979 if matches!(
2980 ty,
2981 DataType::ReliableAck
2982 | DataType::ReliablePartialAck
2983 | DataType::ReliablePacketRequest
2984 | DataType::P2pMessage
2985 ) {
2986 return true;
2987 }
2988
2989 #[cfg(feature = "timesync")]
2990 if matches!(
2991 ty,
2992 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2993 ) {
2994 return true;
2995 }
2996
2997 #[cfg(feature = "discovery")]
2998 if discovery::is_discovery_type(ty) {
2999 return true;
3000 }
3001
3002 let _ = ty;
3003 false
3004}
3005
3006fn with_retries<F>(
3008 this: &Router,
3009 dest: DataEndpoint,
3010 data: &RouterItem,
3011 pkt_for_ctx: Option<&Packet>,
3012 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
3013 called_from_queue: bool,
3014 run: F,
3015) -> TelemetryResult<()>
3016where
3017 F: Fn() -> TelemetryResult<()>,
3018{
3019 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
3020 Ok(((), attempts)) => {
3021 if attempts > 1 {
3022 let mut st = this.state.lock();
3023 st.total_handler_retries = st
3024 .total_handler_retries
3025 .saturating_add(attempts.saturating_sub(1) as u64);
3026 }
3027 Ok(())
3028 }
3029 Err((e, attempts)) => {
3030 {
3031 let mut st = this.state.lock();
3032 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3033 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3034 }
3035
3036 this.remove_pkt_id(data);
3038
3039 if let Some(pkt) = pkt_for_ctx {
3041 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
3042 } else if let Some(env) = env_for_ctx {
3043 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
3044 }
3045
3046 Err(TelemetryError::HandlerError("local handler failed"))
3047 }
3048 }
3049}
3050impl Router {
3052 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
3053 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
3054
3055 #[inline]
3056 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
3057 st.sides
3058 .get(side)
3059 .and_then(|side| side.as_ref())
3060 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
3061 }
3062
3063 fn note_side_tx_success(
3064 &self,
3065 side: RouterSideId,
3066 ty: DataType,
3067 bytes: usize,
3068 relayed: bool,
3069 attempts: usize,
3070 ) {
3071 let mut st = self.state.lock();
3072 #[cfg(feature = "discovery")]
3073 if ty == crate::DataType::DiscoveryAnnounce
3074 && !relayed
3075 && let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
3076 {
3077 throttle.last_liveness_tx_ms = self.clock.now_ms();
3078 }
3079 let entry = st.side_runtime_stats.entry(side).or_default();
3080 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
3081 }
3082
3083 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
3084 let mut st = self.state.lock();
3085 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3086 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3087 let entry = st.side_runtime_stats.entry(side).or_default();
3088 entry.note_tx_failure(ty, attempts);
3089 }
3090
3091 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
3092 let mut st = self.state.lock();
3093 let entry = st.side_runtime_stats.entry(side).or_default();
3094 entry.note_rx(ty, bytes, relayed);
3095 }
3096
3097 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
3098 let mut st = self.state.lock();
3099 let entry = st.side_runtime_stats.entry(side).or_default();
3100 entry.note_local_delivery(ty);
3101 }
3102
3103 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
3104 let mut st = self.state.lock();
3105 let entry = st.side_runtime_stats.entry(side).or_default();
3106 entry.note_local_handler_failure(ty, retries);
3107 }
3108
3109 fn cache_managed_variable_packet(
3110 &self,
3111 pkt: &Packet,
3112 notify_handlers: bool,
3113 ) -> TelemetryResult<()> {
3114 let sender_address = pkt
3115 .sender()
3116 .strip_prefix("@addr:")
3117 .and_then(|value| value.parse::<u32>().ok())
3118 .unwrap_or_else(|| sender_address_u32(pkt.sender()));
3119 let handlers = {
3120 let mut st = self.state.lock();
3121 if !st
3122 .managed_variable_types
3123 .contains(&pkt.data_type().as_u32())
3124 {
3125 return Ok(());
3126 }
3127 let existing = st.managed_variable_latest.get(&pkt.data_type().as_u32());
3128 if existing.is_some_and(|entry| {
3135 let current = &entry.packet;
3136 sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
3137 }) {
3138 return Ok(());
3139 }
3140 let changed = existing.is_none_or(|entry| entry.packet != *pkt);
3141 let updated = ManagedVariableCacheEntry {
3145 packet: pkt.clone(),
3146 sender_address,
3147 cached_at_ms: self.clock.now_ms(),
3148 };
3149 if let Some(entry) = st
3150 .managed_variable_latest
3151 .get_mut(&pkt.data_type().as_u32())
3152 {
3153 **entry = updated;
3154 } else {
3155 st.managed_variable_latest
3156 .insert(pkt.data_type().as_u32(), Box::new(updated));
3157 }
3158 if notify_handlers && changed {
3159 st.network_variable_update_handlers
3160 .get(&pkt.data_type().as_u32())
3161 .cloned()
3162 .unwrap_or_default()
3163 } else {
3164 Vec::new()
3165 }
3166 };
3167 for handler in handlers {
3168 (handler.handler)(pkt)?;
3169 }
3170 Ok(())
3171 }
3172
3173 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
3174 self.cache_managed_variable_packet(pkt, true)
3175 }
3176
3177 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
3178 let st = self.state.lock();
3179 st.managed_variable_latest
3180 .get(&ty.as_u32())
3181 .map(|entry| entry.packet.clone())
3182 }
3183
3184 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
3185 let now_ms = self.clock.now_ms();
3186 let st = self.state.lock();
3187 st.managed_variable_latest.get(&ty.as_u32()).map(|entry| {
3188 (
3189 entry.packet.clone(),
3190 now_ms.saturating_sub(entry.cached_at_ms),
3191 )
3192 })
3193 }
3194
3195 fn is_managed_variable_type(&self, ty: DataType) -> bool {
3196 let st = self.state.lock();
3197 st.managed_variable_types.contains(&ty.as_u32())
3198 }
3199
3200 fn managed_variable_permissions_locked(
3201 st: &RouterInner,
3202 ty: DataType,
3203 ) -> NetworkVariablePermissions {
3204 st.managed_variable_permissions
3205 .get(&ty.as_u32())
3206 .copied()
3207 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
3208 }
3209
3210 fn can_read_managed_variable(&self, ty: DataType) -> bool {
3211 let st = self.state.lock();
3212 Self::managed_variable_permissions_locked(&st, ty).read
3213 }
3214
3215 fn can_write_managed_variable(&self, ty: DataType) -> bool {
3216 let st = self.state.lock();
3217 Self::managed_variable_permissions_locked(&st, ty).write
3218 }
3219
3220 #[inline]
3221 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
3222 let st = self.state.lock();
3223 let side_ref = Self::side_ref(&st, side)?;
3224 if side_ref.opts.ingress_enabled {
3225 Ok(())
3226 } else {
3227 Err(TelemetryError::HandlerError(
3228 "router: ingress disabled for side id",
3229 ))
3230 }
3231 }
3232
3233 #[inline]
3234 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
3235 src != Some(dst)
3236 }
3237
3238 #[inline]
3239 fn route_allowed_locked(
3240 &self,
3241 st: &RouterInner,
3242 src: Option<RouterSideId>,
3243 ty: Option<DataType>,
3244 dst: RouterSideId,
3245 ) -> bool {
3246 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
3247 return false;
3248 };
3249 if !dst_side.opts.egress_enabled {
3250 return false;
3251 }
3252 if let Some(src_id) = src {
3253 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
3254 return false;
3255 };
3256 if !src_side.opts.ingress_enabled {
3257 return false;
3258 }
3259 }
3260 let base_allowed = st
3261 .route_overrides
3262 .get(&(src, dst))
3263 .copied()
3264 .unwrap_or_else(|| self.default_route_enabled(src, dst));
3265 if !base_allowed {
3266 return false;
3267 }
3268
3269 let Some(ty) = ty else {
3270 return true;
3271 };
3272 if st
3273 .typed_route_overrides
3274 .keys()
3275 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3276 {
3277 return st
3278 .typed_route_overrides
3279 .get(&(src, ty.as_u32(), dst))
3280 .copied()
3281 .unwrap_or(false);
3282 }
3283 true
3284 }
3285
3286 fn has_typed_route_overrides_locked(
3287 st: &RouterInner,
3288 src: Option<RouterSideId>,
3289 ty: DataType,
3290 ) -> bool {
3291 st.typed_route_overrides
3292 .keys()
3293 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3294 }
3295
3296 #[cfg(feature = "discovery")]
3297 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
3298 where
3299 I: IntoIterator<Item = DataEndpoint>,
3300 {
3301 let mut overlap = 0usize;
3302 for ep in reachable {
3303 if eps.contains(&ep) {
3304 overlap = overlap.saturating_add(1);
3305 }
3306 }
3307 overlap
3308 }
3309
3310 #[inline]
3311 fn preferred_scoring_endpoints<'a>(
3312 &self,
3313 eps: &'a [DataEndpoint],
3314 prefer_nonlocal: bool,
3315 ) -> alloc::borrow::Cow<'a, [DataEndpoint]> {
3316 if !prefer_nonlocal || eps.iter().all(|&ep| !self.cfg.is_local_endpoint(ep)) {
3317 return alloc::borrow::Cow::Borrowed(eps);
3318 }
3319 let nonlocal: Vec<DataEndpoint> = eps
3320 .iter()
3321 .copied()
3322 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
3323 .collect();
3324 if nonlocal.is_empty() {
3325 alloc::borrow::Cow::Borrowed(eps)
3326 } else {
3327 alloc::borrow::Cow::Owned(nonlocal)
3328 }
3329 }
3330
3331 fn pop_next_queued_control_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
3332 {
3333 let mut isr_rx = self.isr_rx_queue.try_lock()?;
3334 let idx = isr_rx.iter().position(Self::queued_rx_item_is_control);
3335 if let Some(idx) = idx {
3336 return Ok(isr_rx.remove_pos(idx));
3337 }
3338 }
3339
3340 let mut st = self.state.lock();
3341 let idx = st
3342 .received_queue
3343 .iter()
3344 .position(Self::queued_rx_item_is_control);
3345 if let Some(idx) = idx {
3346 return Ok(st.received_queue.remove_pos(idx));
3347 }
3348 Ok(None)
3349 }
3350
3351 fn queued_rx_item_is_control(item: &RouterRxItem) -> bool {
3352 if item.priority == crate::transport_priority(DataType::ManagedVariableValue) {
3355 return true;
3356 }
3357 let ty = match &item.data {
3358 RouterItem::Packet(pkt) => Some(pkt.data_type()),
3359 RouterItem::Packed(bytes) => wire_format::scheduling_type(bytes.as_ref()),
3360 };
3361 match ty {
3362 Some(
3363 DataType::ReliableAck
3364 | DataType::ReliablePartialAck
3365 | DataType::ReliablePacketRequest,
3366 ) => true,
3367 #[cfg(feature = "discovery")]
3368 Some(ty) => discovery::is_discovery_type(ty),
3369 _ => false,
3370 }
3371 }
3372
3373 fn drain_queued_control_rx_before_tx(
3374 &self,
3375 start: u64,
3376 timeout_ms: u32,
3377 ) -> TelemetryResult<bool> {
3378 let mut did_any = false;
3379 while let Some(item) = self.pop_next_queued_control_rx_item()? {
3380 self.process_rx_queue_item(item)?;
3381 did_any = true;
3382 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms) {
3385 break;
3386 }
3387 }
3388 Ok(did_any)
3389 }
3390
3391 fn eligible_side_ids_locked(
3392 &self,
3393 st: &RouterInner,
3394 src: Option<RouterSideId>,
3395 ty: Option<DataType>,
3396 restrict_link_local: bool,
3397 ) -> Vec<RouterSideId> {
3398 st.sides
3399 .iter()
3400 .enumerate()
3401 .filter_map(|(side_id, side)| {
3402 let side = side.as_ref()?;
3403 if restrict_link_local && !side.opts.link_local_enabled {
3404 return None;
3405 }
3406 if self.route_allowed_locked(st, src, ty, side_id) {
3407 Some(side_id)
3408 } else {
3409 None
3410 }
3411 })
3412 .collect()
3413 }
3414
3415 fn apply_route_selection_locked(
3416 &self,
3417 st: &mut RouterInner,
3418 src: Option<RouterSideId>,
3419 mut sides: Vec<RouterSideId>,
3420 origin: RouteSelectionOrigin,
3421 ) -> Vec<RouterSideId> {
3422 if sides.len() <= 1 {
3423 return sides;
3424 }
3425
3426 let selection_mode = st.source_route_modes.get(&src).copied();
3427 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
3428 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
3429 }
3430
3431 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
3432 RouteSelectionMode::Fanout => sides,
3433 RouteSelectionMode::Weighted => {
3434 crate::small_sort::sort(&mut sides);
3435 let total_weight = sides.iter().fold(0_u64, |acc, side| {
3436 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
3437 });
3438 if total_weight == 0 {
3439 return Vec::new();
3440 }
3441 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
3442 let pick = *cursor % total_weight;
3443 *cursor = cursor.wrapping_add(1);
3444 let mut remaining = pick;
3445 for side in sides {
3446 let weight =
3447 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
3448 if remaining < weight {
3449 return vec![side];
3450 }
3451 remaining -= weight;
3452 }
3453 Vec::new()
3454 }
3455 RouteSelectionMode::Failover => {
3456 sides.sort_by_key(|side| {
3457 (
3458 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
3459 *side,
3460 )
3461 });
3462 sides.truncate(1);
3463 sides
3464 }
3465 }
3466 }
3467
3468 fn apply_adaptive_discovery_selection_locked(
3469 &self,
3470 st: &mut RouterInner,
3471 src: Option<RouterSideId>,
3472 mut sides: Vec<RouterSideId>,
3473 ) -> Vec<RouterSideId> {
3474 crate::small_sort::sort(&mut sides);
3475 let mut unmeasured: Vec<_> = sides
3476 .iter()
3477 .copied()
3478 .filter(|side| !st.adaptive_route_stats.contains_key(side))
3479 .collect();
3480 if !unmeasured.is_empty() {
3481 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
3482 let pick = (*cursor as usize) % unmeasured.len();
3483 *cursor = cursor.wrapping_add(1);
3484 return vec![unmeasured.swap_remove(pick)];
3485 }
3486
3487 let now_ms = self.clock.now_ms();
3488 let total_weight = sides.iter().fold(0_u64, |acc, side| {
3489 acc + st
3490 .adaptive_route_stats
3491 .get(side)
3492 .map(|stats| stats.weight(now_ms))
3493 .unwrap_or(1)
3494 });
3495 if total_weight == 0 {
3496 sides.truncate(1);
3497 return sides;
3498 }
3499
3500 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
3501 let pick = *cursor % total_weight;
3502 *cursor = cursor.wrapping_add(1);
3503 let mut remaining = pick;
3504 for side in sides {
3505 let weight = st
3506 .adaptive_route_stats
3507 .get(&side)
3508 .map(|stats| stats.weight(now_ms))
3509 .unwrap_or(1);
3510 if remaining < weight {
3511 return vec![side];
3512 }
3513 remaining -= weight;
3514 }
3515 Vec::new()
3516 }
3517
3518 fn record_side_tx_sample(
3519 &self,
3520 side: RouterSideId,
3521 bytes: usize,
3522 started_ms: u64,
3523 ended_ms: u64,
3524 ) {
3525 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
3526 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
3527 .min(u128::from(u64::MAX)) as u64;
3528 let mut st = self.state.lock();
3529 st.adaptive_route_stats
3530 .entry(side)
3531 .or_default()
3532 .observe(bytes, sample_bps, ended_ms);
3533 }
3534
3535 pub fn note_side_link_probe_sample(
3540 &self,
3541 side: RouterSideId,
3542 bytes: usize,
3543 duration_ms: u64,
3544 ) -> TelemetryResult<()> {
3545 {
3546 let st = self.state.lock();
3547 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
3548 }
3549 let ended_ms = self.clock.now_ms();
3550 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
3551 Ok(())
3552 }
3553
3554 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
3555 match data {
3556 RouterItem::Packet(pkt) => Ok(wire_format::packet_wire_size(pkt)),
3557 RouterItem::Packed(bytes) => Ok(bytes.len()),
3558 }
3559 }
3560
3561 #[inline]
3571 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
3572 match data {
3573 RouterItem::Packet(pkt) => {
3574 if !Self::is_end_to_end_ack_packet(pkt) {
3575 return Ok(None);
3576 }
3577 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
3578 }
3579 RouterItem::Packed(bytes) => {
3580 if wire_format::peek_routing_frame_info(bytes.as_ref())
3581 .ok()
3582 .is_some_and(|frame| {
3583 frame.envelope.ty != DataType::ReliableAck || frame.ack_only()
3584 })
3585 {
3586 return Ok(None);
3587 }
3588 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
3589 if !Self::is_end_to_end_ack_packet(&pkt) {
3590 return Ok(None);
3591 }
3592 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
3593 }
3594 }
3595 }
3596
3597 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
3598 if payload.len() != 8 {
3599 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
3600 }
3601 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
3602 }
3603
3604 #[inline]
3605 fn sender_hash(sender: &str) -> u64 {
3606 if let Some(address) = sender
3607 .strip_prefix("@addr:")
3608 .and_then(|v| v.parse::<u32>().ok())
3609 {
3610 return u64::from(address);
3611 }
3612 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
3613 }
3614
3615 #[inline]
3616 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
3617 let hash = Self::sender_hash(hostname);
3618 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
3619 if address == 0 {
3620 address = 1;
3621 }
3622 address
3623 }
3624
3625 fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
3630 let Some(address) = sender
3631 .strip_prefix("@addr:")
3632 .and_then(|value| value.parse::<u32>().ok())
3633 else {
3634 return sender.to_string();
3635 };
3636 Self::hostname_for_address_locked(st, address).unwrap_or_else(|| sender.to_string())
3637 }
3638
3639 fn hostname_for_address_locked(st: &RouterInner, address: NodeAddress) -> Option<String> {
3646 Self::hostname_for_address_borrowed_locked(st, address).map(str::to_string)
3647 }
3648
3649 fn hostname_for_address_borrowed_locked<'a>(
3650 st: &'a RouterInner,
3651 address: NodeAddress,
3652 ) -> Option<&'a str> {
3653 if let Some(hostname) = st.address_by_value.get(&address) {
3654 return Some(hostname.as_str());
3655 }
3656
3657 let mut found: Option<&'a str> = None;
3658 let mut collision = false;
3659 let mut consider = |candidate: &'a str| {
3660 if sender_address_u32(candidate) != address {
3661 return;
3662 }
3663 match found {
3664 None => found = Some(candidate),
3665 Some(existing) if existing == candidate => {}
3666 Some(_) => collision = true,
3670 }
3671 };
3672
3673 for entry in st.address_book.values() {
3674 consider(entry.hostname.as_ref());
3675 }
3676 #[cfg(feature = "discovery")]
3677 for route in st.discovery_routes.values() {
3678 for (announcer, state) in &route.announcers {
3679 consider(announcer);
3680 for board in state.topology_boards.iter() {
3681 consider(&board.sender_id);
3682 for peer in &board.connections {
3688 consider(peer);
3689 }
3690 }
3691 }
3692 }
3693 if collision { None } else { found }
3694 }
3695
3696 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
3697 match mode {
3698 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
3699 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
3700 _ => AddressAssignmentMode::Dynamic,
3701 }
3702 }
3703
3704 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
3705 let rank = match entry.mode {
3706 AddressAssignmentMode::Static(_) => 2,
3707 AddressAssignmentMode::Requested(_) => 1,
3708 AddressAssignmentMode::Dynamic => 0,
3709 };
3710 (
3711 rank,
3712 core::cmp::Reverse(entry.birth_ms),
3713 u64::MAX - entry.owner_hash,
3714 )
3715 }
3716
3717 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
3718 (
3719 core::cmp::Reverse(entry.birth_ms),
3720 u64::MAX - entry.owner_hash,
3721 )
3722 }
3723
3724 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
3725 let mut candidate = if seed == 0 { 1 } else { seed };
3726 for _ in 0..u32::MAX {
3727 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
3728 return candidate;
3729 }
3730 candidate = candidate.wrapping_add(1);
3731 if candidate == 0 {
3732 candidate = 1;
3733 }
3734 }
3735 1
3736 }
3737
3738 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
3739 if !st.address_book.contains_key(base) {
3740 return base.to_string();
3741 }
3742 let stem = if base.is_empty() { "node" } else { base };
3743 let mut candidate = format!("{stem}-{owner_hash:08x}");
3744 let mut suffix = 1u32;
3745 while st.address_book.contains_key(&candidate) {
3746 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
3747 suffix = suffix.saturating_add(1);
3748 }
3749 candidate
3750 }
3751
3752 fn update_local_identity_locked(
3753 &self,
3754 st: &mut RouterInner,
3755 mut local: AddressBookEntry,
3756 reason: AddressChangeReason,
3757 ) -> AddressChange {
3758 let old = st.local_address.clone();
3759 st.address_book.remove(old.hostname.as_ref());
3760 st.address_by_value.remove(&old.address);
3761 local.last_seen_ms = self.clock.now_ms();
3762 st.address_by_value
3763 .insert(local.address, local.hostname.to_string());
3764 st.address_book
3765 .insert(local.hostname.to_string(), local.clone());
3766 st.local_address = local.clone();
3767 *self.sender.lock() = local.hostname.clone();
3768 #[cfg(feature = "discovery")]
3769 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
3770 AddressChange {
3771 old_hostname: old.hostname,
3772 new_hostname: local.hostname,
3773 old_address: old.address,
3774 new_address: local.address,
3775 reason,
3776 }
3777 }
3778
3779 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
3780 for handler in self.cfg.address_change_handlers.iter() {
3781 (handler.handler)(change.clone())?;
3782 }
3783 Ok(())
3784 }
3785
3786 #[cfg(feature = "discovery")]
3787 fn local_address_advertisement(
3788 &self,
3789 reachable_endpoints: Vec<DataEndpoint>,
3790 reachable_network_variables: Vec<DataType>,
3791 reachable_timesync_sources: Vec<String>,
3792 link_capabilities: discovery::LinkCapabilities,
3793 state: u8,
3794 ) -> discovery::AddressAdvertisement {
3795 let st = self.state.lock();
3796 discovery::AddressAdvertisement {
3797 hostname: st.local_address.hostname.to_string(),
3798 address: st.local_address.address,
3799 requested_address: st.local_address.requested_address,
3800 mode: st.local_address.mode.mode_code(),
3801 state,
3802 birth_ms: st.local_address.birth_ms,
3803 owner_hash: st.local_address.owner_hash,
3804 reachable_endpoints,
3805 reachable_network_variables,
3806 reachable_timesync_sources,
3807 link_capabilities,
3808 }
3809 }
3810
3811 #[cfg(feature = "discovery")]
3812 fn ingest_address_advertisement(
3813 &self,
3814 ad: discovery::AddressAdvertisement,
3815 ) -> TelemetryResult<bool> {
3816 let now_ms = self.clock.now_ms();
3817 let mut remote = AddressBookEntry {
3818 hostname: Arc::from(ad.hostname.as_str()),
3819 address: ad.address,
3820 requested_address: ad.requested_address,
3821 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
3822 birth_ms: ad.birth_ms,
3823 owner_hash: ad.owner_hash,
3824 last_seen_ms: now_ms,
3825 };
3826 let mut change = None;
3827 let mut changed = false;
3828 {
3829 let mut st = self.state.lock();
3830 let local = st.local_address.clone();
3831 let hostname_conflict =
3832 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
3833 let address_conflict =
3834 remote.address == local.address && remote.owner_hash != local.owner_hash;
3835
3836 if hostname_conflict
3837 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
3838 {
3839 let mut next = local.clone();
3840 next.hostname = Arc::from(Self::unique_hostname_locked(
3841 &st,
3842 local.hostname.as_ref(),
3843 local.owner_hash,
3844 ));
3845 change = Some(self.update_local_identity_locked(
3846 &mut st,
3847 next,
3848 AddressChangeReason::HostnameConflict,
3849 ));
3850 changed = true;
3851 } else if address_conflict
3852 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
3853 {
3854 let mut next = local.clone();
3855 let seed = next
3856 .requested_address
3857 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
3858 next.address = Self::allocate_free_address_locked(&st, seed);
3859 let reason = match next.mode {
3860 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
3861 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
3862 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
3863 };
3864 change = Some(self.update_local_identity_locked(&mut st, next, reason));
3865 changed = true;
3866 } else if hostname_conflict {
3867 remote.hostname = Arc::from(Self::unique_hostname_locked(
3868 &st,
3869 remote.hostname.as_ref(),
3870 remote.owner_hash,
3871 ));
3872 changed = true;
3873 } else if address_conflict {
3874 let seed = remote
3875 .requested_address
3876 .max(Self::fallback_address_for_hostname(
3877 remote.hostname.as_ref(),
3878 ));
3879 remote.address = Self::allocate_free_address_locked(&st, seed);
3880 changed = true;
3881 }
3882
3883 st.address_book.remove(remote.hostname.as_ref());
3884 st.address_by_value.remove(&remote.address);
3885 st.address_by_value
3886 .insert(remote.address, remote.hostname.to_string());
3887 st.address_book
3888 .insert(remote.hostname.to_string(), remote.clone());
3889 }
3890 if let Some(change) = change {
3891 self.notify_address_change(change)?;
3892 }
3893 Ok(changed)
3894 }
3895
3896 #[inline]
3897 fn is_end_to_end_ack_sender(sender: &str) -> bool {
3898 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
3899 }
3900
3901 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
3902 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
3903 && !ack_sender.is_empty()
3904 {
3905 return Some(Self::sender_hash(ack_sender));
3906 }
3907 None
3908 }
3909
3910 fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
3916 if pkt.data_type() != DataType::ReliableAck {
3917 return None;
3918 }
3919 pkt.wire_target_senders()
3920 .first()
3921 .copied()
3922 .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
3923 }
3924
3925 fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
3926 Self::end_to_end_ack_sender_hash(pkt).is_some()
3927 && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
3928 }
3929
3930 fn encode_end_to_end_ack_sender(&self) -> String {
3931 let sender = self.sender_arc();
3932 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
3933 }
3934
3935 #[cfg(feature = "discovery")]
3936 fn is_end_to_end_destination_sender(sender: &str) -> bool {
3937 sender != "RELAY" && !Self::is_end_to_end_ack_sender(sender)
3938 }
3939
3940 fn encode_end_to_end_reliable_ack(packet_id: u64) -> crate::SharedBytes {
3941 let mut payload = Vec::with_capacity(8);
3942 payload.extend_from_slice(&packet_id.to_le_bytes());
3943 crate::shared_bytes::convert(Arc::<[u8]>::from(payload))
3944 }
3945
3946 fn encode_p2p_payload(
3947 src_hostname: &str,
3948 src_address: NodeAddress,
3949 src_port: P2pPort,
3950 dst_port: P2pPort,
3951 payload: &[u8],
3952 ) -> TelemetryResult<crate::SharedBytes> {
3953 let host_len = u16::try_from(src_hostname.len())
3954 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
3955 let payload_len = u32::try_from(payload.len())
3956 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
3957 let mut out = Vec::with_capacity(
3958 15usize
3959 .saturating_add(src_hostname.len())
3960 .saturating_add(payload.len()),
3961 );
3962 out.push(1);
3963 out.extend_from_slice(&dst_port.to_le_bytes());
3964 out.extend_from_slice(&src_port.to_le_bytes());
3965 out.extend_from_slice(&src_address.to_le_bytes());
3966 out.extend_from_slice(&host_len.to_le_bytes());
3967 out.extend_from_slice(&payload_len.to_le_bytes());
3968 out.extend_from_slice(src_hostname.as_bytes());
3969 out.extend_from_slice(payload);
3970 Ok(out.into())
3971 }
3972
3973 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
3974 if payload.len() < 15 {
3975 return Err(TelemetryError::Unpack("p2p frame short"));
3976 }
3977 if payload[0] != 1 {
3978 return Err(TelemetryError::Unpack("p2p frame version"));
3979 }
3980 let destination_port =
3981 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
3982 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
3983 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
3984 let host_len =
3985 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
3986 let body_len =
3987 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
3988 let host_start = 15usize;
3989 let host_end = host_start.saturating_add(host_len);
3990 let body_end = host_end.saturating_add(body_len);
3991 if host_end > payload.len() || body_end != payload.len() {
3992 return Err(TelemetryError::Unpack("p2p frame length"));
3993 }
3994 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
3995 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
3996 Ok(P2pDecoded {
3997 source_hostname,
3998 source_address,
3999 source_port,
4000 destination_port,
4001 payload: &payload[host_end..body_end],
4002 })
4003 }
4004
4005 fn encode_p2p_stream_payload(
4006 flags: u8,
4007 source_stream_id: P2pStreamId,
4008 destination_stream_id: P2pStreamId,
4009 sequence: u32,
4010 payload: &[u8],
4011 ) -> TelemetryResult<crate::SharedBytes> {
4012 let payload_len = u32::try_from(payload.len())
4013 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
4014 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
4015 out.extend_from_slice(&P2P_STREAM_MAGIC);
4016 out.push(P2P_STREAM_VERSION);
4017 out.push(flags);
4018 out.extend_from_slice(&source_stream_id.to_le_bytes());
4019 out.extend_from_slice(&destination_stream_id.to_le_bytes());
4020 out.extend_from_slice(&sequence.to_le_bytes());
4021 out.extend_from_slice(&payload_len.to_le_bytes());
4022 out.extend_from_slice(payload);
4023 Ok(out.into())
4024 }
4025
4026 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
4027 if !payload.starts_with(&P2P_STREAM_MAGIC) {
4028 return Ok(None);
4029 }
4030 if payload.len() < 22 {
4031 return Err(TelemetryError::Unpack("p2p stream frame short"));
4032 }
4033 if payload[4] != P2P_STREAM_VERSION {
4034 return Err(TelemetryError::Unpack("p2p stream frame version"));
4035 }
4036 let flags = payload[5];
4037 let source_stream_id =
4038 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
4039 let destination_stream_id =
4040 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
4041 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
4042 let body_len =
4043 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
4044 let body_end = 22usize.saturating_add(body_len);
4045 if body_end != payload.len() {
4046 return Err(TelemetryError::Unpack("p2p stream frame length"));
4047 }
4048 Ok(Some(P2pStreamDecoded {
4049 flags,
4050 source_stream_id,
4051 destination_stream_id,
4052 sequence,
4053 payload: &payload[22..],
4054 }))
4055 }
4056
4057 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
4058 for _ in 0..u32::MAX {
4059 let id = st.next_p2p_stream_id.max(1);
4060 st.next_p2p_stream_id = st.next_p2p_stream_id.wrapping_add(1).max(1);
4061 if !st.p2p_stream_sessions.contains_key(&id) {
4062 return id;
4063 }
4064 }
4065 0
4066 }
4067
4068 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
4069 if pkt.data_type() != DataType::P2pMessage {
4070 return Ok(());
4071 }
4072 let decoded = Self::decode_p2p_payload(pkt.payload())?;
4073 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
4074 return self.dispatch_p2p_stream_frame(&decoded, &stream);
4075 }
4076 let handlers = {
4077 let st = self.state.lock();
4078 st.p2p_port_handlers
4079 .get(&decoded.destination_port)
4080 .cloned()
4081 .unwrap_or_default()
4082 };
4083 for handler in handlers {
4084 (handler.handler)(P2pMessage {
4085 source_hostname: decoded.source_hostname,
4086 source_address: decoded.source_address,
4087 source_port: decoded.source_port,
4088 destination_port: decoded.destination_port,
4089 payload: decoded.payload,
4090 })?;
4091 }
4092 Ok(())
4093 }
4094
4095 fn dispatch_p2p_stream_frame(
4096 &self,
4097 msg: &P2pDecoded<'_>,
4098 frame: &P2pStreamDecoded<'_>,
4099 ) -> TelemetryResult<()> {
4100 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
4101 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, crate::SharedBytes)> = None;
4102 {
4103 let mut st = self.state.lock();
4104 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
4105 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
4106 let existing_id = st.p2p_stream_sessions.iter().find_map(|(id, session)| {
4107 (session.peer_stream_id == frame.source_stream_id
4108 && session.local_port == msg.destination_port
4109 && session.peer_port == msg.source_port
4110 && session.peer_address == msg.source_address
4111 && session.peer_hostname.as_ref() == msg.source_hostname)
4112 .then_some(*id)
4113 });
4114 let local_id = if let Some(local_id) = existing_id {
4115 local_id
4116 } else {
4117 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
4118 if local_id == 0 {
4119 return Err(TelemetryError::Io("p2p stream id exhausted"));
4120 }
4121 st.p2p_stream_sessions.insert(
4122 local_id,
4123 P2pStreamSession {
4124 peer_hostname: peer_hostname.clone(),
4125 peer_address: msg.source_address,
4126 local_port: msg.destination_port,
4127 peer_port: msg.source_port,
4128 peer_stream_id: frame.source_stream_id,
4129 next_sequence: 1,
4130 connected: true,
4131 },
4132 );
4133 let handlers = st
4134 .p2p_stream_handlers
4135 .get(&msg.destination_port)
4136 .cloned()
4137 .unwrap_or_default();
4138 events.push(PendingP2pStreamEvent {
4139 handlers,
4140 kind: P2pStreamEventKind::Accepted,
4141 stream_id: local_id,
4142 peer_stream_id: frame.source_stream_id,
4143 peer_hostname: peer_hostname.clone(),
4144 peer_address: msg.source_address,
4145 local_port: msg.destination_port,
4146 peer_port: msg.source_port,
4147 });
4148 local_id
4149 };
4150 let dst =
4151 st.address_book
4152 .get(msg.source_hostname)
4153 .cloned()
4154 .unwrap_or(AddressBookEntry {
4155 hostname: peer_hostname,
4156 address: msg.source_address,
4157 requested_address: msg.source_address,
4158 mode: AddressAssignmentMode::Dynamic,
4159 birth_ms: self.clock.now_ms(),
4160 owner_hash: Self::sender_hash(msg.source_hostname),
4161 last_seen_ms: self.clock.now_ms(),
4162 });
4163 let payload = Self::encode_p2p_stream_payload(
4164 P2P_STREAM_SYN | P2P_STREAM_ACK,
4165 local_id,
4166 frame.source_stream_id,
4167 0,
4168 &[],
4169 )?;
4170 reply = Some((dst, msg.source_port, msg.destination_port, payload));
4171 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
4172 if let Some(session) = st.p2p_stream_sessions.get_mut(&frame.destination_stream_id)
4173 {
4174 session.peer_stream_id = frame.source_stream_id;
4175 session.connected = true;
4176 let peer_hostname = session.peer_hostname.clone();
4177 let peer_address = session.peer_address;
4178 let local_port = session.local_port;
4179 let peer_port = session.peer_port;
4180 let handlers = st
4181 .p2p_stream_handlers
4182 .get(&local_port)
4183 .cloned()
4184 .unwrap_or_default();
4185 events.push(PendingP2pStreamEvent {
4186 handlers,
4187 kind: P2pStreamEventKind::Connected,
4188 stream_id: frame.destination_stream_id,
4189 peer_stream_id: frame.source_stream_id,
4190 peer_hostname,
4191 peer_address,
4192 local_port,
4193 peer_port,
4194 });
4195 }
4196 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
4197 let kind = if frame.flags & P2P_STREAM_RST != 0 {
4198 P2pStreamEventKind::Reset
4199 } else if frame.flags & P2P_STREAM_FIN != 0 {
4200 P2pStreamEventKind::Closed
4201 } else {
4202 P2pStreamEventKind::Data
4203 };
4204 let session_id = if frame.destination_stream_id != 0 {
4205 Some(frame.destination_stream_id)
4206 } else {
4207 st.p2p_stream_sessions.iter().find_map(|(id, session)| {
4208 (session.peer_stream_id == frame.source_stream_id
4209 && session.local_port == msg.destination_port
4210 && session.peer_port == msg.source_port
4211 && session.peer_address == msg.source_address
4212 && session.peer_hostname.as_ref() == msg.source_hostname)
4213 .then_some(*id)
4214 })
4215 };
4216 if let Some(session_id) = session_id
4217 && let Some(session) = st.p2p_stream_sessions.get(&session_id)
4218 {
4219 let handlers = st
4220 .p2p_stream_handlers
4221 .get(&session.local_port)
4222 .cloned()
4223 .unwrap_or_default();
4224 events.push(PendingP2pStreamEvent {
4225 handlers,
4226 kind,
4227 stream_id: session_id,
4228 peer_stream_id: frame.source_stream_id,
4229 peer_hostname: session.peer_hostname.clone(),
4230 peer_address: session.peer_address,
4231 local_port: session.local_port,
4232 peer_port: session.peer_port,
4233 });
4234 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
4235 st.p2p_stream_sessions.remove(&session_id);
4236 }
4237 }
4238 }
4239 }
4240 if let Some((dst, dst_port, src_port, payload)) = reply {
4241 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
4242 }
4243 for pending in events {
4244 for handler in pending.handlers {
4245 (handler.handler)(P2pStreamEvent {
4246 kind: pending.kind,
4247 stream_id: pending.stream_id,
4248 peer_stream_id: pending.peer_stream_id,
4249 sequence: frame.sequence,
4250 peer_hostname: pending.peer_hostname.as_ref(),
4251 peer_address: pending.peer_address,
4252 local_port: pending.local_port,
4253 peer_port: pending.peer_port,
4254 payload: frame.payload,
4255 })?;
4256 }
4257 }
4258 Ok(())
4259 }
4260
4261 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) -> bool {
4267 let mut st = self.state.lock();
4268 if st.end_to_end_reliable_tx.contains_key(&packet_id)
4272 && st
4273 .reliable_return_routes
4274 .get(&packet_id)
4275 .is_some_and(|route| route.upstream_contract.is_some())
4276 {
4277 return false;
4278 }
4279 let now = self.clock.now_ms();
4283 let retry_ms = runtime_reliable_retransmit_ms();
4284 let duplicate = st.recent_rx.contains(&packet_id);
4285 let last_replay_ms = if duplicate {
4286 now.saturating_sub(retry_ms)
4287 } else {
4288 now
4289 };
4290 let replace_side = !duplicate
4296 || st
4297 .reliable_return_routes
4298 .get(&packet_id)
4299 .is_none_or(|route| {
4300 now.saturating_sub(route.last_replay_ms) >= retry_ms
4301 || !Self::side_ref(&st, route.side)
4302 .is_ok_and(|side| side.opts.egress_enabled)
4303 });
4304 if !Self::remember_reliable_return_route_locked(&mut st, packet_id) {
4305 return true;
4306 }
4307 st.reliable_return_routes
4308 .entry(packet_id)
4309 .and_modify(|route| {
4310 if replace_side {
4311 route.side = side;
4312 }
4313 })
4314 .or_insert(ReliableReturnRouteState {
4315 side,
4316 last_replay_ms,
4317 upstream_contract: None,
4318 });
4319 true
4320 }
4321
4322 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) -> bool {
4328 let cap = runtime_reliable_max_return_routes().max(1);
4329 st.reliable_return_route_order
4330 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
4331 while st.reliable_return_route_order.len() >= cap {
4332 let Some(index) = st
4333 .reliable_return_route_order
4334 .iter()
4335 .position(|id| !st.end_to_end_reliable_tx.contains_key(id))
4336 else {
4337 return false;
4338 };
4339 if let Some(oldest) = st.reliable_return_route_order.remove(index) {
4340 st.reliable_return_routes.remove(&oldest);
4341 }
4342 }
4343 st.reliable_return_route_order.push_back(packet_id);
4344 true
4345 }
4346
4347 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
4348 let cap = runtime_reliable_max_end_to_end_pending().max(1);
4349 st.end_to_end_reliable_tx_order
4350 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
4351 while st.end_to_end_reliable_tx_order.len() >= cap {
4352 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
4353 st.end_to_end_reliable_tx.remove(&oldest);
4354 } else {
4355 break;
4356 }
4357 }
4358 st.end_to_end_reliable_tx_order.push_back(packet_id);
4359 }
4360
4361 #[cfg(feature = "discovery")]
4362 fn expected_end_to_end_destinations_locked(
4363 &self,
4364 st: &RouterInner,
4365 data: &RouterItem,
4366 exclude: Option<RouterSideId>,
4367 managed: bool,
4368 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
4369 let (eps, ty) = self.item_route_info(data)?;
4370 let now_ms = self.clock.now_ms();
4371 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
4372 if managed || st.managed_variable_types.contains(&ty.as_u32()) {
4381 let mut out = BTreeMap::new();
4382 for (&side, route) in st.discovery_routes.iter() {
4383 if Some(side) == exclude {
4384 continue;
4385 }
4386 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4387 continue;
4388 }
4389 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
4390 continue;
4391 };
4392 if restrict_link_local && !side_ref.opts.link_local_enabled {
4393 continue;
4394 }
4395 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
4396 continue;
4397 }
4398 for (sender_id, sender_state) in route.announcers.iter() {
4399 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
4400 || !sender_state.reachable_network_variables.contains(&ty)
4401 || !Self::is_end_to_end_destination_sender(sender_id)
4402 {
4403 continue;
4404 }
4405 out.insert(Self::sender_hash(sender_id), side);
4406 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
4407 return Ok(out);
4408 }
4409 }
4410 }
4411 return Ok(out);
4412 }
4413 let prefer_best_overlap =
4414 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
4415 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
4416 let mut best_overlap = 0usize;
4417 let limit = runtime_reliable_max_end_to_end_pending().max(1);
4418 let mut out = BTreeMap::new();
4419 for (&side, route) in st.discovery_routes.iter() {
4420 if Some(side) == exclude {
4421 continue;
4422 }
4423 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4424 continue;
4425 }
4426 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
4427 continue;
4428 };
4429 if restrict_link_local && !side_ref.opts.link_local_enabled {
4430 continue;
4431 }
4432 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
4433 continue;
4434 }
4435 for sender_state in route.announcers.values() {
4436 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4437 continue;
4438 }
4439 for board in sender_state.topology_boards.iter() {
4440 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
4441 continue;
4442 }
4443 let overlap = Self::endpoint_overlap_count(
4444 board.reachable_endpoints.iter().copied(),
4445 &scoring_eps,
4446 );
4447 if overlap > 0 {
4448 if prefer_best_overlap {
4449 if overlap < best_overlap {
4450 continue;
4451 }
4452 if overlap > best_overlap {
4453 out.clear();
4454 best_overlap = overlap;
4455 }
4456 if out.len() >= limit {
4460 continue;
4461 }
4462 }
4463 out.insert(Self::sender_hash(&board.sender_id), side);
4464 if !prefer_best_overlap && out.len() >= limit {
4465 return Ok(out);
4466 }
4467 }
4468 }
4469 }
4470 }
4471
4472 Ok(out)
4473 }
4474
4475 #[cfg(feature = "discovery")]
4476 #[allow(clippy::too_many_arguments)]
4477 fn discovered_route_candidates_locked(
4478 &self,
4479 st: &RouterInner,
4480 exclude: Option<RouterSideId>,
4481 ty: DataType,
4482 eps: &[DataEndpoint],
4483 target_senders: &[u64],
4484 prefer_nonlocal: bool,
4485 preferred_timesync_source: Option<&str>,
4486 ) -> Vec<DiscoveryCandidateMatch> {
4487 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
4488 let now_ms = self.clock.now_ms();
4489 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
4490 let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
4491 exclude != Some(side)
4492 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
4493 && route.reachable_network_variables.contains(&ty)
4494 });
4495 let mut out = Vec::new();
4496 let mut unique_endpoint_fallbacks = Vec::new();
4497
4498 for (&side, route) in st.discovery_routes.iter() {
4499 if exclude == Some(side)
4500 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
4501 {
4502 continue;
4503 }
4504 if restrict_link_local
4505 && st
4506 .sides
4507 .get(side)
4508 .and_then(|s| s.as_ref())
4509 .map(|s| !s.opts.link_local_enabled)
4510 .unwrap_or(true)
4511 {
4512 continue;
4513 }
4514 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
4515 continue;
4516 }
4517 if !target_senders.is_empty()
4518 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
4519 {
4520 if scoring_eps
4521 .iter()
4522 .copied()
4523 .any(|ep| route.reachable.contains(&ep))
4524 {
4525 unique_endpoint_fallbacks.push(side);
4526 }
4527 continue;
4528 }
4529 if !target_senders.is_empty() {
4530 out.push(DiscoveryCandidateMatch {
4531 side,
4532 overlap: usize::MAX,
4533 });
4534 continue;
4535 }
4536 if has_variable_owner {
4537 if route.reachable_network_variables.contains(&ty) {
4538 out.push(DiscoveryCandidateMatch {
4539 side,
4540 overlap: usize::MAX,
4541 });
4542 }
4543 continue;
4544 }
4545 if preferred_timesync_source
4546 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
4547 {
4548 out.push(DiscoveryCandidateMatch {
4549 side,
4550 overlap: usize::MAX,
4551 });
4552 continue;
4553 }
4554 let overlap =
4555 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
4556 if overlap > 0 {
4557 out.push(DiscoveryCandidateMatch { side, overlap });
4558 }
4559 }
4560 if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
4561 out.push(DiscoveryCandidateMatch {
4562 side: unique_endpoint_fallbacks[0],
4563 overlap: usize::MAX,
4564 });
4565 }
4566 out
4567 }
4568
4569 #[cfg(feature = "discovery")]
4575 fn retain_shortest_discovery_candidates_locked(
4576 st: &RouterInner,
4577 matches: &mut Vec<DiscoveryCandidateMatch>,
4578 endpoints: &[DataEndpoint],
4579 target_senders: &[u64],
4580 now_ms: u64,
4581 ) {
4582 if matches.len() <= 1 || endpoints.is_empty() {
4583 return;
4584 }
4585
4586 let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
4587 let route = st.discovery_routes.get(&side)?;
4588 route
4589 .announcers
4590 .iter()
4591 .filter(|(_, state)| {
4592 now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
4593 })
4594 .filter_map(|(announcer, state)| {
4595 let targets: BTreeSet<&str> = state
4596 .topology_boards
4597 .iter()
4598 .filter(|board| {
4599 target_sender
4600 .map(|target| Self::sender_hash(&board.sender_id) == target)
4601 .unwrap_or_else(|| {
4602 endpoints.iter().any(|endpoint| {
4603 board.reachable_endpoints.contains(endpoint)
4604 })
4605 })
4606 })
4607 .map(|board| board.sender_id.as_str())
4608 .collect();
4609 if targets.is_empty() {
4610 return if let Some(target) = target_sender {
4611 state
4612 .topology_boards
4613 .iter()
4614 .any(|board| Self::sender_hash(&board.sender_id) == target)
4615 .then_some(usize::MAX / 2)
4616 } else {
4617 state
4618 .reachable
4619 .iter()
4620 .any(|endpoint| endpoints.contains(endpoint))
4621 .then_some(usize::MAX / 2)
4622 };
4623 }
4624
4625 let mut distances: BTreeMap<&str, usize> = BTreeMap::new();
4626 let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
4627 while let Some((sender, distance)) = pending.pop_front() {
4628 if distances.contains_key(sender) {
4629 continue;
4630 }
4631 distances.insert(sender, distance);
4632 if targets.contains(sender) {
4633 return Some(distance);
4634 }
4635 for board in state.topology_boards.iter() {
4636 if board.sender_id == sender {
4637 for peer in &board.connections {
4638 if !distances.contains_key(peer.as_str()) {
4639 pending.push_back((peer.as_str(), distance + 1));
4640 }
4641 }
4642 } else if board.connections.iter().any(|peer| peer == sender)
4643 && !distances.contains_key(board.sender_id.as_str())
4644 {
4645 pending.push_back((board.sender_id.as_str(), distance + 1));
4646 }
4647 }
4648 }
4649 None
4650 })
4651 .min()
4652 };
4653
4654 let route_targets: Vec<Option<u64>> = if target_senders.is_empty() {
4655 vec![None]
4656 } else {
4657 target_senders.iter().copied().map(Some).collect()
4658 };
4659 let mut selected = BTreeSet::new();
4660 for target in route_targets {
4661 let distances: Vec<(RouterSideId, usize)> = matches
4662 .iter()
4663 .filter_map(|candidate| {
4664 route_distance(candidate.side, target)
4665 .map(|distance| (candidate.side, distance))
4666 })
4667 .collect();
4668 let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
4669 continue;
4670 };
4671 selected.extend(
4672 distances
4673 .into_iter()
4674 .filter(|(_, distance)| *distance == best)
4675 .map(|(side, _)| side),
4676 );
4677 }
4678 if !selected.is_empty() {
4679 matches.retain(|candidate| selected.contains(&candidate.side));
4680 }
4681 }
4682
4683 #[cfg(feature = "discovery")]
4684 fn select_discovered_candidate_sides_locked(
4685 &self,
4686 st: &mut RouterInner,
4687 exclude: Option<RouterSideId>,
4688 ty: DataType,
4689 target_senders: &[u64],
4690 prefer_best_overlap: bool,
4691 matches: Vec<DiscoveryCandidateMatch>,
4692 ) -> Vec<RouterSideId> {
4693 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
4694 || !target_senders.is_empty()
4695 {
4696 RouteSelectionOrigin::Flood
4697 } else {
4698 RouteSelectionOrigin::Discovered
4699 };
4700
4701 let mut matches = matches;
4702 if matches.iter().any(|m| m.overlap == usize::MAX) {
4703 matches.retain(|m| m.overlap == usize::MAX);
4704 }
4705
4706 let selected: Vec<RouterSideId> = if prefer_best_overlap {
4707 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
4708 matches
4709 .into_iter()
4710 .filter(|m| m.overlap == best_overlap)
4711 .map(|m| m.side)
4712 .collect()
4713 } else {
4714 matches.into_iter().map(|m| m.side).collect()
4715 };
4716
4717 let has_distributed_variable_owners = selected.iter().any(|side| {
4724 st.discovery_routes
4725 .get(side)
4726 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
4727 });
4728 if DataType::try_named("HEARTBEAT") == Some(ty) || has_distributed_variable_owners {
4729 selected
4730 } else {
4731 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
4732 }
4733 }
4734
4735 #[allow(unused_mut)]
4736 fn register_end_to_end_reliable_tx(
4737 &self,
4738 data: &RouterItem,
4739 destinations: Option<BTreeMap<u64, RouterSideId>>,
4740 ) -> TelemetryResult<bool> {
4741 let packet_id = Self::get_hash(data);
4742 let now_ms = self.clock.now_ms();
4743 let ty = match data {
4744 RouterItem::Packet(pkt) => pkt.data_type(),
4745 RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
4746 };
4747 let mut st = self.state.lock();
4748 #[cfg(feature = "discovery")]
4749 let mut pending_destinations = match destinations {
4750 Some(destinations) => destinations,
4751 None => self.expected_end_to_end_destinations_locked(&st, data, None, false)?,
4752 };
4753 #[cfg(not(feature = "discovery"))]
4754 let mut pending_destinations = destinations.unwrap_or_default();
4755 let frozen = match data {
4759 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
4760 RouterItem::Packed(bytes) => {
4761 wire_format::peek_routing_envelope(bytes)?
4762 .target_senders()
4763 .len()
4764 != 0
4765 }
4766 };
4767 self.filter_trackable_end_to_end_destinations_locked(
4768 &st,
4769 ty,
4770 &mut pending_destinations,
4771 frozen,
4772 );
4773 let tracked_destinations = !pending_destinations.is_empty();
4774 if let Some(existing) = st.end_to_end_reliable_tx.get(&packet_id) {
4775 return Ok(!existing.waiting_for_predecessor);
4776 }
4777 if st.end_to_end_reliable_tx.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
4779 return Err(TelemetryError::HandlerError(
4780 "end-to-end reliable window full",
4781 ));
4782 }
4783 let ordered_type =
4784 matches!(reliable_mode(ty), crate::ReliableMode::Ordered).then_some(ty.as_u32());
4785 let waiting_for_predecessor = ordered_type.is_some()
4786 && st
4787 .end_to_end_reliable_tx
4788 .values()
4789 .any(|sent| sent.ordered_type == ordered_type);
4790 #[cfg(feature = "std")]
4791 if reliable_trace_enabled() {
4792 eprintln!(
4793 "[sedsnet-reliable] register time={now_ms} type={} id={packet_id:#x} waiting={waiting_for_predecessor} targets={pending_destinations:?}",
4794 ty.as_u32()
4795 );
4796 }
4797 let mut retained = data.clone();
4798 #[cfg(feature = "compact-packet-store")]
4799 retained.park_for_queue()?;
4800 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
4801 st.end_to_end_reliable_tx.insert(
4802 packet_id,
4803 EndToEndReliableSent {
4804 ordered_type,
4805 waiting_for_predecessor,
4806 data: retained,
4807 pending_destinations,
4808 tracked_destinations,
4809 last_send_ms: now_ms,
4810 retries: 0,
4811 queued: false,
4812 },
4813 );
4814 Ok(!waiting_for_predecessor)
4815 }
4816
4817 fn release_ordered_successors(&self) -> TelemetryResult<()> {
4820 let ready = {
4821 let mut st = self.state.lock();
4822 let RouterInner {
4823 end_to_end_reliable_tx_order: order,
4824 end_to_end_reliable_tx: pending,
4825 ..
4826 } = &mut *st;
4827 let mut ready = Vec::new();
4828 for (index, id) in order.iter().copied().enumerate() {
4829 let Some(sent) = pending.get(&id) else {
4830 continue;
4831 };
4832 if !sent.waiting_for_predecessor {
4833 continue;
4834 }
4835 let Some(ty) = sent.ordered_type else {
4836 continue;
4837 };
4838 if order.iter().take(index).any(|earlier| {
4842 pending
4843 .get(earlier)
4844 .is_some_and(|sent| sent.ordered_type == Some(ty))
4845 }) {
4846 continue;
4847 }
4848 let sent = pending.get_mut(&id).expect("retained ordered entry");
4849 sent.waiting_for_predecessor = false;
4850 sent.last_send_ms = self.clock.now_ms();
4851 ready.push(id);
4852 }
4853 ready
4854 };
4855 for id in ready {
4856 self.queue_end_to_end_reliable_retransmit(id)?;
4857 }
4858 Ok(())
4859 }
4860
4861 #[cfg(feature = "discovery")]
4862 fn reconcile_end_to_end_reliable_destinations_locked(
4863 &self,
4864 st: &mut RouterInner,
4865 ) -> TelemetryResult<()> {
4866 if st.end_to_end_reliable_tx.is_empty() {
4867 return Ok(());
4868 }
4869 let active_destinations = self.active_end_to_end_destinations_locked(st);
4870 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
4871 let mut completed = Vec::new();
4872
4873 for packet_id in packet_ids {
4874 let Some(data) = st
4875 .end_to_end_reliable_tx
4876 .get(&packet_id)
4877 .map(|sent| sent.data.clone())
4878 else {
4879 continue;
4880 };
4881 let excluded = st
4882 .reliable_return_routes
4883 .get(&packet_id)
4884 .filter(|route| route.upstream_contract.is_some())
4885 .map(|route| route.side);
4886 let expected =
4887 self.expected_end_to_end_destinations_locked(st, &data, excluded, false)?;
4888 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
4889 continue;
4890 };
4891 if !sent.tracked_destinations {
4892 continue;
4893 }
4894 if sent.ordered_type.is_some() {
4896 for (sender, side) in &mut sent.pending_destinations {
4897 if let Some(next_side) = expected.get(sender).or_else(|| {
4898 active_destinations
4899 .get(sender)
4900 .filter(|side| Some(**side) != excluded)
4901 }) {
4902 *side = *next_side;
4903 }
4904 }
4905 continue;
4906 }
4907 sent.pending_destinations.retain(|sender_hash, side| {
4908 match (
4909 expected.get(sender_hash),
4910 active_destinations
4911 .get(sender_hash)
4912 .filter(|side| Some(**side) != excluded),
4913 ) {
4914 (Some(next_side), _) | (None, Some(next_side)) => {
4915 *side = *next_side;
4916 true
4917 }
4918 (None, None) => false,
4919 }
4920 });
4921 if sent.pending_destinations.is_empty() {
4922 completed.push(packet_id);
4923 }
4924 }
4925
4926 for packet_id in completed {
4927 st.end_to_end_reliable_tx.remove(&packet_id);
4928 }
4929
4930 Ok(())
4931 }
4932
4933 #[cfg(feature = "discovery")]
4934 fn active_end_to_end_destinations_locked(
4935 &self,
4936 st: &RouterInner,
4937 ) -> BTreeMap<u64, RouterSideId> {
4938 let now_ms = self.clock.now_ms();
4939 let mut out = BTreeMap::new();
4940 for (&side, route) in st.discovery_routes.iter() {
4941 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4942 continue;
4943 }
4944 for (sender_name, sender_state) in &route.announcers {
4945 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4946 continue;
4947 }
4948 if Self::is_end_to_end_destination_sender(sender_name) {
4949 out.insert(Self::sender_hash(sender_name), side);
4950 }
4951 for board in sender_state.topology_boards.iter() {
4952 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
4953 continue;
4954 }
4955 out.insert(Self::sender_hash(&board.sender_id), side);
4956 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
4957 return out;
4958 }
4959 }
4960 }
4961 }
4962 out
4963 }
4964
4965 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
4966 matches!(
4967 st.sides
4968 .get(side)
4969 .and_then(Option::as_ref)
4970 .map(|side| &side.tx_handler),
4971 Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
4972 )
4973 }
4974
4975 fn filter_trackable_end_to_end_destinations_locked(
4976 &self,
4977 st: &RouterInner,
4978 ty: DataType,
4979 pending: &mut BTreeMap<u64, RouterSideId>,
4980 frozen: bool,
4981 ) {
4982 if DataType::try_named("HEARTBEAT") == Some(ty) {
4988 pending.clear();
4989 return;
4990 }
4991 if frozen {
4992 return;
4993 }
4994 let now_ms = self.clock.now_ms();
4995 pending.retain(|_, side| {
4996 Self::side_supports_end_to_end_tracking_locked(st, *side)
4997 && (st.managed_variable_types.contains(&ty.as_u32())
4998 || is_reliable_type(ty)
4999 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
5000 });
5001 }
5002
5003 #[cfg(feature = "discovery")]
5004 fn side_has_multiple_announcers_locked(
5005 &self,
5006 st: &RouterInner,
5007 side: RouterSideId,
5008 now_ms: u64,
5009 ) -> bool {
5010 st.discovery_routes
5011 .get(&side)
5012 .map(|route| {
5013 route
5014 .announcers
5015 .values()
5016 .filter(|sender| {
5017 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5018 })
5019 .take(2)
5020 .count()
5021 > 1
5022 })
5023 .unwrap_or(false)
5024 }
5025
5026 #[cfg(not(feature = "discovery"))]
5027 fn side_has_multiple_announcers_locked(
5028 &self,
5029 _st: &RouterInner,
5030 _side: RouterSideId,
5031 _now_ms: u64,
5032 ) -> bool {
5033 false
5034 }
5035
5036 fn queue_end_to_end_reliable_ack(
5037 &self,
5038 pkt: &Packet,
5039 called_from_queue: bool,
5040 ) -> TelemetryResult<()> {
5041 self.queue_end_to_end_reliable_ack_for_packet_id(
5042 pkt.packet_id(),
5043 pkt.sender(),
5044 called_from_queue,
5045 )
5046 }
5047
5048 fn queue_end_to_end_reliable_ack_for_packet_id(
5049 &self,
5050 packet_id: u64,
5051 original_sender: &str,
5052 called_from_queue: bool,
5053 ) -> TelemetryResult<()> {
5054 if !self.cfg.reliable_enabled() {
5055 return Ok(());
5056 }
5057 let original_sender_hash = {
5058 let st = self.state.lock();
5059 Self::sender_hash(&Self::canonical_sender_locked(&st, original_sender))
5060 };
5061 self.queue_end_to_end_reliable_ack_for_sender_hash(
5062 packet_id,
5063 original_sender_hash,
5064 called_from_queue,
5065 )
5066 }
5067
5068 fn queue_end_to_end_reliable_ack_for_sender_hash(
5069 &self,
5070 packet_id: u64,
5071 original_sender_hash: u64,
5072 called_from_queue: bool,
5073 ) -> TelemetryResult<()> {
5074 if !self.cfg.reliable_enabled() {
5075 return Ok(());
5076 }
5077 let ack_sender = self.encode_end_to_end_ack_sender();
5078 let ack = Packet::new(
5079 DataType::ReliableAck,
5080 message_meta(DataType::ReliableAck).endpoints_ref(),
5081 ack_sender.as_str(),
5082 self.packet_timestamp_ms(),
5083 crate::shared_bytes::convert(Self::encode_end_to_end_reliable_ack(packet_id)),
5084 )?;
5085 let local_sender = self.sender_arc();
5086 let ack = self.attach_wire_contract_to_item(
5090 RouterItem::Packet(ack),
5091 &[
5092 Self::sender_hash(local_sender.as_ref()),
5093 original_sender_hash,
5094 ],
5095 )?;
5096 let return_side = {
5100 let st = self.state.lock();
5101 st.reliable_return_routes
5102 .get(&packet_id)
5103 .map(|route| route.side)
5104 };
5105 let item = match return_side {
5106 Some(dst) => RouterTxItem::ToSide {
5107 src: None,
5108 dst,
5109 data: ack,
5110 },
5111 None => RouterTxItem::Broadcast(ack),
5112 };
5113 self.emit_internal_tx(item, true, called_from_queue)
5114 }
5115
5116 fn emit_internal_tx(
5117 &self,
5118 item: RouterTxItem,
5119 ignore_local: bool,
5120 called_from_queue: bool,
5121 ) -> TelemetryResult<()> {
5122 if called_from_queue {
5123 self.tx_queue_item_with_flags(item, ignore_local)
5124 } else {
5125 self.tx_item_impl(item, ignore_local, false)
5126 }
5127 }
5128
5129 fn emit_internal_tx_with_priority(
5130 &self,
5131 item: RouterTxItem,
5132 ignore_local: bool,
5133 priority: u8,
5134 called_from_queue: bool,
5135 ) -> TelemetryResult<()> {
5136 if called_from_queue {
5137 self.tx_queue_item_with_priority(item, ignore_local, priority)
5138 } else {
5139 self.tx_item_impl(item, ignore_local, false)
5140 }
5141 }
5142
5143 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
5144 {
5145 let mut st = self.state.lock();
5146 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
5147 return Ok(());
5148 };
5149 if sent.queued {
5150 return Ok(());
5151 }
5152 sent.queued = true;
5153 }
5154 let result = self.tx_queue_item_with_priority(
5155 RouterTxItem::EndToEndReplay { packet_id },
5156 true,
5157 Self::router_item_priority_bumped(DataType::ReliableAck),
5158 );
5159 if result.is_err()
5160 && let Some(sent) = self.state.lock().end_to_end_reliable_tx.get_mut(&packet_id)
5161 {
5162 sent.queued = false;
5163 }
5164 result
5165 }
5166
5167 fn end_to_end_retransmit_sides(
5168 &self,
5169 packet_id: u64,
5170 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
5171 let mut st = self.state.lock();
5172 let (data, tracked_destinations, mut sides) = {
5173 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
5174 #[cfg(feature = "std")]
5175 if reliable_trace_enabled() {
5176 eprintln!(
5177 "[sedsnet-reliable] replay time={} id={packet_id:#x} retries={} targets={:?}",
5178 self.clock.now_ms(),
5179 sent.retries,
5180 sent.pending_destinations
5181 );
5182 }
5183 sent.queued = false;
5184 sent.last_send_ms = self.clock.now_ms();
5185 let data = sent.data.clone();
5186 let tracked_destinations = sent.tracked_destinations;
5187 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
5188 (data, tracked_destinations, sides)
5189 };
5190 if tracked_destinations && sides.is_empty() {
5191 st.end_to_end_reliable_tx.remove(&packet_id);
5192 return None;
5193 }
5194 crate::small_sort::sort(&mut sides);
5195 sides.dedup();
5196 Some((data, sides))
5197 }
5198
5199 fn item_type_priority(data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
5200 let (ty, control) = match data {
5201 RouterItem::Packet(pkt) => (
5202 pkt.data_type(),
5203 pkt.payload().len() <= 16
5204 && pkt.endpoints().len() <= 3
5205 && crate::small_frozen_control(
5206 pkt.data_type(),
5207 pkt.wire_target_senders().len(),
5208 pkt.wire_shape(),
5209 ),
5210 ),
5211 RouterItem::Packed(bytes) => {
5212 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
5213 let control = bytes.len() <= 128
5214 && bytes.first().is_some_and(|flags| flags & 0x11 == 0)
5215 && env.endpoints().take(4).count() <= 3
5216 && crate::small_frozen_control(
5217 env.ty,
5218 env.target_senders().len(),
5219 env.wire_shape,
5220 );
5221 (env.ty, control)
5222 }
5223 };
5224 let priority = crate::scheduler_priority(ty);
5225 Ok((ty, if control { priority.max(254) } else { priority }))
5226 }
5227
5228 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
5229 Self::item_type_priority(data).map(|(_, priority)| priority)
5230 }
5231
5232 fn item_service_priority(&self, data: &RouterItem) -> TelemetryResult<u8> {
5233 let (ty, priority) = Self::item_type_priority(data)?;
5234 if self.is_managed_variable_type(ty) {
5235 Ok(crate::transport_priority(DataType::ManagedVariableValue))
5236 } else {
5237 Ok(priority)
5238 }
5239 }
5240
5241 #[inline]
5242 fn router_item_priority_bumped(ty: DataType) -> u8 {
5243 crate::scheduler_priority(ty).saturating_add(16)
5244 }
5245
5246 #[inline]
5247 fn is_transient_tx_error(err: &TelemetryError) -> bool {
5248 matches!(err, TelemetryError::Io("side tx busy" | "memory pressure"))
5249 }
5250
5251 #[cfg(feature = "timesync")]
5252 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
5253 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
5254 || st
5255 .clock
5256 .current_time(now_mono_ns)
5257 .and_then(|reading| reading.unix_time_ms)
5258 .is_some()
5259 }
5260
5261 #[cfg(feature = "timesync")]
5262 fn reconcile_pending_timesync_request_locked(
5263 st: &mut TimeSyncRuntime,
5264 leader: &Option<TimeSyncLeader>,
5265 now_ms: u64,
5266 ) {
5267 let active_remote = match leader {
5268 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
5269 _ => None,
5270 };
5271 let should_clear = st
5272 .pending_request
5273 .as_ref()
5274 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
5275 if should_clear {
5276 st.pending_request = None;
5277 st.next_request_mono_ms = now_ms;
5278 }
5279 }
5280
5281 #[inline]
5283 fn enqueue_to_sides(
5284 &self,
5285 data: RouterItem,
5286 exclude: Option<RouterSideId>,
5287 ignore_local: bool,
5288 ) -> TelemetryResult<()> {
5289 let Some(data) = self.clear_intermediate_hop_targets(data, true)? else {
5290 return Ok(());
5291 };
5292 let plan = self.remote_side_plan(&data, exclude)?;
5293 let mut st = self.state.lock();
5294 let ty = Self::item_data_type(&data)?;
5295 let priority = if st.managed_variable_types.contains(&ty.as_u32()) {
5296 crate::transport_priority(DataType::ManagedVariableValue)
5297 } else {
5298 Self::router_item_priority(&data)?
5299 };
5300
5301 let RemoteSidePlan::Target(sides) = plan;
5302 for idx in sides {
5303 st.push_transmit(TxQueued {
5304 item: RouterTxItem::ToSide {
5305 src: exclude,
5306 dst: idx,
5307 data: data.clone(),
5308 },
5309 ignore_local,
5310 priority,
5311 retry_after_ms: 0,
5312 })?;
5313 }
5314
5315 Ok(())
5316 }
5317
5318 fn relay_send(
5319 &self,
5320 data: RouterItem,
5321 src: Option<RouterSideId>,
5322 called_from_queue: bool,
5323 ) -> TelemetryResult<()> {
5324 if called_from_queue {
5325 return self.enqueue_to_sides(data, src, true);
5326 }
5327
5328 let Some(data) = self.clear_intermediate_hop_targets(data, false)? else {
5329 return Ok(());
5330 };
5331 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
5332 for side in sides {
5333 self.tx_item_impl(
5334 RouterTxItem::ToSide {
5335 src,
5336 dst: side,
5337 data: data.clone(),
5338 },
5339 true,
5340 false,
5341 )?;
5342 }
5343
5344 Ok(())
5345 }
5346
5347 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
5348 match data {
5349 RouterItem::Packet(pkt) => {
5350 pkt.validate()?;
5351 let mut eps = pkt.endpoints().to_vec();
5352 crate::small_sort::sort(&mut eps);
5353 eps.dedup();
5354 Ok((eps, pkt.data_type()))
5355 }
5356 RouterItem::Packed(bytes) => {
5357 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
5358 let mut eps: Vec<DataEndpoint> = env.endpoints().collect();
5359 crate::small_sort::sort(&mut eps);
5360 eps.dedup();
5361 Ok((eps, env.ty))
5362 }
5363 }
5364 }
5365
5366 fn unpack_packet_for_role(&self, bytes: &[u8]) -> TelemetryResult<Packet> {
5367 let managed = crate::memory_admission::supports_work_kinds()
5368 && wire_format::scheduling_type(bytes).is_some_and(|ty| {
5369 !is_internal_control_type(ty) && self.is_managed_variable_type(ty)
5370 });
5371 wire_format::unpack_packet_with_role(bytes, managed)
5372 }
5373
5374 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
5375 match data {
5376 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
5377 RouterItem::Packed(bytes) => Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.ty),
5378 }
5379 }
5380
5381 fn e2e_crypto_supported(&self) -> bool {
5382 #[cfg(feature = "cryptography")]
5383 {
5384 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
5385 && crate::crypto::registered_crypto_available()
5386 }
5387 #[cfg(not(feature = "cryptography"))]
5388 {
5389 false
5390 }
5391 }
5392
5393 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
5394 if is_internal_control_type(ty) {
5395 return false;
5396 }
5397 match self.cfg.e2e_encryption() {
5398 RouterE2eEncryptionMode::Disabled => {
5399 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
5400 }
5401 RouterE2eEncryptionMode::RequiredOnly => {
5402 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
5403 }
5404 RouterE2eEncryptionMode::Preferred => matches!(
5405 message_e2e_encryption_policy(ty),
5406 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
5407 ),
5408 RouterE2eEncryptionMode::ForceAll => true,
5409 }
5410 }
5411
5412 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
5413 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
5414 return Err(TelemetryError::BadArg);
5415 }
5416 Ok(())
5417 }
5418
5419 #[cfg(feature = "cryptography")]
5420 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
5421 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
5422 Some(wire_format::E2eSealConfig {
5423 key_id: self.cfg.e2e_key_id(),
5424 })
5425 } else {
5426 None
5427 }
5428 }
5429
5430 #[inline]
5431 fn pack_packet_for_router(
5432 &self,
5433 pkt: &Packet,
5434 reliable: Option<wire_format::ReliableHeader>,
5435 ) -> TelemetryResult<crate::SharedBytes> {
5436 #[cfg(feature = "cryptography")]
5437 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
5438 return (wire_format::pack_packet_with_wire_contract_e2e(
5439 pkt,
5440 reliable,
5441 pkt.wire_shape(),
5442 pkt.wire_target_senders(),
5443 e2e,
5444 ))
5445 .map(crate::shared_bytes::convert);
5446 }
5447 Ok(match reliable {
5448 Some(hdr) => {
5449 crate::shared_bytes::convert(wire_format::pack_packet_with_reliable(pkt, hdr))
5450 }
5451 None => crate::shared_bytes::convert(wire_format::pack_packet(pkt)),
5452 })
5453 }
5454
5455 #[inline]
5456 fn pack_packet_for_contract(
5457 &self,
5458 pkt: &Packet,
5459 reliable: Option<wire_format::ReliableHeader>,
5460 shape: Option<MessageElement>,
5461 target_senders: &[u64],
5462 ) -> TelemetryResult<crate::SharedBytes> {
5463 #[cfg(feature = "cryptography")]
5464 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
5465 return (wire_format::pack_packet_with_wire_contract_e2e(
5466 pkt,
5467 reliable,
5468 shape,
5469 target_senders,
5470 e2e,
5471 ))
5472 .map(crate::shared_bytes::convert);
5473 }
5474 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
5475 .map(crate::shared_bytes::convert)
5476 }
5477
5478 #[cfg(feature = "cryptography")]
5479 #[inline]
5480 fn prepare_packed_for_remote(
5481 &self,
5482 bytes: crate::SharedBytes,
5483 reliable_override: Option<Option<wire_format::ReliableHeader>>,
5484 ) -> TelemetryResult<crate::SharedBytes> {
5485 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
5486 if frame.ack_only() || self.e2e_seal_config_for_type(frame.envelope.ty).is_none() {
5487 return Ok(bytes);
5488 }
5489 let reliable = reliable_override.unwrap_or(frame.reliable);
5490 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
5491 self.pack_packet_for_contract(
5492 &pkt,
5493 reliable,
5494 frame.envelope.wire_shape,
5495 &frame.envelope.target_senders,
5496 )
5497 }
5498
5499 fn item_requires_end_to_end_tracking(
5503 &self,
5504 data: &RouterItem,
5505 ty: DataType,
5506 ) -> TelemetryResult<bool> {
5507 if is_reliable_type(ty) || self.is_managed_variable_type(ty) {
5508 return Ok(true);
5509 }
5510 match data {
5511 RouterItem::Packet(packet) => Ok(!packet.wire_target_senders().is_empty()),
5512 RouterItem::Packed(bytes) => {
5513 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
5514 Ok(frame.reliable.is_some() || frame.envelope.target_senders().len() != 0)
5515 }
5516 }
5517 }
5518
5519 fn item_target_senders<'a>(
5520 &self,
5521 data: &'a RouterItem,
5522 ) -> TelemetryResult<alloc::borrow::Cow<'a, [u64]>> {
5523 match data {
5524 RouterItem::Packet(pkt) => Ok(alloc::borrow::Cow::Borrowed(pkt.wire_target_senders())),
5525 RouterItem::Packed(bytes) => {
5526 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
5527 if env.target_senders().len() == 0 {
5528 Ok(alloc::borrow::Cow::Borrowed(&[]))
5529 } else {
5530 Ok(alloc::borrow::Cow::Owned(env.target_senders().collect()))
5531 }
5532 }
5533 }
5534 }
5535
5536 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
5537 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
5538 match data {
5539 RouterItem::Packet(pkt) => Ok(pkt.wire_target_senders().is_empty()
5540 || pkt.wire_target_senders().contains(&local_hash)),
5541 RouterItem::Packed(bytes) => {
5542 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
5543 Ok(env.target_senders().len() == 0
5544 || env.target_senders().any(|target| target == local_hash))
5545 }
5546 }
5547 }
5548
5549 fn item_targets_router_as_intermediate_hop(
5550 &self,
5551 data: &RouterItem,
5552 endpoints: &[DataEndpoint],
5553 ) -> TelemetryResult<bool> {
5554 let local_target = match data {
5557 RouterItem::Packet(pkt) => {
5558 if pkt.wire_target_senders().is_empty() || is_internal_control_type(pkt.data_type())
5559 {
5560 return Ok(false);
5561 }
5562 pkt.wire_target_senders()
5563 .contains(&Self::sender_hash(self.sender_arc().as_ref()))
5564 }
5565 RouterItem::Packed(bytes) => {
5566 let env = wire_format::peek_routing_envelope(bytes)?;
5567 if env.target_senders().len() == 0 || is_internal_control_type(env.ty) {
5568 return Ok(false);
5569 }
5570 let local = Self::sender_hash(self.sender_arc().as_ref());
5571 env.target_senders().any(|owner| owner == local)
5572 }
5573 };
5574 Ok(local_target
5575 && !endpoints.iter().copied().any(|endpoint| {
5576 self.endpoint_has_packet_handler(endpoint)
5577 || self.endpoint_has_packed_handler(endpoint)
5578 }))
5579 }
5580
5581 fn clear_intermediate_hop_targets(
5582 &self,
5583 data: RouterItem,
5584 called_from_queue: bool,
5585 ) -> TelemetryResult<Option<RouterItem>> {
5586 let has_targets = match &data {
5590 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
5591 RouterItem::Packed(bytes) => {
5592 wire_format::peek_routing_envelope(bytes)?
5593 .target_senders()
5594 .len()
5595 != 0
5596 }
5597 };
5598 if !has_targets {
5599 return Ok(Some(data));
5600 }
5601 let (endpoints, _) = self.item_route_info(&data)?;
5602 if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
5603 return Ok(Some(data));
5604 }
5605 let pkt = match &data {
5606 RouterItem::Packet(pkt) => pkt.clone(),
5607 RouterItem::Packed(bytes) => self.unpack_packet_for_role(bytes.as_ref())?,
5608 };
5609 let reliable = match &data {
5610 RouterItem::Packet(_) => None,
5611 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
5612 };
5613 let shape = match &data {
5614 RouterItem::Packet(pkt) => pkt.wire_shape(),
5615 RouterItem::Packed(bytes) => {
5616 wire_format::peek_routing_envelope(bytes.as_ref())?.wire_shape
5617 }
5618 };
5619 let upstream_id = Self::get_hash(&data);
5620 #[cfg(feature = "discovery")]
5621 let destinations = {
5622 let st = self.state.lock();
5623 let upstream = st
5624 .reliable_return_routes
5625 .get(&upstream_id)
5626 .map(|route| route.side);
5627 let mut pending =
5628 self.expected_end_to_end_destinations_locked(&st, &data, upstream, true)?;
5629 if pending.is_empty() {
5630 pending =
5631 self.expected_end_to_end_destinations_locked(&st, &data, upstream, false)?;
5632 }
5633 pending
5634 };
5635 #[cfg(feature = "discovery")]
5636 let targets: Vec<u64> = destinations.keys().copied().collect();
5637 #[cfg(not(feature = "discovery"))]
5638 let targets: Vec<u64> = Vec::new();
5639 #[cfg(feature = "discovery")]
5640 if targets.is_empty() {
5641 if self.packet_has_local_delivery(&pkt) {
5642 self.queue_end_to_end_reliable_ack_for_packet_id(
5643 upstream_id,
5644 pkt.sender(),
5645 called_from_queue,
5646 )?;
5647 return Ok(Some(data));
5648 }
5649 if !is_reliable_type(pkt.data_type()) && reliable.is_none() {
5650 return Err(TelemetryError::Io("no discovered route"));
5651 }
5652 }
5653 let forwarded = self.pack_packet_for_contract(&pkt, reliable, shape, &targets)?;
5657 let downstream_id = wire_format::packet_id_from_wire(forwarded.as_ref())?;
5658 let forwarded = RouterItem::Packed(forwarded);
5659 #[cfg(feature = "discovery")]
5660 let already_pending = self
5661 .state
5662 .lock()
5663 .end_to_end_reliable_tx
5664 .contains_key(&downstream_id);
5665 #[cfg(feature = "discovery")]
5666 let ready = self.register_end_to_end_reliable_tx(&forwarded, Some(destinations))?;
5667 #[cfg(not(feature = "discovery"))]
5668 let (ready, already_pending) = {
5669 let _ = called_from_queue;
5670 (true, false)
5671 };
5672 #[cfg(feature = "std")]
5673 if reliable_trace_enabled() {
5674 eprintln!(
5675 "[sedsnet-reliable] proxy retain upstream={upstream_id:#x} downstream={downstream_id:#x} existing={already_pending} ready={ready}"
5676 );
5677 }
5678 {
5679 let mut st = self.state.lock();
5680 let Some(mut route) = st.reliable_return_routes.get(&upstream_id).cloned() else {
5681 st.end_to_end_reliable_tx.remove(&downstream_id);
5682 return Err(TelemetryError::Io("no reliable return route"));
5683 };
5684 route.upstream_contract = Some((
5685 upstream_id,
5686 Self::sender_hash(&Self::canonical_sender_locked(&st, pkt.sender())),
5687 ));
5688 if !Self::remember_reliable_return_route_locked(&mut st, downstream_id) {
5689 st.end_to_end_reliable_tx.remove(&downstream_id);
5690 return Err(TelemetryError::Io("reliable return route window full"));
5691 }
5692 st.reliable_return_routes.insert(downstream_id, route);
5693 }
5694 Ok((ready && !already_pending).then_some(forwarded))
5695 }
5696
5697 #[cfg(feature = "discovery")]
5698 fn side_matches_target_senders_locked(
5699 st: &RouterInner,
5700 side: RouterSideId,
5701 target_senders: &[u64],
5702 now_ms: u64,
5703 ) -> bool {
5704 st.discovery_routes
5705 .get(&side)
5706 .map(|route| {
5707 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5708 return false;
5709 }
5710 route.announcers.iter().any(|(announcer, sender_state)| {
5711 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5712 return false;
5713 }
5714 target_senders.contains(&Self::sender_hash(announcer))
5715 || sender_state
5716 .advertised_address
5717 .is_some_and(|address| target_senders.contains(&u64::from(address)))
5718 || sender_state.topology_boards.iter().any(|board| {
5719 target_senders.contains(&Self::sender_hash(&board.sender_id))
5720 })
5721 })
5722 })
5723 .unwrap_or(false)
5724 }
5725
5726 fn attach_wire_contract_to_item(
5727 &self,
5728 data: RouterItem,
5729 target_senders: &[u64],
5730 ) -> TelemetryResult<RouterItem> {
5731 match data {
5732 RouterItem::Packet(pkt) => {
5733 let reliable = if is_reliable_type(pkt.data_type()) {
5734 Some(wire_format::ReliableHeader {
5735 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
5736 seq: 0,
5737 ack: 0,
5738 })
5739 } else {
5740 None
5741 };
5742 let bytes = self.pack_packet_for_contract(
5743 &pkt,
5744 reliable,
5745 Some(message_meta(pkt.data_type()).element),
5746 target_senders,
5747 )?;
5748 Ok(RouterItem::Packed(bytes))
5749 }
5750 RouterItem::Packed(bytes) => Ok(RouterItem::Packed(bytes)),
5751 }
5752 }
5753
5754 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
5755 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
5756 }
5757
5758 fn should_route_remote(
5759 &self,
5760 data: &RouterItem,
5761 exclude: Option<RouterSideId>,
5762 ) -> TelemetryResult<bool> {
5763 #[cfg(feature = "discovery")]
5764 {
5765 let RemoteSidePlan::Target(sides) = self.remote_side_plan(data, exclude)?;
5766 Ok(!sides.is_empty())
5767 }
5768
5769 #[cfg(not(feature = "discovery"))]
5770 {
5771 let (eps, ty) = self.item_route_info(data)?;
5772 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
5773 return Ok(false);
5774 }
5775 let st = self.state.lock();
5776 Ok(!self
5777 .eligible_side_ids_locked(
5778 &st,
5779 exclude,
5780 Some(ty),
5781 Self::endpoints_are_link_local_only(&eps),
5782 )
5783 .is_empty())
5784 }
5785 }
5786
5787 #[cfg(feature = "discovery")]
5788 fn has_explicit_route_policy_locked(
5789 st: &RouterInner,
5790 src: Option<RouterSideId>,
5791 ty: DataType,
5792 ) -> bool {
5793 st.route_overrides
5794 .keys()
5795 .any(|(route_src, _)| *route_src == src)
5796 || Self::has_typed_route_overrides_locked(st, src, ty)
5797 }
5798
5799 fn remote_side_plan(
5800 &self,
5801 data: &RouterItem,
5802 exclude: Option<RouterSideId>,
5803 ) -> TelemetryResult<RemoteSidePlan> {
5804 #[cfg(feature = "discovery")]
5805 {
5806 let (eps, ty) = self.item_route_info(data)?;
5807 let target_senders = self.item_target_senders(data)?;
5808 let resolve_beyond_local_hop =
5809 self.item_targets_router_as_intermediate_hop(data, &eps)?;
5810 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
5811 let routing_target_senders: &[u64] = if preferred_packet_id.is_some() {
5812 target_senders.get(1..).unwrap_or(&[])
5815 } else if resolve_beyond_local_hop {
5816 &[]
5817 } else {
5818 target_senders.as_ref()
5819 };
5820 if discovery::is_discovery_type(ty) {
5821 let mut st = self.state.lock();
5822 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
5823 return Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
5824 &mut st,
5825 exclude,
5826 sides,
5827 RouteSelectionOrigin::Flood,
5828 )));
5829 }
5830 #[cfg(feature = "timesync")]
5831 if ty == DataType::TimeSyncAnnounce {
5832 let mut st = self.state.lock();
5839 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
5840 let sides =
5841 Self::filter_timesync_sides_locked(&mut st, ty, self.clock.now_ms(), sides);
5842 return Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
5843 &mut st,
5844 exclude,
5845 sides,
5846 RouteSelectionOrigin::Flood,
5847 )));
5848 }
5849 let mut st = self.state.lock();
5850 let now_ms = self.clock.now_ms();
5855 let has_remote_variable_owner = st.managed_variable_types.contains(&ty.as_u32())
5856 && st.discovery_routes.iter().any(|(&side, route)| {
5857 exclude != Some(side)
5858 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5859 && route.reachable_network_variables.contains(&ty)
5860 });
5861 if !(has_nonlocal_endpoint(&eps, &self.cfg)
5862 || force_remote_for_type(ty)
5863 || !target_senders.is_empty()
5864 || has_remote_variable_owner)
5865 {
5866 return Ok(RemoteSidePlan::Target(Vec::new()));
5867 }
5868
5869 #[cfg(feature = "timesync")]
5870 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
5871 #[cfg(not(feature = "timesync"))]
5872 let preferred_timesync_source: Option<String> = None;
5873
5874 if let Some(packet_id) = preferred_packet_id {
5875 let target_side = st
5876 .reliable_return_routes
5877 .get(&packet_id)
5878 .map(|route| route.side);
5879 if let Some(side) = target_side
5880 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
5881 {
5882 #[cfg(feature = "timesync")]
5883 if !Self::timesync_allowed_for_side_locked(
5884 &mut st,
5885 side,
5886 ty,
5887 self.clock.now_ms(),
5888 ) {
5889 return Ok(RemoteSidePlan::Target(Vec::new()));
5890 }
5891 return Ok(RemoteSidePlan::Target(vec![side]));
5892 }
5893 }
5899 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
5900 let prefer_best_overlap = is_reliable_type(ty)
5901 && routing_target_senders.is_empty()
5902 && preferred_packet_id.is_none();
5903 if st.discovery_routes.is_empty() {
5904 let mut fallback =
5905 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
5906 #[cfg(feature = "timesync")]
5907 {
5908 fallback = Self::filter_timesync_sides_locked(
5909 &mut st,
5910 ty,
5911 self.clock.now_ms(),
5912 fallback,
5913 );
5914 }
5915 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
5916 fallback
5917 } else {
5918 Vec::new()
5919 }));
5920 }
5921 let mut matches = self.discovered_route_candidates_locked(
5922 &st,
5923 exclude,
5924 ty,
5925 &eps,
5926 routing_target_senders,
5927 prefer_best_overlap,
5928 preferred_timesync_source.as_deref(),
5929 );
5930 #[cfg(feature = "timesync")]
5931 {
5932 matches =
5933 Self::filter_timesync_matches_locked(&mut st, ty, self.clock.now_ms(), matches);
5934 }
5935 let has_distributed_variable_owners = matches.iter().any(|candidate| {
5941 st.discovery_routes
5942 .get(&candidate.side)
5943 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
5944 });
5945 if DataType::try_named("HEARTBEAT") != Some(ty) && !has_distributed_variable_owners {
5946 Self::retain_shortest_discovery_candidates_locked(
5947 &st,
5948 &mut matches,
5949 &eps,
5950 routing_target_senders,
5951 self.clock.now_ms(),
5952 );
5953 }
5954
5955 if !matches.is_empty() {
5956 Ok(RemoteSidePlan::Target(
5957 self.select_discovered_candidate_sides_locked(
5958 &mut st,
5959 exclude,
5960 ty,
5961 routing_target_senders,
5962 prefer_best_overlap,
5963 matches,
5964 ),
5965 ))
5966 } else if preferred_packet_id.is_some() {
5967 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
5973 Ok(RemoteSidePlan::Target(
5974 if exclude.is_some() && sides.len() == 1 {
5975 sides
5976 } else {
5977 Vec::new()
5978 },
5979 ))
5980 } else if prefer_best_overlap {
5981 Ok(RemoteSidePlan::Target(Vec::new()))
5982 } else {
5983 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
5984 let mut sides =
5985 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
5986 #[cfg(feature = "timesync")]
5987 {
5988 sides = Self::filter_timesync_sides_locked(
5989 &mut st,
5990 ty,
5991 self.clock.now_ms(),
5992 sides,
5993 );
5994 }
5995 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
5996 &mut st,
5997 exclude,
5998 sides,
5999 RouteSelectionOrigin::Flood,
6000 )))
6001 } else {
6002 Ok(RemoteSidePlan::Target(Vec::new()))
6003 }
6004 }
6005 }
6006 #[cfg(not(feature = "discovery"))]
6007 {
6008 let (_, ty) = self.item_route_info(data)?;
6009 let mut st = self.state.lock();
6010 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
6011 let target_side = st
6012 .reliable_return_routes
6013 .get(&packet_id)
6014 .map(|route| route.side);
6015 if let Some(side) = target_side
6016 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
6017 {
6018 return Ok(RemoteSidePlan::Target(vec![side]));
6019 }
6020 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6021 return Ok(RemoteSidePlan::Target(
6022 if exclude.is_some() && sides.len() == 1 {
6023 sides
6024 } else {
6025 Vec::new()
6026 },
6027 ));
6028 }
6029 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6030 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
6031 &mut st,
6032 exclude,
6033 sides,
6034 RouteSelectionOrigin::Flood,
6035 )))
6036 }
6037 }
6038
6039 #[cfg(feature = "discovery")]
6040 fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
6041 let mut eps: Vec<DataEndpoint> = self.cfg.handlers.iter().map(|h| h.endpoint).collect();
6042 let _ = st;
6048 #[cfg(feature = "timesync")]
6049 if self.cfg.timesync_config().is_some() {
6050 eps.push(DataEndpoint::TimeSync);
6051 }
6052 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
6053 crate::small_sort::sort(&mut eps);
6054 eps.dedup();
6055 eps
6056 }
6057
6058 #[cfg(feature = "discovery")]
6059 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
6060 #[cfg(feature = "timesync")]
6061 {
6062 let st = self.timesync.lock();
6063 if let Some(tracker) = st.tracker.as_ref()
6064 && tracker.should_serve(
6065 now_ms,
6066 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
6067 )
6068 {
6069 return vec![self.sender_arc().to_string()];
6070 }
6071 }
6072 Vec::new()
6073 }
6074
6075 #[cfg(all(feature = "discovery", feature = "timesync"))]
6076 fn preferred_timesync_route_source(
6077 &self,
6078 data: &RouterItem,
6079 ty: DataType,
6080 ) -> TelemetryResult<Option<String>> {
6081 if !matches!(
6082 ty,
6083 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
6084 ) {
6085 return Ok(None);
6086 }
6087
6088 match data {
6089 RouterItem::Packet(pkt) => match ty {
6090 DataType::TimeSyncRequest => {
6091 let local_sender = self.sender_arc();
6092 if pkt.sender() == local_sender.as_ref() {
6093 Ok(self.timesync.lock().tracker.as_ref().and_then(|tracker| {
6094 tracker.current_source().map(|src| src.sender.clone())
6095 }))
6096 } else {
6097 Ok(None)
6098 }
6099 }
6100 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
6101 Ok(Some(pkt.sender().to_owned()))
6102 }
6103 _ => Ok(None),
6104 },
6105 RouterItem::Packed(bytes) => {
6106 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
6107 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
6108 }
6109 }
6110 }
6111
6112 #[cfg(feature = "discovery")]
6113 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
6114 st.discovery_cadence.on_topology_change(now_ms);
6115 for side_id in st
6120 .sides
6121 .iter()
6122 .enumerate()
6123 .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
6124 {
6125 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
6126 throttle.pending_incremental = true;
6127 throttle.incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
6128 }
6129 for transport in st.side_transport.values_mut() {
6133 transport.refresh_transmit_header_templates();
6134 }
6135 }
6136
6137 #[cfg(feature = "discovery")]
6138 fn sender_topology_board_mut<'a>(
6139 sender_state: &'a mut DiscoverySenderState,
6140 sender_id: &str,
6141 ) -> &'a mut TopologyBoardNode {
6142 let boards = Arc::make_mut(&mut sender_state.topology_boards);
6143 if let Some(idx) = boards.iter().position(|board| board.sender_id == sender_id) {
6144 return &mut boards[idx];
6145 }
6146 boards.push(TopologyBoardNode {
6147 sender_id: sender_id.to_string(),
6148 reachable_endpoints: Vec::new(),
6149 reachable_timesync_sources: Vec::new(),
6150 connections: Vec::new(),
6151 });
6152 boards.last_mut().expect("inserted above")
6153 }
6154
6155 #[cfg(feature = "discovery")]
6156 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
6157 if let Some(boards) = Arc::get_mut(&mut sender_state.topology_boards) {
6158 discovery::normalize_topology_boards(boards);
6159 }
6160 let (mut reachable, mut reachable_timesync_sources) =
6161 discovery::summarize_topology_boards(&sender_state.topology_boards);
6162 reachable.extend(sender_state.advertised_reachable.iter().copied());
6163 crate::small_sort::sort(&mut reachable);
6164 reachable.dedup();
6165 reachable_timesync_sources.extend(
6166 sender_state
6167 .advertised_reachable_timesync_sources
6168 .iter()
6169 .cloned(),
6170 );
6171 reachable_timesync_sources.sort_unstable();
6172 reachable_timesync_sources.dedup();
6173 sender_state.reachable = reachable;
6174 sender_state.reachable_timesync_sources = reachable_timesync_sources;
6175 }
6176
6177 #[cfg(feature = "discovery")]
6178 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
6179 let mut reachable = Vec::new();
6180 let mut reachable_network_variables = Vec::new();
6181 let mut reachable_timesync_sources = Vec::new();
6182 let mut last_seen_ms = 0u64;
6183 for sender in route.announcers.values() {
6184 reachable.extend(sender.reachable.iter().copied());
6185 reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
6186 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
6187 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
6188 }
6189 reachable_network_variables
6190 .extend(route.requested_network_variables.iter().map(|(ty, _)| *ty));
6191 last_seen_ms = last_seen_ms.max(
6192 route
6193 .requested_network_variables
6194 .iter()
6195 .map(|(_, last_seen_ms)| *last_seen_ms)
6196 .max()
6197 .unwrap_or(0),
6198 );
6199 crate::small_sort::sort(&mut reachable);
6200 reachable.dedup();
6201 crate::small_sort::sort(&mut reachable_network_variables);
6202 reachable_network_variables.dedup();
6203 reachable_timesync_sources.sort_unstable();
6204 reachable_timesync_sources.dedup();
6205 route.reachable = reachable;
6206 route.reachable_network_variables = reachable_network_variables;
6207 route.reachable_timesync_sources = reachable_timesync_sources;
6208 route.last_seen_ms = last_seen_ms;
6209 }
6210
6211 #[cfg(feature = "discovery")]
6212 fn local_discovery_topology_board(
6213 &self,
6214 st: &RouterInner,
6215 now_ms: u64,
6216 link_local_enabled: bool,
6217 ) -> TopologyBoardNode {
6218 let mut reachable_endpoints = self.local_discovery_endpoints(st);
6219 if !link_local_enabled {
6220 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
6221 }
6222 let mut connections = Vec::new();
6223 for route in st.discovery_routes.values() {
6224 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6225 continue;
6226 }
6227 for (sender, sender_state) in route.announcers.iter() {
6228 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
6229 connections.push(sender.clone());
6230 }
6231 }
6232 }
6233 connections.sort_unstable();
6234 connections.dedup();
6235 let sender = self.sender_arc();
6236 TopologyBoardNode {
6237 sender_id: sender.to_string(),
6238 reachable_endpoints,
6239 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
6240 connections,
6241 }
6242 }
6243
6244 #[cfg(feature = "discovery")]
6245 fn advertised_discovery_topology_for_link_locked(
6246 &self,
6247 st: &RouterInner,
6248 now_ms: u64,
6249 link_local_enabled: bool,
6250 exclude_side: Option<RouterSideId>,
6251 ) -> Vec<TopologyBoardNode> {
6252 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
6253 for (&route_side, route) in st.discovery_routes.iter() {
6254 if exclude_side == Some(route_side) {
6255 continue;
6256 }
6257 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6258 continue;
6259 }
6260 for (announcer, sender_state) in route.announcers.iter() {
6261 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6262 continue;
6263 }
6264 if sender_state.topology_boards.is_empty() {
6265 let sender = self.sender_arc();
6266 let fallback = TopologyBoardNode {
6267 sender_id: announcer.clone(),
6268 reachable_endpoints: sender_state.reachable.clone(),
6269 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
6270 connections: vec![sender.to_string()],
6271 };
6272 discovery::merge_topology_boards(&mut boards, core::slice::from_ref(&fallback));
6273 } else {
6274 discovery::merge_topology_boards(&mut boards, &sender_state.topology_boards);
6277 if let Some(board) = boards
6278 .iter_mut()
6279 .find(|board| board.sender_id == *announcer)
6280 {
6281 let sender = self.sender_arc();
6282 if !board.connections.iter().any(|peer| peer == sender.as_ref()) {
6283 board.connections.push(sender.to_string());
6284 }
6285 }
6286 }
6287 }
6288 }
6289 if !link_local_enabled {
6290 for board in &mut boards {
6291 board
6292 .reachable_endpoints
6293 .retain(|ep| !ep.is_link_local_only());
6294 }
6295 }
6296 discovery::normalize_topology_boards(&mut boards);
6297 boards
6298 }
6299
6300 #[cfg(feature = "discovery")]
6301 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
6302 let mut changed = false;
6305 st.discovery_routes.retain(|_, route| {
6306 let before_peers = route.announcers.len();
6307 let before_requests = route.requested_network_variables.len();
6308 route.announcers.retain(|_, sender| {
6309 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6310 });
6311 route
6312 .requested_network_variables
6313 .retain(|(_, last_seen_ms)| {
6314 now_ms.saturating_sub(*last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6315 });
6316 if before_peers != route.announcers.len()
6317 || before_requests != route.requested_network_variables.len()
6318 {
6319 Self::recompute_discovery_side_state(route);
6320 changed = true;
6321 }
6322 let keep =
6323 !route.announcers.is_empty() || !route.requested_network_variables.is_empty();
6324 changed |= !keep;
6325 keep
6326 });
6327 changed
6328 }
6329
6330 #[cfg(feature = "discovery")]
6331 fn advertised_summaries_from_topology_locked(
6332 &self,
6333 st: &RouterInner,
6334 now_ms: u64,
6335 link_local_enabled: bool,
6336 exclude_side: Option<RouterSideId>,
6337 topology: &[TopologyBoardNode],
6338 ) -> (Vec<crate::DataEndpoint>, Vec<String>) {
6339 let (mut endpoints, mut sources) = discovery::summarize_topology_boards(topology);
6340 for (&side, route) in &st.discovery_routes {
6341 if exclude_side == Some(side)
6342 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
6343 {
6344 continue;
6345 }
6346 for peer in route.announcers.values() {
6347 if now_ms.saturating_sub(peer.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6348 continue;
6349 }
6350 for endpoint in &peer.advertised_reachable {
6351 if !discovery::is_discovery_endpoint(*endpoint)
6352 && (link_local_enabled || !endpoint.is_link_local_only())
6353 && !endpoints.contains(endpoint)
6354 {
6355 endpoints.push(*endpoint);
6356 }
6357 }
6358 for source in &peer.advertised_reachable_timesync_sources {
6359 if !sources.contains(source) {
6360 sources.push(source.clone());
6361 }
6362 }
6363 }
6364 }
6365 endpoints.retain(|endpoint| {
6366 !discovery::is_discovery_endpoint(*endpoint)
6367 && (link_local_enabled || !endpoint.is_link_local_only())
6368 });
6369 crate::small_sort::sort(&mut endpoints);
6370 sources.sort_unstable();
6371 (endpoints, sources)
6372 }
6373
6374 #[cfg(feature = "discovery")]
6375 fn advertised_discovery_endpoints_for_link_locked(
6376 &self,
6377 st: &RouterInner,
6378 now_ms: u64,
6379 link_local_enabled: bool,
6380 exclude_side: Option<RouterSideId>,
6381 ) -> Vec<DataEndpoint> {
6382 let (mut reachable_endpoints, _) = discovery::summarize_topology_boards(
6383 &self.advertised_discovery_topology_for_link_locked(
6384 st,
6385 now_ms,
6386 link_local_enabled,
6387 exclude_side,
6388 ),
6389 );
6390 for (&side, route) in &st.discovery_routes {
6393 if exclude_side == Some(side)
6394 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
6395 {
6396 continue;
6397 }
6398 for sender in route.announcers.values() {
6399 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
6400 reachable_endpoints.extend(sender.advertised_reachable.iter().copied());
6401 }
6402 }
6403 }
6404 crate::small_sort::sort(&mut reachable_endpoints);
6405 reachable_endpoints.dedup();
6406 reachable_endpoints
6407 .into_iter()
6408 .filter(|ep| {
6409 !discovery::is_discovery_endpoint(*ep)
6410 && (link_local_enabled || !ep.is_link_local_only())
6411 })
6412 .collect()
6413 }
6414
6415 #[cfg(all(test, feature = "discovery"))]
6416 fn compact_summary_fixture(&self, endpoint: DataEndpoint) {
6417 let mut sender = DiscoverySenderState {
6418 advertised_reachable: vec![endpoint],
6419 advertised_reachable_timesync_sources: vec!["clock-behind-bridge".to_string()],
6420 last_seen_ms: self.clock.now_ms(),
6421 topology_boards: Arc::new(vec![TopologyBoardNode {
6422 sender_id: "bridge".to_string(),
6423 reachable_endpoints: Vec::new(),
6424 reachable_timesync_sources: Vec::new(),
6425 connections: Vec::new(),
6426 }]),
6427 ..Default::default()
6428 };
6429 Self::refresh_sender_topology_state(&mut sender);
6430 let mut route = DiscoverySideState::default();
6431 route.announcers.insert("bridge".to_string(), sender);
6432 Self::recompute_discovery_side_state(&mut route);
6433 self.state.lock().discovery_routes.insert(0, route);
6434 }
6435
6436 #[cfg(feature = "discovery")]
6437 fn advertised_discovery_timesync_sources_for_link_locked(
6438 &self,
6439 st: &RouterInner,
6440 now_ms: u64,
6441 exclude_side: Option<RouterSideId>,
6442 ) -> Vec<String> {
6443 let (_, mut sources) = discovery::summarize_topology_boards(
6444 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
6445 );
6446 for (&side, route) in &st.discovery_routes {
6447 if exclude_side == Some(side)
6448 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
6449 {
6450 continue;
6451 }
6452 for sender in route.announcers.values() {
6453 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
6454 sources.extend(sender.advertised_reachable_timesync_sources.iter().cloned());
6455 }
6456 }
6457 }
6458 sources.sort_unstable();
6459 sources.dedup();
6460 sources
6461 }
6462
6463 #[cfg(feature = "discovery")]
6464 fn advertised_network_variables_for_link_locked(
6465 &self,
6466 st: &RouterInner,
6467 now_ms: u64,
6468 exclude_side: Option<RouterSideId>,
6469 ) -> Vec<DataType> {
6470 let mut types: Vec<DataType> = st
6471 .managed_variable_types
6472 .iter()
6473 .copied()
6474 .map(DataType)
6475 .collect();
6476 for (&route_side, route) in st.discovery_routes.iter() {
6477 if exclude_side == Some(route_side) {
6478 continue;
6479 }
6480 if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
6481 types.extend(route.reachable_network_variables.iter().copied());
6482 }
6483 }
6484 crate::small_sort::sort(&mut types);
6485 types.dedup();
6486 types
6487 }
6488
6489 #[cfg(feature = "discovery")]
6490 fn preferred_discovery_master_present_locked<'a>(
6491 &self,
6492 st: &'a RouterInner,
6493 now_ms: u64,
6494 ) -> Option<&'a str> {
6495 if let Some(preferred) = st.preferred_discovery_master.as_deref()
6496 && (preferred == self.sender_arc().as_ref()
6497 || st
6498 .discovery_routes
6499 .values()
6500 .filter(|route| {
6501 now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6502 })
6503 .any(|route| {
6504 route
6505 .announcers
6506 .iter()
6507 .filter(|(_, peer)| {
6508 now_ms.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6509 })
6510 .any(|(name, peer)| {
6511 name == preferred
6512 || peer.topology_boards.iter().any(|board| {
6513 board.sender_id == preferred
6514 || board
6515 .connections
6516 .iter()
6517 .any(|name| name == preferred)
6518 })
6519 })
6520 }))
6521 {
6522 return Some(preferred);
6525 }
6526 None
6527 }
6528
6529 #[cfg(feature = "discovery")]
6530 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
6531 if let Some(preferred) = self.preferred_discovery_master_present_locked(st, now_ms) {
6532 return preferred.to_string();
6533 }
6534 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
6535 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
6536 }
6537
6538 #[cfg(all(feature = "discovery", feature = "timesync"))]
6539 #[inline]
6540 fn side_is_slow_control_link_locked(
6541 st: &RouterInner,
6542 side_id: RouterSideId,
6543 now_ms: u64,
6544 ) -> bool {
6545 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
6546 let recent_slow = stats.last_slow_observed_ms > 0
6547 && now_ms.saturating_sub(stats.last_slow_observed_ms)
6548 <= discovery::DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
6549 stats.sample_count > 0
6550 && ((stats.estimated_bandwidth_bps > 0
6551 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
6552 || recent_slow)
6553 })
6554 }
6555
6556 #[cfg(feature = "discovery")]
6557 fn discovery_level_for_side_locked(
6558 st: &mut RouterInner,
6559 side_id: RouterSideId,
6560 now_ms: u64,
6561 ) -> Option<DiscoveryAdvertiseLevel> {
6562 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
6563 if !throttle.has_sent_full {
6564 throttle.has_sent_full = true;
6565 throttle.pending_incremental = false;
6566 throttle.incremental_retries_remaining = 0;
6567 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
6568 return Some(DiscoveryAdvertiseLevel::Full);
6569 }
6570 if throttle.pending_incremental {
6571 throttle.incremental_retries_remaining =
6572 throttle.incremental_retries_remaining.saturating_sub(1);
6573 throttle.pending_incremental = throttle.incremental_retries_remaining > 0;
6574 return Some(DiscoveryAdvertiseLevel::Incremental);
6575 }
6576 if throttle.liveness_due(now_ms) {
6579 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
6580 return Some(DiscoveryAdvertiseLevel::MinimalPing);
6581 }
6582 None
6583 }
6584
6585 #[cfg(all(feature = "discovery", feature = "timesync"))]
6586 #[inline]
6587 fn is_timesync_type(ty: DataType) -> bool {
6588 matches!(
6589 ty,
6590 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
6591 )
6592 }
6593
6594 #[cfg(all(feature = "discovery", feature = "timesync"))]
6595 fn timesync_allowed_for_side_locked(
6596 st: &mut RouterInner,
6597 side_id: RouterSideId,
6598 ty: DataType,
6599 now_ms: u64,
6600 ) -> bool {
6601 if !Self::is_timesync_type(ty) {
6602 return true;
6603 }
6604 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
6605 st.timesync_side_throttle.remove(&side_id);
6606 return true;
6607 }
6608
6609 let throttle = st.timesync_side_throttle.entry(side_id).or_default();
6610 if now_ms >= throttle.next_allowed_ms {
6611 throttle.next_allowed_ms = now_ms.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
6612 return true;
6613 }
6614 false
6615 }
6616
6617 #[cfg(all(feature = "discovery", feature = "timesync"))]
6618 fn filter_timesync_sides_locked(
6619 st: &mut RouterInner,
6620 ty: DataType,
6621 now_ms: u64,
6622 sides: Vec<RouterSideId>,
6623 ) -> Vec<RouterSideId> {
6624 sides
6625 .into_iter()
6626 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
6627 .collect()
6628 }
6629
6630 #[cfg(all(feature = "discovery", feature = "timesync"))]
6631 fn filter_timesync_matches_locked(
6632 st: &mut RouterInner,
6633 ty: DataType,
6634 now_ms: u64,
6635 matches: Vec<DiscoveryCandidateMatch>,
6636 ) -> Vec<DiscoveryCandidateMatch> {
6637 matches
6638 .into_iter()
6639 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
6640 .collect()
6641 }
6642
6643 #[cfg(feature = "discovery")]
6644 fn refresh_known_discovery_liveness(
6645 &self,
6646 pkt: &Packet,
6647 side: RouterSideId,
6648 ) -> TelemetryResult<bool> {
6649 if pkt.data_type() == crate::DataType::DiscoveryTopology {
6650 crate::memory_admission::check(512, 128)?;
6654 let now = self.clock.now_ms();
6655 let mut st = self.state.lock();
6656 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
6657 let Some(route) = st.discovery_routes.get_mut(&side) else {
6658 return Ok(false);
6659 };
6660 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
6661 return Ok(false);
6662 };
6663 let Some(full) =
6664 discovery::topology_payload_matches_retained(pkt.payload(), &peer.topology_boards)
6665 else {
6666 return Ok(false);
6667 };
6668 peer.has_full_topology |= full;
6669 peer.last_seen_ms = now;
6670 route.last_seen_ms = route.last_seen_ms.max(now);
6671 return Ok(true);
6672 }
6673 let empty_ping =
6674 pkt.data_type() == crate::DataType::DiscoveryAnnounce && pkt.payload().is_empty();
6675 if !empty_ping && pkt.data_type() != crate::DataType::DiscoveryAddress {
6676 return Ok(false);
6677 }
6678 crate::memory_admission::check(
6679 if empty_ping {
6680 512
6681 } else {
6682 1024 + pkt.payload().len().saturating_mul(4)
6683 },
6684 pkt.payload().len().saturating_mul(2).max(128),
6685 )?;
6686 let mut address = if empty_ping {
6687 None
6688 } else {
6689 Some(discovery::decode_discovery_address(pkt)?)
6690 };
6691 let now = self.clock.now_ms();
6692 let mut st = self.state.lock();
6693 if let Some(ad) = &mut address {
6694 let link_local = st
6695 .sides
6696 .get(side)
6697 .and_then(|entry| entry.as_ref())
6698 .is_some_and(|entry| entry.opts.link_local_enabled);
6699 if !link_local {
6700 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
6701 }
6702 let Some(entry) = st.address_book.get(ad.hostname.as_str()) else {
6703 return Ok(false);
6704 };
6705 if entry.address != ad.address
6706 || entry.requested_address != ad.requested_address
6707 || entry.mode != Self::address_mode_from_code(ad.mode, ad.requested_address)
6708 || entry.owner_hash != ad.owner_hash
6709 {
6710 return Ok(false);
6711 }
6712 if entry.birth_ms != ad.birth_ms
6713 && st.address_book.iter().any(|(hostname, other)| {
6714 hostname != &ad.hostname
6715 && [ad.address, ad.requested_address].iter().any(|candidate| {
6716 *candidate != 0
6717 && (other.address == *candidate
6718 || other.requested_address == *candidate)
6719 })
6720 })
6721 {
6722 return Ok(false);
6726 }
6727 }
6728 let canonical;
6729 let sender = if let Some(ad) = &address {
6730 ad.hostname.as_str()
6731 } else {
6732 canonical = Self::canonical_sender_locked(&st, pkt.sender());
6733 canonical.as_str()
6734 };
6735 let Some(route) = st.discovery_routes.get_mut(&side) else {
6736 return Ok(false);
6737 };
6738 let Some(peer) = route.announcers.get_mut(sender) else {
6739 return Ok(false);
6740 };
6741 if let Some(ad) = &address
6742 && (peer.advertised_address != Some(ad.address)
6743 || peer.advertised_reachable != ad.reachable_endpoints
6744 || peer.advertised_reachable_timesync_sources != ad.reachable_timesync_sources
6745 || peer.reachable_network_variables != ad.reachable_network_variables
6746 || peer.link_capabilities != Some(ad.link_capabilities))
6747 {
6748 return Ok(false);
6749 }
6750 peer.last_seen_ms = now;
6753 route.last_seen_ms = route.last_seen_ms.max(now);
6754 if let Some(ad) = &address
6755 && let Some(entry) = st.address_book.get_mut(ad.hostname.as_str())
6756 {
6757 entry.birth_ms = ad.birth_ms;
6760 entry.last_seen_ms = now;
6761 }
6762 Ok(true)
6763 }
6764
6765 #[cfg(feature = "discovery")]
6766 fn queue_pressure_keepalives(
6767 &self,
6768 requested_side: Option<RouterSideId>,
6769 ) -> TelemetryResult<()> {
6770 let now = self.clock.now_ms();
6771 let sender = self.sender_arc();
6772 crate::memory_admission::check(
6773 256 + sender.len().saturating_mul(4),
6774 sender.len().max(256),
6775 )?;
6776 let side_count = self.state.lock().sides.len();
6777 for side in 0..side_count {
6778 let due = {
6779 let st = self.state.lock();
6780 requested_side.is_none_or(|wanted| wanted == side)
6781 && st.sides[side].is_some()
6782 && self.route_allowed_locked(
6783 &st,
6784 None,
6785 Some(crate::DataType::DiscoveryAnnounce),
6786 side,
6787 )
6788 && st
6789 .discovery_side_throttle
6790 .get(&side)
6791 .is_none_or(|throttle| now >= throttle.next_ping_ms)
6792 };
6793 if !due {
6794 continue;
6795 }
6796 let summary = {
6799 let st = self.state.lock();
6800 let extra = st
6801 .discovery_routes
6802 .iter()
6803 .filter(|(id, _)| **id != side)
6804 .fold(0usize, |total, (_, route)| {
6805 total
6806 .saturating_add(route.reachable.len().saturating_mul(32))
6807 .saturating_add(
6808 route.reachable_network_variables.len().saturating_mul(32),
6809 )
6810 .saturating_add(
6811 route
6812 .reachable_timesync_sources
6813 .iter()
6814 .map(|source| source.len().saturating_add(32).saturating_mul(4))
6815 .sum::<usize>(),
6816 )
6817 });
6818 if crate::memory_admission::check(4096usize.saturating_add(extra), 2048).is_ok() {
6819 let opts = st.sides[side].as_ref().expect("checked above").opts.clone();
6820 let mut endpoints = self.local_discovery_endpoints(&st);
6821 let mut variables =
6822 self.advertised_network_variables_for_link_locked(&st, now, Some(side));
6823 let mut sources = self.local_discovery_timesync_sources(now);
6824 for (&route_side, route) in &st.discovery_routes {
6825 if route_side == side {
6826 continue;
6827 }
6828 for peer in route.announcers.values() {
6829 if now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
6830 endpoints.extend(peer.reachable.iter().copied());
6831 sources.extend(peer.reachable_timesync_sources.iter().cloned());
6832 }
6833 }
6834 }
6835 endpoints.retain(|ep| {
6836 !discovery::is_discovery_endpoint(*ep)
6837 && (opts.link_local_enabled || !ep.is_link_local_only())
6838 });
6839 crate::small_sort::sort(&mut endpoints);
6840 endpoints.dedup();
6841 crate::small_sort::sort(&mut variables);
6842 variables.dedup();
6843 sources.sort_unstable();
6844 sources.dedup();
6845 Some((endpoints, variables, sources, opts.link_capabilities()))
6846 } else {
6847 None
6848 }
6849 };
6850 let pkt = if let Some((endpoints, variables, sources, capabilities)) = summary {
6851 let address = self.local_address_advertisement(
6852 endpoints,
6853 variables,
6854 sources,
6855 capabilities,
6856 discovery::ADDRESS_STATE_REQUEST,
6857 );
6858 discovery::build_discovery_address(sender.as_ref(), now, &address)?
6859 } else {
6860 discovery::build_discovery_announce(sender.as_ref(), now, &[])?
6861 };
6862 self.emit_internal_tx(
6863 RouterTxItem::ToSide {
6864 src: None,
6865 dst: side,
6866 data: RouterItem::Packet(pkt),
6867 },
6868 true,
6869 true,
6870 )?;
6871 self.state
6872 .lock()
6873 .discovery_side_throttle
6874 .entry(side)
6875 .or_default()
6876 .next_ping_ms = now.saturating_add(discovery::DISCOVERY_SLOW_INTERVAL_MS);
6877 }
6878 Ok(())
6879 }
6880
6881 #[cfg(feature = "discovery")]
6882 fn emit_discovery_snapshot(
6883 &self,
6884 called_from_queue: bool,
6885 include_schema: bool,
6886 include_topology: bool,
6887 skip_in_flight_sides: bool,
6888 ) -> TelemetryResult<()> {
6889 self.emit_discovery_snapshot_on_side(
6890 called_from_queue,
6891 include_schema,
6892 include_topology,
6893 skip_in_flight_sides,
6894 None,
6895 )
6896 }
6897
6898 #[cfg(feature = "discovery")]
6899 fn emit_discovery_snapshot_on_side(
6900 &self,
6901 called_from_queue: bool,
6902 include_schema: bool,
6903 include_topology: bool,
6904 skip_in_flight_sides: bool,
6905 requested_side: Option<RouterSideId>,
6906 ) -> TelemetryResult<()> {
6907 if requested_side.is_none() {
6908 let count = self.state.lock().sides.len();
6909 for side in 0..count {
6910 if self.state.lock().sides[side].is_some() {
6911 self.emit_discovery_snapshot_on_side(
6912 called_from_queue,
6913 include_schema,
6914 include_topology,
6915 skip_in_flight_sides,
6916 Some(side),
6917 )?;
6918 }
6919 }
6920 return Ok(());
6921 }
6922 let (scratch, largest) = {
6923 let now = self.clock.now_ms();
6924 let sender = self.sender_arc();
6925 let st = self.state.lock();
6926 let election_required = self
6927 .preferred_discovery_master_present_locked(&st, now)
6928 .is_none();
6929 let snapshot = discovery::topology_snapshot_scratch(
6930 sender.as_ref(),
6931 self.cfg.handlers.len(),
6932 st.managed_variable_types.len().saturating_add(
6933 st.discovery_routes
6934 .iter()
6935 .filter(|(side, _)| Some(**side) != requested_side)
6936 .map(|(_, r)| r.reachable_network_variables.len())
6937 .sum::<usize>(),
6938 ),
6939 4,
6940 true,
6941 st.discovery_routes
6942 .iter()
6943 .filter(|(side, route)| {
6944 (Some(**side) != requested_side)
6945 && now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6946 })
6947 .flat_map(|(_, route)| route.announcers.iter())
6948 .filter(|(_, peer)| {
6949 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6950 })
6951 .map(|(name, peer)| {
6952 (
6953 name.as_str(),
6954 peer.reachable.as_slice(),
6955 peer.reachable_timesync_sources.as_slice(),
6956 peer.topology_boards.as_slice(),
6957 )
6958 }),
6959 );
6960 if !election_required {
6961 snapshot
6962 } else {
6963 let election = discovery::topology_snapshot_scratch(
6964 sender.as_ref(),
6965 self.cfg.handlers.len(),
6966 0,
6967 2,
6968 false,
6969 st.discovery_routes
6970 .values()
6971 .filter(|route| {
6972 now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6973 })
6974 .flat_map(|route| route.announcers.iter())
6975 .filter(|(_, peer)| {
6976 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6977 })
6978 .map(|(name, peer)| {
6979 (
6980 name.as_str(),
6981 peer.reachable.as_slice(),
6982 peer.reachable_timesync_sources.as_slice(),
6983 peer.topology_boards.as_slice(),
6984 )
6985 }),
6986 );
6987 (snapshot.0.max(election.0), snapshot.1.max(election.1))
6988 }
6989 };
6990 let (scratch, largest) = if include_schema {
6993 (scratch.max(8192), largest.max(2048))
6994 } else {
6995 (scratch, largest)
6996 };
6997 while let Err(err) = crate::memory_admission::check(scratch, largest) {
7002 if matches!(err, TelemetryError::Io("memory pressure"))
7003 && crate::memory_admission::check(512, 256).is_ok()
7004 {
7005 let mut st = self.state.lock();
7006 if let Some(victim) = st.lower_priority_shared_queue(199)
7007 && st.pop_shared_queue_item(victim)
7008 {
7009 continue;
7010 }
7011 }
7012 if !include_schema && matches!(err, TelemetryError::Io("memory pressure")) {
7013 return self.queue_pressure_keepalives(requested_side);
7014 }
7015 return Err(err);
7016 }
7017 let now_ms = self.clock.now_ms();
7018 let per_side = {
7019 let mut st = self.state.lock();
7020 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
7021 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7022 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7023 }
7024 st.fit_discovery_budget();
7025 let local_is_master =
7026 self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
7027 let mut per_side = None;
7029 for side_id in 0..st.sides.len() {
7030 let Some(side) = st.sides[side_id].as_ref() else {
7031 continue;
7032 };
7033 let opts = side.opts.clone();
7034 let link_local_enabled = opts.link_local_enabled;
7035 if requested_side.is_some_and(|requested| requested != side_id) {
7036 continue;
7037 }
7038 let discovery_in_flight = skip_in_flight_sides
7039 && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
7040 *pending_side == side_id
7041 && !tx_state.sent.is_empty()
7042 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
7043 });
7044 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
7045 {
7046 continue;
7047 }
7048 let level = if discovery_in_flight {
7052 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
7053 if !throttle.liveness_due(now_ms) {
7054 None
7055 } else {
7056 throttle.next_ping_ms =
7057 now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7058 Some(DiscoveryAdvertiseLevel::MinimalPing)
7059 }
7060 } else {
7061 Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
7062 };
7063 let Some(level) = level else {
7064 continue;
7065 };
7066 let include_side_topology = include_topology
7067 && self.route_allowed_locked(
7068 &st,
7069 None,
7070 Some(DataType::DiscoveryTopology),
7071 side_id,
7072 );
7073 let capabilities = opts.link_capabilities();
7074 if level == DiscoveryAdvertiseLevel::MinimalPing {
7075 per_side = Some((
7076 side_id,
7077 level,
7078 Vec::new(),
7079 Vec::new(),
7080 Vec::new(),
7081 Vec::new(),
7082 Vec::new(),
7083 capabilities,
7084 local_is_master,
7085 ));
7086 continue;
7087 }
7088 let current_topology = if include_side_topology {
7089 self.advertised_discovery_topology_for_link_locked(
7090 &st,
7091 now_ms,
7092 link_local_enabled,
7093 Some(side_id),
7094 )
7095 } else {
7096 Vec::new()
7097 };
7098 let (endpoints, timesync_sources) = if include_side_topology {
7099 self.advertised_summaries_from_topology_locked(
7100 &st,
7101 now_ms,
7102 link_local_enabled,
7103 Some(side_id),
7104 ¤t_topology,
7105 )
7106 } else {
7107 (
7108 self.advertised_discovery_endpoints_for_link_locked(
7109 &st,
7110 now_ms,
7111 link_local_enabled,
7112 Some(side_id),
7113 ),
7114 self.advertised_discovery_timesync_sources_for_link_locked(
7115 &st,
7116 now_ms,
7117 Some(side_id),
7118 ),
7119 )
7120 };
7121 let (topology, removed_topology) = match level {
7122 DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
7123 DiscoveryAdvertiseLevel::Incremental => {
7124 let previous = &st
7125 .discovery_side_throttle
7126 .get(&side_id)
7127 .expect("discovery throttle exists")
7128 .last_topology;
7129 discovery::topology_delta(previous, ¤t_topology)
7130 }
7131 DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
7132 };
7133 let throttle = st
7134 .discovery_side_throttle
7135 .get_mut(&side_id)
7136 .expect("discovery throttle exists");
7137 if level == DiscoveryAdvertiseLevel::Full
7138 || (level == DiscoveryAdvertiseLevel::Incremental
7139 && throttle.incremental_retries_remaining == 0)
7140 {
7141 throttle.last_topology = current_topology;
7142 }
7143 per_side = Some((
7144 side_id,
7145 level,
7146 endpoints,
7147 self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
7148 timesync_sources,
7149 topology,
7150 removed_topology,
7151 capabilities,
7152 local_is_master,
7153 ));
7154 }
7155 per_side
7156 };
7157 if let Some((
7158 side_id,
7159 level,
7160 endpoints,
7161 network_variables,
7162 timesync_sources,
7163 topology,
7164 removed_topology,
7165 capabilities,
7166 local_is_master,
7167 )) = per_side
7168 {
7169 let sender = self.sender_arc();
7170 if matches!(
7171 level,
7172 DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
7173 ) {
7174 let address = self.local_address_advertisement(
7175 endpoints.clone(),
7176 network_variables.clone(),
7177 timesync_sources.clone(),
7178 capabilities,
7179 if local_is_master {
7180 discovery::ADDRESS_STATE_APPROVED
7181 } else {
7182 discovery::ADDRESS_STATE_REQUEST
7183 },
7184 );
7185 let pkt = discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?;
7186 self.emit_internal_tx(
7187 RouterTxItem::ToSide {
7188 src: None,
7189 dst: side_id,
7190 data: RouterItem::Packet(pkt),
7191 },
7192 true,
7193 called_from_queue,
7194 )?;
7195 }
7196 if level == DiscoveryAdvertiseLevel::MinimalPing {
7197 let pkt = discovery::build_discovery_announce(
7198 sender.as_ref(),
7199 now_ms,
7200 endpoints.as_slice(),
7201 )?;
7202 self.emit_internal_tx(
7203 RouterTxItem::ToSide {
7204 src: None,
7205 dst: side_id,
7206 data: RouterItem::Packet(pkt),
7207 },
7208 true,
7209 called_from_queue,
7210 )?;
7211 }
7212 if include_topology
7213 && (!topology.is_empty() || !removed_topology.is_empty())
7214 && level != DiscoveryAdvertiseLevel::MinimalPing
7215 {
7216 let pkt = if level == DiscoveryAdvertiseLevel::Full {
7217 discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
7218 } else {
7219 discovery::build_discovery_topology_delta(
7220 sender.as_ref(),
7221 now_ms,
7222 &topology,
7223 &removed_topology,
7224 )?
7225 };
7226 self.emit_internal_tx(
7227 RouterTxItem::ToSide {
7228 src: None,
7229 dst: side_id,
7230 data: RouterItem::Packet(pkt),
7231 },
7232 true,
7233 called_from_queue,
7234 )?;
7235 }
7236 drop((
7239 endpoints,
7240 network_variables,
7241 timesync_sources,
7242 topology,
7243 removed_topology,
7244 ));
7245 if include_schema && level == DiscoveryAdvertiseLevel::Full {
7250 let pkt = discovery::build_discovery_schema(sender.as_ref(), now_ms)?;
7251 self.emit_internal_tx(
7252 RouterTxItem::ToSide {
7253 src: None,
7254 dst: side_id,
7255 data: RouterItem::Packet(pkt),
7256 },
7257 true,
7258 called_from_queue,
7259 )?;
7260 }
7261 }
7262 Ok(())
7263 }
7264
7265 #[cfg(feature = "discovery")]
7266 fn queue_discovery_announce(
7267 &self,
7268 include_schema: bool,
7269 skip_in_flight_sides: bool,
7270 ) -> TelemetryResult<()> {
7271 let now_ms = self.clock.now_ms();
7272 {
7273 let mut st = self.state.lock();
7274 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
7275 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7276 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7277 }
7278 st.fit_discovery_budget();
7279 if st.sides.iter().all(|side| side.is_none()) {
7280 return Ok(());
7281 }
7282 if include_schema {
7285 for throttle in st.discovery_side_throttle.values_mut() {
7286 throttle.has_sent_full = false;
7287 }
7288 }
7289 st.discovery_cadence.on_announce_sent(now_ms);
7290 }
7291 self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
7292 }
7293
7294 #[cfg(feature = "discovery")]
7295 fn note_schema_received(&self, side: RouterSideId, sender: &str) {
7296 let mut st = self.state.lock();
7297 let address = sender
7301 .strip_prefix("@addr:")
7302 .and_then(|value| value.parse::<u32>().ok());
7303 if let Some(route) = st.discovery_routes.get_mut(&side) {
7304 for (name, peer) in &mut route.announcers {
7305 if name == sender
7306 || address.is_some_and(|address| {
7307 peer.advertised_address == Some(address)
7308 || sender_address_u32(name) == address
7309 })
7310 {
7311 peer.has_schema = true;
7312 break;
7313 }
7314 }
7315 }
7316 }
7317
7318 #[cfg(feature = "discovery")]
7319 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
7320 let now_ms = self.clock.now_ms();
7321 let recovery_sides = {
7325 let mut st = self.state.lock();
7326 let mut due = Vec::new();
7327 for side in 0..st.sides.len() {
7328 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
7329 route.announcers.iter().any(|(name, peer)| {
7330 !peer.has_full_topology
7331 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
7332 && !route.announcers.values().any(|bridge| {
7333 bridge.has_full_topology
7334 && now_ms.saturating_sub(bridge.last_seen_ms)
7335 < DISCOVERY_ROUTE_TTL_MS
7336 && bridge
7337 .topology_boards
7338 .iter()
7339 .any(|board| board.sender_id == *name)
7340 })
7341 })
7342 });
7343 if !missing {
7344 continue;
7345 }
7346 let throttle = st.discovery_side_throttle.entry(side).or_default();
7347 if throttle.next_topology_request_ms == 0 {
7348 throttle.next_topology_request_ms =
7349 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
7350 } else if now_ms >= throttle.next_topology_request_ms {
7351 throttle.next_topology_request_ms =
7352 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
7353 due.push(side);
7354 }
7355 }
7356 due
7357 };
7358 let recovering = !recovery_sides.is_empty();
7359 for side in recovery_sides {
7360 let pkt =
7361 discovery::build_discovery_topology_request(self.sender_arc().as_ref(), now_ms)?;
7362 self.emit_internal_tx(
7363 RouterTxItem::ToSide {
7364 src: None,
7365 dst: side,
7366 data: RouterItem::Packet(pkt),
7367 },
7368 true,
7369 true,
7370 )?;
7371 }
7372 let schema_recovery_sides = {
7373 let mut st = self.state.lock();
7374 let mut due = Vec::new();
7375 for side in 0..st.sides.len() {
7376 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
7377 route.announcers.values().any(|peer| {
7378 peer.has_full_topology
7379 && !peer.has_schema
7380 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
7381 })
7382 });
7383 if !missing {
7384 continue;
7385 }
7386 let throttle = st.discovery_side_throttle.entry(side).or_default();
7387 if throttle.next_schema_request_ms == 0 {
7388 throttle.next_schema_request_ms =
7389 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
7390 } else if now_ms >= throttle.next_schema_request_ms {
7391 throttle.next_schema_request_ms =
7392 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
7393 due.push(side);
7394 }
7395 }
7396 due
7397 };
7398 let recovering = recovering || !schema_recovery_sides.is_empty();
7399 for side in schema_recovery_sides {
7400 let pkt =
7401 discovery::build_discovery_schema_request(self.sender_arc().as_ref(), now_ms)?;
7402 self.emit_internal_tx(
7403 RouterTxItem::ToSide {
7404 src: None,
7405 dst: side,
7406 data: RouterItem::Packet(pkt),
7407 },
7408 true,
7409 true,
7410 )?;
7411 }
7412 let due = {
7413 let mut st = self.state.lock();
7414 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
7415 if removed {
7416 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7417 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7418 }
7419 st.fit_discovery_budget();
7420 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
7421 let Some(side) = side.as_ref() else {
7422 return false;
7423 };
7424 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
7425 {
7426 return false;
7427 }
7428 let _ = side;
7429 let in_flight = st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
7430 *pending_side == side_id
7431 && !tx_state.sent.is_empty()
7432 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
7433 });
7434 st.discovery_side_throttle
7435 .get(&side_id)
7436 .is_none_or(|throttle| {
7437 (!in_flight && (!throttle.has_sent_full || throttle.pending_incremental))
7438 || throttle.liveness_due(now_ms)
7439 })
7440 });
7441 if st.sides.is_empty() || !has_any {
7442 return Ok(recovering);
7443 }
7444 st.discovery_cadence.due(now_ms)
7445 };
7446 if !due {
7447 return Ok(recovering);
7448 }
7449 self.queue_discovery_announce(false, true)?;
7458 Ok(true)
7459 }
7460
7461 #[cfg(feature = "discovery")]
7462 fn learn_discovery_packet(
7463 &self,
7464 pkt: &Packet,
7465 src: Option<RouterSideId>,
7466 called_from_queue: bool,
7467 ) -> TelemetryResult<bool> {
7468 if !discovery::is_discovery_type(pkt.data_type()) {
7469 return Ok(false);
7470 }
7471 let Some(side) = src else {
7472 return Ok(true);
7473 };
7474 if self.refresh_known_discovery_liveness(pkt, side)? {
7475 return Ok(true);
7476 }
7477 #[cfg(not(feature = "std"))]
7478 if pkt.data_type() == DataType::DiscoverySchema
7479 && discovery::discovery_schema_payload_is_fully_known(pkt.payload())?
7480 {
7481 self.note_schema_received(side, pkt.sender());
7482 return Ok(true);
7483 }
7484 if matches!(
7488 pkt.data_type(),
7489 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
7490 ) {
7491 crate::memory_admission::check(1024, 256)?;
7492 if let Some(transport) = self.state.lock().side_transport.get_mut(&side) {
7497 transport.refresh_transmit_header_templates();
7498 }
7499 }
7500 if matches!(
7501 pkt.data_type(),
7502 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
7503 ) {
7504 let now_ms = self.clock.now_ms();
7505 {
7506 let mut st = self.state.lock();
7507 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
7508 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7509 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7510 }
7511 let throttle = st.discovery_side_throttle.entry(side).or_default();
7512 throttle.has_sent_full = false;
7513 throttle.last_topology = Vec::new();
7517 }
7518 self.emit_discovery_snapshot_on_side(
7523 called_from_queue,
7524 pkt.data_type() == DataType::DiscoverySchemaRequest,
7525 true,
7526 false,
7527 Some(side),
7528 )?;
7529 return Ok(true);
7530 }
7531 let small_topology_scratch =
7532 if pkt.data_type() == DataType::DiscoveryTopology && pkt.payload().len() <= 128 {
7533 let st = self.state.lock();
7534 let sender = pkt
7535 .sender()
7536 .strip_prefix("@addr:")
7537 .and_then(|value| value.parse::<u32>().ok())
7538 .and_then(|address| Self::hostname_for_address_borrowed_locked(&st, address))
7539 .unwrap_or(pkt.sender());
7540 st.discovery_routes.get(&side).and_then(|route| {
7541 let peer = route.announcers.get(sender)?;
7542 let mut shallow = discovery::CloneScratch::default();
7543 shallow.buffer(route.reachable.len().saturating_mul(4));
7544 shallow.buffer(route.reachable_network_variables.len().saturating_mul(4));
7545 shallow.buffer(
7546 route
7547 .requested_network_variables
7548 .len()
7549 .saturating_mul(core::mem::size_of::<(DataType, u64)>()),
7550 );
7551 shallow.strings(&route.reachable_timesync_sources);
7552 for (name, value) in &route.announcers {
7553 shallow.buffer(name.len());
7554 shallow.buffer(value.reachable.len().saturating_mul(4));
7555 shallow.buffer(value.advertised_reachable.len().saturating_mul(4));
7556 shallow.strings(&value.reachable_timesync_sources);
7557 shallow.strings(&value.advertised_reachable_timesync_sources);
7558 shallow.buffer(value.reachable_network_variables.len().saturating_mul(4));
7559 }
7560 let entry = core::mem::size_of::<String>()
7561 + core::mem::size_of::<DiscoverySenderState>();
7562 shallow.bytes = shallow
7563 .bytes
7564 .saturating_add(route.announcers.len().saturating_mul(entry));
7565 shallow.largest = shallow
7566 .largest
7567 .max(entry.saturating_mul(11).saturating_add(128));
7568 let mut deep = discovery::CloneScratch::default();
7569 if discovery::topology_payload_is_incremental(pkt.payload()) {
7570 deep.graph(&peer.topology_boards, pkt.payload().len() / 16);
7571 }
7572 Some((
7573 2048usize
7574 .saturating_add(shallow.bytes.saturating_mul(3))
7575 .saturating_add(deep.bytes)
7576 .saturating_add(pkt.payload().len().saturating_mul(4)),
7577 shallow.largest.max(deep.largest).max(2048),
7578 ))
7579 })
7580 } else {
7581 None
7582 };
7583 let retained_summary = if small_topology_scratch.is_none()
7584 && matches!(
7585 pkt.data_type(),
7586 DataType::DiscoveryAddress
7587 | DataType::DiscoveryTopology
7588 | DataType::DiscoveryAnnounce
7589 | DataType::DiscoveryTimeSyncSources
7590 | DataType::ManagedVariableRequest
7591 ) {
7592 self.state
7593 .lock()
7594 .discovery_routes
7595 .get(&side)
7596 .map_or(0, |route| {
7597 RouterInner::discovery_route_byte_cost(side, route)
7598 })
7599 } else {
7600 0
7601 };
7602 let (decode_bytes, largest_block) = if let Some(scratch) = small_topology_scratch {
7603 scratch
7604 } else if pkt.data_type() == DataType::ManagedVariableRequest {
7605 let ty = discovery::decode_managed_variable_request(pkt)?;
7606 let existing = self
7607 .state
7608 .lock()
7609 .discovery_routes
7610 .get(&side)
7611 .is_some_and(|route| {
7612 route
7613 .requested_network_variables
7614 .iter()
7615 .any(|(id, _)| *id == ty)
7616 });
7617 if existing {
7618 (1024, 256)
7619 } else {
7620 (
7623 2048usize.saturating_add(retained_summary.saturating_mul(2)),
7624 retained_summary
7625 .checked_next_power_of_two()
7626 .unwrap_or(usize::MAX)
7627 .max(1024),
7628 )
7629 }
7630 } else {
7631 let scratch = if pkt.data_type() == DataType::DiscoveryAddress {
7632 4096usize.saturating_add(retained_summary)
7633 } else {
7634 8192usize.saturating_add(retained_summary)
7635 };
7636 let decode_bytes = pkt
7637 .payload()
7638 .len()
7639 .saturating_mul(4)
7640 .saturating_add(scratch);
7641 let largest_block = pkt.payload().len().saturating_mul(2).max(2048);
7642 (decode_bytes, largest_block)
7643 };
7644 while let Err(err) = crate::memory_admission::check(decode_bytes, largest_block) {
7645 if !matches!(err, TelemetryError::Io("memory pressure")) {
7646 return Err(err);
7647 }
7648 let mut st = self.state.lock();
7649 let Some(victim) = st.lower_priority_shared_queue(255) else {
7650 return Err(err);
7651 };
7652 if !st.pop_shared_queue_item(victim) {
7653 return Err(err);
7654 }
7655 }
7656 if pkt.data_type() == DataType::DiscoveryAddress {
7657 let mut ad = discovery::decode_discovery_address(pkt)?;
7658 let identity = ad.clone();
7659 let mut changed = false;
7660 let sender_id = if ad.hostname.is_empty() {
7666 pkt.sender()
7667 } else {
7668 ad.hostname.as_str()
7669 };
7670 let now_ms = self.clock.now_ms();
7671 let mut st = self.state.lock();
7672 let side_link_local_enabled = st
7673 .sides
7674 .get(side)
7675 .and_then(|entry| entry.as_ref())
7676 .map(|side_ref| side_ref.opts.link_local_enabled)
7677 .unwrap_or(false);
7678 if !side_link_local_enabled {
7679 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
7680 }
7681 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
7682 let previously_reachable_network_variables = route.reachable_network_variables.clone();
7683 if pkt.sender() != sender_id {
7684 route.announcers.remove(pkt.sender());
7685 }
7686 let mut sender_state =
7687 core::mem::take(route.announcers.entry(sender_id.to_string()).or_default());
7688 if sender_state.advertised_address != Some(ad.address) {
7694 sender_state.advertised_address = Some(ad.address);
7695 changed = true;
7696 }
7697 if sender_state.advertised_reachable != ad.reachable_endpoints {
7698 sender_state.advertised_reachable = ad.reachable_endpoints;
7699 changed = true;
7700 }
7701 if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
7702 sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
7703 changed = true;
7704 }
7705 if sender_state.reachable_network_variables != ad.reachable_network_variables {
7706 sender_state.reachable_network_variables = ad.reachable_network_variables;
7707 changed = true;
7708 }
7709 if sender_state.link_capabilities != Some(ad.link_capabilities) {
7710 sender_state.link_capabilities = Some(ad.link_capabilities);
7711 changed = true;
7712 }
7713 Self::refresh_sender_topology_state(&mut sender_state);
7714 sender_state.last_seen_ms = now_ms;
7715 route.announcers.insert(sender_id.to_string(), sender_state);
7716 Self::recompute_discovery_side_state(&mut route);
7717 let newly_reachable_network_variables: Vec<DataType> = route
7718 .reachable_network_variables
7719 .iter()
7720 .copied()
7721 .filter(|ty| !previously_reachable_network_variables.contains(ty))
7722 .collect();
7723 st.commit_discovery_route(side, route)?;
7724 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7725 if changed {
7726 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
7727 }
7728 let replay: Vec<Packet> = newly_reachable_network_variables
7734 .iter()
7735 .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
7736 .filter_map(|ty| {
7737 st.managed_variable_latest
7738 .get(&ty.as_u32())
7739 .map(|entry| entry.packet.clone())
7740 })
7741 .collect();
7742 drop(st);
7743 self.ingest_address_advertisement(identity)?;
7744 for value in replay {
7745 self.emit_internal_tx_with_priority(
7746 RouterTxItem::ToSide {
7747 src: None,
7748 dst: side,
7749 data: RouterItem::Packet(value),
7750 },
7751 true,
7752 crate::transport_priority(DataType::ManagedVariableValue),
7753 called_from_queue,
7754 )?;
7755 }
7756 return Ok(true);
7757 }
7758 let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
7759 Some(discovery::decode_discovery_topology_update(pkt)?)
7760 } else {
7761 None
7762 };
7763 let packet_sender = {
7764 let st = self.state.lock();
7765 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
7766 if canonical != pkt.sender() {
7767 canonical
7768 } else if let Some(address) = pkt
7769 .sender()
7770 .strip_prefix("@addr:")
7771 .and_then(|value| value.parse::<u32>().ok())
7772 {
7773 decoded_topology
7779 .as_ref()
7780 .and_then(|update| {
7781 update
7782 .boards
7783 .iter()
7784 .find(|board| sender_address_u32(&board.sender_id) == address)
7785 })
7786 .map(|board| board.sender_id.clone())
7787 .unwrap_or(canonical)
7788 } else {
7789 canonical
7790 }
7791 };
7792 let local_sender = self.sender_arc();
7793 if packet_sender == local_sender.as_ref() {
7794 return Ok(true);
7795 }
7796
7797 if pkt.data_type() == DataType::ManagedVariableRequest {
7798 let ty = discovery::decode_managed_variable_request(pkt)?;
7799 {
7805 let now_ms = self.clock.now_ms();
7806 let mut st = self.state.lock();
7807 let existing = st.discovery_routes.get_mut(&side).and_then(|route| {
7811 let (_, last_seen_ms) = route
7812 .requested_network_variables
7813 .iter_mut()
7814 .find(|(requested, _)| *requested == ty)?;
7815 *last_seen_ms = now_ms;
7816 route.last_seen_ms = route.last_seen_ms.max(now_ms);
7817 Some(())
7818 });
7819 let newly_reachable = existing.is_none();
7820 if newly_reachable {
7821 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
7822 route.requested_network_variables.push((ty, now_ms));
7823 Self::recompute_discovery_side_state(&mut route);
7824 st.commit_discovery_route(side, route)?;
7825 }
7826 if newly_reachable {
7827 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7828 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7829 }
7830 }
7831 if !self.can_write_managed_variable(ty) {
7832 return Ok(false);
7837 }
7838 if let Some(value) = self.managed_variable_latest(ty) {
7839 self.emit_internal_tx(
7840 RouterTxItem::ToSide {
7841 src: None,
7842 dst: side,
7843 data: RouterItem::Packet(value),
7844 },
7845 true,
7846 called_from_queue,
7847 )?;
7848 return Ok(true);
7849 }
7850 return Ok(false);
7853 }
7854 if pkt.data_type() == DataType::DiscoverySchema {
7855 let snapshot = discovery::decode_discovery_schema(pkt)?;
7856 let mut st = self.state.lock();
7865 let budget = st.schema_growth_budget();
7866 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
7867 if report.changed() {
7868 st.fit_discovery_budget();
7869 }
7870 drop(st);
7871 self.note_schema_received(side, &packet_sender);
7872 return Ok(true);
7873 }
7874 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
7875 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
7876 return Ok(true);
7877 }
7878 let mut announce = if pkt.data_type() == DataType::DiscoveryAnnounce {
7879 Some(discovery::decode_discovery_announce(pkt)?)
7880 } else {
7881 None
7882 };
7883 let mut time_sources = if pkt.data_type() == DataType::DiscoveryTimeSyncSources {
7884 Some(discovery::decode_discovery_timesync_sources(pkt)?)
7885 } else {
7886 None
7887 };
7888 let mut st = self.state.lock();
7889 let now_ms = self.clock.now_ms();
7890 if pkt.data_type() == DataType::DiscoveryLeave {
7891 let leaving = packet_sender.as_str();
7892 let before = st.discovery_routes.clone();
7893 for route in st.discovery_routes.values_mut() {
7894 route.announcers.remove(leaving);
7895 for sender_state in route.announcers.values_mut() {
7896 Arc::make_mut(&mut sender_state.topology_boards)
7897 .retain(|board| board.sender_id != leaving);
7898 for board in Arc::make_mut(&mut sender_state.topology_boards).iter_mut() {
7899 board.connections.retain(|peer| peer != leaving);
7900 }
7901 Self::refresh_sender_topology_state(sender_state);
7902 }
7903 Self::recompute_discovery_side_state(route);
7904 }
7905 st.discovery_routes
7906 .retain(|_, route| !route.announcers.is_empty());
7907 if st.discovery_routes != before {
7908 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7909 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7910 }
7911 return Ok(true);
7912 }
7913 if let Some(update) = decoded_topology.as_mut() {
7915 for board in update.boards.iter_mut() {
7916 board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
7917 for peer in board.connections.iter_mut() {
7918 *peer = Self::canonical_sender_locked(&st, peer);
7919 }
7920 }
7921 }
7922 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
7923 let alias_state = if packet_sender != pkt.sender() {
7924 route.announcers.remove(pkt.sender())
7925 } else {
7926 None
7927 };
7928 let side_link_local_enabled = st
7929 .sides
7930 .get(side)
7931 .and_then(|entry| entry.as_ref())
7932 .map(|side_ref| side_ref.opts.link_local_enabled)
7933 .unwrap_or(false);
7934 let mut sender_state = route
7935 .announcers
7936 .get_mut(&packet_sender)
7937 .map(core::mem::take)
7938 .or(alias_state)
7939 .unwrap_or_default();
7940 let changed = match pkt.data_type() {
7941 DataType::DiscoveryAnnounce => {
7942 let mut reachable = announce.take().expect("decoded above");
7943 if reachable.is_empty() {
7948 false
7949 } else {
7950 if !side_link_local_enabled {
7951 reachable.retain(|ep| !ep.is_link_local_only());
7952 }
7953 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
7954 let changed = board.reachable_endpoints != reachable;
7955 board.reachable_endpoints = reachable;
7956 Self::refresh_sender_topology_state(&mut sender_state);
7957 changed
7958 }
7959 }
7960 DataType::DiscoveryTimeSyncSources => {
7961 let sources = time_sources.take().expect("decoded above");
7962 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
7963 let changed = board.reachable_timesync_sources != sources;
7964 board.reachable_timesync_sources = sources;
7965 Self::refresh_sender_topology_state(&mut sender_state);
7966 changed
7967 }
7968 DataType::DiscoveryTopology => {
7969 let mut update = decoded_topology
7970 .take()
7971 .expect("topology packet was decoded before route selection");
7972 if !side_link_local_enabled {
7973 for board in update.boards.iter_mut() {
7974 board
7975 .reachable_endpoints
7976 .retain(|ep| !ep.is_link_local_only());
7977 }
7978 }
7979 if !update.incremental {
7980 sender_state.has_full_topology = true;
7981 }
7982 let changed = discovery::apply_shared_topology_update(
7983 &mut sender_state.topology_boards,
7984 update,
7985 );
7986 Self::refresh_sender_topology_state(&mut sender_state);
7987 changed
7988 }
7989 DataType::DiscoverySchema => false,
7990 _ => false,
7991 };
7992 sender_state.last_seen_ms = now_ms;
7993 route.announcers.insert(packet_sender, sender_state);
7994 Self::recompute_discovery_side_state(&mut route);
7995 st.commit_discovery_route(side, route)?;
7996 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7997 if changed {
7998 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7999 }
8000 Ok(true)
8001 }
8002
8003 #[cfg(not(feature = "discovery"))]
8004 fn queue_discovery_announce(&self) -> TelemetryResult<()> {
8005 Ok(())
8006 }
8007
8008 #[cfg(not(feature = "discovery"))]
8009 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
8010 Ok(false)
8011 }
8012
8013 #[cfg(not(feature = "discovery"))]
8014 fn learn_discovery_packet(
8015 &self,
8016 _pkt: &Packet,
8017 _src: Option<RouterSideId>,
8018 _called_from_queue: bool,
8019 ) -> TelemetryResult<bool> {
8020 Ok(false)
8021 }
8022
8023 #[inline]
8024 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
8025 (side, ty.as_u32())
8026 }
8027
8028 fn reliable_tx_state_mut<'a>(
8029 &'a self,
8030 st: &'a mut RouterInner,
8031 side: RouterSideId,
8032 ty: DataType,
8033 ) -> &'a mut ReliableTxState {
8034 let key = Self::reliable_key(side, ty);
8035 st.reliable_tx
8036 .entry(key)
8037 .or_insert_with(|| ReliableTxState {
8038 next_seq: 1,
8039 sent_order: VecDeque::new(),
8040 sent: BTreeMap::new(),
8041 })
8042 }
8043
8044 fn reliable_rx_state_mut<'a>(
8045 &'a self,
8046 st: &'a mut RouterInner,
8047 side: RouterSideId,
8048 ty: DataType,
8049 ) -> &'a mut ReliableRxState {
8050 let key = Self::reliable_key(side, ty);
8051 st.reliable_rx
8052 .entry(key)
8053 .or_insert_with(|| ReliableRxState {
8054 expected_seq: 1,
8055 buffered: BTreeMap::new(),
8056 })
8057 }
8058
8059 fn reliable_control_packet(
8060 &self,
8061 control_ty: DataType,
8062 ty: DataType,
8063 seq: u32,
8064 ) -> TelemetryResult<Packet> {
8065 let sender = self.sender_arc();
8066 Packet::new(
8067 control_ty,
8068 message_meta(control_ty).endpoints_ref(),
8069 sender.as_ref(),
8070 self.packet_timestamp_ms(),
8071 encode_slice_le(&[ty.as_u32(), seq]),
8072 )
8073 }
8074
8075 fn queue_reliable_ack(
8076 &self,
8077 side: RouterSideId,
8078 ty: DataType,
8079 seq: u32,
8080 called_from_queue: bool,
8081 ) -> TelemetryResult<()> {
8082 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
8083 self.emit_internal_tx_with_priority(
8084 RouterTxItem::ToSide {
8085 src: None,
8086 dst: side,
8087 data: RouterItem::Packet(pkt),
8088 },
8089 true,
8090 message_priority(DataType::ReliableAck),
8091 called_from_queue,
8092 )
8093 }
8094
8095 fn queue_reliable_packet_request(
8096 &self,
8097 side: RouterSideId,
8098 ty: DataType,
8099 seq: u32,
8100 called_from_queue: bool,
8101 ) -> TelemetryResult<()> {
8102 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
8103 self.emit_internal_tx_with_priority(
8104 RouterTxItem::ToSide {
8105 src: None,
8106 dst: side,
8107 data: RouterItem::Packet(pkt),
8108 },
8109 true,
8110 message_priority(DataType::ReliablePacketRequest),
8111 called_from_queue,
8112 )
8113 }
8114
8115 fn queue_reliable_partial_ack(
8116 &self,
8117 side: RouterSideId,
8118 ty: DataType,
8119 seq: u32,
8120 called_from_queue: bool,
8121 ) -> TelemetryResult<()> {
8122 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
8123 self.emit_internal_tx_with_priority(
8124 RouterTxItem::ToSide {
8125 src: None,
8126 dst: side,
8127 data: RouterItem::Packet(pkt),
8128 },
8129 true,
8130 message_priority(DataType::ReliablePartialAck),
8131 called_from_queue,
8132 )
8133 }
8134
8135 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
8136 let mut st = self.state.lock();
8137 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8138 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
8139 tx_state.sent.remove(&ack);
8140 tx_state.sent_order.retain(|seq| *seq != ack);
8141 return;
8142 }
8143
8144 while let Some(seq) = tx_state.sent_order.front().copied() {
8145 if seq > ack {
8146 break;
8147 }
8148 tx_state.sent_order.pop_front();
8149 tx_state.sent.remove(&seq);
8150 }
8151 }
8152
8153 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
8154 let mut st = self.state.lock();
8155 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8156 if let Some(sent) = tx_state.sent.get_mut(&seq) {
8157 sent.partial_acked = true;
8158 }
8159 }
8160
8161 fn queue_reliable_retransmit(
8162 &self,
8163 side: RouterSideId,
8164 ty: DataType,
8165 seq: u32,
8166 called_from_queue: bool,
8167 ) -> TelemetryResult<()> {
8168 let mut queued = None;
8169 {
8170 let mut st = self.state.lock();
8171 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8172 if let Some(sent) = tx_state.sent.get_mut(&seq)
8173 && !sent.queued
8174 {
8175 sent.queued = true;
8176 sent.partial_acked = false;
8177 queued = Some(sent.bytes.clone());
8178 }
8179 }
8180
8181 if let Some(bytes) = queued {
8182 if called_from_queue {
8183 self.tx_queue_item_with_priority(
8184 RouterTxItem::ReliableReplay { dst: side, bytes },
8185 true,
8186 Self::router_item_priority_bumped(ty),
8187 )?;
8188 } else {
8189 self.tx_item_impl(
8190 RouterTxItem::ReliableReplay { dst: side, bytes },
8191 true,
8192 false,
8193 )?;
8194 }
8195 }
8196
8197 Ok(())
8198 }
8199
8200 fn send_reliable_raw_to_side(
8201 &self,
8202 side: RouterSideId,
8203 bytes: crate::SharedBytes,
8204 relayed: bool,
8205 ) -> TelemetryResult<()> {
8206 let handler = {
8207 let st = self.state.lock();
8208 let side_ref = Self::side_ref(&st, side)?;
8209 if !side_ref.opts.egress_enabled {
8210 return Ok(());
8211 }
8212 (side_ref.tx_handler.clone(), side_ref.opts.clone())
8213 };
8214
8215 let (handler, opts) = handler;
8216
8217 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
8218 return Err(TelemetryError::Io("side tx busy"));
8219 };
8220 let started_ms = self.clock.now_ms();
8221 let (ty, priority) = Self::item_type_priority(&RouterItem::Packed(bytes.clone()))?;
8222 let priority = priority.max(self.effective_transport_priority(ty));
8223 let result = match handler {
8224 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
8225 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
8226 let mut attempts_total = 0usize;
8227 let mut sent_bytes = 0usize;
8228 self.send_side_frames_resumable(side, frames, |frame| {
8229 match self.retry_with_attempts(runtime_max_handler_retries(), || {
8230 Self::packed_handler_call(&packed, frame, priority)
8231 }) {
8232 Ok((_, attempts)) => {
8233 attempts_total = attempts_total.saturating_add(attempts);
8234 sent_bytes = sent_bytes.saturating_add(frame.len());
8235 }
8236 Err((err, attempts)) => {
8237 self.note_side_tx_failure(
8238 side,
8239 ty,
8240 attempts_total.saturating_add(attempts),
8241 );
8242 return Err(err);
8243 }
8244 }
8245
8246 Ok(())
8247 })?;
8248 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
8249 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
8250 return Ok(());
8251 }
8252 RouterTxHandlerFn::Packet(f) => {
8253 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
8254 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
8255 }
8256 };
8257 match result {
8258 Ok((_, attempts)) => {
8259 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
8260 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
8261 Ok(())
8262 }
8263 Err((err, attempts)) => {
8264 self.note_side_tx_failure(side, ty, attempts);
8265 Err(err)
8266 }
8267 }
8268 }
8269
8270 #[allow(unused_mut)]
8271 fn send_reliable_to_side(
8272 &self,
8273 side: RouterSideId,
8274 data: RouterItem,
8275 relayed: bool,
8276 ) -> TelemetryResult<()> {
8277 let end_to_end_ordered = !self.item_target_senders(&data)?.is_empty()
8278 && matches!(
8279 reliable_mode(self.item_route_info(&data)?.1),
8280 crate::ReliableMode::Ordered
8281 );
8282 let (handler, opts, hop_reliable_enabled) = {
8283 let st = self.state.lock();
8284 let side_ref = Self::side_ref(&st, side)?;
8285 let opts = side_ref.opts.clone();
8286 let hop_reliable_enabled = opts.reliable_enabled
8287 && !end_to_end_ordered
8288 && self.cfg.reliable_enabled()
8289 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
8290 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
8291 };
8292
8293 if !matches!(
8294 &handler,
8295 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
8296 ) {
8297 return self.call_side_tx_handler(side, &handler, &data, relayed);
8298 }
8299
8300 if !hop_reliable_enabled {
8301 let mut adjusted_opts = opts;
8302 adjusted_opts.reliable_enabled = false;
8303 let preserve_end_to_end_ack = true;
8308 if let Some(adjusted) =
8309 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
8310 {
8311 return self.call_side_tx_handler(side, &handler, &adjusted, relayed);
8312 }
8313 return Ok(());
8314 }
8315
8316 let ty = match &data {
8317 RouterItem::Packet(pkt) => pkt.data_type(),
8318 RouterItem::Packed(bytes) => {
8319 wire_format::peek_routing_frame_info(bytes.as_ref())?
8320 .envelope
8321 .ty
8322 }
8323 };
8324 let priority = self.item_service_priority(&data)?;
8325
8326 if !is_reliable_type(ty) {
8327 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
8328 self.call_side_tx_handler(side, &handler, &adjusted, relayed)?;
8329 }
8330 return Ok(());
8331 }
8332
8333 let (seq, flags) = {
8334 let mut st = self.state.lock();
8335 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8336 if tx_state.sent.len() >= runtime_reliable_max_pending() {
8337 return Err(TelemetryError::PacketTooLarge(
8338 "router reliable history full",
8339 ));
8340 }
8341 let seq = tx_state.next_seq;
8342 let next = tx_state.next_seq.wrapping_add(1);
8343 tx_state.next_seq = if next == 0 { 1 } else { next };
8344 let flags = match reliable_mode(ty) {
8345 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
8346 _ => 0,
8347 };
8348 (seq, flags)
8349 };
8350
8351 let mut bytes: crate::SharedBytes = match data {
8352 RouterItem::Packet(pkt) => self.pack_packet_for_router(
8353 &pkt,
8354 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
8355 )?,
8356 RouterItem::Packed(bytes) => {
8357 #[cfg(feature = "cryptography")]
8358 if self.e2e_seal_config_for_type(ty).is_some() {
8359 self.prepare_packed_for_remote(
8360 bytes,
8361 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
8362 )?
8363 } else {
8364 let Some(rewritten) =
8365 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
8366 else {
8367 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
8368 return Err(TelemetryError::Io("side tx busy"));
8369 };
8370 let started_ms = self.clock.now_ms();
8371 let frames =
8372 self.encode_side_transport_frames(side, opts, bytes.clone())?;
8373 let mut attempts_total = 0usize;
8374 let mut sent_bytes = 0usize;
8375 self.send_side_frames_resumable(side, frames, |frame| {
8376 match self.retry_with_attempts(runtime_max_handler_retries(), || {
8377 Self::packed_handler_call(&handler, frame, priority)
8378 }) {
8379 Ok((_, attempts)) => {
8380 attempts_total = attempts_total.saturating_add(attempts);
8381 sent_bytes = sent_bytes.saturating_add(frame.len());
8382 }
8383 Err((err, attempts)) => {
8384 self.note_side_tx_failure(
8385 side,
8386 ty,
8387 attempts_total.saturating_add(attempts),
8388 );
8389 return Err(err);
8390 }
8391 }
8392
8393 Ok(())
8394 })?;
8395 self.record_side_tx_sample(
8396 side,
8397 sent_bytes,
8398 started_ms,
8399 self.clock.now_ms(),
8400 );
8401 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
8402 return Ok(());
8403 };
8404 crate::shared_bytes::convert(rewritten)
8405 }
8406 #[cfg(not(feature = "cryptography"))]
8407 {
8408 let Some(rewritten) =
8409 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
8410 else {
8411 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
8412 return Err(TelemetryError::Io("side tx busy"));
8413 };
8414 let started_ms = self.clock.now_ms();
8415 let frames =
8416 self.encode_side_transport_frames(side, opts, bytes.clone())?;
8417 let mut attempts_total = 0usize;
8418 let mut sent_bytes = 0usize;
8419 self.send_side_frames_resumable(side, frames, |frame| {
8420 match self.retry_with_attempts(runtime_max_handler_retries(), || {
8421 Self::packed_handler_call(&handler, frame, priority)
8422 }) {
8423 Ok((_, attempts)) => {
8424 attempts_total = attempts_total.saturating_add(attempts);
8425 sent_bytes = sent_bytes.saturating_add(frame.len());
8426 }
8427 Err((err, attempts)) => {
8428 self.note_side_tx_failure(
8429 side,
8430 ty,
8431 attempts_total.saturating_add(attempts),
8432 );
8433 return Err(err);
8434 }
8435 }
8436
8437 Ok(())
8438 })?;
8439 self.record_side_tx_sample(
8440 side,
8441 sent_bytes,
8442 started_ms,
8443 self.clock.now_ms(),
8444 );
8445 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
8446 return Ok(());
8447 };
8448 rewritten.into()
8449 }
8450 }
8451 };
8452
8453 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
8454 return Err(TelemetryError::Io("side tx busy"));
8455 };
8456 let started_ms = self.clock.now_ms();
8457 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
8458 let mut attempts_total = 0usize;
8459 let mut sent_bytes = 0usize;
8460 self.send_side_frames_resumable(side, frames, |frame| {
8461 match self.retry_with_attempts(runtime_max_handler_retries(), || {
8462 Self::packed_handler_call(&handler, frame, priority)
8463 }) {
8464 Ok((_, attempts)) => {
8465 attempts_total = attempts_total.saturating_add(attempts);
8466 sent_bytes = sent_bytes.saturating_add(frame.len());
8467 }
8468 Err((err, attempts)) => {
8469 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
8470 return Err(err);
8471 }
8472 }
8473
8474 Ok(())
8475 })?;
8476 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
8477 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
8478
8479 #[cfg(feature = "compact-packet-store")]
8480 {
8481 let _ = bytes.park_for_queue();
8482 }
8483
8484 {
8485 let mut st = self.state.lock();
8486 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
8487 tx_state.sent_order.push_back(seq);
8488 tx_state.sent.insert(
8489 seq,
8490 ReliableSent {
8491 bytes: bytes.clone(),
8492 last_send_ms: self.clock.now_ms(),
8493 retries: 0,
8494 queued: false,
8495 partial_acked: false,
8496 },
8497 );
8498 }
8499
8500 Ok(())
8501 }
8502
8503 fn run_side_chunk_transfer<T>(
8504 &self,
8505 side: RouterSideId,
8506 key: u64,
8507 send: impl FnOnce(&mut usize) -> TelemetryResult<T>,
8508 ) -> TelemetryResult<T> {
8509 let mut next = {
8510 let mut st = self.state.lock();
8511 let transport = st
8512 .side_transport
8513 .get_mut(&side)
8514 .ok_or(TelemetryError::BadArg)?;
8515 if transport.tx_chunk_progress.is_none() {
8516 crate::memory_admission::check(
8517 512,
8518 core::mem::size_of::<crate::side_transport::SideTxProgress>(),
8519 )?;
8520 transport.tx_chunk_progress =
8521 Some(Box::new(crate::side_transport::SideTxProgress::default()));
8522 }
8523 transport
8524 .tx_chunk_progress
8525 .as_mut()
8526 .expect("initialized")
8527 .begin(key, self.clock.now_ms())?
8528 };
8529 let result = send(&mut next);
8530 if let Some(progress) = self
8531 .state
8532 .lock()
8533 .side_transport
8534 .get_mut(&side)
8535 .and_then(|transport| transport.tx_chunk_progress.as_mut())
8536 {
8537 progress.finish(key, next, self.clock.now_ms(), &result);
8538 }
8539 result
8540 }
8541
8542 fn send_side_frames_resumable(
8543 &self,
8544 side: RouterSideId,
8545 mut frames: SideTransportFrames,
8546 mut send: impl FnMut(&[u8]) -> TelemetryResult<()>,
8547 ) -> TelemetryResult<()> {
8548 let bootstraps_template = frames.bootstraps_template();
8549 let result = if frames.len() == 1 {
8550 frames.send_resuming(&mut 0, send)
8551 } else {
8552 let key = frames.progress_key();
8553 self.run_side_chunk_transfer(side, key, |next| frames.send_resuming(next, &mut send))
8554 };
8555 if result.is_err() && bootstraps_template {
8556 if let Some(state) = self.state.lock().side_transport.get_mut(&side) {
8561 state.refresh_transmit_header_templates();
8562 }
8563 }
8564 result
8565 }
8566
8567 fn split_side_transport_frame(
8568 &self,
8569 _side: RouterSideId,
8570 frame: crate::SharedBytes,
8571 max_frame_bytes: usize,
8572 ) -> TelemetryResult<SideTransportFrames> {
8573 SideTransportFrames::split(frame, max_frame_bytes, self.sender_arc().as_bytes())
8574 }
8575
8576 fn encode_side_transport_frames(
8577 &self,
8578 side: RouterSideId,
8579 opts: RouterSideOptions,
8580 raw: crate::SharedBytes,
8581 ) -> TelemetryResult<SideTransportFrames> {
8582 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
8583 return Ok(SideTransportFrames::single(raw));
8584 }
8585
8586 let raw_len = raw.len();
8587 if opts.max_frame_bytes != 0 && raw_len > opts.max_frame_bytes {
8588 let frames = self.split_side_transport_frame(side, raw, opts.max_frame_bytes)?;
8591 let mut st = self.state.lock();
8592 let stats = st.side_runtime_stats.entry(side).or_default();
8593 stats.note_side_transport_full(raw_len, frames.wire_len());
8594 stats.note_side_transport_chunks(frames.len());
8595 return Ok(frames);
8596 }
8597
8598 if opts.header_template_enabled {
8599 let frame = wire_format::peek_routing_frame_info(raw.as_ref())?;
8603 let without_dictionary = opts.max_side_transport_templates == 0 || {
8604 let st = self.state.lock();
8605 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
8606 };
8607 let reliable = is_reliable_type(frame.envelope.ty)
8608 || frame.reliable.is_some()
8609 || frame.envelope.target_senders().len() != 0
8615 || matches!(
8616 frame.envelope.ty,
8617 crate::DataType::ReliableAck
8618 | crate::DataType::ReliablePartialAck
8619 | crate::DataType::ReliablePacketRequest
8620 );
8621 if without_dictionary || reliable {
8622 let mut st = self.state.lock();
8623 if without_dictionary {
8624 let removed = st
8625 .side_transport
8626 .get_mut(&side)
8627 .ok_or(TelemetryError::BadArg)?
8628 .trim_tx_templates_to(0);
8629 for _ in 0..removed {
8630 st.side_runtime_stats
8631 .entry(side)
8632 .or_default()
8633 .note_side_transport_template_eviction();
8634 }
8635 }
8636 st.side_runtime_stats
8637 .entry(side)
8638 .or_default()
8639 .note_side_transport_full(raw_len, raw_len);
8640 return Ok(SideTransportFrames::single(raw));
8641 }
8642 }
8643
8644 #[cfg(any(feature = "std", feature = "fair-admission"))]
8645 if opts.header_template_enabled && crate::memory_admission::supports_work_kinds() {
8646 let (template, ..) = extract_side_header_template_view(raw.as_ref())?;
8647 let cached = self
8648 .state
8649 .lock()
8650 .side_transport
8651 .get(&side)
8652 .is_some_and(|transport| {
8653 transport.tx_template_ids.contains_key(&template.hash)
8654 && transport.tx_templates.contains_key(&template.hash)
8655 });
8656 if !cached
8657 && matches!(
8658 crate::memory_admission::check_receive(raw.as_ref()),
8659 Err(TelemetryError::Io("memory pressure"))
8660 )
8661 {
8662 self.state
8666 .lock()
8667 .side_runtime_stats
8668 .entry(side)
8669 .or_default()
8670 .note_side_transport_full(raw_len, raw_len);
8671 return Ok(SideTransportFrames::single(raw));
8672 }
8673 }
8674
8675 let mut compact_payload_len = None;
8676 let mut used_compact = false;
8677 let used_timestamp_delta = false;
8678 let omitted_timestamp = false;
8679 let wrapped = if opts.header_template_enabled {
8680 let (template, _ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
8681 extract_side_header_template_view(raw.as_ref())?;
8682 let (template_id, use_compact, _previous_timestamp) = {
8685 let mut st = self.state.lock();
8686 let mut effective_template_limit = opts.max_side_transport_templates;
8691 #[cfg(feature = "discovery")]
8692 {
8693 if let Some(route) = st.discovery_routes.get(&side) {
8694 let active_peers = route
8695 .announcers
8696 .values()
8697 .filter(|sender| {
8698 self.clock.now_ms().saturating_sub(sender.last_seen_ms)
8699 <= DISCOVERY_ROUTE_TTL_MS
8700 })
8701 .count();
8702 if active_peers > 1 {
8710 effective_template_limit = 0;
8711 } else {
8712 effective_template_limit = route
8713 .announcers
8714 .values()
8715 .filter_map(|sender| sender.link_capabilities)
8716 .filter(|caps| {
8717 caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
8718 && caps.max_side_transport_templates > 0
8719 })
8720 .map(|caps| caps.max_side_transport_templates as usize)
8721 .fold(effective_template_limit, core::cmp::min);
8722 }
8723 }
8724 }
8725 let trimmed = st
8726 .side_transport
8727 .get_mut(&side)
8728 .ok_or(TelemetryError::BadArg)?
8729 .trim_tx_templates_to(effective_template_limit);
8730 for _ in 0..trimmed {
8731 st.side_runtime_stats
8732 .entry(side)
8733 .or_default()
8734 .note_side_transport_template_eviction();
8735 }
8736 let side_state = st
8737 .side_transport
8738 .get_mut(&side)
8739 .ok_or(TelemetryError::BadArg)?;
8740 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
8741 let previous = side_state.tx_last_timestamps.get(&id).copied();
8742 const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
8745 let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
8746 #[cfg(feature = "discovery")]
8748 let discovery_refresh = discovery::is_discovery_type(_ty);
8749 #[cfg(not(feature = "discovery"))]
8750 let discovery_refresh = false;
8751 let use_compact =
8752 *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES && !discovery_refresh;
8753 if use_compact {
8754 *compact_uses = compact_uses.saturating_add(1);
8755 } else {
8756 *compact_uses = 0;
8757 }
8758 (id, use_compact, previous)
8759 } else {
8760 let template_id = ((template.hash >> 32) as u32 ^ template.hash as u32).max(1);
8766 let evicted = side_state.insert_tx_template(
8767 template.into_owned(),
8768 template_id,
8769 effective_template_limit,
8770 );
8771 if evicted {
8772 st.side_runtime_stats
8773 .entry(side)
8774 .or_default()
8775 .note_side_transport_template_eviction();
8776 }
8777 if let Some(side_state) = st.side_transport.get_mut(&side)
8778 && effective_template_limit > 0
8779 {
8780 side_state.tx_last_timestamps.insert(template_id, timestamp);
8781 }
8782 (template_id, false, None)
8783 }
8784 };
8785 if use_compact {
8786 used_compact = true;
8787 compact_payload_len = Some(payload.len());
8788 let timestamp_field = Some(timestamp);
8791 let mut body = Vec::with_capacity(payload.len() + 32);
8792 body.push(flags);
8793 write_uleb128_local(u64::from(template_id), &mut body);
8794 write_uleb128_local(data_size, &mut body);
8795 if let Some(timestamp_field) = timestamp_field {
8796 write_uleb128_local(timestamp_field, &mut body);
8797 }
8798 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
8799 write_uleb128_local(u64::from(nonce), &mut body);
8800 }
8801 if let Some((seq, ack)) = reliable_seq_ack {
8802 write_uleb128_local(u64::from(seq), &mut body);
8803 write_uleb128_local(u64::from(ack), &mut body);
8804 }
8805 body.extend_from_slice(payload);
8806 {
8807 let mut st = self.state.lock();
8808 if let Some(side_state) = st.side_transport.get_mut(&side) {
8809 side_state.tx_last_timestamps.insert(template_id, timestamp);
8810 }
8811 }
8812 let kind = if omitted_timestamp {
8813 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
8814 } else if used_timestamp_delta {
8815 SIDE_TRANSPORT_KIND_COMPACT_DELTA
8816 } else {
8817 SIDE_TRANSPORT_KIND_COMPACT
8818 };
8819 wrap_side_transport_frame(kind, &body)
8820 } else {
8821 let mut body = Vec::with_capacity(raw.len() + 4);
8822 write_uleb128_local(u64::from(template_id), &mut body);
8823 body.extend_from_slice(raw.as_ref());
8824 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
8825 }
8826 } else {
8827 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
8828 };
8829
8830 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
8831 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
8832 } else {
8833 Ok(SideTransportFrames::single(wrapped))
8834 }?;
8835 let wire_len = frames.wire_len();
8836 let mut st = self.state.lock();
8837 let stats = st.side_runtime_stats.entry(side).or_default();
8838 if used_compact {
8839 let overhead = compact_payload_len
8840 .map(|payload_len| wire_len.saturating_sub(payload_len))
8841 .unwrap_or(wire_len);
8842 stats.note_side_transport_compact(
8843 raw_len,
8844 wire_len,
8845 overhead,
8846 used_timestamp_delta,
8847 omitted_timestamp,
8848 );
8849 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
8850 {
8851 stats.note_side_transport_compact_target_miss();
8852 }
8853 } else {
8854 stats.note_side_transport_full(raw_len, wire_len);
8855 }
8856 if frames.len() > 1 {
8857 stats.note_side_transport_chunks(frames.len());
8858 }
8859 Ok(frames)
8860 }
8861
8862 fn expire_side_chunks(&self) {
8863 let now = self.clock.now_ms();
8864 let mut st = self.state.lock();
8865 for side in st.side_transport.values_mut() {
8866 let side = side.as_mut();
8867 let reclaimed =
8868 SideChunkAssembly::expire(&mut side.rx_chunks, &mut side.rx_chunk_bytes, now);
8869 side.rx_chunk_reclaims = side.rx_chunk_reclaims.saturating_add(reclaimed);
8870 }
8871 }
8872
8873 #[cfg(feature = "discovery")]
8877 fn refresh_ingress_peer_liveness(&self, side: RouterSideId, bytes: &[u8]) {
8878 let Ok(frame) = wire_format::peek_routing_frame_info(bytes) else {
8879 return;
8880 };
8881 let address = frame.envelope.source_address;
8882 self.refresh_peer_liveness(side, address);
8883 }
8884
8885 #[cfg(feature = "discovery")]
8886 #[cfg_attr(not(feature = "std"), inline(never))]
8887 fn refresh_peer_liveness(&self, side: RouterSideId, address: u32) {
8888 let now = self.clock.now_ms();
8889 let mut st = self.state.lock();
8890 let Some(route) = st.discovery_routes.get_mut(&side) else {
8891 return;
8892 };
8893 let mut found = None;
8894 for (name, peer) in &mut route.announcers {
8895 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
8896 if found.is_some() {
8898 return;
8899 }
8900 found = Some(peer);
8901 }
8902 }
8903 let Some(peer) = found else {
8904 return;
8905 };
8906 peer.last_seen_ms = now;
8907 route.last_seen_ms = route.last_seen_ms.max(now);
8908 }
8909
8910 #[cfg(feature = "discovery")]
8914 fn consume_known_packed_topology(
8915 &self,
8916 side: RouterSideId,
8917 bytes: &[u8],
8918 ) -> TelemetryResult<bool> {
8919 let Some((address, payload)) =
8920 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryTopology)?
8921 else {
8922 return Ok(false);
8923 };
8924 let now = self.clock.now_ms();
8925 let mut st = self.state.lock();
8926 let Some(route) = st.discovery_routes.get_mut(&side) else {
8927 return Ok(false);
8928 };
8929 let mut found = None;
8930 for (name, peer) in &mut route.announcers {
8931 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
8932 if found.is_some() {
8933 return Ok(false);
8934 }
8935 found = Some(peer);
8936 }
8937 }
8938 let Some(peer) = found else {
8939 return Ok(false);
8940 };
8941 let Some(full) =
8942 discovery::topology_payload_matches_retained(payload, &peer.topology_boards)
8943 else {
8944 return Ok(false);
8945 };
8946 peer.has_full_topology |= full;
8947 peer.last_seen_ms = now;
8948 route.last_seen_ms = route.last_seen_ms.max(now);
8949 Ok(true)
8950 }
8951
8952 #[cfg(all(feature = "discovery", not(feature = "std")))]
8953 fn consume_known_packed_schema(
8954 &self,
8955 side: RouterSideId,
8956 bytes: &[u8],
8957 ) -> TelemetryResult<bool> {
8958 let Some((address, payload)) =
8959 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoverySchema)?
8960 else {
8961 return Ok(false);
8962 };
8963 if !discovery::discovery_schema_payload_is_fully_known(payload)? {
8964 return Ok(false);
8965 }
8966 let mut st = self.state.lock();
8969 if let Some(route) = st.discovery_routes.get_mut(&side) {
8970 for (name, peer) in &mut route.announcers {
8971 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
8972 peer.has_schema = true;
8973 break;
8974 }
8975 }
8976 }
8977 Ok(true)
8978 }
8979
8980 #[cfg(feature = "discovery")]
8983 fn cached_transit_ready(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<bool> {
8984 if bytes.len() > 128 || bytes.first().is_none_or(|flags| flags & 0x11 != 0) {
8985 return Ok(false);
8986 }
8987 let frame = wire_format::peek_routing_frame_info(bytes)?;
8988 let ty = frame.envelope.ty;
8989 if is_internal_control_type(ty)
8990 || frame.envelope.is_reliable
8991 || frame.reliable.is_some()
8992 || frame.envelope.target_senders().len() != 0
8993 || frame.envelope.wire_shape.is_some_and(|shape| {
8997 shape != crate::message_meta(ty).element
8998 })
8999 || !(1..=3).contains(&frame.envelope.endpoints().take(4).count())
9000 {
9001 return Ok(false);
9002 }
9003 if self.is_managed_variable_type(ty)
9004 || frame.envelope.endpoints().any(|ep| {
9005 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
9006 })
9007 {
9008 return Ok(false);
9009 }
9010
9011 let (template, ..) = extract_side_header_template_view(bytes)?;
9012 let st = self.state.lock();
9013 let active: usize = st.sides.iter().filter(|entry| entry.is_some()).count();
9014 if active != 2 {
9015 return Ok(false);
9016 }
9017 for (id, entry) in st
9018 .sides
9019 .iter()
9020 .enumerate()
9021 .filter_map(|(id, entry)| entry.as_ref().map(|e| (id, e)))
9022 {
9023 if entry.opts.reliable_enabled
9024 || !matches!(
9025 entry.tx_handler,
9026 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
9027 )
9028 || !st
9029 .side_runtime_stats
9030 .get(&id)
9031 .is_some_and(|stats| stats.data_types.contains_key(&ty.as_u32()))
9032 {
9033 return Ok(false);
9034 }
9035 if id == side {
9036 continue;
9037 }
9038 #[cfg(feature = "discovery")]
9039 if !st.discovery_routes.get(&id).is_some_and(|route| {
9040 self.clock.now_ms().saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9041 && frame
9042 .envelope
9043 .endpoints()
9044 .any(|ep| route.reachable.contains(&ep))
9045 }) {
9046 return Ok(false);
9047 }
9048 if entry.opts.header_template_enabled
9049 && entry.opts.max_frame_bytes != 0
9050 && entry.opts.max_side_transport_templates != 0
9051 && !self.side_has_multiple_announcers_locked(&st, id, self.clock.now_ms())
9052 && !st.side_transport.get(&id).is_some_and(|transport| {
9053 transport.tx_template_ids.contains_key(&template.hash)
9054 && transport.tx_templates.contains_key(&template.hash)
9055 })
9056 {
9057 #[cfg(any(feature = "std", feature = "fair-admission"))]
9062 if (entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes)
9063 && st.discovery_routes.get(&side).is_some_and(|route| {
9064 route.announcers.iter().any(|(name, peer)| {
9065 peer.advertised_address == Some(frame.envelope.source_address)
9066 || sender_address_u32(name) == frame.envelope.source_address
9067 })
9068 })
9069 {
9070 continue;
9071 }
9072 return Ok(false);
9073 }
9074 }
9075 Ok(true)
9076 }
9077 #[cfg(not(feature = "discovery"))]
9078 fn cached_transit_ready(&self, _side: RouterSideId, _bytes: &[u8]) -> TelemetryResult<bool> {
9079 Ok(false)
9080 }
9081 #[cfg(any(feature = "std", feature = "fair-admission"))]
9082 fn reclaim_for_receive(&self, bytes: &[u8], victim: &mut Option<(u32, u8)>) -> bool {
9083 let Some((incoming, priority)) = crate::admission_fairness::incoming(bytes) else {
9084 return false;
9085 };
9086 self.reclaim_for_incoming(incoming, priority, victim)
9087 }
9088 #[cfg(any(feature = "std", feature = "fair-admission"))]
9089 fn reclaim_for_incoming(
9090 &self,
9091 incoming: u32,
9092 priority: u8,
9093 victim: &mut Option<(u32, u8)>,
9094 ) -> bool {
9095 let mut st = self.state.lock();
9096 let (owner, priority) = if let Some(selected) = *victim {
9097 selected
9098 } else {
9099 let Some(owner) = st.competing_bulk_owner(incoming, priority) else {
9100 return false;
9101 };
9102 *victim = Some((owner, priority));
9103 (owner, priority)
9104 };
9105 st.reclaim_bulk_from(owner, priority, None)
9106 }
9107 fn admit_received_frame(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<()> {
9108 if !crate::memory_admission::has_probe() {
9109 return Ok(());
9110 }
9111 #[cfg(any(feature = "std", feature = "fair-admission"))]
9112 let mut fair_victim = None;
9113 if crate::memory_admission::supports_work_kinds()
9114 && self.cached_transit_ready(side, bytes)?
9115 {
9116 let result = crate::memory_admission::check_cached_transit(bytes.len());
9117 #[cfg(any(feature = "std", feature = "fair-admission"))]
9118 let result = {
9119 let mut result = result;
9120 while matches!(result, Err(TelemetryError::Io("memory pressure"))) {
9121 if !self.reclaim_for_receive(bytes, &mut fair_victim) {
9122 break;
9123 }
9124 result = crate::memory_admission::check_cached_transit(bytes.len());
9125 }
9126 result
9127 };
9128 result
9129 } else {
9130 let managed = wire_format::scheduling_type(bytes)
9131 .is_some_and(|ty| self.is_managed_variable_type(ty));
9132 let mut protected = None;
9133 loop {
9134 match crate::memory_admission::check_receive_with_role(bytes, managed) {
9135 Err(TelemetryError::Io("memory pressure")) => {
9136 let protected = *protected.get_or_insert_with(|| {
9137 wire_format::peek_routing_frame_info(bytes)
9138 .ok()
9139 .is_some_and(|frame| {
9140 managed
9141 || message_priority(frame.envelope.ty) >= 200
9142 || crate::small_frozen_control(
9143 frame.envelope.ty,
9144 frame.envelope.target_senders().len(),
9145 frame.envelope.wire_shape,
9146 )
9147 })
9148 });
9149 if !protected {
9150 #[cfg(any(feature = "std", feature = "fair-admission"))]
9151 if self.reclaim_for_receive(bytes, &mut fair_victim) {
9152 continue;
9153 }
9154 return Err(TelemetryError::Io("memory pressure"));
9155 }
9156 let mut st = self.state.lock();
9160 let reclaimed = st
9161 .lower_priority_shared_queue(199)
9162 .is_some_and(|queue| st.pop_shared_queue_item(queue));
9163 if !reclaimed {
9164 return Err(TelemetryError::Io("memory pressure"));
9165 }
9166 }
9167 result => return result,
9168 }
9169 }
9170 }
9171 }
9172
9173 fn decode_side_transport_frame(
9174 &self,
9175 side: RouterSideId,
9176 bytes: &[u8],
9177 ) -> TelemetryResult<Option<crate::SharedBytes>> {
9178 self.expire_side_chunks();
9180 let Some((kind, body)) = parse_side_transport_wrapper(bytes)? else {
9181 #[cfg(feature = "discovery")]
9182 self.refresh_ingress_peer_liveness(side, bytes);
9183 #[cfg(all(feature = "discovery", not(feature = "std")))]
9184 if self.consume_known_packed_schema(side, bytes)? {
9185 return Ok(None);
9186 }
9187 #[cfg(feature = "discovery")]
9188 if self.consume_known_packed_topology(side, bytes)? {
9189 return Ok(None);
9190 }
9191 self.admit_received_frame(side, bytes)?;
9192 return Ok(Some(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes))));
9193 };
9194 match kind {
9195 SIDE_TRANSPORT_KIND_FULL => {
9196 let header_templates_enabled = self
9197 .state
9198 .lock()
9199 .sides
9200 .get(side)
9201 .and_then(|entry| entry.as_ref())
9202 .map(|side| side.opts.header_template_enabled)
9203 .unwrap_or(false);
9204 let mut off = 0usize;
9205 let template_id = if header_templates_enabled {
9206 Some(
9207 u32::try_from(read_uleb128_local(body, &mut off)?)
9208 .map_err(|_| TelemetryError::Unpack("side template id too large"))?,
9209 )
9210 } else {
9211 None
9212 };
9213 #[cfg(feature = "discovery")]
9214 self.refresh_ingress_peer_liveness(side, &body[off..]);
9215 #[cfg(all(feature = "discovery", not(feature = "std")))]
9216 if self.consume_known_packed_schema(side, &body[off..])? {
9217 return Ok(None);
9218 }
9219 self.admit_received_frame(side, &body[off..])?;
9220 let raw = Arc::<[u8]>::from(&body[off..]);
9221 if let (Some(template_id), Ok((template, _, _, _, timestamp, _, _, _))) =
9222 (template_id, extract_side_header_template(raw.as_ref()))
9223 {
9224 let mut st = self.state.lock();
9225 let max_templates = st
9226 .sides
9227 .get(side)
9228 .and_then(|side| side.as_ref())
9229 .map(|side| side.opts.max_side_transport_templates)
9230 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
9231 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
9232 let evicted =
9233 side_state.insert_rx_template(template_id, template, max_templates);
9234 if max_templates > 0 {
9235 side_state.rx_last_timestamps.insert(template_id, timestamp);
9236 }
9237 evicted
9238 });
9239 if evicted {
9240 st.side_runtime_stats
9241 .entry(side)
9242 .or_default()
9243 .note_side_transport_template_eviction();
9244 }
9245 }
9246 Ok(Some(crate::shared_bytes::convert(raw)))
9247 }
9248 SIDE_TRANSPORT_KIND_COMPACT
9249 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
9250 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
9251 if body.is_empty() {
9252 return Err(TelemetryError::Unpack("short side compact frame"));
9253 }
9254 let mut off = 1usize;
9255 let template_id = u32::try_from(read_uleb128_local(body, &mut off)?)
9256 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
9257 let (template, timestamp_base) = {
9258 let st = self.state.lock();
9259 let state = st.side_transport.get(&side);
9260 let template = state
9261 .and_then(|state| state.rx_templates_by_id.get(&template_id))
9262 .cloned();
9263 let timestamp_base = if matches!(
9264 kind,
9265 SIDE_TRANSPORT_KIND_COMPACT_DELTA
9266 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
9267 ) {
9268 state
9269 .and_then(|state| state.rx_last_timestamps.get(&template_id))
9270 .copied()
9271 } else {
9272 None
9273 };
9274 (template, timestamp_base)
9275 };
9276 let Some(template) = template else {
9280 return Ok(None);
9281 };
9282 #[cfg(feature = "discovery")]
9285 if let Some(address) = template.source_address() {
9286 self.refresh_peer_liveness(side, address);
9287 }
9288 let reconstructed_bound = body
9289 .len()
9290 .saturating_add(template.prefix().len())
9291 .saturating_add(template.between().len())
9292 .saturating_add(32);
9293 let protected = template.data_type().is_some_and(|ty| {
9297 message_priority(ty) >= 200 || self.is_managed_variable_type(ty)
9298 });
9299 let ack = template.is_ack();
9300 #[cfg(any(feature = "std", feature = "fair-admission"))]
9301 let mut fair_incoming = None;
9302 #[cfg(any(feature = "std", feature = "fair-admission"))]
9303 let mut fair_victim = None;
9304 loop {
9305 match crate::memory_admission::check_compact_reconstruction(
9306 reconstructed_bound,
9307 ack,
9308 ) {
9309 Err(TelemetryError::Io("memory pressure")) if protected => {
9310 let mut st = self.state.lock();
9311 let reclaimed = st
9312 .lower_priority_shared_queue(199)
9313 .is_some_and(|queue| st.pop_shared_queue_item(queue));
9314 if !reclaimed {
9315 return Err(TelemetryError::Io("memory pressure"));
9316 }
9317 }
9318 Err(TelemetryError::Io("memory pressure")) => {
9319 #[cfg(any(feature = "std", feature = "fair-admission"))]
9323 if let Some((incoming, priority)) =
9324 *fair_incoming.get_or_insert_with(|| {
9325 crate::admission_fairness::compact_incoming(&template)
9326 })
9327 && self.reclaim_for_incoming(incoming, priority, &mut fair_victim)
9328 {
9329 continue;
9330 }
9331 return Err(TelemetryError::Io("memory pressure"));
9332 }
9333 result => {
9334 result?;
9335 break;
9336 }
9337 }
9338 }
9339 let timestamp_mode = match kind {
9340 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
9341 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
9342 _ => SideCompactTimestampMode::Absolute,
9343 };
9344 let (frame, timestamp) = reconstruct_side_compact_frame(
9345 &template,
9346 body[0],
9347 &body[off..],
9348 timestamp_mode,
9349 timestamp_base,
9350 )?;
9351 {
9352 let mut st = self.state.lock();
9353 if let Some(side_state) = st.side_transport.get_mut(&side) {
9354 side_state.rx_last_timestamps.insert(template_id, timestamp);
9355 }
9356 }
9357 if crate::memory_admission::supports_work_kinds() {
9361 self.admit_received_frame(side, frame.as_ref())?;
9362 }
9363 Ok(Some(frame))
9364 }
9365 SIDE_TRANSPORT_KIND_CHUNK => {
9366 crate::memory_admission::check_receive(bytes)?;
9367 if body.len() < 8 {
9368 return Err(TelemetryError::Unpack("short side chunk frame"));
9369 }
9370 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
9371 let index = u16::from_le_bytes([body[4], body[5]]);
9372 let total = u16::from_le_bytes([body[6], body[7]]);
9373 if total == 0 || index >= total {
9374 return Err(TelemetryError::Unpack("invalid side chunk index"));
9375 }
9376 let assembled = {
9377 let mut st = self.state.lock();
9378 let now_ms = self.clock.now_ms();
9379 st.make_side_chunk_room(side, transfer_id, index, total, body.len() - 8)?;
9380 let side_state = st
9381 .side_transport
9382 .get_mut(&side)
9383 .ok_or(TelemetryError::BadArg)?;
9384 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
9385 entry.last_seen_ms = now_ms;
9386 entry.total = total;
9387 let added = entry.insert_with(index, || {
9388 crate::shared_bytes::convert(Arc::<[u8]>::from(&body[8..]))
9389 });
9390 side_state.rx_chunk_bytes = side_state.rx_chunk_bytes.saturating_add(added);
9391 if entry.received.len() == usize::from(total) {
9392 let entry = side_state
9393 .take_chunk(transfer_id, false)
9394 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
9395 Some(entry.assemble().inspect_err(|_| {
9396 side_state.rx_chunk_reclaims =
9397 side_state.rx_chunk_reclaims.saturating_add(1);
9398 })?)
9399 } else {
9400 None
9401 }
9402 };
9403 match assembled {
9404 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
9405 None => Ok(None),
9406 }
9407 }
9408 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
9409 }
9410 }
9411
9412 fn packed_handler_call(
9413 handler: &RouterTxHandlerFn,
9414 bytes: &[u8],
9415 priority: u8,
9416 ) -> TelemetryResult<()> {
9417 match handler {
9418 RouterTxHandlerFn::Packed(handler) => handler(bytes),
9419 RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
9420 RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
9421 }
9422 }
9423
9424 fn effective_transport_priority(&self, ty: DataType) -> u8 {
9425 if self.is_managed_variable_type(ty) {
9426 crate::transport_priority(DataType::ManagedVariableValue)
9427 } else {
9428 crate::transport_priority(ty)
9429 }
9430 }
9431
9432 fn call_side_tx_handler(
9433 &self,
9434 side: RouterSideId,
9435 handler: &RouterTxHandlerFn,
9436 data: &RouterItem,
9437 relayed: bool,
9438 ) -> TelemetryResult<()> {
9439 let opts = {
9440 let st = self.state.lock();
9441 Self::side_ref(&st, side)?.opts.clone()
9442 };
9443 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
9444 return Err(TelemetryError::Io("side tx busy"));
9445 };
9446 let started_ms = self.clock.now_ms();
9447 let ty = match data {
9448 RouterItem::Packet(pkt) => pkt.data_type(),
9449 RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
9450 };
9451 let priority = self.item_service_priority(data)?;
9452 let result = match (handler, data) {
9453 (
9454 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
9455 RouterItem::Packed(bytes),
9456 ) => {
9457 #[cfg(feature = "cryptography")]
9458 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
9459 #[cfg(not(feature = "cryptography"))]
9460 let send_bytes = bytes.clone();
9461 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
9462 let mut attempts_total = 0usize;
9463 let mut sent_bytes = 0usize;
9464 self.send_side_frames_resumable(side, frames, |frame| {
9465 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9466 Self::packed_handler_call(packed, frame, priority)
9467 }) {
9468 Ok((_, attempts)) => {
9469 attempts_total = attempts_total.saturating_add(attempts);
9470 sent_bytes = sent_bytes.saturating_add(frame.len());
9471 }
9472 Err((err, attempts)) => {
9473 self.note_side_tx_failure(
9474 side,
9475 ty,
9476 attempts_total.saturating_add(attempts),
9477 );
9478 return Err(err);
9479 }
9480 }
9481
9482 Ok(())
9483 })?;
9484 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9485 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9486 return Ok(());
9487 }
9488 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
9489 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
9490 }
9491 (
9492 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
9493 RouterItem::Packet(pkt),
9494 ) => {
9495 if ty == DataType::DiscoverySchema
9499 && opts.max_frame_bytes != 0
9500 && pkt.payload().len() > opts.max_frame_bytes
9501 {
9502 let mut attempts_total = 0usize;
9503 let mut sent_bytes = 0usize;
9504 let key = crate::packet::hash_bytes_u64(
9505 pkt.packet_id(),
9506 &opts.max_frame_bytes.to_le_bytes(),
9507 );
9508 let result = self.run_side_chunk_transfer(side, key, |next| {
9509 crate::side_transport::send_packet_chunks(
9510 pkt,
9511 opts.max_frame_bytes,
9512 self.sender_arc().as_bytes(),
9513 next,
9514 |frame| match self
9515 .retry_with_attempts(runtime_max_handler_retries(), || {
9516 Self::packed_handler_call(packed, frame, priority)
9517 }) {
9518 Ok((_, attempts)) => {
9519 attempts_total += attempts;
9520 sent_bytes += frame.len();
9521 Ok(())
9522 }
9523 Err((err, attempts)) => {
9524 attempts_total += attempts;
9525 Err(err)
9526 }
9527 },
9528 )
9529 });
9530 let (raw_len, wire_len, chunks) = match result {
9531 Ok(stats) => stats,
9532 Err(err) => {
9533 self.note_side_tx_failure(side, ty, attempts_total);
9534 return Err(err);
9535 }
9536 };
9537 {
9538 let mut st = self.state.lock();
9539 let stats = st.side_runtime_stats.entry(side).or_default();
9540 stats.note_side_transport_full(raw_len, wire_len);
9541 if chunks > 1 {
9542 stats.note_side_transport_chunks(chunks);
9543 }
9544 }
9545 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9546 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9547 return Ok(());
9548 }
9549 let owned = self.pack_packet_for_router(pkt, None)?;
9550 let frames = self.encode_side_transport_frames(side, opts, owned)?;
9551 let mut attempts_total = 0usize;
9552 let mut sent_bytes = 0usize;
9553 self.send_side_frames_resumable(side, frames, |frame| {
9554 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9555 Self::packed_handler_call(packed, frame, priority)
9556 }) {
9557 Ok((_, attempts)) => {
9558 attempts_total = attempts_total.saturating_add(attempts);
9559 sent_bytes = sent_bytes.saturating_add(frame.len());
9560 }
9561 Err((err, attempts)) => {
9562 self.note_side_tx_failure(
9563 side,
9564 ty,
9565 attempts_total.saturating_add(attempts),
9566 );
9567 return Err(err);
9568 }
9569 }
9570
9571 Ok(())
9572 })?;
9573 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9574 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9575 return Ok(());
9576 }
9577 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
9578 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
9579 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
9580 }
9581 };
9582 match result {
9583 Ok((_, attempts)) => {
9584 if let Ok(bytes) = Self::router_item_wire_len(data) {
9585 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
9586 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
9587 }
9588 Ok(())
9589 }
9590 Err((err, attempts)) => {
9591 self.note_side_tx_failure(side, ty, attempts);
9592 Err(err)
9593 }
9594 }
9595 }
9596
9597 fn adjust_reliable_for_side(
9598 &self,
9599 opts: RouterSideOptions,
9600 data: RouterItem,
9601 preserve_end_to_end_ack: bool,
9602 ) -> TelemetryResult<Option<RouterItem>> {
9603 if opts.reliable_enabled {
9604 return Ok(Some(data));
9605 }
9606
9607 match data {
9608 RouterItem::Packed(bytes) => {
9609 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
9610 if is_reliable_type(frame.envelope.ty)
9611 && let Some(hdr) = frame.reliable
9612 {
9613 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
9614 return Ok(None);
9615 }
9616 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
9617 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
9618 bytes.as_ref(),
9619 wire_format::RELIABLE_FLAG_UNSEQUENCED,
9620 hdr.seq,
9621 0,
9622 )?
9623 else {
9624 return Ok(Some(RouterItem::Packed(bytes)));
9625 };
9626 return Ok(Some(RouterItem::Packed(crate::shared_bytes::convert(
9627 rewritten,
9628 ))));
9629 }
9630 }
9631 Ok(Some(RouterItem::Packed(bytes)))
9632 }
9633 RouterItem::Packet(pkt) => {
9634 if matches!(
9635 pkt.data_type(),
9636 DataType::ReliableAck
9637 | DataType::ReliablePartialAck
9638 | DataType::ReliablePacketRequest
9639 ) {
9640 if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
9641 return Ok(Some(RouterItem::Packet(pkt)));
9642 }
9643 return Ok(None);
9644 }
9645 Ok(Some(RouterItem::Packet(pkt)))
9646 }
9647 }
9648 }
9649
9650 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
9651 let now = self.clock.now_ms();
9652 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
9653
9654 {
9655 let mut st = self.state.lock();
9656 if st.reliable_tx.is_empty() {
9657 return Ok(());
9658 }
9659
9660 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
9661 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
9662 continue;
9663 };
9664 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
9665 for seq in sent_order {
9666 let Some(sent) = tx_state.sent.get_mut(&seq) else {
9667 continue;
9668 };
9669 if sent.queued
9670 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
9671 {
9672 continue;
9673 }
9674 if sent.partial_acked {
9675 sent.partial_acked = false;
9682 sent.last_send_ms = now;
9683 continue;
9684 }
9685 if sent.retries >= runtime_reliable_max_retries() {
9686 tx_state.sent.remove(&seq);
9687 tx_state.sent_order.retain(|existing| *existing != seq);
9688 continue;
9689 }
9690 sent.retries += 1;
9691 requeue.push((*side, ty, seq));
9692 }
9693 }
9694 }
9695
9696 for (side, ty, seq) in requeue {
9697 self.queue_reliable_retransmit(side, ty, seq, true)?;
9698 }
9699
9700 Ok(())
9701 }
9702
9703 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
9704 let now = self.clock.now_ms();
9705 let mut requeue = Vec::new();
9706 let mut failed_ordered_types = BTreeSet::new();
9707
9708 {
9709 let mut st = self.state.lock();
9710 #[cfg(feature = "discovery")]
9711 {
9712 if Self::prune_discovery_routes_locked(&mut st, now) {
9713 Self::note_discovery_topology_change_locked(&mut st, now);
9714 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9715 }
9716 }
9721 st.end_to_end_reliable_tx.retain(|&packet_id, sent| {
9724 if sent.waiting_for_predecessor
9725 || sent.queued
9726 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
9727 {
9728 return true;
9729 }
9730 if sent.retries >= runtime_reliable_max_retries() {
9731 if let Some(ty) = sent.ordered_type {
9732 failed_ordered_types.insert(ty);
9733 }
9734 return false;
9735 }
9736 sent.retries += 1;
9737 requeue.push(packet_id);
9738 true
9739 });
9740 st.end_to_end_reliable_tx.retain(|_, sent| {
9742 !sent
9743 .ordered_type
9744 .is_some_and(|ty| failed_ordered_types.contains(&ty))
9745 });
9746 }
9747
9748 for packet_id in requeue {
9749 self.queue_end_to_end_reliable_retransmit(packet_id)?;
9750 }
9751 self.release_ordered_successors()?;
9752 if !failed_ordered_types.is_empty() {
9753 return Err(TelemetryError::HandlerError(
9754 "ordered end-to-end delivery exhausted retries",
9755 ));
9756 }
9757 Ok(())
9758 }
9759
9760 #[cfg(feature = "timesync")]
9761 #[inline]
9762 fn monotonic_now_ns(&self) -> u64 {
9763 self.clock.now_ns()
9764 }
9765
9766 #[cfg(feature = "timesync")]
9767 #[inline]
9768 fn monotonic_now_ms(&self) -> u64 {
9769 self.clock.now_ms()
9770 }
9771
9772 #[cfg(feature = "timesync")]
9773 fn refresh_timesync_state(&self, now_mono_ms: u64) {
9774 let now_mono_ns = self.monotonic_now_ns();
9775 let mut st = self.timesync.lock();
9776 st.clock.prune_expired(now_mono_ms);
9777 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
9778 st.remote_sources
9779 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
9780 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
9781 let leader = if let Some(tracker) = st.tracker.as_mut() {
9782 let _ = tracker.refresh(now_mono_ms);
9783 tracker.leader(now_mono_ms, has_usable_time)
9784 } else {
9785 None
9786 };
9787 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
9788 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
9789 let target_ms = st
9790 .remote_sources
9791 .get(remote.sender.as_str())
9792 .map(|src| src.sample_unix_ms);
9793 if let Some(target_ms) = target_ms {
9794 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
9795 }
9796 }
9797 }
9798
9799 #[cfg(feature = "timesync")]
9800 pub fn update_network_time_source(
9802 &self,
9803 source: &str,
9804 priority: u64,
9805 time: PartialNetworkTime,
9806 ttl_ms: Option<u64>,
9807 ) {
9808 let now_ms = self.monotonic_now_ms();
9809 let now_ns = self.monotonic_now_ns();
9810 let mut st = self.timesync.lock();
9811 st.clock
9812 .update_source(source, priority, time, now_ms, now_ns, ttl_ms);
9813 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
9814 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
9815 }
9816 }
9817
9818 #[cfg(feature = "timesync")]
9819 fn set_network_time_source_impl(
9820 &self,
9821 source: &str,
9822 priority: u64,
9823 time: PartialNetworkTime,
9824 ttl_ms: Option<u64>,
9825 ) {
9826 let observed_mono_ms = self.monotonic_now_ms();
9827 let observed_mono_ns = self.monotonic_now_ns();
9828 let mut st = self.timesync.lock();
9829 let commit_mono_ms = self.monotonic_now_ms();
9830 let commit_mono_ns = self.monotonic_now_ns();
9831 let adjusted = if let Some(base) = time.to_network_time() {
9832 let elapsed_ns = commit_mono_ns.saturating_sub(observed_mono_ns);
9833 advance_network_time(base, elapsed_ns)
9834 .map(PartialNetworkTime::from)
9835 .unwrap_or(time)
9836 } else {
9837 time
9838 };
9839 let adjusted_mono_ms =
9840 observed_mono_ms.saturating_add(commit_mono_ms.saturating_sub(observed_mono_ms));
9841 st.clock.update_source(
9842 source,
9843 priority,
9844 adjusted,
9845 commit_mono_ms.max(adjusted_mono_ms),
9846 commit_mono_ns,
9847 ttl_ms,
9848 );
9849 if let Some(unix_ms) = adjusted.to_network_time().and_then(|t| t.as_unix_ms()) {
9850 st.disciplined_clock.steer_unix_ms(commit_mono_ns, unix_ms);
9851 }
9852 }
9853
9854 #[cfg(feature = "timesync")]
9855 fn local_network_time_priority(&self) -> u64 {
9856 let st = self.timesync.lock();
9857 st.cfg.map(|cfg| cfg.priority).unwrap_or(0)
9858 }
9859
9860 #[cfg(feature = "timesync")]
9861 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
9863 let priority = self.local_network_time_priority();
9864 if time.is_complete_date() && time.is_complete_time() {
9865 self.set_network_time_source_impl(LOCAL_TIMESYNC_FULL_SOURCE_ID, priority, time, None);
9866 let mut st = self.timesync.lock();
9867 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
9868 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
9869 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
9870 return;
9871 }
9872
9873 {
9874 let mut st = self.timesync.lock();
9875 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
9876 }
9877
9878 if time.year.is_some() || time.month.is_some() || time.day.is_some() {
9879 self.set_network_time_source_impl(
9880 LOCAL_TIMESYNC_DATE_SOURCE_ID,
9881 priority,
9882 PartialNetworkTime {
9883 year: time.year,
9884 month: time.month,
9885 day: time.day,
9886 ..Default::default()
9887 },
9888 None,
9889 );
9890 }
9891
9892 if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
9893 self.set_network_time_source_impl(
9894 LOCAL_TIMESYNC_TOD_SOURCE_ID,
9895 priority,
9896 PartialNetworkTime {
9897 hour: time.hour,
9898 minute: time.minute,
9899 second: time.second,
9900 nanosecond: time.nanosecond,
9901 ..Default::default()
9902 },
9903 None,
9904 );
9905 }
9906
9907 if time.nanosecond.is_some() {
9908 self.set_network_time_source_impl(
9909 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
9910 priority,
9911 PartialNetworkTime {
9912 nanosecond: time.nanosecond,
9913 ..Default::default()
9914 },
9915 None,
9916 );
9917 }
9918 }
9919
9920 #[cfg(feature = "timesync")]
9921 pub fn clear_local_network_time(&self) {
9923 let mut st = self.timesync.lock();
9924 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
9925 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
9926 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
9927 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
9928 }
9929
9930 #[cfg(feature = "timesync")]
9931 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
9933 self.set_local_network_time(PartialNetworkTime {
9934 year: Some(year),
9935 month: Some(month),
9936 day: Some(day),
9937 ..Default::default()
9938 });
9939 }
9940
9941 #[cfg(feature = "timesync")]
9942 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
9944 self.set_local_network_time(PartialNetworkTime {
9945 hour: Some(hour),
9946 minute: Some(minute),
9947 ..Default::default()
9948 });
9949 }
9950
9951 #[cfg(feature = "timesync")]
9952 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
9954 self.set_local_network_time(PartialNetworkTime {
9955 hour: Some(hour),
9956 minute: Some(minute),
9957 second: Some(second),
9958 ..Default::default()
9959 });
9960 }
9961
9962 #[cfg(feature = "timesync")]
9963 pub fn set_local_network_time_hms_millis(
9965 &self,
9966 hour: u8,
9967 minute: u8,
9968 second: u8,
9969 millisecond: u16,
9970 ) {
9971 self.set_local_network_time(PartialNetworkTime {
9972 hour: Some(hour),
9973 minute: Some(minute),
9974 second: Some(second),
9975 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
9976 ..Default::default()
9977 });
9978 }
9979
9980 #[cfg(feature = "timesync")]
9981 pub fn set_local_network_time_hms_nanos(
9983 &self,
9984 hour: u8,
9985 minute: u8,
9986 second: u8,
9987 nanosecond: u32,
9988 ) {
9989 self.set_local_network_time(PartialNetworkTime {
9990 hour: Some(hour),
9991 minute: Some(minute),
9992 second: Some(second),
9993 nanosecond: Some(nanosecond),
9994 ..Default::default()
9995 });
9996 }
9997
9998 #[cfg(feature = "timesync")]
9999 pub fn set_local_network_datetime(
10001 &self,
10002 year: i32,
10003 month: u8,
10004 day: u8,
10005 hour: u8,
10006 minute: u8,
10007 second: u8,
10008 ) {
10009 self.set_local_network_time(PartialNetworkTime {
10010 year: Some(year),
10011 month: Some(month),
10012 day: Some(day),
10013 hour: Some(hour),
10014 minute: Some(minute),
10015 second: Some(second),
10016 ..Default::default()
10017 });
10018 }
10019
10020 #[cfg(feature = "timesync")]
10021 #[allow(clippy::too_many_arguments)]
10022 pub fn set_local_network_datetime_millis(
10024 &self,
10025 year: i32,
10026 month: u8,
10027 day: u8,
10028 hour: u8,
10029 minute: u8,
10030 second: u8,
10031 millisecond: u16,
10032 ) {
10033 self.set_local_network_time(PartialNetworkTime {
10034 year: Some(year),
10035 month: Some(month),
10036 day: Some(day),
10037 hour: Some(hour),
10038 minute: Some(minute),
10039 second: Some(second),
10040 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
10041 });
10042 }
10043
10044 #[cfg(feature = "timesync")]
10045 #[allow(clippy::too_many_arguments)]
10046 pub fn set_local_network_datetime_nanos(
10048 &self,
10049 year: i32,
10050 month: u8,
10051 day: u8,
10052 hour: u8,
10053 minute: u8,
10054 second: u8,
10055 nanosecond: u32,
10056 ) {
10057 self.set_local_network_time(PartialNetworkTime {
10058 year: Some(year),
10059 month: Some(month),
10060 day: Some(day),
10061 hour: Some(hour),
10062 minute: Some(minute),
10063 second: Some(second),
10064 nanosecond: Some(nanosecond),
10065 });
10066 }
10067
10068 #[cfg(feature = "timesync")]
10069 pub fn clear_network_time_source(&self, source: &str) {
10071 let mut st = self.timesync.lock();
10072 st.clock.remove_source(source);
10073 }
10074
10075 #[cfg(feature = "timesync")]
10076 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
10078 let mut st = self.timesync.lock();
10079 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
10080 st.cfg = cfg;
10081 st.tracker = cfg.map(TimeSyncTracker::new);
10082 st.disciplined_clock = SlewedNetworkClock::new(
10083 cfg.map(|c| c.max_slew_ppm)
10084 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
10085 );
10086 st.remote_sources.clear();
10087 st.next_seq = 1;
10088 st.next_announce_mono_ms = 0;
10089 st.next_request_mono_ms = 0;
10090 st.pending_request = None;
10091 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
10092 for source in stale_remote_sources {
10093 st.clock.remove_source(&source);
10094 }
10095 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
10096 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
10097 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
10098 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
10099 }
10100
10101 #[cfg(feature = "timesync")]
10102 pub fn network_time(&self) -> Option<NetworkTimeReading> {
10104 let now_ms = self.monotonic_now_ms();
10105 let now_ns = self.monotonic_now_ns();
10106 self.refresh_timesync_state(now_ms);
10107 let st = self.timesync.lock();
10108 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
10109 return Some(NetworkTimeReading {
10110 time: PartialNetworkTime::from_unix_ms(unix_ms),
10111 unix_time_ms: Some(unix_ms),
10112 });
10113 }
10114 st.clock.current_time(now_ns)
10115 }
10116
10117 #[cfg(feature = "timesync")]
10118 pub fn network_time_ms(&self) -> Option<u64> {
10120 self.network_time().and_then(|t| t.unix_time_ms)
10121 }
10122
10123 #[cfg(feature = "timesync")]
10124 fn packet_timestamp_ms(&self) -> u64 {
10125 self.network_time_ms()
10126 .unwrap_or_else(|| self.monotonic_now_ms())
10127 }
10128
10129 #[cfg(not(feature = "timesync"))]
10130 fn packet_timestamp_ms(&self) -> u64 {
10131 self.clock.now_ms()
10132 }
10133
10134 #[cfg(feature = "timesync")]
10135 fn queue_internal_timesync_request(
10136 &self,
10137 seq: u64,
10138 t1_mono_ms: u64,
10139 called_from_queue: bool,
10140 ) -> TelemetryResult<()> {
10141 let pkt_ts = self.packet_timestamp_ms();
10142 if called_from_queue {
10143 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
10144 } else {
10145 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
10146 }
10147 }
10148
10149 #[cfg(feature = "timesync")]
10150 fn queue_internal_timesync_response(
10151 &self,
10152 seq: u64,
10153 t1_mono_ms: u64,
10154 t2_network_ms: u64,
10155 t3_network_ms: u64,
10156 dst: Option<RouterSideId>,
10157 called_from_queue: bool,
10158 ) -> TelemetryResult<()> {
10159 let pkt_ts = self.packet_timestamp_ms();
10160 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
10161 let sender = self.sender_arc();
10162 let pkt = Packet::new(
10163 DataType::TimeSyncResponse,
10164 &[DataEndpoint::TimeSync],
10165 sender.as_ref(),
10166 pkt_ts,
10167 payload,
10168 )?;
10169 match dst {
10170 Some(dst) => self.emit_internal_tx(
10171 RouterTxItem::ToSide {
10172 src: None,
10173 dst,
10174 data: RouterItem::Packet(pkt),
10175 },
10176 true,
10177 called_from_queue,
10178 ),
10179 None => self.emit_internal_tx(
10180 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
10181 true,
10182 called_from_queue,
10183 ),
10184 }
10185 }
10186
10187 #[cfg(feature = "timesync")]
10188 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
10190 let now_ms = self.monotonic_now_ms();
10191 let now_ns = self.monotonic_now_ns();
10192 let mut queued_any = false;
10193 let mut announce_priority = None;
10194 let mut request = None;
10195
10196 {
10197 let mut st = self.timesync.lock();
10198 st.clock.prune_expired(now_ms);
10199 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
10200 st.remote_sources
10201 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
10202 let Some(cfg) = st.cfg else {
10203 return Ok(false);
10204 };
10205
10206 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
10207 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
10208 let _ = tracker.refresh(now_ms);
10209 (
10210 tracker.leader(now_ms, has_usable_time),
10211 tracker.local_announce_priority(now_ms, has_usable_time),
10212 )
10213 } else {
10214 (None, None)
10215 };
10216 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
10217
10218 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
10219 let target_ms = st
10220 .remote_sources
10221 .get(&remote.sender)
10222 .map(|src| src.sample_unix_ms);
10223 if let Some(target_ms) = target_ms {
10224 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
10225 }
10226 }
10227
10228 if let Some(priority) = announce_prio
10229 && now_ms >= st.next_announce_mono_ms
10230 {
10231 announce_priority = Some(priority);
10232 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
10233 }
10234
10235 if let Some(TimeSyncLeader::Remote(remote)) = leader
10236 && now_ms >= st.next_request_mono_ms
10237 && st.pending_request.is_none()
10238 {
10239 let seq = st.next_seq;
10240 let next = st.next_seq.wrapping_add(1);
10241 st.next_seq = if next == 0 { 1 } else { next };
10242 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
10243 st.pending_request = Some(PendingTimeSyncRequest {
10244 seq,
10245 t1_mono_ms: now_ms,
10246 source: remote.sender,
10247 });
10248 request = Some((seq, now_ms));
10249 }
10250 }
10251
10252 if let Some(priority) = announce_priority {
10253 let time_ms = self.packet_timestamp_ms();
10254 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
10255 queued_any = true;
10256 }
10257 if let Some((seq, t1_mono_ms)) = request {
10258 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
10259 queued_any = true;
10260 }
10261
10262 Ok(queued_any)
10263 }
10264
10265 #[cfg(feature = "timesync")]
10266 fn handle_internal_timesync_packet(
10267 &self,
10268 pkt: &Packet,
10269 src: Option<RouterSideId>,
10270 called_from_queue: bool,
10271 ) -> TelemetryResult<bool> {
10272 let Some(cfg) = self.cfg.timesync_config() else {
10273 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
10274 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
10275 }
10276 return Ok(true);
10277 };
10278
10279 let now_mono_ms = self.monotonic_now_ms();
10280 let now_mono_ns = self.monotonic_now_ns();
10281 let mut response = None;
10282 let mut poll_after = false;
10283
10284 {
10285 let mut st = self.timesync.lock();
10286 st.clock.prune_expired(now_mono_ms);
10287 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
10288 st.remote_sources
10289 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
10290 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
10291 if st.tracker.is_none() {
10292 return Ok(true);
10293 }
10294
10295 match pkt.data_type() {
10296 DataType::TimeSyncAnnounce => {
10297 let ann = decode_timesync_announce(pkt)?;
10298 let should_steer = {
10299 let tracker = st.tracker.as_mut().expect("tracker checked above");
10300 let _ = tracker.handle_announce(pkt, now_mono_ms)?;
10301 matches!(
10302 tracker.leader(now_mono_ms, has_usable_time),
10303 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender()
10304 )
10305 };
10306 st.remote_sources.insert(
10307 pkt.sender().to_owned(),
10308 RemoteTimeSyncSource {
10309 priority: ann.priority,
10310 last_sample_mono_ms: now_mono_ms,
10311 sample_unix_ms: ann.time_ms,
10312 },
10313 );
10314 st.clock.update_source(
10315 pkt.sender(),
10316 ann.priority,
10317 PartialNetworkTime::from_unix_ms(ann.time_ms),
10318 now_mono_ms,
10319 now_mono_ns,
10320 Some(cfg.source_timeout_ms),
10321 );
10322 if should_steer {
10323 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
10324 }
10325 poll_after = true;
10326 }
10327 DataType::TimeSyncRequest => {
10328 let should_serve = {
10329 let tracker = st.tracker.as_ref().expect("tracker checked above");
10330 tracker.should_serve(now_mono_ms, has_usable_time)
10331 };
10332 if should_serve {
10333 let req = decode_timesync_request(pkt)?;
10334 let network_now = st
10335 .disciplined_clock
10336 .read_unix_ms(now_mono_ns)
10337 .or_else(|| {
10338 st.clock
10339 .current_time(now_mono_ns)
10340 .and_then(|t| t.unix_time_ms)
10341 })
10342 .unwrap_or(now_mono_ms);
10343 let t2 = network_now;
10344 let t3 = network_now;
10345 response = Some((req.seq, req.t1_ms, t2, t3, src));
10346 }
10347 }
10348 DataType::TimeSyncResponse => {
10349 let resp = decode_timesync_response(pkt)?;
10350 let pending = st.pending_request.clone();
10351 if let Some(pending) = pending
10352 && pending.seq == resp.seq
10353 && pending.source == pkt.sender()
10354 {
10355 let source_priority = {
10356 let tracker = st.tracker.as_ref().expect("tracker checked above");
10357 tracker
10358 .best_active_source(now_mono_ms)
10359 .map(|s| s.priority)
10360 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
10361 .unwrap_or(cfg.priority)
10362 };
10363 let (estimate_ms, _delay_ms) = compute_network_time_sample(
10364 pending.t1_mono_ms,
10365 resp.t2_ms,
10366 resp.t3_ms,
10367 now_mono_ms,
10368 );
10369 st.remote_sources.insert(
10370 pkt.sender().to_owned(),
10371 RemoteTimeSyncSource {
10372 priority: source_priority,
10373 last_sample_mono_ms: now_mono_ms,
10374 sample_unix_ms: estimate_ms,
10375 },
10376 );
10377 st.clock.update_source(
10378 pkt.sender(),
10379 source_priority,
10380 PartialNetworkTime::from_unix_ms(estimate_ms),
10381 now_mono_ms,
10382 now_mono_ns,
10383 Some(cfg.source_timeout_ms),
10384 );
10385 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
10386 st.pending_request = None;
10387 }
10388 }
10389 _ => {}
10390 }
10391 }
10392
10393 if let Some((seq, t1, t2, t3, dst)) = response {
10394 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
10395 }
10396 if poll_after {
10397 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
10398 }
10399
10400 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
10401 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
10402 }
10403
10404 Ok(true)
10405 }
10406
10407 #[cfg(feature = "std")]
10409 pub fn new(cfg: RouterConfig) -> Self {
10410 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
10411 }
10412
10413 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
10415 #[cfg(feature = "timesync")]
10416 let timesync_cfg = cfg.timesync_config();
10417 #[cfg(feature = "discovery")]
10418 let preferred_discovery_master = cfg.preferred_discovery_master.clone();
10419 let memory = cfg.memory_config();
10420 let hostname: Arc<str> = Arc::from(cfg.sender());
10421 let address_mode = cfg.address_mode();
10422 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
10423 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
10424 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
10425 let requested = address_mode.requested_address();
10426 let address = if requested == 0 { fallback } else { requested };
10427 let local_address = AddressBookEntry {
10428 hostname: hostname.clone(),
10429 address,
10430 requested_address: requested,
10431 mode: address_mode,
10432 birth_ms: clock
10433 .now_ms()
10434 .saturating_mul(1_000_000)
10435 .saturating_add(instance_seq),
10436 owner_hash,
10437 last_seen_ms: clock.now_ms(),
10438 };
10439 let mut address_book = BTreeMap::new();
10440 address_book.insert(hostname.to_string(), local_address.clone());
10441 let mut address_by_value = BTreeMap::new();
10442 address_by_value.insert(address, hostname.to_string());
10443 Self {
10444 sender: RouterMutex::new(hostname),
10445 cfg,
10446 state: RouterMutex::new(RouterInner {
10447 memory,
10448 sides: Vec::new(),
10449 route_overrides: BTreeMap::new(),
10450 typed_route_overrides: BTreeMap::new(),
10451 route_weights: BTreeMap::new(),
10452 route_priorities: BTreeMap::new(),
10453 source_route_modes: BTreeMap::new(),
10454 route_selection_cursors: BTreeMap::new(),
10455 adaptive_route_stats: BTreeMap::new(),
10456 side_runtime_stats: BTreeMap::new(),
10457 side_transport: BTreeMap::new(),
10458 managed_variable_types: BTreeSet::new(),
10459 managed_variable_permissions: BTreeMap::new(),
10460 managed_variable_latest: BTreeMap::new(),
10461 network_variable_update_handlers: BTreeMap::new(),
10462 local_address,
10463 address_book,
10464 address_by_value,
10465 p2p_port_handlers: BTreeMap::new(),
10466 p2p_stream_handlers: BTreeMap::new(),
10467 p2p_stream_sessions: BTreeMap::new(),
10468 next_p2p_stream_id: 1,
10469 received_queue: BoundedDeque::new(
10470 memory.max_queue_budget,
10471 memory.starting_queue_size,
10472 memory.queue_grow_step,
10473 ),
10474 transmit_queue: BoundedDeque::new(
10475 memory.max_queue_budget,
10476 memory.starting_queue_size,
10477 memory.queue_grow_step,
10478 ),
10479 tx_priority_burst: 0,
10480 recent_rx: BoundedDeque::new(
10481 memory.recent_rx_queue_bytes(),
10482 memory.recent_rx_queue_bytes(),
10483 memory.queue_grow_step,
10484 ),
10485 reliable_tx: BTreeMap::new(),
10486 reliable_rx: BTreeMap::new(),
10487 reliable_return_routes: BTreeMap::new(),
10488 reliable_return_route_order: VecDeque::new(),
10489 end_to_end_reliable_tx: BTreeMap::new(),
10490 end_to_end_reliable_tx_order: VecDeque::new(),
10491 total_handler_failures: 0,
10492 total_handler_retries: 0,
10493 #[cfg(feature = "discovery")]
10494 discovery_routes: BTreeMap::new(),
10495 #[cfg(feature = "discovery")]
10496 discovery_cadence: DiscoveryCadenceState::default(),
10497 #[cfg(feature = "discovery")]
10498 discovery_side_throttle: BTreeMap::new(),
10499 #[cfg(feature = "discovery")]
10500 preferred_discovery_master,
10501 #[cfg(all(feature = "discovery", feature = "timesync"))]
10502 timesync_side_throttle: BTreeMap::new(),
10503 }),
10504 isr_rx_queue: IsrRxQueue::new(
10505 memory.max_queue_budget,
10506 memory.starting_queue_size,
10507 memory.queue_grow_step,
10508 ),
10509 side_tx_gate: ReentryGate::new(),
10510 clock,
10511 #[cfg(feature = "timesync")]
10512 timesync: RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)),
10513 }
10514 }
10515
10516 #[inline]
10517 fn sender_arc(&self) -> Arc<str> {
10518 self.sender.lock().clone()
10519 }
10520
10521 #[inline]
10522 pub fn sender(&self) -> Arc<str> {
10523 self.sender_arc()
10524 }
10525
10526 pub fn current_address(&self) -> NodeAddress {
10527 self.state.lock().local_address.address
10528 }
10529
10530 pub fn hostname(&self) -> Arc<str> {
10531 self.sender_arc()
10532 }
10533
10534 pub fn set_preferred_discovery_master(&self, hostname: Option<&str>) {
10539 #[cfg(feature = "discovery")]
10540 {
10541 let preferred = hostname
10542 .filter(|value| !value.is_empty())
10543 .map(Arc::<str>::from);
10544 let mut st = self.state.lock();
10545 if st.preferred_discovery_master != preferred {
10546 st.preferred_discovery_master = preferred;
10547 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
10548 }
10549 }
10550 #[cfg(not(feature = "discovery"))]
10551 let _ = hostname;
10552 }
10553
10554 pub fn preferred_discovery_master(&self) -> Option<Arc<str>> {
10556 #[cfg(feature = "discovery")]
10557 {
10558 return self.state.lock().preferred_discovery_master.clone();
10559 }
10560 #[cfg(not(feature = "discovery"))]
10561 None
10562 }
10563
10564 pub fn address_book(&self) -> Vec<AddressBookEntry> {
10565 self.state.lock().address_book.values().cloned().collect()
10566 }
10567
10568 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
10569 let now_ms = self.clock.now_ms();
10570 #[allow(unused_mut)]
10571 let mut st = self.state.lock();
10572 #[cfg(feature = "discovery")]
10573 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
10574 Self::note_discovery_topology_change_locked(&mut st, now_ms);
10575 }
10576 if let Some(entry) = st.address_book.get(hostname) {
10577 return Some(entry.clone());
10578 }
10579 let address = sender_address_u32(hostname);
10585 if Self::hostname_for_address_locked(&st, address).as_deref() != Some(hostname) {
10586 return None;
10587 }
10588 Some(AddressBookEntry {
10589 hostname: Arc::from(hostname),
10590 address,
10591 requested_address: 0,
10592 mode: AddressAssignmentMode::Dynamic,
10593 birth_ms: 0,
10594 owner_hash: Self::sender_hash(hostname),
10595 last_seen_ms: now_ms,
10596 })
10597 }
10598
10599 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
10600 let st = self.state.lock();
10601 let direct = st
10602 .address_by_value
10603 .get(&address)
10604 .and_then(|hostname| st.address_book.get(hostname))
10605 .cloned();
10606 if direct.is_some() {
10607 return direct;
10608 }
10609
10610 let hostname = Self::hostname_for_address_locked(&st, address)?;
10611 if let Some(entry) = st.address_book.get(&hostname) {
10612 return Some(entry.clone());
10613 }
10614
10615 let hostname: Arc<str> = Arc::from(hostname);
10620 Some(AddressBookEntry {
10621 hostname: hostname.clone(),
10622 address,
10623 requested_address: 0,
10624 mode: AddressAssignmentMode::Dynamic,
10625 birth_ms: 0,
10626 owner_hash: Self::sender_hash(hostname.as_ref()),
10627 last_seen_ms: self.clock.now_ms(),
10628 })
10629 }
10630
10631 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
10632 where
10633 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
10634 {
10635 if port == 0 {
10636 return Err(TelemetryError::BadArg);
10637 }
10638 let mut st = self.state.lock();
10639 st.p2p_port_handlers
10640 .entry(port)
10641 .or_default()
10642 .push(P2pPortHandler {
10643 handler: Arc::new(f),
10644 });
10645 Ok(())
10646 }
10647
10648 pub fn clear_p2p_port(&self, port: P2pPort) {
10649 self.state.lock().p2p_port_handlers.remove(&port);
10650 }
10651
10652 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
10653 where
10654 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
10655 {
10656 if port == 0 {
10657 return Err(TelemetryError::BadArg);
10658 }
10659 let mut st = self.state.lock();
10660 st.p2p_stream_handlers
10661 .entry(port)
10662 .or_default()
10663 .push(P2pStreamHandler {
10664 handler: Arc::new(f),
10665 });
10666 Ok(())
10667 }
10668
10669 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
10670 self.state.lock().p2p_stream_handlers.remove(&port);
10671 }
10672
10673 fn send_p2p_to_entry(
10674 &self,
10675 dst: AddressBookEntry,
10676 dst_port: P2pPort,
10677 src_port: P2pPort,
10678 payload: &[u8],
10679 ) -> TelemetryResult<()> {
10680 if dst_port == 0 {
10681 return Err(TelemetryError::BadArg);
10682 }
10683 let local = self.state.lock().local_address.clone();
10684 let payload = Self::encode_p2p_payload(
10685 local.hostname.as_ref(),
10686 local.address,
10687 src_port,
10688 dst_port,
10689 payload,
10690 )?;
10691 let pkt = Packet::new(
10692 DataType::P2pMessage,
10693 &[DataEndpoint::Discovery],
10694 local.hostname.as_ref(),
10695 self.packet_timestamp_ms(),
10696 crate::shared_bytes::convert(payload),
10697 )?;
10698 let target = Self::sender_hash(dst.hostname.as_ref());
10699 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
10700 if target == Self::sender_hash(self.sender_arc().as_ref()) {
10701 self.dispatch_p2p_packet(&pkt)?;
10702 return Ok(());
10703 }
10704 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
10705 }
10706
10707 pub fn send_p2p_to_hostname(
10708 &self,
10709 hostname: &str,
10710 dst_port: P2pPort,
10711 src_port: P2pPort,
10712 payload: &[u8],
10713 ) -> TelemetryResult<()> {
10714 let dst = self
10715 .resolve_hostname(hostname)
10716 .ok_or(TelemetryError::BadArg)?;
10717 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
10718 }
10719
10720 pub fn send_p2p_to_address(
10721 &self,
10722 address: NodeAddress,
10723 dst_port: P2pPort,
10724 src_port: P2pPort,
10725 payload: &[u8],
10726 ) -> TelemetryResult<()> {
10727 let dst = self
10728 .resolve_address(address)
10729 .ok_or(TelemetryError::BadArg)?;
10730 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
10731 }
10732
10733 fn open_p2p_stream_to_entry(
10734 &self,
10735 dst: AddressBookEntry,
10736 dst_port: P2pPort,
10737 src_port: P2pPort,
10738 ) -> TelemetryResult<P2pStreamId> {
10739 if dst_port == 0 || src_port == 0 {
10740 return Err(TelemetryError::BadArg);
10741 }
10742 let stream_id = {
10743 let mut st = self.state.lock();
10744 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
10745 if stream_id == 0 {
10746 return Err(TelemetryError::Io("p2p stream id exhausted"));
10747 }
10748 st.p2p_stream_sessions.insert(
10749 stream_id,
10750 P2pStreamSession {
10751 peer_hostname: dst.hostname.clone(),
10752 peer_address: dst.address,
10753 local_port: src_port,
10754 peer_port: dst_port,
10755 peer_stream_id: 0,
10756 next_sequence: 1,
10757 connected: false,
10758 },
10759 );
10760 stream_id
10761 };
10762 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
10763 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
10764 Ok(stream_id)
10765 }
10766
10767 pub fn open_p2p_stream_to_hostname(
10768 &self,
10769 hostname: &str,
10770 dst_port: P2pPort,
10771 src_port: P2pPort,
10772 ) -> TelemetryResult<P2pStreamId> {
10773 let dst = self
10774 .resolve_hostname(hostname)
10775 .ok_or(TelemetryError::BadArg)?;
10776 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
10777 }
10778
10779 pub fn open_p2p_stream_to_address(
10780 &self,
10781 address: NodeAddress,
10782 dst_port: P2pPort,
10783 src_port: P2pPort,
10784 ) -> TelemetryResult<P2pStreamId> {
10785 let dst = self
10786 .resolve_address(address)
10787 .ok_or(TelemetryError::BadArg)?;
10788 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
10789 }
10790
10791 fn send_p2p_stream_control(
10792 &self,
10793 stream_id: P2pStreamId,
10794 flags: u8,
10795 payload: &[u8],
10796 ) -> TelemetryResult<()> {
10797 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
10798 let mut st = self.state.lock();
10799 let Some(session) = st.p2p_stream_sessions.get_mut(&stream_id) else {
10800 return Err(TelemetryError::BadArg);
10801 };
10802 let peer_hostname = session.peer_hostname.clone();
10803 let peer_address = session.peer_address;
10804 let peer_port = session.peer_port;
10805 let local_port = session.local_port;
10806 let peer_stream_id = session.peer_stream_id;
10807 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
10808 let seq = session.next_sequence;
10809 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
10810 let dst = st
10811 .address_book
10812 .get(peer_hostname.as_ref())
10813 .cloned()
10814 .unwrap_or(AddressBookEntry {
10815 hostname: peer_hostname.clone(),
10816 address: peer_address,
10817 requested_address: peer_address,
10818 mode: AddressAssignmentMode::Dynamic,
10819 birth_ms: self.clock.now_ms(),
10820 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
10821 last_seen_ms: self.clock.now_ms(),
10822 });
10823 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
10824 };
10825 if refresh_syn {
10826 let syn_payload =
10827 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
10828 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
10829 }
10830 let stream_payload =
10831 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
10832 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
10833 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
10834 self.state.lock().p2p_stream_sessions.remove(&stream_id);
10835 }
10836 Ok(())
10837 }
10838
10839 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
10840 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
10841 }
10842
10843 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
10844 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
10845 }
10846
10847 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
10848 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
10849 }
10850
10851 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
10852 let hostname: Arc<str> = Arc::from(sender.as_ref());
10853 let mut st = self.state.lock();
10854 let mut local = st.local_address.clone();
10855 local.hostname = hostname.clone();
10856 local.owner_hash = Self::sender_hash(hostname.as_ref());
10857 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
10858 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
10859 local.requested_address = 0;
10860 }
10861 let change =
10862 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
10863 drop(st);
10864 let _ = self.notify_address_change(change);
10865 }
10866
10867 pub fn set_address_assignment(
10868 &self,
10869 mode: AddressAssignmentMode,
10870 ) -> TelemetryResult<AddressChange> {
10871 let mut st = self.state.lock();
10872 let mut local = st.local_address.clone();
10873 local.mode = mode;
10874 local.requested_address = mode.requested_address();
10875 local.address = match mode {
10876 AddressAssignmentMode::Dynamic => {
10877 Self::fallback_address_for_hostname(local.hostname.as_ref())
10878 }
10879 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
10880 if address == 0 {
10881 1
10882 } else {
10883 address
10884 }
10885 }
10886 };
10887 let change =
10888 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
10889 drop(st);
10890 self.notify_address_change(change.clone())?;
10891 Ok(change)
10892 }
10893
10894 pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
10903 where
10904 N: AsRef<str>,
10905 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
10906 {
10907 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
10908 }
10909
10910 pub fn add_side_packed_small_packets<N, F>(
10914 &self,
10915 name: N,
10916 tx: F,
10917 max_frame_bytes: usize,
10918 ) -> RouterSideId
10919 where
10920 N: AsRef<str>,
10921 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
10922 {
10923 self.add_side_packed_with_options(
10924 name,
10925 tx,
10926 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
10927 )
10928 }
10929
10930 pub fn add_side_packed_with_options<N, F>(
10939 &self,
10940 name: N,
10941 tx: F,
10942 opts: RouterSideOptions,
10943 ) -> RouterSideId
10944 where
10945 N: AsRef<str>,
10946 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
10947 {
10948 let mut st = self.state.lock();
10949 let side = Some(RouterSide {
10950 name: Arc::from(name.as_ref()),
10951 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
10952 opts,
10953 });
10954 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
10955 st.sides[id] = side;
10956 id
10957 } else {
10958 let id = st.sides.len();
10959 st.sides.push(side);
10960 id
10961 };
10962 st.side_runtime_stats
10963 .insert(id, Box::<SideRuntimeStatsInner>::default());
10964 st.side_transport
10965 .insert(id, Box::<SideTransportState>::default());
10966 #[cfg(feature = "discovery")]
10967 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
10968 id
10969 }
10970
10971 pub fn add_side_packed_with_priority_and_options<N, F>(
10980 &self,
10981 name: N,
10982 tx: F,
10983 opts: RouterSideOptions,
10984 ) -> RouterSideId
10985 where
10986 N: AsRef<str>,
10987 F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
10988 {
10989 let mut st = self.state.lock();
10990 let side = Some(RouterSide {
10991 name: Arc::from(name.as_ref()),
10992 tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
10993 opts,
10994 });
10995 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
10996 st.sides[id] = side;
10997 id
10998 } else {
10999 let id = st.sides.len();
11000 st.sides.push(side);
11001 id
11002 };
11003 st.side_runtime_stats
11004 .insert(id, Box::<SideRuntimeStatsInner>::default());
11005 st.side_transport
11006 .insert(id, Box::<SideTransportState>::default());
11007 #[cfg(feature = "discovery")]
11008 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11009 id
11010 }
11011
11012 pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
11017 where
11018 N: AsRef<str>,
11019 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
11020 {
11021 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
11022 }
11023
11024 pub fn add_side_packet_with_options<N, F>(
11030 &self,
11031 name: N,
11032 tx: F,
11033 opts: RouterSideOptions,
11034 ) -> RouterSideId
11035 where
11036 N: AsRef<str>,
11037 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
11038 {
11039 let mut st = self.state.lock();
11040 let side = Some(RouterSide {
11041 name: Arc::from(name.as_ref()),
11042 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
11043 opts,
11044 });
11045 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
11046 st.sides[id] = side;
11047 id
11048 } else {
11049 let id = st.sides.len();
11050 st.sides.push(side);
11051 id
11052 };
11053 st.side_runtime_stats
11054 .insert(id, Box::<SideRuntimeStatsInner>::default());
11055 st.side_transport
11056 .insert(id, Box::<SideTransportState>::default());
11057 #[cfg(feature = "discovery")]
11058 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11059 id
11060 }
11061
11062 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
11067 let now_ms = self.clock.now_ms();
11068 {
11069 let mut st = self.state.lock();
11070 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
11071 if slot.is_none() {
11072 return Err(TelemetryError::BadArg);
11073 }
11074 *slot = None;
11075 while st.sides.last().is_some_and(Option::is_none) {
11076 st.sides.pop();
11077 }
11078 st.route_overrides
11083 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
11084 st.typed_route_overrides
11085 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
11086 st.route_weights
11087 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
11088 st.route_priorities
11089 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
11090 st.source_route_modes.remove(&Some(side));
11091 st.route_selection_cursors.remove(&Some(side));
11092 st.adaptive_route_stats.remove(&side);
11093 #[cfg(feature = "discovery")]
11094 st.discovery_side_throttle.remove(&side);
11095 #[cfg(all(feature = "discovery", feature = "timesync"))]
11096 st.timesync_side_throttle.remove(&side);
11097 st.side_runtime_stats.remove(&side);
11098 st.side_transport.remove(&side);
11099 st.reliable_return_routes
11100 .retain(|_, route| route.side != side);
11101 st.transmit_queue.retain(
11102 |queued| {
11103 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
11104 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
11105 },
11106 );
11107 st.received_queue.retain(|queued| queued.src != Some(side));
11108 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
11109 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
11110 #[cfg(feature = "discovery")]
11111 {
11112 st.discovery_routes.remove(&side);
11113 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11114 }
11115 }
11116 let mut isr_rx = self.isr_rx_queue.try_lock()?;
11117 isr_rx.retain(|queued| queued.src != Some(side));
11118 Ok(())
11119 }
11120
11121 pub fn set_side_ingress_enabled(
11126 &self,
11127 side: RouterSideId,
11128 enabled: bool,
11129 ) -> TelemetryResult<()> {
11130 let now_ms = self.clock.now_ms();
11131 let mut st = self.state.lock();
11132 let side_ref = st
11133 .sides
11134 .get_mut(side)
11135 .and_then(|side| side.as_mut())
11136 .ok_or(TelemetryError::BadArg)?;
11137 side_ref.opts.ingress_enabled = enabled;
11138 #[cfg(feature = "discovery")]
11139 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11140 Ok(())
11141 }
11142
11143 pub fn set_side_egress_enabled(
11147 &self,
11148 side: RouterSideId,
11149 enabled: bool,
11150 ) -> TelemetryResult<()> {
11151 let now_ms = self.clock.now_ms();
11152 let mut st = self.state.lock();
11153 let side_ref = st
11154 .sides
11155 .get_mut(side)
11156 .and_then(|side| side.as_mut())
11157 .ok_or(TelemetryError::BadArg)?;
11158 side_ref.opts.egress_enabled = enabled;
11159 if !enabled {
11160 st.transmit_queue.retain(
11161 |queued| {
11162 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
11163 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
11164 },
11165 );
11166 }
11167 #[cfg(feature = "discovery")]
11168 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11169 Ok(())
11170 }
11171
11172 pub fn set_source_route_mode(
11178 &self,
11179 src: Option<RouterSideId>,
11180 mode: RouteSelectionMode,
11181 ) -> TelemetryResult<()> {
11182 let now_ms = self.clock.now_ms();
11183 let mut st = self.state.lock();
11184 if let Some(src) = src {
11185 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11186 }
11187 st.source_route_modes.insert(src, mode);
11191 st.route_selection_cursors.remove(&src);
11192 #[cfg(feature = "discovery")]
11193 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11194 Ok(())
11195 }
11196
11197 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
11199 let now_ms = self.clock.now_ms();
11200 let mut st = self.state.lock();
11201 if let Some(src) = src {
11202 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11203 }
11204 st.source_route_modes.remove(&src);
11205 st.route_selection_cursors.remove(&src);
11206 #[cfg(feature = "discovery")]
11207 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11208 Ok(())
11209 }
11210
11211 pub fn set_route_weight(
11216 &self,
11217 src: Option<RouterSideId>,
11218 dst: RouterSideId,
11219 weight: u32,
11220 ) -> TelemetryResult<()> {
11221 let now_ms = self.clock.now_ms();
11222 let mut st = self.state.lock();
11223 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11224 if let Some(src) = src {
11225 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11226 }
11227 st.route_weights.insert((src, dst), weight);
11228 st.route_selection_cursors.remove(&src);
11229 #[cfg(feature = "discovery")]
11230 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11231 Ok(())
11232 }
11233
11234 pub fn clear_route_weight(
11236 &self,
11237 src: Option<RouterSideId>,
11238 dst: RouterSideId,
11239 ) -> TelemetryResult<()> {
11240 let now_ms = self.clock.now_ms();
11241 let mut st = self.state.lock();
11242 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11243 if let Some(src) = src {
11244 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11245 }
11246 st.route_weights.remove(&(src, dst));
11247 st.route_selection_cursors.remove(&src);
11248 #[cfg(feature = "discovery")]
11249 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11250 Ok(())
11251 }
11252
11253 pub fn set_route_priority(
11258 &self,
11259 src: Option<RouterSideId>,
11260 dst: RouterSideId,
11261 priority: u32,
11262 ) -> TelemetryResult<()> {
11263 let now_ms = self.clock.now_ms();
11264 let mut st = self.state.lock();
11265 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11266 if let Some(src) = src {
11267 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11268 }
11269 st.route_priorities.insert((src, dst), priority);
11270 #[cfg(feature = "discovery")]
11271 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11272 Ok(())
11273 }
11274
11275 pub fn clear_route_priority(
11277 &self,
11278 src: Option<RouterSideId>,
11279 dst: RouterSideId,
11280 ) -> TelemetryResult<()> {
11281 let now_ms = self.clock.now_ms();
11282 let mut st = self.state.lock();
11283 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11284 if let Some(src) = src {
11285 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11286 }
11287 st.route_priorities.remove(&(src, dst));
11288 #[cfg(feature = "discovery")]
11289 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11290 Ok(())
11291 }
11292
11293 pub fn set_route(
11298 &self,
11299 src: Option<RouterSideId>,
11300 dst: RouterSideId,
11301 enabled: bool,
11302 ) -> TelemetryResult<()> {
11303 let now_ms = self.clock.now_ms();
11304 let mut st = self.state.lock();
11305 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11306 if let Some(src) = src {
11307 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11308 }
11309 st.route_overrides.insert((src, dst), enabled);
11310 #[cfg(feature = "discovery")]
11311 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11312 Ok(())
11313 }
11314
11315 pub fn set_typed_route(
11320 &self,
11321 src: Option<RouterSideId>,
11322 ty: DataType,
11323 dst: RouterSideId,
11324 enabled: bool,
11325 ) -> TelemetryResult<()> {
11326 let now_ms = self.clock.now_ms();
11327 let mut st = self.state.lock();
11328 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11329 if let Some(src) = src {
11330 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11331 }
11332 st.typed_route_overrides
11333 .insert((src, ty.as_u32(), dst), enabled);
11334 #[cfg(feature = "discovery")]
11335 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11336 Ok(())
11337 }
11338
11339 pub fn clear_typed_route(
11341 &self,
11342 src: Option<RouterSideId>,
11343 ty: DataType,
11344 dst: RouterSideId,
11345 ) -> TelemetryResult<()> {
11346 let now_ms = self.clock.now_ms();
11347 let mut st = self.state.lock();
11348 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11349 if let Some(src) = src {
11350 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11351 }
11352 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
11353 #[cfg(feature = "discovery")]
11354 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11355 Ok(())
11356 }
11357
11358 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
11360 let now_ms = self.clock.now_ms();
11361 let mut st = self.state.lock();
11362 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
11363 if let Some(src) = src {
11364 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
11365 }
11366 st.route_overrides.remove(&(src, dst));
11367 #[cfg(feature = "discovery")]
11368 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11369 Ok(())
11370 }
11371
11372 #[cfg(feature = "discovery")]
11374 pub fn announce_discovery(&self) -> TelemetryResult<()> {
11375 self.queue_discovery_announce(true, false)
11376 }
11377
11378 #[cfg(feature = "discovery")]
11380 pub fn announce_leave(&self) -> TelemetryResult<()> {
11381 let sender = self.sender_arc();
11382 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
11383 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11384 }
11385
11386 #[cfg(feature = "discovery")]
11388 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
11389 self.poll_discovery_announce()
11390 }
11391
11392 #[cfg(feature = "discovery")]
11393 pub fn request_topology(&self) -> TelemetryResult<()> {
11394 let sender = self.sender_arc();
11395 let pkt =
11396 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
11397 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11398 }
11399
11400 #[cfg(feature = "discovery")]
11401 pub fn request_schema(&self) -> TelemetryResult<()> {
11402 let sender = self.sender_arc();
11403 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
11404 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11405 }
11406
11407 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
11414 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
11415 }
11416
11417 pub fn enable_network_variable(
11422 &self,
11423 ty: DataType,
11424 permissions: NetworkVariablePermissions,
11425 ) -> TelemetryResult<()> {
11426 if is_internal_control_type(ty) {
11427 return Err(TelemetryError::InvalidType);
11428 }
11429 let mut st = self.state.lock();
11430 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
11431 st.managed_variable_permissions
11432 .insert(ty.as_u32(), permissions);
11433 #[cfg(feature = "discovery")]
11434 if newly_advertised {
11435 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11436 }
11437 Ok(())
11438 }
11439
11440 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
11446 where
11447 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
11448 {
11449 if is_internal_control_type(ty) {
11450 return Err(TelemetryError::InvalidType);
11451 }
11452 let mut st = self.state.lock();
11453 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
11454 st.network_variable_update_handlers
11455 .entry(ty.as_u32())
11456 .or_default()
11457 .push(NetworkVariableUpdateHandler {
11458 handler: Arc::new(f),
11459 });
11460 #[cfg(feature = "discovery")]
11461 if newly_advertised {
11462 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11463 }
11464 Ok(())
11465 }
11466
11467 pub fn disable_managed_variable(&self, ty: DataType) {
11468 let mut st = self.state.lock();
11469 let was_advertised = st.managed_variable_types.remove(&ty.as_u32());
11470 st.managed_variable_permissions.remove(&ty.as_u32());
11471 st.managed_variable_latest.remove(&ty.as_u32());
11472 st.network_variable_update_handlers.remove(&ty.as_u32());
11473 #[cfg(feature = "discovery")]
11474 if was_advertised {
11475 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11476 }
11477 }
11478
11479 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
11480 if is_internal_control_type(pkt.data_type()) {
11481 return Err(TelemetryError::InvalidType);
11482 }
11483 pkt.validate()?;
11484 {
11485 let mut st = self.state.lock();
11486 let newly_advertised = st.managed_variable_types.insert(pkt.data_type().as_u32());
11487 #[cfg(feature = "discovery")]
11488 if newly_advertised {
11489 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
11490 }
11491 }
11492 self.cache_managed_variable_packet(&pkt, false)
11493 }
11494
11495 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
11496 self.managed_variable_latest(ty)
11497 }
11498
11499 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
11505 if is_internal_control_type(pkt.data_type()) {
11506 return Err(TelemetryError::InvalidType);
11507 }
11508 pkt.validate()?;
11509 let ty = pkt.data_type();
11510 {
11511 let mut st = self.state.lock();
11512 st.managed_variable_types.insert(ty.as_u32());
11513 let perms = Self::managed_variable_permissions_locked(&st, ty);
11514 if !perms.write {
11515 return Err(TelemetryError::PermissionDenied);
11516 }
11517 }
11518 self.cache_managed_variable_packet(&pkt, false)?;
11519 #[cfg(feature = "discovery")]
11520 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
11521 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
11522 if self.side_tx_active() {
11523 return self.tx_queue_item_with_priority(
11524 item,
11525 false,
11526 crate::transport_priority(DataType::ManagedVariableValue),
11527 );
11528 }
11529 self.tx_item(item)
11530 }
11531
11532 #[cfg(feature = "discovery")]
11539 pub fn get_network_variable(
11540 &self,
11541 ty: DataType,
11542 stale_after_ms: Option<u64>,
11543 ) -> TelemetryResult<Option<Packet>> {
11544 if is_internal_control_type(ty) {
11545 return Err(TelemetryError::InvalidType);
11546 }
11547 {
11548 let mut st = self.state.lock();
11549 st.managed_variable_types.insert(ty.as_u32());
11550 let perms = Self::managed_variable_permissions_locked(&st, ty);
11551 if !perms.read {
11552 return Err(TelemetryError::PermissionDenied);
11553 }
11554 }
11555 let cached = self.managed_variable_latest_with_age(ty);
11556 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
11557 (None, _) => true,
11558 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
11559 (Some(_), None) => false,
11560 };
11561 if needs_refresh {
11562 self.request_managed_variable(ty)?;
11563 }
11564 Ok(cached.map(|(pkt, _age_ms)| pkt))
11565 }
11566
11567 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
11569 if is_internal_control_type(ty) {
11570 return Err(TelemetryError::InvalidType);
11571 }
11572 if !self.can_read_managed_variable(ty) {
11573 return Err(TelemetryError::PermissionDenied);
11574 }
11575 Ok(self.managed_variable_latest(ty))
11576 }
11577
11578 #[cfg(feature = "discovery")]
11579 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
11580 if is_internal_control_type(ty) {
11581 return Err(TelemetryError::InvalidType);
11582 }
11583 if !self.can_read_managed_variable(ty) {
11584 return Err(TelemetryError::PermissionDenied);
11585 }
11586 let sender = self.sender_arc();
11587 let pkt =
11588 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
11589 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11590 }
11591
11592 #[cfg(feature = "discovery")]
11593 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
11594 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
11595 self.request_managed_variable(ty)
11596 }
11597
11598 #[cfg(feature = "discovery")]
11600 pub fn export_topology(&self) -> TopologySnapshot {
11601 let now_ms = self.clock.now_ms();
11602 let mut st = self.state.lock();
11603 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
11604 Self::note_discovery_topology_change_locked(&mut st, now_ms);
11605 }
11606 let routes = st
11607 .discovery_routes
11608 .iter()
11609 .filter_map(|(&side_id, route)| {
11610 let side = st.sides.get(side_id)?.as_ref()?;
11611 let announcers = route
11612 .announcers
11613 .iter()
11614 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
11615 sender_id: sender_id.clone(),
11616 reachable_endpoints: sender_state
11617 .reachable
11618 .iter()
11619 .copied()
11620 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
11621 .collect(),
11622 reachable_network_variables: sender_state
11623 .reachable_network_variables
11624 .clone(),
11625 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
11626 routers: sender_state.topology_boards.as_ref().clone(),
11627 last_seen_ms: sender_state.last_seen_ms,
11628 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
11629 })
11630 .collect();
11631 Some(TopologySideRoute {
11632 side_id,
11633 side_name: side.name.to_string(),
11634 reachable_endpoints: route
11635 .reachable
11636 .iter()
11637 .copied()
11638 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
11639 .collect(),
11640 reachable_network_variables: route.reachable_network_variables.clone(),
11641 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
11642 announcers,
11643 last_seen_ms: route.last_seen_ms,
11644 age_ms: now_ms.saturating_sub(route.last_seen_ms),
11645 })
11646 })
11647 .collect();
11648 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
11649 let advertised_endpoints =
11650 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
11651 let advertised_timesync_sources =
11652 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
11653 let links = discovery::topology_links_from_boards(&routers);
11654 TopologySnapshot {
11655 advertised_endpoints,
11656 advertised_timesync_sources,
11657 routers,
11658 links,
11659 routes,
11660 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
11661 next_announce_ms: st.discovery_cadence.next_announce_ms,
11662 }
11663 }
11664
11665 #[cfg(feature = "discovery")]
11666 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
11667 let now_ms = self.clock.now_ms();
11668 let st = self.state.lock();
11669 let mut side_ids = Vec::new();
11670 let mut side_names = Vec::new();
11671 let mut last_seen_ms = None::<u64>;
11672 let mut reachable_endpoints = Vec::new();
11673 let mut reachable_timesync_sources = Vec::new();
11674 let mut packets_sent = 0u64;
11675 let mut packets_received = 0u64;
11676 let mut bytes_sent = 0u64;
11677 let mut bytes_received = 0u64;
11678
11679 for (side_id, route) in &st.discovery_routes {
11680 let Some(sender_state) = route.announcers.get(sender_id) else {
11681 continue;
11682 };
11683 side_ids.push(*side_id);
11684 if let Some(side_name) = st
11685 .sides
11686 .get(*side_id)
11687 .and_then(|side| side.as_ref())
11688 .map(|side| side.name.clone())
11689 {
11690 side_names.push(side_name.to_string());
11691 }
11692 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
11693 reachable_endpoints.extend(sender_state.reachable.iter().copied());
11694 reachable_timesync_sources
11695 .extend(sender_state.reachable_timesync_sources.iter().cloned());
11696 if let Some(stats) = st.side_runtime_stats.get(side_id) {
11697 packets_sent = packets_sent.saturating_add(stats.tx_packets);
11698 packets_received = packets_received.saturating_add(stats.rx_packets);
11699 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
11700 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
11701 }
11702 }
11703
11704 if side_ids.is_empty() {
11705 return None;
11706 }
11707 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
11708 crate::small_sort::sort(&mut reachable_endpoints);
11709 reachable_endpoints.dedup();
11710 reachable_timesync_sources.sort_unstable();
11711 reachable_timesync_sources.dedup();
11712 crate::small_sort::sort(&mut side_ids);
11713 side_ids.dedup();
11714 side_names.sort_unstable();
11715 side_names.dedup();
11716 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
11717 Some(ClientStatsSnapshot {
11718 sender_id: sender_id.to_string(),
11719 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
11720 side_ids,
11721 side_names,
11722 last_seen_ms,
11723 age_ms,
11724 reachable_endpoints,
11725 reachable_timesync_sources,
11726 packets_sent,
11727 packets_received,
11728 bytes_sent,
11729 bytes_received,
11730 })
11731 }
11732
11733 pub fn copy_reliable_pending_stats(
11737 &self,
11738 first: usize,
11739 rows: &mut [crate::diagnostics::ReliablePendingRow],
11740 ) -> usize {
11741 rows.fill(crate::diagnostics::ReliablePendingRow::default());
11742 let st = self.state.lock();
11743 for (row, (&id, sent)) in rows
11744 .iter_mut()
11745 .zip(st.end_to_end_reliable_tx.iter().skip(first))
11746 {
11747 row.packet_id = id;
11748 row.last_send_ms = sent.last_send_ms;
11749 row.data_type = Self::item_data_type(&sent.data).map_or(0, DataType::as_u32);
11750 row.pending_count = sent.pending_destinations.len().min(u32::MAX as usize) as u32;
11751 row.retries = sent.retries;
11752 row.flags = u32::from(sent.queued)
11753 | (u32::from(sent.waiting_for_predecessor) << 1)
11754 | (u32::from(
11755 st.reliable_return_routes
11756 .get(&id)
11757 .is_some_and(|route| route.upstream_contract.is_some()),
11758 ) << 2)
11759 | (u32::from(sent.tracked_destinations) << 3);
11760 for (out, owner) in row
11761 .pending_owners
11762 .iter_mut()
11763 .zip(sent.pending_destinations.keys())
11764 {
11765 *out = *owner;
11766 }
11767 }
11768 st.end_to_end_reliable_tx.len()
11769 }
11770
11771 pub fn export_reliable_pending_stats(&self) -> Vec<crate::diagnostics::ReliablePendingStats> {
11775 let st = self.state.lock();
11776 st.end_to_end_reliable_tx
11777 .iter()
11778 .filter_map(|(&packet_id, sent)| {
11779 Some(crate::diagnostics::ReliablePendingStats {
11780 packet_id,
11781 data_type: Self::item_data_type(&sent.data).ok()?,
11782 pending_destinations: sent.pending_destinations.keys().copied().collect(),
11783 retries: sent.retries,
11784 queued: sent.queued,
11785 waiting_for_predecessor: sent.waiting_for_predecessor,
11786 last_send_ms: sent.last_send_ms,
11787 })
11788 })
11789 .collect()
11790 }
11791
11792 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
11793 let now_ms = self.clock.now_ms();
11794 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
11795 let st = self.state.lock();
11796
11797 let mut sides = Vec::new();
11798 for (side_id, side) in st.sides.iter().enumerate() {
11799 let Some(side) = side.as_ref() else { continue };
11800 let empty_stats = SideRuntimeStatsInner::default();
11801 let stats = st
11802 .side_runtime_stats
11803 .get(&side_id)
11804 .map(Box::as_ref)
11805 .unwrap_or(&empty_stats);
11806 let adaptive = st
11807 .adaptive_route_stats
11808 .get(&side_id)
11809 .cloned()
11810 .unwrap_or_default()
11811 .snapshot(now_ms, true);
11812 let (tx_template_count, rx_template_count) = st
11813 .side_transport
11814 .get(&side_id)
11815 .map(|state| (state.tx_template_count(), state.rx_template_count()))
11816 .unwrap_or((0, 0));
11817 let mut data_types: Vec<RuntimeTypeStats> = stats
11818 .data_types
11819 .iter()
11820 .map(|(&ty, item)| {
11821 let item = item.counters();
11822 RuntimeTypeStats {
11823 data_type: DataType(ty),
11824 tx_packets: item.tx_packets,
11825 tx_bytes: item.tx_bytes,
11826 rx_packets: item.rx_packets,
11827 rx_bytes: item.rx_bytes,
11828 relayed_tx_packets: item.relayed_tx_packets,
11829 relayed_tx_bytes: item.relayed_tx_bytes,
11830 relayed_rx_packets: item.relayed_rx_packets,
11831 relayed_rx_bytes: item.relayed_rx_bytes,
11832 tx_retries: item.tx_retries,
11833 handler_failures: item.handler_failures,
11834 }
11835 })
11836 .collect();
11837 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
11838 sides.push(RuntimeSideStats {
11839 side_id,
11840 side_name: side.name.to_string(),
11841 reliable_enabled: side.opts.reliable_enabled,
11842 link_local_enabled: side.opts.link_local_enabled,
11843 header_template_enabled: side.opts.header_template_enabled,
11844 max_frame_bytes: side.opts.max_frame_bytes,
11845 compact_header_target_bytes: side.opts.compact_header_target_bytes,
11846 side_transport_profile: side.opts.effective_transport_profile().as_str(),
11847 ingress_enabled: side.opts.ingress_enabled,
11848 egress_enabled: side.opts.egress_enabled,
11849 tx_packets: stats.tx_packets,
11850 tx_bytes: stats.tx_bytes,
11851 rx_packets: stats.rx_packets,
11852 rx_bytes: stats.rx_bytes,
11853 relayed_tx_packets: stats.relayed_tx_packets,
11854 relayed_tx_bytes: stats.relayed_tx_bytes,
11855 relayed_rx_packets: stats.relayed_rx_packets,
11856 relayed_rx_bytes: stats.relayed_rx_bytes,
11857 local_delivery_packets: stats.local_delivery_packets,
11858 tx_retries: stats.tx_retries,
11859 tx_handler_failures: stats.tx_handler_failures,
11860 local_handler_failures: stats.local_handler_failures,
11861 total_handler_retries: stats.total_handler_retries,
11862 side_transport_full_frames: stats.side_transport_full_frames,
11863 side_transport_compact_frames: stats.side_transport_compact_frames,
11864 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
11865 side_transport_compact_omitted_timestamp_frames: stats
11866 .side_transport_compact_omitted_timestamp_frames,
11867 side_transport_chunk_frames: stats.side_transport_chunk_frames,
11868 side_transport_chunk_reclaims: st
11869 .side_transport
11870 .get(&side_id)
11871 .map_or(0, |state| u64::from(state.rx_chunk_reclaims)),
11872 side_transport_raw_bytes: stats.side_transport_raw_bytes,
11873 side_transport_wire_bytes: stats.side_transport_wire_bytes,
11874 side_transport_bytes_saved: stats.side_transport_bytes_saved,
11875 side_transport_min_compact_overhead_bytes: stats
11876 .side_transport_min_compact_overhead_bytes,
11877 side_transport_max_compact_overhead_bytes: stats
11878 .side_transport_max_compact_overhead_bytes,
11879 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
11880 side_transport_template_evictions: stats.side_transport_template_evictions,
11881 side_transport_tx_template_count: tx_template_count,
11882 side_transport_rx_template_count: rx_template_count,
11883 max_side_transport_templates: side.opts.max_side_transport_templates,
11884 adaptive,
11885 data_types,
11886 });
11887 }
11888
11889 let mut route_modes: Vec<RouteModeStats> = st
11890 .route_selection_cursors
11891 .iter()
11892 .map(|(src, cursor)| RouteModeStats {
11893 src_side_id: *src,
11894 selection_mode: st.source_route_modes.get(src).copied(),
11895 cursor: *cursor,
11896 })
11897 .collect();
11898 for src in st.source_route_modes.keys() {
11899 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
11900 route_modes.push(RouteModeStats {
11901 src_side_id: *src,
11902 selection_mode: st.source_route_modes.get(src).copied(),
11903 cursor: 0,
11904 });
11905 }
11906 }
11907 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
11908
11909 let mut route_overrides: Vec<RouteOverrideStats> = st
11910 .route_overrides
11911 .iter()
11912 .map(|((src, dst), enabled)| RouteOverrideStats {
11913 src_side_id: *src,
11914 dst_side_id: *dst,
11915 enabled: *enabled,
11916 })
11917 .collect();
11918 route_overrides.sort_unstable_by_key(|item| {
11919 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
11920 });
11921
11922 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
11923 .typed_route_overrides
11924 .iter()
11925 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
11926 src_side_id: *src,
11927 data_type: DataType(*ty),
11928 dst_side_id: *dst,
11929 enabled: *enabled,
11930 })
11931 .collect();
11932 typed_route_overrides.sort_unstable_by_key(|item| {
11933 (
11934 item.src_side_id.unwrap_or(usize::MAX),
11935 item.data_type.as_u32(),
11936 item.dst_side_id,
11937 )
11938 });
11939
11940 let mut route_weights: Vec<RouteWeightStats> = st
11941 .route_weights
11942 .iter()
11943 .map(|((src, dst), weight)| RouteWeightStats {
11944 src_side_id: *src,
11945 dst_side_id: *dst,
11946 weight: *weight,
11947 })
11948 .collect();
11949 route_weights.sort_unstable_by_key(|item| {
11950 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
11951 });
11952
11953 let mut route_priorities: Vec<RoutePriorityStats> = st
11954 .route_priorities
11955 .iter()
11956 .map(|((src, dst), priority)| RoutePriorityStats {
11957 src_side_id: *src,
11958 dst_side_id: *dst,
11959 priority: *priority,
11960 })
11961 .collect();
11962 route_priorities.sort_unstable_by_key(|item| {
11963 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
11964 });
11965
11966 #[cfg(feature = "discovery")]
11967 let discovery = DiscoveryRuntimeStats {
11968 route_count: st.discovery_routes.len(),
11969 announcer_count: st
11970 .discovery_routes
11971 .values()
11972 .map(|route| route.announcers.len())
11973 .sum(),
11974 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
11975 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
11976 };
11977 #[cfg(not(feature = "discovery"))]
11978 let discovery = DiscoveryRuntimeStats {
11979 route_count: 0,
11980 announcer_count: 0,
11981 current_announce_interval_ms: None,
11982 next_announce_ms: None,
11983 };
11984
11985 RuntimeStatsSnapshot {
11986 sides,
11987 route_modes,
11988 route_overrides,
11989 typed_route_overrides,
11990 route_weights,
11991 route_priorities,
11992 queues: QueueRuntimeStats {
11993 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
11994 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
11995 tx_len: st.transmit_queue.len(),
11996 tx_bytes: st.transmit_queue.bytes_used(),
11997 replay_len: 0,
11998 replay_bytes: 0,
11999 recent_rx_len: st.recent_rx.len(),
12000 recent_rx_bytes: st.recent_rx.bytes_used(),
12001 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
12002 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
12003 shared_queue_bytes_used: st.shared_queue_bytes_used(),
12004 },
12005 reliable: ReliableRuntimeStats {
12006 reliable_return_route_count: st.reliable_return_routes.len(),
12007 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
12008 end_to_end_pending_destination_count: st
12009 .end_to_end_reliable_tx
12010 .values()
12011 .map(|sent| sent.pending_destinations.len())
12012 .sum(),
12013 end_to_end_acked_cache_count: 0,
12014 },
12015 discovery,
12016 total_handler_failures: st.total_handler_failures,
12017 total_handler_retries: st.total_handler_retries,
12018 }
12019 }
12020
12021 pub fn memory_counters(&self) -> [u32; 16] {
12025 let st = self.state.lock();
12026 #[cfg(feature = "discovery")]
12027 let discovery = st.discovery_bytes_used();
12028 #[cfg(not(feature = "discovery"))]
12029 let discovery = 0;
12030 let templates = st
12031 .side_transport
12032 .values()
12033 .map(|side| side.tx_templates.len() + side.rx_templates_by_id.len())
12034 .sum::<usize>();
12035 let template_bytes = st
12036 .side_transport
12037 .values()
12038 .flat_map(|side| {
12039 side.tx_templates
12040 .values()
12041 .chain(side.rx_templates_by_id.values())
12042 })
12043 .map(|t| size_of::<SideHeaderTemplate>() + t.prefix().len() + t.between().len())
12044 .sum::<usize>();
12045 let chunks = st
12046 .side_transport
12047 .values()
12048 .flat_map(|side| side.rx_chunks.values())
12049 .flat_map(|assembly| assembly.received.values())
12050 .map(|bytes| bytes.len())
12051 .sum::<usize>();
12052 let type_stats = st
12053 .side_runtime_stats
12054 .values()
12055 .map(|side| side.data_types.len())
12056 .sum::<usize>();
12057 let managed = st
12058 .managed_variable_latest
12059 .values()
12060 .map(|entry| entry.packet.byte_cost())
12061 .sum::<usize>();
12062 let pending = st
12063 .end_to_end_reliable_tx
12064 .values()
12065 .map(|entry| match &entry.data {
12066 RouterItem::Packet(packet) => packet.byte_cost(),
12067 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
12068 })
12069 .sum::<usize>();
12070 [
12071 1,
12072 st.memory.max_queue_budget,
12073 st.shared_queue_bytes_used(),
12074 st.received_queue.bytes_used(),
12075 st.transmit_queue.bytes_used(),
12076 st.received_queue.capacity() * size_of::<RouterRxItem>()
12077 + st.transmit_queue.capacity() * size_of::<TxQueued>()
12078 + st.recent_rx.capacity() * size_of::<u64>(),
12079 st.reliable_rx_buffered_bytes(),
12080 discovery,
12081 crate::config::schema_bytes_used(),
12082 managed,
12083 pending,
12084 templates,
12085 template_bytes,
12086 chunks,
12087 type_stats,
12088 st.side_runtime_stats
12089 .values()
12090 .flat_map(|side| side.data_types.values())
12091 .map(|stats| size_of::<TypeRuntimeStatsInner>() + stats.heap_bytes())
12092 .sum::<usize>(),
12093 ]
12094 .map(|value| u32::try_from(value).unwrap_or(u32::MAX))
12095 }
12096
12097 pub fn export_memory_layout_json(&self) -> String {
12099 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
12100 let st = self.state.lock();
12101 #[cfg(feature = "discovery")]
12102 let discovery_bytes = st.discovery_bytes_used();
12103 #[cfg(not(feature = "discovery"))]
12104 let discovery_bytes = 0usize;
12105 let schema_bytes = crate::config::schema_bytes_used();
12106 let network_variable_cache_bytes = st
12107 .managed_variable_latest
12108 .values()
12109 .map(|entry| entry.packet.byte_cost())
12110 .sum::<usize>();
12111 let mut out = String::new();
12112 let memory = st.memory;
12113 let _ = fmt::Write::write_fmt(
12114 &mut out,
12115 format_args!(
12116 "{{\"kind\":\"router\",\
12117 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
12118 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
12119 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
12120 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
12121 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
12122 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
12123 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
12124 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
12125 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
12126 st.shared_queue_bytes_used(),
12127 memory.max_queue_budget,
12128 st.received_queue.bytes_used(),
12129 st.received_queue.max_bytes(),
12130 st.received_queue.len(),
12131 isr_rx.1,
12132 memory.max_queue_budget,
12133 isr_rx.0,
12134 st.transmit_queue.bytes_used(),
12135 st.transmit_queue.max_bytes(),
12136 st.transmit_queue.len(),
12137 st.recent_rx.bytes_used(),
12138 st.recent_rx.max_bytes(),
12139 st.recent_rx.len(),
12140 st.reliable_rx_buffered_bytes(),
12141 memory.max_queue_budget,
12142 st.reliable_rx_buffer_len(),
12143 discovery_bytes,
12144 memory.max_queue_budget,
12145 schema_bytes,
12146 memory.max_queue_budget,
12147 network_variable_cache_bytes,
12148 memory.max_queue_budget,
12149 st.managed_variable_latest.len(),
12150 ),
12151 );
12152 out
12153 }
12154
12155 #[cfg(test)]
12156 pub(crate) fn debug_end_to_end_pending_destination_count(
12157 &self,
12158 packet_id: u64,
12159 ) -> Option<usize> {
12160 let st = self.state.lock();
12161 st.end_to_end_reliable_tx
12162 .get(&packet_id)
12163 .map(|sent| sent.pending_destinations.len())
12164 }
12165
12166 #[cfg(test)]
12167 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
12168 let st = self.state.lock();
12169 st.end_to_end_reliable_tx.len()
12170 }
12171
12172 #[cfg(test)]
12173 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
12174 let st = self.state.lock();
12175 st.reliable_return_routes.len()
12176 }
12177
12178 #[cfg(test)]
12179 pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
12180 let st = self.state.lock();
12181 (st.sides.len(), st.sides.capacity())
12182 }
12183
12184 #[cfg(test)]
12185 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
12186 let st = self.state.lock();
12187 (
12188 st.received_queue.len(),
12189 st.transmit_queue.len(),
12190 st.recent_rx.len(),
12191 )
12192 }
12193
12194 #[cfg(test)]
12195 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
12196 let st = self.state.lock();
12197 st.shared_queue_bytes_used()
12198 }
12199
12200 #[cfg(test)]
12201 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
12202 let st = self.state.lock();
12203 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
12204 }
12205
12206 fn get_hash(item: &RouterItem) -> u64 {
12210 match item {
12211 RouterItem::Packet(pkt) => pkt.packet_id(),
12212 RouterItem::Packed(bytes) => {
12213 match wire_format::packet_id_from_wire(bytes.as_ref()) {
12214 Ok(id) => id,
12215 Err(_e) => {
12216 let h: u64 = 0x9E37_79B9_7F4A_7C15;
12219 hash_bytes_u64(h, bytes.as_ref())
12220 }
12221 }
12222 }
12223 }
12224 }
12225
12226 fn remove_pkt_id(&self, item: &RouterItem) {
12228 let hash = Self::get_hash(item);
12229 let mut st = self.state.lock();
12230 st.recent_rx.remove_value(&hash);
12231 }
12232
12233 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
12236 let id = Self::get_hash(item);
12237 let mut st = self.state.lock();
12238 if st.recent_rx.contains(&id) {
12239 Ok(true)
12240 } else {
12241 st.push_recent_rx(id)?;
12242 Ok(false)
12243 }
12244 }
12245
12246 fn handle_callback_error(
12251 &self,
12252 pkt: &Packet,
12253 dest: Option<DataEndpoint>,
12254 e: TelemetryError,
12255 called_from_queue: bool,
12256 ) -> TelemetryResult<()> {
12257 if matches!(&e, TelemetryError::Io("memory pressure")) {
12260 return Err(e);
12261 }
12262
12263 let device = self.sender_arc();
12264 let error_msg = match dest {
12265 Some(failed_local) => format!(
12266 "Handler for endpoint {:?} failed on device {:?}: {:?}",
12267 failed_local, device, e
12268 ),
12269 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
12270 };
12271
12272 let mut recipients: Vec<DataEndpoint> = pkt
12273 .endpoints()
12274 .iter()
12275 .copied()
12276 .filter(|&ep| self.cfg.is_local_endpoint(ep))
12277 .collect();
12278 crate::small_sort::sort(&mut recipients);
12279 recipients.dedup();
12280
12281 if let Some(failed_local) = dest {
12282 recipients.retain(|&ep| ep != failed_local);
12283 }
12284
12285 if recipients.is_empty() && dest.is_none() {
12289 fallback_stdout(&error_msg);
12290 return Ok(());
12291 }
12292
12293 if recipients.is_empty() {
12296 recipients = pkt.endpoints().to_vec();
12297 crate::small_sort::sort(&mut recipients);
12298 recipients.dedup();
12299 if let Some(failed_local) = dest {
12300 recipients.retain(|&ep| ep != failed_local);
12301 }
12302 }
12303
12304 if recipients.is_empty() {
12305 fallback_stdout(&error_msg);
12306 return Ok(());
12307 }
12308
12309 let payload = make_error_payload(&error_msg);
12310
12311 let sender = self.sender_arc();
12312 let error_pkt = Packet::new(
12313 DataType::TelemetryError,
12314 &recipients,
12315 sender.as_ref(),
12316 self.packet_timestamp_ms(),
12317 crate::shared_bytes::convert(payload),
12318 )?;
12319
12320 self.emit_internal_tx(
12321 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
12322 false,
12323 called_from_queue,
12324 )
12325 }
12326
12327 #[inline]
12331 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
12332 self.process_tx_queue_with_timeout(0)
12333 }
12334
12335 #[inline]
12337 pub fn process_all_queues(&self) -> TelemetryResult<()> {
12338 self.process_all_queues_with_timeout(0)
12339 }
12340
12341 #[inline]
12343 pub fn clear_queues(&self) {
12344 let mut st = self.state.lock();
12345 st.transmit_queue.clear();
12346 st.received_queue.clear();
12347 drop(st);
12348 let _ = self.isr_rx_queue.clear();
12349 }
12350
12351 #[inline]
12353 pub fn clear_rx_queue(&self) {
12354 let mut st = self.state.lock();
12355 st.received_queue.clear();
12356 drop(st);
12357 let _ = self.isr_rx_queue.clear();
12358 }
12359
12360 #[inline]
12362 pub fn clear_tx_queue(&self) {
12363 let mut st = self.state.lock();
12364 st.transmit_queue.clear();
12365 st.tx_priority_burst = 0;
12366 }
12367
12368 fn pop_transmit_queue_locked(st: &mut RouterInner, now_ms: u64) -> Option<TxQueued> {
12369 st.transmit_queue.pop_front_fair_if(
12370 &mut st.tx_priority_burst,
12371 MAX_PRIORITY_BURST,
12372 |item| item.priority,
12373 |item| item.retry_after_ms <= now_ms,
12374 )
12375 }
12376
12377 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
12380 self.expire_side_chunks();
12381 let start = self.clock.now_ms();
12382 loop {
12383 let received_control = crate::memory_admission::allow_drain(
12384 self.drain_queued_control_rx_before_tx(start, timeout_ms),
12385 )?;
12386 let control_budget_used = received_control
12387 && timeout_ms != 0
12388 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
12389 if !control_budget_used {
12392 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
12393 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
12394 }
12395
12396 let pkt_opt = {
12397 let mut st = self.state.lock();
12398 Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms())
12399 };
12400 let Some(pkt) = pkt_opt else { break };
12401 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
12402 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
12403 break;
12404 }
12405 }
12406 Ok(())
12407 }
12408
12409 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
12412 #[cfg(feature = "timesync")]
12413 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
12414 #[cfg(feature = "discovery")]
12415 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12416 self.process_tx_queue_with_timeout_impl(timeout_ms)
12417 }
12418
12419 pub fn dispatch_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
12425 self.process_tx_queue_with_timeout_impl(timeout_ms)
12426 }
12427
12428 #[inline]
12430 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
12431 self.rx_item(&item, true)
12432 }
12433
12434 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
12437 self.expire_side_chunks();
12438 let start = self.clock.now_ms();
12439 loop {
12440 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
12441 let mut st = self.state.lock();
12442 st.received_queue.pop_front()
12443 });
12444 let Some(item) = item_opt else { break };
12445 self.process_rx_queue_item(item)?;
12446 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
12447 break;
12448 }
12449 }
12450 Ok(())
12451 }
12452
12453 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
12456 #[cfg(feature = "timesync")]
12457 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
12458 #[cfg(feature = "discovery")]
12459 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12460 self.process_rx_queue_with_timeout_impl(timeout_ms)
12461 }
12462
12463 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
12466 self.expire_side_chunks();
12467 let start = self.clock.now_ms();
12472 loop {
12473 let received_control = crate::memory_admission::allow_drain(
12474 self.drain_queued_control_rx_before_tx(start, timeout_ms),
12475 )?;
12476 let mut did_any = received_control;
12477 let control_budget_used = received_control
12478 && timeout_ms != 0
12479 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
12480 if !control_budget_used {
12481 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
12482 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
12483 }
12484
12485 if let Some(pkt) = {
12486 let mut st = self.state.lock();
12487 let now = self.clock.now_ms();
12488 let inner = &mut *st;
12489 inner.transmit_queue.pop_front_fair_if(
12490 &mut inner.tx_priority_burst,
12491 MAX_PRIORITY_BURST,
12492 |item| item.priority,
12493 |item| {
12494 item.retry_after_ms <= now && (!control_budget_used || item.priority >= 200)
12495 },
12496 )
12497 } {
12498 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
12499 did_any = true;
12500 }
12501 if control_budget_used {
12503 return Ok(());
12504 }
12505 if let Some(item) = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
12506 let mut st = self.state.lock();
12507 st.received_queue.pop_front()
12508 }) {
12509 self.process_rx_queue_item(item)?;
12510 did_any = true;
12511 }
12512 if !did_any
12513 || (timeout_ms != 0
12514 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms))
12515 {
12516 break;
12517 }
12518 }
12519
12520 Ok(())
12521 }
12522
12523 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
12526 #[cfg(feature = "timesync")]
12527 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
12528 #[cfg(feature = "discovery")]
12529 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12530 self.process_all_queues_with_timeout_impl(timeout_ms)
12531 }
12532
12533 pub fn drain_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
12538 self.process_all_queues_with_timeout_impl(timeout_ms)
12539 }
12540
12541 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
12546 #[cfg(feature = "timesync")]
12547 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
12548
12549 #[cfg(feature = "discovery")]
12550 {
12551 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12552 }
12553
12554 self.process_all_queues_with_timeout_impl(timeout_ms)
12555 }
12556
12557 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
12562 #[cfg(feature = "discovery")]
12563 {
12564 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
12565 }
12566
12567 self.process_all_queues_with_timeout_impl(timeout_ms)
12568 }
12569
12570 #[inline]
12572 fn tx_queue_item_with_flags(
12573 &self,
12574 item: RouterTxItem,
12575 ignore_local: bool,
12576 ) -> TelemetryResult<()> {
12577 self.tx_queue_item_with_flags_at(item, ignore_local, 0)
12578 }
12579
12580 fn tx_queue_item_for_retry(
12581 &self,
12582 item: RouterTxItem,
12583 ignore_local: bool,
12584 ) -> TelemetryResult<()> {
12585 self.tx_queue_item_with_flags_at(item, ignore_local, self.clock.now_ms().saturating_add(1))
12586 }
12587
12588 fn tx_queue_item_with_flags_at(
12589 &self,
12590 item: RouterTxItem,
12591 ignore_local: bool,
12592 retry_after_ms: u64,
12593 ) -> TelemetryResult<()> {
12594 let priority = match &item {
12595 RouterTxItem::Broadcast(data) => self.item_service_priority(data)?,
12596 RouterTxItem::EndToEndReplay { .. } => {
12597 Self::router_item_priority_bumped(DataType::ReliableAck)
12598 }
12599 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
12600 RouterTxItem::ReliableReplay { bytes, .. } => {
12601 let ty = wire_format::peek_routing_envelope(bytes.as_ref())?.ty;
12602 Self::router_item_priority_bumped(ty)
12603 }
12604 };
12605 self.tx_queue_item_with_priority_at(item, ignore_local, priority, retry_after_ms)
12606 }
12607
12608 #[inline]
12609 fn tx_queue_item_with_priority(
12610 &self,
12611 item: RouterTxItem,
12612 ignore_local: bool,
12613 priority: u8,
12614 ) -> TelemetryResult<()> {
12615 self.tx_queue_item_with_priority_at(item, ignore_local, priority, 0)
12616 }
12617
12618 fn tx_queue_item_with_priority_at(
12619 &self,
12620 item: RouterTxItem,
12621 ignore_local: bool,
12622 priority: u8,
12623 retry_after_ms: u64,
12624 ) -> TelemetryResult<()> {
12625 let mut st = self.state.lock();
12626 st.push_transmit(TxQueued {
12627 item,
12628 ignore_local,
12629 priority,
12630 retry_after_ms,
12631 })?;
12632 Ok(())
12633 }
12634
12635 #[inline]
12637 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
12638 self.tx_queue_item_with_flags(item, false)
12639 }
12640
12641 #[inline]
12642 fn try_enter_side_tx(&self) -> Option<crate::lock::ReentryGuard<'_>> {
12643 self.side_tx_gate.try_enter()
12644 }
12645
12646 #[inline]
12647 fn side_tx_active(&self) -> bool {
12648 self.side_tx_gate.is_active()
12649 }
12650
12651 #[inline]
12655 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
12656 self.process_rx_queue_with_timeout(0)
12657 }
12658
12659 #[inline]
12661 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
12662 crate::memory_admission::check_receive(bytes)?;
12663 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
12664 let priority = self.item_service_priority(&data)?;
12665 let mut st = self.state.lock();
12666 st.push_received(RouterRxItem {
12667 src: None,
12668 data,
12669 priority,
12670 })?;
12671 Ok(())
12672 }
12673
12674 #[inline]
12679 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
12680 crate::memory_admission::check_receive(bytes)?;
12681 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
12682 let priority = Self::router_item_priority(&data)?;
12683 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
12684 src: None,
12685 data,
12686 priority,
12687 })
12688 }
12689
12690 #[inline]
12692 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
12693 pkt.validate()?;
12694 let data = RouterItem::Packet(pkt);
12695 let priority = self.item_service_priority(&data)?;
12696 let mut st = self.state.lock();
12697 st.push_received(RouterRxItem {
12698 src: None,
12699 data,
12700 priority,
12701 })?;
12702 Ok(())
12703 }
12704
12705 #[inline]
12710 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
12711 pkt.validate()?;
12712 let data = RouterItem::Packet(pkt);
12713 let priority = Self::router_item_priority(&data)?;
12714 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
12715 src: None,
12716 data,
12717 priority,
12718 })
12719 }
12720
12721 #[inline]
12723 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
12724 self.ensure_side_ingress_enabled(side)?;
12725 pkt.validate()?;
12726 let data = RouterItem::Packet(pkt);
12727 let priority = self.item_service_priority(&data)?;
12728 let mut st = self.state.lock();
12729 st.push_received(RouterRxItem {
12730 src: Some(side),
12731 data,
12732 priority,
12733 })?;
12734 Ok(())
12735 }
12736
12737 #[inline]
12742 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
12743 self.ensure_side_ingress_enabled(side)?;
12744 pkt.validate()?;
12745 let data = RouterItem::Packet(pkt);
12746 let priority = Self::router_item_priority(&data)?;
12747 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
12748 src: Some(side),
12749 data,
12750 priority,
12751 })
12752 }
12753
12754 #[inline]
12756 pub fn rx_packed_queue_from_side(
12757 &self,
12758 bytes: &[u8],
12759 side: RouterSideId,
12760 ) -> TelemetryResult<()> {
12761 self.ensure_side_ingress_enabled(side)?;
12762 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
12763 return Ok(());
12764 };
12765 let data = RouterItem::Packed(decoded);
12766 let priority = self.item_service_priority(&data)?;
12767 let mut st = self.state.lock();
12768 st.push_received(RouterRxItem {
12769 src: Some(side),
12770 data,
12771 priority,
12772 })?;
12773 Ok(())
12774 }
12775
12776 #[inline]
12781 pub fn rx_packed_queue_from_side_isr(
12782 &self,
12783 bytes: &[u8],
12784 side: RouterSideId,
12785 ) -> TelemetryResult<()> {
12786 self.ensure_side_ingress_enabled(side)?;
12787 crate::memory_admission::check_receive(bytes)?;
12788 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
12789 let priority = Self::router_item_priority(&data)?;
12790 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
12791 src: Some(side),
12792 data,
12793 priority,
12794 })
12795 }
12796
12797 fn retry_with_attempts<F, T>(
12798 &self,
12799 times: usize,
12800 f: F,
12801 ) -> Result<(T, usize), (TelemetryError, usize)>
12802 where
12803 F: Fn() -> TelemetryResult<T>,
12804 {
12805 let mut last_err = None;
12806 for attempt in 0..times {
12807 match f() {
12808 Ok(v) => return Ok((v, attempt + 1)),
12809 Err(e) if Self::is_transient_tx_error(&e) => return Err((e, attempt + 1)),
12812 Err(e) => last_err = Some((e, attempt + 1)),
12813 }
12814 }
12815 Err(last_err.expect("times > 0"))
12816 }
12817
12818 #[inline]
12820 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
12821 self.cfg
12822 .handlers
12823 .iter()
12824 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
12825 }
12826
12827 #[inline]
12829 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
12830 self.cfg
12831 .handlers
12832 .iter()
12833 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
12834 }
12835
12836 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
12837 pkt.endpoints()
12838 .iter()
12839 .copied()
12840 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
12841 }
12842
12843 fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
12850 self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
12851 }
12852
12853 fn call_handler_with_retries(
12859 &self,
12860 dest: DataEndpoint,
12861 handler: &EndpointHandler,
12862 data: Option<&[u8]>,
12863 pkt_for_ctx: Option<&Packet>,
12864 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
12865 called_from_queue: bool,
12866 ) -> TelemetryResult<()> {
12867 let owned_tmp: Option<RouterItem>;
12868
12869 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
12870 (Some(d), _) => {
12871 owned_tmp = Some(RouterItem::Packed(crate::shared_bytes::convert(
12872 Arc::<[u8]>::from(d),
12873 )));
12874 owned_tmp.as_ref().unwrap()
12875 }
12876 (None, Some(pkt)) => {
12877 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
12878 owned_tmp.as_ref().unwrap()
12879 }
12880 (None, None) => {
12881 debug_assert!(
12882 false,
12883 "call_handler_with_retries called without data or packet context"
12884 );
12885 return Ok(());
12886 }
12887 };
12888
12889 match (&handler.handler, data) {
12890 (EndpointHandlerFn::Packet(f), _) => {
12891 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
12892 with_retries(
12893 self,
12894 dest,
12895 item_for_ctx,
12896 pkt_for_ctx,
12897 env_for_ctx,
12898 called_from_queue,
12899 || f(pkt),
12900 )
12901 }
12902
12903 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
12904 self,
12905 dest,
12906 item_for_ctx,
12907 pkt_for_ctx,
12908 env_for_ctx,
12909 called_from_queue,
12910 || f(bytes),
12911 ),
12912
12913 (EndpointHandlerFn::Packed(_), None) => Ok(()),
12914 }
12915 }
12916
12917 fn handle_callback_error_from_env(
12922 &self,
12923 env: &wire_format::TelemetryEnvelope,
12924 dest: Option<DataEndpoint>,
12925 e: TelemetryError,
12926 called_from_queue: bool,
12927 ) -> TelemetryResult<()> {
12928 if matches!(&e, TelemetryError::Io("memory pressure")) {
12931 return Err(e);
12932 }
12933
12934 let mut recipients: Vec<DataEndpoint> = env
12935 .endpoints
12936 .iter()
12937 .copied()
12938 .filter(|&ep| self.cfg.is_local_endpoint(ep))
12939 .collect();
12940 crate::small_sort::sort(&mut recipients);
12941 recipients.dedup();
12942 if let Some(failed) = dest {
12943 recipients.retain(|&ep| ep != failed);
12944 }
12945
12946 let device = self.sender_arc();
12947 let error_msg = format!(
12948 "Handler for endpoint {:?} failed on device {:?}: {:?}",
12949 dest, device, e
12950 );
12951
12952 if recipients.is_empty() && dest.is_none() {
12953 fallback_stdout(&error_msg);
12954 return Ok(());
12955 }
12956
12957 if recipients.is_empty() {
12958 recipients = env.endpoints.to_vec();
12959 crate::small_sort::sort(&mut recipients);
12960 recipients.dedup();
12961 if let Some(failed) = dest {
12962 recipients.retain(|&ep| ep != failed);
12963 }
12964 }
12965
12966 if recipients.is_empty() {
12967 fallback_stdout(&error_msg);
12968 return Ok(());
12969 }
12970
12971 let payload = make_error_payload(&error_msg);
12972
12973 let error_pkt = Packet::new(
12974 DataType::TelemetryError,
12975 &recipients,
12976 &env.sender.clone(),
12977 env.timestamp_ms,
12978 crate::shared_bytes::convert(payload),
12979 )?;
12980 self.emit_internal_tx(
12981 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
12982 false,
12983 called_from_queue,
12984 )
12985 }
12986
12987 fn handle_internal_reliable_packet(
12988 &self,
12989 pkt: &Packet,
12990 src: Option<RouterSideId>,
12991 called_from_queue: bool,
12992 ) -> TelemetryResult<bool> {
12993 if !matches!(
12994 pkt.data_type(),
12995 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
12996 ) {
12997 return Ok(false);
12998 }
12999
13000 let Some(src) = src else {
13001 return Ok(false);
13002 };
13003
13004 if Self::is_end_to_end_ack_packet(pkt)
13005 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
13006 {
13007 let alias = {
13008 let st = self.state.lock();
13009 st.reliable_return_routes
13010 .get(&packet_id)
13011 .filter(|route| route.upstream_contract.is_some())
13012 .cloned()
13013 };
13014 if let Some(alias) = alias {
13015 let (upstream_id, publisher_hash) = alias.upstream_contract.expect("checked alias");
13016 #[cfg(feature = "std")]
13017 if reliable_trace_enabled() {
13018 eprintln!(
13019 "[sedsnet-reliable] proxy receipt id={packet_id:#x} upstream={upstream_id:#x} owner={:?}",
13020 Self::end_to_end_ack_sender_hash(pkt)
13021 );
13022 }
13023 let completed = {
13024 let mut st = self.state.lock();
13025 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
13026 if let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
13027 sent.pending_destinations.remove(&owner);
13028 }
13029 if !sent.pending_destinations.is_empty() {
13030 return Ok(true);
13031 }
13032 st.end_to_end_reliable_tx.remove(&packet_id);
13033 true
13034 } else {
13035 return Ok(true);
13038 }
13039 };
13040 if completed {
13041 let mut st = self.state.lock();
13042 if Self::remember_reliable_return_route_locked(&mut st, upstream_id) {
13043 st.reliable_return_routes.insert(
13044 upstream_id,
13045 ReliableReturnRouteState {
13046 side: alias.side,
13047 last_replay_ms: alias.last_replay_ms,
13048 upstream_contract: None,
13049 },
13050 );
13051 }
13052 }
13053 self.queue_end_to_end_reliable_ack_for_sender_hash(
13054 upstream_id,
13055 publisher_hash,
13056 called_from_queue,
13057 )?;
13058 self.release_ordered_successors()?;
13059 return Ok(true);
13060 }
13061 let targets_local_sender = pkt.wire_target_senders().is_empty()
13068 || pkt
13069 .wire_target_senders()
13070 .contains(&Self::sender_hash(self.sender_arc().as_ref()));
13071 let mut st = self.state.lock();
13072 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
13073 if sent.waiting_for_predecessor {
13074 return Ok(targets_local_sender);
13075 }
13076 if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
13077 let _removed = sent.pending_destinations.remove(&sender_hash);
13078 #[cfg(feature = "std")]
13079 if reliable_trace_enabled() {
13080 eprintln!(
13081 "[sedsnet-reliable] receipt time={} id={packet_id:#x} sender={sender_hash:#x} matched={} remaining={:?}",
13082 self.clock.now_ms(),
13083 _removed.is_some(),
13084 sent.pending_destinations
13085 );
13086 }
13087 if sent.pending_destinations.is_empty() {
13088 st.end_to_end_reliable_tx.remove(&packet_id);
13089 }
13090 drop(st);
13091 self.release_ordered_successors()?;
13092 return Ok(targets_local_sender);
13093 }
13094 st.end_to_end_reliable_tx.remove(&packet_id);
13095 drop(st);
13096 self.release_ordered_successors()?;
13097 return Ok(targets_local_sender);
13098 }
13099 return Ok(targets_local_sender);
13102 }
13103
13104 let vals = pkt.data_as_u32()?;
13105 if vals.len() != 2 {
13106 return Err(TelemetryError::Unpack("bad reliable control payload"));
13107 }
13108 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
13109 let seq = vals[1];
13110
13111 match pkt.data_type() {
13112 DataType::ReliableAck => {
13113 self.handle_reliable_ack(src, ty, seq);
13114 Ok(true)
13115 }
13116 DataType::ReliablePartialAck => {
13117 self.handle_reliable_partial_ack(src, ty, seq);
13118 Ok(true)
13119 }
13120 DataType::ReliablePacketRequest => {
13121 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
13122 Ok(true)
13123 }
13124 _ => Ok(false),
13125 }
13126 }
13127
13128 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
13134 if crate::memory_admission::supports_work_kinds()
13135 && let (Some(side), RouterItem::Packed(bytes)) = (item.src, &item.data)
13136 {
13137 self.admit_received_frame(side, bytes)?;
13138 }
13139 if let Some(src) = item.src {
13140 self.ensure_side_ingress_enabled(src)?;
13141 match &item.data {
13142 RouterItem::Packet(pkt) => {
13143 let bytes = wire_format::packet_wire_size(pkt);
13144 self.note_side_rx(src, pkt.data_type(), bytes, true);
13145 }
13146 RouterItem::Packed(bytes) => {
13147 if let Ok(env) = wire_format::peek_routing_envelope(bytes.as_ref()) {
13148 self.note_side_rx(src, env.ty, bytes.len(), true);
13149 }
13150 }
13151 }
13152 match &item.data {
13153 RouterItem::Packet(pkt) => {
13154 if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
13155 && !is_internal_control_type(pkt.data_type())
13156 && !self.note_reliable_return_route(src, pkt.packet_id())
13157 {
13158 return Ok(());
13159 }
13160 }
13161 RouterItem::Packed(bytes) => {
13162 if let Ok(env) = wire_format::peek_routing_envelope(bytes.as_ref())
13163 && (is_reliable_type(env.ty) || env.target_senders().len() != 0)
13164 && !is_internal_control_type(env.ty)
13165 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
13166 && !self.note_reliable_return_route(src, packet_id)
13167 {
13168 return Ok(());
13169 }
13170 }
13171 }
13172 }
13173 match &item.data {
13174 RouterItem::Packet(pkt) => {
13175 if !is_internal_control_type(pkt.data_type()) {
13176 self.remember_managed_variable_packet(pkt)?;
13177 }
13178 }
13179 RouterItem::Packed(bytes) => {
13180 if let Ok(env) = wire_format::peek_routing_envelope(bytes.as_ref())
13181 && !is_internal_control_type(env.ty)
13182 && self.is_managed_variable_type(env.ty)
13183 {
13184 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13185 pkt.validate()?;
13186 self.remember_managed_variable_packet(&pkt)?;
13187 }
13188 }
13189 }
13190 let mut released_buffered: Vec<crate::SharedBytes> = Vec::new();
13191 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
13192 let (_opts, handler_is_packed, hop_reliable_enabled) = {
13193 let st = self.state.lock();
13194 let side_ref = Self::side_ref(&st, src)?;
13195 let opts = side_ref.opts.clone();
13196 (
13197 opts.clone(),
13198 matches!(
13199 side_ref.tx_handler,
13200 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
13201 ),
13202 opts.reliable_enabled
13203 && self.cfg.reliable_enabled()
13204 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
13205 )
13206 };
13207
13208 if hop_reliable_enabled && handler_is_packed {
13209 let frame = match wire_format::peek_routing_frame_info(bytes.as_ref()) {
13210 Ok(frame) => frame,
13211 Err(e) => {
13212 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
13213 if let Ok(frame) =
13214 wire_format::peek_frame_info_unchecked(bytes.as_ref())
13215 && is_reliable_type(frame.envelope.ty)
13216 && let Some(hdr) = frame.reliable
13217 {
13218 let unordered =
13219 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
13220 let unsequenced =
13221 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
13222
13223 if !unsequenced {
13224 let requested = if unordered {
13225 hdr.seq
13226 } else {
13227 let mut st = self.state.lock();
13228 let rx_state = self.reliable_rx_state_mut(
13229 &mut st,
13230 src,
13231 frame.envelope.ty,
13232 );
13233 rx_state.expected_seq.min(hdr.seq)
13234 };
13235 self.queue_reliable_packet_request(
13236 src,
13237 frame.envelope.ty,
13238 requested,
13239 called_from_queue,
13240 )?;
13241 }
13242 }
13243 return Ok(());
13244 }
13245 return Err(e);
13246 }
13247 };
13248 if is_reliable_type(frame.envelope.ty)
13249 && let Some(hdr) = frame.reliable
13250 {
13251 if frame.ack_only() {
13252 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
13253 return Ok(());
13254 }
13255 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
13256 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
13257
13258 if !unsequenced {
13259 if unordered {
13260 self.queue_reliable_ack(
13261 src,
13262 frame.envelope.ty,
13263 hdr.seq,
13264 called_from_queue,
13265 )?;
13266 } else {
13267 let mut release: Vec<crate::SharedBytes> = Vec::new();
13268 let mut last_delivered = None;
13269 let mut ack_old = None;
13270 let mut request_missing = None;
13271 let mut partial_ack = None;
13272 {
13273 let mut st = self.state.lock();
13274 let rx_state =
13275 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
13276 let expected_seq = rx_state.expected_seq;
13277 if hdr.seq < expected_seq {
13278 ack_old = Some(expected_seq.saturating_sub(1));
13279 } else if hdr.seq > expected_seq {
13280 request_missing = Some(expected_seq);
13281 partial_ack = Some(hdr.seq);
13282 st.buffer_reliable_rx(
13283 src,
13284 frame.envelope.ty,
13285 hdr.seq,
13286 bytes.clone(),
13287 )?;
13288 } else {
13289 release.push(bytes.clone());
13290 last_delivered = Some(hdr.seq);
13291 let mut next_expected = hdr.seq.wrapping_add(1);
13292 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
13293 release.push(buf);
13294 last_delivered = Some(next_expected);
13295 let next = next_expected.wrapping_add(1);
13296 next_expected = if next == 0 { 1 } else { next };
13297 }
13298 rx_state.expected_seq = next_expected;
13299 }
13300 }
13301
13302 if let Some(ack_seq) = ack_old {
13303 self.queue_reliable_ack(
13304 src,
13305 frame.envelope.ty,
13306 ack_seq,
13307 called_from_queue,
13308 )?;
13309 return Ok(());
13310 }
13311 if let Some(request_seq) = request_missing {
13312 if let Some(partial_seq) = partial_ack {
13313 self.queue_reliable_partial_ack(
13314 src,
13315 frame.envelope.ty,
13316 partial_seq,
13317 called_from_queue,
13318 )?;
13319 }
13320 self.queue_reliable_packet_request(
13321 src,
13322 frame.envelope.ty,
13323 request_seq,
13324 called_from_queue,
13325 )?;
13326 return Ok(());
13327 }
13328
13329 if let Some(ack_seq) = last_delivered {
13330 self.queue_reliable_ack(
13331 src,
13332 frame.envelope.ty,
13333 ack_seq,
13334 called_from_queue,
13335 )?;
13336 }
13337
13338 released_buffered.extend(release.into_iter().skip(1));
13339 }
13340 }
13341 }
13342 } else {
13343 match wire_format::peek_routing_frame_info(bytes.as_ref()) {
13344 Ok(frame) => {
13345 if frame.ack_only() {
13346 return Ok(());
13347 }
13348 }
13349 Err(e) => {
13350 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
13351 return Ok(());
13352 }
13353 return Err(e);
13354 }
13355 }
13356 }
13357 }
13358
13359 if self.is_duplicate_pkt(&item.data)? {
13360 if (item.src.is_some()
13363 && self
13364 .item_target_senders(&item.data)?
13365 .iter()
13366 .any(|target| *target != Self::sender_hash(self.sender_arc().as_ref())))
13367 || (item.src.is_some() && self.should_route_remote(&item.data, item.src)? && {
13368 let (endpoints, _) = self.item_route_info(&item.data)?;
13369 self.item_targets_router_as_intermediate_hop(&item.data, &endpoints)?
13370 })
13371 {
13372 let id = Self::get_hash(&item.data);
13373 let replay = {
13374 let mut st = self.state.lock();
13375 st.reliable_return_routes.get_mut(&id).is_some_and(|route| {
13376 let now = self.clock.now_ms();
13377 if now.saturating_sub(route.last_replay_ms)
13378 < runtime_reliable_retransmit_ms()
13379 {
13380 return false;
13381 }
13382 route.last_replay_ms = now;
13383 true
13384 })
13385 };
13386 if replay {
13387 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
13388 }
13389 }
13390 if item.src.is_some() {
13391 let local_sender = self.sender_arc();
13392 match &item.data {
13393 RouterItem::Packet(pkt)
13394 if (is_reliable_type(pkt.data_type())
13395 || !pkt.wire_target_senders().is_empty())
13396 && pkt.sender() != local_sender.as_ref()
13397 && self.item_targets_local_sender(&item.data)?
13398 && self.packet_has_local_delivery(pkt)
13399 && !(self.should_route_remote(&item.data, item.src)?
13400 && self.item_targets_router_as_intermediate_hop(
13401 &item.data,
13402 pkt.endpoints(),
13403 )?) =>
13404 {
13405 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
13406 }
13407 RouterItem::Packed(bytes) => {
13408 if let Ok(pkt) = self.unpack_packet_for_role(bytes.as_ref())
13409 && (is_reliable_type(pkt.data_type())
13410 || !pkt.wire_target_senders().is_empty())
13411 && pkt.sender() != local_sender.as_ref()
13412 && self.item_targets_local_sender(&item.data)?
13413 && self.packet_has_local_delivery(&pkt)
13414 && !(self.should_route_remote(&item.data, item.src)?
13415 && self.item_targets_router_as_intermediate_hop(
13416 &item.data,
13417 pkt.endpoints(),
13418 )?)
13419 {
13420 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
13421 .unwrap_or_else(|_| pkt.packet_id());
13422 self.queue_end_to_end_reliable_ack_for_packet_id(
13423 packet_id,
13424 pkt.sender(),
13425 called_from_queue,
13426 )?;
13427 }
13428 }
13429 _ => {}
13430 }
13431 }
13432 return Ok(());
13433 }
13434
13435 self.dispatch_rx_data(item, called_from_queue)?;
13436
13437 for release_bytes in released_buffered {
13438 let release_data = RouterItem::Packed(release_bytes.clone());
13439 if self.is_duplicate_pkt(&release_data)? {
13440 continue;
13441 }
13442 let release_item = RouterRxItem {
13443 src: item.src,
13444 priority: Self::router_item_priority(&release_data)?,
13445 data: release_data,
13446 };
13447 self.dispatch_rx_data(&release_item, called_from_queue)?;
13448 }
13449
13450 Ok(())
13451 }
13452
13453 fn dispatch_rx_data(
13454 &self,
13455 item: &RouterRxItem,
13456 called_from_queue: bool,
13457 ) -> TelemetryResult<()> {
13458 match &item.data {
13459 RouterItem::Packet(pkt) => {
13460 pkt.validate()?;
13461 #[cfg(feature = "discovery")]
13462 if let Some(side) = item.src {
13463 self.refresh_peer_liveness(side, sender_address_u32(pkt.sender()));
13464 }
13465
13466 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
13467 return Ok(());
13468 }
13469
13470 if pkt.data_type() == DataType::P2pMessage {
13471 if self.item_targets_local_sender(&item.data)? {
13472 self.dispatch_p2p_packet(pkt)?;
13473 if item.src.is_some() {
13474 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
13475 }
13476 }
13477 if self.should_route_remote(&item.data, item.src)? {
13478 self.relay_send(
13479 RouterItem::Packet(pkt.to_owned()),
13480 item.src,
13481 called_from_queue,
13482 )?;
13483 }
13484 return Ok(());
13485 }
13486
13487 #[cfg(feature = "timesync")]
13488 if matches!(
13489 pkt.data_type(),
13490 DataType::TimeSyncAnnounce
13491 | DataType::TimeSyncRequest
13492 | DataType::TimeSyncResponse
13493 ) {
13494 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
13495 return Ok(());
13496 }
13497
13498 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
13499 if !discovery_is_hop_local_advertisement(pkt.data_type())
13504 && self.should_route_remote(&item.data, item.src)?
13505 {
13506 self.relay_send(
13507 RouterItem::Packet(pkt.to_owned()),
13508 item.src,
13509 called_from_queue,
13510 )?;
13511 }
13512 return Ok(());
13513 }
13514
13515 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
13516 crate::small_sort::sort(&mut eps);
13517 eps.dedup();
13518 let had_local_handler = eps.iter().copied().any(|ep| {
13519 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
13520 });
13521 let had_local_delivery =
13522 had_local_handler || self.is_managed_variable_type(pkt.data_type());
13523
13524 let has_remote = self.should_route_remote(&item.data, item.src)?;
13525 let targets_local = self.item_targets_local_sender(&item.data)?;
13526
13527 let has_packed_local = eps
13528 .iter()
13529 .copied()
13530 .any(|ep| self.endpoint_has_packed_handler(ep));
13531 let bytes_opt = if has_packed_local {
13532 Some(wire_format::pack_packet(pkt))
13533 } else {
13534 None
13535 };
13536
13537 if targets_local {
13538 for dest in eps {
13539 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
13540 let result = match (&h.handler, &bytes_opt) {
13541 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
13542 .call_handler_with_retries(
13543 dest,
13544 h,
13545 Some(bytes.as_ref()),
13546 Some(pkt),
13547 None,
13548 called_from_queue,
13549 ),
13550 (EndpointHandlerFn::Packed(_), None) => {
13551 let bytes = wire_format::pack_packet(pkt);
13552 self.call_handler_with_retries(
13553 dest,
13554 h,
13555 Some(bytes.as_ref()),
13556 Some(pkt),
13557 None,
13558 called_from_queue,
13559 )
13560 }
13561 (EndpointHandlerFn::Packet(_), _) => self
13562 .call_handler_with_retries(
13563 dest,
13564 h,
13565 None,
13566 Some(pkt),
13567 None,
13568 called_from_queue,
13569 ),
13570 };
13571 if result.is_err()
13572 && let Some(src) = item.src
13573 {
13574 self.note_side_local_handler_failure(
13575 src,
13576 pkt.data_type(),
13577 runtime_max_handler_retries(),
13578 );
13579 }
13580 }
13581 }
13582 }
13583
13584 if let Some(src) = item.src
13585 && had_local_delivery
13586 && targets_local
13587 {
13588 self.note_side_local_delivery(src, pkt.data_type());
13589 }
13590
13591 if item.src.is_some()
13592 && had_local_delivery
13593 && targets_local
13594 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
13595 && !(has_remote
13596 && self
13597 .item_targets_router_as_intermediate_hop(&item.data, pkt.endpoints())?)
13598 {
13599 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
13600 }
13601
13602 if has_remote {
13603 let relay_item = RouterItem::Packet(pkt.to_owned());
13604 self.relay_send(relay_item, item.src, called_from_queue)?;
13605 }
13606
13607 Ok(())
13608 }
13609 RouterItem::Packed(bytes) => {
13610 let routing = wire_format::peek_routing_envelope(bytes.as_ref())?;
13614 if !is_internal_control_type(routing.ty)
13615 && !routing.is_reliable
13616 && routing.target_senders().len() == 0
13617 && !self.is_managed_variable_type(routing.ty)
13618 && !routing.endpoints().any(|ep| {
13619 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
13620 })
13621 {
13622 return self.relay_send(item.data.clone(), item.src, called_from_queue);
13626 }
13627
13628 let env = wire_format::peek_envelope(bytes.as_ref())?;
13629
13630 if matches!(
13631 env.ty,
13632 DataType::ReliableAck
13633 | DataType::ReliablePartialAck
13634 | DataType::ReliablePacketRequest
13635 ) {
13636 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13637 pkt.validate()?;
13638 let handled =
13639 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
13640 if !handled && self.should_route_remote(&item.data, item.src)? {
13643 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
13644 }
13645 return Ok(());
13646 }
13647
13648 if env.ty == DataType::P2pMessage {
13649 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13650 pkt.validate()?;
13651 if self.item_targets_local_sender(&item.data)? {
13652 self.dispatch_p2p_packet(&pkt)?;
13653 if item.src.is_some() {
13654 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
13655 .unwrap_or_else(|_| pkt.packet_id());
13656 self.queue_end_to_end_reliable_ack_for_packet_id(
13657 packet_id,
13658 pkt.sender(),
13659 called_from_queue,
13660 )?;
13661 }
13662 }
13663 if self.should_route_remote(&item.data, item.src)? {
13664 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
13665 }
13666 return Ok(());
13667 }
13668
13669 #[cfg(feature = "timesync")]
13670 if matches!(
13671 env.ty,
13672 DataType::TimeSyncAnnounce
13673 | DataType::TimeSyncRequest
13674 | DataType::TimeSyncResponse
13675 ) {
13676 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13677 pkt.validate()?;
13678 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
13679 return Ok(());
13680 }
13681
13682 #[cfg(feature = "discovery")]
13683 if discovery::is_discovery_type(env.ty) {
13684 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13685 pkt.validate()?;
13686 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
13687 if !discovery_is_hop_local_advertisement(env.ty)
13688 && self.should_route_remote(&item.data, item.src)?
13689 {
13690 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
13691 }
13692 return Ok(());
13693 }
13694
13695 let any_packet_needed = env
13696 .endpoints
13697 .iter()
13698 .copied()
13699 .any(|ep| self.endpoint_has_packet_handler(ep));
13700
13701 let mut pkt_opt = if any_packet_needed {
13702 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13703 pkt.validate()?;
13704 Some(pkt)
13705 } else {
13706 None
13707 };
13708
13709 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
13710 crate::small_sort::sort(&mut eps);
13711 eps.dedup();
13712 let had_local_handler = eps.iter().copied().any(|ep| {
13713 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
13714 });
13715 let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
13716
13717 let has_remote = self.should_route_remote(&item.data, item.src)?;
13718 let targets_local = self.item_targets_local_sender(&item.data)?;
13719
13720 if targets_local {
13721 for dest in eps {
13722 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
13723 let result = match &h.handler {
13724 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
13725 dest,
13726 h,
13727 Some(bytes.as_ref()),
13728 pkt_opt.as_ref(),
13729 Some(&env),
13730 called_from_queue,
13731 ),
13732 EndpointHandlerFn::Packet(_) => {
13733 if pkt_opt.is_none() {
13734 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13735 pkt.validate()?;
13736 pkt_opt = Some(pkt);
13737 }
13738 let pkt_ref = pkt_opt.as_ref().expect("just set");
13739 self.call_handler_with_retries(
13740 dest,
13741 h,
13742 None,
13743 Some(pkt_ref),
13744 Some(&env),
13745 called_from_queue,
13746 )
13747 }
13748 };
13749 if result.is_err()
13750 && let Some(src) = item.src
13751 {
13752 self.note_side_local_handler_failure(
13753 src,
13754 env.ty,
13755 runtime_max_handler_retries(),
13756 );
13757 }
13758 }
13759 }
13760 }
13761
13762 if item.src.is_some()
13763 && had_local_delivery
13764 && targets_local
13765 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
13766 && !(has_remote
13767 && self
13768 .item_targets_router_as_intermediate_hop(&item.data, &env.endpoints)?)
13769 {
13770 let packet_id =
13771 wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
13772 pkt_opt
13773 .as_ref()
13774 .map(Packet::packet_id)
13775 .ok_or(TelemetryError::Unpack("missing packet id"))
13776 })?;
13777 self.queue_end_to_end_reliable_ack_for_packet_id(
13778 packet_id,
13779 &env.sender,
13780 called_from_queue,
13781 )?;
13782 }
13783
13784 if has_remote {
13785 let relay_item = match pkt_opt {
13786 Some(ref p) => RouterItem::Packet(p.clone()),
13787 None => RouterItem::Packed(bytes.clone()),
13788 };
13789 self.relay_send(relay_item, item.src, called_from_queue)?;
13790 }
13791
13792 Ok(())
13793 }
13794 }
13795 }
13796
13797 fn dispatch_local_for_item(
13798 &self,
13799 item: &RouterItem,
13800 called_from_queue: bool,
13801 ) -> TelemetryResult<()> {
13802 match item {
13803 RouterItem::Packet(pkt) => {
13804 pkt.validate()?;
13805 if is_internal_control_type(pkt.data_type()) {
13806 return Ok(());
13807 }
13808 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
13809 if !self.item_targets_local_sender(item)? {
13810 return Ok(());
13811 }
13812
13813 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
13814 crate::small_sort::sort(&mut eps);
13815 eps.dedup();
13816
13817 let has_packed_local = eps
13818 .iter()
13819 .copied()
13820 .any(|ep| self.endpoint_has_packed_handler(ep));
13821 let bytes_opt = if has_packed_local {
13822 Some(wire_format::pack_packet(pkt))
13823 } else {
13824 None
13825 };
13826
13827 for dest in eps {
13828 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
13829 match (&h.handler, &bytes_opt) {
13830 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
13831 self.call_handler_with_retries(
13832 dest,
13833 h,
13834 Some(bytes.as_ref()),
13835 Some(pkt),
13836 None,
13837 called_from_queue,
13838 )?;
13839 }
13840 (EndpointHandlerFn::Packed(_), None) => {
13841 let bytes = wire_format::pack_packet(pkt);
13842 self.call_handler_with_retries(
13843 dest,
13844 h,
13845 Some(bytes.as_ref()),
13846 Some(pkt),
13847 None,
13848 called_from_queue,
13849 )?;
13850 }
13851 (EndpointHandlerFn::Packet(_), _) => {
13852 self.call_handler_with_retries(
13853 dest,
13854 h,
13855 None,
13856 Some(pkt),
13857 None,
13858 called_from_queue,
13859 )?;
13860 }
13861 }
13862 }
13863 }
13864 }
13865 RouterItem::Packed(bytes) => {
13866 let env = wire_format::peek_envelope(bytes.as_ref())?;
13867 if is_internal_control_type(env.ty) {
13868 return Ok(());
13869 }
13870 self.ensure_e2e_policy_supported_for_type(env.ty)?;
13871 if !self.item_targets_local_sender(item)? {
13872 return Ok(());
13873 }
13874
13875 let any_packet_needed = env
13876 .endpoints
13877 .iter()
13878 .copied()
13879 .any(|ep| self.endpoint_has_packet_handler(ep));
13880
13881 let mut pkt_opt = if any_packet_needed {
13882 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13883 pkt.validate()?;
13884 Some(pkt)
13885 } else {
13886 None
13887 };
13888
13889 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
13890 crate::small_sort::sort(&mut eps);
13891 eps.dedup();
13892
13893 for dest in eps {
13894 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
13895 match &h.handler {
13896 EndpointHandlerFn::Packed(_) => {
13897 self.call_handler_with_retries(
13898 dest,
13899 h,
13900 Some(bytes.as_ref()),
13901 pkt_opt.as_ref(),
13902 Some(&env),
13903 called_from_queue,
13904 )?;
13905 }
13906 EndpointHandlerFn::Packet(_) => {
13907 if pkt_opt.is_none() {
13908 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13909 pkt.validate()?;
13910 pkt_opt = Some(pkt);
13911 }
13912 let pkt_ref = pkt_opt.as_ref().expect("just set");
13913 self.call_handler_with_retries(
13914 dest,
13915 h,
13916 None,
13917 Some(pkt_ref),
13918 Some(&env),
13919 called_from_queue,
13920 )?;
13921 }
13922 }
13923 }
13924 }
13925 }
13926 }
13927
13928 Ok(())
13929 }
13930
13931 fn tx_item_impl(
13937 &self,
13938 item: RouterTxItem,
13939 ignore_local: bool,
13940 called_from_queue: bool,
13941 ) -> TelemetryResult<()> {
13942 match item {
13943 RouterTxItem::Broadcast(data) => {
13944 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
13945 if let RouterItem::Packet(pkt) = &data
13946 && !is_internal_control_type(pkt.data_type())
13947 {
13948 self.remember_managed_variable_packet(pkt)?;
13949 }
13950 #[cfg(feature = "discovery")]
13951 let is_discovery = matches!(&data, RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()))
13952 || matches!(&data, RouterItem::Packed(bytes)
13953 if wire_format::peek_envelope(bytes.as_ref())
13954 .map(|env| discovery::is_discovery_type(env.ty))
13955 .unwrap_or(false));
13956 if !ignore_local {
13957 if self.is_duplicate_pkt(&data)? {
13958 return Ok(());
13959 }
13960 #[cfg(feature = "discovery")]
13961 if !is_discovery
13962 && !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
13963 && !matches!(&data, RouterItem::Packed(bytes)
13964 if wire_format::peek_routing_envelope(bytes.as_ref())
13965 .map(|env| is_internal_control_type(env.ty))
13966 .unwrap_or(false))
13967 {
13968 self.dispatch_local_for_item(&data, called_from_queue)?;
13969 }
13970 #[cfg(not(feature = "discovery"))]
13971 if !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
13972 && !matches!(&data, RouterItem::Packed(bytes)
13973 if wire_format::peek_routing_envelope(bytes.as_ref())
13974 .map(|env| is_internal_control_type(env.ty))
13975 .unwrap_or(false))
13976 {
13977 self.dispatch_local_for_item(&data, called_from_queue)?;
13978 }
13979 }
13980
13981 let send_remote = match &data {
13982 RouterItem::Packet(pkt) => {
13983 pkt.validate()?;
13984 self.should_route_remote(&data, None)?
13985 }
13986 RouterItem::Packed(bytes) => {
13987 let _ = wire_format::peek_envelope(bytes.as_ref())?;
13988 self.should_route_remote(&data, None)?
13989 }
13990 };
13991
13992 if !send_remote {
13993 let (endpoints, ty) = self.item_route_info(&data)?;
13994 if !ignore_local
13995 && !is_internal_control_type(ty)
13996 && is_reliable_type(ty)
13997 && has_nonlocal_endpoint(&endpoints, &self.cfg)
13998 && !self
13999 .state
14000 .lock()
14001 .managed_variable_types
14002 .contains(&ty.as_u32())
14003 {
14004 self.remove_pkt_id(&data);
14008 return Err(TelemetryError::Io("no discovered route"));
14009 }
14010 return Ok(());
14011 }
14012 let mut data = data;
14013 let ty = match &data {
14014 RouterItem::Packet(pkt) => pkt.data_type(),
14015 RouterItem::Packed(bytes) => {
14016 wire_format::peek_routing_envelope(bytes.as_ref())?.ty
14017 }
14018 };
14019 if !ignore_local
14020 && !is_internal_control_type(ty)
14021 && self.item_requires_end_to_end_tracking(&data, ty)?
14022 {
14023 #[cfg(feature = "discovery")]
14024 {
14025 let pending = {
14026 let st = self.state.lock();
14027 let mut pending = self
14028 .expected_end_to_end_destinations_locked(&st, &data, None, false)?;
14029 self.filter_trackable_end_to_end_destinations_locked(
14030 &st,
14031 ty,
14032 &mut pending,
14033 false,
14034 );
14035 pending
14036 };
14037 if !pending.is_empty() {
14038 let original_data = data.clone();
14039 let mut targets: Vec<u64> = pending.keys().copied().collect();
14040 crate::small_sort::sort(&mut targets);
14041 targets.dedup();
14042 data = self.attach_wire_contract_to_item(data, &targets)?;
14043 match self.register_end_to_end_reliable_tx(&data, Some(pending)) {
14044 Ok(false) => return Ok(()),
14045 Ok(true) => {}
14046 Err(error) => {
14047 self.remove_pkt_id(&original_data);
14048 return Err(error);
14049 }
14050 }
14051 }
14052 }
14053 }
14054 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, None)?;
14055 for side in sides.iter().copied() {
14056 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
14057 if Self::is_transient_tx_error(&e) {
14058 self.tx_queue_item_for_retry(
14059 RouterTxItem::ToSide {
14060 src: None,
14061 dst: side,
14062 data: data.clone(),
14063 },
14064 true,
14065 )?;
14066 continue;
14067 }
14068 match &data {
14069 RouterItem::Packet(pkt) => {
14070 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
14071 }
14072 RouterItem::Packed(bytes) => {
14073 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
14074 let _ = self.handle_callback_error_from_env(
14075 &env,
14076 None,
14077 e,
14078 called_from_queue,
14079 );
14080 }
14081 }
14082 }
14083 return Err(TelemetryError::HandlerError("tx handler failed"));
14084 }
14085 }
14086 }
14087 RouterTxItem::ToSide { src, dst, data } => {
14088 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
14089 if let RouterItem::Packet(pkt) = &data
14090 && !is_internal_control_type(pkt.data_type())
14091 {
14092 self.remember_managed_variable_packet(pkt)?;
14093 }
14094 if !ignore_local {
14095 if self.is_duplicate_pkt(&data)? {
14096 return Ok(());
14097 }
14098 let suppress_local = matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
14099 || matches!(&data, RouterItem::Packed(bytes)
14100 if wire_format::peek_routing_envelope(bytes.as_ref())
14101 .map(|env| is_internal_control_type(env.ty))
14102 .unwrap_or(false));
14103 if !suppress_local {
14104 self.dispatch_local_for_item(&data, called_from_queue)?;
14105 }
14106 }
14107 let allowed = {
14108 let mut st = self.state.lock();
14109 let ty = match &data {
14110 RouterItem::Packet(pkt) => Some(pkt.data_type()),
14111 RouterItem::Packed(bytes) => {
14112 Some(wire_format::peek_routing_envelope(bytes.as_ref())?.ty)
14113 }
14114 };
14115 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
14116 #[cfg(all(feature = "discovery", feature = "timesync"))]
14117 let timesync_allowed = ty
14118 .map(|ty| {
14119 Self::timesync_allowed_for_side_locked(
14120 &mut st,
14121 dst,
14122 ty,
14123 self.clock.now_ms(),
14124 )
14125 })
14126 .unwrap_or(true);
14127 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
14128 let timesync_allowed = true;
14129 route_allowed && timesync_allowed
14130 };
14131 if !allowed {
14132 return Ok(());
14133 }
14134 if let Err(e) = self.send_reliable_to_side(dst, data.clone(), src.is_some()) {
14135 if Self::is_transient_tx_error(&e) {
14136 self.tx_queue_item_for_retry(
14137 RouterTxItem::ToSide { src, dst, data },
14138 true,
14139 )?;
14140 return Ok(());
14141 }
14142 match &data {
14143 RouterItem::Packet(pkt) => {
14144 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
14145 }
14146 RouterItem::Packed(bytes) => {
14147 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
14148 let _ = self.handle_callback_error_from_env(
14149 &env,
14150 None,
14151 e,
14152 called_from_queue,
14153 );
14154 }
14155 }
14156 }
14157 return Err(TelemetryError::HandlerError("tx handler failed"));
14158 }
14159 }
14160 RouterTxItem::EndToEndReplay { packet_id } => {
14161 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
14162 return Ok(());
14163 };
14164 if sides.is_empty() {
14165 let RemoteSidePlan::Target(fallback_sides) =
14166 self.remote_side_plan(&data, None)?;
14167 sides = fallback_sides;
14168 }
14169 for side in sides.iter().copied() {
14170 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
14171 if Self::is_transient_tx_error(&e) {
14172 self.tx_queue_item_for_retry(
14173 RouterTxItem::ToSide {
14174 src: None,
14175 dst: side,
14176 data: data.clone(),
14177 },
14178 true,
14179 )?;
14180 continue;
14181 }
14182 match &data {
14183 RouterItem::Packet(pkt) => {
14184 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
14185 }
14186 RouterItem::Packed(bytes) => {
14187 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
14188 let _ = self.handle_callback_error_from_env(
14189 &env,
14190 None,
14191 e,
14192 called_from_queue,
14193 );
14194 }
14195 }
14196 }
14197 return Err(TelemetryError::HandlerError("tx handler failed"));
14198 }
14199 }
14200 }
14201 RouterTxItem::ReliableReplay { dst, bytes } => {
14202 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
14203 let ty = frame.envelope.ty;
14204 let Some(hdr) = frame.reliable else {
14205 return Ok(());
14206 };
14207 {
14208 let mut st = self.state.lock();
14209 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
14210 if !tx_state.sent.contains_key(&hdr.seq) {
14211 return Ok(());
14212 }
14213 }
14214 if let Err(e) = self.send_reliable_raw_to_side(dst, bytes.clone(), false) {
14215 if Self::is_transient_tx_error(&e) {
14216 self.tx_queue_item_for_retry(
14217 RouterTxItem::ReliableReplay { dst, bytes },
14218 true,
14219 )?;
14220 return Ok(());
14221 }
14222 return Err(e);
14223 }
14224 let mut st = self.state.lock();
14225 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
14226 if let Some(sent) = tx_state.sent.get_mut(&hdr.seq) {
14227 sent.last_send_ms = self.clock.now_ms();
14228 sent.queued = false;
14229 }
14230 }
14231 }
14232
14233 Ok(())
14234 }
14235
14236 #[inline]
14238 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
14239 self.tx_item_impl(item, false, false)
14240 }
14241
14242 #[inline]
14249 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
14250 if self.side_tx_active() {
14251 return self.rx_packed_queue(bytes);
14252 }
14253 crate::memory_admission::check_receive(bytes)?;
14254 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
14255 let item = RouterRxItem {
14256 src: None,
14257 priority: Self::router_item_priority(&data)?,
14258 data,
14259 };
14260 self.rx_item(&item, false)
14261 }
14262
14263 #[inline]
14268 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
14269 if self.side_tx_active() {
14270 return self.rx_queue(pkt.clone());
14271 }
14272 let data = RouterItem::Packet(pkt.clone());
14273 let item = RouterRxItem {
14274 src: None,
14275 priority: Self::router_item_priority(&data)?,
14276 data,
14277 };
14278 self.rx_item(&item, false)
14279 }
14280
14281 #[inline]
14286 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
14287 if self.side_tx_active() {
14288 return self.rx_queue_from_side(pkt.clone(), side);
14289 }
14290 self.ensure_side_ingress_enabled(side)?;
14291 let data = RouterItem::Packet(pkt.clone());
14292 let item = RouterRxItem {
14293 src: Some(side),
14294 priority: Self::router_item_priority(&data)?,
14295 data,
14296 };
14297 self.rx_item(&item, false)
14298 }
14299
14300 #[inline]
14305 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
14306 if self.side_tx_active() {
14307 return self.rx_packed_queue_from_side(bytes, side);
14308 }
14309 self.ensure_side_ingress_enabled(side)?;
14310 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
14311 return Ok(());
14312 };
14313 let data = RouterItem::Packed(decoded);
14314 let item = RouterRxItem {
14315 src: Some(side),
14316 priority: Self::router_item_priority(&data)?,
14317 data,
14318 };
14319 self.rx_item(&item, false)
14320 }
14321
14322 #[inline]
14329 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
14330 #[cfg(feature = "discovery")]
14331 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14332 if self.side_tx_active() {
14333 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
14334 }
14335 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
14336 }
14337
14338 #[inline]
14343 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
14344 #[cfg(feature = "discovery")]
14345 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14346 if self.side_tx_active() {
14347 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
14348 crate::shared_bytes::convert(pkt),
14349 )));
14350 }
14351 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(
14352 crate::shared_bytes::convert(pkt),
14353 )))
14354 }
14355
14356 #[inline]
14362 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
14363 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
14364 }
14365
14366 #[inline]
14368 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
14369 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
14370 crate::shared_bytes::convert(data),
14371 )))
14372 }
14373
14374 #[inline]
14381 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
14382 #[cfg(feature = "discovery")]
14383 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14384 if self.side_tx_active() {
14385 return self.log_queue(ty, data);
14386 }
14387 let sender = self.sender_arc();
14388 log_raw(
14389 sender.as_ref(),
14390 ty,
14391 data,
14392 self.packet_timestamp_ms(),
14393 |pkt| self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
14394 )
14395 }
14396
14397 #[inline]
14399 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
14400 let sender = self.sender_arc();
14401 log_raw(
14402 sender.as_ref(),
14403 ty,
14404 data,
14405 self.packet_timestamp_ms(),
14406 |pkt| self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
14407 )
14408 }
14409
14410 #[inline]
14412 pub fn log_ts<T: LeBytes>(
14413 &self,
14414 ty: DataType,
14415 timestamp: u64,
14416 data: &[T],
14417 ) -> TelemetryResult<()> {
14418 #[cfg(feature = "discovery")]
14419 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
14420 if self.side_tx_active() {
14421 return self.log_queue_ts(ty, timestamp, data);
14422 }
14423 let sender = self.sender_arc();
14424 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
14425 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
14426 })
14427 }
14428
14429 #[inline]
14431 pub fn log_queue_ts<T: LeBytes>(
14432 &self,
14433 ty: DataType,
14434 timestamp: u64,
14435 data: &[T],
14436 ) -> TelemetryResult<()> {
14437 let sender = self.sender_arc();
14438 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
14439 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
14440 })
14441 }
14442}
14443
14444#[cfg(all(test, feature = "std", feature = "discovery"))]
14445mod queued_schema_identity_tests {
14446 use super::*;
14447
14448 fn catalog(timestamp: u64, byte: u8) -> Packet {
14449 Packet::new(
14450 crate::DataType::DiscoverySchema,
14451 &[DataEndpoint::Discovery],
14452 "SCHEMA_PEER",
14453 timestamp,
14454 crate::shared_bytes::convert(Arc::<[u8]>::from(vec![byte; 1024])),
14455 )
14456 .unwrap()
14457 }
14458
14459 #[test]
14460 fn repeated_schema_preserves_partial_transfer_but_updates_and_other_sides_do_not_coalesce() {
14461 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
14462 let side = router.add_side_packed("A", |_| Ok(()));
14463 let other_side = router.add_side_packed("B", |_| Ok(()));
14464 let original = catalog(1, 42);
14465 let duplicate = catalog(2, 42);
14466 assert_ne!(original.packet_id(), duplicate.packet_id());
14467 let mut st = router.state.lock();
14468 st.transmit_queue.clear();
14469 let queued = |dst, pkt| TxQueued {
14470 item: RouterTxItem::ToSide {
14471 src: None,
14472 dst,
14473 data: RouterItem::Packet(pkt),
14474 },
14475 ignore_local: true,
14476 priority: 255,
14477 retry_after_ms: 123,
14478 };
14479 st.push_transmit(queued(side, original.clone())).unwrap();
14480 st.push_transmit(queued(side, duplicate.clone())).unwrap();
14481 assert_eq!(st.transmit_queue.len(), 1);
14482 let pending = st.transmit_queue.iter().next().unwrap();
14483 assert_eq!(pending.retry_after_ms, 123);
14484 assert!(
14485 matches!(&pending.item, RouterTxItem::ToSide { data: RouterItem::Packet(pkt), .. }
14486 if pkt.packet_id() == original.packet_id())
14487 );
14488 st.push_transmit(queued(other_side, duplicate)).unwrap();
14489 assert_eq!(st.transmit_queue.len(), 2);
14490 let changed = catalog(3, 43);
14491 st.push_transmit(queued(side, changed.clone())).unwrap();
14492 assert_eq!(st.transmit_queue.len(), 2);
14493 assert!(st.transmit_queue.iter().any(|queued| matches!(&queued.item,
14494 RouterTxItem::ToSide { dst, data: RouterItem::Packet(pkt), .. }
14495 if *dst == side && pkt.packet_id() == changed.packet_id())));
14496 }
14497
14498 #[test]
14499 fn repeated_announcements_complete_one_fragment_per_dispatch() {
14500 use core::sync::atomic::{AtomicUsize, Ordering};
14501 let allowance = Arc::new(AtomicUsize::new(0));
14502 let accepted = Arc::new(std::sync::Mutex::new(Vec::<Arc<[u8]>>::new()));
14503 let available = allowance.clone();
14504 let frames = accepted.clone();
14505 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 1u64));
14506 let side = node.add_side_packed_with_options(
14507 "bounded CAN",
14508 move |bytes| {
14509 if available
14510 .try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
14511 count.checked_sub(1)
14512 })
14513 .is_err()
14514 {
14515 return Err(TelemetryError::Io("side tx busy"));
14516 }
14517 frames.lock().unwrap().push(Arc::<[u8]>::from(bytes));
14518 Ok(())
14519 },
14520 RouterSideOptions::default().with_small_packet_transport(128),
14521 );
14522 let handler = node.state.lock().sides[side]
14523 .as_ref()
14524 .unwrap()
14525 .tx_handler
14526 .clone();
14527 let mut complete = false;
14528 for timestamp in 1..100 {
14529 let mut state = 0x517cc1b7u32;
14531 let payload: Vec<u8> = (0..3600)
14532 .map(|_| {
14533 state ^= state << 13;
14534 state ^= state >> 17;
14535 state ^= state << 5;
14536 state as u8
14537 })
14538 .collect();
14539 let packet = Packet::new(
14540 crate::DataType::DiscoverySchema,
14541 &[crate::DataEndpoint::Discovery],
14542 "SCHEMA_PEER",
14543 timestamp,
14544 crate::shared_bytes::convert(Arc::<[u8]>::from(payload)),
14545 )
14546 .unwrap();
14547 node.state
14548 .lock()
14549 .push_transmit(TxQueued {
14550 item: RouterTxItem::ToSide {
14551 src: None,
14552 dst: side,
14553 data: RouterItem::Packet(packet),
14554 },
14555 ignore_local: true,
14556 priority: 255,
14557 retry_after_ms: 0,
14558 })
14559 .unwrap();
14560 let data = match &node.state.lock().transmit_queue.iter().next().unwrap().item {
14561 RouterTxItem::ToSide { data, .. } => data.clone(),
14562 _ => panic!("unexpected item"),
14563 };
14564 allowance.store(1, Ordering::Relaxed);
14565 match node.call_side_tx_handler(side, &handler, &data, false) {
14566 Ok(()) => {
14567 complete = true;
14568 break;
14569 }
14570 Err(TelemetryError::Io(_)) => {}
14571 other => panic!("unexpected result: {other:?}"),
14572 }
14573 }
14574 assert!(
14575 complete,
14576 "new announcement timestamps must not replay the prefix forever"
14577 );
14578 let received = accepted.lock().unwrap();
14579 assert!(
14582 received.len() > 1,
14583 "fixture must exercise fragmentation, not a compressed single frame"
14584 );
14585 let first = &received[0];
14586 assert_eq!(&first[..4], &[83, 68, 84, 3]);
14587 assert_eq!(
14588 u16::from_le_bytes([first[10], first[11]]) as usize,
14589 received.len()
14590 );
14591 let transfer_id = &first[4..8];
14592 for (index, frame) in received.iter().enumerate() {
14593 assert_eq!(&frame[4..8], transfer_id);
14594 assert_eq!(u16::from_le_bytes([frame[8], frame[9]]) as usize, index);
14595 }
14596 }
14597 #[test]
14598 fn schema_receipt_resolves_assigned_numeric_address_without_adding_peers() {
14599 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
14600 let side = router.add_side_packed("A", |_| Ok(()));
14601 let assigned = 0x1234u32;
14602 assert_ne!(sender_address_u32("SCHEMA_PEER"), assigned);
14603 {
14604 let mut st = router.state.lock();
14605 st.discovery_routes
14606 .entry(side)
14607 .or_default()
14608 .announcers
14609 .insert(
14610 "SCHEMA_PEER".into(),
14611 DiscoverySenderState {
14612 advertised_address: Some(assigned),
14613 ..DiscoverySenderState::default()
14614 },
14615 );
14616 }
14617 router.note_schema_received(side, "@addr:4660");
14618 router.note_schema_received(side, "@addr:4661");
14619 let st = router.state.lock();
14620 let peers = &st.discovery_routes.get(&side).unwrap().announcers;
14621 assert_eq!(peers.len(), 1);
14622 assert!(peers["SCHEMA_PEER"].has_schema);
14623 }
14624 #[test]
14625 fn busy_handler_returns_after_one_attempt_and_other_errors_keep_retry_budget() {
14626 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
14627 let attempts = core::cell::Cell::new(0);
14628 let result: Result<((), usize), _> = router.retry_with_attempts(8, || {
14629 attempts.set(attempts.get() + 1);
14630 Err(TelemetryError::Io("side tx busy"))
14631 });
14632 assert!(matches!(
14633 result,
14634 Err((TelemetryError::Io("side tx busy"), 1))
14635 ));
14636 assert_eq!(attempts.get(), 1);
14637 attempts.set(0);
14638 let result: Result<((), usize), _> = router.retry_with_attempts(3, || {
14639 attempts.set(attempts.get() + 1);
14640 Err(TelemetryError::Io("permanent handler failure"))
14641 });
14642 assert!(matches!(
14643 result,
14644 Err((TelemetryError::Io("permanent handler failure"), 3))
14645 ));
14646 assert_eq!(attempts.get(), 3);
14647 }
14648 #[test]
14649 fn rejected_first_template_frame_is_retried_self_describing() {
14650 crate::tests::ensure_common_test_schema();
14651 let ty = crate::config::register_data_type_with_description(
14652 "ROUTER_BUSY_FIRST_TEMPLATE",
14653 "",
14654 crate::MessageElement::Static(
14655 1,
14656 crate::MessageDataType::UInt8,
14657 crate::MessageClass::Data,
14658 ),
14659 &[crate::DataEndpoint::named("RADIO")],
14660 crate::ReliableMode::None,
14661 1,
14662 )
14663 .unwrap();
14664 let tx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
14665 let rx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
14666 let opts = RouterSideOptions::default().with_small_packet_transport(0);
14667 let a = tx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
14668 let b = rx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
14669 let packet = Packet::new(
14670 ty,
14671 &[crate::DataEndpoint::named("RADIO")],
14672 "BUSY_PEER",
14673 1,
14674 crate::shared_bytes::convert(Arc::<[u8]>::from([42u8])),
14675 )
14676 .unwrap();
14677 let raw = wire_format::pack_packet(&packet);
14678 let first = tx
14680 .encode_side_transport_frames(
14681 a,
14682 opts.clone(),
14683 crate::shared_bytes::convert(raw.clone()),
14684 )
14685 .unwrap();
14686 assert!(
14687 tx.send_side_frames_resumable(a, first, |_| Err(TelemetryError::Io("side tx busy")))
14688 .is_err()
14689 );
14690 let retry = tx
14691 .encode_side_transport_frames(
14692 a,
14693 opts.clone(),
14694 crate::shared_bytes::convert(raw.clone()),
14695 )
14696 .unwrap();
14697 let mut received = None;
14698 tx.send_side_frames_resumable(a, retry, |frame| {
14699 assert_eq!(
14700 frame[3], SIDE_TRANSPORT_KIND_FULL,
14701 "retry requires the unaccepted dictionary bootstrap"
14702 );
14703 received = rx.decode_side_transport_frame(b, frame)?;
14704 Ok(())
14705 })
14706 .unwrap();
14707 assert_eq!(received, Some(crate::shared_bytes::convert(raw.clone())));
14708 let next = tx
14710 .encode_side_transport_frames(a, opts, crate::shared_bytes::convert(raw.clone()))
14711 .unwrap();
14712 tx.send_side_frames_resumable(a, next, |frame| {
14713 assert_eq!(frame[3], SIDE_TRANSPORT_KIND_COMPACT);
14714 assert_eq!(
14715 rx.decode_side_transport_frame(b, frame)?,
14716 Some(crate::shared_bytes::convert(raw.clone()))
14717 );
14718 Ok(())
14719 })
14720 .unwrap();
14721 }
14722 #[test]
14723 fn already_queued_receipt_precedes_ordered_retry_exhaustion() {
14724 crate::tests::ensure_common_test_schema();
14725 let ep = DataEndpoint::named("RADIO");
14726 let ty = crate::config::register_data_type_with_description(
14727 "QUEUED_RECEIPT_DEADLINE",
14728 "",
14729 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
14730 &[ep],
14731 crate::ReliableMode::Ordered,
14732 1,
14733 )
14734 .unwrap();
14735 for isr in [false, true] {
14736 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
14737 let clock = now.clone();
14738 let router = Router::new_with_clock(
14739 RouterConfig::new([]).with_hostname("FC"),
14740 Box::new(move || clock.load(Ordering::Relaxed)),
14741 );
14742 let side = router.add_side_packed("CAN", |_| Ok(()));
14743 let packet = Packet::new(ty, &[ep], "FC", 1, Arc::<[u8]>::from([42u8])).unwrap();
14744 let id = packet.packet_id();
14745 let owner = Router::sender_hash("GS");
14746 let mut destinations = BTreeMap::new();
14747 destinations.insert(owner, side);
14748 router.state.lock().end_to_end_reliable_tx.insert(
14749 id,
14750 EndToEndReliableSent {
14751 ordered_type: Some(ty.as_u32()),
14752 waiting_for_predecessor: false,
14753 data: RouterItem::Packet(packet),
14754 pending_destinations: destinations,
14755 tracked_destinations: true,
14756 last_send_ms: 0,
14757 retries: runtime_reliable_max_retries(),
14758 queued: false,
14759 },
14760 );
14761 let snapshot = router.export_reliable_pending_stats();
14762 assert_eq!(snapshot.len(), 1);
14763 assert_eq!(snapshot[0].packet_id, id);
14764 assert_eq!(snapshot[0].data_type, ty);
14765 assert_eq!(snapshot[0].pending_destinations, vec![owner]);
14766 assert_eq!(snapshot[0].retries, runtime_reliable_max_retries());
14767 assert!(!snapshot[0].queued);
14768 assert!(!snapshot[0].waiting_for_predecessor);
14769 let ack = Packet::new_with_wire_contract(
14770 DataType::ReliableAck,
14771 message_meta(DataType::ReliableAck).endpoints_ref(),
14772 "E2EACK:GS",
14773 1,
14774 1,
14775 Arc::<[u8]>::from(id.to_le_bytes()),
14776 None,
14777 Arc::from([owner, Router::sender_hash("FC")]),
14778 )
14779 .unwrap();
14780 let packed = wire_format::pack_packet(&ack);
14781 if isr {
14782 router.rx_packed_queue_from_side_isr(&packed, side).unwrap();
14783 } else {
14784 router.rx_packed_queue_from_side(&packed, side).unwrap();
14785 }
14786 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
14787 router
14788 .drain_queues_with_timeout(1)
14789 .expect("a received ACK must be processed before cancelling its ordered delivery");
14790 assert!(!router.state.lock().end_to_end_reliable_tx.contains_key(&id));
14791 assert!(router.export_reliable_pending_stats().is_empty());
14792 }
14793 }
14794 #[test]
14795 fn embedded_gateway_receipt_with_numeric_sender_clears_host_pending_owner() {
14796 crate::tests::ensure_common_test_schema();
14797 let bytes = [
14800 0x0c, 0x01, 0x01, 0x08, 0xd4, 0xe5, 0xea, 0xb6, 0xb7, 0x33, 0xff, 0x90, 0x02, 0xb4,
14801 0xe6, 0xdd, 0x94, 0x07, 0x14, 0x03, 0x44, 0x02, 0x02, 0x06, 0x25, 0xd2, 0x59, 0x1b,
14802 0xbb, 0xfc, 0xa7, 0x5f, 0x43, 0x92, 0x9e, 0x12, 0xbb, 0xfc, 0xa7, 0x26, 0xa2, 0xa0,
14803 0xbe, 0x39, 0x65, 0xe2, 0xd8, 0x61, 0x68, 0x22, 0xfd,
14804 ];
14805 let ack = wire_format::unpack_packet(&bytes).unwrap();
14806 let owner = Router::sender_hash("GB");
14807 assert_eq!(Router::end_to_end_ack_sender_hash(&ack), Some(owner));
14808 let id = Router::decode_end_to_end_reliable_ack(ack.payload()).unwrap();
14809 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
14810 let side = router.add_side_packed("Pico", |_| Ok(()));
14811 let data = ack.clone();
14812 router.state.lock().end_to_end_reliable_tx.insert(
14813 id,
14814 EndToEndReliableSent {
14815 ordered_type: None,
14816 waiting_for_predecessor: false,
14817 data: RouterItem::Packet(data),
14818 pending_destinations: BTreeMap::from([(owner, side)]),
14819 tracked_destinations: true,
14820 last_send_ms: 0,
14821 retries: 0,
14822 queued: false,
14823 },
14824 );
14825 router.rx_packed_queue_from_side(&bytes, side).unwrap();
14826 router.drain_queues_with_timeout(1).unwrap();
14827 assert!(router.export_reliable_pending_stats().is_empty());
14828 }
14829
14830 #[test]
14831 fn frozen_delivery_contract_bypasses_dictionary_even_for_best_effort_schema() {
14832 crate::tests::ensure_common_test_schema();
14833 let ep = DataEndpoint::named("RADIO");
14834 let ty = crate::config::register_data_type_with_description(
14835 "MANAGED_CONTROL_WITH_FROZEN_OWNERS",
14836 "",
14837 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
14838 &[ep],
14839 crate::ReliableMode::None,
14840 16,
14841 )
14842 .unwrap();
14843 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
14844 let opts = RouterSideOptions::default().with_template_transport();
14845 let side = router.add_side_packed_with_options("radio", |_| Ok(()), opts.clone());
14846 let packet = Packet::new_with_wire_contract(
14847 ty,
14848 &[ep],
14849 "GS",
14850 1,
14851 1,
14852 Arc::<[u8]>::from([1u8]),
14853 None,
14854 Arc::from([Router::sender_hash("RF")]),
14855 )
14856 .unwrap();
14857 let raw = wire_format::pack_packet(&packet);
14858 assert_eq!(
14859 router
14860 .item_service_priority(&RouterItem::Packet(packet.clone()))
14861 .unwrap(),
14862 254
14863 );
14864 assert_eq!(
14865 router
14866 .item_service_priority(&RouterItem::Packed(crate::shared_bytes::convert(
14867 raw.clone()
14868 )))
14869 .unwrap(),
14870 254
14871 );
14872 assert!(
14873 !wire_format::peek_routing_frame_info(&raw)
14874 .unwrap()
14875 .envelope
14876 .is_reliable
14877 );
14878 for _ in 0..3 {
14879 let frames = router
14880 .encode_side_transport_frames(
14881 side,
14882 opts.clone(),
14883 crate::shared_bytes::convert(raw.clone()),
14884 )
14885 .unwrap();
14886 let mut count = 0;
14887 router
14888 .send_side_frames_resumable(side, frames, |bytes| {
14889 assert_eq!(
14890 bytes,
14891 raw.as_ref(),
14892 "delivery contract must remain independently decodable"
14893 );
14894 count += 1;
14895 Ok(())
14896 })
14897 .unwrap();
14898 assert_eq!(count, 1);
14899 }
14900 }
14901
14902 #[test]
14903 fn managed_receive_queues_keep_service_priority_with_low_schema_priority() {
14904 crate::tests::ensure_common_test_schema();
14905 let ep = DataEndpoint::named("RADIO");
14906 let ty = crate::config::register_data_type_with_description(
14907 "MANAGED_RX_SERVICE_PRIORITY",
14908 "",
14909 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
14910 &[ep],
14911 crate::ReliableMode::None,
14912 16,
14913 )
14914 .unwrap();
14915 let router = Router::new(RouterConfig::new([]).with_hostname("RF"));
14916 router
14917 .enable_network_variable(ty, NetworkVariablePermissions::READ_ONLY)
14918 .unwrap();
14919 let side = router.add_side_packed("radio", |_| Ok(()));
14920 router.clear_queues();
14921 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
14922 let wire = wire_format::pack_packet(&packet);
14923 router.rx_queue(packet.clone()).unwrap();
14924 router.rx_queue_from_side(packet.clone(), side).unwrap();
14925 router.rx_packed_queue(&wire).unwrap();
14926 router.rx_packed_queue_from_side(&wire, side).unwrap();
14927 for _ in 0..4 {
14928 let queued = router
14929 .pop_next_queued_control_rx_item()
14930 .unwrap()
14931 .expect("registered managed controls must precede queued bulk work");
14932 assert_eq!(
14933 queued.priority,
14934 crate::transport_priority(DataType::ManagedVariableValue)
14935 );
14936 }
14937 assert!(router.pop_next_queued_control_rx_item().unwrap().is_none());
14938 }
14939
14940 #[test]
14941 fn managed_value_keeps_service_priority_after_side_backpressure() {
14942 crate::tests::ensure_common_test_schema();
14943 let ep = DataEndpoint::named("RADIO");
14944 let ty = crate::config::register_data_type_with_description(
14945 "MANAGED_RETRY_SERVICE_PRIORITY",
14946 "",
14947 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
14948 &[ep],
14949 crate::ReliableMode::None,
14950 16,
14951 )
14952 .unwrap();
14953 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
14954 router
14955 .enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
14956 .unwrap();
14957 let side = router.add_side_packed("radio", |_| Err(TelemetryError::Io("side tx busy")));
14958 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
14959 let item = RouterItem::Packet(packet);
14960 for queued in [
14961 RouterTxItem::Broadcast(item.clone()),
14962 RouterTxItem::ToSide {
14963 src: None,
14964 dst: side,
14965 data: item.clone(),
14966 },
14967 ] {
14968 router.tx_queue_item_for_retry(queued, true).unwrap();
14969 let queued = router.state.lock().transmit_queue.pop_front().unwrap();
14970 assert_eq!(
14971 queued.priority,
14972 crate::transport_priority(DataType::ManagedVariableValue),
14973 "a busy side must not demote managed controls to bulk application priority"
14974 );
14975 assert!(queued.retry_after_ms > 0);
14976 }
14977 }
14978 #[test]
14979 fn receipt_budget_does_not_starve_transmit() {
14980 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
14981 crate::tests::ensure_common_test_schema();
14982 let now = Arc::new(AtomicU64::new(0));
14983 let clock = now.clone();
14984 let count = Arc::new(AtomicUsize::new(0));
14985 let accepted = count.clone();
14986 let node = Router::new_with_clock(
14987 RouterConfig::default(),
14988 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
14989 );
14990 let side = node.add_side_packed("wire", move |_| {
14991 accepted.fetch_add(1, Ordering::Relaxed);
14992 Ok(())
14993 });
14994 node.clear_queues();
14995 let packet = Packet::new(
14996 crate::DataType::ReliableAck,
14997 &[crate::DataEndpoint::TelemetryError],
14998 "E2EACK:PEER",
14999 1,
15000 Arc::<[u8]>::from([0u8; 8]),
15001 )
15002 .unwrap();
15003 node.state
15004 .lock()
15005 .push_transmit(TxQueued {
15006 item: RouterTxItem::ToSide {
15007 src: None,
15008 dst: side,
15009 data: RouterItem::Packet(packet.clone()),
15010 },
15011 ignore_local: true,
15012 priority: 250,
15013 retry_after_ms: 0,
15014 })
15015 .unwrap();
15016 node.rx_packed_queue_from_side(
15017 &wire_format::pack_packet(
15018 &Packet::new(
15019 crate::DataType::ReliableAck,
15020 &[crate::DataEndpoint::TelemetryError],
15021 "RX_PEER",
15022 2,
15023 Arc::<[u8]>::from([0u8; 8]),
15024 )
15025 .unwrap(),
15026 ),
15027 side,
15028 )
15029 .unwrap();
15030 node.drain_queues_with_timeout(1).unwrap();
15031 assert_eq!(
15032 count.load(Ordering::Relaxed),
15033 1,
15034 "control RX exhausting its cooperative budget must still allow one queued TX"
15035 );
15036 }
15037}
15038
15039#[cfg(all(test, feature = "discovery", feature = "std"))]
15040fn untemplated_test_node() -> Router {
15041 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0))
15042}
15043#[cfg(all(test, feature = "discovery", feature = "std"))]
15044fn untemplated_test_options() -> RouterSideOptions {
15045 RouterSideOptions::default().with_small_packet_transport(64)
15046}
15047#[cfg(all(test, feature = "discovery", feature = "std"))]
15048#[allow(clippy::duplicate_mod)]
15049#[path = "tests/untemplated_transport.rs"]
15050mod untemplated_transport_tests;
15051
15052#[cfg(all(test, feature = "std", feature = "discovery"))]
15053mod queued_producer_maintenance_tests {
15054 use super::*;
15055 #[test]
15056 fn queued_publication_defers_discovery_to_worker_maintenance() {
15057 crate::tests::ensure_common_test_schema();
15058 let ty = DataType::named("GPS_DATA");
15059 for variant in 0..4 {
15060 let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
15061 let output = sent.clone();
15062 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
15063 router.add_side_packed("wire", move |bytes| {
15064 output
15065 .lock()
15066 .unwrap()
15067 .push(wire_format::unpack_packet(bytes)?.data_type());
15068 Ok(())
15069 });
15070 router.clear_queues();
15071 let packet =
15072 Packet::from_f32_slice(ty, &[1.0, 2.0, 3.0], message_meta(ty).endpoints_ref(), 42)
15073 .unwrap();
15074 match variant {
15075 0 => router.tx_queue(packet).unwrap(),
15076 1 => router
15077 .tx_packed_queue(wire_format::pack_packet(&packet))
15078 .unwrap(),
15079 2 => router.log_queue(ty, &[1.0f32, 2.0, 3.0]).unwrap(),
15080 _ => router.log_queue_ts(ty, 42, &[1.0f32, 2.0, 3.0]).unwrap(),
15081 }
15082 assert!(
15083 sent.lock().unwrap().is_empty(),
15084 "producer invoked a transport"
15085 );
15086 assert_eq!(
15087 router.state.lock().transmit_queue.len(),
15088 1,
15089 "queued API performed discovery on the producer stack (variant {variant})"
15090 );
15091 router.periodic(0).unwrap();
15092 let sent = sent.lock().unwrap();
15093 assert!(sent.contains(&ty));
15094 assert!(
15095 sent.iter().copied().any(discovery::is_discovery_type),
15096 "worker maintenance must still announce the board"
15097 );
15098 }
15099 }
15100}
15101
15102#[cfg(all(test, feature = "std"))]
15103mod bounded_fair_queue_tests {
15104 use super::*;
15105 #[test]
15106 fn small_canonical_reports_survive_fast_senders_fixed_ring_bursts() {
15107 let ep = crate::config::register_endpoint("ROUTER_SMALL_QUEUE_DEST", false).unwrap();
15108 let ty = crate::config::register_data_type_with_description(
15109 "ROUTER_SMALL_QUEUE_REPORT",
15110 "",
15111 crate::MessageElement::Static(
15112 4,
15113 crate::MessageDataType::UInt8,
15114 crate::MessageClass::Data,
15115 ),
15116 &[ep],
15117 crate::ReliableMode::None,
15118 0,
15119 )
15120 .unwrap();
15121 for tx in [false, true] {
15122 for slow_last in [false, true] {
15123 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
15124 let make = |sender| {
15125 RouterItem::Packed(crate::shared_bytes::convert(wire_format::pack_packet(
15126 &Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 4])).unwrap(),
15127 )))
15128 };
15129 let rx = |data| RouterRxItem {
15130 src: None,
15131 data,
15132 priority: 0,
15133 };
15134 let tx_item = |data| TxQueued {
15135 item: RouterTxItem::Broadcast(data),
15136 ignore_local: true,
15137 priority: 0,
15138 retry_after_ms: 0,
15139 };
15140 let mut st = node.state.lock();
15141 st.memory.max_queue_budget = usize::MAX;
15142 if tx {
15143 st.transmit_queue = BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
15144 } else {
15145 st.received_queue = BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
15146 }
15147 let order = if slow_last {
15148 ["FAST", "FAST", "SLOW"]
15149 } else {
15150 ["SLOW", "FAST", "FAST"]
15151 };
15152 for sender in order.into_iter().chain(core::iter::repeat_n("FAST", 32)) {
15153 if tx {
15154 st.push_transmit(tx_item(make(sender))).unwrap();
15155 } else {
15156 st.push_received(rx(make(sender))).unwrap();
15157 }
15158 }
15159 let slow = wire_format::source_address_for_sender("SLOW");
15160 let retained = if tx {
15161 st.transmit_queue
15162 .iter()
15163 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
15164 } else {
15165 st.received_queue.iter().any(|item| {
15166 RouterInner::fair_bulk_owner(&item.data, item.priority) == Some(slow)
15167 })
15168 };
15169 assert!(
15170 retained,
15171 "small report was displaced: tx={tx}, slow_last={slow_last}"
15172 );
15173 }
15174 }
15175 }
15176
15177 #[test]
15178 fn slow_stream_survives_same_board_fast_stream_bursts() {
15179 let ep = crate::config::register_endpoint("ROUTER_STREAM_QUEUE_DEST", false).unwrap();
15180 let ty = crate::config::register_data_type_with_description(
15181 "ROUTER_STREAM_QUEUE_FAST",
15182 "",
15183 crate::MessageElement::Static(
15184 64,
15185 crate::MessageDataType::UInt8,
15186 crate::MessageClass::Data,
15187 ),
15188 &[ep],
15189 crate::ReliableMode::None,
15190 0,
15191 )
15192 .unwrap();
15193 let slow_ty = crate::config::register_data_type_with_description(
15194 "ROUTER_STREAM_QUEUE_SLOW",
15195 "",
15196 crate::MessageElement::Static(
15197 64,
15198 crate::MessageDataType::UInt8,
15199 crate::MessageClass::Data,
15200 ),
15201 &[ep],
15202 crate::ReliableMode::None,
15203 0,
15204 )
15205 .unwrap();
15206 for mode in 0..3 {
15207 for tx in [false, true] {
15208 for slow_last in [false, true] {
15209 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
15210 let make = |sender| {
15211 let packet = Packet::new(
15212 if sender == "SLOW" { slow_ty } else { ty },
15213 &[ep],
15214 "SAME_BOARD",
15215 1,
15216 Arc::<[u8]>::from([1; 64]),
15217 )
15218 .unwrap();
15219 if mode == 1 {
15220 RouterItem::Packet(packet)
15221 } else {
15222 RouterItem::Packed(crate::shared_bytes::convert(
15223 wire_format::pack_packet(&packet),
15224 ))
15225 }
15226 };
15227 let rx = |data| RouterRxItem {
15228 src: None,
15229 data,
15230 priority: 0,
15231 };
15232 let tx_item = |data| TxQueued {
15233 item: RouterTxItem::Broadcast(data),
15234 ignore_local: true,
15235 priority: 0,
15236 retry_after_ms: 0,
15237 };
15238 let mut st = node.state.lock();
15239 st.memory.max_queue_budget = usize::MAX;
15240 if mode == 0 {
15241 if tx {
15242 st.transmit_queue =
15243 BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
15244 } else {
15245 st.received_queue =
15246 BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
15247 }
15248 } else if mode == 1 {
15249 let cost = if tx {
15250 tx_item(make("SLOW")).byte_cost()
15251 + 2 * tx_item(make("FAST")).byte_cost()
15252 } else {
15253 rx(make("SLOW")).byte_cost() + 2 * rx(make("FAST")).byte_cost()
15254 };
15255 if tx {
15256 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
15257 } else {
15258 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
15259 }
15260 }
15261 let order = if slow_last {
15262 ["FAST", "FAST", "SLOW"]
15263 } else {
15264 ["SLOW", "FAST", "FAST"]
15265 };
15266 for (index, sender) in order
15267 .into_iter()
15268 .chain(core::iter::repeat_n("FAST", 32))
15269 .enumerate()
15270 {
15271 if mode == 2 && index == 3 {
15272 st.memory.max_queue_budget = st.shared_queue_bytes_used();
15273 }
15274 if tx {
15275 st.push_transmit(tx_item(make(sender))).unwrap();
15276 } else {
15277 st.push_received(rx(make(sender))).unwrap();
15278 }
15279 }
15280 let is_slow = |data: &RouterItem| match data {
15281 RouterItem::Packet(p) => p.data_type() == slow_ty,
15282 RouterItem::Packed(b) => {
15283 wire_format::scheduling_type(b.as_ref()) == Some(slow_ty)
15284 }
15285 };
15286 let retained = if tx {
15287 st.transmit_queue.iter().any(|item| {
15288 let data = match &item.item {
15289 RouterTxItem::Broadcast(data)
15290 | RouterTxItem::ToSide { data, .. } => data,
15291 _ => return false,
15292 };
15293 is_slow(data)
15294 })
15295 } else {
15296 st.received_queue.iter().any(|item| is_slow(&item.data))
15297 };
15298 assert!(
15299 retained,
15300 "slow stream displaced: mode={mode}, tx={tx}, slow_last={slow_last}"
15301 );
15302 }
15303 }
15304 }
15305 }
15306
15307 #[test]
15308 fn fast_producer_preserves_slow_reports_at_each_queue_boundary() {
15309 let ep = crate::config::register_endpoint("QUEUE_FAIR_DEST", false).unwrap();
15310 let ty = crate::config::register_data_type_with_description(
15311 "QUEUE_FAIR_REPORT",
15312 "",
15313 MessageElement::Static(64, crate::MessageDataType::UInt8, crate::MessageClass::Data),
15314 &[ep],
15315 crate::ReliableMode::None,
15316 0,
15317 )
15318 .unwrap();
15319 for mode in 0..3 {
15320 for tx in [false, true] {
15321 for slow_last in [false, true] {
15322 for next_sender in ["FAST", "NEW_SLOW"] {
15323 let router =
15324 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
15325 let make = |sender| {
15326 RouterItem::Packet(
15327 Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 64]))
15328 .unwrap(),
15329 )
15330 };
15331 let rx = |data| RouterRxItem {
15332 src: None,
15333 data,
15334 priority: 0,
15335 };
15336 let tx_item = |data| TxQueued {
15337 item: RouterTxItem::Broadcast(data),
15338 ignore_local: true,
15339 priority: 0,
15340 retry_after_ms: 0,
15341 };
15342 let order = if slow_last {
15343 ["FAST", "FAST", "SLOW"]
15344 } else {
15345 ["SLOW", "FAST", "FAST"]
15346 };
15347 let mut st = router.state.lock();
15348 st.memory.max_queue_budget = usize::MAX;
15349 let cost = if tx {
15350 order.iter().map(|s| tx_item(make(s)).byte_cost()).sum()
15351 } else {
15352 order.iter().map(|s| rx(make(s)).byte_cost()).sum()
15353 };
15354 if mode == 0 {
15355 if tx {
15356 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
15357 } else {
15358 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
15359 }
15360 } else if mode == 1 {
15361 if tx {
15362 st.transmit_queue =
15363 BoundedDeque::new(cost * 10, 3 * size_of::<TxQueued>(), 1.0);
15364 } else {
15365 st.received_queue = BoundedDeque::new(
15366 cost * 10,
15367 3 * size_of::<RouterRxItem>(),
15368 1.0,
15369 );
15370 }
15371 }
15372 for sender in order {
15373 if tx {
15374 st.push_transmit(tx_item(make(sender))).unwrap();
15375 } else {
15376 st.push_received(rx(make(sender))).unwrap();
15377 }
15378 }
15379 if mode == 2 {
15380 st.memory.max_queue_budget = st.shared_queue_bytes_used();
15381 }
15382 if tx {
15383 st.push_transmit(tx_item(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
15384 } else {
15385 st.push_received(rx(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
15386 }
15387 let slow = wire_format::source_address_for_sender("SLOW");
15388 let retained = if tx {
15389 st.transmit_queue
15390 .iter()
15391 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
15392 } else {
15393 st.received_queue.iter().any(|item| {
15394 RouterInner::fair_bulk_owner(&item.data, item.priority)
15395 == Some(slow)
15396 })
15397 };
15398 assert!(
15399 retained,
15400 "slow report evicted: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"
15401 );
15402 }
15403 }
15404 }
15405 }
15406 }
15407}
15408
15409#[cfg(all(test, feature = "discovery"))]
15410#[test]
15411fn cow_discovery_rejected_growth_keeps_original_graph_and_routes() {
15412 crate::tests::ensure_common_test_schema();
15413 let node = Router::new_with_clock(RouterConfig::default().with_sender("GB"), Box::new(|| 0));
15414 let side = node.add_side_packed("can", |_| Ok(()));
15415 let baseline = discovery::build_discovery_topology(
15416 "PEER",
15417 0,
15418 &[TopologyBoardNode {
15419 sender_id: "PEER".into(),
15420 reachable_endpoints: Vec::new(),
15421 reachable_timesync_sources: Vec::new(),
15422 connections: Vec::new(),
15423 }],
15424 )
15425 .unwrap();
15426 node.rx_from_side(&baseline, side).unwrap();
15427 node.process_all_queues().unwrap();
15428 node.clear_queues();
15429 let before = node.export_topology();
15430 let original = {
15431 let mut st = node.state.lock();
15432 let graph = st.discovery_routes[&side].announcers["PEER"]
15433 .topology_boards
15434 .clone();
15435 st.memory.max_queue_budget = st.shared_queue_bytes_used();
15436 graph
15437 };
15438 let update = discovery::build_discovery_topology_delta(
15439 "PEER",
15440 1,
15441 &[TopologyBoardNode {
15442 sender_id: "PEER".into(),
15443 reachable_endpoints: Vec::new(),
15444 reachable_timesync_sources: vec!["NEW_CLOCK".into()],
15445 connections: vec!["NEW_NEIGHBOR".into()],
15446 }],
15447 &[],
15448 )
15449 .unwrap();
15450 assert!(
15451 node.rx_from_side(&update, side)
15452 .and_then(|_| node.process_all_queues())
15453 .is_err(),
15454 "growth must fail when no queue budget can be reclaimed"
15455 );
15456 assert_eq!(node.export_topology().routes, before.routes);
15457 let st = node.state.lock();
15458 assert!(Arc::ptr_eq(
15459 &original,
15460 &st.discovery_routes[&side].announcers["PEER"].topology_boards
15461 ));
15462 assert!(original[0].connections.is_empty());
15463 assert!(original[0].reachable_timesync_sources.is_empty());
15464}