1#[cfg(test)]
12#[path = "tests/relay_legacy_contracts.rs"]
13mod relay_legacy_contracts;
14
15#[cfg(all(test, feature = "std", feature = "discovery"))]
16#[path = "tests/shared_routing_roles.rs"]
17mod shared_routing_roles;
18
19#[cfg(all(test, feature = "std", feature = "discovery"))]
20#[path = "tests/schema_prefix_admission.rs"]
21mod schema_prefix_admission;
22
23#[cfg(all(test, feature = "std", feature = "discovery"))]
24#[path = "tests/schema_receive_progress.rs"]
25mod schema_receive_progress;
26
27#[cfg(all(test, feature = "std", feature = "discovery"))]
28#[path = "tests/radio_trace_decoder.rs"]
29mod radio_trace_decoder;
30
31#[cfg(all(test, feature = "std", feature = "discovery"))]
32#[path = "tests/schema_response_admission.rs"]
33mod schema_response_admission_tests;
34
35#[cfg(all(test, feature = "std", feature = "discovery"))]
36#[path = "tests/accepted_route_ownership.rs"]
37mod accepted_route_ownership;
38
39#[cfg(all(test, feature = "std"))]
40#[path = "tests/independent_side_tx_router.rs"]
41mod independent_side_tx;
42#[cfg(all(test, feature = "std", feature = "discovery"))]
43#[path = "tests/ingress_target_filter.rs"]
44mod ingress_target_filter_tests;
45#[cfg(all(test, feature = "std", feature = "discovery", feature = "compression"))]
46#[path = "tests/payload_codec_negotiation.rs"]
47mod payload_codec_negotiation;
48#[cfg(all(test, feature = "std", feature = "discovery"))]
49#[path = "tests/reliable_tx_window_backpressure.rs"]
50mod reliable_tx_window_backpressure_tests;
51#[cfg(all(test, feature = "std", feature = "discovery"))]
52#[path = "tests/shared_chunk_budget_router.rs"]
53mod shared_chunk_budget_tests;
54#[cfg(test)]
56#[allow(clippy::duplicate_mod)]
57#[path = "tests/side_template_index.rs"]
58mod side_template_index_tests;
59#[cfg(feature = "discovery")]
60use crate::config::runtime_reliable_max_end_to_end_pending;
61#[cfg(all(test, feature = "std", feature = "discovery"))]
62#[path = "tests/replay_ownership_router.rs"]
63mod replay_ownership_tests;
64use crate::side_transport::{
65 SIDE_TRANSPORT_FLAG_PACKET_NONCE, SIDE_TRANSPORT_KIND_CHUNK, SIDE_TRANSPORT_KIND_COMPACT,
66 SIDE_TRANSPORT_KIND_COMPACT_DELTA, SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP,
67 SIDE_TRANSPORT_KIND_FULL, SideChunkAssembly, SideCompactTimestampMode, SideHeaderTemplate,
68 SideTransportFrames, extract_side_header_template, extract_side_header_template_view,
69 parse_side_transport_wrapper, read_uleb128_local, reconstruct_side_compact_frame,
70 wrap_side_transport_frame, write_uleb128_local,
71};
72#[cfg(all(test, feature = "discovery"))]
73#[path = "tests/ack_return_identity.rs"]
74mod ack_return_identity;
75#[cfg(all(test, feature = "discovery"))]
76#[path = "tests/ack_split_horizon.rs"]
77mod ack_split_horizon;
78
79#[cfg(all(test, feature = "discovery"))]
80#[path = "tests/discovery_liveness.rs"]
81mod discovery_liveness;
82
83#[cfg(all(test, feature = "discovery"))]
84mod compact_summary_tests {
85 use super::*;
86
87 #[test]
88 fn ordered_pending_window_is_bounded_and_exhaustion_cancels_the_tail() {
89 crate::tests::ensure_common_test_schema();
90 let ep = DataEndpoint::named("RADIO");
91 let ty = crate::config::register_data_type_with_description(
92 "ORDERED_WINDOW_REGRESSION",
93 "bounded ordered window",
94 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
95 &[ep],
96 crate::ReliableMode::Ordered,
97 200,
98 )
99 .unwrap();
100 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
101 let clock = now.clone();
102 let router = Router::new_with_clock(
103 RouterConfig::new([]),
104 Box::new(move || clock.load(Ordering::Relaxed)),
105 );
106 let capacity = runtime_reliable_max_end_to_end_pending().max(1);
107 for index in 0..capacity {
108 let item = RouterItem::Packet(
109 Packet::new(
110 ty,
111 &[ep],
112 "SOURCE",
113 index as u64 + 1,
114 Arc::<[u8]>::from([index as u8]),
115 )
116 .unwrap(),
117 );
118 assert_eq!(
119 router.register_end_to_end_reliable_tx(&item, None).unwrap(),
120 index == 0
121 );
122 }
123 let rejected = RouterItem::Packet(
124 Packet::new(
125 ty,
126 &[ep],
127 "SOURCE",
128 capacity as u64 + 1,
129 Arc::<[u8]>::from([0]),
130 )
131 .unwrap(),
132 );
133 assert!(
134 router
135 .register_end_to_end_reliable_tx(&rejected, None)
136 .is_err()
137 );
138 {
139 let mut state = router.state.lock();
140 assert_eq!(state.end_to_end_reliable_tx.len(), capacity);
141 let first = state.end_to_end_reliable_tx_order[0];
142 state
143 .end_to_end_reliable_tx
144 .get_mut(&first)
145 .unwrap()
146 .retries = runtime_reliable_max_retries();
147 }
148 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
149 assert!(router.process_end_to_end_reliable_timeouts().is_err());
150 assert!(router.state.lock().end_to_end_reliable_tx.is_empty());
151 }
152 #[test]
153 fn compact_reachability_survives_incomplete_detailed_topology() {
154 crate::tests::ensure_common_test_schema();
155 let router = Router::new(RouterConfig::new([]).with_sender("GB"));
156 let endpoint = DataEndpoint::try_named("GROUND_STATION")
157 .or_else(|| DataEndpoint::try_named("RADIO"))
158 .unwrap();
159 router.compact_summary_fixture(endpoint);
160 let state = router.state.lock();
161 let now = router.clock.now_ms();
162 let advertised =
163 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(1));
164 assert!(
165 advertised.contains(&endpoint),
166 "compact subscriber route was lost at the bridge"
167 );
168 let reflected =
169 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(0));
170 assert!(
171 !reflected.contains(&endpoint),
172 "split horizon must still exclude ingress"
173 );
174 assert_eq!(
175 router.advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(1)),
176 vec!["clock-behind-bridge".to_string()]
177 );
178 assert!(
179 router
180 .advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(0))
181 .is_empty()
182 );
183 assert!(
184 !router
185 .advertised_discovery_endpoints_for_link_locked(
186 &state,
187 now + DISCOVERY_ROUTE_TTL_MS + 1,
188 true,
189 Some(1)
190 )
191 .contains(&endpoint)
192 );
193 assert!(
194 router
195 .advertised_discovery_timesync_sources_for_link_locked(
196 &state,
197 now + DISCOVERY_ROUTE_TTL_MS + 1,
198 Some(1)
199 )
200 .is_empty()
201 );
202 }
203}
204
205use crate::config::RuntimeMemoryConfig;
206use crate::diagnostics::{
207 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
208 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
209 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
210};
211#[cfg(feature = "discovery")]
212use crate::discovery::{
213 self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
214 DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState,
215 TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS, TopologyAnnouncerRoute, TopologyBoardNode,
216 TopologySideRoute, TopologySnapshot,
217};
218use crate::packet::{hash_bytes_u64, sender_address_u32};
219use crate::queue::{BoundedDeque, ByteCost};
220#[cfg(all(not(feature = "std"), target_os = "none"))]
221use crate::seds_error_msg;
222#[cfg(feature = "timesync")]
223use crate::timesync::{
224 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
225 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
226 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
227 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
228 decode_timesync_request, decode_timesync_response,
229};
230use crate::{
231 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
232 config::{
233 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
234 runtime_reliable_max_pending, runtime_reliable_max_retries,
235 runtime_reliable_max_return_routes, runtime_reliable_retransmit_ms,
236 },
237 get_needed_message_size, impl_letype_num, is_reliable_type,
238 lock::{ReentryGate, RouterMutex},
239 message_e2e_encryption_policy, message_meta, message_priority,
240 packet::Packet,
241 reliable_mode, wire_format,
242};
243use alloc::string::{String, ToString};
244
245#[cfg(any(test, all(feature = "embedded", not(feature = "std"))))]
246use crate::small_map::SmallMap as RouterMap;
247#[cfg(not(any(test, all(feature = "embedded", not(feature = "std")))))]
248use alloc::collections::BTreeMap as RouterMap;
249
250use alloc::collections::BTreeMap;
251
252use alloc::{
253 borrow::ToOwned,
254 boxed::Box,
255 collections::{BTreeSet, VecDeque},
256 format,
257 sync::Arc,
258 vec,
259 vec::Vec,
260};
261use core::cell::UnsafeCell;
262use core::fmt;
263use core::fmt::{Debug, Formatter};
264use core::mem::size_of;
265use core::ops::{Deref, DerefMut};
266use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
267#[cfg(feature = "std")]
268use std::time::Instant;
269
270pub type RouterSideId = usize;
272
273const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
274pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
275pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
276pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
277static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
278
279#[derive(Clone, Copy, Debug, PartialEq, Eq)]
280pub enum SideTransportProfile {
281 Canonical,
282 Template,
283 Ipv6Like,
284 Ipv4Like,
285}
286
287impl SideTransportProfile {
288 #[inline]
289 pub const fn as_str(self) -> &'static str {
290 match self {
291 Self::Canonical => "canonical",
292 Self::Template => "template",
293 Self::Ipv6Like => "ipv6_like",
294 Self::Ipv4Like => "ipv4_like",
295 }
296 }
297
298 #[cfg(feature = "discovery")]
299 #[inline]
300 pub const fn discovery_code(self) -> u8 {
301 match self {
302 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
303 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
304 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
305 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
306 }
307 }
308}
309
310#[cfg(test)]
311mod chunk_assembly_memory_tests {
312 use super::*;
313 #[test]
314 fn full_chunk_budget_reclaims_other_partial_transfers_before_active_schema() {
315 let mut state = SideTransportState::default();
316 for id in 1..=4 {
317 let mut assembly = SideChunkAssembly {
318 total: 28,
319 last_seen_ms: id as u64,
320 ..Default::default()
321 };
322 for index in 0..23 {
323 assembly.insert(
324 index,
325 crate::shared_bytes::convert(Arc::<[u8]>::from([id as u8; 112])),
326 );
327 }
328 state.rx_chunks.insert(id, assembly);
329 }
330 state.rx_chunks.get_mut(&1).unwrap().insert(
331 23,
332 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 56])),
333 );
334 state.refresh_rx_chunk_bytes();
336 SideChunkAssembly::make_room(
337 &mut state.rx_chunks,
338 &mut state.rx_chunk_bytes,
339 4,
340 23,
341 28,
342 112,
343 16384,
344 )
345 .unwrap();
346 assert!(!state.rx_chunks.contains_key(&1));
347 assert_eq!(state.rx_chunks[&4].received_len(), 23);
348 for index in 23..28 {
349 state.refresh_rx_chunk_bytes();
350 SideChunkAssembly::make_room(
351 &mut state.rx_chunks,
352 &mut state.rx_chunk_bytes,
353 4,
354 index,
355 28,
356 112,
357 16384,
358 )
359 .unwrap();
360 state.rx_chunks.get_mut(&4).unwrap().insert(
361 index,
362 crate::shared_bytes::convert(Arc::<[u8]>::from([4u8; 112])),
363 );
364 }
365 assert_eq!(
366 state.rx_chunks.remove(&4).unwrap().assemble().unwrap(),
367 vec![4u8; 28 * 112]
368 );
369 assert!(
370 state
371 .rx_chunks
372 .values()
373 .flat_map(|a| a.received.iter().flatten())
374 .map(|p| p.len() + 64)
375 .sum::<usize>()
376 <= 16384
377 );
378 }
379
380 #[test]
381 fn duplicate_chunk_keeps_partial_transfers_but_oversize_transfer_is_rejected() {
382 let mut state = SideTransportState::default();
383 let mut assembly = SideChunkAssembly {
384 total: 2,
385 ..Default::default()
386 };
387 assembly.insert(
388 0,
389 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 112])),
390 );
391 state.rx_chunks.insert(1, assembly.clone());
392 state.rx_chunks.insert(2, assembly);
393 state.refresh_rx_chunk_bytes();
394 SideChunkAssembly::make_room(
395 &mut state.rx_chunks,
396 &mut state.rx_chunk_bytes,
397 1,
398 0,
399 2,
400 112,
401 352,
402 )
403 .unwrap();
404 assert_eq!(state.rx_chunks.len(), 2);
405 state.refresh_rx_chunk_bytes();
406 assert!(
407 SideChunkAssembly::make_room(
408 &mut state.rx_chunks,
409 &mut state.rx_chunk_bytes,
410 1,
411 1,
412 2,
413 400,
414 352
415 )
416 .is_err()
417 );
418 assert!(!state.rx_chunks.contains_key(&1));
419 assert!(state.rx_chunks.contains_key(&2));
420 }
421
422 #[test]
423 fn schema_sized_reassembly_allocates_exactly_and_preserves_order() {
424 let payload: Vec<u8> = (0..3652).map(|i| (i % 251) as u8).collect();
425 let mut assembly = SideChunkAssembly {
426 total: payload.len().div_ceil(56) as u16,
427 ..Default::default()
428 };
429 for (index, chunk) in payload.chunks(56).enumerate().rev() {
430 assembly.insert(
431 index as u16,
432 crate::shared_bytes::convert(Arc::<[u8]>::from(chunk)),
433 );
434 }
435 assert_eq!(assembly.received_len(), usize::from(assembly.total));
436 let out = assembly.assemble().unwrap();
437 assert_eq!(out, payload);
438 assert_eq!(
439 out.capacity(),
440 payload.len(),
441 "do not double discovery reassembly capacity"
442 );
443 }
444 #[test]
445 fn incomplete_or_noncontiguous_chunks_are_rejected() {
446 let mut assembly = SideChunkAssembly {
447 total: 2,
448 ..Default::default()
449 };
450 assembly.insert(0, crate::shared_bytes::convert(Arc::<[u8]>::from([1u8])));
451 assert!(assembly.clone().assemble().is_err());
452 assert!(
453 assembly
454 .insert_with(2, 0, || Arc::<[u8]>::from([2u8]))
455 .is_err()
456 );
457 assert!(assembly.assemble().is_err());
458 }
459}
460
461#[derive(Clone, Debug, Default)]
462struct SideTransportState {
463 #[cfg(feature = "std")]
464 tx_callback_gate: Arc<ReentryGate>,
465 tx_chunk_progress: Option<Box<crate::side_transport::SideTxProgress>>,
466 #[cfg(all(feature = "compression", feature = "discovery"))]
467 payload_codec: crate::payload_codec::Ledger,
468 #[cfg(all(feature = "compression", feature = "discovery"))]
469 payload_codec_choice: u64,
470 tx_template_ids: RouterMap<u64, u32>,
471 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
472 tx_last_timestamps: RouterMap<u32, u64>,
473 tx_compact_uses: RouterMap<u32, u8>,
474 rx_templates: RouterMap<u64, u32>,
475 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
476 rx_last_timestamps: RouterMap<u32, u64>,
477 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
478 rx_chunk_reclaims: u32,
479 rx_chunk_bytes: usize,
480}
481
482impl SideTransportState {
483 #[inline(never)]
484 fn take_chunk(&mut self, id: u32, reclaimed: bool) -> Option<SideChunkAssembly> {
485 let value = SideChunkAssembly::remove(&mut self.rx_chunks, &mut self.rx_chunk_bytes, id)?;
486 if reclaimed {
487 self.rx_chunk_reclaims = self.rx_chunk_reclaims.saturating_add(1);
488 }
489 Some(value)
490 }
491 #[cfg(test)]
493 #[inline(never)]
494 fn refresh_rx_chunk_bytes(&mut self) {
495 self.rx_chunk_bytes = self
496 .rx_chunks
497 .values()
498 .map(SideChunkAssembly::byte_cost)
499 .sum();
500 }
501 fn refresh_transmit_header_templates(&mut self) {
502 for uses in self.tx_compact_uses.values_mut() {
506 *uses = u8::MAX;
507 }
508 }
511
512 fn tx_template_count(&self) -> usize {
513 self.tx_template_ids.len()
514 }
515
516 fn rx_template_count(&self) -> usize {
517 self.rx_templates_by_id.len()
518 }
519
520 fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
521 let mut removed = 0usize;
522 while self.tx_template_ids.len() > limit {
523 let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
524 break;
525 };
526 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
527 self.tx_last_timestamps.remove(&old_id);
528 self.tx_compact_uses.remove(&old_id);
529 }
530 self.tx_templates.remove(&old_hash);
531 removed += 1;
532 }
533 removed
534 }
535
536 fn insert_tx_template(
537 &mut self,
538 template: SideHeaderTemplate,
539 template_id: u32,
540 max_templates: usize,
541 ) -> bool {
542 if max_templates == 0 {
543 return false;
544 }
545 let evicted = !self.tx_template_ids.contains_key(&template.hash)
546 && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
547 self.tx_template_ids.insert(template.hash, template_id);
548 self.tx_templates.insert(template.hash, template);
549 self.tx_compact_uses.insert(template_id, 0);
550 evicted
551 }
552
553 fn insert_rx_template(
554 &mut self,
555 template_id: u32,
556 template: SideHeaderTemplate,
557 max_templates: usize,
558 ) -> bool {
559 if max_templates == 0 {
560 return false;
561 }
562 if let Some(old_id) = self.rx_templates.get(&template.hash).copied()
565 && old_id != template_id
566 {
567 self.rx_templates_by_id.remove(&old_id);
568 self.rx_last_timestamps.remove(&old_id);
569 self.rx_templates.remove(&template.hash);
570 }
571 if let Some(previous) = self.rx_templates_by_id.get(&template_id)
572 && previous.hash != template.hash
573 {
574 self.rx_templates.remove(&previous.hash);
575 self.rx_last_timestamps.remove(&template_id);
576 }
577 let mut evicted = false;
578 if self.rx_templates_by_id.len() >= max_templates
579 && !self.rx_templates_by_id.contains_key(&template_id)
580 && let Some((&old_hash, &old_id)) = self.rx_templates.first_key_value()
583 {
584 self.rx_templates.remove(&old_hash);
585 self.rx_templates_by_id.remove(&old_id);
586 self.rx_last_timestamps.remove(&old_id);
587 evicted = true;
588 }
589 self.rx_templates.insert(template.hash, template_id);
590 self.rx_templates_by_id.insert(template_id, template);
591 evicted
592 }
593}
594
595#[derive(Clone, Debug, PartialEq, Eq)]
596pub struct CompactTimestampOmissionPolicy {
597 all: bool,
598 types: Option<Arc<[DataType]>>,
599}
600impl CompactTimestampOmissionPolicy {
601 pub const fn none() -> Self {
602 Self {
603 all: false,
604 types: None,
605 }
606 }
607 pub const fn all() -> Self {
608 Self {
609 all: true,
610 types: None,
611 }
612 }
613 pub fn with_data_type(mut self, ty: DataType) -> Self {
614 self.insert(ty);
615 self
616 }
617 pub fn insert(&mut self, ty: DataType) {
618 if self.contains(ty) {
619 return;
620 }
621 let current = self.types.as_deref().unwrap_or(&[]);
622 let mut types = Vec::with_capacity(current.len() + 1);
623 types.extend_from_slice(current);
624 let index = types.binary_search(&ty).unwrap_err();
625 types.insert(index, ty);
626 self.types = Some(Arc::from(types));
627 }
628 pub fn contains(&self, ty: DataType) -> bool {
629 self.all
630 || self
631 .types
632 .as_deref()
633 .is_some_and(|types| types.binary_search(&ty).is_ok())
634 }
635 pub fn is_empty(&self) -> bool {
636 !self.all && self.types.is_none()
637 }
638}
639
640impl Default for CompactTimestampOmissionPolicy {
641 #[inline]
642 fn default() -> Self {
643 Self::none()
644 }
645}
646
647#[derive(Clone, Debug, PartialEq, Eq)]
648pub enum RouterItem {
649 Packet(Packet),
650 Packed(crate::SharedBytes),
651}
652
653#[cfg(feature = "compact-packet-store")]
654impl RouterItem {
655 fn park_for_queue(&mut self) -> TelemetryResult<()> {
656 match self {
657 Self::Packet(packet) => packet.park_for_queue()?,
658 Self::Packed(bytes) => bytes.park_for_queue()?,
659 }
660 Ok(())
661 }
662}
663
664#[derive(Clone, Debug, PartialEq, Eq)]
665struct RouterRxItem {
666 src: Option<RouterSideId>,
667 data: RouterItem,
668 priority: u8,
669}
670
671#[derive(Clone, Debug, PartialEq, Eq)]
672enum RouterTxItem {
673 Broadcast(RouterItem),
674 EndToEndReplay {
675 packet_id: u64,
676 },
677 ToSide {
678 src: Option<RouterSideId>,
679 dst: RouterSideId,
680 data: RouterItem,
681 },
682 ReliableReplay {
683 dst: RouterSideId,
684 ty: DataType,
685 seq: u32,
686 bytes: crate::SharedBytes,
687 },
688}
689
690impl ByteCost for RouterRxItem {
691 #[cfg(feature = "compact-packet-store")]
692 fn park_for_queue(&mut self) -> TelemetryResult<()> {
693 self.data.park_for_queue()
694 }
695 #[inline]
696 fn byte_cost(&self) -> usize {
697 match &self.data {
698 RouterItem::Packet(pkt) => pkt.byte_cost(),
699 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
700 }
701 }
702}
703
704impl ByteCost for RouterTxItem {
705 #[cfg(feature = "compact-packet-store")]
706 fn park_for_queue(&mut self) -> TelemetryResult<()> {
707 match self {
708 Self::Broadcast(item) | Self::ToSide { data: item, .. } => item.park_for_queue(),
709 Self::ReliableReplay { bytes, .. } => bytes.park_for_queue(),
710 _ => Ok(()),
711 }
712 }
713 #[inline]
714 fn byte_cost(&self) -> usize {
715 match self {
716 RouterTxItem::Broadcast(data) => match data {
717 RouterItem::Packet(pkt) => pkt.byte_cost(),
718 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
719 },
720 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
721 RouterTxItem::ToSide { data, .. } => match data {
722 RouterItem::Packet(pkt) => pkt.byte_cost(),
723 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
724 },
725 RouterTxItem::ReliableReplay { bytes, .. } => {
726 size_of::<crate::SharedBytes>() + bytes.len()
727 }
728 }
729 }
730}
731
732#[derive(Clone, Copy)]
733enum ReplayMarkerKey {
734 EndToEnd(u64),
735 Hop(RouterSideId, DataType, u32),
736}
737impl RouterTxItem {
738 fn replay_marker_key(&self) -> Option<ReplayMarkerKey> {
739 match self {
740 Self::EndToEndReplay { packet_id } => Some(ReplayMarkerKey::EndToEnd(*packet_id)),
741 Self::ReliableReplay { dst, ty, seq, .. } => {
742 Some(ReplayMarkerKey::Hop(*dst, *ty, *seq))
743 }
744 _ => None,
745 }
746 }
747}
748
749#[repr(u8)]
750#[derive(Clone, Copy, Debug, PartialEq, Eq)]
751enum QueuedDelivery {
752 Untracked,
753 OrderedTail,
754 Ordered,
755 DeferredOrdered,
756 DeferredUnordered,
757 Unordered,
758 ClaimedOrdered,
759 ClaimedUnordered,
760}
761
762#[derive(Clone, Debug, PartialEq, Eq)]
766struct TxQueued {
767 disposable: bool,
769 item: RouterTxItem,
770 ignore_local: bool,
771 priority: u8,
772 delivery_state: QueuedDelivery,
773 accepted_type: u32,
774 deferred_since_ms: u64,
775 retry_after_ms: u64,
776}
777
778#[derive(Clone, Copy)]
779struct TxClaim {
780 token: u32,
781 ordered: bool,
782}
783#[derive(Default)]
784struct DispatchContext {
785 priority: u8,
787 claim: Option<TxClaim>,
788 since: Option<u64>,
789 completion_flags: u8,
790 refused: bool,
791 admitted: bool,
792}
793impl DispatchContext {
794 const TERMINAL: u8 = 1;
795 const OWNER_TRANSFERRED: u8 = 2;
796
797 fn is_terminal(&self) -> bool {
798 self.completion_flags & Self::TERMINAL != 0
799 }
800 fn owner_transferred(&self) -> bool {
801 self.completion_flags & Self::OWNER_TRANSFERRED != 0
802 }
803 fn mark_terminal(&mut self) {
804 self.completion_flags |= Self::TERMINAL;
805 }
806 fn mark_owner_transferred(&mut self) {
807 self.completion_flags |= Self::OWNER_TRANSFERRED;
808 }
809}
810const _: () = assert!(
814 size_of::<DispatchContext>() == size_of::<(Option<TxClaim>, Option<u64>, bool, bool)>()
815);
816
817struct TxWork {
818 queued: TxQueued,
819 claim: Option<TxClaim>,
820 owed_turn: bool,
821}
822
823const MAX_PRIORITY_BURST: u8 = 8;
828
829const _: () = {
835 assert!(
836 size_of::<TxQueued>()
837 == size_of::<(RouterTxItem, bool, u8, QueuedDelivery, u32, u64, u64)>()
838 );
839 let metadata_bytes = 2 * size_of::<u64>()
840 + size_of::<u32>()
841 + size_of::<QueuedDelivery>()
842 + 2 * size_of::<bool>()
843 + size_of::<u8>();
844 let alignment = core::mem::align_of::<TxQueued>();
845 let expected = (size_of::<RouterTxItem>() + metadata_bytes).div_ceil(alignment) * alignment;
846 assert!(size_of::<TxQueued>() == expected);
847};
848
849impl ByteCost for TxQueued {
850 #[cfg(feature = "compact-packet-store")]
851 fn park_for_queue(&mut self) -> TelemetryResult<()> {
852 self.item.park_for_queue()
853 }
854 #[inline]
856 fn byte_cost(&self) -> usize {
857 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>() + 2 * size_of::<u64>()
858 }
859}
860
861impl ByteCost for u64 {
863 #[inline]
865 fn byte_cost(&self) -> usize {
866 size_of::<u64>()
867 }
868}
869
870#[derive(Debug, Clone)]
873struct ReliableTxState {
874 next_seq: u32,
875 sent_order: VecDeque<u32>,
876 sent: BTreeMap<u32, ReliableSent>,
877}
878
879#[derive(Debug, Clone)]
880struct ReliableSent {
881 bytes: crate::SharedBytes,
882 last_send_ms: u64,
883 retries: u32,
884 queued: bool,
885 partial_acked: bool,
886}
887
888#[derive(Debug, Clone)]
889struct EndToEndReliableSent {
890 ordered_type: Option<u32>,
891 waiting_for_predecessor: bool,
892 data: RouterItem,
893 pending_destinations: BTreeMap<u64, RouterSideId>,
894 tracked_destinations: bool,
895 last_send_ms: u64,
896 retries: u32,
897 queued: bool,
898}
899
900impl EndToEndReliableSent {
901 fn try_box(self) -> TelemetryResult<Box<Self>> {
905 let layout = core::alloc::Layout::new::<Self>();
906 let ptr = unsafe { alloc::alloc::alloc(layout).cast::<Self>() };
910 if ptr.is_null() {
911 return Err(TelemetryError::Io("memory pressure"));
912 }
913 unsafe {
914 ptr.write(self);
915 Ok(Box::from_raw(ptr))
916 }
917 }
918}
919
920#[derive(Debug, Clone)]
921struct ReliableRxState {
922 expected_seq: u32,
923 buffered: RouterMap<u32, crate::SharedBytes>,
924}
925
926#[derive(Debug, Clone)]
927struct ReliableReturnRouteState {
928 side: RouterSideId,
929 delivery_completed: bool,
932 last_replay_ms: u64,
933 upstream_contract: Option<(u64, u64)>,
937}
938
939#[cfg(feature = "discovery")]
940#[derive(Debug, Clone, Default, PartialEq, Eq)]
941struct DiscoverySenderState {
942 advertised_address: Option<u32>,
945 reachable: Vec<DataEndpoint>,
946 advertised_reachable: Vec<DataEndpoint>,
947 reachable_network_variables: Vec<DataType>,
948 reachable_timesync_sources: Vec<String>,
949 advertised_reachable_timesync_sources: Vec<String>,
950 link_capabilities: Option<discovery::LinkCapabilities>,
951 topology_boards: Arc<Vec<TopologyBoardNode>>,
952 has_full_topology: bool,
953 has_schema: bool,
954 last_seen_ms: u64,
955}
956
957#[cfg(feature = "discovery")]
958#[derive(Debug, Clone, Default, PartialEq, Eq)]
959struct DiscoverySideState {
960 reachable: Vec<DataEndpoint>,
961 reachable_network_variables: Vec<DataType>,
962 requested_network_variables: Vec<(DataType, u64)>,
963 reachable_timesync_sources: Vec<String>,
964 last_seen_ms: u64,
965 announcers: BTreeMap<String, DiscoverySenderState>,
966}
967
968#[cfg(feature = "discovery")]
969#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
970#[repr(u8)]
971enum SchemaResponsePhase {
972 #[default]
973 Idle,
974 PrefixBuilding,
975 CatalogWaiting,
976 CatalogBuilding,
977 Owned,
978}
979
980#[cfg(feature = "discovery")]
981#[derive(Debug, Clone, Default)]
982struct DiscoverySideThrottleState {
983 next_topology_request_ms: u64,
984 next_schema_request_ms: u64,
985 next_schema_response_ms: u64,
988 schema_response_packet_id: u64,
989 schema_response_active: bool,
990 schema_response_phase: SchemaResponsePhase,
991 schema_response_started: bool,
992 schema_response_pending: bool,
993 next_ping_ms: u64,
994 last_liveness_tx_ms: u64,
996 pending_incremental: bool,
997 incremental_retries_remaining: u8,
998 has_sent_full: bool,
999 last_topology: Vec<TopologyBoardNode>,
1000}
1001
1002#[cfg(feature = "discovery")]
1003impl DiscoverySideThrottleState {
1004 fn liveness_due(&self, now_ms: u64) -> bool {
1005 now_ms >= self.next_ping_ms
1006 && now_ms.saturating_sub(self.last_liveness_tx_ms)
1007 >= DISCOVERY_SLOW_LINK_PING_INTERVAL_MS
1008 }
1009}
1010
1011#[cfg(all(feature = "discovery", feature = "timesync"))]
1012#[derive(Debug, Clone, Default)]
1013struct TimeSyncSideThrottleState {
1014 next_allowed_ms: u64,
1015}
1016
1017#[cfg(feature = "discovery")]
1018#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1019enum DiscoveryAdvertiseLevel {
1020 MinimalPing,
1021 Incremental,
1022 Full,
1023}
1024
1025#[derive(Debug, Clone, Default)]
1026struct AdaptiveRouteStats {
1027 estimated_bandwidth_bps: u64,
1028 peak_bandwidth_bps: u64,
1029 last_observed_ms: u64,
1030 last_slow_observed_ms: u64,
1031 sample_count: u64,
1032 window_started_ms: u64,
1033 window_bytes: u64,
1034 peak_usage_bps: u64,
1035}
1036
1037impl AdaptiveRouteStats {
1038 #[inline]
1039 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
1040 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
1041 sample_bps
1042 } else if sample_bps >= self.estimated_bandwidth_bps {
1043 self.estimated_bandwidth_bps
1044 .saturating_mul(3)
1045 .saturating_add(sample_bps.saturating_mul(5))
1046 / 8
1047 } else {
1048 self.estimated_bandwidth_bps
1049 .saturating_mul(7)
1050 .saturating_add(sample_bps)
1051 / 8
1052 };
1053 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
1054 self.last_observed_ms = now_ms;
1055 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
1056 self.last_slow_observed_ms = now_ms;
1057 }
1058 self.sample_count = self.sample_count.saturating_add(1);
1059 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
1060 self.window_started_ms = now_ms;
1061 self.window_bytes = 0;
1062 }
1063 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
1064 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
1065 }
1066
1067 #[inline]
1068 fn current_usage_bps(&self, now_ms: u64) -> u64 {
1069 if self.window_started_ms == 0 {
1070 return 0;
1071 }
1072 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
1073 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
1074 .min(u128::from(u64::MAX)) as u64
1075 }
1076
1077 #[inline]
1078 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
1079 let capacity = self
1080 .estimated_bandwidth_bps
1081 .max(self.peak_bandwidth_bps)
1082 .max(1);
1083 capacity.saturating_sub(self.current_usage_bps(now_ms))
1084 }
1085
1086 #[inline]
1087 fn weight(&self, now_ms: u64) -> u64 {
1088 self.available_headroom_bps(now_ms).max(1)
1089 }
1090
1091 #[inline]
1092 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
1093 let current_usage_bps = self.current_usage_bps(now_ms);
1094 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
1095 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
1096 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
1097 AdaptiveLinkStats {
1098 auto_balancing_enabled,
1099 estimated_capacity_bps,
1100 peak_capacity_bps,
1101 current_usage_bps,
1102 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
1103 available_headroom_bps,
1104 effective_weight: available_headroom_bps.max(1),
1105 last_observed_ms: self.last_observed_ms,
1106 sample_count: self.sample_count,
1107 }
1108 }
1109}
1110
1111use crate::type_stats::TypeRuntimeStatsInner;
1112
1113#[derive(Debug, Clone, Default)]
1114struct SideRuntimeStatsInner {
1115 tx_packets: u64,
1116 tx_bytes: u64,
1117 rx_packets: u64,
1118 rx_bytes: u64,
1119 relayed_tx_packets: u64,
1120 relayed_tx_bytes: u64,
1121 relayed_rx_packets: u64,
1122 relayed_rx_bytes: u64,
1123 local_delivery_packets: u64,
1124 tx_retries: u64,
1125 tx_handler_failures: u64,
1126 local_handler_failures: u64,
1127 total_handler_retries: u64,
1128 side_transport_full_frames: u64,
1129 side_transport_compact_frames: u64,
1130 side_transport_compact_delta_frames: u64,
1131 side_transport_compact_omitted_timestamp_frames: u64,
1132 side_transport_chunk_frames: u64,
1133 side_transport_raw_bytes: u64,
1134 side_transport_wire_bytes: u64,
1135 side_transport_bytes_saved: u64,
1136 side_transport_min_compact_overhead_bytes: Option<usize>,
1137 side_transport_max_compact_overhead_bytes: Option<usize>,
1138 side_transport_compact_target_misses: u64,
1139 side_transport_template_evictions: u64,
1140 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
1141}
1142
1143impl SideRuntimeStatsInner {
1144 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
1145 self.data_types.entry(ty.as_u32()).or_default()
1146 }
1147
1148 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
1149 self.tx_packets = self.tx_packets.saturating_add(1);
1150 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
1151 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1152 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1153 if relayed {
1154 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
1155 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
1156 }
1157 self.type_stats_mut(ty)
1158 .tx(bytes as u64, relayed, retries as u64);
1159 }
1160
1161 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
1162 self.rx_packets = self.rx_packets.saturating_add(1);
1163 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
1164 if relayed {
1165 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
1166 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
1167 }
1168 self.type_stats_mut(ty).rx(bytes as u64, relayed);
1169 }
1170
1171 fn note_local_delivery(&mut self, ty: DataType) {
1172 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
1173 self.type_stats_mut(ty).rx(0, false);
1174 }
1175
1176 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
1177 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
1178 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1179 self.type_stats_mut(ty).failure(0);
1180 }
1181
1182 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
1183 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
1184 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1185 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1186 self.type_stats_mut(ty).failure(retries as u64);
1187 }
1188
1189 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
1190 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
1191 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1192 }
1193
1194 fn note_side_transport_compact(
1195 &mut self,
1196 raw_bytes: usize,
1197 wire_bytes: usize,
1198 compact_overhead_bytes: usize,
1199 used_timestamp_delta: bool,
1200 omitted_timestamp: bool,
1201 ) {
1202 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
1203 if used_timestamp_delta {
1204 self.side_transport_compact_delta_frames =
1205 self.side_transport_compact_delta_frames.saturating_add(1);
1206 }
1207 if omitted_timestamp {
1208 self.side_transport_compact_omitted_timestamp_frames = self
1209 .side_transport_compact_omitted_timestamp_frames
1210 .saturating_add(1);
1211 }
1212 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1213 self.side_transport_min_compact_overhead_bytes = Some(
1214 self.side_transport_min_compact_overhead_bytes
1215 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
1216 );
1217 self.side_transport_max_compact_overhead_bytes = Some(
1218 self.side_transport_max_compact_overhead_bytes
1219 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
1220 );
1221 }
1222
1223 fn note_side_transport_chunks(&mut self, chunks: usize) {
1224 self.side_transport_chunk_frames = self
1225 .side_transport_chunk_frames
1226 .saturating_add(chunks as u64);
1227 }
1228
1229 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
1230 self.side_transport_raw_bytes = self
1231 .side_transport_raw_bytes
1232 .saturating_add(raw_bytes as u64);
1233 self.side_transport_wire_bytes = self
1234 .side_transport_wire_bytes
1235 .saturating_add(wire_bytes as u64);
1236 if raw_bytes > wire_bytes {
1237 self.side_transport_bytes_saved = self
1238 .side_transport_bytes_saved
1239 .saturating_add((raw_bytes - wire_bytes) as u64);
1240 }
1241 }
1242
1243 fn note_side_transport_compact_target_miss(&mut self) {
1244 self.side_transport_compact_target_misses =
1245 self.side_transport_compact_target_misses.saturating_add(1);
1246 }
1247
1248 fn note_side_transport_template_eviction(&mut self) {
1249 self.side_transport_template_evictions =
1250 self.side_transport_template_evictions.saturating_add(1);
1251 }
1252}
1253
1254#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1255enum RouteSelectionOrigin {
1256 Flood,
1257 Discovered,
1258}
1259
1260type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1263
1264type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
1266type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
1267
1268#[derive(Clone)]
1274pub enum EndpointHandlerFn {
1275 Packet(Arc<PacketHandlerFn>),
1276 Packed(Arc<PackedHandlerFn>),
1277}
1278
1279pub struct EndpointHandler {
1281 endpoint: DataEndpoint,
1282 handler: EndpointHandlerFn,
1283}
1284
1285impl Debug for EndpointHandlerFn {
1287 #[inline]
1288 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1289 match self {
1290 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
1291 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
1292 }
1293 }
1294}
1295
1296#[derive(Clone)]
1298pub enum RouterTxHandlerFn {
1299 Packed(Arc<PackedHandlerFn>),
1300 PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
1301 Packet(Arc<PacketHandlerFn>),
1302}
1303
1304impl Debug for RouterTxHandlerFn {
1305 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1306 match self {
1307 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
1308 RouterTxHandlerFn::PackedWithPriority(_) => f
1309 .debug_tuple("PackedWithPriority")
1310 .field(&"<handler>")
1311 .finish(),
1312 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
1313 }
1314 }
1315}
1316
1317pub use crate::routing_config::SideOptions as RouterSideOptions;
1318
1319#[derive(Clone, Debug)]
1321pub struct RouterSide {
1322 pub name: Arc<str>,
1323 pub tx_handler: RouterTxHandlerFn,
1324 pub opts: RouterSideOptions,
1325}
1326
1327impl Debug for EndpointHandler {
1329 #[inline]
1330 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1331 f.debug_struct("EndpointHandler")
1332 .field("endpoint", &self.endpoint)
1333 .field("handler", &self.handler)
1334 .finish()
1335 }
1336}
1337
1338#[inline]
1339pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1340 #[cfg(feature = "timesync")]
1341 if matches!(endpoint, DataEndpoint::TimeSync) {
1342 return true;
1343 }
1344 #[cfg(feature = "discovery")]
1345 {
1346 discovery::is_discovery_endpoint(endpoint)
1347 }
1348 #[cfg(not(feature = "discovery"))]
1349 {
1350 let _ = endpoint;
1351 false
1352 }
1353}
1354
1355impl EndpointHandler {
1356 #[inline]
1360 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1361 where
1362 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1363 {
1364 assert!(
1365 !endpoint_is_router_internal(endpoint),
1366 "reserved internal endpoint handlers must not be user-registered"
1367 );
1368 #[cfg(feature = "std")]
1369 crate::config::ensure_endpoint_id(endpoint, false)
1370 .expect("endpoint handler endpoint registration failed");
1371 Self {
1372 endpoint,
1373 handler: EndpointHandlerFn::Packet(Arc::new(f)),
1374 }
1375 }
1376
1377 #[inline]
1379 #[cfg(feature = "std")]
1380 pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1381 where
1382 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1383 {
1384 Self::new_packet_handler(endpoint.id, f)
1385 }
1386
1387 #[cfg(not(feature = "std"))]
1389 #[inline]
1390 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1391 where
1392 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1393 {
1394 Self::new_packet_handler(endpoint.id, f)
1395 }
1396
1397 #[cfg(feature = "std")]
1399 #[inline]
1400 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1401 where
1402 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1403 {
1404 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1405 .ok_or(TelemetryError::BadArg)?;
1406 Ok(Self::new_packet_handler(endpoint.id, f))
1407 }
1408
1409 #[inline]
1413 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1414 where
1415 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1416 {
1417 assert!(
1418 !endpoint_is_router_internal(endpoint),
1419 "reserved internal endpoint handlers must not be user-registered"
1420 );
1421 #[cfg(feature = "std")]
1422 crate::config::ensure_endpoint_id(endpoint, false)
1423 .expect("endpoint handler endpoint registration failed");
1424 Self {
1425 endpoint,
1426 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1427 }
1428 }
1429
1430 #[inline]
1432 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1433 where
1434 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1435 {
1436 Self::new_packed_handler(endpoint.id, f)
1437 }
1438
1439 #[cfg(feature = "std")]
1441 #[inline]
1442 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1443 where
1444 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1445 {
1446 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1447 .ok_or(TelemetryError::BadArg)?;
1448 Ok(Self::new_packed_handler(endpoint.id, f))
1449 }
1450
1451 #[inline]
1453 pub fn get_endpoint(&self) -> DataEndpoint {
1454 self.endpoint
1455 }
1456
1457 #[inline]
1459 pub fn get_handler(&self) -> &EndpointHandlerFn {
1460 &self.handler
1461 }
1462}
1463
1464pub trait Clock {
1465 fn now_ms(&self) -> u64;
1467
1468 fn now_ns(&self) -> u64 {
1472 self.now_ms().saturating_mul(1_000_000)
1473 }
1474}
1475
1476impl<T: Fn() -> u64> Clock for T {
1477 #[inline]
1478 fn now_ms(&self) -> u64 {
1479 self()
1480 }
1481}
1482
1483#[cfg(feature = "std")]
1484#[derive(Debug)]
1485struct StdMonotonicClock {
1486 start: Instant,
1487}
1488
1489#[cfg(feature = "std")]
1490impl Default for StdMonotonicClock {
1491 fn default() -> Self {
1492 Self {
1493 start: Instant::now(),
1494 }
1495 }
1496}
1497
1498#[cfg(feature = "std")]
1499impl Clock for StdMonotonicClock {
1500 fn now_ms(&self) -> u64 {
1501 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1502 }
1503
1504 fn now_ns(&self) -> u64 {
1505 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1506 }
1507}
1508
1509#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1511pub enum RouterE2eEncryptionMode {
1512 Disabled,
1514 RequiredOnly,
1516 Preferred,
1518 ForceAll,
1520}
1521
1522pub type NodeAddress = u32;
1523pub type P2pPort = u16;
1524pub type P2pStreamId = u32;
1525
1526const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1527const P2P_STREAM_VERSION: u8 = 1;
1528const P2P_STREAM_SYN: u8 = 0x01;
1529const P2P_STREAM_ACK: u8 = 0x02;
1530const P2P_STREAM_FIN: u8 = 0x04;
1531const P2P_STREAM_RST: u8 = 0x08;
1532const P2P_STREAM_DATA: u8 = 0x10;
1533
1534#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1535pub enum AddressAssignmentMode {
1536 Dynamic,
1537 Requested(NodeAddress),
1538 Static(NodeAddress),
1539}
1540
1541impl AddressAssignmentMode {
1542 #[inline]
1543 #[cfg(feature = "discovery")]
1544 fn mode_code(self) -> u8 {
1545 match self {
1546 Self::Dynamic => 0,
1547 Self::Requested(_) => 1,
1548 Self::Static(_) => 2,
1549 }
1550 }
1551
1552 #[inline]
1553 fn requested_address(self) -> NodeAddress {
1554 match self {
1555 Self::Dynamic => 0,
1556 Self::Requested(addr) | Self::Static(addr) => addr,
1557 }
1558 }
1559}
1560
1561#[derive(Debug, Clone, PartialEq, Eq)]
1562pub enum AddressChangeReason {
1563 Configured,
1564 DynamicConflict,
1565 RequestedConflict,
1566 StaticConflict,
1567 HostnameConflict,
1568}
1569
1570#[derive(Debug, Clone, PartialEq, Eq)]
1571pub struct AddressChange {
1572 pub old_hostname: Arc<str>,
1573 pub new_hostname: Arc<str>,
1574 pub old_address: NodeAddress,
1575 pub new_address: NodeAddress,
1576 pub reason: AddressChangeReason,
1577}
1578
1579#[derive(Debug, Clone, PartialEq, Eq)]
1580pub struct AddressBookEntry {
1581 pub hostname: Arc<str>,
1582 pub address: NodeAddress,
1583 pub requested_address: NodeAddress,
1584 pub mode: AddressAssignmentMode,
1585 pub birth_ms: u64,
1586 pub owner_hash: u64,
1587 pub last_seen_ms: u64,
1588}
1589
1590pub struct P2pMessage<'a> {
1591 pub source_hostname: &'a str,
1592 pub source_address: NodeAddress,
1593 pub source_port: P2pPort,
1594 pub destination_port: P2pPort,
1595 pub payload: &'a [u8],
1596}
1597
1598#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1599pub enum P2pStreamEventKind {
1600 Accepted,
1601 Connected,
1602 Data,
1603 Closed,
1604 Reset,
1605}
1606
1607pub struct P2pStreamEvent<'a> {
1608 pub kind: P2pStreamEventKind,
1609 pub stream_id: P2pStreamId,
1610 pub peer_stream_id: P2pStreamId,
1611 pub sequence: u32,
1612 pub peer_hostname: &'a str,
1613 pub peer_address: NodeAddress,
1614 pub local_port: P2pPort,
1615 pub peer_port: P2pPort,
1616 pub payload: &'a [u8],
1617}
1618
1619struct P2pDecoded<'a> {
1620 source_hostname: &'a str,
1621 source_address: NodeAddress,
1622 source_port: P2pPort,
1623 destination_port: P2pPort,
1624 payload: &'a [u8],
1625}
1626
1627#[derive(Debug, Clone)]
1628struct P2pStreamDecoded<'a> {
1629 flags: u8,
1630 source_stream_id: P2pStreamId,
1631 destination_stream_id: P2pStreamId,
1632 sequence: u32,
1633 payload: &'a [u8],
1634}
1635
1636type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1637
1638#[derive(Clone)]
1639struct AddressChangeHandler {
1640 handler: Arc<AddressChangeFn>,
1641}
1642
1643impl Debug for AddressChangeHandler {
1644 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1645 f.write_str("AddressChangeHandler(<handler>)")
1646 }
1647}
1648
1649type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1650
1651#[derive(Clone)]
1652struct P2pPortHandler {
1653 handler: Arc<P2pPortHandlerFn>,
1654}
1655
1656impl Debug for P2pPortHandler {
1657 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1658 f.write_str("P2pPortHandler(<handler>)")
1659 }
1660}
1661
1662type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1663
1664#[derive(Clone)]
1665struct P2pStreamHandler {
1666 handler: Arc<P2pStreamHandlerFn>,
1667}
1668
1669impl Debug for P2pStreamHandler {
1670 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1671 f.write_str("P2pStreamHandler(<handler>)")
1672 }
1673}
1674
1675#[derive(Debug, Clone)]
1676struct P2pStreamSession {
1677 peer_hostname: Arc<str>,
1678 peer_address: NodeAddress,
1679 local_port: P2pPort,
1680 peer_port: P2pPort,
1681 peer_stream_id: P2pStreamId,
1682 next_sequence: u32,
1683 connected: bool,
1684}
1685
1686#[derive(Debug, Clone)]
1687struct PendingP2pStreamEvent {
1688 handlers: Vec<P2pStreamHandler>,
1689 kind: P2pStreamEventKind,
1690 stream_id: P2pStreamId,
1691 peer_stream_id: P2pStreamId,
1692 peer_hostname: Arc<str>,
1693 peer_address: NodeAddress,
1694 local_port: P2pPort,
1695 peer_port: P2pPort,
1696}
1697
1698#[derive(Debug, Clone)]
1699pub struct RouterConfig {
1700 role: crate::routing_config::RoutingRole,
1701 handlers: Arc<[EndpointHandler]>,
1703 reliable_enabled: bool,
1705 sender: Option<Arc<str>>,
1707 address_mode: AddressAssignmentMode,
1709 address_change_handlers: Arc<[AddressChangeHandler]>,
1711 e2e_encryption: RouterE2eEncryptionMode,
1713 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
1715 e2e_key_id: u32,
1716 memory: RuntimeMemoryConfig,
1717 #[cfg(feature = "discovery")]
1718 preferred_discovery_master: Option<Arc<str>>,
1719 #[cfg(feature = "timesync")]
1720 timesync: Option<TimeSyncConfig>,
1721}
1722
1723impl RouterConfig {
1724 pub(crate) fn relay() -> Self {
1725 let mut cfg = Self::default().with_sender("RELAY");
1726 cfg.role = crate::routing_config::RoutingRole::Relay;
1727 cfg
1728 }
1729
1730 pub(crate) fn configured_sender(&self) -> Arc<str> {
1731 self.sender
1732 .clone()
1733 .unwrap_or_else(|| Arc::from(runtime_device_identifier()))
1734 }
1735
1736 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
1741 #[cfg(feature = "cryptography")]
1742 {
1743 RouterE2eEncryptionMode::Preferred
1744 }
1745 #[cfg(not(feature = "cryptography"))]
1746 {
1747 RouterE2eEncryptionMode::Disabled
1748 }
1749 }
1750
1751 pub fn new<H>(handlers: H) -> Self
1753 where
1754 H: Into<Arc<[EndpointHandler]>>,
1755 {
1756 let handlers: Arc<[EndpointHandler]> = handlers.into();
1757 assert!(
1758 handlers
1759 .iter()
1760 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
1761 "reserved internal endpoint handlers must not be user-registered"
1762 );
1763 Self {
1764 role: crate::routing_config::RoutingRole::Router,
1765 handlers,
1766 reliable_enabled: true,
1767 sender: None,
1768 address_mode: AddressAssignmentMode::Dynamic,
1769 address_change_handlers: Arc::from([]),
1770 e2e_encryption: Self::default_e2e_encryption_mode(),
1771 e2e_key_id: 0,
1772 memory: RuntimeMemoryConfig::default(),
1773 #[cfg(feature = "discovery")]
1774 preferred_discovery_master: None,
1775 #[cfg(feature = "timesync")]
1776 timesync: None,
1777 }
1778 }
1779
1780 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
1782 self.reliable_enabled = enabled;
1783 self
1784 }
1785
1786 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
1788 self.sender = Some(Arc::from(sender.as_ref()));
1789 self
1790 }
1791
1792 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
1794 self.with_sender(hostname)
1795 }
1796
1797 pub fn with_dynamic_address(mut self) -> Self {
1799 self.address_mode = AddressAssignmentMode::Dynamic;
1800 self
1801 }
1802
1803 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
1805 self.address_mode = AddressAssignmentMode::Requested(address);
1806 self
1807 }
1808
1809 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
1811 self.address_mode = AddressAssignmentMode::Static(address);
1812 self
1813 }
1814
1815 pub fn on_address_change<F>(mut self, f: F) -> Self
1817 where
1818 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
1819 {
1820 let mut handlers = self.address_change_handlers.to_vec();
1821 handlers.push(AddressChangeHandler {
1822 handler: Arc::new(f),
1823 });
1824 self.address_change_handlers = Arc::from(handlers);
1825 self
1826 }
1827
1828 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
1830 self.e2e_encryption = mode;
1831 self
1832 }
1833
1834 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
1836 self.e2e_key_id = key_id;
1837 self
1838 }
1839
1840 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
1842 memory.validate()?;
1843 self.memory = memory;
1844 Ok(self)
1845 }
1846
1847 #[cfg(feature = "discovery")]
1853 pub fn with_preferred_discovery_master<S: AsRef<str>>(mut self, hostname: S) -> Self {
1854 let hostname = hostname.as_ref();
1855 self.preferred_discovery_master = if hostname.is_empty() {
1856 None
1857 } else {
1858 Some(Arc::from(hostname))
1859 };
1860 self
1861 }
1862
1863 #[cfg(feature = "timesync")]
1864 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
1866 self.timesync = Some(cfg);
1867 self
1868 }
1869
1870 #[inline]
1871 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1873 if endpoint_is_router_internal(ep) {
1874 return false;
1875 }
1876 self.handlers.iter().any(|h| h.endpoint == ep)
1877 }
1878
1879 #[inline]
1880 fn reliable_enabled(&self) -> bool {
1881 self.reliable_enabled
1882 }
1883
1884 #[inline]
1885 fn address_mode(&self) -> AddressAssignmentMode {
1886 self.address_mode
1887 }
1888
1889 #[inline]
1890 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1891 self.e2e_encryption
1892 }
1893
1894 #[cfg(feature = "cryptography")]
1895 #[inline]
1896 fn e2e_key_id(&self) -> u32 {
1897 self.e2e_key_id
1898 }
1899
1900 #[cfg(feature = "timesync")]
1901 #[inline]
1902 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1903 self.timesync
1904 }
1905
1906 #[inline]
1907 pub(crate) fn memory_config(&self) -> RuntimeMemoryConfig {
1908 self.memory
1909 }
1910}
1911
1912impl Default for RouterConfig {
1913 fn default() -> Self {
1914 Self {
1915 role: crate::routing_config::RoutingRole::Router,
1916 handlers: Arc::from([]),
1917 reliable_enabled: true,
1918 sender: None,
1919 address_mode: AddressAssignmentMode::Dynamic,
1920 address_change_handlers: Arc::from([]),
1921 e2e_encryption: Self::default_e2e_encryption_mode(),
1922 e2e_key_id: 0,
1923 memory: RuntimeMemoryConfig::default(),
1924 #[cfg(feature = "discovery")]
1925 preferred_discovery_master: None,
1926 #[cfg(feature = "timesync")]
1927 timesync: None,
1928 }
1929 }
1930}
1931
1932#[derive(Debug)]
1936struct ActiveRoutingConfig {
1937 role: crate::routing_config::RoutingRole,
1938 reliable_enabled: bool,
1939 local: Option<Box<RouterConfig>>,
1940}
1941impl ActiveRoutingConfig {
1942 fn new(cfg: RouterConfig) -> Self {
1943 let role = cfg.role;
1944 Self {
1945 role,
1946 reliable_enabled: cfg.reliable_enabled(),
1947 local: role.local_services().then(|| Box::new(cfg)),
1948 }
1949 }
1950 fn handlers(&self) -> &[EndpointHandler] {
1951 self.local.as_ref().map_or(&[], |cfg| cfg.handlers.as_ref())
1952 }
1953 fn address_change_handlers(&self) -> &[AddressChangeHandler] {
1954 self.local
1955 .as_ref()
1956 .map_or(&[], |cfg| cfg.address_change_handlers.as_ref())
1957 }
1958 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1959 self.local
1960 .as_ref()
1961 .is_some_and(|cfg| cfg.is_local_endpoint(ep))
1962 }
1963 fn reliable_enabled(&self) -> bool {
1964 self.reliable_enabled
1965 }
1966 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1967 self.local
1968 .as_ref()
1969 .map_or(RouterE2eEncryptionMode::Disabled, |cfg| {
1970 cfg.e2e_encryption()
1971 })
1972 }
1973 #[cfg(feature = "cryptography")]
1974 fn e2e_key_id(&self) -> u32 {
1975 self.local.as_ref().map_or(0, |cfg| cfg.e2e_key_id())
1976 }
1977 #[cfg(feature = "timesync")]
1978 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1979 self.local.as_ref().and_then(|cfg| cfg.timesync_config())
1980 }
1981}
1982
1983pub trait LeBytes: Copy + Sized {
1986 const WIDTH: usize;
1987 fn write_le(self, out: &mut [u8]);
1988 fn from_le_slice(bytes: &[u8]) -> Self;
1989}
1990
1991impl_letype_num!(u8, 1);
1992impl_letype_num!(u16, 2);
1993impl_letype_num!(u32, 4);
1994impl_letype_num!(u64, 8);
1995impl_letype_num!(u128, 16);
1996impl_letype_num!(i8, 1);
1997impl_letype_num!(i16, 2);
1998impl_letype_num!(i32, 4);
1999impl_letype_num!(i64, 8);
2000impl_letype_num!(i128, 16);
2001impl_letype_num!(f32, 4);
2002impl_letype_num!(f64, 8);
2003
2004pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
2006 let total = data.len() * T::WIDTH;
2007 let mut buf = vec![0u8; total];
2008
2009 for (i, v) in data.iter().copied().enumerate() {
2010 let start = i * T::WIDTH;
2011 v.write_le(&mut buf[start..start + T::WIDTH]);
2012 }
2013
2014 Arc::from(buf)
2015}
2016
2017fn make_error_payload(msg: &str) -> crate::SharedBytes {
2020 let meta = message_meta(DataType::TelemetryError);
2021 match meta.element {
2022 MessageElement::Static(_, _, _) => {
2023 let max = get_needed_message_size(DataType::TelemetryError);
2024 let bytes = msg.as_bytes();
2025 let n = core::cmp::min(max, bytes.len());
2026 let mut buf = vec![0u8; max];
2027 if n > 0 {
2028 buf[..n].copy_from_slice(&bytes[..n]);
2029 }
2030 crate::shared_bytes::convert(Arc::<[u8]>::from(buf))
2031 }
2032 MessageElement::Dynamic(_, _) => {
2033 crate::shared_bytes::convert(Arc::<[u8]>::from(msg.as_bytes()))
2034 }
2035 }
2036}
2037
2038fn log_raw<T, F>(
2042 sender: Arc<str>,
2043 ty: DataType,
2044 data: &[T],
2045 timestamp: u64,
2046 mut tx_function: F,
2047) -> TelemetryResult<()>
2048where
2049 T: LeBytes,
2050 F: FnMut(Packet) -> TelemetryResult<()>,
2051{
2052 let meta = message_meta(ty);
2053 let got = data
2054 .len()
2055 .checked_mul(T::WIDTH)
2056 .ok_or(TelemetryError::PacketTooLarge(
2057 "typed payload size overflow",
2058 ))?;
2059 if T::WIDTH == 0 {
2060 return Err(TelemetryError::BadArg);
2061 }
2062
2063 match meta.element {
2064 MessageElement::Static(_, _, _) => {
2065 let expected = crate::data_type_size(meta.element.data_type()) * meta.element;
2066 if got != expected {
2067 return Err(TelemetryError::SizeMismatch { expected, got });
2068 }
2069 }
2070 MessageElement::Dynamic(_, _) => {
2071 }
2074 }
2075
2076 const INLINE_ENCODE_BYTES: usize = if crate::config::STACK_PAYLOAD_SIZE < u8::MAX as usize {
2077 crate::config::STACK_PAYLOAD_SIZE
2078 } else {
2079 u8::MAX as usize
2080 };
2081 let payload = if got <= INLINE_ENCODE_BYTES {
2082 let mut bytes = [0u8; INLINE_ENCODE_BYTES];
2085 for (value, destination) in data
2086 .iter()
2087 .copied()
2088 .zip(bytes[..got].chunks_exact_mut(T::WIDTH))
2089 {
2090 value.write_le(destination);
2091 }
2092 crate::config::StandardSmallPayload::new(&bytes[..got])
2093 } else {
2094 crate::config::StandardSmallPayload::from_arc(encode_slice_le(data))
2095 };
2096 let pkt = Packet::new_local(ty, meta.endpoints_owned(), sender, timestamp, payload)?;
2097 tx_function(pkt)
2098}
2099
2100#[cfg(feature = "std")]
2102fn reliable_trace_enabled() -> bool {
2103 static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
2104 *ENABLED.get_or_init(|| {
2105 matches!(
2106 std::env::var("SEDSNET_TRACE_RELIABLE").as_deref(),
2107 Ok("1" | "true")
2108 )
2109 })
2110}
2111
2112fn fallback_stdout(_msg: &str) {
2116 #[cfg(feature = "std")]
2117 {
2118 eprintln!("{}", _msg);
2119 }
2120 #[cfg(all(not(feature = "std"), target_os = "none"))]
2121 {
2122 unsafe {
2125 seds_error_msg(_msg.as_ptr(), _msg.len());
2126 }
2127 }
2128}
2129
2130#[derive(Debug, Clone)]
2135struct LocalServicesState {
2138 managed_variable_types: BTreeSet<u32>,
2139 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
2140 managed_variable_latest: BTreeMap<u32, Box<ManagedVariableCacheEntry>>,
2141 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
2142 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
2143 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
2144 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
2145 next_p2p_stream_id: P2pStreamId,
2146}
2147
2148#[derive(Default)]
2149struct ForwardedReceiptCache {
2150 destinations: BTreeMap<u64, BTreeSet<u64>>,
2151 order: VecDeque<u64>,
2152}
2153
2154struct RouterInner {
2155 #[cfg(feature = "compression")]
2156 codec_reserved_bytes: usize,
2157 memory: RuntimeMemoryConfig,
2158 sides: Vec<Option<RouterSide>>,
2159 route_overrides: RouterMap<(Option<RouterSideId>, RouterSideId), bool>,
2160 typed_route_overrides: RouterMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
2161 route_weights: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2162 route_priorities: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2163 source_route_modes: RouterMap<Option<RouterSideId>, RouteSelectionMode>,
2164 route_selection_cursors: RouterMap<Option<RouterSideId>, u64>,
2165 adaptive_route_stats: RouterMap<RouterSideId, AdaptiveRouteStats>,
2166 side_runtime_stats: RouterMap<RouterSideId, Box<SideRuntimeStatsInner>>,
2169 side_transport: RouterMap<RouterSideId, Box<SideTransportState>>,
2170 local_services: Option<Box<LocalServicesState>>,
2171 local_address: AddressBookEntry,
2172 address_book: BTreeMap<String, AddressBookEntry>,
2173 address_by_value: BTreeMap<NodeAddress, String>,
2174 received_queue: BoundedDeque<RouterRxItem>,
2175 transmit_queue: BoundedDeque<TxQueued>,
2176 tx_priority_burst: u8,
2177 has_ordered_retry: bool,
2178 ordered_claim_seq: u32,
2179 rx_priority_burst: u8,
2180 recent_rx: BoundedDeque<u64>,
2181 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
2182 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
2183 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
2184 reliable_return_route_order: VecDeque<u64>,
2185 forwarded_receipts: Option<Box<ForwardedReceiptCache>>,
2186 end_to_end_reliable_tx: BTreeMap<u64, Box<EndToEndReliableSent>>,
2187 end_to_end_reliable_tx_order: VecDeque<u64>,
2188 total_handler_failures: u64,
2189 total_handler_retries: u64,
2190 #[cfg(feature = "discovery")]
2191 discovery_routes: RouterMap<RouterSideId, DiscoverySideState>,
2192 #[cfg(feature = "discovery")]
2193 discovery_cadence: DiscoveryCadenceState,
2194 #[cfg(feature = "discovery")]
2195 discovery_side_throttle: RouterMap<RouterSideId, DiscoverySideThrottleState>,
2196 #[cfg(feature = "discovery")]
2197 schema_response_sequence: u64,
2198 #[cfg(feature = "discovery")]
2199 preferred_discovery_master: Option<Arc<str>>,
2200 #[cfg(all(feature = "discovery", feature = "timesync"))]
2201 timesync_side_throttle: RouterMap<RouterSideId, TimeSyncSideThrottleState>,
2202}
2203#[derive(Debug, Clone)]
2204struct ManagedVariableCacheEntry {
2205 packet: Packet,
2206 sender_address: u32,
2207 #[cfg(feature = "discovery")]
2208 cached_at_ms: u64,
2209}
2210
2211type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
2212
2213#[derive(Clone)]
2214struct NetworkVariableUpdateHandler {
2215 handler: Arc<NetworkVariableUpdateFn>,
2216}
2217
2218impl Debug for NetworkVariableUpdateHandler {
2219 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2220 f.write_str("NetworkVariableUpdateHandler(<handler>)")
2221 }
2222}
2223
2224#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2226pub struct NetworkVariablePermissions {
2227 pub read: bool,
2228 pub write: bool,
2229}
2230
2231impl NetworkVariablePermissions {
2232 pub const NONE: Self = Self {
2233 read: false,
2234 write: false,
2235 };
2236 pub const READ_ONLY: Self = Self {
2237 read: true,
2238 write: false,
2239 };
2240 pub const WRITE_ONLY: Self = Self {
2241 read: false,
2242 write: true,
2243 };
2244 pub const READ_WRITE: Self = Self {
2245 read: true,
2246 write: true,
2247 };
2248}
2249
2250#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2251enum RouterQueueKind {
2252 Received,
2253 Transmit,
2254}
2255
2256#[cfg(feature = "compression")]
2257impl crate::payload_codec::QueueBudget for RouterInner {
2258 fn codec_reserved(&mut self) -> &mut usize {
2259 &mut self.codec_reserved_bytes
2260 }
2261 fn codec_used(&self) -> usize {
2262 self.shared_queue_bytes_used()
2263 }
2264 fn codec_limit(&self) -> usize {
2265 self.memory.max_queue_budget
2266 }
2267}
2268
2269impl RouterInner {
2270 fn is_managed_variable(&self, ty: u32) -> bool {
2271 self.local_services
2272 .as_ref()
2273 .is_some_and(|local| local.managed_variable_types.contains(&ty))
2274 }
2275
2276 fn local_cache_bytes(&self) -> usize {
2277 self.local_services
2278 .iter()
2279 .flat_map(|local| local.managed_variable_latest.values())
2280 .map(|entry| entry.packet.byte_cost())
2281 .sum()
2282 }
2283
2284 fn replay_counters(&self) -> (usize, usize) {
2285 self.transmit_queue
2286 .iter()
2287 .filter(|q| q.item.replay_marker_key().is_some())
2288 .fold((0usize, 0usize), |(n, bytes), q| {
2289 (n + 1, bytes.saturating_add(q.byte_cost()))
2290 })
2291 }
2292
2293 fn local_cache_len(&self) -> usize {
2294 self.local_services
2295 .as_ref()
2296 .map_or(0, |local| local.managed_variable_latest.len())
2297 }
2298
2299 #[cfg(all(feature = "compression", feature = "discovery"))]
2300 fn reclaim_codec_metadata(&mut self) -> bool {
2301 self.side_transport.values_mut().any(|transport| {
2302 transport
2303 .payload_codec
2304 .evict_for_budget(DISCOVERY_ROUTE_TTL_MS)
2305 })
2306 }
2307
2308 #[cfg(feature = "compression")]
2313 fn prepare_codec_queue_owner(&mut self, side: Option<usize>, data: &mut RouterItem) {
2314 let RouterItem::Packed(bytes) = data else {
2315 return;
2316 };
2317 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
2318 return;
2319 }
2320 let capable = {
2321 #[cfg(feature = "discovery")]
2322 {
2323 side.and_then(|side| self.side_transport.get(&side))
2324 .is_some_and(|transport| {
2325 transport.payload_codec.retention_can_stay_compressed()
2326 })
2327 }
2328 #[cfg(not(feature = "discovery"))]
2329 {
2330 let _ = side;
2331 false
2332 }
2333 };
2334 if capable {
2335 return;
2336 }
2337 let Ok(required) = wire_format::decoded_frame_upper_bound(bytes.as_ref()) else {
2338 return;
2339 };
2340 if self
2341 .shared_queue_bytes_used()
2342 .checked_add(required)
2343 .and_then(|used| {
2344 used.checked_add(
2345 size_of::<crate::SharedBytes>()
2346 + size_of::<bool>()
2347 + size_of::<u8>()
2348 + size_of::<u64>(),
2349 )
2350 })
2351 .is_none_or(|used| used > self.memory.max_queue_budget)
2352 {
2353 return;
2354 }
2355 if let Ok(Some(raw)) = wire_format::payload_encoding_for_retention(
2356 bytes.as_ref(),
2357 self.memory.max_queue_budget,
2358 ) {
2359 *bytes = crate::shared_bytes::convert(raw);
2360 }
2361 }
2362
2363 #[cfg(feature = "discovery")]
2364 fn schema_response_owner(item: &RouterTxItem) -> Option<(RouterSideId, u64)> {
2365 match item {
2366 RouterTxItem::ToSide { dst, data, .. }
2367 if Router::item_data_type(data).ok() == Some(DataType::DiscoverySchema) =>
2368 {
2369 Some((*dst, Router::get_hash(data)))
2370 }
2371 _ => None,
2372 }
2373 }
2374
2375 #[cfg(feature = "discovery")]
2376 fn schema_response_owner_lost(
2377 responses: &mut RouterMap<RouterSideId, DiscoverySideThrottleState>,
2378 owner: Option<(RouterSideId, u64)>,
2379 ) {
2380 if let Some((side, id)) = owner
2381 && let Some(response) = responses.get_mut(&side)
2382 && response.schema_response_active
2383 && response.schema_response_phase == SchemaResponsePhase::Owned
2384 && response.schema_response_packet_id == id
2385 {
2386 response.schema_response_packet_id = 0;
2390 response.schema_response_started = false;
2391 response.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
2392 }
2393 }
2394
2395 fn retain_transmit<F>(&mut self, keep: F)
2396 where
2397 F: FnMut(&TxQueued) -> bool,
2398 {
2399 let hop = &mut self.reliable_tx;
2400 let end = &mut self.end_to_end_reliable_tx;
2401 #[cfg(feature = "discovery")]
2402 let responses = &mut self.discovery_side_throttle;
2403 self.transmit_queue.retain_observed(keep, |evicted| {
2404 #[cfg(feature = "discovery")]
2405 Self::schema_response_owner_lost(responses, Self::schema_response_owner(&evicted.item));
2406 if let Some(key) = evicted.item.replay_marker_key() {
2407 Self::set_replay_queued(hop, end, key, false);
2408 }
2409 });
2410 }
2411
2412 #[inline(never)]
2413 fn claim_index(&self, token: u32) -> Option<usize> {
2414 self.transmit_queue.iter().position(|item| {
2415 matches!(
2416 item.delivery_state,
2417 QueuedDelivery::ClaimedOrdered | QueuedDelivery::ClaimedUnordered
2418 ) && item.deferred_since_ms == u64::from(token)
2419 })
2420 }
2421
2422 #[inline(never)]
2423 fn refresh_retry_hint(&mut self) {
2424 self.has_ordered_retry = self.transmit_queue.iter().any(|item| {
2425 matches!(
2426 item.delivery_state,
2427 QueuedDelivery::ClaimedOrdered
2428 | QueuedDelivery::ClaimedUnordered
2429 | QueuedDelivery::DeferredOrdered
2430 )
2431 });
2432 }
2433
2434 fn cancel_accepted_type(&mut self, ty: u32) {
2435 self.retain_transmit(|item| {
2436 !matches!(&item.item,
2437 RouterTxItem::Broadcast(_) if item.accepted_type == ty)
2438 });
2439 }
2440
2441 fn replay_is_queued(&self, key: ReplayMarkerKey) -> Option<bool> {
2442 match key {
2443 ReplayMarkerKey::EndToEnd(id) => self.end_to_end_reliable_tx.get(&id).map(|s| s.queued),
2444 ReplayMarkerKey::Hop(side, ty, seq) => self
2445 .reliable_tx
2446 .get(&(side, ty.as_u32()))
2447 .and_then(|tx| tx.sent.get(&seq))
2448 .map(|s| s.queued),
2449 }
2450 }
2451 fn set_replay_queued(
2452 hop: &mut BTreeMap<(RouterSideId, u32), ReliableTxState>,
2453 end: &mut BTreeMap<u64, Box<EndToEndReliableSent>>,
2454 key: ReplayMarkerKey,
2455 queued: bool,
2456 ) {
2457 let flag = match key {
2458 ReplayMarkerKey::EndToEnd(id) => end.get_mut(&id).map(|s| &mut s.queued),
2459 ReplayMarkerKey::Hop(side, ty, seq) => hop
2460 .get_mut(&(side, ty.as_u32()))
2461 .and_then(|tx| tx.sent.get_mut(&seq))
2462 .map(|s| &mut s.queued),
2463 };
2464 if let Some(flag) = flag {
2465 *flag = queued;
2466 }
2467 }
2468
2469 #[cfg(any(feature = "std", feature = "fair-admission"))]
2470 fn fair_bulk_owner(data: &RouterItem, priority: u8) -> Option<u32> {
2471 match data {
2472 RouterItem::Packet(packet) => crate::admission_fairness::packet_owner(packet, priority),
2473 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2474 crate::admission_fairness::packed_owner(view, priority)
2475 })
2476 .ok()
2477 .flatten(),
2478 }
2479 }
2480 #[cfg(any(feature = "std", feature = "fair-admission"))]
2481 fn tx_bulk_owner(item: &TxQueued) -> Option<u32> {
2482 match &item.item {
2483 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2484 Self::fair_bulk_owner(data, item.priority)
2485 }
2486 _ => None,
2487 }
2488 }
2489 #[cfg(any(feature = "std", feature = "fair-admission"))]
2490 fn bulk_stream(data: &RouterItem) -> Option<u32> {
2491 match data {
2492 RouterItem::Packet(packet) => Some(packet.data_type().as_u32()),
2493 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2494 wire_format::scheduling_type(view).map(|ty| ty.as_u32())
2495 })
2496 .ok()
2497 .flatten(),
2498 }
2499 }
2500 #[cfg(any(feature = "std", feature = "fair-admission"))]
2501 fn tx_bulk_stream(item: &TxQueued) -> Option<u32> {
2502 match &item.item {
2503 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2504 Self::bulk_stream(data)
2505 }
2506 _ => None,
2507 }
2508 }
2509 #[cfg(any(feature = "std", feature = "fair-admission"))]
2510 fn bulk_loads(&self, priority: u8) -> crate::admission_fairness::Loads {
2511 let mut loads = crate::admission_fairness::Loads::new();
2512 for item in self.received_queue.iter() {
2513 if item.priority <= priority
2514 && let Some(owner) = Self::fair_bulk_owner(&item.data, item.priority)
2515 {
2516 loads.record(owner, item.byte_cost());
2517 }
2518 }
2519 for item in self.transmit_queue.iter() {
2520 if item.priority <= priority
2521 && let Some(owner) = Self::tx_bulk_owner(item)
2522 {
2523 loads.record(owner, item.byte_cost());
2524 }
2525 }
2526 loads
2527 }
2528 #[cfg(any(feature = "std", feature = "fair-admission"))]
2529 fn competing_bulk_owner(&self, incoming: u32, priority: u8) -> Option<u32> {
2530 self.bulk_loads(priority).dominant_other(incoming)
2531 }
2532 #[cfg(any(feature = "std", feature = "fair-admission"))]
2533 fn reclaim_bulk_from(
2534 &mut self,
2535 owner: u32,
2536 priority: u8,
2537 incoming_stream: Option<u32>,
2538 ) -> bool {
2539 let mut streams = crate::admission_fairness::Loads::new();
2540 for item in self.received_queue.iter() {
2541 if item.priority <= priority
2542 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2543 && let Some(ty) = Self::bulk_stream(&item.data)
2544 {
2545 streams.record(ty, item.byte_cost());
2546 }
2547 }
2548 for item in self.transmit_queue.iter() {
2549 if item.priority <= priority
2550 && Self::tx_bulk_owner(item) == Some(owner)
2551 && let Some(ty) = Self::tx_bulk_stream(item)
2552 {
2553 streams.record(ty, item.byte_cost());
2554 }
2555 }
2556 let stream = streams.queue_victim(incoming_stream);
2557 let rx = self
2558 .received_queue
2559 .iter()
2560 .enumerate()
2561 .filter(|(_, item)| {
2562 item.priority <= priority
2563 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2564 && stream.is_none_or(|ty| Self::bulk_stream(&item.data) == Some(ty))
2565 })
2566 .min_by_key(|(index, item)| (item.priority, *index))
2567 .map(|(index, item)| (index, item.priority));
2568 let tx = self
2569 .transmit_queue
2570 .iter()
2571 .enumerate()
2572 .filter(|(_, item)| {
2573 item.priority <= priority
2574 && Self::tx_bulk_owner(item) == Some(owner)
2575 && stream.is_none_or(|ty| Self::tx_bulk_stream(item) == Some(ty))
2576 })
2577 .min_by_key(|(index, item)| (item.priority, *index))
2578 .map(|(index, item)| (index, item.priority));
2579 match (rx, tx) {
2580 (Some((index, rp)), Some((_, tp))) if rp <= tp => {
2581 self.received_queue.remove_pos(index).is_some()
2582 }
2583 (Some((index, _)), None) => self.received_queue.remove_pos(index).is_some(),
2584 (_, Some((index, _))) => self.transmit_queue.remove_pos(index).is_some(),
2585 _ => false,
2586 }
2587 }
2588
2589 #[cfg(feature = "discovery")]
2590 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
2591 board
2592 .sender_id
2593 .len()
2594 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
2595 .saturating_add(
2596 board
2597 .reachable_timesync_sources
2598 .iter()
2599 .map(|s| s.len())
2600 .sum::<usize>(),
2601 )
2602 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
2603 }
2604
2605 #[cfg(feature = "discovery")]
2606 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
2607 sender
2608 .len()
2609 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
2610 .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
2611 .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
2612 .saturating_add(
2613 state
2614 .reachable_timesync_sources
2615 .iter()
2616 .map(|s| s.len())
2617 .sum::<usize>(),
2618 )
2619 .saturating_add(
2620 state
2621 .advertised_reachable_timesync_sources
2622 .iter()
2623 .map(|s| s.len())
2624 .sum::<usize>(),
2625 )
2626 .saturating_add(
2627 state
2628 .topology_boards
2629 .iter()
2630 .map(Self::topology_board_byte_cost)
2631 .sum::<usize>(),
2632 )
2633 .saturating_add(size_of::<DiscoverySenderState>())
2634 }
2635
2636 #[cfg(feature = "discovery")]
2637 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
2638 size_of::<RouterSideId>()
2639 .saturating_add(size_of::<DiscoverySideState>())
2640 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
2641 .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
2642 .saturating_add(
2643 route.requested_network_variables.len()
2644 * (size_of::<DataType>() + size_of::<u64>()),
2645 )
2646 .saturating_add(
2647 route
2648 .reachable_timesync_sources
2649 .iter()
2650 .map(|s| s.len())
2651 .sum::<usize>(),
2652 )
2653 .saturating_add(
2654 route
2655 .announcers
2656 .iter()
2657 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
2658 .sum::<usize>(),
2659 )
2660 .saturating_add(side.saturating_sub(side))
2661 }
2662
2663 #[cfg(feature = "discovery")]
2664 fn discovery_bytes_used(&self) -> usize {
2665 self.discovery_routes
2666 .iter()
2667 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
2668 .sum()
2669 }
2670
2671 #[inline]
2672 fn reliable_rx_buffered_bytes(&self) -> usize {
2673 self.reliable_rx
2674 .values()
2675 .flat_map(|state| state.buffered.values())
2676 .map(|bytes| size_of::<crate::SharedBytes>() + bytes.len())
2677 .sum()
2678 }
2679
2680 fn reliable_rx_buffer_len(&self) -> usize {
2681 self.reliable_rx
2682 .values()
2683 .map(|state| state.buffered.len())
2684 .sum()
2685 }
2686
2687 #[inline(never)]
2688 fn side_chunk_bytes_used(&self) -> usize {
2689 self.side_transport
2690 .values()
2691 .map(|side| side.rx_chunk_bytes)
2692 .sum()
2693 }
2694 fn discard_side_chunk(&mut self, side: RouterSideId, id: u32) -> bool {
2695 self.side_transport
2696 .get_mut(&side)
2697 .is_some_and(|state| state.take_chunk(id, true).is_some())
2698 }
2699 fn reclaim_oldest_side_chunk(&mut self, keep: Option<(RouterSideId, u32)>) -> bool {
2700 let mut victim = None;
2701 let mut time = u64::MAX;
2702 for (&side, state) in &self.side_transport {
2703 let preserved = match keep {
2704 Some((owner, id)) if owner == side => Some(id),
2705 _ => None,
2706 };
2707 if let Some((id, seen)) = SideChunkAssembly::oldest(&state.rx_chunks, preserved)
2708 && (victim.is_none() || seen < time)
2709 {
2710 time = seen;
2711 victim = Some((side, id));
2712 }
2713 }
2714 match victim {
2715 Some((side, id)) => self.discard_side_chunk(side, id),
2716 None => false,
2717 }
2718 }
2719 fn make_side_chunk_room(
2720 &mut self,
2721 side: RouterSideId,
2722 id: u32,
2723 index: u16,
2724 total: u16,
2725 payload_len: usize,
2726 ) -> TelemetryResult<()> {
2727 let budget = self
2728 .memory
2729 .max_queue_budget
2730 .saturating_sub(self.protected_metadata_bytes_used());
2731 let state = self
2732 .side_transport
2733 .get_mut(&side)
2734 .ok_or(TelemetryError::BadArg)?;
2735 let existing = state.rx_chunks.get(&id).map(|a| a.total);
2736 if existing.is_some_and(|old| old != 0 && old != total) {
2737 state.take_chunk(id, true);
2738 return Err(TelemetryError::Unpack("side chunk total mismatch"));
2739 }
2740 let before = state.rx_chunks.len();
2741 let room = SideChunkAssembly::make_room(
2742 &mut state.rx_chunks,
2743 &mut state.rx_chunk_bytes,
2744 id,
2745 index,
2746 total,
2747 payload_len,
2748 budget,
2749 );
2750 if before != state.rx_chunks.len() {
2751 state.rx_chunk_reclaims = state
2752 .rx_chunk_reclaims
2753 .saturating_add((before - state.rx_chunks.len()) as u32);
2754 }
2755 let cost = room?;
2756 if cost == 0 {
2757 return Ok(());
2758 }
2759 if existing.is_none()
2760 && state.rx_chunks.len() >= 4
2761 && let Some((oldest, _)) = SideChunkAssembly::oldest(&state.rx_chunks, None)
2762 {
2763 state.take_chunk(oldest, true);
2764 }
2765 while self.shared_queue_bytes_used().saturating_add(cost) > self.memory.max_queue_budget {
2766 #[cfg(any(feature = "std", feature = "fair-admission"))]
2767 if let Some(owner) = self.bulk_loads(199).queue_victim(None)
2768 && self.reclaim_bulk_from(owner, 199, None)
2769 {
2770 continue;
2771 }
2772 if self.reclaim_oldest_side_chunk(Some((side, id))) {
2773 continue;
2774 }
2775 return Err(TelemetryError::Io("memory pressure"));
2778 }
2779 Ok(())
2780 }
2781
2782 #[inline(never)]
2783 fn protected_metadata_bytes_used(&self) -> usize {
2784 let codec_bytes = {
2785 #[cfg(all(feature = "compression", feature = "discovery"))]
2786 {
2787 self.side_transport
2788 .values()
2789 .map(|side| side.payload_codec.heap_bytes())
2790 .sum::<usize>()
2791 }
2792 #[cfg(not(all(feature = "compression", feature = "discovery")))]
2793 {
2794 0usize
2795 }
2796 };
2797 codec_bytes.saturating_add(
2798 self.recent_rx
2799 .max_bytes()
2800 .saturating_add(self.reliable_rx_buffered_bytes())
2801 .saturating_add(crate::config::schema_bytes_used())
2802 .saturating_add({
2803 #[cfg(feature = "discovery")]
2804 {
2805 self.discovery_bytes_used()
2806 }
2807 #[cfg(not(feature = "discovery"))]
2808 {
2809 0
2810 }
2811 }),
2812 )
2813 }
2814 #[inline]
2815 fn shared_queue_bytes_used(&self) -> usize {
2816 self.received_queue
2817 .bytes_used()
2818 .saturating_add(self.transmit_queue.bytes_used())
2819 .saturating_add(self.side_chunk_bytes_used())
2820 .saturating_add(self.protected_metadata_bytes_used())
2821 .saturating_add({
2822 #[cfg(feature = "compression")]
2823 {
2824 self.codec_reserved_bytes
2825 }
2826 #[cfg(not(feature = "compression"))]
2827 {
2828 0
2829 }
2830 })
2831 }
2832
2833 fn classify_transmit_disposable(item: &TxQueued) -> bool {
2837 let data = match &item.item {
2838 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => data,
2839 _ => return true,
2840 };
2841 match data {
2842 RouterItem::Packet(pkt) => {
2843 is_internal_control_type(pkt.data_type())
2844 || (!is_reliable_type(pkt.data_type()) && pkt.wire_target_senders().is_empty())
2845 }
2846 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2847 wire_format::peek_routing_frame_info(view).map_or(true, |frame| {
2848 is_internal_control_type(frame.envelope.ty)
2849 || (!is_reliable_type(frame.envelope.ty)
2850 && frame.reliable.is_none()
2851 && frame.envelope.target_senders().len() == 0)
2852 })
2853 })
2854 .unwrap_or(false),
2855 }
2856 }
2857
2858 fn transmit_eviction_candidate(queue: &BoundedDeque<TxQueued>) -> Option<usize> {
2859 queue
2860 .iter()
2861 .enumerate()
2862 .filter(|(_, queued)| queued.disposable)
2863 .min_by_key(|(index, queued)| (queued.priority, core::cmp::Reverse(*index)))
2864 .map(|(index, _)| index)
2865 }
2866
2867 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
2868 match preferred {
2869 RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
2874 RouterQueueKind::Transmit => {
2875 let Some(index) = Self::transmit_eviction_candidate(&self.transmit_queue) else {
2876 return false;
2877 };
2878 let Some(item) = self.transmit_queue.remove_pos(index) else {
2879 return false;
2880 };
2881 #[cfg(feature = "discovery")]
2882 Self::schema_response_owner_lost(
2883 &mut self.discovery_side_throttle,
2884 Self::schema_response_owner(&item.item),
2885 );
2886 if let Some(key) = item.item.replay_marker_key() {
2887 Self::set_replay_queued(
2888 &mut self.reliable_tx,
2889 &mut self.end_to_end_reliable_tx,
2890 key,
2891 false,
2892 );
2893 }
2894 true
2895 }
2896 }
2897 }
2898
2899 fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
2900 let received = self
2901 .received_queue
2902 .lowest_priority(|queued| queued.priority)
2903 .map(|priority| (RouterQueueKind::Received, priority));
2904 let transmit = self
2905 .transmit_queue
2906 .iter()
2907 .filter(|queued| queued.disposable)
2908 .map(|queued| queued.priority)
2909 .min()
2910 .map(|priority| (RouterQueueKind::Transmit, priority));
2911 [received, transmit]
2912 .into_iter()
2913 .flatten()
2914 .filter(|(_, priority)| {
2915 *priority < incoming_priority
2916 || (incoming_priority != 254 && *priority == incoming_priority)
2917 })
2918 .min_by_key(|(kind, priority)| {
2919 (
2920 *priority,
2921 if *kind == RouterQueueKind::Received {
2922 0
2923 } else {
2924 1
2925 },
2926 )
2927 })
2928 .map(|(kind, _)| kind)
2929 }
2930
2931 fn make_shared_queue_room(
2932 &mut self,
2933 incoming_cost: usize,
2934 incoming_priority: u8,
2935 _incoming: Option<&RouterItem>,
2936 ) -> TelemetryResult<()> {
2937 if incoming_cost > self.memory.max_queue_budget {
2938 return Err(TelemetryError::PacketTooLarge(
2939 "Item exceeds maximum shared queue budget",
2940 ));
2941 }
2942
2943 #[cfg(any(feature = "std", feature = "fair-admission"))]
2944 let mut fair_victim = None;
2945 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
2946 > self.memory.max_queue_budget
2947 {
2948 #[cfg(all(feature = "compression", feature = "discovery"))]
2949 if self.reclaim_codec_metadata() {
2950 continue;
2951 }
2952
2953 #[cfg(any(feature = "std", feature = "fair-admission"))]
2954 {
2955 let incoming =
2956 _incoming.and_then(|data| Self::fair_bulk_owner(data, incoming_priority));
2957 let victim = *fair_victim.get_or_insert_with(|| {
2958 self.bulk_loads(incoming_priority.min(199))
2959 .queue_victim(incoming)
2960 });
2961 if let Some(victim) = victim
2962 && self.reclaim_bulk_from(
2963 victim,
2964 incoming_priority.min(199),
2965 (incoming == Some(victim))
2966 .then(|| _incoming.and_then(Self::bulk_stream))
2967 .flatten(),
2968 )
2969 {
2970 continue;
2971 }
2972 if incoming.is_some() {
2973 return Err(TelemetryError::Io("shared fair queue saturated"));
2974 }
2975 }
2976 let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
2977 if incoming_priority >= 200 && self.reclaim_oldest_side_chunk(None) {
2978 continue;
2979 }
2980 return Err(TelemetryError::Io("shared priority queue saturated"));
2981 };
2982 if !self.pop_shared_queue_item(victim) {
2983 return Err(TelemetryError::PacketTooLarge(
2984 "Item exceeds maximum shared queue budget",
2985 ));
2986 }
2987 }
2988
2989 Ok(())
2990 }
2991
2992 #[cfg(feature = "discovery")]
2993 fn schema_growth_budget(&self) -> usize {
2994 let protected = self
2997 .recent_rx
2998 .max_bytes()
2999 .saturating_add(self.discovery_bytes_used())
3000 .saturating_add(self.reliable_rx_buffered_bytes())
3001 .saturating_add({
3002 #[cfg(feature = "compression")]
3003 {
3004 self.codec_reserved_bytes
3005 }
3006 #[cfg(not(feature = "compression"))]
3007 {
3008 0
3009 }
3010 });
3011 self.memory.max_queue_budget.saturating_sub(protected)
3012 }
3013
3014 #[cfg(feature = "discovery")]
3015 fn commit_discovery_route(
3016 &mut self,
3017 side: RouterSideId,
3018 route: DiscoverySideState,
3019 ) -> TelemetryResult<()> {
3020 let previous = self
3021 .discovery_routes
3022 .get(&side)
3023 .map_or(0, |route| Self::discovery_route_byte_cost(side, route));
3024 let incoming = Self::discovery_route_byte_cost(side, &route);
3025 let growth = incoming.saturating_sub(previous);
3026 self.make_shared_queue_room(growth, 255, None)?;
3027 self.discovery_routes.insert(side, route);
3028 Ok(())
3029 }
3030
3031 #[cfg(feature = "discovery")]
3032 fn fit_discovery_budget(&mut self) {
3033 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
3034 #[cfg(all(feature = "compression", feature = "discovery"))]
3035 if self.reclaim_codec_metadata() {
3036 continue;
3037 }
3038
3039 if let Some(victim) = self.lower_priority_shared_queue(255)
3042 && self.pop_shared_queue_item(victim)
3043 {
3044 continue;
3045 }
3046 break;
3049 }
3050 }
3051
3052 fn reliable_original_type(item: &TxQueued) -> Option<DataType> {
3053 let RouterTxItem::Broadcast(data) = &item.item else {
3054 return None;
3055 };
3056 let _ = data;
3057 let ty = DataType(item.accepted_type);
3058 (item.delivery_state != QueuedDelivery::Untracked).then_some(ty)
3059 }
3060
3061 fn ordered_original_type(item: &TxQueued) -> Option<DataType> {
3062 let RouterTxItem::Broadcast(data) = &item.item else {
3063 return None;
3064 };
3065 let _ = data;
3066 let ty = DataType(item.accepted_type);
3067 matches!(
3068 item.delivery_state,
3069 QueuedDelivery::Ordered
3070 | QueuedDelivery::OrderedTail
3071 | QueuedDelivery::DeferredOrdered
3072 | QueuedDelivery::ClaimedOrdered
3073 )
3074 .then_some(ty)
3075 }
3076
3077 fn pop_received_fair(&mut self) -> Option<RouterRxItem> {
3078 self.received_queue.pop_front_bulk_fair(
3079 &mut self.rx_priority_burst,
3080 MAX_PRIORITY_BURST,
3081 |item| item.priority,
3082 )
3083 }
3084
3085 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
3086 self.make_shared_queue_room(item.byte_cost(), item.priority, Some(&item.data))?;
3087 #[cfg(any(feature = "std", feature = "fair-admission"))]
3088 {
3089 #[cfg(any(feature = "std", feature = "fair-admission"))]
3090 let mut fair_victim = None;
3091 self.received_queue.push_back_prioritized_select(
3092 item,
3093 |queued| queued.priority,
3094 |queue, _incoming| {
3095 #[cfg(any(feature = "std", feature = "fair-admission"))]
3096 {
3097 crate::admission_fairness::queue_victim(
3098 queue,
3099 _incoming,
3100 |item| Self::fair_bulk_owner(&item.data, item.priority),
3101 |item| item.priority,
3102 |item| Self::bulk_stream(&item.data),
3103 &mut fair_victim,
3104 )
3105 }
3106 },
3107 core::mem::drop,
3108 )
3109 }
3110 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3111 {
3112 self.received_queue
3113 .push_back_prioritized(item, |queued| queued.priority)
3114 }
3115 }
3116
3117 fn push_transmit(&mut self, mut item: TxQueued) -> TelemetryResult<()> {
3118 #[cfg(feature = "compression")]
3119 match &mut item.item {
3120 RouterTxItem::Broadcast(data) => self.prepare_codec_queue_owner(None, data),
3121 RouterTxItem::ToSide { dst, data, .. } => {
3122 self.prepare_codec_queue_owner(Some(*dst), data)
3123 }
3124 _ => {}
3125 }
3126 if let RouterTxItem::Broadcast(data) = &item.item {
3127 item.accepted_type = match data {
3128 RouterItem::Packet(packet) => packet.data_type().as_u32(),
3129 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3130 wire_format::scheduling_type(view)
3131 })
3132 .ok()
3133 .flatten()
3134 .map_or(0, |ty| ty.as_u32()),
3135 };
3136 }
3137 if item.delivery_state == QueuedDelivery::Untracked
3138 && matches!(&item.item, RouterTxItem::Broadcast(_))
3139 {
3140 item.delivery_state = match reliable_mode(DataType(item.accepted_type)) {
3141 crate::ReliableMode::Ordered => QueuedDelivery::Ordered,
3142 crate::ReliableMode::Unordered => QueuedDelivery::Unordered,
3143 crate::ReliableMode::None => QueuedDelivery::Untracked,
3144 };
3145 }
3146 if matches!(
3149 item.delivery_state,
3150 QueuedDelivery::Ordered | QueuedDelivery::DeferredOrdered
3151 ) && let Some(priority) = self.transmit_queue.iter().find_map(|queued| {
3152 (Self::ordered_original_type(queued).is_some()
3153 && queued.accepted_type == item.accepted_type)
3154 .then_some(queued.priority)
3155 }) {
3156 item.priority = priority;
3157 item.delivery_state = QueuedDelivery::OrderedTail;
3158 }
3159 item.disposable = if item.delivery_state != QueuedDelivery::Untracked {
3160 false
3161 } else {
3162 Self::classify_transmit_disposable(&item)
3163 };
3164 let ordered_retry = Self::ordered_original_type(&item).is_some()
3165 && item.delivery_state == QueuedDelivery::DeferredOrdered;
3166 let replay = item.item.replay_marker_key();
3167 if replay.is_some_and(|key| self.replay_is_queued(key).is_none_or(|queued| queued)) {
3170 return Ok(());
3171 }
3172 #[cfg(feature = "discovery")]
3173 if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
3174 if let RouterItem::Packet(incoming) = data
3177 && self.transmit_queue.iter().any(|queued| {
3178 matches!(&queued.item,
3179 RouterTxItem::ToSide { dst: queued_dst, data: RouterItem::Packet(existing), .. }
3180 if queued_dst == dst && discovery::same_schema_snapshot(existing, incoming))
3181 })
3182 {
3183 return Ok(());
3184 }
3185 let incoming = match data {
3186 RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
3187 Some((pkt.data_type(), pkt.sender().to_owned()))
3188 }
3189 RouterItem::Packet(_) | RouterItem::Packed(_) => None,
3190 };
3191 if let Some((incoming_ty, incoming_sender)) = incoming {
3192 self.retain_transmit(|queued| {
3193 let RouterTxItem::ToSide {
3194 dst: queued_dst,
3195 data: queued_data,
3196 ..
3197 } = &queued.item
3198 else {
3199 return true;
3200 };
3201 let same_snapshot = match queued_data {
3202 RouterItem::Packet(pkt) => {
3203 pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
3204 }
3205 RouterItem::Packed(_) => false,
3206 };
3207 *queued_dst != *dst || !same_snapshot
3208 });
3209 }
3210 }
3211 self.make_shared_queue_room(
3212 item.byte_cost(),
3213 item.priority,
3214 match &item.item {
3215 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => Some(data),
3216 _ => None,
3217 },
3218 )?;
3219 let reliable_tx = &mut self.reliable_tx;
3220 let end_to_end = &mut self.end_to_end_reliable_tx;
3221 #[cfg(feature = "discovery")]
3222 let responses = &mut self.discovery_side_throttle;
3223 let result = {
3224 #[cfg(any(feature = "std", feature = "fair-admission"))]
3225 {
3226 #[cfg(any(feature = "std", feature = "fair-admission"))]
3227 let mut fair_victim = None;
3228 self.transmit_queue.push_back_prioritized_select(
3229 item,
3230 |queued| queued.priority,
3231 |queue, _incoming| {
3232 #[cfg(any(feature = "std", feature = "fair-admission"))]
3233 {
3234 if Self::tx_bulk_owner(_incoming).is_some() {
3235 crate::admission_fairness::queue_victim(
3236 queue,
3237 _incoming,
3238 Self::tx_bulk_owner,
3239 |item| item.priority,
3240 Self::tx_bulk_stream,
3241 &mut fair_victim,
3242 )
3243 } else {
3244 Self::transmit_eviction_candidate(queue)
3245 }
3246 }
3247 },
3248 |evicted: TxQueued| {
3249 #[cfg(feature = "discovery")]
3250 Self::schema_response_owner_lost(
3251 responses,
3252 Self::schema_response_owner(&evicted.item),
3253 );
3254 if let Some(key) = evicted.item.replay_marker_key() {
3255 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3256 }
3257 },
3258 )
3259 }
3260 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3261 {
3262 self.transmit_queue.push_back_prioritized_select(
3263 item,
3264 |queued| queued.priority,
3265 |queue, _| Self::transmit_eviction_candidate(queue),
3266 |evicted: TxQueued| {
3267 #[cfg(feature = "discovery")]
3268 Self::schema_response_owner_lost(
3269 responses,
3270 Self::schema_response_owner(&evicted.item),
3271 );
3272 if let Some(key) = evicted.item.replay_marker_key() {
3273 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3274 }
3275 },
3276 )
3277 }
3278 };
3279 if result.is_ok() && ordered_retry {
3280 self.has_ordered_retry = true;
3281 }
3282 if result.is_ok()
3283 && let Some(key) = replay
3284 {
3285 Self::set_replay_queued(reliable_tx, end_to_end, key, true);
3286 }
3287 result
3288 }
3289
3290 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
3291 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
3292 let _ = self.recent_rx.pop_front();
3293 }
3294 self.make_shared_queue_room(0, 0, None)?;
3295 self.recent_rx.push_back(id)
3296 }
3297
3298 #[allow(unused_mut)]
3299 fn buffer_reliable_rx(
3300 &mut self,
3301 side: RouterSideId,
3302 ty: DataType,
3303 seq: u32,
3304 mut bytes: crate::SharedBytes,
3305 ) -> TelemetryResult<()> {
3306 let key = Router::reliable_key(side, ty);
3307 if self
3308 .reliable_rx
3309 .get(&key)
3310 .is_some_and(|state| state.buffered.contains_key(&seq))
3311 {
3312 return Ok(());
3313 }
3314 let cost = size_of::<crate::SharedBytes>() + bytes.len();
3315 self.make_shared_queue_room(cost, crate::transport_priority(ty), None)?;
3316 #[cfg(feature = "compact-packet-store")]
3317 bytes.park_for_queue()?;
3318 let rx_state = self
3319 .reliable_rx
3320 .entry(key)
3321 .or_insert_with(|| ReliableRxState {
3322 expected_seq: 1,
3323 buffered: Default::default(),
3324 });
3325 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
3326 let _ = rx_state.buffered.pop_first();
3327 }
3328 rx_state.buffered.insert(seq, bytes);
3329 Ok(())
3330 }
3331}
3332
3333struct IsrRxQueue {
3338 busy: AtomicBool,
3339 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
3340}
3341
3342unsafe impl Send for IsrRxQueue {}
3343unsafe impl Sync for IsrRxQueue {}
3344
3345struct IsrRxQueueGuard<'a> {
3346 owner: &'a IsrRxQueue,
3347}
3348
3349impl Deref for IsrRxQueueGuard<'_> {
3350 type Target = BoundedDeque<RouterRxItem>;
3351
3352 #[inline]
3353 fn deref(&self) -> &Self::Target {
3354 unsafe { &*self.owner.q.get() }
3355 }
3356}
3357
3358impl DerefMut for IsrRxQueueGuard<'_> {
3359 #[inline]
3360 fn deref_mut(&mut self) -> &mut Self::Target {
3361 unsafe { &mut *self.owner.q.get() }
3362 }
3363}
3364
3365impl Drop for IsrRxQueueGuard<'_> {
3366 #[inline]
3367 fn drop(&mut self) {
3368 self.owner.busy.store(false, Ordering::Release);
3369 }
3370}
3371
3372impl IsrRxQueue {
3373 #[inline]
3374 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
3375 Self {
3376 busy: AtomicBool::new(false),
3377 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
3378 }
3379 }
3380
3381 #[inline]
3382 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
3383 match self
3384 .busy
3385 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
3386 {
3387 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
3388 Err(_) => Err(TelemetryError::Io("rx queue busy")),
3389 }
3390 }
3391
3392 #[allow(dead_code)]
3393 #[inline]
3394 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
3395 let mut g = self.try_lock()?;
3396 g.push_back(item)
3397 }
3398
3399 #[inline]
3400 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
3401 let mut g = self.try_lock()?;
3402 g.push_back_prioritized(item, |queued| queued.priority)
3403 }
3404
3405 #[inline]
3406 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
3407 let mut g = self.try_lock()?;
3408 Ok(g.pop_front())
3409 }
3410
3411 #[inline]
3412 fn clear(&self) -> TelemetryResult<()> {
3413 let mut g = self.try_lock()?;
3414 g.clear();
3415 Ok(())
3416 }
3417
3418 #[inline]
3419 fn snapshot(&self) -> Option<(usize, usize)> {
3420 let g = self.try_lock().ok()?;
3421 Some((g.len(), g.bytes_used()))
3422 }
3423}
3424
3425pub struct Router {
3429 sender: RouterMutex<Arc<str>>,
3430 cfg: ActiveRoutingConfig,
3431 state: RouterMutex<RouterInner>,
3432 isr_rx_queue: Option<Box<IsrRxQueue>>,
3433 #[cfg(not(feature = "std"))]
3434 side_tx_gate: ReentryGate,
3435 #[cfg(feature = "std")]
3436 side_tx_activity: crate::lock::SideTxActivity,
3437 #[cfg(feature = "std")]
3438 active_queued_tx: core::sync::atomic::AtomicUsize,
3439 clock: Box<dyn Clock + Send + Sync>,
3440 #[cfg(feature = "timesync")]
3441 timesync: Option<Box<RouterMutex<TimeSyncRuntime>>>,
3442}
3443
3444#[cfg(feature = "std")]
3445struct QueuedTxActivityGuard<'a>(&'a core::sync::atomic::AtomicUsize);
3446#[cfg(feature = "std")]
3447impl Drop for QueuedTxActivityGuard<'_> {
3448 fn drop(&mut self) {
3449 self.0.fetch_sub(1, core::sync::atomic::Ordering::Release);
3450 }
3451}
3452
3453#[cfg(feature = "timesync")]
3454#[derive(Debug, Clone)]
3455struct PendingTimeSyncRequest {
3456 seq: u64,
3457 t1_mono_ms: u64,
3458 source: String,
3459}
3460
3461#[cfg(feature = "timesync")]
3462#[derive(Debug, Clone)]
3463struct RemoteTimeSyncSource {
3464 priority: u64,
3465 last_sample_mono_ms: u64,
3466 sample_unix_ms: u64,
3467}
3468
3469#[cfg(feature = "timesync")]
3470#[derive(Debug, Clone)]
3471struct TimeSyncRuntime {
3472 cfg: Option<TimeSyncConfig>,
3473 tracker: Option<TimeSyncTracker>,
3474 clock: NetworkClock,
3475 disciplined_clock: SlewedNetworkClock,
3476 remote_sources: BTreeMap<String, RemoteTimeSyncSource>,
3477 next_seq: u64,
3478 next_announce_mono_ms: u64,
3479 next_request_mono_ms: u64,
3480 pending_request: Option<PendingTimeSyncRequest>,
3481}
3482
3483#[cfg(feature = "timesync")]
3484impl TimeSyncRuntime {
3485 fn new(cfg: Option<TimeSyncConfig>) -> Self {
3486 Self {
3487 tracker: cfg.map(TimeSyncTracker::new),
3488 cfg,
3489 clock: NetworkClock::default(),
3490 disciplined_clock: SlewedNetworkClock::new(
3491 cfg.map(|c| c.max_slew_ppm)
3492 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
3493 ),
3494 remote_sources: Default::default(),
3495 next_seq: 1,
3496 next_announce_mono_ms: 0,
3497 next_request_mono_ms: 0,
3498 pending_request: None,
3499 }
3500 }
3501}
3502
3503enum RemoteSidePlan {
3504 Target(Vec<RouterSideId>),
3505}
3506
3507#[cfg(feature = "discovery")]
3508#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
3509struct DiscoveryCandidateMatch {
3510 side: RouterSideId,
3511 overlap: usize,
3512}
3513
3514#[cfg(feature = "discovery")]
3515#[inline]
3516const fn discovery_is_hop_local_advertisement(ty: DataType, transparent: bool) -> bool {
3517 matches!(
3518 ty,
3519 DataType::DiscoveryAnnounce
3520 | DataType::DiscoveryTimeSyncSources
3521 | DataType::DiscoveryTopology
3522 | DataType::DiscoveryLinkCapabilities
3523 ) || (matches!(ty, DataType::DiscoveryAddress) && !transparent)
3524}
3525
3526#[cfg(not(feature = "discovery"))]
3527const fn discovery_is_hop_local_advertisement(_ty: DataType, _transparent: bool) -> bool {
3528 false
3529}
3530
3531impl Debug for Router {
3532 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
3533 let sender = self.sender();
3534 f.debug_struct("Router")
3535 .field("sender", &sender)
3536 .field("cfg", &self.cfg)
3537 .field("state", &"<mutex>")
3538 .field("clock", &"Clock")
3539 .finish()
3540 }
3541}
3542
3543#[inline]
3546fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &ActiveRoutingConfig) -> bool {
3547 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
3548}
3549
3550#[inline]
3551fn force_remote_for_type(ty: DataType) -> bool {
3552 matches!(
3553 ty,
3554 DataType::ReliableAck
3555 | DataType::ReliablePartialAck
3556 | DataType::ReliablePacketRequest
3557 | DataType::P2pMessage
3558 ) || {
3559 #[cfg(feature = "timesync")]
3560 {
3561 matches!(
3562 ty,
3563 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
3564 )
3565 }
3566 #[cfg(not(feature = "timesync"))]
3567 {
3568 false
3569 }
3570 }
3571}
3572
3573#[inline]
3574fn is_internal_control_type(ty: DataType) -> bool {
3575 if matches!(
3576 ty,
3577 DataType::ReliableAck
3578 | DataType::ReliablePartialAck
3579 | DataType::ReliablePacketRequest
3580 | DataType::P2pMessage
3581 ) {
3582 return true;
3583 }
3584
3585 #[cfg(feature = "timesync")]
3586 if matches!(
3587 ty,
3588 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
3589 ) {
3590 return true;
3591 }
3592
3593 #[cfg(feature = "discovery")]
3594 if discovery::is_discovery_type(ty) {
3595 return true;
3596 }
3597
3598 let _ = ty;
3599 false
3600}
3601
3602fn with_retries<F>(
3604 this: &Router,
3605 dest: DataEndpoint,
3606 data: &RouterItem,
3607 pkt_for_ctx: Option<&Packet>,
3608 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
3609 called_from_queue: bool,
3610 run: F,
3611) -> TelemetryResult<()>
3612where
3613 F: Fn() -> TelemetryResult<()>,
3614{
3615 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
3616 Ok(((), attempts)) => {
3617 if attempts > 1 {
3618 let mut st = this.state.lock();
3619 st.total_handler_retries = st
3620 .total_handler_retries
3621 .saturating_add(attempts.saturating_sub(1) as u64);
3622 }
3623 Ok(())
3624 }
3625 Err((e, attempts)) => {
3626 {
3627 let mut st = this.state.lock();
3628 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3629 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3630 }
3631
3632 this.remove_pkt_id(data);
3634
3635 if let Some(pkt) = pkt_for_ctx {
3637 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
3638 } else if let Some(env) = env_for_ctx {
3639 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
3640 }
3641
3642 Err(TelemetryError::HandlerError("local handler failed"))
3643 }
3644 }
3645}
3646impl Router {
3648 #[cfg(feature = "discovery")]
3649 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
3650 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
3651
3652 #[inline]
3653 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
3654 st.sides
3655 .get(side)
3656 .and_then(|side| side.as_ref())
3657 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
3658 }
3659
3660 fn note_side_tx_success(
3661 &self,
3662 side: RouterSideId,
3663 ty: DataType,
3664 bytes: usize,
3665 relayed: bool,
3666 attempts: usize,
3667 ) {
3668 let mut st = self.state.lock();
3669 #[cfg(feature = "discovery")]
3670 if ty == crate::DataType::DiscoveryAnnounce
3671 && !relayed
3672 && let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
3673 {
3674 throttle.last_liveness_tx_ms = self.clock.now_ms();
3675 }
3676 #[cfg(all(feature = "discovery", feature = "timesync"))]
3677 if Self::is_timesync_type(ty) {
3678 let now = self.clock.now_ms();
3679 if Self::side_is_slow_control_link_locked(&st, side, now) {
3680 st.timesync_side_throttle
3681 .entry(side)
3682 .or_default()
3683 .next_allowed_ms = now.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
3684 } else {
3685 st.timesync_side_throttle.remove(&side);
3686 }
3687 }
3688 let entry = st.side_runtime_stats.entry(side).or_default();
3689 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
3690 }
3691
3692 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
3693 let mut st = self.state.lock();
3694 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
3695 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
3696 let entry = st.side_runtime_stats.entry(side).or_default();
3697 entry.note_tx_failure(ty, attempts);
3698 }
3699
3700 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
3701 let mut st = self.state.lock();
3702 let entry = st.side_runtime_stats.entry(side).or_default();
3703 entry.note_rx(ty, bytes, relayed);
3704 }
3705
3706 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
3707 let mut st = self.state.lock();
3708 let entry = st.side_runtime_stats.entry(side).or_default();
3709 entry.note_local_delivery(ty);
3710 }
3711
3712 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
3713 let mut st = self.state.lock();
3714 let entry = st.side_runtime_stats.entry(side).or_default();
3715 entry.note_local_handler_failure(ty, retries);
3716 }
3717
3718 fn cache_managed_variable_packet(
3719 &self,
3720 pkt: &Packet,
3721 notify_handlers: bool,
3722 ) -> TelemetryResult<()> {
3723 let sender_address = pkt
3724 .sender()
3725 .strip_prefix("@addr:")
3726 .and_then(|value| value.parse::<u32>().ok())
3727 .unwrap_or_else(|| sender_address_u32(pkt.sender()));
3728 let handlers = {
3729 let mut st = self.state.lock();
3730 if !st.is_managed_variable(pkt.data_type().as_u32()) {
3731 return Ok(());
3732 }
3733 let existing = st
3734 .local_services
3735 .as_ref()
3736 .expect("local services enabled")
3737 .managed_variable_latest
3738 .get(&pkt.data_type().as_u32());
3739 if existing.is_some_and(|entry| {
3746 let current = &entry.packet;
3747 sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
3748 }) {
3749 return Ok(());
3750 }
3751 let changed = existing.is_none_or(|entry| entry.packet != *pkt);
3752 let updated = ManagedVariableCacheEntry {
3756 packet: pkt.clone(),
3757 sender_address,
3758 #[cfg(feature = "discovery")]
3759 cached_at_ms: self.clock.now_ms(),
3760 };
3761 if let Some(entry) = st
3762 .local_services
3763 .as_mut()
3764 .expect("local services enabled")
3765 .managed_variable_latest
3766 .get_mut(&pkt.data_type().as_u32())
3767 {
3768 **entry = updated;
3769 } else {
3770 st.local_services
3771 .as_mut()
3772 .expect("local services enabled")
3773 .managed_variable_latest
3774 .insert(pkt.data_type().as_u32(), Box::new(updated));
3775 }
3776 if notify_handlers && changed {
3777 st.local_services
3778 .as_ref()
3779 .expect("local services enabled")
3780 .network_variable_update_handlers
3781 .get(&pkt.data_type().as_u32())
3782 .cloned()
3783 .unwrap_or_default()
3784 } else {
3785 Vec::new()
3786 }
3787 };
3788 for handler in handlers {
3789 (handler.handler)(pkt)?;
3790 }
3791 Ok(())
3792 }
3793
3794 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
3795 self.cache_managed_variable_packet(pkt, true)
3796 }
3797
3798 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
3799 let st = self.state.lock();
3800 st.local_services
3801 .as_ref()?
3802 .managed_variable_latest
3803 .get(&ty.as_u32())
3804 .map(|entry| entry.packet.clone())
3805 }
3806
3807 #[cfg(feature = "discovery")]
3808 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
3809 let now_ms = self.clock.now_ms();
3810 let st = self.state.lock();
3811 st.local_services
3812 .as_ref()?
3813 .managed_variable_latest
3814 .get(&ty.as_u32())
3815 .map(|entry| {
3816 (
3817 entry.packet.clone(),
3818 now_ms.saturating_sub(entry.cached_at_ms),
3819 )
3820 })
3821 }
3822
3823 fn is_managed_variable_type(&self, ty: DataType) -> bool {
3824 let st = self.state.lock();
3825 st.is_managed_variable(ty.as_u32())
3826 }
3827
3828 fn managed_variable_permissions_locked(
3829 st: &RouterInner,
3830 ty: DataType,
3831 ) -> NetworkVariablePermissions {
3832 let Some(local) = st.local_services.as_ref() else {
3833 return NetworkVariablePermissions::NONE;
3834 };
3835 local
3836 .managed_variable_permissions
3837 .get(&ty.as_u32())
3838 .copied()
3839 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
3840 }
3841
3842 fn can_read_managed_variable(&self, ty: DataType) -> bool {
3843 let st = self.state.lock();
3844 Self::managed_variable_permissions_locked(&st, ty).read
3845 }
3846
3847 #[cfg(feature = "discovery")]
3848 fn can_write_managed_variable(&self, ty: DataType) -> bool {
3849 let st = self.state.lock();
3850 Self::managed_variable_permissions_locked(&st, ty).write
3851 }
3852
3853 #[inline]
3854 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
3855 let st = self.state.lock();
3856 let side_ref = Self::side_ref(&st, side)?;
3857 if side_ref.opts.ingress_enabled {
3858 Ok(())
3859 } else {
3860 Err(TelemetryError::HandlerError(
3861 "router: ingress disabled for side id",
3862 ))
3863 }
3864 }
3865
3866 #[inline]
3867 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
3868 src != Some(dst)
3869 }
3870
3871 #[inline]
3872 fn route_allowed_locked(
3873 &self,
3874 st: &RouterInner,
3875 src: Option<RouterSideId>,
3876 ty: Option<DataType>,
3877 dst: RouterSideId,
3878 ) -> bool {
3879 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
3880 return false;
3881 };
3882 if !dst_side.opts.egress_enabled {
3883 return false;
3884 }
3885 if let Some(src_id) = src {
3886 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
3887 return false;
3888 };
3889 if !src_side.opts.ingress_enabled {
3890 return false;
3891 }
3892 }
3893 let base_allowed = st
3894 .route_overrides
3895 .get(&(src, dst))
3896 .copied()
3897 .unwrap_or_else(|| self.default_route_enabled(src, dst));
3898 if !base_allowed {
3899 return false;
3900 }
3901
3902 let Some(ty) = ty else {
3903 return true;
3904 };
3905 if st
3906 .typed_route_overrides
3907 .keys()
3908 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3909 {
3910 return st
3911 .typed_route_overrides
3912 .get(&(src, ty.as_u32(), dst))
3913 .copied()
3914 .unwrap_or(false);
3915 }
3916 true
3917 }
3918
3919 #[cfg(feature = "discovery")]
3920 fn has_typed_route_overrides_locked(
3921 st: &RouterInner,
3922 src: Option<RouterSideId>,
3923 ty: DataType,
3924 ) -> bool {
3925 st.typed_route_overrides
3926 .keys()
3927 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
3928 }
3929
3930 #[cfg(feature = "discovery")]
3931 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
3932 where
3933 I: IntoIterator<Item = DataEndpoint>,
3934 {
3935 let mut overlap = 0usize;
3936 for ep in reachable {
3937 if eps.contains(&ep) {
3938 overlap = overlap.saturating_add(1);
3939 }
3940 }
3941 overlap
3942 }
3943
3944 #[inline]
3945 #[cfg(feature = "discovery")]
3946 fn preferred_scoring_endpoints<'a>(
3947 &self,
3948 eps: &'a [DataEndpoint],
3949 prefer_nonlocal: bool,
3950 ) -> alloc::borrow::Cow<'a, [DataEndpoint]> {
3951 if !prefer_nonlocal || eps.iter().all(|&ep| !self.cfg.is_local_endpoint(ep)) {
3952 return alloc::borrow::Cow::Borrowed(eps);
3953 }
3954 let nonlocal: Vec<DataEndpoint> = eps
3955 .iter()
3956 .copied()
3957 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
3958 .collect();
3959 if nonlocal.is_empty() {
3960 alloc::borrow::Cow::Borrowed(eps)
3961 } else {
3962 alloc::borrow::Cow::Owned(nonlocal)
3963 }
3964 }
3965
3966 fn pop_next_queued_control_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
3967 if let Some(queue) = self.isr_rx_queue.as_ref() {
3968 let mut isr_rx = queue.try_lock()?;
3969 let idx = isr_rx
3970 .iter()
3971 .position(|item| self.queued_rx_item_is_control(item));
3972 if let Some(idx) = idx {
3973 return Ok(isr_rx.remove_pos(idx));
3974 }
3975 }
3976
3977 let mut st = self.state.lock();
3978 let idx = st
3979 .received_queue
3980 .iter()
3981 .position(|item| self.queued_rx_item_is_control(item));
3982 if let Some(idx) = idx {
3983 return Ok(st.received_queue.remove_pos(idx));
3984 }
3985 Ok(None)
3986 }
3987
3988 fn queued_rx_item_is_control(&self, item: &RouterRxItem) -> bool {
3989 if !self.cfg.role.local_services()
3993 && item.priority < crate::transport_priority(DataType::ReliableAck)
3994 {
3995 return false;
3996 }
3997 if item.priority < crate::transport_priority(DataType::ManagedVariableValue) {
4001 return false;
4002 }
4003 if item.priority == crate::transport_priority(DataType::ManagedVariableValue) {
4006 return true;
4007 }
4008 let ty = match &item.data {
4009 RouterItem::Packet(pkt) => Some(pkt.data_type()),
4010 RouterItem::Packed(bytes) => {
4011 if bytes.len() <= 128
4012 && wire_format::peek_routing_frame_info(bytes)
4013 .is_ok_and(|frame| frame.ack_only())
4014 {
4015 return true;
4016 }
4017 wire_format::scheduling_type(bytes.as_ref())
4018 }
4019 };
4020 match ty {
4021 Some(
4022 DataType::ReliableAck
4023 | DataType::ReliablePartialAck
4024 | DataType::ReliablePacketRequest,
4025 ) => true,
4026 #[cfg(feature = "discovery")]
4027 Some(ty) => self.cfg.role.local_services() && discovery::is_discovery_type(ty),
4028 _ => false,
4029 }
4030 }
4031
4032 fn drain_queued_control_rx_before_tx(
4033 &self,
4034 start: u64,
4035 timeout_ms: u32,
4036 ) -> TelemetryResult<bool> {
4037 let mut did_any = false;
4038 while let Some(item) = self.pop_next_queued_control_rx_item()? {
4039 self.process_rx_queue_item(item)?;
4040 did_any = true;
4041 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms) {
4044 break;
4045 }
4046 }
4047 Ok(did_any)
4048 }
4049
4050 fn eligible_side_ids_locked(
4051 &self,
4052 st: &RouterInner,
4053 src: Option<RouterSideId>,
4054 ty: Option<DataType>,
4055 restrict_link_local: bool,
4056 ) -> Vec<RouterSideId> {
4057 st.sides
4058 .iter()
4059 .enumerate()
4060 .filter_map(|(side_id, side)| {
4061 let side = side.as_ref()?;
4062 if restrict_link_local && !side.opts.link_local_enabled {
4063 return None;
4064 }
4065 if self.route_allowed_locked(st, src, ty, side_id) {
4066 Some(side_id)
4067 } else {
4068 None
4069 }
4070 })
4071 .collect()
4072 }
4073
4074 fn apply_route_selection_locked(
4075 &self,
4076 st: &mut RouterInner,
4077 src: Option<RouterSideId>,
4078 mut sides: Vec<RouterSideId>,
4079 origin: RouteSelectionOrigin,
4080 ) -> Vec<RouterSideId> {
4081 if sides.len() <= 1 {
4082 return sides;
4083 }
4084
4085 let selection_mode = st.source_route_modes.get(&src).copied();
4086 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
4087 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
4088 }
4089
4090 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
4091 RouteSelectionMode::Fanout => sides,
4092 RouteSelectionMode::Weighted => {
4093 crate::small_sort::sort(&mut sides);
4094 let total_weight = sides.iter().fold(0_u64, |acc, side| {
4095 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
4096 });
4097 if total_weight == 0 {
4098 return Vec::new();
4099 }
4100 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4101 let pick = *cursor % total_weight;
4102 *cursor = cursor.wrapping_add(1);
4103 let mut remaining = pick;
4104 for side in sides {
4105 let weight =
4106 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
4107 if remaining < weight {
4108 return vec![side];
4109 }
4110 remaining -= weight;
4111 }
4112 Vec::new()
4113 }
4114 RouteSelectionMode::Failover => {
4115 sides.sort_by_key(|side| {
4116 (
4117 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
4118 *side,
4119 )
4120 });
4121 sides.truncate(1);
4122 sides
4123 }
4124 }
4125 }
4126
4127 fn apply_adaptive_discovery_selection_locked(
4128 &self,
4129 st: &mut RouterInner,
4130 src: Option<RouterSideId>,
4131 mut sides: Vec<RouterSideId>,
4132 ) -> Vec<RouterSideId> {
4133 crate::small_sort::sort(&mut sides);
4134 let mut unmeasured: Vec<_> = sides
4135 .iter()
4136 .copied()
4137 .filter(|side| !st.adaptive_route_stats.contains_key(side))
4138 .collect();
4139 if !unmeasured.is_empty() {
4140 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4141 let pick = (*cursor as usize) % unmeasured.len();
4142 *cursor = cursor.wrapping_add(1);
4143 return vec![unmeasured.swap_remove(pick)];
4144 }
4145
4146 let now_ms = self.clock.now_ms();
4147 let total_weight = sides.iter().fold(0_u64, |acc, side| {
4148 acc + st
4149 .adaptive_route_stats
4150 .get(side)
4151 .map(|stats| stats.weight(now_ms))
4152 .unwrap_or(1)
4153 });
4154 if total_weight == 0 {
4155 sides.truncate(1);
4156 return sides;
4157 }
4158
4159 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4160 let pick = *cursor % total_weight;
4161 *cursor = cursor.wrapping_add(1);
4162 let mut remaining = pick;
4163 for side in sides {
4164 let weight = st
4165 .adaptive_route_stats
4166 .get(&side)
4167 .map(|stats| stats.weight(now_ms))
4168 .unwrap_or(1);
4169 if remaining < weight {
4170 return vec![side];
4171 }
4172 remaining -= weight;
4173 }
4174 Vec::new()
4175 }
4176
4177 fn record_side_tx_sample(
4178 &self,
4179 side: RouterSideId,
4180 bytes: usize,
4181 started_ms: u64,
4182 ended_ms: u64,
4183 ) {
4184 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
4185 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
4186 .min(u128::from(u64::MAX)) as u64;
4187 let mut st = self.state.lock();
4188 st.adaptive_route_stats
4189 .entry(side)
4190 .or_default()
4191 .observe(bytes, sample_bps, ended_ms);
4192 }
4193
4194 pub fn note_side_link_probe_sample(
4199 &self,
4200 side: RouterSideId,
4201 bytes: usize,
4202 duration_ms: u64,
4203 ) -> TelemetryResult<()> {
4204 {
4205 let st = self.state.lock();
4206 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
4207 }
4208 let ended_ms = self.clock.now_ms();
4209 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
4210 Ok(())
4211 }
4212
4213 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
4214 match data {
4215 RouterItem::Packet(pkt) => Ok(wire_format::packet_wire_size(pkt)),
4216 RouterItem::Packed(bytes) => Ok(bytes.len()),
4217 }
4218 }
4219
4220 #[inline]
4230 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
4231 match data {
4232 RouterItem::Packet(pkt) => {
4233 if !Self::is_end_to_end_ack_packet(pkt) {
4234 return Ok(None);
4235 }
4236 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4237 }
4238 RouterItem::Packed(bytes) => {
4239 if wire_format::peek_routing_frame_info(bytes.as_ref())
4240 .ok()
4241 .is_some_and(|frame| {
4242 frame.envelope.ty != DataType::ReliableAck || frame.ack_only()
4243 })
4244 {
4245 return Ok(None);
4246 }
4247 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4248 if !Self::is_end_to_end_ack_packet(&pkt) {
4249 return Ok(None);
4250 }
4251 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4252 }
4253 }
4254 }
4255
4256 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
4257 if payload.len() != 8 {
4258 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
4259 }
4260 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
4261 }
4262
4263 #[inline]
4264 fn sender_hash(sender: &str) -> u64 {
4265 if let Some(address) = sender
4266 .strip_prefix("@addr:")
4267 .and_then(|v| v.parse::<u32>().ok())
4268 {
4269 return u64::from(address);
4270 }
4271 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
4272 }
4273
4274 #[inline]
4275 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
4276 let hash = Self::sender_hash(hostname);
4277 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
4278 if address == 0 {
4279 address = 1;
4280 }
4281 address
4282 }
4283
4284 fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
4289 let Some(address) = sender
4290 .strip_prefix("@addr:")
4291 .and_then(|value| value.parse::<u32>().ok())
4292 else {
4293 return sender.to_string();
4294 };
4295 Self::hostname_for_address_locked(st, address).unwrap_or_else(|| sender.to_string())
4296 }
4297
4298 fn hostname_for_address_locked(st: &RouterInner, address: NodeAddress) -> Option<String> {
4305 Self::hostname_for_address_borrowed_locked(st, address).map(str::to_string)
4306 }
4307
4308 fn hostname_for_address_borrowed_locked<'a>(
4309 st: &'a RouterInner,
4310 address: NodeAddress,
4311 ) -> Option<&'a str> {
4312 if let Some(hostname) = st.address_by_value.get(&address) {
4313 return Some(hostname.as_str());
4314 }
4315
4316 let mut found: Option<&'a str> = None;
4317 let mut collision = false;
4318 let mut consider = |candidate: &'a str| {
4319 if sender_address_u32(candidate) != address {
4320 return;
4321 }
4322 match found {
4323 None => found = Some(candidate),
4324 Some(existing) if existing == candidate => {}
4325 Some(_) => collision = true,
4329 }
4330 };
4331
4332 for entry in st.address_book.values() {
4333 consider(entry.hostname.as_ref());
4334 }
4335 #[cfg(feature = "discovery")]
4336 for route in st.discovery_routes.values() {
4337 for (announcer, state) in &route.announcers {
4338 consider(announcer);
4339 for board in state.topology_boards.iter() {
4340 consider(&board.sender_id);
4341 for peer in &board.connections {
4347 consider(peer);
4348 }
4349 }
4350 }
4351 }
4352 if collision { None } else { found }
4353 }
4354
4355 #[cfg(feature = "discovery")]
4356 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
4357 match mode {
4358 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
4359 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
4360 _ => AddressAssignmentMode::Dynamic,
4361 }
4362 }
4363
4364 #[cfg(feature = "discovery")]
4365 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
4366 let rank = match entry.mode {
4367 AddressAssignmentMode::Static(_) => 2,
4368 AddressAssignmentMode::Requested(_) => 1,
4369 AddressAssignmentMode::Dynamic => 0,
4370 };
4371 (
4372 rank,
4373 core::cmp::Reverse(entry.birth_ms),
4374 u64::MAX - entry.owner_hash,
4375 )
4376 }
4377
4378 #[cfg(feature = "discovery")]
4379 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
4380 (
4381 core::cmp::Reverse(entry.birth_ms),
4382 u64::MAX - entry.owner_hash,
4383 )
4384 }
4385
4386 #[cfg(feature = "discovery")]
4387 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
4388 let mut candidate = if seed == 0 { 1 } else { seed };
4389 for _ in 0..u32::MAX {
4390 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
4391 return candidate;
4392 }
4393 candidate = candidate.wrapping_add(1);
4394 if candidate == 0 {
4395 candidate = 1;
4396 }
4397 }
4398 1
4399 }
4400
4401 #[cfg(feature = "discovery")]
4402 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
4403 if !st.address_book.contains_key(base) {
4404 return base.to_string();
4405 }
4406 let stem = if base.is_empty() { "node" } else { base };
4407 let mut candidate = format!("{stem}-{owner_hash:08x}");
4408 let mut suffix = 1u32;
4409 while st.address_book.contains_key(&candidate) {
4410 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
4411 suffix = suffix.saturating_add(1);
4412 }
4413 candidate
4414 }
4415
4416 fn update_local_identity_locked(
4417 &self,
4418 st: &mut RouterInner,
4419 mut local: AddressBookEntry,
4420 reason: AddressChangeReason,
4421 ) -> AddressChange {
4422 let old = st.local_address.clone();
4423 st.address_book.remove(old.hostname.as_ref());
4424 st.address_by_value.remove(&old.address);
4425 local.last_seen_ms = self.clock.now_ms();
4426 st.address_by_value
4427 .insert(local.address, local.hostname.to_string());
4428 st.address_book
4429 .insert(local.hostname.to_string(), local.clone());
4430 st.local_address = local.clone();
4431 *self.sender.lock() = local.hostname.clone();
4432 #[cfg(feature = "discovery")]
4433 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
4434 AddressChange {
4435 old_hostname: old.hostname,
4436 new_hostname: local.hostname,
4437 old_address: old.address,
4438 new_address: local.address,
4439 reason,
4440 }
4441 }
4442
4443 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
4444 for handler in self.cfg.address_change_handlers().iter() {
4445 (handler.handler)(change.clone())?;
4446 }
4447 Ok(())
4448 }
4449
4450 #[cfg(feature = "discovery")]
4451 fn local_address_advertisement(
4452 &self,
4453 reachable_endpoints: Vec<DataEndpoint>,
4454 reachable_network_variables: Vec<DataType>,
4455 reachable_timesync_sources: Vec<String>,
4456 link_capabilities: discovery::LinkCapabilities,
4457 state: u8,
4458 ) -> discovery::AddressAdvertisement {
4459 let st = self.state.lock();
4460 discovery::AddressAdvertisement {
4461 hostname: st.local_address.hostname.to_string(),
4462 address: st.local_address.address,
4463 requested_address: st.local_address.requested_address,
4464 mode: st.local_address.mode.mode_code(),
4465 state,
4466 birth_ms: st.local_address.birth_ms,
4467 owner_hash: st.local_address.owner_hash,
4468 reachable_endpoints,
4469 reachable_network_variables,
4470 reachable_timesync_sources,
4471 link_capabilities,
4472 }
4473 }
4474
4475 #[cfg(feature = "discovery")]
4476 fn ingest_address_advertisement(
4477 &self,
4478 ad: discovery::AddressAdvertisement,
4479 ) -> TelemetryResult<bool> {
4480 let now_ms = self.clock.now_ms();
4481 let mut remote = AddressBookEntry {
4482 hostname: Arc::from(ad.hostname.as_str()),
4483 address: ad.address,
4484 requested_address: ad.requested_address,
4485 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
4486 birth_ms: ad.birth_ms,
4487 owner_hash: ad.owner_hash,
4488 last_seen_ms: now_ms,
4489 };
4490 let mut change = None;
4491 let mut changed = false;
4492 {
4493 let mut st = self.state.lock();
4494 let local = st.local_address.clone();
4495 let hostname_conflict =
4496 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
4497 let address_conflict =
4498 remote.address == local.address && remote.owner_hash != local.owner_hash;
4499
4500 if hostname_conflict
4501 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
4502 {
4503 let mut next = local.clone();
4504 next.hostname = Arc::from(Self::unique_hostname_locked(
4505 &st,
4506 local.hostname.as_ref(),
4507 local.owner_hash,
4508 ));
4509 change = Some(self.update_local_identity_locked(
4510 &mut st,
4511 next,
4512 AddressChangeReason::HostnameConflict,
4513 ));
4514 changed = true;
4515 } else if address_conflict
4516 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
4517 {
4518 let mut next = local.clone();
4519 let seed = next
4520 .requested_address
4521 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
4522 next.address = Self::allocate_free_address_locked(&st, seed);
4523 let reason = match next.mode {
4524 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
4525 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
4526 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
4527 };
4528 change = Some(self.update_local_identity_locked(&mut st, next, reason));
4529 changed = true;
4530 } else if hostname_conflict {
4531 remote.hostname = Arc::from(Self::unique_hostname_locked(
4532 &st,
4533 remote.hostname.as_ref(),
4534 remote.owner_hash,
4535 ));
4536 changed = true;
4537 } else if address_conflict {
4538 let seed = remote
4539 .requested_address
4540 .max(Self::fallback_address_for_hostname(
4541 remote.hostname.as_ref(),
4542 ));
4543 remote.address = Self::allocate_free_address_locked(&st, seed);
4544 changed = true;
4545 }
4546
4547 st.address_book.remove(remote.hostname.as_ref());
4548 st.address_by_value.remove(&remote.address);
4549 st.address_by_value
4550 .insert(remote.address, remote.hostname.to_string());
4551 st.address_book
4552 .insert(remote.hostname.to_string(), remote.clone());
4553 }
4554 if let Some(change) = change {
4555 self.notify_address_change(change)?;
4556 }
4557 Ok(changed)
4558 }
4559
4560 #[inline]
4561 #[cfg(feature = "discovery")]
4562 fn is_end_to_end_ack_sender(sender: &str) -> bool {
4563 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
4564 }
4565
4566 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
4567 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
4568 && !ack_sender.is_empty()
4569 {
4570 return Some(Self::sender_hash(ack_sender));
4571 }
4572 None
4573 }
4574
4575 fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
4581 if pkt.data_type() != DataType::ReliableAck {
4582 return None;
4583 }
4584 pkt.wire_target_senders()
4585 .first()
4586 .copied()
4587 .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
4588 }
4589
4590 fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
4591 Self::end_to_end_ack_sender_hash(pkt).is_some()
4592 && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
4593 }
4594
4595 fn encode_end_to_end_ack_sender(&self) -> String {
4596 let sender = self.sender_arc();
4597 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
4598 }
4599
4600 #[cfg(feature = "discovery")]
4601 fn is_end_to_end_destination_sender(&self, sender: &str) -> bool {
4602 sender != self.sender_arc().as_ref() && !Self::is_end_to_end_ack_sender(sender)
4603 }
4604
4605 fn encode_p2p_payload(
4606 src_hostname: &str,
4607 src_address: NodeAddress,
4608 src_port: P2pPort,
4609 dst_port: P2pPort,
4610 payload: &[u8],
4611 ) -> TelemetryResult<crate::SharedBytes> {
4612 let host_len = u16::try_from(src_hostname.len())
4613 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
4614 let payload_len = u32::try_from(payload.len())
4615 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
4616 let mut out = Vec::with_capacity(
4617 15usize
4618 .saturating_add(src_hostname.len())
4619 .saturating_add(payload.len()),
4620 );
4621 out.push(1);
4622 out.extend_from_slice(&dst_port.to_le_bytes());
4623 out.extend_from_slice(&src_port.to_le_bytes());
4624 out.extend_from_slice(&src_address.to_le_bytes());
4625 out.extend_from_slice(&host_len.to_le_bytes());
4626 out.extend_from_slice(&payload_len.to_le_bytes());
4627 out.extend_from_slice(src_hostname.as_bytes());
4628 out.extend_from_slice(payload);
4629 Ok(out.into())
4630 }
4631
4632 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
4633 if payload.len() < 15 {
4634 return Err(TelemetryError::Unpack("p2p frame short"));
4635 }
4636 if payload[0] != 1 {
4637 return Err(TelemetryError::Unpack("p2p frame version"));
4638 }
4639 let destination_port =
4640 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
4641 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
4642 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
4643 let host_len =
4644 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
4645 let body_len =
4646 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
4647 let host_start = 15usize;
4648 let host_end = host_start.saturating_add(host_len);
4649 let body_end = host_end.saturating_add(body_len);
4650 if host_end > payload.len() || body_end != payload.len() {
4651 return Err(TelemetryError::Unpack("p2p frame length"));
4652 }
4653 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
4654 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
4655 Ok(P2pDecoded {
4656 source_hostname,
4657 source_address,
4658 source_port,
4659 destination_port,
4660 payload: &payload[host_end..body_end],
4661 })
4662 }
4663
4664 fn encode_p2p_stream_payload(
4665 flags: u8,
4666 source_stream_id: P2pStreamId,
4667 destination_stream_id: P2pStreamId,
4668 sequence: u32,
4669 payload: &[u8],
4670 ) -> TelemetryResult<crate::SharedBytes> {
4671 let payload_len = u32::try_from(payload.len())
4672 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
4673 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
4674 out.extend_from_slice(&P2P_STREAM_MAGIC);
4675 out.push(P2P_STREAM_VERSION);
4676 out.push(flags);
4677 out.extend_from_slice(&source_stream_id.to_le_bytes());
4678 out.extend_from_slice(&destination_stream_id.to_le_bytes());
4679 out.extend_from_slice(&sequence.to_le_bytes());
4680 out.extend_from_slice(&payload_len.to_le_bytes());
4681 out.extend_from_slice(payload);
4682 Ok(out.into())
4683 }
4684
4685 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
4686 if !payload.starts_with(&P2P_STREAM_MAGIC) {
4687 return Ok(None);
4688 }
4689 if payload.len() < 22 {
4690 return Err(TelemetryError::Unpack("p2p stream frame short"));
4691 }
4692 if payload[4] != P2P_STREAM_VERSION {
4693 return Err(TelemetryError::Unpack("p2p stream frame version"));
4694 }
4695 let flags = payload[5];
4696 let source_stream_id =
4697 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
4698 let destination_stream_id =
4699 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
4700 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
4701 let body_len =
4702 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
4703 let body_end = 22usize.saturating_add(body_len);
4704 if body_end != payload.len() {
4705 return Err(TelemetryError::Unpack("p2p stream frame length"));
4706 }
4707 Ok(Some(P2pStreamDecoded {
4708 flags,
4709 source_stream_id,
4710 destination_stream_id,
4711 sequence,
4712 payload: &payload[22..],
4713 }))
4714 }
4715
4716 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
4717 for _ in 0..u32::MAX {
4718 let id = st
4719 .local_services
4720 .as_ref()
4721 .expect("local services enabled")
4722 .next_p2p_stream_id
4723 .max(1);
4724 st.local_services
4725 .as_mut()
4726 .expect("local services enabled")
4727 .next_p2p_stream_id = st
4728 .local_services
4729 .as_ref()
4730 .expect("local services enabled")
4731 .next_p2p_stream_id
4732 .wrapping_add(1)
4733 .max(1);
4734 if !st
4735 .local_services
4736 .as_ref()
4737 .expect("local services enabled")
4738 .p2p_stream_sessions
4739 .contains_key(&id)
4740 {
4741 return id;
4742 }
4743 }
4744 0
4745 }
4746
4747 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
4748 if pkt.data_type() != DataType::P2pMessage {
4749 return Ok(());
4750 }
4751 let decoded = Self::decode_p2p_payload(pkt.payload())?;
4752 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
4753 return self.dispatch_p2p_stream_frame(&decoded, &stream);
4754 }
4755 let handlers = {
4756 let st = self.state.lock();
4757 st.local_services
4758 .as_ref()
4759 .expect("local services enabled")
4760 .p2p_port_handlers
4761 .get(&decoded.destination_port)
4762 .cloned()
4763 .unwrap_or_default()
4764 };
4765 for handler in handlers {
4766 (handler.handler)(P2pMessage {
4767 source_hostname: decoded.source_hostname,
4768 source_address: decoded.source_address,
4769 source_port: decoded.source_port,
4770 destination_port: decoded.destination_port,
4771 payload: decoded.payload,
4772 })?;
4773 }
4774 Ok(())
4775 }
4776
4777 fn dispatch_p2p_stream_frame(
4778 &self,
4779 msg: &P2pDecoded<'_>,
4780 frame: &P2pStreamDecoded<'_>,
4781 ) -> TelemetryResult<()> {
4782 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
4783 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, crate::SharedBytes)> = None;
4784 {
4785 let mut st = self.state.lock();
4786 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
4787 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
4788 let existing_id = st
4789 .local_services
4790 .as_ref()
4791 .expect("local services enabled")
4792 .p2p_stream_sessions
4793 .iter()
4794 .find_map(|(id, session)| {
4795 (session.peer_stream_id == frame.source_stream_id
4796 && session.local_port == msg.destination_port
4797 && session.peer_port == msg.source_port
4798 && session.peer_address == msg.source_address
4799 && session.peer_hostname.as_ref() == msg.source_hostname)
4800 .then_some(*id)
4801 });
4802 let local_id = if let Some(local_id) = existing_id {
4803 local_id
4804 } else {
4805 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
4806 if local_id == 0 {
4807 return Err(TelemetryError::Io("p2p stream id exhausted"));
4808 }
4809 st.local_services
4810 .as_mut()
4811 .expect("local services enabled")
4812 .p2p_stream_sessions
4813 .insert(
4814 local_id,
4815 P2pStreamSession {
4816 peer_hostname: peer_hostname.clone(),
4817 peer_address: msg.source_address,
4818 local_port: msg.destination_port,
4819 peer_port: msg.source_port,
4820 peer_stream_id: frame.source_stream_id,
4821 next_sequence: 1,
4822 connected: true,
4823 },
4824 );
4825 let handlers = st
4826 .local_services
4827 .as_ref()
4828 .expect("local services enabled")
4829 .p2p_stream_handlers
4830 .get(&msg.destination_port)
4831 .cloned()
4832 .unwrap_or_default();
4833 events.push(PendingP2pStreamEvent {
4834 handlers,
4835 kind: P2pStreamEventKind::Accepted,
4836 stream_id: local_id,
4837 peer_stream_id: frame.source_stream_id,
4838 peer_hostname: peer_hostname.clone(),
4839 peer_address: msg.source_address,
4840 local_port: msg.destination_port,
4841 peer_port: msg.source_port,
4842 });
4843 local_id
4844 };
4845 let dst =
4846 st.address_book
4847 .get(msg.source_hostname)
4848 .cloned()
4849 .unwrap_or(AddressBookEntry {
4850 hostname: peer_hostname,
4851 address: msg.source_address,
4852 requested_address: msg.source_address,
4853 mode: AddressAssignmentMode::Dynamic,
4854 birth_ms: self.clock.now_ms(),
4855 owner_hash: Self::sender_hash(msg.source_hostname),
4856 last_seen_ms: self.clock.now_ms(),
4857 });
4858 let payload = Self::encode_p2p_stream_payload(
4859 P2P_STREAM_SYN | P2P_STREAM_ACK,
4860 local_id,
4861 frame.source_stream_id,
4862 0,
4863 &[],
4864 )?;
4865 reply = Some((dst, msg.source_port, msg.destination_port, payload));
4866 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
4867 if let Some(session) = st
4868 .local_services
4869 .as_mut()
4870 .expect("local services enabled")
4871 .p2p_stream_sessions
4872 .get_mut(&frame.destination_stream_id)
4873 {
4874 session.peer_stream_id = frame.source_stream_id;
4875 session.connected = true;
4876 let peer_hostname = session.peer_hostname.clone();
4877 let peer_address = session.peer_address;
4878 let local_port = session.local_port;
4879 let peer_port = session.peer_port;
4880 let handlers = st
4881 .local_services
4882 .as_ref()
4883 .expect("local services enabled")
4884 .p2p_stream_handlers
4885 .get(&local_port)
4886 .cloned()
4887 .unwrap_or_default();
4888 events.push(PendingP2pStreamEvent {
4889 handlers,
4890 kind: P2pStreamEventKind::Connected,
4891 stream_id: frame.destination_stream_id,
4892 peer_stream_id: frame.source_stream_id,
4893 peer_hostname,
4894 peer_address,
4895 local_port,
4896 peer_port,
4897 });
4898 }
4899 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
4900 let kind = if frame.flags & P2P_STREAM_RST != 0 {
4901 P2pStreamEventKind::Reset
4902 } else if frame.flags & P2P_STREAM_FIN != 0 {
4903 P2pStreamEventKind::Closed
4904 } else {
4905 P2pStreamEventKind::Data
4906 };
4907 let session_id = if frame.destination_stream_id != 0 {
4908 Some(frame.destination_stream_id)
4909 } else {
4910 st.local_services
4911 .as_ref()
4912 .expect("local services enabled")
4913 .p2p_stream_sessions
4914 .iter()
4915 .find_map(|(id, session)| {
4916 (session.peer_stream_id == frame.source_stream_id
4917 && session.local_port == msg.destination_port
4918 && session.peer_port == msg.source_port
4919 && session.peer_address == msg.source_address
4920 && session.peer_hostname.as_ref() == msg.source_hostname)
4921 .then_some(*id)
4922 })
4923 };
4924 if let Some(session_id) = session_id
4925 && let Some(session) = st
4926 .local_services
4927 .as_ref()
4928 .expect("local services enabled")
4929 .p2p_stream_sessions
4930 .get(&session_id)
4931 {
4932 let handlers = st
4933 .local_services
4934 .as_ref()
4935 .expect("local services enabled")
4936 .p2p_stream_handlers
4937 .get(&session.local_port)
4938 .cloned()
4939 .unwrap_or_default();
4940 events.push(PendingP2pStreamEvent {
4941 handlers,
4942 kind,
4943 stream_id: session_id,
4944 peer_stream_id: frame.source_stream_id,
4945 peer_hostname: session.peer_hostname.clone(),
4946 peer_address: session.peer_address,
4947 local_port: session.local_port,
4948 peer_port: session.peer_port,
4949 });
4950 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
4951 st.local_services
4952 .as_mut()
4953 .expect("local services enabled")
4954 .p2p_stream_sessions
4955 .remove(&session_id);
4956 }
4957 }
4958 }
4959 }
4960 if let Some((dst, dst_port, src_port, payload)) = reply {
4961 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
4962 }
4963 for pending in events {
4964 for handler in pending.handlers {
4965 (handler.handler)(P2pStreamEvent {
4966 kind: pending.kind,
4967 stream_id: pending.stream_id,
4968 peer_stream_id: pending.peer_stream_id,
4969 sequence: frame.sequence,
4970 peer_hostname: pending.peer_hostname.as_ref(),
4971 peer_address: pending.peer_address,
4972 local_port: pending.local_port,
4973 peer_port: pending.peer_port,
4974 payload: frame.payload,
4975 })?;
4976 }
4977 }
4978 Ok(())
4979 }
4980
4981 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) -> bool {
4987 let mut st = self.state.lock();
4988 if st.end_to_end_reliable_tx.contains_key(&packet_id)
4992 && st
4993 .reliable_return_routes
4994 .get(&packet_id)
4995 .is_some_and(|route| route.upstream_contract.is_some())
4996 {
4997 return false;
4998 }
4999 let now = self.clock.now_ms();
5003 let retry_ms = runtime_reliable_retransmit_ms();
5004 let duplicate = st.recent_rx.contains(&packet_id)
5005 || st
5006 .reliable_return_routes
5007 .get(&packet_id)
5008 .is_some_and(|route| route.delivery_completed);
5009 let last_replay_ms = if duplicate {
5010 now.saturating_sub(retry_ms)
5011 } else {
5012 now
5013 };
5014 let replace_side = !duplicate
5020 || st
5021 .reliable_return_routes
5022 .get(&packet_id)
5023 .is_none_or(|route| {
5024 now.saturating_sub(route.last_replay_ms) >= retry_ms
5025 || !Self::side_ref(&st, route.side)
5026 .is_ok_and(|side| side.opts.egress_enabled)
5027 });
5028 if !Self::remember_reliable_return_route_locked(&mut st, packet_id) {
5029 return true;
5030 }
5031 st.reliable_return_routes
5032 .entry(packet_id)
5033 .and_modify(|route| {
5034 if replace_side {
5035 route.side = side;
5036 }
5037 })
5038 .or_insert(ReliableReturnRouteState {
5039 side,
5040 delivery_completed: false,
5041 last_replay_ms,
5042 upstream_contract: None,
5043 });
5044 true
5045 }
5046
5047 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) -> bool {
5053 let cap = runtime_reliable_max_return_routes().max(1);
5054 st.reliable_return_route_order
5055 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
5056 while st.reliable_return_route_order.len() >= cap {
5057 let Some(index) = st
5058 .reliable_return_route_order
5059 .iter()
5060 .position(|id| !st.end_to_end_reliable_tx.contains_key(id))
5061 else {
5062 return false;
5063 };
5064 if let Some(oldest) = st.reliable_return_route_order.remove(index) {
5065 st.reliable_return_routes.remove(&oldest);
5066 }
5067 }
5068 st.reliable_return_route_order.push_back(packet_id);
5069 true
5070 }
5071
5072 #[cfg(feature = "discovery")]
5073 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
5074 let cap = runtime_reliable_max_end_to_end_pending().max(1);
5075 st.end_to_end_reliable_tx_order
5076 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
5077 while st.end_to_end_reliable_tx_order.len() >= cap {
5078 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
5079 st.end_to_end_reliable_tx.remove(&oldest);
5080 } else {
5081 break;
5082 }
5083 }
5084 st.end_to_end_reliable_tx_order.push_back(packet_id);
5085 }
5086
5087 #[cfg(feature = "discovery")]
5088 fn expected_end_to_end_destinations_locked(
5089 &self,
5090 st: &RouterInner,
5091 data: &RouterItem,
5092 exclude: Option<RouterSideId>,
5093 managed: bool,
5094 resolve_proxy_segment: bool,
5095 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
5096 let (eps, ty) = self.item_route_info(data)?;
5097 let frozen_targets = if resolve_proxy_segment {
5100 wire_format::TargetSenderSet::Hashes(&[])
5101 } else {
5102 match data {
5103 RouterItem::Packet(packet) => {
5104 wire_format::TargetSenderSet::Hashes(packet.wire_target_senders())
5105 }
5106 RouterItem::Packed(bytes) => {
5107 wire_format::peek_routing_envelope(bytes)?.target_sender_set()
5108 }
5109 }
5110 };
5111 let now_ms = self.clock.now_ms();
5112 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
5113 if managed || st.is_managed_variable(ty.as_u32()) {
5122 let mut out = BTreeMap::new();
5123 for (&side, route) in st.discovery_routes.iter() {
5124 if Some(side) == exclude {
5125 continue;
5126 }
5127 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5128 continue;
5129 }
5130 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5131 continue;
5132 };
5133 if restrict_link_local && !side_ref.opts.link_local_enabled {
5134 continue;
5135 }
5136 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5137 continue;
5138 }
5139 for (sender_id, sender_state) in route.announcers.iter() {
5140 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
5141 || !sender_state.reachable_network_variables.contains(&ty)
5142 || !self.is_end_to_end_destination_sender(sender_id)
5143 {
5144 continue;
5145 }
5146 let owner = Self::sender_hash(sender_id);
5147 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5148 continue;
5149 }
5150 out.insert(owner, side);
5151 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5152 return Ok(out);
5153 }
5154 }
5155 }
5156 return Ok(out);
5157 }
5158 let prefer_best_overlap =
5159 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
5160 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
5161 let mut best_overlap = 0usize;
5162 let limit = runtime_reliable_max_end_to_end_pending().max(1);
5163 let mut out = BTreeMap::new();
5164 for (&side, route) in st.discovery_routes.iter() {
5165 if Some(side) == exclude {
5166 continue;
5167 }
5168 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5169 continue;
5170 }
5171 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5172 continue;
5173 };
5174 if restrict_link_local && !side_ref.opts.link_local_enabled {
5175 continue;
5176 }
5177 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5178 continue;
5179 }
5180 for sender_state in route.announcers.values() {
5181 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5182 continue;
5183 }
5184 for board in sender_state.topology_boards.iter() {
5185 if !self.is_end_to_end_destination_sender(&board.sender_id) {
5186 continue;
5187 }
5188 let owner = Self::sender_hash(&board.sender_id);
5189 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5190 continue;
5191 }
5192 let overlap = Self::endpoint_overlap_count(
5193 board.reachable_endpoints.iter().copied(),
5194 &scoring_eps,
5195 );
5196 if overlap > 0 {
5197 if prefer_best_overlap {
5198 if overlap < best_overlap {
5199 continue;
5200 }
5201 if overlap > best_overlap {
5202 out.clear();
5203 best_overlap = overlap;
5204 }
5205 if out.len() >= limit {
5209 continue;
5210 }
5211 }
5212 out.insert(owner, side);
5213 if !prefer_best_overlap && out.len() >= limit {
5214 return Ok(out);
5215 }
5216 }
5217 }
5218 }
5219 }
5220
5221 Ok(out)
5222 }
5223
5224 #[cfg(feature = "discovery")]
5225 #[allow(clippy::too_many_arguments)]
5226 fn discovered_route_candidates_locked(
5227 &self,
5228 st: &RouterInner,
5229 exclude: Option<RouterSideId>,
5230 ty: DataType,
5231 eps: &[DataEndpoint],
5232 target_senders: wire_format::TargetSenderSet<'_>,
5233 prefer_nonlocal: bool,
5234 preferred_timesync_source: Option<&str>,
5235 ) -> Vec<DiscoveryCandidateMatch> {
5236 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
5237 let now_ms = self.clock.now_ms();
5238 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
5239 let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
5240 exclude != Some(side)
5241 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5242 && route.reachable_network_variables.contains(&ty)
5243 });
5244 let mut out = Vec::new();
5245 let mut unique_endpoint_fallbacks = Vec::new();
5246
5247 for (&side, route) in st.discovery_routes.iter() {
5248 if exclude == Some(side)
5249 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
5250 {
5251 continue;
5252 }
5253 if restrict_link_local
5254 && st
5255 .sides
5256 .get(side)
5257 .and_then(|s| s.as_ref())
5258 .map(|s| !s.opts.link_local_enabled)
5259 .unwrap_or(true)
5260 {
5261 continue;
5262 }
5263 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5264 continue;
5265 }
5266 if !target_senders.is_empty()
5267 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
5268 {
5269 if scoring_eps
5270 .iter()
5271 .copied()
5272 .any(|ep| route.reachable.contains(&ep))
5273 {
5274 unique_endpoint_fallbacks.push(side);
5275 }
5276 continue;
5277 }
5278 if !target_senders.is_empty() {
5279 out.push(DiscoveryCandidateMatch {
5280 side,
5281 overlap: usize::MAX,
5282 });
5283 continue;
5284 }
5285 if has_variable_owner {
5286 if route.reachable_network_variables.contains(&ty) {
5287 out.push(DiscoveryCandidateMatch {
5288 side,
5289 overlap: usize::MAX,
5290 });
5291 }
5292 continue;
5293 }
5294 if preferred_timesync_source
5295 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
5296 {
5297 out.push(DiscoveryCandidateMatch {
5298 side,
5299 overlap: usize::MAX,
5300 });
5301 continue;
5302 }
5303 let overlap =
5304 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
5305 if overlap > 0 {
5306 out.push(DiscoveryCandidateMatch { side, overlap });
5307 }
5308 }
5309 if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
5310 out.push(DiscoveryCandidateMatch {
5311 side: unique_endpoint_fallbacks[0],
5312 overlap: usize::MAX,
5313 });
5314 }
5315 out
5316 }
5317
5318 #[cfg(feature = "discovery")]
5324 fn retain_shortest_discovery_candidates_locked(
5325 st: &RouterInner,
5326 matches: &mut Vec<DiscoveryCandidateMatch>,
5327 endpoints: &[DataEndpoint],
5328 target_senders: wire_format::TargetSenderSet<'_>,
5329 now_ms: u64,
5330 ) {
5331 if matches.len() <= 1 || endpoints.is_empty() {
5332 return;
5333 }
5334
5335 let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
5336 let route = st.discovery_routes.get(&side)?;
5337 route
5338 .announcers
5339 .iter()
5340 .filter(|(_, state)| {
5341 now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5342 })
5343 .filter_map(|(announcer, state)| {
5344 let is_target = |board: &TopologyBoardNode| {
5345 target_sender
5346 .map(|target| Self::sender_hash(&board.sender_id) == target)
5347 .unwrap_or_else(|| {
5348 endpoints
5349 .iter()
5350 .any(|endpoint| board.reachable_endpoints.contains(endpoint))
5351 })
5352 };
5353 if !state.topology_boards.iter().any(is_target) {
5356 return if let Some(target) = target_sender {
5357 state
5358 .topology_boards
5359 .iter()
5360 .any(|board| Self::sender_hash(&board.sender_id) == target)
5361 .then_some(usize::MAX / 2)
5362 } else {
5363 state
5364 .reachable
5365 .iter()
5366 .any(|endpoint| endpoints.contains(endpoint))
5367 .then_some(usize::MAX / 2)
5368 };
5369 }
5370
5371 let mut visited: BTreeSet<&str> = BTreeSet::new();
5372 let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
5373 while let Some((sender, distance)) = pending.pop_front() {
5374 if !visited.insert(sender) {
5375 continue;
5376 }
5377 for board in state.topology_boards.iter() {
5378 if board.sender_id == sender {
5379 if is_target(board) {
5380 return Some(distance);
5381 }
5382 for peer in &board.connections {
5383 if !visited.contains(peer.as_str()) {
5384 pending.push_back((peer.as_str(), distance + 1));
5385 }
5386 }
5387 } else if board.connections.iter().any(|peer| peer == sender)
5388 && !visited.contains(board.sender_id.as_str())
5389 {
5390 pending.push_back((board.sender_id.as_str(), distance + 1));
5391 }
5392 }
5393 }
5394 None
5395 })
5396 .min()
5397 };
5398
5399 let route_targets = target_senders
5400 .iter()
5401 .map(Some)
5402 .chain(core::iter::once(None).filter(|_| target_senders.is_empty()));
5403 let mut selected = BTreeSet::new();
5404 for target in route_targets {
5405 let distances: Vec<(RouterSideId, usize)> = matches
5406 .iter()
5407 .filter_map(|candidate| {
5408 route_distance(candidate.side, target)
5409 .map(|distance| (candidate.side, distance))
5410 })
5411 .collect();
5412 let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
5413 continue;
5414 };
5415 selected.extend(
5416 distances
5417 .into_iter()
5418 .filter(|(_, distance)| *distance == best)
5419 .map(|(side, _)| side),
5420 );
5421 }
5422 if !selected.is_empty() {
5423 matches.retain(|candidate| selected.contains(&candidate.side));
5424 }
5425 }
5426
5427 #[cfg(feature = "discovery")]
5431 fn select_endpoint_role_sides_locked(
5432 &self,
5433 st: &mut RouterInner,
5434 exclude: Option<RouterSideId>,
5435 endpoints: &[DataEndpoint],
5436 matches: &[DiscoveryCandidateMatch],
5437 now_ms: u64,
5438 ) -> Vec<RouterSideId> {
5439 let mut selected = Vec::new();
5440 for endpoint in endpoints {
5441 let mut candidates: Vec<_> = matches
5442 .iter()
5443 .copied()
5444 .filter(|candidate| {
5445 st.discovery_routes
5446 .get(&candidate.side)
5447 .is_some_and(|route| route.reachable.contains(endpoint))
5448 })
5449 .collect();
5450 Self::retain_shortest_discovery_candidates_locked(
5451 st,
5452 &mut candidates,
5453 core::slice::from_ref(endpoint),
5454 wire_format::TargetSenderSet::Hashes(&[]),
5455 now_ms,
5456 );
5457 if candidates
5458 .iter()
5459 .any(|candidate| selected.contains(&candidate.side))
5460 {
5461 continue;
5462 }
5463 let sides = self.apply_route_selection_locked(
5464 st,
5465 exclude,
5466 candidates
5467 .into_iter()
5468 .map(|candidate| candidate.side)
5469 .collect(),
5470 RouteSelectionOrigin::Discovered,
5471 );
5472 for side in sides {
5473 if !selected.contains(&side) {
5474 selected.push(side);
5475 }
5476 }
5477 }
5478 selected
5479 }
5480
5481 #[cfg(feature = "discovery")]
5482 fn select_discovered_candidate_sides_locked(
5483 &self,
5484 st: &mut RouterInner,
5485 packet: (&RouterItem, DataType),
5486 exclude: Option<RouterSideId>,
5487 target_senders: wire_format::TargetSenderSet<'_>,
5488 prefer_best_overlap: bool,
5489 matches: Vec<DiscoveryCandidateMatch>,
5490 ) -> Vec<RouterSideId> {
5491 let (data, ty) = packet;
5492 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
5493 || !target_senders.is_empty()
5494 || (!self.cfg.role.local_services() && is_reliable_type(ty))
5495 {
5496 RouteSelectionOrigin::Flood
5497 } else {
5498 RouteSelectionOrigin::Discovered
5499 };
5500
5501 let mut matches = matches;
5502 if matches.iter().any(|m| m.overlap == usize::MAX) {
5503 matches.retain(|m| m.overlap == usize::MAX);
5504 }
5505
5506 let selected: Vec<RouterSideId> = if prefer_best_overlap {
5507 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
5508 matches
5509 .into_iter()
5510 .filter(|m| m.overlap == best_overlap)
5511 .map(|m| m.side)
5512 .collect()
5513 } else {
5514 matches.into_iter().map(|m| m.side).collect()
5515 };
5516
5517 let selected = if st.forwarded_receipts.is_some() && is_reliable_type(ty) {
5518 self.filter_end_to_end_satisfied_sides_locked(
5519 st,
5520 data,
5521 selected,
5522 &self
5523 .item_route_info(data)
5524 .map(|info| info.0)
5525 .unwrap_or_default(),
5526 ty,
5527 )
5528 } else {
5529 selected
5530 };
5531
5532 let has_distributed_variable_owners = selected.iter().any(|side| {
5536 st.discovery_routes
5537 .get(side)
5538 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
5539 });
5540 if has_distributed_variable_owners {
5541 selected
5542 } else {
5543 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
5544 }
5545 }
5546
5547 fn note_end_to_end_acked_destination_locked(
5548 st: &mut RouterInner,
5549 packet_id: u64,
5550 sender_hash: u64,
5551 ) {
5552 let cache = st
5553 .forwarded_receipts
5554 .get_or_insert_with(|| Box::new(ForwardedReceiptCache::default()));
5555 let entry_cap = crate::config::runtime_reliable_max_end_to_end_ack_cache().max(1);
5556 cache
5557 .order
5558 .retain(|id| cache.destinations.contains_key(id) && *id != packet_id);
5559 while cache.order.len() >= entry_cap {
5560 if let Some(oldest) = cache.order.pop_front() {
5561 cache.destinations.remove(&oldest);
5562 } else {
5563 break;
5564 }
5565 }
5566 cache.order.push_back(packet_id);
5567
5568 let acked = cache.destinations.entry(packet_id).or_default();
5569 let sender_cap = crate::config::runtime_reliable_max_end_to_end_pending().max(1);
5570 if acked.len() < sender_cap || acked.contains(&sender_hash) {
5571 acked.insert(sender_hash);
5572 }
5573 }
5574
5575 #[cfg(feature = "discovery")]
5576 fn filter_end_to_end_satisfied_sides_locked(
5577 &self,
5578 st: &RouterInner,
5579 data: &RouterItem,
5580 sides: Vec<RouterSideId>,
5581 eps: &[crate::DataEndpoint],
5582 ty: crate::DataType,
5583 ) -> Vec<RouterSideId> {
5584 if !is_reliable_type(ty)
5585 || Self::reliable_control_target_packet_id(data)
5586 .ok()
5587 .flatten()
5588 .is_some()
5589 {
5590 return sides;
5591 }
5592 let packet_id = match data {
5593 RouterItem::Packet(pkt) => pkt.packet_id(),
5594 RouterItem::Packed(bytes) => match wire_format::packet_id_from_wire(bytes.as_ref()) {
5595 Ok(packet_id) => packet_id,
5596 Err(TelemetryError::Unpack("reliable control frame")) => return sides,
5597 Err(_) => return sides,
5598 },
5599 };
5600 let Some(acked) = st
5601 .forwarded_receipts
5602 .as_ref()
5603 .and_then(|cache| cache.destinations.get(&packet_id))
5604 else {
5605 return sides;
5606 };
5607 let now_ms = self.clock.now_ms();
5608 let mut filtered = Vec::new();
5609 for side in sides {
5610 let Some(route) = st.discovery_routes.get(&side) else {
5611 filtered.push(side);
5612 continue;
5613 };
5614 let mut still_pending = false;
5615 let mut had_destination_board = false;
5616 for sender_state in route.announcers.values() {
5617 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5618 continue;
5619 }
5620 for board in sender_state.topology_boards.iter() {
5621 if !self.is_end_to_end_destination_sender(&board.sender_id) {
5622 continue;
5623 }
5624 had_destination_board = true;
5625 let sender_hash = Self::sender_hash(&board.sender_id);
5626 if acked.contains(&sender_hash) {
5627 continue;
5628 }
5629 if eps
5630 .iter()
5631 .copied()
5632 .any(|ep| board.reachable_endpoints.contains(&ep))
5633 {
5634 still_pending = true;
5635 break;
5636 }
5637 still_pending = true;
5640 break;
5641 }
5642 if still_pending {
5643 break;
5644 }
5645 }
5646 if still_pending || !had_destination_board {
5647 filtered.push(side);
5648 }
5649 }
5650 filtered
5651 }
5652
5653 #[cfg(feature = "discovery")]
5654 fn reconcile_forwarded_receipts_locked(&self, st: &mut RouterInner) {
5655 if st.forwarded_receipts.is_none() {
5656 return;
5657 }
5658 let mut active_senders = BTreeSet::new();
5659 for route in st.discovery_routes.values() {
5660 for sender_state in route.announcers.values() {
5661 for board in sender_state.topology_boards.iter() {
5662 if self.is_end_to_end_destination_sender(&board.sender_id) {
5663 active_senders.insert(Self::sender_hash(&board.sender_id));
5664 }
5665 }
5666 }
5667 }
5668 st.forwarded_receipts
5669 .as_mut()
5670 .expect("checked cache")
5671 .destinations
5672 .retain(|_, acked| {
5673 acked.retain(|sender_hash| active_senders.contains(sender_hash));
5674 !acked.is_empty()
5675 });
5676 }
5677
5678 #[cfg(feature = "discovery")]
5679 fn register_end_to_end_reliable_tx(
5680 &self,
5681 data: &RouterItem,
5682 destinations: Option<BTreeMap<u64, RouterSideId>>,
5683 ) -> TelemetryResult<bool> {
5684 let packet_id = Self::get_hash(data);
5685 let now_ms = self.clock.now_ms();
5686 let ty = match data {
5687 RouterItem::Packet(pkt) => pkt.data_type(),
5688 RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
5689 };
5690 let mut st = self.state.lock();
5691 #[cfg(feature = "discovery")]
5692 let mut pending_destinations = match destinations {
5693 Some(destinations) => destinations,
5694 None => self.expected_end_to_end_destinations_locked(&st, data, None, false, false)?,
5695 };
5696 #[cfg(not(feature = "discovery"))]
5697 let mut pending_destinations = destinations.unwrap_or_default();
5698 let frozen = match data {
5702 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
5703 RouterItem::Packed(bytes) => {
5704 wire_format::peek_routing_envelope(bytes)?
5705 .target_senders()
5706 .len()
5707 != 0
5708 }
5709 };
5710 self.filter_trackable_end_to_end_destinations_locked(
5711 &st,
5712 ty,
5713 &mut pending_destinations,
5714 frozen,
5715 );
5716 let tracked_destinations = !pending_destinations.is_empty();
5717 if let Some(existing) = st.end_to_end_reliable_tx.get(&packet_id) {
5718 return Ok(!existing.waiting_for_predecessor);
5719 }
5720 if st.end_to_end_reliable_tx.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5722 return Err(TelemetryError::Io("reliable window full"));
5723 }
5724 let ordered_type =
5725 matches!(reliable_mode(ty), crate::ReliableMode::Ordered).then_some(ty.as_u32());
5726 let waiting_for_predecessor = ordered_type.is_some()
5727 && st
5728 .end_to_end_reliable_tx
5729 .values()
5730 .any(|sent| sent.ordered_type == ordered_type);
5731 #[cfg(feature = "std")]
5732 if reliable_trace_enabled() {
5733 eprintln!(
5734 "[sedsnet-reliable] register time={now_ms} type={} id={packet_id:#x} waiting={waiting_for_predecessor} targets={pending_destinations:?}",
5735 ty.as_u32()
5736 );
5737 }
5738 #[cfg(feature = "compression")]
5739 let retained = match data {
5740 RouterItem::Packed(bytes) => RouterItem::Packed(
5741 wire_format::payload_encoding_for_retention(
5742 bytes.as_ref(),
5743 st.memory.max_queue_budget,
5744 )?
5745 .map(crate::shared_bytes::convert)
5746 .unwrap_or_else(|| bytes.clone()),
5747 ),
5748 packet => packet.clone(),
5749 };
5750 #[cfg(not(feature = "compression"))]
5751 let retained = data.clone();
5752 #[cfg(feature = "compact-packet-store")]
5753 let mut retained = retained;
5754 #[cfg(feature = "compact-packet-store")]
5755 retained.park_for_queue()?;
5756 let sent = EndToEndReliableSent {
5757 ordered_type,
5758 waiting_for_predecessor,
5759 data: retained,
5760 pending_destinations,
5761 tracked_destinations,
5762 last_send_ms: now_ms,
5763 retries: 0,
5764 queued: false,
5765 }
5766 .try_box()?;
5767 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
5768 st.end_to_end_reliable_tx.insert(packet_id, sent);
5769 Ok(!waiting_for_predecessor)
5770 }
5771
5772 fn release_ordered_successors(&self) -> TelemetryResult<()> {
5775 let ready = {
5776 let mut st = self.state.lock();
5777 let RouterInner {
5778 end_to_end_reliable_tx_order: order,
5779 end_to_end_reliable_tx: pending,
5780 ..
5781 } = &mut *st;
5782 let mut ready = Vec::new();
5783 for (index, id) in order.iter().copied().enumerate() {
5784 let Some(sent) = pending.get(&id) else {
5785 continue;
5786 };
5787 if !sent.waiting_for_predecessor {
5788 continue;
5789 }
5790 let Some(ty) = sent.ordered_type else {
5791 continue;
5792 };
5793 if order.iter().take(index).any(|earlier| {
5797 pending
5798 .get(earlier)
5799 .is_some_and(|sent| sent.ordered_type == Some(ty))
5800 }) {
5801 continue;
5802 }
5803 let sent = pending.get_mut(&id).expect("retained ordered entry");
5804 sent.waiting_for_predecessor = false;
5805 sent.last_send_ms = self.clock.now_ms();
5806 ready.push(id);
5807 }
5808 ready
5809 };
5810 for id in ready {
5811 self.queue_end_to_end_reliable_retransmit(id)?;
5812 }
5813 Ok(())
5814 }
5815
5816 #[cfg(feature = "discovery")]
5817 fn reconcile_end_to_end_reliable_destinations_locked(
5818 &self,
5819 st: &mut RouterInner,
5820 ) -> TelemetryResult<()> {
5821 self.reconcile_forwarded_receipts_locked(st);
5822 if st.end_to_end_reliable_tx.is_empty() {
5823 return Ok(());
5824 }
5825 let active_destinations = self.active_end_to_end_destinations_locked(st);
5826 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
5827
5828 for packet_id in packet_ids {
5829 let Some(data) = st
5830 .end_to_end_reliable_tx
5831 .get(&packet_id)
5832 .map(|sent| sent.data.clone())
5833 else {
5834 continue;
5835 };
5836 let excluded = st
5837 .reliable_return_routes
5838 .get(&packet_id)
5839 .filter(|route| route.upstream_contract.is_some())
5840 .map(|route| route.side);
5841 let expected =
5842 self.expected_end_to_end_destinations_locked(st, &data, excluded, false, false)?;
5843 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
5844 continue;
5845 };
5846 if !sent.tracked_destinations {
5847 continue;
5848 }
5849 for (sender, side) in &mut sent.pending_destinations {
5853 if let Some(next_side) = expected.get(sender).or_else(|| {
5854 active_destinations
5855 .get(sender)
5856 .filter(|side| Some(**side) != excluded)
5857 }) {
5858 *side = *next_side;
5859 }
5860 }
5861 }
5862
5863 Ok(())
5864 }
5865
5866 #[cfg(feature = "discovery")]
5867 fn active_end_to_end_destinations_locked(
5868 &self,
5869 st: &RouterInner,
5870 ) -> BTreeMap<u64, RouterSideId> {
5871 let now_ms = self.clock.now_ms();
5872 let mut out = BTreeMap::new();
5873 for (&side, route) in st.discovery_routes.iter() {
5874 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5875 continue;
5876 }
5877 for (sender_name, sender_state) in &route.announcers {
5878 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5879 continue;
5880 }
5881 if self.is_end_to_end_destination_sender(sender_name) {
5882 out.insert(Self::sender_hash(sender_name), side);
5883 }
5884 for board in sender_state.topology_boards.iter() {
5885 if !self.is_end_to_end_destination_sender(&board.sender_id) {
5886 continue;
5887 }
5888 out.insert(Self::sender_hash(&board.sender_id), side);
5889 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5890 return out;
5891 }
5892 }
5893 }
5894 }
5895 out
5896 }
5897
5898 #[cfg(feature = "discovery")]
5899 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
5900 matches!(
5901 st.sides
5902 .get(side)
5903 .and_then(Option::as_ref)
5904 .map(|side| &side.tx_handler),
5905 Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
5906 )
5907 }
5908
5909 #[cfg(feature = "discovery")]
5910 fn filter_trackable_end_to_end_destinations_locked(
5911 &self,
5912 st: &RouterInner,
5913 ty: DataType,
5914 pending: &mut BTreeMap<u64, RouterSideId>,
5915 frozen: bool,
5916 ) {
5917 if frozen {
5918 return;
5919 }
5920 let now_ms = self.clock.now_ms();
5921 pending.retain(|_, side| {
5922 Self::side_supports_end_to_end_tracking_locked(st, *side)
5923 && (st.is_managed_variable(ty.as_u32())
5924 || is_reliable_type(ty)
5925 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
5926 });
5927 }
5928
5929 #[cfg(feature = "discovery")]
5930 fn side_has_multiple_announcers_locked(
5931 &self,
5932 st: &RouterInner,
5933 side: RouterSideId,
5934 now_ms: u64,
5935 ) -> bool {
5936 st.discovery_routes
5937 .get(&side)
5938 .map(|route| {
5939 route
5940 .announcers
5941 .values()
5942 .filter(|sender| {
5943 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5944 })
5945 .take(2)
5946 .count()
5947 > 1
5948 })
5949 .unwrap_or(false)
5950 }
5951
5952 #[cfg(not(feature = "discovery"))]
5953 fn side_has_multiple_announcers_locked(
5954 &self,
5955 _st: &RouterInner,
5956 _side: RouterSideId,
5957 _now_ms: u64,
5958 ) -> bool {
5959 false
5960 }
5961
5962 fn queue_end_to_end_reliable_ack(
5963 &self,
5964 pkt: &Packet,
5965 called_from_queue: bool,
5966 ) -> TelemetryResult<()> {
5967 self.queue_end_to_end_reliable_ack_for_packet_id(
5968 pkt.packet_id(),
5969 pkt.sender(),
5970 called_from_queue,
5971 )
5972 }
5973
5974 fn queue_end_to_end_reliable_ack_for_packet_id(
5975 &self,
5976 packet_id: u64,
5977 original_sender: &str,
5978 called_from_queue: bool,
5979 ) -> TelemetryResult<()> {
5980 if !self.cfg.reliable_enabled() {
5981 return Ok(());
5982 }
5983 let original_sender_hash = {
5984 let mut st = self.state.lock();
5985 if let Some(route) = st.reliable_return_routes.get_mut(&packet_id) {
5990 route.delivery_completed = true;
5991 }
5992 Self::sender_hash(&Self::canonical_sender_locked(&st, original_sender))
5993 };
5994 self.queue_end_to_end_reliable_ack_for_sender_hash(
5995 packet_id,
5996 original_sender_hash,
5997 called_from_queue,
5998 )
5999 }
6000
6001 fn queue_end_to_end_reliable_ack_for_sender_hash(
6002 &self,
6003 packet_id: u64,
6004 original_sender_hash: u64,
6005 called_from_queue: bool,
6006 ) -> TelemetryResult<()> {
6007 if !self.cfg.reliable_enabled() {
6008 return Ok(());
6009 }
6010 let ack_sender = self.encode_end_to_end_ack_sender();
6011 let ack = Packet::new_local(
6012 DataType::ReliableAck,
6013 message_meta(DataType::ReliableAck).endpoints_owned(),
6014 Arc::from(ack_sender),
6015 self.packet_timestamp_ms(),
6016 crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
6017 )?;
6018 let local_sender = self.sender_arc();
6019 let ack = self.attach_wire_contract_to_item(
6023 RouterItem::Packet(ack),
6024 &[
6025 Self::sender_hash(local_sender.as_ref()),
6026 original_sender_hash,
6027 ],
6028 )?;
6029 let return_side = {
6033 let st = self.state.lock();
6034 st.reliable_return_routes
6035 .get(&packet_id)
6036 .map(|route| route.side)
6037 };
6038 let item = match return_side {
6039 Some(dst) => RouterTxItem::ToSide {
6040 src: None,
6041 dst,
6042 data: ack,
6043 },
6044 None => RouterTxItem::Broadcast(ack),
6045 };
6046 self.emit_internal_tx(item, true, called_from_queue)
6047 }
6048
6049 fn emit_internal_tx(
6050 &self,
6051 item: RouterTxItem,
6052 ignore_local: bool,
6053 called_from_queue: bool,
6054 ) -> TelemetryResult<()> {
6055 if called_from_queue {
6056 self.tx_queue_item_with_flags(item, ignore_local)
6057 } else {
6058 self.tx_item_impl(item, ignore_local, false)
6059 }
6060 }
6061
6062 fn emit_internal_tx_with_priority(
6063 &self,
6064 item: RouterTxItem,
6065 ignore_local: bool,
6066 priority: u8,
6067 called_from_queue: bool,
6068 ) -> TelemetryResult<()> {
6069 if called_from_queue {
6070 self.tx_queue_item_with_priority(item, ignore_local, priority)
6071 } else {
6072 self.tx_item_impl(item, ignore_local, false)
6073 }
6074 }
6075
6076 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
6077 let now = self.clock.now_ms();
6078 let result = self.tx_queue_item_with_priority(
6079 RouterTxItem::EndToEndReplay { packet_id },
6080 true,
6081 Self::router_item_priority_bumped(DataType::ReliableAck),
6082 );
6083 if result.is_ok()
6087 && let Some(sent) = self.state.lock().end_to_end_reliable_tx.get_mut(&packet_id)
6088 {
6089 sent.last_send_ms = sent.last_send_ms.max(now);
6090 }
6091 result
6092 }
6093
6094 fn end_to_end_retransmit_sides(
6095 &self,
6096 packet_id: u64,
6097 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
6098 let mut st = self.state.lock();
6099 let (data, tracked_destinations, mut sides) = {
6100 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
6101 #[cfg(feature = "std")]
6102 if reliable_trace_enabled() {
6103 eprintln!(
6104 "[sedsnet-reliable] replay time={} id={packet_id:#x} retries={} targets={:?}",
6105 self.clock.now_ms(),
6106 sent.retries,
6107 sent.pending_destinations
6108 );
6109 }
6110 sent.queued = false;
6111 sent.last_send_ms = self.clock.now_ms();
6112 let data = sent.data.clone();
6113 let tracked_destinations = sent.tracked_destinations;
6114 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
6115 (data, tracked_destinations, sides)
6116 };
6117 if tracked_destinations && sides.is_empty() {
6118 st.end_to_end_reliable_tx.remove(&packet_id);
6119 return None;
6120 }
6121 crate::small_sort::sort(&mut sides);
6122 sides.dedup();
6123 Some((data, sides))
6124 }
6125
6126 fn item_type_priority(data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
6127 let (ty, control) = match data {
6128 RouterItem::Packet(pkt) => (
6129 pkt.data_type(),
6130 pkt.payload().len() <= 16
6131 && pkt.endpoints().len() <= 3
6132 && crate::small_frozen_control(
6133 pkt.data_type(),
6134 pkt.wire_target_senders().len(),
6135 pkt.wire_shape(),
6136 ),
6137 ),
6138 RouterItem::Packed(bytes) => {
6139 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
6140 let env = &frame.envelope;
6141 if bytes.len() <= 128 && frame.ack_only() {
6144 return Ok((
6145 env.ty,
6146 crate::transport_priority(crate::DataType::ReliableAck),
6147 ));
6148 }
6149 let control = bytes.len() <= 128
6150 && bytes.first().is_some_and(|flags| flags & 0x11 == 0)
6151 && env.endpoints().take(4).count() <= 3
6152 && crate::small_frozen_control(
6153 env.ty,
6154 env.target_senders().len(),
6155 env.wire_shape,
6156 );
6157 (env.ty, control)
6158 }
6159 };
6160 let priority = crate::scheduler_priority(ty);
6161 Ok((ty, if control { priority.max(254) } else { priority }))
6162 }
6163
6164 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
6165 Self::item_type_priority(data).map(|(_, priority)| priority)
6166 }
6167
6168 fn item_service_type_priority(&self, data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
6169 let (ty, priority) = Self::item_type_priority(data)?;
6170 let priority = if self.is_managed_variable_type(ty) {
6171 crate::transport_priority(DataType::ManagedVariableValue)
6172 } else {
6173 priority
6174 };
6175 Ok((ty, priority))
6176 }
6177
6178 fn item_service_priority(&self, data: &RouterItem) -> TelemetryResult<u8> {
6179 self.item_service_type_priority(data)
6180 .map(|(_, priority)| priority)
6181 }
6182
6183 #[inline]
6184 fn router_item_priority_bumped(ty: DataType) -> u8 {
6185 crate::scheduler_priority(ty).saturating_add(16)
6186 }
6187
6188 #[inline]
6189 fn is_transient_tx_error(err: &TelemetryError) -> bool {
6190 matches!(
6191 err,
6192 TelemetryError::Io("side tx busy" | "memory pressure" | "reliable window full")
6193 )
6194 }
6195
6196 #[cfg(feature = "timesync")]
6197 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
6198 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
6199 || st
6200 .clock
6201 .current_time(now_mono_ns)
6202 .and_then(|reading| reading.unix_time_ms)
6203 .is_some()
6204 }
6205
6206 #[cfg(feature = "timesync")]
6207 fn reconcile_pending_timesync_request_locked(
6208 st: &mut TimeSyncRuntime,
6209 leader: &Option<TimeSyncLeader>,
6210 now_ms: u64,
6211 ) {
6212 let active_remote = match leader {
6213 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
6214 _ => None,
6215 };
6216 let should_clear = st
6217 .pending_request
6218 .as_ref()
6219 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
6220 if should_clear {
6221 st.pending_request = None;
6222 st.next_request_mono_ms = now_ms;
6223 }
6224 }
6225
6226 #[inline]
6228 fn enqueue_to_sides(
6229 &self,
6230 data: RouterItem,
6231 exclude: Option<RouterSideId>,
6232 ignore_local: bool,
6233 ) -> TelemetryResult<()> {
6234 let Some(data) = self.clear_intermediate_hop_targets(data, true)? else {
6235 return Ok(());
6236 };
6237 let plan = self.remote_side_plan(&data, exclude)?;
6238 let mut st = self.state.lock();
6239 let ty = Self::item_data_type(&data)?;
6240 let priority = if st.is_managed_variable(ty.as_u32()) {
6241 crate::transport_priority(DataType::ManagedVariableValue)
6242 } else {
6243 Self::router_item_priority(&data)?
6244 };
6245
6246 let RemoteSidePlan::Target(sides) = plan;
6247 for idx in sides {
6248 st.push_transmit(TxQueued {
6249 disposable: true,
6250 item: RouterTxItem::ToSide {
6251 src: exclude,
6252 dst: idx,
6253 data: data.clone(),
6254 },
6255 ignore_local,
6256 priority,
6257 delivery_state: QueuedDelivery::Untracked,
6258 accepted_type: 0,
6259 deferred_since_ms: 0,
6260 retry_after_ms: 0,
6261 })?;
6262 }
6263
6264 Ok(())
6265 }
6266
6267 fn relay_send(
6268 &self,
6269 data: RouterItem,
6270 src: Option<RouterSideId>,
6271 called_from_queue: bool,
6272 ) -> TelemetryResult<()> {
6273 if called_from_queue {
6274 return self.enqueue_to_sides(data, src, true);
6275 }
6276
6277 let Some(data) = self.clear_intermediate_hop_targets(data, false)? else {
6278 return Ok(());
6279 };
6280 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
6281 for side in sides {
6282 self.tx_item_impl(
6283 RouterTxItem::ToSide {
6284 src,
6285 dst: side,
6286 data: data.clone(),
6287 },
6288 true,
6289 false,
6290 )?;
6291 }
6292
6293 Ok(())
6294 }
6295
6296 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
6297 match data {
6298 RouterItem::Packet(pkt) => {
6299 pkt.validate()?;
6300 let mut eps = pkt.endpoints().to_vec();
6301 crate::small_sort::sort(&mut eps);
6302 eps.dedup();
6303 Ok((eps, pkt.data_type()))
6304 }
6305 RouterItem::Packed(bytes) => {
6306 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6307 let mut eps: Vec<DataEndpoint> = env.endpoints().collect();
6308 crate::small_sort::sort(&mut eps);
6309 eps.dedup();
6310 Ok((eps, env.ty))
6311 }
6312 }
6313 }
6314
6315 fn unpack_packet_for_role(&self, bytes: &[u8]) -> TelemetryResult<Packet> {
6316 let managed = crate::memory_admission::supports_work_kinds()
6317 && wire_format::scheduling_type(bytes).is_some_and(|ty| {
6318 !is_internal_control_type(ty) && self.is_managed_variable_type(ty)
6319 });
6320 wire_format::unpack_packet_with_role(bytes, managed)
6321 }
6322
6323 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
6324 match data {
6325 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
6326 RouterItem::Packed(bytes) => Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.ty),
6327 }
6328 }
6329
6330 fn e2e_crypto_supported(&self) -> bool {
6331 #[cfg(feature = "cryptography")]
6332 {
6333 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
6334 && crate::crypto::registered_crypto_available()
6335 }
6336 #[cfg(not(feature = "cryptography"))]
6337 {
6338 false
6339 }
6340 }
6341
6342 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
6343 if is_internal_control_type(ty) {
6344 return false;
6345 }
6346 match self.cfg.e2e_encryption() {
6347 RouterE2eEncryptionMode::Disabled => {
6348 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
6349 }
6350 RouterE2eEncryptionMode::RequiredOnly => {
6351 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
6352 }
6353 RouterE2eEncryptionMode::Preferred => matches!(
6354 message_e2e_encryption_policy(ty),
6355 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
6356 ),
6357 RouterE2eEncryptionMode::ForceAll => true,
6358 }
6359 }
6360
6361 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
6362 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
6363 return Err(TelemetryError::BadArg);
6364 }
6365 Ok(())
6366 }
6367
6368 #[cfg(feature = "cryptography")]
6369 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
6370 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
6371 Some(wire_format::E2eSealConfig {
6372 key_id: self.cfg.e2e_key_id(),
6373 })
6374 } else {
6375 None
6376 }
6377 }
6378
6379 #[inline]
6380 fn pack_packet_for_router(
6381 &self,
6382 pkt: &Packet,
6383 reliable: Option<wire_format::ReliableHeader>,
6384 ) -> TelemetryResult<crate::SharedBytes> {
6385 #[cfg(feature = "compression")]
6386 {
6387 #[cfg(feature = "cryptography")]
6388 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
6389 #[cfg(not(feature = "cryptography"))]
6390 let e2e = None;
6391 let reliable = reliable.or_else(|| {
6392 is_reliable_type(pkt.data_type()).then_some(wire_format::ReliableHeader {
6393 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
6394 seq: 0,
6395 ack: 0,
6396 })
6397 });
6398 wire_format::pack_routed_packet(
6399 pkt,
6400 reliable,
6401 pkt.wire_shape(),
6402 pkt.wire_target_senders(),
6403 e2e,
6404 )
6405 .map(crate::shared_bytes::convert)
6406 }
6407 #[cfg(not(feature = "compression"))]
6408 {
6409 #[cfg(feature = "cryptography")]
6410 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
6411 return (wire_format::pack_packet_with_wire_contract_e2e(
6412 pkt,
6413 reliable,
6414 pkt.wire_shape(),
6415 pkt.wire_target_senders(),
6416 e2e,
6417 ))
6418 .map(crate::shared_bytes::convert);
6419 }
6420 Ok(match reliable {
6421 Some(hdr) => {
6422 crate::shared_bytes::convert(wire_format::pack_packet_with_reliable(pkt, hdr))
6423 }
6424 None => crate::shared_bytes::convert(wire_format::pack_packet(pkt)),
6425 })
6426 }
6427 }
6428
6429 #[inline]
6430 fn pack_packet_for_contract(
6431 &self,
6432 pkt: &Packet,
6433 reliable: Option<wire_format::ReliableHeader>,
6434 shape: Option<MessageElement>,
6435 target_senders: &[u64],
6436 ) -> TelemetryResult<crate::SharedBytes> {
6437 #[cfg(feature = "compression")]
6438 {
6439 #[cfg(feature = "cryptography")]
6440 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
6441 #[cfg(not(feature = "cryptography"))]
6442 let e2e = None;
6443 wire_format::pack_routed_packet(pkt, reliable, shape, target_senders, e2e)
6444 .map(crate::shared_bytes::convert)
6445 }
6446 #[cfg(not(feature = "compression"))]
6447 {
6448 #[cfg(feature = "cryptography")]
6449 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
6450 return (wire_format::pack_packet_with_wire_contract_e2e(
6451 pkt,
6452 reliable,
6453 shape,
6454 target_senders,
6455 e2e,
6456 ))
6457 .map(crate::shared_bytes::convert);
6458 }
6459 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
6460 .map(crate::shared_bytes::convert)
6461 }
6462 }
6463
6464 #[cfg(feature = "cryptography")]
6465 #[inline]
6466 fn prepare_packed_for_remote(
6467 &self,
6468 bytes: crate::SharedBytes,
6469 reliable_override: Option<Option<wire_format::ReliableHeader>>,
6470 ) -> TelemetryResult<crate::SharedBytes> {
6471 let routing = wire_format::peek_routing_frame_info(bytes.as_ref())?;
6474 if routing.ack_only() || self.e2e_seal_config_for_type(routing.envelope.ty).is_none() {
6475 return Ok(bytes);
6476 }
6477 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
6478 let reliable = reliable_override.unwrap_or(frame.reliable);
6479 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
6480 self.pack_packet_for_contract(
6481 &pkt,
6482 reliable,
6483 frame.envelope.wire_shape,
6484 &frame.envelope.target_senders,
6485 )
6486 }
6487
6488 fn item_requires_end_to_end_tracking(
6492 &self,
6493 data: &RouterItem,
6494 ty: DataType,
6495 ) -> TelemetryResult<bool> {
6496 if is_reliable_type(ty) || self.is_managed_variable_type(ty) {
6497 return Ok(true);
6498 }
6499 match data {
6500 RouterItem::Packet(packet) => Ok(!packet.wire_target_senders().is_empty()),
6501 RouterItem::Packed(bytes) => {
6502 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
6503 Ok(frame.reliable.is_some() || frame.envelope.target_senders().len() != 0)
6504 }
6505 }
6506 }
6507
6508 #[cfg(feature = "discovery")]
6509 fn item_target_senders<'a>(
6510 &self,
6511 data: &'a RouterItem,
6512 ) -> TelemetryResult<wire_format::TargetSenderSet<'a>> {
6513 match data {
6514 RouterItem::Packet(pkt) => Ok(wire_format::TargetSenderSet::Hashes(
6515 pkt.wire_target_senders(),
6516 )),
6517 RouterItem::Packed(bytes) => {
6518 Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.target_sender_set())
6519 }
6520 }
6521 }
6522
6523 fn item_targets_another_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
6524 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
6525 match data {
6526 RouterItem::Packet(pkt) => {
6527 Ok(pkt.wire_target_senders().iter().any(|v| *v != local_hash))
6528 }
6529 RouterItem::Packed(bytes) => {
6530 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6531 Ok(env.target_senders().any(|target| target != local_hash))
6532 }
6533 }
6534 }
6535
6536 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
6537 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
6538 match data {
6539 RouterItem::Packet(pkt) => Ok(pkt.wire_target_senders().is_empty()
6540 || pkt.wire_target_senders().contains(&local_hash)),
6541 RouterItem::Packed(bytes) => {
6542 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
6543 Ok(env.target_senders().len() == 0
6544 || env.target_senders().any(|target| target == local_hash))
6545 }
6546 }
6547 }
6548
6549 fn item_targets_router_as_intermediate_hop(
6550 &self,
6551 data: &RouterItem,
6552 endpoints: &[DataEndpoint],
6553 ) -> TelemetryResult<bool> {
6554 let local_target = match data {
6557 RouterItem::Packet(pkt) => {
6558 if pkt.wire_target_senders().is_empty() || is_internal_control_type(pkt.data_type())
6559 {
6560 return Ok(false);
6561 }
6562 pkt.wire_target_senders()
6563 .contains(&Self::sender_hash(self.sender_arc().as_ref()))
6564 }
6565 RouterItem::Packed(bytes) => {
6566 let env = wire_format::peek_routing_envelope(bytes)?;
6567 if env.target_senders().len() == 0 || is_internal_control_type(env.ty) {
6568 return Ok(false);
6569 }
6570 let local = Self::sender_hash(self.sender_arc().as_ref());
6571 env.target_senders().any(|owner| owner == local)
6572 }
6573 };
6574 Ok(local_target
6575 && !endpoints.iter().copied().any(|endpoint| {
6576 self.endpoint_has_packet_handler(endpoint)
6577 || self.endpoint_has_packed_handler(endpoint)
6578 }))
6579 }
6580
6581 fn clear_intermediate_hop_targets(
6582 &self,
6583 data: RouterItem,
6584 called_from_queue: bool,
6585 ) -> TelemetryResult<Option<RouterItem>> {
6586 let has_targets = match &data {
6590 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
6591 RouterItem::Packed(bytes) => {
6592 wire_format::peek_routing_envelope(bytes)?
6593 .target_senders()
6594 .len()
6595 != 0
6596 }
6597 };
6598 if !has_targets {
6599 return Ok(Some(data));
6600 }
6601 let (endpoints, _) = self.item_route_info(&data)?;
6602 if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
6603 return Ok(Some(data));
6604 }
6605 let pkt = match &data {
6606 RouterItem::Packet(pkt) => pkt.clone(),
6607 RouterItem::Packed(bytes) => self.unpack_packet_for_role(bytes.as_ref())?,
6608 };
6609 let reliable = match &data {
6610 RouterItem::Packet(_) => None,
6611 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
6612 };
6613 let shape = match &data {
6614 RouterItem::Packet(pkt) => pkt.wire_shape(),
6615 RouterItem::Packed(bytes) => {
6616 wire_format::peek_routing_envelope(bytes.as_ref())?.wire_shape
6617 }
6618 };
6619 let upstream_id = Self::get_hash(&data);
6620 let completed_contract = {
6624 let st = self.state.lock();
6625 st.reliable_return_routes
6626 .get(&upstream_id)
6627 .filter(|route| route.delivery_completed)
6628 .and_then(|route| route.upstream_contract)
6629 };
6630 if let Some((packet_id, publisher_hash)) = completed_contract {
6631 self.queue_end_to_end_reliable_ack_for_sender_hash(
6632 packet_id,
6633 publisher_hash,
6634 called_from_queue,
6635 )?;
6636 return Ok(None);
6637 }
6638 #[cfg(feature = "discovery")]
6639 let destinations = {
6640 let st = self.state.lock();
6641 let upstream = st
6642 .reliable_return_routes
6643 .get(&upstream_id)
6644 .map(|route| route.side);
6645 let mut pending =
6646 self.expected_end_to_end_destinations_locked(&st, &data, upstream, true, true)?;
6647 if pending.is_empty() {
6648 pending = self
6649 .expected_end_to_end_destinations_locked(&st, &data, upstream, false, true)?;
6650 }
6651 pending
6652 };
6653 #[cfg(feature = "discovery")]
6654 let targets: Vec<u64> = destinations.keys().copied().collect();
6655 #[cfg(not(feature = "discovery"))]
6656 let targets: Vec<u64> = Vec::new();
6657 #[cfg(feature = "discovery")]
6658 if targets.is_empty() {
6659 if self.packet_has_local_delivery(&pkt) {
6660 self.queue_end_to_end_reliable_ack_for_packet_id(
6661 upstream_id,
6662 pkt.sender(),
6663 called_from_queue,
6664 )?;
6665 return Ok(Some(data));
6666 }
6667 if !is_reliable_type(pkt.data_type()) && reliable.is_none() {
6668 return Err(TelemetryError::Io("no discovered route"));
6669 }
6670 }
6671 let forwarded = self.pack_packet_for_contract(&pkt, reliable, shape, &targets)?;
6675 let downstream_id = wire_format::packet_id_from_wire(forwarded.as_ref())?;
6676 let forwarded = RouterItem::Packed(forwarded);
6677 #[cfg(feature = "discovery")]
6678 let already_pending = self
6679 .state
6680 .lock()
6681 .end_to_end_reliable_tx
6682 .contains_key(&downstream_id);
6683 #[cfg(feature = "discovery")]
6684 let ready = self.register_end_to_end_reliable_tx(&forwarded, Some(destinations))?;
6685 #[cfg(not(feature = "discovery"))]
6686 let (ready, already_pending) = {
6687 let _ = called_from_queue;
6688 (true, false)
6689 };
6690 #[cfg(feature = "std")]
6691 if reliable_trace_enabled() {
6692 eprintln!(
6693 "[sedsnet-reliable] proxy retain upstream={upstream_id:#x} downstream={downstream_id:#x} existing={already_pending} ready={ready}"
6694 );
6695 }
6696 {
6697 let mut st = self.state.lock();
6698 let Some(mut route) = st.reliable_return_routes.get(&upstream_id).cloned() else {
6699 st.end_to_end_reliable_tx.remove(&downstream_id);
6700 return Err(TelemetryError::Io("no reliable return route"));
6701 };
6702 route.upstream_contract = Some((
6703 upstream_id,
6704 Self::sender_hash(&Self::canonical_sender_locked(&st, pkt.sender())),
6705 ));
6706 if !Self::remember_reliable_return_route_locked(&mut st, downstream_id) {
6707 st.end_to_end_reliable_tx.remove(&downstream_id);
6708 return Err(TelemetryError::Io("reliable return route window full"));
6709 }
6710 st.reliable_return_routes.insert(downstream_id, route);
6711 }
6712 Ok((ready && !already_pending).then_some(forwarded))
6713 }
6714
6715 #[cfg(feature = "discovery")]
6716 fn side_matches_target_senders_locked(
6717 st: &RouterInner,
6718 side: RouterSideId,
6719 target_senders: wire_format::TargetSenderSet<'_>,
6720 now_ms: u64,
6721 ) -> bool {
6722 st.discovery_routes
6723 .get(&side)
6724 .map(|route| {
6725 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6726 return false;
6727 }
6728 route.announcers.iter().any(|(announcer, sender_state)| {
6729 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6730 return false;
6731 }
6732 target_senders.contains(Self::sender_hash(announcer))
6733 || sender_state
6734 .advertised_address
6735 .is_some_and(|address| target_senders.contains(u64::from(address)))
6736 || sender_state.topology_boards.iter().any(|board| {
6737 target_senders.contains(Self::sender_hash(&board.sender_id))
6738 })
6739 })
6740 })
6741 .unwrap_or(false)
6742 }
6743
6744 fn attach_wire_contract_to_item(
6745 &self,
6746 data: RouterItem,
6747 target_senders: &[u64],
6748 ) -> TelemetryResult<RouterItem> {
6749 let pkt = match data {
6750 RouterItem::Packet(packet) => packet,
6751 RouterItem::Packed(bytes) => {
6752 if wire_format::peek_routing_envelope(&bytes)?
6755 .target_senders()
6756 .len()
6757 != 0
6758 {
6759 return Ok(RouterItem::Packed(bytes));
6760 }
6761 self.unpack_packet_for_role(&bytes)?
6762 }
6763 };
6764 let reliable = if is_reliable_type(pkt.data_type()) {
6765 Some(wire_format::ReliableHeader {
6766 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
6767 seq: 0,
6768 ack: 0,
6769 })
6770 } else {
6771 None
6772 };
6773 let targets = if pkt.wire_target_senders().is_empty() {
6776 target_senders
6777 } else {
6778 pkt.wire_target_senders()
6779 };
6780 let bytes = self.pack_packet_for_contract(
6781 &pkt,
6782 reliable,
6783 pkt.wire_shape()
6784 .or_else(|| Some(message_meta(pkt.data_type()).element)),
6785 targets,
6786 )?;
6787 Ok(RouterItem::Packed(bytes))
6788 }
6789
6790 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
6791 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
6792 }
6793
6794 fn should_route_remote(
6795 &self,
6796 data: &RouterItem,
6797 exclude: Option<RouterSideId>,
6798 ) -> TelemetryResult<bool> {
6799 #[cfg(feature = "discovery")]
6800 {
6801 let RemoteSidePlan::Target(sides) =
6802 self.remote_side_plan_with_selection(data, exclude, false)?;
6803 Ok(!sides.is_empty())
6804 }
6805
6806 #[cfg(not(feature = "discovery"))]
6807 {
6808 let (eps, ty) = self.item_route_info(data)?;
6809 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
6810 return Ok(false);
6811 }
6812 let st = self.state.lock();
6813 Ok(!self
6814 .eligible_side_ids_locked(
6815 &st,
6816 exclude,
6817 Some(ty),
6818 Self::endpoints_are_link_local_only(&eps),
6819 )
6820 .is_empty())
6821 }
6822 }
6823
6824 #[cfg(feature = "discovery")]
6825 fn has_explicit_route_policy_locked(
6826 st: &RouterInner,
6827 src: Option<RouterSideId>,
6828 ty: DataType,
6829 ) -> bool {
6830 st.route_overrides
6831 .keys()
6832 .any(|(route_src, _)| *route_src == src)
6833 || Self::has_typed_route_overrides_locked(st, src, ty)
6834 }
6835
6836 fn remote_side_plan(
6837 &self,
6838 data: &RouterItem,
6839 exclude: Option<RouterSideId>,
6840 ) -> TelemetryResult<RemoteSidePlan> {
6841 self.remote_side_plan_with_selection(data, exclude, true)
6842 }
6843
6844 #[inline(never)]
6847 fn remote_side_plan_with_selection(
6848 &self,
6849 data: &RouterItem,
6850 exclude: Option<RouterSideId>,
6851 select_routes: bool,
6852 ) -> TelemetryResult<RemoteSidePlan> {
6853 #[cfg(feature = "discovery")]
6854 let (eps, ty) = self.item_route_info(data)?;
6855 let select_sides = |state: &mut RouterInner, sides: Vec<RouterSideId>, origin| {
6856 #[cfg(feature = "discovery")]
6857 let sides = self.filter_end_to_end_satisfied_sides_locked(state, data, sides, &eps, ty);
6858 if select_routes {
6859 self.apply_route_selection_locked(state, exclude, sides, origin)
6860 } else {
6861 sides
6862 }
6863 };
6864 #[cfg(feature = "discovery")]
6865 {
6866 let target_senders = self.item_target_senders(data)?;
6867 let resolve_beyond_local_hop =
6868 self.item_targets_router_as_intermediate_hop(data, &eps)?;
6869 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
6870 let routing_target_senders = if preferred_packet_id.is_some() {
6871 target_senders.skip(1)
6874 } else if resolve_beyond_local_hop {
6875 wire_format::TargetSenderSet::Hashes(&[])
6876 } else {
6877 target_senders
6878 };
6879 if discovery::is_discovery_type(ty) {
6880 let mut st = self.state.lock();
6881 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6882 return Ok(RemoteSidePlan::Target(select_sides(
6883 &mut st,
6884 sides,
6885 RouteSelectionOrigin::Flood,
6886 )));
6887 }
6888 #[cfg(feature = "timesync")]
6889 if ty == DataType::TimeSyncAnnounce {
6890 let mut st = self.state.lock();
6897 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6898 let sides = if select_routes {
6899 Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
6900 } else {
6901 sides
6902 };
6903 return Ok(RemoteSidePlan::Target(select_sides(
6904 &mut st,
6905 sides,
6906 RouteSelectionOrigin::Flood,
6907 )));
6908 }
6909 let mut st = self.state.lock();
6910 let now_ms = self.clock.now_ms();
6915 let has_remote_variable_owner = st.is_managed_variable(ty.as_u32())
6916 && st.discovery_routes.iter().any(|(&side, route)| {
6917 exclude != Some(side)
6918 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6919 && route.reachable_network_variables.contains(&ty)
6920 });
6921 if !(has_nonlocal_endpoint(&eps, &self.cfg)
6922 || force_remote_for_type(ty)
6923 || !target_senders.is_empty()
6924 || has_remote_variable_owner)
6925 {
6926 return Ok(RemoteSidePlan::Target(Vec::new()));
6927 }
6928
6929 #[cfg(feature = "timesync")]
6930 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
6931 #[cfg(not(feature = "timesync"))]
6932 let preferred_timesync_source: Option<String> = None;
6933
6934 if let Some(packet_id) = preferred_packet_id {
6935 let target_side = st
6936 .reliable_return_routes
6937 .get(&packet_id)
6938 .map(|route| route.side);
6939 if let Some(side) = target_side
6940 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
6941 {
6942 #[cfg(feature = "timesync")]
6943 if select_routes
6944 && !Self::timesync_allowed_for_side_locked(
6945 &st,
6946 side,
6947 ty,
6948 self.clock.now_ms(),
6949 )
6950 {
6951 return Ok(RemoteSidePlan::Target(Vec::new()));
6952 }
6953 return Ok(RemoteSidePlan::Target(vec![side]));
6954 }
6955 let eligible = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
6958 let exact: Vec<_> = eligible
6959 .iter()
6960 .copied()
6961 .filter(|&side| {
6962 !routing_target_senders.is_empty()
6963 && Self::side_matches_target_senders_locked(
6964 &st,
6965 side,
6966 routing_target_senders,
6967 now_ms,
6968 )
6969 })
6970 .collect();
6971 if !exact.is_empty() {
6972 return Ok(RemoteSidePlan::Target(select_sides(
6973 &mut st,
6974 exact,
6975 RouteSelectionOrigin::Discovered,
6976 )));
6977 }
6978 let publisher_known = !routing_target_senders.is_empty()
6983 && st.discovery_routes.keys().any(|&side| {
6984 Self::side_matches_target_senders_locked(
6985 &st,
6986 side,
6987 routing_target_senders,
6988 now_ms,
6989 )
6990 });
6991 return Ok(RemoteSidePlan::Target(
6992 if !publisher_known && exclude.is_some() && eligible.len() == 1 {
6993 eligible
6994 } else {
6995 Vec::new()
6996 },
6997 ));
6998 }
6999 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
7000 let prefer_best_overlap = is_reliable_type(ty)
7001 && routing_target_senders.is_empty()
7002 && preferred_packet_id.is_none();
7003 if st.discovery_routes.is_empty() {
7004 let mut fallback =
7005 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
7006 #[cfg(feature = "timesync")]
7007 {
7008 if select_routes {
7009 fallback = Self::filter_timesync_sides_locked(
7010 &st,
7011 ty,
7012 self.clock.now_ms(),
7013 fallback,
7014 );
7015 }
7016 }
7017 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
7018 fallback
7019 } else {
7020 Vec::new()
7021 }));
7022 }
7023 let mut matches = self.discovered_route_candidates_locked(
7024 &st,
7025 exclude,
7026 ty,
7027 &eps,
7028 routing_target_senders,
7029 prefer_best_overlap,
7030 preferred_timesync_source.as_deref(),
7031 );
7032 #[cfg(feature = "timesync")]
7033 {
7034 if select_routes {
7035 matches =
7036 Self::filter_timesync_matches_locked(&st, ty, self.clock.now_ms(), matches);
7037 }
7038 }
7039 let has_distributed_variable_owners = matches.iter().any(|candidate| {
7042 st.discovery_routes
7043 .get(&candidate.side)
7044 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
7045 });
7046 if !has_distributed_variable_owners {
7047 if self.cfg.role.local_services()
7051 && eps.len() > 1
7052 && matches.len() > 1
7053 && !is_reliable_type(ty)
7054 && !st.is_managed_variable(ty.as_u32())
7055 && routing_target_senders.is_empty()
7056 && preferred_packet_id.is_none()
7057 && preferred_timesync_source.is_none()
7058 && !Self::has_explicit_route_policy_locked(&st, exclude, ty)
7059 && !st.source_route_modes.contains_key(&exclude)
7060 {
7061 if !select_routes {
7062 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
7063 }
7064 let sides = self.select_endpoint_role_sides_locked(
7065 &mut st,
7066 exclude,
7067 &eps,
7068 &matches,
7069 self.clock.now_ms(),
7070 );
7071 return Ok(RemoteSidePlan::Target(sides));
7072 }
7073 Self::retain_shortest_discovery_candidates_locked(
7074 &st,
7075 &mut matches,
7076 &eps,
7077 routing_target_senders,
7078 self.clock.now_ms(),
7079 );
7080 }
7081
7082 if !matches.is_empty() {
7083 if !select_routes {
7084 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
7087 }
7088 Ok(RemoteSidePlan::Target(
7089 self.select_discovered_candidate_sides_locked(
7090 &mut st,
7091 (data, ty),
7092 exclude,
7093 routing_target_senders,
7094 prefer_best_overlap,
7095 matches,
7096 ),
7097 ))
7098 } else if !self.cfg.role.local_services() && exclude.is_some() {
7099 let fallback =
7100 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
7101 let known_elsewhere = !routing_target_senders.is_empty()
7102 && st.discovery_routes.keys().any(|&side| {
7103 Self::side_matches_target_senders_locked(
7104 &st,
7105 side,
7106 routing_target_senders,
7107 self.clock.now_ms(),
7108 )
7109 });
7110 Ok(RemoteSidePlan::Target(
7111 if fallback.len() == 1 && !known_elsewhere {
7112 select_sides(&mut st, fallback, RouteSelectionOrigin::Flood)
7113 } else {
7114 Vec::new()
7115 },
7116 ))
7117 } else if prefer_best_overlap {
7118 Ok(RemoteSidePlan::Target(Vec::new()))
7119 } else {
7120 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
7121 let mut sides =
7122 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
7123 #[cfg(feature = "timesync")]
7124 {
7125 if select_routes {
7126 sides = Self::filter_timesync_sides_locked(
7127 &st,
7128 ty,
7129 self.clock.now_ms(),
7130 sides,
7131 );
7132 }
7133 }
7134 Ok(RemoteSidePlan::Target(select_sides(
7135 &mut st,
7136 sides,
7137 RouteSelectionOrigin::Flood,
7138 )))
7139 } else {
7140 Ok(RemoteSidePlan::Target(Vec::new()))
7141 }
7142 }
7143 }
7144 #[cfg(not(feature = "discovery"))]
7145 {
7146 let ty = Self::item_data_type(data)?;
7147 let mut st = self.state.lock();
7148 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
7149 let target_side = st
7150 .reliable_return_routes
7151 .get(&packet_id)
7152 .map(|route| route.side);
7153 if let Some(side) = target_side
7154 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
7155 {
7156 return Ok(RemoteSidePlan::Target(vec![side]));
7157 }
7158 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7159 return Ok(RemoteSidePlan::Target(
7160 if exclude.is_some() && sides.len() == 1 {
7161 sides
7162 } else {
7163 Vec::new()
7164 },
7165 ));
7166 }
7167 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7168 Ok(RemoteSidePlan::Target(select_sides(
7169 &mut st,
7170 sides,
7171 RouteSelectionOrigin::Flood,
7172 )))
7173 }
7174 }
7175
7176 #[cfg(feature = "discovery")]
7177 fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
7178 let mut eps: Vec<DataEndpoint> = self.cfg.handlers().iter().map(|h| h.endpoint).collect();
7179 let _ = st;
7185 #[cfg(feature = "timesync")]
7186 if self.cfg.timesync_config().is_some() {
7187 eps.push(DataEndpoint::TimeSync);
7188 }
7189 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
7190 crate::small_sort::sort(&mut eps);
7191 eps.dedup();
7192 eps
7193 }
7194
7195 #[cfg(feature = "discovery")]
7196 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
7197 #[cfg(feature = "timesync")]
7198 if self.timesync.is_none() {
7199 return Vec::new();
7200 }
7201
7202 #[cfg(feature = "timesync")]
7203 {
7204 let st = self
7205 .timesync
7206 .as_ref()
7207 .expect("local time service enabled")
7208 .lock();
7209 if let Some(tracker) = st.tracker.as_ref()
7210 && tracker.should_serve(
7211 now_ms,
7212 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
7213 )
7214 {
7215 return vec![self.sender_arc().to_string()];
7216 }
7217 }
7218 Vec::new()
7219 }
7220
7221 #[cfg(all(feature = "discovery", feature = "timesync"))]
7222 fn preferred_timesync_route_source(
7223 &self,
7224 data: &RouterItem,
7225 ty: DataType,
7226 ) -> TelemetryResult<Option<String>> {
7227 if !matches!(
7228 ty,
7229 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
7230 ) {
7231 return Ok(None);
7232 }
7233
7234 match data {
7235 RouterItem::Packet(pkt) => match ty {
7236 DataType::TimeSyncRequest => {
7237 let local_sender = self.sender_arc();
7238 if pkt.sender() == local_sender.as_ref() {
7239 Ok(self.timesync.as_ref().and_then(|runtime| {
7240 runtime.lock().tracker.as_ref().and_then(|tracker| {
7241 tracker.current_source().map(|src| src.sender.clone())
7242 })
7243 }))
7244 } else {
7245 Ok(None)
7246 }
7247 }
7248 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
7249 Ok(Some(pkt.sender().to_owned()))
7250 }
7251 _ => Ok(None),
7252 },
7253 RouterItem::Packed(bytes) => {
7254 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
7255 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
7256 }
7257 }
7258 }
7259
7260 #[cfg(feature = "discovery")]
7261 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
7262 st.discovery_cadence.on_topology_change(now_ms);
7263 for side_id in st
7268 .sides
7269 .iter()
7270 .enumerate()
7271 .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
7272 {
7273 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
7274 throttle.pending_incremental = true;
7275 throttle.incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
7276 }
7277 for transport in st.side_transport.values_mut() {
7281 transport.refresh_transmit_header_templates();
7282 }
7283 }
7284
7285 #[cfg(feature = "discovery")]
7286 fn sender_topology_board_mut<'a>(
7287 sender_state: &'a mut DiscoverySenderState,
7288 sender_id: &str,
7289 ) -> &'a mut TopologyBoardNode {
7290 let boards = Arc::make_mut(&mut sender_state.topology_boards);
7291 if let Some(idx) = boards.iter().position(|board| board.sender_id == sender_id) {
7292 return &mut boards[idx];
7293 }
7294 boards.push(TopologyBoardNode {
7295 sender_id: sender_id.to_string(),
7296 reachable_endpoints: Vec::new(),
7297 reachable_timesync_sources: Vec::new(),
7298 connections: Vec::new(),
7299 });
7300 boards.last_mut().expect("inserted above")
7301 }
7302
7303 #[cfg(feature = "discovery")]
7304 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
7305 if let Some(boards) = Arc::get_mut(&mut sender_state.topology_boards) {
7306 discovery::normalize_topology_boards(boards);
7307 }
7308 let (mut reachable, mut reachable_timesync_sources) =
7309 discovery::summarize_topology_boards(&sender_state.topology_boards);
7310 reachable.extend(sender_state.advertised_reachable.iter().copied());
7311 crate::small_sort::sort(&mut reachable);
7312 reachable.dedup();
7313 reachable_timesync_sources.extend(
7314 sender_state
7315 .advertised_reachable_timesync_sources
7316 .iter()
7317 .cloned(),
7318 );
7319 reachable_timesync_sources.sort_unstable();
7320 reachable_timesync_sources.dedup();
7321 sender_state.reachable = reachable;
7322 sender_state.reachable_timesync_sources = reachable_timesync_sources;
7323 }
7324
7325 #[cfg(feature = "discovery")]
7326 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
7327 let mut reachable = Vec::new();
7328 let mut reachable_network_variables = Vec::new();
7329 let mut reachable_timesync_sources = Vec::new();
7330 let mut last_seen_ms = 0u64;
7331 for sender in route.announcers.values() {
7332 reachable.extend(sender.reachable.iter().copied());
7333 reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
7334 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
7335 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
7336 }
7337 reachable_network_variables
7338 .extend(route.requested_network_variables.iter().map(|(ty, _)| *ty));
7339 last_seen_ms = last_seen_ms.max(
7340 route
7341 .requested_network_variables
7342 .iter()
7343 .map(|(_, last_seen_ms)| *last_seen_ms)
7344 .max()
7345 .unwrap_or(0),
7346 );
7347 crate::small_sort::sort(&mut reachable);
7348 reachable.dedup();
7349 crate::small_sort::sort(&mut reachable_network_variables);
7350 reachable_network_variables.dedup();
7351 reachable_timesync_sources.sort_unstable();
7352 reachable_timesync_sources.dedup();
7353 route.reachable = reachable;
7354 route.reachable_network_variables = reachable_network_variables;
7355 route.reachable_timesync_sources = reachable_timesync_sources;
7356 route.last_seen_ms = last_seen_ms;
7357 }
7358
7359 #[cfg(feature = "discovery")]
7360 fn local_discovery_topology_board(
7361 &self,
7362 st: &RouterInner,
7363 now_ms: u64,
7364 link_local_enabled: bool,
7365 ) -> TopologyBoardNode {
7366 let mut reachable_endpoints = self.local_discovery_endpoints(st);
7367 if !link_local_enabled {
7368 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
7369 }
7370 let mut connections = Vec::new();
7371 for route in st.discovery_routes.values() {
7372 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7373 continue;
7374 }
7375 for (sender, sender_state) in route.announcers.iter() {
7376 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7377 connections.push(sender.clone());
7378 }
7379 }
7380 }
7381 connections.sort_unstable();
7382 connections.dedup();
7383 let sender = self.sender_arc();
7384 TopologyBoardNode {
7385 sender_id: sender.to_string(),
7386 reachable_endpoints,
7387 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
7388 connections,
7389 }
7390 }
7391
7392 #[cfg(feature = "discovery")]
7393 fn advertised_discovery_topology_for_link_locked(
7394 &self,
7395 st: &RouterInner,
7396 now_ms: u64,
7397 link_local_enabled: bool,
7398 exclude_side: Option<RouterSideId>,
7399 ) -> Vec<TopologyBoardNode> {
7400 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
7401 for (&route_side, route) in st.discovery_routes.iter() {
7402 if exclude_side == Some(route_side) {
7403 continue;
7404 }
7405 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7406 continue;
7407 }
7408 for (announcer, sender_state) in route.announcers.iter() {
7409 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7410 continue;
7411 }
7412 if sender_state.topology_boards.is_empty() {
7413 let sender = self.sender_arc();
7414 let fallback = TopologyBoardNode {
7415 sender_id: announcer.clone(),
7416 reachable_endpoints: sender_state.reachable.clone(),
7417 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
7418 connections: vec![sender.to_string()],
7419 };
7420 discovery::merge_topology_boards(&mut boards, core::slice::from_ref(&fallback));
7421 } else {
7422 discovery::merge_topology_boards(&mut boards, &sender_state.topology_boards);
7425 if let Some(board) = boards
7426 .iter_mut()
7427 .find(|board| board.sender_id == *announcer)
7428 {
7429 let sender = self.sender_arc();
7430 if !board.connections.iter().any(|peer| peer == sender.as_ref()) {
7431 board.connections.push(sender.to_string());
7432 }
7433 }
7434 }
7435 }
7436 }
7437 if !link_local_enabled {
7438 for board in &mut boards {
7439 board
7440 .reachable_endpoints
7441 .retain(|ep| !ep.is_link_local_only());
7442 }
7443 }
7444 discovery::normalize_topology_boards(&mut boards);
7445 boards
7446 }
7447
7448 #[cfg(feature = "discovery")]
7449 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
7450 let mut changed = false;
7453 st.discovery_routes.retain(|_, route| {
7454 let before_peers = route.announcers.len();
7455 let before_requests = route.requested_network_variables.len();
7456 route.announcers.retain(|_, sender| {
7457 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7458 });
7459 route
7460 .requested_network_variables
7461 .retain(|(_, last_seen_ms)| {
7462 now_ms.saturating_sub(*last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7463 });
7464 if before_peers != route.announcers.len()
7465 || before_requests != route.requested_network_variables.len()
7466 {
7467 Self::recompute_discovery_side_state(route);
7468 changed = true;
7469 }
7470 let keep =
7471 !route.announcers.is_empty() || !route.requested_network_variables.is_empty();
7472 changed |= !keep;
7473 keep
7474 });
7475 changed
7476 }
7477
7478 #[cfg(feature = "discovery")]
7479 fn advertised_summaries_from_topology_locked(
7480 &self,
7481 st: &RouterInner,
7482 now_ms: u64,
7483 link_local_enabled: bool,
7484 exclude_side: Option<RouterSideId>,
7485 topology: &[TopologyBoardNode],
7486 ) -> (Vec<crate::DataEndpoint>, Vec<String>) {
7487 let (mut endpoints, mut sources) = discovery::summarize_topology_boards(topology);
7488 for (&side, route) in &st.discovery_routes {
7489 if exclude_side == Some(side)
7490 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7491 {
7492 continue;
7493 }
7494 for peer in route.announcers.values() {
7495 if now_ms.saturating_sub(peer.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7496 continue;
7497 }
7498 for endpoint in &peer.advertised_reachable {
7499 if !discovery::is_discovery_endpoint(*endpoint)
7500 && (link_local_enabled || !endpoint.is_link_local_only())
7501 && !endpoints.contains(endpoint)
7502 {
7503 endpoints.push(*endpoint);
7504 }
7505 }
7506 for source in &peer.advertised_reachable_timesync_sources {
7507 if !sources.contains(source) {
7508 sources.push(source.clone());
7509 }
7510 }
7511 }
7512 }
7513 endpoints.retain(|endpoint| {
7514 !discovery::is_discovery_endpoint(*endpoint)
7515 && (link_local_enabled || !endpoint.is_link_local_only())
7516 });
7517 crate::small_sort::sort(&mut endpoints);
7518 sources.sort_unstable();
7519 (endpoints, sources)
7520 }
7521
7522 #[cfg(feature = "discovery")]
7523 fn advertised_discovery_endpoints_for_link_locked(
7524 &self,
7525 st: &RouterInner,
7526 now_ms: u64,
7527 link_local_enabled: bool,
7528 exclude_side: Option<RouterSideId>,
7529 ) -> Vec<DataEndpoint> {
7530 let (mut reachable_endpoints, _) = discovery::summarize_topology_boards(
7531 &self.advertised_discovery_topology_for_link_locked(
7532 st,
7533 now_ms,
7534 link_local_enabled,
7535 exclude_side,
7536 ),
7537 );
7538 for (&side, route) in &st.discovery_routes {
7541 if exclude_side == Some(side)
7542 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7543 {
7544 continue;
7545 }
7546 for sender in route.announcers.values() {
7547 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7548 reachable_endpoints.extend(sender.advertised_reachable.iter().copied());
7549 }
7550 }
7551 }
7552 crate::small_sort::sort(&mut reachable_endpoints);
7553 reachable_endpoints.dedup();
7554 reachable_endpoints
7555 .into_iter()
7556 .filter(|ep| {
7557 !discovery::is_discovery_endpoint(*ep)
7558 && (link_local_enabled || !ep.is_link_local_only())
7559 })
7560 .collect()
7561 }
7562
7563 #[cfg(all(test, feature = "discovery"))]
7564 fn compact_summary_fixture(&self, endpoint: DataEndpoint) {
7565 let mut sender = DiscoverySenderState {
7566 advertised_reachable: vec![endpoint],
7567 advertised_reachable_timesync_sources: vec!["clock-behind-bridge".to_string()],
7568 last_seen_ms: self.clock.now_ms(),
7569 topology_boards: Arc::new(vec![TopologyBoardNode {
7570 sender_id: "bridge".to_string(),
7571 reachable_endpoints: Vec::new(),
7572 reachable_timesync_sources: Vec::new(),
7573 connections: Vec::new(),
7574 }]),
7575 ..Default::default()
7576 };
7577 Self::refresh_sender_topology_state(&mut sender);
7578 let mut route = DiscoverySideState::default();
7579 route.announcers.insert("bridge".to_string(), sender);
7580 Self::recompute_discovery_side_state(&mut route);
7581 self.state.lock().discovery_routes.insert(0, route);
7582 }
7583
7584 #[cfg(feature = "discovery")]
7585 fn advertised_discovery_timesync_sources_for_link_locked(
7586 &self,
7587 st: &RouterInner,
7588 now_ms: u64,
7589 exclude_side: Option<RouterSideId>,
7590 ) -> Vec<String> {
7591 let (_, mut sources) = discovery::summarize_topology_boards(
7592 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
7593 );
7594 for (&side, route) in &st.discovery_routes {
7595 if exclude_side == Some(side)
7596 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
7597 {
7598 continue;
7599 }
7600 for sender in route.announcers.values() {
7601 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7602 sources.extend(sender.advertised_reachable_timesync_sources.iter().cloned());
7603 }
7604 }
7605 }
7606 sources.sort_unstable();
7607 sources.dedup();
7608 sources
7609 }
7610
7611 #[cfg(feature = "discovery")]
7612 fn advertised_network_variables_for_link_locked(
7613 &self,
7614 st: &RouterInner,
7615 now_ms: u64,
7616 exclude_side: Option<RouterSideId>,
7617 ) -> Vec<DataType> {
7618 let mut types: Vec<DataType> = st
7619 .local_services
7620 .iter()
7621 .flat_map(|local| local.managed_variable_types.iter())
7622 .copied()
7623 .map(DataType)
7624 .collect();
7625 for (&route_side, route) in st.discovery_routes.iter() {
7626 if exclude_side == Some(route_side) {
7627 continue;
7628 }
7629 if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7630 types.extend(route.reachable_network_variables.iter().copied());
7631 }
7632 }
7633 crate::small_sort::sort(&mut types);
7634 types.dedup();
7635 types
7636 }
7637
7638 #[cfg(feature = "discovery")]
7639 fn preferred_discovery_master_present_locked<'a>(
7640 &self,
7641 st: &'a RouterInner,
7642 now_ms: u64,
7643 ) -> Option<&'a str> {
7644 if let Some(preferred) = st.preferred_discovery_master.as_deref()
7645 && (preferred == self.sender_arc().as_ref()
7646 || st
7647 .discovery_routes
7648 .values()
7649 .filter(|route| {
7650 now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7651 })
7652 .any(|route| {
7653 route
7654 .announcers
7655 .iter()
7656 .filter(|(_, peer)| {
7657 now_ms.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7658 })
7659 .any(|(name, peer)| {
7660 name == preferred
7661 || peer.topology_boards.iter().any(|board| {
7662 board.sender_id == preferred
7663 || board
7664 .connections
7665 .iter()
7666 .any(|name| name == preferred)
7667 })
7668 })
7669 }))
7670 {
7671 return Some(preferred);
7674 }
7675 None
7676 }
7677
7678 #[cfg(feature = "discovery")]
7679 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
7680 if let Some(preferred) = self.preferred_discovery_master_present_locked(st, now_ms) {
7681 return preferred.to_string();
7682 }
7683 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
7684 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
7685 }
7686
7687 #[cfg(all(feature = "discovery", feature = "timesync"))]
7688 #[inline]
7689 fn side_is_slow_control_link_locked(
7690 st: &RouterInner,
7691 side_id: RouterSideId,
7692 now_ms: u64,
7693 ) -> bool {
7694 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
7695 let recent_slow = stats.last_slow_observed_ms > 0
7696 && now_ms.saturating_sub(stats.last_slow_observed_ms)
7697 <= discovery::DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
7698 stats.sample_count > 0
7699 && ((stats.estimated_bandwidth_bps > 0
7700 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
7701 || recent_slow)
7702 })
7703 }
7704
7705 #[cfg(feature = "discovery")]
7706 fn discovery_level_for_side_locked(
7707 st: &mut RouterInner,
7708 side_id: RouterSideId,
7709 now_ms: u64,
7710 ) -> Option<DiscoveryAdvertiseLevel> {
7711 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
7712 if !throttle.has_sent_full {
7713 throttle.has_sent_full = true;
7714 throttle.pending_incremental = false;
7715 throttle.incremental_retries_remaining = 0;
7716 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7717 return Some(DiscoveryAdvertiseLevel::Full);
7718 }
7719 if throttle.pending_incremental {
7720 throttle.incremental_retries_remaining =
7721 throttle.incremental_retries_remaining.saturating_sub(1);
7722 throttle.pending_incremental = throttle.incremental_retries_remaining > 0;
7723 return Some(DiscoveryAdvertiseLevel::Incremental);
7724 }
7725 if throttle.liveness_due(now_ms) {
7728 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
7729 return Some(DiscoveryAdvertiseLevel::MinimalPing);
7730 }
7731 None
7732 }
7733
7734 #[cfg(all(feature = "discovery", feature = "timesync"))]
7735 #[inline]
7736 fn is_timesync_type(ty: DataType) -> bool {
7737 matches!(
7738 ty,
7739 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
7740 )
7741 }
7742
7743 #[cfg(all(feature = "discovery", feature = "timesync"))]
7744 #[inline(never)]
7745 fn timesync_allowed_for_side_locked(
7746 st: &RouterInner,
7747 side_id: RouterSideId,
7748 ty: DataType,
7749 now_ms: u64,
7750 ) -> bool {
7751 if !Self::is_timesync_type(ty) {
7752 return true;
7753 }
7754 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
7755 return true;
7756 }
7757 st.timesync_side_throttle
7760 .get(&side_id)
7761 .is_none_or(|throttle| now_ms >= throttle.next_allowed_ms)
7762 }
7763
7764 #[cfg(all(feature = "discovery", feature = "timesync"))]
7765 fn filter_timesync_sides_locked(
7766 st: &RouterInner,
7767 ty: DataType,
7768 now_ms: u64,
7769 sides: Vec<RouterSideId>,
7770 ) -> Vec<RouterSideId> {
7771 sides
7772 .into_iter()
7773 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
7774 .collect()
7775 }
7776
7777 #[cfg(all(feature = "discovery", feature = "timesync"))]
7778 fn filter_timesync_matches_locked(
7779 st: &RouterInner,
7780 ty: DataType,
7781 now_ms: u64,
7782 matches: Vec<DiscoveryCandidateMatch>,
7783 ) -> Vec<DiscoveryCandidateMatch> {
7784 matches
7785 .into_iter()
7786 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
7787 .collect()
7788 }
7789
7790 #[cfg(feature = "discovery")]
7791 fn refresh_known_discovery_liveness(
7792 &self,
7793 pkt: &Packet,
7794 side: RouterSideId,
7795 ) -> TelemetryResult<bool> {
7796 if pkt.data_type() == crate::DataType::DiscoveryAddress {
7797 return self.refresh_known_address_payload(pkt.payload(), side);
7798 }
7799 if pkt.data_type() == crate::DataType::DiscoveryTopology {
7800 crate::memory_admission::check(512, 128)?;
7804 let now = self.clock.now_ms();
7805 let mut st = self.state.lock();
7806 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
7807 let Some(route) = st.discovery_routes.get_mut(&side) else {
7808 return Ok(false);
7809 };
7810 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
7811 return Ok(false);
7812 };
7813 let Some(full) =
7814 discovery::topology_payload_matches_retained(pkt.payload(), &peer.topology_boards)
7815 else {
7816 return Ok(false);
7817 };
7818 peer.has_full_topology |= full;
7819 peer.last_seen_ms = now;
7820 route.last_seen_ms = route.last_seen_ms.max(now);
7821 return Ok(true);
7822 }
7823 if pkt.data_type() != crate::DataType::DiscoveryAnnounce || !pkt.payload().is_empty() {
7824 return Ok(false);
7825 }
7826 crate::memory_admission::check(512, 128)?;
7827 let now = self.clock.now_ms();
7828 let mut st = self.state.lock();
7829 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
7830 let Some(route) = st.discovery_routes.get_mut(&side) else {
7831 return Ok(false);
7832 };
7833 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
7834 return Ok(false);
7835 };
7836 peer.last_seen_ms = now;
7837 route.last_seen_ms = route.last_seen_ms.max(now);
7838 Ok(true)
7839 }
7840
7841 #[cfg(feature = "discovery")]
7842 fn refresh_known_address_payload(
7843 &self,
7844 payload: &[u8],
7845 side: RouterSideId,
7846 ) -> TelemetryResult<bool> {
7847 let ad = discovery::decode_address_view(payload)?;
7848 let now = self.clock.now_ms();
7849 let mut st = self.state.lock();
7850 let link_local = st
7851 .sides
7852 .get(side)
7853 .and_then(|entry| entry.as_ref())
7854 .is_some_and(|entry| entry.opts.link_local_enabled);
7855 {
7856 let Some(entry) = st.address_book.get(ad.hostname) else {
7857 return Ok(false);
7858 };
7859 if entry.address != ad.address
7860 || entry.requested_address != ad.requested_address
7861 || entry.mode != Self::address_mode_from_code(ad.mode, ad.requested_address)
7862 || entry.owner_hash != ad.owner_hash
7863 {
7864 return Ok(false);
7865 }
7866 if entry.birth_ms != ad.birth_ms
7867 && st.address_book.iter().any(|(hostname, other)| {
7868 hostname.as_str() != ad.hostname
7869 && [ad.address, ad.requested_address].iter().any(|candidate| {
7870 *candidate != 0
7871 && (other.address == *candidate
7872 || other.requested_address == *candidate)
7873 })
7874 })
7875 {
7876 return Ok(false);
7880 }
7881 }
7882 let sender = ad.hostname;
7883 let Some(route) = st.discovery_routes.get_mut(&side) else {
7884 return Ok(false);
7885 };
7886 let Some(peer) = route.announcers.get_mut(sender) else {
7887 return Ok(false);
7888 };
7889 if peer.advertised_address != Some(ad.address)
7890 || !ad
7891 .endpoints()
7892 .filter(|ep| link_local || !ep.is_link_local_only())
7893 .eq(peer.advertised_reachable.iter().copied())
7894 || !ad.sources().eq(peer
7895 .advertised_reachable_timesync_sources
7896 .iter()
7897 .map(String::as_str))
7898 || !ad
7899 .variables()
7900 .eq(peer.reachable_network_variables.iter().copied())
7901 || peer.link_capabilities != Some(ad.link_capabilities)
7902 {
7903 return Ok(false);
7904 }
7905 peer.last_seen_ms = now;
7908 route.last_seen_ms = route.last_seen_ms.max(now);
7909 if let Some(entry) = st.address_book.get_mut(ad.hostname) {
7910 entry.birth_ms = ad.birth_ms;
7913 entry.last_seen_ms = now;
7914 }
7915 #[cfg(feature = "compression")]
7916 {
7917 drop(st);
7918 self.note_payload_codec_address(side, payload, true);
7919 }
7920 Ok(true)
7921 }
7922
7923 #[cfg(feature = "discovery")]
7924 fn queue_pressure_keepalives(
7925 &self,
7926 requested_side: Option<RouterSideId>,
7927 ) -> TelemetryResult<()> {
7928 let now = self.clock.now_ms();
7929 let sender = self.sender_arc();
7930 crate::memory_admission::check(
7931 256 + sender.len().saturating_mul(4),
7932 sender.len().max(256),
7933 )?;
7934 let side_count = self.state.lock().sides.len();
7935 for side in 0..side_count {
7936 let due = {
7937 let st = self.state.lock();
7938 requested_side.is_none_or(|wanted| wanted == side)
7939 && st.sides[side].is_some()
7940 && self.route_allowed_locked(
7941 &st,
7942 None,
7943 Some(crate::DataType::DiscoveryAnnounce),
7944 side,
7945 )
7946 && st
7947 .discovery_side_throttle
7948 .get(&side)
7949 .is_none_or(|throttle| now >= throttle.next_ping_ms)
7950 };
7951 if !due {
7952 continue;
7953 }
7954 let summary = {
7957 let st = self.state.lock();
7958 let extra = st
7959 .discovery_routes
7960 .iter()
7961 .filter(|(id, _)| **id != side)
7962 .fold(0usize, |total, (_, route)| {
7963 total
7964 .saturating_add(route.reachable.len().saturating_mul(32))
7965 .saturating_add(
7966 route.reachable_network_variables.len().saturating_mul(32),
7967 )
7968 .saturating_add(
7969 route
7970 .reachable_timesync_sources
7971 .iter()
7972 .map(|source| source.len().saturating_add(32).saturating_mul(4))
7973 .sum::<usize>(),
7974 )
7975 });
7976 if crate::memory_admission::check(4096usize.saturating_add(extra), 2048).is_ok() {
7977 let opts = st.sides[side].as_ref().expect("checked above").opts.clone();
7978 let mut endpoints = self.local_discovery_endpoints(&st);
7979 let mut variables =
7980 self.advertised_network_variables_for_link_locked(&st, now, Some(side));
7981 let mut sources = self.local_discovery_timesync_sources(now);
7982 for (&route_side, route) in &st.discovery_routes {
7983 if route_side == side {
7984 continue;
7985 }
7986 for peer in route.announcers.values() {
7987 if now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
7988 endpoints.extend(peer.reachable.iter().copied());
7989 sources.extend(peer.reachable_timesync_sources.iter().cloned());
7990 }
7991 }
7992 }
7993 endpoints.retain(|ep| {
7994 !discovery::is_discovery_endpoint(*ep)
7995 && (opts.link_local_enabled || !ep.is_link_local_only())
7996 });
7997 crate::small_sort::sort(&mut endpoints);
7998 endpoints.dedup();
7999 crate::small_sort::sort(&mut variables);
8000 variables.dedup();
8001 sources.sort_unstable();
8002 sources.dedup();
8003 Some((endpoints, variables, sources, opts.link_capabilities()))
8004 } else {
8005 None
8006 }
8007 };
8008 let pkt = if let Some((endpoints, variables, sources, capabilities)) = summary {
8009 let address = self.local_address_advertisement(
8010 endpoints,
8011 variables,
8012 sources,
8013 capabilities,
8014 discovery::ADDRESS_STATE_REQUEST,
8015 );
8016 discovery::build_discovery_address(sender.as_ref(), now, &address)?
8017 } else {
8018 discovery::build_discovery_announce(sender.as_ref(), now, &[])?
8019 };
8020 self.emit_internal_tx(
8021 RouterTxItem::ToSide {
8022 src: None,
8023 dst: side,
8024 data: RouterItem::Packet(pkt),
8025 },
8026 true,
8027 true,
8028 )?;
8029 self.state
8030 .lock()
8031 .discovery_side_throttle
8032 .entry(side)
8033 .or_default()
8034 .next_ping_ms = now.saturating_add(discovery::DISCOVERY_SLOW_INTERVAL_MS);
8035 }
8036 Ok(())
8037 }
8038
8039 #[cfg(feature = "discovery")]
8040 fn emit_discovery_snapshot(
8041 &self,
8042 called_from_queue: bool,
8043 include_schema: bool,
8044 include_topology: bool,
8045 skip_in_flight_sides: bool,
8046 ) -> TelemetryResult<()> {
8047 self.emit_discovery_snapshot_on_side(
8048 called_from_queue,
8049 include_schema,
8050 include_topology,
8051 skip_in_flight_sides,
8052 None,
8053 )
8054 }
8055
8056 #[cfg(feature = "discovery")]
8057 fn emit_discovery_snapshot_on_side(
8058 &self,
8059 called_from_queue: bool,
8060 include_schema: bool,
8061 include_topology: bool,
8062 skip_in_flight_sides: bool,
8063 requested_side: Option<RouterSideId>,
8064 ) -> TelemetryResult<()> {
8065 self.emit_discovery_snapshot_on_side_claimed(
8066 called_from_queue,
8067 include_schema,
8068 include_topology,
8069 skip_in_flight_sides,
8070 requested_side,
8071 None,
8072 )
8073 }
8074
8075 #[cfg(feature = "discovery")]
8076 fn emit_discovery_snapshot_on_side_claimed(
8077 &self,
8078 called_from_queue: bool,
8079 include_schema: bool,
8080 include_topology: bool,
8081 skip_in_flight_sides: bool,
8082 requested_side: Option<RouterSideId>,
8083 prefix_claim: Option<u64>,
8084 ) -> TelemetryResult<()> {
8085 if requested_side.is_none() {
8086 let count = self.state.lock().sides.len();
8087 let mut first_error = None;
8088 for side in 0..count {
8089 if self.state.lock().sides[side].is_none() {
8091 continue;
8092 }
8093 if let Err(error) = self.emit_discovery_snapshot_on_side(
8094 called_from_queue,
8095 include_schema,
8096 include_topology,
8097 skip_in_flight_sides,
8098 Some(side),
8099 ) && first_error.is_none()
8100 {
8101 first_error = Some(error);
8102 }
8103 }
8104 return first_error.map_or(Ok(()), Err);
8105 }
8106 if include_schema {
8107 return self
8108 .start_schema_response_generation(
8109 requested_side.expect("single discovery side"),
8110 true,
8111 called_from_queue,
8112 include_topology,
8113 skip_in_flight_sides,
8114 )
8115 .map(|_| ());
8116 }
8117 let (scratch, largest) = {
8118 let now = self.clock.now_ms();
8119 let sender = self.sender_arc();
8120 let st = self.state.lock();
8121 let election_required = self.cfg.role.local_services()
8122 && self
8123 .preferred_discovery_master_present_locked(&st, now)
8124 .is_none();
8125 let snapshot = discovery::topology_snapshot_scratch(
8126 sender.as_ref(),
8127 self.cfg.handlers().len(),
8128 st.local_services
8129 .as_ref()
8130 .map_or(0, |local| local.managed_variable_types.len())
8131 .saturating_add(
8132 st.discovery_routes
8133 .iter()
8134 .filter(|(side, _)| Some(**side) != requested_side)
8135 .map(|(_, r)| r.reachable_network_variables.len())
8136 .sum::<usize>(),
8137 ),
8138 4,
8139 true,
8140 st.discovery_routes
8141 .iter()
8142 .filter(|(side, route)| {
8143 (Some(**side) != requested_side)
8144 && now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8145 })
8146 .flat_map(|(_, route)| route.announcers.iter())
8147 .filter(|(_, peer)| {
8148 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8149 })
8150 .map(|(name, peer)| {
8151 (
8152 name.as_str(),
8153 peer.reachable.as_slice(),
8154 peer.reachable_timesync_sources.as_slice(),
8155 peer.topology_boards.as_slice(),
8156 )
8157 }),
8158 );
8159 if !election_required {
8160 snapshot
8161 } else {
8162 let election = discovery::topology_snapshot_scratch(
8163 sender.as_ref(),
8164 self.cfg.handlers().len(),
8165 0,
8166 2,
8167 false,
8168 st.discovery_routes
8169 .values()
8170 .filter(|route| {
8171 now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8172 })
8173 .flat_map(|route| route.announcers.iter())
8174 .filter(|(_, peer)| {
8175 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8176 })
8177 .map(|(name, peer)| {
8178 (
8179 name.as_str(),
8180 peer.reachable.as_slice(),
8181 peer.reachable_timesync_sources.as_slice(),
8182 peer.topology_boards.as_slice(),
8183 )
8184 }),
8185 );
8186 (snapshot.0.max(election.0), snapshot.1.max(election.1))
8187 }
8188 };
8189 let (scratch, largest) = if include_schema {
8192 (scratch.max(8192), largest.max(2048))
8193 } else {
8194 (scratch, largest)
8195 };
8196 while let Err(err) = crate::memory_admission::check(scratch, largest) {
8201 if matches!(err, TelemetryError::Io("memory pressure"))
8202 && crate::memory_admission::check(512, 256).is_ok()
8203 {
8204 let mut st = self.state.lock();
8205 if let Some(victim) = st.lower_priority_shared_queue(199)
8206 && st.pop_shared_queue_item(victim)
8207 {
8208 continue;
8209 }
8210 }
8211 if !include_schema && matches!(err, TelemetryError::Io("memory pressure")) {
8212 return self.queue_pressure_keepalives(requested_side);
8213 }
8214 return Err(err);
8215 }
8216 let now_ms = self.clock.now_ms();
8217 let per_side = {
8218 let mut st = self.state.lock();
8219 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
8220 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8221 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8222 }
8223 st.fit_discovery_budget();
8224 let local_is_master = self.cfg.role.local_services()
8225 && self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
8226 let mut per_side = None;
8228 for side_id in 0..st.sides.len() {
8229 let Some(side) = st.sides[side_id].as_ref() else {
8230 continue;
8231 };
8232 let opts = side.opts.clone();
8233 let link_local_enabled = opts.link_local_enabled;
8234 if requested_side.is_some_and(|requested| requested != side_id) {
8235 continue;
8236 }
8237 if let Some(claim) = prefix_claim
8238 && !st
8239 .discovery_side_throttle
8240 .get(&side_id)
8241 .is_some_and(|throttle| {
8242 throttle.schema_response_active
8243 && throttle.schema_response_phase
8244 == SchemaResponsePhase::PrefixBuilding
8245 && throttle.schema_response_packet_id == claim
8246 })
8247 {
8248 return Ok(());
8249 }
8250 let discovery_in_flight = skip_in_flight_sides
8251 && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
8252 *pending_side == side_id
8253 && !tx_state.sent.is_empty()
8254 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
8255 });
8256 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
8257 {
8258 continue;
8259 }
8260 let level = if discovery_in_flight {
8264 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
8265 if !throttle.liveness_due(now_ms) {
8266 None
8267 } else {
8268 throttle.next_ping_ms =
8269 now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
8270 Some(DiscoveryAdvertiseLevel::MinimalPing)
8271 }
8272 } else {
8273 Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
8274 };
8275 let Some(level) = level else {
8276 continue;
8277 };
8278 let include_side_topology = include_topology
8279 && self.route_allowed_locked(
8280 &st,
8281 None,
8282 Some(DataType::DiscoveryTopology),
8283 side_id,
8284 );
8285 let capabilities = opts.link_capabilities();
8286 if level == DiscoveryAdvertiseLevel::MinimalPing {
8287 per_side = Some((
8288 side_id,
8289 level,
8290 Vec::new(),
8291 Vec::new(),
8292 Vec::new(),
8293 Vec::new(),
8294 Vec::new(),
8295 capabilities,
8296 local_is_master,
8297 ));
8298 continue;
8299 }
8300 let current_topology = if include_side_topology {
8301 self.advertised_discovery_topology_for_link_locked(
8302 &st,
8303 now_ms,
8304 link_local_enabled,
8305 Some(side_id),
8306 )
8307 } else {
8308 Vec::new()
8309 };
8310 let (endpoints, timesync_sources) = if include_side_topology {
8311 self.advertised_summaries_from_topology_locked(
8312 &st,
8313 now_ms,
8314 link_local_enabled,
8315 Some(side_id),
8316 ¤t_topology,
8317 )
8318 } else {
8319 (
8320 self.advertised_discovery_endpoints_for_link_locked(
8321 &st,
8322 now_ms,
8323 link_local_enabled,
8324 Some(side_id),
8325 ),
8326 self.advertised_discovery_timesync_sources_for_link_locked(
8327 &st,
8328 now_ms,
8329 Some(side_id),
8330 ),
8331 )
8332 };
8333 let (topology, removed_topology) = match level {
8334 DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
8335 DiscoveryAdvertiseLevel::Incremental => {
8336 let previous = &st
8337 .discovery_side_throttle
8338 .get(&side_id)
8339 .expect("discovery throttle exists")
8340 .last_topology;
8341 discovery::topology_delta(previous, ¤t_topology)
8342 }
8343 DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
8344 };
8345 let throttle = st
8346 .discovery_side_throttle
8347 .get_mut(&side_id)
8348 .expect("discovery throttle exists");
8349 if level == DiscoveryAdvertiseLevel::Full
8350 || (level == DiscoveryAdvertiseLevel::Incremental
8351 && throttle.incremental_retries_remaining == 0)
8352 {
8353 throttle.last_topology = current_topology;
8354 }
8355 per_side = Some((
8356 side_id,
8357 level,
8358 endpoints,
8359 self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
8360 timesync_sources,
8361 topology,
8362 removed_topology,
8363 capabilities,
8364 local_is_master,
8365 ));
8366 }
8367 per_side
8368 };
8369 if let Some((
8370 side_id,
8371 level,
8372 endpoints,
8373 network_variables,
8374 timesync_sources,
8375 topology,
8376 removed_topology,
8377 capabilities,
8378 local_is_master,
8379 )) = per_side
8380 {
8381 let sender = self.sender_arc();
8382 if matches!(
8383 level,
8384 DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
8385 ) && (self.cfg.role.local_services() || level == DiscoveryAdvertiseLevel::Full)
8386 {
8387 let pkt = if self.cfg.role.local_services() {
8388 let address = self.local_address_advertisement(
8389 endpoints.clone(),
8390 network_variables.clone(),
8391 timesync_sources.clone(),
8392 capabilities,
8393 if local_is_master {
8394 discovery::ADDRESS_STATE_APPROVED
8395 } else {
8396 discovery::ADDRESS_STATE_REQUEST
8397 },
8398 );
8399 discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?
8400 } else {
8401 discovery::build_discovery_link_capabilities(
8402 sender.as_ref(),
8403 now_ms,
8404 capabilities,
8405 )?
8406 };
8407 self.emit_discovery_prefix_item(
8408 RouterTxItem::ToSide {
8409 src: None,
8410 dst: side_id,
8411 data: RouterItem::Packet(pkt),
8412 },
8413 true,
8414 called_from_queue,
8415 prefix_claim.map(|claim| (side_id, claim)),
8416 )?;
8417 }
8418 if level == DiscoveryAdvertiseLevel::MinimalPing
8419 || (!self.cfg.role.local_services()
8420 && (!endpoints.is_empty() || level != DiscoveryAdvertiseLevel::Full))
8421 {
8422 let pkt = discovery::build_discovery_announce(
8423 sender.as_ref(),
8424 now_ms,
8425 endpoints.as_slice(),
8426 )?;
8427 self.emit_discovery_prefix_item(
8428 RouterTxItem::ToSide {
8429 src: None,
8430 dst: side_id,
8431 data: RouterItem::Packet(pkt),
8432 },
8433 true,
8434 called_from_queue,
8435 prefix_claim.map(|claim| (side_id, claim)),
8436 )?;
8437 }
8438 if !self.cfg.role.local_services()
8439 && level != DiscoveryAdvertiseLevel::MinimalPing
8440 && !timesync_sources.is_empty()
8441 {
8442 let pkt = discovery::build_discovery_timesync_sources(
8443 sender.as_ref(),
8444 now_ms,
8445 ×ync_sources,
8446 )?;
8447 self.emit_discovery_prefix_item(
8448 RouterTxItem::ToSide {
8449 src: None,
8450 dst: side_id,
8451 data: RouterItem::Packet(pkt),
8452 },
8453 true,
8454 called_from_queue,
8455 prefix_claim.map(|claim| (side_id, claim)),
8456 )?;
8457 }
8458 if include_topology
8459 && (!topology.is_empty() || !removed_topology.is_empty())
8460 && level != DiscoveryAdvertiseLevel::MinimalPing
8461 {
8462 let pkt = if level == DiscoveryAdvertiseLevel::Full {
8463 discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
8464 } else {
8465 discovery::build_discovery_topology_delta(
8466 sender.as_ref(),
8467 now_ms,
8468 &topology,
8469 &removed_topology,
8470 )?
8471 };
8472 self.emit_discovery_prefix_item(
8473 RouterTxItem::ToSide {
8474 src: None,
8475 dst: side_id,
8476 data: RouterItem::Packet(pkt),
8477 },
8478 true,
8479 called_from_queue,
8480 prefix_claim.map(|claim| (side_id, claim)),
8481 )?;
8482 }
8483 drop((
8486 endpoints,
8487 network_variables,
8488 timesync_sources,
8489 topology,
8490 removed_topology,
8491 ));
8492 }
8493 Ok(())
8494 }
8495
8496 #[cfg(feature = "discovery")]
8497 fn emit_discovery_prefix_item(
8498 &self,
8499 item: RouterTxItem,
8500 ignore_local: bool,
8501 called_from_queue: bool,
8502 claim: Option<(RouterSideId, u64)>,
8503 ) -> TelemetryResult<()> {
8504 let Some((side, claim)) = claim else {
8505 return self.emit_internal_tx(item, ignore_local, called_from_queue);
8506 };
8507 let priority = match &item {
8508 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
8509 _ => return Err(TelemetryError::BadArg),
8510 };
8511 let mut st = self.state.lock();
8512 if !st
8513 .discovery_side_throttle
8514 .get(&side)
8515 .is_some_and(|throttle| {
8516 throttle.schema_response_active
8517 && throttle.schema_response_phase == SchemaResponsePhase::PrefixBuilding
8518 && throttle.schema_response_packet_id == claim
8519 })
8520 {
8521 return Err(TelemetryError::Io("schema prefix cancelled"));
8522 }
8523 st.push_transmit(TxQueued {
8524 disposable: true,
8525 item,
8526 ignore_local,
8527 priority,
8528 delivery_state: QueuedDelivery::Untracked,
8529 accepted_type: 0,
8530 deferred_since_ms: 0,
8531 retry_after_ms: 0,
8532 })
8533 }
8534
8535 #[cfg(feature = "discovery")]
8536 fn queue_discovery_announce(
8537 &self,
8538 include_schema: bool,
8539 skip_in_flight_sides: bool,
8540 ) -> TelemetryResult<()> {
8541 let now_ms = self.clock.now_ms();
8542 {
8543 let mut st = self.state.lock();
8544 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
8545 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8546 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8547 }
8548 st.fit_discovery_budget();
8549 if st.sides.iter().all(|side| side.is_none()) {
8550 return Ok(());
8551 }
8552 st.discovery_cadence.on_announce_sent(now_ms);
8555 }
8556 self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
8557 }
8558
8559 #[cfg(feature = "discovery")]
8560 fn note_schema_received(&self, side: RouterSideId, sender: &str) {
8561 let mut st = self.state.lock();
8562 let address = sender
8566 .strip_prefix("@addr:")
8567 .and_then(|value| value.parse::<u32>().ok());
8568 if let Some(route) = st.discovery_routes.get_mut(&side) {
8569 for (name, peer) in &mut route.announcers {
8570 if name == sender
8571 || address.is_some_and(|address| {
8572 peer.advertised_address == Some(address)
8573 || sender_address_u32(name) == address
8574 })
8575 {
8576 peer.has_schema = true;
8577 break;
8578 }
8579 }
8580 }
8581 }
8582
8583 #[cfg(feature = "discovery")]
8586 fn start_schema_response_generation(
8587 &self,
8588 side: RouterSideId,
8589 explicit: bool,
8590 called_from_queue: bool,
8591 include_topology: bool,
8592 skip_in_flight_sides: bool,
8593 ) -> TelemetryResult<bool> {
8594 let now = self.clock.now_ms();
8595 let claim = {
8596 let mut st = self.state.lock();
8597 let throttle = st.discovery_side_throttle.entry(side).or_default();
8598 if throttle.schema_response_active {
8599 if explicit {
8600 throttle.schema_response_pending = true;
8601 }
8602 return Ok(false);
8603 }
8604 if !explicit
8605 && (!throttle.schema_response_pending || now < throttle.next_schema_response_ms)
8606 {
8607 return Ok(false);
8608 }
8609 let claim = st
8610 .schema_response_sequence
8611 .checked_add(1)
8612 .ok_or(TelemetryError::Io("schema generation exhausted"))?;
8613 st.schema_response_sequence = claim;
8614 let throttle = st
8615 .discovery_side_throttle
8616 .get_mut(&side)
8617 .expect("schema throttle");
8618 throttle.schema_response_pending = false;
8619 throttle.schema_response_active = true;
8620 throttle.schema_response_phase = SchemaResponsePhase::PrefixBuilding;
8621 throttle.schema_response_packet_id = claim;
8622 throttle.schema_response_started = false;
8623 throttle.next_schema_response_ms =
8624 now.saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
8625 throttle.has_sent_full = false;
8626 throttle.last_topology = Vec::new();
8627 if let Some(transport) = st.side_transport.get_mut(&side) {
8628 transport.refresh_transmit_header_templates();
8629 }
8630 claim
8631 };
8632 let prefix = self.emit_discovery_snapshot_on_side_claimed(
8636 called_from_queue,
8637 false,
8638 include_topology,
8639 skip_in_flight_sides,
8640 Some(side),
8641 Some(claim),
8642 );
8643 {
8644 let mut st = self.state.lock();
8645 let throttle = st
8646 .discovery_side_throttle
8647 .get_mut(&side)
8648 .ok_or(TelemetryError::BadArg)?;
8649 if !throttle.schema_response_active
8650 || throttle.schema_response_phase != SchemaResponsePhase::PrefixBuilding
8651 || throttle.schema_response_packet_id != claim
8652 {
8653 return Ok(false);
8654 }
8655 if let Err(error) = prefix {
8656 throttle.schema_response_active = false;
8657 throttle.schema_response_phase = SchemaResponsePhase::Idle;
8658 throttle.schema_response_pending = true;
8659 return Err(error);
8660 }
8661 if !throttle.has_sent_full {
8662 throttle.schema_response_active = false;
8664 throttle.schema_response_phase = SchemaResponsePhase::Idle;
8665 return Ok(true);
8666 }
8667 throttle.schema_response_packet_id = 0;
8668 throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
8669 }
8670 self.admit_schema_response_catalog(side)?;
8671 Ok(true)
8672 }
8673
8674 #[cfg(feature = "discovery")]
8675 fn admit_schema_response_catalog(&self, side: RouterSideId) -> TelemetryResult<bool> {
8676 self.admit_schema_response_catalog_with(side, || {
8677 discovery::build_discovery_schema(self.sender_arc().as_ref(), self.clock.now_ms())
8678 })
8679 }
8680
8681 #[cfg(feature = "discovery")]
8682 fn admit_schema_response_catalog_with(
8683 &self,
8684 side: RouterSideId,
8685 build: impl FnOnce() -> TelemetryResult<Packet>,
8686 ) -> TelemetryResult<bool> {
8687 let claim = {
8688 let mut st = self.state.lock();
8689 let throttle = st
8690 .discovery_side_throttle
8691 .get(&side)
8692 .ok_or(TelemetryError::BadArg)?;
8693 if !throttle.schema_response_active
8694 || throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting
8695 {
8696 return Ok(false);
8697 }
8698 let protected_full = st.transmit_queue.len() >= st.transmit_queue.capacity()
8699 && !RouterInner::transmit_eviction_candidate(&st.transmit_queue)
8700 .and_then(|index| st.transmit_queue.get(index))
8701 .is_some_and(|queued| {
8702 queued.priority < crate::transport_priority(DataType::DiscoverySchema)
8703 });
8704 if protected_full
8705 || !self.route_allowed_locked(&st, None, Some(DataType::DiscoverySchema), side)
8706 {
8707 return Ok(false);
8708 }
8709 let claim = st
8710 .schema_response_sequence
8711 .checked_add(1)
8712 .ok_or(TelemetryError::Io("schema generation exhausted"))?;
8713 st.schema_response_sequence = claim;
8714 let throttle = st
8715 .discovery_side_throttle
8716 .get_mut(&side)
8717 .expect("schema throttle");
8718 throttle.schema_response_phase = SchemaResponsePhase::CatalogBuilding;
8719 throttle.schema_response_packet_id = claim;
8720 claim
8721 };
8722 let built = crate::memory_admission::check(8192, 2048).and_then(|_| build());
8726 let prepared = built.and_then(|packet| {
8727 let item = RouterTxItem::ToSide {
8728 src: None,
8729 dst: side,
8730 data: RouterItem::Packet(packet),
8731 };
8732 let priority = match &item {
8733 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
8734 _ => unreachable!(),
8735 };
8736 Ok((item, priority))
8737 });
8738 let mut st = self.state.lock();
8739 let throttle = st
8740 .discovery_side_throttle
8741 .get(&side)
8742 .ok_or(TelemetryError::BadArg)?;
8743 if !throttle.schema_response_active
8744 || throttle.schema_response_phase != SchemaResponsePhase::CatalogBuilding
8745 || throttle.schema_response_packet_id != claim
8746 {
8747 return Ok(false);
8748 }
8749 let result = prepared.and_then(|(item, priority)| {
8752 let packet_id = Self::get_hash(match &item {
8753 RouterTxItem::ToSide { data, .. } => data,
8754 _ => unreachable!(),
8755 });
8756 let throttle = st
8757 .discovery_side_throttle
8758 .get_mut(&side)
8759 .expect("schema throttle");
8760 throttle.schema_response_packet_id = packet_id;
8761 throttle.schema_response_phase = SchemaResponsePhase::Owned;
8762 st.push_transmit(TxQueued {
8763 disposable: true,
8764 item,
8765 ignore_local: true,
8766 priority,
8767 delivery_state: QueuedDelivery::Untracked,
8768 accepted_type: 0,
8769 deferred_since_ms: 0,
8770 retry_after_ms: 0,
8771 })
8772 });
8773 let throttle = st
8774 .discovery_side_throttle
8775 .get_mut(&side)
8776 .expect("schema throttle");
8777 match result {
8778 Ok(()) => Ok(true),
8779 Err(TelemetryError::Io(_)) => {
8780 throttle.schema_response_packet_id = 0;
8781 throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
8782 Ok(false)
8783 }
8784 Err(error) => {
8785 throttle.schema_response_active = false;
8786 throttle.schema_response_phase = SchemaResponsePhase::Idle;
8787 throttle.schema_response_pending = true;
8788 Err(error)
8789 }
8790 }
8791 }
8792
8793 #[cfg(feature = "discovery")]
8794 fn poll_schema_response_on_side(&self, side: RouterSideId) -> TelemetryResult<bool> {
8795 let waiting = {
8796 let st = self.state.lock();
8797 let Some(throttle) = st.discovery_side_throttle.get(&side) else {
8798 return Ok(false);
8799 };
8800 if throttle.schema_response_active {
8801 if throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting {
8802 return Ok(false);
8803 }
8804 true
8805 } else {
8806 if !throttle.schema_response_pending
8807 || self.clock.now_ms() < throttle.next_schema_response_ms
8808 {
8809 return Ok(false);
8810 }
8811 if st.transmit_queue.iter().any(|entry| {
8813 matches!(&entry.item,
8814 RouterTxItem::ToSide {dst, data, ..} if *dst == side
8815 && Self::item_data_type(data).ok() == Some(DataType::DiscoverySchema))
8816 }) {
8817 return Ok(false);
8818 }
8819 false
8820 }
8821 };
8822 if waiting {
8823 self.admit_schema_response_catalog(side)
8824 } else {
8825 self.start_schema_response_generation(side, false, true, true, false)
8826 }
8827 }
8828
8829 #[cfg(feature = "discovery")]
8830 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
8831 let now_ms = self.clock.now_ms();
8832 let recovery_sides = {
8836 let mut st = self.state.lock();
8837 let mut due = Vec::new();
8838 for side in 0..st.sides.len() {
8839 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
8840 route.announcers.iter().any(|(name, peer)| {
8841 !peer.has_full_topology
8842 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
8843 && !route.announcers.values().any(|bridge| {
8844 bridge.has_full_topology
8845 && now_ms.saturating_sub(bridge.last_seen_ms)
8846 < DISCOVERY_ROUTE_TTL_MS
8847 && bridge
8848 .topology_boards
8849 .iter()
8850 .any(|board| board.sender_id == *name)
8851 })
8852 })
8853 });
8854 if !missing {
8855 continue;
8856 }
8857 let throttle = st.discovery_side_throttle.entry(side).or_default();
8858 if throttle.next_topology_request_ms == 0 {
8859 throttle.next_topology_request_ms =
8860 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
8861 } else if now_ms >= throttle.next_topology_request_ms {
8862 throttle.next_topology_request_ms =
8863 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
8864 due.push(side);
8865 }
8866 }
8867 due
8868 };
8869 let recovering = !recovery_sides.is_empty();
8870 for side in recovery_sides {
8871 let pkt =
8872 discovery::build_discovery_topology_request(self.sender_arc().as_ref(), now_ms)?;
8873 self.emit_internal_tx(
8874 RouterTxItem::ToSide {
8875 src: None,
8876 dst: side,
8877 data: RouterItem::Packet(pkt),
8878 },
8879 true,
8880 true,
8881 )?;
8882 }
8883 let schema_recovery_sides = {
8884 let mut st = self.state.lock();
8885 let mut due = Vec::new();
8886 for side in 0..st.sides.len() {
8887 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
8888 route.announcers.iter().any(|(name, peer)| {
8889 peer.has_full_topology
8890 && !peer.has_schema
8891 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
8892 && !st.side_transport.get(&side).is_some_and(|transport| {
8896 transport.rx_chunks.values().any(|assembly| {
8897 assembly.schema_progress_for(sender_address_u32(name), now_ms)
8900 || peer.advertised_address.is_some_and(|address| {
8901 assembly.schema_progress_for(address, now_ms)
8902 })
8903 })
8904 })
8905 })
8906 });
8907 if !missing {
8908 continue;
8909 }
8910 let throttle = st.discovery_side_throttle.entry(side).or_default();
8911 if throttle.next_schema_request_ms == 0 {
8912 throttle.next_schema_request_ms =
8913 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
8914 } else if now_ms >= throttle.next_schema_request_ms {
8915 throttle.next_schema_request_ms =
8916 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
8917 due.push(side);
8918 }
8919 }
8920 due
8921 };
8922 let recovering = recovering || !schema_recovery_sides.is_empty();
8923 for side in schema_recovery_sides {
8924 let pkt =
8925 discovery::build_discovery_schema_request(self.sender_arc().as_ref(), now_ms)?;
8926 self.emit_internal_tx(
8927 RouterTxItem::ToSide {
8928 src: None,
8929 dst: side,
8930 data: RouterItem::Packet(pkt),
8931 },
8932 true,
8933 true,
8934 )?;
8935 }
8936 let due = {
8937 let mut st = self.state.lock();
8938 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
8939 if removed {
8940 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
8941 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8942 }
8943 st.fit_discovery_budget();
8944 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
8945 let Some(side) = side.as_ref() else {
8946 return false;
8947 };
8948 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
8949 {
8950 return false;
8951 }
8952 let _ = side;
8953 let in_flight = st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
8954 *pending_side == side_id
8955 && !tx_state.sent.is_empty()
8956 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
8957 });
8958 st.discovery_side_throttle
8959 .get(&side_id)
8960 .is_none_or(|throttle| {
8961 (!in_flight && (!throttle.has_sent_full || throttle.pending_incremental))
8962 || throttle.liveness_due(now_ms)
8963 })
8964 });
8965 if st.sides.is_empty() || !has_any {
8966 return Ok(recovering);
8967 }
8968 st.discovery_cadence.due(now_ms)
8969 };
8970 if !due {
8971 return Ok(recovering);
8972 }
8973 self.queue_discovery_announce(false, true)?;
8982 Ok(true)
8983 }
8984
8985 #[cfg(feature = "discovery")]
8986 fn learn_discovery_packet(
8987 &self,
8988 pkt: &Packet,
8989 src: Option<RouterSideId>,
8990 called_from_queue: bool,
8991 ) -> TelemetryResult<bool> {
8992 if !discovery::is_discovery_type(pkt.data_type()) {
8993 return Ok(false);
8994 }
8995 let Some(side) = src else {
8996 return Ok(true);
8997 };
8998 #[cfg(feature = "compression")]
8999 self.note_payload_codec_packet(side, pkt, false);
9000 if self.refresh_known_discovery_liveness(pkt, side)? {
9001 return Ok(true);
9002 }
9003 #[cfg(not(feature = "std"))]
9004 if pkt.data_type() == crate::DataType::DiscoverySchema {
9005 self.merge_schema_changes(pkt.payload())?;
9006 self.note_schema_received(side, pkt.sender());
9007 return Ok(true);
9008 }
9009 if pkt.data_type() == DataType::DiscoverySchemaRequest {
9010 self.state
9011 .lock()
9012 .discovery_side_throttle
9013 .entry(side)
9014 .or_default()
9015 .schema_response_pending = true;
9016 self.poll_schema_response_on_side(side)?;
9017 return Ok(true);
9018 }
9019 if matches!(
9023 pkt.data_type(),
9024 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
9025 ) {
9026 crate::memory_admission::check(1024, 256)?;
9027 if let Some(transport) = self.state.lock().side_transport.get_mut(&side) {
9032 transport.refresh_transmit_header_templates();
9033 }
9034 }
9035 if matches!(
9036 pkt.data_type(),
9037 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
9038 ) {
9039 let now_ms = self.clock.now_ms();
9040 {
9041 let mut st = self.state.lock();
9042 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9043 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9044 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9045 }
9046 let throttle = st.discovery_side_throttle.entry(side).or_default();
9047 throttle.has_sent_full = false;
9048 throttle.last_topology = Vec::new();
9052 }
9053 self.emit_discovery_snapshot_on_side(
9058 called_from_queue,
9059 pkt.data_type() == DataType::DiscoverySchemaRequest,
9060 true,
9061 false,
9062 Some(side),
9063 )?;
9064 return Ok(true);
9065 }
9066 let small_topology_scratch =
9067 if pkt.data_type() == DataType::DiscoveryTopology && pkt.payload().len() <= 128 {
9068 let st = self.state.lock();
9069 let sender = pkt
9070 .sender()
9071 .strip_prefix("@addr:")
9072 .and_then(|value| value.parse::<u32>().ok())
9073 .and_then(|address| Self::hostname_for_address_borrowed_locked(&st, address))
9074 .unwrap_or(pkt.sender());
9075 st.discovery_routes.get(&side).and_then(|route| {
9076 let peer = route.announcers.get(sender)?;
9077 let mut shallow = discovery::CloneScratch::default();
9078 shallow.buffer(route.reachable.len().saturating_mul(4));
9079 shallow.buffer(route.reachable_network_variables.len().saturating_mul(4));
9080 shallow.buffer(
9081 route
9082 .requested_network_variables
9083 .len()
9084 .saturating_mul(core::mem::size_of::<(DataType, u64)>()),
9085 );
9086 shallow.strings(&route.reachable_timesync_sources);
9087 for (name, value) in &route.announcers {
9088 shallow.buffer(name.len());
9089 shallow.buffer(value.reachable.len().saturating_mul(4));
9090 shallow.buffer(value.advertised_reachable.len().saturating_mul(4));
9091 shallow.strings(&value.reachable_timesync_sources);
9092 shallow.strings(&value.advertised_reachable_timesync_sources);
9093 shallow.buffer(value.reachable_network_variables.len().saturating_mul(4));
9094 }
9095 let entry = core::mem::size_of::<String>()
9096 + core::mem::size_of::<DiscoverySenderState>();
9097 shallow.bytes = shallow
9098 .bytes
9099 .saturating_add(route.announcers.len().saturating_mul(entry));
9100 shallow.largest = shallow
9101 .largest
9102 .max(entry.saturating_mul(11).saturating_add(128));
9103 let mut deep = discovery::CloneScratch::default();
9104 if discovery::topology_payload_is_incremental(pkt.payload()) {
9105 deep.graph(&peer.topology_boards, pkt.payload().len() / 16);
9106 }
9107 Some((
9108 2048usize
9109 .saturating_add(shallow.bytes.saturating_mul(3))
9110 .saturating_add(deep.bytes)
9111 .saturating_add(pkt.payload().len().saturating_mul(4)),
9112 shallow.largest.max(deep.largest).max(2048),
9113 ))
9114 })
9115 } else {
9116 None
9117 };
9118 let retained_summary = if small_topology_scratch.is_none()
9119 && matches!(
9120 pkt.data_type(),
9121 DataType::DiscoveryAddress
9122 | DataType::DiscoveryTopology
9123 | DataType::DiscoveryAnnounce
9124 | DataType::DiscoveryTimeSyncSources
9125 | DataType::ManagedVariableRequest
9126 ) {
9127 self.state
9128 .lock()
9129 .discovery_routes
9130 .get(&side)
9131 .map_or(0, |route| {
9132 RouterInner::discovery_route_byte_cost(side, route)
9133 })
9134 } else {
9135 0
9136 };
9137 let (decode_bytes, largest_block) = if let Some(scratch) = small_topology_scratch {
9138 scratch
9139 } else if pkt.data_type() == DataType::ManagedVariableRequest {
9140 let ty = discovery::decode_managed_variable_request(pkt)?;
9141 let existing = self
9142 .state
9143 .lock()
9144 .discovery_routes
9145 .get(&side)
9146 .is_some_and(|route| {
9147 route
9148 .requested_network_variables
9149 .iter()
9150 .any(|(id, _)| *id == ty)
9151 });
9152 if existing {
9153 (1024, 256)
9154 } else {
9155 (
9158 2048usize.saturating_add(retained_summary.saturating_mul(2)),
9159 retained_summary
9160 .checked_next_power_of_two()
9161 .unwrap_or(usize::MAX)
9162 .max(1024),
9163 )
9164 }
9165 } else {
9166 let scratch = if pkt.data_type() == DataType::DiscoveryAddress {
9167 4096usize.saturating_add(retained_summary)
9168 } else {
9169 8192usize.saturating_add(retained_summary)
9170 };
9171 let decode_bytes = pkt
9172 .payload()
9173 .len()
9174 .saturating_mul(4)
9175 .saturating_add(scratch);
9176 let largest_block = pkt.payload().len().saturating_mul(2).max(2048);
9177 (decode_bytes, largest_block)
9178 };
9179 while let Err(err) = crate::memory_admission::check(decode_bytes, largest_block) {
9180 if !matches!(err, TelemetryError::Io("memory pressure")) {
9181 return Err(err);
9182 }
9183 let mut st = self.state.lock();
9184 #[cfg(feature = "compression")]
9185 if st.reclaim_codec_metadata() {
9186 continue;
9187 }
9188
9189 let Some(victim) = st.lower_priority_shared_queue(255) else {
9190 return Err(err);
9191 };
9192 if !st.pop_shared_queue_item(victim) {
9193 return Err(err);
9194 }
9195 }
9196 if pkt.data_type() == DataType::DiscoveryAddress {
9197 let mut ad = discovery::decode_discovery_address(pkt)?;
9198 let identity = ad.clone();
9199 let mut changed = false;
9200 let sender_id = if ad.hostname.is_empty() {
9206 pkt.sender()
9207 } else {
9208 ad.hostname.as_str()
9209 };
9210 let now_ms = self.clock.now_ms();
9211 let mut st = self.state.lock();
9212 let side_link_local_enabled = st
9213 .sides
9214 .get(side)
9215 .and_then(|entry| entry.as_ref())
9216 .map(|side_ref| side_ref.opts.link_local_enabled)
9217 .unwrap_or(false);
9218 if !side_link_local_enabled {
9219 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
9220 }
9221 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
9222 let previously_reachable_network_variables = route.reachable_network_variables.clone();
9223 if pkt.sender() != sender_id {
9224 route.announcers.remove(pkt.sender());
9225 }
9226 let mut sender_state =
9227 core::mem::take(route.announcers.entry(sender_id.to_string()).or_default());
9228 if sender_state.advertised_address != Some(ad.address) {
9234 sender_state.advertised_address = Some(ad.address);
9235 changed = true;
9236 }
9237 if sender_state.advertised_reachable != ad.reachable_endpoints {
9238 sender_state.advertised_reachable = ad.reachable_endpoints;
9239 changed = true;
9240 }
9241 if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
9242 sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
9243 changed = true;
9244 }
9245 if sender_state.reachable_network_variables != ad.reachable_network_variables {
9246 sender_state.reachable_network_variables = ad.reachable_network_variables;
9247 changed = true;
9248 }
9249 if sender_state.link_capabilities != Some(ad.link_capabilities) {
9250 sender_state.link_capabilities = Some(ad.link_capabilities);
9251 changed = true;
9252 }
9253 Self::refresh_sender_topology_state(&mut sender_state);
9254 sender_state.last_seen_ms = now_ms;
9255 route.announcers.insert(sender_id.to_string(), sender_state);
9256 Self::recompute_discovery_side_state(&mut route);
9257 let newly_reachable_network_variables: Vec<DataType> = route
9258 .reachable_network_variables
9259 .iter()
9260 .copied()
9261 .filter(|ty| !previously_reachable_network_variables.contains(ty))
9262 .collect();
9263 st.commit_discovery_route(side, route)?;
9264 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9265 if changed {
9266 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
9267 }
9268 let replay: Vec<Packet> = newly_reachable_network_variables
9274 .iter()
9275 .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
9276 .filter_map(|ty| {
9277 st.local_services
9278 .as_ref()?
9279 .managed_variable_latest
9280 .get(&ty.as_u32())
9281 .map(|entry| entry.packet.clone())
9282 })
9283 .collect();
9284 drop(st);
9285 self.ingest_address_advertisement(identity)?;
9286 #[cfg(feature = "compression")]
9287 self.note_payload_codec_packet(side, pkt, true);
9288 for value in replay {
9289 self.emit_internal_tx_with_priority(
9290 RouterTxItem::ToSide {
9291 src: None,
9292 dst: side,
9293 data: RouterItem::Packet(value),
9294 },
9295 true,
9296 crate::transport_priority(DataType::ManagedVariableValue),
9297 called_from_queue,
9298 )?;
9299 }
9300 return Ok(true);
9301 }
9302 let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
9303 Some(discovery::decode_discovery_topology_update(pkt)?)
9304 } else {
9305 None
9306 };
9307 let packet_sender = {
9308 let st = self.state.lock();
9309 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
9310 if canonical != pkt.sender() {
9311 canonical
9312 } else if let Some(address) = pkt
9313 .sender()
9314 .strip_prefix("@addr:")
9315 .and_then(|value| value.parse::<u32>().ok())
9316 {
9317 decoded_topology
9323 .as_ref()
9324 .and_then(|update| {
9325 update
9326 .boards
9327 .iter()
9328 .find(|board| sender_address_u32(&board.sender_id) == address)
9329 })
9330 .map(|board| board.sender_id.clone())
9331 .unwrap_or(canonical)
9332 } else {
9333 canonical
9334 }
9335 };
9336 let local_sender = self.sender_arc();
9337 if packet_sender == local_sender.as_ref() {
9338 return Ok(true);
9339 }
9340
9341 if pkt.data_type() == DataType::ManagedVariableRequest {
9342 let ty = discovery::decode_managed_variable_request(pkt)?;
9343 {
9349 let now_ms = self.clock.now_ms();
9350 let mut st = self.state.lock();
9351 let existing = st.discovery_routes.get_mut(&side).and_then(|route| {
9355 let (_, last_seen_ms) = route
9356 .requested_network_variables
9357 .iter_mut()
9358 .find(|(requested, _)| *requested == ty)?;
9359 *last_seen_ms = now_ms;
9360 route.last_seen_ms = route.last_seen_ms.max(now_ms);
9361 Some(())
9362 });
9363 let newly_reachable = existing.is_none();
9364 if newly_reachable {
9365 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
9366 route.requested_network_variables.push((ty, now_ms));
9367 Self::recompute_discovery_side_state(&mut route);
9368 st.commit_discovery_route(side, route)?;
9369 }
9370 if newly_reachable {
9371 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9372 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9373 }
9374 }
9375 if !self.can_write_managed_variable(ty) {
9376 return Ok(false);
9381 }
9382 if let Some(value) = self.managed_variable_latest(ty) {
9383 self.emit_internal_tx(
9384 RouterTxItem::ToSide {
9385 src: None,
9386 dst: side,
9387 data: RouterItem::Packet(value),
9388 },
9389 true,
9390 called_from_queue,
9391 )?;
9392 return Ok(true);
9393 }
9394 return Ok(false);
9397 }
9398 #[cfg(feature = "std")]
9399 if pkt.data_type() == DataType::DiscoverySchema {
9400 #[cfg(feature = "compression")]
9401 let snapshot =
9402 self.decode_required_schema(|| discovery::decode_discovery_schema(pkt))?;
9403 #[cfg(not(feature = "compression"))]
9404 let snapshot = discovery::decode_discovery_schema(pkt)?;
9405 let mut st = self.state.lock();
9414 let budget = st.schema_growth_budget();
9415 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
9416 if report.changed() {
9417 st.fit_discovery_budget();
9418 }
9419 drop(st);
9420 self.note_schema_received(side, &packet_sender);
9421 return Ok(true);
9422 }
9423 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
9424 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
9425 #[cfg(feature = "compression")]
9426 self.note_payload_codec_packet(side, pkt, true);
9427 return Ok(true);
9428 }
9429 let mut announce = if pkt.data_type() == DataType::DiscoveryAnnounce {
9430 Some(discovery::decode_discovery_announce(pkt)?)
9431 } else {
9432 None
9433 };
9434 let mut time_sources = if pkt.data_type() == DataType::DiscoveryTimeSyncSources {
9435 Some(discovery::decode_discovery_timesync_sources(pkt)?)
9436 } else {
9437 None
9438 };
9439 let mut st = self.state.lock();
9440 let now_ms = self.clock.now_ms();
9441 if pkt.data_type() == DataType::DiscoveryLeave {
9442 let leaving = packet_sender.as_str();
9443 let before = st.discovery_routes.clone();
9444 for route in st.discovery_routes.values_mut() {
9445 route.announcers.remove(leaving);
9446 for sender_state in route.announcers.values_mut() {
9447 Arc::make_mut(&mut sender_state.topology_boards)
9448 .retain(|board| board.sender_id != leaving);
9449 for board in Arc::make_mut(&mut sender_state.topology_boards).iter_mut() {
9450 board.connections.retain(|peer| peer != leaving);
9451 }
9452 Self::refresh_sender_topology_state(sender_state);
9453 }
9454 Self::recompute_discovery_side_state(route);
9455 }
9456 st.discovery_routes
9457 .retain(|_, route| !route.announcers.is_empty());
9458 if st.discovery_routes != before {
9459 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9460 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9461 }
9462 #[cfg(feature = "compression")]
9463 {
9464 drop(st);
9465 self.note_payload_codec_packet(side, pkt, true);
9466 }
9467 return Ok(true);
9468 }
9469 if let Some(update) = decoded_topology.as_mut() {
9471 for board in update.boards.iter_mut() {
9472 board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
9473 for peer in board.connections.iter_mut() {
9474 *peer = Self::canonical_sender_locked(&st, peer);
9475 }
9476 }
9477 }
9478 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
9479 let alias_state = if packet_sender != pkt.sender() {
9480 route.announcers.remove(pkt.sender())
9481 } else {
9482 None
9483 };
9484 let side_link_local_enabled = st
9485 .sides
9486 .get(side)
9487 .and_then(|entry| entry.as_ref())
9488 .map(|side_ref| side_ref.opts.link_local_enabled)
9489 .unwrap_or(false);
9490 let mut sender_state = route
9491 .announcers
9492 .get_mut(&packet_sender)
9493 .map(core::mem::take)
9494 .or(alias_state)
9495 .unwrap_or_default();
9496 let changed = match pkt.data_type() {
9497 DataType::DiscoveryAnnounce => {
9498 let mut reachable = announce.take().expect("decoded above");
9499 if reachable.is_empty() {
9504 false
9505 } else {
9506 if !side_link_local_enabled {
9507 reachable.retain(|ep| !ep.is_link_local_only());
9508 }
9509 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
9510 let changed = board.reachable_endpoints != reachable;
9511 board.reachable_endpoints = reachable;
9512 Self::refresh_sender_topology_state(&mut sender_state);
9513 changed
9514 }
9515 }
9516 DataType::DiscoveryTimeSyncSources => {
9517 let sources = time_sources.take().expect("decoded above");
9518 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
9519 let changed = board.reachable_timesync_sources != sources;
9520 board.reachable_timesync_sources = sources;
9521 Self::refresh_sender_topology_state(&mut sender_state);
9522 changed
9523 }
9524 DataType::DiscoveryTopology => {
9525 let mut update = decoded_topology
9526 .take()
9527 .expect("topology packet was decoded before route selection");
9528 if !side_link_local_enabled {
9529 for board in update.boards.iter_mut() {
9530 board
9531 .reachable_endpoints
9532 .retain(|ep| !ep.is_link_local_only());
9533 }
9534 }
9535 if !update.incremental {
9536 sender_state.has_full_topology = true;
9537 }
9538 let changed = discovery::apply_shared_topology_update(
9539 &mut sender_state.topology_boards,
9540 update,
9541 );
9542 Self::refresh_sender_topology_state(&mut sender_state);
9543 changed
9544 }
9545 DataType::DiscoverySchema => false,
9546 _ => false,
9547 };
9548 sender_state.last_seen_ms = now_ms;
9549 route.announcers.insert(packet_sender, sender_state);
9550 Self::recompute_discovery_side_state(&mut route);
9551 st.commit_discovery_route(side, route)?;
9552 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9553 if changed {
9554 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9555 }
9556 Ok(true)
9557 }
9558
9559 #[cfg(not(feature = "discovery"))]
9560 fn learn_discovery_packet(
9561 &self,
9562 _pkt: &Packet,
9563 _src: Option<RouterSideId>,
9564 _called_from_queue: bool,
9565 ) -> TelemetryResult<bool> {
9566 Ok(false)
9567 }
9568
9569 #[inline]
9570 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
9571 (side, ty.as_u32())
9572 }
9573
9574 fn reliable_tx_state_mut<'a>(
9575 &'a self,
9576 st: &'a mut RouterInner,
9577 side: RouterSideId,
9578 ty: DataType,
9579 ) -> &'a mut ReliableTxState {
9580 let key = Self::reliable_key(side, ty);
9581 st.reliable_tx
9582 .entry(key)
9583 .or_insert_with(|| ReliableTxState {
9584 next_seq: 1,
9585 sent_order: VecDeque::new(),
9586 sent: Default::default(),
9587 })
9588 }
9589
9590 fn reliable_rx_state_mut<'a>(
9591 &'a self,
9592 st: &'a mut RouterInner,
9593 side: RouterSideId,
9594 ty: DataType,
9595 ) -> &'a mut ReliableRxState {
9596 let key = Self::reliable_key(side, ty);
9597 st.reliable_rx
9598 .entry(key)
9599 .or_insert_with(|| ReliableRxState {
9600 expected_seq: 1,
9601 buffered: Default::default(),
9602 })
9603 }
9604
9605 fn reliable_control_packet(
9606 &self,
9607 control_ty: DataType,
9608 ty: DataType,
9609 seq: u32,
9610 ) -> TelemetryResult<Packet> {
9611 let mut bytes = [0u8; 8];
9612 bytes[..4].copy_from_slice(&ty.as_u32().to_le_bytes());
9613 bytes[4..].copy_from_slice(&seq.to_le_bytes());
9614 Packet::new_local(
9615 control_ty,
9616 message_meta(control_ty).endpoints_owned(),
9617 self.sender_arc(),
9618 self.packet_timestamp_ms(),
9619 crate::config::StandardSmallPayload::new(&bytes),
9620 )
9621 }
9622
9623 fn queue_reliable_ack(
9624 &self,
9625 side: RouterSideId,
9626 ty: DataType,
9627 seq: u32,
9628 called_from_queue: bool,
9629 ) -> TelemetryResult<()> {
9630 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
9631 self.emit_internal_tx_with_priority(
9632 RouterTxItem::ToSide {
9633 src: None,
9634 dst: side,
9635 data: RouterItem::Packet(pkt),
9636 },
9637 true,
9638 message_priority(DataType::ReliableAck),
9639 called_from_queue,
9640 )
9641 }
9642
9643 fn queue_reliable_packet_request(
9644 &self,
9645 side: RouterSideId,
9646 ty: DataType,
9647 seq: u32,
9648 called_from_queue: bool,
9649 ) -> TelemetryResult<()> {
9650 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
9651 self.emit_internal_tx_with_priority(
9652 RouterTxItem::ToSide {
9653 src: None,
9654 dst: side,
9655 data: RouterItem::Packet(pkt),
9656 },
9657 true,
9658 message_priority(DataType::ReliablePacketRequest),
9659 called_from_queue,
9660 )
9661 }
9662
9663 fn queue_reliable_partial_ack(
9664 &self,
9665 side: RouterSideId,
9666 ty: DataType,
9667 seq: u32,
9668 called_from_queue: bool,
9669 ) -> TelemetryResult<()> {
9670 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
9671 self.emit_internal_tx_with_priority(
9672 RouterTxItem::ToSide {
9673 src: None,
9674 dst: side,
9675 data: RouterItem::Packet(pkt),
9676 },
9677 true,
9678 message_priority(DataType::ReliablePartialAck),
9679 called_from_queue,
9680 )
9681 }
9682
9683 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
9684 let mut st = self.state.lock();
9685 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
9686 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
9687 tx_state.sent.remove(&ack);
9688 tx_state.sent_order.retain(|seq| *seq != ack);
9689 return;
9690 }
9691
9692 while let Some(seq) = tx_state.sent_order.front().copied() {
9693 if seq > ack {
9694 break;
9695 }
9696 tx_state.sent_order.pop_front();
9697 tx_state.sent.remove(&seq);
9698 }
9699 }
9700
9701 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
9702 let mut st = self.state.lock();
9703 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
9704 if let Some(sent) = tx_state.sent.get_mut(&seq) {
9705 sent.partial_acked = true;
9706 }
9707 }
9708
9709 fn queue_reliable_retransmit(
9710 &self,
9711 side: RouterSideId,
9712 ty: DataType,
9713 seq: u32,
9714 called_from_queue: bool,
9715 ) -> TelemetryResult<()> {
9716 let mut queued = None;
9717 {
9718 let mut st = self.state.lock();
9719 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
9720 if let Some(sent) = tx_state.sent.get_mut(&seq)
9721 && !sent.queued
9722 {
9723 sent.partial_acked = false;
9724 queued = Some(sent.bytes.clone());
9725 }
9726 }
9727
9728 if let Some(bytes) = queued {
9729 let now = self.clock.now_ms();
9730 if called_from_queue {
9731 self.tx_queue_item_with_priority(
9732 RouterTxItem::ReliableReplay {
9733 dst: side,
9734 ty,
9735 seq,
9736 bytes,
9737 },
9738 true,
9739 Self::router_item_priority_bumped(ty),
9740 )?;
9741 } else {
9742 self.tx_item_impl(
9743 RouterTxItem::ReliableReplay {
9744 dst: side,
9745 ty,
9746 seq,
9747 bytes,
9748 },
9749 true,
9750 false,
9751 )?;
9752 }
9753 let mut st = self.state.lock();
9754 if let Some(sent) = st
9755 .reliable_tx
9756 .get_mut(&Self::reliable_key(side, ty))
9757 .and_then(|tx| tx.sent.get_mut(&seq))
9758 {
9759 sent.last_send_ms = sent.last_send_ms.max(now);
9760 }
9761 }
9762
9763 Ok(())
9764 }
9765
9766 fn send_reliable_raw_to_side(
9767 &self,
9768 side: RouterSideId,
9769 bytes: crate::SharedBytes,
9770 relayed: bool,
9771 admitted: &mut bool,
9772 ) -> TelemetryResult<()> {
9773 let handler = {
9774 let st = self.state.lock();
9775 let side_ref = Self::side_ref(&st, side)?;
9776 if !side_ref.opts.egress_enabled {
9777 return Ok(());
9778 }
9779 (side_ref.tx_handler.clone(), side_ref.opts.clone())
9780 };
9781
9782 let (handler, opts) = handler;
9783
9784 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
9785 return Err(TelemetryError::Io("side tx busy"));
9786 };
9787 let started_ms = self.clock.now_ms();
9788 let (ty, priority) = Self::item_type_priority(&RouterItem::Packed(bytes.clone()))?;
9789 let priority = priority.max(self.effective_transport_priority(ty));
9790 let result = match handler {
9791 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
9792 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
9793 let mut attempts_total = 0usize;
9794 let mut sent_bytes = 0usize;
9795 self.send_side_frames_resumable(side, frames, |frame| {
9796 match self.retry_with_attempts(runtime_max_handler_retries(), || {
9797 Self::packed_handler_call(&packed, frame, priority)
9798 }) {
9799 Ok((_, attempts)) => {
9800 *admitted = true;
9801 attempts_total = attempts_total.saturating_add(attempts);
9802 sent_bytes = sent_bytes.saturating_add(frame.len());
9803 }
9804 Err((err, attempts)) => {
9805 self.note_side_tx_failure(
9806 side,
9807 ty,
9808 attempts_total.saturating_add(attempts),
9809 );
9810 return Err(err);
9811 }
9812 }
9813
9814 Ok(())
9815 })?;
9816 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
9817 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
9818 return Ok(());
9819 }
9820 RouterTxHandlerFn::Packet(f) => {
9821 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
9822 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
9823 }
9824 };
9825 match result {
9826 Ok((_, attempts)) => {
9827 *admitted = true;
9828 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
9829 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
9830 Ok(())
9831 }
9832 Err((err, attempts)) => {
9833 self.note_side_tx_failure(side, ty, attempts);
9834 Err(err)
9835 }
9836 }
9837 }
9838
9839 #[allow(unused_mut)]
9840 #[cfg(feature = "compression")]
9841 #[cfg(test)]
9842 fn send_queued_to_side(
9843 &self,
9844 side: RouterSideId,
9845 data: &mut RouterItem,
9846 relayed: bool,
9847 ) -> (
9848 TelemetryResult<()>,
9849 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
9850 ) {
9851 self.send_queued_to_side_with_progress(side, data, relayed, &mut false)
9852 }
9853
9854 #[cfg(feature = "compression")]
9855 fn send_queued_to_side_with_progress(
9856 &self,
9857 side: RouterSideId,
9858 data: &mut RouterItem,
9859 relayed: bool,
9860 admitted: &mut bool,
9861 ) -> (
9862 TelemetryResult<()>,
9863 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
9864 ) {
9865 let reservation = match self.prepare_queued_payload(side, data) {
9866 Ok(reservation) => reservation,
9867 Err(error) => return (Err(error), None),
9868 };
9869 (
9870 self.send_reliable_to_side_with_progress(side, data.clone(), relayed, admitted),
9871 reservation,
9872 )
9873 }
9874
9875 #[cfg(test)]
9876 fn send_reliable_to_side(
9877 &self,
9878 side: RouterSideId,
9879 data: RouterItem,
9880 relayed: bool,
9881 ) -> TelemetryResult<()> {
9882 self.send_reliable_to_side_with_progress(side, data, relayed, &mut false)
9883 }
9884
9885 fn send_reliable_to_side_with_progress(
9886 &self,
9887 side: RouterSideId,
9888 data: RouterItem,
9889 relayed: bool,
9890 admitted: &mut bool,
9891 ) -> TelemetryResult<()> {
9892 #[cfg(feature = "discovery")]
9893 let schema_id = (Self::item_data_type(&data).ok() == Some(DataType::DiscoverySchema))
9894 .then(|| Self::get_hash(&data));
9895 #[cfg(feature = "discovery")]
9896 if let Some(id) = schema_id {
9897 let mut st = self.state.lock();
9898 if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
9899 && throttle.schema_response_active
9900 && throttle.schema_response_phase == SchemaResponsePhase::Owned
9901 && throttle.schema_response_packet_id == id
9902 && !throttle.schema_response_started
9903 {
9904 throttle.schema_response_started = true;
9905 throttle.next_schema_response_ms = self
9906 .clock
9907 .now_ms()
9908 .saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
9909 if let Some(transport) = st.side_transport.get_mut(&side) {
9914 transport.refresh_transmit_header_templates();
9915 }
9916 }
9917 }
9918 let result = self.send_reliable_to_side_impl(side, data, relayed, admitted);
9919 #[cfg(feature = "discovery")]
9920 if let Some(id) = schema_id {
9921 let mut st = self.state.lock();
9922 if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
9923 && throttle.schema_response_active
9924 && throttle.schema_response_phase == SchemaResponsePhase::Owned
9925 && throttle.schema_response_packet_id == id
9926 {
9927 if !result.as_ref().is_err_and(Self::is_transient_tx_error) {
9930 throttle.schema_response_active = false;
9931 throttle.schema_response_phase = SchemaResponsePhase::Idle;
9932 throttle.schema_response_pending |= result.is_err();
9933 }
9934 }
9935 }
9936 result
9937 }
9938
9939 fn send_reliable_to_side_impl(
9940 &self,
9941 side: RouterSideId,
9942 data: RouterItem,
9943 relayed: bool,
9944 admitted: &mut bool,
9945 ) -> TelemetryResult<()> {
9946 let (ty, addressed) = match &data {
9949 RouterItem::Packet(pkt) => (pkt.data_type(), !pkt.wire_target_senders().is_empty()),
9950 RouterItem::Packed(bytes) => {
9951 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
9952 (env.ty, env.target_senders().len() != 0)
9953 }
9954 };
9955 let end_to_end_ordered =
9956 addressed && matches!(reliable_mode(ty), crate::ReliableMode::Ordered);
9957 let (handler, opts, hop_reliable_enabled) = {
9958 let st = self.state.lock();
9959 let side_ref = Self::side_ref(&st, side)?;
9960 let opts = side_ref.opts.clone();
9961 let hop_reliable_enabled = opts.reliable_enabled
9962 && !end_to_end_ordered
9963 && self.cfg.reliable_enabled()
9964 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
9965 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
9966 };
9967
9968 if !matches!(
9969 &handler,
9970 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
9971 ) {
9972 return self
9973 .call_side_tx_handler_with_progress(side, &handler, &data, relayed, admitted);
9974 }
9975
9976 if !hop_reliable_enabled {
9977 let mut adjusted_opts = opts;
9978 adjusted_opts.reliable_enabled = false;
9979 let preserve_end_to_end_ack = true;
9984 if let Some(adjusted) =
9985 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
9986 {
9987 return self.call_side_tx_handler_with_progress(
9988 side, &handler, &adjusted, relayed, admitted,
9989 );
9990 }
9991 return Ok(());
9992 }
9993
9994 if let RouterItem::Packed(bytes) = &data {
9995 wire_format::peek_routing_frame_info(bytes.as_ref())?;
9996 }
9997 let priority = self.item_service_priority(&data)?;
9998
9999 if !is_reliable_type(ty) {
10000 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
10001 self.call_side_tx_handler_with_progress(
10002 side, &handler, &adjusted, relayed, admitted,
10003 )?;
10004 }
10005 return Ok(());
10006 }
10007
10008 #[cfg(feature = "compression")]
10009 let data = match data {
10010 RouterItem::Packed(bytes) => {
10011 let budget = self.state.lock().memory.max_queue_budget;
10012 let normalized =
10013 wire_format::payload_encoding_for_retention(bytes.as_ref(), budget)?;
10014 RouterItem::Packed(
10015 normalized
10016 .map(crate::shared_bytes::convert)
10017 .unwrap_or(bytes),
10018 )
10019 }
10020 packet => packet,
10021 };
10022 let (seq, flags) = {
10023 let mut st = self.state.lock();
10024 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10025 if tx_state.sent.len() >= runtime_reliable_max_pending() {
10026 return Err(TelemetryError::Io("reliable window full"));
10027 }
10028 let seq = tx_state.next_seq;
10029 let next = tx_state.next_seq.wrapping_add(1);
10030 tx_state.next_seq = if next == 0 { 1 } else { next };
10031 let flags = match reliable_mode(ty) {
10032 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
10033 _ => 0,
10034 };
10035 (seq, flags)
10036 };
10037
10038 let bytes: crate::SharedBytes = match data {
10039 RouterItem::Packet(pkt) => self.pack_packet_for_router(
10040 &pkt,
10041 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
10042 )?,
10043 RouterItem::Packed(bytes) => {
10044 #[cfg(feature = "cryptography")]
10045 if self.e2e_seal_config_for_type(ty).is_some() {
10046 self.prepare_packed_for_remote(
10047 bytes,
10048 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
10049 )?
10050 } else {
10051 let Some(rewritten) =
10052 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
10053 else {
10054 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10055 return Err(TelemetryError::Io("side tx busy"));
10056 };
10057 let started_ms = self.clock.now_ms();
10058 let frames =
10059 self.encode_side_transport_frames(side, opts, bytes.clone())?;
10060 let mut attempts_total = 0usize;
10061 let mut sent_bytes = 0usize;
10062 self.send_side_frames_resumable(side, frames, |frame| {
10063 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10064 Self::packed_handler_call(&handler, frame, priority)
10065 }) {
10066 Ok((_, attempts)) => {
10067 *admitted = true;
10068 attempts_total = attempts_total.saturating_add(attempts);
10069 sent_bytes = sent_bytes.saturating_add(frame.len());
10070 }
10071 Err((err, attempts)) => {
10072 self.note_side_tx_failure(
10073 side,
10074 ty,
10075 attempts_total.saturating_add(attempts),
10076 );
10077 return Err(err);
10078 }
10079 }
10080
10081 Ok(())
10082 })?;
10083 self.record_side_tx_sample(
10084 side,
10085 sent_bytes,
10086 started_ms,
10087 self.clock.now_ms(),
10088 );
10089 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10090 return Ok(());
10091 };
10092 crate::shared_bytes::convert(rewritten)
10093 }
10094 #[cfg(not(feature = "cryptography"))]
10095 {
10096 let Some(rewritten) =
10097 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
10098 else {
10099 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10100 return Err(TelemetryError::Io("side tx busy"));
10101 };
10102 let started_ms = self.clock.now_ms();
10103 let frames =
10104 self.encode_side_transport_frames(side, opts, bytes.clone())?;
10105 let mut attempts_total = 0usize;
10106 let mut sent_bytes = 0usize;
10107 self.send_side_frames_resumable(side, frames, |frame| {
10108 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10109 Self::packed_handler_call(&handler, frame, priority)
10110 }) {
10111 Ok((_, attempts)) => {
10112 *admitted = true;
10113 attempts_total = attempts_total.saturating_add(attempts);
10114 sent_bytes = sent_bytes.saturating_add(frame.len());
10115 }
10116 Err((err, attempts)) => {
10117 self.note_side_tx_failure(
10118 side,
10119 ty,
10120 attempts_total.saturating_add(attempts),
10121 );
10122 return Err(err);
10123 }
10124 }
10125
10126 Ok(())
10127 })?;
10128 self.record_side_tx_sample(
10129 side,
10130 sent_bytes,
10131 started_ms,
10132 self.clock.now_ms(),
10133 );
10134 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10135 return Ok(());
10136 };
10137 crate::shared_bytes::convert(rewritten)
10138 }
10139 }
10140 };
10141
10142 #[cfg(feature = "compact-packet-store")]
10143 let mut bytes = bytes;
10144
10145 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10146 return Err(TelemetryError::Io("side tx busy"));
10147 };
10148 let started_ms = self.clock.now_ms();
10149 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
10150 let mut attempts_total = 0usize;
10151 let mut sent_bytes = 0usize;
10152 self.send_side_frames_resumable(side, frames, |frame| {
10153 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10154 Self::packed_handler_call(&handler, frame, priority)
10155 }) {
10156 Ok((_, attempts)) => {
10157 *admitted = true;
10158 attempts_total = attempts_total.saturating_add(attempts);
10159 sent_bytes = sent_bytes.saturating_add(frame.len());
10160 }
10161 Err((err, attempts)) => {
10162 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
10163 return Err(err);
10164 }
10165 }
10166
10167 Ok(())
10168 })?;
10169 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10170 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10171
10172 #[cfg(feature = "compact-packet-store")]
10173 {
10174 let _ = bytes.park_for_queue();
10175 }
10176
10177 {
10178 let mut st = self.state.lock();
10179 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10180 tx_state.sent_order.push_back(seq);
10181 tx_state.sent.insert(
10182 seq,
10183 ReliableSent {
10184 bytes: bytes.clone(),
10185 last_send_ms: self.clock.now_ms(),
10186 retries: 0,
10187 queued: false,
10188 partial_acked: false,
10189 },
10190 );
10191 }
10192
10193 Ok(())
10194 }
10195
10196 fn run_side_chunk_transfer<T>(
10197 &self,
10198 side: RouterSideId,
10199 key: u64,
10200 send: impl FnOnce(&mut usize) -> TelemetryResult<T>,
10201 ) -> TelemetryResult<T> {
10202 let mut next = {
10203 let mut st = self.state.lock();
10204 let transport = st
10205 .side_transport
10206 .get_mut(&side)
10207 .ok_or(TelemetryError::BadArg)?;
10208 if transport.tx_chunk_progress.is_none() {
10209 crate::memory_admission::check(
10210 512,
10211 core::mem::size_of::<crate::side_transport::SideTxProgress>(),
10212 )?;
10213 transport.tx_chunk_progress =
10214 Some(Box::new(crate::side_transport::SideTxProgress::default()));
10215 }
10216 transport
10217 .tx_chunk_progress
10218 .as_mut()
10219 .expect("initialized")
10220 .begin(key, self.clock.now_ms())?
10221 };
10222 let result = send(&mut next);
10223 if let Some(progress) = self
10224 .state
10225 .lock()
10226 .side_transport
10227 .get_mut(&side)
10228 .and_then(|transport| transport.tx_chunk_progress.as_mut())
10229 {
10230 progress.finish(key, next, self.clock.now_ms(), &result);
10231 }
10232 result
10233 }
10234
10235 fn send_side_frames_resumable(
10236 &self,
10237 side: RouterSideId,
10238 mut frames: SideTransportFrames,
10239 mut send: impl FnMut(&[u8]) -> TelemetryResult<()>,
10240 ) -> TelemetryResult<()> {
10241 let bootstraps_template = frames.bootstraps_template();
10242 let result = if frames.len() == 1 {
10243 frames.send_resuming(&mut 0, send)
10244 } else {
10245 let key = frames.progress_key();
10246 self.run_side_chunk_transfer(side, key, |next| frames.send_resuming(next, &mut send))
10247 };
10248 if result.is_err() && bootstraps_template {
10249 if let Some(state) = self.state.lock().side_transport.get_mut(&side) {
10254 state.refresh_transmit_header_templates();
10255 }
10256 }
10257 result
10258 }
10259
10260 fn split_side_transport_frame(
10261 &self,
10262 _side: RouterSideId,
10263 frame: crate::SharedBytes,
10264 max_frame_bytes: usize,
10265 ) -> TelemetryResult<SideTransportFrames> {
10266 SideTransportFrames::split(frame, max_frame_bytes, self.sender_arc().as_bytes())
10267 }
10268
10269 #[cfg(all(feature = "compression", feature = "discovery"))]
10270 fn note_payload_codec_packet(&self, side: RouterSideId, pkt: &Packet, accepted: bool) {
10271 let now = self.clock.now_ms();
10272 let mut st = self.state.lock();
10273 if !st
10274 .sides
10275 .get(side)
10276 .and_then(|entry| entry.as_ref())
10277 .is_some_and(|entry| {
10278 matches!(
10279 entry.tx_handler,
10280 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
10281 )
10282 })
10283 {
10284 return;
10285 }
10286 if accepted && pkt.data_type() == crate::DataType::DiscoveryLeave {
10287 for transport in st.side_transport.values_mut() {
10288 let _ = transport.payload_codec.discovery_packet(
10289 pkt,
10290 true,
10291 now,
10292 DISCOVERY_ROUTE_TTL_MS,
10293 );
10294 }
10295 } else if let Some(transport) = st.side_transport.get_mut(&side) {
10296 let _ = transport.payload_codec.discovery_packet(
10297 pkt,
10298 accepted,
10299 now,
10300 DISCOVERY_ROUTE_TTL_MS,
10301 );
10302 }
10303 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
10304 && let Some(transport) = st.side_transport.get_mut(&side)
10305 {
10306 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
10307 }
10308 }
10309 #[cfg(all(feature = "compression", feature = "discovery"))]
10310 fn note_payload_codec_address(&self, side: RouterSideId, payload: &[u8], accepted: bool) {
10311 let now = self.clock.now_ms();
10312 let mut st = self.state.lock();
10313 if !st
10314 .sides
10315 .get(side)
10316 .and_then(|entry| entry.as_ref())
10317 .is_some_and(|entry| {
10318 matches!(
10319 entry.tx_handler,
10320 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
10321 )
10322 })
10323 {
10324 return;
10325 }
10326 if let Some(transport) = st.side_transport.get_mut(&side) {
10327 let _ = transport.payload_codec.address_payload(
10328 payload,
10329 accepted,
10330 now,
10331 DISCOVERY_ROUTE_TTL_MS,
10332 );
10333 }
10334 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
10335 && let Some(transport) = st.side_transport.get_mut(&side)
10336 {
10337 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
10338 }
10339 }
10340
10341 #[cfg(feature = "compression")]
10342 fn payload_for_transport_side(
10343 &self,
10344 side: RouterSideId,
10345 raw: crate::SharedBytes,
10346 ) -> TelemetryResult<crate::SharedBytes> {
10347 #[cfg(not(feature = "discovery"))]
10348 let _ = side;
10349 let (allowed, budget) = {
10350 #[cfg(feature = "discovery")]
10351 let mut st = self.state.lock();
10352 #[cfg(not(feature = "discovery"))]
10353 let st = self.state.lock();
10354 let budget = st.memory.max_queue_budget;
10355 #[cfg(feature = "discovery")]
10356 let allowed = {
10357 let transport = st
10358 .side_transport
10359 .get_mut(&side)
10360 .ok_or(TelemetryError::BadArg)?;
10361 let generation = transport.payload_codec.generation();
10362 if transport.payload_codec_choice != generation {
10363 transport.payload_codec_choice = generation;
10364 transport.tx_chunk_progress = None;
10365 transport.refresh_transmit_header_templates();
10366 }
10367 transport.payload_codec.allowed(self.clock.now_ms())
10368 };
10369 #[cfg(not(feature = "discovery"))]
10370 let allowed = false;
10371 (allowed, budget)
10372 };
10373 Ok(
10374 match wire_format::payload_encoding_for_side(raw.as_ref(), allowed, budget)? {
10375 Some(frame) => crate::shared_bytes::convert(frame),
10376 None => raw,
10377 },
10378 )
10379 }
10380
10381 #[cfg(feature = "compression")]
10385 fn prepare_queued_payload(
10386 &self,
10387 side: RouterSideId,
10388 data: &mut RouterItem,
10389 ) -> TelemetryResult<Option<crate::payload_codec::QueueReservation<'_, RouterInner>>> {
10390 let RouterItem::Packed(bytes) = data else {
10391 return Ok(None);
10392 };
10393 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
10394 return Ok(None);
10395 }
10396 let capable = {
10397 #[cfg(feature = "discovery")]
10398 {
10399 self.state
10400 .lock()
10401 .side_transport
10402 .get(&side)
10403 .is_some_and(|transport| transport.payload_codec.allowed(self.clock.now_ms()))
10404 }
10405 #[cfg(not(feature = "discovery"))]
10406 {
10407 false
10408 }
10409 };
10410 if capable {
10411 return Ok(None);
10412 }
10413 let required = wire_format::decoded_frame_upper_bound(bytes.as_ref())?
10414 .checked_add(
10415 size_of::<crate::SharedBytes>()
10416 + size_of::<bool>()
10417 + size_of::<u8>()
10418 + size_of::<u64>(),
10419 )
10420 .ok_or(TelemetryError::Io("memory pressure"))?;
10421 let reservation = crate::payload_codec::QueueReservation::acquire(&self.state, required)?;
10422 *bytes = self.payload_for_transport_side(side, bytes.clone())?;
10423 Ok(Some(reservation))
10424 }
10425
10426 fn encode_side_transport_frames(
10427 &self,
10428 side: RouterSideId,
10429 opts: RouterSideOptions,
10430 raw: crate::SharedBytes,
10431 ) -> TelemetryResult<SideTransportFrames> {
10432 #[cfg(feature = "compression")]
10433 let raw = self.payload_for_transport_side(side, raw)?;
10434 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
10435 return Ok(SideTransportFrames::single(raw));
10436 }
10437
10438 let raw_len = raw.len();
10439 if opts.max_frame_bytes != 0 && raw_len > opts.max_frame_bytes {
10440 let frames = self.split_side_transport_frame(side, raw, opts.max_frame_bytes)?;
10443 let mut st = self.state.lock();
10444 let stats = st.side_runtime_stats.entry(side).or_default();
10445 stats.note_side_transport_full(raw_len, frames.wire_len());
10446 stats.note_side_transport_chunks(frames.len());
10447 return Ok(frames);
10448 }
10449
10450 if opts.header_template_enabled {
10451 let frame = wire_format::peek_routing_frame_info(raw.as_ref())?;
10455 let without_dictionary = opts.max_side_transport_templates == 0 || {
10456 let st = self.state.lock();
10457 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
10458 };
10459 let reliable = is_reliable_type(frame.envelope.ty)
10460 || frame.reliable.is_some()
10461 || frame.envelope.target_senders().len() != 0
10467 || matches!(
10468 frame.envelope.ty,
10469 crate::DataType::ReliableAck
10470 | crate::DataType::ReliablePartialAck
10471 | crate::DataType::ReliablePacketRequest
10472 );
10473 if without_dictionary || reliable {
10474 let mut st = self.state.lock();
10475 if without_dictionary {
10476 let removed = st
10477 .side_transport
10478 .get_mut(&side)
10479 .ok_or(TelemetryError::BadArg)?
10480 .trim_tx_templates_to(0);
10481 for _ in 0..removed {
10482 st.side_runtime_stats
10483 .entry(side)
10484 .or_default()
10485 .note_side_transport_template_eviction();
10486 }
10487 }
10488 st.side_runtime_stats
10489 .entry(side)
10490 .or_default()
10491 .note_side_transport_full(raw_len, raw_len);
10492 return Ok(SideTransportFrames::single(raw));
10493 }
10494 }
10495
10496 if opts.header_template_enabled && crate::memory_admission::supports_work_kinds() {
10497 let (template, ..) = extract_side_header_template_view(raw.as_ref())?;
10498 let cached = self
10499 .state
10500 .lock()
10501 .side_transport
10502 .get(&side)
10503 .is_some_and(|transport| {
10504 transport.tx_template_ids.contains_key(&template.hash)
10505 && transport.tx_templates.contains_key(&template.hash)
10506 });
10507 if !cached
10508 && matches!(
10509 crate::memory_admission::check_receive(raw.as_ref()),
10510 Err(TelemetryError::Io("memory pressure"))
10511 )
10512 {
10513 self.state
10517 .lock()
10518 .side_runtime_stats
10519 .entry(side)
10520 .or_default()
10521 .note_side_transport_full(raw_len, raw_len);
10522 return Ok(SideTransportFrames::single(raw));
10523 }
10524 }
10525
10526 let mut compact_payload_len = None;
10527 let mut used_compact = false;
10528 let used_timestamp_delta = false;
10529 let omitted_timestamp = false;
10530 let wrapped = if opts.header_template_enabled {
10531 let (template, _ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
10532 extract_side_header_template_view(raw.as_ref())?;
10533 let (template_id, use_compact, _previous_timestamp) = {
10536 let mut st = self.state.lock();
10537 #[cfg(feature = "discovery")]
10542 let mut effective_template_limit = opts.max_side_transport_templates;
10543 #[cfg(not(feature = "discovery"))]
10544 let effective_template_limit = opts.max_side_transport_templates;
10545 #[cfg(feature = "discovery")]
10546 {
10547 if let Some(route) = st.discovery_routes.get(&side) {
10548 let active_peers = route
10549 .announcers
10550 .values()
10551 .filter(|sender| {
10552 self.clock.now_ms().saturating_sub(sender.last_seen_ms)
10553 <= DISCOVERY_ROUTE_TTL_MS
10554 })
10555 .count();
10556 if active_peers > 1 {
10564 effective_template_limit = 0;
10565 } else {
10566 effective_template_limit = route
10567 .announcers
10568 .values()
10569 .filter_map(|sender| sender.link_capabilities)
10570 .filter(|caps| {
10571 caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
10572 && caps.max_side_transport_templates > 0
10573 })
10574 .map(|caps| caps.max_side_transport_templates as usize)
10575 .fold(effective_template_limit, core::cmp::min);
10576 }
10577 }
10578 }
10579 let trimmed = st
10580 .side_transport
10581 .get_mut(&side)
10582 .ok_or(TelemetryError::BadArg)?
10583 .trim_tx_templates_to(effective_template_limit);
10584 for _ in 0..trimmed {
10585 st.side_runtime_stats
10586 .entry(side)
10587 .or_default()
10588 .note_side_transport_template_eviction();
10589 }
10590 let side_state = st
10591 .side_transport
10592 .get_mut(&side)
10593 .ok_or(TelemetryError::BadArg)?;
10594 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
10595 let previous = side_state.tx_last_timestamps.get(&id).copied();
10596 const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
10599 let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
10600 #[cfg(feature = "discovery")]
10602 let discovery_refresh = discovery::is_discovery_type(_ty);
10603 #[cfg(not(feature = "discovery"))]
10604 let discovery_refresh = false;
10605 let use_compact =
10606 *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES && !discovery_refresh;
10607 if use_compact {
10608 *compact_uses = compact_uses.saturating_add(1);
10609 } else {
10610 *compact_uses = 0;
10611 }
10612 (id, use_compact, previous)
10613 } else {
10614 let template_id = ((template.hash >> 32) as u32 ^ template.hash as u32).max(1);
10620 let evicted = side_state.insert_tx_template(
10621 template.into_owned(),
10622 template_id,
10623 effective_template_limit,
10624 );
10625 if evicted {
10626 st.side_runtime_stats
10627 .entry(side)
10628 .or_default()
10629 .note_side_transport_template_eviction();
10630 }
10631 if let Some(side_state) = st.side_transport.get_mut(&side)
10632 && effective_template_limit > 0
10633 {
10634 side_state.tx_last_timestamps.insert(template_id, timestamp);
10635 }
10636 (template_id, false, None)
10637 }
10638 };
10639 if use_compact {
10640 used_compact = true;
10641 compact_payload_len = Some(payload.len());
10642 let timestamp_field = Some(timestamp);
10645 let mut body = Vec::with_capacity(payload.len() + 32);
10646 body.push(flags);
10647 write_uleb128_local(u64::from(template_id), &mut body);
10648 write_uleb128_local(data_size, &mut body);
10649 if let Some(timestamp_field) = timestamp_field {
10650 write_uleb128_local(timestamp_field, &mut body);
10651 }
10652 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
10653 write_uleb128_local(u64::from(nonce), &mut body);
10654 }
10655 if let Some((seq, ack)) = reliable_seq_ack {
10656 write_uleb128_local(u64::from(seq), &mut body);
10657 write_uleb128_local(u64::from(ack), &mut body);
10658 }
10659 body.extend_from_slice(payload);
10660 {
10661 let mut st = self.state.lock();
10662 if let Some(side_state) = st.side_transport.get_mut(&side) {
10663 side_state.tx_last_timestamps.insert(template_id, timestamp);
10664 }
10665 }
10666 let kind = if omitted_timestamp {
10667 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
10668 } else if used_timestamp_delta {
10669 SIDE_TRANSPORT_KIND_COMPACT_DELTA
10670 } else {
10671 SIDE_TRANSPORT_KIND_COMPACT
10672 };
10673 wrap_side_transport_frame(kind, &body)
10674 } else {
10675 let mut body = Vec::with_capacity(raw.len() + 4);
10676 write_uleb128_local(u64::from(template_id), &mut body);
10677 body.extend_from_slice(raw.as_ref());
10678 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
10679 }
10680 } else {
10681 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
10682 };
10683
10684 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
10685 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
10686 } else {
10687 Ok(SideTransportFrames::single(wrapped))
10688 }?;
10689 let wire_len = frames.wire_len();
10690 let mut st = self.state.lock();
10691 let stats = st.side_runtime_stats.entry(side).or_default();
10692 if used_compact {
10693 let overhead = compact_payload_len
10694 .map(|payload_len| wire_len.saturating_sub(payload_len))
10695 .unwrap_or(wire_len);
10696 stats.note_side_transport_compact(
10697 raw_len,
10698 wire_len,
10699 overhead,
10700 used_timestamp_delta,
10701 omitted_timestamp,
10702 );
10703 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
10704 {
10705 stats.note_side_transport_compact_target_miss();
10706 }
10707 } else {
10708 stats.note_side_transport_full(raw_len, wire_len);
10709 }
10710 if frames.len() > 1 {
10711 stats.note_side_transport_chunks(frames.len());
10712 }
10713 Ok(frames)
10714 }
10715
10716 fn expire_side_chunks(&self) {
10717 let now = self.clock.now_ms();
10718 let mut st = self.state.lock();
10719 for side in st.side_transport.values_mut() {
10720 let side = side.as_mut();
10721 #[cfg(all(feature = "compression", feature = "discovery"))]
10722 side.payload_codec.expire(now, DISCOVERY_ROUTE_TTL_MS);
10723 let reclaimed =
10724 SideChunkAssembly::expire(&mut side.rx_chunks, &mut side.rx_chunk_bytes, now);
10725 side.rx_chunk_reclaims = side.rx_chunk_reclaims.saturating_add(reclaimed);
10726 }
10727 }
10728
10729 #[cfg(feature = "discovery")]
10733 fn refresh_ingress_peer_liveness<'a>(
10734 &self,
10735 side: RouterSideId,
10736 bytes: &'a [u8],
10737 ) -> Option<wire_format::RoutingFrameInfo<'a>> {
10738 let frame = wire_format::peek_routing_frame_info(bytes).ok()?;
10739 self.refresh_peer_liveness(side, frame.envelope.source_address);
10740 #[cfg(feature = "compression")]
10741 if frame.envelope.ty == crate::DataType::DiscoveryAddress
10742 && let Ok(Some((_, payload))) =
10743 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryAddress)
10744 {
10745 self.note_payload_codec_address(side, payload, false);
10746 }
10747 Some(frame)
10748 }
10749
10750 #[cfg(feature = "discovery")]
10751 #[cfg_attr(not(feature = "std"), inline(never))]
10752 fn refresh_peer_liveness(&self, side: RouterSideId, address: u32) {
10753 let now = self.clock.now_ms();
10754 let mut st = self.state.lock();
10755 #[cfg(feature = "compression")]
10756 if let Some(transport) = st.side_transport.get_mut(&side) {
10757 transport
10758 .payload_codec
10759 .observe(address, now, DISCOVERY_ROUTE_TTL_MS);
10760 }
10761 let Some(route) = st.discovery_routes.get_mut(&side) else {
10762 return;
10763 };
10764 let mut found = None;
10765 for (name, peer) in &mut route.announcers {
10766 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
10767 if found.is_some() {
10769 return;
10770 }
10771 found = Some(peer);
10772 }
10773 }
10774 let Some(peer) = found else {
10775 return;
10776 };
10777 peer.last_seen_ms = now;
10778 route.last_seen_ms = route.last_seen_ms.max(now);
10779 }
10780
10781 #[cfg(feature = "discovery")]
10785 fn consume_known_packed_topology(
10786 &self,
10787 side: RouterSideId,
10788 bytes: &[u8],
10789 ) -> TelemetryResult<bool> {
10790 let Some((address, payload)) =
10791 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryTopology)?
10792 else {
10793 return Ok(false);
10794 };
10795 let now = self.clock.now_ms();
10796 let mut st = self.state.lock();
10797 let Some(route) = st.discovery_routes.get_mut(&side) else {
10798 return Ok(false);
10799 };
10800 let mut found = None;
10801 for (name, peer) in &mut route.announcers {
10802 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
10803 if found.is_some() {
10804 return Ok(false);
10805 }
10806 found = Some(peer);
10807 }
10808 }
10809 let Some(peer) = found else {
10810 return Ok(false);
10811 };
10812 let Some(full) =
10813 discovery::topology_payload_matches_retained(payload, &peer.topology_boards)
10814 else {
10815 return Ok(false);
10816 };
10817 peer.has_full_topology |= full;
10818 peer.last_seen_ms = now;
10819 route.last_seen_ms = route.last_seen_ms.max(now);
10820 Ok(true)
10821 }
10822
10823 #[cfg(all(feature = "discovery", not(feature = "std")))]
10824 fn consume_known_packed_schema(
10825 &self,
10826 side: RouterSideId,
10827 bytes: &[u8],
10828 ) -> TelemetryResult<bool> {
10829 let Some((address, payload)) =
10830 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoverySchema)?
10831 else {
10832 return Ok(false);
10833 };
10834 self.refresh_peer_liveness(side, address);
10835 self.merge_schema_changes(payload)?;
10836 let mut st = self.state.lock();
10837 if let Some(route) = st.discovery_routes.get_mut(&side) {
10838 for (name, peer) in &mut route.announcers {
10839 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
10840 peer.has_schema = true;
10841 break;
10842 }
10843 }
10844 }
10845 Ok(true)
10846 }
10847
10848 #[cfg(all(feature = "compression", feature = "discovery"))]
10849 fn decode_required_schema<T>(
10850 &self,
10851 mut decode: impl FnMut() -> TelemetryResult<T>,
10852 ) -> TelemetryResult<T> {
10853 loop {
10854 match decode() {
10855 Err(error @ TelemetryError::Io("memory pressure")) => {
10856 if !self.state.lock().reclaim_codec_metadata() {
10857 return Err(error);
10858 }
10859 }
10860 result => return result,
10861 }
10862 }
10863 }
10864
10865 #[cfg(all(feature = "discovery", not(feature = "std")))]
10866 #[inline(never)]
10867 fn merge_schema_changes(&self, payload: &[u8]) -> TelemetryResult<()> {
10868 #[cfg(feature = "compression")]
10871 let snapshot = self.decode_required_schema(|| discovery::decode_schema_changes(payload))?;
10872 #[cfg(not(feature = "compression"))]
10873 let snapshot = discovery::decode_schema_changes(payload)?;
10874 let mut st = self.state.lock();
10875 let budget = st.schema_growth_budget();
10876 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
10877 if report.changed() {
10878 st.fit_discovery_budget();
10879 }
10880 Ok(())
10881 }
10882
10883 #[cfg(all(
10888 feature = "discovery",
10889 any(feature = "std", feature = "ingress-target-filter")
10890 ))]
10891 fn targets_only_ingress(
10892 &self,
10893 side: RouterSideId,
10894 frame: &wire_format::RoutingFrameInfo<'_>,
10895 ) -> bool {
10896 let ty = frame.envelope.ty;
10897 if is_internal_control_type(ty)
10900 || frame.envelope.target_senders().len() == 0
10901 || frame.reliable.is_some_and(|h| {
10902 h.flags != wire_format::RELIABLE_FLAG_UNSEQUENCED || h.seq != 0 || h.ack != 0
10903 })
10904 {
10905 return false;
10906 }
10907 let Some(meta) = crate::config::try_get_message_meta(ty) else {
10908 return false;
10909 };
10910 if frame
10911 .envelope
10912 .wire_shape
10913 .is_some_and(|shape| shape != meta.element)
10914 {
10915 return false;
10916 }
10917 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
10918 let st = self.state.lock();
10919 if st.is_managed_variable(ty.as_u32())
10920 || Self::has_explicit_route_policy_locked(&st, Some(side), ty)
10921 {
10922 return false;
10923 }
10924 let now = self.clock.now_ms();
10925 frame.envelope.target_senders().all(|target| {
10926 target != local_hash
10927 && target != u64::from(st.local_address.address)
10928 && Self::side_matches_target_senders_locked(
10929 &st,
10930 side,
10931 wire_format::TargetSenderSet::Hashes(&[target]),
10932 now,
10933 )
10934 && !st.sides.iter().enumerate().any(|(other, entry)| {
10935 other != side
10936 && entry.as_ref().is_some_and(|e| e.opts.egress_enabled)
10937 && Self::side_matches_target_senders_locked(
10938 &st,
10939 other,
10940 wire_format::TargetSenderSet::Hashes(&[target]),
10941 now,
10942 )
10943 })
10944 })
10945 }
10946
10947 #[cfg(feature = "discovery")]
10950 fn cached_transit_ready(
10951 &self,
10952 side: RouterSideId,
10953 bytes: &[u8],
10954 view: Option<&wire_format::RoutingFrameInfo<'_>>,
10955 ) -> TelemetryResult<bool> {
10956 if bytes.len() > 128 || bytes.first().is_none_or(|flags| flags & 0x10 != 0) {
10961 return Ok(false);
10962 }
10963 let parsed;
10964 let frame = match view {
10965 Some(frame) => frame,
10966 None => {
10967 parsed = wire_format::peek_routing_frame_info(bytes)?;
10968 &parsed
10969 }
10970 };
10971 let ty = frame.envelope.ty;
10972 if is_internal_control_type(ty)
10973 || frame.envelope.is_reliable
10974 || frame.reliable.is_some()
10975 || frame.envelope.target_senders().len() != 0
10976 || frame.envelope.wire_shape.is_some_and(|shape| {
10980 shape != crate::message_meta(ty).element
10981 })
10982 || !(1..=3).contains(&frame.envelope.endpoints().take(4).count())
10983 {
10984 return Ok(false);
10985 }
10986 if self.is_managed_variable_type(ty)
10987 || frame.envelope.endpoints().any(|ep| {
10988 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
10989 })
10990 {
10991 return Ok(false);
10992 }
10993
10994 let canonical_possible = {
10998 let st = self.state.lock();
10999 st.discovery_routes.get(&side).is_some_and(|route| {
11000 route.announcers.iter().any(|(name, peer)| {
11001 peer.advertised_address == Some(frame.envelope.source_address)
11002 || sender_address_u32(name) == frame.envelope.source_address
11003 })
11004 }) && st.sides.iter().flatten().all(|entry| {
11005 entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes
11006 })
11007 };
11008 let template_hash = if canonical_possible {
11009 None
11010 } else {
11011 Some(extract_side_header_template_view(bytes)?.0.hash)
11012 };
11013 let st = self.state.lock();
11014 let active: usize = st.sides.iter().filter(|entry| entry.is_some()).count();
11015 if active != 2 {
11016 return Ok(false);
11017 }
11018 for (id, entry) in st
11019 .sides
11020 .iter()
11021 .enumerate()
11022 .filter_map(|(id, entry)| entry.as_ref().map(|e| (id, e)))
11023 {
11024 if entry.opts.reliable_enabled
11025 || !matches!(
11026 entry.tx_handler,
11027 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11028 )
11029 || !st
11030 .side_runtime_stats
11031 .get(&id)
11032 .is_some_and(|stats| stats.data_types.contains_key(&ty.as_u32()))
11033 {
11034 return Ok(false);
11035 }
11036 if id == side {
11037 continue;
11038 }
11039 #[cfg(feature = "discovery")]
11040 if !st.discovery_routes.get(&id).is_some_and(|route| {
11041 self.clock.now_ms().saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
11042 && frame
11043 .envelope
11044 .endpoints()
11045 .any(|ep| route.reachable.contains(&ep))
11046 }) {
11047 return Ok(false);
11048 }
11049 if entry.opts.header_template_enabled
11050 && entry.opts.max_frame_bytes != 0
11051 && entry.opts.max_side_transport_templates != 0
11052 && !self.side_has_multiple_announcers_locked(&st, id, self.clock.now_ms())
11053 && !template_hash.is_some_and(|hash| {
11054 st.side_transport.get(&id).is_some_and(|transport| {
11055 transport.tx_template_ids.contains_key(&hash)
11056 && transport.tx_templates.contains_key(&hash)
11057 })
11058 })
11059 {
11060 if (entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes)
11065 && st.discovery_routes.get(&side).is_some_and(|route| {
11066 route.announcers.iter().any(|(name, peer)| {
11067 peer.advertised_address == Some(frame.envelope.source_address)
11068 || sender_address_u32(name) == frame.envelope.source_address
11069 })
11070 })
11071 {
11072 continue;
11073 }
11074 return Ok(false);
11075 }
11076 }
11077 Ok(true)
11078 }
11079 #[cfg(not(feature = "discovery"))]
11080 fn cached_transit_ready(
11081 &self,
11082 _side: RouterSideId,
11083 _bytes: &[u8],
11084 _view: Option<&wire_format::RoutingFrameInfo<'_>>,
11085 ) -> TelemetryResult<bool> {
11086 Ok(false)
11087 }
11088 #[cfg(any(feature = "std", feature = "fair-admission"))]
11089 fn reclaim_for_receive(&self, bytes: &[u8], victim: &mut Option<(u32, u8)>) -> bool {
11090 let Some((incoming, priority)) = crate::admission_fairness::incoming(bytes) else {
11091 return false;
11092 };
11093 self.reclaim_for_incoming(incoming, priority, victim)
11094 }
11095 #[cfg(any(feature = "std", feature = "fair-admission"))]
11096 fn reclaim_for_incoming(
11097 &self,
11098 incoming: u32,
11099 priority: u8,
11100 victim: &mut Option<(u32, u8)>,
11101 ) -> bool {
11102 let mut st = self.state.lock();
11103 let (owner, priority) = if let Some(selected) = *victim {
11104 selected
11105 } else {
11106 let Some(owner) = st.competing_bulk_owner(incoming, priority) else {
11107 return false;
11108 };
11109 *victim = Some((owner, priority));
11110 (owner, priority)
11111 };
11112 st.reclaim_bulk_from(owner, priority, None)
11113 }
11114 fn admit_received_frame(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<()> {
11115 self.admit_received_frame_with_view(side, bytes, None)
11116 }
11117
11118 fn admit_received_frame_with_view(
11121 &self,
11122 side: RouterSideId,
11123 bytes: &[u8],
11124 view: Option<&wire_format::RoutingFrameInfo<'_>>,
11125 ) -> TelemetryResult<()> {
11126 if !crate::memory_admission::has_probe() {
11127 return Ok(());
11128 }
11129 #[cfg(any(feature = "std", feature = "fair-admission"))]
11130 let mut fair_victim = None;
11131 if crate::memory_admission::supports_work_kinds()
11132 && self.cached_transit_ready(side, bytes, view)?
11133 {
11134 let result = crate::memory_admission::check_cached_transit(bytes.len());
11135 #[cfg(all(feature = "compression", feature = "discovery"))]
11136 let result = {
11137 let mut result = result;
11138 while matches!(result, Err(TelemetryError::Io("memory pressure")))
11139 && self.state.lock().reclaim_codec_metadata()
11140 {
11141 result = crate::memory_admission::check_cached_transit(bytes.len());
11142 }
11143 result
11144 };
11145
11146 #[cfg(any(feature = "std", feature = "fair-admission"))]
11147 let result = {
11148 let mut result = result;
11149 while matches!(result, Err(TelemetryError::Io("memory pressure"))) {
11150 if !self.reclaim_for_receive(bytes, &mut fair_victim) {
11151 break;
11152 }
11153 result = crate::memory_admission::check_cached_transit(bytes.len());
11154 }
11155 result
11156 };
11157 result
11158 } else {
11159 let managed = wire_format::scheduling_type(bytes)
11160 .is_some_and(|ty| self.is_managed_variable_type(ty));
11161 let mut protected = None;
11162 loop {
11163 match crate::memory_admission::check_receive_with_role(bytes, managed) {
11164 Err(TelemetryError::Io("memory pressure")) => {
11165 #[cfg(all(feature = "compression", feature = "discovery"))]
11166 if self.state.lock().reclaim_codec_metadata() {
11167 continue;
11168 }
11169
11170 let protected = *protected.get_or_insert_with(|| {
11171 wire_format::peek_routing_frame_info(bytes)
11172 .ok()
11173 .is_some_and(|frame| {
11174 managed
11175 || message_priority(frame.envelope.ty) >= 200
11176 || crate::small_frozen_control(
11177 frame.envelope.ty,
11178 frame.envelope.target_senders().len(),
11179 frame.envelope.wire_shape,
11180 )
11181 })
11182 });
11183 if !protected {
11184 #[cfg(any(feature = "std", feature = "fair-admission"))]
11185 if self.reclaim_for_receive(bytes, &mut fair_victim) {
11186 continue;
11187 }
11188 return Err(TelemetryError::Io("memory pressure"));
11189 }
11190 let mut st = self.state.lock();
11194 let reclaimed = st
11195 .lower_priority_shared_queue(199)
11196 .is_some_and(|queue| st.pop_shared_queue_item(queue));
11197 if !reclaimed {
11198 return Err(TelemetryError::Io("memory pressure"));
11199 }
11200 }
11201 result => return result,
11202 }
11203 }
11204 }
11205 }
11206
11207 #[cfg(feature = "discovery")]
11208 fn consume_known_packed_discovery(
11209 &self,
11210 side: RouterSideId,
11211 bytes: &[u8],
11212 ) -> TelemetryResult<bool> {
11213 let result = match wire_format::scheduling_type(bytes) {
11216 #[cfg(not(feature = "std"))]
11217 Some(crate::DataType::DiscoverySchema) => self.consume_known_packed_schema(side, bytes),
11218 Some(crate::DataType::DiscoveryTopology) => {
11219 self.consume_known_packed_topology(side, bytes)
11220 }
11221 Some(crate::DataType::DiscoveryAddress) => {
11222 match wire_format::peek_plain_discovery_payload(
11223 bytes,
11224 crate::DataType::DiscoveryAddress,
11225 ) {
11226 Ok(Some((address, payload))) => {
11227 self.refresh_peer_liveness(side, address);
11228 self.refresh_known_address_payload(payload, side)
11229 .map(|known| known && self.cfg.role.local_services())
11230 }
11231 Ok(None) => Ok(false),
11232 Err(error) => Err(error),
11233 }
11234 }
11235 _ => Ok(false),
11236 };
11237 if result.is_err() {
11238 let _ = self.refresh_ingress_peer_liveness(side, bytes);
11242 }
11243 result
11244 }
11245
11246 fn decode_side_transport_frame(
11247 &self,
11248 side: RouterSideId,
11249 bytes: &[u8],
11250 ) -> TelemetryResult<Option<crate::SharedBytes>> {
11251 self.expire_side_chunks();
11253 let Some((kind, body)) = parse_side_transport_wrapper(bytes)? else {
11254 #[cfg(feature = "discovery")]
11255 if self.consume_known_packed_discovery(side, bytes)? {
11256 return Ok(None);
11257 }
11258 #[cfg(feature = "discovery")]
11259 let view = self.refresh_ingress_peer_liveness(side, bytes);
11260 #[cfg(not(feature = "discovery"))]
11261 let view = None;
11262 #[cfg(all(
11263 feature = "discovery",
11264 any(feature = "std", feature = "ingress-target-filter")
11265 ))]
11266 if bytes.first().is_some_and(|flags| flags & 0x10 == 0)
11267 && view
11268 .as_ref()
11269 .is_some_and(|frame| self.targets_only_ingress(side, frame))
11270 {
11271 return Ok(None);
11272 }
11273 self.admit_received_frame_with_view(side, bytes, view.as_ref())?;
11274 return Ok(Some(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes))));
11275 };
11276 match kind {
11277 SIDE_TRANSPORT_KIND_FULL => {
11278 let header_templates_enabled = self
11279 .state
11280 .lock()
11281 .sides
11282 .get(side)
11283 .and_then(|entry| entry.as_ref())
11284 .map(|side| side.opts.header_template_enabled)
11285 .unwrap_or(false);
11286 let mut off = 0usize;
11287 let template_id = if header_templates_enabled {
11288 Some(
11289 u32::try_from(read_uleb128_local(body, &mut off)?)
11290 .map_err(|_| TelemetryError::Unpack("side template id too large"))?,
11291 )
11292 } else {
11293 None
11294 };
11295 #[cfg(feature = "discovery")]
11296 if self.consume_known_packed_discovery(side, &body[off..])? {
11297 return Ok(None);
11298 }
11299 #[cfg(feature = "discovery")]
11300 let view = self.refresh_ingress_peer_liveness(side, &body[off..]);
11301 #[cfg(not(feature = "discovery"))]
11302 let view = None;
11303 self.admit_received_frame_with_view(side, &body[off..], view.as_ref())?;
11304 let raw = Arc::<[u8]>::from(&body[off..]);
11305 if let (Some(template_id), Ok((template, _, _, _, timestamp, _, _, _))) =
11306 (template_id, extract_side_header_template(raw.as_ref()))
11307 {
11308 let mut st = self.state.lock();
11309 let max_templates = st
11310 .sides
11311 .get(side)
11312 .and_then(|side| side.as_ref())
11313 .map(|side| side.opts.max_side_transport_templates)
11314 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
11315 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
11316 let evicted =
11317 side_state.insert_rx_template(template_id, template, max_templates);
11318 if max_templates > 0 {
11319 side_state.rx_last_timestamps.insert(template_id, timestamp);
11320 }
11321 evicted
11322 });
11323 if evicted {
11324 st.side_runtime_stats
11325 .entry(side)
11326 .or_default()
11327 .note_side_transport_template_eviction();
11328 }
11329 }
11330 Ok(Some(crate::shared_bytes::convert(raw)))
11331 }
11332 SIDE_TRANSPORT_KIND_COMPACT
11333 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
11334 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
11335 if body.is_empty() {
11336 return Err(TelemetryError::Unpack("short side compact frame"));
11337 }
11338 let mut off = 1usize;
11339 let template_id = u32::try_from(read_uleb128_local(body, &mut off)?)
11340 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
11341 let (template, timestamp_base) = {
11342 let st = self.state.lock();
11343 let state = st.side_transport.get(&side);
11344 let template = state
11345 .and_then(|state| state.rx_templates_by_id.get(&template_id))
11346 .cloned();
11347 let timestamp_base = if matches!(
11348 kind,
11349 SIDE_TRANSPORT_KIND_COMPACT_DELTA
11350 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
11351 ) {
11352 state
11353 .and_then(|state| state.rx_last_timestamps.get(&template_id))
11354 .copied()
11355 } else {
11356 None
11357 };
11358 (template, timestamp_base)
11359 };
11360 let Some(template) = template else {
11364 if !self.cfg.role.local_services() {
11365 return Err(TelemetryError::Unpack("unknown side compact template"));
11366 }
11367 return Ok(None);
11368 };
11369 #[cfg(feature = "discovery")]
11372 if let Some(address) = template.source_address() {
11373 self.refresh_peer_liveness(side, address);
11374 }
11375 let reconstructed_bound = body
11376 .len()
11377 .saturating_add(template.prefix().len())
11378 .saturating_add(template.between().len())
11379 .saturating_add(32);
11380 let protected = template.data_type().is_some_and(|ty| {
11384 message_priority(ty) >= 200 || self.is_managed_variable_type(ty)
11385 });
11386 let ack = template.is_ack();
11387 #[cfg(any(feature = "std", feature = "fair-admission"))]
11388 let mut fair_incoming = None;
11389 #[cfg(any(feature = "std", feature = "fair-admission"))]
11390 let mut fair_victim = None;
11391 loop {
11392 match crate::memory_admission::check_compact_reconstruction(
11393 reconstructed_bound,
11394 ack,
11395 ) {
11396 Err(TelemetryError::Io("memory pressure")) if protected => {
11397 let mut st = self.state.lock();
11398 let reclaimed = st
11399 .lower_priority_shared_queue(199)
11400 .is_some_and(|queue| st.pop_shared_queue_item(queue));
11401 if !reclaimed {
11402 return Err(TelemetryError::Io("memory pressure"));
11403 }
11404 }
11405 Err(TelemetryError::Io("memory pressure")) => {
11406 #[cfg(any(feature = "std", feature = "fair-admission"))]
11410 if let Some((incoming, priority)) =
11411 *fair_incoming.get_or_insert_with(|| {
11412 crate::admission_fairness::compact_incoming(&template)
11413 })
11414 && self.reclaim_for_incoming(incoming, priority, &mut fair_victim)
11415 {
11416 continue;
11417 }
11418 return Err(TelemetryError::Io("memory pressure"));
11419 }
11420 result => {
11421 result?;
11422 break;
11423 }
11424 }
11425 }
11426 let timestamp_mode = match kind {
11427 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
11428 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
11429 _ => SideCompactTimestampMode::Absolute,
11430 };
11431 let (frame, timestamp) = reconstruct_side_compact_frame(
11432 &template,
11433 body[0],
11434 &body[off..],
11435 timestamp_mode,
11436 timestamp_base,
11437 )?;
11438 {
11439 let mut st = self.state.lock();
11440 if let Some(side_state) = st.side_transport.get_mut(&side) {
11441 side_state.rx_last_timestamps.insert(template_id, timestamp);
11442 }
11443 }
11444 if crate::memory_admission::supports_work_kinds() {
11448 self.admit_received_frame(side, frame.as_ref())?;
11449 }
11450 Ok(Some(frame))
11451 }
11452 SIDE_TRANSPORT_KIND_CHUNK => {
11453 if body.len() < 8 {
11454 return Err(TelemetryError::Unpack("short side chunk frame"));
11455 }
11456 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
11457 let index = u16::from_le_bytes([body[4], body[5]]);
11458 let total = u16::from_le_bytes([body[6], body[7]]);
11459 if total == 0 || index >= total {
11460 return Err(TelemetryError::Unpack("invalid side chunk index"));
11461 }
11462 let assembled = {
11463 let mut st = self.state.lock();
11464 let now_ms = self.clock.now_ms();
11465 st.make_side_chunk_room(side, transfer_id, index, total, body.len() - 8)?;
11466 if let Some(entry) = st
11470 .side_transport
11471 .get_mut(&side)
11472 .and_then(|state| state.rx_chunks.get_mut(&transfer_id))
11473 && entry.contains_index(index)
11474 {
11475 return Ok(None);
11478 }
11479 let retained = st
11482 .side_transport
11483 .get(&side)
11484 .and_then(|state| state.rx_chunks.get(&transfer_id))
11485 .map(|entry| entry.received_len());
11486 loop {
11487 match crate::memory_admission::check_retained_fragment(bytes, retained) {
11488 Err(TelemetryError::Io("memory pressure")) => {
11489 if st
11490 .lower_priority_shared_queue(199)
11491 .is_some_and(|queue| st.pop_shared_queue_item(queue))
11492 || st.reclaim_oldest_side_chunk(Some((side, transfer_id)))
11493 {
11494 continue;
11495 }
11496 return Err(TelemetryError::Io("memory pressure"));
11497 }
11498 result => {
11499 result?;
11500 break;
11501 }
11502 }
11503 }
11504 let side_state = st
11505 .side_transport
11506 .get_mut(&side)
11507 .ok_or(TelemetryError::BadArg)?;
11508 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
11509 entry.total = total;
11510 let added =
11511 entry.insert_with(index, now_ms, || Arc::<[u8]>::from(&body[8..]))?;
11512 side_state.rx_chunk_bytes = side_state.rx_chunk_bytes.saturating_add(added);
11513 if entry.received_len() == usize::from(total) {
11514 let entry = side_state
11515 .take_chunk(transfer_id, false)
11516 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
11517 Some(entry.assemble().inspect_err(|_| {
11518 side_state.rx_chunk_reclaims =
11519 side_state.rx_chunk_reclaims.saturating_add(1);
11520 })?)
11521 } else {
11522 None
11523 }
11524 };
11525 match assembled {
11526 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
11527 None => Ok(None),
11528 }
11529 }
11530 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
11531 }
11532 }
11533
11534 fn packed_handler_call(
11535 handler: &RouterTxHandlerFn,
11536 bytes: &[u8],
11537 priority: u8,
11538 ) -> TelemetryResult<()> {
11539 match handler {
11540 RouterTxHandlerFn::Packed(handler) => handler(bytes),
11541 RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
11542 RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
11543 }
11544 }
11545
11546 fn effective_transport_priority(&self, ty: DataType) -> u8 {
11547 if self.is_managed_variable_type(ty) {
11548 crate::transport_priority(DataType::ManagedVariableValue)
11549 } else {
11550 crate::transport_priority(ty)
11551 }
11552 }
11553
11554 #[cfg(test)]
11555 fn call_side_tx_handler(
11556 &self,
11557 side: RouterSideId,
11558 handler: &RouterTxHandlerFn,
11559 data: &RouterItem,
11560 relayed: bool,
11561 ) -> TelemetryResult<()> {
11562 self.call_side_tx_handler_with_progress(side, handler, data, relayed, &mut false)
11563 }
11564
11565 fn call_side_tx_handler_with_progress(
11566 &self,
11567 side: RouterSideId,
11568 handler: &RouterTxHandlerFn,
11569 data: &RouterItem,
11570 relayed: bool,
11571 admitted: &mut bool,
11572 ) -> TelemetryResult<()> {
11573 let opts = {
11574 let st = self.state.lock();
11575 Self::side_ref(&st, side)?.opts.clone()
11576 };
11577 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11578 return Err(TelemetryError::Io("side tx busy"));
11579 };
11580 let started_ms = self.clock.now_ms();
11581 let (ty, priority) = self.item_service_type_priority(data)?;
11583 #[cfg(all(feature = "discovery", feature = "timesync"))]
11584 if !Self::timesync_allowed_for_side_locked(
11585 &self.state.lock(),
11586 side,
11587 ty,
11588 self.clock.now_ms(),
11589 ) {
11590 return Ok(());
11591 }
11592 let result = match (handler, data) {
11593 (
11594 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
11595 RouterItem::Packed(bytes),
11596 ) => {
11597 #[cfg(feature = "cryptography")]
11598 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
11599 #[cfg(not(feature = "cryptography"))]
11600 let send_bytes = bytes.clone();
11601 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
11602 let mut attempts_total = 0usize;
11603 let mut sent_bytes = 0usize;
11604 self.send_side_frames_resumable(side, frames, |frame| {
11605 match self.retry_with_attempts(runtime_max_handler_retries(), || {
11606 Self::packed_handler_call(packed, frame, priority)
11607 }) {
11608 Ok((_, attempts)) => {
11609 *admitted = true;
11610 attempts_total = attempts_total.saturating_add(attempts);
11611 sent_bytes = sent_bytes.saturating_add(frame.len());
11612 }
11613 Err((err, attempts)) => {
11614 self.note_side_tx_failure(
11615 side,
11616 ty,
11617 attempts_total.saturating_add(attempts),
11618 );
11619 return Err(err);
11620 }
11621 }
11622
11623 Ok(())
11624 })?;
11625 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
11626 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11627 return Ok(());
11628 }
11629 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
11630 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
11631 }
11632 (
11633 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
11634 RouterItem::Packet(pkt),
11635 ) => {
11636 if ty == DataType::DiscoverySchema
11640 && opts.max_frame_bytes != 0
11641 && pkt.payload().len() > opts.max_frame_bytes
11642 {
11643 let mut attempts_total = 0usize;
11644 let mut sent_bytes = 0usize;
11645 let key = crate::packet::hash_bytes_u64(
11646 pkt.packet_id(),
11647 &opts.max_frame_bytes.to_le_bytes(),
11648 );
11649 let result = self.run_side_chunk_transfer(side, key, |next| {
11650 crate::side_transport::send_packet_chunks(
11651 pkt,
11652 opts.max_frame_bytes,
11653 self.sender_arc().as_bytes(),
11654 next,
11655 |frame| match self
11656 .retry_with_attempts(runtime_max_handler_retries(), || {
11657 Self::packed_handler_call(packed, frame, priority)
11658 }) {
11659 Ok((_, attempts)) => {
11660 *admitted = true;
11661 attempts_total += attempts;
11662 sent_bytes += frame.len();
11663 Ok(())
11664 }
11665 Err((err, attempts)) => {
11666 attempts_total += attempts;
11667 Err(err)
11668 }
11669 },
11670 )
11671 });
11672 let (raw_len, wire_len, chunks) = match result {
11673 Ok(stats) => stats,
11674 Err(err) => {
11675 self.note_side_tx_failure(side, ty, attempts_total);
11676 return Err(err);
11677 }
11678 };
11679 {
11680 let mut st = self.state.lock();
11681 let stats = st.side_runtime_stats.entry(side).or_default();
11682 stats.note_side_transport_full(raw_len, wire_len);
11683 if chunks > 1 {
11684 stats.note_side_transport_chunks(chunks);
11685 }
11686 }
11687 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
11688 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11689 return Ok(());
11690 }
11691 let owned = self.pack_packet_for_router(pkt, None)?;
11692 let frames = self.encode_side_transport_frames(side, opts, owned)?;
11693 let mut attempts_total = 0usize;
11694 let mut sent_bytes = 0usize;
11695 self.send_side_frames_resumable(side, frames, |frame| {
11696 match self.retry_with_attempts(runtime_max_handler_retries(), || {
11697 Self::packed_handler_call(packed, frame, priority)
11698 }) {
11699 Ok((_, attempts)) => {
11700 *admitted = true;
11701 attempts_total = attempts_total.saturating_add(attempts);
11702 sent_bytes = sent_bytes.saturating_add(frame.len());
11703 }
11704 Err((err, attempts)) => {
11705 self.note_side_tx_failure(
11706 side,
11707 ty,
11708 attempts_total.saturating_add(attempts),
11709 );
11710 return Err(err);
11711 }
11712 }
11713
11714 Ok(())
11715 })?;
11716 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
11717 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11718 return Ok(());
11719 }
11720 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
11721 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
11722 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
11723 }
11724 };
11725 match result {
11726 Ok((_, attempts)) => {
11727 *admitted = true;
11728 if let Ok(bytes) = Self::router_item_wire_len(data) {
11729 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
11730 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
11731 }
11732 Ok(())
11733 }
11734 Err((err, attempts)) => {
11735 self.note_side_tx_failure(side, ty, attempts);
11736 Err(err)
11737 }
11738 }
11739 }
11740
11741 fn adjust_reliable_for_side(
11742 &self,
11743 opts: RouterSideOptions,
11744 data: RouterItem,
11745 preserve_end_to_end_ack: bool,
11746 ) -> TelemetryResult<Option<RouterItem>> {
11747 if opts.reliable_enabled {
11748 return Ok(Some(data));
11749 }
11750
11751 match data {
11752 RouterItem::Packed(bytes) => {
11753 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
11754 if is_reliable_type(frame.envelope.ty)
11755 && let Some(hdr) = frame.reliable
11756 {
11757 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
11758 return Ok(None);
11759 }
11760 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
11761 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
11762 bytes.as_ref(),
11763 wire_format::RELIABLE_FLAG_UNSEQUENCED,
11764 hdr.seq,
11765 0,
11766 )?
11767 else {
11768 return Ok(Some(RouterItem::Packed(bytes)));
11769 };
11770 return Ok(Some(RouterItem::Packed(crate::shared_bytes::convert(
11771 rewritten,
11772 ))));
11773 }
11774 }
11775 Ok(Some(RouterItem::Packed(bytes)))
11776 }
11777 RouterItem::Packet(pkt) => {
11778 if matches!(
11779 pkt.data_type(),
11780 DataType::ReliableAck
11781 | DataType::ReliablePartialAck
11782 | DataType::ReliablePacketRequest
11783 ) {
11784 if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
11785 return Ok(Some(RouterItem::Packet(pkt)));
11786 }
11787 return Ok(None);
11788 }
11789 Ok(Some(RouterItem::Packet(pkt)))
11790 }
11791 }
11792 }
11793
11794 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
11795 let now = self.clock.now_ms();
11796 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
11797
11798 {
11799 let mut st = self.state.lock();
11800 if st.reliable_tx.is_empty() {
11801 return Ok(());
11802 }
11803
11804 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
11805 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
11806 continue;
11807 };
11808 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
11809 for seq in sent_order {
11810 let Some(sent) = tx_state.sent.get_mut(&seq) else {
11811 continue;
11812 };
11813 if sent.queued
11814 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
11815 {
11816 continue;
11817 }
11818 if sent.partial_acked {
11819 sent.partial_acked = false;
11826 sent.last_send_ms = now;
11827 continue;
11828 }
11829 if sent.retries >= runtime_reliable_max_retries() {
11830 tx_state.sent.remove(&seq);
11831 tx_state.sent_order.retain(|existing| *existing != seq);
11832 continue;
11833 }
11834 sent.retries += 1;
11835 requeue.push((*side, ty, seq));
11836 }
11837 }
11838 }
11839
11840 for (side, ty, seq) in requeue {
11841 self.queue_reliable_retransmit(side, ty, seq, true)?;
11842 }
11843
11844 Ok(())
11845 }
11846
11847 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
11848 let now = self.clock.now_ms();
11849 let mut requeue = Vec::new();
11850 let mut failed_ordered_types = BTreeSet::new();
11851
11852 {
11853 let mut st = self.state.lock();
11854 #[cfg(feature = "discovery")]
11855 {
11856 if Self::prune_discovery_routes_locked(&mut st, now) {
11857 Self::note_discovery_topology_change_locked(&mut st, now);
11858 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
11859 }
11860 }
11865 st.end_to_end_reliable_tx.retain(|&packet_id, sent| {
11868 if sent.waiting_for_predecessor
11869 || sent.queued
11870 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
11871 {
11872 return true;
11873 }
11874 if sent.retries >= runtime_reliable_max_retries() {
11875 if let Some(ty) = sent.ordered_type {
11876 failed_ordered_types.insert(ty);
11877 }
11878 return false;
11879 }
11880 sent.retries += 1;
11881 requeue.push(packet_id);
11882 true
11883 });
11884 st.end_to_end_reliable_tx.retain(|_, sent| {
11886 !sent
11887 .ordered_type
11888 .is_some_and(|ty| failed_ordered_types.contains(&ty))
11889 });
11890 }
11891
11892 for packet_id in requeue {
11893 self.queue_end_to_end_reliable_retransmit(packet_id)?;
11894 }
11895 self.release_ordered_successors()?;
11896 if !failed_ordered_types.is_empty() {
11897 return Err(TelemetryError::HandlerError(
11898 "ordered end-to-end delivery exhausted retries",
11899 ));
11900 }
11901 Ok(())
11902 }
11903
11904 #[cfg(feature = "timesync")]
11905 #[inline]
11906 fn monotonic_now_ns(&self) -> u64 {
11907 self.clock.now_ns()
11908 }
11909
11910 #[cfg(feature = "timesync")]
11911 #[inline]
11912 fn monotonic_now_ms(&self) -> u64 {
11913 self.clock.now_ms()
11914 }
11915
11916 #[cfg(feature = "timesync")]
11917 fn refresh_timesync_state(&self, now_mono_ms: u64) {
11918 if self.timesync.is_none() {
11919 return;
11920 }
11921
11922 let now_mono_ns = self.monotonic_now_ns();
11923 let mut st = self
11924 .timesync
11925 .as_ref()
11926 .expect("local time service enabled")
11927 .lock();
11928 st.clock.prune_expired(now_mono_ms);
11929 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
11930 st.remote_sources
11931 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
11932 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
11933 let leader = if let Some(tracker) = st.tracker.as_mut() {
11934 let _ = tracker.refresh(now_mono_ms);
11935 tracker.leader(now_mono_ms, has_usable_time)
11936 } else {
11937 None
11938 };
11939 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
11940 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
11941 let target_ms = st.remote_sources.get(remote.sender.as_str()).map(|src| {
11944 src.sample_unix_ms
11945 .saturating_add(now_mono_ms.saturating_sub(src.last_sample_mono_ms))
11946 });
11947 if let Some(target_ms) = target_ms {
11948 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
11949 }
11950 }
11951 }
11952
11953 #[cfg(feature = "timesync")]
11954 pub fn update_network_time_source(
11956 &self,
11957 source: &str,
11958 priority: u64,
11959 time: PartialNetworkTime,
11960 ttl_ms: Option<u64>,
11961 ) {
11962 if self.timesync.is_none() {
11963 return;
11964 }
11965
11966 let now_ms = self.monotonic_now_ms();
11967 let now_ns = self.monotonic_now_ns();
11968 let mut st = self
11969 .timesync
11970 .as_ref()
11971 .expect("local time service enabled")
11972 .lock();
11973 st.clock
11974 .update_source(source, priority, time, now_ms, now_ns, ttl_ms);
11975 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
11976 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
11977 }
11978 }
11979
11980 #[cfg(feature = "timesync")]
11981 fn set_network_time_source_impl(
11982 &self,
11983 source: &str,
11984 priority: u64,
11985 time: PartialNetworkTime,
11986 ttl_ms: Option<u64>,
11987 ) {
11988 if self.timesync.is_none() {
11989 return;
11990 }
11991
11992 let observed_mono_ms = self.monotonic_now_ms();
11993 let observed_mono_ns = self.monotonic_now_ns();
11994 let mut st = self
11995 .timesync
11996 .as_ref()
11997 .expect("local time service enabled")
11998 .lock();
11999 let commit_mono_ms = self.monotonic_now_ms();
12000 let commit_mono_ns = self.monotonic_now_ns();
12001 let adjusted = if let Some(base) = time.to_network_time() {
12002 let elapsed_ns = commit_mono_ns.saturating_sub(observed_mono_ns);
12003 advance_network_time(base, elapsed_ns)
12004 .map(PartialNetworkTime::from)
12005 .unwrap_or(time)
12006 } else {
12007 time
12008 };
12009 let adjusted_mono_ms =
12010 observed_mono_ms.saturating_add(commit_mono_ms.saturating_sub(observed_mono_ms));
12011 st.clock.update_source(
12012 source,
12013 priority,
12014 adjusted,
12015 commit_mono_ms.max(adjusted_mono_ms),
12016 commit_mono_ns,
12017 ttl_ms,
12018 );
12019 if let Some(unix_ms) = adjusted.to_network_time().and_then(|t| t.as_unix_ms()) {
12020 st.disciplined_clock.steer_unix_ms(commit_mono_ns, unix_ms);
12021 }
12022 }
12023
12024 #[cfg(feature = "timesync")]
12025 fn local_network_time_priority(&self) -> u64 {
12026 if self.timesync.is_none() {
12027 return 0;
12028 }
12029
12030 let st = self
12031 .timesync
12032 .as_ref()
12033 .expect("local time service enabled")
12034 .lock();
12035 st.cfg.map(|cfg| cfg.priority).unwrap_or(0)
12036 }
12037
12038 #[cfg(feature = "timesync")]
12039 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
12041 if self.timesync.is_none() {
12042 return;
12043 }
12044
12045 let priority = self.local_network_time_priority();
12046 if time.is_complete_date() && time.is_complete_time() {
12047 self.set_network_time_source_impl(LOCAL_TIMESYNC_FULL_SOURCE_ID, priority, time, None);
12048 let mut st = self
12049 .timesync
12050 .as_ref()
12051 .expect("local time service enabled")
12052 .lock();
12053 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
12054 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
12055 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
12056 return;
12057 }
12058
12059 {
12060 let mut st = self
12061 .timesync
12062 .as_ref()
12063 .expect("local time service enabled")
12064 .lock();
12065 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
12066 }
12067
12068 if time.year.is_some() || time.month.is_some() || time.day.is_some() {
12069 self.set_network_time_source_impl(
12070 LOCAL_TIMESYNC_DATE_SOURCE_ID,
12071 priority,
12072 PartialNetworkTime {
12073 year: time.year,
12074 month: time.month,
12075 day: time.day,
12076 ..Default::default()
12077 },
12078 None,
12079 );
12080 }
12081
12082 if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
12083 self.set_network_time_source_impl(
12084 LOCAL_TIMESYNC_TOD_SOURCE_ID,
12085 priority,
12086 PartialNetworkTime {
12087 hour: time.hour,
12088 minute: time.minute,
12089 second: time.second,
12090 nanosecond: time.nanosecond,
12091 ..Default::default()
12092 },
12093 None,
12094 );
12095 }
12096
12097 if time.nanosecond.is_some() {
12098 self.set_network_time_source_impl(
12099 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
12100 priority,
12101 PartialNetworkTime {
12102 nanosecond: time.nanosecond,
12103 ..Default::default()
12104 },
12105 None,
12106 );
12107 }
12108 }
12109
12110 #[cfg(feature = "timesync")]
12111 pub fn clear_local_network_time(&self) {
12113 if self.timesync.is_none() {
12114 return;
12115 }
12116
12117 let mut st = self
12118 .timesync
12119 .as_ref()
12120 .expect("local time service enabled")
12121 .lock();
12122 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
12123 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
12124 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
12125 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
12126 }
12127
12128 #[cfg(feature = "timesync")]
12129 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
12131 self.set_local_network_time(PartialNetworkTime {
12132 year: Some(year),
12133 month: Some(month),
12134 day: Some(day),
12135 ..Default::default()
12136 });
12137 }
12138
12139 #[cfg(feature = "timesync")]
12140 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
12142 self.set_local_network_time(PartialNetworkTime {
12143 hour: Some(hour),
12144 minute: Some(minute),
12145 ..Default::default()
12146 });
12147 }
12148
12149 #[cfg(feature = "timesync")]
12150 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
12152 self.set_local_network_time(PartialNetworkTime {
12153 hour: Some(hour),
12154 minute: Some(minute),
12155 second: Some(second),
12156 ..Default::default()
12157 });
12158 }
12159
12160 #[cfg(feature = "timesync")]
12161 pub fn set_local_network_time_hms_millis(
12163 &self,
12164 hour: u8,
12165 minute: u8,
12166 second: u8,
12167 millisecond: u16,
12168 ) {
12169 self.set_local_network_time(PartialNetworkTime {
12170 hour: Some(hour),
12171 minute: Some(minute),
12172 second: Some(second),
12173 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
12174 ..Default::default()
12175 });
12176 }
12177
12178 #[cfg(feature = "timesync")]
12179 pub fn set_local_network_time_hms_nanos(
12181 &self,
12182 hour: u8,
12183 minute: u8,
12184 second: u8,
12185 nanosecond: u32,
12186 ) {
12187 self.set_local_network_time(PartialNetworkTime {
12188 hour: Some(hour),
12189 minute: Some(minute),
12190 second: Some(second),
12191 nanosecond: Some(nanosecond),
12192 ..Default::default()
12193 });
12194 }
12195
12196 #[cfg(feature = "timesync")]
12197 pub fn set_local_network_datetime(
12199 &self,
12200 year: i32,
12201 month: u8,
12202 day: u8,
12203 hour: u8,
12204 minute: u8,
12205 second: u8,
12206 ) {
12207 self.set_local_network_time(PartialNetworkTime {
12208 year: Some(year),
12209 month: Some(month),
12210 day: Some(day),
12211 hour: Some(hour),
12212 minute: Some(minute),
12213 second: Some(second),
12214 ..Default::default()
12215 });
12216 }
12217
12218 #[cfg(feature = "timesync")]
12219 #[allow(clippy::too_many_arguments)]
12220 pub fn set_local_network_datetime_millis(
12222 &self,
12223 year: i32,
12224 month: u8,
12225 day: u8,
12226 hour: u8,
12227 minute: u8,
12228 second: u8,
12229 millisecond: u16,
12230 ) {
12231 self.set_local_network_time(PartialNetworkTime {
12232 year: Some(year),
12233 month: Some(month),
12234 day: Some(day),
12235 hour: Some(hour),
12236 minute: Some(minute),
12237 second: Some(second),
12238 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
12239 });
12240 }
12241
12242 #[cfg(feature = "timesync")]
12243 #[allow(clippy::too_many_arguments)]
12244 pub fn set_local_network_datetime_nanos(
12246 &self,
12247 year: i32,
12248 month: u8,
12249 day: u8,
12250 hour: u8,
12251 minute: u8,
12252 second: u8,
12253 nanosecond: u32,
12254 ) {
12255 self.set_local_network_time(PartialNetworkTime {
12256 year: Some(year),
12257 month: Some(month),
12258 day: Some(day),
12259 hour: Some(hour),
12260 minute: Some(minute),
12261 second: Some(second),
12262 nanosecond: Some(nanosecond),
12263 });
12264 }
12265
12266 #[cfg(feature = "timesync")]
12267 pub fn clear_network_time_source(&self, source: &str) {
12269 if self.timesync.is_none() {
12270 return;
12271 }
12272
12273 let mut st = self
12274 .timesync
12275 .as_ref()
12276 .expect("local time service enabled")
12277 .lock();
12278 st.clock.remove_source(source);
12279 }
12280
12281 #[cfg(feature = "timesync")]
12282 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
12284 if self.timesync.is_none() {
12285 return;
12286 }
12287
12288 let mut st = self
12289 .timesync
12290 .as_ref()
12291 .expect("local time service enabled")
12292 .lock();
12293 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
12294 st.cfg = cfg;
12295 st.tracker = cfg.map(TimeSyncTracker::new);
12296 st.disciplined_clock = SlewedNetworkClock::new(
12297 cfg.map(|c| c.max_slew_ppm)
12298 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
12299 );
12300 st.remote_sources.clear();
12301 st.next_seq = 1;
12302 st.next_announce_mono_ms = 0;
12303 st.next_request_mono_ms = 0;
12304 st.pending_request = None;
12305 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
12306 for source in stale_remote_sources {
12307 st.clock.remove_source(&source);
12308 }
12309 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
12310 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
12311 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
12312 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
12313 }
12314
12315 #[cfg(feature = "timesync")]
12316 pub fn network_time(&self) -> Option<NetworkTimeReading> {
12318 let runtime = self.timesync.as_ref()?;
12319
12320 let now_ms = self.monotonic_now_ms();
12321 let now_ns = self.monotonic_now_ns();
12322 self.refresh_timesync_state(now_ms);
12323 let st = runtime.lock();
12324 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
12325 return Some(NetworkTimeReading {
12326 time: PartialNetworkTime::from_unix_ms(unix_ms),
12327 unix_time_ms: Some(unix_ms),
12328 });
12329 }
12330 st.clock.current_time(now_ns)
12331 }
12332
12333 #[cfg(feature = "timesync")]
12334 pub fn network_time_ms(&self) -> Option<u64> {
12336 self.network_time().and_then(|t| t.unix_time_ms)
12337 }
12338
12339 #[cfg(feature = "timesync")]
12340 fn packet_timestamp_ms(&self) -> u64 {
12341 self.network_time_ms()
12342 .unwrap_or_else(|| self.monotonic_now_ms())
12343 }
12344
12345 #[cfg(not(feature = "timesync"))]
12346 fn packet_timestamp_ms(&self) -> u64 {
12347 self.clock.now_ms()
12348 }
12349
12350 #[cfg(feature = "timesync")]
12351 fn queue_internal_timesync_request(
12352 &self,
12353 seq: u64,
12354 t1_mono_ms: u64,
12355 called_from_queue: bool,
12356 ) -> TelemetryResult<()> {
12357 let pkt_ts = self.packet_timestamp_ms();
12358 if called_from_queue {
12359 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
12360 } else {
12361 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
12362 }
12363 }
12364
12365 #[cfg(feature = "timesync")]
12366 fn queue_internal_timesync_response(
12367 &self,
12368 seq: u64,
12369 t1_mono_ms: u64,
12370 t2_network_ms: u64,
12371 t3_network_ms: u64,
12372 dst: Option<RouterSideId>,
12373 called_from_queue: bool,
12374 ) -> TelemetryResult<()> {
12375 let pkt_ts = self.packet_timestamp_ms();
12376 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
12377 let sender = self.sender_arc();
12378 let pkt = Packet::new(
12379 DataType::TimeSyncResponse,
12380 &[DataEndpoint::TimeSync],
12381 sender.as_ref(),
12382 pkt_ts,
12383 payload,
12384 )?;
12385 match dst {
12386 Some(dst) => self.emit_internal_tx(
12387 RouterTxItem::ToSide {
12388 src: None,
12389 dst,
12390 data: RouterItem::Packet(pkt),
12391 },
12392 true,
12393 called_from_queue,
12394 ),
12395 None => self.emit_internal_tx(
12396 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
12397 true,
12398 called_from_queue,
12399 ),
12400 }
12401 }
12402
12403 #[cfg(feature = "timesync")]
12404 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
12406 if self.timesync.is_none() {
12407 return Ok(false);
12408 }
12409
12410 let now_ms = self.monotonic_now_ms();
12411 let now_ns = self.monotonic_now_ns();
12412 let mut queued_any = false;
12413 let mut announce_priority = None;
12414 let mut request = None;
12415
12416 {
12417 let mut st = self
12418 .timesync
12419 .as_ref()
12420 .expect("local time service enabled")
12421 .lock();
12422 st.clock.prune_expired(now_ms);
12423 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
12424 st.remote_sources
12425 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
12426 let Some(cfg) = st.cfg else {
12427 return Ok(false);
12428 };
12429
12430 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
12431 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
12432 let _ = tracker.refresh(now_ms);
12433 (
12434 tracker.leader(now_ms, has_usable_time),
12435 tracker.local_announce_priority(now_ms, has_usable_time),
12436 )
12437 } else {
12438 (None, None)
12439 };
12440 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
12441
12442 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
12443 let target_ms = st.remote_sources.get(&remote.sender).map(|src| {
12444 src.sample_unix_ms
12445 .saturating_add(now_ms.saturating_sub(src.last_sample_mono_ms))
12446 });
12447 if let Some(target_ms) = target_ms {
12448 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
12449 }
12450 }
12451
12452 if let Some(priority) = announce_prio
12453 && now_ms >= st.next_announce_mono_ms
12454 {
12455 announce_priority = Some(priority);
12456 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
12457 }
12458
12459 if let Some(TimeSyncLeader::Remote(remote)) = leader
12460 && now_ms >= st.next_request_mono_ms
12461 && st.pending_request.is_none()
12462 {
12463 let seq = st.next_seq;
12464 let next = st.next_seq.wrapping_add(1);
12465 st.next_seq = if next == 0 { 1 } else { next };
12466 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
12467 st.pending_request = Some(PendingTimeSyncRequest {
12468 seq,
12469 t1_mono_ms: now_ms,
12470 source: remote.sender,
12471 });
12472 request = Some((seq, now_ms));
12473 }
12474 }
12475
12476 if let Some(priority) = announce_priority {
12477 let time_ms = self.packet_timestamp_ms();
12478 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
12479 queued_any = true;
12480 }
12481 if let Some((seq, t1_mono_ms)) = request {
12482 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
12483 queued_any = true;
12484 }
12485
12486 Ok(queued_any)
12487 }
12488
12489 #[cfg(feature = "timesync")]
12490 fn handle_internal_timesync_packet(
12491 &self,
12492 pkt: &Packet,
12493 src: Option<RouterSideId>,
12494 called_from_queue: bool,
12495 ) -> TelemetryResult<bool> {
12496 let Some(cfg) = self.cfg.timesync_config() else {
12497 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
12498 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
12499 }
12500 return Ok(true);
12501 };
12502
12503 let now_mono_ms = self.monotonic_now_ms();
12504 let now_mono_ns = self.monotonic_now_ns();
12505 let mut response = None;
12506 let mut poll_after = false;
12507
12508 {
12509 let mut st = self
12510 .timesync
12511 .as_ref()
12512 .expect("local time service enabled")
12513 .lock();
12514 st.clock.prune_expired(now_mono_ms);
12515 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
12516 st.remote_sources
12517 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
12518 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
12519 if st.tracker.is_none() {
12520 return Ok(true);
12521 }
12522
12523 match pkt.data_type() {
12524 DataType::TimeSyncAnnounce => {
12525 let ann = decode_timesync_announce(pkt)?;
12526 let should_steer = {
12527 let tracker = st.tracker.as_mut().expect("tracker checked above");
12528 let _ = tracker.handle_announce(pkt, now_mono_ms)?;
12529 matches!(
12530 tracker.leader(now_mono_ms, has_usable_time),
12531 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender()
12532 )
12533 };
12534 st.remote_sources.insert(
12535 pkt.sender().to_owned(),
12536 RemoteTimeSyncSource {
12537 priority: ann.priority,
12538 last_sample_mono_ms: now_mono_ms,
12539 sample_unix_ms: ann.time_ms,
12540 },
12541 );
12542 st.clock.update_source(
12543 pkt.sender(),
12544 ann.priority,
12545 PartialNetworkTime::from_unix_ms(ann.time_ms),
12546 now_mono_ms,
12547 now_mono_ns,
12548 Some(cfg.source_timeout_ms),
12549 );
12550 if should_steer {
12551 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
12552 }
12553 poll_after = true;
12554 }
12555 DataType::TimeSyncRequest => {
12556 let should_serve = {
12557 let tracker = st.tracker.as_ref().expect("tracker checked above");
12558 tracker.should_serve(now_mono_ms, has_usable_time)
12559 };
12560 if should_serve {
12561 let req = decode_timesync_request(pkt)?;
12562 let network_now = st
12563 .disciplined_clock
12564 .read_unix_ms(now_mono_ns)
12565 .or_else(|| {
12566 st.clock
12567 .current_time(now_mono_ns)
12568 .and_then(|t| t.unix_time_ms)
12569 })
12570 .unwrap_or(now_mono_ms);
12571 let t2 = network_now;
12572 let t3 = network_now;
12573 response = Some((req.seq, req.t1_ms, t2, t3, src));
12574 }
12575 }
12576 DataType::TimeSyncResponse => {
12577 let resp = decode_timesync_response(pkt)?;
12578 let pending = st.pending_request.clone();
12579 if let Some(pending) = pending
12580 && pending.seq == resp.seq
12581 && pending.source == pkt.sender()
12582 {
12583 let source_priority = {
12584 let tracker = st.tracker.as_ref().expect("tracker checked above");
12585 tracker
12586 .best_active_source(now_mono_ms)
12587 .map(|s| s.priority)
12588 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
12589 .unwrap_or(cfg.priority)
12590 };
12591 let (estimate_ms, _delay_ms) = compute_network_time_sample(
12592 pending.t1_mono_ms,
12593 resp.t2_ms,
12594 resp.t3_ms,
12595 now_mono_ms,
12596 );
12597 st.remote_sources.insert(
12598 pkt.sender().to_owned(),
12599 RemoteTimeSyncSource {
12600 priority: source_priority,
12601 last_sample_mono_ms: now_mono_ms,
12602 sample_unix_ms: estimate_ms,
12603 },
12604 );
12605 st.clock.update_source(
12606 pkt.sender(),
12607 source_priority,
12608 PartialNetworkTime::from_unix_ms(estimate_ms),
12609 now_mono_ms,
12610 now_mono_ns,
12611 Some(cfg.source_timeout_ms),
12612 );
12613 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
12614 st.pending_request = None;
12615 }
12616 }
12617 _ => {}
12618 }
12619 }
12620
12621 if let Some((seq, t1, t2, t3, dst)) = response {
12622 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
12623 }
12624 if poll_after {
12625 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
12626 }
12627
12628 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
12629 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
12630 }
12631
12632 Ok(true)
12633 }
12634
12635 #[cfg(feature = "std")]
12637 pub fn new(cfg: RouterConfig) -> Self {
12638 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
12639 }
12640
12641 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
12643 #[cfg(feature = "timesync")]
12644 let timesync_cfg = cfg.timesync_config();
12645 #[cfg(feature = "discovery")]
12646 let preferred_discovery_master = cfg.preferred_discovery_master.clone();
12647 let memory = cfg.memory_config();
12648 let local_services_enabled = cfg.role.local_services();
12649 let hostname: Arc<str> = cfg.configured_sender();
12650 let address_mode = cfg.address_mode();
12651 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
12652 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
12653 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
12654 let requested = address_mode.requested_address();
12655 let address = if requested == 0 { fallback } else { requested };
12656 let local_address = AddressBookEntry {
12657 hostname: hostname.clone(),
12658 address,
12659 requested_address: requested,
12660 mode: address_mode,
12661 birth_ms: clock
12662 .now_ms()
12663 .saturating_mul(1_000_000)
12664 .saturating_add(instance_seq),
12665 owner_hash,
12666 last_seen_ms: clock.now_ms(),
12667 };
12668 let mut address_book = BTreeMap::new();
12669 if local_services_enabled {
12670 address_book.insert(hostname.to_string(), local_address.clone());
12671 }
12672 let mut address_by_value = BTreeMap::new();
12673 if local_services_enabled {
12674 address_by_value.insert(address, hostname.to_string());
12675 }
12676 Self {
12677 sender: RouterMutex::new(hostname),
12678 cfg: ActiveRoutingConfig::new(cfg),
12679 state: RouterMutex::new(RouterInner {
12680 #[cfg(feature = "compression")]
12681 codec_reserved_bytes: 0,
12682 memory,
12683 sides: Vec::new(),
12684 route_overrides: Default::default(),
12685 typed_route_overrides: Default::default(),
12686 route_weights: Default::default(),
12687 route_priorities: Default::default(),
12688 source_route_modes: Default::default(),
12689 route_selection_cursors: Default::default(),
12690 adaptive_route_stats: Default::default(),
12691 side_runtime_stats: Default::default(),
12692 side_transport: Default::default(),
12693 local_services: local_services_enabled.then(|| {
12694 Box::new(LocalServicesState {
12695 managed_variable_types: BTreeSet::new(),
12696 managed_variable_permissions: Default::default(),
12697 managed_variable_latest: Default::default(),
12698 network_variable_update_handlers: Default::default(),
12699 p2p_port_handlers: Default::default(),
12700 p2p_stream_handlers: Default::default(),
12701 p2p_stream_sessions: Default::default(),
12702 next_p2p_stream_id: 1,
12703 })
12704 }),
12705 local_address,
12706 address_book,
12707 address_by_value,
12708 received_queue: BoundedDeque::new(
12709 memory.max_queue_budget,
12710 memory.starting_queue_size,
12711 memory.queue_grow_step,
12712 ),
12713 transmit_queue: BoundedDeque::new(
12714 memory.max_queue_budget,
12715 memory.starting_queue_size,
12716 memory.queue_grow_step,
12717 ),
12718 tx_priority_burst: 0,
12719 has_ordered_retry: false,
12720 ordered_claim_seq: 0,
12721 rx_priority_burst: 0,
12722 recent_rx: BoundedDeque::new(
12723 memory.recent_rx_queue_bytes(),
12724 memory.recent_rx_queue_bytes(),
12725 memory.queue_grow_step,
12726 ),
12727 reliable_tx: Default::default(),
12728 reliable_rx: Default::default(),
12729 reliable_return_routes: Default::default(),
12730 reliable_return_route_order: VecDeque::new(),
12731 forwarded_receipts: None,
12732 end_to_end_reliable_tx: Default::default(),
12733 end_to_end_reliable_tx_order: VecDeque::new(),
12734 total_handler_failures: 0,
12735 total_handler_retries: 0,
12736 #[cfg(feature = "discovery")]
12737 discovery_routes: Default::default(),
12738 #[cfg(feature = "discovery")]
12739 discovery_cadence: DiscoveryCadenceState::default(),
12740 #[cfg(feature = "discovery")]
12741 discovery_side_throttle: Default::default(),
12742 #[cfg(feature = "discovery")]
12743 schema_response_sequence: 0,
12744 #[cfg(feature = "discovery")]
12745 preferred_discovery_master,
12746 #[cfg(all(feature = "discovery", feature = "timesync"))]
12747 timesync_side_throttle: Default::default(),
12748 }),
12749 isr_rx_queue: local_services_enabled.then(|| {
12750 Box::new(IsrRxQueue::new(
12751 memory.max_queue_budget,
12752 memory.starting_queue_size,
12753 memory.queue_grow_step,
12754 ))
12755 }),
12756 #[cfg(not(feature = "std"))]
12757 side_tx_gate: ReentryGate::new(),
12758 #[cfg(feature = "std")]
12759 side_tx_activity: crate::lock::SideTxActivity::default(),
12760 #[cfg(feature = "std")]
12761 active_queued_tx: core::sync::atomic::AtomicUsize::new(0),
12762 clock,
12763 #[cfg(feature = "timesync")]
12764 timesync: local_services_enabled
12765 .then(|| Box::new(RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)))),
12766 }
12767 }
12768
12769 #[inline]
12770 fn sender_arc(&self) -> Arc<str> {
12771 self.sender.lock().clone()
12772 }
12773
12774 #[inline]
12775 pub fn sender(&self) -> Arc<str> {
12776 self.sender_arc()
12777 }
12778
12779 pub fn current_address(&self) -> NodeAddress {
12780 self.state.lock().local_address.address
12781 }
12782
12783 pub fn hostname(&self) -> Arc<str> {
12784 self.sender_arc()
12785 }
12786
12787 pub fn set_preferred_discovery_master(&self, hostname: Option<&str>) {
12792 #[cfg(feature = "discovery")]
12793 {
12794 let preferred = hostname
12795 .filter(|value| !value.is_empty())
12796 .map(Arc::<str>::from);
12797 let mut st = self.state.lock();
12798 if st.preferred_discovery_master != preferred {
12799 st.preferred_discovery_master = preferred;
12800 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
12801 }
12802 }
12803 #[cfg(not(feature = "discovery"))]
12804 let _ = hostname;
12805 }
12806
12807 pub fn preferred_discovery_master(&self) -> Option<Arc<str>> {
12809 #[cfg(feature = "discovery")]
12810 {
12811 return self.state.lock().preferred_discovery_master.clone();
12812 }
12813 #[cfg(not(feature = "discovery"))]
12814 None
12815 }
12816
12817 pub fn address_book(&self) -> Vec<AddressBookEntry> {
12818 self.state.lock().address_book.values().cloned().collect()
12819 }
12820
12821 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
12822 let now_ms = self.clock.now_ms();
12823 #[allow(unused_mut)]
12824 let mut st = self.state.lock();
12825 #[cfg(feature = "discovery")]
12826 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
12827 Self::note_discovery_topology_change_locked(&mut st, now_ms);
12828 }
12829 if let Some(entry) = st.address_book.get(hostname) {
12830 return Some(entry.clone());
12831 }
12832 let address = sender_address_u32(hostname);
12838 if Self::hostname_for_address_locked(&st, address).as_deref() != Some(hostname) {
12839 return None;
12840 }
12841 Some(AddressBookEntry {
12842 hostname: Arc::from(hostname),
12843 address,
12844 requested_address: 0,
12845 mode: AddressAssignmentMode::Dynamic,
12846 birth_ms: 0,
12847 owner_hash: Self::sender_hash(hostname),
12848 last_seen_ms: now_ms,
12849 })
12850 }
12851
12852 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
12853 let st = self.state.lock();
12854 let direct = st
12855 .address_by_value
12856 .get(&address)
12857 .and_then(|hostname| st.address_book.get(hostname))
12858 .cloned();
12859 if direct.is_some() {
12860 return direct;
12861 }
12862
12863 let hostname = Self::hostname_for_address_locked(&st, address)?;
12864 if let Some(entry) = st.address_book.get(&hostname) {
12865 return Some(entry.clone());
12866 }
12867
12868 let hostname: Arc<str> = Arc::from(hostname);
12873 Some(AddressBookEntry {
12874 hostname: hostname.clone(),
12875 address,
12876 requested_address: 0,
12877 mode: AddressAssignmentMode::Dynamic,
12878 birth_ms: 0,
12879 owner_hash: Self::sender_hash(hostname.as_ref()),
12880 last_seen_ms: self.clock.now_ms(),
12881 })
12882 }
12883
12884 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
12885 where
12886 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
12887 {
12888 if port == 0 {
12889 return Err(TelemetryError::BadArg);
12890 }
12891 let mut st = self.state.lock();
12892 st.local_services
12893 .as_mut()
12894 .expect("local services enabled")
12895 .p2p_port_handlers
12896 .entry(port)
12897 .or_default()
12898 .push(P2pPortHandler {
12899 handler: Arc::new(f),
12900 });
12901 Ok(())
12902 }
12903
12904 pub fn clear_p2p_port(&self, port: P2pPort) {
12905 self.state
12906 .lock()
12907 .local_services
12908 .as_mut()
12909 .expect("local services enabled")
12910 .p2p_port_handlers
12911 .remove(&port);
12912 }
12913
12914 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
12915 where
12916 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
12917 {
12918 if port == 0 {
12919 return Err(TelemetryError::BadArg);
12920 }
12921 let mut st = self.state.lock();
12922 st.local_services
12923 .as_mut()
12924 .expect("local services enabled")
12925 .p2p_stream_handlers
12926 .entry(port)
12927 .or_default()
12928 .push(P2pStreamHandler {
12929 handler: Arc::new(f),
12930 });
12931 Ok(())
12932 }
12933
12934 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
12935 self.state
12936 .lock()
12937 .local_services
12938 .as_mut()
12939 .expect("local services enabled")
12940 .p2p_stream_handlers
12941 .remove(&port);
12942 }
12943
12944 fn send_p2p_to_entry(
12945 &self,
12946 dst: AddressBookEntry,
12947 dst_port: P2pPort,
12948 src_port: P2pPort,
12949 payload: &[u8],
12950 ) -> TelemetryResult<()> {
12951 if dst_port == 0 {
12952 return Err(TelemetryError::BadArg);
12953 }
12954 let local = self.state.lock().local_address.clone();
12955 let payload = Self::encode_p2p_payload(
12956 local.hostname.as_ref(),
12957 local.address,
12958 src_port,
12959 dst_port,
12960 payload,
12961 )?;
12962 let pkt = Packet::new(
12963 DataType::P2pMessage,
12964 &[DataEndpoint::Discovery],
12965 local.hostname.as_ref(),
12966 self.packet_timestamp_ms(),
12967 crate::shared_bytes::convert(payload),
12968 )?;
12969 let target = Self::sender_hash(dst.hostname.as_ref());
12970 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
12971 if target == Self::sender_hash(self.sender_arc().as_ref()) {
12972 self.dispatch_p2p_packet(&pkt)?;
12973 return Ok(());
12974 }
12975 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
12976 }
12977
12978 pub fn send_p2p_to_hostname(
12979 &self,
12980 hostname: &str,
12981 dst_port: P2pPort,
12982 src_port: P2pPort,
12983 payload: &[u8],
12984 ) -> TelemetryResult<()> {
12985 let dst = self
12986 .resolve_hostname(hostname)
12987 .ok_or(TelemetryError::BadArg)?;
12988 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
12989 }
12990
12991 pub fn send_p2p_to_address(
12992 &self,
12993 address: NodeAddress,
12994 dst_port: P2pPort,
12995 src_port: P2pPort,
12996 payload: &[u8],
12997 ) -> TelemetryResult<()> {
12998 let dst = self
12999 .resolve_address(address)
13000 .ok_or(TelemetryError::BadArg)?;
13001 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
13002 }
13003
13004 fn open_p2p_stream_to_entry(
13005 &self,
13006 dst: AddressBookEntry,
13007 dst_port: P2pPort,
13008 src_port: P2pPort,
13009 ) -> TelemetryResult<P2pStreamId> {
13010 if dst_port == 0 || src_port == 0 {
13011 return Err(TelemetryError::BadArg);
13012 }
13013 let stream_id = {
13014 let mut st = self.state.lock();
13015 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
13016 if stream_id == 0 {
13017 return Err(TelemetryError::Io("p2p stream id exhausted"));
13018 }
13019 st.local_services
13020 .as_mut()
13021 .expect("local services enabled")
13022 .p2p_stream_sessions
13023 .insert(
13024 stream_id,
13025 P2pStreamSession {
13026 peer_hostname: dst.hostname.clone(),
13027 peer_address: dst.address,
13028 local_port: src_port,
13029 peer_port: dst_port,
13030 peer_stream_id: 0,
13031 next_sequence: 1,
13032 connected: false,
13033 },
13034 );
13035 stream_id
13036 };
13037 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
13038 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
13039 Ok(stream_id)
13040 }
13041
13042 pub fn open_p2p_stream_to_hostname(
13043 &self,
13044 hostname: &str,
13045 dst_port: P2pPort,
13046 src_port: P2pPort,
13047 ) -> TelemetryResult<P2pStreamId> {
13048 let dst = self
13049 .resolve_hostname(hostname)
13050 .ok_or(TelemetryError::BadArg)?;
13051 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
13052 }
13053
13054 pub fn open_p2p_stream_to_address(
13055 &self,
13056 address: NodeAddress,
13057 dst_port: P2pPort,
13058 src_port: P2pPort,
13059 ) -> TelemetryResult<P2pStreamId> {
13060 let dst = self
13061 .resolve_address(address)
13062 .ok_or(TelemetryError::BadArg)?;
13063 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
13064 }
13065
13066 fn send_p2p_stream_control(
13067 &self,
13068 stream_id: P2pStreamId,
13069 flags: u8,
13070 payload: &[u8],
13071 ) -> TelemetryResult<()> {
13072 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
13073 let mut st = self.state.lock();
13074 let Some(session) = st
13075 .local_services
13076 .as_mut()
13077 .expect("local services enabled")
13078 .p2p_stream_sessions
13079 .get_mut(&stream_id)
13080 else {
13081 return Err(TelemetryError::BadArg);
13082 };
13083 let peer_hostname = session.peer_hostname.clone();
13084 let peer_address = session.peer_address;
13085 let peer_port = session.peer_port;
13086 let local_port = session.local_port;
13087 let peer_stream_id = session.peer_stream_id;
13088 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
13089 let seq = session.next_sequence;
13090 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
13091 let dst = st
13092 .address_book
13093 .get(peer_hostname.as_ref())
13094 .cloned()
13095 .unwrap_or(AddressBookEntry {
13096 hostname: peer_hostname.clone(),
13097 address: peer_address,
13098 requested_address: peer_address,
13099 mode: AddressAssignmentMode::Dynamic,
13100 birth_ms: self.clock.now_ms(),
13101 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
13102 last_seen_ms: self.clock.now_ms(),
13103 });
13104 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
13105 };
13106 if refresh_syn {
13107 let syn_payload =
13108 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
13109 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
13110 }
13111 let stream_payload =
13112 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
13113 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
13114 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
13115 self.state
13116 .lock()
13117 .local_services
13118 .as_mut()
13119 .expect("local services enabled")
13120 .p2p_stream_sessions
13121 .remove(&stream_id);
13122 }
13123 Ok(())
13124 }
13125
13126 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
13127 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
13128 }
13129
13130 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
13131 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
13132 }
13133
13134 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
13135 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
13136 }
13137
13138 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
13139 let hostname: Arc<str> = Arc::from(sender.as_ref());
13140 if !self.cfg.role.local_services() {
13141 *self.sender.lock() = hostname;
13144 return;
13145 }
13146 let mut st = self.state.lock();
13147 let mut local = st.local_address.clone();
13148 local.hostname = hostname.clone();
13149 local.owner_hash = Self::sender_hash(hostname.as_ref());
13150 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
13151 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
13152 local.requested_address = 0;
13153 }
13154 let change =
13155 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
13156 drop(st);
13157 let _ = self.notify_address_change(change);
13158 }
13159
13160 pub fn set_address_assignment(
13161 &self,
13162 mode: AddressAssignmentMode,
13163 ) -> TelemetryResult<AddressChange> {
13164 let mut st = self.state.lock();
13165 let mut local = st.local_address.clone();
13166 local.mode = mode;
13167 local.requested_address = mode.requested_address();
13168 local.address = match mode {
13169 AddressAssignmentMode::Dynamic => {
13170 Self::fallback_address_for_hostname(local.hostname.as_ref())
13171 }
13172 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
13173 if address == 0 {
13174 1
13175 } else {
13176 address
13177 }
13178 }
13179 };
13180 let change =
13181 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
13182 drop(st);
13183 self.notify_address_change(change.clone())?;
13184 Ok(change)
13185 }
13186
13187 pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
13196 where
13197 N: AsRef<str>,
13198 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
13199 {
13200 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
13201 }
13202
13203 pub fn add_side_packed_small_packets<N, F>(
13207 &self,
13208 name: N,
13209 tx: F,
13210 max_frame_bytes: usize,
13211 ) -> RouterSideId
13212 where
13213 N: AsRef<str>,
13214 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
13215 {
13216 self.add_side_packed_with_options(
13217 name,
13218 tx,
13219 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
13220 )
13221 }
13222
13223 pub fn add_side_packed_with_options<N, F>(
13232 &self,
13233 name: N,
13234 tx: F,
13235 opts: RouterSideOptions,
13236 ) -> RouterSideId
13237 where
13238 N: AsRef<str>,
13239 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
13240 {
13241 let mut st = self.state.lock();
13242 let side = Some(RouterSide {
13243 name: Arc::from(name.as_ref()),
13244 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
13245 opts,
13246 });
13247 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
13248 st.sides[id] = side;
13249 id
13250 } else {
13251 let id = st.sides.len();
13252 st.sides.push(side);
13253 id
13254 };
13255 st.side_runtime_stats
13256 .insert(id, Box::<SideRuntimeStatsInner>::default());
13257 st.side_transport
13258 .insert(id, Box::<SideTransportState>::default());
13259 #[cfg(feature = "discovery")]
13260 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13261 id
13262 }
13263
13264 pub fn add_side_packed_with_priority_and_options<N, F>(
13273 &self,
13274 name: N,
13275 tx: F,
13276 opts: RouterSideOptions,
13277 ) -> RouterSideId
13278 where
13279 N: AsRef<str>,
13280 F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
13281 {
13282 let mut st = self.state.lock();
13283 let side = Some(RouterSide {
13284 name: Arc::from(name.as_ref()),
13285 tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
13286 opts,
13287 });
13288 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
13289 st.sides[id] = side;
13290 id
13291 } else {
13292 let id = st.sides.len();
13293 st.sides.push(side);
13294 id
13295 };
13296 st.side_runtime_stats
13297 .insert(id, Box::<SideRuntimeStatsInner>::default());
13298 st.side_transport
13299 .insert(id, Box::<SideTransportState>::default());
13300 #[cfg(feature = "discovery")]
13301 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13302 id
13303 }
13304
13305 pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
13310 where
13311 N: AsRef<str>,
13312 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
13313 {
13314 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
13315 }
13316
13317 pub fn add_side_packet_with_options<N, F>(
13323 &self,
13324 name: N,
13325 tx: F,
13326 opts: RouterSideOptions,
13327 ) -> RouterSideId
13328 where
13329 N: AsRef<str>,
13330 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
13331 {
13332 let mut st = self.state.lock();
13333 let side = Some(RouterSide {
13334 name: Arc::from(name.as_ref()),
13335 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
13336 opts,
13337 });
13338 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
13339 st.sides[id] = side;
13340 id
13341 } else {
13342 let id = st.sides.len();
13343 st.sides.push(side);
13344 id
13345 };
13346 st.side_runtime_stats
13347 .insert(id, Box::<SideRuntimeStatsInner>::default());
13348 st.side_transport
13349 .insert(id, Box::<SideTransportState>::default());
13350 #[cfg(feature = "discovery")]
13351 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13352 id
13353 }
13354
13355 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
13360 #[cfg(feature = "discovery")]
13361 let now_ms = self.clock.now_ms();
13362 {
13363 let mut st = self.state.lock();
13364 #[cfg(feature = "std")]
13365 if st
13366 .side_transport
13367 .get(&side)
13368 .is_some_and(|state| state.tx_callback_gate.is_active())
13369 {
13370 return Err(TelemetryError::Io("side tx busy"));
13371 }
13372 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
13373 if slot.is_none() {
13374 return Err(TelemetryError::BadArg);
13375 }
13376 *slot = None;
13377 while st.sides.last().is_some_and(Option::is_none) {
13378 st.sides.pop();
13379 }
13380 st.route_overrides
13385 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
13386 st.typed_route_overrides
13387 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
13388 st.route_weights
13389 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
13390 st.route_priorities
13391 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
13392 st.source_route_modes.remove(&Some(side));
13393 st.route_selection_cursors.remove(&Some(side));
13394 st.adaptive_route_stats.remove(&side);
13395 #[cfg(feature = "discovery")]
13396 st.discovery_side_throttle.remove(&side);
13397 #[cfg(all(feature = "discovery", feature = "timesync"))]
13398 st.timesync_side_throttle.remove(&side);
13399 st.side_runtime_stats.remove(&side);
13400 st.side_transport.remove(&side);
13401 st.reliable_return_routes
13402 .retain(|_, route| route.side != side);
13403 st.retain_transmit(
13404 |queued| {
13405 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
13406 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
13407 },
13408 );
13409 st.received_queue.retain(|queued| queued.src != Some(side));
13410 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
13411 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
13412 #[cfg(feature = "discovery")]
13413 {
13414 st.discovery_routes.remove(&side);
13415 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13416 }
13417 }
13418 if let Some(queue) = self.isr_rx_queue.as_ref() {
13419 let mut isr_rx = queue.try_lock()?;
13420 isr_rx.retain(|queued| queued.src != Some(side));
13421 }
13422 Ok(())
13423 }
13424
13425 pub fn set_side_ingress_enabled(
13430 &self,
13431 side: RouterSideId,
13432 enabled: bool,
13433 ) -> TelemetryResult<()> {
13434 #[cfg(feature = "discovery")]
13435 let now_ms = self.clock.now_ms();
13436 let mut st = self.state.lock();
13437 let side_ref = st
13438 .sides
13439 .get_mut(side)
13440 .and_then(|side| side.as_mut())
13441 .ok_or(TelemetryError::BadArg)?;
13442 side_ref.opts.ingress_enabled = enabled;
13443 #[cfg(feature = "discovery")]
13444 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13445 Ok(())
13446 }
13447
13448 pub fn set_side_egress_enabled(
13452 &self,
13453 side: RouterSideId,
13454 enabled: bool,
13455 ) -> TelemetryResult<()> {
13456 #[cfg(feature = "discovery")]
13457 let now_ms = self.clock.now_ms();
13458 let mut st = self.state.lock();
13459 let side_ref = st
13460 .sides
13461 .get_mut(side)
13462 .and_then(|side| side.as_mut())
13463 .ok_or(TelemetryError::BadArg)?;
13464 side_ref.opts.egress_enabled = enabled;
13465 if !enabled {
13466 st.retain_transmit(
13467 |queued| {
13468 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
13469 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
13470 },
13471 );
13472 }
13473 #[cfg(feature = "discovery")]
13474 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13475 Ok(())
13476 }
13477
13478 pub fn set_source_route_mode(
13484 &self,
13485 src: Option<RouterSideId>,
13486 mode: RouteSelectionMode,
13487 ) -> TelemetryResult<()> {
13488 #[cfg(feature = "discovery")]
13489 let now_ms = self.clock.now_ms();
13490 let mut st = self.state.lock();
13491 if let Some(src) = src {
13492 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13493 }
13494 st.source_route_modes.insert(src, mode);
13498 st.route_selection_cursors.remove(&src);
13499 #[cfg(feature = "discovery")]
13500 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13501 Ok(())
13502 }
13503
13504 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
13506 #[cfg(feature = "discovery")]
13507 let now_ms = self.clock.now_ms();
13508 let mut st = self.state.lock();
13509 if let Some(src) = src {
13510 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13511 }
13512 st.source_route_modes.remove(&src);
13513 st.route_selection_cursors.remove(&src);
13514 #[cfg(feature = "discovery")]
13515 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13516 Ok(())
13517 }
13518
13519 pub fn set_route_weight(
13524 &self,
13525 src: Option<RouterSideId>,
13526 dst: RouterSideId,
13527 weight: u32,
13528 ) -> TelemetryResult<()> {
13529 #[cfg(feature = "discovery")]
13530 let now_ms = self.clock.now_ms();
13531 let mut st = self.state.lock();
13532 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13533 if let Some(src) = src {
13534 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13535 }
13536 st.route_weights.insert((src, dst), weight);
13537 st.route_selection_cursors.remove(&src);
13538 #[cfg(feature = "discovery")]
13539 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13540 Ok(())
13541 }
13542
13543 pub fn clear_route_weight(
13545 &self,
13546 src: Option<RouterSideId>,
13547 dst: RouterSideId,
13548 ) -> TelemetryResult<()> {
13549 #[cfg(feature = "discovery")]
13550 let now_ms = self.clock.now_ms();
13551 let mut st = self.state.lock();
13552 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13553 if let Some(src) = src {
13554 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13555 }
13556 st.route_weights.remove(&(src, dst));
13557 st.route_selection_cursors.remove(&src);
13558 #[cfg(feature = "discovery")]
13559 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13560 Ok(())
13561 }
13562
13563 pub fn set_route_priority(
13568 &self,
13569 src: Option<RouterSideId>,
13570 dst: RouterSideId,
13571 priority: u32,
13572 ) -> TelemetryResult<()> {
13573 #[cfg(feature = "discovery")]
13574 let now_ms = self.clock.now_ms();
13575 let mut st = self.state.lock();
13576 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13577 if let Some(src) = src {
13578 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13579 }
13580 st.route_priorities.insert((src, dst), priority);
13581 #[cfg(feature = "discovery")]
13582 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13583 Ok(())
13584 }
13585
13586 pub fn clear_route_priority(
13588 &self,
13589 src: Option<RouterSideId>,
13590 dst: RouterSideId,
13591 ) -> TelemetryResult<()> {
13592 #[cfg(feature = "discovery")]
13593 let now_ms = self.clock.now_ms();
13594 let mut st = self.state.lock();
13595 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13596 if let Some(src) = src {
13597 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13598 }
13599 st.route_priorities.remove(&(src, dst));
13600 #[cfg(feature = "discovery")]
13601 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13602 Ok(())
13603 }
13604
13605 pub fn set_route(
13610 &self,
13611 src: Option<RouterSideId>,
13612 dst: RouterSideId,
13613 enabled: bool,
13614 ) -> TelemetryResult<()> {
13615 #[cfg(feature = "discovery")]
13616 let now_ms = self.clock.now_ms();
13617 let mut st = self.state.lock();
13618 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13619 if let Some(src) = src {
13620 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13621 }
13622 st.route_overrides.insert((src, dst), enabled);
13623 #[cfg(feature = "discovery")]
13624 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13625 Ok(())
13626 }
13627
13628 pub fn set_typed_route(
13633 &self,
13634 src: Option<RouterSideId>,
13635 ty: DataType,
13636 dst: RouterSideId,
13637 enabled: bool,
13638 ) -> TelemetryResult<()> {
13639 #[cfg(feature = "discovery")]
13640 let now_ms = self.clock.now_ms();
13641 let mut st = self.state.lock();
13642 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13643 if let Some(src) = src {
13644 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13645 }
13646 st.typed_route_overrides
13647 .insert((src, ty.as_u32(), dst), enabled);
13648 #[cfg(feature = "discovery")]
13649 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13650 Ok(())
13651 }
13652
13653 pub fn clear_typed_route(
13655 &self,
13656 src: Option<RouterSideId>,
13657 ty: DataType,
13658 dst: RouterSideId,
13659 ) -> TelemetryResult<()> {
13660 #[cfg(feature = "discovery")]
13661 let now_ms = self.clock.now_ms();
13662 let mut st = self.state.lock();
13663 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13664 if let Some(src) = src {
13665 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13666 }
13667 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
13668 #[cfg(feature = "discovery")]
13669 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13670 Ok(())
13671 }
13672
13673 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
13675 #[cfg(feature = "discovery")]
13676 let now_ms = self.clock.now_ms();
13677 let mut st = self.state.lock();
13678 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
13679 if let Some(src) = src {
13680 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
13681 }
13682 st.route_overrides.remove(&(src, dst));
13683 #[cfg(feature = "discovery")]
13684 Self::note_discovery_topology_change_locked(&mut st, now_ms);
13685 Ok(())
13686 }
13687
13688 #[cfg(feature = "discovery")]
13690 pub fn announce_discovery(&self) -> TelemetryResult<()> {
13691 self.queue_discovery_announce(true, false)
13692 }
13693
13694 #[cfg(feature = "discovery")]
13696 pub fn announce_leave(&self) -> TelemetryResult<()> {
13697 let sender = self.sender_arc();
13698 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
13699 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
13700 }
13701
13702 #[cfg(feature = "discovery")]
13704 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
13705 let mut emitted = self.poll_discovery_announce()?;
13707 let side_count = self.state.lock().sides.len();
13708 for side in 0..side_count {
13709 if self.state.lock().sides[side].is_some() {
13710 emitted |= self.poll_schema_response_on_side(side)?;
13711 }
13712 }
13713 Ok(emitted)
13714 }
13715
13716 #[cfg(feature = "discovery")]
13717 pub fn request_topology(&self) -> TelemetryResult<()> {
13718 let sender = self.sender_arc();
13719 let pkt =
13720 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
13721 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
13722 }
13723
13724 #[cfg(feature = "discovery")]
13725 pub fn request_schema(&self) -> TelemetryResult<()> {
13726 let sender = self.sender_arc();
13727 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
13728 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
13729 }
13730
13731 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
13738 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
13739 }
13740
13741 pub fn enable_network_variable(
13746 &self,
13747 ty: DataType,
13748 permissions: NetworkVariablePermissions,
13749 ) -> TelemetryResult<()> {
13750 if is_internal_control_type(ty) {
13751 return Err(TelemetryError::InvalidType);
13752 }
13753 let mut st = self.state.lock();
13754 #[cfg(feature = "discovery")]
13755 let newly_advertised = st
13756 .local_services
13757 .as_mut()
13758 .expect("local services enabled")
13759 .managed_variable_types
13760 .insert(ty.as_u32());
13761 #[cfg(not(feature = "discovery"))]
13762 st.local_services
13763 .as_mut()
13764 .expect("local services enabled")
13765 .managed_variable_types
13766 .insert(ty.as_u32());
13767 st.local_services
13768 .as_mut()
13769 .expect("local services enabled")
13770 .managed_variable_permissions
13771 .insert(ty.as_u32(), permissions);
13772 #[cfg(feature = "discovery")]
13773 if newly_advertised {
13774 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13775 }
13776 Ok(())
13777 }
13778
13779 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
13785 where
13786 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
13787 {
13788 if is_internal_control_type(ty) {
13789 return Err(TelemetryError::InvalidType);
13790 }
13791 let mut st = self.state.lock();
13792 #[cfg(feature = "discovery")]
13793 let newly_advertised = st
13794 .local_services
13795 .as_mut()
13796 .expect("local services enabled")
13797 .managed_variable_types
13798 .insert(ty.as_u32());
13799 #[cfg(not(feature = "discovery"))]
13800 st.local_services
13801 .as_mut()
13802 .expect("local services enabled")
13803 .managed_variable_types
13804 .insert(ty.as_u32());
13805 st.local_services
13806 .as_mut()
13807 .expect("local services enabled")
13808 .network_variable_update_handlers
13809 .entry(ty.as_u32())
13810 .or_default()
13811 .push(NetworkVariableUpdateHandler {
13812 handler: Arc::new(f),
13813 });
13814 #[cfg(feature = "discovery")]
13815 if newly_advertised {
13816 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13817 }
13818 Ok(())
13819 }
13820
13821 pub fn disable_managed_variable(&self, ty: DataType) {
13822 let mut st = self.state.lock();
13823 #[cfg(feature = "discovery")]
13824 let was_advertised = st
13825 .local_services
13826 .as_mut()
13827 .expect("local services enabled")
13828 .managed_variable_types
13829 .remove(&ty.as_u32());
13830 #[cfg(not(feature = "discovery"))]
13831 st.local_services
13832 .as_mut()
13833 .expect("local services enabled")
13834 .managed_variable_types
13835 .remove(&ty.as_u32());
13836 st.local_services
13837 .as_mut()
13838 .expect("local services enabled")
13839 .managed_variable_permissions
13840 .remove(&ty.as_u32());
13841 st.local_services
13842 .as_mut()
13843 .expect("local services enabled")
13844 .managed_variable_latest
13845 .remove(&ty.as_u32());
13846 st.local_services
13847 .as_mut()
13848 .expect("local services enabled")
13849 .network_variable_update_handlers
13850 .remove(&ty.as_u32());
13851 #[cfg(feature = "discovery")]
13852 if was_advertised {
13853 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13854 }
13855 }
13856
13857 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
13858 if is_internal_control_type(pkt.data_type()) {
13859 return Err(TelemetryError::InvalidType);
13860 }
13861 pkt.validate()?;
13862 {
13863 let mut st = self.state.lock();
13864 #[cfg(feature = "discovery")]
13865 let newly_advertised = st
13866 .local_services
13867 .as_mut()
13868 .expect("local services enabled")
13869 .managed_variable_types
13870 .insert(pkt.data_type().as_u32());
13871 #[cfg(not(feature = "discovery"))]
13872 st.local_services
13873 .as_mut()
13874 .expect("local services enabled")
13875 .managed_variable_types
13876 .insert(pkt.data_type().as_u32());
13877 #[cfg(feature = "discovery")]
13878 if newly_advertised {
13879 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
13880 }
13881 }
13882 self.cache_managed_variable_packet(&pkt, false)
13883 }
13884
13885 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
13886 self.managed_variable_latest(ty)
13887 }
13888
13889 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
13895 if is_internal_control_type(pkt.data_type()) {
13896 return Err(TelemetryError::InvalidType);
13897 }
13898 pkt.validate()?;
13899 let ty = pkt.data_type();
13900 {
13901 let mut st = self.state.lock();
13902 st.local_services
13903 .as_mut()
13904 .expect("local services enabled")
13905 .managed_variable_types
13906 .insert(ty.as_u32());
13907 let perms = Self::managed_variable_permissions_locked(&st, ty);
13908 if !perms.write {
13909 return Err(TelemetryError::PermissionDenied);
13910 }
13911 }
13912 self.cache_managed_variable_packet(&pkt, false)?;
13913 #[cfg(feature = "discovery")]
13914 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
13915 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
13916 if self.side_tx_active() {
13917 return self.tx_queue_item_with_priority(
13918 item,
13919 false,
13920 crate::transport_priority(DataType::ManagedVariableValue),
13921 );
13922 }
13923 self.tx_item(item)
13924 }
13925
13926 #[cfg(feature = "discovery")]
13933 pub fn get_network_variable(
13934 &self,
13935 ty: DataType,
13936 stale_after_ms: Option<u64>,
13937 ) -> TelemetryResult<Option<Packet>> {
13938 if is_internal_control_type(ty) {
13939 return Err(TelemetryError::InvalidType);
13940 }
13941 {
13942 let mut st = self.state.lock();
13943 st.local_services
13944 .as_mut()
13945 .expect("local services enabled")
13946 .managed_variable_types
13947 .insert(ty.as_u32());
13948 let perms = Self::managed_variable_permissions_locked(&st, ty);
13949 if !perms.read {
13950 return Err(TelemetryError::PermissionDenied);
13951 }
13952 }
13953 let cached = self.managed_variable_latest_with_age(ty);
13954 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
13955 (None, _) => true,
13956 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
13957 (Some(_), None) => false,
13958 };
13959 if needs_refresh {
13960 self.request_managed_variable(ty)?;
13961 }
13962 Ok(cached.map(|(pkt, _age_ms)| pkt))
13963 }
13964
13965 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
13967 if is_internal_control_type(ty) {
13968 return Err(TelemetryError::InvalidType);
13969 }
13970 if !self.can_read_managed_variable(ty) {
13971 return Err(TelemetryError::PermissionDenied);
13972 }
13973 Ok(self.managed_variable_latest(ty))
13974 }
13975
13976 #[cfg(feature = "discovery")]
13977 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
13978 if is_internal_control_type(ty) {
13979 return Err(TelemetryError::InvalidType);
13980 }
13981 if !self.can_read_managed_variable(ty) {
13982 return Err(TelemetryError::PermissionDenied);
13983 }
13984 let sender = self.sender_arc();
13985 let pkt =
13986 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
13987 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
13988 }
13989
13990 #[cfg(feature = "discovery")]
13991 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
13992 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
13993 self.request_managed_variable(ty)
13994 }
13995
13996 #[cfg(feature = "discovery")]
13998 pub fn export_topology(&self) -> TopologySnapshot {
13999 let now_ms = self.clock.now_ms();
14000 let mut st = self.state.lock();
14001 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
14002 Self::note_discovery_topology_change_locked(&mut st, now_ms);
14003 }
14004 let routes = st
14005 .discovery_routes
14006 .iter()
14007 .filter_map(|(&side_id, route)| {
14008 let side = st.sides.get(side_id)?.as_ref()?;
14009 let announcers = route
14010 .announcers
14011 .iter()
14012 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
14013 sender_id: sender_id.clone(),
14014 reachable_endpoints: sender_state
14015 .reachable
14016 .iter()
14017 .copied()
14018 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
14019 .collect(),
14020 reachable_network_variables: sender_state
14021 .reachable_network_variables
14022 .clone(),
14023 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
14024 routers: sender_state.topology_boards.as_ref().clone(),
14025 last_seen_ms: sender_state.last_seen_ms,
14026 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
14027 })
14028 .collect();
14029 Some(TopologySideRoute {
14030 side_id,
14031 side_name: side.name.to_string(),
14032 reachable_endpoints: route
14033 .reachable
14034 .iter()
14035 .copied()
14036 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
14037 .collect(),
14038 reachable_network_variables: route.reachable_network_variables.clone(),
14039 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
14040 announcers,
14041 last_seen_ms: route.last_seen_ms,
14042 age_ms: now_ms.saturating_sub(route.last_seen_ms),
14043 })
14044 })
14045 .collect();
14046 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
14047 let advertised_endpoints =
14048 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
14049 let advertised_timesync_sources =
14050 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
14051 let links = discovery::topology_links_from_boards(&routers);
14052 TopologySnapshot {
14053 advertised_endpoints,
14054 advertised_timesync_sources,
14055 routers,
14056 links,
14057 routes,
14058 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
14059 next_announce_ms: st.discovery_cadence.next_announce_ms,
14060 }
14061 }
14062
14063 #[cfg(feature = "discovery")]
14064 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
14065 let now_ms = self.clock.now_ms();
14066 let st = self.state.lock();
14067 let mut side_ids = Vec::new();
14068 let mut side_names = Vec::new();
14069 let mut last_seen_ms = None::<u64>;
14070 let mut reachable_endpoints = Vec::new();
14071 let mut reachable_timesync_sources = Vec::new();
14072 let mut packets_sent = 0u64;
14073 let mut packets_received = 0u64;
14074 let mut bytes_sent = 0u64;
14075 let mut bytes_received = 0u64;
14076
14077 for (side_id, route) in &st.discovery_routes {
14078 let Some(sender_state) = route.announcers.get(sender_id) else {
14079 continue;
14080 };
14081 side_ids.push(*side_id);
14082 if let Some(side_name) = st
14083 .sides
14084 .get(*side_id)
14085 .and_then(|side| side.as_ref())
14086 .map(|side| side.name.clone())
14087 {
14088 side_names.push(side_name.to_string());
14089 }
14090 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
14091 reachable_endpoints.extend(sender_state.reachable.iter().copied());
14092 reachable_timesync_sources
14093 .extend(sender_state.reachable_timesync_sources.iter().cloned());
14094 if let Some(stats) = st.side_runtime_stats.get(side_id) {
14095 packets_sent = packets_sent.saturating_add(stats.tx_packets);
14096 packets_received = packets_received.saturating_add(stats.rx_packets);
14097 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
14098 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
14099 }
14100 }
14101
14102 if side_ids.is_empty() {
14103 return None;
14104 }
14105 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
14106 crate::small_sort::sort(&mut reachable_endpoints);
14107 reachable_endpoints.dedup();
14108 reachable_timesync_sources.sort_unstable();
14109 reachable_timesync_sources.dedup();
14110 crate::small_sort::sort(&mut side_ids);
14111 side_ids.dedup();
14112 side_names.sort_unstable();
14113 side_names.dedup();
14114 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
14115 Some(ClientStatsSnapshot {
14116 sender_id: sender_id.to_string(),
14117 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
14118 side_ids,
14119 side_names,
14120 last_seen_ms,
14121 age_ms,
14122 reachable_endpoints,
14123 reachable_timesync_sources,
14124 packets_sent,
14125 packets_received,
14126 bytes_sent,
14127 bytes_received,
14128 })
14129 }
14130
14131 pub fn copy_reliable_pending_stats(
14135 &self,
14136 first: usize,
14137 rows: &mut [crate::diagnostics::ReliablePendingRow],
14138 ) -> usize {
14139 rows.fill(crate::diagnostics::ReliablePendingRow::default());
14140 let st = self.state.lock();
14141 for (row, (&id, sent)) in rows
14142 .iter_mut()
14143 .zip(st.end_to_end_reliable_tx.iter().skip(first))
14144 {
14145 row.packet_id = id;
14146 row.last_send_ms = sent.last_send_ms;
14147 row.data_type = Self::item_data_type(&sent.data).map_or(0, DataType::as_u32);
14148 row.pending_count = sent.pending_destinations.len().min(u32::MAX as usize) as u32;
14149 row.retries = sent.retries;
14150 row.flags = u32::from(sent.queued)
14151 | (u32::from(sent.waiting_for_predecessor) << 1)
14152 | (u32::from(
14153 st.reliable_return_routes
14154 .get(&id)
14155 .is_some_and(|route| route.upstream_contract.is_some()),
14156 ) << 2)
14157 | (u32::from(sent.tracked_destinations) << 3);
14158 for (out, owner) in row
14159 .pending_owners
14160 .iter_mut()
14161 .zip(sent.pending_destinations.keys())
14162 {
14163 *out = *owner;
14164 }
14165 }
14166 st.end_to_end_reliable_tx.len()
14167 }
14168
14169 pub fn export_reliable_pending_stats(&self) -> Vec<crate::diagnostics::ReliablePendingStats> {
14173 let st = self.state.lock();
14174 st.end_to_end_reliable_tx
14175 .iter()
14176 .filter_map(|(&packet_id, sent)| {
14177 Some(crate::diagnostics::ReliablePendingStats {
14178 packet_id,
14179 data_type: Self::item_data_type(&sent.data).ok()?,
14180 pending_destinations: sent.pending_destinations.keys().copied().collect(),
14181 retries: sent.retries,
14182 queued: sent.queued,
14183 waiting_for_predecessor: sent.waiting_for_predecessor,
14184 last_send_ms: sent.last_send_ms,
14185 })
14186 })
14187 .collect()
14188 }
14189
14190 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
14191 let now_ms = self.clock.now_ms();
14192 let isr_rx = self
14193 .isr_rx_queue
14194 .as_ref()
14195 .and_then(|queue| queue.snapshot())
14196 .unwrap_or((0, 0));
14197 let st = self.state.lock();
14198
14199 let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
14200 (0, 0)
14201 } else {
14202 st.replay_counters()
14203 };
14204 let mut sides = Vec::new();
14205 for (side_id, side) in st.sides.iter().enumerate() {
14206 let Some(side) = side.as_ref() else { continue };
14207 let empty_stats = SideRuntimeStatsInner::default();
14208 let stats = st
14209 .side_runtime_stats
14210 .get(&side_id)
14211 .map(Box::as_ref)
14212 .unwrap_or(&empty_stats);
14213 let adaptive = st
14214 .adaptive_route_stats
14215 .get(&side_id)
14216 .cloned()
14217 .unwrap_or_default()
14218 .snapshot(now_ms, true);
14219 let (tx_template_count, rx_template_count) = st
14220 .side_transport
14221 .get(&side_id)
14222 .map(|state| (state.tx_template_count(), state.rx_template_count()))
14223 .unwrap_or((0, 0));
14224 let mut data_types: Vec<RuntimeTypeStats> = stats
14225 .data_types
14226 .iter()
14227 .map(|(&ty, item)| {
14228 let item = item.counters();
14229 RuntimeTypeStats {
14230 data_type: DataType(ty),
14231 tx_packets: item.tx_packets,
14232 tx_bytes: item.tx_bytes,
14233 rx_packets: item.rx_packets,
14234 rx_bytes: item.rx_bytes,
14235 relayed_tx_packets: item.relayed_tx_packets,
14236 relayed_tx_bytes: item.relayed_tx_bytes,
14237 relayed_rx_packets: item.relayed_rx_packets,
14238 relayed_rx_bytes: item.relayed_rx_bytes,
14239 tx_retries: item.tx_retries,
14240 handler_failures: item.handler_failures,
14241 }
14242 })
14243 .collect();
14244 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
14245 sides.push(RuntimeSideStats {
14246 side_id,
14247 side_name: side.name.to_string(),
14248 reliable_enabled: side.opts.reliable_enabled,
14249 link_local_enabled: side.opts.link_local_enabled,
14250 header_template_enabled: side.opts.header_template_enabled,
14251 max_frame_bytes: side.opts.max_frame_bytes,
14252 compact_header_target_bytes: side.opts.compact_header_target_bytes,
14253 side_transport_profile: side.opts.effective_transport_profile().as_str(),
14254 ingress_enabled: side.opts.ingress_enabled,
14255 egress_enabled: side.opts.egress_enabled,
14256 tx_packets: stats.tx_packets,
14257 tx_bytes: stats.tx_bytes,
14258 rx_packets: stats.rx_packets,
14259 rx_bytes: stats.rx_bytes,
14260 relayed_tx_packets: stats.relayed_tx_packets,
14261 relayed_tx_bytes: stats.relayed_tx_bytes,
14262 relayed_rx_packets: stats.relayed_rx_packets,
14263 relayed_rx_bytes: stats.relayed_rx_bytes,
14264 local_delivery_packets: stats.local_delivery_packets,
14265 tx_retries: stats.tx_retries,
14266 tx_handler_failures: stats.tx_handler_failures,
14267 local_handler_failures: stats.local_handler_failures,
14268 total_handler_retries: stats.total_handler_retries,
14269 side_transport_full_frames: stats.side_transport_full_frames,
14270 side_transport_compact_frames: stats.side_transport_compact_frames,
14271 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
14272 side_transport_compact_omitted_timestamp_frames: stats
14273 .side_transport_compact_omitted_timestamp_frames,
14274 side_transport_chunk_frames: stats.side_transport_chunk_frames,
14275 side_transport_chunk_reclaims: st
14276 .side_transport
14277 .get(&side_id)
14278 .map_or(0, |state| u64::from(state.rx_chunk_reclaims)),
14279 side_transport_raw_bytes: stats.side_transport_raw_bytes,
14280 side_transport_wire_bytes: stats.side_transport_wire_bytes,
14281 side_transport_bytes_saved: stats.side_transport_bytes_saved,
14282 side_transport_min_compact_overhead_bytes: stats
14283 .side_transport_min_compact_overhead_bytes,
14284 side_transport_max_compact_overhead_bytes: stats
14285 .side_transport_max_compact_overhead_bytes,
14286 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
14287 side_transport_template_evictions: stats.side_transport_template_evictions,
14288 side_transport_tx_template_count: tx_template_count,
14289 side_transport_rx_template_count: rx_template_count,
14290 max_side_transport_templates: side.opts.max_side_transport_templates,
14291 adaptive,
14292 data_types,
14293 });
14294 }
14295
14296 let mut route_modes: Vec<RouteModeStats> = st
14297 .route_selection_cursors
14298 .iter()
14299 .map(|(src, cursor)| RouteModeStats {
14300 src_side_id: *src,
14301 selection_mode: st.source_route_modes.get(src).copied(),
14302 cursor: *cursor,
14303 })
14304 .collect();
14305 for src in st.source_route_modes.keys() {
14306 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
14307 route_modes.push(RouteModeStats {
14308 src_side_id: *src,
14309 selection_mode: st.source_route_modes.get(src).copied(),
14310 cursor: 0,
14311 });
14312 }
14313 }
14314 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
14315
14316 let mut route_overrides: Vec<RouteOverrideStats> = st
14317 .route_overrides
14318 .iter()
14319 .map(|((src, dst), enabled)| RouteOverrideStats {
14320 src_side_id: *src,
14321 dst_side_id: *dst,
14322 enabled: *enabled,
14323 })
14324 .collect();
14325 route_overrides.sort_unstable_by_key(|item| {
14326 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
14327 });
14328
14329 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
14330 .typed_route_overrides
14331 .iter()
14332 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
14333 src_side_id: *src,
14334 data_type: DataType(*ty),
14335 dst_side_id: *dst,
14336 enabled: *enabled,
14337 })
14338 .collect();
14339 typed_route_overrides.sort_unstable_by_key(|item| {
14340 (
14341 item.src_side_id.unwrap_or(usize::MAX),
14342 item.data_type.as_u32(),
14343 item.dst_side_id,
14344 )
14345 });
14346
14347 let mut route_weights: Vec<RouteWeightStats> = st
14348 .route_weights
14349 .iter()
14350 .map(|((src, dst), weight)| RouteWeightStats {
14351 src_side_id: *src,
14352 dst_side_id: *dst,
14353 weight: *weight,
14354 })
14355 .collect();
14356 route_weights.sort_unstable_by_key(|item| {
14357 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
14358 });
14359
14360 let mut route_priorities: Vec<RoutePriorityStats> = st
14361 .route_priorities
14362 .iter()
14363 .map(|((src, dst), priority)| RoutePriorityStats {
14364 src_side_id: *src,
14365 dst_side_id: *dst,
14366 priority: *priority,
14367 })
14368 .collect();
14369 route_priorities.sort_unstable_by_key(|item| {
14370 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
14371 });
14372
14373 #[cfg(feature = "discovery")]
14374 let discovery = DiscoveryRuntimeStats {
14375 route_count: st.discovery_routes.len(),
14376 announcer_count: st
14377 .discovery_routes
14378 .values()
14379 .map(|route| route.announcers.len())
14380 .sum(),
14381 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
14382 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
14383 };
14384 #[cfg(not(feature = "discovery"))]
14385 let discovery = DiscoveryRuntimeStats {
14386 route_count: 0,
14387 announcer_count: 0,
14388 current_announce_interval_ms: None,
14389 next_announce_ms: None,
14390 };
14391
14392 RuntimeStatsSnapshot {
14393 sides,
14394 route_modes,
14395 route_overrides,
14396 typed_route_overrides,
14397 route_weights,
14398 route_priorities,
14399 queues: QueueRuntimeStats {
14400 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
14401 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
14402 tx_len: st.transmit_queue.len().saturating_sub(replay_len),
14403 tx_bytes: st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
14404 replay_len,
14405 replay_bytes,
14406 recent_rx_len: st.recent_rx.len(),
14407 recent_rx_bytes: st.recent_rx.bytes_used(),
14408 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
14409 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
14410 shared_queue_bytes_used: st.shared_queue_bytes_used(),
14411 },
14412 reliable: ReliableRuntimeStats {
14413 reliable_return_route_count: st.reliable_return_routes.len(),
14414 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
14415 end_to_end_pending_destination_count: st
14416 .end_to_end_reliable_tx
14417 .values()
14418 .map(|sent| sent.pending_destinations.len())
14419 .sum(),
14420 end_to_end_acked_cache_count: st
14421 .forwarded_receipts
14422 .as_ref()
14423 .map_or(0, |cache| cache.destinations.len()),
14424 },
14425 discovery,
14426 total_handler_failures: st.total_handler_failures,
14427 total_handler_retries: st.total_handler_retries,
14428 }
14429 }
14430
14431 #[cfg(feature = "std")]
14440 pub fn has_pending_reliable_tx_type(&self, ty: DataType) -> bool {
14441 let st = self.state.lock();
14442 if self
14443 .active_queued_tx
14444 .load(core::sync::atomic::Ordering::Acquire)
14445 != 0
14446 {
14447 return true;
14448 }
14449 let matches_data = |data: &RouterItem| match data {
14450 RouterItem::Packet(packet) => packet.data_type() == ty,
14451 RouterItem::Packed(_) => true,
14452 };
14453 st.transmit_queue.iter().any(|queued| {
14454 if queued.accepted_type != 0 {
14455 return queued.accepted_type == ty.as_u32();
14456 }
14457 match &queued.item {
14458 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
14459 matches_data(data)
14460 }
14461 RouterTxItem::ReliableReplay { ty: found, .. } => *found == ty,
14462 RouterTxItem::EndToEndReplay { packet_id } => st
14463 .end_to_end_reliable_tx
14464 .get(packet_id)
14465 .is_some_and(|owner| matches_data(&owner.data)),
14466 }
14467 }) || st
14468 .end_to_end_reliable_tx
14469 .values()
14470 .any(|owner| matches_data(&owner.data))
14471 || st
14472 .reliable_tx
14473 .iter()
14474 .any(|((_, found), owner)| *found == ty.as_u32() && !owner.sent.is_empty())
14475 }
14476
14477 #[cfg(feature = "std")]
14480 fn track_queued_tx(&self, work: Option<TxWork>) -> Option<(TxWork, QueuedTxActivityGuard<'_>)> {
14481 work.map(|work| {
14482 self.active_queued_tx
14483 .fetch_add(1, core::sync::atomic::Ordering::AcqRel);
14484 (work, QueuedTxActivityGuard(&self.active_queued_tx))
14485 })
14486 }
14487
14488 pub fn has_queued_work(&self) -> bool {
14495 if self
14496 .isr_rx_queue
14497 .as_ref()
14498 .is_some_and(|queue| queue.try_lock().map_or(true, |queue| !queue.is_empty()))
14499 {
14500 return true;
14501 }
14502 let state = self.state.lock();
14503 !state.received_queue.is_empty() || !state.transmit_queue.is_empty()
14504 }
14505
14506 pub fn queue_counters(&self) -> [u32; 16] {
14512 let now = self.clock.now_ms();
14513 let st = self.state.lock();
14514 let ty_of = |item: &RouterTxItem| match item {
14515 RouterTxItem::Broadcast(RouterItem::Packet(packet))
14516 | RouterTxItem::ToSide {
14517 data: RouterItem::Packet(packet),
14518 ..
14519 } => Some(packet.data_type()),
14520 RouterTxItem::Broadcast(RouterItem::Packed(_))
14521 | RouterTxItem::ToSide {
14522 data: RouterItem::Packed(_),
14523 ..
14524 }
14525 | RouterTxItem::ReliableReplay { .. } => None,
14526 RouterTxItem::EndToEndReplay { packet_id } => st
14527 .end_to_end_reliable_tx
14528 .get(packet_id)
14529 .and_then(|pending| match &pending.data {
14530 RouterItem::Packet(packet) => Some(packet.data_type()),
14531 RouterItem::Packed(_) => None,
14532 }),
14533 };
14534 let sum_cost = |total: usize, cost: usize| total.saturating_add(cost);
14535 let front = st.transmit_queue.iter().next();
14536 let ack_count = st
14537 .transmit_queue
14538 .iter()
14539 .filter(|q| {
14540 matches!(
14541 ty_of(&q.item),
14542 Some(
14543 DataType::ReliableAck
14544 | DataType::ReliablePartialAck
14545 | DataType::ReliablePacketRequest
14546 )
14547 )
14548 })
14549 .count();
14550 let schema_count = st
14551 .transmit_queue
14552 .iter()
14553 .filter(|q| ty_of(&q.item) == Some(DataType::DiscoverySchema))
14554 .count();
14555 [
14556 1,
14557 st.received_queue.len() as u32,
14558 st.transmit_queue.len() as u32,
14559 st.received_queue
14560 .iter()
14561 .map(ByteCost::byte_cost)
14562 .fold(0, sum_cost) as u32,
14563 st.transmit_queue
14564 .iter()
14565 .map(ByteCost::byte_cost)
14566 .fold(0, sum_cost) as u32,
14567 front
14568 .and_then(|q| ty_of(&q.item))
14569 .map_or(0, DataType::as_u32),
14570 front.map_or(0, |q| q.priority.into()),
14571 front.map_or(0, |q| {
14572 q.retry_after_ms.saturating_sub(now).min(u32::MAX.into()) as u32
14573 }),
14574 ack_count as u32,
14575 schema_count as u32,
14576 st.transmit_queue
14577 .iter()
14578 .map(|q| u32::from(q.priority))
14579 .min()
14580 .unwrap_or(0),
14581 st.end_to_end_reliable_tx.len() as u32,
14582 st.end_to_end_reliable_tx
14583 .values()
14584 .map(|entry| match &entry.data {
14585 RouterItem::Packet(packet) => packet.byte_cost(),
14586 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
14587 })
14588 .fold(0, sum_cost) as u32,
14589 front.map_or(0, |q| match q.item {
14590 RouterTxItem::Broadcast(_) => 1,
14591 RouterTxItem::ToSide { .. } => 2,
14592 RouterTxItem::ReliableReplay { .. } => 3,
14593 RouterTxItem::EndToEndReplay { .. } => 4,
14594 }),
14595 now as u32,
14596 (now >> 32) as u32,
14597 ]
14598 }
14599
14600 pub fn memory_counters(&self) -> [u32; 16] {
14601 let st = self.state.lock();
14602 #[cfg(feature = "discovery")]
14603 let discovery = st.discovery_bytes_used();
14604 #[cfg(not(feature = "discovery"))]
14605 let discovery = 0;
14606 let templates = st
14607 .side_transport
14608 .values()
14609 .map(|side| side.tx_templates.len() + side.rx_templates_by_id.len())
14610 .sum::<usize>();
14611 let template_bytes = st
14612 .side_transport
14613 .values()
14614 .flat_map(|side| {
14615 side.tx_templates
14616 .values()
14617 .chain(side.rx_templates_by_id.values())
14618 })
14619 .map(|t| size_of::<SideHeaderTemplate>() + t.prefix().len() + t.between().len())
14620 .sum::<usize>();
14621 let chunks = st
14622 .side_transport
14623 .values()
14624 .flat_map(|side| side.rx_chunks.values())
14625 .flat_map(|assembly| assembly.received.iter().flatten())
14626 .map(|bytes| bytes.len())
14627 .sum::<usize>();
14628 let type_stats = st
14629 .side_runtime_stats
14630 .values()
14631 .map(|side| side.data_types.len())
14632 .sum::<usize>();
14633 let managed = st.local_cache_bytes();
14634 let pending = st
14635 .end_to_end_reliable_tx
14636 .values()
14637 .map(|entry| match &entry.data {
14638 RouterItem::Packet(packet) => packet.byte_cost(),
14639 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
14640 })
14641 .sum::<usize>();
14642 [
14643 1,
14644 st.memory.max_queue_budget,
14645 st.shared_queue_bytes_used(),
14646 st.received_queue.bytes_used(),
14647 st.transmit_queue.bytes_used(),
14648 st.received_queue.capacity() * size_of::<RouterRxItem>()
14649 + st.transmit_queue.capacity() * size_of::<TxQueued>()
14650 + st.recent_rx.capacity() * size_of::<u64>(),
14651 st.reliable_rx_buffered_bytes(),
14652 discovery,
14653 crate::config::schema_bytes_used(),
14654 managed,
14655 pending,
14656 templates,
14657 template_bytes,
14658 chunks,
14659 type_stats,
14660 st.side_runtime_stats
14661 .values()
14662 .flat_map(|side| side.data_types.values())
14663 .map(|stats| size_of::<TypeRuntimeStatsInner>() + stats.heap_bytes())
14664 .sum::<usize>(),
14665 ]
14666 .map(|value| u32::try_from(value).unwrap_or(u32::MAX))
14667 }
14668
14669 pub fn export_memory_layout_json(&self) -> String {
14671 let isr_rx = self
14672 .isr_rx_queue
14673 .as_ref()
14674 .and_then(|queue| queue.snapshot())
14675 .unwrap_or((0, 0));
14676 let st = self.state.lock();
14677 #[cfg(feature = "discovery")]
14678 let discovery_bytes = st.discovery_bytes_used();
14679 #[cfg(not(feature = "discovery"))]
14680 let discovery_bytes = 0usize;
14681 let schema_bytes = crate::config::schema_bytes_used();
14682 let network_variable_cache_bytes = st.local_cache_bytes();
14683 let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
14684 (0, 0)
14685 } else {
14686 st.replay_counters()
14687 };
14688 let mut out = String::new();
14689 let memory = st.memory;
14690 let _ = fmt::Write::write_fmt(
14691 &mut out,
14692 format_args!(
14693 "{{\"kind\":\"{}\",\
14694 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
14695 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
14696 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
14697 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
14698 \"replay_queue_bytes_used\":{},\"replay_queue_bytes_allocated\":{},\"replay_queue_len\":{},\
14699 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
14700 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
14701 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
14702 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
14703 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
14704 if self.cfg.role.local_services() {
14705 "router"
14706 } else {
14707 "relay"
14708 },
14709 st.shared_queue_bytes_used(),
14710 memory.max_queue_budget,
14711 st.received_queue.bytes_used(),
14712 st.received_queue.max_bytes(),
14713 st.received_queue.len(),
14714 isr_rx.1,
14715 memory.max_queue_budget,
14716 isr_rx.0,
14717 st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
14718 st.transmit_queue.max_bytes(),
14719 st.transmit_queue.len().saturating_sub(replay_len),
14720 replay_bytes,
14721 memory.max_queue_budget,
14722 replay_len,
14723 st.recent_rx.bytes_used(),
14724 st.recent_rx.max_bytes(),
14725 st.recent_rx.len(),
14726 st.reliable_rx_buffered_bytes(),
14727 memory.max_queue_budget,
14728 st.reliable_rx_buffer_len(),
14729 discovery_bytes,
14730 memory.max_queue_budget,
14731 schema_bytes,
14732 memory.max_queue_budget,
14733 network_variable_cache_bytes,
14734 memory.max_queue_budget,
14735 st.local_cache_len(),
14736 ),
14737 );
14738 out
14739 }
14740
14741 #[cfg(test)]
14742 #[cfg(feature = "discovery")]
14743 pub(crate) fn debug_end_to_end_pending_destination_count(
14744 &self,
14745 packet_id: u64,
14746 ) -> Option<usize> {
14747 let st = self.state.lock();
14748 st.end_to_end_reliable_tx
14749 .get(&packet_id)
14750 .map(|sent| sent.pending_destinations.len())
14751 }
14752
14753 #[cfg(test)]
14754 #[cfg(feature = "discovery")]
14755 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
14756 let st = self.state.lock();
14757 st.end_to_end_reliable_tx.len()
14758 }
14759
14760 #[cfg(test)]
14761 #[cfg(feature = "discovery")]
14762 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
14763 let st = self.state.lock();
14764 st.reliable_return_routes.len()
14765 }
14766
14767 #[cfg(test)]
14768 pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
14769 let st = self.state.lock();
14770 (st.sides.len(), st.sides.capacity())
14771 }
14772
14773 #[cfg(test)]
14774 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
14775 let st = self.state.lock();
14776 (
14777 st.received_queue.len(),
14778 st.transmit_queue.len(),
14779 st.recent_rx.len(),
14780 )
14781 }
14782
14783 #[cfg(test)]
14784 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
14785 let st = self.state.lock();
14786 st.shared_queue_bytes_used()
14787 }
14788
14789 #[cfg(test)]
14790 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
14791 let st = self.state.lock();
14792 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
14793 }
14794
14795 fn get_hash(item: &RouterItem) -> u64 {
14799 match item {
14800 RouterItem::Packet(pkt) => pkt.packet_id(),
14801 RouterItem::Packed(bytes) => {
14802 match wire_format::packet_id_from_wire(bytes.as_ref()) {
14803 Ok(id) => id,
14804 Err(_e) => {
14805 let h: u64 = 0x9E37_79B9_7F4A_7C15;
14808 hash_bytes_u64(h, bytes.as_ref())
14809 }
14810 }
14811 }
14812 }
14813 }
14814
14815 fn remove_pkt_id(&self, item: &RouterItem) {
14817 let hash = Self::get_hash(item);
14818 let mut st = self.state.lock();
14819 st.recent_rx.remove_value(&hash);
14820 }
14821
14822 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
14825 let id = Self::get_hash(item);
14826 let mut st = self.state.lock();
14827 if st.recent_rx.contains(&id)
14828 || st
14829 .reliable_return_routes
14830 .get(&id)
14831 .is_some_and(|route| route.delivery_completed)
14832 {
14833 Ok(true)
14834 } else {
14835 st.push_recent_rx(id)?;
14836 Ok(false)
14837 }
14838 }
14839
14840 fn handle_callback_error(
14845 &self,
14846 pkt: &Packet,
14847 dest: Option<DataEndpoint>,
14848 e: TelemetryError,
14849 called_from_queue: bool,
14850 ) -> TelemetryResult<()> {
14851 if !self.cfg.role.local_services() {
14852 return Err(e);
14853 }
14854
14855 if matches!(&e, TelemetryError::Io("memory pressure")) {
14858 return Err(e);
14859 }
14860
14861 let device = self.sender_arc();
14862 let error_msg = match dest {
14863 Some(failed_local) => format!(
14864 "Handler for endpoint {:?} failed on device {:?}: {:?}",
14865 failed_local, device, e
14866 ),
14867 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
14868 };
14869
14870 let mut recipients: Vec<DataEndpoint> = pkt
14871 .endpoints()
14872 .iter()
14873 .copied()
14874 .filter(|&ep| self.cfg.is_local_endpoint(ep))
14875 .collect();
14876 crate::small_sort::sort(&mut recipients);
14877 recipients.dedup();
14878
14879 if let Some(failed_local) = dest {
14880 recipients.retain(|&ep| ep != failed_local);
14881 }
14882
14883 if recipients.is_empty() && dest.is_none() {
14887 fallback_stdout(&error_msg);
14888 return Ok(());
14889 }
14890
14891 if recipients.is_empty() {
14894 recipients = pkt.endpoints().to_vec();
14895 crate::small_sort::sort(&mut recipients);
14896 recipients.dedup();
14897 if let Some(failed_local) = dest {
14898 recipients.retain(|&ep| ep != failed_local);
14899 }
14900 }
14901
14902 if recipients.is_empty() {
14903 fallback_stdout(&error_msg);
14904 return Ok(());
14905 }
14906
14907 let payload = make_error_payload(&error_msg);
14908
14909 let sender = self.sender_arc();
14910 let error_pkt = Packet::new(
14911 DataType::TelemetryError,
14912 &recipients,
14913 sender.as_ref(),
14914 self.packet_timestamp_ms(),
14915 crate::shared_bytes::convert(payload),
14916 )?;
14917
14918 self.emit_internal_tx(
14919 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
14920 false,
14921 called_from_queue,
14922 )
14923 }
14924
14925 #[inline]
14929 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
14930 self.process_tx_queue_with_timeout(0)
14931 }
14932
14933 #[inline]
14935 pub fn process_all_queues(&self) -> TelemetryResult<()> {
14936 self.process_all_queues_with_timeout(0)
14937 }
14938
14939 #[inline]
14941 pub fn clear_queues(&self) {
14942 let mut st = self.state.lock();
14943 st.retain_transmit(|_| false);
14944 st.transmit_queue.cancel_deferred_turn();
14945 #[cfg(feature = "discovery")]
14946 for response in st.discovery_side_throttle.values_mut() {
14947 if response.schema_response_active {
14948 response.schema_response_active = false;
14949 response.schema_response_phase = SchemaResponsePhase::Idle;
14950 response.schema_response_pending = true;
14951 }
14952 }
14953 st.has_ordered_retry = false;
14954 st.received_queue.clear();
14955 st.rx_priority_burst = 0;
14956 drop(st);
14957 let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
14958 }
14959
14960 #[inline]
14962 pub fn clear_rx_queue(&self) {
14963 let mut st = self.state.lock();
14964 st.received_queue.clear();
14965 st.rx_priority_burst = 0;
14966 drop(st);
14967 let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
14968 }
14969
14970 #[inline]
14972 pub fn clear_tx_queue(&self) {
14973 let mut st = self.state.lock();
14974 st.retain_transmit(|_| false);
14975 st.transmit_queue.cancel_deferred_turn();
14976 #[cfg(feature = "discovery")]
14977 for response in st.discovery_side_throttle.values_mut() {
14978 if response.schema_response_active {
14979 response.schema_response_active = false;
14980 response.schema_response_phase = SchemaResponsePhase::Idle;
14981 response.schema_response_pending = true;
14982 }
14983 }
14984 st.tx_priority_burst = 0;
14985 st.has_ordered_retry = false;
14986 }
14987
14988 #[inline(never)]
14989 fn pop_transmit_queue_locked(
14990 st: &mut RouterInner,
14991 now_ms: u64,
14992 control_only: bool,
14993 ) -> Option<TxWork> {
14994 let slow = st.has_ordered_retry;
14995 let mut next_token = st.ordered_claim_seq.wrapping_add(1).max(1);
14996 if slow {
14997 while st.claim_index(next_token).is_some() {
14998 next_token = next_token.wrapping_add(1).max(1);
14999 }
15000 }
15001 let (selected, owed_turn) = st.transmit_queue.select_front_fair_deferred(
15002 &mut st.tx_priority_burst,
15003 MAX_PRIORITY_BURST,
15004 |item| item.priority,
15005 |_queue, item| {
15006 !matches!(
15007 item.delivery_state,
15008 QueuedDelivery::ClaimedOrdered
15009 | QueuedDelivery::ClaimedUnordered
15010 | QueuedDelivery::OrderedTail
15011 ) && item.retry_after_ms <= now_ms
15012 && (!control_only || item.priority >= 200)
15013 },
15014 )?;
15015 let current = st.transmit_queue.get(selected)?;
15016 let reliable = RouterInner::reliable_original_type(current).is_some();
15017 let ordered = reliable && RouterInner::ordered_original_type(current).is_some();
15018 let (queued, claim) = if reliable {
15019 let queued = current.clone();
15020 st.transmit_queue.update_metadata(selected, |item| {
15021 item.delivery_state = if ordered {
15022 QueuedDelivery::ClaimedOrdered
15023 } else {
15024 QueuedDelivery::ClaimedUnordered
15025 };
15026 item.deferred_since_ms = u64::from(next_token);
15027 });
15028 st.ordered_claim_seq = next_token;
15029 (
15030 queued,
15031 Some(TxClaim {
15032 token: next_token,
15033 ordered,
15034 }),
15035 )
15036 } else {
15037 (st.transmit_queue.remove_pos(selected)?, None)
15038 };
15039 if claim.is_some() {
15040 st.has_ordered_retry = true;
15041 } else if slow {
15042 st.refresh_retry_hint();
15043 }
15044 Some(TxWork {
15045 queued,
15046 claim,
15047 owed_turn,
15048 })
15049 }
15050
15051 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
15054 self.expire_side_chunks();
15055 let start = self.clock.now_ms();
15056 loop {
15057 let received_control = crate::memory_admission::allow_drain(
15058 self.drain_queued_control_rx_before_tx(start, timeout_ms),
15059 )?;
15060 let control_budget_used = received_control
15061 && timeout_ms != 0
15062 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
15063 if !control_budget_used {
15066 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
15067 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
15068 }
15069
15070 let pkt_opt = {
15071 let mut st = self.state.lock();
15072 let work = Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false);
15073 #[cfg(feature = "std")]
15074 let work = self.track_queued_tx(work);
15075 work
15076 };
15077 let Some(pkt) = pkt_opt else { break };
15078 #[cfg(feature = "std")]
15079 let (pkt, _activity) = pkt;
15080 self.dispatch_queued_item(pkt)?;
15081 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
15082 break;
15083 }
15084 }
15085 Ok(())
15086 }
15087
15088 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
15091 #[cfg(feature = "timesync")]
15092 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
15093 #[cfg(feature = "discovery")]
15094 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15095 self.process_tx_queue_with_timeout_impl(timeout_ms)
15096 }
15097
15098 pub fn dispatch_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
15104 self.process_tx_queue_with_timeout_impl(timeout_ms)
15105 }
15106
15107 #[inline]
15109 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
15110 self.rx_item(&item, true)
15111 }
15112
15113 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
15116 self.expire_side_chunks();
15117 let start = self.clock.now_ms();
15118 loop {
15119 let item_opt = self
15120 .isr_rx_queue
15121 .as_ref()
15122 .and_then(|queue| queue.pop_front().unwrap_or(None))
15123 .or_else(|| {
15124 let mut st = self.state.lock();
15125 st.pop_received_fair()
15126 });
15127 let Some(item) = item_opt else { break };
15128 self.process_rx_queue_item(item)?;
15129 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
15130 break;
15131 }
15132 }
15133 Ok(())
15134 }
15135
15136 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
15139 #[cfg(feature = "timesync")]
15140 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
15141 #[cfg(feature = "discovery")]
15142 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15143 self.process_rx_queue_with_timeout_impl(timeout_ms)
15144 }
15145
15146 fn process_one_queued_rx(&self) -> TelemetryResult<bool> {
15149 let item = self
15150 .isr_rx_queue
15151 .as_ref()
15152 .and_then(|queue| queue.pop_front().unwrap_or(None))
15153 .or_else(|| self.state.lock().pop_received_fair());
15154 let Some(item) = item else {
15155 return Ok(false);
15156 };
15157 self.process_rx_queue_item(item)?;
15158 Ok(true)
15159 }
15160
15161 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
15162 self.expire_side_chunks();
15163 let receive_first = !self.cfg.role.local_services();
15168 let start = self.clock.now_ms();
15169 loop {
15170 let received_control = crate::memory_admission::allow_drain(
15171 self.drain_queued_control_rx_before_tx(start, timeout_ms),
15172 )?;
15173 let mut did_any = received_control;
15174 let control_budget_used = received_control
15175 && timeout_ms != 0
15176 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
15177 if !control_budget_used {
15178 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
15179 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
15180 }
15181
15182 if receive_first && !control_budget_used {
15183 did_any |= self.process_one_queued_rx()?;
15184 }
15185
15186 if let Some(pkt) = {
15187 let mut st = self.state.lock();
15188 let now = self.clock.now_ms();
15189 let work = Self::pop_transmit_queue_locked(&mut st, now, false);
15192 #[cfg(feature = "std")]
15193 let work = self.track_queued_tx(work);
15194 work
15195 } {
15196 #[cfg(feature = "std")]
15197 let (pkt, _activity) = pkt;
15198 self.dispatch_queued_item(pkt)?;
15199 did_any = true;
15200 }
15201 if control_budget_used {
15203 return Ok(());
15204 }
15205 if !receive_first {
15206 did_any |= self.process_one_queued_rx()?;
15207 }
15208 if !did_any
15209 || (timeout_ms != 0
15210 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms))
15211 {
15212 break;
15213 }
15214 }
15215
15216 Ok(())
15217 }
15218
15219 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
15222 #[cfg(feature = "timesync")]
15223 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
15224 #[cfg(feature = "discovery")]
15225 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15226 self.process_all_queues_with_timeout_impl(timeout_ms)
15227 }
15228
15229 pub fn drain_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
15234 self.process_all_queues_with_timeout_impl(timeout_ms)
15235 }
15236
15237 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
15242 #[cfg(feature = "timesync")]
15243 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
15244
15245 #[cfg(feature = "discovery")]
15246 {
15247 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15248 }
15249
15250 self.process_all_queues_with_timeout_impl(timeout_ms)
15251 }
15252
15253 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
15258 #[cfg(feature = "discovery")]
15259 {
15260 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15261 }
15262
15263 self.process_all_queues_with_timeout_impl(timeout_ms)
15264 }
15265
15266 #[inline]
15268 fn tx_queue_item_with_flags(
15269 &self,
15270 item: RouterTxItem,
15271 ignore_local: bool,
15272 ) -> TelemetryResult<()> {
15273 self.tx_queue_item_with_flags_at(item, ignore_local, 0)
15274 }
15275
15276 #[cfg(feature = "compression")]
15277 fn retry_codec_owner(
15278 &self,
15279 item: RouterTxItem,
15280 ignore_local: bool,
15281 reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
15282 ) -> TelemetryResult<()> {
15283 self.retry_codec_owner_at_priority(item, ignore_local, reservation, 0)
15284 }
15285
15286 #[cfg(feature = "compression")]
15287 fn retry_codec_owner_at_priority(
15288 &self,
15289 item: RouterTxItem,
15290 ignore_local: bool,
15291 reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
15292 priority_floor: u8,
15293 ) -> TelemetryResult<()> {
15294 let priority = priority_floor.max(match &item {
15295 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
15296 _ => return Err(TelemetryError::BadArg),
15297 });
15298 #[cfg(feature = "discovery")]
15299 let schema_owner = RouterInner::schema_response_owner(&item);
15300 let mut state = self.state.lock();
15301 if let Some(reservation) = reservation {
15302 reservation.release_locked(&mut state);
15303 }
15304 let result = state.push_transmit(TxQueued {
15305 disposable: true,
15306 item,
15307 ignore_local,
15308 priority,
15309 delivery_state: QueuedDelivery::Untracked,
15310 accepted_type: 0,
15311 deferred_since_ms: 0,
15312 retry_after_ms: self.clock.now_ms().saturating_add(1),
15313 });
15314 #[cfg(feature = "discovery")]
15315 if result.is_err() {
15316 RouterInner::schema_response_owner_lost(
15317 &mut state.discovery_side_throttle,
15318 schema_owner,
15319 );
15320 }
15321 result
15322 }
15323
15324 fn claim_schema_is_current(
15325 &self,
15326 ty: DataType,
15327 claim: Option<TxClaim>,
15328 ) -> TelemetryResult<bool> {
15329 let Some(claim) = claim else {
15330 return Ok(true);
15331 };
15332 let expected = if claim.ordered {
15333 crate::ReliableMode::Ordered
15334 } else {
15335 crate::ReliableMode::Unordered
15336 };
15337 let current = reliable_mode(ty);
15338 let mut st = self.state.lock();
15339 let owned = st.claim_index(claim.token).is_some();
15340 if !owned {
15341 return Ok(false);
15342 }
15343 if current == expected {
15344 return Ok(true);
15345 }
15346 st.cancel_accepted_type(ty.as_u32());
15347 st.refresh_retry_hint();
15348 Err(TelemetryError::HandlerError(
15349 "accepted reliable schema changed before dispatch",
15350 ))
15351 }
15352
15353 fn mark_claim_local_delivered(&self, claim: Option<TxClaim>) -> bool {
15354 let Some(claim) = claim else {
15355 return true;
15356 };
15357 let mut st = self.state.lock();
15358 let index = st.claim_index(claim.token);
15359 let Some(index) = index else {
15360 return false;
15361 };
15362 st.transmit_queue
15363 .update_metadata(index, |item| item.ignore_local = true);
15364 true
15365 }
15366
15367 fn dispatch_queued_item(&self, work: TxWork) -> TelemetryResult<()> {
15368 let owed_turn = work.owed_turn;
15369 let queued = work.queued;
15370 let since = matches!(
15371 queued.delivery_state,
15372 QueuedDelivery::DeferredOrdered | QueuedDelivery::DeferredUnordered
15373 )
15374 .then_some(queued.deferred_since_ms);
15375 let mut context = DispatchContext {
15376 priority: queued.priority,
15377 claim: work.claim,
15378 since,
15379 completion_flags: 0,
15380 refused: false,
15381 admitted: false,
15382 };
15383 let schema = self.claim_schema_is_current(DataType(queued.accepted_type), work.claim);
15384 if !matches!(schema, Ok(true)) {
15385 self.finish_dispatch_turn(owed_turn, &context);
15386 return schema.map(|_| ());
15387 }
15388 let mut result =
15389 self.tx_item_impl_with_retry(queued.item, queued.ignore_local, true, &mut context);
15390 if let Some(claim) = work.claim {
15391 let mut st = self.state.lock();
15392 if result.is_err() && !context.is_terminal() && !context.owner_transferred() {
15393 let Some(index) = st.claim_index(claim.token) else {
15394 drop(st);
15395 self.finish_dispatch_turn(owed_turn, &context);
15396 return result;
15397 };
15398 let original = st.transmit_queue.get(index).expect("claimed owner");
15399 if !original.ignore_local
15400 && let RouterTxItem::Broadcast(data) = &original.item
15401 {
15402 let id = Self::get_hash(data);
15403 st.recent_rx.remove_value(&id);
15404 }
15405 drop(st);
15406 result = self.defer_accepted_original(&mut context);
15407 if result.is_ok() {
15408 self.finish_dispatch_turn(owed_turn, &context);
15409 return Ok(());
15410 }
15411 st = self.state.lock();
15412 }
15413 if let Some(index) = st.claim_index(claim.token) {
15414 if context.is_terminal() && claim.ordered {
15415 let ty = st
15416 .transmit_queue
15417 .get(index)
15418 .expect("claimed owner")
15419 .accepted_type;
15420 st.cancel_accepted_type(ty);
15421 } else {
15422 let ty = st
15423 .transmit_queue
15424 .get(index)
15425 .expect("claimed owner")
15426 .accepted_type;
15427 let _ = st.transmit_queue.remove_pos(index);
15428 if claim.ordered {
15429 let next = st.transmit_queue.iter().position(|item| {
15432 item.delivery_state == QueuedDelivery::OrderedTail
15433 && item.accepted_type == ty
15434 });
15435 if let Some(next) = next {
15436 st.transmit_queue.update_metadata(next, |item| {
15437 item.delivery_state = QueuedDelivery::Ordered
15438 });
15439 }
15440 }
15441 }
15442 }
15443 st.refresh_retry_hint();
15444 }
15445 self.finish_dispatch_turn(owed_turn, &context);
15446 result
15447 }
15448
15449 fn finish_dispatch_turn(&self, owed_turn: bool, context: &DispatchContext) {
15450 let mut guard = self.state.lock();
15451 let st = &mut *guard;
15452 st.transmit_queue.finish_deferred_turn(
15453 context.priority,
15454 owed_turn,
15455 context.admitted,
15456 context.refused,
15457 &mut st.tx_priority_burst,
15458 );
15459 }
15460
15461 fn expire_deferred_original(
15462 &self,
15463 ty: DataType,
15464 since: u64,
15465 ordered: bool,
15466 ) -> TelemetryResult<()> {
15467 let lifetime = runtime_reliable_retransmit_ms()
15468 .saturating_mul(u64::from(runtime_reliable_max_retries()) + 1);
15469 let now = self.clock.now_ms();
15470 if now >= since && now - since < lifetime {
15471 return Ok(());
15472 }
15473 if ordered {
15474 let mut st = self.state.lock();
15475 st.cancel_accepted_type(ty.as_u32());
15476 st.refresh_retry_hint();
15477 }
15478 Err(TelemetryError::HandlerError(
15479 "accepted reliable delivery expired before dispatch",
15480 ))
15481 }
15482
15483 fn defer_accepted_original(&self, context: &mut DispatchContext) -> TelemetryResult<()> {
15484 context.refused = true;
15485 let Some(claim) = context.claim else {
15486 return Err(TelemetryError::Io("reliable owner unavailable"));
15487 };
15488 let now = self.clock.now_ms();
15489 let Some(retry_after_ms) = now.checked_add(1) else {
15490 context.mark_terminal();
15491 return Err(TelemetryError::HandlerError(
15492 "accepted reliable clock exhausted",
15493 ));
15494 };
15495 let mut st = self.state.lock();
15496 let Some(index) = st.claim_index(claim.token) else {
15497 return Ok(());
15498 };
15499 st.transmit_queue.update_metadata(index, |item| {
15500 item.delivery_state = if claim.ordered {
15501 QueuedDelivery::DeferredOrdered
15502 } else {
15503 QueuedDelivery::DeferredUnordered
15504 };
15505 item.deferred_since_ms = context.since.unwrap_or(now);
15506 item.retry_after_ms = retry_after_ms;
15507 });
15508 st.has_ordered_retry = true;
15509 Ok(())
15510 }
15511
15512 #[cfg(feature = "discovery")]
15513 #[allow(unused_mut)]
15514 fn replace_claim_payload(&self, claim: TxClaim, mut data: RouterItem) -> TelemetryResult<()> {
15515 #[cfg(feature = "compact-packet-store")]
15516 data.park_for_queue()?;
15517 let payload_cost = match &data {
15518 RouterItem::Packet(packet) => packet.byte_cost(),
15519 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
15520 };
15521 let mut st = self.state.lock();
15522 let Some(index) = st.claim_index(claim.token) else {
15523 return Ok(());
15524 };
15525 let original = st.transmit_queue.get(index).expect("claimed owner");
15526 let old_cost = original.byte_cost();
15527 let new_cost = old_cost
15528 .saturating_sub(original.item.byte_cost())
15529 .saturating_add(payload_cost);
15530 let priority = original.priority;
15531 let growth = new_cost.saturating_sub(old_cost);
15532 if growth != 0 {
15533 st.make_shared_queue_room(growth, priority, None)?;
15534 }
15535 let Some(index) = st.claim_index(claim.token) else {
15536 return Ok(());
15537 };
15538 st.transmit_queue.update_admitted(index, new_cost, |item| {
15539 item.item = RouterTxItem::Broadcast(data)
15540 })
15541 }
15542
15543 fn tx_queue_item_for_retry(
15544 &self,
15545 item: RouterTxItem,
15546 ignore_local: bool,
15547 ) -> TelemetryResult<()> {
15548 self.tx_queue_item_with_flags_at(item, ignore_local, self.clock.now_ms().saturating_add(1))
15549 }
15550
15551 #[cfg(not(feature = "compression"))]
15552 fn tx_queue_item_for_retry_at_priority(
15553 &self,
15554 item: RouterTxItem,
15555 ignore_local: bool,
15556 priority: u8,
15557 ) -> TelemetryResult<()> {
15558 #[cfg(feature = "discovery")]
15559 let schema_owner = RouterInner::schema_response_owner(&item);
15560 let result = self.tx_queue_item_with_priority_at(
15561 item,
15562 ignore_local,
15563 priority,
15564 self.clock.now_ms().saturating_add(1),
15565 );
15566 #[cfg(feature = "discovery")]
15567 if result.is_err() {
15568 RouterInner::schema_response_owner_lost(
15569 &mut self.state.lock().discovery_side_throttle,
15570 schema_owner,
15571 );
15572 }
15573 result
15574 }
15575
15576 fn tx_queue_item_with_flags_at(
15577 &self,
15578 item: RouterTxItem,
15579 ignore_local: bool,
15580 retry_after_ms: u64,
15581 ) -> TelemetryResult<()> {
15582 let priority = match &item {
15583 RouterTxItem::Broadcast(data) => self.item_service_priority(data)?,
15584 RouterTxItem::EndToEndReplay { .. } => {
15585 Self::router_item_priority_bumped(DataType::ReliableAck)
15586 }
15587 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
15588 RouterTxItem::ReliableReplay { ty, .. } => Self::router_item_priority_bumped(*ty),
15589 };
15590 self.tx_queue_item_with_priority_at(item, ignore_local, priority, retry_after_ms)
15591 }
15592
15593 #[inline]
15594 fn tx_queue_item_with_priority(
15595 &self,
15596 item: RouterTxItem,
15597 ignore_local: bool,
15598 priority: u8,
15599 ) -> TelemetryResult<()> {
15600 self.tx_queue_item_with_priority_at(item, ignore_local, priority, 0)
15601 }
15602
15603 fn tx_queue_item_with_priority_at(
15604 &self,
15605 item: RouterTxItem,
15606 ignore_local: bool,
15607 priority: u8,
15608 retry_after_ms: u64,
15609 ) -> TelemetryResult<()> {
15610 let mut st = self.state.lock();
15611 st.push_transmit(TxQueued {
15612 disposable: true,
15613 item,
15614 ignore_local,
15615 priority,
15616 delivery_state: QueuedDelivery::Untracked,
15617 accepted_type: 0,
15618 deferred_since_ms: 0,
15619 retry_after_ms,
15620 })?;
15621 Ok(())
15622 }
15623
15624 #[inline]
15626 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
15627 self.tx_queue_item_with_flags(item, false)
15628 }
15629
15630 #[cfg(feature = "std")]
15631 #[inline]
15632 fn try_enter_side_tx(&self, side: usize) -> Option<crate::lock::ActiveSideTxGuard<'_>> {
15633 let gate = self
15634 .state
15635 .lock()
15636 .side_transport
15637 .get(&side)?
15638 .tx_callback_gate
15639 .clone();
15640 self.side_tx_activity.try_enter(gate)
15641 }
15642 #[cfg(not(feature = "std"))]
15643 #[inline]
15644 fn try_enter_side_tx(&self, _side: usize) -> Option<crate::lock::ReentryGuard<'_>> {
15645 self.side_tx_gate.try_enter()
15646 }
15647 #[inline]
15648 fn side_tx_active(&self) -> bool {
15649 #[cfg(feature = "std")]
15650 {
15651 self.side_tx_activity.is_active()
15652 }
15653 #[cfg(not(feature = "std"))]
15654 {
15655 self.side_tx_gate.is_active()
15656 }
15657 }
15658
15659 #[inline]
15663 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
15664 self.process_rx_queue_with_timeout(0)
15665 }
15666
15667 #[inline]
15669 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
15670 crate::memory_admission::check_receive(bytes)?;
15671 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
15672 let priority = self.item_service_priority(&data)?;
15673 let mut st = self.state.lock();
15674 st.push_received(RouterRxItem {
15675 src: None,
15676 data,
15677 priority,
15678 })?;
15679 Ok(())
15680 }
15681
15682 #[inline]
15687 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
15688 crate::memory_admission::check_receive(bytes)?;
15689 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
15690 let priority = Self::router_item_priority(&data)?;
15691 self.isr_rx_queue
15692 .as_ref()
15693 .ok_or(TelemetryError::PermissionDenied)?
15694 .push_back_prioritized(RouterRxItem {
15695 src: None,
15696 data,
15697 priority,
15698 })
15699 }
15700
15701 #[inline]
15703 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
15704 pkt.validate()?;
15705 let data = RouterItem::Packet(pkt);
15706 let priority = self.item_service_priority(&data)?;
15707 let mut st = self.state.lock();
15708 st.push_received(RouterRxItem {
15709 src: None,
15710 data,
15711 priority,
15712 })?;
15713 Ok(())
15714 }
15715
15716 #[inline]
15721 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
15722 pkt.validate()?;
15723 let data = RouterItem::Packet(pkt);
15724 let priority = Self::router_item_priority(&data)?;
15725 self.isr_rx_queue
15726 .as_ref()
15727 .ok_or(TelemetryError::PermissionDenied)?
15728 .push_back_prioritized(RouterRxItem {
15729 src: None,
15730 data,
15731 priority,
15732 })
15733 }
15734
15735 #[inline]
15737 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
15738 self.ensure_side_ingress_enabled(side)?;
15739 pkt.validate()?;
15740 let data = RouterItem::Packet(pkt);
15741 let priority = self.item_service_priority(&data)?;
15742 let mut st = self.state.lock();
15743 st.push_received(RouterRxItem {
15744 src: Some(side),
15745 data,
15746 priority,
15747 })?;
15748 Ok(())
15749 }
15750
15751 #[inline]
15756 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
15757 self.ensure_side_ingress_enabled(side)?;
15758 pkt.validate()?;
15759 let data = RouterItem::Packet(pkt);
15760 let priority = Self::router_item_priority(&data)?;
15761 self.isr_rx_queue
15762 .as_ref()
15763 .ok_or(TelemetryError::PermissionDenied)?
15764 .push_back_prioritized(RouterRxItem {
15765 src: Some(side),
15766 data,
15767 priority,
15768 })
15769 }
15770
15771 #[inline]
15773 pub fn rx_packed_queue_from_side(
15774 &self,
15775 bytes: &[u8],
15776 side: RouterSideId,
15777 ) -> TelemetryResult<()> {
15778 self.ensure_side_ingress_enabled(side)?;
15779 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
15780 return Ok(());
15781 };
15782 let data = RouterItem::Packed(decoded);
15783 let priority = self.item_service_priority(&data)?;
15784 let mut st = self.state.lock();
15785 st.push_received(RouterRxItem {
15786 src: Some(side),
15787 data,
15788 priority,
15789 })?;
15790 Ok(())
15791 }
15792
15793 #[inline]
15798 pub fn rx_packed_queue_from_side_isr(
15799 &self,
15800 bytes: &[u8],
15801 side: RouterSideId,
15802 ) -> TelemetryResult<()> {
15803 self.ensure_side_ingress_enabled(side)?;
15804 crate::memory_admission::check_receive(bytes)?;
15805 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
15806 let priority = Self::router_item_priority(&data)?;
15807 self.isr_rx_queue
15808 .as_ref()
15809 .ok_or(TelemetryError::PermissionDenied)?
15810 .push_back_prioritized(RouterRxItem {
15811 src: Some(side),
15812 data,
15813 priority,
15814 })
15815 }
15816
15817 fn retry_with_attempts<F, T>(
15818 &self,
15819 times: usize,
15820 f: F,
15821 ) -> Result<(T, usize), (TelemetryError, usize)>
15822 where
15823 F: Fn() -> TelemetryResult<T>,
15824 {
15825 let times = if self.cfg.role.local_services() {
15826 times
15827 } else {
15828 times.min(1)
15829 };
15830 let mut last_err = None;
15831 for attempt in 0..times {
15832 match f() {
15833 Ok(v) => return Ok((v, attempt + 1)),
15834 Err(e) if Self::is_transient_tx_error(&e) => return Err((e, attempt + 1)),
15837 Err(e) => last_err = Some((e, attempt + 1)),
15838 }
15839 }
15840 Err(last_err.expect("times > 0"))
15841 }
15842
15843 #[inline]
15845 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
15846 self.cfg
15847 .handlers()
15848 .iter()
15849 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
15850 }
15851
15852 #[inline]
15854 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
15855 self.cfg
15856 .handlers()
15857 .iter()
15858 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
15859 }
15860
15861 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
15862 pkt.endpoints()
15863 .iter()
15864 .copied()
15865 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
15866 }
15867
15868 fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
15875 self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
15876 }
15877
15878 fn call_handler_with_retries(
15884 &self,
15885 dest: DataEndpoint,
15886 handler: &EndpointHandler,
15887 data: Option<&[u8]>,
15888 pkt_for_ctx: Option<&Packet>,
15889 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
15890 called_from_queue: bool,
15891 ) -> TelemetryResult<()> {
15892 let owned_tmp: Option<RouterItem>;
15893
15894 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
15895 (Some(d), _) => {
15896 owned_tmp = Some(RouterItem::Packed(crate::shared_bytes::convert(
15897 Arc::<[u8]>::from(d),
15898 )));
15899 owned_tmp.as_ref().unwrap()
15900 }
15901 (None, Some(pkt)) => {
15902 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
15903 owned_tmp.as_ref().unwrap()
15904 }
15905 (None, None) => {
15906 debug_assert!(
15907 false,
15908 "call_handler_with_retries called without data or packet context"
15909 );
15910 return Ok(());
15911 }
15912 };
15913
15914 match (&handler.handler, data) {
15915 (EndpointHandlerFn::Packet(f), _) => {
15916 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
15917 with_retries(
15918 self,
15919 dest,
15920 item_for_ctx,
15921 pkt_for_ctx,
15922 env_for_ctx,
15923 called_from_queue,
15924 || f(pkt),
15925 )
15926 }
15927
15928 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
15929 self,
15930 dest,
15931 item_for_ctx,
15932 pkt_for_ctx,
15933 env_for_ctx,
15934 called_from_queue,
15935 || f(bytes),
15936 ),
15937
15938 (EndpointHandlerFn::Packed(_), None) => Ok(()),
15939 }
15940 }
15941
15942 fn handle_callback_error_from_env(
15947 &self,
15948 env: &wire_format::TelemetryEnvelope,
15949 dest: Option<DataEndpoint>,
15950 e: TelemetryError,
15951 called_from_queue: bool,
15952 ) -> TelemetryResult<()> {
15953 if !self.cfg.role.local_services() {
15954 return Err(e);
15955 }
15956
15957 if matches!(&e, TelemetryError::Io("memory pressure")) {
15960 return Err(e);
15961 }
15962
15963 let mut recipients: Vec<DataEndpoint> = env
15964 .endpoints
15965 .iter()
15966 .copied()
15967 .filter(|&ep| self.cfg.is_local_endpoint(ep))
15968 .collect();
15969 crate::small_sort::sort(&mut recipients);
15970 recipients.dedup();
15971 if let Some(failed) = dest {
15972 recipients.retain(|&ep| ep != failed);
15973 }
15974
15975 let device = self.sender_arc();
15976 let error_msg = format!(
15977 "Handler for endpoint {:?} failed on device {:?}: {:?}",
15978 dest, device, e
15979 );
15980
15981 if recipients.is_empty() && dest.is_none() {
15982 fallback_stdout(&error_msg);
15983 return Ok(());
15984 }
15985
15986 if recipients.is_empty() {
15987 recipients = env.endpoints.to_vec();
15988 crate::small_sort::sort(&mut recipients);
15989 recipients.dedup();
15990 if let Some(failed) = dest {
15991 recipients.retain(|&ep| ep != failed);
15992 }
15993 }
15994
15995 if recipients.is_empty() {
15996 fallback_stdout(&error_msg);
15997 return Ok(());
15998 }
15999
16000 let payload = make_error_payload(&error_msg);
16001
16002 let error_pkt = Packet::new(
16003 DataType::TelemetryError,
16004 &recipients,
16005 &env.sender.clone(),
16006 env.timestamp_ms,
16007 crate::shared_bytes::convert(payload),
16008 )?;
16009 self.emit_internal_tx(
16010 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
16011 false,
16012 called_from_queue,
16013 )
16014 }
16015
16016 fn handle_internal_reliable_packet(
16017 &self,
16018 pkt: &Packet,
16019 src: Option<RouterSideId>,
16020 called_from_queue: bool,
16021 ) -> TelemetryResult<bool> {
16022 if !matches!(
16023 pkt.data_type(),
16024 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
16025 ) {
16026 return Ok(false);
16027 }
16028
16029 let Some(src) = src else {
16030 return Ok(false);
16031 };
16032
16033 if Self::is_end_to_end_ack_packet(pkt)
16034 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
16035 {
16036 let targets_local_sender = self.cfg.role.local_services()
16037 && (pkt.wire_target_senders().is_empty()
16038 || pkt
16039 .wire_target_senders()
16040 .contains(&Self::sender_hash(self.sender_arc().as_ref())));
16041 if !targets_local_sender && let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
16042 Self::note_end_to_end_acked_destination_locked(
16043 &mut self.state.lock(),
16044 packet_id,
16045 owner,
16046 );
16047 }
16048 let alias = {
16049 let st = self.state.lock();
16050 st.reliable_return_routes
16051 .get(&packet_id)
16052 .filter(|route| route.upstream_contract.is_some())
16053 .cloned()
16054 };
16055 if let Some(alias) = alias {
16056 let (upstream_id, publisher_hash) = alias.upstream_contract.expect("checked alias");
16057 #[cfg(feature = "std")]
16058 if reliable_trace_enabled() {
16059 eprintln!(
16060 "[sedsnet-reliable] proxy receipt id={packet_id:#x} upstream={upstream_id:#x} owner={:?}",
16061 Self::end_to_end_ack_sender_hash(pkt)
16062 );
16063 }
16064 let completed = {
16065 let mut st = self.state.lock();
16066 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
16067 if let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
16068 sent.pending_destinations.remove(&owner);
16069 }
16070 if !sent.pending_destinations.is_empty() {
16071 return Ok(true);
16072 }
16073 st.end_to_end_reliable_tx.remove(&packet_id);
16074 true
16075 } else {
16076 return Ok(true);
16079 }
16080 };
16081 if completed {
16082 let mut st = self.state.lock();
16083 if Self::remember_reliable_return_route_locked(&mut st, upstream_id) {
16084 st.reliable_return_routes.insert(
16085 upstream_id,
16086 ReliableReturnRouteState {
16087 side: alias.side,
16088 delivery_completed: true,
16089 last_replay_ms: alias.last_replay_ms,
16090 upstream_contract: Some((upstream_id, publisher_hash)),
16091 },
16092 );
16093 }
16094 }
16095 self.queue_end_to_end_reliable_ack_for_sender_hash(
16096 upstream_id,
16097 publisher_hash,
16098 called_from_queue,
16099 )?;
16100 self.release_ordered_successors()?;
16101 return Ok(true);
16102 }
16103 let mut st = self.state.lock();
16110 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
16111 if sent.waiting_for_predecessor {
16112 return Ok(targets_local_sender);
16113 }
16114 if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
16115 let _removed = sent.pending_destinations.remove(&sender_hash);
16116 #[cfg(feature = "std")]
16117 if reliable_trace_enabled() {
16118 eprintln!(
16119 "[sedsnet-reliable] receipt time={} id={packet_id:#x} sender={sender_hash:#x} matched={} remaining={:?}",
16120 self.clock.now_ms(),
16121 _removed.is_some(),
16122 sent.pending_destinations
16123 );
16124 }
16125 if sent.pending_destinations.is_empty() {
16126 st.end_to_end_reliable_tx.remove(&packet_id);
16127 }
16128 drop(st);
16129 self.release_ordered_successors()?;
16130 return Ok(targets_local_sender);
16131 }
16132 st.end_to_end_reliable_tx.remove(&packet_id);
16133 drop(st);
16134 self.release_ordered_successors()?;
16135 return Ok(targets_local_sender);
16136 }
16137 return Ok(targets_local_sender);
16140 }
16141
16142 let vals = pkt.data_as_u32()?;
16143 if vals.len() != 2 {
16144 return Err(TelemetryError::Unpack("bad reliable control payload"));
16145 }
16146 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
16147 let seq = vals[1];
16148
16149 match pkt.data_type() {
16150 DataType::ReliableAck => {
16151 self.handle_reliable_ack(src, ty, seq);
16152 Ok(true)
16153 }
16154 DataType::ReliablePartialAck => {
16155 self.handle_reliable_partial_ack(src, ty, seq);
16156 Ok(true)
16157 }
16158 DataType::ReliablePacketRequest => {
16159 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
16160 Ok(true)
16161 }
16162 _ => Ok(false),
16163 }
16164 }
16165
16166 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
16172 let packed_frame = match &item.data {
16176 RouterItem::Packed(bytes) => Some(wire_format::peek_routing_frame_info(bytes.as_ref())),
16177 RouterItem::Packet(_) => None,
16178 };
16179 if crate::memory_admission::supports_work_kinds()
16180 && let (Some(side), RouterItem::Packed(bytes)) = (item.src, &item.data)
16181 {
16182 self.admit_received_frame_with_view(
16183 side,
16184 bytes,
16185 packed_frame.as_ref().and_then(|frame| frame.as_ref().ok()),
16186 )?;
16187 }
16188 let packed_routing = packed_frame
16189 .as_ref()
16190 .and_then(|frame| frame.as_ref().ok())
16191 .map(|frame| &frame.envelope);
16192 if let Some(src) = item.src {
16193 self.ensure_side_ingress_enabled(src)?;
16194 match &item.data {
16195 RouterItem::Packet(pkt) => {
16196 let bytes = wire_format::packet_wire_size(pkt);
16197 self.note_side_rx(src, pkt.data_type(), bytes, true);
16198 }
16199 RouterItem::Packed(bytes) => {
16200 if let Some(env) = packed_routing.as_ref() {
16201 self.note_side_rx(src, env.ty, bytes.len(), true);
16202 }
16203 }
16204 }
16205 match &item.data {
16206 RouterItem::Packet(pkt) => {
16207 if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
16208 && !is_internal_control_type(pkt.data_type())
16209 && !self.note_reliable_return_route(src, pkt.packet_id())
16210 {
16211 return Ok(());
16212 }
16213 }
16214 RouterItem::Packed(bytes) => {
16215 if let Some(env) = packed_routing.as_ref()
16216 && (is_reliable_type(env.ty) || env.target_senders().len() != 0)
16217 && !is_internal_control_type(env.ty)
16218 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
16219 && !self.note_reliable_return_route(src, packet_id)
16220 {
16221 return Ok(());
16222 }
16223 }
16224 }
16225 }
16226 match &item.data {
16227 RouterItem::Packet(pkt) => {
16228 if !is_internal_control_type(pkt.data_type()) {
16229 self.remember_managed_variable_packet(pkt)?;
16230 }
16231 }
16232 RouterItem::Packed(bytes) => {
16233 if let Some(env) = packed_routing.as_ref()
16234 && !is_internal_control_type(env.ty)
16235 && self.is_managed_variable_type(env.ty)
16236 {
16237 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16238 pkt.validate()?;
16239 self.remember_managed_variable_packet(&pkt)?;
16240 }
16241 }
16242 }
16243 let mut released_buffered: Vec<crate::SharedBytes> = Vec::new();
16244 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
16245 let (_opts, handler_is_packed, hop_reliable_enabled) = {
16246 let st = self.state.lock();
16247 let side_ref = Self::side_ref(&st, src)?;
16248 let opts = side_ref.opts.clone();
16249 (
16250 opts.clone(),
16251 matches!(
16252 side_ref.tx_handler,
16253 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
16254 ),
16255 opts.reliable_enabled
16256 && self.cfg.reliable_enabled()
16257 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
16258 )
16259 };
16260
16261 if hop_reliable_enabled && handler_is_packed {
16262 let frame = match packed_frame
16263 .as_ref()
16264 .expect("packed frame")
16265 .as_ref()
16266 .map_err(Clone::clone)
16267 {
16268 Ok(frame) => frame,
16269 Err(e) => {
16270 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
16271 if let Ok(frame) =
16272 wire_format::peek_frame_info_unchecked(bytes.as_ref())
16273 && is_reliable_type(frame.envelope.ty)
16274 && let Some(hdr) = frame.reliable
16275 {
16276 let unordered =
16277 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
16278 let unsequenced =
16279 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
16280
16281 if !unsequenced {
16282 let requested = if unordered {
16283 hdr.seq
16284 } else {
16285 let mut st = self.state.lock();
16286 let rx_state = self.reliable_rx_state_mut(
16287 &mut st,
16288 src,
16289 frame.envelope.ty,
16290 );
16291 rx_state.expected_seq.min(hdr.seq)
16292 };
16293 self.queue_reliable_packet_request(
16294 src,
16295 frame.envelope.ty,
16296 requested,
16297 called_from_queue,
16298 )?;
16299 }
16300 }
16301 return Ok(());
16302 }
16303 return Err(e);
16304 }
16305 };
16306 if is_reliable_type(frame.envelope.ty)
16307 && let Some(hdr) = frame.reliable
16308 {
16309 if frame.ack_only() {
16310 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
16311 return Ok(());
16312 }
16313 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
16314 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
16315
16316 if !unsequenced {
16317 if unordered {
16318 self.queue_reliable_ack(
16319 src,
16320 frame.envelope.ty,
16321 hdr.seq,
16322 called_from_queue,
16323 )?;
16324 } else {
16325 let mut release: Vec<crate::SharedBytes> = Vec::new();
16326 let mut last_delivered = None;
16327 let mut ack_old = None;
16328 let mut request_missing = None;
16329 let mut partial_ack = None;
16330 {
16331 let mut st = self.state.lock();
16332 let rx_state =
16333 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
16334 let expected_seq = rx_state.expected_seq;
16335 if hdr.seq < expected_seq {
16336 ack_old = Some(expected_seq.saturating_sub(1));
16337 } else if hdr.seq > expected_seq {
16338 request_missing = Some(expected_seq);
16339 partial_ack = Some(hdr.seq);
16340 st.buffer_reliable_rx(
16341 src,
16342 frame.envelope.ty,
16343 hdr.seq,
16344 bytes.clone(),
16345 )?;
16346 } else {
16347 release.push(bytes.clone());
16348 last_delivered = Some(hdr.seq);
16349 let mut next_expected = hdr.seq.wrapping_add(1);
16350 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
16351 release.push(buf);
16352 last_delivered = Some(next_expected);
16353 let next = next_expected.wrapping_add(1);
16354 next_expected = if next == 0 { 1 } else { next };
16355 }
16356 rx_state.expected_seq = next_expected;
16357 }
16358 }
16359
16360 if let Some(ack_seq) = ack_old {
16361 self.queue_reliable_ack(
16362 src,
16363 frame.envelope.ty,
16364 ack_seq,
16365 called_from_queue,
16366 )?;
16367 return Ok(());
16368 }
16369 if let Some(request_seq) = request_missing {
16370 if let Some(partial_seq) = partial_ack {
16371 self.queue_reliable_partial_ack(
16372 src,
16373 frame.envelope.ty,
16374 partial_seq,
16375 called_from_queue,
16376 )?;
16377 }
16378 self.queue_reliable_packet_request(
16379 src,
16380 frame.envelope.ty,
16381 request_seq,
16382 called_from_queue,
16383 )?;
16384 return Ok(());
16385 }
16386
16387 if let Some(ack_seq) = last_delivered {
16388 self.queue_reliable_ack(
16389 src,
16390 frame.envelope.ty,
16391 ack_seq,
16392 called_from_queue,
16393 )?;
16394 }
16395
16396 released_buffered.extend(release.into_iter().skip(1));
16397 }
16398 }
16399 }
16400 } else {
16401 match packed_frame
16402 .as_ref()
16403 .expect("packed frame")
16404 .as_ref()
16405 .map_err(Clone::clone)
16406 {
16407 Ok(frame) => {
16408 if frame.ack_only() {
16409 return Ok(());
16410 }
16411 }
16412 Err(e) => {
16413 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
16414 return Ok(());
16415 }
16416 return Err(e);
16417 }
16418 }
16419 }
16420 }
16421
16422 if self.is_duplicate_pkt(&item.data)? {
16423 #[cfg(feature = "discovery")]
16424 let shared_pending = if item.src.is_some()
16425 && Self::item_data_type(&item.data).is_ok_and(|ty| {
16430 is_reliable_type(ty) && !is_internal_control_type(ty)
16431 })
16432 && self.item_target_senders(&item.data)?.is_empty()
16433 {
16434 let RemoteSidePlan::Target(sides) =
16435 self.remote_side_plan_with_selection(&item.data, item.src, false)?;
16436 let st = self.state.lock();
16437 sides.into_iter().any(|side| {
16438 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
16439 })
16440 } else {
16441 false
16442 };
16443 #[cfg(not(feature = "discovery"))]
16444 let shared_pending = false;
16445 if shared_pending {
16446 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
16447 }
16448
16449 if !shared_pending
16452 && ((item.src.is_some() && self.item_targets_another_sender(&item.data)?)
16453 || (item.src.is_some() && self.should_route_remote(&item.data, item.src)? && {
16454 let (endpoints, _) = self.item_route_info(&item.data)?;
16455 self.item_targets_router_as_intermediate_hop(&item.data, &endpoints)?
16456 }))
16457 {
16458 let id = Self::get_hash(&item.data);
16459 let replay = {
16460 let mut st = self.state.lock();
16461 st.reliable_return_routes.get_mut(&id).is_some_and(|route| {
16462 let now = self.clock.now_ms();
16463 if now.saturating_sub(route.last_replay_ms)
16464 < runtime_reliable_retransmit_ms()
16465 {
16466 return false;
16467 }
16468 route.last_replay_ms = now;
16469 true
16470 })
16471 };
16472 if replay {
16473 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
16474 }
16475 }
16476 if item.src.is_some() {
16477 let local_sender = self.sender_arc();
16478 match &item.data {
16479 RouterItem::Packet(pkt)
16480 if (is_reliable_type(pkt.data_type())
16481 || !pkt.wire_target_senders().is_empty())
16482 && pkt.sender() != local_sender.as_ref()
16483 && self.item_targets_local_sender(&item.data)?
16484 && self.packet_has_local_delivery(pkt)
16485 && !(self.should_route_remote(&item.data, item.src)?
16486 && self.item_targets_router_as_intermediate_hop(
16487 &item.data,
16488 pkt.endpoints(),
16489 )?) =>
16490 {
16491 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
16492 }
16493 RouterItem::Packed(bytes) => {
16494 if let Ok(pkt) = self.unpack_packet_for_role(bytes.as_ref())
16495 && (is_reliable_type(pkt.data_type())
16496 || !pkt.wire_target_senders().is_empty())
16497 && pkt.sender() != local_sender.as_ref()
16498 && self.item_targets_local_sender(&item.data)?
16499 && self.packet_has_local_delivery(&pkt)
16500 && !(self.should_route_remote(&item.data, item.src)?
16501 && self.item_targets_router_as_intermediate_hop(
16502 &item.data,
16503 pkt.endpoints(),
16504 )?)
16505 {
16506 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
16507 .unwrap_or_else(|_| pkt.packet_id());
16508 self.queue_end_to_end_reliable_ack_for_packet_id(
16509 packet_id,
16510 pkt.sender(),
16511 called_from_queue,
16512 )?;
16513 }
16514 }
16515 _ => {}
16516 }
16517 }
16518 return Ok(());
16519 }
16520
16521 self.dispatch_rx_data(item, called_from_queue)?;
16522
16523 for release_bytes in released_buffered {
16524 let release_data = RouterItem::Packed(release_bytes.clone());
16525 if self.is_duplicate_pkt(&release_data)? {
16526 continue;
16527 }
16528 let release_item = RouterRxItem {
16529 src: item.src,
16530 priority: Self::router_item_priority(&release_data)?,
16531 data: release_data,
16532 };
16533 self.dispatch_rx_data(&release_item, called_from_queue)?;
16534 }
16535
16536 Ok(())
16537 }
16538
16539 fn dispatch_rx_data(
16540 &self,
16541 item: &RouterRxItem,
16542 called_from_queue: bool,
16543 ) -> TelemetryResult<()> {
16544 match &item.data {
16545 RouterItem::Packet(pkt) => {
16546 pkt.validate()?;
16547 #[cfg(feature = "discovery")]
16548 if let Some(side) = item.src {
16549 self.refresh_peer_liveness(side, sender_address_u32(pkt.sender()));
16550 }
16551
16552 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
16553 return Ok(());
16554 }
16555
16556 if pkt.data_type() == DataType::P2pMessage {
16557 if self.item_targets_local_sender(&item.data)? {
16558 self.dispatch_p2p_packet(pkt)?;
16559 if item.src.is_some() {
16560 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
16561 }
16562 }
16563 if self.should_route_remote(&item.data, item.src)? {
16564 self.relay_send(
16565 RouterItem::Packet(pkt.to_owned()),
16566 item.src,
16567 called_from_queue,
16568 )?;
16569 }
16570 return Ok(());
16571 }
16572
16573 #[cfg(feature = "timesync")]
16574 if matches!(
16575 pkt.data_type(),
16576 DataType::TimeSyncAnnounce
16577 | DataType::TimeSyncRequest
16578 | DataType::TimeSyncResponse
16579 ) {
16580 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
16581 return Ok(());
16582 }
16583
16584 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
16585 if !discovery_is_hop_local_advertisement(
16590 pkt.data_type(),
16591 !self.cfg.role.local_services(),
16592 ) && self.should_route_remote(&item.data, item.src)?
16593 {
16594 self.relay_send(
16595 RouterItem::Packet(pkt.to_owned()),
16596 item.src,
16597 called_from_queue,
16598 )?;
16599 }
16600 return Ok(());
16601 }
16602
16603 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
16604 crate::small_sort::sort(&mut eps);
16605 eps.dedup();
16606 let had_local_handler = eps.iter().copied().any(|ep| {
16607 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
16608 });
16609 let had_local_delivery =
16610 had_local_handler || self.is_managed_variable_type(pkt.data_type());
16611
16612 let has_remote = self.should_route_remote(&item.data, item.src)?;
16613 let targets_local = self.item_targets_local_sender(&item.data)?;
16614
16615 let has_packed_local = eps
16616 .iter()
16617 .copied()
16618 .any(|ep| self.endpoint_has_packed_handler(ep));
16619 let bytes_opt = if has_packed_local {
16620 Some(wire_format::pack_packet(pkt))
16621 } else {
16622 None
16623 };
16624
16625 let mut local_delivery_error = None;
16626 if targets_local {
16627 for dest in eps {
16628 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
16629 let result = match (&h.handler, &bytes_opt) {
16630 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
16631 .call_handler_with_retries(
16632 dest,
16633 h,
16634 Some(bytes.as_ref()),
16635 Some(pkt),
16636 None,
16637 called_from_queue,
16638 ),
16639 (EndpointHandlerFn::Packed(_), None) => {
16640 let bytes = wire_format::pack_packet(pkt);
16641 self.call_handler_with_retries(
16642 dest,
16643 h,
16644 Some(bytes.as_ref()),
16645 Some(pkt),
16646 None,
16647 called_from_queue,
16648 )
16649 }
16650 (EndpointHandlerFn::Packet(_), _) => self
16651 .call_handler_with_retries(
16652 dest,
16653 h,
16654 None,
16655 Some(pkt),
16656 None,
16657 called_from_queue,
16658 ),
16659 };
16660 if let Err(error) = result {
16661 if let Some(src) = item.src {
16662 self.note_side_local_handler_failure(
16663 src,
16664 pkt.data_type(),
16665 runtime_max_handler_retries(),
16666 );
16667 }
16668 local_delivery_error.get_or_insert(error);
16669 }
16670 }
16671 }
16672 }
16673
16674 if let Some(src) = item.src
16675 && had_local_delivery
16676 && local_delivery_error.is_none()
16677 && targets_local
16678 {
16679 self.note_side_local_delivery(src, pkt.data_type());
16680 }
16681
16682 if item.src.is_some()
16683 && had_local_delivery
16684 && local_delivery_error.is_none()
16685 && targets_local
16686 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
16687 && !(has_remote
16688 && self
16689 .item_targets_router_as_intermediate_hop(&item.data, pkt.endpoints())?)
16690 {
16691 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
16692 }
16693
16694 if has_remote {
16695 let relay_item = RouterItem::Packet(pkt.to_owned());
16696 self.relay_send(relay_item, item.src, called_from_queue)?;
16697 }
16698
16699 local_delivery_error.map_or(Ok(()), Err)
16703 }
16704 RouterItem::Packed(bytes) => {
16705 let routing = wire_format::peek_routing_envelope(bytes.as_ref())?;
16709 if !is_internal_control_type(routing.ty)
16710 && !routing.is_reliable
16711 && routing.target_senders().len() == 0
16712 && !self.is_managed_variable_type(routing.ty)
16713 && !routing.endpoints().any(|ep| {
16714 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
16715 })
16716 {
16717 return self.relay_send(item.data.clone(), item.src, called_from_queue);
16721 }
16722
16723 let env = wire_format::peek_envelope(bytes.as_ref())?;
16724
16725 if matches!(
16726 env.ty,
16727 DataType::ReliableAck
16728 | DataType::ReliablePartialAck
16729 | DataType::ReliablePacketRequest
16730 ) {
16731 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16732 pkt.validate()?;
16733 let handled =
16734 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
16735 if !handled && self.should_route_remote(&item.data, item.src)? {
16738 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
16739 }
16740 return Ok(());
16741 }
16742
16743 if env.ty == DataType::P2pMessage {
16744 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16745 pkt.validate()?;
16746 if self.item_targets_local_sender(&item.data)? {
16747 self.dispatch_p2p_packet(&pkt)?;
16748 if item.src.is_some() {
16749 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
16750 .unwrap_or_else(|_| pkt.packet_id());
16751 self.queue_end_to_end_reliable_ack_for_packet_id(
16752 packet_id,
16753 pkt.sender(),
16754 called_from_queue,
16755 )?;
16756 }
16757 }
16758 if self.should_route_remote(&item.data, item.src)? {
16759 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
16760 }
16761 return Ok(());
16762 }
16763
16764 #[cfg(feature = "timesync")]
16765 if matches!(
16766 env.ty,
16767 DataType::TimeSyncAnnounce
16768 | DataType::TimeSyncRequest
16769 | DataType::TimeSyncResponse
16770 ) {
16771 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16772 pkt.validate()?;
16773 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
16774 return Ok(());
16775 }
16776
16777 #[cfg(feature = "discovery")]
16778 if discovery::is_discovery_type(env.ty) {
16779 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16780 pkt.validate()?;
16781 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
16782 if !discovery_is_hop_local_advertisement(
16783 env.ty,
16784 !self.cfg.role.local_services(),
16785 ) && self.should_route_remote(&item.data, item.src)?
16786 {
16787 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
16788 }
16789 return Ok(());
16790 }
16791
16792 let any_packet_needed = env
16793 .endpoints
16794 .iter()
16795 .copied()
16796 .any(|ep| self.endpoint_has_packet_handler(ep));
16797
16798 let mut pkt_opt = if any_packet_needed {
16799 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16800 pkt.validate()?;
16801 Some(pkt)
16802 } else {
16803 None
16804 };
16805
16806 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
16807 crate::small_sort::sort(&mut eps);
16808 eps.dedup();
16809 let had_local_handler = eps.iter().copied().any(|ep| {
16810 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
16811 });
16812 let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
16813
16814 let has_remote = self.should_route_remote(&item.data, item.src)?;
16815 let targets_local = self.item_targets_local_sender(&item.data)?;
16816
16817 let mut local_delivery_error = None;
16818 if targets_local {
16819 for dest in eps {
16820 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
16821 let result = match &h.handler {
16822 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
16823 dest,
16824 h,
16825 Some(bytes.as_ref()),
16826 pkt_opt.as_ref(),
16827 Some(&env),
16828 called_from_queue,
16829 ),
16830 EndpointHandlerFn::Packet(_) => {
16831 if pkt_opt.is_none() {
16832 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16833 pkt.validate()?;
16834 pkt_opt = Some(pkt);
16835 }
16836 let pkt_ref = pkt_opt.as_ref().expect("just set");
16837 self.call_handler_with_retries(
16838 dest,
16839 h,
16840 None,
16841 Some(pkt_ref),
16842 Some(&env),
16843 called_from_queue,
16844 )
16845 }
16846 };
16847 if let Err(error) = result {
16848 if let Some(src) = item.src {
16849 self.note_side_local_handler_failure(
16850 src,
16851 env.ty,
16852 runtime_max_handler_retries(),
16853 );
16854 }
16855 local_delivery_error.get_or_insert(error);
16856 }
16857 }
16858 }
16859 }
16860
16861 if item.src.is_some()
16862 && had_local_delivery
16863 && local_delivery_error.is_none()
16864 && targets_local
16865 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
16866 && !(has_remote
16867 && self
16868 .item_targets_router_as_intermediate_hop(&item.data, &env.endpoints)?)
16869 {
16870 let packet_id =
16871 wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
16872 pkt_opt
16873 .as_ref()
16874 .map(Packet::packet_id)
16875 .ok_or(TelemetryError::Unpack("missing packet id"))
16876 })?;
16877 self.queue_end_to_end_reliable_ack_for_packet_id(
16878 packet_id,
16879 &env.sender,
16880 called_from_queue,
16881 )?;
16882 }
16883
16884 if has_remote {
16885 let relay_item = match pkt_opt {
16886 Some(ref p) => RouterItem::Packet(p.clone()),
16887 None => RouterItem::Packed(bytes.clone()),
16888 };
16889 self.relay_send(relay_item, item.src, called_from_queue)?;
16890 }
16891
16892 local_delivery_error.map_or(Ok(()), Err)
16896 }
16897 }
16898 }
16899
16900 fn dispatch_local_for_item(
16901 &self,
16902 item: &RouterItem,
16903 called_from_queue: bool,
16904 ) -> TelemetryResult<()> {
16905 match item {
16906 RouterItem::Packet(pkt) => {
16907 pkt.validate()?;
16908 if is_internal_control_type(pkt.data_type()) {
16909 return Ok(());
16910 }
16911 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
16912 if !self.item_targets_local_sender(item)? {
16913 return Ok(());
16914 }
16915
16916 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
16917 crate::small_sort::sort(&mut eps);
16918 eps.dedup();
16919
16920 let has_packed_local = eps
16921 .iter()
16922 .copied()
16923 .any(|ep| self.endpoint_has_packed_handler(ep));
16924 let bytes_opt = if has_packed_local {
16925 Some(wire_format::pack_packet(pkt))
16926 } else {
16927 None
16928 };
16929
16930 for dest in eps {
16931 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
16932 match (&h.handler, &bytes_opt) {
16933 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
16934 self.call_handler_with_retries(
16935 dest,
16936 h,
16937 Some(bytes.as_ref()),
16938 Some(pkt),
16939 None,
16940 called_from_queue,
16941 )?;
16942 }
16943 (EndpointHandlerFn::Packed(_), None) => {
16944 let bytes = wire_format::pack_packet(pkt);
16945 self.call_handler_with_retries(
16946 dest,
16947 h,
16948 Some(bytes.as_ref()),
16949 Some(pkt),
16950 None,
16951 called_from_queue,
16952 )?;
16953 }
16954 (EndpointHandlerFn::Packet(_), _) => {
16955 self.call_handler_with_retries(
16956 dest,
16957 h,
16958 None,
16959 Some(pkt),
16960 None,
16961 called_from_queue,
16962 )?;
16963 }
16964 }
16965 }
16966 }
16967 }
16968 RouterItem::Packed(bytes) => {
16969 let env = wire_format::peek_envelope(bytes.as_ref())?;
16970 if is_internal_control_type(env.ty) {
16971 return Ok(());
16972 }
16973 self.ensure_e2e_policy_supported_for_type(env.ty)?;
16974 if !self.item_targets_local_sender(item)? {
16975 return Ok(());
16976 }
16977
16978 let any_packet_needed = env
16979 .endpoints
16980 .iter()
16981 .copied()
16982 .any(|ep| self.endpoint_has_packet_handler(ep));
16983
16984 let mut pkt_opt = if any_packet_needed {
16985 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
16986 pkt.validate()?;
16987 Some(pkt)
16988 } else {
16989 None
16990 };
16991
16992 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
16993 crate::small_sort::sort(&mut eps);
16994 eps.dedup();
16995
16996 for dest in eps {
16997 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
16998 match &h.handler {
16999 EndpointHandlerFn::Packed(_) => {
17000 self.call_handler_with_retries(
17001 dest,
17002 h,
17003 Some(bytes.as_ref()),
17004 pkt_opt.as_ref(),
17005 Some(&env),
17006 called_from_queue,
17007 )?;
17008 }
17009 EndpointHandlerFn::Packet(_) => {
17010 if pkt_opt.is_none() {
17011 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
17012 pkt.validate()?;
17013 pkt_opt = Some(pkt);
17014 }
17015 let pkt_ref = pkt_opt.as_ref().expect("just set");
17016 self.call_handler_with_retries(
17017 dest,
17018 h,
17019 None,
17020 Some(pkt_ref),
17021 Some(&env),
17022 called_from_queue,
17023 )?;
17024 }
17025 }
17026 }
17027 }
17028 }
17029 }
17030
17031 Ok(())
17032 }
17033
17034 fn tx_item_impl(
17040 &self,
17041 item: RouterTxItem,
17042 ignore_local: bool,
17043 called_from_queue: bool,
17044 ) -> TelemetryResult<()> {
17045 self.tx_item_impl_with_retry(
17046 item,
17047 ignore_local,
17048 called_from_queue,
17049 &mut DispatchContext::default(),
17050 )
17051 }
17052
17053 fn tx_item_impl_with_retry(
17054 &self,
17055 item: RouterTxItem,
17056 ignore_local: bool,
17057 called_from_queue: bool,
17058 context: &mut DispatchContext,
17059 ) -> TelemetryResult<()> {
17060 match item {
17061 RouterTxItem::Broadcast(data) => {
17062 let claim = context.claim;
17063 let deferred_since = context.since;
17064 let ty = Self::item_data_type(&data)?;
17065
17066 if let Some(since) = deferred_since
17067 && let Err(error) = self.expire_deferred_original(
17068 ty,
17069 since,
17070 claim.is_some_and(|claim| claim.ordered),
17071 )
17072 {
17073 context.mark_terminal();
17074 return Err(error);
17075 }
17076 if !called_from_queue
17077 && !ignore_local
17078 && matches!(reliable_mode(ty), crate::ReliableMode::Ordered)
17079 && self
17080 .state
17081 .lock()
17082 .transmit_queue
17083 .iter()
17084 .any(|queued| RouterInner::ordered_original_type(queued) == Some(ty))
17085 {
17086 return self.tx_queue_item(RouterTxItem::Broadcast(data));
17087 }
17088
17089 self.ensure_e2e_policy_supported_for_type(ty)?;
17090 if let RouterItem::Packet(pkt) = &data
17091 && !is_internal_control_type(pkt.data_type())
17092 {
17093 self.remember_managed_variable_packet(pkt)?;
17094 }
17095 #[cfg(feature = "discovery")]
17096 let is_discovery = discovery::is_discovery_type(ty);
17097 if !ignore_local {
17098 if self.is_duplicate_pkt(&data)? {
17099 return Ok(());
17100 }
17101 #[cfg(feature = "discovery")]
17102 if !is_discovery
17103 && !is_internal_control_type(ty)
17104 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
17105 {
17106 if claim.is_some() {
17108 context.mark_terminal();
17109 return Err(TelemetryError::HandlerError(
17110 "accepted local delivery failed",
17111 ));
17112 }
17113 return Err(error);
17114 }
17115 #[cfg(not(feature = "discovery"))]
17116 if !is_internal_control_type(ty)
17117 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
17118 {
17119 if claim.is_some() {
17121 context.mark_terminal();
17122 return Err(TelemetryError::HandlerError(
17123 "accepted local delivery failed",
17124 ));
17125 }
17126 return Err(error);
17127 }
17128 }
17129
17130 if !ignore_local && !self.mark_claim_local_delivered(claim) {
17131 return Ok(());
17133 }
17134
17135 if !self.claim_schema_is_current(ty, claim)? {
17136 return Ok(());
17137 }
17138
17139 match &data {
17140 RouterItem::Packet(packet) => packet.validate()?,
17141 RouterItem::Packed(_) => {}
17143 }
17144 let freezes_contract = !is_internal_control_type(ty)
17147 && self.item_requires_end_to_end_tracking(&data, ty)?;
17148 #[cfg(feature = "discovery")]
17154 let cached_plan = if freezes_contract {
17155 None
17156 } else {
17157 Some(self.remote_side_plan(&data, None)?)
17158 };
17159 #[cfg(feature = "discovery")]
17160 let send_remote = match &cached_plan {
17161 Some(RemoteSidePlan::Target(sides)) => !sides.is_empty(),
17162 None => self.should_route_remote(&data, None)?,
17163 };
17164 #[cfg(not(feature = "discovery"))]
17165 let send_remote = self.should_route_remote(&data, None)?;
17166
17167 if !send_remote {
17168 let (endpoints, ty) = self.item_route_info(&data)?;
17169 if !is_internal_control_type(ty)
17170 && is_reliable_type(ty)
17171 && has_nonlocal_endpoint(&endpoints, &self.cfg)
17172 && !self.state.lock().is_managed_variable(ty.as_u32())
17173 {
17174 self.remove_pkt_id(&data);
17178 if called_from_queue {
17179 return self.defer_accepted_original(context);
17180 }
17181 return Err(TelemetryError::Io("no discovered route"));
17182 }
17183 return Ok(());
17184 }
17185 #[cfg(feature = "discovery")]
17186 let mut data = data;
17187 if freezes_contract {
17188 #[cfg(feature = "discovery")]
17189 {
17190 let pending = {
17191 let st = self.state.lock();
17192 let mut pending = self.expected_end_to_end_destinations_locked(
17193 &st, &data, None, false, false,
17194 )?;
17195 self.filter_trackable_end_to_end_destinations_locked(
17196 &st,
17197 ty,
17198 &mut pending,
17199 false,
17200 );
17201 pending
17202 };
17203 if !pending.is_empty() {
17204 let targets: Vec<u64> = pending.keys().copied().collect();
17206 data = self.attach_wire_contract_to_item(data, &targets)?;
17207 match self.register_end_to_end_reliable_tx(&data, Some(pending)) {
17208 Ok(ready) => {
17209 context.mark_owner_transferred();
17210 if !ready {
17211 return Ok(());
17212 }
17213 }
17214 Err(error) => {
17215 self.remove_pkt_id(&data);
17218 if called_from_queue
17219 && matches!(
17220 error,
17221 TelemetryError::Io(
17222 "reliable window full" | "memory pressure"
17223 )
17224 )
17225 {
17226 if let Some(claim) = claim {
17230 self.replace_claim_payload(claim, data)?;
17231 }
17232 self.defer_accepted_original(context)?;
17233 return Ok(());
17234 }
17235 return Err(error);
17236 }
17237 }
17238 }
17239 }
17240 }
17241 #[cfg(feature = "discovery")]
17242 let plan = match cached_plan {
17243 Some(plan) => plan,
17244 None => self.remote_side_plan(&data, None)?,
17245 };
17246 #[cfg(not(feature = "discovery"))]
17247 let plan = self.remote_side_plan(&data, None)?;
17248 let RemoteSidePlan::Target(sides) = plan;
17249 for side in sides.iter().copied() {
17250 #[cfg(feature = "compression")]
17251 let mut side_data = data.clone();
17252 #[cfg(feature = "compression")]
17253 let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
17254 side,
17255 &mut side_data,
17256 false,
17257 &mut context.admitted,
17258 );
17259 #[cfg(not(feature = "compression"))]
17260 let send_result = self.send_reliable_to_side_with_progress(
17261 side,
17262 data.clone(),
17263 false,
17264 &mut context.admitted,
17265 );
17266 if let Err(e) = send_result {
17267 if Self::is_transient_tx_error(&e) {
17268 context.refused = true;
17269 #[cfg(feature = "compression")]
17270 self.retry_codec_owner(
17271 RouterTxItem::ToSide {
17272 src: None,
17273 dst: side,
17274 data: side_data,
17275 },
17276 true,
17277 &mut reservation,
17278 )?;
17279 #[cfg(not(feature = "compression"))]
17280 self.tx_queue_item_for_retry(
17281 RouterTxItem::ToSide {
17282 src: None,
17283 dst: side,
17284 data: data.clone(),
17285 },
17286 true,
17287 )?;
17288 continue;
17289 }
17290 match &data {
17291 RouterItem::Packet(pkt) => {
17292 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
17293 }
17294 RouterItem::Packed(bytes) => {
17295 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
17296 let _ = self.handle_callback_error_from_env(
17297 &env,
17298 None,
17299 e,
17300 called_from_queue,
17301 );
17302 }
17303 }
17304 }
17305 return Err(TelemetryError::HandlerError("tx handler failed"));
17306 }
17307 }
17308 }
17309 RouterTxItem::ToSide { src, dst, data } => {
17310 #[cfg(feature = "compression")]
17311 let mut data = data;
17312 let ty = Self::item_data_type(&data)?;
17313 self.ensure_e2e_policy_supported_for_type(ty)?;
17314 if let RouterItem::Packet(pkt) = &data
17315 && !is_internal_control_type(pkt.data_type())
17316 {
17317 self.remember_managed_variable_packet(pkt)?;
17318 }
17319 if !ignore_local {
17320 if self.is_duplicate_pkt(&data)? {
17321 return Ok(());
17322 }
17323 let suppress_local = is_internal_control_type(ty);
17324 if !suppress_local {
17325 self.dispatch_local_for_item(&data, called_from_queue)?;
17326 }
17327 }
17328 let allowed = {
17329 let st = self.state.lock();
17330 let ty = Some(ty);
17331 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
17332 #[cfg(all(feature = "discovery", feature = "timesync"))]
17333 let timesync_allowed = ty
17334 .map(|ty| {
17335 Self::timesync_allowed_for_side_locked(
17336 &st,
17337 dst,
17338 ty,
17339 self.clock.now_ms(),
17340 )
17341 })
17342 .unwrap_or(true);
17343 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
17344 let timesync_allowed = true;
17345 route_allowed && timesync_allowed
17346 };
17347 if !allowed {
17348 #[cfg(feature = "discovery")]
17349 if ty == DataType::DiscoverySchema {
17350 RouterInner::schema_response_owner_lost(
17351 &mut self.state.lock().discovery_side_throttle,
17352 Some((dst, Self::get_hash(&data))),
17353 );
17354 }
17355 return Ok(());
17356 }
17357 #[cfg(feature = "compression")]
17358 let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
17359 dst,
17360 &mut data,
17361 src.is_some(),
17362 &mut context.admitted,
17363 );
17364 #[cfg(not(feature = "compression"))]
17365 let send_result = self.send_reliable_to_side_with_progress(
17366 dst,
17367 data.clone(),
17368 src.is_some(),
17369 &mut context.admitted,
17370 );
17371 if let Err(e) = send_result {
17372 if Self::is_transient_tx_error(&e) {
17373 context.refused = true;
17374 let retry_priority =
17375 context.priority.max(self.item_service_priority(&data)?);
17376 #[cfg(feature = "compression")]
17377 self.retry_codec_owner_at_priority(
17378 RouterTxItem::ToSide { src, dst, data },
17379 true,
17380 &mut reservation,
17381 retry_priority,
17382 )?;
17383 #[cfg(not(feature = "compression"))]
17384 self.tx_queue_item_for_retry_at_priority(
17385 RouterTxItem::ToSide { src, dst, data },
17386 true,
17387 retry_priority,
17388 )?;
17389 return Ok(());
17390 }
17391 if !self.cfg.role.local_services() {
17392 return Err(e);
17393 }
17394 match &data {
17395 RouterItem::Packet(pkt) => {
17396 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
17397 }
17398 RouterItem::Packed(bytes) => {
17399 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
17400 let _ = self.handle_callback_error_from_env(
17401 &env,
17402 None,
17403 e,
17404 called_from_queue,
17405 );
17406 }
17407 }
17408 }
17409 return Err(TelemetryError::HandlerError("tx handler failed"));
17410 }
17411 }
17412 RouterTxItem::EndToEndReplay { packet_id } => {
17413 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
17414 return Ok(());
17415 };
17416 if sides.is_empty() {
17417 let RemoteSidePlan::Target(fallback_sides) =
17418 self.remote_side_plan(&data, None)?;
17419 sides = fallback_sides;
17420 }
17421 for side in sides.iter().copied() {
17422 #[cfg(feature = "compression")]
17423 let mut side_data = data.clone();
17424 #[cfg(feature = "compression")]
17425 let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
17426 side,
17427 &mut side_data,
17428 false,
17429 &mut context.admitted,
17430 );
17431 #[cfg(not(feature = "compression"))]
17432 let send_result = self.send_reliable_to_side_with_progress(
17433 side,
17434 data.clone(),
17435 false,
17436 &mut context.admitted,
17437 );
17438 if let Err(e) = send_result {
17439 if Self::is_transient_tx_error(&e) {
17440 context.refused = true;
17441 let retry_priority =
17442 Self::router_item_priority_bumped(DataType::ReliableAck);
17443 #[cfg(feature = "compression")]
17444 self.retry_codec_owner_at_priority(
17445 RouterTxItem::ToSide {
17446 src: None,
17447 dst: side,
17448 data: side_data,
17449 },
17450 true,
17451 &mut reservation,
17452 retry_priority,
17453 )?;
17454 #[cfg(not(feature = "compression"))]
17455 self.tx_queue_item_for_retry_at_priority(
17456 RouterTxItem::ToSide {
17457 src: None,
17458 dst: side,
17459 data: data.clone(),
17460 },
17461 true,
17462 retry_priority,
17463 )?;
17464 continue;
17465 }
17466 match &data {
17467 RouterItem::Packet(pkt) => {
17468 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
17469 }
17470 RouterItem::Packed(bytes) => {
17471 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
17472 let _ = self.handle_callback_error_from_env(
17473 &env,
17474 None,
17475 e,
17476 called_from_queue,
17477 );
17478 }
17479 }
17480 }
17481 return Err(TelemetryError::HandlerError("tx handler failed"));
17482 }
17483 }
17484 }
17485 RouterTxItem::ReliableReplay {
17486 dst,
17487 ty,
17488 seq,
17489 bytes,
17490 } => {
17491 {
17492 let mut st = self.state.lock();
17493 let Some(sent) = st
17494 .reliable_tx
17495 .get_mut(&Self::reliable_key(dst, ty))
17496 .and_then(|tx| tx.sent.get_mut(&seq))
17497 else {
17498 return Ok(());
17499 };
17500 sent.queued = false;
17501 sent.last_send_ms = self.clock.now_ms();
17502 }
17503 if let Err(e) =
17504 self.send_reliable_raw_to_side(dst, bytes.clone(), false, &mut context.admitted)
17505 {
17506 if Self::is_transient_tx_error(&e) {
17507 context.refused = true;
17508 self.tx_queue_item_for_retry(
17509 RouterTxItem::ReliableReplay {
17510 dst,
17511 ty,
17512 seq,
17513 bytes,
17514 },
17515 true,
17516 )?;
17517 return Ok(());
17518 }
17519 return Err(e);
17520 }
17521 let mut st = self.state.lock();
17522 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
17523 if let Some(sent) = tx_state.sent.get_mut(&seq) {
17524 sent.last_send_ms = self.clock.now_ms();
17525 }
17526 }
17527 }
17528
17529 Ok(())
17530 }
17531
17532 #[inline]
17534 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
17535 self.tx_item_impl(item, false, false)
17536 }
17537
17538 #[inline]
17545 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
17546 if self.side_tx_active() {
17547 return self.rx_packed_queue(bytes);
17548 }
17549 crate::memory_admission::check_receive(bytes)?;
17550 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17551 let item = RouterRxItem {
17552 src: None,
17553 priority: Self::router_item_priority(&data)?,
17554 data,
17555 };
17556 self.rx_item(&item, false)
17557 }
17558
17559 #[inline]
17564 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
17565 if self.side_tx_active() {
17566 return self.rx_queue(pkt.clone());
17567 }
17568 let data = RouterItem::Packet(pkt.clone());
17569 let item = RouterRxItem {
17570 src: None,
17571 priority: Self::router_item_priority(&data)?,
17572 data,
17573 };
17574 self.rx_item(&item, false)
17575 }
17576
17577 #[inline]
17582 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
17583 if self.side_tx_active() {
17584 return self.rx_queue_from_side(pkt.clone(), side);
17585 }
17586 self.ensure_side_ingress_enabled(side)?;
17587 let data = RouterItem::Packet(pkt.clone());
17588 let item = RouterRxItem {
17589 src: Some(side),
17590 priority: Self::router_item_priority(&data)?,
17591 data,
17592 };
17593 self.rx_item(&item, false)
17594 }
17595
17596 #[inline]
17601 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
17602 if self.side_tx_active() {
17603 return self.rx_packed_queue_from_side(bytes, side);
17604 }
17605 self.ensure_side_ingress_enabled(side)?;
17606 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
17607 return Ok(());
17608 };
17609 let data = RouterItem::Packed(decoded);
17610 let item = RouterRxItem {
17611 src: Some(side),
17612 priority: Self::router_item_priority(&data)?,
17613 data,
17614 };
17615 self.rx_item(&item, false)
17616 }
17617
17618 fn ensure_application_origin_allowed(&self) -> TelemetryResult<()> {
17619 if self.cfg.role.local_services() {
17620 Ok(())
17621 } else {
17622 Err(TelemetryError::PermissionDenied)
17623 }
17624 }
17625
17626 #[cfg(all(test, feature = "discovery"))]
17627 pub(crate) fn debug_end_to_end_acked_destination_count(&self, id: u64) -> Option<usize> {
17628 self.state
17629 .lock()
17630 .forwarded_receipts
17631 .as_ref()?
17632 .destinations
17633 .get(&id)
17634 .map(BTreeSet::len)
17635 }
17636 #[cfg(all(test, feature = "discovery"))]
17637 pub(crate) fn debug_end_to_end_acked_packet_count(&self) -> usize {
17638 self.state
17639 .lock()
17640 .forwarded_receipts
17641 .as_ref()
17642 .map_or(0, |cache| cache.destinations.len())
17643 }
17644
17645 #[inline]
17652 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
17653 self.ensure_application_origin_allowed()?;
17654 #[cfg(feature = "discovery")]
17655 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17656 if self.side_tx_active() {
17657 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
17658 }
17659 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
17660 }
17661
17662 #[inline]
17667 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
17668 self.ensure_application_origin_allowed()?;
17669 #[cfg(feature = "discovery")]
17670 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17671 if self.side_tx_active() {
17672 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
17673 crate::shared_bytes::convert(pkt),
17674 )));
17675 }
17676 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(
17677 crate::shared_bytes::convert(pkt),
17678 )))
17679 }
17680
17681 #[inline]
17687 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
17688 self.ensure_application_origin_allowed()?;
17689 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
17690 }
17691
17692 #[inline]
17694 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
17695 self.ensure_application_origin_allowed()?;
17696 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
17697 crate::shared_bytes::convert(data),
17698 )))
17699 }
17700
17701 #[inline(never)]
17707 fn log_dispatch<T: LeBytes>(
17708 &self,
17709 ty: DataType,
17710 data: &[T],
17711 timestamp: Option<u64>,
17712 mut queued: bool,
17713 ) -> TelemetryResult<()> {
17714 self.ensure_application_origin_allowed()?;
17715 if !queued {
17716 #[cfg(feature = "discovery")]
17717 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17718 queued = self.side_tx_active();
17719 }
17720 let timestamp = timestamp.unwrap_or_else(|| self.packet_timestamp_ms());
17721 log_raw(self.sender_arc(), ty, data, timestamp, |pkt| {
17722 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
17723 if queued {
17724 self.tx_queue_item(item)
17725 } else {
17726 self.tx_item(item)
17727 }
17728 })
17729 }
17730
17731 #[inline]
17734 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
17735 self.log_dispatch(ty, data, None, false)
17736 }
17737
17738 #[inline]
17740 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
17741 self.log_dispatch(ty, data, None, true)
17742 }
17743
17744 #[inline]
17746 pub fn log_ts<T: LeBytes>(
17747 &self,
17748 ty: DataType,
17749 timestamp: u64,
17750 data: &[T],
17751 ) -> TelemetryResult<()> {
17752 self.log_dispatch(ty, data, Some(timestamp), false)
17753 }
17754
17755 #[inline]
17757 pub fn log_queue_ts<T: LeBytes>(
17758 &self,
17759 ty: DataType,
17760 timestamp: u64,
17761 data: &[T],
17762 ) -> TelemetryResult<()> {
17763 self.log_dispatch(ty, data, Some(timestamp), true)
17764 }
17765}
17766
17767#[cfg(all(test, feature = "std", feature = "discovery"))]
17768mod queued_schema_identity_tests {
17769 use super::*;
17770
17771 fn catalog(timestamp: u64, byte: u8) -> Packet {
17772 Packet::new(
17773 crate::DataType::DiscoverySchema,
17774 &[DataEndpoint::Discovery],
17775 "SCHEMA_PEER",
17776 timestamp,
17777 crate::shared_bytes::convert(Arc::<[u8]>::from(vec![byte; 1024])),
17778 )
17779 .unwrap()
17780 }
17781
17782 #[test]
17783 fn inline_and_large_typed_publications_share_headers_and_keep_le_bytes() {
17784 crate::tests::ensure_common_test_schema();
17785 let endpoint = DataEndpoint::named("SD_CARD");
17786 let ty = crate::config::register_data_type_with_description(
17787 "INLINE_TYPED_BOUNDARY",
17788 "",
17789 MessageElement::Dynamic(crate::MessageDataType::UInt32, crate::MessageClass::Data),
17790 &[endpoint],
17791 crate::ReliableMode::None,
17792 16,
17793 )
17794 .unwrap();
17795 let sender: Arc<str> = Arc::from("DAQ_SOURCE");
17796 let endpoints = message_meta(ty);
17797 for count in [
17798 0,
17799 1,
17800 crate::config::STACK_PAYLOAD_SIZE / 4,
17801 crate::config::STACK_PAYLOAD_SIZE / 4 + 1,
17802 64,
17803 ] {
17804 let values = (0..count)
17805 .map(|i| 0x1234_5678u32.wrapping_add(i as u32))
17806 .collect::<Vec<_>>();
17807 log_raw(sender.clone(), ty, &values, 42, |packet| {
17808 let bytes = values
17809 .iter()
17810 .flat_map(|v| v.to_le_bytes())
17811 .collect::<Vec<_>>();
17812 assert_eq!(packet.payload(), bytes);
17813 assert_eq!(packet.sender().as_ptr(), sender.as_ptr());
17814 assert_eq!(
17815 packet.endpoints().as_ptr(),
17816 endpoints.endpoints_ref().as_ptr()
17817 );
17818 packet.validate().unwrap();
17819 Ok(())
17820 })
17821 .unwrap();
17822 }
17823 #[derive(Clone, Copy)]
17824 struct BadWidth;
17825 impl LeBytes for BadWidth {
17826 const WIDTH: usize = 0;
17827 fn write_le(self, _: &mut [u8]) {
17828 unreachable!()
17829 }
17830 fn from_le_slice(_: &[u8]) -> Self {
17831 Self
17832 }
17833 }
17834 assert!(matches!(
17835 log_raw(sender.clone(), ty, &[BadWidth], 42, |_| Ok(())),
17836 Err(TelemetryError::BadArg)
17837 ));
17838 #[derive(Clone, Copy)]
17839 struct HugeWidth;
17840 impl LeBytes for HugeWidth {
17841 const WIDTH: usize = usize::MAX;
17842 fn write_le(self, _: &mut [u8]) {
17843 unreachable!()
17844 }
17845 fn from_le_slice(_: &[u8]) -> Self {
17846 Self
17847 }
17848 }
17849 assert!(matches!(
17850 log_raw(sender, ty, &[HugeWidth, HugeWidth], 42, |_| Ok(())),
17851 Err(TelemetryError::PacketTooLarge(_))
17852 ));
17853 }
17854
17855 #[test]
17856 fn queue_accounting_reports_live_cost_without_draining_or_pinning() {
17857 crate::tests::ensure_common_test_schema();
17858 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 42u64));
17859 let packet = Packet::new(
17860 crate::DataType::named("IMU_DATA"),
17861 &[DataEndpoint::named("SD_CARD")],
17862 "SOURCE",
17863 1,
17864 crate::shared_bytes::convert(Arc::<[u8]>::from([0u8; 24])),
17865 )
17866 .unwrap();
17867 router.tx_queue(packet).unwrap();
17868 assert!(router.has_queued_work());
17869 let expected = router.memory_counters();
17870 let a = router.queue_counters();
17871 assert_eq!(a[0], 1);
17872 assert_eq!(a[1], 0);
17873 assert_eq!(a[2], 1);
17874 assert_eq!(a[3], expected[3]);
17875 assert_eq!(a[4], expected[4]);
17876 assert_eq!(a[5], crate::DataType::named("IMU_DATA").as_u32());
17877 assert_eq!(a[14], 42);
17878 assert_eq!(a[15], 0);
17879 assert_eq!(router.queue_counters(), a);
17880 assert_eq!(router.memory_counters(), expected);
17881 router.drain_queues_with_timeout(0).unwrap();
17882 let empty = router.queue_counters();
17883 assert_eq!(empty[2], 0);
17884 assert_eq!(empty[4], 0);
17885 assert_eq!(router.memory_counters()[4], 0);
17886 assert!(!router.has_queued_work());
17887 }
17888
17889 #[test]
17890 fn repeated_schema_preserves_partial_transfer_but_updates_and_other_sides_do_not_coalesce() {
17891 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
17892 let side = router.add_side_packed("A", |_| Ok(()));
17893 let other_side = router.add_side_packed("B", |_| Ok(()));
17894 let original = catalog(1, 42);
17895 let duplicate = catalog(2, 42);
17896 assert_ne!(original.packet_id(), duplicate.packet_id());
17897 let mut st = router.state.lock();
17898 st.transmit_queue.clear();
17899 let queued = |dst, pkt| TxQueued {
17900 disposable: true,
17901 item: RouterTxItem::ToSide {
17902 src: None,
17903 dst,
17904 data: RouterItem::Packet(pkt),
17905 },
17906 ignore_local: true,
17907 priority: 255,
17908 delivery_state: QueuedDelivery::Untracked,
17909 accepted_type: 0,
17910 deferred_since_ms: 0,
17911 retry_after_ms: 123,
17912 };
17913 st.push_transmit(queued(side, original.clone())).unwrap();
17914 st.push_transmit(queued(side, duplicate.clone())).unwrap();
17915 assert_eq!(st.transmit_queue.len(), 1);
17916 let pending = st.transmit_queue.iter().next().unwrap();
17917 assert_eq!(pending.retry_after_ms, 123);
17918 assert!(
17919 matches!(&pending.item, RouterTxItem::ToSide { data: RouterItem::Packet(pkt), .. }
17920 if pkt.packet_id() == original.packet_id())
17921 );
17922 st.push_transmit(queued(other_side, duplicate)).unwrap();
17923 assert_eq!(st.transmit_queue.len(), 2);
17924 let changed = catalog(3, 43);
17925 st.push_transmit(queued(side, changed.clone())).unwrap();
17926 assert_eq!(st.transmit_queue.len(), 2);
17927 assert!(st.transmit_queue.iter().any(|queued| matches!(&queued.item,
17928 RouterTxItem::ToSide { dst, data: RouterItem::Packet(pkt), .. }
17929 if *dst == side && pkt.packet_id() == changed.packet_id())));
17930 }
17931
17932 #[test]
17933 fn repeated_announcements_complete_one_fragment_per_dispatch() {
17934 use core::sync::atomic::{AtomicUsize, Ordering};
17935 let allowance = Arc::new(AtomicUsize::new(0));
17936 let accepted = Arc::new(std::sync::Mutex::new(Vec::<Arc<[u8]>>::new()));
17937 let available = allowance.clone();
17938 let frames = accepted.clone();
17939 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 1u64));
17940 let side = node.add_side_packed_with_options(
17941 "bounded CAN",
17942 move |bytes| {
17943 if available
17944 .try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
17945 count.checked_sub(1)
17946 })
17947 .is_err()
17948 {
17949 return Err(TelemetryError::Io("side tx busy"));
17950 }
17951 frames.lock().unwrap().push(Arc::<[u8]>::from(bytes));
17952 Ok(())
17953 },
17954 RouterSideOptions::default().with_small_packet_transport(128),
17955 );
17956 let handler = node.state.lock().sides[side]
17957 .as_ref()
17958 .unwrap()
17959 .tx_handler
17960 .clone();
17961 let mut complete = false;
17962 for timestamp in 1..100 {
17963 let mut state = 0x517cc1b7u32;
17965 let payload: Vec<u8> = (0..3600)
17966 .map(|_| {
17967 state ^= state << 13;
17968 state ^= state >> 17;
17969 state ^= state << 5;
17970 state as u8
17971 })
17972 .collect();
17973 let packet = Packet::new(
17974 crate::DataType::DiscoverySchema,
17975 &[crate::DataEndpoint::Discovery],
17976 "SCHEMA_PEER",
17977 timestamp,
17978 crate::shared_bytes::convert(Arc::<[u8]>::from(payload)),
17979 )
17980 .unwrap();
17981 node.state
17982 .lock()
17983 .push_transmit(TxQueued {
17984 disposable: true,
17985 item: RouterTxItem::ToSide {
17986 src: None,
17987 dst: side,
17988 data: RouterItem::Packet(packet),
17989 },
17990 ignore_local: true,
17991 priority: 255,
17992 delivery_state: QueuedDelivery::Untracked,
17993 accepted_type: 0,
17994 deferred_since_ms: 0,
17995 retry_after_ms: 0,
17996 })
17997 .unwrap();
17998 let data = match &node.state.lock().transmit_queue.iter().next().unwrap().item {
17999 RouterTxItem::ToSide { data, .. } => data.clone(),
18000 _ => panic!("unexpected item"),
18001 };
18002 allowance.store(1, Ordering::Relaxed);
18003 match node.call_side_tx_handler(side, &handler, &data, false) {
18004 Ok(()) => {
18005 complete = true;
18006 break;
18007 }
18008 Err(TelemetryError::Io(_)) => {}
18009 other => panic!("unexpected result: {other:?}"),
18010 }
18011 }
18012 assert!(
18013 complete,
18014 "new announcement timestamps must not replay the prefix forever"
18015 );
18016 let received = accepted.lock().unwrap();
18017 assert!(
18020 received.len() > 1,
18021 "fixture must exercise fragmentation, not a compressed single frame"
18022 );
18023 let first = &received[0];
18024 assert_eq!(&first[..4], &[83, 68, 84, 3]);
18025 assert_eq!(
18026 u16::from_le_bytes([first[10], first[11]]) as usize,
18027 received.len()
18028 );
18029 let transfer_id = &first[4..8];
18030 for (index, frame) in received.iter().enumerate() {
18031 assert_eq!(&frame[4..8], transfer_id);
18032 assert_eq!(u16::from_le_bytes([frame[8], frame[9]]) as usize, index);
18033 }
18034 }
18035 #[test]
18036 fn schema_receipt_resolves_assigned_numeric_address_without_adding_peers() {
18037 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
18038 let side = router.add_side_packed("A", |_| Ok(()));
18039 let assigned = 0x1234u32;
18040 assert_ne!(sender_address_u32("SCHEMA_PEER"), assigned);
18041 {
18042 let mut st = router.state.lock();
18043 st.discovery_routes
18044 .entry(side)
18045 .or_default()
18046 .announcers
18047 .insert(
18048 "SCHEMA_PEER".into(),
18049 DiscoverySenderState {
18050 advertised_address: Some(assigned),
18051 ..DiscoverySenderState::default()
18052 },
18053 );
18054 }
18055 router.note_schema_received(side, "@addr:4660");
18056 router.note_schema_received(side, "@addr:4661");
18057 let st = router.state.lock();
18058 let peers = &st.discovery_routes.get(&side).unwrap().announcers;
18059 assert_eq!(peers.len(), 1);
18060 assert!(peers["SCHEMA_PEER"].has_schema);
18061 }
18062 #[test]
18063 fn busy_handler_returns_after_one_attempt_and_other_errors_keep_retry_budget() {
18064 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
18065 let attempts = core::cell::Cell::new(0);
18066 let result: Result<((), usize), _> = router.retry_with_attempts(8, || {
18067 attempts.set(attempts.get() + 1);
18068 Err(TelemetryError::Io("side tx busy"))
18069 });
18070 assert!(matches!(
18071 result,
18072 Err((TelemetryError::Io("side tx busy"), 1))
18073 ));
18074 assert_eq!(attempts.get(), 1);
18075 attempts.set(0);
18076 let result: Result<((), usize), _> = router.retry_with_attempts(3, || {
18077 attempts.set(attempts.get() + 1);
18078 Err(TelemetryError::Io("permanent handler failure"))
18079 });
18080 assert!(matches!(
18081 result,
18082 Err((TelemetryError::Io("permanent handler failure"), 3))
18083 ));
18084 assert_eq!(attempts.get(), 3);
18085 }
18086 #[test]
18087 fn rejected_first_template_frame_is_retried_self_describing() {
18088 crate::tests::ensure_common_test_schema();
18089 let ty = crate::config::register_data_type_with_description(
18090 "ROUTER_BUSY_FIRST_TEMPLATE",
18091 "",
18092 crate::MessageElement::Static(
18093 1,
18094 crate::MessageDataType::UInt8,
18095 crate::MessageClass::Data,
18096 ),
18097 &[crate::DataEndpoint::named("RADIO")],
18098 crate::ReliableMode::None,
18099 1,
18100 )
18101 .unwrap();
18102 let tx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
18103 let rx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
18104 let opts = RouterSideOptions::default().with_small_packet_transport(0);
18105 let a = tx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
18106 let b = rx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
18107 let packet = Packet::new(
18108 ty,
18109 &[crate::DataEndpoint::named("RADIO")],
18110 "BUSY_PEER",
18111 1,
18112 crate::shared_bytes::convert(Arc::<[u8]>::from([42u8])),
18113 )
18114 .unwrap();
18115 let raw = wire_format::pack_packet(&packet);
18116 let first = tx
18118 .encode_side_transport_frames(
18119 a,
18120 opts.clone(),
18121 crate::shared_bytes::convert(raw.clone()),
18122 )
18123 .unwrap();
18124 assert!(
18125 tx.send_side_frames_resumable(a, first, |_| Err(TelemetryError::Io("side tx busy")))
18126 .is_err()
18127 );
18128 let retry = tx
18129 .encode_side_transport_frames(
18130 a,
18131 opts.clone(),
18132 crate::shared_bytes::convert(raw.clone()),
18133 )
18134 .unwrap();
18135 let mut received = None;
18136 tx.send_side_frames_resumable(a, retry, |frame| {
18137 assert_eq!(
18138 frame[3], SIDE_TRANSPORT_KIND_FULL,
18139 "retry requires the unaccepted dictionary bootstrap"
18140 );
18141 received = rx.decode_side_transport_frame(b, frame)?;
18142 Ok(())
18143 })
18144 .unwrap();
18145 assert_eq!(received, Some(crate::shared_bytes::convert(raw.clone())));
18146 let next = tx
18148 .encode_side_transport_frames(a, opts, crate::shared_bytes::convert(raw.clone()))
18149 .unwrap();
18150 tx.send_side_frames_resumable(a, next, |frame| {
18151 assert_eq!(frame[3], SIDE_TRANSPORT_KIND_COMPACT);
18152 assert_eq!(
18153 rx.decode_side_transport_frame(b, frame)?,
18154 Some(crate::shared_bytes::convert(raw.clone()))
18155 );
18156 Ok(())
18157 })
18158 .unwrap();
18159 }
18160 #[test]
18161 fn already_queued_receipt_precedes_ordered_retry_exhaustion() {
18162 crate::tests::ensure_common_test_schema();
18163 let ep = DataEndpoint::named("RADIO");
18164 let ty = crate::config::register_data_type_with_description(
18165 "QUEUED_RECEIPT_DEADLINE",
18166 "",
18167 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
18168 &[ep],
18169 crate::ReliableMode::Ordered,
18170 1,
18171 )
18172 .unwrap();
18173 for isr in [false, true] {
18174 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
18175 let clock = now.clone();
18176 let router = Router::new_with_clock(
18177 RouterConfig::new([]).with_hostname("FC"),
18178 Box::new(move || clock.load(Ordering::Relaxed)),
18179 );
18180 let side = router.add_side_packed("CAN", |_| Ok(()));
18181 let packet = Packet::new(ty, &[ep], "FC", 1, Arc::<[u8]>::from([42u8])).unwrap();
18182 let id = packet.packet_id();
18183 let owner = Router::sender_hash("GS");
18184 let mut destinations = BTreeMap::new();
18185 destinations.insert(owner, side);
18186 router.state.lock().end_to_end_reliable_tx.insert(
18187 id,
18188 EndToEndReliableSent {
18189 ordered_type: Some(ty.as_u32()),
18190 waiting_for_predecessor: false,
18191 data: RouterItem::Packet(packet),
18192 pending_destinations: destinations,
18193 tracked_destinations: true,
18194 last_send_ms: 0,
18195 retries: runtime_reliable_max_retries(),
18196 queued: false,
18197 }
18198 .try_box()
18199 .unwrap(),
18200 );
18201 let snapshot = router.export_reliable_pending_stats();
18202 assert_eq!(snapshot.len(), 1);
18203 assert_eq!(snapshot[0].packet_id, id);
18204 assert_eq!(snapshot[0].data_type, ty);
18205 assert_eq!(snapshot[0].pending_destinations, vec![owner]);
18206 assert_eq!(snapshot[0].retries, runtime_reliable_max_retries());
18207 assert!(!snapshot[0].queued);
18208 assert!(!snapshot[0].waiting_for_predecessor);
18209 let ack = Packet::new_with_wire_contract(
18210 DataType::ReliableAck,
18211 message_meta(DataType::ReliableAck).endpoints_ref(),
18212 "E2EACK:GS",
18213 1,
18214 1,
18215 Arc::<[u8]>::from(id.to_le_bytes()),
18216 None,
18217 Arc::from([owner, Router::sender_hash("FC")]),
18218 )
18219 .unwrap();
18220 let packed = wire_format::pack_packet(&ack);
18221 if isr {
18222 router.rx_packed_queue_from_side_isr(&packed, side).unwrap();
18223 } else {
18224 router.rx_packed_queue_from_side(&packed, side).unwrap();
18225 }
18226 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
18227 router
18228 .drain_queues_with_timeout(1)
18229 .expect("a received ACK must be processed before cancelling its ordered delivery");
18230 assert!(!router.state.lock().end_to_end_reliable_tx.contains_key(&id));
18231 assert!(router.export_reliable_pending_stats().is_empty());
18232 }
18233 }
18234 #[test]
18235 fn embedded_gateway_receipt_with_numeric_sender_clears_host_pending_owner() {
18236 crate::tests::ensure_common_test_schema();
18237 let bytes = [
18240 0x0c, 0x01, 0x01, 0x08, 0xd4, 0xe5, 0xea, 0xb6, 0xb7, 0x33, 0xff, 0x90, 0x02, 0xb4,
18241 0xe6, 0xdd, 0x94, 0x07, 0x14, 0x03, 0x44, 0x02, 0x02, 0x06, 0x25, 0xd2, 0x59, 0x1b,
18242 0xbb, 0xfc, 0xa7, 0x5f, 0x43, 0x92, 0x9e, 0x12, 0xbb, 0xfc, 0xa7, 0x26, 0xa2, 0xa0,
18243 0xbe, 0x39, 0x65, 0xe2, 0xd8, 0x61, 0x68, 0x22, 0xfd,
18244 ];
18245 let ack = wire_format::unpack_packet(&bytes).unwrap();
18246 let owner = Router::sender_hash("GB");
18247 assert_eq!(Router::end_to_end_ack_sender_hash(&ack), Some(owner));
18248 let id = Router::decode_end_to_end_reliable_ack(ack.payload()).unwrap();
18249 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
18250 let side = router.add_side_packed("Pico", |_| Ok(()));
18251 let data = ack.clone();
18252 router.state.lock().end_to_end_reliable_tx.insert(
18253 id,
18254 EndToEndReliableSent {
18255 ordered_type: None,
18256 waiting_for_predecessor: false,
18257 data: RouterItem::Packet(data),
18258 pending_destinations: BTreeMap::from([(owner, side)]),
18259 tracked_destinations: true,
18260 last_send_ms: 0,
18261 retries: 0,
18262 queued: false,
18263 }
18264 .try_box()
18265 .unwrap(),
18266 );
18267 router.rx_packed_queue_from_side(&bytes, side).unwrap();
18268 router.drain_queues_with_timeout(1).unwrap();
18269 assert!(router.export_reliable_pending_stats().is_empty());
18270 }
18271
18272 #[test]
18273 fn frozen_delivery_contract_bypasses_dictionary_even_for_best_effort_schema() {
18274 crate::tests::ensure_common_test_schema();
18275 let ep = DataEndpoint::named("RADIO");
18276 let ty = crate::config::register_data_type_with_description(
18277 "MANAGED_CONTROL_WITH_FROZEN_OWNERS",
18278 "",
18279 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
18280 &[ep],
18281 crate::ReliableMode::None,
18282 16,
18283 )
18284 .unwrap();
18285 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
18286 let opts = RouterSideOptions::default().with_template_transport();
18287 let side = router.add_side_packed_with_options("radio", |_| Ok(()), opts.clone());
18288 let packet = Packet::new_with_wire_contract(
18289 ty,
18290 &[ep],
18291 "GS",
18292 1,
18293 1,
18294 Arc::<[u8]>::from([1u8]),
18295 None,
18296 Arc::from([Router::sender_hash("RF")]),
18297 )
18298 .unwrap();
18299 let raw = wire_format::pack_packet(&packet);
18300 assert_eq!(
18301 router
18302 .item_service_priority(&RouterItem::Packet(packet.clone()))
18303 .unwrap(),
18304 254
18305 );
18306 assert_eq!(
18307 router
18308 .item_service_priority(&RouterItem::Packed(crate::shared_bytes::convert(
18309 raw.clone()
18310 )))
18311 .unwrap(),
18312 254
18313 );
18314 assert!(
18315 !wire_format::peek_routing_frame_info(&raw)
18316 .unwrap()
18317 .envelope
18318 .is_reliable
18319 );
18320 for _ in 0..3 {
18321 let frames = router
18322 .encode_side_transport_frames(
18323 side,
18324 opts.clone(),
18325 crate::shared_bytes::convert(raw.clone()),
18326 )
18327 .unwrap();
18328 let mut count = 0;
18329 router
18330 .send_side_frames_resumable(side, frames, |bytes| {
18331 assert_eq!(
18332 bytes,
18333 raw.as_ref(),
18334 "delivery contract must remain independently decodable"
18335 );
18336 count += 1;
18337 Ok(())
18338 })
18339 .unwrap();
18340 assert_eq!(count, 1);
18341 }
18342 }
18343
18344 #[test]
18345 fn managed_receive_queues_keep_service_priority_with_low_schema_priority() {
18346 crate::tests::ensure_common_test_schema();
18347 let ep = DataEndpoint::named("RADIO");
18348 let ty = crate::config::register_data_type_with_description(
18349 "MANAGED_RX_SERVICE_PRIORITY",
18350 "",
18351 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
18352 &[ep],
18353 crate::ReliableMode::None,
18354 16,
18355 )
18356 .unwrap();
18357 let router = Router::new(RouterConfig::new([]).with_hostname("RF"));
18358 router
18359 .enable_network_variable(ty, NetworkVariablePermissions::READ_ONLY)
18360 .unwrap();
18361 let side = router.add_side_packed("radio", |_| Ok(()));
18362 router.clear_queues();
18363 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
18364 let wire = wire_format::pack_packet(&packet);
18365 router.rx_queue(packet.clone()).unwrap();
18366 router.rx_queue_from_side(packet.clone(), side).unwrap();
18367 router.rx_packed_queue(&wire).unwrap();
18368 router.rx_packed_queue_from_side(&wire, side).unwrap();
18369 for _ in 0..4 {
18370 let queued = router
18371 .pop_next_queued_control_rx_item()
18372 .unwrap()
18373 .expect("registered managed controls must precede queued bulk work");
18374 assert_eq!(
18375 queued.priority,
18376 crate::transport_priority(DataType::ManagedVariableValue)
18377 );
18378 }
18379 assert!(router.pop_next_queued_control_rx_item().unwrap().is_none());
18380 }
18381
18382 #[test]
18383 fn managed_value_keeps_service_priority_after_side_backpressure() {
18384 crate::tests::ensure_common_test_schema();
18385 let ep = DataEndpoint::named("RADIO");
18386 let ty = crate::config::register_data_type_with_description(
18387 "MANAGED_RETRY_SERVICE_PRIORITY",
18388 "",
18389 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
18390 &[ep],
18391 crate::ReliableMode::None,
18392 16,
18393 )
18394 .unwrap();
18395 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
18396 router
18397 .enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
18398 .unwrap();
18399 let side = router.add_side_packed("radio", |_| Err(TelemetryError::Io("side tx busy")));
18400 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
18401 let item = RouterItem::Packet(packet);
18402 for queued in [
18403 RouterTxItem::Broadcast(item.clone()),
18404 RouterTxItem::ToSide {
18405 src: None,
18406 dst: side,
18407 data: item.clone(),
18408 },
18409 ] {
18410 router.tx_queue_item_for_retry(queued, true).unwrap();
18411 let queued = router.state.lock().transmit_queue.pop_front().unwrap();
18412 assert_eq!(
18413 queued.priority,
18414 crate::transport_priority(DataType::ManagedVariableValue),
18415 "a busy side must not demote managed controls to bulk application priority"
18416 );
18417 assert!(queued.retry_after_ms > 0);
18418 }
18419 }
18420 #[test]
18421 fn receipt_budget_does_not_starve_transmit() {
18422 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
18423 crate::tests::ensure_common_test_schema();
18424 let now = Arc::new(AtomicU64::new(0));
18425 let clock = now.clone();
18426 let count = Arc::new(AtomicUsize::new(0));
18427 let accepted = count.clone();
18428 let node = Router::new_with_clock(
18429 RouterConfig::default(),
18430 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
18431 );
18432 let side = node.add_side_packed("wire", move |_| {
18433 accepted.fetch_add(1, Ordering::Relaxed);
18434 Ok(())
18435 });
18436 node.clear_queues();
18437 let packet = Packet::new(
18438 crate::DataType::ReliableAck,
18439 &[crate::DataEndpoint::TelemetryError],
18440 "E2EACK:PEER",
18441 1,
18442 Arc::<[u8]>::from([0u8; 8]),
18443 )
18444 .unwrap();
18445 node.state
18446 .lock()
18447 .push_transmit(TxQueued {
18448 disposable: true,
18449 item: RouterTxItem::ToSide {
18450 src: None,
18451 dst: side,
18452 data: RouterItem::Packet(packet.clone()),
18453 },
18454 ignore_local: true,
18455 priority: 250,
18456 delivery_state: QueuedDelivery::Untracked,
18457 accepted_type: 0,
18458 deferred_since_ms: 0,
18459 retry_after_ms: 0,
18460 })
18461 .unwrap();
18462 node.rx_packed_queue_from_side(
18463 &wire_format::pack_packet(
18464 &Packet::new(
18465 crate::DataType::ReliableAck,
18466 &[crate::DataEndpoint::TelemetryError],
18467 "RX_PEER",
18468 2,
18469 Arc::<[u8]>::from([0u8; 8]),
18470 )
18471 .unwrap(),
18472 ),
18473 side,
18474 )
18475 .unwrap();
18476 node.drain_queues_with_timeout(1).unwrap();
18477 assert_eq!(
18478 count.load(Ordering::Relaxed),
18479 1,
18480 "control RX exhausting its cooperative budget must still allow one queued TX"
18481 );
18482 }
18483 #[test]
18484 fn receipt_budget_does_not_starve_bulk_transmit() {
18485 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
18486 crate::tests::ensure_common_test_schema();
18487 let now = Arc::new(AtomicU64::new(0));
18488 let clock = now.clone();
18489 let count = Arc::new(AtomicUsize::new(0));
18490 let accepted = count.clone();
18491 let node = Router::new_with_clock(
18492 RouterConfig::default(),
18493 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
18494 );
18495 let side = node.add_side_packed("wire", move |_| {
18496 accepted.fetch_add(1, Ordering::Relaxed);
18497 Ok(())
18498 });
18499 node.clear_queues();
18500 let packet = Packet::new(
18501 crate::DataType::ReliableAck,
18502 &[crate::DataEndpoint::TelemetryError],
18503 "E2EACK:PEER",
18504 1,
18505 Arc::<[u8]>::from([0u8; 8]),
18506 )
18507 .unwrap();
18508 node.state
18509 .lock()
18510 .push_transmit(TxQueued {
18511 disposable: true,
18512 item: RouterTxItem::ToSide {
18513 src: None,
18514 dst: side,
18515 data: RouterItem::Packet(packet.clone()),
18516 },
18517 ignore_local: true,
18518 priority: 1,
18519 delivery_state: QueuedDelivery::Untracked,
18520 accepted_type: 0,
18521 deferred_since_ms: 0,
18522 retry_after_ms: 0,
18523 })
18524 .unwrap();
18525 node.rx_packed_queue_from_side(
18526 &wire_format::pack_packet(
18527 &Packet::new(
18528 crate::DataType::ReliableAck,
18529 &[crate::DataEndpoint::TelemetryError],
18530 "RX_PEER",
18531 2,
18532 Arc::<[u8]>::from([0u8; 8]),
18533 )
18534 .unwrap(),
18535 ),
18536 side,
18537 )
18538 .unwrap();
18539 node.process_all_queues_with_timeout_impl(1).unwrap();
18540 assert_eq!(
18541 count.load(Ordering::Relaxed),
18542 1,
18543 "control RX exhausting its cooperative budget must still allow one queued bulk TX"
18544 );
18545 }
18546}
18547
18548#[cfg(all(test, feature = "discovery", feature = "std"))]
18549fn untemplated_test_node() -> Router {
18550 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0))
18551}
18552#[cfg(all(test, feature = "discovery", feature = "std"))]
18553fn untemplated_test_options() -> RouterSideOptions {
18554 RouterSideOptions::default().with_small_packet_transport(64)
18555}
18556#[cfg(all(test, feature = "discovery", feature = "std"))]
18557#[allow(clippy::duplicate_mod)]
18558#[path = "tests/untemplated_transport.rs"]
18559mod untemplated_transport_tests;
18560
18561#[cfg(all(test, feature = "std", feature = "discovery"))]
18562mod queued_producer_maintenance_tests {
18563 use super::*;
18564 #[test]
18565 fn queued_publication_defers_discovery_to_worker_maintenance() {
18566 crate::tests::ensure_common_test_schema();
18567 let ty = DataType::named("GPS_DATA");
18568 for variant in 0..4 {
18569 let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
18570 let output = sent.clone();
18571 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
18572 router.add_side_packed("wire", move |bytes| {
18573 output
18574 .lock()
18575 .unwrap()
18576 .push(wire_format::unpack_packet(bytes)?.data_type());
18577 Ok(())
18578 });
18579 router.clear_queues();
18580 let packet =
18581 Packet::from_f32_slice(ty, &[1.0, 2.0, 3.0], message_meta(ty).endpoints_ref(), 42)
18582 .unwrap();
18583 match variant {
18584 0 => router.tx_queue(packet).unwrap(),
18585 1 => router
18586 .tx_packed_queue(wire_format::pack_packet(&packet))
18587 .unwrap(),
18588 2 => router.log_queue(ty, &[1.0f32, 2.0, 3.0]).unwrap(),
18589 _ => router.log_queue_ts(ty, 42, &[1.0f32, 2.0, 3.0]).unwrap(),
18590 }
18591 assert!(
18592 sent.lock().unwrap().is_empty(),
18593 "producer invoked a transport"
18594 );
18595 assert_eq!(
18596 router.state.lock().transmit_queue.len(),
18597 1,
18598 "queued API performed discovery on the producer stack (variant {variant})"
18599 );
18600 router.periodic(0).unwrap();
18601 let sent = sent.lock().unwrap();
18602 assert!(sent.contains(&ty));
18603 assert!(
18604 sent.iter().copied().any(discovery::is_discovery_type),
18605 "worker maintenance must still announce the board"
18606 );
18607 }
18608 }
18609}
18610
18611#[cfg(all(test, feature = "std"))]
18612mod bounded_fair_queue_tests {
18613 use super::*;
18614 #[test]
18615 fn small_canonical_reports_survive_fast_senders_fixed_ring_bursts() {
18616 let ep = crate::config::register_endpoint("ROUTER_SMALL_QUEUE_DEST", false).unwrap();
18617 let ty = crate::config::register_data_type_with_description(
18618 "ROUTER_SMALL_QUEUE_REPORT",
18619 "",
18620 crate::MessageElement::Static(
18621 4,
18622 crate::MessageDataType::UInt8,
18623 crate::MessageClass::Data,
18624 ),
18625 &[ep],
18626 crate::ReliableMode::None,
18627 0,
18628 )
18629 .unwrap();
18630 for tx in [false, true] {
18631 for slow_last in [false, true] {
18632 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
18633 let make = |sender| {
18634 RouterItem::Packed(crate::shared_bytes::convert(wire_format::pack_packet(
18635 &Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 4])).unwrap(),
18636 )))
18637 };
18638 let rx = |data| RouterRxItem {
18639 src: None,
18640 data,
18641 priority: 0,
18642 };
18643 let tx_item = |data| TxQueued {
18644 disposable: true,
18645 item: RouterTxItem::Broadcast(data),
18646 ignore_local: true,
18647 priority: 0,
18648 delivery_state: QueuedDelivery::Untracked,
18649 accepted_type: 0,
18650 deferred_since_ms: 0,
18651 retry_after_ms: 0,
18652 };
18653 let mut st = node.state.lock();
18654 st.memory.max_queue_budget = usize::MAX;
18655 if tx {
18656 st.transmit_queue = BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
18657 } else {
18658 st.received_queue = BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
18659 }
18660 let order = if slow_last {
18661 ["FAST", "FAST", "SLOW"]
18662 } else {
18663 ["SLOW", "FAST", "FAST"]
18664 };
18665 for sender in order.into_iter().chain(core::iter::repeat_n("FAST", 32)) {
18666 if tx {
18667 st.push_transmit(tx_item(make(sender))).unwrap();
18668 } else {
18669 st.push_received(rx(make(sender))).unwrap();
18670 }
18671 }
18672 let slow = wire_format::source_address_for_sender("SLOW");
18673 let retained = if tx {
18674 st.transmit_queue
18675 .iter()
18676 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
18677 } else {
18678 st.received_queue.iter().any(|item| {
18679 RouterInner::fair_bulk_owner(&item.data, item.priority) == Some(slow)
18680 })
18681 };
18682 assert!(
18683 retained,
18684 "small report was displaced: tx={tx}, slow_last={slow_last}"
18685 );
18686 }
18687 }
18688 }
18689
18690 #[test]
18691 fn slow_stream_survives_same_board_fast_stream_bursts() {
18692 let ep = crate::config::register_endpoint("ROUTER_STREAM_QUEUE_DEST", false).unwrap();
18693 let ty = crate::config::register_data_type_with_description(
18694 "ROUTER_STREAM_QUEUE_FAST",
18695 "",
18696 crate::MessageElement::Static(
18697 64,
18698 crate::MessageDataType::UInt8,
18699 crate::MessageClass::Data,
18700 ),
18701 &[ep],
18702 crate::ReliableMode::None,
18703 0,
18704 )
18705 .unwrap();
18706 let slow_ty = crate::config::register_data_type_with_description(
18707 "ROUTER_STREAM_QUEUE_SLOW",
18708 "",
18709 crate::MessageElement::Static(
18710 64,
18711 crate::MessageDataType::UInt8,
18712 crate::MessageClass::Data,
18713 ),
18714 &[ep],
18715 crate::ReliableMode::None,
18716 0,
18717 )
18718 .unwrap();
18719 for mode in 0..3 {
18720 for tx in [false, true] {
18721 for slow_last in [false, true] {
18722 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
18723 let make = |sender| {
18724 let packet = Packet::new(
18725 if sender == "SLOW" { slow_ty } else { ty },
18726 &[ep],
18727 "SAME_BOARD",
18728 1,
18729 Arc::<[u8]>::from([1; 64]),
18730 )
18731 .unwrap();
18732 if mode == 1 {
18733 RouterItem::Packet(packet)
18734 } else {
18735 RouterItem::Packed(crate::shared_bytes::convert(
18736 wire_format::pack_packet(&packet),
18737 ))
18738 }
18739 };
18740 let rx = |data| RouterRxItem {
18741 src: None,
18742 data,
18743 priority: 0,
18744 };
18745 let tx_item = |data| TxQueued {
18746 disposable: true,
18747 item: RouterTxItem::Broadcast(data),
18748 ignore_local: true,
18749 priority: 0,
18750 delivery_state: QueuedDelivery::Untracked,
18751 accepted_type: 0,
18752 deferred_since_ms: 0,
18753 retry_after_ms: 0,
18754 };
18755 let mut st = node.state.lock();
18756 st.memory.max_queue_budget = usize::MAX;
18757 if mode == 0 {
18758 if tx {
18759 st.transmit_queue =
18760 BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
18761 } else {
18762 st.received_queue =
18763 BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
18764 }
18765 } else if mode == 1 {
18766 let cost = if tx {
18767 tx_item(make("SLOW")).byte_cost()
18768 + 2 * tx_item(make("FAST")).byte_cost()
18769 } else {
18770 rx(make("SLOW")).byte_cost() + 2 * rx(make("FAST")).byte_cost()
18771 };
18772 if tx {
18773 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
18774 } else {
18775 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
18776 }
18777 }
18778 let order = if slow_last {
18779 ["FAST", "FAST", "SLOW"]
18780 } else {
18781 ["SLOW", "FAST", "FAST"]
18782 };
18783 for (index, sender) in order
18784 .into_iter()
18785 .chain(core::iter::repeat_n("FAST", 32))
18786 .enumerate()
18787 {
18788 if mode == 2 && index == 3 {
18789 st.memory.max_queue_budget = st.shared_queue_bytes_used();
18790 }
18791 if tx {
18792 st.push_transmit(tx_item(make(sender))).unwrap();
18793 } else {
18794 st.push_received(rx(make(sender))).unwrap();
18795 }
18796 }
18797 let is_slow = |data: &RouterItem| match data {
18798 RouterItem::Packet(p) => p.data_type() == slow_ty,
18799 RouterItem::Packed(b) => {
18800 wire_format::scheduling_type(b.as_ref()) == Some(slow_ty)
18801 }
18802 };
18803 let retained = if tx {
18804 st.transmit_queue.iter().any(|item| {
18805 let data = match &item.item {
18806 RouterTxItem::Broadcast(data)
18807 | RouterTxItem::ToSide { data, .. } => data,
18808 _ => return false,
18809 };
18810 is_slow(data)
18811 })
18812 } else {
18813 st.received_queue.iter().any(|item| is_slow(&item.data))
18814 };
18815 assert!(
18816 retained,
18817 "slow stream displaced: mode={mode}, tx={tx}, slow_last={slow_last}"
18818 );
18819 }
18820 }
18821 }
18822 }
18823
18824 #[test]
18825 fn fast_producer_preserves_slow_reports_at_each_queue_boundary() {
18826 let ep = crate::config::register_endpoint("QUEUE_FAIR_DEST", false).unwrap();
18827 let ty = crate::config::register_data_type_with_description(
18828 "QUEUE_FAIR_REPORT",
18829 "",
18830 MessageElement::Static(64, crate::MessageDataType::UInt8, crate::MessageClass::Data),
18831 &[ep],
18832 crate::ReliableMode::None,
18833 0,
18834 )
18835 .unwrap();
18836 for mode in 0..3 {
18837 for tx in [false, true] {
18838 for slow_last in [false, true] {
18839 for next_sender in ["FAST", "NEW_SLOW"] {
18840 let router =
18841 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
18842 let make = |sender| {
18843 RouterItem::Packet(
18844 Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 64]))
18845 .unwrap(),
18846 )
18847 };
18848 let rx = |data| RouterRxItem {
18849 src: None,
18850 data,
18851 priority: 0,
18852 };
18853 let tx_item = |data| TxQueued {
18854 disposable: true,
18855 item: RouterTxItem::Broadcast(data),
18856 ignore_local: true,
18857 priority: 0,
18858 delivery_state: QueuedDelivery::Untracked,
18859 accepted_type: 0,
18860 deferred_since_ms: 0,
18861 retry_after_ms: 0,
18862 };
18863 let order = if slow_last {
18864 ["FAST", "FAST", "SLOW"]
18865 } else {
18866 ["SLOW", "FAST", "FAST"]
18867 };
18868 let mut st = router.state.lock();
18869 st.memory.max_queue_budget = usize::MAX;
18870 let cost = if tx {
18871 order.iter().map(|s| tx_item(make(s)).byte_cost()).sum()
18872 } else {
18873 order.iter().map(|s| rx(make(s)).byte_cost()).sum()
18874 };
18875 if mode == 0 {
18876 if tx {
18877 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
18878 } else {
18879 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
18880 }
18881 } else if mode == 1 {
18882 if tx {
18883 st.transmit_queue =
18884 BoundedDeque::new(cost * 10, 3 * size_of::<TxQueued>(), 1.0);
18885 } else {
18886 st.received_queue = BoundedDeque::new(
18887 cost * 10,
18888 3 * size_of::<RouterRxItem>(),
18889 1.0,
18890 );
18891 }
18892 }
18893 for sender in order {
18894 if tx {
18895 st.push_transmit(tx_item(make(sender))).unwrap();
18896 } else {
18897 st.push_received(rx(make(sender))).unwrap();
18898 }
18899 }
18900 if mode == 2 {
18901 st.memory.max_queue_budget = st.shared_queue_bytes_used();
18902 }
18903 if tx {
18904 st.push_transmit(tx_item(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
18905 } else {
18906 st.push_received(rx(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
18907 }
18908 let slow = wire_format::source_address_for_sender("SLOW");
18909 let retained = if tx {
18910 st.transmit_queue
18911 .iter()
18912 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
18913 } else {
18914 st.received_queue.iter().any(|item| {
18915 RouterInner::fair_bulk_owner(&item.data, item.priority)
18916 == Some(slow)
18917 })
18918 };
18919 assert!(
18920 retained,
18921 "slow report evicted: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"
18922 );
18923 }
18924 }
18925 }
18926 }
18927 }
18928}
18929
18930#[cfg(all(test, feature = "discovery"))]
18931#[test]
18932fn cow_discovery_rejected_growth_keeps_original_graph_and_routes() {
18933 crate::tests::ensure_common_test_schema();
18934 let node = Router::new_with_clock(RouterConfig::default().with_sender("GB"), Box::new(|| 0));
18935 let side = node.add_side_packed("can", |_| Ok(()));
18936 let baseline = discovery::build_discovery_topology(
18937 "PEER",
18938 0,
18939 &[TopologyBoardNode {
18940 sender_id: "PEER".into(),
18941 reachable_endpoints: Vec::new(),
18942 reachable_timesync_sources: Vec::new(),
18943 connections: Vec::new(),
18944 }],
18945 )
18946 .unwrap();
18947 node.rx_from_side(&baseline, side).unwrap();
18948 node.process_all_queues().unwrap();
18949 node.clear_queues();
18950 let before = node.export_topology();
18951 let original = {
18952 let mut st = node.state.lock();
18953 let graph = st.discovery_routes[&side].announcers["PEER"]
18954 .topology_boards
18955 .clone();
18956 st.memory.max_queue_budget = st.shared_queue_bytes_used();
18957 graph
18958 };
18959 let update = discovery::build_discovery_topology_delta(
18960 "PEER",
18961 1,
18962 &[TopologyBoardNode {
18963 sender_id: "PEER".into(),
18964 reachable_endpoints: Vec::new(),
18965 reachable_timesync_sources: vec!["NEW_CLOCK".into()],
18966 connections: vec!["NEW_NEIGHBOR".into()],
18967 }],
18968 &[],
18969 )
18970 .unwrap();
18971 assert!(
18972 node.rx_from_side(&update, side)
18973 .and_then(|_| node.process_all_queues())
18974 .is_err(),
18975 "growth must fail when no queue budget can be reclaimed"
18976 );
18977 assert_eq!(node.export_topology().routes, before.routes);
18978 let st = node.state.lock();
18979 assert!(Arc::ptr_eq(
18980 &original,
18981 &st.discovery_routes[&side].announcers["PEER"].topology_boards
18982 ));
18983 assert!(original[0].connections.is_empty());
18984 assert!(original[0].reachable_timesync_sources.is_empty());
18985}
18986
18987#[cfg(all(test, feature = "std"))]
18988mod bounded_rx_priority_tests {
18989 use super::*;
18990 #[test]
18991 fn retained_slow_report_cannot_starve_behind_continuous_fast_data() {
18992 let ep =
18993 crate::config::register_endpoint("RouterRxItem_FAIR_EP_RouterInner", false).unwrap();
18994 let ty = crate::config::register_data_type_with_description(
18995 "RouterRxItem_FAIR_DATA_RouterInner",
18996 "",
18997 crate::MessageElement::Static(
18998 1,
18999 crate::MessageDataType::Float32,
19000 crate::MessageClass::Data,
19001 ),
19002 &[ep],
19003 crate::ReliableMode::None,
19004 0,
19005 )
19006 .unwrap();
19007 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
19008 let mut st = node.state.lock();
19009 st.memory.max_queue_budget = 65536;
19010 st.received_queue = BoundedDeque::new(32768, 16 * size_of::<RouterRxItem>(), 1.0);
19011 let make = |sender: &str, stamp, priority| {
19012 let packet = Packet::new(ty, &[ep], sender, stamp, Arc::<[u8]>::from([0; 4])).unwrap();
19013 RouterRxItem {
19014 src: None,
19015 data: RouterItem::Packet(packet),
19016 priority,
19017 }
19018 };
19019 st.push_received(make("VALVE", 0, 0)).unwrap();
19020 let mut delivered = false;
19021 for tick in 1..=32u64 {
19022 for index in 0..3 {
19023 st.push_received(make("DAQ", tick * 3 + index, 16)).unwrap();
19024 }
19025 let item = st.pop_received_fair().unwrap();
19026 let sender = match &item.data {
19027 RouterItem::Packet(packet) => packet.sender(),
19028 _ => panic!("expected packet"),
19029 };
19030 if sender == "VALVE" {
19031 delivered = true;
19032 assert!(tick <= u64::from(MAX_PRIORITY_BURST) + 1);
19033 break;
19034 }
19035 }
19036 assert!(
19037 delivered,
19038 "retention alone does not deliver low-rate reports under a continuous higher-priority source"
19039 );
19040 st.received_queue.clear();
19041 st.rx_priority_burst = MAX_PRIORITY_BURST;
19042 st.push_received(make("VALVE", 100, 0)).unwrap();
19043 st.push_received(make("VALVE", 101, 0)).unwrap();
19044 st.push_received(make("DAQ", 102, 16)).unwrap();
19045 st.push_received(make("COMMAND", 103, 254)).unwrap();
19046 assert_eq!(
19047 st.pop_received_fair().unwrap().priority,
19048 254,
19049 "a fairness turn must never delay protected control"
19050 );
19051 let mut first_slow = None;
19052 for tick in 0..=MAX_PRIORITY_BURST {
19053 st.push_received(make("DAQ", 200 + u64::from(tick), 16))
19054 .unwrap();
19055 let item = st.pop_received_fair().unwrap();
19056 if let RouterItem::Packet(packet) = &item.data
19057 && packet.sender() == "VALVE"
19058 {
19059 first_slow = Some(packet.timestamp());
19060 break;
19061 }
19062 }
19063 assert_eq!(
19064 first_slow,
19065 Some(100),
19066 "fairness preserves same-priority FIFO order"
19067 );
19068 st.rx_priority_burst = MAX_PRIORITY_BURST;
19069 drop(st);
19070 node.clear_rx_queue();
19071 assert_eq!(node.state.lock().rx_priority_burst, 0);
19072 }
19073}
19074
19075#[cfg(all(test, feature = "std", feature = "discovery"))]
19076#[path = "tests/accepted_tx_ownership.rs"]
19077mod accepted_tx_ownership;