1#[cfg(all(test, feature = "std", feature = "discovery"))]
12#[path = "tests/scoped_schema_recovery_audit.rs"]
13mod scoped_schema_recovery_audit;
14
15#[cfg(test)]
16#[path = "tests/relay_legacy_contracts.rs"]
17mod relay_legacy_contracts;
18
19#[cfg(all(test, feature = "std", feature = "discovery"))]
20#[path = "tests/shared_routing_roles.rs"]
21mod shared_routing_roles;
22
23#[cfg(all(test, feature = "std", feature = "discovery"))]
24#[path = "tests/schema_prefix_admission.rs"]
25mod schema_prefix_admission;
26
27#[cfg(all(test, feature = "std", feature = "discovery"))]
28#[path = "tests/schema_receive_progress.rs"]
29mod schema_receive_progress;
30
31#[cfg(all(test, feature = "std", feature = "discovery"))]
32#[path = "tests/radio_trace_decoder.rs"]
33mod radio_trace_decoder;
34
35#[cfg(all(test, feature = "std", feature = "discovery"))]
36#[path = "tests/schema_response_admission.rs"]
37mod schema_response_admission_tests;
38
39#[cfg(all(test, feature = "std", feature = "discovery"))]
40#[path = "tests/accepted_route_ownership.rs"]
41mod accepted_route_ownership;
42
43#[cfg(all(test, feature = "std"))]
44#[path = "tests/independent_side_tx_router.rs"]
45mod independent_side_tx;
46#[cfg(all(test, feature = "std", feature = "discovery"))]
47#[path = "tests/ingress_target_filter.rs"]
48mod ingress_target_filter_tests;
49#[cfg(all(test, feature = "std", feature = "discovery", feature = "compression"))]
50#[path = "tests/payload_codec_negotiation.rs"]
51mod payload_codec_negotiation;
52#[cfg(all(test, feature = "std", feature = "discovery"))]
53#[path = "tests/reliable_tx_window_backpressure.rs"]
54mod reliable_tx_window_backpressure_tests;
55#[cfg(all(test, feature = "std", feature = "discovery"))]
56#[path = "tests/shared_chunk_budget_router.rs"]
57mod shared_chunk_budget_tests;
58#[cfg(test)]
60#[allow(clippy::duplicate_mod)]
61#[path = "tests/side_template_index.rs"]
62mod side_template_index_tests;
63#[cfg(feature = "discovery")]
64use crate::config::runtime_reliable_max_end_to_end_pending;
65#[cfg(all(test, feature = "std", feature = "discovery"))]
66#[path = "tests/replay_ownership_router.rs"]
67mod replay_ownership_tests;
68use crate::side_transport::{
69 SIDE_TRANSPORT_FLAG_PACKET_NONCE, SIDE_TRANSPORT_KIND_CHUNK, SIDE_TRANSPORT_KIND_COMPACT,
70 SIDE_TRANSPORT_KIND_COMPACT_DELTA, SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP,
71 SIDE_TRANSPORT_KIND_FULL, SideChunkAssembly, SideCompactTimestampMode, SideHeaderTemplate,
72 SideTransportFrames, extract_side_header_template, extract_side_header_template_view,
73 parse_side_transport_wrapper, read_uleb128_local, reconstruct_side_compact_frame,
74 wrap_side_transport_frame, write_uleb128_local,
75};
76#[cfg(all(test, feature = "discovery"))]
77#[path = "tests/ack_return_identity.rs"]
78mod ack_return_identity;
79#[cfg(all(test, feature = "discovery"))]
80#[path = "tests/ack_split_horizon.rs"]
81mod ack_split_horizon;
82
83#[cfg(all(test, feature = "discovery"))]
84#[path = "tests/discovery_liveness.rs"]
85mod discovery_liveness;
86
87#[cfg(all(test, feature = "discovery"))]
88mod compact_summary_tests {
89 use super::*;
90
91 #[test]
92 fn ordered_pending_window_is_bounded_and_exhaustion_cancels_the_tail() {
93 crate::tests::ensure_common_test_schema();
94 let ep = DataEndpoint::named("RADIO");
95 let ty = crate::config::register_data_type_with_description(
96 "ORDERED_WINDOW_REGRESSION",
97 "bounded ordered window",
98 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
99 &[ep],
100 crate::ReliableMode::Ordered,
101 200,
102 )
103 .unwrap();
104 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
105 let clock = now.clone();
106 let router = Router::new_with_clock(
107 RouterConfig::new([]),
108 Box::new(move || clock.load(Ordering::Relaxed)),
109 );
110 let capacity = runtime_reliable_max_end_to_end_pending().max(1);
111 for index in 0..capacity {
112 let item = RouterItem::Packet(
113 Packet::new(
114 ty,
115 &[ep],
116 "SOURCE",
117 index as u64 + 1,
118 Arc::<[u8]>::from([index as u8]),
119 )
120 .unwrap(),
121 );
122 assert_eq!(
123 router.register_end_to_end_reliable_tx(&item, None).unwrap(),
124 index == 0
125 );
126 }
127 let rejected = RouterItem::Packet(
128 Packet::new(
129 ty,
130 &[ep],
131 "SOURCE",
132 capacity as u64 + 1,
133 Arc::<[u8]>::from([0]),
134 )
135 .unwrap(),
136 );
137 assert!(
138 router
139 .register_end_to_end_reliable_tx(&rejected, None)
140 .is_err()
141 );
142 {
143 let mut state = router.state.lock();
144 assert_eq!(state.end_to_end_reliable_tx.len(), capacity);
145 let first = state.end_to_end_reliable_tx_order[0];
146 state
147 .end_to_end_reliable_tx
148 .get_mut(&first)
149 .unwrap()
150 .retries = runtime_reliable_max_retries();
151 }
152 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
153 assert!(router.process_end_to_end_reliable_timeouts().is_err());
154 assert!(router.state.lock().end_to_end_reliable_tx.is_empty());
155 }
156 #[test]
157 fn compact_reachability_survives_incomplete_detailed_topology() {
158 crate::tests::ensure_common_test_schema();
159 let router = Router::new(RouterConfig::new([]).with_sender("GB"));
160 let endpoint = DataEndpoint::try_named("GROUND_STATION")
161 .or_else(|| DataEndpoint::try_named("RADIO"))
162 .unwrap();
163 router.compact_summary_fixture(endpoint);
164 let state = router.state.lock();
165 let now = router.clock.now_ms();
166 let advertised =
167 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(1));
168 assert!(
169 advertised.contains(&endpoint),
170 "compact subscriber route was lost at the bridge"
171 );
172 let reflected =
173 router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(0));
174 assert!(
175 !reflected.contains(&endpoint),
176 "split horizon must still exclude ingress"
177 );
178 assert_eq!(
179 router.advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(1)),
180 vec!["clock-behind-bridge".to_string()]
181 );
182 assert!(
183 router
184 .advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(0))
185 .is_empty()
186 );
187 assert!(
188 !router
189 .advertised_discovery_endpoints_for_link_locked(
190 &state,
191 now + DISCOVERY_ROUTE_TTL_MS + 1,
192 true,
193 Some(1)
194 )
195 .contains(&endpoint)
196 );
197 assert!(
198 router
199 .advertised_discovery_timesync_sources_for_link_locked(
200 &state,
201 now + DISCOVERY_ROUTE_TTL_MS + 1,
202 Some(1)
203 )
204 .is_empty()
205 );
206 }
207}
208
209use crate::config::RuntimeMemoryConfig;
210use crate::diagnostics::{
211 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
212 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
213 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
214};
215#[cfg(all(feature = "discovery", feature = "timesync"))]
216use crate::discovery::TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS;
217#[cfg(feature = "discovery")]
218use crate::discovery::{
219 self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
220 DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState, TopologyAnnouncerRoute,
221 TopologyBoardNode, TopologySideRoute, TopologySnapshot,
222};
223use crate::packet::{hash_bytes_u64, sender_address_u32};
224use crate::queue::{BoundedDeque, ByteCost};
225#[cfg(all(not(feature = "std"), target_os = "none"))]
226use crate::seds_error_msg;
227#[cfg(feature = "timesync")]
228use crate::timesync::{
229 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
230 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
231 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
232 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
233 decode_timesync_request, decode_timesync_response,
234};
235use crate::{
236 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
237 config::{
238 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
239 runtime_reliable_max_pending, runtime_reliable_max_retries,
240 runtime_reliable_max_return_routes, runtime_reliable_retransmit_ms,
241 },
242 get_needed_message_size, impl_letype_num, is_reliable_type,
243 lock::{ReentryGate, RouterMutex},
244 message_e2e_encryption_policy, message_meta, message_priority,
245 packet::Packet,
246 reliable_mode, wire_format,
247};
248use alloc::string::{String, ToString};
249
250#[cfg(any(test, all(feature = "embedded", not(feature = "std"))))]
251use crate::small_map::SmallMap as RouterMap;
252#[cfg(not(any(test, all(feature = "embedded", not(feature = "std")))))]
253use alloc::collections::BTreeMap as RouterMap;
254
255use alloc::collections::BTreeMap;
256
257use alloc::{
258 borrow::ToOwned,
259 boxed::Box,
260 collections::{BTreeSet, VecDeque},
261 format,
262 sync::Arc,
263 vec,
264 vec::Vec,
265};
266use core::cell::UnsafeCell;
267use core::fmt;
268use core::fmt::{Debug, Formatter};
269use core::mem::size_of;
270use core::ops::{Deref, DerefMut};
271use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
272#[cfg(feature = "std")]
273use std::time::Instant;
274
275pub type RouterSideId = usize;
277
278const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
279pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
280pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
281pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
282static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
283
284#[derive(Clone, Copy, Debug, PartialEq, Eq)]
285pub enum SideTransportProfile {
286 Canonical,
287 Template,
288 Ipv6Like,
289 Ipv4Like,
290}
291
292impl SideTransportProfile {
293 #[inline]
294 pub const fn as_str(self) -> &'static str {
295 match self {
296 Self::Canonical => "canonical",
297 Self::Template => "template",
298 Self::Ipv6Like => "ipv6_like",
299 Self::Ipv4Like => "ipv4_like",
300 }
301 }
302
303 #[cfg(feature = "discovery")]
304 #[inline]
305 pub const fn discovery_code(self) -> u8 {
306 match self {
307 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
308 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
309 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
310 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
311 }
312 }
313}
314
315#[cfg(test)]
316mod chunk_assembly_memory_tests {
317 use super::*;
318 #[test]
319 fn full_chunk_budget_reclaims_other_partial_transfers_before_active_schema() {
320 let mut state = SideTransportState::default();
321 for id in 1..=4 {
322 let mut assembly = SideChunkAssembly {
323 total: 28,
324 last_seen_ms: id as u64,
325 ..Default::default()
326 };
327 for index in 0..23 {
328 assembly.insert(
329 index,
330 crate::shared_bytes::convert(Arc::<[u8]>::from([id as u8; 112])),
331 );
332 }
333 state.rx_chunks.insert(id, assembly);
334 }
335 state.rx_chunks.get_mut(&1).unwrap().insert(
336 23,
337 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 56])),
338 );
339 state.refresh_rx_chunk_bytes();
341 SideChunkAssembly::make_room(
342 &mut state.rx_chunks,
343 &mut state.rx_chunk_bytes,
344 4,
345 23,
346 28,
347 112,
348 16384,
349 )
350 .unwrap();
351 assert!(!state.rx_chunks.contains_key(&1));
352 assert_eq!(state.rx_chunks[&4].received_len(), 23);
353 for index in 23..28 {
354 state.refresh_rx_chunk_bytes();
355 SideChunkAssembly::make_room(
356 &mut state.rx_chunks,
357 &mut state.rx_chunk_bytes,
358 4,
359 index,
360 28,
361 112,
362 16384,
363 )
364 .unwrap();
365 state.rx_chunks.get_mut(&4).unwrap().insert(
366 index,
367 crate::shared_bytes::convert(Arc::<[u8]>::from([4u8; 112])),
368 );
369 }
370 assert_eq!(
371 state.rx_chunks.remove(&4).unwrap().assemble().unwrap(),
372 vec![4u8; 28 * 112]
373 );
374 assert!(
375 state
376 .rx_chunks
377 .values()
378 .flat_map(|a| a.received.iter().flatten())
379 .map(|p| p.len() + 64)
380 .sum::<usize>()
381 <= 16384
382 );
383 }
384
385 #[test]
386 fn duplicate_chunk_keeps_partial_transfers_but_oversize_transfer_is_rejected() {
387 let mut state = SideTransportState::default();
388 let mut assembly = SideChunkAssembly {
389 total: 2,
390 ..Default::default()
391 };
392 assembly.insert(
393 0,
394 crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 112])),
395 );
396 state.rx_chunks.insert(1, assembly.clone());
397 state.rx_chunks.insert(2, assembly);
398 state.refresh_rx_chunk_bytes();
399 SideChunkAssembly::make_room(
400 &mut state.rx_chunks,
401 &mut state.rx_chunk_bytes,
402 1,
403 0,
404 2,
405 112,
406 352,
407 )
408 .unwrap();
409 assert_eq!(state.rx_chunks.len(), 2);
410 state.refresh_rx_chunk_bytes();
411 assert!(
412 SideChunkAssembly::make_room(
413 &mut state.rx_chunks,
414 &mut state.rx_chunk_bytes,
415 1,
416 1,
417 2,
418 400,
419 352
420 )
421 .is_err()
422 );
423 assert!(!state.rx_chunks.contains_key(&1));
424 assert!(state.rx_chunks.contains_key(&2));
425 }
426
427 #[test]
428 fn schema_sized_reassembly_allocates_exactly_and_preserves_order() {
429 let payload: Vec<u8> = (0..3652).map(|i| (i % 251) as u8).collect();
430 let mut assembly = SideChunkAssembly {
431 total: payload.len().div_ceil(56) as u16,
432 ..Default::default()
433 };
434 for (index, chunk) in payload.chunks(56).enumerate().rev() {
435 assembly.insert(
436 index as u16,
437 crate::shared_bytes::convert(Arc::<[u8]>::from(chunk)),
438 );
439 }
440 assert_eq!(assembly.received_len(), usize::from(assembly.total));
441 let out = assembly.assemble().unwrap();
442 assert_eq!(out, payload);
443 assert_eq!(
444 out.capacity(),
445 payload.len(),
446 "do not double discovery reassembly capacity"
447 );
448 }
449 #[test]
450 fn incomplete_or_noncontiguous_chunks_are_rejected() {
451 let mut assembly = SideChunkAssembly {
452 total: 2,
453 ..Default::default()
454 };
455 assembly.insert(0, crate::shared_bytes::convert(Arc::<[u8]>::from([1u8])));
456 assert!(assembly.clone().assemble().is_err());
457 assert!(
458 assembly
459 .insert_with(2, 0, || Arc::<[u8]>::from([2u8]))
460 .is_err()
461 );
462 assert!(assembly.assemble().is_err());
463 }
464}
465
466#[derive(Clone, Debug, Default)]
467struct SideTransportState {
468 #[cfg(feature = "std")]
469 tx_callback_gate: Arc<ReentryGate>,
470 tx_chunk_progress: Option<Box<crate::side_transport::SideTxProgress>>,
471 #[cfg(all(feature = "compression", feature = "discovery"))]
472 payload_codec: crate::payload_codec::Ledger,
473 #[cfg(all(feature = "compression", feature = "discovery"))]
474 payload_codec_choice: u64,
475 tx_template_ids: RouterMap<u64, u32>,
476 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
477 tx_last_timestamps: RouterMap<u32, u64>,
478 tx_compact_uses: RouterMap<u32, u8>,
479 rx_templates: RouterMap<u64, u32>,
480 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
481 rx_last_timestamps: RouterMap<u32, u64>,
482 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
483 rx_chunk_reclaims: u32,
484 rx_chunk_bytes: usize,
485}
486
487impl SideTransportState {
488 #[inline(never)]
489 fn take_chunk(&mut self, id: u32, reclaimed: bool) -> Option<SideChunkAssembly> {
490 let value = SideChunkAssembly::remove(&mut self.rx_chunks, &mut self.rx_chunk_bytes, id)?;
491 if reclaimed {
492 self.rx_chunk_reclaims = self.rx_chunk_reclaims.saturating_add(1);
493 }
494 Some(value)
495 }
496 #[cfg(test)]
498 #[inline(never)]
499 fn refresh_rx_chunk_bytes(&mut self) {
500 self.rx_chunk_bytes = self
501 .rx_chunks
502 .values()
503 .map(SideChunkAssembly::byte_cost)
504 .sum();
505 }
506 fn refresh_transmit_header_templates(&mut self) {
507 for uses in self.tx_compact_uses.values_mut() {
511 *uses = u8::MAX;
512 }
513 }
516
517 fn tx_template_count(&self) -> usize {
518 self.tx_template_ids.len()
519 }
520
521 fn rx_template_count(&self) -> usize {
522 self.rx_templates_by_id.len()
523 }
524
525 fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
526 let mut removed = 0usize;
527 while self.tx_template_ids.len() > limit {
528 let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
529 break;
530 };
531 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
532 self.tx_last_timestamps.remove(&old_id);
533 self.tx_compact_uses.remove(&old_id);
534 }
535 self.tx_templates.remove(&old_hash);
536 removed += 1;
537 }
538 removed
539 }
540
541 fn insert_tx_template(
542 &mut self,
543 template: SideHeaderTemplate,
544 template_id: u32,
545 max_templates: usize,
546 ) -> bool {
547 if max_templates == 0 {
548 return false;
549 }
550 let evicted = !self.tx_template_ids.contains_key(&template.hash)
551 && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
552 self.tx_template_ids.insert(template.hash, template_id);
553 self.tx_templates.insert(template.hash, template);
554 self.tx_compact_uses.insert(template_id, 0);
555 evicted
556 }
557
558 fn insert_rx_template(
559 &mut self,
560 template_id: u32,
561 template: SideHeaderTemplate,
562 max_templates: usize,
563 ) -> bool {
564 if max_templates == 0 {
565 return false;
566 }
567 if let Some(old_id) = self.rx_templates.get(&template.hash).copied()
570 && old_id != template_id
571 {
572 self.rx_templates_by_id.remove(&old_id);
573 self.rx_last_timestamps.remove(&old_id);
574 self.rx_templates.remove(&template.hash);
575 }
576 if let Some(previous) = self.rx_templates_by_id.get(&template_id)
577 && previous.hash != template.hash
578 {
579 self.rx_templates.remove(&previous.hash);
580 self.rx_last_timestamps.remove(&template_id);
581 }
582 let mut evicted = false;
583 if self.rx_templates_by_id.len() >= max_templates
584 && !self.rx_templates_by_id.contains_key(&template_id)
585 && let Some((&old_hash, &old_id)) = self.rx_templates.first_key_value()
588 {
589 self.rx_templates.remove(&old_hash);
590 self.rx_templates_by_id.remove(&old_id);
591 self.rx_last_timestamps.remove(&old_id);
592 evicted = true;
593 }
594 self.rx_templates.insert(template.hash, template_id);
595 self.rx_templates_by_id.insert(template_id, template);
596 evicted
597 }
598}
599
600#[derive(Clone, Debug, PartialEq, Eq)]
601pub struct CompactTimestampOmissionPolicy {
602 all: bool,
603 types: Option<Arc<[DataType]>>,
604}
605impl CompactTimestampOmissionPolicy {
606 pub const fn none() -> Self {
607 Self {
608 all: false,
609 types: None,
610 }
611 }
612 pub const fn all() -> Self {
613 Self {
614 all: true,
615 types: None,
616 }
617 }
618 pub fn with_data_type(mut self, ty: DataType) -> Self {
619 self.insert(ty);
620 self
621 }
622 pub fn insert(&mut self, ty: DataType) {
623 if self.contains(ty) {
624 return;
625 }
626 let current = self.types.as_deref().unwrap_or(&[]);
627 let mut types = Vec::with_capacity(current.len() + 1);
628 types.extend_from_slice(current);
629 let index = types.binary_search(&ty).unwrap_err();
630 types.insert(index, ty);
631 self.types = Some(Arc::from(types));
632 }
633 pub fn contains(&self, ty: DataType) -> bool {
634 self.all
635 || self
636 .types
637 .as_deref()
638 .is_some_and(|types| types.binary_search(&ty).is_ok())
639 }
640 pub fn is_empty(&self) -> bool {
641 !self.all && self.types.is_none()
642 }
643}
644
645impl Default for CompactTimestampOmissionPolicy {
646 #[inline]
647 fn default() -> Self {
648 Self::none()
649 }
650}
651
652#[derive(Clone, Debug, PartialEq, Eq)]
653pub enum RouterItem {
654 Packet(Packet),
655 Packed(crate::SharedBytes),
656}
657
658#[cfg(feature = "compact-packet-store")]
659impl RouterItem {
660 fn park_for_queue(&mut self) -> TelemetryResult<()> {
661 match self {
662 Self::Packet(packet) => packet.park_for_queue()?,
663 Self::Packed(bytes) => bytes.park_for_queue()?,
664 }
665 Ok(())
666 }
667}
668
669#[derive(Clone, Debug, PartialEq, Eq)]
670struct RouterRxItem {
671 src: Option<RouterSideId>,
672 data: RouterItem,
673 priority: u8,
674}
675
676#[derive(Clone, Debug, PartialEq, Eq)]
677enum RouterTxItem {
678 Broadcast(RouterItem),
679 EndToEndReplay {
680 packet_id: u64,
681 },
682 ToSide {
683 src: Option<RouterSideId>,
684 dst: RouterSideId,
685 data: RouterItem,
686 },
687 ReliableReplay {
688 dst: RouterSideId,
689 ty: DataType,
690 seq: u32,
691 bytes: crate::SharedBytes,
692 },
693}
694
695impl ByteCost for RouterRxItem {
696 #[cfg(feature = "compact-packet-store")]
697 fn park_for_queue(&mut self) -> TelemetryResult<()> {
698 self.data.park_for_queue()
699 }
700 #[inline]
701 fn byte_cost(&self) -> usize {
702 match &self.data {
703 RouterItem::Packet(pkt) => pkt.byte_cost(),
704 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
705 }
706 }
707}
708
709impl ByteCost for RouterTxItem {
710 #[cfg(feature = "compact-packet-store")]
711 fn park_for_queue(&mut self) -> TelemetryResult<()> {
712 match self {
713 Self::Broadcast(item) | Self::ToSide { data: item, .. } => item.park_for_queue(),
714 Self::ReliableReplay { bytes, .. } => bytes.park_for_queue(),
715 _ => Ok(()),
716 }
717 }
718 #[inline]
719 fn byte_cost(&self) -> usize {
720 match self {
721 RouterTxItem::Broadcast(data) => match data {
722 RouterItem::Packet(pkt) => pkt.byte_cost(),
723 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
724 },
725 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
726 RouterTxItem::ToSide { data, .. } => match data {
727 RouterItem::Packet(pkt) => pkt.byte_cost(),
728 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
729 },
730 RouterTxItem::ReliableReplay { bytes, .. } => {
731 size_of::<crate::SharedBytes>() + bytes.len()
732 }
733 }
734 }
735}
736
737#[derive(Clone, Copy)]
738enum ReplayMarkerKey {
739 EndToEnd(u64),
740 Hop(RouterSideId, DataType, u32),
741}
742impl RouterTxItem {
743 fn replay_marker_key(&self) -> Option<ReplayMarkerKey> {
744 match self {
745 Self::EndToEndReplay { packet_id } => Some(ReplayMarkerKey::EndToEnd(*packet_id)),
746 Self::ReliableReplay { dst, ty, seq, .. } => {
747 Some(ReplayMarkerKey::Hop(*dst, *ty, *seq))
748 }
749 _ => None,
750 }
751 }
752}
753
754#[repr(u8)]
755#[derive(Clone, Copy, Debug, PartialEq, Eq)]
756enum QueuedDelivery {
757 Untracked,
758 OrderedTail,
759 Ordered,
760 DeferredOrdered,
761 DeferredUnordered,
762 Unordered,
763 ClaimedOrdered,
764 ClaimedUnordered,
765}
766
767const TX_DISPOSABLE: u8 = 1;
768const TX_E2E_CHILD: u8 = 8;
769#[cfg(feature = "std")]
770const TX_E2E_RETRY: u8 = 16;
771#[cfg(feature = "latest-value-egress")]
772const TX_LATEST_PENDING: u8 = 2;
773#[cfg(feature = "latest-value-egress")]
774const TX_HISTORY_OWNED: u8 = 4;
775#[cfg(feature = "latest-value-egress")]
776const TX_CANONICAL_OWED: u8 = 32;
777#[cfg(feature = "latest-value-egress")]
778const TX_CANONICAL_SERVED: u8 = 64;
779#[cfg(feature = "latest-value-egress")]
780const TX_CANONICAL_TURN: u8 = TX_CANONICAL_OWED | TX_CANONICAL_SERVED;
781#[cfg(feature = "latest-value-egress")]
782const TX_LATEST_TURN: u8 = 128;
783#[cfg(feature = "latest-value-egress")]
784const TX_FRESH: u8 = TX_DISPOSABLE | TX_LATEST_PENDING;
785#[cfg(not(feature = "latest-value-egress"))]
786const TX_FRESH: u8 = TX_DISPOSABLE;
787
788#[derive(Clone, Debug, PartialEq, Eq)]
792struct TxQueued {
793 disposable: u8,
795 item: RouterTxItem,
796 ignore_local: bool,
797 priority: u8,
798 delivery_state: QueuedDelivery,
799 accepted_type: u32,
800 deferred_since_ms: u64,
801 retry_after_ms: u64,
802}
803
804#[derive(Clone, Copy)]
805struct TxClaim {
806 token: u32,
807 ordered: bool,
808 side_retry: bool,
809}
810const _: () = {
811 assert!(size_of::<TxClaim>() == size_of::<(u32, bool)>());
812 assert!(size_of::<Option<TxClaim>>() == size_of::<Option<(u32, bool)>>());
813};
814#[derive(Clone, Copy, Default)]
815struct EgressService {
816 #[cfg(feature = "latest-value-egress")]
817 token: u32,
818 #[cfg(feature = "latest-value-egress")]
819 side: u8,
820 #[cfg(feature = "latest-value-egress")]
821 kind: u8,
822}
823#[derive(Default)]
824struct DispatchContext {
825 priority: u8,
827 claim: Option<TxClaim>,
828 since: Option<u64>,
829 completion_flags: u8,
830 admitted: bool,
831 #[cfg(feature = "latest-value-egress")]
832 latest_service: EgressService,
833}
834impl DispatchContext {
835 const TERMINAL: u8 = 1;
836 const OWNER_TRANSFERRED: u8 = 2;
837 const REFUSED: u8 = 4;
838 fn refused(&self) -> bool {
839 self.completion_flags & Self::REFUSED != 0
840 }
841 fn mark_refused(&mut self) {
842 self.completion_flags |= Self::REFUSED;
843 }
844
845 fn egress_service(&self) -> EgressService {
846 #[cfg(feature = "latest-value-egress")]
847 {
848 self.latest_service
849 }
850 #[cfg(not(feature = "latest-value-egress"))]
851 {
852 EgressService::default()
853 }
854 }
855 fn yielded_canonical_turn(&self) -> bool {
856 #[cfg(feature = "latest-value-egress")]
857 {
858 self.latest_service.side != 0
859 && self.completion_flags & TX_CANONICAL_OWED != 0
860 && self.admitted
861 }
862 #[cfg(not(feature = "latest-value-egress"))]
863 {
864 false
865 }
866 }
867 fn canonical_turn_flags(&self) -> u8 {
868 #[cfg(feature = "latest-value-egress")]
869 if self.completion_flags & TX_LATEST_TURN != 0 {
870 return TX_LATEST_TURN;
871 }
872 #[cfg(feature = "latest-value-egress")]
873 if self.completion_flags & TX_CANONICAL_OWED != 0 {
874 return if self.admitted {
875 TX_CANONICAL_SERVED
876 } else {
877 self.completion_flags & TX_CANONICAL_TURN
878 };
879 }
880 0
881 }
882
883 fn is_terminal(&self) -> bool {
884 self.completion_flags & Self::TERMINAL != 0
885 }
886 fn owner_transferred(&self) -> bool {
887 self.completion_flags & Self::OWNER_TRANSFERRED != 0
888 }
889 fn mark_terminal(&mut self) {
890 self.completion_flags |= Self::TERMINAL;
891 }
892 #[cfg(feature = "discovery")]
893 fn mark_owner_transferred(&mut self) {
894 self.completion_flags |= Self::OWNER_TRANSFERRED;
895 }
896}
897const _: () = assert!(
901 size_of::<DispatchContext>()
902 <= size_of::<(Option<TxClaim>, Option<u64>, bool, bool)>() + size_of::<u64>()
903);
904
905struct TxWork {
906 queued: TxQueued,
907 claim: Option<TxClaim>,
908 owed_turn: bool,
909 #[cfg(feature = "latest-value-egress")]
910 service: EgressService,
911}
912
913const MAX_PRIORITY_BURST: u8 = 8;
918
919const _: () = {
925 assert!(
926 size_of::<TxQueued>()
927 == size_of::<(RouterTxItem, bool, u8, QueuedDelivery, u32, u64, u64)>()
928 );
929 let metadata_bytes = 2 * size_of::<u64>()
930 + size_of::<u32>()
931 + size_of::<QueuedDelivery>()
932 + 2 * size_of::<bool>()
933 + size_of::<u8>();
934 let alignment = core::mem::align_of::<TxQueued>();
935 let expected = (size_of::<RouterTxItem>() + metadata_bytes).div_ceil(alignment) * alignment;
936 assert!(size_of::<TxQueued>() == expected);
937};
938
939impl ByteCost for TxQueued {
940 #[cfg(feature = "compact-packet-store")]
941 fn park_for_queue(&mut self) -> TelemetryResult<()> {
942 self.item.park_for_queue()
943 }
944 #[inline]
946 fn byte_cost(&self) -> usize {
947 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>() + 2 * size_of::<u64>()
948 }
949}
950
951impl ByteCost for u64 {
953 #[inline]
955 fn byte_cost(&self) -> usize {
956 size_of::<u64>()
957 }
958}
959
960#[derive(Debug, Clone)]
963struct ReliableTxState {
964 next_seq: u32,
965 sent_order: VecDeque<u32>,
966 sent: BTreeMap<u32, ReliableSent>,
967}
968
969#[derive(Debug, Clone)]
970struct ReliableSent {
971 bytes: crate::SharedBytes,
972 last_send_ms: u64,
973 retries: u32,
974 queued: bool,
975 partial_acked: bool,
976}
977
978#[cfg(feature = "std")]
981const PENDING_TYPE_METADATA_BYTES: usize = size_of::<u64>();
982
983#[derive(Debug, Clone)]
984struct EndToEndReliableSent {
985 #[cfg(feature = "std")]
986 completion_epoch: u64,
987 #[cfg(feature = "std")]
990 pending_type: u32,
991 ordered_type: Option<u32>,
992 waiting_for_predecessor: bool,
993 data: RouterItem,
994 pending_destinations: BTreeMap<u64, RouterSideId>,
995 tracked_destinations: bool,
996 last_send_ms: u64,
997 retries: u32,
998 queued: bool,
999}
1000
1001impl EndToEndReliableSent {
1002 #[cfg(feature = "discovery")]
1006 fn try_box(self) -> TelemetryResult<Box<Self>> {
1007 let layout = core::alloc::Layout::new::<Self>();
1008 let ptr = unsafe { alloc::alloc::alloc(layout).cast::<Self>() };
1012 if ptr.is_null() {
1013 return Err(TelemetryError::Io("memory pressure"));
1014 }
1015 unsafe {
1016 ptr.write(self);
1017 Ok(Box::from_raw(ptr))
1018 }
1019 }
1020}
1021
1022#[derive(Debug, Clone)]
1023struct ReliableRxState {
1024 expected_seq: u32,
1025 buffered: RouterMap<u32, crate::SharedBytes>,
1026}
1027
1028const RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES: usize = size_of::<u64>();
1032
1033#[derive(Debug, Clone)]
1034struct ReliableReturnRouteState {
1035 side: RouterSideId,
1036 delivery_completed: bool,
1039 local_admission_completed: bool,
1040 local_admission_token: u64,
1041 admission_cancelled: bool,
1042 admission_committing: bool,
1043 last_replay_ms: u64,
1044 upstream_contract: Option<(u64, u64)>,
1048}
1049
1050struct ReliableLocalAdmissionGuard<'a> {
1053 router: &'a Router,
1054 packet_id: u64,
1055 token: u64,
1056}
1057#[cfg(all(test, feature = "std"))]
1058std::thread_local! {
1059 static LOCAL_ADMISSION_CHECKS: core::cell::Cell<usize> = const {core::cell::Cell::new(0)};
1060}
1061impl ReliableLocalAdmissionGuard<'_> {
1062 fn check_locked(&self, state: &RouterInner) -> TelemetryResult<()> {
1063 #[cfg(all(test, feature = "std"))]
1064 LOCAL_ADMISSION_CHECKS.with(|count| count.set(count.get() + 1));
1065 if state
1066 .reliable_return_routes
1067 .get(&self.packet_id)
1068 .is_some_and(|route| {
1069 route.local_admission_token == self.token && !route.admission_cancelled
1070 })
1071 {
1072 Ok(())
1073 } else {
1074 Err(TelemetryError::Io("side changed during rx"))
1075 }
1076 }
1077 fn check(&self) -> TelemetryResult<()> {
1078 self.check_locked(&self.router.state.lock())
1079 }
1080 fn begin_commit(&self, local_accepted: bool) -> TelemetryResult<()> {
1081 let mut state = self.router.state.lock();
1082 self.check_locked(&state)?;
1083 let route = state
1084 .reliable_return_routes
1085 .get_mut(&self.packet_id)
1086 .expect("checked lease");
1087 route.admission_committing = true;
1088 route.local_admission_completed |= local_accepted;
1089 Ok(())
1090 }
1091 fn end_commit(&self) {
1092 let mut state = self.router.state.lock();
1093 if let Some(route) = state.reliable_return_routes.get_mut(&self.packet_id)
1094 && route.local_admission_token == self.token
1095 {
1096 route.admission_committing = false;
1097 }
1098 }
1099}
1100
1101impl Drop for ReliableLocalAdmissionGuard<'_> {
1102 fn drop(&mut self) {
1103 let mut state = self.router.state.lock();
1104 let cancelled = if let Some(route) = state.reliable_return_routes.get_mut(&self.packet_id)
1105 && route.local_admission_token == self.token
1106 {
1107 route.local_admission_token = 0;
1108 route.admission_committing = false;
1109 route.admission_cancelled
1110 } else {
1111 false
1112 };
1113 if cancelled {
1114 state.reliable_return_routes.remove(&self.packet_id);
1115 state
1116 .reliable_return_route_order
1117 .retain(|id| *id != self.packet_id);
1118 }
1119 }
1120}
1121
1122#[cfg(feature = "discovery")]
1123#[derive(Debug, Clone, Default, PartialEq, Eq)]
1124struct DiscoverySenderState {
1125 advertised_address: Option<u32>,
1128 reachable: Vec<DataEndpoint>,
1129 advertised_reachable: Vec<DataEndpoint>,
1130 reachable_network_variables: Vec<DataType>,
1131 reachable_timesync_sources: Vec<String>,
1132 advertised_reachable_timesync_sources: Vec<String>,
1133 link_capabilities: Option<discovery::LinkCapabilities>,
1134 topology_boards: Arc<Vec<TopologyBoardNode>>,
1135 has_full_topology: bool,
1136 has_schema: bool,
1137 last_seen_ms: u64,
1138}
1139
1140#[cfg(feature = "discovery")]
1143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144struct ManagedVariableSubscription {
1145 last_seen_ms: u64,
1146 next_refresh_ms: u64,
1147 request_id: u64,
1148 value_id: u64,
1149 upstream: RouterSideId,
1150 ty: DataType,
1151 requester: u32,
1152 writer: u32,
1153 refresh_pending: bool,
1154}
1155
1156#[cfg(feature = "discovery")]
1157fn reserve_managed_subscriptions(
1158 requests: &mut Vec<ManagedVariableSubscription>,
1159 additional: usize,
1160) -> TelemetryResult<()> {
1161 let needed = requests
1162 .len()
1163 .checked_add(additional)
1164 .ok_or(TelemetryError::Io("memory pressure"))?;
1165 if needed <= requests.capacity() {
1166 return Ok(());
1167 }
1168 let bytes = needed
1169 .checked_mul(size_of::<ManagedVariableSubscription>())
1170 .ok_or(TelemetryError::Io("memory pressure"))?;
1171 crate::memory_admission::check(bytes, bytes)?;
1172 requests
1173 .try_reserve_exact(additional)
1174 .map_err(|_| TelemetryError::Io("memory pressure"))
1175}
1176
1177#[cfg(feature = "discovery")]
1178#[derive(Debug, Clone, Default, PartialEq, Eq)]
1179struct DiscoverySideState {
1180 reachable: Vec<DataEndpoint>,
1181 reachable_network_variables: Vec<DataType>,
1182 requested_network_variables: Vec<ManagedVariableSubscription>,
1183 reachable_timesync_sources: Vec<String>,
1184 last_seen_ms: u64,
1185 announcers: BTreeMap<String, Box<DiscoverySenderState>>,
1186}
1187
1188#[cfg(feature = "discovery")]
1189impl DiscoverySideState {
1190 fn try_clone(&self) -> TelemetryResult<Self> {
1191 let announcers = crate::peer_record::try_clone_map(&self.announcers)?;
1195 let mut requests = Vec::new();
1196 reserve_managed_subscriptions(&mut requests, self.requested_network_variables.len())?;
1197 requests.extend_from_slice(&self.requested_network_variables);
1198 Ok(Self {
1199 reachable: self.reachable.clone(),
1200 reachable_network_variables: self.reachable_network_variables.clone(),
1201 requested_network_variables: requests,
1202 reachable_timesync_sources: self.reachable_timesync_sources.clone(),
1203 last_seen_ms: self.last_seen_ms,
1204 announcers,
1205 })
1206 }
1207}
1208
1209#[cfg(feature = "discovery")]
1210#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1211#[repr(u8)]
1212enum SchemaResponsePhase {
1213 #[default]
1214 Idle,
1215 PrefixBuilding,
1216 CatalogWaiting,
1217 CatalogBuilding,
1218 Owned,
1219}
1220
1221#[cfg(feature = "discovery")]
1222#[derive(Debug, Clone, Default)]
1223struct DiscoverySideThrottleState {
1224 next_topology_request_ms: u64,
1225 next_schema_request_ms: u64,
1226 schema_request_cursor: u32,
1228 next_schema_response_ms: u64,
1231 schema_response_packet_id: u64,
1232 schema_response_phase: SchemaResponsePhase,
1233 flags: u8,
1234 next_ping_ms: u64,
1235 last_liveness_tx_ms: u64,
1237 incremental_retries_remaining: u8,
1238 last_topology: Vec<TopologyBoardNode>,
1239}
1240
1241#[cfg(feature = "discovery")]
1242impl DiscoverySideThrottleState {
1243 #[inline]
1244 fn flag(&self, mask: u8) -> bool {
1245 self.flags & mask != 0
1246 }
1247 #[inline]
1248 fn set_flag(&mut self, mask: u8, value: bool) {
1249 if value {
1250 self.flags |= mask;
1251 } else {
1252 self.flags &= !mask;
1253 }
1254 }
1255 #[inline]
1256 fn schema_response_active(&self) -> bool {
1257 self.flag(1 << 0)
1258 }
1259 #[inline]
1260 fn set_schema_response_active(&mut self, value: bool) {
1261 self.set_flag(1 << 0, value)
1262 }
1263 #[inline]
1264 fn schema_response_started(&self) -> bool {
1265 self.flag(1 << 1)
1266 }
1267 #[inline]
1268 fn set_schema_response_started(&mut self, value: bool) {
1269 self.set_flag(1 << 1, value)
1270 }
1271 #[inline]
1272 fn schema_response_pending(&self) -> bool {
1273 self.flag(1 << 2)
1274 }
1275 #[inline]
1276 fn set_schema_response_pending(&mut self, value: bool) {
1277 self.set_flag(1 << 2, value)
1278 }
1279 #[inline]
1280 fn pending_incremental(&self) -> bool {
1281 self.flag(1 << 3)
1282 }
1283 #[inline]
1284 fn set_pending_incremental(&mut self, value: bool) {
1285 self.set_flag(1 << 3, value)
1286 }
1287 #[inline]
1288 fn has_sent_full(&self) -> bool {
1289 self.flag(1 << 4)
1290 }
1291 #[inline]
1292 fn set_has_sent_full(&mut self, value: bool) {
1293 self.set_flag(1 << 4, value)
1294 }
1295
1296 fn liveness_due(&self, now_ms: u64) -> bool {
1297 now_ms >= self.next_ping_ms
1298 && now_ms.saturating_sub(self.last_liveness_tx_ms)
1299 >= DISCOVERY_SLOW_LINK_PING_INTERVAL_MS
1300 }
1301}
1302
1303#[cfg(all(feature = "discovery", feature = "timesync"))]
1304#[derive(Debug, Clone, Default)]
1305struct TimeSyncSideThrottleState {
1306 next_allowed_ms: u64,
1307}
1308
1309#[cfg(feature = "discovery")]
1310#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1311enum DiscoveryAdvertiseLevel {
1312 MinimalPing,
1313 Incremental,
1314 Full,
1315}
1316
1317#[derive(Debug, Clone, Default)]
1318struct AdaptiveRouteStats {
1319 estimated_bandwidth_bps: u64,
1320 peak_bandwidth_bps: u64,
1321 last_observed_ms: u64,
1322 last_slow_observed_ms: u64,
1323 sample_count: u64,
1324 window_started_ms: u64,
1325 window_bytes: u64,
1326 peak_usage_bps: u64,
1327}
1328
1329impl AdaptiveRouteStats {
1330 #[inline]
1331 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
1332 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
1333 sample_bps
1334 } else if sample_bps >= self.estimated_bandwidth_bps {
1335 self.estimated_bandwidth_bps
1336 .saturating_mul(3)
1337 .saturating_add(sample_bps.saturating_mul(5))
1338 / 8
1339 } else {
1340 self.estimated_bandwidth_bps
1341 .saturating_mul(7)
1342 .saturating_add(sample_bps)
1343 / 8
1344 };
1345 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
1346 self.last_observed_ms = now_ms;
1347 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
1348 self.last_slow_observed_ms = now_ms;
1349 }
1350 self.sample_count = self.sample_count.saturating_add(1);
1351 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
1352 self.window_started_ms = now_ms;
1353 self.window_bytes = 0;
1354 }
1355 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
1356 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
1357 }
1358
1359 #[inline]
1360 fn current_usage_bps(&self, now_ms: u64) -> u64 {
1361 if self.window_started_ms == 0 {
1362 return 0;
1363 }
1364 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
1365 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
1366 .min(u128::from(u64::MAX)) as u64
1367 }
1368
1369 #[inline]
1370 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
1371 let capacity = self
1372 .estimated_bandwidth_bps
1373 .max(self.peak_bandwidth_bps)
1374 .max(1);
1375 capacity.saturating_sub(self.current_usage_bps(now_ms))
1376 }
1377
1378 #[inline]
1379 fn weight(&self, now_ms: u64) -> u64 {
1380 self.available_headroom_bps(now_ms).max(1)
1381 }
1382
1383 #[inline]
1384 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
1385 let current_usage_bps = self.current_usage_bps(now_ms);
1386 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
1387 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
1388 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
1389 AdaptiveLinkStats {
1390 auto_balancing_enabled,
1391 estimated_capacity_bps,
1392 peak_capacity_bps,
1393 current_usage_bps,
1394 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
1395 available_headroom_bps,
1396 effective_weight: available_headroom_bps.max(1),
1397 last_observed_ms: self.last_observed_ms,
1398 sample_count: self.sample_count,
1399 }
1400 }
1401}
1402
1403use crate::type_stats::TypeRuntimeStatsInner;
1404
1405#[derive(Debug, Clone, Default)]
1406struct SideRuntimeStatsInner {
1407 tx_packets: u64,
1408 tx_bytes: u64,
1409 rx_packets: u64,
1410 rx_bytes: u64,
1411 relayed_tx_packets: u64,
1412 relayed_tx_bytes: u64,
1413 relayed_rx_packets: u64,
1414 relayed_rx_bytes: u64,
1415 local_delivery_packets: u64,
1416 tx_retries: u64,
1417 tx_handler_failures: u64,
1418 local_handler_failures: u64,
1419 total_handler_retries: u64,
1420 side_transport_full_frames: u64,
1421 side_transport_compact_frames: u64,
1422 side_transport_compact_delta_frames: u64,
1423 side_transport_compact_omitted_timestamp_frames: u64,
1424 side_transport_chunk_frames: u64,
1425 side_transport_raw_bytes: u64,
1426 side_transport_wire_bytes: u64,
1427 side_transport_bytes_saved: u64,
1428 side_transport_min_compact_overhead_bytes: Option<usize>,
1429 side_transport_max_compact_overhead_bytes: Option<usize>,
1430 side_transport_compact_target_misses: u64,
1431 side_transport_template_evictions: u64,
1432 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
1433}
1434
1435impl SideRuntimeStatsInner {
1436 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
1437 self.data_types.entry(ty.as_u32()).or_default()
1438 }
1439
1440 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
1441 self.tx_packets = self.tx_packets.saturating_add(1);
1442 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
1443 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1444 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1445 if relayed {
1446 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
1447 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
1448 }
1449 self.type_stats_mut(ty)
1450 .tx(bytes as u64, relayed, retries as u64);
1451 }
1452
1453 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
1454 self.rx_packets = self.rx_packets.saturating_add(1);
1455 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
1456 if relayed {
1457 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
1458 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
1459 }
1460 self.type_stats_mut(ty).rx(bytes as u64, relayed);
1461 }
1462
1463 fn note_local_delivery(&mut self, ty: DataType) {
1464 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
1465 self.type_stats_mut(ty).rx(0, false);
1466 }
1467
1468 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
1469 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
1470 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1471 self.type_stats_mut(ty).failure(0);
1472 }
1473
1474 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
1475 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
1476 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1477 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1478 self.type_stats_mut(ty).failure(retries as u64);
1479 }
1480
1481 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
1482 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
1483 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1484 }
1485
1486 fn note_side_transport_compact(
1487 &mut self,
1488 raw_bytes: usize,
1489 wire_bytes: usize,
1490 compact_overhead_bytes: usize,
1491 used_timestamp_delta: bool,
1492 omitted_timestamp: bool,
1493 ) {
1494 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
1495 if used_timestamp_delta {
1496 self.side_transport_compact_delta_frames =
1497 self.side_transport_compact_delta_frames.saturating_add(1);
1498 }
1499 if omitted_timestamp {
1500 self.side_transport_compact_omitted_timestamp_frames = self
1501 .side_transport_compact_omitted_timestamp_frames
1502 .saturating_add(1);
1503 }
1504 self.note_side_transport_bytes(raw_bytes, wire_bytes);
1505 self.side_transport_min_compact_overhead_bytes = Some(
1506 self.side_transport_min_compact_overhead_bytes
1507 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
1508 );
1509 self.side_transport_max_compact_overhead_bytes = Some(
1510 self.side_transport_max_compact_overhead_bytes
1511 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
1512 );
1513 }
1514
1515 fn note_side_transport_chunks(&mut self, chunks: usize) {
1516 self.side_transport_chunk_frames = self
1517 .side_transport_chunk_frames
1518 .saturating_add(chunks as u64);
1519 }
1520
1521 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
1522 self.side_transport_raw_bytes = self
1523 .side_transport_raw_bytes
1524 .saturating_add(raw_bytes as u64);
1525 self.side_transport_wire_bytes = self
1526 .side_transport_wire_bytes
1527 .saturating_add(wire_bytes as u64);
1528 if raw_bytes > wire_bytes {
1529 self.side_transport_bytes_saved = self
1530 .side_transport_bytes_saved
1531 .saturating_add((raw_bytes - wire_bytes) as u64);
1532 }
1533 }
1534
1535 fn note_side_transport_compact_target_miss(&mut self) {
1536 self.side_transport_compact_target_misses =
1537 self.side_transport_compact_target_misses.saturating_add(1);
1538 }
1539
1540 fn note_side_transport_template_eviction(&mut self) {
1541 self.side_transport_template_evictions =
1542 self.side_transport_template_evictions.saturating_add(1);
1543 }
1544}
1545
1546#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1547enum RouteSelectionOrigin {
1548 Flood,
1549 Discovered,
1550}
1551
1552type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1555
1556type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
1558type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
1559
1560#[derive(Clone)]
1566pub enum EndpointHandlerFn {
1567 Packet(Arc<PacketHandlerFn>),
1568 Packed(Arc<PackedHandlerFn>),
1569}
1570
1571pub struct EndpointHandler {
1573 endpoint: DataEndpoint,
1574 handler: EndpointHandlerFn,
1575}
1576
1577impl Debug for EndpointHandlerFn {
1579 #[inline]
1580 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1581 match self {
1582 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
1583 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
1584 }
1585 }
1586}
1587
1588#[derive(Clone)]
1590pub enum RouterTxHandlerFn {
1591 Packed(Arc<PackedHandlerFn>),
1592 PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
1593 Packet(Arc<PacketHandlerFn>),
1594}
1595
1596impl Debug for RouterTxHandlerFn {
1597 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1598 match self {
1599 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
1600 RouterTxHandlerFn::PackedWithPriority(_) => f
1601 .debug_tuple("PackedWithPriority")
1602 .field(&"<handler>")
1603 .finish(),
1604 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
1605 }
1606 }
1607}
1608
1609pub use crate::routing_config::SideOptions as RouterSideOptions;
1610
1611#[derive(Clone, Debug)]
1613pub struct RouterSide {
1614 pub name: Arc<str>,
1615 pub tx_handler: RouterTxHandlerFn,
1616 pub opts: RouterSideOptions,
1617}
1618
1619impl Debug for EndpointHandler {
1621 #[inline]
1622 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1623 f.debug_struct("EndpointHandler")
1624 .field("endpoint", &self.endpoint)
1625 .field("handler", &self.handler)
1626 .finish()
1627 }
1628}
1629
1630#[inline]
1631pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1632 #[cfg(feature = "timesync")]
1633 if matches!(endpoint, DataEndpoint::TimeSync) {
1634 return true;
1635 }
1636 #[cfg(feature = "discovery")]
1637 {
1638 discovery::is_discovery_endpoint(endpoint)
1639 }
1640 #[cfg(not(feature = "discovery"))]
1641 {
1642 let _ = endpoint;
1643 false
1644 }
1645}
1646
1647impl EndpointHandler {
1648 #[inline]
1652 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1653 where
1654 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1655 {
1656 assert!(
1657 !endpoint_is_router_internal(endpoint),
1658 "reserved internal endpoint handlers must not be user-registered"
1659 );
1660 #[cfg(feature = "std")]
1661 crate::config::ensure_endpoint_id(endpoint, false)
1662 .expect("endpoint handler endpoint registration failed");
1663 Self {
1664 endpoint,
1665 handler: EndpointHandlerFn::Packet(Arc::new(f)),
1666 }
1667 }
1668
1669 #[inline]
1671 #[cfg(feature = "std")]
1672 pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1673 where
1674 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1675 {
1676 Self::new_packet_handler(endpoint.id, f)
1677 }
1678
1679 #[cfg(not(feature = "std"))]
1681 #[inline]
1682 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1683 where
1684 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1685 {
1686 Self::new_packet_handler(endpoint.id, f)
1687 }
1688
1689 #[cfg(feature = "std")]
1691 #[inline]
1692 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1693 where
1694 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1695 {
1696 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1697 .ok_or(TelemetryError::BadArg)?;
1698 Ok(Self::new_packet_handler(endpoint.id, f))
1699 }
1700
1701 #[inline]
1705 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1706 where
1707 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1708 {
1709 assert!(
1710 !endpoint_is_router_internal(endpoint),
1711 "reserved internal endpoint handlers must not be user-registered"
1712 );
1713 #[cfg(feature = "std")]
1714 crate::config::ensure_endpoint_id(endpoint, false)
1715 .expect("endpoint handler endpoint registration failed");
1716 Self {
1717 endpoint,
1718 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1719 }
1720 }
1721
1722 #[inline]
1724 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1725 where
1726 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1727 {
1728 Self::new_packed_handler(endpoint.id, f)
1729 }
1730
1731 #[cfg(feature = "std")]
1733 #[inline]
1734 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1735 where
1736 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1737 {
1738 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1739 .ok_or(TelemetryError::BadArg)?;
1740 Ok(Self::new_packed_handler(endpoint.id, f))
1741 }
1742
1743 #[inline]
1745 pub fn get_endpoint(&self) -> DataEndpoint {
1746 self.endpoint
1747 }
1748
1749 #[inline]
1751 pub fn get_handler(&self) -> &EndpointHandlerFn {
1752 &self.handler
1753 }
1754}
1755
1756pub trait Clock {
1757 fn now_ms(&self) -> u64;
1759
1760 fn now_ns(&self) -> u64 {
1764 self.now_ms().saturating_mul(1_000_000)
1765 }
1766}
1767
1768impl<T: Fn() -> u64> Clock for T {
1769 #[inline]
1770 fn now_ms(&self) -> u64 {
1771 self()
1772 }
1773}
1774
1775#[cfg(feature = "std")]
1776#[derive(Debug)]
1777struct StdMonotonicClock {
1778 start: Instant,
1779}
1780
1781#[cfg(feature = "std")]
1782impl Default for StdMonotonicClock {
1783 fn default() -> Self {
1784 Self {
1785 start: Instant::now(),
1786 }
1787 }
1788}
1789
1790#[cfg(feature = "std")]
1791impl Clock for StdMonotonicClock {
1792 fn now_ms(&self) -> u64 {
1793 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1794 }
1795
1796 fn now_ns(&self) -> u64 {
1797 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1798 }
1799}
1800
1801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1803pub enum RouterE2eEncryptionMode {
1804 Disabled,
1806 RequiredOnly,
1808 Preferred,
1810 ForceAll,
1812}
1813
1814pub type NodeAddress = u32;
1815pub type P2pPort = u16;
1816pub type P2pStreamId = u32;
1817
1818const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1819const P2P_STREAM_VERSION: u8 = 1;
1820const P2P_STREAM_SYN: u8 = 0x01;
1821const P2P_STREAM_ACK: u8 = 0x02;
1822const P2P_STREAM_FIN: u8 = 0x04;
1823const P2P_STREAM_RST: u8 = 0x08;
1824const P2P_STREAM_DATA: u8 = 0x10;
1825
1826#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1827pub enum AddressAssignmentMode {
1828 Dynamic,
1829 Requested(NodeAddress),
1830 Static(NodeAddress),
1831}
1832
1833impl AddressAssignmentMode {
1834 #[inline]
1835 #[cfg(feature = "discovery")]
1836 fn mode_code(self) -> u8 {
1837 match self {
1838 Self::Dynamic => 0,
1839 Self::Requested(_) => 1,
1840 Self::Static(_) => 2,
1841 }
1842 }
1843
1844 #[inline]
1845 fn requested_address(self) -> NodeAddress {
1846 match self {
1847 Self::Dynamic => 0,
1848 Self::Requested(addr) | Self::Static(addr) => addr,
1849 }
1850 }
1851}
1852
1853#[derive(Debug, Clone, PartialEq, Eq)]
1854pub enum AddressChangeReason {
1855 Configured,
1856 DynamicConflict,
1857 RequestedConflict,
1858 StaticConflict,
1859 HostnameConflict,
1860}
1861
1862#[derive(Debug, Clone, PartialEq, Eq)]
1863pub struct AddressChange {
1864 pub old_hostname: Arc<str>,
1865 pub new_hostname: Arc<str>,
1866 pub old_address: NodeAddress,
1867 pub new_address: NodeAddress,
1868 pub reason: AddressChangeReason,
1869}
1870
1871#[derive(Debug, Clone, PartialEq, Eq)]
1872pub struct AddressBookEntry {
1873 pub hostname: Arc<str>,
1874 pub address: NodeAddress,
1875 pub requested_address: NodeAddress,
1876 pub mode: AddressAssignmentMode,
1877 pub birth_ms: u64,
1878 pub owner_hash: u64,
1879 pub last_seen_ms: u64,
1880}
1881
1882#[cfg(feature = "discovery")]
1883struct PreparedAddressRecords {
1884 peer: Option<Box<AddressBookEntry>>,
1885 local: Option<Box<AddressBookEntry>>,
1886}
1887
1888pub struct P2pMessage<'a> {
1889 pub source_hostname: &'a str,
1890 pub source_address: NodeAddress,
1891 pub source_port: P2pPort,
1892 pub destination_port: P2pPort,
1893 pub payload: &'a [u8],
1894}
1895
1896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1897pub enum P2pStreamEventKind {
1898 Accepted,
1899 Connected,
1900 Data,
1901 Closed,
1902 Reset,
1903}
1904
1905pub struct P2pStreamEvent<'a> {
1906 pub kind: P2pStreamEventKind,
1907 pub stream_id: P2pStreamId,
1908 pub peer_stream_id: P2pStreamId,
1909 pub sequence: u32,
1910 pub peer_hostname: &'a str,
1911 pub peer_address: NodeAddress,
1912 pub local_port: P2pPort,
1913 pub peer_port: P2pPort,
1914 pub payload: &'a [u8],
1915}
1916
1917struct P2pDecoded<'a> {
1918 source_hostname: &'a str,
1919 source_address: NodeAddress,
1920 source_port: P2pPort,
1921 destination_port: P2pPort,
1922 payload: &'a [u8],
1923}
1924
1925#[derive(Debug, Clone)]
1926struct P2pStreamDecoded<'a> {
1927 flags: u8,
1928 source_stream_id: P2pStreamId,
1929 destination_stream_id: P2pStreamId,
1930 sequence: u32,
1931 payload: &'a [u8],
1932}
1933
1934type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1935
1936#[derive(Clone)]
1937struct AddressChangeHandler {
1938 handler: Arc<AddressChangeFn>,
1939}
1940
1941impl Debug for AddressChangeHandler {
1942 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1943 f.write_str("AddressChangeHandler(<handler>)")
1944 }
1945}
1946
1947type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1948
1949#[derive(Clone)]
1950struct P2pPortHandler {
1951 handler: Arc<P2pPortHandlerFn>,
1952}
1953
1954impl Debug for P2pPortHandler {
1955 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1956 f.write_str("P2pPortHandler(<handler>)")
1957 }
1958}
1959
1960type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1961
1962#[derive(Clone)]
1963struct P2pStreamHandler {
1964 handler: Arc<P2pStreamHandlerFn>,
1965}
1966
1967impl Debug for P2pStreamHandler {
1968 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1969 f.write_str("P2pStreamHandler(<handler>)")
1970 }
1971}
1972
1973#[derive(Debug, Clone)]
1974struct P2pStreamSession {
1975 peer_hostname: Arc<str>,
1976 peer_address: NodeAddress,
1977 local_port: P2pPort,
1978 peer_port: P2pPort,
1979 peer_stream_id: P2pStreamId,
1980 next_sequence: u32,
1981 connected: bool,
1982}
1983
1984#[derive(Debug, Clone)]
1985struct PendingP2pStreamEvent {
1986 handlers: Vec<P2pStreamHandler>,
1987 kind: P2pStreamEventKind,
1988 stream_id: P2pStreamId,
1989 peer_stream_id: P2pStreamId,
1990 peer_hostname: Arc<str>,
1991 peer_address: NodeAddress,
1992 local_port: P2pPort,
1993 peer_port: P2pPort,
1994}
1995
1996#[derive(Debug, Clone)]
1997pub struct RouterConfig {
1998 role: crate::routing_config::RoutingRole,
1999 handlers: Arc<[EndpointHandler]>,
2001 reliable_enabled: bool,
2003 sender: Option<Arc<str>>,
2005 address_mode: AddressAssignmentMode,
2007 address_change_handlers: Arc<[AddressChangeHandler]>,
2009 e2e_encryption: RouterE2eEncryptionMode,
2011 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
2013 e2e_key_id: u32,
2014 memory: RuntimeMemoryConfig,
2015 #[cfg(feature = "discovery")]
2016 preferred_discovery_master: Option<Arc<str>>,
2017 #[cfg(feature = "timesync")]
2018 timesync: Option<TimeSyncConfig>,
2019}
2020
2021impl RouterConfig {
2022 pub(crate) fn relay() -> Self {
2023 let mut cfg = Self::default().with_sender("RELAY");
2024 cfg.role = crate::routing_config::RoutingRole::Relay;
2025 cfg
2026 }
2027
2028 pub(crate) fn configured_sender(&self) -> Arc<str> {
2029 self.sender
2030 .clone()
2031 .unwrap_or_else(|| Arc::from(runtime_device_identifier()))
2032 }
2033
2034 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
2039 #[cfg(feature = "cryptography")]
2040 {
2041 RouterE2eEncryptionMode::Preferred
2042 }
2043 #[cfg(not(feature = "cryptography"))]
2044 {
2045 RouterE2eEncryptionMode::Disabled
2046 }
2047 }
2048
2049 pub fn new<H>(handlers: H) -> Self
2051 where
2052 H: Into<Arc<[EndpointHandler]>>,
2053 {
2054 let handlers: Arc<[EndpointHandler]> = handlers.into();
2055 assert!(
2056 handlers
2057 .iter()
2058 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
2059 "reserved internal endpoint handlers must not be user-registered"
2060 );
2061 Self {
2062 role: crate::routing_config::RoutingRole::Router,
2063 handlers,
2064 reliable_enabled: true,
2065 sender: None,
2066 address_mode: AddressAssignmentMode::Dynamic,
2067 address_change_handlers: Arc::from([]),
2068 e2e_encryption: Self::default_e2e_encryption_mode(),
2069 e2e_key_id: 0,
2070 memory: RuntimeMemoryConfig::default(),
2071 #[cfg(feature = "discovery")]
2072 preferred_discovery_master: None,
2073 #[cfg(feature = "timesync")]
2074 timesync: None,
2075 }
2076 }
2077
2078 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
2080 self.reliable_enabled = enabled;
2081 self
2082 }
2083
2084 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
2086 self.sender = Some(Arc::from(sender.as_ref()));
2087 self
2088 }
2089
2090 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
2092 self.with_sender(hostname)
2093 }
2094
2095 pub fn with_dynamic_address(mut self) -> Self {
2097 self.address_mode = AddressAssignmentMode::Dynamic;
2098 self
2099 }
2100
2101 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
2103 self.address_mode = AddressAssignmentMode::Requested(address);
2104 self
2105 }
2106
2107 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
2109 self.address_mode = AddressAssignmentMode::Static(address);
2110 self
2111 }
2112
2113 pub fn on_address_change<F>(mut self, f: F) -> Self
2115 where
2116 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
2117 {
2118 let mut handlers = self.address_change_handlers.to_vec();
2119 handlers.push(AddressChangeHandler {
2120 handler: Arc::new(f),
2121 });
2122 self.address_change_handlers = Arc::from(handlers);
2123 self
2124 }
2125
2126 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
2128 self.e2e_encryption = mode;
2129 self
2130 }
2131
2132 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
2134 self.e2e_key_id = key_id;
2135 self
2136 }
2137
2138 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
2140 memory.validate()?;
2141 self.memory = memory;
2142 Ok(self)
2143 }
2144
2145 #[cfg(feature = "discovery")]
2151 pub fn with_preferred_discovery_master<S: AsRef<str>>(mut self, hostname: S) -> Self {
2152 let hostname = hostname.as_ref();
2153 self.preferred_discovery_master = if hostname.is_empty() {
2154 None
2155 } else {
2156 Some(Arc::from(hostname))
2157 };
2158 self
2159 }
2160
2161 #[cfg(feature = "timesync")]
2162 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
2164 self.timesync = Some(cfg);
2165 self
2166 }
2167
2168 #[inline]
2169 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
2171 if endpoint_is_router_internal(ep) {
2172 return false;
2173 }
2174 self.handlers.iter().any(|h| h.endpoint == ep)
2175 }
2176
2177 #[inline]
2178 fn reliable_enabled(&self) -> bool {
2179 self.reliable_enabled
2180 }
2181
2182 #[inline]
2183 fn address_mode(&self) -> AddressAssignmentMode {
2184 self.address_mode
2185 }
2186
2187 #[inline]
2188 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
2189 self.e2e_encryption
2190 }
2191
2192 #[cfg(feature = "cryptography")]
2193 #[inline]
2194 fn e2e_key_id(&self) -> u32 {
2195 self.e2e_key_id
2196 }
2197
2198 #[cfg(feature = "timesync")]
2199 #[inline]
2200 fn timesync_config(&self) -> Option<TimeSyncConfig> {
2201 self.timesync
2202 }
2203
2204 #[inline]
2205 pub(crate) fn memory_config(&self) -> RuntimeMemoryConfig {
2206 self.memory
2207 }
2208}
2209
2210impl Default for RouterConfig {
2211 fn default() -> Self {
2212 Self {
2213 role: crate::routing_config::RoutingRole::Router,
2214 handlers: Arc::from([]),
2215 reliable_enabled: true,
2216 sender: None,
2217 address_mode: AddressAssignmentMode::Dynamic,
2218 address_change_handlers: Arc::from([]),
2219 e2e_encryption: Self::default_e2e_encryption_mode(),
2220 e2e_key_id: 0,
2221 memory: RuntimeMemoryConfig::default(),
2222 #[cfg(feature = "discovery")]
2223 preferred_discovery_master: None,
2224 #[cfg(feature = "timesync")]
2225 timesync: None,
2226 }
2227 }
2228}
2229
2230#[derive(Debug)]
2234struct ActiveRoutingConfig {
2235 role: crate::routing_config::RoutingRole,
2236 reliable_enabled: bool,
2237 local: Option<Box<RouterConfig>>,
2238}
2239impl ActiveRoutingConfig {
2240 fn new(cfg: RouterConfig) -> Self {
2241 let role = cfg.role;
2242 Self {
2243 role,
2244 reliable_enabled: cfg.reliable_enabled(),
2245 local: role.local_services().then(|| Box::new(cfg)),
2246 }
2247 }
2248 fn handlers(&self) -> &[EndpointHandler] {
2249 self.local.as_ref().map_or(&[], |cfg| cfg.handlers.as_ref())
2250 }
2251 fn address_change_handlers(&self) -> &[AddressChangeHandler] {
2252 self.local
2253 .as_ref()
2254 .map_or(&[], |cfg| cfg.address_change_handlers.as_ref())
2255 }
2256 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
2257 self.local
2258 .as_ref()
2259 .is_some_and(|cfg| cfg.is_local_endpoint(ep))
2260 }
2261 fn reliable_enabled(&self) -> bool {
2262 self.reliable_enabled
2263 }
2264 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
2265 self.local
2266 .as_ref()
2267 .map_or(RouterE2eEncryptionMode::Disabled, |cfg| {
2268 cfg.e2e_encryption()
2269 })
2270 }
2271 #[cfg(feature = "cryptography")]
2272 fn e2e_key_id(&self) -> u32 {
2273 self.local.as_ref().map_or(0, |cfg| cfg.e2e_key_id())
2274 }
2275 #[cfg(feature = "timesync")]
2276 fn timesync_config(&self) -> Option<TimeSyncConfig> {
2277 self.local.as_ref().and_then(|cfg| cfg.timesync_config())
2278 }
2279}
2280
2281pub trait LeBytes: Copy + Sized {
2284 const WIDTH: usize;
2285 fn write_le(self, out: &mut [u8]);
2286 fn from_le_slice(bytes: &[u8]) -> Self;
2287}
2288
2289impl_letype_num!(u8, 1);
2290impl_letype_num!(u16, 2);
2291impl_letype_num!(u32, 4);
2292impl_letype_num!(u64, 8);
2293impl_letype_num!(u128, 16);
2294impl_letype_num!(i8, 1);
2295impl_letype_num!(i16, 2);
2296impl_letype_num!(i32, 4);
2297impl_letype_num!(i64, 8);
2298impl_letype_num!(i128, 16);
2299impl_letype_num!(f32, 4);
2300impl_letype_num!(f64, 8);
2301
2302pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
2304 let total = data.len() * T::WIDTH;
2305 let mut buf = vec![0u8; total];
2306
2307 for (i, v) in data.iter().copied().enumerate() {
2308 let start = i * T::WIDTH;
2309 v.write_le(&mut buf[start..start + T::WIDTH]);
2310 }
2311
2312 Arc::from(buf)
2313}
2314
2315fn make_error_payload(msg: &str) -> crate::SharedBytes {
2318 let meta = message_meta(DataType::TelemetryError);
2319 match meta.element {
2320 MessageElement::Static(_, _, _) => {
2321 let max = get_needed_message_size(DataType::TelemetryError);
2322 let bytes = msg.as_bytes();
2323 let n = core::cmp::min(max, bytes.len());
2324 let mut buf = vec![0u8; max];
2325 if n > 0 {
2326 buf[..n].copy_from_slice(&bytes[..n]);
2327 }
2328 crate::shared_bytes::convert(Arc::<[u8]>::from(buf))
2329 }
2330 MessageElement::Dynamic(_, _) => {
2331 crate::shared_bytes::convert(Arc::<[u8]>::from(msg.as_bytes()))
2332 }
2333 }
2334}
2335
2336fn log_raw<T, F>(
2340 sender: Arc<str>,
2341 ty: DataType,
2342 data: &[T],
2343 timestamp: u64,
2344 mut tx_function: F,
2345) -> TelemetryResult<()>
2346where
2347 T: LeBytes,
2348 F: FnMut(Packet) -> TelemetryResult<()>,
2349{
2350 let meta = message_meta(ty);
2351 let got = data
2352 .len()
2353 .checked_mul(T::WIDTH)
2354 .ok_or(TelemetryError::PacketTooLarge(
2355 "typed payload size overflow",
2356 ))?;
2357 if T::WIDTH == 0 {
2358 return Err(TelemetryError::BadArg);
2359 }
2360
2361 match meta.element {
2362 MessageElement::Static(_, _, _) => {
2363 let expected = crate::data_type_size(meta.element.data_type()) * meta.element;
2364 if got != expected {
2365 return Err(TelemetryError::SizeMismatch { expected, got });
2366 }
2367 }
2368 MessageElement::Dynamic(_, _) => {
2369 }
2372 }
2373
2374 const INLINE_ENCODE_BYTES: usize = if crate::config::STACK_PAYLOAD_SIZE < u8::MAX as usize {
2375 crate::config::STACK_PAYLOAD_SIZE
2376 } else {
2377 u8::MAX as usize
2378 };
2379 let payload = if got <= INLINE_ENCODE_BYTES {
2380 let mut bytes = [0u8; INLINE_ENCODE_BYTES];
2383 for (value, destination) in data
2384 .iter()
2385 .copied()
2386 .zip(bytes[..got].chunks_exact_mut(T::WIDTH))
2387 {
2388 value.write_le(destination);
2389 }
2390 crate::config::StandardSmallPayload::new(&bytes[..got])
2391 } else {
2392 crate::config::StandardSmallPayload::from_arc(encode_slice_le(data))
2393 };
2394 let pkt = Packet::new_local(ty, meta.endpoints_owned(), sender, timestamp, payload)?;
2395 tx_function(pkt)
2396}
2397
2398#[cfg(feature = "std")]
2400fn reliable_trace_enabled() -> bool {
2401 static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
2402 *ENABLED.get_or_init(|| {
2403 matches!(
2404 std::env::var("SEDSNET_TRACE_RELIABLE").as_deref(),
2405 Ok("1" | "true")
2406 )
2407 })
2408}
2409
2410fn fallback_stdout(_msg: &str) {
2414 #[cfg(feature = "std")]
2415 {
2416 eprintln!("{}", _msg);
2417 }
2418 #[cfg(all(not(feature = "std"), target_os = "none"))]
2419 {
2420 unsafe {
2423 seds_error_msg(_msg.as_ptr(), _msg.len());
2424 }
2425 }
2426}
2427
2428#[derive(Debug, Clone)]
2433struct LocalServicesState {
2436 managed_variable_types: BTreeSet<u32>,
2437 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
2438 managed_variable_latest: BTreeMap<u32, Box<ManagedVariableCacheEntry>>,
2439 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
2440 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
2441 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
2442 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
2443 next_p2p_stream_id: P2pStreamId,
2444}
2445
2446#[derive(Default)]
2447struct ForwardedReceiptCache {
2448 destinations: BTreeMap<u64, BTreeSet<u64>>,
2449 order: VecDeque<u64>,
2450}
2451
2452struct RouterInner {
2453 #[cfg(feature = "compression")]
2454 codec_reserved_bytes: usize,
2455 memory: RuntimeMemoryConfig,
2456 sides: Vec<Option<RouterSide>>,
2457 route_overrides: RouterMap<(Option<RouterSideId>, RouterSideId), bool>,
2458 typed_route_overrides: RouterMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
2459 route_weights: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2460 route_priorities: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2461 source_route_modes: RouterMap<Option<RouterSideId>, RouteSelectionMode>,
2462 route_selection_cursors: RouterMap<Option<RouterSideId>, u64>,
2463 adaptive_route_stats: RouterMap<RouterSideId, AdaptiveRouteStats>,
2464 side_runtime_stats: RouterMap<RouterSideId, Box<SideRuntimeStatsInner>>,
2467 side_transport: RouterMap<RouterSideId, Box<SideTransportState>>,
2468 local_services: Option<Box<LocalServicesState>>,
2469 local_address: AddressBookEntry,
2470 address_book: BTreeMap<String, Box<AddressBookEntry>>,
2471 address_by_value: BTreeMap<NodeAddress, String>,
2472 received_queue: BoundedDeque<RouterRxItem>,
2473 transmit_queue: BoundedDeque<TxQueued>,
2474 #[cfg(feature = "std")]
2475 active_queued_types: [(u32, u32); ACTIVE_QUEUED_TYPE_SLOTS],
2476 tx_priority_burst: u8,
2477 tx_policy_flags: u8,
2478 #[cfg(feature = "latest-value-egress")]
2479 latest_turn_phase: u8,
2480 #[cfg(feature = "latest-value-egress")]
2481 latest_turn_credit: i32,
2482 #[cfg(feature = "latest-value-egress")]
2483 latest_service_seq: u32,
2484 ordered_claim_seq: u32,
2485 rx_priority_burst: u8,
2486 recent_rx: BoundedDeque<u64>,
2487 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
2488 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
2489 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
2490 local_admission_sequence: u64,
2491 reliable_return_route_order: VecDeque<u64>,
2492 forwarded_receipts: Option<Box<ForwardedReceiptCache>>,
2493 end_to_end_reliable_tx: BTreeMap<u64, Box<EndToEndReliableSent>>,
2494 end_to_end_reliable_tx_order: VecDeque<u64>,
2495 total_handler_failures: u64,
2496 total_handler_retries: u64,
2497 #[cfg(feature = "discovery")]
2498 discovery_routes: RouterMap<RouterSideId, DiscoverySideState>,
2499 #[cfg(feature = "discovery")]
2500 discovery_cadence: DiscoveryCadenceState,
2501 #[cfg(feature = "discovery")]
2502 discovery_side_throttle: RouterMap<RouterSideId, DiscoverySideThrottleState>,
2503 #[cfg(feature = "discovery")]
2504 schema_response_sequence: u64,
2505 #[cfg(feature = "discovery")]
2506 preferred_discovery_master: Option<Arc<str>>,
2507 #[cfg(all(feature = "discovery", feature = "timesync"))]
2508 timesync_side_throttle: RouterMap<RouterSideId, TimeSyncSideThrottleState>,
2509}
2510#[derive(Debug, Clone)]
2511struct ManagedVariableCacheEntry {
2512 packet: Packet,
2513 sender_address: u32,
2514 #[cfg(feature = "discovery")]
2515 cached_at_ms: u64,
2516 #[cfg(feature = "discovery")]
2520 upstream: Option<core::num::NonZeroUsize>,
2521}
2522
2523type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
2524
2525#[derive(Clone)]
2526struct NetworkVariableUpdateHandler {
2527 handler: Arc<NetworkVariableUpdateFn>,
2528}
2529
2530impl Debug for NetworkVariableUpdateHandler {
2531 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2532 f.write_str("NetworkVariableUpdateHandler(<handler>)")
2533 }
2534}
2535
2536#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2538pub struct NetworkVariablePermissions {
2539 pub read: bool,
2540 pub write: bool,
2541}
2542
2543impl NetworkVariablePermissions {
2544 pub const NONE: Self = Self {
2545 read: false,
2546 write: false,
2547 };
2548 pub const READ_ONLY: Self = Self {
2549 read: true,
2550 write: false,
2551 };
2552 pub const WRITE_ONLY: Self = Self {
2553 read: false,
2554 write: true,
2555 };
2556 pub const READ_WRITE: Self = Self {
2557 read: true,
2558 write: true,
2559 };
2560}
2561
2562#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2563enum RouterQueueKind {
2564 Received,
2565 Transmit,
2566}
2567
2568#[cfg(feature = "compression")]
2569impl crate::payload_codec::QueueBudget for RouterInner {
2570 fn codec_reserved(&mut self) -> &mut usize {
2571 &mut self.codec_reserved_bytes
2572 }
2573 fn codec_used(&self) -> usize {
2574 self.shared_queue_bytes_used()
2575 }
2576 fn codec_limit(&self) -> usize {
2577 self.memory.max_queue_budget
2578 }
2579}
2580
2581impl RouterInner {
2582 fn is_managed_variable(&self, ty: u32) -> bool {
2583 self.local_services
2584 .as_ref()
2585 .is_some_and(|local| local.managed_variable_types.contains(&ty))
2586 }
2587
2588 fn local_cache_bytes(&self) -> usize {
2589 self.local_services
2590 .iter()
2591 .flat_map(|local| local.managed_variable_latest.values())
2592 .map(|entry| entry.packet.byte_cost())
2593 .sum()
2594 }
2595
2596 fn replay_counters(&self) -> (usize, usize) {
2597 self.transmit_queue
2598 .iter()
2599 .filter(|q| q.item.replay_marker_key().is_some())
2600 .fold((0usize, 0usize), |(n, bytes), q| {
2601 (n + 1, bytes.saturating_add(q.byte_cost()))
2602 })
2603 }
2604
2605 fn local_cache_len(&self) -> usize {
2606 self.local_services
2607 .as_ref()
2608 .map_or(0, |local| local.managed_variable_latest.len())
2609 }
2610
2611 #[cfg(all(feature = "compression", feature = "discovery"))]
2612 fn reclaim_codec_metadata(&mut self) -> bool {
2613 self.side_transport.values_mut().any(|transport| {
2614 transport
2615 .payload_codec
2616 .evict_for_budget(DISCOVERY_ROUTE_TTL_MS)
2617 })
2618 }
2619
2620 #[cfg(feature = "compression")]
2625 fn prepare_codec_queue_owner(&mut self, side: Option<usize>, data: &mut RouterItem) {
2626 let RouterItem::Packed(bytes) = data else {
2627 return;
2628 };
2629 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
2630 return;
2631 }
2632 let capable = {
2633 #[cfg(feature = "discovery")]
2634 {
2635 side.and_then(|side| self.side_transport.get(&side))
2636 .is_some_and(|transport| {
2637 transport.payload_codec.retention_can_stay_compressed()
2638 })
2639 }
2640 #[cfg(not(feature = "discovery"))]
2641 {
2642 let _ = side;
2643 false
2644 }
2645 };
2646 if capable {
2647 return;
2648 }
2649 let Ok(required) = wire_format::decoded_frame_upper_bound(bytes.as_ref()) else {
2650 return;
2651 };
2652 if self
2653 .shared_queue_bytes_used()
2654 .checked_add(required)
2655 .and_then(|used| {
2656 used.checked_add(
2657 size_of::<crate::SharedBytes>()
2658 + size_of::<bool>()
2659 + size_of::<u8>()
2660 + size_of::<u64>(),
2661 )
2662 })
2663 .is_none_or(|used| used > self.memory.max_queue_budget)
2664 {
2665 return;
2666 }
2667 if let Ok(Some(raw)) = wire_format::payload_encoding_for_retention(
2668 bytes.as_ref(),
2669 self.memory.max_queue_budget,
2670 ) {
2671 *bytes = crate::shared_bytes::convert(raw);
2672 }
2673 }
2674
2675 #[cfg(feature = "discovery")]
2676 fn schema_response_owner(item: &RouterTxItem) -> Option<(RouterSideId, u64)> {
2677 match item {
2678 RouterTxItem::ToSide { dst, data, .. }
2679 if Router::item_data_type(data).ok() == Some(DataType::DiscoverySchema) =>
2680 {
2681 Some((*dst, Router::get_hash(data)))
2682 }
2683 _ => None,
2684 }
2685 }
2686
2687 #[cfg(feature = "discovery")]
2688 fn schema_response_owner_lost(
2689 responses: &mut RouterMap<RouterSideId, DiscoverySideThrottleState>,
2690 owner: Option<(RouterSideId, u64)>,
2691 ) {
2692 if let Some((side, id)) = owner
2693 && let Some(response) = responses.get_mut(&side)
2694 && response.schema_response_active()
2695 && response.schema_response_phase == SchemaResponsePhase::Owned
2696 && response.schema_response_packet_id == id
2697 {
2698 response.schema_response_packet_id = 0;
2702 response.set_schema_response_started(false);
2703 response.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
2704 }
2705 }
2706
2707 fn retain_transmit<F>(&mut self, keep: F)
2708 where
2709 F: FnMut(&TxQueued) -> bool,
2710 {
2711 let hop = &mut self.reliable_tx;
2712 let end = &mut self.end_to_end_reliable_tx;
2713 #[cfg(feature = "discovery")]
2714 let responses = &mut self.discovery_side_throttle;
2715 self.transmit_queue.retain_observed(keep, |evicted| {
2716 #[cfg(feature = "discovery")]
2717 Self::schema_response_owner_lost(responses, Self::schema_response_owner(&evicted.item));
2718 if let Some(key) = evicted.item.replay_marker_key() {
2719 Self::set_replay_queued(hop, end, key, false);
2720 }
2721 });
2722 }
2723
2724 #[inline(never)]
2725 fn claim_index(&self, token: u32) -> Option<usize> {
2726 self.transmit_queue.iter().position(|item| {
2727 matches!(
2728 item.delivery_state,
2729 QueuedDelivery::ClaimedOrdered | QueuedDelivery::ClaimedUnordered
2730 ) && if item.disposable & TX_E2E_CHILD != 0 {
2731 matches!(&item.item, RouterTxItem::ToSide { src: Some(value), .. } if *value == token as usize)
2732 } else { item.deferred_since_ms == u64::from(token) }
2733 })
2734 }
2735
2736 #[inline(never)]
2737 fn refresh_retry_hint(&mut self) {
2738 let pending = self.transmit_queue.iter().any(|item| {
2739 matches!(
2740 item.delivery_state,
2741 QueuedDelivery::ClaimedOrdered
2742 | QueuedDelivery::ClaimedUnordered
2743 | QueuedDelivery::DeferredOrdered
2744 )
2745 });
2746 self.tx_policy_flags = (self.tx_policy_flags & !1) | u8::from(pending);
2747 }
2748
2749 fn cancel_accepted_type(&mut self, ty: u32) {
2750 self.retain_transmit(|item| {
2751 !matches!(&item.item,
2752 RouterTxItem::Broadcast(_) if item.accepted_type == ty)
2753 });
2754 }
2755
2756 fn replay_is_queued(&self, key: ReplayMarkerKey) -> Option<bool> {
2757 match key {
2758 ReplayMarkerKey::EndToEnd(id) => self.end_to_end_reliable_tx.get(&id).map(|s| s.queued),
2759 ReplayMarkerKey::Hop(side, ty, seq) => self
2760 .reliable_tx
2761 .get(&(side, ty.as_u32()))
2762 .and_then(|tx| tx.sent.get(&seq))
2763 .map(|s| s.queued),
2764 }
2765 }
2766 fn set_replay_queued(
2767 hop: &mut BTreeMap<(RouterSideId, u32), ReliableTxState>,
2768 end: &mut BTreeMap<u64, Box<EndToEndReliableSent>>,
2769 key: ReplayMarkerKey,
2770 queued: bool,
2771 ) {
2772 let flag = match key {
2773 ReplayMarkerKey::EndToEnd(id) => end.get_mut(&id).map(|s| &mut s.queued),
2774 ReplayMarkerKey::Hop(side, ty, seq) => hop
2775 .get_mut(&(side, ty.as_u32()))
2776 .and_then(|tx| tx.sent.get_mut(&seq))
2777 .map(|s| &mut s.queued),
2778 };
2779 if let Some(flag) = flag {
2780 *flag = queued;
2781 }
2782 }
2783
2784 #[cfg(any(feature = "std", feature = "fair-admission"))]
2785 fn fair_bulk_owner(data: &RouterItem, priority: u8) -> Option<u32> {
2786 match data {
2787 RouterItem::Packet(packet) => crate::admission_fairness::packet_owner(packet, priority),
2788 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2789 crate::admission_fairness::packed_owner(view, priority)
2790 })
2791 .ok()
2792 .flatten(),
2793 }
2794 }
2795 #[cfg(any(feature = "std", feature = "fair-admission"))]
2796 fn tx_bulk_owner(item: &TxQueued) -> Option<u32> {
2797 if item.disposable & TX_E2E_CHILD != 0 {
2798 return None;
2799 }
2800 #[cfg(feature = "latest-value-egress")]
2801 if item.disposable & TX_HISTORY_OWNED != 0 {
2802 return None;
2803 }
2804 match &item.item {
2805 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2806 Self::fair_bulk_owner(data, item.priority)
2807 }
2808 _ => None,
2809 }
2810 }
2811 #[cfg(any(feature = "std", feature = "fair-admission"))]
2812 fn bulk_stream(data: &RouterItem) -> Option<u32> {
2813 match data {
2814 RouterItem::Packet(packet) => Some(packet.data_type().as_u32()),
2815 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2816 wire_format::scheduling_type(view).map(|ty| ty.as_u32())
2817 })
2818 .ok()
2819 .flatten(),
2820 }
2821 }
2822 #[cfg(any(feature = "std", feature = "fair-admission"))]
2823 fn tx_bulk_stream(item: &TxQueued) -> Option<u32> {
2824 match &item.item {
2825 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2826 Self::bulk_stream(data)
2827 }
2828 _ => None,
2829 }
2830 }
2831 #[cfg(any(feature = "std", feature = "fair-admission"))]
2832 fn bulk_loads(&self, priority: u8) -> crate::admission_fairness::Loads {
2833 let mut loads = crate::admission_fairness::Loads::new();
2834 for item in self.received_queue.iter() {
2835 if item.priority <= priority
2836 && let Some(owner) = Self::fair_bulk_owner(&item.data, item.priority)
2837 {
2838 loads.record(owner, item.byte_cost());
2839 }
2840 }
2841 for item in self.transmit_queue.iter() {
2842 if item.priority <= priority
2843 && let Some(owner) = Self::tx_bulk_owner(item)
2844 {
2845 loads.record(owner, item.byte_cost());
2846 }
2847 }
2848 loads
2849 }
2850 #[cfg(any(feature = "std", feature = "fair-admission"))]
2851 fn competing_bulk_owner(&self, incoming: u32, priority: u8) -> Option<u32> {
2852 self.bulk_loads(priority).dominant_other(incoming)
2853 }
2854 #[cfg(any(feature = "std", feature = "fair-admission"))]
2855 fn reclaim_bulk_from(
2856 &mut self,
2857 owner: u32,
2858 priority: u8,
2859 incoming_stream: Option<u32>,
2860 ) -> bool {
2861 let mut streams = crate::admission_fairness::Loads::new();
2862 for item in self.received_queue.iter() {
2863 if item.priority <= priority
2864 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2865 && let Some(ty) = Self::bulk_stream(&item.data)
2866 {
2867 streams.record(ty, item.byte_cost());
2868 }
2869 }
2870 for item in self.transmit_queue.iter() {
2871 if item.priority <= priority
2872 && Self::tx_bulk_owner(item) == Some(owner)
2873 && let Some(ty) = Self::tx_bulk_stream(item)
2874 {
2875 streams.record(ty, item.byte_cost());
2876 }
2877 }
2878 let stream = streams.queue_victim(incoming_stream);
2879 let rx = self
2880 .received_queue
2881 .iter()
2882 .enumerate()
2883 .filter(|(_, item)| {
2884 item.priority <= priority
2885 && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2886 && stream.is_none_or(|ty| Self::bulk_stream(&item.data) == Some(ty))
2887 })
2888 .min_by_key(|(index, item)| (item.priority, *index))
2889 .map(|(index, item)| (index, item.priority));
2890 let tx = self
2891 .transmit_queue
2892 .iter()
2893 .enumerate()
2894 .filter(|(_, item)| {
2895 item.priority <= priority
2896 && Self::tx_bulk_owner(item) == Some(owner)
2897 && stream.is_none_or(|ty| Self::tx_bulk_stream(item) == Some(ty))
2898 })
2899 .min_by_key(|(index, item)| (item.priority, *index))
2900 .map(|(index, item)| (index, item.priority));
2901 match (rx, tx) {
2902 (Some((index, rp)), Some((_, tp))) if rp <= tp => {
2903 self.received_queue.remove_pos(index).is_some()
2904 }
2905 (Some((index, _)), None) => self.received_queue.remove_pos(index).is_some(),
2906 (_, Some((index, _))) => self.transmit_queue.remove_pos(index).is_some(),
2907 _ => false,
2908 }
2909 }
2910
2911 #[cfg(feature = "discovery")]
2912 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
2913 board
2914 .sender_id
2915 .len()
2916 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
2917 .saturating_add(
2918 board
2919 .reachable_timesync_sources
2920 .iter()
2921 .map(|s| s.len())
2922 .sum::<usize>(),
2923 )
2924 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
2925 }
2926
2927 #[cfg(feature = "discovery")]
2928 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
2929 sender
2930 .len()
2931 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
2932 .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
2933 .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
2934 .saturating_add(
2935 state
2936 .reachable_timesync_sources
2937 .iter()
2938 .map(|s| s.len())
2939 .sum::<usize>(),
2940 )
2941 .saturating_add(
2942 state
2943 .advertised_reachable_timesync_sources
2944 .iter()
2945 .map(|s| s.len())
2946 .sum::<usize>(),
2947 )
2948 .saturating_add(
2949 state
2950 .topology_boards
2951 .iter()
2952 .map(Self::topology_board_byte_cost)
2953 .sum::<usize>(),
2954 )
2955 .saturating_add(crate::peer_record::allocation_bytes::<DiscoverySenderState>())
2956 }
2957
2958 #[cfg(feature = "discovery")]
2959 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
2960 size_of::<RouterSideId>()
2961 .saturating_add(size_of::<DiscoverySideState>())
2962 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
2963 .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
2964 .saturating_add(
2965 route.requested_network_variables.len() * size_of::<ManagedVariableSubscription>(),
2966 )
2967 .saturating_add(
2968 route
2969 .reachable_timesync_sources
2970 .iter()
2971 .map(|s| s.len())
2972 .sum::<usize>(),
2973 )
2974 .saturating_add(
2975 route
2976 .announcers
2977 .iter()
2978 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
2979 .sum::<usize>(),
2980 )
2981 .saturating_add(side.saturating_sub(side))
2982 }
2983
2984 #[cfg(feature = "discovery")]
2985 fn discovery_bytes_used(&self) -> usize {
2986 self.discovery_routes
2987 .iter()
2988 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
2989 .sum()
2990 }
2991
2992 #[inline]
2993 fn reliable_rx_buffered_bytes(&self) -> usize {
2994 self.reliable_rx
2995 .values()
2996 .flat_map(|state| state.buffered.values())
2997 .map(|bytes| size_of::<crate::SharedBytes>() + bytes.len())
2998 .sum()
2999 }
3000
3001 fn reliable_rx_buffer_len(&self) -> usize {
3002 self.reliable_rx
3003 .values()
3004 .map(|state| state.buffered.len())
3005 .sum()
3006 }
3007
3008 #[inline(never)]
3009 fn side_chunk_bytes_used(&self) -> usize {
3010 self.side_transport
3011 .values()
3012 .map(|side| side.rx_chunk_bytes)
3013 .sum()
3014 }
3015 fn discard_side_chunk(&mut self, side: RouterSideId, id: u32) -> bool {
3016 self.side_transport
3017 .get_mut(&side)
3018 .is_some_and(|state| state.take_chunk(id, true).is_some())
3019 }
3020 fn reclaim_oldest_side_chunk(&mut self, keep: Option<(RouterSideId, u32)>) -> bool {
3021 let mut victim = None;
3022 let mut time = u64::MAX;
3023 for (&side, state) in &self.side_transport {
3024 let preserved = match keep {
3025 Some((owner, id)) if owner == side => Some(id),
3026 _ => None,
3027 };
3028 if let Some((id, seen)) = SideChunkAssembly::oldest(&state.rx_chunks, preserved)
3029 && (victim.is_none() || seen < time)
3030 {
3031 time = seen;
3032 victim = Some((side, id));
3033 }
3034 }
3035 match victim {
3036 Some((side, id)) => self.discard_side_chunk(side, id),
3037 None => false,
3038 }
3039 }
3040 fn make_side_chunk_room(
3041 &mut self,
3042 side: RouterSideId,
3043 id: u32,
3044 index: u16,
3045 total: u16,
3046 payload_len: usize,
3047 ) -> TelemetryResult<()> {
3048 let budget = self
3049 .memory
3050 .max_queue_budget
3051 .saturating_sub(self.protected_metadata_bytes_used());
3052 let state = self
3053 .side_transport
3054 .get_mut(&side)
3055 .ok_or(TelemetryError::BadArg)?;
3056 let existing = state.rx_chunks.get(&id).map(|a| a.total);
3057 if existing.is_some_and(|old| old != 0 && old != total) {
3058 state.take_chunk(id, true);
3059 return Err(TelemetryError::Unpack("side chunk total mismatch"));
3060 }
3061 let before = state.rx_chunks.len();
3062 let room = SideChunkAssembly::make_room(
3063 &mut state.rx_chunks,
3064 &mut state.rx_chunk_bytes,
3065 id,
3066 index,
3067 total,
3068 payload_len,
3069 budget,
3070 );
3071 if before != state.rx_chunks.len() {
3072 state.rx_chunk_reclaims = state
3073 .rx_chunk_reclaims
3074 .saturating_add((before - state.rx_chunks.len()) as u32);
3075 }
3076 let cost = room?;
3077 if cost == 0 {
3078 return Ok(());
3079 }
3080 if existing.is_none()
3081 && state.rx_chunks.len() >= 4
3082 && let Some((oldest, _)) = SideChunkAssembly::oldest(&state.rx_chunks, None)
3083 {
3084 state.take_chunk(oldest, true);
3085 }
3086 while self.shared_queue_bytes_used().saturating_add(cost) > self.memory.max_queue_budget {
3087 #[cfg(any(feature = "std", feature = "fair-admission"))]
3088 if let Some(owner) = self.bulk_loads(199).queue_victim(None)
3089 && self.reclaim_bulk_from(owner, 199, None)
3090 {
3091 continue;
3092 }
3093 if self.reclaim_oldest_side_chunk(Some((side, id))) {
3094 continue;
3095 }
3096 return Err(TelemetryError::Io("memory pressure"));
3099 }
3100 Ok(())
3101 }
3102
3103 #[inline(never)]
3104 fn protected_metadata_bytes_used(&self) -> usize {
3105 let codec_bytes = {
3106 #[cfg(all(feature = "compression", feature = "discovery"))]
3107 {
3108 self.side_transport
3109 .values()
3110 .map(|side| side.payload_codec.heap_bytes())
3111 .sum::<usize>()
3112 }
3113 #[cfg(not(all(feature = "compression", feature = "discovery")))]
3114 {
3115 0usize
3116 }
3117 };
3118 let completion_bytes = {
3119 #[cfg(feature = "std")]
3120 {
3121 self.end_to_end_reliable_tx
3122 .values()
3123 .map(|sent| {
3124 PENDING_TYPE_METADATA_BYTES
3125 + usize::from(sent.completion_epoch != 0) * size_of::<u64>()
3126 })
3127 .sum::<usize>()
3128 }
3129 #[cfg(not(feature = "std"))]
3130 {
3131 0usize
3132 }
3133 };
3134 let return_route_growth = self
3135 .reliable_return_routes
3136 .len()
3137 .saturating_mul(RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES);
3138 codec_bytes
3139 .saturating_add(completion_bytes)
3140 .saturating_add(return_route_growth)
3141 .saturating_add(
3142 self.recent_rx
3143 .max_bytes()
3144 .saturating_add(self.reliable_rx_buffered_bytes())
3145 .saturating_add(crate::config::schema_bytes_used())
3146 .saturating_add({
3147 #[cfg(feature = "discovery")]
3148 {
3149 self.discovery_bytes_used()
3150 }
3151 #[cfg(not(feature = "discovery"))]
3152 {
3153 0
3154 }
3155 }),
3156 )
3157 }
3158 #[inline]
3159 fn shared_queue_bytes_used(&self) -> usize {
3160 self.received_queue
3161 .bytes_used()
3162 .saturating_add(self.transmit_queue.bytes_used())
3163 .saturating_add(self.side_chunk_bytes_used())
3164 .saturating_add(self.protected_metadata_bytes_used())
3165 .saturating_add({
3166 #[cfg(feature = "compression")]
3167 {
3168 self.codec_reserved_bytes
3169 }
3170 #[cfg(not(feature = "compression"))]
3171 {
3172 0
3173 }
3174 })
3175 }
3176
3177 fn classify_transmit_disposable(item: &TxQueued) -> bool {
3181 let data = match &item.item {
3182 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => data,
3183 _ => return true,
3184 };
3185 match data {
3186 RouterItem::Packet(pkt) => {
3187 is_internal_control_type(pkt.data_type())
3188 || (!is_reliable_type(pkt.data_type()) && pkt.wire_target_senders().is_empty())
3189 }
3190 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3191 wire_format::peek_routing_frame_info(view).map_or(true, |frame| {
3192 is_internal_control_type(frame.envelope.ty)
3193 || (!is_reliable_type(frame.envelope.ty)
3194 && frame.reliable.is_none()
3195 && frame.envelope.target_senders().len() == 0)
3196 })
3197 })
3198 .unwrap_or(false),
3199 }
3200 }
3201
3202 fn transmit_eviction_candidate(queue: &BoundedDeque<TxQueued>) -> Option<usize> {
3203 queue
3204 .iter()
3205 .enumerate()
3206 .filter(|(_, queued)| queued.disposable & TX_DISPOSABLE != 0)
3207 .min_by_key(|(index, queued)| (queued.priority, core::cmp::Reverse(*index)))
3208 .map(|(index, _)| index)
3209 }
3210
3211 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
3212 match preferred {
3213 RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
3218 RouterQueueKind::Transmit => {
3219 let Some(index) = Self::transmit_eviction_candidate(&self.transmit_queue) else {
3220 return false;
3221 };
3222 let Some(item) = self.transmit_queue.remove_pos(index) else {
3223 return false;
3224 };
3225 #[cfg(feature = "discovery")]
3226 Self::schema_response_owner_lost(
3227 &mut self.discovery_side_throttle,
3228 Self::schema_response_owner(&item.item),
3229 );
3230 if let Some(key) = item.item.replay_marker_key() {
3231 Self::set_replay_queued(
3232 &mut self.reliable_tx,
3233 &mut self.end_to_end_reliable_tx,
3234 key,
3235 false,
3236 );
3237 }
3238 true
3239 }
3240 }
3241 }
3242
3243 fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
3244 let received = self
3245 .received_queue
3246 .lowest_priority(|queued| queued.priority)
3247 .map(|priority| (RouterQueueKind::Received, priority));
3248 let transmit = self
3249 .transmit_queue
3250 .iter()
3251 .filter(|queued| queued.disposable & TX_DISPOSABLE != 0)
3252 .map(|queued| queued.priority)
3253 .min()
3254 .map(|priority| (RouterQueueKind::Transmit, priority));
3255 [received, transmit]
3256 .into_iter()
3257 .flatten()
3258 .filter(|(_, priority)| {
3259 *priority < incoming_priority
3260 || (incoming_priority != 254 && *priority == incoming_priority)
3261 })
3262 .min_by_key(|(kind, priority)| {
3263 (
3264 *priority,
3265 if *kind == RouterQueueKind::Received {
3266 0
3267 } else {
3268 1
3269 },
3270 )
3271 })
3272 .map(|(kind, _)| kind)
3273 }
3274
3275 fn make_shared_queue_room(
3276 &mut self,
3277 incoming_cost: usize,
3278 incoming_priority: u8,
3279 _incoming: Option<&RouterItem>,
3280 ) -> TelemetryResult<()> {
3281 if incoming_cost > self.memory.max_queue_budget {
3282 return Err(TelemetryError::PacketTooLarge(
3283 "Item exceeds maximum shared queue budget",
3284 ));
3285 }
3286
3287 #[cfg(any(feature = "std", feature = "fair-admission"))]
3288 let mut fair_victim = None;
3289 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
3290 > self.memory.max_queue_budget
3291 {
3292 #[cfg(all(feature = "compression", feature = "discovery"))]
3293 if self.reclaim_codec_metadata() {
3294 continue;
3295 }
3296
3297 #[cfg(any(feature = "std", feature = "fair-admission"))]
3298 {
3299 let incoming =
3300 _incoming.and_then(|data| Self::fair_bulk_owner(data, incoming_priority));
3301 let victim = *fair_victim.get_or_insert_with(|| {
3302 self.bulk_loads(incoming_priority.min(199))
3303 .queue_victim(incoming)
3304 });
3305 if let Some(victim) = victim
3306 && self.reclaim_bulk_from(
3307 victim,
3308 incoming_priority.min(199),
3309 (incoming == Some(victim))
3310 .then(|| _incoming.and_then(Self::bulk_stream))
3311 .flatten(),
3312 )
3313 {
3314 continue;
3315 }
3316 if incoming.is_some() {
3317 return Err(TelemetryError::Io("shared fair queue saturated"));
3318 }
3319 }
3320 let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
3321 if incoming_priority >= 200 && self.reclaim_oldest_side_chunk(None) {
3322 continue;
3323 }
3324 return Err(TelemetryError::Io("shared priority queue saturated"));
3325 };
3326 if !self.pop_shared_queue_item(victim) {
3327 return Err(TelemetryError::PacketTooLarge(
3328 "Item exceeds maximum shared queue budget",
3329 ));
3330 }
3331 }
3332
3333 Ok(())
3334 }
3335
3336 #[cfg(feature = "discovery")]
3337 fn schema_growth_budget(&self) -> usize {
3338 let protected = self
3341 .recent_rx
3342 .max_bytes()
3343 .saturating_add(self.discovery_bytes_used())
3344 .saturating_add(self.reliable_rx_buffered_bytes())
3345 .saturating_add({
3346 #[cfg(feature = "compression")]
3347 {
3348 self.codec_reserved_bytes
3349 }
3350 #[cfg(not(feature = "compression"))]
3351 {
3352 0
3353 }
3354 });
3355 self.memory.max_queue_budget.saturating_sub(protected)
3356 }
3357
3358 #[cfg(feature = "discovery")]
3359 fn cancel_managed_refresh_side(&mut self, side: RouterSideId) {
3360 for (&ingress, route) in &mut self.discovery_routes {
3361 for request in &mut route.requested_network_variables {
3362 if ingress == side || request.upstream == side {
3363 request.refresh_pending = false;
3364 }
3365 }
3366 }
3367 if let Some(local) = self.local_services.as_mut() {
3368 for entry in local.managed_variable_latest.values_mut() {
3369 if entry
3370 .upstream
3371 .is_some_and(|upstream| upstream.get() - 1 == side)
3372 {
3373 entry.upstream = None;
3374 }
3375 }
3376 }
3377 }
3378
3379 #[cfg(feature = "discovery")]
3380 fn commit_discovery_route(
3381 &mut self,
3382 side: RouterSideId,
3383 route: DiscoverySideState,
3384 ) -> TelemetryResult<()> {
3385 let previous = self
3386 .discovery_routes
3387 .get(&side)
3388 .map_or(0, |route| Self::discovery_route_byte_cost(side, route));
3389 let incoming = Self::discovery_route_byte_cost(side, &route);
3390 let growth = incoming.saturating_sub(previous);
3391 self.make_shared_queue_room(growth, 255, None)?;
3392 self.discovery_routes.insert(side, route);
3393 Ok(())
3394 }
3395
3396 #[cfg(feature = "discovery")]
3397 fn fit_discovery_budget(&mut self) {
3398 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
3399 #[cfg(all(feature = "compression", feature = "discovery"))]
3400 if self.reclaim_codec_metadata() {
3401 continue;
3402 }
3403
3404 if let Some(victim) = self.lower_priority_shared_queue(255)
3407 && self.pop_shared_queue_item(victim)
3408 {
3409 continue;
3410 }
3411 break;
3414 }
3415 }
3416
3417 fn reliable_original_type(item: &TxQueued) -> Option<DataType> {
3418 let RouterTxItem::Broadcast(data) = &item.item else {
3419 return None;
3420 };
3421 let _ = data;
3422 let ty = DataType(item.accepted_type);
3423 (item.delivery_state != QueuedDelivery::Untracked).then_some(ty)
3424 }
3425
3426 fn ordered_original_type(item: &TxQueued) -> Option<DataType> {
3427 let RouterTxItem::Broadcast(data) = &item.item else {
3428 return None;
3429 };
3430 let _ = data;
3431 let ty = DataType(item.accepted_type);
3432 matches!(
3433 item.delivery_state,
3434 QueuedDelivery::Ordered
3435 | QueuedDelivery::OrderedTail
3436 | QueuedDelivery::DeferredOrdered
3437 | QueuedDelivery::ClaimedOrdered
3438 )
3439 .then_some(ty)
3440 }
3441
3442 fn pop_received_fair(&mut self) -> Option<RouterRxItem> {
3443 self.received_queue.pop_front_bulk_fair(
3444 &mut self.rx_priority_burst,
3445 MAX_PRIORITY_BURST,
3446 |item| item.priority,
3447 )
3448 }
3449
3450 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
3451 self.make_shared_queue_room(item.byte_cost(), item.priority, Some(&item.data))?;
3452 #[cfg(any(feature = "std", feature = "fair-admission"))]
3453 {
3454 #[cfg(any(feature = "std", feature = "fair-admission"))]
3455 let mut fair_victim = None;
3456 self.received_queue.push_back_prioritized_select(
3457 item,
3458 |queued| queued.priority,
3459 |queue, _incoming| {
3460 #[cfg(any(feature = "std", feature = "fair-admission"))]
3461 {
3462 crate::admission_fairness::queue_victim(
3463 queue,
3464 _incoming,
3465 |item| Self::fair_bulk_owner(&item.data, item.priority),
3466 |item| item.priority,
3467 |item| Self::bulk_stream(&item.data),
3468 &mut fair_victim,
3469 )
3470 }
3471 },
3472 core::mem::drop,
3473 )
3474 }
3475 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3476 {
3477 self.received_queue
3478 .push_back_prioritized(item, |queued| queued.priority)
3479 }
3480 }
3481
3482 fn push_transmit(&mut self, mut item: TxQueued) -> TelemetryResult<()> {
3483 let child = item.disposable & TX_E2E_CHILD != 0;
3484 #[cfg(feature = "latest-value-egress")]
3485 let turn_flags = if !child
3486 && item.delivery_state == QueuedDelivery::Untracked
3487 && self.canonical_catalog_owner(&item)
3488 {
3489 item.disposable & TX_CANONICAL_TURN
3490 } else if !child
3491 && item.delivery_state == QueuedDelivery::Untracked
3492 && matches!(&item.item, RouterTxItem::ToSide { dst, .. }
3493 if self.latest_side() == Some(*dst))
3494 {
3495 item.disposable & TX_LATEST_TURN
3496 } else {
3497 0
3498 };
3499 #[cfg(feature = "std")]
3500 let retry_origin = item.disposable & TX_E2E_RETRY;
3501 #[cfg(feature = "latest-value-egress")]
3502 if item.disposable & TX_LATEST_PENDING != 0 && !self.latest_entry_eligible(&item) {
3503 item.disposable &= !TX_LATEST_PENDING;
3504 }
3505 #[cfg(feature = "compression")]
3506 match &mut item.item {
3507 RouterTxItem::Broadcast(data) => self.prepare_codec_queue_owner(None, data),
3508 RouterTxItem::ToSide { dst, data, .. } => {
3509 self.prepare_codec_queue_owner(Some(*dst), data)
3510 }
3511 _ => {}
3512 }
3513 let type_source = match &item.item {
3514 RouterTxItem::Broadcast(data) => Some(data),
3515 #[cfg(feature = "std")]
3518 RouterTxItem::ToSide { data, .. } if item.accepted_type == 0 => Some(data),
3519 _ => None,
3520 };
3521 if let Some(data) = type_source {
3522 item.accepted_type = match data {
3523 RouterItem::Packet(packet) => packet.data_type().as_u32(),
3524 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3525 wire_format::scheduling_type(view)
3526 })
3527 .ok()
3528 .flatten()
3529 .map_or(0, |ty| ty.as_u32()),
3530 };
3531 }
3532 if item.delivery_state == QueuedDelivery::Untracked
3533 && matches!(&item.item, RouterTxItem::Broadcast(_))
3534 {
3535 item.delivery_state = match reliable_mode(DataType(item.accepted_type)) {
3536 crate::ReliableMode::Ordered => QueuedDelivery::Ordered,
3537 crate::ReliableMode::Unordered => QueuedDelivery::Unordered,
3538 crate::ReliableMode::None => QueuedDelivery::Untracked,
3539 };
3540 }
3541 if matches!(
3544 item.delivery_state,
3545 QueuedDelivery::Ordered | QueuedDelivery::DeferredOrdered
3546 ) && let Some(priority) = self.transmit_queue.iter().find_map(|queued| {
3547 (Self::ordered_original_type(queued).is_some()
3548 && queued.accepted_type == item.accepted_type)
3549 .then_some(queued.priority)
3550 }) {
3551 item.priority = priority;
3552 item.delivery_state = QueuedDelivery::OrderedTail;
3553 }
3554 #[cfg(not(feature = "latest-value-egress"))]
3555 {
3556 item.disposable = if item.delivery_state != QueuedDelivery::Untracked {
3557 0
3558 } else {
3559 u8::from(Self::classify_transmit_disposable(&item))
3560 };
3561 }
3562 #[cfg(feature = "latest-value-egress")]
3563 {
3564 let history_owned = item.disposable & TX_HISTORY_OWNED != 0
3565 || (item.retry_after_ms != 0
3566 && matches!(&item.item,
3567 RouterTxItem::ToSide { dst, .. } if self.latest_side() == Some(*dst)));
3568 item.disposable = if history_owned {
3569 TX_HISTORY_OWNED
3570 } else if item.delivery_state != QueuedDelivery::Untracked {
3571 0
3572 } else {
3573 u8::from(Self::classify_transmit_disposable(&item))
3574 | (item.disposable & TX_LATEST_PENDING)
3575 };
3576 item.disposable |= turn_flags;
3577 if item.disposable & TX_LATEST_PENDING != 0 && self.replace_latest_pending(&mut item)? {
3578 return Ok(());
3579 }
3580 }
3581 if child {
3582 item.disposable = (item.disposable & !TX_DISPOSABLE) | TX_E2E_CHILD;
3583 #[cfg(feature = "std")]
3584 {
3585 item.disposable |= retry_origin;
3586 }
3587 }
3588 let ordered_retry = Self::ordered_original_type(&item).is_some()
3589 && item.delivery_state == QueuedDelivery::DeferredOrdered;
3590 let replay = item.item.replay_marker_key();
3591 if replay.is_some_and(|key| self.replay_is_queued(key).is_none_or(|queued| queued)) {
3594 return Ok(());
3595 }
3596 #[cfg(feature = "discovery")]
3597 if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
3598 if let RouterItem::Packet(incoming) = data
3601 && self.transmit_queue.iter().any(|queued| {
3602 matches!(&queued.item,
3603 RouterTxItem::ToSide { dst: queued_dst, data: RouterItem::Packet(existing), .. }
3604 if queued_dst == dst && discovery::same_schema_snapshot(existing, incoming))
3605 })
3606 {
3607 return Ok(());
3608 }
3609 let incoming = match data {
3610 RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
3611 Some((pkt.data_type(), pkt.sender().to_owned()))
3612 }
3613 RouterItem::Packet(_) | RouterItem::Packed(_) => None,
3614 };
3615 if let Some((incoming_ty, incoming_sender)) = incoming {
3616 self.retain_transmit(|queued| {
3617 let RouterTxItem::ToSide {
3618 dst: queued_dst,
3619 data: queued_data,
3620 ..
3621 } = &queued.item
3622 else {
3623 return true;
3624 };
3625 let same_snapshot = match queued_data {
3626 RouterItem::Packet(pkt) => {
3627 pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
3628 }
3629 RouterItem::Packed(_) => false,
3630 };
3631 *queued_dst != *dst || !same_snapshot
3632 });
3633 }
3634 }
3635 self.make_shared_queue_room(
3636 item.byte_cost(),
3637 item.priority,
3638 match &item.item {
3639 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => Some(data),
3640 _ => None,
3641 },
3642 )?;
3643 let reliable_tx = &mut self.reliable_tx;
3644 let end_to_end = &mut self.end_to_end_reliable_tx;
3645 #[cfg(feature = "discovery")]
3646 let responses = &mut self.discovery_side_throttle;
3647 let result = {
3648 #[cfg(any(feature = "std", feature = "fair-admission"))]
3649 {
3650 #[cfg(any(feature = "std", feature = "fair-admission"))]
3651 let mut fair_victim = None;
3652 self.transmit_queue.push_back_prioritized_select(
3653 item,
3654 |queued| queued.priority,
3655 |queue, _incoming| {
3656 #[cfg(any(feature = "std", feature = "fair-admission"))]
3657 {
3658 if Self::tx_bulk_owner(_incoming).is_some() {
3659 crate::admission_fairness::queue_victim(
3660 queue,
3661 _incoming,
3662 Self::tx_bulk_owner,
3663 |item| item.priority,
3664 Self::tx_bulk_stream,
3665 &mut fair_victim,
3666 )
3667 } else {
3668 Self::transmit_eviction_candidate(queue)
3669 }
3670 }
3671 },
3672 |evicted: TxQueued| {
3673 #[cfg(feature = "discovery")]
3674 Self::schema_response_owner_lost(
3675 responses,
3676 Self::schema_response_owner(&evicted.item),
3677 );
3678 if let Some(key) = evicted.item.replay_marker_key() {
3679 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3680 }
3681 },
3682 )
3683 }
3684 #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3685 {
3686 self.transmit_queue.push_back_prioritized_select(
3687 item,
3688 |queued| queued.priority,
3689 |queue, _| Self::transmit_eviction_candidate(queue),
3690 |evicted: TxQueued| {
3691 #[cfg(feature = "discovery")]
3692 Self::schema_response_owner_lost(
3693 responses,
3694 Self::schema_response_owner(&evicted.item),
3695 );
3696 if let Some(key) = evicted.item.replay_marker_key() {
3697 Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3698 }
3699 },
3700 )
3701 }
3702 };
3703 if result.is_ok() && ordered_retry {
3704 self.tx_policy_flags |= 1;
3705 }
3706 if result.is_ok()
3707 && let Some(key) = replay
3708 {
3709 Self::set_replay_queued(reliable_tx, end_to_end, key, true);
3710 }
3711 result
3712 }
3713
3714 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
3715 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
3716 let _ = self.recent_rx.pop_front();
3717 }
3718 self.make_shared_queue_room(0, 0, None)?;
3719 self.recent_rx.push_back(id)
3720 }
3721
3722 #[allow(unused_mut)]
3723 fn buffer_reliable_rx(
3724 &mut self,
3725 side: RouterSideId,
3726 ty: DataType,
3727 seq: u32,
3728 mut bytes: crate::SharedBytes,
3729 ) -> TelemetryResult<()> {
3730 let key = Router::reliable_key(side, ty);
3731 if self
3732 .reliable_rx
3733 .get(&key)
3734 .is_some_and(|state| state.buffered.contains_key(&seq))
3735 {
3736 return Ok(());
3737 }
3738 let cost = size_of::<crate::SharedBytes>() + bytes.len();
3739 self.make_shared_queue_room(cost, crate::transport_priority(ty), None)?;
3740 #[cfg(feature = "compact-packet-store")]
3741 bytes.park_for_queue()?;
3742 let rx_state = self
3743 .reliable_rx
3744 .entry(key)
3745 .or_insert_with(|| ReliableRxState {
3746 expected_seq: 1,
3747 buffered: Default::default(),
3748 });
3749 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
3750 let _ = rx_state.buffered.pop_first();
3751 }
3752 rx_state.buffered.insert(seq, bytes);
3753 Ok(())
3754 }
3755}
3756
3757struct IsrRxQueue {
3762 busy: AtomicBool,
3763 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
3764}
3765
3766unsafe impl Send for IsrRxQueue {}
3767unsafe impl Sync for IsrRxQueue {}
3768
3769struct IsrRxQueueGuard<'a> {
3770 owner: &'a IsrRxQueue,
3771}
3772
3773impl Deref for IsrRxQueueGuard<'_> {
3774 type Target = BoundedDeque<RouterRxItem>;
3775
3776 #[inline]
3777 fn deref(&self) -> &Self::Target {
3778 unsafe { &*self.owner.q.get() }
3779 }
3780}
3781
3782impl DerefMut for IsrRxQueueGuard<'_> {
3783 #[inline]
3784 fn deref_mut(&mut self) -> &mut Self::Target {
3785 unsafe { &mut *self.owner.q.get() }
3786 }
3787}
3788
3789impl Drop for IsrRxQueueGuard<'_> {
3790 #[inline]
3791 fn drop(&mut self) {
3792 self.owner.busy.store(false, Ordering::Release);
3793 }
3794}
3795
3796impl IsrRxQueue {
3797 #[inline]
3798 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
3799 Self {
3800 busy: AtomicBool::new(false),
3801 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
3802 }
3803 }
3804
3805 #[inline]
3806 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
3807 match self
3808 .busy
3809 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
3810 {
3811 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
3812 Err(_) => Err(TelemetryError::Io("rx queue busy")),
3813 }
3814 }
3815
3816 #[allow(dead_code)]
3817 #[inline]
3818 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
3819 let mut g = self.try_lock()?;
3820 g.push_back(item)
3821 }
3822
3823 #[inline]
3824 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
3825 let mut g = self.try_lock()?;
3826 g.push_back_prioritized(item, |queued| queued.priority)
3827 }
3828
3829 #[inline]
3830 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
3831 let mut g = self.try_lock()?;
3832 Ok(g.pop_front())
3833 }
3834
3835 #[inline]
3836 fn clear(&self) -> TelemetryResult<()> {
3837 let mut g = self.try_lock()?;
3838 g.clear();
3839 Ok(())
3840 }
3841
3842 #[inline]
3843 fn snapshot(&self) -> Option<(usize, usize)> {
3844 let g = self.try_lock().ok()?;
3845 Some((g.len(), g.bytes_used()))
3846 }
3847}
3848
3849pub struct Router {
3853 #[cfg(feature = "std")]
3854 retry_trackers: RouterMutex<Vec<std::sync::Weak<completion::TrackerInner>>>,
3855 sender: RouterMutex<Arc<str>>,
3856 cfg: ActiveRoutingConfig,
3857 state: RouterMutex<RouterInner>,
3858 isr_rx_queue: Option<Box<IsrRxQueue>>,
3859 #[cfg(not(feature = "std"))]
3860 side_tx_gate: ReentryGate,
3861 #[cfg(feature = "std")]
3862 side_tx_activity: crate::lock::SideTxActivity,
3863 #[cfg(feature = "std")]
3864 active_queued_tx: core::sync::atomic::AtomicUsize,
3865 clock: Box<dyn Clock + Send + Sync>,
3866 #[cfg(feature = "timesync")]
3867 timesync: Option<Box<RouterMutex<TimeSyncRuntime>>>,
3868}
3869
3870#[cfg(feature = "std")]
3871const ACTIVE_QUEUED_TYPE_SLOTS: usize = 16;
3872#[cfg(feature = "std")]
3873#[derive(Clone, Copy)]
3874struct ActiveQueuedTypeKey {
3875 slot: core::num::NonZeroU8,
3876 kind: u32,
3877}
3878#[cfg(feature = "std")]
3879struct QueuedTxActivityGuard<'a> {
3880 router: &'a Router,
3881 key: Option<ActiveQueuedTypeKey>,
3882}
3883#[cfg(feature = "std")]
3884impl Drop for QueuedTxActivityGuard<'_> {
3885 fn drop(&mut self) {
3886 let mut state = self.router.state.lock();
3887 if let Some(key) = self.key {
3888 let slot = &mut state.active_queued_types[usize::from(key.slot.get() - 1)];
3889 assert_eq!(slot.0, key.kind, "active queued type guard changed owner");
3890 slot.1 = slot.1.checked_sub(1).expect("active queued type underflow");
3891 if slot.1 == 0 {
3892 slot.0 = 0;
3893 }
3894 }
3895 let total = self
3896 .router
3897 .active_queued_tx
3898 .load(core::sync::atomic::Ordering::Acquire);
3899 self.router.active_queued_tx.store(
3900 total.checked_sub(1).expect("active queued TX underflow"),
3901 core::sync::atomic::Ordering::Release,
3902 );
3903 }
3904}
3905
3906#[cfg(feature = "timesync")]
3907#[derive(Debug, Clone)]
3908struct PendingTimeSyncRequest {
3909 seq: u64,
3910 t1_mono_ms: u64,
3911 source: String,
3912}
3913
3914#[cfg(feature = "timesync")]
3915#[derive(Debug, Clone)]
3916struct RemoteTimeSyncSource {
3917 priority: u64,
3918 last_sample_mono_ms: u64,
3919 sample_unix_ms: u64,
3920}
3921
3922#[cfg(feature = "timesync")]
3923#[derive(Debug, Clone)]
3924struct TimeSyncRuntime {
3925 cfg: Option<TimeSyncConfig>,
3926 tracker: Option<TimeSyncTracker>,
3927 clock: NetworkClock,
3928 disciplined_clock: SlewedNetworkClock,
3929 remote_sources: BTreeMap<String, Box<RemoteTimeSyncSource>>,
3930 next_seq: u64,
3931 next_announce_mono_ms: u64,
3932 next_request_mono_ms: u64,
3933 pending_request: Option<PendingTimeSyncRequest>,
3934}
3935
3936#[cfg(feature = "timesync")]
3937impl TimeSyncRuntime {
3938 fn new(cfg: Option<TimeSyncConfig>) -> Self {
3939 Self {
3940 tracker: cfg.map(TimeSyncTracker::new),
3941 cfg,
3942 clock: NetworkClock::default(),
3943 disciplined_clock: SlewedNetworkClock::new(
3944 cfg.map(|c| c.max_slew_ppm)
3945 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
3946 ),
3947 remote_sources: Default::default(),
3948 next_seq: 1,
3949 next_announce_mono_ms: 0,
3950 next_request_mono_ms: 0,
3951 pending_request: None,
3952 }
3953 }
3954}
3955
3956enum RemoteSidePlan {
3957 Target(Vec<RouterSideId>),
3958}
3959
3960#[cfg(feature = "discovery")]
3961#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
3962struct DiscoveryCandidateMatch {
3963 side: RouterSideId,
3964 overlap: usize,
3965}
3966
3967#[cfg(feature = "discovery")]
3968#[inline]
3969const fn discovery_is_hop_local_advertisement(ty: DataType, transparent: bool) -> bool {
3970 matches!(
3971 ty,
3972 DataType::DiscoveryAnnounce
3973 | DataType::DiscoveryTimeSyncSources
3974 | DataType::DiscoveryTopology
3975 | DataType::DiscoveryLinkCapabilities
3976 ) || (matches!(ty, DataType::DiscoveryAddress) && !transparent)
3977}
3978
3979#[cfg(not(feature = "discovery"))]
3980const fn discovery_is_hop_local_advertisement(_ty: DataType, _transparent: bool) -> bool {
3981 false
3982}
3983
3984impl Debug for Router {
3985 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
3986 let sender = self.sender();
3987 f.debug_struct("Router")
3988 .field("sender", &sender)
3989 .field("cfg", &self.cfg)
3990 .field("state", &"<mutex>")
3991 .field("clock", &"Clock")
3992 .finish()
3993 }
3994}
3995
3996#[inline]
3999fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &ActiveRoutingConfig) -> bool {
4000 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
4001}
4002
4003#[inline]
4004fn force_remote_for_type(ty: DataType) -> bool {
4005 matches!(
4006 ty,
4007 DataType::ReliableAck
4008 | DataType::ReliablePartialAck
4009 | DataType::ReliablePacketRequest
4010 | DataType::P2pMessage
4011 ) || {
4012 #[cfg(feature = "timesync")]
4013 {
4014 matches!(
4015 ty,
4016 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4017 )
4018 }
4019 #[cfg(not(feature = "timesync"))]
4020 {
4021 false
4022 }
4023 }
4024}
4025
4026#[inline]
4027fn is_internal_control_type(ty: DataType) -> bool {
4028 if matches!(
4029 ty,
4030 DataType::ReliableAck
4031 | DataType::ReliablePartialAck
4032 | DataType::ReliablePacketRequest
4033 | DataType::P2pMessage
4034 ) {
4035 return true;
4036 }
4037
4038 #[cfg(feature = "timesync")]
4039 if matches!(
4040 ty,
4041 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4042 ) {
4043 return true;
4044 }
4045
4046 #[cfg(feature = "discovery")]
4047 if discovery::is_discovery_type(ty) {
4048 return true;
4049 }
4050
4051 let _ = ty;
4052 false
4053}
4054
4055fn with_retries<F>(
4057 this: &Router,
4058 dest: DataEndpoint,
4059 data: &RouterItem,
4060 pkt_for_ctx: Option<&Packet>,
4061 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
4062 called_from_queue: bool,
4063 run: F,
4064) -> TelemetryResult<()>
4065where
4066 F: Fn() -> TelemetryResult<()>,
4067{
4068 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
4069 Ok(((), attempts)) => {
4070 if attempts > 1 {
4071 let mut st = this.state.lock();
4072 st.total_handler_retries = st
4073 .total_handler_retries
4074 .saturating_add(attempts.saturating_sub(1) as u64);
4075 }
4076 Ok(())
4077 }
4078 Err((e, attempts)) => {
4079 {
4080 let mut st = this.state.lock();
4081 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
4082 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
4083 }
4084
4085 this.remove_pkt_id(data);
4087
4088 if let Some(pkt) = pkt_for_ctx {
4090 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
4091 } else if let Some(env) = env_for_ctx {
4092 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
4093 }
4094
4095 Err(TelemetryError::HandlerError("local handler failed"))
4096 }
4097 }
4098}
4099impl Router {
4101 #[cfg(feature = "discovery")]
4102 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
4103 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
4104
4105 #[inline]
4106 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
4107 st.sides
4108 .get(side)
4109 .and_then(|side| side.as_ref())
4110 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
4111 }
4112
4113 fn note_side_tx_success(
4114 &self,
4115 side: RouterSideId,
4116 ty: DataType,
4117 bytes: usize,
4118 relayed: bool,
4119 attempts: usize,
4120 ) {
4121 let mut st = self.state.lock();
4122 #[cfg(feature = "discovery")]
4123 if ty == crate::DataType::DiscoveryAnnounce
4124 && !relayed
4125 && let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
4126 {
4127 throttle.last_liveness_tx_ms = self.clock.now_ms();
4128 }
4129 #[cfg(all(feature = "discovery", feature = "timesync"))]
4130 if Self::is_timesync_type(ty) {
4131 let now = self.clock.now_ms();
4132 if Self::side_is_slow_control_link_locked(&st, side, now) {
4133 st.timesync_side_throttle
4134 .entry(side)
4135 .or_default()
4136 .next_allowed_ms = now.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
4137 } else {
4138 st.timesync_side_throttle.remove(&side);
4139 }
4140 }
4141 let entry = st.side_runtime_stats.entry(side).or_default();
4142 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
4143 }
4144
4145 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
4146 let mut st = self.state.lock();
4147 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
4148 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
4149 let entry = st.side_runtime_stats.entry(side).or_default();
4150 entry.note_tx_failure(ty, attempts);
4151 }
4152
4153 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
4154 let mut st = self.state.lock();
4155 let entry = st.side_runtime_stats.entry(side).or_default();
4156 entry.note_rx(ty, bytes, relayed);
4157 }
4158
4159 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
4160 let mut st = self.state.lock();
4161 let entry = st.side_runtime_stats.entry(side).or_default();
4162 entry.note_local_delivery(ty);
4163 }
4164
4165 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
4166 let mut st = self.state.lock();
4167 let entry = st.side_runtime_stats.entry(side).or_default();
4168 entry.note_local_handler_failure(ty, retries);
4169 }
4170
4171 fn cache_managed_variable_packet(
4172 &self,
4173 pkt: &Packet,
4174 notify_handlers: bool,
4175 source_side: Option<RouterSideId>,
4176 ) -> TelemetryResult<()> {
4177 #[cfg(not(feature = "discovery"))]
4178 let _ = source_side;
4179 let sender_address = pkt
4180 .sender()
4181 .strip_prefix("@addr:")
4182 .and_then(|value| value.parse::<u32>().ok())
4183 .unwrap_or_else(|| sender_address_u32(pkt.sender()));
4184 let handlers = {
4185 let mut st = self.state.lock();
4186 if !st.is_managed_variable(pkt.data_type().as_u32()) {
4187 return Ok(());
4188 }
4189 let existing = st
4190 .local_services
4191 .as_ref()
4192 .expect("local services enabled")
4193 .managed_variable_latest
4194 .get(&pkt.data_type().as_u32());
4195 if existing.is_some_and(|entry| {
4202 let current = &entry.packet;
4203 sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
4204 }) {
4205 return Ok(());
4206 }
4207 let changed = existing.is_none_or(|entry| entry.packet.packet_id() != pkt.packet_id());
4211 #[cfg(feature = "discovery")]
4212 let upstream = existing
4213 .filter(|_| !changed)
4214 .and_then(|entry| entry.upstream)
4215 .or_else(|| {
4216 source_side
4217 .and_then(|side| side.checked_add(1))
4218 .and_then(core::num::NonZeroUsize::new)
4219 });
4220 let updated = ManagedVariableCacheEntry {
4224 packet: pkt.clone(),
4225 sender_address,
4226 #[cfg(feature = "discovery")]
4227 cached_at_ms: self.clock.now_ms(),
4228 #[cfg(feature = "discovery")]
4229 upstream,
4230 };
4231 if let Some(entry) = st
4232 .local_services
4233 .as_mut()
4234 .expect("local services enabled")
4235 .managed_variable_latest
4236 .get_mut(&pkt.data_type().as_u32())
4237 {
4238 **entry = updated;
4239 } else {
4240 st.local_services
4241 .as_mut()
4242 .expect("local services enabled")
4243 .managed_variable_latest
4244 .insert(pkt.data_type().as_u32(), Box::new(updated));
4245 }
4246 if notify_handlers && changed {
4247 st.local_services
4248 .as_ref()
4249 .expect("local services enabled")
4250 .network_variable_update_handlers
4251 .get(&pkt.data_type().as_u32())
4252 .cloned()
4253 .unwrap_or_default()
4254 } else {
4255 Vec::new()
4256 }
4257 };
4258 for handler in handlers {
4259 (handler.handler)(pkt)?;
4260 }
4261 Ok(())
4262 }
4263
4264 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
4265 self.cache_managed_variable_packet(pkt, true, None)
4266 }
4267
4268 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
4269 let st = self.state.lock();
4270 st.local_services
4271 .as_ref()?
4272 .managed_variable_latest
4273 .get(&ty.as_u32())
4274 .map(|entry| entry.packet.clone())
4275 }
4276
4277 #[cfg(feature = "discovery")]
4278 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
4279 let now_ms = self.clock.now_ms();
4280 let st = self.state.lock();
4281 st.local_services
4282 .as_ref()?
4283 .managed_variable_latest
4284 .get(&ty.as_u32())
4285 .map(|entry| {
4286 (
4287 entry.packet.clone(),
4288 now_ms.saturating_sub(entry.cached_at_ms),
4289 )
4290 })
4291 }
4292
4293 fn is_managed_variable_type(&self, ty: DataType) -> bool {
4294 let st = self.state.lock();
4295 st.is_managed_variable(ty.as_u32())
4296 }
4297
4298 fn managed_variable_permissions_locked(
4299 st: &RouterInner,
4300 ty: DataType,
4301 ) -> NetworkVariablePermissions {
4302 let Some(local) = st.local_services.as_ref() else {
4303 return NetworkVariablePermissions::NONE;
4304 };
4305 local
4306 .managed_variable_permissions
4307 .get(&ty.as_u32())
4308 .copied()
4309 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
4310 }
4311
4312 fn can_read_managed_variable(&self, ty: DataType) -> bool {
4313 let st = self.state.lock();
4314 Self::managed_variable_permissions_locked(&st, ty).read
4315 }
4316
4317 #[cfg(feature = "discovery")]
4318 fn can_write_managed_variable(&self, ty: DataType) -> bool {
4319 let st = self.state.lock();
4320 Self::managed_variable_permissions_locked(&st, ty).write
4321 }
4322
4323 #[inline]
4324 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
4325 let st = self.state.lock();
4326 let side_ref = Self::side_ref(&st, side)?;
4327 if side_ref.opts.ingress_enabled {
4328 Ok(())
4329 } else {
4330 Err(TelemetryError::HandlerError(
4331 "router: ingress disabled for side id",
4332 ))
4333 }
4334 }
4335
4336 #[inline]
4337 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
4338 src != Some(dst)
4339 }
4340
4341 #[inline]
4342 fn route_allowed_locked(
4343 &self,
4344 st: &RouterInner,
4345 src: Option<RouterSideId>,
4346 ty: Option<DataType>,
4347 dst: RouterSideId,
4348 ) -> bool {
4349 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
4350 return false;
4351 };
4352 if !dst_side.opts.egress_enabled {
4353 return false;
4354 }
4355 if let Some(src_id) = src {
4356 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
4357 return false;
4358 };
4359 if !src_side.opts.ingress_enabled {
4360 return false;
4361 }
4362 }
4363 let base_allowed = st
4364 .route_overrides
4365 .get(&(src, dst))
4366 .copied()
4367 .unwrap_or_else(|| self.default_route_enabled(src, dst));
4368 if !base_allowed {
4369 return false;
4370 }
4371
4372 let Some(ty) = ty else {
4373 return true;
4374 };
4375 if st
4376 .typed_route_overrides
4377 .keys()
4378 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
4379 {
4380 return st
4381 .typed_route_overrides
4382 .get(&(src, ty.as_u32(), dst))
4383 .copied()
4384 .unwrap_or(false);
4385 }
4386 true
4387 }
4388
4389 #[cfg(feature = "discovery")]
4390 fn has_typed_route_overrides_locked(
4391 st: &RouterInner,
4392 src: Option<RouterSideId>,
4393 ty: DataType,
4394 ) -> bool {
4395 st.typed_route_overrides
4396 .keys()
4397 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
4398 }
4399
4400 #[cfg(feature = "discovery")]
4401 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
4402 where
4403 I: IntoIterator<Item = DataEndpoint>,
4404 {
4405 let mut overlap = 0usize;
4406 for ep in reachable {
4407 if eps.contains(&ep) {
4408 overlap = overlap.saturating_add(1);
4409 }
4410 }
4411 overlap
4412 }
4413
4414 #[inline]
4415 #[cfg(feature = "discovery")]
4416 fn preferred_scoring_endpoints<'a>(
4417 &self,
4418 eps: &'a [DataEndpoint],
4419 prefer_nonlocal: bool,
4420 ) -> alloc::borrow::Cow<'a, [DataEndpoint]> {
4421 if !prefer_nonlocal || eps.iter().all(|&ep| !self.cfg.is_local_endpoint(ep)) {
4422 return alloc::borrow::Cow::Borrowed(eps);
4423 }
4424 let nonlocal: Vec<DataEndpoint> = eps
4425 .iter()
4426 .copied()
4427 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
4428 .collect();
4429 if nonlocal.is_empty() {
4430 alloc::borrow::Cow::Borrowed(eps)
4431 } else {
4432 alloc::borrow::Cow::Owned(nonlocal)
4433 }
4434 }
4435
4436 fn pop_next_queued_control_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
4437 if let Some(queue) = self.isr_rx_queue.as_ref() {
4438 let mut isr_rx = queue.try_lock()?;
4439 let idx = isr_rx
4440 .iter()
4441 .position(|item| self.queued_rx_item_is_control(item));
4442 if let Some(idx) = idx {
4443 return Ok(isr_rx.remove_pos(idx));
4444 }
4445 }
4446
4447 let mut st = self.state.lock();
4448 let idx = st
4449 .received_queue
4450 .iter()
4451 .position(|item| self.queued_rx_item_is_control(item));
4452 if let Some(idx) = idx {
4453 return Ok(st.received_queue.remove_pos(idx));
4454 }
4455 Ok(None)
4456 }
4457
4458 fn queued_rx_item_is_control(&self, item: &RouterRxItem) -> bool {
4459 if !self.cfg.role.local_services()
4463 && item.priority < crate::transport_priority(DataType::ReliableAck)
4464 {
4465 return false;
4466 }
4467 if item.priority < crate::transport_priority(DataType::ManagedVariableValue) {
4471 return false;
4472 }
4473 if item.priority == crate::transport_priority(DataType::ManagedVariableValue) {
4476 return true;
4477 }
4478 let ty = match &item.data {
4479 RouterItem::Packet(pkt) => Some(pkt.data_type()),
4480 RouterItem::Packed(bytes) => {
4481 if bytes.len() <= 128
4482 && wire_format::peek_routing_frame_info(bytes)
4483 .is_ok_and(|frame| frame.ack_only())
4484 {
4485 return true;
4486 }
4487 wire_format::scheduling_type(bytes.as_ref())
4488 }
4489 };
4490 match ty {
4491 Some(
4492 DataType::ReliableAck
4493 | DataType::ReliablePartialAck
4494 | DataType::ReliablePacketRequest,
4495 ) => true,
4496 #[cfg(feature = "discovery")]
4497 Some(ty) => self.cfg.role.local_services() && discovery::is_discovery_type(ty),
4498 _ => false,
4499 }
4500 }
4501
4502 fn drain_queued_control_rx_before_tx(
4503 &self,
4504 start: u64,
4505 timeout_ms: u32,
4506 ) -> TelemetryResult<bool> {
4507 let mut did_any = false;
4508 while let Some(item) = self.pop_next_queued_control_rx_item()? {
4509 self.process_rx_queue_item(item)?;
4510 did_any = true;
4511 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms) {
4514 break;
4515 }
4516 }
4517 Ok(did_any)
4518 }
4519
4520 fn eligible_side_ids_locked(
4521 &self,
4522 st: &RouterInner,
4523 src: Option<RouterSideId>,
4524 ty: Option<DataType>,
4525 restrict_link_local: bool,
4526 ) -> Vec<RouterSideId> {
4527 st.sides
4528 .iter()
4529 .enumerate()
4530 .filter_map(|(side_id, side)| {
4531 let side = side.as_ref()?;
4532 if restrict_link_local && !side.opts.link_local_enabled {
4533 return None;
4534 }
4535 if self.route_allowed_locked(st, src, ty, side_id) {
4536 Some(side_id)
4537 } else {
4538 None
4539 }
4540 })
4541 .collect()
4542 }
4543
4544 fn apply_route_selection_locked(
4545 &self,
4546 st: &mut RouterInner,
4547 src: Option<RouterSideId>,
4548 mut sides: Vec<RouterSideId>,
4549 origin: RouteSelectionOrigin,
4550 ) -> Vec<RouterSideId> {
4551 if sides.len() <= 1 {
4552 return sides;
4553 }
4554
4555 let selection_mode = st.source_route_modes.get(&src).copied();
4556 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
4557 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
4558 }
4559
4560 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
4561 RouteSelectionMode::Fanout => sides,
4562 RouteSelectionMode::Weighted => {
4563 crate::small_sort::sort(&mut sides);
4564 let total_weight = sides.iter().fold(0_u64, |acc, side| {
4565 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
4566 });
4567 if total_weight == 0 {
4568 return Vec::new();
4569 }
4570 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4571 let pick = *cursor % total_weight;
4572 *cursor = cursor.wrapping_add(1);
4573 let mut remaining = pick;
4574 for side in sides {
4575 let weight =
4576 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
4577 if remaining < weight {
4578 return vec![side];
4579 }
4580 remaining -= weight;
4581 }
4582 Vec::new()
4583 }
4584 RouteSelectionMode::Failover => {
4585 sides.sort_by_key(|side| {
4586 (
4587 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
4588 *side,
4589 )
4590 });
4591 sides.truncate(1);
4592 sides
4593 }
4594 }
4595 }
4596
4597 fn apply_adaptive_discovery_selection_locked(
4598 &self,
4599 st: &mut RouterInner,
4600 src: Option<RouterSideId>,
4601 mut sides: Vec<RouterSideId>,
4602 ) -> Vec<RouterSideId> {
4603 crate::small_sort::sort(&mut sides);
4604 let mut unmeasured: Vec<_> = sides
4605 .iter()
4606 .copied()
4607 .filter(|side| !st.adaptive_route_stats.contains_key(side))
4608 .collect();
4609 if !unmeasured.is_empty() {
4610 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4611 let pick = (*cursor as usize) % unmeasured.len();
4612 *cursor = cursor.wrapping_add(1);
4613 return vec![unmeasured.swap_remove(pick)];
4614 }
4615
4616 let now_ms = self.clock.now_ms();
4617 let total_weight = sides.iter().fold(0_u64, |acc, side| {
4618 acc + st
4619 .adaptive_route_stats
4620 .get(side)
4621 .map(|stats| stats.weight(now_ms))
4622 .unwrap_or(1)
4623 });
4624 if total_weight == 0 {
4625 sides.truncate(1);
4626 return sides;
4627 }
4628
4629 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4630 let pick = *cursor % total_weight;
4631 *cursor = cursor.wrapping_add(1);
4632 let mut remaining = pick;
4633 for side in sides {
4634 let weight = st
4635 .adaptive_route_stats
4636 .get(&side)
4637 .map(|stats| stats.weight(now_ms))
4638 .unwrap_or(1);
4639 if remaining < weight {
4640 return vec![side];
4641 }
4642 remaining -= weight;
4643 }
4644 Vec::new()
4645 }
4646
4647 fn record_side_tx_sample(
4648 &self,
4649 side: RouterSideId,
4650 bytes: usize,
4651 started_ms: u64,
4652 ended_ms: u64,
4653 ) {
4654 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
4655 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
4656 .min(u128::from(u64::MAX)) as u64;
4657 let mut st = self.state.lock();
4658 st.adaptive_route_stats
4659 .entry(side)
4660 .or_default()
4661 .observe(bytes, sample_bps, ended_ms);
4662 }
4663
4664 pub fn note_side_link_probe_sample(
4669 &self,
4670 side: RouterSideId,
4671 bytes: usize,
4672 duration_ms: u64,
4673 ) -> TelemetryResult<()> {
4674 {
4675 let st = self.state.lock();
4676 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
4677 }
4678 let ended_ms = self.clock.now_ms();
4679 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
4680 Ok(())
4681 }
4682
4683 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
4684 match data {
4685 RouterItem::Packet(pkt) => Ok(wire_format::packet_wire_size(pkt)),
4686 RouterItem::Packed(bytes) => Ok(bytes.len()),
4687 }
4688 }
4689
4690 #[inline]
4700 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
4701 match data {
4702 RouterItem::Packet(pkt) => {
4703 if !Self::is_end_to_end_ack_packet(pkt) {
4704 return Ok(None);
4705 }
4706 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4707 }
4708 RouterItem::Packed(bytes) => {
4709 if wire_format::peek_routing_frame_info(bytes.as_ref())
4710 .ok()
4711 .is_some_and(|frame| {
4712 frame.envelope.ty != DataType::ReliableAck || frame.ack_only()
4713 })
4714 {
4715 return Ok(None);
4716 }
4717 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4718 if !Self::is_end_to_end_ack_packet(&pkt) {
4719 return Ok(None);
4720 }
4721 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4722 }
4723 }
4724 }
4725
4726 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
4727 if payload.len() != 8 {
4728 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
4729 }
4730 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
4731 }
4732
4733 #[inline]
4734 fn sender_hash(sender: &str) -> u64 {
4735 if let Some(address) = sender
4736 .strip_prefix("@addr:")
4737 .and_then(|v| v.parse::<u32>().ok())
4738 {
4739 return u64::from(address);
4740 }
4741 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
4742 }
4743
4744 #[inline]
4745 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
4746 let hash = Self::sender_hash(hostname);
4747 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
4748 if address == 0 {
4749 address = 1;
4750 }
4751 address
4752 }
4753
4754 fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
4759 let Some(address) = sender
4760 .strip_prefix("@addr:")
4761 .and_then(|value| value.parse::<u32>().ok())
4762 else {
4763 return sender.to_string();
4764 };
4765 Self::hostname_for_address_locked(st, address).unwrap_or_else(|| sender.to_string())
4766 }
4767
4768 fn hostname_for_address_locked(st: &RouterInner, address: NodeAddress) -> Option<String> {
4775 Self::hostname_for_address_borrowed_locked(st, address).map(str::to_string)
4776 }
4777
4778 fn hostname_for_address_borrowed_locked<'a>(
4779 st: &'a RouterInner,
4780 address: NodeAddress,
4781 ) -> Option<&'a str> {
4782 if let Some(hostname) = st.address_by_value.get(&address) {
4783 return Some(hostname.as_str());
4784 }
4785
4786 let mut found: Option<&'a str> = None;
4787 let mut collision = false;
4788 let mut consider = |candidate: &'a str| {
4789 if sender_address_u32(candidate) != address {
4790 return;
4791 }
4792 match found {
4793 None => found = Some(candidate),
4794 Some(existing) if existing == candidate => {}
4795 Some(_) => collision = true,
4799 }
4800 };
4801
4802 for entry in st.address_book.values() {
4803 consider(entry.hostname.as_ref());
4804 }
4805 #[cfg(feature = "discovery")]
4806 for route in st.discovery_routes.values() {
4807 for (announcer, state) in &route.announcers {
4808 consider(announcer);
4809 for board in state.topology_boards.iter() {
4810 consider(&board.sender_id);
4811 for peer in &board.connections {
4817 consider(peer);
4818 }
4819 }
4820 }
4821 }
4822 if collision { None } else { found }
4823 }
4824
4825 #[cfg(feature = "discovery")]
4826 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
4827 match mode {
4828 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
4829 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
4830 _ => AddressAssignmentMode::Dynamic,
4831 }
4832 }
4833
4834 #[cfg(feature = "discovery")]
4835 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
4836 let rank = match entry.mode {
4837 AddressAssignmentMode::Static(_) => 2,
4838 AddressAssignmentMode::Requested(_) => 1,
4839 AddressAssignmentMode::Dynamic => 0,
4840 };
4841 (
4842 rank,
4843 core::cmp::Reverse(entry.birth_ms),
4844 u64::MAX - entry.owner_hash,
4845 )
4846 }
4847
4848 #[cfg(feature = "discovery")]
4849 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
4850 (
4851 core::cmp::Reverse(entry.birth_ms),
4852 u64::MAX - entry.owner_hash,
4853 )
4854 }
4855
4856 #[cfg(feature = "discovery")]
4857 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
4858 let mut candidate = if seed == 0 { 1 } else { seed };
4859 for _ in 0..u32::MAX {
4860 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
4861 return candidate;
4862 }
4863 candidate = candidate.wrapping_add(1);
4864 if candidate == 0 {
4865 candidate = 1;
4866 }
4867 }
4868 1
4869 }
4870
4871 #[cfg(feature = "discovery")]
4872 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
4873 if !st.address_book.contains_key(base) && st.local_address.hostname.as_ref() != base {
4874 return base.to_string();
4875 }
4876 let stem = if base.is_empty() { "node" } else { base };
4877 let mut candidate = format!("{stem}-{owner_hash:08x}");
4878 let mut suffix = 1u32;
4879 while st.address_book.contains_key(&candidate)
4880 || st.local_address.hostname.as_ref() == candidate
4881 {
4882 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
4883 suffix = suffix.saturating_add(1);
4884 }
4885 candidate
4886 }
4887
4888 fn update_local_identity_locked(
4889 &self,
4890 st: &mut RouterInner,
4891 mut local: AddressBookEntry,
4892 reason: AddressChangeReason,
4893 spare: Option<Box<AddressBookEntry>>,
4894 ) -> TelemetryResult<AddressChange> {
4895 let old = st.local_address.clone();
4896 let mut record = match st.address_book.remove(old.hostname.as_ref()) {
4899 Some(record) => record,
4900 None => spare.ok_or(TelemetryError::Io("local identity changed"))?,
4901 };
4902 st.address_by_value.remove(&old.address);
4903 local.last_seen_ms = self.clock.now_ms();
4904 st.address_by_value
4905 .insert(local.address, local.hostname.to_string());
4906 *record = local.clone();
4907 st.address_book.insert(local.hostname.to_string(), record);
4908 st.local_address = local.clone();
4909 *self.sender.lock() = local.hostname.clone();
4910 #[cfg(feature = "discovery")]
4911 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
4912 Ok(AddressChange {
4913 old_hostname: old.hostname,
4914 new_hostname: local.hostname,
4915 old_address: old.address,
4916 new_address: local.address,
4917 reason,
4918 })
4919 }
4920
4921 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
4922 for handler in self.cfg.address_change_handlers().iter() {
4923 (handler.handler)(change.clone())?;
4924 }
4925 Ok(())
4926 }
4927
4928 #[cfg(feature = "discovery")]
4929 fn local_address_advertisement(
4930 &self,
4931 reachable_endpoints: Vec<DataEndpoint>,
4932 reachable_network_variables: Vec<DataType>,
4933 reachable_timesync_sources: Vec<String>,
4934 link_capabilities: discovery::LinkCapabilities,
4935 state: u8,
4936 ) -> discovery::AddressAdvertisement {
4937 let st = self.state.lock();
4938 discovery::AddressAdvertisement {
4939 hostname: st.local_address.hostname.to_string(),
4940 address: st.local_address.address,
4941 requested_address: st.local_address.requested_address,
4942 mode: st.local_address.mode.mode_code(),
4943 state,
4944 birth_ms: st.local_address.birth_ms,
4945 owner_hash: st.local_address.owner_hash,
4946 reachable_endpoints,
4947 reachable_network_variables,
4948 reachable_timesync_sources,
4949 link_capabilities,
4950 }
4951 }
4952
4953 #[cfg(feature = "discovery")]
4954 fn prepare_address_record(
4955 st: &RouterInner,
4956 ad: &discovery::AddressAdvertisement,
4957 now_ms: u64,
4958 ) -> TelemetryResult<PreparedAddressRecords> {
4959 let remote = AddressBookEntry {
4960 hostname: Arc::from(ad.hostname.as_str()),
4961 address: ad.address,
4962 requested_address: ad.requested_address,
4963 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
4964 birth_ms: ad.birth_ms,
4965 owner_hash: ad.owner_hash,
4966 last_seen_ms: now_ms,
4967 };
4968 let peer = if st
4969 .address_book
4970 .get(ad.hostname.as_str())
4971 .is_some_and(|entry| entry.owner_hash == ad.owner_hash)
4972 {
4973 None
4974 } else {
4975 Some(crate::peer_record::try_box(remote.clone())?)
4976 };
4977 let local = &st.local_address;
4978 let moves_local = remote.owner_hash != local.owner_hash
4979 && ((remote.hostname == local.hostname
4980 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(local))
4981 || (remote.address == local.address
4982 && Self::address_winner_pref(&remote) > Self::address_winner_pref(local)));
4983 let spare = if moves_local && !st.address_book.contains_key(local.hostname.as_ref()) {
4984 Some(crate::peer_record::try_box(local.clone())?)
4985 } else {
4986 None
4987 };
4988 Ok(PreparedAddressRecords { peer, local: spare })
4989 }
4990
4991 #[cfg(all(feature = "discovery", feature = "timesync", test))]
4992 fn ingest_address_advertisement(
4993 &self,
4994 ad: discovery::AddressAdvertisement,
4995 ) -> TelemetryResult<bool> {
4996 let prepared = Self::prepare_address_record(&self.state.lock(), &ad, self.clock.now_ms())?;
4997 self.ingest_prepared_address(ad, prepared)
4998 }
4999
5000 #[cfg(feature = "discovery")]
5001 fn ingest_prepared_address(
5002 &self,
5003 ad: discovery::AddressAdvertisement,
5004 prepared: PreparedAddressRecords,
5005 ) -> TelemetryResult<bool> {
5006 let PreparedAddressRecords {
5007 peer: mut fresh_peer,
5008 local: mut local_spare,
5009 } = prepared;
5010 let now_ms = self.clock.now_ms();
5011 let mut remote = match fresh_peer.as_ref() {
5012 Some(record) => record.as_ref().clone(),
5013 None => AddressBookEntry {
5014 hostname: Arc::from(ad.hostname.as_str()),
5015 address: ad.address,
5016 requested_address: ad.requested_address,
5017 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
5018 birth_ms: ad.birth_ms,
5019 owner_hash: ad.owner_hash,
5020 last_seen_ms: now_ms,
5021 },
5022 };
5023 remote.last_seen_ms = now_ms;
5024 let mut change = None;
5025 let mut changed = false;
5026 {
5027 let mut st = self.state.lock();
5028 let local = st.local_address.clone();
5029 if fresh_peer.is_none()
5033 && !st
5034 .address_book
5035 .get(remote.hostname.as_ref())
5036 .is_some_and(|entry| entry.owner_hash == remote.owner_hash)
5037 {
5038 return Err(TelemetryError::Io("peer identity changed"));
5039 }
5040 let hostname_conflict =
5041 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
5042 let address_conflict =
5043 remote.address == local.address && remote.owner_hash != local.owner_hash;
5044
5045 if hostname_conflict
5046 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
5047 {
5048 let mut next = local.clone();
5049 next.hostname = Arc::from(Self::unique_hostname_locked(
5050 &st,
5051 local.hostname.as_ref(),
5052 local.owner_hash,
5053 ));
5054 change = Some(self.update_local_identity_locked(
5055 &mut st,
5056 next,
5057 AddressChangeReason::HostnameConflict,
5058 local_spare.take(),
5059 )?);
5060 changed = true;
5061 } else if address_conflict
5062 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
5063 {
5064 let mut next = local.clone();
5065 let seed = next
5066 .requested_address
5067 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
5068 next.address = Self::allocate_free_address_locked(&st, seed);
5069 let reason = match next.mode {
5070 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
5071 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
5072 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
5073 };
5074 change = Some(self.update_local_identity_locked(
5075 &mut st,
5076 next,
5077 reason,
5078 local_spare.take(),
5079 )?);
5080 changed = true;
5081 } else if hostname_conflict {
5082 remote.hostname = Arc::from(Self::unique_hostname_locked(
5083 &st,
5084 remote.hostname.as_ref(),
5085 remote.owner_hash,
5086 ));
5087 changed = true;
5088 } else if address_conflict {
5089 let seed = remote
5090 .requested_address
5091 .max(Self::fallback_address_for_hostname(
5092 remote.hostname.as_ref(),
5093 ));
5094 remote.address = Self::allocate_free_address_locked(&st, seed);
5095 changed = true;
5096 }
5097
5098 let mut record = fresh_peer.take().unwrap_or_else(|| {
5099 st.address_book
5100 .remove(ad.hostname.as_str())
5101 .expect("known peer remains present")
5102 });
5103 *record = remote.clone();
5104 st.address_by_value.remove(&remote.address);
5105 st.address_by_value
5106 .insert(remote.address, remote.hostname.to_string());
5107 st.address_book.insert(remote.hostname.to_string(), record);
5108 }
5109 if let Some(change) = change {
5110 self.notify_address_change(change)?;
5111 }
5112 Ok(changed)
5113 }
5114
5115 #[inline]
5116 #[cfg(feature = "discovery")]
5117 fn is_end_to_end_ack_sender(sender: &str) -> bool {
5118 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
5119 }
5120
5121 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
5122 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
5123 && !ack_sender.is_empty()
5124 {
5125 return Some(Self::sender_hash(ack_sender));
5126 }
5127 None
5128 }
5129
5130 fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
5136 if pkt.data_type() != DataType::ReliableAck {
5137 return None;
5138 }
5139 pkt.wire_target_senders()
5140 .first()
5141 .copied()
5142 .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
5143 }
5144
5145 fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
5146 Self::end_to_end_ack_sender_hash(pkt).is_some()
5147 && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
5148 }
5149
5150 fn encode_end_to_end_ack_sender(&self) -> String {
5151 let sender = self.sender_arc();
5152 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
5153 }
5154
5155 #[cfg(feature = "discovery")]
5156 fn is_end_to_end_destination_sender(&self, sender: &str) -> bool {
5157 sender != self.sender_arc().as_ref() && !Self::is_end_to_end_ack_sender(sender)
5158 }
5159
5160 fn encode_p2p_payload(
5161 src_hostname: &str,
5162 src_address: NodeAddress,
5163 src_port: P2pPort,
5164 dst_port: P2pPort,
5165 payload: &[u8],
5166 ) -> TelemetryResult<crate::SharedBytes> {
5167 let host_len = u16::try_from(src_hostname.len())
5168 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
5169 let payload_len = u32::try_from(payload.len())
5170 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
5171 let mut out = Vec::with_capacity(
5172 15usize
5173 .saturating_add(src_hostname.len())
5174 .saturating_add(payload.len()),
5175 );
5176 out.push(1);
5177 out.extend_from_slice(&dst_port.to_le_bytes());
5178 out.extend_from_slice(&src_port.to_le_bytes());
5179 out.extend_from_slice(&src_address.to_le_bytes());
5180 out.extend_from_slice(&host_len.to_le_bytes());
5181 out.extend_from_slice(&payload_len.to_le_bytes());
5182 out.extend_from_slice(src_hostname.as_bytes());
5183 out.extend_from_slice(payload);
5184 Ok(out.into())
5185 }
5186
5187 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
5188 if payload.len() < 15 {
5189 return Err(TelemetryError::Unpack("p2p frame short"));
5190 }
5191 if payload[0] != 1 {
5192 return Err(TelemetryError::Unpack("p2p frame version"));
5193 }
5194 let destination_port =
5195 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
5196 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
5197 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
5198 let host_len =
5199 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
5200 let body_len =
5201 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
5202 let host_start = 15usize;
5203 let host_end = host_start.saturating_add(host_len);
5204 let body_end = host_end.saturating_add(body_len);
5205 if host_end > payload.len() || body_end != payload.len() {
5206 return Err(TelemetryError::Unpack("p2p frame length"));
5207 }
5208 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
5209 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
5210 Ok(P2pDecoded {
5211 source_hostname,
5212 source_address,
5213 source_port,
5214 destination_port,
5215 payload: &payload[host_end..body_end],
5216 })
5217 }
5218
5219 fn encode_p2p_stream_payload(
5220 flags: u8,
5221 source_stream_id: P2pStreamId,
5222 destination_stream_id: P2pStreamId,
5223 sequence: u32,
5224 payload: &[u8],
5225 ) -> TelemetryResult<crate::SharedBytes> {
5226 let payload_len = u32::try_from(payload.len())
5227 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
5228 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
5229 out.extend_from_slice(&P2P_STREAM_MAGIC);
5230 out.push(P2P_STREAM_VERSION);
5231 out.push(flags);
5232 out.extend_from_slice(&source_stream_id.to_le_bytes());
5233 out.extend_from_slice(&destination_stream_id.to_le_bytes());
5234 out.extend_from_slice(&sequence.to_le_bytes());
5235 out.extend_from_slice(&payload_len.to_le_bytes());
5236 out.extend_from_slice(payload);
5237 Ok(out.into())
5238 }
5239
5240 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
5241 if !payload.starts_with(&P2P_STREAM_MAGIC) {
5242 return Ok(None);
5243 }
5244 if payload.len() < 22 {
5245 return Err(TelemetryError::Unpack("p2p stream frame short"));
5246 }
5247 if payload[4] != P2P_STREAM_VERSION {
5248 return Err(TelemetryError::Unpack("p2p stream frame version"));
5249 }
5250 let flags = payload[5];
5251 let source_stream_id =
5252 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
5253 let destination_stream_id =
5254 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
5255 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
5256 let body_len =
5257 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
5258 let body_end = 22usize.saturating_add(body_len);
5259 if body_end != payload.len() {
5260 return Err(TelemetryError::Unpack("p2p stream frame length"));
5261 }
5262 Ok(Some(P2pStreamDecoded {
5263 flags,
5264 source_stream_id,
5265 destination_stream_id,
5266 sequence,
5267 payload: &payload[22..],
5268 }))
5269 }
5270
5271 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
5272 for _ in 0..u32::MAX {
5273 let id = st
5274 .local_services
5275 .as_ref()
5276 .expect("local services enabled")
5277 .next_p2p_stream_id
5278 .max(1);
5279 st.local_services
5280 .as_mut()
5281 .expect("local services enabled")
5282 .next_p2p_stream_id = st
5283 .local_services
5284 .as_ref()
5285 .expect("local services enabled")
5286 .next_p2p_stream_id
5287 .wrapping_add(1)
5288 .max(1);
5289 if !st
5290 .local_services
5291 .as_ref()
5292 .expect("local services enabled")
5293 .p2p_stream_sessions
5294 .contains_key(&id)
5295 {
5296 return id;
5297 }
5298 }
5299 0
5300 }
5301
5302 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
5303 if pkt.data_type() != DataType::P2pMessage {
5304 return Ok(());
5305 }
5306 let decoded = Self::decode_p2p_payload(pkt.payload())?;
5307 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
5308 return self.dispatch_p2p_stream_frame(&decoded, &stream);
5309 }
5310 let handlers = {
5311 let st = self.state.lock();
5312 st.local_services
5313 .as_ref()
5314 .expect("local services enabled")
5315 .p2p_port_handlers
5316 .get(&decoded.destination_port)
5317 .cloned()
5318 .unwrap_or_default()
5319 };
5320 for handler in handlers {
5321 (handler.handler)(P2pMessage {
5322 source_hostname: decoded.source_hostname,
5323 source_address: decoded.source_address,
5324 source_port: decoded.source_port,
5325 destination_port: decoded.destination_port,
5326 payload: decoded.payload,
5327 })?;
5328 }
5329 Ok(())
5330 }
5331
5332 fn dispatch_p2p_stream_frame(
5333 &self,
5334 msg: &P2pDecoded<'_>,
5335 frame: &P2pStreamDecoded<'_>,
5336 ) -> TelemetryResult<()> {
5337 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
5338 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, crate::SharedBytes)> = None;
5339 {
5340 let mut st = self.state.lock();
5341 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
5342 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
5343 let existing_id = st
5344 .local_services
5345 .as_ref()
5346 .expect("local services enabled")
5347 .p2p_stream_sessions
5348 .iter()
5349 .find_map(|(id, session)| {
5350 (session.peer_stream_id == frame.source_stream_id
5351 && session.local_port == msg.destination_port
5352 && session.peer_port == msg.source_port
5353 && session.peer_address == msg.source_address
5354 && session.peer_hostname.as_ref() == msg.source_hostname)
5355 .then_some(*id)
5356 });
5357 let local_id = if let Some(local_id) = existing_id {
5358 local_id
5359 } else {
5360 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
5361 if local_id == 0 {
5362 return Err(TelemetryError::Io("p2p stream id exhausted"));
5363 }
5364 st.local_services
5365 .as_mut()
5366 .expect("local services enabled")
5367 .p2p_stream_sessions
5368 .insert(
5369 local_id,
5370 P2pStreamSession {
5371 peer_hostname: peer_hostname.clone(),
5372 peer_address: msg.source_address,
5373 local_port: msg.destination_port,
5374 peer_port: msg.source_port,
5375 peer_stream_id: frame.source_stream_id,
5376 next_sequence: 1,
5377 connected: true,
5378 },
5379 );
5380 let handlers = st
5381 .local_services
5382 .as_ref()
5383 .expect("local services enabled")
5384 .p2p_stream_handlers
5385 .get(&msg.destination_port)
5386 .cloned()
5387 .unwrap_or_default();
5388 events.push(PendingP2pStreamEvent {
5389 handlers,
5390 kind: P2pStreamEventKind::Accepted,
5391 stream_id: local_id,
5392 peer_stream_id: frame.source_stream_id,
5393 peer_hostname: peer_hostname.clone(),
5394 peer_address: msg.source_address,
5395 local_port: msg.destination_port,
5396 peer_port: msg.source_port,
5397 });
5398 local_id
5399 };
5400 let dst = st
5401 .address_book
5402 .get(msg.source_hostname)
5403 .map(|entry| entry.as_ref().clone())
5404 .unwrap_or(AddressBookEntry {
5405 hostname: peer_hostname,
5406 address: msg.source_address,
5407 requested_address: msg.source_address,
5408 mode: AddressAssignmentMode::Dynamic,
5409 birth_ms: self.clock.now_ms(),
5410 owner_hash: Self::sender_hash(msg.source_hostname),
5411 last_seen_ms: self.clock.now_ms(),
5412 });
5413 let payload = Self::encode_p2p_stream_payload(
5414 P2P_STREAM_SYN | P2P_STREAM_ACK,
5415 local_id,
5416 frame.source_stream_id,
5417 0,
5418 &[],
5419 )?;
5420 reply = Some((dst, msg.source_port, msg.destination_port, payload));
5421 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
5422 if let Some(session) = st
5423 .local_services
5424 .as_mut()
5425 .expect("local services enabled")
5426 .p2p_stream_sessions
5427 .get_mut(&frame.destination_stream_id)
5428 {
5429 session.peer_stream_id = frame.source_stream_id;
5430 session.connected = true;
5431 let peer_hostname = session.peer_hostname.clone();
5432 let peer_address = session.peer_address;
5433 let local_port = session.local_port;
5434 let peer_port = session.peer_port;
5435 let handlers = st
5436 .local_services
5437 .as_ref()
5438 .expect("local services enabled")
5439 .p2p_stream_handlers
5440 .get(&local_port)
5441 .cloned()
5442 .unwrap_or_default();
5443 events.push(PendingP2pStreamEvent {
5444 handlers,
5445 kind: P2pStreamEventKind::Connected,
5446 stream_id: frame.destination_stream_id,
5447 peer_stream_id: frame.source_stream_id,
5448 peer_hostname,
5449 peer_address,
5450 local_port,
5451 peer_port,
5452 });
5453 }
5454 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
5455 let kind = if frame.flags & P2P_STREAM_RST != 0 {
5456 P2pStreamEventKind::Reset
5457 } else if frame.flags & P2P_STREAM_FIN != 0 {
5458 P2pStreamEventKind::Closed
5459 } else {
5460 P2pStreamEventKind::Data
5461 };
5462 let session_id = if frame.destination_stream_id != 0 {
5463 Some(frame.destination_stream_id)
5464 } else {
5465 st.local_services
5466 .as_ref()
5467 .expect("local services enabled")
5468 .p2p_stream_sessions
5469 .iter()
5470 .find_map(|(id, session)| {
5471 (session.peer_stream_id == frame.source_stream_id
5472 && session.local_port == msg.destination_port
5473 && session.peer_port == msg.source_port
5474 && session.peer_address == msg.source_address
5475 && session.peer_hostname.as_ref() == msg.source_hostname)
5476 .then_some(*id)
5477 })
5478 };
5479 if let Some(session_id) = session_id
5480 && let Some(session) = st
5481 .local_services
5482 .as_ref()
5483 .expect("local services enabled")
5484 .p2p_stream_sessions
5485 .get(&session_id)
5486 {
5487 let handlers = st
5488 .local_services
5489 .as_ref()
5490 .expect("local services enabled")
5491 .p2p_stream_handlers
5492 .get(&session.local_port)
5493 .cloned()
5494 .unwrap_or_default();
5495 events.push(PendingP2pStreamEvent {
5496 handlers,
5497 kind,
5498 stream_id: session_id,
5499 peer_stream_id: frame.source_stream_id,
5500 peer_hostname: session.peer_hostname.clone(),
5501 peer_address: session.peer_address,
5502 local_port: session.local_port,
5503 peer_port: session.peer_port,
5504 });
5505 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
5506 st.local_services
5507 .as_mut()
5508 .expect("local services enabled")
5509 .p2p_stream_sessions
5510 .remove(&session_id);
5511 }
5512 }
5513 }
5514 }
5515 if let Some((dst, dst_port, src_port, payload)) = reply {
5516 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
5517 }
5518 for pending in events {
5519 for handler in pending.handlers {
5520 (handler.handler)(P2pStreamEvent {
5521 kind: pending.kind,
5522 stream_id: pending.stream_id,
5523 peer_stream_id: pending.peer_stream_id,
5524 sequence: frame.sequence,
5525 peer_hostname: pending.peer_hostname.as_ref(),
5526 peer_address: pending.peer_address,
5527 local_port: pending.local_port,
5528 peer_port: pending.peer_port,
5529 payload: frame.payload,
5530 })?;
5531 }
5532 }
5533 Ok(())
5534 }
5535
5536 fn queue_duplicate_proxy_edge_receipt(
5539 &self,
5540 item: &RouterRxItem,
5541 packet_id: u64,
5542 ) -> TelemetryResult<()> {
5543 let Some(src) = item.src else { return Ok(()) };
5544 let eligible = |state: &RouterInner| {
5545 state.end_to_end_reliable_tx.contains_key(&packet_id)
5546 && state
5547 .reliable_return_routes
5548 .get(&packet_id)
5549 .is_some_and(|route| {
5550 route.side != src
5551 && route.local_admission_completed
5552 && !route.admission_cancelled
5553 && route.local_admission_token == 0
5554 && route.upstream_contract.is_some()
5555 })
5556 };
5557 if !eligible(&self.state.lock()) {
5558 return Ok(());
5559 }
5560 let local_sender = self.sender_arc();
5561 let local_hash = Self::sender_hash(local_sender.as_ref());
5562 let source_address = match &item.data {
5563 RouterItem::Packet(packet) => {
5564 packet.validate()?;
5565 if !packet.wire_target_senders().contains(&local_hash) {
5566 return Ok(());
5567 }
5568 sender_address_u32(packet.sender())
5569 }
5570 RouterItem::Packed(bytes) => {
5571 let address = crate::shared_bytes::with_slice(bytes, |view| {
5572 let frame = wire_format::peek_routing_frame_info(view)?;
5573 Ok::<_, TelemetryError>(
5574 frame
5575 .envelope
5576 .target_senders()
5577 .any(|target| target == local_hash)
5578 .then_some(frame.envelope.source_address),
5579 )
5580 })??;
5581 let Some(address) = address else {
5582 return Ok(());
5583 };
5584 address
5585 }
5586 };
5587 let publisher_hash = {
5591 let state = self.state.lock();
5592 if !eligible(&state) {
5593 return Ok(());
5594 }
5595 let retained = &state
5596 .end_to_end_reliable_tx
5597 .get(&packet_id)
5598 .expect("eligible owner")
5599 .data;
5600 let retained_address = match retained {
5601 RouterItem::Packet(packet) => sender_address_u32(packet.sender()),
5602 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
5603 wire_format::peek_routing_envelope(view).map(|envelope| envelope.source_address)
5604 })??,
5605 };
5606 if retained_address != source_address {
5607 return Ok(());
5608 }
5609 let Some((original_id, publisher)) = state
5610 .reliable_return_routes
5611 .get(&packet_id)
5612 .and_then(|route| route.upstream_contract)
5613 else {
5614 return Ok(());
5615 };
5616 if original_id != packet_id {
5617 return Ok(());
5618 };
5619 publisher
5620 };
5621 let ack = Packet::new_local(
5622 DataType::ReliableAck,
5623 message_meta(DataType::ReliableAck).endpoints_owned(),
5624 Arc::from(self.encode_end_to_end_ack_sender()),
5625 self.packet_timestamp_ms(),
5626 crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
5627 )?;
5628 let ack = self
5629 .attach_wire_contract_to_item(RouterItem::Packet(ack), &[local_hash, publisher_hash])?;
5630 let priority = self.item_service_priority(&ack)?;
5631 let mut state = self.state.lock();
5632 if !eligible(&state)
5633 || !Self::side_ref(&state, src).is_ok_and(|side| side.opts.egress_enabled)
5634 || state
5635 .reliable_return_routes
5636 .get(&packet_id)
5637 .and_then(|route| route.upstream_contract)
5638 != Some((packet_id, publisher_hash))
5639 {
5640 return Ok(());
5641 }
5642 let already_queued = state.transmit_queue.iter().any(|queued| {
5643 let RouterTxItem::ToSide { dst, data, .. } = &queued.item else {
5644 return false;
5645 };
5646 if *dst != src {
5647 return false;
5648 }
5649 match data {
5650 RouterItem::Packet(packet) => {
5651 packet.data_type() == DataType::ReliableAck
5652 && packet.payload() == packet_id.to_le_bytes()
5653 }
5654 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
5655 wire_format::scheduling_type(view) == Some(DataType::ReliableAck)
5656 && wire_format::unpack_packet(view)
5657 .is_ok_and(|packet| packet.payload() == packet_id.to_le_bytes())
5658 })
5659 .unwrap_or(false),
5660 }
5661 });
5662 if !already_queued {
5663 state.push_transmit(TxQueued {
5664 disposable: TX_DISPOSABLE,
5665 item: RouterTxItem::ToSide {
5666 src: None,
5667 dst: src,
5668 data: ack,
5669 },
5670 ignore_local: true,
5671 priority,
5672 delivery_state: QueuedDelivery::Untracked,
5673 accepted_type: DataType::ReliableAck.as_u32(),
5674 deferred_since_ms: 0,
5675 retry_after_ms: 0,
5676 })?;
5677 }
5678 Ok(())
5679 }
5680
5681 #[cfg(test)]
5684 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) -> bool {
5685 self.begin_reliable_local_admission(side, packet_id)
5686 .is_ok_and(|guard| guard.is_some())
5687 }
5688
5689 fn begin_reliable_local_admission(
5695 &self,
5696 side: RouterSideId,
5697 packet_id: u64,
5698 ) -> TelemetryResult<Option<ReliableLocalAdmissionGuard<'_>>> {
5699 self.begin_reliable_local_admission_with_parent(side, packet_id, None)
5700 }
5701
5702 fn begin_reliable_local_admission_with_parent(
5703 &self,
5704 side: RouterSideId,
5705 packet_id: u64,
5706 parent: Option<&ReliableLocalAdmissionGuard<'_>>,
5707 ) -> TelemetryResult<Option<ReliableLocalAdmissionGuard<'_>>> {
5708 let mut st = self.state.lock();
5709 if let Some(parent) = parent {
5710 parent.check_locked(&st)?;
5711 }
5712
5713 if !Self::side_ref(&st, side)?.opts.ingress_enabled {
5714 return Err(TelemetryError::HandlerError(
5715 "router: ingress disabled for side id",
5716 ));
5717 }
5718
5719 if st
5720 .reliable_return_routes
5721 .get(&packet_id)
5722 .is_some_and(|route| route.local_admission_token != 0)
5723 {
5724 return Err(TelemetryError::Io("rx packet busy"));
5725 }
5726 if st.end_to_end_reliable_tx.contains_key(&packet_id)
5730 && st
5731 .reliable_return_routes
5732 .get(&packet_id)
5733 .is_some_and(|route| route.upstream_contract.is_some())
5734 {
5735 return Ok(None);
5736 }
5737 let now = self.clock.now_ms();
5741 let retry_ms = runtime_reliable_retransmit_ms();
5742 let duplicate = st.recent_rx.contains(&packet_id)
5743 || st
5744 .reliable_return_routes
5745 .get(&packet_id)
5746 .is_some_and(|route| route.delivery_completed);
5747 let last_replay_ms = if duplicate {
5748 now.saturating_sub(retry_ms)
5749 } else {
5750 now
5751 };
5752 let replace_side = !duplicate
5758 || st
5759 .reliable_return_routes
5760 .get(&packet_id)
5761 .is_none_or(|route| {
5762 now.saturating_sub(route.last_replay_ms) >= retry_ms
5763 || !Self::side_ref(&st, route.side)
5764 .is_ok_and(|side| side.opts.egress_enabled)
5765 });
5766 if !Self::can_admit_reliable_return_route_growth_locked(&st, packet_id) {
5767 return Err(TelemetryError::Io("memory pressure"));
5768 }
5769 let token = st
5770 .local_admission_sequence
5771 .checked_add(1)
5772 .ok_or(TelemetryError::Io("local admission sequence exhausted"))?;
5773 if !Self::remember_reliable_return_route_locked(&mut st, packet_id) {
5774 return Err(TelemetryError::Io("reliable return route window full"));
5775 }
5776 st.local_admission_sequence = token;
5777 st.reliable_return_routes
5778 .entry(packet_id)
5779 .and_modify(|route| {
5780 if replace_side {
5781 route.side = side;
5782 }
5783 })
5784 .or_insert(ReliableReturnRouteState {
5785 side,
5786 delivery_completed: false,
5787 local_admission_completed: false,
5788 local_admission_token: 0,
5789 admission_cancelled: false,
5790 admission_committing: false,
5791 last_replay_ms,
5792 upstream_contract: None,
5793 });
5794 st.reliable_return_routes
5795 .get_mut(&packet_id)
5796 .expect("reserved route")
5797 .local_admission_token = token;
5798 Ok(Some(ReliableLocalAdmissionGuard {
5799 router: self,
5800 packet_id,
5801 token,
5802 }))
5803 }
5804
5805 fn can_admit_reliable_return_route_growth_locked(st: &RouterInner, packet_id: u64) -> bool {
5811 if st.reliable_return_routes.contains_key(&packet_id)
5812 || st.reliable_return_routes.len() >= runtime_reliable_max_return_routes().max(1)
5813 {
5814 return true;
5817 }
5818 st.shared_queue_bytes_used()
5819 .checked_add(RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES)
5820 .is_some_and(|bytes| bytes <= st.memory.max_queue_budget)
5821 }
5822
5823 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) -> bool {
5824 if !Self::can_admit_reliable_return_route_growth_locked(st, packet_id) {
5825 return false;
5826 }
5827 let cap = runtime_reliable_max_return_routes().max(1);
5828 st.reliable_return_route_order
5829 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
5830 while st.reliable_return_route_order.len() >= cap {
5831 let Some(index) = st.reliable_return_route_order.iter().position(|id| {
5832 !st.end_to_end_reliable_tx.contains_key(id)
5833 && !st
5834 .reliable_return_routes
5835 .get(id)
5836 .is_some_and(|route| route.local_admission_token != 0)
5837 }) else {
5838 return false;
5839 };
5840 if let Some(oldest) = st.reliable_return_route_order.remove(index) {
5841 st.reliable_return_routes.remove(&oldest);
5842 }
5843 }
5844 st.reliable_return_route_order.push_back(packet_id);
5845 true
5846 }
5847
5848 #[cfg(feature = "discovery")]
5849 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
5850 let cap = runtime_reliable_max_end_to_end_pending().max(1);
5851 st.end_to_end_reliable_tx_order
5852 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
5853 while st.end_to_end_reliable_tx_order.len() >= cap {
5854 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
5855 st.end_to_end_reliable_tx.remove(&oldest);
5856 } else {
5857 break;
5858 }
5859 }
5860 st.end_to_end_reliable_tx_order.push_back(packet_id);
5861 }
5862
5863 #[cfg(feature = "discovery")]
5864 fn expected_end_to_end_destinations_locked(
5865 &self,
5866 st: &RouterInner,
5867 data: &RouterItem,
5868 exclude: Option<RouterSideId>,
5869 managed: bool,
5870 resolve_proxy_segment: bool,
5871 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
5872 let (eps, ty) = self.item_route_info(data)?;
5873 let original_publisher = if resolve_proxy_segment {
5878 match data {
5879 RouterItem::Packet(packet) => {
5880 if let Some(address) = packet
5881 .sender()
5882 .strip_prefix("@addr:")
5883 .and_then(|value| value.parse::<u32>().ok())
5884 {
5885 Self::hostname_for_address_borrowed_locked(st, address)
5886 .map(Self::sender_hash)
5887 } else {
5888 Some(Self::sender_hash(packet.sender()))
5889 }
5890 }
5891 RouterItem::Packed(bytes) => {
5892 let address = crate::shared_bytes::with_slice(bytes, |view| {
5893 wire_format::peek_routing_envelope(view)
5894 .map(|envelope| envelope.source_address)
5895 })??;
5896 Self::hostname_for_address_borrowed_locked(st, address).map(Self::sender_hash)
5897 }
5898 }
5899 } else {
5900 None
5901 };
5902
5903 let frozen_targets = if resolve_proxy_segment {
5906 wire_format::TargetSenderSet::Hashes(&[])
5907 } else {
5908 match data {
5909 RouterItem::Packet(packet) => {
5910 wire_format::TargetSenderSet::Hashes(packet.wire_target_senders())
5911 }
5912 RouterItem::Packed(bytes) => {
5913 wire_format::peek_routing_envelope(bytes)?.target_sender_set()
5914 }
5915 }
5916 };
5917 let now_ms = self.clock.now_ms();
5918 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
5919 if managed || st.is_managed_variable(ty.as_u32()) {
5928 let mut out = BTreeMap::new();
5929 for (&side, route) in st.discovery_routes.iter() {
5930 if Some(side) == exclude {
5931 continue;
5932 }
5933 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5934 continue;
5935 }
5936 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5937 continue;
5938 };
5939 if restrict_link_local && !side_ref.opts.link_local_enabled {
5940 continue;
5941 }
5942 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5943 continue;
5944 }
5945 for (sender_id, sender_state) in route.announcers.iter() {
5946 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
5947 || !sender_state.reachable_network_variables.contains(&ty)
5948 || !self.is_end_to_end_destination_sender(sender_id)
5949 || original_publisher == Some(Self::sender_hash(sender_id))
5950 {
5951 continue;
5952 }
5953 let owner = Self::sender_hash(sender_id);
5954 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5955 continue;
5956 }
5957 out.insert(owner, side);
5958 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5959 return Ok(out);
5960 }
5961 }
5962 }
5963 return Ok(out);
5964 }
5965 let prefer_best_overlap =
5966 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
5967 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
5968 let mut best_overlap = 0usize;
5969 let limit = runtime_reliable_max_end_to_end_pending().max(1);
5970 let mut out = BTreeMap::new();
5971 for (&side, route) in st.discovery_routes.iter() {
5972 if Some(side) == exclude {
5973 continue;
5974 }
5975 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5976 continue;
5977 }
5978 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5979 continue;
5980 };
5981 if restrict_link_local && !side_ref.opts.link_local_enabled {
5982 continue;
5983 }
5984 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5985 continue;
5986 }
5987 for sender_state in route.announcers.values() {
5988 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5989 continue;
5990 }
5991 for board in sender_state.topology_boards.iter() {
5992 if !self.is_end_to_end_destination_sender(&board.sender_id)
5993 || original_publisher == Some(Self::sender_hash(&board.sender_id))
5994 {
5995 continue;
5996 }
5997 let owner = Self::sender_hash(&board.sender_id);
5998 if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5999 continue;
6000 }
6001 let overlap = Self::endpoint_overlap_count(
6002 board.reachable_endpoints.iter().copied(),
6003 &scoring_eps,
6004 );
6005 if overlap > 0 {
6006 if prefer_best_overlap {
6007 if overlap < best_overlap {
6008 continue;
6009 }
6010 if overlap > best_overlap {
6011 out.clear();
6012 best_overlap = overlap;
6013 }
6014 if out.len() >= limit {
6018 continue;
6019 }
6020 }
6021 out.insert(owner, side);
6022 if !prefer_best_overlap && out.len() >= limit {
6023 return Ok(out);
6024 }
6025 }
6026 }
6027 }
6028 }
6029
6030 Ok(out)
6031 }
6032
6033 #[cfg(feature = "discovery")]
6034 #[allow(clippy::too_many_arguments)]
6035 fn discovered_route_candidates_locked(
6036 &self,
6037 st: &RouterInner,
6038 exclude: Option<RouterSideId>,
6039 ty: DataType,
6040 eps: &[DataEndpoint],
6041 target_senders: wire_format::TargetSenderSet<'_>,
6042 prefer_nonlocal: bool,
6043 preferred_timesync_source: Option<&str>,
6044 ) -> Vec<DiscoveryCandidateMatch> {
6045 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
6046 let now_ms = self.clock.now_ms();
6047 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
6048 let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
6049 exclude != Some(side)
6050 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6051 && route.reachable_network_variables.contains(&ty)
6052 });
6053 let mut out = Vec::new();
6054 let mut unique_endpoint_fallbacks = Vec::new();
6055
6056 for (&side, route) in st.discovery_routes.iter() {
6057 if exclude == Some(side)
6058 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
6059 {
6060 continue;
6061 }
6062 if restrict_link_local
6063 && st
6064 .sides
6065 .get(side)
6066 .and_then(|s| s.as_ref())
6067 .map(|s| !s.opts.link_local_enabled)
6068 .unwrap_or(true)
6069 {
6070 continue;
6071 }
6072 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
6073 continue;
6074 }
6075 if !target_senders.is_empty()
6076 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
6077 {
6078 if scoring_eps
6079 .iter()
6080 .copied()
6081 .any(|ep| route.reachable.contains(&ep))
6082 {
6083 unique_endpoint_fallbacks.push(side);
6084 }
6085 continue;
6086 }
6087 if !target_senders.is_empty() {
6088 out.push(DiscoveryCandidateMatch {
6089 side,
6090 overlap: usize::MAX,
6091 });
6092 continue;
6093 }
6094 if has_variable_owner {
6095 if route.reachable_network_variables.contains(&ty) {
6096 out.push(DiscoveryCandidateMatch {
6097 side,
6098 overlap: usize::MAX,
6099 });
6100 }
6101 continue;
6102 }
6103 if preferred_timesync_source
6104 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
6105 {
6106 out.push(DiscoveryCandidateMatch {
6107 side,
6108 overlap: usize::MAX,
6109 });
6110 continue;
6111 }
6112 let overlap =
6113 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
6114 if overlap > 0 {
6115 out.push(DiscoveryCandidateMatch { side, overlap });
6116 }
6117 }
6118 if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
6119 out.push(DiscoveryCandidateMatch {
6120 side: unique_endpoint_fallbacks[0],
6121 overlap: usize::MAX,
6122 });
6123 }
6124 out
6125 }
6126
6127 #[cfg(feature = "discovery")]
6133 fn retain_shortest_discovery_candidates_locked(
6134 st: &RouterInner,
6135 matches: &mut Vec<DiscoveryCandidateMatch>,
6136 endpoints: &[DataEndpoint],
6137 target_senders: wire_format::TargetSenderSet<'_>,
6138 now_ms: u64,
6139 ) {
6140 if matches.len() <= 1 || endpoints.is_empty() {
6141 return;
6142 }
6143
6144 let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
6145 let route = st.discovery_routes.get(&side)?;
6146 route
6147 .announcers
6148 .iter()
6149 .filter(|(_, state)| {
6150 now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6151 })
6152 .filter_map(|(announcer, state)| {
6153 let is_target = |board: &TopologyBoardNode| {
6154 target_sender
6155 .map(|target| Self::sender_hash(&board.sender_id) == target)
6156 .unwrap_or_else(|| {
6157 endpoints
6158 .iter()
6159 .any(|endpoint| board.reachable_endpoints.contains(endpoint))
6160 })
6161 };
6162 if !state.topology_boards.iter().any(is_target) {
6165 return if let Some(target) = target_sender {
6166 state
6167 .topology_boards
6168 .iter()
6169 .any(|board| Self::sender_hash(&board.sender_id) == target)
6170 .then_some(usize::MAX / 2)
6171 } else {
6172 state
6173 .reachable
6174 .iter()
6175 .any(|endpoint| endpoints.contains(endpoint))
6176 .then_some(usize::MAX / 2)
6177 };
6178 }
6179
6180 let mut visited: BTreeSet<&str> = BTreeSet::new();
6181 let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
6182 while let Some((sender, distance)) = pending.pop_front() {
6183 if !visited.insert(sender) {
6184 continue;
6185 }
6186 for board in state.topology_boards.iter() {
6187 if board.sender_id == sender {
6188 if is_target(board) {
6189 return Some(distance);
6190 }
6191 for peer in &board.connections {
6192 if !visited.contains(peer.as_str()) {
6193 pending.push_back((peer.as_str(), distance + 1));
6194 }
6195 }
6196 } else if board.connections.iter().any(|peer| peer == sender)
6197 && !visited.contains(board.sender_id.as_str())
6198 {
6199 pending.push_back((board.sender_id.as_str(), distance + 1));
6200 }
6201 }
6202 }
6203 None
6204 })
6205 .min()
6206 };
6207
6208 let route_targets = target_senders
6209 .iter()
6210 .map(Some)
6211 .chain(core::iter::once(None).filter(|_| target_senders.is_empty()));
6212 let mut selected = BTreeSet::new();
6213 for target in route_targets {
6214 let distances: Vec<(RouterSideId, usize)> = matches
6215 .iter()
6216 .filter_map(|candidate| {
6217 route_distance(candidate.side, target)
6218 .map(|distance| (candidate.side, distance))
6219 })
6220 .collect();
6221 let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
6222 continue;
6223 };
6224 selected.extend(
6225 distances
6226 .into_iter()
6227 .filter(|(_, distance)| *distance == best)
6228 .map(|(side, _)| side),
6229 );
6230 }
6231 if !selected.is_empty() {
6232 matches.retain(|candidate| selected.contains(&candidate.side));
6233 }
6234 }
6235
6236 #[cfg(feature = "discovery")]
6240 fn select_endpoint_role_sides_locked(
6241 &self,
6242 st: &mut RouterInner,
6243 exclude: Option<RouterSideId>,
6244 endpoints: &[DataEndpoint],
6245 matches: &[DiscoveryCandidateMatch],
6246 now_ms: u64,
6247 ) -> Vec<RouterSideId> {
6248 let mut selected = Vec::new();
6249 for endpoint in endpoints {
6250 let mut candidates: Vec<_> = matches
6251 .iter()
6252 .copied()
6253 .filter(|candidate| {
6254 st.discovery_routes
6255 .get(&candidate.side)
6256 .is_some_and(|route| route.reachable.contains(endpoint))
6257 })
6258 .collect();
6259 Self::retain_shortest_discovery_candidates_locked(
6260 st,
6261 &mut candidates,
6262 core::slice::from_ref(endpoint),
6263 wire_format::TargetSenderSet::Hashes(&[]),
6264 now_ms,
6265 );
6266 if candidates
6267 .iter()
6268 .any(|candidate| selected.contains(&candidate.side))
6269 {
6270 continue;
6271 }
6272 let sides = self.apply_route_selection_locked(
6273 st,
6274 exclude,
6275 candidates
6276 .into_iter()
6277 .map(|candidate| candidate.side)
6278 .collect(),
6279 RouteSelectionOrigin::Discovered,
6280 );
6281 for side in sides {
6282 if !selected.contains(&side) {
6283 selected.push(side);
6284 }
6285 }
6286 }
6287 selected
6288 }
6289
6290 #[cfg(feature = "discovery")]
6291 fn select_discovered_candidate_sides_locked(
6292 &self,
6293 st: &mut RouterInner,
6294 packet: (&RouterItem, DataType),
6295 exclude: Option<RouterSideId>,
6296 target_senders: wire_format::TargetSenderSet<'_>,
6297 prefer_best_overlap: bool,
6298 matches: Vec<DiscoveryCandidateMatch>,
6299 ) -> Vec<RouterSideId> {
6300 let (data, ty) = packet;
6301 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
6302 || !target_senders.is_empty()
6303 || (!self.cfg.role.local_services() && is_reliable_type(ty))
6304 {
6305 RouteSelectionOrigin::Flood
6306 } else {
6307 RouteSelectionOrigin::Discovered
6308 };
6309
6310 let mut matches = matches;
6311 if matches.iter().any(|m| m.overlap == usize::MAX) {
6312 matches.retain(|m| m.overlap == usize::MAX);
6313 }
6314
6315 let selected: Vec<RouterSideId> = if prefer_best_overlap {
6316 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
6317 matches
6318 .into_iter()
6319 .filter(|m| m.overlap == best_overlap)
6320 .map(|m| m.side)
6321 .collect()
6322 } else {
6323 matches.into_iter().map(|m| m.side).collect()
6324 };
6325
6326 let selected = if st.forwarded_receipts.is_some() && is_reliable_type(ty) {
6327 self.filter_end_to_end_satisfied_sides_locked(
6328 st,
6329 data,
6330 selected,
6331 &self
6332 .item_route_info(data)
6333 .map(|info| info.0)
6334 .unwrap_or_default(),
6335 ty,
6336 )
6337 } else {
6338 selected
6339 };
6340
6341 let has_distributed_variable_owners = selected.iter().any(|side| {
6345 st.discovery_routes
6346 .get(side)
6347 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
6348 });
6349 if has_distributed_variable_owners {
6350 selected
6351 } else {
6352 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
6353 }
6354 }
6355
6356 fn note_end_to_end_acked_destination_locked(
6357 st: &mut RouterInner,
6358 packet_id: u64,
6359 sender_hash: u64,
6360 ) {
6361 let cache = st
6362 .forwarded_receipts
6363 .get_or_insert_with(|| Box::new(ForwardedReceiptCache::default()));
6364 let entry_cap = crate::config::runtime_reliable_max_end_to_end_ack_cache().max(1);
6365 cache
6366 .order
6367 .retain(|id| cache.destinations.contains_key(id) && *id != packet_id);
6368 while cache.order.len() >= entry_cap {
6369 if let Some(oldest) = cache.order.pop_front() {
6370 cache.destinations.remove(&oldest);
6371 } else {
6372 break;
6373 }
6374 }
6375 cache.order.push_back(packet_id);
6376
6377 let acked = cache.destinations.entry(packet_id).or_default();
6378 let sender_cap = crate::config::runtime_reliable_max_end_to_end_pending().max(1);
6379 if acked.len() < sender_cap || acked.contains(&sender_hash) {
6380 acked.insert(sender_hash);
6381 }
6382 }
6383
6384 #[cfg(feature = "discovery")]
6385 fn filter_end_to_end_satisfied_sides_locked(
6386 &self,
6387 st: &RouterInner,
6388 data: &RouterItem,
6389 sides: Vec<RouterSideId>,
6390 eps: &[crate::DataEndpoint],
6391 ty: crate::DataType,
6392 ) -> Vec<RouterSideId> {
6393 if !is_reliable_type(ty)
6394 || Self::reliable_control_target_packet_id(data)
6395 .ok()
6396 .flatten()
6397 .is_some()
6398 {
6399 return sides;
6400 }
6401 let packet_id = match data {
6402 RouterItem::Packet(pkt) => pkt.packet_id(),
6403 RouterItem::Packed(bytes) => match wire_format::packet_id_from_wire(bytes.as_ref()) {
6404 Ok(packet_id) => packet_id,
6405 Err(TelemetryError::Unpack("reliable control frame")) => return sides,
6406 Err(_) => return sides,
6407 },
6408 };
6409 let Some(acked) = st
6410 .forwarded_receipts
6411 .as_ref()
6412 .and_then(|cache| cache.destinations.get(&packet_id))
6413 else {
6414 return sides;
6415 };
6416 let now_ms = self.clock.now_ms();
6417 let mut filtered = Vec::new();
6418 for side in sides {
6419 let Some(route) = st.discovery_routes.get(&side) else {
6420 filtered.push(side);
6421 continue;
6422 };
6423 let mut still_pending = false;
6424 let mut had_destination_board = false;
6425 for sender_state in route.announcers.values() {
6426 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6427 continue;
6428 }
6429 for board in sender_state.topology_boards.iter() {
6430 if !self.is_end_to_end_destination_sender(&board.sender_id) {
6431 continue;
6432 }
6433 had_destination_board = true;
6434 let sender_hash = Self::sender_hash(&board.sender_id);
6435 if acked.contains(&sender_hash) {
6436 continue;
6437 }
6438 if eps
6439 .iter()
6440 .copied()
6441 .any(|ep| board.reachable_endpoints.contains(&ep))
6442 {
6443 still_pending = true;
6444 break;
6445 }
6446 still_pending = true;
6449 break;
6450 }
6451 if still_pending {
6452 break;
6453 }
6454 }
6455 if still_pending || !had_destination_board {
6456 filtered.push(side);
6457 }
6458 }
6459 filtered
6460 }
6461
6462 #[cfg(feature = "discovery")]
6463 fn reconcile_forwarded_receipts_locked(&self, st: &mut RouterInner) {
6464 if st.forwarded_receipts.is_none() {
6465 return;
6466 }
6467 let mut active_senders = BTreeSet::new();
6468 for route in st.discovery_routes.values() {
6469 for sender_state in route.announcers.values() {
6470 for board in sender_state.topology_boards.iter() {
6471 if self.is_end_to_end_destination_sender(&board.sender_id) {
6472 active_senders.insert(Self::sender_hash(&board.sender_id));
6473 }
6474 }
6475 }
6476 }
6477 st.forwarded_receipts
6478 .as_mut()
6479 .expect("checked cache")
6480 .destinations
6481 .retain(|_, acked| {
6482 acked.retain(|sender_hash| active_senders.contains(sender_hash));
6483 !acked.is_empty()
6484 });
6485 }
6486
6487 #[cfg(feature = "discovery")]
6488 fn register_end_to_end_reliable_tx(
6489 &self,
6490 data: &RouterItem,
6491 destinations: Option<BTreeMap<u64, RouterSideId>>,
6492 ) -> TelemetryResult<bool> {
6493 let packet_id = Self::get_hash(data);
6494 #[cfg(feature = "std")]
6495 let completion_epoch = self.retry_birth();
6496 let now_ms = self.clock.now_ms();
6497 let ty = match data {
6498 RouterItem::Packet(pkt) => pkt.data_type(),
6499 RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
6500 };
6501 let mut st = self.state.lock();
6502 #[cfg(feature = "discovery")]
6503 let mut pending_destinations = match destinations {
6504 Some(destinations) => destinations,
6505 None => self.expected_end_to_end_destinations_locked(&st, data, None, false, false)?,
6506 };
6507 #[cfg(not(feature = "discovery"))]
6508 let mut pending_destinations = destinations.unwrap_or_default();
6509 let frozen = match data {
6513 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
6514 RouterItem::Packed(bytes) => {
6515 wire_format::peek_routing_envelope(bytes)?
6516 .target_senders()
6517 .len()
6518 != 0
6519 }
6520 };
6521 self.filter_trackable_end_to_end_destinations_locked(
6522 &st,
6523 ty,
6524 &mut pending_destinations,
6525 frozen,
6526 );
6527 let tracked_destinations = !pending_destinations.is_empty();
6528 if let Some(existing) = st.end_to_end_reliable_tx.get(&packet_id) {
6529 return Ok(!existing.waiting_for_predecessor);
6530 }
6531 if st.end_to_end_reliable_tx.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
6533 return Err(TelemetryError::Io("reliable window full"));
6534 }
6535 #[cfg(feature = "std")]
6536 if st
6537 .shared_queue_bytes_used()
6538 .saturating_add(PENDING_TYPE_METADATA_BYTES)
6539 .saturating_add(usize::from(completion_epoch != 0) * size_of::<u64>())
6540 > st.memory.max_queue_budget
6541 {
6542 return Err(TelemetryError::Io("memory pressure"));
6543 }
6544 let ordered_type =
6545 matches!(reliable_mode(ty), crate::ReliableMode::Ordered).then_some(ty.as_u32());
6546 let waiting_for_predecessor = ordered_type.is_some()
6547 && st
6548 .end_to_end_reliable_tx
6549 .values()
6550 .any(|sent| sent.ordered_type == ordered_type);
6551 #[cfg(feature = "std")]
6552 if reliable_trace_enabled() {
6553 eprintln!(
6554 "[sedsnet-reliable] register time={now_ms} type={} id={packet_id:#x} waiting={waiting_for_predecessor} targets={pending_destinations:?}",
6555 ty.as_u32()
6556 );
6557 }
6558 #[cfg(feature = "compression")]
6559 let retained = match data {
6560 RouterItem::Packed(bytes) => RouterItem::Packed(
6561 wire_format::payload_encoding_for_retention(
6562 bytes.as_ref(),
6563 st.memory.max_queue_budget,
6564 )?
6565 .map(crate::shared_bytes::convert)
6566 .unwrap_or_else(|| bytes.clone()),
6567 ),
6568 packet => packet.clone(),
6569 };
6570 #[cfg(not(feature = "compression"))]
6571 let retained = data.clone();
6572 #[cfg(feature = "compact-packet-store")]
6573 let mut retained = retained;
6574 #[cfg(feature = "compact-packet-store")]
6575 retained.park_for_queue()?;
6576 let sent = EndToEndReliableSent {
6577 #[cfg(feature = "std")]
6578 completion_epoch,
6579 #[cfg(feature = "std")]
6580 pending_type: ty.as_u32(),
6581 ordered_type,
6582 waiting_for_predecessor,
6583 data: retained,
6584 pending_destinations,
6585 tracked_destinations,
6586 last_send_ms: now_ms,
6587 retries: 0,
6588 queued: false,
6589 }
6590 .try_box()?;
6591 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
6592 st.end_to_end_reliable_tx.insert(packet_id, sent);
6593 Ok(!waiting_for_predecessor)
6594 }
6595
6596 fn release_ordered_successors(&self) -> TelemetryResult<()> {
6599 let ready = {
6600 let mut st = self.state.lock();
6601 let RouterInner {
6602 end_to_end_reliable_tx_order: order,
6603 end_to_end_reliable_tx: pending,
6604 ..
6605 } = &mut *st;
6606 let mut ready = Vec::new();
6607 for (index, id) in order.iter().copied().enumerate() {
6608 let Some(sent) = pending.get(&id) else {
6609 continue;
6610 };
6611 if !sent.waiting_for_predecessor {
6612 continue;
6613 }
6614 let Some(ty) = sent.ordered_type else {
6615 continue;
6616 };
6617 if order.iter().take(index).any(|earlier| {
6621 pending
6622 .get(earlier)
6623 .is_some_and(|sent| sent.ordered_type == Some(ty))
6624 }) {
6625 continue;
6626 }
6627 let sent = pending.get_mut(&id).expect("retained ordered entry");
6628 sent.waiting_for_predecessor = false;
6629 sent.last_send_ms = self.clock.now_ms();
6630 ready.push(id);
6631 }
6632 ready
6633 };
6634 for id in ready {
6635 self.queue_end_to_end_reliable_retransmit(id)?;
6636 }
6637 Ok(())
6638 }
6639
6640 #[cfg(feature = "discovery")]
6641 fn reconcile_end_to_end_reliable_destinations_locked(
6642 &self,
6643 st: &mut RouterInner,
6644 ) -> TelemetryResult<()> {
6645 self.reconcile_forwarded_receipts_locked(st);
6646 if st.end_to_end_reliable_tx.is_empty() {
6647 return Ok(());
6648 }
6649 let active_destinations = self.active_end_to_end_destinations_locked(st);
6650 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
6651
6652 for packet_id in packet_ids {
6653 let Some(data) = st
6654 .end_to_end_reliable_tx
6655 .get(&packet_id)
6656 .map(|sent| sent.data.clone())
6657 else {
6658 continue;
6659 };
6660 let excluded = st
6661 .reliable_return_routes
6662 .get(&packet_id)
6663 .filter(|route| route.upstream_contract.is_some())
6664 .map(|route| route.side);
6665 let expected =
6666 self.expected_end_to_end_destinations_locked(st, &data, excluded, false, false)?;
6667 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
6668 continue;
6669 };
6670 if !sent.tracked_destinations {
6671 continue;
6672 }
6673 for (sender, side) in &mut sent.pending_destinations {
6677 if let Some(next_side) = expected.get(sender).or_else(|| {
6678 active_destinations
6679 .get(sender)
6680 .filter(|side| Some(**side) != excluded)
6681 }) {
6682 *side = *next_side;
6683 }
6684 }
6685 }
6686
6687 Ok(())
6688 }
6689
6690 #[cfg(feature = "discovery")]
6691 fn active_end_to_end_destinations_locked(
6692 &self,
6693 st: &RouterInner,
6694 ) -> BTreeMap<u64, RouterSideId> {
6695 let now_ms = self.clock.now_ms();
6696 let mut out = BTreeMap::new();
6697 for (&side, route) in st.discovery_routes.iter() {
6698 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6699 continue;
6700 }
6701 for (sender_name, sender_state) in &route.announcers {
6702 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6703 continue;
6704 }
6705 if self.is_end_to_end_destination_sender(sender_name) {
6706 out.insert(Self::sender_hash(sender_name), side);
6707 }
6708 for board in sender_state.topology_boards.iter() {
6709 if !self.is_end_to_end_destination_sender(&board.sender_id) {
6710 continue;
6711 }
6712 out.insert(Self::sender_hash(&board.sender_id), side);
6713 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
6714 return out;
6715 }
6716 }
6717 }
6718 }
6719 out
6720 }
6721
6722 #[cfg(feature = "discovery")]
6723 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
6724 matches!(
6725 st.sides
6726 .get(side)
6727 .and_then(Option::as_ref)
6728 .map(|side| &side.tx_handler),
6729 Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
6730 )
6731 }
6732
6733 #[cfg(feature = "discovery")]
6734 fn filter_trackable_end_to_end_destinations_locked(
6735 &self,
6736 st: &RouterInner,
6737 ty: DataType,
6738 pending: &mut BTreeMap<u64, RouterSideId>,
6739 frozen: bool,
6740 ) {
6741 if frozen {
6742 return;
6743 }
6744 let now_ms = self.clock.now_ms();
6745 pending.retain(|_, side| {
6746 Self::side_supports_end_to_end_tracking_locked(st, *side)
6747 && (st.is_managed_variable(ty.as_u32())
6748 || is_reliable_type(ty)
6749 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
6750 });
6751 }
6752
6753 #[cfg(feature = "discovery")]
6754 fn side_has_multiple_announcers_locked(
6755 &self,
6756 st: &RouterInner,
6757 side: RouterSideId,
6758 now_ms: u64,
6759 ) -> bool {
6760 st.discovery_routes
6761 .get(&side)
6762 .map(|route| {
6763 route
6764 .announcers
6765 .values()
6766 .filter(|sender| {
6767 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6768 })
6769 .take(2)
6770 .count()
6771 > 1
6772 })
6773 .unwrap_or(false)
6774 }
6775
6776 #[cfg(not(feature = "discovery"))]
6777 fn side_has_multiple_announcers_locked(
6778 &self,
6779 _st: &RouterInner,
6780 _side: RouterSideId,
6781 _now_ms: u64,
6782 ) -> bool {
6783 false
6784 }
6785
6786 #[cfg(test)]
6789 fn queue_end_to_end_reliable_ack(
6790 &self,
6791 pkt: &Packet,
6792 called_from_queue: bool,
6793 ) -> TelemetryResult<()> {
6794 self.queue_end_to_end_reliable_ack_for_packet_id(
6795 pkt.packet_id(),
6796 pkt.sender(),
6797 called_from_queue,
6798 )
6799 }
6800
6801 fn queue_end_to_end_reliable_ack_for_packet_id(
6802 &self,
6803 packet_id: u64,
6804 original_sender: &str,
6805 called_from_queue: bool,
6806 ) -> TelemetryResult<()> {
6807 if !self.cfg.reliable_enabled() {
6808 return Ok(());
6809 }
6810 let original_sender_hash = {
6811 let mut st = self.state.lock();
6812 if let Some(route) = st.reliable_return_routes.get_mut(&packet_id) {
6817 route.delivery_completed = true;
6818 }
6819 Self::sender_hash(&Self::canonical_sender_locked(&st, original_sender))
6820 };
6821 self.queue_end_to_end_reliable_ack_for_sender_hash(
6822 packet_id,
6823 original_sender_hash,
6824 called_from_queue,
6825 )
6826 }
6827
6828 fn build_end_to_end_ack_data(
6829 &self,
6830 packet_id: u64,
6831 original_sender_hash: u64,
6832 ) -> TelemetryResult<RouterItem> {
6833 let ack_sender = self.encode_end_to_end_ack_sender();
6834 let ack = Packet::new_local(
6835 DataType::ReliableAck,
6836 message_meta(DataType::ReliableAck).endpoints_owned(),
6837 Arc::from(ack_sender),
6838 self.packet_timestamp_ms(),
6839 crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
6840 )?;
6841 let local_sender = self.sender_arc();
6842 self.attach_wire_contract_to_item(
6846 RouterItem::Packet(ack),
6847 &[
6848 Self::sender_hash(local_sender.as_ref()),
6849 original_sender_hash,
6850 ],
6851 )
6852 }
6853
6854 fn queue_admitted_end_to_end_ack(
6855 &self,
6856 packet_id: u64,
6857 original_sender: &str,
6858 called_from_queue: bool,
6859 admission: Option<&ReliableLocalAdmissionGuard<'_>>,
6860 ) -> TelemetryResult<()> {
6861 let Some(guard) = admission else {
6862 return self.queue_end_to_end_reliable_ack_for_packet_id(
6863 packet_id,
6864 original_sender,
6865 called_from_queue,
6866 );
6867 };
6868 if !self.cfg.reliable_enabled() {
6869 return Ok(());
6870 }
6871 if guard.packet_id != packet_id || !core::ptr::eq(guard.router, self) {
6872 return Err(TelemetryError::BadArg);
6873 }
6874 let publisher_hash = {
6875 let state = self.state.lock();
6876 guard.check_locked(&state)?;
6877 let dst = state
6878 .reliable_return_routes
6879 .get(&packet_id)
6880 .expect("checked lease")
6881 .side;
6882 if !Self::side_ref(&state, dst)?.opts.egress_enabled {
6885 return Ok(());
6886 }
6887 Self::sender_hash(&Self::canonical_sender_locked(&state, original_sender))
6888 };
6889 let data = self.build_end_to_end_ack_data(packet_id, publisher_hash)?;
6890 let priority = self.item_service_priority(&data)?;
6891 let ack_id = Self::get_hash(&data);
6894 let mut state = self.state.lock();
6895 guard.check_locked(&state)?;
6896 let dst = state
6897 .reliable_return_routes
6898 .get(&packet_id)
6899 .expect("checked lease")
6900 .side;
6901 if !Self::side_ref(&state, dst)?.opts.egress_enabled {
6902 return Ok(());
6903 }
6904 state.push_transmit(TxQueued {
6905 disposable: TX_DISPOSABLE,
6906 item: RouterTxItem::ToSide {
6907 src: None,
6908 dst,
6909 data,
6910 },
6911 ignore_local: true,
6912 priority,
6913 delivery_state: QueuedDelivery::Untracked,
6914 accepted_type: DataType::ReliableAck.as_u32(),
6915 deferred_since_ms: 0,
6916 retry_after_ms: 0,
6917 })?;
6918 state
6920 .reliable_return_routes
6921 .get_mut(&packet_id)
6922 .expect("checked lease")
6923 .delivery_completed = true;
6924 #[cfg(feature = "std")]
6928 let can_claim = self
6929 .active_queued_tx
6930 .load(core::sync::atomic::Ordering::Acquire)
6931 != usize::MAX;
6932 #[cfg(not(feature = "std"))]
6933 let can_claim = true;
6934 let work = if !called_from_queue && can_claim {
6935 let index = state.transmit_queue.iter().position(|queued| {
6936 if queued.accepted_type != DataType::ReliableAck.as_u32() {
6937 return false;
6938 }
6939 let RouterTxItem::ToSide {
6940 dst: side, data, ..
6941 } = &queued.item
6942 else {
6943 return false;
6944 };
6945 if *side != dst {
6946 return false;
6947 }
6948 match data {
6949 RouterItem::Packet(packet) => packet.packet_id() == ack_id,
6950 RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
6951 wire_format::packet_id_from_wire(view).is_ok_and(|id| id == ack_id)
6952 })
6953 .unwrap_or(false),
6954 }
6955 });
6956 index
6957 .and_then(|index| state.transmit_queue.remove_pos(index))
6958 .map(|queued| {
6959 #[cfg(feature = "latest-value-egress")]
6960 let service = EgressService {
6961 token: state.latest_service_seq,
6962 side: state.latest_dispatch_tag(&queued),
6963 kind: 0,
6964 };
6965 TxWork {
6966 queued,
6967 claim: None,
6968 owed_turn: false,
6969 #[cfg(feature = "latest-value-egress")]
6970 service,
6971 }
6972 })
6973 } else {
6974 None
6975 };
6976 #[cfg(feature = "std")]
6977 let work = self.track_queued_tx(&mut state, work);
6978 drop(state);
6979 if let Some(work) = work {
6980 #[cfg(feature = "std")]
6981 let (work, _activity) = work;
6982 self.dispatch_queued_item(work)?;
6983 }
6984 Ok(())
6985 }
6986
6987 fn queue_end_to_end_reliable_ack_for_sender_hash(
6988 &self,
6989 packet_id: u64,
6990 original_sender_hash: u64,
6991 called_from_queue: bool,
6992 ) -> TelemetryResult<()> {
6993 if !self.cfg.reliable_enabled() {
6994 return Ok(());
6995 }
6996 let ack = self.build_end_to_end_ack_data(packet_id, original_sender_hash)?;
6997 let return_side = {
7001 let st = self.state.lock();
7002 st.reliable_return_routes
7003 .get(&packet_id)
7004 .map(|route| route.side)
7005 };
7006 let item = match return_side {
7007 Some(dst) => RouterTxItem::ToSide {
7008 src: None,
7009 dst,
7010 data: ack,
7011 },
7012 None => RouterTxItem::Broadcast(ack),
7013 };
7014 self.emit_internal_tx(item, true, called_from_queue)
7015 }
7016
7017 fn emit_internal_tx(
7018 &self,
7019 item: RouterTxItem,
7020 ignore_local: bool,
7021 called_from_queue: bool,
7022 ) -> TelemetryResult<()> {
7023 if called_from_queue {
7024 self.tx_queue_item_with_flags(item, ignore_local)
7025 } else {
7026 self.tx_item_impl(item, ignore_local, false)
7027 }
7028 }
7029
7030 fn emit_internal_tx_with_priority(
7031 &self,
7032 item: RouterTxItem,
7033 ignore_local: bool,
7034 priority: u8,
7035 called_from_queue: bool,
7036 ) -> TelemetryResult<()> {
7037 if called_from_queue {
7038 self.tx_queue_item_with_priority(item, ignore_local, priority)
7039 } else {
7040 self.tx_item_impl(item, ignore_local, false)
7041 }
7042 }
7043
7044 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
7045 let now = self.clock.now_ms();
7046 let result = self.tx_queue_item_with_priority(
7047 RouterTxItem::EndToEndReplay { packet_id },
7048 true,
7049 Self::router_item_priority_bumped(DataType::ReliableAck),
7050 );
7051 if result.is_ok()
7055 && let Some(sent) = self.state.lock().end_to_end_reliable_tx.get_mut(&packet_id)
7056 {
7057 sent.last_send_ms = sent.last_send_ms.max(now);
7058 }
7059 result
7060 }
7061
7062 fn end_to_end_retransmit_sides(
7063 &self,
7064 packet_id: u64,
7065 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
7066 let mut st = self.state.lock();
7067 let (data, tracked_destinations, mut sides) = {
7068 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
7069 #[cfg(feature = "std")]
7070 if reliable_trace_enabled() {
7071 eprintln!(
7072 "[sedsnet-reliable] replay time={} id={packet_id:#x} retries={} targets={:?}",
7073 self.clock.now_ms(),
7074 sent.retries,
7075 sent.pending_destinations
7076 );
7077 }
7078 sent.last_send_ms = self.clock.now_ms();
7079 let data = sent.data.clone();
7080 let tracked_destinations = sent.tracked_destinations;
7081 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
7082 (data, tracked_destinations, sides)
7083 };
7084 if tracked_destinations && sides.is_empty() {
7085 st.end_to_end_reliable_tx.remove(&packet_id);
7086 return None;
7087 }
7088 crate::small_sort::sort(&mut sides);
7089 sides.dedup();
7090 Some((data, sides))
7091 }
7092
7093 fn item_type_priority(data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
7094 let (ty, control) = match data {
7095 RouterItem::Packet(pkt) => (
7096 pkt.data_type(),
7097 pkt.payload().len() <= 16
7098 && pkt.endpoints().len() <= 3
7099 && crate::small_frozen_control(
7100 pkt.data_type(),
7101 pkt.wire_target_senders().len(),
7102 pkt.wire_shape(),
7103 ),
7104 ),
7105 RouterItem::Packed(bytes) => {
7106 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7107 let env = &frame.envelope;
7108 if bytes.len() <= 128 && frame.ack_only() {
7111 return Ok((
7112 env.ty,
7113 crate::transport_priority(crate::DataType::ReliableAck),
7114 ));
7115 }
7116 let control = bytes.len() <= 128
7117 && bytes.first().is_some_and(|flags| flags & 0x11 == 0)
7118 && env.endpoints().take(4).count() <= 3
7119 && crate::small_frozen_control(
7120 env.ty,
7121 env.target_senders().len(),
7122 env.wire_shape,
7123 );
7124 (env.ty, control)
7125 }
7126 };
7127 let priority = crate::scheduler_priority(ty);
7128 Ok((ty, if control { priority.max(254) } else { priority }))
7129 }
7130
7131 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
7132 Self::item_type_priority(data).map(|(_, priority)| priority)
7133 }
7134
7135 fn item_service_type_priority(&self, data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
7136 let (ty, priority) = Self::item_type_priority(data)?;
7137 let priority = if self.is_managed_variable_type(ty) {
7138 crate::transport_priority(DataType::ManagedVariableValue)
7139 } else {
7140 priority
7141 };
7142 Ok((ty, priority))
7143 }
7144
7145 fn item_service_priority(&self, data: &RouterItem) -> TelemetryResult<u8> {
7146 self.item_service_type_priority(data)
7147 .map(|(_, priority)| priority)
7148 }
7149
7150 #[inline]
7151 fn router_item_priority_bumped(ty: DataType) -> u8 {
7152 crate::scheduler_priority(ty).saturating_add(16)
7153 }
7154
7155 #[inline]
7156 fn is_transient_tx_error(err: &TelemetryError) -> bool {
7157 matches!(
7158 err,
7159 TelemetryError::Io("side tx busy" | "memory pressure" | "reliable window full")
7160 )
7161 }
7162
7163 #[cfg(feature = "timesync")]
7164 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
7165 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
7166 || st
7167 .clock
7168 .current_time(now_mono_ns)
7169 .and_then(|reading| reading.unix_time_ms)
7170 .is_some()
7171 }
7172
7173 #[cfg(feature = "timesync")]
7174 fn reconcile_pending_timesync_request_locked(
7175 st: &mut TimeSyncRuntime,
7176 leader: &Option<TimeSyncLeader>,
7177 now_ms: u64,
7178 ) {
7179 let active_remote = match leader {
7180 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
7181 _ => None,
7182 };
7183 let should_clear = st
7184 .pending_request
7185 .as_ref()
7186 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
7187 if should_clear {
7188 st.pending_request = None;
7189 st.next_request_mono_ms = now_ms;
7190 }
7191 }
7192
7193 #[inline]
7195 fn enqueue_to_sides(
7196 &self,
7197 data: RouterItem,
7198 exclude: Option<RouterSideId>,
7199 ignore_local: bool,
7200 ) -> TelemetryResult<()> {
7201 let Some(data) = self.clear_intermediate_hop_targets(data, true)? else {
7202 return Ok(());
7203 };
7204 let plan = self.remote_side_plan(&data, exclude)?;
7205 let mut st = self.state.lock();
7206 let ty = Self::item_data_type(&data)?;
7207 let priority = if st.is_managed_variable(ty.as_u32()) {
7208 crate::transport_priority(DataType::ManagedVariableValue)
7209 } else {
7210 Self::router_item_priority(&data)?
7211 };
7212
7213 let RemoteSidePlan::Target(sides) = plan;
7214 for idx in sides {
7215 st.push_transmit(TxQueued {
7216 disposable: TX_FRESH,
7217 item: RouterTxItem::ToSide {
7218 src: exclude,
7219 dst: idx,
7220 data: data.clone(),
7221 },
7222 ignore_local,
7223 priority,
7224 delivery_state: QueuedDelivery::Untracked,
7225 accepted_type: 0,
7226 deferred_since_ms: 0,
7227 retry_after_ms: 0,
7228 })?;
7229 }
7230
7231 Ok(())
7232 }
7233
7234 fn relay_send(
7235 &self,
7236 data: RouterItem,
7237 src: Option<RouterSideId>,
7238 called_from_queue: bool,
7239 ) -> TelemetryResult<()> {
7240 if called_from_queue {
7241 return self.enqueue_to_sides(data, src, true);
7242 }
7243
7244 let Some(data) = self.clear_intermediate_hop_targets(data, false)? else {
7245 return Ok(());
7246 };
7247 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
7248 for side in sides {
7249 #[cfg(feature = "latest-value-egress")]
7250 if self.queue_latest_egress(side, src, &data)? {
7251 continue;
7252 }
7253 self.tx_item_impl(
7254 RouterTxItem::ToSide {
7255 src,
7256 dst: side,
7257 data: data.clone(),
7258 },
7259 true,
7260 false,
7261 )?;
7262 }
7263
7264 Ok(())
7265 }
7266
7267 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
7268 match data {
7269 RouterItem::Packet(pkt) => {
7270 pkt.validate()?;
7271 let mut eps = pkt.endpoints().to_vec();
7272 crate::small_sort::sort(&mut eps);
7273 eps.dedup();
7274 Ok((eps, pkt.data_type()))
7275 }
7276 RouterItem::Packed(bytes) => {
7277 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7278 let mut eps: Vec<DataEndpoint> = env.endpoints().collect();
7279 crate::small_sort::sort(&mut eps);
7280 eps.dedup();
7281 Ok((eps, env.ty))
7282 }
7283 }
7284 }
7285
7286 fn unpack_packet_for_role(&self, bytes: &[u8]) -> TelemetryResult<Packet> {
7287 let managed = crate::memory_admission::supports_work_kinds()
7288 && wire_format::scheduling_type(bytes).is_some_and(|ty| {
7289 !is_internal_control_type(ty) && self.is_managed_variable_type(ty)
7290 });
7291 wire_format::unpack_packet_with_role(bytes, managed)
7292 }
7293
7294 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
7295 match data {
7296 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
7297 RouterItem::Packed(bytes) => Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.ty),
7298 }
7299 }
7300
7301 fn e2e_crypto_supported(&self) -> bool {
7302 #[cfg(feature = "cryptography")]
7303 {
7304 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
7305 && crate::crypto::registered_crypto_available()
7306 }
7307 #[cfg(not(feature = "cryptography"))]
7308 {
7309 false
7310 }
7311 }
7312
7313 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
7314 if is_internal_control_type(ty) {
7315 return false;
7316 }
7317 match self.cfg.e2e_encryption() {
7318 RouterE2eEncryptionMode::Disabled => {
7319 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
7320 }
7321 RouterE2eEncryptionMode::RequiredOnly => {
7322 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
7323 }
7324 RouterE2eEncryptionMode::Preferred => matches!(
7325 message_e2e_encryption_policy(ty),
7326 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
7327 ),
7328 RouterE2eEncryptionMode::ForceAll => true,
7329 }
7330 }
7331
7332 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
7333 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
7334 return Err(TelemetryError::BadArg);
7335 }
7336 Ok(())
7337 }
7338
7339 #[cfg(feature = "cryptography")]
7340 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
7341 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
7342 Some(wire_format::E2eSealConfig {
7343 key_id: self.cfg.e2e_key_id(),
7344 })
7345 } else {
7346 None
7347 }
7348 }
7349
7350 #[inline]
7351 fn pack_packet_for_router(
7352 &self,
7353 pkt: &Packet,
7354 reliable: Option<wire_format::ReliableHeader>,
7355 ) -> TelemetryResult<crate::SharedBytes> {
7356 #[cfg(feature = "compression")]
7357 {
7358 #[cfg(feature = "cryptography")]
7359 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
7360 #[cfg(not(feature = "cryptography"))]
7361 let e2e = None;
7362 let reliable = reliable.or_else(|| {
7363 is_reliable_type(pkt.data_type()).then_some(wire_format::ReliableHeader {
7364 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
7365 seq: 0,
7366 ack: 0,
7367 })
7368 });
7369 wire_format::pack_routed_packet(
7370 pkt,
7371 reliable,
7372 pkt.wire_shape(),
7373 pkt.wire_target_senders(),
7374 e2e,
7375 )
7376 .map(crate::shared_bytes::convert)
7377 }
7378 #[cfg(not(feature = "compression"))]
7379 {
7380 #[cfg(feature = "cryptography")]
7381 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
7382 return (wire_format::pack_packet_with_wire_contract_e2e(
7383 pkt,
7384 reliable,
7385 pkt.wire_shape(),
7386 pkt.wire_target_senders(),
7387 e2e,
7388 ))
7389 .map(crate::shared_bytes::convert);
7390 }
7391 Ok(match reliable {
7392 Some(hdr) => {
7393 crate::shared_bytes::convert(wire_format::pack_packet_with_reliable(pkt, hdr))
7394 }
7395 None => crate::shared_bytes::convert(wire_format::pack_packet(pkt)),
7396 })
7397 }
7398 }
7399
7400 #[inline]
7401 fn pack_packet_for_contract(
7402 &self,
7403 pkt: &Packet,
7404 reliable: Option<wire_format::ReliableHeader>,
7405 shape: Option<MessageElement>,
7406 target_senders: &[u64],
7407 ) -> TelemetryResult<crate::SharedBytes> {
7408 #[cfg(feature = "compression")]
7409 {
7410 #[cfg(feature = "cryptography")]
7411 let e2e = self.e2e_seal_config_for_type(pkt.data_type());
7412 #[cfg(not(feature = "cryptography"))]
7413 let e2e = None;
7414 wire_format::pack_routed_packet(pkt, reliable, shape, target_senders, e2e)
7415 .map(crate::shared_bytes::convert)
7416 }
7417 #[cfg(not(feature = "compression"))]
7418 {
7419 #[cfg(feature = "cryptography")]
7420 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
7421 return (wire_format::pack_packet_with_wire_contract_e2e(
7422 pkt,
7423 reliable,
7424 shape,
7425 target_senders,
7426 e2e,
7427 ))
7428 .map(crate::shared_bytes::convert);
7429 }
7430 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
7431 .map(crate::shared_bytes::convert)
7432 }
7433 }
7434
7435 #[cfg(feature = "cryptography")]
7436 #[inline]
7437 fn prepare_packed_for_remote(
7438 &self,
7439 bytes: crate::SharedBytes,
7440 reliable_override: Option<Option<wire_format::ReliableHeader>>,
7441 ) -> TelemetryResult<crate::SharedBytes> {
7442 let routing = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7445 if routing.ack_only() || self.e2e_seal_config_for_type(routing.envelope.ty).is_none() {
7446 return Ok(bytes);
7447 }
7448 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
7449 let reliable = reliable_override.unwrap_or(frame.reliable);
7450 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
7451 self.pack_packet_for_contract(
7452 &pkt,
7453 reliable,
7454 frame.envelope.wire_shape,
7455 &frame.envelope.target_senders,
7456 )
7457 }
7458
7459 fn item_requires_end_to_end_tracking(
7463 &self,
7464 data: &RouterItem,
7465 ty: DataType,
7466 ) -> TelemetryResult<bool> {
7467 if is_reliable_type(ty) || self.is_managed_variable_type(ty) {
7468 return Ok(true);
7469 }
7470 match data {
7471 RouterItem::Packet(packet) => Ok(!packet.wire_target_senders().is_empty()),
7472 RouterItem::Packed(bytes) => {
7473 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7474 Ok(frame.reliable.is_some() || frame.envelope.target_senders().len() != 0)
7475 }
7476 }
7477 }
7478
7479 #[cfg(feature = "discovery")]
7480 fn item_target_senders<'a>(
7481 &self,
7482 data: &'a RouterItem,
7483 ) -> TelemetryResult<wire_format::TargetSenderSet<'a>> {
7484 match data {
7485 RouterItem::Packet(pkt) => Ok(wire_format::TargetSenderSet::Hashes(
7486 pkt.wire_target_senders(),
7487 )),
7488 RouterItem::Packed(bytes) => {
7489 Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.target_sender_set())
7490 }
7491 }
7492 }
7493
7494 fn item_targets_another_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
7495 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
7496 match data {
7497 RouterItem::Packet(pkt) => {
7498 Ok(pkt.wire_target_senders().iter().any(|v| *v != local_hash))
7499 }
7500 RouterItem::Packed(bytes) => {
7501 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7502 Ok(env.target_senders().any(|target| target != local_hash))
7503 }
7504 }
7505 }
7506
7507 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
7508 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
7509 match data {
7510 RouterItem::Packet(pkt) => Ok(pkt.wire_target_senders().is_empty()
7511 || pkt.wire_target_senders().contains(&local_hash)),
7512 RouterItem::Packed(bytes) => {
7513 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7514 Ok(env.target_senders().len() == 0
7515 || env.target_senders().any(|target| target == local_hash))
7516 }
7517 }
7518 }
7519
7520 fn item_targets_router_as_intermediate_hop(
7521 &self,
7522 data: &RouterItem,
7523 endpoints: &[DataEndpoint],
7524 ) -> TelemetryResult<bool> {
7525 let local_target = match data {
7528 RouterItem::Packet(pkt) => {
7529 if pkt.wire_target_senders().is_empty() || is_internal_control_type(pkt.data_type())
7530 {
7531 return Ok(false);
7532 }
7533 pkt.wire_target_senders()
7534 .contains(&Self::sender_hash(self.sender_arc().as_ref()))
7535 }
7536 RouterItem::Packed(bytes) => {
7537 let env = wire_format::peek_routing_envelope(bytes)?;
7538 if env.target_senders().len() == 0 || is_internal_control_type(env.ty) {
7539 return Ok(false);
7540 }
7541 let local = Self::sender_hash(self.sender_arc().as_ref());
7542 env.target_senders().any(|owner| owner == local)
7543 }
7544 };
7545 Ok(local_target
7546 && !endpoints.iter().copied().any(|endpoint| {
7547 self.endpoint_has_packet_handler(endpoint)
7548 || self.endpoint_has_packed_handler(endpoint)
7549 }))
7550 }
7551
7552 fn clear_intermediate_hop_targets(
7553 &self,
7554 data: RouterItem,
7555 called_from_queue: bool,
7556 ) -> TelemetryResult<Option<RouterItem>> {
7557 let has_targets = match &data {
7561 RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
7562 RouterItem::Packed(bytes) => {
7563 wire_format::peek_routing_envelope(bytes)?
7564 .target_senders()
7565 .len()
7566 != 0
7567 }
7568 };
7569 if !has_targets {
7570 return Ok(Some(data));
7571 }
7572 let (endpoints, _) = self.item_route_info(&data)?;
7573 if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
7574 return Ok(Some(data));
7575 }
7576 let pkt = match &data {
7577 RouterItem::Packet(pkt) => pkt.clone(),
7578 RouterItem::Packed(bytes) => self.unpack_packet_for_role(bytes.as_ref())?,
7579 };
7580 let reliable = match &data {
7581 RouterItem::Packet(_) => None,
7582 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
7583 };
7584 let shape = match &data {
7585 RouterItem::Packet(pkt) => pkt.wire_shape(),
7586 RouterItem::Packed(bytes) => {
7587 wire_format::peek_routing_envelope(bytes.as_ref())?.wire_shape
7588 }
7589 };
7590 let upstream_id = Self::get_hash(&data);
7591 let completed_contract = {
7595 let st = self.state.lock();
7596 st.reliable_return_routes
7597 .get(&upstream_id)
7598 .filter(|route| route.delivery_completed)
7599 .and_then(|route| route.upstream_contract)
7600 };
7601 if let Some((packet_id, publisher_hash)) = completed_contract {
7602 self.queue_end_to_end_reliable_ack_for_sender_hash(
7603 packet_id,
7604 publisher_hash,
7605 called_from_queue,
7606 )?;
7607 return Ok(None);
7608 }
7609 #[cfg(feature = "discovery")]
7610 let destinations = {
7611 let st = self.state.lock();
7612 let upstream = st
7613 .reliable_return_routes
7614 .get(&upstream_id)
7615 .map(|route| route.side);
7616 let mut pending =
7617 self.expected_end_to_end_destinations_locked(&st, &data, upstream, true, true)?;
7618 if pending.is_empty() {
7619 pending = self
7620 .expected_end_to_end_destinations_locked(&st, &data, upstream, false, true)?;
7621 }
7622 pending
7623 };
7624 #[cfg(feature = "discovery")]
7625 let targets: Vec<u64> = destinations.keys().copied().collect();
7626 #[cfg(not(feature = "discovery"))]
7627 let targets: Vec<u64> = Vec::new();
7628 #[cfg(feature = "discovery")]
7629 if targets.is_empty() {
7630 if self.packet_has_local_delivery(&pkt) {
7631 self.queue_end_to_end_reliable_ack_for_packet_id(
7632 upstream_id,
7633 pkt.sender(),
7634 called_from_queue,
7635 )?;
7636 return Ok(Some(data));
7637 }
7638 if !is_reliable_type(pkt.data_type()) && reliable.is_none() {
7639 return Err(TelemetryError::Io("no discovered route"));
7640 }
7641 }
7642 let forwarded = self.pack_packet_for_contract(&pkt, reliable, shape, &targets)?;
7646 let downstream_id = wire_format::packet_id_from_wire(forwarded.as_ref())?;
7647 let forwarded = RouterItem::Packed(forwarded);
7648 #[cfg(feature = "discovery")]
7649 let already_pending = self
7650 .state
7651 .lock()
7652 .end_to_end_reliable_tx
7653 .contains_key(&downstream_id);
7654 #[cfg(feature = "discovery")]
7655 let ready = self.register_end_to_end_reliable_tx(&forwarded, Some(destinations))?;
7656 #[cfg(not(feature = "discovery"))]
7657 let (ready, already_pending) = {
7658 let _ = called_from_queue;
7659 (true, false)
7660 };
7661 #[cfg(feature = "std")]
7662 if reliable_trace_enabled() {
7663 eprintln!(
7664 "[sedsnet-reliable] proxy retain upstream={upstream_id:#x} downstream={downstream_id:#x} existing={already_pending} ready={ready}"
7665 );
7666 }
7667 {
7668 let mut st = self.state.lock();
7669 let Some(mut route) = st.reliable_return_routes.get(&upstream_id).cloned() else {
7670 st.end_to_end_reliable_tx.remove(&downstream_id);
7671 return Err(TelemetryError::Io("no reliable return route"));
7672 };
7673 route.upstream_contract = Some((
7674 upstream_id,
7675 Self::sender_hash(&Self::canonical_sender_locked(&st, pkt.sender())),
7676 ));
7677 if !Self::remember_reliable_return_route_locked(&mut st, downstream_id) {
7678 st.end_to_end_reliable_tx.remove(&downstream_id);
7679 return Err(TelemetryError::Io("reliable return route window full"));
7680 }
7681 st.reliable_return_routes.insert(downstream_id, route);
7682 }
7683 Ok((ready && !already_pending).then_some(forwarded))
7684 }
7685
7686 #[cfg(feature = "discovery")]
7687 fn side_matches_target_senders_locked(
7688 st: &RouterInner,
7689 side: RouterSideId,
7690 target_senders: wire_format::TargetSenderSet<'_>,
7691 now_ms: u64,
7692 ) -> bool {
7693 st.discovery_routes
7694 .get(&side)
7695 .map(|route| {
7696 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7697 return false;
7698 }
7699 route.announcers.iter().any(|(announcer, sender_state)| {
7700 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7701 return false;
7702 }
7703 target_senders.contains(Self::sender_hash(announcer))
7704 || sender_state
7705 .advertised_address
7706 .is_some_and(|address| target_senders.contains(u64::from(address)))
7707 || sender_state.topology_boards.iter().any(|board| {
7708 target_senders.contains(Self::sender_hash(&board.sender_id))
7709 })
7710 })
7711 })
7712 .unwrap_or(false)
7713 }
7714
7715 fn attach_wire_contract_to_item(
7716 &self,
7717 data: RouterItem,
7718 target_senders: &[u64],
7719 ) -> TelemetryResult<RouterItem> {
7720 let pkt = match data {
7721 RouterItem::Packet(packet) => packet,
7722 RouterItem::Packed(bytes) => {
7723 if wire_format::peek_routing_envelope(&bytes)?
7726 .target_senders()
7727 .len()
7728 != 0
7729 {
7730 return Ok(RouterItem::Packed(bytes));
7731 }
7732 self.unpack_packet_for_role(&bytes)?
7733 }
7734 };
7735 let reliable = if is_reliable_type(pkt.data_type()) {
7736 Some(wire_format::ReliableHeader {
7737 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
7738 seq: 0,
7739 ack: 0,
7740 })
7741 } else {
7742 None
7743 };
7744 let targets = if pkt.wire_target_senders().is_empty() {
7747 target_senders
7748 } else {
7749 pkt.wire_target_senders()
7750 };
7751 let bytes = self.pack_packet_for_contract(
7752 &pkt,
7753 reliable,
7754 pkt.wire_shape()
7755 .or_else(|| Some(message_meta(pkt.data_type()).element)),
7756 targets,
7757 )?;
7758 Ok(RouterItem::Packed(bytes))
7759 }
7760
7761 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
7762 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
7763 }
7764
7765 fn should_route_remote(
7766 &self,
7767 data: &RouterItem,
7768 exclude: Option<RouterSideId>,
7769 ) -> TelemetryResult<bool> {
7770 #[cfg(feature = "discovery")]
7771 {
7772 let RemoteSidePlan::Target(sides) =
7773 self.remote_side_plan_with_selection(data, exclude, false)?;
7774 Ok(!sides.is_empty())
7775 }
7776
7777 #[cfg(not(feature = "discovery"))]
7778 {
7779 let (eps, ty) = self.item_route_info(data)?;
7780 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
7781 return Ok(false);
7782 }
7783 let st = self.state.lock();
7784 Ok(!self
7785 .eligible_side_ids_locked(
7786 &st,
7787 exclude,
7788 Some(ty),
7789 Self::endpoints_are_link_local_only(&eps),
7790 )
7791 .is_empty())
7792 }
7793 }
7794
7795 #[cfg(feature = "discovery")]
7796 fn has_explicit_route_policy_locked(
7797 st: &RouterInner,
7798 src: Option<RouterSideId>,
7799 ty: DataType,
7800 ) -> bool {
7801 st.route_overrides
7802 .keys()
7803 .any(|(route_src, _)| *route_src == src)
7804 || Self::has_typed_route_overrides_locked(st, src, ty)
7805 }
7806
7807 fn remote_side_plan(
7808 &self,
7809 data: &RouterItem,
7810 exclude: Option<RouterSideId>,
7811 ) -> TelemetryResult<RemoteSidePlan> {
7812 self.remote_side_plan_with_selection(data, exclude, true)
7813 }
7814
7815 #[inline(never)]
7818 fn remote_side_plan_with_selection(
7819 &self,
7820 data: &RouterItem,
7821 exclude: Option<RouterSideId>,
7822 select_routes: bool,
7823 ) -> TelemetryResult<RemoteSidePlan> {
7824 #[cfg(feature = "discovery")]
7825 let (eps, ty) = self.item_route_info(data)?;
7826 let select_sides = |state: &mut RouterInner, sides: Vec<RouterSideId>, origin| {
7827 #[cfg(feature = "discovery")]
7828 let sides = self.filter_end_to_end_satisfied_sides_locked(state, data, sides, &eps, ty);
7829 if select_routes {
7830 self.apply_route_selection_locked(state, exclude, sides, origin)
7831 } else {
7832 sides
7833 }
7834 };
7835 #[cfg(feature = "discovery")]
7836 {
7837 let target_senders = self.item_target_senders(data)?;
7838 let resolve_beyond_local_hop =
7839 self.item_targets_router_as_intermediate_hop(data, &eps)?;
7840 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
7841 let routing_target_senders = if preferred_packet_id.is_some() {
7842 target_senders.skip(1)
7845 } else if resolve_beyond_local_hop {
7846 wire_format::TargetSenderSet::Hashes(&[])
7847 } else {
7848 target_senders
7849 };
7850 if discovery::is_discovery_type(ty) {
7851 let mut st = self.state.lock();
7852 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7853 return Ok(RemoteSidePlan::Target(select_sides(
7854 &mut st,
7855 sides,
7856 RouteSelectionOrigin::Flood,
7857 )));
7858 }
7859 #[cfg(feature = "timesync")]
7860 if ty == DataType::TimeSyncAnnounce {
7861 let mut st = self.state.lock();
7868 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7869 let sides = if select_routes {
7870 Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
7871 } else {
7872 sides
7873 };
7874 return Ok(RemoteSidePlan::Target(select_sides(
7875 &mut st,
7876 sides,
7877 RouteSelectionOrigin::Flood,
7878 )));
7879 }
7880 let mut st = self.state.lock();
7881 let now_ms = self.clock.now_ms();
7886 let has_remote_variable_owner = st.is_managed_variable(ty.as_u32())
7887 && st.discovery_routes.iter().any(|(&side, route)| {
7888 exclude != Some(side)
7889 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7890 && route.reachable_network_variables.contains(&ty)
7891 });
7892 if !(has_nonlocal_endpoint(&eps, &self.cfg)
7893 || force_remote_for_type(ty)
7894 || !target_senders.is_empty()
7895 || has_remote_variable_owner)
7896 {
7897 return Ok(RemoteSidePlan::Target(Vec::new()));
7898 }
7899
7900 #[cfg(feature = "timesync")]
7901 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
7902 #[cfg(not(feature = "timesync"))]
7903 let preferred_timesync_source: Option<String> = None;
7904
7905 if let Some(packet_id) = preferred_packet_id {
7906 let target_side = st
7907 .reliable_return_routes
7908 .get(&packet_id)
7909 .map(|route| route.side);
7910 if let Some(side) = target_side
7911 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
7912 {
7913 #[cfg(feature = "timesync")]
7914 if select_routes
7915 && !Self::timesync_allowed_for_side_locked(
7916 &st,
7917 side,
7918 ty,
7919 self.clock.now_ms(),
7920 )
7921 {
7922 return Ok(RemoteSidePlan::Target(Vec::new()));
7923 }
7924 return Ok(RemoteSidePlan::Target(vec![side]));
7925 }
7926 let eligible = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7929 let exact: Vec<_> = eligible
7930 .iter()
7931 .copied()
7932 .filter(|&side| {
7933 !routing_target_senders.is_empty()
7934 && Self::side_matches_target_senders_locked(
7935 &st,
7936 side,
7937 routing_target_senders,
7938 now_ms,
7939 )
7940 })
7941 .collect();
7942 if !exact.is_empty() {
7943 return Ok(RemoteSidePlan::Target(select_sides(
7944 &mut st,
7945 exact,
7946 RouteSelectionOrigin::Discovered,
7947 )));
7948 }
7949 let publisher_known = !routing_target_senders.is_empty()
7954 && st.discovery_routes.keys().any(|&side| {
7955 Self::side_matches_target_senders_locked(
7956 &st,
7957 side,
7958 routing_target_senders,
7959 now_ms,
7960 )
7961 });
7962 return Ok(RemoteSidePlan::Target(
7963 if !publisher_known && exclude.is_some() && eligible.len() == 1 {
7964 eligible
7965 } else {
7966 Vec::new()
7967 },
7968 ));
7969 }
7970 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
7971 let prefer_best_overlap = is_reliable_type(ty)
7972 && routing_target_senders.is_empty()
7973 && preferred_packet_id.is_none();
7974 if st.discovery_routes.is_empty() {
7975 let fallback =
7976 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
7977 #[cfg(feature = "timesync")]
7978 let fallback = if select_routes {
7979 Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), fallback)
7980 } else {
7981 fallback
7982 };
7983 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
7984 fallback
7985 } else {
7986 Vec::new()
7987 }));
7988 }
7989 let mut matches = self.discovered_route_candidates_locked(
7990 &st,
7991 exclude,
7992 ty,
7993 &eps,
7994 routing_target_senders,
7995 prefer_best_overlap,
7996 preferred_timesync_source.as_deref(),
7997 );
7998 #[cfg(feature = "timesync")]
7999 {
8000 if select_routes {
8001 matches =
8002 Self::filter_timesync_matches_locked(&st, ty, self.clock.now_ms(), matches);
8003 }
8004 }
8005 let has_distributed_variable_owners = matches.iter().any(|candidate| {
8008 st.discovery_routes
8009 .get(&candidate.side)
8010 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
8011 });
8012 if !has_distributed_variable_owners {
8013 if self.cfg.role.local_services()
8017 && eps.len() > 1
8018 && matches.len() > 1
8019 && !is_reliable_type(ty)
8020 && !st.is_managed_variable(ty.as_u32())
8021 && routing_target_senders.is_empty()
8022 && preferred_packet_id.is_none()
8023 && preferred_timesync_source.is_none()
8024 && !Self::has_explicit_route_policy_locked(&st, exclude, ty)
8025 && !st.source_route_modes.contains_key(&exclude)
8026 {
8027 if !select_routes {
8028 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
8029 }
8030 let sides = self.select_endpoint_role_sides_locked(
8031 &mut st,
8032 exclude,
8033 &eps,
8034 &matches,
8035 self.clock.now_ms(),
8036 );
8037 return Ok(RemoteSidePlan::Target(sides));
8038 }
8039 Self::retain_shortest_discovery_candidates_locked(
8040 &st,
8041 &mut matches,
8042 &eps,
8043 routing_target_senders,
8044 self.clock.now_ms(),
8045 );
8046 }
8047
8048 if !matches.is_empty() {
8049 if !select_routes {
8050 return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
8053 }
8054 Ok(RemoteSidePlan::Target(
8055 self.select_discovered_candidate_sides_locked(
8056 &mut st,
8057 (data, ty),
8058 exclude,
8059 routing_target_senders,
8060 prefer_best_overlap,
8061 matches,
8062 ),
8063 ))
8064 } else if !self.cfg.role.local_services() && exclude.is_some() {
8065 let fallback =
8066 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
8067 let known_elsewhere = !routing_target_senders.is_empty()
8068 && st.discovery_routes.keys().any(|&side| {
8069 Self::side_matches_target_senders_locked(
8070 &st,
8071 side,
8072 routing_target_senders,
8073 self.clock.now_ms(),
8074 )
8075 });
8076 Ok(RemoteSidePlan::Target(
8077 if fallback.len() == 1 && !known_elsewhere {
8078 select_sides(&mut st, fallback, RouteSelectionOrigin::Flood)
8079 } else {
8080 Vec::new()
8081 },
8082 ))
8083 } else if prefer_best_overlap {
8084 Ok(RemoteSidePlan::Target(Vec::new()))
8085 } else {
8086 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
8087 let sides =
8088 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
8089 #[cfg(feature = "timesync")]
8090 let sides = if select_routes {
8091 Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
8092 } else {
8093 sides
8094 };
8095 Ok(RemoteSidePlan::Target(select_sides(
8096 &mut st,
8097 sides,
8098 RouteSelectionOrigin::Flood,
8099 )))
8100 } else {
8101 Ok(RemoteSidePlan::Target(Vec::new()))
8102 }
8103 }
8104 }
8105 #[cfg(not(feature = "discovery"))]
8106 {
8107 let ty = Self::item_data_type(data)?;
8108 let mut st = self.state.lock();
8109 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
8110 let target_side = st
8111 .reliable_return_routes
8112 .get(&packet_id)
8113 .map(|route| route.side);
8114 if let Some(side) = target_side
8115 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
8116 {
8117 return Ok(RemoteSidePlan::Target(vec![side]));
8118 }
8119 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
8120 return Ok(RemoteSidePlan::Target(
8121 if exclude.is_some() && sides.len() == 1 {
8122 sides
8123 } else {
8124 Vec::new()
8125 },
8126 ));
8127 }
8128 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
8129 Ok(RemoteSidePlan::Target(select_sides(
8130 &mut st,
8131 sides,
8132 RouteSelectionOrigin::Flood,
8133 )))
8134 }
8135 }
8136
8137 #[cfg(feature = "discovery")]
8138 fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
8139 let mut eps: Vec<DataEndpoint> = self.cfg.handlers().iter().map(|h| h.endpoint).collect();
8140 let _ = st;
8146 #[cfg(feature = "timesync")]
8147 if self.cfg.timesync_config().is_some() {
8148 eps.push(DataEndpoint::TimeSync);
8149 }
8150 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
8151 crate::small_sort::sort(&mut eps);
8152 eps.dedup();
8153 eps
8154 }
8155
8156 #[cfg(all(feature = "discovery", not(feature = "timesync")))]
8157 fn local_discovery_timesync_sources(&self, _now_ms: u64) -> Vec<String> {
8158 Vec::new()
8159 }
8160
8161 #[cfg(all(feature = "discovery", feature = "timesync"))]
8162 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
8163 #[cfg(feature = "timesync")]
8164 if self.timesync.is_none() {
8165 return Vec::new();
8166 }
8167
8168 #[cfg(feature = "timesync")]
8169 {
8170 let st = self
8171 .timesync
8172 .as_ref()
8173 .expect("local time service enabled")
8174 .lock();
8175 if let Some(tracker) = st.tracker.as_ref()
8176 && tracker.should_serve(
8177 now_ms,
8178 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
8179 )
8180 {
8181 return vec![self.sender_arc().to_string()];
8182 }
8183 }
8184 Vec::new()
8185 }
8186
8187 #[cfg(all(feature = "discovery", feature = "timesync"))]
8188 fn preferred_timesync_route_source(
8189 &self,
8190 data: &RouterItem,
8191 ty: DataType,
8192 ) -> TelemetryResult<Option<String>> {
8193 if !matches!(
8194 ty,
8195 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
8196 ) {
8197 return Ok(None);
8198 }
8199
8200 match data {
8201 RouterItem::Packet(pkt) => match ty {
8202 DataType::TimeSyncRequest => {
8203 let local_sender = self.sender_arc();
8204 if pkt.sender() == local_sender.as_ref() {
8205 Ok(self.timesync.as_ref().and_then(|runtime| {
8206 runtime.lock().tracker.as_ref().and_then(|tracker| {
8207 tracker.current_source().map(|src| src.sender.clone())
8208 })
8209 }))
8210 } else {
8211 Ok(None)
8212 }
8213 }
8214 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
8215 Ok(Some(pkt.sender().to_owned()))
8216 }
8217 _ => Ok(None),
8218 },
8219 RouterItem::Packed(bytes) => {
8220 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
8221 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
8222 }
8223 }
8224 }
8225
8226 #[cfg(feature = "discovery")]
8227 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
8228 st.discovery_cadence.on_topology_change(now_ms);
8229 for side_id in st
8234 .sides
8235 .iter()
8236 .enumerate()
8237 .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
8238 {
8239 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
8240 throttle.set_pending_incremental(true);
8241 throttle.incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
8242 }
8243 for transport in st.side_transport.values_mut() {
8247 transport.refresh_transmit_header_templates();
8248 }
8249 }
8250
8251 #[cfg(feature = "discovery")]
8252 fn sender_topology_board_mut<'a>(
8253 sender_state: &'a mut DiscoverySenderState,
8254 sender_id: &str,
8255 ) -> &'a mut TopologyBoardNode {
8256 let boards = Arc::make_mut(&mut sender_state.topology_boards);
8257 if let Some(idx) = boards.iter().position(|board| board.sender_id == sender_id) {
8258 return &mut boards[idx];
8259 }
8260 boards.push(TopologyBoardNode {
8261 sender_id: sender_id.to_string(),
8262 reachable_endpoints: Vec::new(),
8263 reachable_timesync_sources: Vec::new(),
8264 connections: Vec::new(),
8265 });
8266 boards.last_mut().expect("inserted above")
8267 }
8268
8269 #[cfg(feature = "discovery")]
8270 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
8271 if let Some(boards) = Arc::get_mut(&mut sender_state.topology_boards) {
8272 discovery::normalize_topology_boards(boards);
8273 }
8274 let (mut reachable, mut reachable_timesync_sources) =
8275 discovery::summarize_topology_boards(&sender_state.topology_boards);
8276 reachable.extend(sender_state.advertised_reachable.iter().copied());
8277 crate::small_sort::sort(&mut reachable);
8278 reachable.dedup();
8279 reachable_timesync_sources.extend(
8280 sender_state
8281 .advertised_reachable_timesync_sources
8282 .iter()
8283 .cloned(),
8284 );
8285 reachable_timesync_sources.sort_unstable();
8286 reachable_timesync_sources.dedup();
8287 sender_state.reachable = reachable;
8288 sender_state.reachable_timesync_sources = reachable_timesync_sources;
8289 }
8290
8291 #[cfg(feature = "discovery")]
8292 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
8293 let mut reachable = Vec::new();
8294 let mut reachable_network_variables = Vec::new();
8295 let mut reachable_timesync_sources = Vec::new();
8296 let mut last_seen_ms = 0u64;
8297 for sender in route.announcers.values() {
8298 reachable.extend(sender.reachable.iter().copied());
8299 reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
8300 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
8301 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
8302 }
8303 reachable_network_variables.extend(
8304 route
8305 .requested_network_variables
8306 .iter()
8307 .map(|request| request.ty),
8308 );
8309 last_seen_ms = last_seen_ms.max(
8310 route
8311 .requested_network_variables
8312 .iter()
8313 .map(|request| request.last_seen_ms)
8314 .max()
8315 .unwrap_or(0),
8316 );
8317 crate::small_sort::sort(&mut reachable);
8318 reachable.dedup();
8319 crate::small_sort::sort(&mut reachable_network_variables);
8320 reachable_network_variables.dedup();
8321 reachable_timesync_sources.sort_unstable();
8322 reachable_timesync_sources.dedup();
8323 route.reachable = reachable;
8324 route.reachable_network_variables = reachable_network_variables;
8325 route.reachable_timesync_sources = reachable_timesync_sources;
8326 route.last_seen_ms = last_seen_ms;
8327 }
8328
8329 #[cfg(feature = "discovery")]
8330 fn local_discovery_topology_board(
8331 &self,
8332 st: &RouterInner,
8333 now_ms: u64,
8334 link_local_enabled: bool,
8335 ) -> TopologyBoardNode {
8336 let mut reachable_endpoints = self.local_discovery_endpoints(st);
8337 if !link_local_enabled {
8338 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
8339 }
8340 let mut connections = Vec::new();
8341 for route in st.discovery_routes.values() {
8342 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8343 continue;
8344 }
8345 for (sender, sender_state) in route.announcers.iter() {
8346 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8347 connections.push(sender.clone());
8348 }
8349 }
8350 }
8351 connections.sort_unstable();
8352 connections.dedup();
8353 let sender = self.sender_arc();
8354 TopologyBoardNode {
8355 sender_id: sender.to_string(),
8356 reachable_endpoints,
8357 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
8358 connections,
8359 }
8360 }
8361
8362 #[cfg(feature = "discovery")]
8363 fn advertised_discovery_topology_for_link_locked(
8364 &self,
8365 st: &RouterInner,
8366 now_ms: u64,
8367 link_local_enabled: bool,
8368 exclude_side: Option<RouterSideId>,
8369 ) -> Vec<TopologyBoardNode> {
8370 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
8371 for (&route_side, route) in st.discovery_routes.iter() {
8372 if exclude_side == Some(route_side) {
8373 continue;
8374 }
8375 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8376 continue;
8377 }
8378 for (announcer, sender_state) in route.announcers.iter() {
8379 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8380 continue;
8381 }
8382 if sender_state.topology_boards.is_empty() {
8383 let sender = self.sender_arc();
8384 let fallback = TopologyBoardNode {
8385 sender_id: announcer.clone(),
8386 reachable_endpoints: sender_state.reachable.clone(),
8387 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
8388 connections: vec![sender.to_string()],
8389 };
8390 discovery::merge_topology_boards(&mut boards, core::slice::from_ref(&fallback));
8391 } else {
8392 discovery::merge_topology_boards(&mut boards, &sender_state.topology_boards);
8395 if let Some(board) = boards
8396 .iter_mut()
8397 .find(|board| board.sender_id == *announcer)
8398 {
8399 let sender = self.sender_arc();
8400 if !board.connections.iter().any(|peer| peer == sender.as_ref()) {
8401 board.connections.push(sender.to_string());
8402 }
8403 }
8404 }
8405 }
8406 }
8407 if !link_local_enabled {
8408 for board in &mut boards {
8409 board
8410 .reachable_endpoints
8411 .retain(|ep| !ep.is_link_local_only());
8412 }
8413 }
8414 discovery::normalize_topology_boards(&mut boards);
8415 boards
8416 }
8417
8418 #[cfg(feature = "discovery")]
8419 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
8420 let mut changed = false;
8423 st.discovery_routes.retain(|_, route| {
8424 let before_peers = route.announcers.len();
8425 let before_requests = route.requested_network_variables.len();
8426 route.announcers.retain(|_, sender| {
8427 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8428 });
8429 route.requested_network_variables.retain(|request| {
8430 now_ms >= request.last_seen_ms
8431 && now_ms - request.last_seen_ms <= DISCOVERY_ROUTE_TTL_MS
8432 });
8433 if before_peers != route.announcers.len()
8434 || before_requests != route.requested_network_variables.len()
8435 {
8436 Self::recompute_discovery_side_state(route);
8437 changed = true;
8438 }
8439 let keep =
8440 !route.announcers.is_empty() || !route.requested_network_variables.is_empty();
8441 changed |= !keep;
8442 keep
8443 });
8444 changed
8445 }
8446
8447 #[cfg(feature = "discovery")]
8448 fn advertised_summaries_from_topology_locked(
8449 &self,
8450 st: &RouterInner,
8451 now_ms: u64,
8452 link_local_enabled: bool,
8453 exclude_side: Option<RouterSideId>,
8454 topology: &[TopologyBoardNode],
8455 ) -> (Vec<crate::DataEndpoint>, Vec<String>) {
8456 let (mut endpoints, mut sources) = discovery::summarize_topology_boards(topology);
8457 for (&side, route) in &st.discovery_routes {
8458 if exclude_side == Some(side)
8459 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8460 {
8461 continue;
8462 }
8463 for peer in route.announcers.values() {
8464 if now_ms.saturating_sub(peer.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8465 continue;
8466 }
8467 for endpoint in &peer.advertised_reachable {
8468 if !discovery::is_discovery_endpoint(*endpoint)
8469 && (link_local_enabled || !endpoint.is_link_local_only())
8470 && !endpoints.contains(endpoint)
8471 {
8472 endpoints.push(*endpoint);
8473 }
8474 }
8475 for source in &peer.advertised_reachable_timesync_sources {
8476 if !sources.contains(source) {
8477 sources.push(source.clone());
8478 }
8479 }
8480 }
8481 }
8482 endpoints.retain(|endpoint| {
8483 !discovery::is_discovery_endpoint(*endpoint)
8484 && (link_local_enabled || !endpoint.is_link_local_only())
8485 });
8486 crate::small_sort::sort(&mut endpoints);
8487 sources.sort_unstable();
8488 (endpoints, sources)
8489 }
8490
8491 #[cfg(feature = "discovery")]
8492 fn advertised_discovery_endpoints_for_link_locked(
8493 &self,
8494 st: &RouterInner,
8495 now_ms: u64,
8496 link_local_enabled: bool,
8497 exclude_side: Option<RouterSideId>,
8498 ) -> Vec<DataEndpoint> {
8499 let (mut reachable_endpoints, _) = discovery::summarize_topology_boards(
8500 &self.advertised_discovery_topology_for_link_locked(
8501 st,
8502 now_ms,
8503 link_local_enabled,
8504 exclude_side,
8505 ),
8506 );
8507 for (&side, route) in &st.discovery_routes {
8510 if exclude_side == Some(side)
8511 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8512 {
8513 continue;
8514 }
8515 for sender in route.announcers.values() {
8516 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8517 reachable_endpoints.extend(sender.advertised_reachable.iter().copied());
8518 }
8519 }
8520 }
8521 crate::small_sort::sort(&mut reachable_endpoints);
8522 reachable_endpoints.dedup();
8523 reachable_endpoints
8524 .into_iter()
8525 .filter(|ep| {
8526 !discovery::is_discovery_endpoint(*ep)
8527 && (link_local_enabled || !ep.is_link_local_only())
8528 })
8529 .collect()
8530 }
8531
8532 #[cfg(all(test, feature = "discovery"))]
8533 fn compact_summary_fixture(&self, endpoint: DataEndpoint) {
8534 let mut sender = DiscoverySenderState {
8535 advertised_reachable: vec![endpoint],
8536 advertised_reachable_timesync_sources: vec!["clock-behind-bridge".to_string()],
8537 last_seen_ms: self.clock.now_ms(),
8538 topology_boards: Arc::new(vec![TopologyBoardNode {
8539 sender_id: "bridge".to_string(),
8540 reachable_endpoints: Vec::new(),
8541 reachable_timesync_sources: Vec::new(),
8542 connections: Vec::new(),
8543 }]),
8544 ..Default::default()
8545 };
8546 Self::refresh_sender_topology_state(&mut sender);
8547 let mut route = DiscoverySideState::default();
8548 route
8549 .announcers
8550 .insert("bridge".to_string(), Box::new(sender));
8551 Self::recompute_discovery_side_state(&mut route);
8552 self.state.lock().discovery_routes.insert(0, route);
8553 }
8554
8555 #[cfg(feature = "discovery")]
8556 fn advertised_discovery_timesync_sources_for_link_locked(
8557 &self,
8558 st: &RouterInner,
8559 now_ms: u64,
8560 exclude_side: Option<RouterSideId>,
8561 ) -> Vec<String> {
8562 let (_, mut sources) = discovery::summarize_topology_boards(
8563 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
8564 );
8565 for (&side, route) in &st.discovery_routes {
8566 if exclude_side == Some(side)
8567 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8568 {
8569 continue;
8570 }
8571 for sender in route.announcers.values() {
8572 if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8573 sources.extend(sender.advertised_reachable_timesync_sources.iter().cloned());
8574 }
8575 }
8576 }
8577 sources.sort_unstable();
8578 sources.dedup();
8579 sources
8580 }
8581
8582 #[cfg(feature = "discovery")]
8583 fn advertised_network_variables_for_link_locked(
8584 &self,
8585 st: &RouterInner,
8586 now_ms: u64,
8587 exclude_side: Option<RouterSideId>,
8588 ) -> Vec<DataType> {
8589 let mut types: Vec<DataType> = st
8590 .local_services
8591 .iter()
8592 .flat_map(|local| local.managed_variable_types.iter())
8593 .copied()
8594 .map(DataType)
8595 .collect();
8596 for (&route_side, route) in st.discovery_routes.iter() {
8597 if exclude_side == Some(route_side) {
8598 continue;
8599 }
8600 if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8601 types.extend(route.reachable_network_variables.iter().copied());
8602 }
8603 }
8604 crate::small_sort::sort(&mut types);
8605 types.dedup();
8606 types
8607 }
8608
8609 #[cfg(feature = "discovery")]
8610 fn preferred_discovery_master_present_locked<'a>(
8611 &self,
8612 st: &'a RouterInner,
8613 now_ms: u64,
8614 ) -> Option<&'a str> {
8615 if let Some(preferred) = st.preferred_discovery_master.as_deref()
8616 && (preferred == self.sender_arc().as_ref()
8617 || st
8618 .discovery_routes
8619 .values()
8620 .filter(|route| {
8621 now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8622 })
8623 .any(|route| {
8624 route
8625 .announcers
8626 .iter()
8627 .filter(|(_, peer)| {
8628 now_ms.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8629 })
8630 .any(|(name, peer)| {
8631 name == preferred
8632 || peer.topology_boards.iter().any(|board| {
8633 board.sender_id == preferred
8634 || board
8635 .connections
8636 .iter()
8637 .any(|name| name == preferred)
8638 })
8639 })
8640 }))
8641 {
8642 return Some(preferred);
8645 }
8646 None
8647 }
8648
8649 #[cfg(feature = "discovery")]
8650 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
8651 if let Some(preferred) = self.preferred_discovery_master_present_locked(st, now_ms) {
8652 return preferred.to_string();
8653 }
8654 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
8655 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
8656 }
8657
8658 #[cfg(all(feature = "discovery", feature = "timesync"))]
8659 #[inline]
8660 fn side_is_slow_control_link_locked(
8661 st: &RouterInner,
8662 side_id: RouterSideId,
8663 now_ms: u64,
8664 ) -> bool {
8665 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
8666 let recent_slow = stats.last_slow_observed_ms > 0
8667 && now_ms.saturating_sub(stats.last_slow_observed_ms)
8668 <= discovery::DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
8669 stats.sample_count > 0
8670 && ((stats.estimated_bandwidth_bps > 0
8671 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
8672 || recent_slow)
8673 })
8674 }
8675
8676 #[cfg(feature = "discovery")]
8677 fn discovery_level_for_side_locked(
8678 st: &mut RouterInner,
8679 side_id: RouterSideId,
8680 now_ms: u64,
8681 ) -> Option<DiscoveryAdvertiseLevel> {
8682 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
8683 if !throttle.has_sent_full() {
8684 throttle.set_has_sent_full(true);
8685 throttle.set_pending_incremental(false);
8686 throttle.incremental_retries_remaining = 0;
8687 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
8688 return Some(DiscoveryAdvertiseLevel::Full);
8689 }
8690 if throttle.pending_incremental() {
8691 throttle.incremental_retries_remaining =
8692 throttle.incremental_retries_remaining.saturating_sub(1);
8693 throttle.set_pending_incremental(throttle.incremental_retries_remaining > 0);
8694 return Some(DiscoveryAdvertiseLevel::Incremental);
8695 }
8696 if throttle.liveness_due(now_ms) {
8699 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
8700 return Some(DiscoveryAdvertiseLevel::MinimalPing);
8701 }
8702 None
8703 }
8704
8705 #[cfg(all(feature = "discovery", feature = "timesync"))]
8706 #[inline]
8707 fn is_timesync_type(ty: DataType) -> bool {
8708 matches!(
8709 ty,
8710 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
8711 )
8712 }
8713
8714 #[cfg(all(feature = "discovery", feature = "timesync"))]
8715 #[inline(never)]
8716 fn timesync_allowed_for_side_locked(
8717 st: &RouterInner,
8718 side_id: RouterSideId,
8719 ty: DataType,
8720 now_ms: u64,
8721 ) -> bool {
8722 if !Self::is_timesync_type(ty) {
8723 return true;
8724 }
8725 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
8726 return true;
8727 }
8728 st.timesync_side_throttle
8731 .get(&side_id)
8732 .is_none_or(|throttle| now_ms >= throttle.next_allowed_ms)
8733 }
8734
8735 #[cfg(all(feature = "discovery", feature = "timesync"))]
8736 fn filter_timesync_sides_locked(
8737 st: &RouterInner,
8738 ty: DataType,
8739 now_ms: u64,
8740 sides: Vec<RouterSideId>,
8741 ) -> Vec<RouterSideId> {
8742 sides
8743 .into_iter()
8744 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
8745 .collect()
8746 }
8747
8748 #[cfg(all(feature = "discovery", feature = "timesync"))]
8749 fn filter_timesync_matches_locked(
8750 st: &RouterInner,
8751 ty: DataType,
8752 now_ms: u64,
8753 matches: Vec<DiscoveryCandidateMatch>,
8754 ) -> Vec<DiscoveryCandidateMatch> {
8755 matches
8756 .into_iter()
8757 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
8758 .collect()
8759 }
8760
8761 #[cfg(feature = "discovery")]
8762 fn refresh_known_discovery_liveness(
8763 &self,
8764 pkt: &Packet,
8765 side: RouterSideId,
8766 ) -> TelemetryResult<bool> {
8767 if pkt.data_type() == crate::DataType::DiscoveryAddress {
8768 return self.refresh_known_address_payload(pkt.payload(), side);
8769 }
8770 if pkt.data_type() == crate::DataType::DiscoveryTopology {
8771 crate::memory_admission::check(512, 128)?;
8775 let now = self.clock.now_ms();
8776 let mut st = self.state.lock();
8777 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
8778 let Some(route) = st.discovery_routes.get_mut(&side) else {
8779 return Ok(false);
8780 };
8781 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
8782 return Ok(false);
8783 };
8784 let Some(full) =
8785 discovery::topology_payload_matches_retained(pkt.payload(), &peer.topology_boards)
8786 else {
8787 return Ok(false);
8788 };
8789 peer.has_full_topology |= full;
8790 peer.last_seen_ms = now;
8791 route.last_seen_ms = route.last_seen_ms.max(now);
8792 return Ok(true);
8793 }
8794 if pkt.data_type() != crate::DataType::DiscoveryAnnounce || !pkt.payload().is_empty() {
8795 return Ok(false);
8796 }
8797 crate::memory_admission::check(512, 128)?;
8798 let now = self.clock.now_ms();
8799 let mut st = self.state.lock();
8800 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
8801 let Some(route) = st.discovery_routes.get_mut(&side) else {
8802 return Ok(false);
8803 };
8804 let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
8805 return Ok(false);
8806 };
8807 peer.last_seen_ms = now;
8808 route.last_seen_ms = route.last_seen_ms.max(now);
8809 Ok(true)
8810 }
8811
8812 #[cfg(feature = "discovery")]
8813 fn refresh_known_address_payload(
8814 &self,
8815 payload: &[u8],
8816 side: RouterSideId,
8817 ) -> TelemetryResult<bool> {
8818 let ad = discovery::decode_address_view(payload)?;
8819 let now = self.clock.now_ms();
8820 let mut st = self.state.lock();
8821 let link_local = st
8822 .sides
8823 .get(side)
8824 .and_then(|entry| entry.as_ref())
8825 .is_some_and(|entry| entry.opts.link_local_enabled);
8826 {
8827 let Some(entry) = st.address_book.get(ad.hostname) else {
8828 return Ok(false);
8829 };
8830 if entry.address != ad.address
8831 || entry.requested_address != ad.requested_address
8832 || entry.mode != Self::address_mode_from_code(ad.mode, ad.requested_address)
8833 || entry.owner_hash != ad.owner_hash
8834 {
8835 return Ok(false);
8836 }
8837 if entry.birth_ms != ad.birth_ms
8838 && st.address_book.iter().any(|(hostname, other)| {
8839 hostname.as_str() != ad.hostname
8840 && [ad.address, ad.requested_address].iter().any(|candidate| {
8841 *candidate != 0
8842 && (other.address == *candidate
8843 || other.requested_address == *candidate)
8844 })
8845 })
8846 {
8847 return Ok(false);
8851 }
8852 }
8853 let sender = ad.hostname;
8854 let Some(route) = st.discovery_routes.get_mut(&side) else {
8855 return Ok(false);
8856 };
8857 let Some(peer) = route.announcers.get_mut(sender) else {
8858 return Ok(false);
8859 };
8860 if peer.advertised_address != Some(ad.address)
8861 || !ad
8862 .endpoints()
8863 .filter(|ep| link_local || !ep.is_link_local_only())
8864 .eq(peer.advertised_reachable.iter().copied())
8865 || !ad.sources().eq(peer
8866 .advertised_reachable_timesync_sources
8867 .iter()
8868 .map(String::as_str))
8869 || !ad
8870 .variables()
8871 .eq(peer.reachable_network_variables.iter().copied())
8872 || peer.link_capabilities != Some(ad.link_capabilities)
8873 {
8874 return Ok(false);
8875 }
8876 peer.last_seen_ms = now;
8879 route.last_seen_ms = route.last_seen_ms.max(now);
8880 if let Some(entry) = st.address_book.get_mut(ad.hostname) {
8881 entry.birth_ms = ad.birth_ms;
8884 entry.last_seen_ms = now;
8885 }
8886 #[cfg(feature = "compression")]
8887 {
8888 drop(st);
8889 self.note_payload_codec_address(side, payload, true);
8890 }
8891 Ok(true)
8892 }
8893
8894 #[cfg(feature = "discovery")]
8895 fn queue_pressure_keepalives(
8896 &self,
8897 requested_side: Option<RouterSideId>,
8898 ) -> TelemetryResult<()> {
8899 let now = self.clock.now_ms();
8900 let sender = self.sender_arc();
8901 crate::memory_admission::check(
8902 256 + sender.len().saturating_mul(4),
8903 sender.len().max(256),
8904 )?;
8905 let side_count = self.state.lock().sides.len();
8906 for side in 0..side_count {
8907 let due = {
8908 let st = self.state.lock();
8909 requested_side.is_none_or(|wanted| wanted == side)
8910 && st.sides[side].is_some()
8911 && self.route_allowed_locked(
8912 &st,
8913 None,
8914 Some(crate::DataType::DiscoveryAnnounce),
8915 side,
8916 )
8917 && st
8918 .discovery_side_throttle
8919 .get(&side)
8920 .is_none_or(|throttle| now >= throttle.next_ping_ms)
8921 };
8922 if !due {
8923 continue;
8924 }
8925 let summary = {
8928 let st = self.state.lock();
8929 let extra = st
8930 .discovery_routes
8931 .iter()
8932 .filter(|(id, _)| **id != side)
8933 .fold(0usize, |total, (_, route)| {
8934 total
8935 .saturating_add(route.reachable.len().saturating_mul(32))
8936 .saturating_add(
8937 route.reachable_network_variables.len().saturating_mul(32),
8938 )
8939 .saturating_add(
8940 route
8941 .reachable_timesync_sources
8942 .iter()
8943 .map(|source| source.len().saturating_add(32).saturating_mul(4))
8944 .sum::<usize>(),
8945 )
8946 });
8947 if crate::memory_admission::check(4096usize.saturating_add(extra), 2048).is_ok() {
8948 let opts = st.sides[side].as_ref().expect("checked above").opts.clone();
8949 let mut endpoints = self.local_discovery_endpoints(&st);
8950 let mut variables =
8951 self.advertised_network_variables_for_link_locked(&st, now, Some(side));
8952 let mut sources = self.local_discovery_timesync_sources(now);
8953 for (&route_side, route) in &st.discovery_routes {
8954 if route_side == side {
8955 continue;
8956 }
8957 for peer in route.announcers.values() {
8958 if now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8959 endpoints.extend(peer.reachable.iter().copied());
8960 sources.extend(peer.reachable_timesync_sources.iter().cloned());
8961 }
8962 }
8963 }
8964 endpoints.retain(|ep| {
8965 !discovery::is_discovery_endpoint(*ep)
8966 && (opts.link_local_enabled || !ep.is_link_local_only())
8967 });
8968 crate::small_sort::sort(&mut endpoints);
8969 endpoints.dedup();
8970 crate::small_sort::sort(&mut variables);
8971 variables.dedup();
8972 sources.sort_unstable();
8973 sources.dedup();
8974 Some((endpoints, variables, sources, opts.link_capabilities()))
8975 } else {
8976 None
8977 }
8978 };
8979 let pkt = if let Some((endpoints, variables, sources, capabilities)) = summary {
8980 let address = self.local_address_advertisement(
8981 endpoints,
8982 variables,
8983 sources,
8984 capabilities,
8985 discovery::ADDRESS_STATE_REQUEST,
8986 );
8987 discovery::build_discovery_address(sender.as_ref(), now, &address)?
8988 } else {
8989 discovery::build_discovery_announce(sender.as_ref(), now, &[])?
8990 };
8991 self.emit_internal_tx(
8992 RouterTxItem::ToSide {
8993 src: None,
8994 dst: side,
8995 data: RouterItem::Packet(pkt),
8996 },
8997 true,
8998 true,
8999 )?;
9000 self.state
9001 .lock()
9002 .discovery_side_throttle
9003 .entry(side)
9004 .or_default()
9005 .next_ping_ms = now.saturating_add(discovery::DISCOVERY_SLOW_INTERVAL_MS);
9006 }
9007 Ok(())
9008 }
9009
9010 #[cfg(feature = "discovery")]
9011 fn emit_discovery_snapshot(
9012 &self,
9013 called_from_queue: bool,
9014 include_schema: bool,
9015 include_topology: bool,
9016 skip_in_flight_sides: bool,
9017 ) -> TelemetryResult<()> {
9018 self.emit_discovery_snapshot_on_side(
9019 called_from_queue,
9020 include_schema,
9021 include_topology,
9022 skip_in_flight_sides,
9023 None,
9024 )
9025 }
9026
9027 #[cfg(feature = "discovery")]
9028 fn emit_discovery_snapshot_on_side(
9029 &self,
9030 called_from_queue: bool,
9031 include_schema: bool,
9032 include_topology: bool,
9033 skip_in_flight_sides: bool,
9034 requested_side: Option<RouterSideId>,
9035 ) -> TelemetryResult<()> {
9036 self.emit_discovery_snapshot_on_side_claimed(
9037 called_from_queue,
9038 include_schema,
9039 include_topology,
9040 skip_in_flight_sides,
9041 requested_side,
9042 None,
9043 )
9044 }
9045
9046 #[cfg(feature = "discovery")]
9047 fn emit_discovery_snapshot_on_side_claimed(
9048 &self,
9049 called_from_queue: bool,
9050 include_schema: bool,
9051 include_topology: bool,
9052 skip_in_flight_sides: bool,
9053 requested_side: Option<RouterSideId>,
9054 prefix_claim: Option<u64>,
9055 ) -> TelemetryResult<()> {
9056 if requested_side.is_none() {
9057 let count = self.state.lock().sides.len();
9058 let mut first_error = None;
9059 for side in 0..count {
9060 if self.state.lock().sides[side].is_none() {
9062 continue;
9063 }
9064 if let Err(error) = self.emit_discovery_snapshot_on_side(
9065 called_from_queue,
9066 include_schema,
9067 include_topology,
9068 skip_in_flight_sides,
9069 Some(side),
9070 ) && first_error.is_none()
9071 {
9072 first_error = Some(error);
9073 }
9074 }
9075 return first_error.map_or(Ok(()), Err);
9076 }
9077 if include_schema {
9078 return self
9079 .start_schema_response_generation(
9080 requested_side.expect("single discovery side"),
9081 true,
9082 called_from_queue,
9083 include_topology,
9084 skip_in_flight_sides,
9085 )
9086 .map(|_| ());
9087 }
9088 let (scratch, largest) = {
9089 let now = self.clock.now_ms();
9090 let sender = self.sender_arc();
9091 let st = self.state.lock();
9092 let election_required = self.cfg.role.local_services()
9093 && self
9094 .preferred_discovery_master_present_locked(&st, now)
9095 .is_none();
9096 let snapshot = discovery::topology_snapshot_scratch(
9097 sender.as_ref(),
9098 self.cfg.handlers().len(),
9099 st.local_services
9100 .as_ref()
9101 .map_or(0, |local| local.managed_variable_types.len())
9102 .saturating_add(
9103 st.discovery_routes
9104 .iter()
9105 .filter(|(side, _)| Some(**side) != requested_side)
9106 .map(|(_, r)| r.reachable_network_variables.len())
9107 .sum::<usize>(),
9108 ),
9109 4,
9110 true,
9111 st.discovery_routes
9112 .iter()
9113 .filter(|(side, route)| {
9114 (Some(**side) != requested_side)
9115 && now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9116 })
9117 .flat_map(|(_, route)| route.announcers.iter())
9118 .filter(|(_, peer)| {
9119 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9120 })
9121 .map(|(name, peer)| {
9122 (
9123 name.as_str(),
9124 peer.reachable.as_slice(),
9125 peer.reachable_timesync_sources.as_slice(),
9126 peer.topology_boards.as_slice(),
9127 )
9128 }),
9129 );
9130 if !election_required {
9131 snapshot
9132 } else {
9133 let election = discovery::topology_snapshot_scratch(
9134 sender.as_ref(),
9135 self.cfg.handlers().len(),
9136 0,
9137 2,
9138 false,
9139 st.discovery_routes
9140 .values()
9141 .filter(|route| {
9142 now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9143 })
9144 .flat_map(|route| route.announcers.iter())
9145 .filter(|(_, peer)| {
9146 now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9147 })
9148 .map(|(name, peer)| {
9149 (
9150 name.as_str(),
9151 peer.reachable.as_slice(),
9152 peer.reachable_timesync_sources.as_slice(),
9153 peer.topology_boards.as_slice(),
9154 )
9155 }),
9156 );
9157 (snapshot.0.max(election.0), snapshot.1.max(election.1))
9158 }
9159 };
9160 let (scratch, largest) = if include_schema {
9163 (scratch.max(8192), largest.max(2048))
9164 } else {
9165 (scratch, largest)
9166 };
9167 while let Err(err) = crate::memory_admission::check(scratch, largest) {
9172 if matches!(err, TelemetryError::Io("memory pressure"))
9173 && crate::memory_admission::check(512, 256).is_ok()
9174 {
9175 let mut st = self.state.lock();
9176 if let Some(victim) = st.lower_priority_shared_queue(199)
9177 && st.pop_shared_queue_item(victim)
9178 {
9179 continue;
9180 }
9181 }
9182 if !include_schema && matches!(err, TelemetryError::Io("memory pressure")) {
9183 return self.queue_pressure_keepalives(requested_side);
9184 }
9185 return Err(err);
9186 }
9187 let now_ms = self.clock.now_ms();
9188 let per_side = {
9189 let mut st = self.state.lock();
9190 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9191 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9192 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9193 }
9194 st.fit_discovery_budget();
9195 let local_is_master = self.cfg.role.local_services()
9196 && self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
9197 let mut per_side = None;
9199 for side_id in 0..st.sides.len() {
9200 let Some(side) = st.sides[side_id].as_ref() else {
9201 continue;
9202 };
9203 let opts = side.opts.clone();
9204 let link_local_enabled = opts.link_local_enabled;
9205 if requested_side.is_some_and(|requested| requested != side_id) {
9206 continue;
9207 }
9208 if let Some(claim) = prefix_claim
9209 && !st
9210 .discovery_side_throttle
9211 .get(&side_id)
9212 .is_some_and(|throttle| {
9213 throttle.schema_response_active()
9214 && throttle.schema_response_phase
9215 == SchemaResponsePhase::PrefixBuilding
9216 && throttle.schema_response_packet_id == claim
9217 })
9218 {
9219 return Ok(());
9220 }
9221 let discovery_in_flight = skip_in_flight_sides
9222 && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
9223 *pending_side == side_id
9224 && !tx_state.sent.is_empty()
9225 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
9226 });
9227 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
9228 {
9229 continue;
9230 }
9231 let level = if discovery_in_flight {
9235 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
9236 if !throttle.liveness_due(now_ms) {
9237 None
9238 } else {
9239 throttle.next_ping_ms =
9240 now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
9241 Some(DiscoveryAdvertiseLevel::MinimalPing)
9242 }
9243 } else {
9244 Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
9245 };
9246 let Some(level) = level else {
9247 continue;
9248 };
9249 let include_side_topology = include_topology
9250 && self.route_allowed_locked(
9251 &st,
9252 None,
9253 Some(DataType::DiscoveryTopology),
9254 side_id,
9255 );
9256 let capabilities = opts.link_capabilities();
9257 if level == DiscoveryAdvertiseLevel::MinimalPing {
9258 per_side = Some((
9259 side_id,
9260 level,
9261 Vec::new(),
9262 Vec::new(),
9263 Vec::new(),
9264 Vec::new(),
9265 Vec::new(),
9266 capabilities,
9267 local_is_master,
9268 ));
9269 continue;
9270 }
9271 let current_topology = if include_side_topology {
9272 self.advertised_discovery_topology_for_link_locked(
9273 &st,
9274 now_ms,
9275 link_local_enabled,
9276 Some(side_id),
9277 )
9278 } else {
9279 Vec::new()
9280 };
9281 let (endpoints, timesync_sources) = if include_side_topology {
9282 self.advertised_summaries_from_topology_locked(
9283 &st,
9284 now_ms,
9285 link_local_enabled,
9286 Some(side_id),
9287 ¤t_topology,
9288 )
9289 } else {
9290 (
9291 self.advertised_discovery_endpoints_for_link_locked(
9292 &st,
9293 now_ms,
9294 link_local_enabled,
9295 Some(side_id),
9296 ),
9297 self.advertised_discovery_timesync_sources_for_link_locked(
9298 &st,
9299 now_ms,
9300 Some(side_id),
9301 ),
9302 )
9303 };
9304 let (topology, removed_topology) = match level {
9305 DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
9306 DiscoveryAdvertiseLevel::Incremental => {
9307 let previous = &st
9308 .discovery_side_throttle
9309 .get(&side_id)
9310 .expect("discovery throttle exists")
9311 .last_topology;
9312 discovery::topology_delta(previous, ¤t_topology)
9313 }
9314 DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
9315 };
9316 let throttle = st
9317 .discovery_side_throttle
9318 .get_mut(&side_id)
9319 .expect("discovery throttle exists");
9320 if level == DiscoveryAdvertiseLevel::Full
9321 || (level == DiscoveryAdvertiseLevel::Incremental
9322 && throttle.incremental_retries_remaining == 0)
9323 {
9324 throttle.last_topology = current_topology;
9325 }
9326 per_side = Some((
9327 side_id,
9328 level,
9329 endpoints,
9330 self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
9331 timesync_sources,
9332 topology,
9333 removed_topology,
9334 capabilities,
9335 local_is_master,
9336 ));
9337 }
9338 per_side
9339 };
9340 if let Some((
9341 side_id,
9342 level,
9343 endpoints,
9344 network_variables,
9345 timesync_sources,
9346 topology,
9347 removed_topology,
9348 capabilities,
9349 local_is_master,
9350 )) = per_side
9351 {
9352 let sender = self.sender_arc();
9353 if matches!(
9354 level,
9355 DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
9356 ) && (self.cfg.role.local_services() || level == DiscoveryAdvertiseLevel::Full)
9357 {
9358 let pkt = if self.cfg.role.local_services() {
9359 let address = self.local_address_advertisement(
9360 endpoints.clone(),
9361 network_variables.clone(),
9362 timesync_sources.clone(),
9363 capabilities,
9364 if local_is_master {
9365 discovery::ADDRESS_STATE_APPROVED
9366 } else {
9367 discovery::ADDRESS_STATE_REQUEST
9368 },
9369 );
9370 discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?
9371 } else {
9372 discovery::build_discovery_link_capabilities(
9373 sender.as_ref(),
9374 now_ms,
9375 capabilities,
9376 )?
9377 };
9378 self.emit_discovery_prefix_item(
9379 RouterTxItem::ToSide {
9380 src: None,
9381 dst: side_id,
9382 data: RouterItem::Packet(pkt),
9383 },
9384 true,
9385 called_from_queue,
9386 prefix_claim.map(|claim| (side_id, claim)),
9387 )?;
9388 }
9389 if level == DiscoveryAdvertiseLevel::MinimalPing
9390 || (!self.cfg.role.local_services()
9391 && (!endpoints.is_empty() || level != DiscoveryAdvertiseLevel::Full))
9392 {
9393 let pkt = discovery::build_discovery_announce(
9394 sender.as_ref(),
9395 now_ms,
9396 endpoints.as_slice(),
9397 )?;
9398 self.emit_discovery_prefix_item(
9399 RouterTxItem::ToSide {
9400 src: None,
9401 dst: side_id,
9402 data: RouterItem::Packet(pkt),
9403 },
9404 true,
9405 called_from_queue,
9406 prefix_claim.map(|claim| (side_id, claim)),
9407 )?;
9408 }
9409 if !self.cfg.role.local_services()
9410 && level != DiscoveryAdvertiseLevel::MinimalPing
9411 && !timesync_sources.is_empty()
9412 {
9413 let pkt = discovery::build_discovery_timesync_sources(
9414 sender.as_ref(),
9415 now_ms,
9416 ×ync_sources,
9417 )?;
9418 self.emit_discovery_prefix_item(
9419 RouterTxItem::ToSide {
9420 src: None,
9421 dst: side_id,
9422 data: RouterItem::Packet(pkt),
9423 },
9424 true,
9425 called_from_queue,
9426 prefix_claim.map(|claim| (side_id, claim)),
9427 )?;
9428 }
9429 if include_topology
9430 && (!topology.is_empty() || !removed_topology.is_empty())
9431 && level != DiscoveryAdvertiseLevel::MinimalPing
9432 {
9433 let pkt = if level == DiscoveryAdvertiseLevel::Full {
9434 discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
9435 } else {
9436 discovery::build_discovery_topology_delta(
9437 sender.as_ref(),
9438 now_ms,
9439 &topology,
9440 &removed_topology,
9441 )?
9442 };
9443 self.emit_discovery_prefix_item(
9444 RouterTxItem::ToSide {
9445 src: None,
9446 dst: side_id,
9447 data: RouterItem::Packet(pkt),
9448 },
9449 true,
9450 called_from_queue,
9451 prefix_claim.map(|claim| (side_id, claim)),
9452 )?;
9453 }
9454 drop((
9457 endpoints,
9458 network_variables,
9459 timesync_sources,
9460 topology,
9461 removed_topology,
9462 ));
9463 }
9464 Ok(())
9465 }
9466
9467 #[cfg(feature = "discovery")]
9468 fn emit_discovery_prefix_item(
9469 &self,
9470 item: RouterTxItem,
9471 ignore_local: bool,
9472 called_from_queue: bool,
9473 claim: Option<(RouterSideId, u64)>,
9474 ) -> TelemetryResult<()> {
9475 let Some((side, claim)) = claim else {
9476 return self.emit_internal_tx(item, ignore_local, called_from_queue);
9477 };
9478 let priority = match &item {
9479 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
9480 _ => return Err(TelemetryError::BadArg),
9481 };
9482 let mut st = self.state.lock();
9483 if !st
9484 .discovery_side_throttle
9485 .get(&side)
9486 .is_some_and(|throttle| {
9487 throttle.schema_response_active()
9488 && throttle.schema_response_phase == SchemaResponsePhase::PrefixBuilding
9489 && throttle.schema_response_packet_id == claim
9490 })
9491 {
9492 return Err(TelemetryError::Io("schema prefix cancelled"));
9493 }
9494 st.push_transmit(TxQueued {
9495 disposable: TX_DISPOSABLE,
9496 item,
9497 ignore_local,
9498 priority,
9499 delivery_state: QueuedDelivery::Untracked,
9500 accepted_type: 0,
9501 deferred_since_ms: 0,
9502 retry_after_ms: 0,
9503 })
9504 }
9505
9506 #[cfg(feature = "discovery")]
9507 fn queue_discovery_announce(
9508 &self,
9509 include_schema: bool,
9510 skip_in_flight_sides: bool,
9511 ) -> TelemetryResult<()> {
9512 let now_ms = self.clock.now_ms();
9513 {
9514 let mut st = self.state.lock();
9515 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9516 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9517 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9518 }
9519 st.fit_discovery_budget();
9520 if st.sides.iter().all(|side| side.is_none()) {
9521 return Ok(());
9522 }
9523 st.discovery_cadence.on_announce_sent(now_ms);
9526 }
9527 self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
9528 }
9529
9530 #[cfg(feature = "discovery")]
9531 fn missing_schema_request_targets(
9532 st: &RouterInner,
9533 side: RouterSideId,
9534 now: u64,
9535 ) -> ([u32; discovery::SCHEMA_REQUEST_MAX_TARGETS], usize, bool) {
9536 let mut targets = [0u32; discovery::SCHEMA_REQUEST_MAX_TARGETS];
9537 let mut count = 0;
9538 let mut legacy_broadcast = false;
9539 let cursor = st
9540 .discovery_side_throttle
9541 .get(&side)
9542 .map_or(0, |v| v.schema_request_cursor);
9543 let order = |address: u32| {
9544 let delta = address.wrapping_sub(cursor);
9545 if delta == 0 {
9546 1u64 << 32
9547 } else {
9548 u64::from(delta)
9549 }
9550 };
9551 if let Some(route) = st.discovery_routes.get(&side) {
9552 for (name, peer) in &route.announcers {
9553 if !peer.has_full_topology
9554 || peer.has_schema
9555 || now.saturating_sub(peer.last_seen_ms) >= DISCOVERY_ROUTE_TTL_MS
9556 {
9557 continue;
9558 }
9559 if st.side_transport.get(&side).is_some_and(|transport| {
9560 transport.rx_chunks.values().any(|assembly| {
9561 assembly.schema_progress_for(sender_address_u32(name), now)
9562 || peer
9563 .advertised_address
9564 .is_some_and(|a| assembly.schema_progress_for(a, now))
9565 })
9566 }) {
9567 continue;
9568 }
9569 let address = peer
9570 .advertised_address
9571 .filter(|a| *a != 0)
9572 .unwrap_or_else(|| sender_address_u32(name));
9573 if address == 0 {
9574 legacy_broadcast = true;
9577 continue;
9578 }
9579 if targets[..count].contains(&address) {
9580 continue;
9581 }
9582 let position = targets[..count]
9583 .iter()
9584 .position(|old| order(address) < order(*old))
9585 .unwrap_or(count);
9586 if position >= targets.len() {
9587 continue;
9588 }
9589 let next = (count + 1).min(targets.len());
9590 for i in ((position + 1)..next).rev() {
9591 targets[i] = targets[i - 1];
9592 }
9593 targets[position] = address;
9594 count = next;
9595 }
9596 }
9597 (targets, count, legacy_broadcast)
9598 }
9599
9600 #[cfg(feature = "discovery")]
9601 fn note_schema_received(&self, side: RouterSideId, sender: &str) {
9602 let mut st = self.state.lock();
9603 let address = sender
9607 .strip_prefix("@addr:")
9608 .and_then(|value| value.parse::<u32>().ok());
9609 if let Some(route) = st.discovery_routes.get_mut(&side) {
9610 for (name, peer) in &mut route.announcers {
9611 if name == sender
9612 || address.is_some_and(|address| {
9613 peer.advertised_address == Some(address)
9614 || sender_address_u32(name) == address
9615 })
9616 {
9617 peer.has_schema = true;
9618 break;
9619 }
9620 }
9621 }
9622 }
9623
9624 #[cfg(feature = "discovery")]
9627 fn start_schema_response_generation(
9628 &self,
9629 side: RouterSideId,
9630 explicit: bool,
9631 called_from_queue: bool,
9632 include_topology: bool,
9633 skip_in_flight_sides: bool,
9634 ) -> TelemetryResult<bool> {
9635 let now = self.clock.now_ms();
9636 let claim = {
9637 let mut st = self.state.lock();
9638 let throttle = st.discovery_side_throttle.entry(side).or_default();
9639 if throttle.schema_response_active() {
9640 if explicit {
9641 throttle.set_schema_response_pending(true);
9642 }
9643 return Ok(false);
9644 }
9645 if !explicit
9646 && (!throttle.schema_response_pending() || now < throttle.next_schema_response_ms)
9647 {
9648 return Ok(false);
9649 }
9650 let claim = st
9651 .schema_response_sequence
9652 .checked_add(1)
9653 .ok_or(TelemetryError::Io("schema generation exhausted"))?;
9654 st.schema_response_sequence = claim;
9655 let throttle = st
9656 .discovery_side_throttle
9657 .get_mut(&side)
9658 .expect("schema throttle");
9659 throttle.set_schema_response_pending(false);
9660 throttle.set_schema_response_active(true);
9661 throttle.schema_response_phase = SchemaResponsePhase::PrefixBuilding;
9662 throttle.schema_response_packet_id = claim;
9663 throttle.set_schema_response_started(false);
9664 throttle.next_schema_response_ms =
9665 now.saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
9666 throttle.set_has_sent_full(false);
9667 throttle.last_topology = Vec::new();
9668 if let Some(transport) = st.side_transport.get_mut(&side) {
9669 transport.refresh_transmit_header_templates();
9670 }
9671 claim
9672 };
9673 let prefix = self.emit_discovery_snapshot_on_side_claimed(
9677 called_from_queue,
9678 false,
9679 include_topology,
9680 skip_in_flight_sides,
9681 Some(side),
9682 Some(claim),
9683 );
9684 {
9685 let mut st = self.state.lock();
9686 let throttle = st
9687 .discovery_side_throttle
9688 .get_mut(&side)
9689 .ok_or(TelemetryError::BadArg)?;
9690 if !throttle.schema_response_active()
9691 || throttle.schema_response_phase != SchemaResponsePhase::PrefixBuilding
9692 || throttle.schema_response_packet_id != claim
9693 {
9694 return Ok(false);
9695 }
9696 if let Err(error) = prefix {
9697 throttle.set_schema_response_active(false);
9698 throttle.schema_response_phase = SchemaResponsePhase::Idle;
9699 throttle.set_schema_response_pending(true);
9700 return Err(error);
9701 }
9702 if !throttle.has_sent_full() {
9703 throttle.set_schema_response_active(false);
9705 throttle.schema_response_phase = SchemaResponsePhase::Idle;
9706 return Ok(true);
9707 }
9708 throttle.schema_response_packet_id = 0;
9709 throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
9710 }
9711 self.admit_schema_response_catalog(side)?;
9712 Ok(true)
9713 }
9714
9715 #[cfg(feature = "discovery")]
9716 fn admit_schema_response_catalog(&self, side: RouterSideId) -> TelemetryResult<bool> {
9717 self.admit_schema_response_catalog_with(side, || {
9718 discovery::build_discovery_schema(self.sender_arc().as_ref(), self.clock.now_ms())
9719 })
9720 }
9721
9722 #[cfg(feature = "discovery")]
9723 fn admit_schema_response_catalog_with(
9724 &self,
9725 side: RouterSideId,
9726 build: impl FnOnce() -> TelemetryResult<Packet>,
9727 ) -> TelemetryResult<bool> {
9728 let claim = {
9729 let mut st = self.state.lock();
9730 let throttle = st
9731 .discovery_side_throttle
9732 .get(&side)
9733 .ok_or(TelemetryError::BadArg)?;
9734 if !throttle.schema_response_active()
9735 || throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting
9736 {
9737 return Ok(false);
9738 }
9739 let protected_full = st.transmit_queue.len() >= st.transmit_queue.capacity()
9740 && !RouterInner::transmit_eviction_candidate(&st.transmit_queue)
9741 .and_then(|index| st.transmit_queue.get(index))
9742 .is_some_and(|queued| {
9743 queued.priority < crate::transport_priority(DataType::DiscoverySchema)
9744 });
9745 if protected_full
9746 || !self.route_allowed_locked(&st, None, Some(DataType::DiscoverySchema), side)
9747 {
9748 return Ok(false);
9749 }
9750 let claim = st
9751 .schema_response_sequence
9752 .checked_add(1)
9753 .ok_or(TelemetryError::Io("schema generation exhausted"))?;
9754 st.schema_response_sequence = claim;
9755 let throttle = st
9756 .discovery_side_throttle
9757 .get_mut(&side)
9758 .expect("schema throttle");
9759 throttle.schema_response_phase = SchemaResponsePhase::CatalogBuilding;
9760 throttle.schema_response_packet_id = claim;
9761 claim
9762 };
9763 let built = crate::memory_admission::check(8192, 2048).and_then(|_| build());
9767 let prepared = built.and_then(|packet| {
9768 let item = RouterTxItem::ToSide {
9769 src: None,
9770 dst: side,
9771 data: RouterItem::Packet(packet),
9772 };
9773 let priority = match &item {
9774 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
9775 _ => unreachable!(),
9776 };
9777 Ok((item, priority))
9778 });
9779 let mut st = self.state.lock();
9780 let throttle = st
9781 .discovery_side_throttle
9782 .get(&side)
9783 .ok_or(TelemetryError::BadArg)?;
9784 if !throttle.schema_response_active()
9785 || throttle.schema_response_phase != SchemaResponsePhase::CatalogBuilding
9786 || throttle.schema_response_packet_id != claim
9787 {
9788 return Ok(false);
9789 }
9790 let result = prepared.and_then(|(item, priority)| {
9793 let packet_id = Self::get_hash(match &item {
9794 RouterTxItem::ToSide { data, .. } => data,
9795 _ => unreachable!(),
9796 });
9797 let throttle = st
9798 .discovery_side_throttle
9799 .get_mut(&side)
9800 .expect("schema throttle");
9801 throttle.schema_response_packet_id = packet_id;
9802 throttle.schema_response_phase = SchemaResponsePhase::Owned;
9803 st.push_transmit(TxQueued {
9804 disposable: TX_DISPOSABLE,
9805 item,
9806 ignore_local: true,
9807 priority,
9808 delivery_state: QueuedDelivery::Untracked,
9809 accepted_type: 0,
9810 deferred_since_ms: 0,
9811 retry_after_ms: 0,
9812 })
9813 });
9814 let throttle = st
9815 .discovery_side_throttle
9816 .get_mut(&side)
9817 .expect("schema throttle");
9818 match result {
9819 Ok(()) => Ok(true),
9820 Err(TelemetryError::Io(_)) => {
9821 throttle.schema_response_packet_id = 0;
9822 throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
9823 Ok(false)
9824 }
9825 Err(error) => {
9826 throttle.set_schema_response_active(false);
9827 throttle.schema_response_phase = SchemaResponsePhase::Idle;
9828 throttle.set_schema_response_pending(true);
9829 Err(error)
9830 }
9831 }
9832 }
9833
9834 #[cfg(feature = "discovery")]
9835 fn poll_schema_response_on_side(&self, side: RouterSideId) -> TelemetryResult<bool> {
9836 let waiting = {
9837 let st = self.state.lock();
9838 let Some(throttle) = st.discovery_side_throttle.get(&side) else {
9839 return Ok(false);
9840 };
9841 if throttle.schema_response_active() {
9842 if throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting {
9843 return Ok(false);
9844 }
9845 true
9846 } else {
9847 if !throttle.schema_response_pending()
9848 || self.clock.now_ms() < throttle.next_schema_response_ms
9849 {
9850 return Ok(false);
9851 }
9852 if st.transmit_queue.iter().any(|entry| {
9854 matches!(&entry.item,
9855 RouterTxItem::ToSide {dst, data, ..} if *dst == side
9856 && Self::item_data_type(data).ok() == Some(DataType::DiscoverySchema))
9857 }) {
9858 return Ok(false);
9859 }
9860 false
9861 }
9862 };
9863 if waiting {
9864 self.admit_schema_response_catalog(side)
9865 } else {
9866 self.start_schema_response_generation(side, false, true, true, false)
9867 }
9868 }
9869
9870 #[cfg(feature = "discovery")]
9871 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
9872 let now_ms = self.clock.now_ms();
9873 let recovery_sides = {
9877 let mut st = self.state.lock();
9878 let mut due = Vec::new();
9879 for side in 0..st.sides.len() {
9880 let missing = st.discovery_routes.get(&side).is_some_and(|route| {
9881 route.announcers.iter().any(|(name, peer)| {
9882 !peer.has_full_topology
9883 && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
9884 && !route.announcers.values().any(|bridge| {
9885 bridge.has_full_topology
9886 && now_ms.saturating_sub(bridge.last_seen_ms)
9887 < DISCOVERY_ROUTE_TTL_MS
9888 && bridge
9889 .topology_boards
9890 .iter()
9891 .any(|board| board.sender_id == *name)
9892 })
9893 })
9894 });
9895 if !missing {
9896 continue;
9897 }
9898 let throttle = st.discovery_side_throttle.entry(side).or_default();
9899 if throttle.next_topology_request_ms == 0 {
9900 throttle.next_topology_request_ms =
9901 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
9902 } else if now_ms >= throttle.next_topology_request_ms {
9903 throttle.next_topology_request_ms =
9904 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
9905 due.push(side);
9906 }
9907 }
9908 due
9909 };
9910 let recovering = !recovery_sides.is_empty();
9911 for side in recovery_sides {
9912 let pkt =
9913 discovery::build_discovery_topology_request(self.sender_arc().as_ref(), now_ms)?;
9914 self.emit_internal_tx(
9915 RouterTxItem::ToSide {
9916 src: None,
9917 dst: side,
9918 data: RouterItem::Packet(pkt),
9919 },
9920 true,
9921 true,
9922 )?;
9923 }
9924 let schema_recovery_sides = {
9925 let mut st = self.state.lock();
9926 let mut due = Vec::new();
9927 for side in 0..st.sides.len() {
9928 if st
9931 .discovery_side_throttle
9932 .get(&side)
9933 .is_some_and(|throttle| {
9934 throttle.next_schema_request_ms != 0
9935 && now_ms < throttle.next_schema_request_ms
9936 })
9937 {
9938 continue;
9939 }
9940 let (_, count, legacy) = Self::missing_schema_request_targets(&st, side, now_ms);
9941 let missing = count != 0 || legacy;
9942 if !missing {
9943 continue;
9944 }
9945 let throttle = st.discovery_side_throttle.entry(side).or_default();
9946 if throttle.next_schema_request_ms == 0 {
9947 throttle.next_schema_request_ms =
9948 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
9949 } else if now_ms >= throttle.next_schema_request_ms {
9950 throttle.next_schema_request_ms =
9951 now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
9952 due.push(side);
9953 }
9954 }
9955 due
9956 };
9957 let recovering = recovering || !schema_recovery_sides.is_empty();
9958 for side in schema_recovery_sides {
9959 let (addresses, count, legacy) = {
9960 let st = self.state.lock();
9961 Self::missing_schema_request_targets(&st, side, now_ms)
9962 };
9963 if count == 0 && !legacy {
9964 continue;
9965 }
9966 let pkt = if legacy {
9967 discovery::build_discovery_schema_request(self.sender_arc().as_ref(), now_ms)?
9968 } else {
9969 discovery::build_scoped_discovery_schema_request(
9970 self.sender_arc().as_ref(),
9971 now_ms,
9972 &addresses[..count],
9973 )?
9974 };
9975 self.emit_internal_tx(
9976 RouterTxItem::ToSide {
9977 src: None,
9978 dst: side,
9979 data: RouterItem::Packet(pkt),
9980 },
9981 true,
9982 true,
9983 )?;
9984 if !legacy
9985 && let Some(throttle) = self.state.lock().discovery_side_throttle.get_mut(&side)
9986 {
9987 throttle.schema_request_cursor = addresses[count - 1];
9988 }
9989 }
9990 let due = {
9991 let mut st = self.state.lock();
9992 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
9993 if removed {
9994 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9995 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9996 }
9997 st.fit_discovery_budget();
9998 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
9999 let Some(side) = side.as_ref() else {
10000 return false;
10001 };
10002 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
10003 {
10004 return false;
10005 }
10006 let _ = side;
10007 let in_flight = st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
10008 *pending_side == side_id
10009 && !tx_state.sent.is_empty()
10010 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
10011 });
10012 st.discovery_side_throttle
10013 .get(&side_id)
10014 .is_none_or(|throttle| {
10015 (!in_flight
10016 && (!throttle.has_sent_full() || throttle.pending_incremental()))
10017 || throttle.liveness_due(now_ms)
10018 })
10019 });
10020 if st.sides.is_empty() || !has_any {
10021 return Ok(recovering);
10022 }
10023 st.discovery_cadence.due(now_ms)
10024 };
10025 if !due {
10026 return Ok(recovering);
10027 }
10028 self.queue_discovery_announce(false, true)?;
10037 Ok(true)
10038 }
10039
10040 #[cfg(feature = "discovery")]
10041 fn learn_discovery_packet(
10042 &self,
10043 pkt: &Packet,
10044 src: Option<RouterSideId>,
10045 called_from_queue: bool,
10046 ) -> TelemetryResult<bool> {
10047 if !discovery::is_discovery_type(pkt.data_type()) {
10048 return Ok(false);
10049 }
10050 let Some(side) = src else {
10051 return Ok(true);
10052 };
10053 #[cfg(feature = "compression")]
10054 self.note_payload_codec_packet(side, pkt, false);
10055 if self.refresh_known_discovery_liveness(pkt, side)? {
10056 return Ok(true);
10057 }
10058 #[cfg(not(feature = "std"))]
10059 if pkt.data_type() == crate::DataType::DiscoverySchema {
10060 self.merge_schema_changes(pkt.payload())?;
10061 self.note_schema_received(side, pkt.sender());
10062 return Ok(true);
10063 }
10064 if pkt.data_type() == DataType::DiscoverySchemaRequest {
10065 if let Some(targets) = discovery::schema_request_targets(pkt.payload()) {
10066 let assigned = self.state.lock().local_address.address;
10067 let legacy = sender_address_u32(self.sender_arc().as_ref());
10068 if !(0..targets.len() / 4).any(|index| {
10069 let bytes = &targets[index * 4..index * 4 + 4];
10070 let address =
10071 u32::from_le_bytes(bytes.try_into().expect("validated address width"));
10072 address == assigned || address == legacy
10073 }) {
10074 return Ok(true);
10077 }
10078 }
10079 self.state
10080 .lock()
10081 .discovery_side_throttle
10082 .entry(side)
10083 .or_default()
10084 .set_schema_response_pending(true);
10085 self.poll_schema_response_on_side(side)?;
10086 return Ok(true);
10087 }
10088 if matches!(
10092 pkt.data_type(),
10093 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
10094 ) {
10095 crate::memory_admission::check(1024, 256)?;
10096 if let Some(transport) = self.state.lock().side_transport.get_mut(&side) {
10101 transport.refresh_transmit_header_templates();
10102 }
10103 }
10104 if matches!(
10105 pkt.data_type(),
10106 DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
10107 ) {
10108 let now_ms = self.clock.now_ms();
10109 {
10110 let mut st = self.state.lock();
10111 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
10112 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10113 Self::note_discovery_topology_change_locked(&mut st, now_ms);
10114 }
10115 let throttle = st.discovery_side_throttle.entry(side).or_default();
10116 throttle.set_has_sent_full(false);
10117 throttle.last_topology = Vec::new();
10121 }
10122 self.emit_discovery_snapshot_on_side(
10127 called_from_queue,
10128 pkt.data_type() == DataType::DiscoverySchemaRequest,
10129 true,
10130 false,
10131 Some(side),
10132 )?;
10133 return Ok(true);
10134 }
10135 let small_topology_scratch =
10136 if pkt.data_type() == DataType::DiscoveryTopology && pkt.payload().len() <= 128 {
10137 let st = self.state.lock();
10138 let sender = pkt
10139 .sender()
10140 .strip_prefix("@addr:")
10141 .and_then(|value| value.parse::<u32>().ok())
10142 .and_then(|address| Self::hostname_for_address_borrowed_locked(&st, address))
10143 .unwrap_or(pkt.sender());
10144 st.discovery_routes.get(&side).and_then(|route| {
10145 let peer = route.announcers.get(sender)?;
10146 let mut shallow = discovery::CloneScratch::default();
10147 shallow.buffer(route.reachable.len().saturating_mul(4));
10148 shallow.buffer(route.reachable_network_variables.len().saturating_mul(4));
10149 shallow.buffer(
10150 route
10151 .requested_network_variables
10152 .len()
10153 .saturating_mul(core::mem::size_of::<ManagedVariableSubscription>()),
10154 );
10155 shallow.strings(&route.reachable_timesync_sources);
10156 for (name, value) in &route.announcers {
10157 shallow.buffer(name.len());
10158 shallow.buffer(value.reachable.len().saturating_mul(4));
10159 shallow.buffer(value.advertised_reachable.len().saturating_mul(4));
10160 shallow.strings(&value.reachable_timesync_sources);
10161 shallow.strings(&value.advertised_reachable_timesync_sources);
10162 shallow.buffer(value.reachable_network_variables.len().saturating_mul(4));
10163 }
10164 let entry = core::mem::size_of::<String>()
10165 + crate::peer_record::allocation_bytes::<DiscoverySenderState>();
10166 shallow.bytes = shallow
10167 .bytes
10168 .saturating_add(route.announcers.len().saturating_mul(entry));
10169 shallow.largest = shallow
10170 .largest
10171 .max(entry.saturating_mul(11).saturating_add(128));
10172 let mut deep = discovery::CloneScratch::default();
10173 if discovery::topology_payload_is_incremental(pkt.payload()) {
10174 deep.graph(&peer.topology_boards, pkt.payload().len() / 16);
10175 }
10176 Some((
10177 2048usize
10178 .saturating_add(shallow.bytes.saturating_mul(3))
10179 .saturating_add(deep.bytes)
10180 .saturating_add(pkt.payload().len().saturating_mul(4)),
10181 shallow.largest.max(deep.largest).max(2048),
10182 ))
10183 })
10184 } else {
10185 None
10186 };
10187 let retained_summary = if small_topology_scratch.is_none()
10188 && matches!(
10189 pkt.data_type(),
10190 DataType::DiscoveryAddress
10191 | DataType::DiscoveryTopology
10192 | DataType::DiscoveryAnnounce
10193 | DataType::DiscoveryTimeSyncSources
10194 | DataType::ManagedVariableRequest
10195 ) {
10196 self.state
10197 .lock()
10198 .discovery_routes
10199 .get(&side)
10200 .map_or(0, |route| {
10201 RouterInner::discovery_route_byte_cost(side, route)
10202 })
10203 } else {
10204 0
10205 };
10206 let (decode_bytes, largest_block) = if let Some(scratch) = small_topology_scratch {
10207 scratch
10208 } else if pkt.data_type() == DataType::ManagedVariableRequest {
10209 let ty = discovery::decode_managed_variable_request(pkt)?;
10210 let existing = self
10211 .state
10212 .lock()
10213 .discovery_routes
10214 .get(&side)
10215 .is_some_and(|route| {
10216 route
10217 .requested_network_variables
10218 .iter()
10219 .any(|request| request.ty == ty)
10220 });
10221 if existing {
10222 (1024, 256)
10223 } else {
10224 (
10227 2048usize.saturating_add(retained_summary.saturating_mul(2)),
10228 retained_summary
10229 .checked_next_power_of_two()
10230 .unwrap_or(usize::MAX)
10231 .max(1024),
10232 )
10233 }
10234 } else {
10235 let scratch = if pkt.data_type() == DataType::DiscoveryAddress {
10236 4096usize.saturating_add(retained_summary)
10237 } else {
10238 8192usize.saturating_add(retained_summary)
10239 };
10240 let decode_bytes = pkt
10241 .payload()
10242 .len()
10243 .saturating_mul(4)
10244 .saturating_add(scratch);
10245 let largest_block = pkt.payload().len().saturating_mul(2).max(2048);
10246 (decode_bytes, largest_block)
10247 };
10248 while let Err(err) = crate::memory_admission::check(decode_bytes, largest_block) {
10249 if !matches!(err, TelemetryError::Io("memory pressure")) {
10250 return Err(err);
10251 }
10252 let mut st = self.state.lock();
10253 #[cfg(feature = "compression")]
10254 if st.reclaim_codec_metadata() {
10255 continue;
10256 }
10257
10258 let Some(victim) = st.lower_priority_shared_queue(255) else {
10259 return Err(err);
10260 };
10261 if !st.pop_shared_queue_item(victim) {
10262 return Err(err);
10263 }
10264 }
10265 if pkt.data_type() == DataType::DiscoveryAddress {
10266 let mut ad = discovery::decode_discovery_address(pkt)?;
10267 let identity = ad.clone();
10268 let mut changed = false;
10269 let sender_id = if ad.hostname.is_empty() {
10275 pkt.sender()
10276 } else {
10277 ad.hostname.as_str()
10278 };
10279 let now_ms = self.clock.now_ms();
10280 let mut st = self.state.lock();
10281 let side_link_local_enabled = st
10282 .sides
10283 .get(side)
10284 .and_then(|entry| entry.as_ref())
10285 .map(|side_ref| side_ref.opts.link_local_enabled)
10286 .unwrap_or(false);
10287 if !side_link_local_enabled {
10288 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
10289 }
10290 let mut route = st
10291 .discovery_routes
10292 .get(&side)
10293 .map(DiscoverySideState::try_clone)
10294 .transpose()?
10295 .unwrap_or_default();
10296 let previously_reachable_network_variables = route.reachable_network_variables.clone();
10297 if pkt.sender() != sender_id {
10298 route.announcers.remove(pkt.sender());
10299 }
10300 let mut sender_state = match route.announcers.remove(sender_id) {
10301 Some(record) => record,
10302 None => crate::peer_record::try_box(DiscoverySenderState::default())?,
10303 };
10304 if sender_state.advertised_address != Some(ad.address) {
10310 sender_state.advertised_address = Some(ad.address);
10311 changed = true;
10312 }
10313 if sender_state.advertised_reachable != ad.reachable_endpoints {
10314 sender_state.advertised_reachable = ad.reachable_endpoints;
10315 changed = true;
10316 }
10317 if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
10318 sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
10319 changed = true;
10320 }
10321 if sender_state.reachable_network_variables != ad.reachable_network_variables {
10322 sender_state.reachable_network_variables = ad.reachable_network_variables;
10323 changed = true;
10324 }
10325 if sender_state.link_capabilities != Some(ad.link_capabilities) {
10326 sender_state.link_capabilities = Some(ad.link_capabilities);
10327 changed = true;
10328 }
10329 Self::refresh_sender_topology_state(&mut sender_state);
10330 sender_state.last_seen_ms = now_ms;
10331 route.announcers.insert(sender_id.to_string(), sender_state);
10332 Self::recompute_discovery_side_state(&mut route);
10333 let newly_reachable_network_variables: Vec<DataType> = route
10334 .reachable_network_variables
10335 .iter()
10336 .copied()
10337 .filter(|ty| !previously_reachable_network_variables.contains(ty))
10338 .collect();
10339 let prepared_address = Self::prepare_address_record(&st, &identity, now_ms)?;
10340 st.commit_discovery_route(side, route)?;
10341 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10342 if changed {
10343 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
10344 }
10345 let replay: Vec<Packet> = newly_reachable_network_variables
10351 .iter()
10352 .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
10353 .filter_map(|ty| {
10354 st.local_services
10355 .as_ref()?
10356 .managed_variable_latest
10357 .get(&ty.as_u32())
10358 .map(|entry| entry.packet.clone())
10359 })
10360 .collect();
10361 drop(st);
10362 self.ingest_prepared_address(identity, prepared_address)?;
10363 #[cfg(feature = "compression")]
10364 self.note_payload_codec_packet(side, pkt, true);
10365 for value in replay {
10366 self.emit_internal_tx_with_priority(
10367 RouterTxItem::ToSide {
10368 src: None,
10369 dst: side,
10370 data: RouterItem::Packet(value),
10371 },
10372 true,
10373 crate::transport_priority(DataType::ManagedVariableValue),
10374 called_from_queue,
10375 )?;
10376 }
10377 return Ok(true);
10378 }
10379 let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
10380 Some(discovery::decode_discovery_topology_update(pkt)?)
10381 } else {
10382 None
10383 };
10384 let packet_sender = {
10385 let st = self.state.lock();
10386 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
10387 if canonical != pkt.sender() {
10388 canonical
10389 } else if let Some(address) = pkt
10390 .sender()
10391 .strip_prefix("@addr:")
10392 .and_then(|value| value.parse::<u32>().ok())
10393 {
10394 decoded_topology
10400 .as_ref()
10401 .and_then(|update| {
10402 update
10403 .boards
10404 .iter()
10405 .find(|board| sender_address_u32(&board.sender_id) == address)
10406 })
10407 .map(|board| board.sender_id.clone())
10408 .unwrap_or(canonical)
10409 } else {
10410 canonical
10411 }
10412 };
10413 let local_sender = self.sender_arc();
10414 if packet_sender == local_sender.as_ref() {
10415 return Ok(true);
10416 }
10417
10418 if pkt.data_type() == DataType::ManagedVariableRequest {
10419 let ty = discovery::decode_managed_variable_request(pkt)?;
10420 {
10426 let now_ms = self.clock.now_ms();
10427 let mut st = self.state.lock();
10428 let grant = st.local_services.as_ref().and_then(|local| {
10432 if Self::managed_variable_permissions_locked(&st, ty).write {
10433 return None;
10434 }
10435 let entry = local.managed_variable_latest.get(&ty.as_u32())?;
10436 let value_id = entry.packet.packet_id();
10437 let upstream = entry.upstream?.get() - 1;
10438 (upstream != side
10439 && Self::side_ref(&st, upstream)
10440 .is_ok_and(|link| link.opts.ingress_enabled))
10441 .then_some((value_id, upstream, entry.sender_address))
10442 });
10443 let mut request = ManagedVariableSubscription {
10444 last_seen_ms: now_ms,
10445 next_refresh_ms: 0,
10446 request_id: pkt.packet_id(),
10447 value_id: 0,
10448 upstream: side,
10449 ty,
10450 requester: sender_address_u32(pkt.sender()),
10451 writer: 0,
10452 refresh_pending: false,
10453 };
10454 if let Some((value_id, upstream, writer)) = grant {
10455 request.value_id = value_id;
10456 request.upstream = upstream;
10457 request.writer = writer;
10458 request.refresh_pending = true;
10459 }
10460 let existing = st.discovery_routes.get_mut(&side).and_then(|route| {
10461 let previous = route
10462 .requested_network_variables
10463 .iter_mut()
10464 .find(|previous| previous.ty == ty)?;
10465 let repeated = previous.request_id == request.request_id
10466 && previous.requester == request.requester;
10467 let backstep = now_ms < previous.last_seen_ms;
10468 let cooling =
10469 previous.next_refresh_ms == u64::MAX || now_ms < previous.next_refresh_ms;
10470 if backstep {
10471 previous.refresh_pending = false;
10472 } else if !repeated && !cooling && !previous.refresh_pending {
10473 request.next_refresh_ms = previous.next_refresh_ms;
10474 *previous = request;
10475 } else {
10476 previous.last_seen_ms = now_ms;
10480 }
10481 route.last_seen_ms = route.last_seen_ms.max(now_ms);
10482 Some(())
10483 });
10484 let newly_reachable = existing.is_none();
10485 if newly_reachable {
10486 st.make_shared_queue_room(size_of::<ManagedVariableSubscription>(), 255, None)?;
10489 let mut route = st
10490 .discovery_routes
10491 .get(&side)
10492 .map(DiscoverySideState::try_clone)
10493 .transpose()?
10494 .unwrap_or_default();
10495 reserve_managed_subscriptions(&mut route.requested_network_variables, 1)?;
10496 route.requested_network_variables.push(request);
10497 Self::recompute_discovery_side_state(&mut route);
10498 st.commit_discovery_route(side, route)?;
10499 }
10500 if newly_reachable {
10501 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10502 Self::note_discovery_topology_change_locked(&mut st, now_ms);
10503 }
10504 }
10505 if !self.can_write_managed_variable(ty) {
10506 return Ok(false);
10511 }
10512 if let Some(value) = self.managed_variable_latest(ty) {
10513 self.emit_internal_tx(
10514 RouterTxItem::ToSide {
10515 src: None,
10516 dst: side,
10517 data: RouterItem::Packet(value),
10518 },
10519 true,
10520 called_from_queue,
10521 )?;
10522 return Ok(true);
10523 }
10524 return Ok(false);
10527 }
10528 #[cfg(feature = "std")]
10529 if pkt.data_type() == DataType::DiscoverySchema {
10530 #[cfg(feature = "compression")]
10531 let snapshot =
10532 self.decode_required_schema(|| discovery::decode_discovery_schema(pkt))?;
10533 #[cfg(not(feature = "compression"))]
10534 let snapshot = discovery::decode_discovery_schema(pkt)?;
10535 let mut st = self.state.lock();
10544 let budget = st.schema_growth_budget();
10545 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
10546 if report.changed() {
10547 st.fit_discovery_budget();
10548 }
10549 drop(st);
10550 self.note_schema_received(side, &packet_sender);
10551 return Ok(true);
10552 }
10553 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
10554 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
10555 #[cfg(feature = "compression")]
10556 self.note_payload_codec_packet(side, pkt, true);
10557 return Ok(true);
10558 }
10559 let mut announce = if pkt.data_type() == DataType::DiscoveryAnnounce {
10560 Some(discovery::decode_discovery_announce(pkt)?)
10561 } else {
10562 None
10563 };
10564 let mut time_sources = if pkt.data_type() == DataType::DiscoveryTimeSyncSources {
10565 Some(discovery::decode_discovery_timesync_sources(pkt)?)
10566 } else {
10567 None
10568 };
10569 let mut st = self.state.lock();
10570 let now_ms = self.clock.now_ms();
10571 if pkt.data_type() == DataType::DiscoveryLeave {
10572 let leaving = packet_sender.as_str();
10573 let before = st
10574 .discovery_routes
10575 .iter()
10576 .map(|(side, route)| Ok((*side, route.try_clone()?)))
10577 .collect::<TelemetryResult<RouterMap<_, _>>>()?;
10578 for route in st.discovery_routes.values_mut() {
10579 route.announcers.remove(leaving);
10580 for sender_state in route.announcers.values_mut() {
10581 Arc::make_mut(&mut sender_state.topology_boards)
10582 .retain(|board| board.sender_id != leaving);
10583 for board in Arc::make_mut(&mut sender_state.topology_boards).iter_mut() {
10584 board.connections.retain(|peer| peer != leaving);
10585 }
10586 Self::refresh_sender_topology_state(sender_state);
10587 }
10588 Self::recompute_discovery_side_state(route);
10589 }
10590 st.discovery_routes
10591 .retain(|_, route| !route.announcers.is_empty());
10592 if st.discovery_routes != before {
10593 Self::note_discovery_topology_change_locked(&mut st, now_ms);
10594 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10595 }
10596 #[cfg(feature = "compression")]
10597 {
10598 drop(st);
10599 self.note_payload_codec_packet(side, pkt, true);
10600 }
10601 return Ok(true);
10602 }
10603 if let Some(update) = decoded_topology.as_mut() {
10605 for board in update.boards.iter_mut() {
10606 board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
10607 for peer in board.connections.iter_mut() {
10608 *peer = Self::canonical_sender_locked(&st, peer);
10609 }
10610 }
10611 }
10612 let mut route = st
10613 .discovery_routes
10614 .get(&side)
10615 .map(DiscoverySideState::try_clone)
10616 .transpose()?
10617 .unwrap_or_default();
10618 let alias_state = if packet_sender != pkt.sender() {
10619 route.announcers.remove(pkt.sender())
10620 } else {
10621 None
10622 };
10623 let side_link_local_enabled = st
10624 .sides
10625 .get(side)
10626 .and_then(|entry| entry.as_ref())
10627 .map(|side_ref| side_ref.opts.link_local_enabled)
10628 .unwrap_or(false);
10629 let mut sender_state = match route.announcers.remove(&packet_sender).or(alias_state) {
10630 Some(record) => record,
10631 None => crate::peer_record::try_box(DiscoverySenderState::default())?,
10632 };
10633 let changed = match pkt.data_type() {
10634 DataType::DiscoveryAnnounce => {
10635 let mut reachable = announce.take().expect("decoded above");
10636 if reachable.is_empty() {
10641 false
10642 } else {
10643 if !side_link_local_enabled {
10644 reachable.retain(|ep| !ep.is_link_local_only());
10645 }
10646 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
10647 let changed = board.reachable_endpoints != reachable;
10648 board.reachable_endpoints = reachable;
10649 Self::refresh_sender_topology_state(&mut sender_state);
10650 changed
10651 }
10652 }
10653 DataType::DiscoveryTimeSyncSources => {
10654 let sources = time_sources.take().expect("decoded above");
10655 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
10656 let changed = board.reachable_timesync_sources != sources;
10657 board.reachable_timesync_sources = sources;
10658 Self::refresh_sender_topology_state(&mut sender_state);
10659 changed
10660 }
10661 DataType::DiscoveryTopology => {
10662 let mut update = decoded_topology
10663 .take()
10664 .expect("topology packet was decoded before route selection");
10665 if !side_link_local_enabled {
10666 for board in update.boards.iter_mut() {
10667 board
10668 .reachable_endpoints
10669 .retain(|ep| !ep.is_link_local_only());
10670 }
10671 }
10672 if !update.incremental {
10673 sender_state.has_full_topology = true;
10674 }
10675 let changed = discovery::apply_shared_topology_update(
10676 &mut sender_state.topology_boards,
10677 update,
10678 );
10679 Self::refresh_sender_topology_state(&mut sender_state);
10680 changed
10681 }
10682 DataType::DiscoverySchema => false,
10683 _ => false,
10684 };
10685 sender_state.last_seen_ms = now_ms;
10686 route.announcers.insert(packet_sender, sender_state);
10687 Self::recompute_discovery_side_state(&mut route);
10688 st.commit_discovery_route(side, route)?;
10689 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10690 if changed {
10691 Self::note_discovery_topology_change_locked(&mut st, now_ms);
10692 }
10693 Ok(true)
10694 }
10695
10696 #[cfg(not(feature = "discovery"))]
10697 fn learn_discovery_packet(
10698 &self,
10699 _pkt: &Packet,
10700 _src: Option<RouterSideId>,
10701 _called_from_queue: bool,
10702 ) -> TelemetryResult<bool> {
10703 Ok(false)
10704 }
10705
10706 #[inline]
10707 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
10708 (side, ty.as_u32())
10709 }
10710
10711 fn reliable_tx_state_mut<'a>(
10712 &'a self,
10713 st: &'a mut RouterInner,
10714 side: RouterSideId,
10715 ty: DataType,
10716 ) -> &'a mut ReliableTxState {
10717 let key = Self::reliable_key(side, ty);
10718 st.reliable_tx
10719 .entry(key)
10720 .or_insert_with(|| ReliableTxState {
10721 next_seq: 1,
10722 sent_order: VecDeque::new(),
10723 sent: Default::default(),
10724 })
10725 }
10726
10727 fn reliable_rx_state_mut<'a>(
10728 &'a self,
10729 st: &'a mut RouterInner,
10730 side: RouterSideId,
10731 ty: DataType,
10732 ) -> &'a mut ReliableRxState {
10733 let key = Self::reliable_key(side, ty);
10734 st.reliable_rx
10735 .entry(key)
10736 .or_insert_with(|| ReliableRxState {
10737 expected_seq: 1,
10738 buffered: Default::default(),
10739 })
10740 }
10741
10742 fn reliable_control_packet(
10743 &self,
10744 control_ty: DataType,
10745 ty: DataType,
10746 seq: u32,
10747 ) -> TelemetryResult<Packet> {
10748 let mut bytes = [0u8; 8];
10749 bytes[..4].copy_from_slice(&ty.as_u32().to_le_bytes());
10750 bytes[4..].copy_from_slice(&seq.to_le_bytes());
10751 Packet::new_local(
10752 control_ty,
10753 message_meta(control_ty).endpoints_owned(),
10754 self.sender_arc(),
10755 self.packet_timestamp_ms(),
10756 crate::config::StandardSmallPayload::new(&bytes),
10757 )
10758 }
10759
10760 fn queue_reliable_ack(
10761 &self,
10762 side: RouterSideId,
10763 ty: DataType,
10764 seq: u32,
10765 called_from_queue: bool,
10766 ) -> TelemetryResult<()> {
10767 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
10768 self.emit_internal_tx_with_priority(
10769 RouterTxItem::ToSide {
10770 src: None,
10771 dst: side,
10772 data: RouterItem::Packet(pkt),
10773 },
10774 true,
10775 message_priority(DataType::ReliableAck),
10776 called_from_queue,
10777 )
10778 }
10779
10780 fn queue_reliable_packet_request(
10781 &self,
10782 side: RouterSideId,
10783 ty: DataType,
10784 seq: u32,
10785 called_from_queue: bool,
10786 ) -> TelemetryResult<()> {
10787 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
10788 self.emit_internal_tx_with_priority(
10789 RouterTxItem::ToSide {
10790 src: None,
10791 dst: side,
10792 data: RouterItem::Packet(pkt),
10793 },
10794 true,
10795 message_priority(DataType::ReliablePacketRequest),
10796 called_from_queue,
10797 )
10798 }
10799
10800 fn queue_reliable_partial_ack(
10801 &self,
10802 side: RouterSideId,
10803 ty: DataType,
10804 seq: u32,
10805 called_from_queue: bool,
10806 ) -> TelemetryResult<()> {
10807 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
10808 self.emit_internal_tx_with_priority(
10809 RouterTxItem::ToSide {
10810 src: None,
10811 dst: side,
10812 data: RouterItem::Packet(pkt),
10813 },
10814 true,
10815 message_priority(DataType::ReliablePartialAck),
10816 called_from_queue,
10817 )
10818 }
10819
10820 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
10821 let mut st = self.state.lock();
10822 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10823 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
10824 tx_state.sent.remove(&ack);
10825 tx_state.sent_order.retain(|seq| *seq != ack);
10826 return;
10827 }
10828
10829 while let Some(seq) = tx_state.sent_order.front().copied() {
10830 if seq > ack {
10831 break;
10832 }
10833 tx_state.sent_order.pop_front();
10834 tx_state.sent.remove(&seq);
10835 }
10836 }
10837
10838 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
10839 let mut st = self.state.lock();
10840 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10841 if let Some(sent) = tx_state.sent.get_mut(&seq) {
10842 sent.partial_acked = true;
10843 }
10844 }
10845
10846 fn queue_reliable_retransmit(
10847 &self,
10848 side: RouterSideId,
10849 ty: DataType,
10850 seq: u32,
10851 called_from_queue: bool,
10852 ) -> TelemetryResult<()> {
10853 let mut queued = None;
10854 {
10855 let mut st = self.state.lock();
10856 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10857 if let Some(sent) = tx_state.sent.get_mut(&seq)
10858 && !sent.queued
10859 {
10860 sent.partial_acked = false;
10861 queued = Some(sent.bytes.clone());
10862 }
10863 }
10864
10865 if let Some(bytes) = queued {
10866 let now = self.clock.now_ms();
10867 if called_from_queue {
10868 self.tx_queue_item_with_priority(
10869 RouterTxItem::ReliableReplay {
10870 dst: side,
10871 ty,
10872 seq,
10873 bytes,
10874 },
10875 true,
10876 Self::router_item_priority_bumped(ty),
10877 )?;
10878 } else {
10879 self.tx_item_impl(
10880 RouterTxItem::ReliableReplay {
10881 dst: side,
10882 ty,
10883 seq,
10884 bytes,
10885 },
10886 true,
10887 false,
10888 )?;
10889 }
10890 let mut st = self.state.lock();
10891 if let Some(sent) = st
10892 .reliable_tx
10893 .get_mut(&Self::reliable_key(side, ty))
10894 .and_then(|tx| tx.sent.get_mut(&seq))
10895 {
10896 sent.last_send_ms = sent.last_send_ms.max(now);
10897 }
10898 }
10899
10900 Ok(())
10901 }
10902
10903 fn send_reliable_raw_to_side(
10904 &self,
10905 side: RouterSideId,
10906 bytes: crate::SharedBytes,
10907 relayed: bool,
10908 admitted: &mut bool,
10909 ) -> TelemetryResult<()> {
10910 let handler = {
10911 let st = self.state.lock();
10912 let side_ref = Self::side_ref(&st, side)?;
10913 if !side_ref.opts.egress_enabled {
10914 return Ok(());
10915 }
10916 (side_ref.tx_handler.clone(), side_ref.opts.clone())
10917 };
10918
10919 let (handler, opts) = handler;
10920
10921 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10922 return Err(TelemetryError::Io("side tx busy"));
10923 };
10924 let started_ms = self.clock.now_ms();
10925 let (ty, priority) = Self::item_type_priority(&RouterItem::Packed(bytes.clone()))?;
10926 let priority = priority.max(self.effective_transport_priority(ty));
10927 let result = match handler {
10928 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
10929 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
10930 let mut attempts_total = 0usize;
10931 let mut sent_bytes = 0usize;
10932 self.send_side_frames_resumable(side, frames, |frame| {
10933 match self.retry_with_attempts(runtime_max_handler_retries(), || {
10934 self.packed_handler_call(
10935 side,
10936 &packed,
10937 frame,
10938 priority,
10939 EgressService::default(),
10940 )
10941 }) {
10942 Ok((_, attempts)) => {
10943 *admitted = true;
10944 attempts_total = attempts_total.saturating_add(attempts);
10945 sent_bytes = sent_bytes.saturating_add(frame.len());
10946 }
10947 Err((err, attempts)) => {
10948 self.note_side_tx_failure(
10949 side,
10950 ty,
10951 attempts_total.saturating_add(attempts),
10952 );
10953 return Err(err);
10954 }
10955 }
10956
10957 Ok(())
10958 })?;
10959 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10960 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10961 return Ok(());
10962 }
10963 RouterTxHandlerFn::Packet(f) => {
10964 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
10965 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
10966 }
10967 };
10968 match result {
10969 Ok((_, attempts)) => {
10970 *admitted = true;
10971 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
10972 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
10973 Ok(())
10974 }
10975 Err((err, attempts)) => {
10976 self.note_side_tx_failure(side, ty, attempts);
10977 Err(err)
10978 }
10979 }
10980 }
10981
10982 #[allow(unused_mut)]
10983 #[cfg(feature = "compression")]
10984 #[cfg(test)]
10985 fn send_queued_to_side(
10986 &self,
10987 side: RouterSideId,
10988 data: &mut RouterItem,
10989 relayed: bool,
10990 ) -> (
10991 TelemetryResult<()>,
10992 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
10993 ) {
10994 self.send_queued_to_side_with_progress(side, data, relayed, &mut false)
10995 }
10996
10997 #[cfg(feature = "compression")]
10998 fn send_queued_to_side_with_progress(
10999 &self,
11000 side: RouterSideId,
11001 data: &mut RouterItem,
11002 relayed: bool,
11003 admitted: &mut bool,
11004 ) -> (
11005 TelemetryResult<()>,
11006 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
11007 ) {
11008 self.send_queued_to_side_with_service(
11009 side,
11010 data,
11011 relayed,
11012 admitted,
11013 EgressService::default(),
11014 )
11015 }
11016 #[cfg(feature = "compression")]
11017 fn send_queued_to_side_with_service(
11018 &self,
11019 side: RouterSideId,
11020 data: &mut RouterItem,
11021 relayed: bool,
11022 admitted: &mut bool,
11023 service: EgressService,
11024 ) -> (
11025 TelemetryResult<()>,
11026 Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
11027 ) {
11028 let reservation = match self.prepare_queued_payload(side, data) {
11029 Ok(reservation) => reservation,
11030 Err(error) => return (Err(error), None),
11031 };
11032 (
11033 self.send_reliable_to_side_with_service(side, data.clone(), relayed, admitted, service),
11034 reservation,
11035 )
11036 }
11037
11038 #[cfg(test)]
11039 fn send_reliable_to_side(
11040 &self,
11041 side: RouterSideId,
11042 data: RouterItem,
11043 relayed: bool,
11044 ) -> TelemetryResult<()> {
11045 self.send_reliable_to_side_with_progress(side, data, relayed, &mut false)
11046 }
11047
11048 #[cfg(any(test, not(feature = "compression")))]
11049 fn send_reliable_to_side_with_progress(
11050 &self,
11051 side: RouterSideId,
11052 data: RouterItem,
11053 relayed: bool,
11054 admitted: &mut bool,
11055 ) -> TelemetryResult<()> {
11056 self.send_reliable_to_side_with_service(
11057 side,
11058 data,
11059 relayed,
11060 admitted,
11061 EgressService::default(),
11062 )
11063 }
11064 fn send_reliable_to_side_with_service(
11065 &self,
11066 side: RouterSideId,
11067 data: RouterItem,
11068 relayed: bool,
11069 admitted: &mut bool,
11070 service: EgressService,
11071 ) -> TelemetryResult<()> {
11072 #[cfg(feature = "discovery")]
11073 let schema_id = (Self::item_data_type(&data).ok() == Some(DataType::DiscoverySchema))
11074 .then(|| Self::get_hash(&data));
11075 #[cfg(feature = "discovery")]
11076 if let Some(id) = schema_id {
11077 let mut st = self.state.lock();
11078 if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
11079 && throttle.schema_response_active()
11080 && throttle.schema_response_phase == SchemaResponsePhase::Owned
11081 && throttle.schema_response_packet_id == id
11082 && !throttle.schema_response_started()
11083 {
11084 throttle.set_schema_response_started(true);
11085 throttle.next_schema_response_ms = self
11086 .clock
11087 .now_ms()
11088 .saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
11089 if let Some(transport) = st.side_transport.get_mut(&side) {
11094 transport.refresh_transmit_header_templates();
11095 }
11096 }
11097 }
11098 let result = self.send_reliable_to_side_impl(side, data, relayed, admitted, service);
11099 #[cfg(feature = "discovery")]
11100 if let Some(id) = schema_id {
11101 let mut st = self.state.lock();
11102 if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
11103 && throttle.schema_response_active()
11104 && throttle.schema_response_phase == SchemaResponsePhase::Owned
11105 && throttle.schema_response_packet_id == id
11106 {
11107 if !result.as_ref().is_err_and(Self::is_transient_tx_error) {
11110 throttle.set_schema_response_active(false);
11111 throttle.schema_response_phase = SchemaResponsePhase::Idle;
11112 throttle.set_schema_response_pending(
11113 throttle.schema_response_pending() || result.is_err(),
11114 );
11115 }
11116 }
11117 }
11118 result
11119 }
11120
11121 fn send_reliable_to_side_impl(
11122 &self,
11123 side: RouterSideId,
11124 data: RouterItem,
11125 relayed: bool,
11126 admitted: &mut bool,
11127 service: EgressService,
11128 ) -> TelemetryResult<()> {
11129 let (ty, addressed) = match &data {
11132 RouterItem::Packet(pkt) => (pkt.data_type(), !pkt.wire_target_senders().is_empty()),
11133 RouterItem::Packed(bytes) => {
11134 let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
11135 (env.ty, env.target_senders().len() != 0)
11136 }
11137 };
11138 let end_to_end_ordered =
11139 addressed && matches!(reliable_mode(ty), crate::ReliableMode::Ordered);
11140 let (handler, opts, hop_reliable_enabled) = {
11141 let st = self.state.lock();
11142 let side_ref = Self::side_ref(&st, side)?;
11143 let opts = side_ref.opts.clone();
11144 let hop_reliable_enabled = opts.reliable_enabled
11145 && !end_to_end_ordered
11146 && self.cfg.reliable_enabled()
11147 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
11148 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
11149 };
11150
11151 if !matches!(
11152 &handler,
11153 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11154 ) {
11155 return self.call_side_tx_handler_with_service(
11156 side, &handler, &data, relayed, admitted, service,
11157 );
11158 }
11159
11160 if !hop_reliable_enabled {
11161 let mut adjusted_opts = opts;
11162 adjusted_opts.reliable_enabled = false;
11163 let preserve_end_to_end_ack = true;
11168 if let Some(adjusted) =
11169 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
11170 {
11171 return self.call_side_tx_handler_with_service(
11172 side, &handler, &adjusted, relayed, admitted, service,
11173 );
11174 }
11175 return Ok(());
11176 }
11177
11178 if let RouterItem::Packed(bytes) = &data {
11179 wire_format::peek_routing_frame_info(bytes.as_ref())?;
11180 }
11181 let priority = self.item_service_priority(&data)?;
11182
11183 if !is_reliable_type(ty) {
11184 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
11185 self.call_side_tx_handler_with_service(
11186 side, &handler, &adjusted, relayed, admitted, service,
11187 )?;
11188 }
11189 return Ok(());
11190 }
11191
11192 #[cfg(feature = "compression")]
11193 let data = match data {
11194 RouterItem::Packed(bytes) => {
11195 let budget = self.state.lock().memory.max_queue_budget;
11196 let normalized =
11197 wire_format::payload_encoding_for_retention(bytes.as_ref(), budget)?;
11198 RouterItem::Packed(
11199 normalized
11200 .map(crate::shared_bytes::convert)
11201 .unwrap_or(bytes),
11202 )
11203 }
11204 packet => packet,
11205 };
11206 let (seq, flags) = {
11207 let mut st = self.state.lock();
11208 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
11209 if tx_state.sent.len() >= runtime_reliable_max_pending() {
11210 return Err(TelemetryError::Io("reliable window full"));
11211 }
11212 let seq = tx_state.next_seq;
11213 let next = tx_state.next_seq.wrapping_add(1);
11214 tx_state.next_seq = if next == 0 { 1 } else { next };
11215 let flags = match reliable_mode(ty) {
11216 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
11217 _ => 0,
11218 };
11219 (seq, flags)
11220 };
11221
11222 let bytes: crate::SharedBytes = match data {
11223 RouterItem::Packet(pkt) => self.pack_packet_for_router(
11224 &pkt,
11225 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
11226 )?,
11227 RouterItem::Packed(bytes) => {
11228 #[cfg(feature = "cryptography")]
11229 if self.e2e_seal_config_for_type(ty).is_some() {
11230 self.prepare_packed_for_remote(
11231 bytes,
11232 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
11233 )?
11234 } else {
11235 let Some(rewritten) =
11236 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
11237 else {
11238 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11239 return Err(TelemetryError::Io("side tx busy"));
11240 };
11241 let started_ms = self.clock.now_ms();
11242 let frames =
11243 self.encode_side_transport_frames(side, opts, bytes.clone())?;
11244 let mut attempts_total = 0usize;
11245 let mut sent_bytes = 0usize;
11246 self.send_side_frames_resumable_with_service(
11247 side,
11248 frames,
11249 service,
11250 |frame| {
11251 match self.retry_with_attempts(
11252 runtime_max_handler_retries(),
11253 || {
11254 self.packed_handler_call(
11255 side, &handler, frame, priority, service,
11256 )
11257 },
11258 ) {
11259 Ok((_, attempts)) => {
11260 *admitted = true;
11261 attempts_total = attempts_total.saturating_add(attempts);
11262 sent_bytes = sent_bytes.saturating_add(frame.len());
11263 }
11264 Err((err, attempts)) => {
11265 self.note_side_tx_failure(
11266 side,
11267 ty,
11268 attempts_total.saturating_add(attempts),
11269 );
11270 return Err(err);
11271 }
11272 }
11273
11274 Ok(())
11275 },
11276 )?;
11277 self.record_side_tx_sample(
11278 side,
11279 sent_bytes,
11280 started_ms,
11281 self.clock.now_ms(),
11282 );
11283 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11284 return Ok(());
11285 };
11286 crate::shared_bytes::convert(rewritten)
11287 }
11288 #[cfg(not(feature = "cryptography"))]
11289 {
11290 let Some(rewritten) =
11291 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
11292 else {
11293 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11294 return Err(TelemetryError::Io("side tx busy"));
11295 };
11296 let started_ms = self.clock.now_ms();
11297 let frames =
11298 self.encode_side_transport_frames(side, opts, bytes.clone())?;
11299 let mut attempts_total = 0usize;
11300 let mut sent_bytes = 0usize;
11301 self.send_side_frames_resumable_with_service(
11302 side,
11303 frames,
11304 service,
11305 |frame| {
11306 match self.retry_with_attempts(
11307 runtime_max_handler_retries(),
11308 || {
11309 self.packed_handler_call(
11310 side, &handler, frame, priority, service,
11311 )
11312 },
11313 ) {
11314 Ok((_, attempts)) => {
11315 *admitted = true;
11316 attempts_total = attempts_total.saturating_add(attempts);
11317 sent_bytes = sent_bytes.saturating_add(frame.len());
11318 }
11319 Err((err, attempts)) => {
11320 self.note_side_tx_failure(
11321 side,
11322 ty,
11323 attempts_total.saturating_add(attempts),
11324 );
11325 return Err(err);
11326 }
11327 }
11328
11329 Ok(())
11330 },
11331 )?;
11332 self.record_side_tx_sample(
11333 side,
11334 sent_bytes,
11335 started_ms,
11336 self.clock.now_ms(),
11337 );
11338 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11339 return Ok(());
11340 };
11341 crate::shared_bytes::convert(rewritten)
11342 }
11343 }
11344 };
11345
11346 #[cfg(feature = "compact-packet-store")]
11347 let mut bytes = bytes;
11348
11349 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11350 return Err(TelemetryError::Io("side tx busy"));
11351 };
11352 let started_ms = self.clock.now_ms();
11353 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
11354 let mut attempts_total = 0usize;
11355 let mut sent_bytes = 0usize;
11356 self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
11357 match self.retry_with_attempts(runtime_max_handler_retries(), || {
11358 self.packed_handler_call(side, &handler, frame, priority, service)
11359 }) {
11360 Ok((_, attempts)) => {
11361 *admitted = true;
11362 attempts_total = attempts_total.saturating_add(attempts);
11363 sent_bytes = sent_bytes.saturating_add(frame.len());
11364 }
11365 Err((err, attempts)) => {
11366 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
11367 return Err(err);
11368 }
11369 }
11370
11371 Ok(())
11372 })?;
11373 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
11374 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11375
11376 #[cfg(feature = "compact-packet-store")]
11377 {
11378 let _ = bytes.park_for_queue();
11379 }
11380
11381 {
11382 let mut st = self.state.lock();
11383 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
11384 tx_state.sent_order.push_back(seq);
11385 tx_state.sent.insert(
11386 seq,
11387 ReliableSent {
11388 bytes: bytes.clone(),
11389 last_send_ms: self.clock.now_ms(),
11390 retries: 0,
11391 queued: false,
11392 partial_acked: false,
11393 },
11394 );
11395 }
11396
11397 Ok(())
11398 }
11399
11400 fn run_side_chunk_transfer<T>(
11401 &self,
11402 side: RouterSideId,
11403 key: u64,
11404 history_dependent: bool,
11405 service: EgressService,
11406 send: impl FnOnce(&mut usize) -> TelemetryResult<T>,
11407 ) -> TelemetryResult<T> {
11408 let mut next = {
11409 let mut st = self.state.lock();
11410 #[cfg(feature = "latest-value-egress")]
11411 let defer_capacity = st.latest_side() == Some(side);
11412 #[cfg(feature = "latest-value-egress")]
11413 let latest_owner = defer_capacity
11414 && service.token != 0
11415 && service.side != 0
11416 && side == usize::from(service.side - 1)
11417 && service.token == st.latest_service_seq
11418 && service.kind == 2
11419 && st.latest_turn_phase == 4;
11420 let transport = st
11421 .side_transport
11422 .get_mut(&side)
11423 .ok_or(TelemetryError::BadArg)?;
11424 if transport.tx_chunk_progress.is_none() {
11425 crate::memory_admission::check(
11426 512,
11427 core::mem::size_of::<crate::side_transport::SideTxProgress>(),
11428 )?;
11429 transport.tx_chunk_progress =
11430 Some(Box::new(crate::side_transport::SideTxProgress::default()));
11431 }
11432 let progress = transport.tx_chunk_progress.as_mut().expect("initialized");
11433 #[cfg(feature = "latest-value-egress")]
11434 if defer_capacity
11435 && !progress.has_available_slot(self.clock.now_ms())
11436 && !progress.has_live_key(key, self.clock.now_ms())
11437 {
11438 return Err(TelemetryError::Io("side tx busy"));
11441 }
11442 #[cfg(feature = "latest-value-egress")]
11443 {
11444 progress.begin_classified(
11445 key,
11446 self.clock.now_ms(),
11447 history_dependent,
11448 latest_owner,
11449 )?
11450 }
11451 #[cfg(not(feature = "latest-value-egress"))]
11452 {
11453 let _ = service;
11454 progress.begin(key, self.clock.now_ms(), history_dependent)?
11455 }
11456 };
11457 let result = send(&mut next);
11458 if let Some(progress) = self
11459 .state
11460 .lock()
11461 .side_transport
11462 .get_mut(&side)
11463 .and_then(|transport| transport.tx_chunk_progress.as_mut())
11464 {
11465 progress.finish(key, next, self.clock.now_ms(), &result);
11466 }
11467 result
11468 }
11469
11470 fn send_side_frames_resumable(
11471 &self,
11472 side: RouterSideId,
11473 frames: SideTransportFrames,
11474 send: impl FnMut(&[u8]) -> TelemetryResult<()>,
11475 ) -> TelemetryResult<()> {
11476 self.send_side_frames_resumable_with_service(side, frames, EgressService::default(), send)
11477 }
11478
11479 fn send_side_frames_resumable_with_service(
11480 &self,
11481 side: RouterSideId,
11482 mut frames: SideTransportFrames,
11483 service: EgressService,
11484 mut send: impl FnMut(&[u8]) -> TelemetryResult<()>,
11485 ) -> TelemetryResult<()> {
11486 let bootstraps_template = frames.bootstraps_template();
11487 let result = if frames.len() == 1 {
11488 frames.send_resuming(&mut 0, send)
11489 } else {
11490 let key = frames.progress_key();
11491 let history_dependent = frames.history_dependent();
11492 self.run_side_chunk_transfer(side, key, history_dependent, service, |next| {
11493 frames.send_resuming(next, &mut send)
11494 })
11495 };
11496 if result.is_err() && bootstraps_template {
11497 if let Some(state) = self.state.lock().side_transport.get_mut(&side) {
11502 state.refresh_transmit_header_templates();
11503 }
11504 }
11505 result
11506 }
11507
11508 fn split_side_transport_frame(
11509 &self,
11510 _side: RouterSideId,
11511 frame: crate::SharedBytes,
11512 max_frame_bytes: usize,
11513 ) -> TelemetryResult<SideTransportFrames> {
11514 SideTransportFrames::split(frame, max_frame_bytes, self.sender_arc().as_bytes())
11515 }
11516
11517 #[cfg(all(feature = "compression", feature = "discovery"))]
11518 fn note_payload_codec_packet(&self, side: RouterSideId, pkt: &Packet, accepted: bool) {
11519 let now = self.clock.now_ms();
11520 let mut st = self.state.lock();
11521 if !st
11522 .sides
11523 .get(side)
11524 .and_then(|entry| entry.as_ref())
11525 .is_some_and(|entry| {
11526 matches!(
11527 entry.tx_handler,
11528 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11529 )
11530 })
11531 {
11532 return;
11533 }
11534 if accepted && pkt.data_type() == crate::DataType::DiscoveryLeave {
11535 for transport in st.side_transport.values_mut() {
11536 let _ = transport.payload_codec.discovery_packet(
11537 pkt,
11538 true,
11539 now,
11540 DISCOVERY_ROUTE_TTL_MS,
11541 );
11542 }
11543 } else if let Some(transport) = st.side_transport.get_mut(&side) {
11544 let _ = transport.payload_codec.discovery_packet(
11545 pkt,
11546 accepted,
11547 now,
11548 DISCOVERY_ROUTE_TTL_MS,
11549 );
11550 }
11551 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
11552 && let Some(transport) = st.side_transport.get_mut(&side)
11553 {
11554 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
11555 }
11556 }
11557 #[cfg(all(feature = "compression", feature = "discovery"))]
11558 fn note_payload_codec_address(&self, side: RouterSideId, payload: &[u8], accepted: bool) {
11559 let now = self.clock.now_ms();
11560 let mut st = self.state.lock();
11561 if !st
11562 .sides
11563 .get(side)
11564 .and_then(|entry| entry.as_ref())
11565 .is_some_and(|entry| {
11566 matches!(
11567 entry.tx_handler,
11568 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11569 )
11570 })
11571 {
11572 return;
11573 }
11574 if let Some(transport) = st.side_transport.get_mut(&side) {
11575 let _ = transport.payload_codec.address_payload(
11576 payload,
11577 accepted,
11578 now,
11579 DISCOVERY_ROUTE_TTL_MS,
11580 );
11581 }
11582 if st.shared_queue_bytes_used() > st.memory.max_queue_budget
11583 && let Some(transport) = st.side_transport.get_mut(&side)
11584 {
11585 transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
11586 }
11587 }
11588
11589 #[cfg(feature = "compression")]
11590 fn payload_for_transport_side(
11591 &self,
11592 side: RouterSideId,
11593 raw: crate::SharedBytes,
11594 ) -> TelemetryResult<crate::SharedBytes> {
11595 #[cfg(not(feature = "discovery"))]
11596 let _ = side;
11597 let (allowed, budget) = {
11598 #[cfg(feature = "discovery")]
11599 let mut st = self.state.lock();
11600 #[cfg(not(feature = "discovery"))]
11601 let st = self.state.lock();
11602 let budget = st.memory.max_queue_budget;
11603 #[cfg(feature = "discovery")]
11604 let allowed = {
11605 let transport = st
11606 .side_transport
11607 .get_mut(&side)
11608 .ok_or(TelemetryError::BadArg)?;
11609 let generation = transport.payload_codec.generation();
11610 if transport.payload_codec_choice != generation {
11611 transport.payload_codec_choice = generation;
11612 transport.tx_chunk_progress = None;
11613 transport.refresh_transmit_header_templates();
11614 }
11615 transport.payload_codec.allowed(self.clock.now_ms())
11616 };
11617 #[cfg(not(feature = "discovery"))]
11618 let allowed = false;
11619 (allowed, budget)
11620 };
11621 Ok(
11622 match wire_format::payload_encoding_for_side(raw.as_ref(), allowed, budget)? {
11623 Some(frame) => crate::shared_bytes::convert(frame),
11624 None => raw,
11625 },
11626 )
11627 }
11628
11629 #[cfg(feature = "compression")]
11633 fn prepare_queued_payload(
11634 &self,
11635 side: RouterSideId,
11636 data: &mut RouterItem,
11637 ) -> TelemetryResult<Option<crate::payload_codec::QueueReservation<'_, RouterInner>>> {
11638 let RouterItem::Packed(bytes) = data else {
11639 return Ok(None);
11640 };
11641 if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
11642 return Ok(None);
11643 }
11644 let capable = {
11645 #[cfg(feature = "discovery")]
11646 {
11647 self.state
11648 .lock()
11649 .side_transport
11650 .get(&side)
11651 .is_some_and(|transport| transport.payload_codec.allowed(self.clock.now_ms()))
11652 }
11653 #[cfg(not(feature = "discovery"))]
11654 {
11655 false
11656 }
11657 };
11658 if capable {
11659 return Ok(None);
11660 }
11661 let required = wire_format::decoded_frame_upper_bound(bytes.as_ref())?
11662 .checked_add(
11663 size_of::<crate::SharedBytes>()
11664 + size_of::<bool>()
11665 + size_of::<u8>()
11666 + size_of::<u64>(),
11667 )
11668 .ok_or(TelemetryError::Io("memory pressure"))?;
11669 let reservation = crate::payload_codec::QueueReservation::acquire(&self.state, required)?;
11670 *bytes = self.payload_for_transport_side(side, bytes.clone())?;
11671 Ok(Some(reservation))
11672 }
11673
11674 fn encode_side_transport_frames(
11675 &self,
11676 side: RouterSideId,
11677 opts: RouterSideOptions,
11678 raw: crate::SharedBytes,
11679 ) -> TelemetryResult<SideTransportFrames> {
11680 #[cfg(feature = "compression")]
11681 let raw = self.payload_for_transport_side(side, raw)?;
11682 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
11683 return Ok(SideTransportFrames::single(raw));
11684 }
11685
11686 let raw_len = raw.len();
11687 if opts.max_frame_bytes != 0 && raw_len > opts.max_frame_bytes {
11688 let frames = self.split_side_transport_frame(side, raw, opts.max_frame_bytes)?;
11691 let mut st = self.state.lock();
11692 let stats = st.side_runtime_stats.entry(side).or_default();
11693 stats.note_side_transport_full(raw_len, frames.wire_len());
11694 stats.note_side_transport_chunks(frames.len());
11695 return Ok(frames);
11696 }
11697
11698 if opts.header_template_enabled {
11699 let frame = wire_format::peek_routing_frame_info(raw.as_ref())?;
11703 let without_dictionary = opts.max_side_transport_templates == 0 || {
11704 let st = self.state.lock();
11705 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
11706 };
11707 let reliable = is_reliable_type(frame.envelope.ty)
11708 || frame.reliable.is_some()
11709 || frame.envelope.target_senders().len() != 0
11715 || matches!(
11716 frame.envelope.ty,
11717 crate::DataType::ReliableAck
11718 | crate::DataType::ReliablePartialAck
11719 | crate::DataType::ReliablePacketRequest
11720 );
11721 if without_dictionary || reliable {
11722 let mut st = self.state.lock();
11723 if without_dictionary {
11724 let removed = st
11725 .side_transport
11726 .get_mut(&side)
11727 .ok_or(TelemetryError::BadArg)?
11728 .trim_tx_templates_to(0);
11729 for _ in 0..removed {
11730 st.side_runtime_stats
11731 .entry(side)
11732 .or_default()
11733 .note_side_transport_template_eviction();
11734 }
11735 }
11736 st.side_runtime_stats
11737 .entry(side)
11738 .or_default()
11739 .note_side_transport_full(raw_len, raw_len);
11740 return Ok(SideTransportFrames::single(raw));
11741 }
11742 }
11743
11744 if opts.header_template_enabled && crate::memory_admission::supports_work_kinds() {
11745 let (template, ..) = extract_side_header_template_view(raw.as_ref())?;
11746 let cached = self
11747 .state
11748 .lock()
11749 .side_transport
11750 .get(&side)
11751 .is_some_and(|transport| {
11752 transport.tx_template_ids.contains_key(&template.hash)
11753 && transport.tx_templates.contains_key(&template.hash)
11754 });
11755 if !cached
11756 && matches!(
11757 crate::memory_admission::check_receive(raw.as_ref()),
11758 Err(TelemetryError::Io("memory pressure"))
11759 )
11760 {
11761 self.state
11765 .lock()
11766 .side_runtime_stats
11767 .entry(side)
11768 .or_default()
11769 .note_side_transport_full(raw_len, raw_len);
11770 return Ok(SideTransportFrames::single(raw));
11771 }
11772 }
11773
11774 let mut compact_payload_len = None;
11775 let mut used_compact = false;
11776 let used_timestamp_delta = false;
11777 let omitted_timestamp = false;
11778 let wrapped = if opts.header_template_enabled {
11779 let (template, _ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
11780 extract_side_header_template_view(raw.as_ref())?;
11781 let (template_id, use_compact, _previous_timestamp) = {
11784 let mut st = self.state.lock();
11785 #[cfg(feature = "discovery")]
11790 let mut effective_template_limit = opts.max_side_transport_templates;
11791 #[cfg(not(feature = "discovery"))]
11792 let effective_template_limit = opts.max_side_transport_templates;
11793 #[cfg(feature = "discovery")]
11794 {
11795 if let Some(route) = st.discovery_routes.get(&side) {
11796 let active_peers = route
11797 .announcers
11798 .values()
11799 .filter(|sender| {
11800 self.clock.now_ms().saturating_sub(sender.last_seen_ms)
11801 <= DISCOVERY_ROUTE_TTL_MS
11802 })
11803 .count();
11804 if active_peers > 1 {
11812 effective_template_limit = 0;
11813 } else {
11814 effective_template_limit = route
11815 .announcers
11816 .values()
11817 .filter_map(|sender| sender.link_capabilities)
11818 .filter(|caps| {
11819 caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
11820 && caps.max_side_transport_templates > 0
11821 })
11822 .map(|caps| caps.max_side_transport_templates as usize)
11823 .fold(effective_template_limit, core::cmp::min);
11824 }
11825 }
11826 }
11827 let trimmed = st
11828 .side_transport
11829 .get_mut(&side)
11830 .ok_or(TelemetryError::BadArg)?
11831 .trim_tx_templates_to(effective_template_limit);
11832 for _ in 0..trimmed {
11833 st.side_runtime_stats
11834 .entry(side)
11835 .or_default()
11836 .note_side_transport_template_eviction();
11837 }
11838 let side_state = st
11839 .side_transport
11840 .get_mut(&side)
11841 .ok_or(TelemetryError::BadArg)?;
11842 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
11843 let previous = side_state.tx_last_timestamps.get(&id).copied();
11844 const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
11847 let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
11848 #[cfg(feature = "discovery")]
11850 let discovery_refresh = discovery::is_discovery_type(_ty);
11851 #[cfg(not(feature = "discovery"))]
11852 let discovery_refresh = false;
11853 let use_compact =
11854 *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES && !discovery_refresh;
11855 if use_compact {
11856 *compact_uses = compact_uses.saturating_add(1);
11857 } else {
11858 *compact_uses = 0;
11859 }
11860 (id, use_compact, previous)
11861 } else {
11862 let template_id = ((template.hash >> 32) as u32 ^ template.hash as u32).max(1);
11868 let evicted = side_state.insert_tx_template(
11869 template.into_owned(),
11870 template_id,
11871 effective_template_limit,
11872 );
11873 if evicted {
11874 st.side_runtime_stats
11875 .entry(side)
11876 .or_default()
11877 .note_side_transport_template_eviction();
11878 }
11879 if let Some(side_state) = st.side_transport.get_mut(&side)
11880 && effective_template_limit > 0
11881 {
11882 side_state.tx_last_timestamps.insert(template_id, timestamp);
11883 }
11884 (template_id, false, None)
11885 }
11886 };
11887 if use_compact {
11888 used_compact = true;
11889 compact_payload_len = Some(payload.len());
11890 let timestamp_field = Some(timestamp);
11893 let mut body = Vec::with_capacity(payload.len() + 32);
11894 body.push(flags);
11895 write_uleb128_local(u64::from(template_id), &mut body);
11896 write_uleb128_local(data_size, &mut body);
11897 if let Some(timestamp_field) = timestamp_field {
11898 write_uleb128_local(timestamp_field, &mut body);
11899 }
11900 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
11901 write_uleb128_local(u64::from(nonce), &mut body);
11902 }
11903 if let Some((seq, ack)) = reliable_seq_ack {
11904 write_uleb128_local(u64::from(seq), &mut body);
11905 write_uleb128_local(u64::from(ack), &mut body);
11906 }
11907 body.extend_from_slice(payload);
11908 {
11909 let mut st = self.state.lock();
11910 if let Some(side_state) = st.side_transport.get_mut(&side) {
11911 side_state.tx_last_timestamps.insert(template_id, timestamp);
11912 }
11913 }
11914 let kind = if omitted_timestamp {
11915 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
11916 } else if used_timestamp_delta {
11917 SIDE_TRANSPORT_KIND_COMPACT_DELTA
11918 } else {
11919 SIDE_TRANSPORT_KIND_COMPACT
11920 };
11921 wrap_side_transport_frame(kind, &body)
11922 } else {
11923 let mut body = Vec::with_capacity(raw.len() + 4);
11924 write_uleb128_local(u64::from(template_id), &mut body);
11925 body.extend_from_slice(raw.as_ref());
11926 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
11927 }
11928 } else {
11929 wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
11930 };
11931
11932 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
11933 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
11934 } else {
11935 Ok(SideTransportFrames::single(wrapped))
11936 }?;
11937 let wire_len = frames.wire_len();
11938 let mut st = self.state.lock();
11939 let stats = st.side_runtime_stats.entry(side).or_default();
11940 if used_compact {
11941 let overhead = compact_payload_len
11942 .map(|payload_len| wire_len.saturating_sub(payload_len))
11943 .unwrap_or(wire_len);
11944 stats.note_side_transport_compact(
11945 raw_len,
11946 wire_len,
11947 overhead,
11948 used_timestamp_delta,
11949 omitted_timestamp,
11950 );
11951 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
11952 {
11953 stats.note_side_transport_compact_target_miss();
11954 }
11955 } else {
11956 stats.note_side_transport_full(raw_len, wire_len);
11957 }
11958 if frames.len() > 1 {
11959 stats.note_side_transport_chunks(frames.len());
11960 }
11961 Ok(frames)
11962 }
11963
11964 fn expire_side_chunks(&self) {
11965 let now = self.clock.now_ms();
11966 let mut st = self.state.lock();
11967 for side in st.side_transport.values_mut() {
11968 let side = side.as_mut();
11969 #[cfg(all(feature = "compression", feature = "discovery"))]
11970 side.payload_codec.expire(now, DISCOVERY_ROUTE_TTL_MS);
11971 let reclaimed =
11972 SideChunkAssembly::expire(&mut side.rx_chunks, &mut side.rx_chunk_bytes, now);
11973 side.rx_chunk_reclaims = side.rx_chunk_reclaims.saturating_add(reclaimed);
11974 }
11975 }
11976
11977 #[cfg(feature = "discovery")]
11981 fn refresh_ingress_peer_liveness<'a>(
11982 &self,
11983 side: RouterSideId,
11984 bytes: &'a [u8],
11985 ) -> Option<wire_format::RoutingFrameInfo<'a>> {
11986 let frame = wire_format::peek_routing_frame_info(bytes).ok()?;
11987 self.refresh_peer_liveness(side, frame.envelope.source_address);
11988 #[cfg(feature = "compression")]
11989 if frame.envelope.ty == crate::DataType::DiscoveryAddress
11990 && let Ok(Some((_, payload))) =
11991 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryAddress)
11992 {
11993 self.note_payload_codec_address(side, payload, false);
11994 }
11995 Some(frame)
11996 }
11997
11998 #[cfg(feature = "discovery")]
11999 #[cfg_attr(not(feature = "std"), inline(never))]
12000 fn refresh_peer_liveness(&self, side: RouterSideId, address: u32) {
12001 let now = self.clock.now_ms();
12002 let mut st = self.state.lock();
12003 #[cfg(feature = "compression")]
12004 if let Some(transport) = st.side_transport.get_mut(&side) {
12005 transport
12006 .payload_codec
12007 .observe(address, now, DISCOVERY_ROUTE_TTL_MS);
12008 }
12009 let Some(route) = st.discovery_routes.get_mut(&side) else {
12010 return;
12011 };
12012 let mut found = None;
12013 for (name, peer) in &mut route.announcers {
12014 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12015 if found.is_some() {
12017 return;
12018 }
12019 found = Some(peer);
12020 }
12021 }
12022 let Some(peer) = found else {
12023 return;
12024 };
12025 peer.last_seen_ms = now;
12026 route.last_seen_ms = route.last_seen_ms.max(now);
12027 }
12028
12029 #[cfg(feature = "discovery")]
12033 fn consume_known_packed_topology(
12034 &self,
12035 side: RouterSideId,
12036 bytes: &[u8],
12037 ) -> TelemetryResult<bool> {
12038 let Some((address, payload)) =
12039 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryTopology)?
12040 else {
12041 return Ok(false);
12042 };
12043 let now = self.clock.now_ms();
12044 let mut st = self.state.lock();
12045 let Some(route) = st.discovery_routes.get_mut(&side) else {
12046 return Ok(false);
12047 };
12048 let mut found = None;
12049 for (name, peer) in &mut route.announcers {
12050 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12051 if found.is_some() {
12052 return Ok(false);
12053 }
12054 found = Some(peer);
12055 }
12056 }
12057 let Some(peer) = found else {
12058 return Ok(false);
12059 };
12060 let Some(full) =
12061 discovery::topology_payload_matches_retained(payload, &peer.topology_boards)
12062 else {
12063 return Ok(false);
12064 };
12065 peer.has_full_topology |= full;
12066 peer.last_seen_ms = now;
12067 route.last_seen_ms = route.last_seen_ms.max(now);
12068 Ok(true)
12069 }
12070
12071 #[cfg(all(feature = "discovery", not(feature = "std")))]
12072 fn consume_known_packed_schema(
12073 &self,
12074 side: RouterSideId,
12075 bytes: &[u8],
12076 ) -> TelemetryResult<bool> {
12077 let Some((address, payload)) =
12078 wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoverySchema)?
12079 else {
12080 return Ok(false);
12081 };
12082 self.refresh_peer_liveness(side, address);
12083 self.merge_schema_changes(payload)?;
12084 let mut st = self.state.lock();
12085 if let Some(route) = st.discovery_routes.get_mut(&side) {
12086 for (name, peer) in &mut route.announcers {
12087 if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12088 peer.has_schema = true;
12089 break;
12090 }
12091 }
12092 }
12093 Ok(true)
12094 }
12095
12096 #[cfg(all(feature = "compression", feature = "discovery"))]
12097 fn decode_required_schema<T>(
12098 &self,
12099 mut decode: impl FnMut() -> TelemetryResult<T>,
12100 ) -> TelemetryResult<T> {
12101 loop {
12102 match decode() {
12103 Err(error @ TelemetryError::Io("memory pressure")) => {
12104 if !self.state.lock().reclaim_codec_metadata() {
12105 return Err(error);
12106 }
12107 }
12108 result => return result,
12109 }
12110 }
12111 }
12112
12113 #[cfg(all(feature = "discovery", not(feature = "std")))]
12114 #[inline(never)]
12115 fn merge_schema_changes(&self, payload: &[u8]) -> TelemetryResult<()> {
12116 #[cfg(feature = "compression")]
12119 let snapshot = self.decode_required_schema(|| discovery::decode_schema_changes(payload))?;
12120 #[cfg(not(feature = "compression"))]
12121 let snapshot = discovery::decode_schema_changes(payload)?;
12122 let mut st = self.state.lock();
12123 let budget = st.schema_growth_budget();
12124 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
12125 if report.changed() {
12126 st.fit_discovery_budget();
12127 }
12128 Ok(())
12129 }
12130
12131 #[cfg(all(
12136 feature = "discovery",
12137 any(feature = "std", feature = "ingress-target-filter")
12138 ))]
12139 fn targets_only_ingress(
12140 &self,
12141 side: RouterSideId,
12142 frame: &wire_format::RoutingFrameInfo<'_>,
12143 ) -> bool {
12144 let ty = frame.envelope.ty;
12145 if is_internal_control_type(ty)
12148 || frame.envelope.target_senders().len() == 0
12149 || frame.reliable.is_some_and(|h| {
12150 h.flags != wire_format::RELIABLE_FLAG_UNSEQUENCED || h.seq != 0 || h.ack != 0
12151 })
12152 {
12153 return false;
12154 }
12155 let Some(meta) = crate::config::try_get_message_meta(ty) else {
12156 return false;
12157 };
12158 if frame
12159 .envelope
12160 .wire_shape
12161 .is_some_and(|shape| shape != meta.element)
12162 {
12163 return false;
12164 }
12165 let local_hash = Self::sender_hash(self.sender_arc().as_ref());
12166 let st = self.state.lock();
12167 if st.is_managed_variable(ty.as_u32())
12168 || Self::has_explicit_route_policy_locked(&st, Some(side), ty)
12169 {
12170 return false;
12171 }
12172 let now = self.clock.now_ms();
12173 frame.envelope.target_senders().all(|target| {
12174 target != local_hash
12175 && target != u64::from(st.local_address.address)
12176 && Self::side_matches_target_senders_locked(
12177 &st,
12178 side,
12179 wire_format::TargetSenderSet::Hashes(&[target]),
12180 now,
12181 )
12182 && !st.sides.iter().enumerate().any(|(other, entry)| {
12183 other != side
12184 && entry.as_ref().is_some_and(|e| e.opts.egress_enabled)
12185 && Self::side_matches_target_senders_locked(
12186 &st,
12187 other,
12188 wire_format::TargetSenderSet::Hashes(&[target]),
12189 now,
12190 )
12191 })
12192 })
12193 }
12194
12195 #[cfg(feature = "discovery")]
12198 fn cached_transit_ready(
12199 &self,
12200 side: RouterSideId,
12201 bytes: &[u8],
12202 view: Option<&wire_format::RoutingFrameInfo<'_>>,
12203 ) -> TelemetryResult<bool> {
12204 if bytes.len() > 128 || bytes.first().is_none_or(|flags| flags & 0x10 != 0) {
12209 return Ok(false);
12210 }
12211 let parsed;
12212 let frame = match view {
12213 Some(frame) => frame,
12214 None => {
12215 parsed = wire_format::peek_routing_frame_info(bytes)?;
12216 &parsed
12217 }
12218 };
12219 let ty = frame.envelope.ty;
12220 if is_internal_control_type(ty)
12221 || frame.envelope.is_reliable
12222 || frame.reliable.is_some()
12223 || frame.envelope.target_senders().len() != 0
12224 || frame.envelope.wire_shape.is_some_and(|shape| {
12228 shape != crate::message_meta(ty).element
12229 })
12230 || !(1..=3).contains(&frame.envelope.endpoints().take(4).count())
12231 {
12232 return Ok(false);
12233 }
12234 if self.is_managed_variable_type(ty)
12235 || frame.envelope.endpoints().any(|ep| {
12236 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
12237 })
12238 {
12239 return Ok(false);
12240 }
12241
12242 let canonical_possible = {
12246 let st = self.state.lock();
12247 st.discovery_routes.get(&side).is_some_and(|route| {
12248 route.announcers.iter().any(|(name, peer)| {
12249 peer.advertised_address == Some(frame.envelope.source_address)
12250 || sender_address_u32(name) == frame.envelope.source_address
12251 })
12252 }) && st.sides.iter().flatten().all(|entry| {
12253 entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes
12254 })
12255 };
12256 let template_hash = if canonical_possible {
12257 None
12258 } else {
12259 Some(extract_side_header_template_view(bytes)?.0.hash)
12260 };
12261 let st = self.state.lock();
12262 let active: usize = st.sides.iter().filter(|entry| entry.is_some()).count();
12263 if active != 2 {
12264 return Ok(false);
12265 }
12266 for (id, entry) in st
12267 .sides
12268 .iter()
12269 .enumerate()
12270 .filter_map(|(id, entry)| entry.as_ref().map(|e| (id, e)))
12271 {
12272 if entry.opts.reliable_enabled
12273 || !matches!(
12274 entry.tx_handler,
12275 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
12276 )
12277 || !st
12278 .side_runtime_stats
12279 .get(&id)
12280 .is_some_and(|stats| stats.data_types.contains_key(&ty.as_u32()))
12281 {
12282 return Ok(false);
12283 }
12284 if id == side {
12285 continue;
12286 }
12287 #[cfg(feature = "discovery")]
12288 if !st.discovery_routes.get(&id).is_some_and(|route| {
12289 self.clock.now_ms().saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
12290 && frame
12291 .envelope
12292 .endpoints()
12293 .any(|ep| route.reachable.contains(&ep))
12294 }) {
12295 return Ok(false);
12296 }
12297 if entry.opts.header_template_enabled
12298 && entry.opts.max_frame_bytes != 0
12299 && entry.opts.max_side_transport_templates != 0
12300 && !self.side_has_multiple_announcers_locked(&st, id, self.clock.now_ms())
12301 && !template_hash.is_some_and(|hash| {
12302 st.side_transport.get(&id).is_some_and(|transport| {
12303 transport.tx_template_ids.contains_key(&hash)
12304 && transport.tx_templates.contains_key(&hash)
12305 })
12306 })
12307 {
12308 if (entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes)
12313 && st.discovery_routes.get(&side).is_some_and(|route| {
12314 route.announcers.iter().any(|(name, peer)| {
12315 peer.advertised_address == Some(frame.envelope.source_address)
12316 || sender_address_u32(name) == frame.envelope.source_address
12317 })
12318 })
12319 {
12320 continue;
12321 }
12322 return Ok(false);
12323 }
12324 }
12325 Ok(true)
12326 }
12327 #[cfg(not(feature = "discovery"))]
12328 fn cached_transit_ready(
12329 &self,
12330 _side: RouterSideId,
12331 _bytes: &[u8],
12332 _view: Option<&wire_format::RoutingFrameInfo<'_>>,
12333 ) -> TelemetryResult<bool> {
12334 Ok(false)
12335 }
12336 #[cfg(any(feature = "std", feature = "fair-admission"))]
12337 fn reclaim_for_receive(&self, bytes: &[u8], victim: &mut Option<(u32, u8)>) -> bool {
12338 let Some((incoming, priority)) = crate::admission_fairness::incoming(bytes) else {
12339 return false;
12340 };
12341 self.reclaim_for_incoming(incoming, priority, victim)
12342 }
12343 #[cfg(any(feature = "std", feature = "fair-admission"))]
12344 fn reclaim_for_incoming(
12345 &self,
12346 incoming: u32,
12347 priority: u8,
12348 victim: &mut Option<(u32, u8)>,
12349 ) -> bool {
12350 let mut st = self.state.lock();
12351 let (owner, priority) = if let Some(selected) = *victim {
12352 selected
12353 } else {
12354 let Some(owner) = st.competing_bulk_owner(incoming, priority) else {
12355 return false;
12356 };
12357 *victim = Some((owner, priority));
12358 (owner, priority)
12359 };
12360 st.reclaim_bulk_from(owner, priority, None)
12361 }
12362 fn admit_received_frame(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<()> {
12363 self.admit_received_frame_with_view(side, bytes, None)
12364 }
12365
12366 fn admit_received_frame_with_view(
12369 &self,
12370 side: RouterSideId,
12371 bytes: &[u8],
12372 view: Option<&wire_format::RoutingFrameInfo<'_>>,
12373 ) -> TelemetryResult<()> {
12374 if !crate::memory_admission::has_probe() {
12375 return Ok(());
12376 }
12377 #[cfg(any(feature = "std", feature = "fair-admission"))]
12378 let mut fair_victim = None;
12379 if crate::memory_admission::supports_work_kinds()
12380 && self.cached_transit_ready(side, bytes, view)?
12381 {
12382 let result = crate::memory_admission::check_cached_transit(bytes.len());
12383 #[cfg(all(feature = "compression", feature = "discovery"))]
12384 let result = {
12385 let mut result = result;
12386 while matches!(result, Err(TelemetryError::Io("memory pressure")))
12387 && self.state.lock().reclaim_codec_metadata()
12388 {
12389 result = crate::memory_admission::check_cached_transit(bytes.len());
12390 }
12391 result
12392 };
12393
12394 #[cfg(any(feature = "std", feature = "fair-admission"))]
12395 let result = {
12396 let mut result = result;
12397 while matches!(result, Err(TelemetryError::Io("memory pressure"))) {
12398 if !self.reclaim_for_receive(bytes, &mut fair_victim) {
12399 break;
12400 }
12401 result = crate::memory_admission::check_cached_transit(bytes.len());
12402 }
12403 result
12404 };
12405 result
12406 } else {
12407 let managed = wire_format::scheduling_type(bytes)
12408 .is_some_and(|ty| self.is_managed_variable_type(ty));
12409 let mut protected = None;
12410 loop {
12411 match crate::memory_admission::check_receive_with_role(bytes, managed) {
12412 Err(TelemetryError::Io("memory pressure")) => {
12413 #[cfg(all(feature = "compression", feature = "discovery"))]
12414 if self.state.lock().reclaim_codec_metadata() {
12415 continue;
12416 }
12417
12418 let protected = *protected.get_or_insert_with(|| {
12419 wire_format::peek_routing_frame_info(bytes)
12420 .ok()
12421 .is_some_and(|frame| {
12422 managed
12423 || message_priority(frame.envelope.ty) >= 200
12424 || crate::small_frozen_control(
12425 frame.envelope.ty,
12426 frame.envelope.target_senders().len(),
12427 frame.envelope.wire_shape,
12428 )
12429 })
12430 });
12431 if !protected {
12432 #[cfg(any(feature = "std", feature = "fair-admission"))]
12433 if self.reclaim_for_receive(bytes, &mut fair_victim) {
12434 continue;
12435 }
12436 return Err(TelemetryError::Io("memory pressure"));
12437 }
12438 let mut st = self.state.lock();
12442 let reclaimed = st
12443 .lower_priority_shared_queue(199)
12444 .is_some_and(|queue| st.pop_shared_queue_item(queue));
12445 if !reclaimed {
12446 return Err(TelemetryError::Io("memory pressure"));
12447 }
12448 }
12449 result => return result,
12450 }
12451 }
12452 }
12453 }
12454
12455 #[cfg(feature = "discovery")]
12456 fn consume_known_packed_discovery(
12457 &self,
12458 side: RouterSideId,
12459 bytes: &[u8],
12460 ) -> TelemetryResult<bool> {
12461 let result = match wire_format::scheduling_type(bytes) {
12464 #[cfg(not(feature = "std"))]
12465 Some(crate::DataType::DiscoverySchema) => self.consume_known_packed_schema(side, bytes),
12466 Some(crate::DataType::DiscoveryTopology) => {
12467 self.consume_known_packed_topology(side, bytes)
12468 }
12469 Some(crate::DataType::DiscoveryAddress) => {
12470 match wire_format::peek_plain_discovery_payload(
12471 bytes,
12472 crate::DataType::DiscoveryAddress,
12473 ) {
12474 Ok(Some((address, payload))) => {
12475 self.refresh_peer_liveness(side, address);
12476 self.refresh_known_address_payload(payload, side)
12477 .map(|known| known && self.cfg.role.local_services())
12478 }
12479 Ok(None) => Ok(false),
12480 Err(error) => Err(error),
12481 }
12482 }
12483 _ => Ok(false),
12484 };
12485 if result.is_err() {
12486 let _ = self.refresh_ingress_peer_liveness(side, bytes);
12490 }
12491 result
12492 }
12493
12494 fn decode_side_transport_frame(
12495 &self,
12496 side: RouterSideId,
12497 bytes: &[u8],
12498 ) -> TelemetryResult<Option<crate::SharedBytes>> {
12499 self.expire_side_chunks();
12501 let Some((kind, body)) = parse_side_transport_wrapper(bytes)? else {
12502 #[cfg(feature = "discovery")]
12503 if self.consume_known_packed_discovery(side, bytes)? {
12504 return Ok(None);
12505 }
12506 #[cfg(feature = "discovery")]
12507 let view = self.refresh_ingress_peer_liveness(side, bytes);
12508 #[cfg(not(feature = "discovery"))]
12509 let view = None;
12510 #[cfg(all(
12511 feature = "discovery",
12512 any(feature = "std", feature = "ingress-target-filter")
12513 ))]
12514 if bytes.first().is_some_and(|flags| flags & 0x10 == 0)
12515 && view
12516 .as_ref()
12517 .is_some_and(|frame| self.targets_only_ingress(side, frame))
12518 {
12519 return Ok(None);
12520 }
12521 self.admit_received_frame_with_view(side, bytes, view.as_ref())?;
12522 return Ok(Some(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes))));
12523 };
12524 match kind {
12525 SIDE_TRANSPORT_KIND_FULL => {
12526 let header_templates_enabled = self
12527 .state
12528 .lock()
12529 .sides
12530 .get(side)
12531 .and_then(|entry| entry.as_ref())
12532 .map(|side| side.opts.header_template_enabled)
12533 .unwrap_or(false);
12534 let mut off = 0usize;
12535 let template_id = if header_templates_enabled {
12536 Some(
12537 u32::try_from(read_uleb128_local(body, &mut off)?)
12538 .map_err(|_| TelemetryError::Unpack("side template id too large"))?,
12539 )
12540 } else {
12541 None
12542 };
12543 #[cfg(feature = "discovery")]
12544 if self.consume_known_packed_discovery(side, &body[off..])? {
12545 return Ok(None);
12546 }
12547 #[cfg(feature = "discovery")]
12548 let view = self.refresh_ingress_peer_liveness(side, &body[off..]);
12549 #[cfg(not(feature = "discovery"))]
12550 let view = None;
12551 self.admit_received_frame_with_view(side, &body[off..], view.as_ref())?;
12552 let raw = Arc::<[u8]>::from(&body[off..]);
12553 if let (Some(template_id), Ok((template, _, _, _, timestamp, _, _, _))) =
12554 (template_id, extract_side_header_template(raw.as_ref()))
12555 {
12556 let mut st = self.state.lock();
12557 let max_templates = st
12558 .sides
12559 .get(side)
12560 .and_then(|side| side.as_ref())
12561 .map(|side| side.opts.max_side_transport_templates)
12562 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
12563 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
12564 let evicted =
12565 side_state.insert_rx_template(template_id, template, max_templates);
12566 if max_templates > 0 {
12567 side_state.rx_last_timestamps.insert(template_id, timestamp);
12568 }
12569 evicted
12570 });
12571 if evicted {
12572 st.side_runtime_stats
12573 .entry(side)
12574 .or_default()
12575 .note_side_transport_template_eviction();
12576 }
12577 }
12578 Ok(Some(crate::shared_bytes::convert(raw)))
12579 }
12580 SIDE_TRANSPORT_KIND_COMPACT
12581 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
12582 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
12583 if body.is_empty() {
12584 return Err(TelemetryError::Unpack("short side compact frame"));
12585 }
12586 let mut off = 1usize;
12587 let template_id = u32::try_from(read_uleb128_local(body, &mut off)?)
12588 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
12589 let (template, timestamp_base) = {
12590 let st = self.state.lock();
12591 let state = st.side_transport.get(&side);
12592 let template = state
12593 .and_then(|state| state.rx_templates_by_id.get(&template_id))
12594 .cloned();
12595 let timestamp_base = if matches!(
12596 kind,
12597 SIDE_TRANSPORT_KIND_COMPACT_DELTA
12598 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
12599 ) {
12600 state
12601 .and_then(|state| state.rx_last_timestamps.get(&template_id))
12602 .copied()
12603 } else {
12604 None
12605 };
12606 (template, timestamp_base)
12607 };
12608 let Some(template) = template else {
12612 if !self.cfg.role.local_services() {
12613 return Err(TelemetryError::Unpack("unknown side compact template"));
12614 }
12615 return Ok(None);
12616 };
12617 #[cfg(feature = "discovery")]
12620 if let Some(address) = template.source_address() {
12621 self.refresh_peer_liveness(side, address);
12622 }
12623 let reconstructed_bound = body
12624 .len()
12625 .saturating_add(template.prefix().len())
12626 .saturating_add(template.between().len())
12627 .saturating_add(32);
12628 let protected = template.data_type().is_some_and(|ty| {
12632 message_priority(ty) >= 200 || self.is_managed_variable_type(ty)
12633 });
12634 let ack = template.is_ack();
12635 #[cfg(any(feature = "std", feature = "fair-admission"))]
12636 let mut fair_incoming = None;
12637 #[cfg(any(feature = "std", feature = "fair-admission"))]
12638 let mut fair_victim = None;
12639 loop {
12640 match crate::memory_admission::check_compact_reconstruction(
12641 reconstructed_bound,
12642 ack,
12643 ) {
12644 Err(TelemetryError::Io("memory pressure")) if protected => {
12645 let mut st = self.state.lock();
12646 let reclaimed = st
12647 .lower_priority_shared_queue(199)
12648 .is_some_and(|queue| st.pop_shared_queue_item(queue));
12649 if !reclaimed {
12650 return Err(TelemetryError::Io("memory pressure"));
12651 }
12652 }
12653 Err(TelemetryError::Io("memory pressure")) => {
12654 #[cfg(any(feature = "std", feature = "fair-admission"))]
12658 if let Some((incoming, priority)) =
12659 *fair_incoming.get_or_insert_with(|| {
12660 crate::admission_fairness::compact_incoming(&template)
12661 })
12662 && self.reclaim_for_incoming(incoming, priority, &mut fair_victim)
12663 {
12664 continue;
12665 }
12666 return Err(TelemetryError::Io("memory pressure"));
12667 }
12668 result => {
12669 result?;
12670 break;
12671 }
12672 }
12673 }
12674 let timestamp_mode = match kind {
12675 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
12676 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
12677 _ => SideCompactTimestampMode::Absolute,
12678 };
12679 let (frame, timestamp) = reconstruct_side_compact_frame(
12680 &template,
12681 body[0],
12682 &body[off..],
12683 timestamp_mode,
12684 timestamp_base,
12685 )?;
12686 {
12687 let mut st = self.state.lock();
12688 if let Some(side_state) = st.side_transport.get_mut(&side) {
12689 side_state.rx_last_timestamps.insert(template_id, timestamp);
12690 }
12691 }
12692 if crate::memory_admission::supports_work_kinds() {
12696 self.admit_received_frame(side, frame.as_ref())?;
12697 }
12698 Ok(Some(frame))
12699 }
12700 SIDE_TRANSPORT_KIND_CHUNK => {
12701 if body.len() < 8 {
12702 return Err(TelemetryError::Unpack("short side chunk frame"));
12703 }
12704 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
12705 let index = u16::from_le_bytes([body[4], body[5]]);
12706 let total = u16::from_le_bytes([body[6], body[7]]);
12707 if total == 0 || index >= total {
12708 return Err(TelemetryError::Unpack("invalid side chunk index"));
12709 }
12710 let assembled = {
12711 let mut st = self.state.lock();
12712 let now_ms = self.clock.now_ms();
12713 st.make_side_chunk_room(side, transfer_id, index, total, body.len() - 8)?;
12714 if let Some(entry) = st
12718 .side_transport
12719 .get_mut(&side)
12720 .and_then(|state| state.rx_chunks.get_mut(&transfer_id))
12721 && entry.contains_index(index)
12722 {
12723 return Ok(None);
12726 }
12727 let retained = st
12730 .side_transport
12731 .get(&side)
12732 .and_then(|state| state.rx_chunks.get(&transfer_id))
12733 .map(|entry| entry.received_len());
12734 loop {
12735 match crate::memory_admission::check_retained_fragment(bytes, retained) {
12736 Err(TelemetryError::Io("memory pressure")) => {
12737 if st
12738 .lower_priority_shared_queue(199)
12739 .is_some_and(|queue| st.pop_shared_queue_item(queue))
12740 || st.reclaim_oldest_side_chunk(Some((side, transfer_id)))
12741 {
12742 continue;
12743 }
12744 return Err(TelemetryError::Io("memory pressure"));
12745 }
12746 result => {
12747 result?;
12748 break;
12749 }
12750 }
12751 }
12752 let side_state = st
12753 .side_transport
12754 .get_mut(&side)
12755 .ok_or(TelemetryError::BadArg)?;
12756 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
12757 entry.total = total;
12758 let added =
12759 entry.insert_with(index, now_ms, || Arc::<[u8]>::from(&body[8..]))?;
12760 side_state.rx_chunk_bytes = side_state.rx_chunk_bytes.saturating_add(added);
12761 if entry.received_len() == usize::from(total) {
12762 let entry = side_state
12763 .take_chunk(transfer_id, false)
12764 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
12765 Some(entry.assemble().inspect_err(|_| {
12766 side_state.rx_chunk_reclaims =
12767 side_state.rx_chunk_reclaims.saturating_add(1);
12768 })?)
12769 } else {
12770 None
12771 }
12772 };
12773 match assembled {
12774 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
12775 None => Ok(None),
12776 }
12777 }
12778 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
12779 }
12780 }
12781
12782 fn packed_handler_call(
12783 &self,
12784 side: RouterSideId,
12785 handler: &RouterTxHandlerFn,
12786 bytes: &[u8],
12787 priority: u8,
12788 _service: EgressService,
12789 ) -> TelemetryResult<()> {
12790 let result = match handler {
12791 RouterTxHandlerFn::Packed(handler) => handler(bytes),
12792 RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
12793 RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
12794 };
12795 #[cfg(feature = "latest-value-egress")]
12796 if result.is_ok() && _service.token != 0 && _service.side != 0 {
12797 let mut st = self.state.lock();
12798 if st.latest_service_seq == _service.token
12799 && st.latest_side() == Some(side)
12800 && side == usize::from(_service.side - 1)
12801 && matches!((_service.kind, st.latest_turn_phase), (1, 3) | (2, 4))
12802 {
12803 let bytes = i32::try_from(bytes.len()).unwrap_or(i32::MAX);
12804 match _service.kind {
12805 1 => {
12806 let cap = st
12807 .sides
12808 .get(side)
12809 .and_then(Option::as_ref)
12810 .map_or(0, |side| {
12811 side.opts
12812 .max_frame_bytes
12813 .saturating_mul(usize::from(MAX_PRIORITY_BURST))
12814 .min(i32::MAX as usize) as i32
12815 });
12816 st.latest_turn_credit =
12817 st.latest_turn_credit.saturating_add(bytes).min(cap);
12818 }
12819 2 => {
12820 st.latest_turn_credit = st
12821 .latest_turn_credit
12822 .saturating_sub(bytes.saturating_mul(2))
12823 }
12824 _ => {}
12825 }
12826 }
12827 }
12828 #[cfg(not(feature = "latest-value-egress"))]
12829 let _ = side;
12830 result
12831 }
12832
12833 fn effective_transport_priority(&self, ty: DataType) -> u8 {
12834 if self.is_managed_variable_type(ty) {
12835 crate::transport_priority(DataType::ManagedVariableValue)
12836 } else {
12837 crate::transport_priority(ty)
12838 }
12839 }
12840
12841 #[cfg(all(test, feature = "std", feature = "discovery"))]
12842 fn call_side_tx_handler(
12843 &self,
12844 side: RouterSideId,
12845 handler: &RouterTxHandlerFn,
12846 data: &RouterItem,
12847 relayed: bool,
12848 ) -> TelemetryResult<()> {
12849 self.call_side_tx_handler_with_progress(side, handler, data, relayed, &mut false)
12850 }
12851
12852 fn canonical_catalog_stream(packet: &Packet, max_frame_bytes: usize) -> bool {
12855 packet.data_type() == DataType::DiscoverySchema
12856 && max_frame_bytes != 0
12857 && packet.data_size() > max_frame_bytes
12858 }
12859
12860 #[cfg(all(test, feature = "std", feature = "discovery"))]
12861 fn call_side_tx_handler_with_progress(
12862 &self,
12863 side: RouterSideId,
12864 handler: &RouterTxHandlerFn,
12865 data: &RouterItem,
12866 relayed: bool,
12867 admitted: &mut bool,
12868 ) -> TelemetryResult<()> {
12869 self.call_side_tx_handler_with_service(
12870 side,
12871 handler,
12872 data,
12873 relayed,
12874 admitted,
12875 EgressService::default(),
12876 )
12877 }
12878 fn call_side_tx_handler_with_service(
12879 &self,
12880 side: RouterSideId,
12881 handler: &RouterTxHandlerFn,
12882 data: &RouterItem,
12883 relayed: bool,
12884 admitted: &mut bool,
12885 service: EgressService,
12886 ) -> TelemetryResult<()> {
12887 let opts = {
12888 let st = self.state.lock();
12889 Self::side_ref(&st, side)?.opts.clone()
12890 };
12891 let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
12892 return Err(TelemetryError::Io("side tx busy"));
12893 };
12894 let started_ms = self.clock.now_ms();
12895 let (ty, priority) = self.item_service_type_priority(data)?;
12897 #[cfg(all(feature = "discovery", feature = "timesync"))]
12898 if !Self::timesync_allowed_for_side_locked(
12899 &self.state.lock(),
12900 side,
12901 ty,
12902 self.clock.now_ms(),
12903 ) {
12904 return Ok(());
12905 }
12906 let result = match (handler, data) {
12907 (
12908 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
12909 RouterItem::Packed(bytes),
12910 ) => {
12911 #[cfg(feature = "cryptography")]
12912 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
12913 #[cfg(not(feature = "cryptography"))]
12914 let send_bytes = bytes.clone();
12915 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
12916 let mut attempts_total = 0usize;
12917 let mut sent_bytes = 0usize;
12918 self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
12919 match self.retry_with_attempts(runtime_max_handler_retries(), || {
12920 self.packed_handler_call(side, packed, frame, priority, service)
12921 }) {
12922 Ok((_, attempts)) => {
12923 *admitted = true;
12924 attempts_total = attempts_total.saturating_add(attempts);
12925 sent_bytes = sent_bytes.saturating_add(frame.len());
12926 }
12927 Err((err, attempts)) => {
12928 self.note_side_tx_failure(
12929 side,
12930 ty,
12931 attempts_total.saturating_add(attempts),
12932 );
12933 return Err(err);
12934 }
12935 }
12936
12937 Ok(())
12938 })?;
12939 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
12940 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
12941 return Ok(());
12942 }
12943 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
12944 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
12945 }
12946 (
12947 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
12948 RouterItem::Packet(pkt),
12949 ) => {
12950 if Self::canonical_catalog_stream(pkt, opts.max_frame_bytes) {
12954 let mut attempts_total = 0usize;
12955 let mut sent_bytes = 0usize;
12956 let key = crate::side_transport::canonical_packet_progress_key(
12957 pkt.packet_id(),
12958 opts.max_frame_bytes,
12959 );
12960 #[cfg(feature = "latest-value-egress")]
12961 let bounded_turn = {
12962 let st = self.state.lock();
12963 service.kind == 1
12964 && service.side != 0
12965 && usize::from(service.side - 1) == side
12966 && service.token == st.latest_service_seq
12967 && st.latest_side() == Some(side)
12968 };
12969 #[cfg(not(feature = "latest-value-egress"))]
12970 let bounded_turn = false;
12971 let mut admitted_chunks = 0usize;
12972 let mut yielded = false;
12973 let result = self.run_side_chunk_transfer(side, key, false, service, |next| {
12974 crate::side_transport::send_packet_chunks(
12975 pkt,
12976 opts.max_frame_bytes,
12977 self.sender_arc().as_bytes(),
12978 next,
12979 |frame| {
12980 #[cfg(feature = "latest-value-egress")]
12981 let fresh_waiting = if bounded_turn && admitted_chunks != 0 {
12982 let st = self.state.lock();
12983 st.latest_side() == Some(side) && st.transmit_queue.iter().any(|queued|
12984 queued.disposable & (TX_LATEST_PENDING | TX_LATEST_TURN) != 0
12985 && matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side))
12986 } else { false };
12987 #[cfg(not(feature = "latest-value-egress"))]
12988 let fresh_waiting = bounded_turn && admitted_chunks != 0;
12989 if fresh_waiting {
12990 yielded = true;
12991 return Err(TelemetryError::Io("side tx busy"));
12992 }
12993 match self.retry_with_attempts(
12994 runtime_max_handler_retries(),
12995 || {
12996 self.packed_handler_call(
12997 side, packed, frame, priority, service,
12998 )
12999 },
13000 ) {
13001 Ok((_, attempts)) => {
13002 *admitted = true;
13003 attempts_total += attempts;
13004 sent_bytes += frame.len();
13005 admitted_chunks += 1;
13006 Ok(())
13007 }
13008 Err((err, attempts)) => {
13009 attempts_total += attempts;
13010 Err(err)
13011 }
13012 }
13013 },
13014 )
13015 });
13016 let (raw_len, wire_len, chunks) = match result {
13017 Ok(stats) => stats,
13018 Err(err) => {
13019 if !yielded {
13020 self.note_side_tx_failure(side, ty, attempts_total);
13021 }
13022 return Err(err);
13023 }
13024 };
13025 {
13026 let mut st = self.state.lock();
13027 let stats = st.side_runtime_stats.entry(side).or_default();
13028 stats.note_side_transport_full(raw_len, wire_len);
13029 if chunks > 1 {
13030 stats.note_side_transport_chunks(chunks);
13031 }
13032 }
13033 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
13034 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
13035 return Ok(());
13036 }
13037 let owned = self.pack_packet_for_router(pkt, None)?;
13038 let frames = self.encode_side_transport_frames(side, opts, owned)?;
13039 let mut attempts_total = 0usize;
13040 let mut sent_bytes = 0usize;
13041 self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
13042 match self.retry_with_attempts(runtime_max_handler_retries(), || {
13043 self.packed_handler_call(side, packed, frame, priority, service)
13044 }) {
13045 Ok((_, attempts)) => {
13046 *admitted = true;
13047 attempts_total = attempts_total.saturating_add(attempts);
13048 sent_bytes = sent_bytes.saturating_add(frame.len());
13049 }
13050 Err((err, attempts)) => {
13051 self.note_side_tx_failure(
13052 side,
13053 ty,
13054 attempts_total.saturating_add(attempts),
13055 );
13056 return Err(err);
13057 }
13058 }
13059
13060 Ok(())
13061 })?;
13062 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
13063 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
13064 return Ok(());
13065 }
13066 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
13067 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13068 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
13069 }
13070 };
13071 match result {
13072 Ok((_, attempts)) => {
13073 *admitted = true;
13074 if let Ok(bytes) = Self::router_item_wire_len(data) {
13075 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
13076 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
13077 }
13078 Ok(())
13079 }
13080 Err((err, attempts)) => {
13081 self.note_side_tx_failure(side, ty, attempts);
13082 Err(err)
13083 }
13084 }
13085 }
13086
13087 fn adjust_reliable_for_side(
13088 &self,
13089 opts: RouterSideOptions,
13090 data: RouterItem,
13091 preserve_end_to_end_ack: bool,
13092 ) -> TelemetryResult<Option<RouterItem>> {
13093 if opts.reliable_enabled {
13094 return Ok(Some(data));
13095 }
13096
13097 match data {
13098 RouterItem::Packed(bytes) => {
13099 let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
13100 if is_reliable_type(frame.envelope.ty)
13101 && let Some(hdr) = frame.reliable
13102 {
13103 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
13104 return Ok(None);
13105 }
13106 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
13107 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
13108 bytes.as_ref(),
13109 wire_format::RELIABLE_FLAG_UNSEQUENCED,
13110 hdr.seq,
13111 0,
13112 )?
13113 else {
13114 return Ok(Some(RouterItem::Packed(bytes)));
13115 };
13116 return Ok(Some(RouterItem::Packed(crate::shared_bytes::convert(
13117 rewritten,
13118 ))));
13119 }
13120 }
13121 Ok(Some(RouterItem::Packed(bytes)))
13122 }
13123 RouterItem::Packet(pkt) => {
13124 if matches!(
13125 pkt.data_type(),
13126 DataType::ReliableAck
13127 | DataType::ReliablePartialAck
13128 | DataType::ReliablePacketRequest
13129 ) {
13130 if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
13131 return Ok(Some(RouterItem::Packet(pkt)));
13132 }
13133 return Ok(None);
13134 }
13135 Ok(Some(RouterItem::Packet(pkt)))
13136 }
13137 }
13138 }
13139
13140 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
13141 let now = self.clock.now_ms();
13142 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
13143
13144 {
13145 let mut st = self.state.lock();
13146 if st.reliable_tx.is_empty() {
13147 return Ok(());
13148 }
13149
13150 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
13151 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
13152 continue;
13153 };
13154 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
13155 for seq in sent_order {
13156 let Some(sent) = tx_state.sent.get_mut(&seq) else {
13157 continue;
13158 };
13159 if sent.queued
13160 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
13161 {
13162 continue;
13163 }
13164 if sent.partial_acked {
13165 sent.partial_acked = false;
13172 sent.last_send_ms = now;
13173 continue;
13174 }
13175 if sent.retries >= runtime_reliable_max_retries() {
13176 tx_state.sent.remove(&seq);
13177 tx_state.sent_order.retain(|existing| *existing != seq);
13178 continue;
13179 }
13180 sent.retries += 1;
13181 requeue.push((*side, ty, seq));
13182 }
13183 }
13184 }
13185
13186 for (side, ty, seq) in requeue {
13187 self.queue_reliable_retransmit(side, ty, seq, true)?;
13188 }
13189
13190 Ok(())
13191 }
13192
13193 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
13194 let now = self.clock.now_ms();
13195 let mut requeue = Vec::new();
13196 let mut failed_ordered_types = BTreeSet::new();
13197
13198 {
13199 let mut st = self.state.lock();
13200 #[cfg(feature = "discovery")]
13201 {
13202 if Self::prune_discovery_routes_locked(&mut st, now) {
13203 Self::note_discovery_topology_change_locked(&mut st, now);
13204 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
13205 }
13206 }
13211 let RouterInner {
13214 transmit_queue,
13215 end_to_end_reliable_tx,
13216 ..
13217 } = &mut *st;
13218 end_to_end_reliable_tx.retain(|&packet_id, sent| {
13219 let covered = !sent.pending_destinations.is_empty()
13220 && sent
13221 .pending_destinations
13222 .values()
13223 .all(|side| Self::e2e_child_owned(transmit_queue, *side, packet_id));
13224 if covered
13225 || sent.waiting_for_predecessor
13226 || sent.queued
13227 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
13228 {
13229 return true;
13230 }
13231 if sent.retries >= runtime_reliable_max_retries() {
13232 if let Some(ty) = sent.ordered_type {
13233 failed_ordered_types.insert(ty);
13234 }
13235 return false;
13236 }
13237 sent.retries += 1;
13238 requeue.push(packet_id);
13239 true
13240 });
13241 st.end_to_end_reliable_tx.retain(|_, sent| {
13243 !sent
13244 .ordered_type
13245 .is_some_and(|ty| failed_ordered_types.contains(&ty))
13246 });
13247 }
13248
13249 for packet_id in requeue {
13250 self.queue_end_to_end_reliable_retransmit(packet_id)?;
13251 }
13252 self.release_ordered_successors()?;
13253 if !failed_ordered_types.is_empty() {
13254 return Err(TelemetryError::HandlerError(
13255 "ordered end-to-end delivery exhausted retries",
13256 ));
13257 }
13258 Ok(())
13259 }
13260
13261 #[cfg(feature = "timesync")]
13262 #[inline]
13263 fn monotonic_now_ns(&self) -> u64 {
13264 self.clock.now_ns()
13265 }
13266
13267 #[cfg(feature = "timesync")]
13268 #[inline]
13269 fn monotonic_now_ms(&self) -> u64 {
13270 self.clock.now_ms()
13271 }
13272
13273 #[cfg(feature = "timesync")]
13274 fn refresh_timesync_state(&self, now_mono_ms: u64) {
13275 if self.timesync.is_none() {
13276 return;
13277 }
13278
13279 let now_mono_ns = self.monotonic_now_ns();
13280 let mut st = self
13281 .timesync
13282 .as_ref()
13283 .expect("local time service enabled")
13284 .lock();
13285 st.clock.prune_expired(now_mono_ms);
13286 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13287 st.remote_sources
13288 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13289 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
13290 let leader = if let Some(tracker) = st.tracker.as_mut() {
13291 let _ = tracker.refresh(now_mono_ms);
13292 tracker.leader(now_mono_ms, has_usable_time)
13293 } else {
13294 None
13295 };
13296 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
13297 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
13298 let target_ms = st.remote_sources.get(remote.sender.as_str()).map(|src| {
13301 src.sample_unix_ms
13302 .saturating_add(now_mono_ms.saturating_sub(src.last_sample_mono_ms))
13303 });
13304 if let Some(target_ms) = target_ms {
13305 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
13306 }
13307 }
13308 }
13309
13310 #[cfg(feature = "timesync")]
13311 pub fn update_network_time_source(
13313 &self,
13314 source: &str,
13315 priority: u64,
13316 time: PartialNetworkTime,
13317 ttl_ms: Option<u64>,
13318 ) {
13319 if self.timesync.is_none() {
13320 return;
13321 }
13322
13323 let now_ms = self.monotonic_now_ms();
13324 let now_ns = self.monotonic_now_ns();
13325 let mut st = self
13326 .timesync
13327 .as_ref()
13328 .expect("local time service enabled")
13329 .lock();
13330 if st
13331 .clock
13332 .try_update_source(source, priority, time, now_ms, now_ns, ttl_ms)
13333 .is_err()
13334 {
13335 return;
13336 }
13337 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
13338 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
13339 }
13340 }
13341
13342 #[cfg(feature = "timesync")]
13343 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
13346 let _ = self.try_set_local_network_time(time);
13347 }
13348
13349 #[cfg(feature = "timesync")]
13350 pub fn try_set_local_network_time(&self, time: PartialNetworkTime) -> TelemetryResult<()> {
13353 let Some(runtime) = self.timesync.as_ref() else {
13354 return Ok(());
13355 };
13356 let observed_ns = self.monotonic_now_ns();
13357 let mut st = runtime.lock();
13358 let now_ms = self.monotonic_now_ms();
13359 let now_ns = self.monotonic_now_ns();
13360 let priority = st.cfg.map(|cfg| cfg.priority).unwrap_or(0);
13361 let time = time
13365 .to_network_time()
13366 .and_then(|base| advance_network_time(base, now_ns.saturating_sub(observed_ns)))
13367 .map(PartialNetworkTime::from)
13368 .unwrap_or(time);
13369 if time.is_complete_date() && time.is_complete_time() {
13370 let adjusted = time;
13371 let update = st.clock.prepare_source(
13372 LOCAL_TIMESYNC_FULL_SOURCE_ID,
13373 priority,
13374 adjusted,
13375 now_ms,
13376 now_ns,
13377 None,
13378 )?;
13379 st.clock.commit_source(update);
13380 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13381 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13382 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13383 if let Some(unix_ms) = adjusted
13384 .to_network_time()
13385 .and_then(|time| time.as_unix_ms())
13386 {
13387 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
13388 }
13389 } else {
13390 let date = if time.year.is_some() || time.month.is_some() || time.day.is_some() {
13393 Some(st.clock.prepare_source(
13394 LOCAL_TIMESYNC_DATE_SOURCE_ID,
13395 priority,
13396 PartialNetworkTime {
13397 year: time.year,
13398 month: time.month,
13399 day: time.day,
13400 ..Default::default()
13401 },
13402 now_ms,
13403 observed_ns,
13404 None,
13405 )?)
13406 } else {
13407 None
13408 };
13409 let tod = if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
13410 Some(st.clock.prepare_source(
13411 LOCAL_TIMESYNC_TOD_SOURCE_ID,
13412 priority,
13413 PartialNetworkTime {
13414 hour: time.hour,
13415 minute: time.minute,
13416 second: time.second,
13417 nanosecond: time.nanosecond,
13418 ..Default::default()
13419 },
13420 now_ms,
13421 observed_ns,
13422 None,
13423 )?)
13424 } else {
13425 None
13426 };
13427 let subsec = if time.nanosecond.is_some() {
13428 Some(st.clock.prepare_source(
13429 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
13430 priority,
13431 PartialNetworkTime {
13432 nanosecond: time.nanosecond,
13433 ..Default::default()
13434 },
13435 now_ms,
13436 observed_ns,
13437 None,
13438 )?)
13439 } else {
13440 None
13441 };
13442 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13443 for update in [date, tod, subsec].into_iter().flatten() {
13444 st.clock.commit_source(update);
13445 }
13446 }
13447 Ok(())
13448 }
13449
13450 #[cfg(feature = "timesync")]
13451 pub fn clear_local_network_time(&self) {
13453 if self.timesync.is_none() {
13454 return;
13455 }
13456
13457 let mut st = self
13458 .timesync
13459 .as_ref()
13460 .expect("local time service enabled")
13461 .lock();
13462 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13463 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13464 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13465 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13466 }
13467
13468 #[cfg(feature = "timesync")]
13469 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
13471 self.set_local_network_time(PartialNetworkTime {
13472 year: Some(year),
13473 month: Some(month),
13474 day: Some(day),
13475 ..Default::default()
13476 });
13477 }
13478
13479 #[cfg(feature = "timesync")]
13480 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
13482 self.set_local_network_time(PartialNetworkTime {
13483 hour: Some(hour),
13484 minute: Some(minute),
13485 ..Default::default()
13486 });
13487 }
13488
13489 #[cfg(feature = "timesync")]
13490 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
13492 self.set_local_network_time(PartialNetworkTime {
13493 hour: Some(hour),
13494 minute: Some(minute),
13495 second: Some(second),
13496 ..Default::default()
13497 });
13498 }
13499
13500 #[cfg(feature = "timesync")]
13501 pub fn set_local_network_time_hms_millis(
13503 &self,
13504 hour: u8,
13505 minute: u8,
13506 second: u8,
13507 millisecond: u16,
13508 ) {
13509 self.set_local_network_time(PartialNetworkTime {
13510 hour: Some(hour),
13511 minute: Some(minute),
13512 second: Some(second),
13513 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
13514 ..Default::default()
13515 });
13516 }
13517
13518 #[cfg(feature = "timesync")]
13519 pub fn set_local_network_time_hms_nanos(
13521 &self,
13522 hour: u8,
13523 minute: u8,
13524 second: u8,
13525 nanosecond: u32,
13526 ) {
13527 self.set_local_network_time(PartialNetworkTime {
13528 hour: Some(hour),
13529 minute: Some(minute),
13530 second: Some(second),
13531 nanosecond: Some(nanosecond),
13532 ..Default::default()
13533 });
13534 }
13535
13536 #[cfg(feature = "timesync")]
13537 pub fn set_local_network_datetime(
13539 &self,
13540 year: i32,
13541 month: u8,
13542 day: u8,
13543 hour: u8,
13544 minute: u8,
13545 second: u8,
13546 ) {
13547 self.set_local_network_time(PartialNetworkTime {
13548 year: Some(year),
13549 month: Some(month),
13550 day: Some(day),
13551 hour: Some(hour),
13552 minute: Some(minute),
13553 second: Some(second),
13554 ..Default::default()
13555 });
13556 }
13557
13558 #[cfg(feature = "timesync")]
13559 #[allow(clippy::too_many_arguments)]
13560 pub fn set_local_network_datetime_millis(
13562 &self,
13563 year: i32,
13564 month: u8,
13565 day: u8,
13566 hour: u8,
13567 minute: u8,
13568 second: u8,
13569 millisecond: u16,
13570 ) {
13571 self.set_local_network_time(PartialNetworkTime {
13572 year: Some(year),
13573 month: Some(month),
13574 day: Some(day),
13575 hour: Some(hour),
13576 minute: Some(minute),
13577 second: Some(second),
13578 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
13579 });
13580 }
13581
13582 #[cfg(feature = "timesync")]
13583 #[allow(clippy::too_many_arguments)]
13584 pub fn set_local_network_datetime_nanos(
13586 &self,
13587 year: i32,
13588 month: u8,
13589 day: u8,
13590 hour: u8,
13591 minute: u8,
13592 second: u8,
13593 nanosecond: u32,
13594 ) {
13595 self.set_local_network_time(PartialNetworkTime {
13596 year: Some(year),
13597 month: Some(month),
13598 day: Some(day),
13599 hour: Some(hour),
13600 minute: Some(minute),
13601 second: Some(second),
13602 nanosecond: Some(nanosecond),
13603 });
13604 }
13605
13606 #[cfg(feature = "timesync")]
13607 pub fn clear_network_time_source(&self, source: &str) {
13609 if self.timesync.is_none() {
13610 return;
13611 }
13612
13613 let mut st = self
13614 .timesync
13615 .as_ref()
13616 .expect("local time service enabled")
13617 .lock();
13618 st.clock.remove_source(source);
13619 }
13620
13621 #[cfg(feature = "timesync")]
13622 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
13624 if self.timesync.is_none() {
13625 return;
13626 }
13627
13628 let mut st = self
13629 .timesync
13630 .as_ref()
13631 .expect("local time service enabled")
13632 .lock();
13633 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
13634 st.cfg = cfg;
13635 st.tracker = cfg.map(TimeSyncTracker::new);
13636 st.disciplined_clock = SlewedNetworkClock::new(
13637 cfg.map(|c| c.max_slew_ppm)
13638 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
13639 );
13640 st.remote_sources.clear();
13641 st.next_seq = 1;
13642 st.next_announce_mono_ms = 0;
13643 st.next_request_mono_ms = 0;
13644 st.pending_request = None;
13645 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
13646 for source in stale_remote_sources {
13647 st.clock.remove_source(&source);
13648 }
13649 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13650 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13651 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13652 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13653 }
13654
13655 #[cfg(feature = "timesync")]
13656 pub fn network_time(&self) -> Option<NetworkTimeReading> {
13658 let runtime = self.timesync.as_ref()?;
13659
13660 let now_ms = self.monotonic_now_ms();
13661 let now_ns = self.monotonic_now_ns();
13662 self.refresh_timesync_state(now_ms);
13663 let st = runtime.lock();
13664 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
13665 return Some(NetworkTimeReading {
13666 time: PartialNetworkTime::from_unix_ms(unix_ms),
13667 unix_time_ms: Some(unix_ms),
13668 });
13669 }
13670 st.clock.current_time(now_ns)
13671 }
13672
13673 #[cfg(feature = "timesync")]
13674 pub fn network_time_ms(&self) -> Option<u64> {
13676 self.network_time().and_then(|t| t.unix_time_ms)
13677 }
13678
13679 #[cfg(feature = "timesync")]
13680 fn packet_timestamp_ms(&self) -> u64 {
13681 self.network_time_ms()
13682 .unwrap_or_else(|| self.monotonic_now_ms())
13683 }
13684
13685 #[cfg(not(feature = "timesync"))]
13686 fn packet_timestamp_ms(&self) -> u64 {
13687 self.clock.now_ms()
13688 }
13689
13690 #[cfg(feature = "timesync")]
13691 fn queue_internal_timesync_request(
13692 &self,
13693 seq: u64,
13694 t1_mono_ms: u64,
13695 called_from_queue: bool,
13696 ) -> TelemetryResult<()> {
13697 let pkt_ts = self.packet_timestamp_ms();
13698 if called_from_queue {
13699 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
13700 } else {
13701 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
13702 }
13703 }
13704
13705 #[cfg(feature = "timesync")]
13706 fn queue_internal_timesync_response(
13707 &self,
13708 seq: u64,
13709 t1_mono_ms: u64,
13710 t2_network_ms: u64,
13711 t3_network_ms: u64,
13712 dst: Option<RouterSideId>,
13713 called_from_queue: bool,
13714 ) -> TelemetryResult<()> {
13715 let pkt_ts = self.packet_timestamp_ms();
13716 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
13717 let sender = self.sender_arc();
13718 let pkt = Packet::new(
13719 DataType::TimeSyncResponse,
13720 &[DataEndpoint::TimeSync],
13721 sender.as_ref(),
13722 pkt_ts,
13723 payload,
13724 )?;
13725 match dst {
13726 Some(dst) => self.emit_internal_tx(
13727 RouterTxItem::ToSide {
13728 src: None,
13729 dst,
13730 data: RouterItem::Packet(pkt),
13731 },
13732 true,
13733 called_from_queue,
13734 ),
13735 None => self.emit_internal_tx(
13736 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
13737 true,
13738 called_from_queue,
13739 ),
13740 }
13741 }
13742
13743 #[cfg(feature = "timesync")]
13744 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
13746 if self.timesync.is_none() {
13747 return Ok(false);
13748 }
13749
13750 let now_ms = self.monotonic_now_ms();
13751 let now_ns = self.monotonic_now_ns();
13752 let mut queued_any = false;
13753 let mut announce_priority = None;
13754 let mut request = None;
13755
13756 {
13757 let mut st = self
13758 .timesync
13759 .as_ref()
13760 .expect("local time service enabled")
13761 .lock();
13762 st.clock.prune_expired(now_ms);
13763 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13764 st.remote_sources
13765 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13766 let Some(cfg) = st.cfg else {
13767 return Ok(false);
13768 };
13769
13770 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
13771 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
13772 let _ = tracker.refresh(now_ms);
13773 (
13774 tracker.leader(now_ms, has_usable_time),
13775 tracker.local_announce_priority(now_ms, has_usable_time),
13776 )
13777 } else {
13778 (None, None)
13779 };
13780 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
13781
13782 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
13783 let target_ms = st.remote_sources.get(&remote.sender).map(|src| {
13784 src.sample_unix_ms
13785 .saturating_add(now_ms.saturating_sub(src.last_sample_mono_ms))
13786 });
13787 if let Some(target_ms) = target_ms {
13788 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
13789 }
13790 }
13791
13792 if let Some(priority) = announce_prio
13793 && now_ms >= st.next_announce_mono_ms
13794 {
13795 announce_priority = Some(priority);
13796 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
13797 }
13798
13799 if let Some(TimeSyncLeader::Remote(remote)) = leader
13800 && now_ms >= st.next_request_mono_ms
13801 && st.pending_request.is_none()
13802 {
13803 let seq = st.next_seq;
13804 let next = st.next_seq.wrapping_add(1);
13805 st.next_seq = if next == 0 { 1 } else { next };
13806 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
13807 st.pending_request = Some(PendingTimeSyncRequest {
13808 seq,
13809 t1_mono_ms: now_ms,
13810 source: remote.sender,
13811 });
13812 request = Some((seq, now_ms));
13813 }
13814 }
13815
13816 if let Some(priority) = announce_priority {
13817 let time_ms = self.packet_timestamp_ms();
13818 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
13819 queued_any = true;
13820 }
13821 if let Some((seq, t1_mono_ms)) = request {
13822 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
13823 queued_any = true;
13824 }
13825
13826 Ok(queued_any)
13827 }
13828
13829 #[cfg(feature = "timesync")]
13830 fn handle_internal_timesync_packet(
13831 &self,
13832 pkt: &Packet,
13833 src: Option<RouterSideId>,
13834 called_from_queue: bool,
13835 ) -> TelemetryResult<bool> {
13836 let Some(cfg) = self.cfg.timesync_config() else {
13837 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
13838 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
13839 }
13840 return Ok(true);
13841 };
13842
13843 let now_mono_ms = self.monotonic_now_ms();
13844 let now_mono_ns = self.monotonic_now_ns();
13845 let mut response = None;
13846 let mut poll_after = false;
13847
13848 {
13849 let mut st = self
13850 .timesync
13851 .as_ref()
13852 .expect("local time service enabled")
13853 .lock();
13854 st.clock.prune_expired(now_mono_ms);
13855 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13856 st.remote_sources
13857 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13858 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
13859 if st.tracker.is_none() {
13860 return Ok(true);
13861 }
13862
13863 match pkt.data_type() {
13864 DataType::TimeSyncAnnounce => {
13865 let ann = decode_timesync_announce(pkt)?;
13866 let tracker_update = st
13870 .tracker
13871 .as_ref()
13872 .expect("tracker checked above")
13873 .prepare_source(pkt.sender(), ann, now_mono_ms)?;
13874 let remote_update = crate::peer_record::prepare(
13875 &st.remote_sources,
13876 pkt.sender(),
13877 RemoteTimeSyncSource {
13878 priority: ann.priority,
13879 last_sample_mono_ms: now_mono_ms,
13880 sample_unix_ms: ann.time_ms,
13881 },
13882 )?;
13883 let clock_update = st.clock.prepare_source(
13884 pkt.sender(),
13885 ann.priority,
13886 PartialNetworkTime::from_unix_ms(ann.time_ms),
13887 now_mono_ms,
13888 now_mono_ns,
13889 Some(cfg.source_timeout_ms),
13890 )?;
13891 remote_update.commit(&mut st.remote_sources);
13892 st.clock.commit_source(clock_update);
13893 let should_steer = {
13894 let tracker = st.tracker.as_mut().expect("tracker checked above");
13895 tracker.commit_source(tracker_update);
13896 matches!(tracker.leader(now_mono_ms, has_usable_time),
13897 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender())
13898 };
13899 if should_steer {
13900 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
13901 }
13902 poll_after = true;
13903 }
13904 DataType::TimeSyncRequest => {
13905 let should_serve = {
13906 let tracker = st.tracker.as_ref().expect("tracker checked above");
13907 tracker.should_serve(now_mono_ms, has_usable_time)
13908 };
13909 if should_serve {
13910 let req = decode_timesync_request(pkt)?;
13911 let network_now = st
13912 .disciplined_clock
13913 .read_unix_ms(now_mono_ns)
13914 .or_else(|| {
13915 st.clock
13916 .current_time(now_mono_ns)
13917 .and_then(|t| t.unix_time_ms)
13918 })
13919 .unwrap_or(now_mono_ms);
13920 let t2 = network_now;
13921 let t3 = network_now;
13922 response = Some((req.seq, req.t1_ms, t2, t3, src));
13923 }
13924 }
13925 DataType::TimeSyncResponse => {
13926 let resp = decode_timesync_response(pkt)?;
13927 let pending = st.pending_request.clone();
13928 if let Some(pending) = pending
13929 && pending.seq == resp.seq
13930 && pending.source == pkt.sender()
13931 {
13932 let source_priority = {
13933 let tracker = st.tracker.as_ref().expect("tracker checked above");
13934 tracker
13935 .best_active_source(now_mono_ms)
13936 .map(|s| s.priority)
13937 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
13938 .unwrap_or(cfg.priority)
13939 };
13940 let (estimate_ms, _delay_ms) = compute_network_time_sample(
13941 pending.t1_mono_ms,
13942 resp.t2_ms,
13943 resp.t3_ms,
13944 now_mono_ms,
13945 );
13946 let remote_update = crate::peer_record::prepare(
13947 &st.remote_sources,
13948 pkt.sender(),
13949 RemoteTimeSyncSource {
13950 priority: source_priority,
13951 last_sample_mono_ms: now_mono_ms,
13952 sample_unix_ms: estimate_ms,
13953 },
13954 )?;
13955 let clock_update = st.clock.prepare_source(
13956 pkt.sender(),
13957 source_priority,
13958 PartialNetworkTime::from_unix_ms(estimate_ms),
13959 now_mono_ms,
13960 now_mono_ns,
13961 Some(cfg.source_timeout_ms),
13962 )?;
13963 remote_update.commit(&mut st.remote_sources);
13964 st.clock.commit_source(clock_update);
13965 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
13966 st.pending_request = None;
13967 }
13968 }
13969 _ => {}
13970 }
13971 }
13972
13973 if let Some((seq, t1, t2, t3, dst)) = response {
13974 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
13975 }
13976 if poll_after {
13977 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
13978 }
13979
13980 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
13981 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
13982 }
13983
13984 Ok(true)
13985 }
13986
13987 #[cfg(feature = "std")]
13989 pub fn new(cfg: RouterConfig) -> Self {
13990 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
13991 }
13992
13993 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
13995 #[cfg(feature = "timesync")]
13996 let timesync_cfg = cfg.timesync_config();
13997 #[cfg(feature = "discovery")]
13998 let preferred_discovery_master = cfg.preferred_discovery_master.clone();
13999 let memory = cfg.memory_config();
14000 let local_services_enabled = cfg.role.local_services();
14001 let hostname: Arc<str> = cfg.configured_sender();
14002 let address_mode = cfg.address_mode();
14003 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
14004 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
14005 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
14006 let requested = address_mode.requested_address();
14007 let address = if requested == 0 { fallback } else { requested };
14008 let local_address = AddressBookEntry {
14009 hostname: hostname.clone(),
14010 address,
14011 requested_address: requested,
14012 mode: address_mode,
14013 birth_ms: clock
14014 .now_ms()
14015 .saturating_mul(1_000_000)
14016 .saturating_add(instance_seq),
14017 owner_hash,
14018 last_seen_ms: clock.now_ms(),
14019 };
14020 let mut address_book = BTreeMap::new();
14021 if local_services_enabled {
14022 address_book.insert(hostname.to_string(), Box::new(local_address.clone()));
14023 }
14024 let mut address_by_value = BTreeMap::new();
14025 if local_services_enabled {
14026 address_by_value.insert(address, hostname.to_string());
14027 }
14028 Self {
14029 #[cfg(feature = "std")]
14030 retry_trackers: RouterMutex::new(Vec::new()),
14031 sender: RouterMutex::new(hostname),
14032 cfg: ActiveRoutingConfig::new(cfg),
14033 state: RouterMutex::new(RouterInner {
14034 #[cfg(feature = "compression")]
14035 codec_reserved_bytes: 0,
14036 memory,
14037 sides: Vec::new(),
14038 route_overrides: Default::default(),
14039 typed_route_overrides: Default::default(),
14040 route_weights: Default::default(),
14041 route_priorities: Default::default(),
14042 source_route_modes: Default::default(),
14043 route_selection_cursors: Default::default(),
14044 adaptive_route_stats: Default::default(),
14045 side_runtime_stats: Default::default(),
14046 side_transport: Default::default(),
14047 local_services: local_services_enabled.then(|| {
14048 Box::new(LocalServicesState {
14049 managed_variable_types: BTreeSet::new(),
14050 managed_variable_permissions: Default::default(),
14051 managed_variable_latest: Default::default(),
14052 network_variable_update_handlers: Default::default(),
14053 p2p_port_handlers: Default::default(),
14054 p2p_stream_handlers: Default::default(),
14055 p2p_stream_sessions: Default::default(),
14056 next_p2p_stream_id: 1,
14057 })
14058 }),
14059 local_address,
14060 address_book,
14061 address_by_value,
14062 received_queue: BoundedDeque::new(
14063 memory.max_queue_budget,
14064 memory.starting_queue_size,
14065 memory.queue_grow_step,
14066 ),
14067 transmit_queue: BoundedDeque::new(
14068 memory.max_queue_budget,
14069 memory.starting_queue_size,
14070 memory.queue_grow_step,
14071 ),
14072 #[cfg(feature = "std")]
14073 active_queued_types: [(0, 0); ACTIVE_QUEUED_TYPE_SLOTS],
14074 tx_priority_burst: 0,
14075 tx_policy_flags: 0,
14076 #[cfg(feature = "latest-value-egress")]
14077 latest_turn_phase: 0,
14078 #[cfg(feature = "latest-value-egress")]
14079 latest_turn_credit: 0,
14080 #[cfg(feature = "latest-value-egress")]
14081 latest_service_seq: 0,
14082 ordered_claim_seq: 0,
14083 rx_priority_burst: 0,
14084 recent_rx: BoundedDeque::new(
14085 memory.recent_rx_queue_bytes(),
14086 memory.recent_rx_queue_bytes(),
14087 memory.queue_grow_step,
14088 ),
14089 reliable_tx: Default::default(),
14090 reliable_rx: Default::default(),
14091 reliable_return_routes: Default::default(),
14092 local_admission_sequence: 0,
14093 reliable_return_route_order: VecDeque::new(),
14094 forwarded_receipts: None,
14095 end_to_end_reliable_tx: Default::default(),
14096 end_to_end_reliable_tx_order: VecDeque::new(),
14097 total_handler_failures: 0,
14098 total_handler_retries: 0,
14099 #[cfg(feature = "discovery")]
14100 discovery_routes: Default::default(),
14101 #[cfg(feature = "discovery")]
14102 discovery_cadence: DiscoveryCadenceState::default(),
14103 #[cfg(feature = "discovery")]
14104 discovery_side_throttle: Default::default(),
14105 #[cfg(feature = "discovery")]
14106 schema_response_sequence: 0,
14107 #[cfg(feature = "discovery")]
14108 preferred_discovery_master,
14109 #[cfg(all(feature = "discovery", feature = "timesync"))]
14110 timesync_side_throttle: Default::default(),
14111 }),
14112 isr_rx_queue: local_services_enabled.then(|| {
14113 Box::new(IsrRxQueue::new(
14114 memory.max_queue_budget,
14115 memory.starting_queue_size,
14116 memory.queue_grow_step,
14117 ))
14118 }),
14119 #[cfg(not(feature = "std"))]
14120 side_tx_gate: ReentryGate::new(),
14121 #[cfg(feature = "std")]
14122 side_tx_activity: crate::lock::SideTxActivity::default(),
14123 #[cfg(feature = "std")]
14124 active_queued_tx: core::sync::atomic::AtomicUsize::new(0),
14125 clock,
14126 #[cfg(feature = "timesync")]
14127 timesync: local_services_enabled
14128 .then(|| Box::new(RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)))),
14129 }
14130 }
14131
14132 #[inline]
14133 fn sender_arc(&self) -> Arc<str> {
14134 self.sender.lock().clone()
14135 }
14136
14137 #[inline]
14138 pub fn sender(&self) -> Arc<str> {
14139 self.sender_arc()
14140 }
14141
14142 pub fn current_address(&self) -> NodeAddress {
14143 self.state.lock().local_address.address
14144 }
14145
14146 pub fn hostname(&self) -> Arc<str> {
14147 self.sender_arc()
14148 }
14149
14150 pub fn set_preferred_discovery_master(&self, hostname: Option<&str>) {
14155 #[cfg(feature = "discovery")]
14156 {
14157 let preferred = hostname
14158 .filter(|value| !value.is_empty())
14159 .map(Arc::<str>::from);
14160 let mut st = self.state.lock();
14161 if st.preferred_discovery_master != preferred {
14162 st.preferred_discovery_master = preferred;
14163 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14164 }
14165 }
14166 #[cfg(not(feature = "discovery"))]
14167 let _ = hostname;
14168 }
14169
14170 pub fn preferred_discovery_master(&self) -> Option<Arc<str>> {
14172 #[cfg(feature = "discovery")]
14173 {
14174 return self.state.lock().preferred_discovery_master.clone();
14175 }
14176 #[cfg(not(feature = "discovery"))]
14177 None
14178 }
14179
14180 pub fn sender_matches_hostname(&self, sender: &str, expected_hostname: &str) -> bool {
14190 if sender.is_empty()
14191 || expected_hostname.is_empty()
14192 || sender.contains('\0')
14193 || expected_hostname.contains('\0')
14194 || expected_hostname.starts_with("@addr:")
14195 {
14196 return false;
14197 }
14198 let Some(value) = sender.strip_prefix("@addr:") else {
14199 return sender == expected_hostname;
14200 };
14201 if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
14202 return false;
14203 }
14204 let Ok(address) = value.parse::<u32>() else {
14205 return false;
14206 };
14207 let state = self.state.lock();
14208 Self::hostname_for_address_borrowed_locked(&state, address) == Some(expected_hostname)
14209 }
14210
14211 pub fn address_book(&self) -> Vec<AddressBookEntry> {
14212 self.state
14213 .lock()
14214 .address_book
14215 .values()
14216 .map(|entry| entry.as_ref().clone())
14217 .collect()
14218 }
14219
14220 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
14221 let now_ms = self.clock.now_ms();
14222 #[allow(unused_mut)]
14223 let mut st = self.state.lock();
14224 #[cfg(feature = "discovery")]
14225 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
14226 Self::note_discovery_topology_change_locked(&mut st, now_ms);
14227 }
14228 if let Some(entry) = st.address_book.get(hostname) {
14229 return Some(entry.as_ref().clone());
14230 }
14231 let address = sender_address_u32(hostname);
14237 if Self::hostname_for_address_locked(&st, address).as_deref() != Some(hostname) {
14238 return None;
14239 }
14240 Some(AddressBookEntry {
14241 hostname: Arc::from(hostname),
14242 address,
14243 requested_address: 0,
14244 mode: AddressAssignmentMode::Dynamic,
14245 birth_ms: 0,
14246 owner_hash: Self::sender_hash(hostname),
14247 last_seen_ms: now_ms,
14248 })
14249 }
14250
14251 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
14252 let st = self.state.lock();
14253 let direct = st
14254 .address_by_value
14255 .get(&address)
14256 .and_then(|hostname| st.address_book.get(hostname))
14257 .map(|entry| entry.as_ref().clone());
14258 if direct.is_some() {
14259 return direct;
14260 }
14261
14262 let hostname = Self::hostname_for_address_locked(&st, address)?;
14263 if let Some(entry) = st.address_book.get(&hostname) {
14264 return Some(entry.as_ref().clone());
14265 }
14266
14267 let hostname: Arc<str> = Arc::from(hostname);
14272 Some(AddressBookEntry {
14273 hostname: hostname.clone(),
14274 address,
14275 requested_address: 0,
14276 mode: AddressAssignmentMode::Dynamic,
14277 birth_ms: 0,
14278 owner_hash: Self::sender_hash(hostname.as_ref()),
14279 last_seen_ms: self.clock.now_ms(),
14280 })
14281 }
14282
14283 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
14284 where
14285 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
14286 {
14287 if port == 0 {
14288 return Err(TelemetryError::BadArg);
14289 }
14290 let mut st = self.state.lock();
14291 st.local_services
14292 .as_mut()
14293 .expect("local services enabled")
14294 .p2p_port_handlers
14295 .entry(port)
14296 .or_default()
14297 .push(P2pPortHandler {
14298 handler: Arc::new(f),
14299 });
14300 Ok(())
14301 }
14302
14303 pub fn clear_p2p_port(&self, port: P2pPort) {
14304 self.state
14305 .lock()
14306 .local_services
14307 .as_mut()
14308 .expect("local services enabled")
14309 .p2p_port_handlers
14310 .remove(&port);
14311 }
14312
14313 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
14314 where
14315 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
14316 {
14317 if port == 0 {
14318 return Err(TelemetryError::BadArg);
14319 }
14320 let mut st = self.state.lock();
14321 st.local_services
14322 .as_mut()
14323 .expect("local services enabled")
14324 .p2p_stream_handlers
14325 .entry(port)
14326 .or_default()
14327 .push(P2pStreamHandler {
14328 handler: Arc::new(f),
14329 });
14330 Ok(())
14331 }
14332
14333 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
14334 self.state
14335 .lock()
14336 .local_services
14337 .as_mut()
14338 .expect("local services enabled")
14339 .p2p_stream_handlers
14340 .remove(&port);
14341 }
14342
14343 fn send_p2p_to_entry(
14344 &self,
14345 dst: AddressBookEntry,
14346 dst_port: P2pPort,
14347 src_port: P2pPort,
14348 payload: &[u8],
14349 ) -> TelemetryResult<()> {
14350 if dst_port == 0 {
14351 return Err(TelemetryError::BadArg);
14352 }
14353 let local = self.state.lock().local_address.clone();
14354 let payload = Self::encode_p2p_payload(
14355 local.hostname.as_ref(),
14356 local.address,
14357 src_port,
14358 dst_port,
14359 payload,
14360 )?;
14361 let pkt = Packet::new(
14362 DataType::P2pMessage,
14363 &[DataEndpoint::Discovery],
14364 local.hostname.as_ref(),
14365 self.packet_timestamp_ms(),
14366 crate::shared_bytes::convert(payload),
14367 )?;
14368 let target = Self::sender_hash(dst.hostname.as_ref());
14369 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
14370 if target == Self::sender_hash(self.sender_arc().as_ref()) {
14371 self.dispatch_p2p_packet(&pkt)?;
14372 return Ok(());
14373 }
14374 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
14375 }
14376
14377 pub fn send_p2p_to_hostname(
14378 &self,
14379 hostname: &str,
14380 dst_port: P2pPort,
14381 src_port: P2pPort,
14382 payload: &[u8],
14383 ) -> TelemetryResult<()> {
14384 let dst = self
14385 .resolve_hostname(hostname)
14386 .ok_or(TelemetryError::BadArg)?;
14387 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
14388 }
14389
14390 pub fn send_p2p_to_address(
14391 &self,
14392 address: NodeAddress,
14393 dst_port: P2pPort,
14394 src_port: P2pPort,
14395 payload: &[u8],
14396 ) -> TelemetryResult<()> {
14397 let dst = self
14398 .resolve_address(address)
14399 .ok_or(TelemetryError::BadArg)?;
14400 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
14401 }
14402
14403 fn open_p2p_stream_to_entry(
14404 &self,
14405 dst: AddressBookEntry,
14406 dst_port: P2pPort,
14407 src_port: P2pPort,
14408 ) -> TelemetryResult<P2pStreamId> {
14409 if dst_port == 0 || src_port == 0 {
14410 return Err(TelemetryError::BadArg);
14411 }
14412 let stream_id = {
14413 let mut st = self.state.lock();
14414 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
14415 if stream_id == 0 {
14416 return Err(TelemetryError::Io("p2p stream id exhausted"));
14417 }
14418 st.local_services
14419 .as_mut()
14420 .expect("local services enabled")
14421 .p2p_stream_sessions
14422 .insert(
14423 stream_id,
14424 P2pStreamSession {
14425 peer_hostname: dst.hostname.clone(),
14426 peer_address: dst.address,
14427 local_port: src_port,
14428 peer_port: dst_port,
14429 peer_stream_id: 0,
14430 next_sequence: 1,
14431 connected: false,
14432 },
14433 );
14434 stream_id
14435 };
14436 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
14437 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
14438 Ok(stream_id)
14439 }
14440
14441 pub fn open_p2p_stream_to_hostname(
14442 &self,
14443 hostname: &str,
14444 dst_port: P2pPort,
14445 src_port: P2pPort,
14446 ) -> TelemetryResult<P2pStreamId> {
14447 let dst = self
14448 .resolve_hostname(hostname)
14449 .ok_or(TelemetryError::BadArg)?;
14450 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
14451 }
14452
14453 pub fn open_p2p_stream_to_address(
14454 &self,
14455 address: NodeAddress,
14456 dst_port: P2pPort,
14457 src_port: P2pPort,
14458 ) -> TelemetryResult<P2pStreamId> {
14459 let dst = self
14460 .resolve_address(address)
14461 .ok_or(TelemetryError::BadArg)?;
14462 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
14463 }
14464
14465 fn send_p2p_stream_control(
14466 &self,
14467 stream_id: P2pStreamId,
14468 flags: u8,
14469 payload: &[u8],
14470 ) -> TelemetryResult<()> {
14471 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
14472 let mut st = self.state.lock();
14473 let Some(session) = st
14474 .local_services
14475 .as_mut()
14476 .expect("local services enabled")
14477 .p2p_stream_sessions
14478 .get_mut(&stream_id)
14479 else {
14480 return Err(TelemetryError::BadArg);
14481 };
14482 let peer_hostname = session.peer_hostname.clone();
14483 let peer_address = session.peer_address;
14484 let peer_port = session.peer_port;
14485 let local_port = session.local_port;
14486 let peer_stream_id = session.peer_stream_id;
14487 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
14488 let seq = session.next_sequence;
14489 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
14490 let dst = st
14491 .address_book
14492 .get(peer_hostname.as_ref())
14493 .map(|entry| entry.as_ref().clone())
14494 .unwrap_or(AddressBookEntry {
14495 hostname: peer_hostname.clone(),
14496 address: peer_address,
14497 requested_address: peer_address,
14498 mode: AddressAssignmentMode::Dynamic,
14499 birth_ms: self.clock.now_ms(),
14500 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
14501 last_seen_ms: self.clock.now_ms(),
14502 });
14503 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
14504 };
14505 if refresh_syn {
14506 let syn_payload =
14507 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
14508 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
14509 }
14510 let stream_payload =
14511 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
14512 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
14513 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
14514 self.state
14515 .lock()
14516 .local_services
14517 .as_mut()
14518 .expect("local services enabled")
14519 .p2p_stream_sessions
14520 .remove(&stream_id);
14521 }
14522 Ok(())
14523 }
14524
14525 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
14526 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
14527 }
14528
14529 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
14530 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
14531 }
14532
14533 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
14534 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
14535 }
14536
14537 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
14538 let hostname: Arc<str> = Arc::from(sender.as_ref());
14539 if !self.cfg.role.local_services() {
14540 *self.sender.lock() = hostname;
14543 return;
14544 }
14545 let mut st = self.state.lock();
14546 let mut local = st.local_address.clone();
14547 local.hostname = hostname.clone();
14548 local.owner_hash = Self::sender_hash(hostname.as_ref());
14549 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
14550 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
14551 local.requested_address = 0;
14552 }
14553 let Ok(change) = self.update_local_identity_locked(
14554 &mut st,
14555 local,
14556 AddressChangeReason::Configured,
14557 None,
14558 ) else {
14559 return;
14560 };
14561 drop(st);
14562 let _ = self.notify_address_change(change);
14563 }
14564
14565 pub fn set_address_assignment(
14566 &self,
14567 mode: AddressAssignmentMode,
14568 ) -> TelemetryResult<AddressChange> {
14569 let mut st = self.state.lock();
14570 let mut local = st.local_address.clone();
14571 local.mode = mode;
14572 local.requested_address = mode.requested_address();
14573 local.address = match mode {
14574 AddressAssignmentMode::Dynamic => {
14575 Self::fallback_address_for_hostname(local.hostname.as_ref())
14576 }
14577 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
14578 if address == 0 {
14579 1
14580 } else {
14581 address
14582 }
14583 }
14584 };
14585 let spare = if st
14586 .address_book
14587 .contains_key(st.local_address.hostname.as_ref())
14588 {
14589 None
14590 } else {
14591 #[cfg(any(feature = "discovery", feature = "timesync"))]
14592 let record = crate::peer_record::try_box(st.local_address.clone())?;
14593 #[cfg(not(any(feature = "discovery", feature = "timesync")))]
14594 let record = Box::new(st.local_address.clone());
14595 Some(record)
14596 };
14597 let change = self.update_local_identity_locked(
14598 &mut st,
14599 local,
14600 AddressChangeReason::Configured,
14601 spare,
14602 )?;
14603 drop(st);
14604 self.notify_address_change(change.clone())?;
14605 Ok(change)
14606 }
14607
14608 pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
14617 where
14618 N: AsRef<str>,
14619 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14620 {
14621 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
14622 }
14623
14624 pub fn add_side_packed_small_packets<N, F>(
14628 &self,
14629 name: N,
14630 tx: F,
14631 max_frame_bytes: usize,
14632 ) -> RouterSideId
14633 where
14634 N: AsRef<str>,
14635 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14636 {
14637 self.add_side_packed_with_options(
14638 name,
14639 tx,
14640 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
14641 )
14642 }
14643
14644 pub fn add_side_packed_with_options<N, F>(
14653 &self,
14654 name: N,
14655 tx: F,
14656 opts: RouterSideOptions,
14657 ) -> RouterSideId
14658 where
14659 N: AsRef<str>,
14660 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14661 {
14662 let mut st = self.state.lock();
14663 let side = Some(RouterSide {
14664 name: Arc::from(name.as_ref()),
14665 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
14666 opts,
14667 });
14668 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14669 st.sides[id] = side;
14670 id
14671 } else {
14672 let id = st.sides.len();
14673 st.sides.push(side);
14674 id
14675 };
14676 st.side_runtime_stats
14677 .insert(id, Box::<SideRuntimeStatsInner>::default());
14678 st.side_transport
14679 .insert(id, Box::<SideTransportState>::default());
14680 #[cfg(feature = "discovery")]
14681 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14682 id
14683 }
14684
14685 pub fn add_side_packed_with_priority_and_options<N, F>(
14694 &self,
14695 name: N,
14696 tx: F,
14697 opts: RouterSideOptions,
14698 ) -> RouterSideId
14699 where
14700 N: AsRef<str>,
14701 F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
14702 {
14703 let mut st = self.state.lock();
14704 let side = Some(RouterSide {
14705 name: Arc::from(name.as_ref()),
14706 tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
14707 opts,
14708 });
14709 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14710 st.sides[id] = side;
14711 id
14712 } else {
14713 let id = st.sides.len();
14714 st.sides.push(side);
14715 id
14716 };
14717 st.side_runtime_stats
14718 .insert(id, Box::<SideRuntimeStatsInner>::default());
14719 st.side_transport
14720 .insert(id, Box::<SideTransportState>::default());
14721 #[cfg(feature = "discovery")]
14722 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14723 id
14724 }
14725
14726 pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
14731 where
14732 N: AsRef<str>,
14733 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
14734 {
14735 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
14736 }
14737
14738 pub fn add_side_packet_with_options<N, F>(
14744 &self,
14745 name: N,
14746 tx: F,
14747 opts: RouterSideOptions,
14748 ) -> RouterSideId
14749 where
14750 N: AsRef<str>,
14751 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
14752 {
14753 let mut st = self.state.lock();
14754 let side = Some(RouterSide {
14755 name: Arc::from(name.as_ref()),
14756 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
14757 opts,
14758 });
14759 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14760 st.sides[id] = side;
14761 id
14762 } else {
14763 let id = st.sides.len();
14764 st.sides.push(side);
14765 id
14766 };
14767 st.side_runtime_stats
14768 .insert(id, Box::<SideRuntimeStatsInner>::default());
14769 st.side_transport
14770 .insert(id, Box::<SideTransportState>::default());
14771 #[cfg(feature = "discovery")]
14772 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14773 id
14774 }
14775
14776 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
14781 #[cfg(feature = "discovery")]
14782 let now_ms = self.clock.now_ms();
14783 {
14784 let mut st = self.state.lock();
14785 if st.reliable_return_routes.values().any(|route| {
14786 route.side == side && route.local_admission_token != 0 && route.admission_committing
14787 }) {
14788 return Err(TelemetryError::Io("side rx commit busy"));
14789 }
14790
14791 #[cfg(feature = "std")]
14792 if st
14793 .side_transport
14794 .get(&side)
14795 .is_some_and(|state| state.tx_callback_gate.is_active())
14796 {
14797 return Err(TelemetryError::Io("side tx busy"));
14798 }
14799 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
14800 if slot.is_none() {
14801 return Err(TelemetryError::BadArg);
14802 }
14803 #[cfg(feature = "std")]
14804 self.invalidate_retry_side(side);
14805 *slot = None;
14806 #[cfg(feature = "latest-value-egress")]
14807 if st.latest_side() == Some(side) {
14808 st.latest_turn_phase = 0;
14809 st.latest_turn_credit = 0;
14810 st.tx_policy_flags &= 1;
14811 }
14812 while st.sides.last().is_some_and(Option::is_none) {
14813 st.sides.pop();
14814 }
14815 st.route_overrides
14820 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14821 st.typed_route_overrides
14822 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
14823 st.route_weights
14824 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14825 st.route_priorities
14826 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14827 st.source_route_modes.remove(&Some(side));
14828 st.route_selection_cursors.remove(&Some(side));
14829 st.adaptive_route_stats.remove(&side);
14830 #[cfg(feature = "discovery")]
14831 st.discovery_side_throttle.remove(&side);
14832 #[cfg(all(feature = "discovery", feature = "timesync"))]
14833 st.timesync_side_throttle.remove(&side);
14834 st.side_runtime_stats.remove(&side);
14835 st.side_transport.remove(&side);
14836 st.reliable_return_routes.retain(|_, route| {
14837 if route.side != side {
14838 return true;
14839 }
14840 if route.local_admission_token != 0 {
14841 route.admission_cancelled = true;
14842 return true;
14843 }
14844 false
14845 });
14846 st.retain_transmit(
14847 |queued| {
14848 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
14849 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
14850 },
14851 );
14852 st.received_queue.retain(|queued| queued.src != Some(side));
14853 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
14854 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
14855 #[cfg(feature = "discovery")]
14856 {
14857 st.cancel_managed_refresh_side(side);
14858 st.discovery_routes.remove(&side);
14859 Self::note_discovery_topology_change_locked(&mut st, now_ms);
14860 }
14861 }
14862 if let Some(queue) = self.isr_rx_queue.as_ref() {
14863 let mut isr_rx = queue.try_lock()?;
14864 isr_rx.retain(|queued| queued.src != Some(side));
14865 }
14866 Ok(())
14867 }
14868
14869 #[cfg(feature = "latest-value-egress")]
14875 pub fn set_latest_value_side(&self, side: Option<RouterSideId>) -> TelemetryResult<()> {
14876 let mut st = self.state.lock();
14877 let encoded = if let Some(side) = side {
14878 if side >= 127 {
14879 return Err(TelemetryError::BadArg);
14880 }
14881 let entry = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
14882 if matches!(entry.tx_handler, RouterTxHandlerFn::Packet(_)) {
14883 return Err(TelemetryError::BadArg);
14884 }
14885 ((side as u8) + 1) << 1
14886 } else {
14887 0
14888 };
14889 if encoded != 0 && st.latest_service_seq == u32::MAX {
14890 return Err(TelemetryError::BadArg);
14891 }
14892 st.latest_service_seq = st.latest_service_seq.saturating_add(1);
14893 st.tx_policy_flags = (st.tx_policy_flags & 1) | encoded;
14894 st.latest_turn_phase = 0;
14895 st.latest_turn_credit = 0;
14896 st.clear_canonical_turn_flags();
14897 Ok(())
14898 }
14899
14900 pub fn set_side_ingress_enabled(
14904 &self,
14905 side: RouterSideId,
14906 enabled: bool,
14907 ) -> TelemetryResult<()> {
14908 #[cfg(feature = "discovery")]
14909 let now_ms = self.clock.now_ms();
14910 let mut st = self.state.lock();
14911 if !enabled
14912 && st.reliable_return_routes.values().any(|route| {
14913 route.side == side && route.local_admission_token != 0 && route.admission_committing
14914 })
14915 {
14916 return Err(TelemetryError::Io("side rx commit busy"));
14917 }
14918
14919 let side_ref = st
14920 .sides
14921 .get_mut(side)
14922 .and_then(|side| side.as_mut())
14923 .ok_or(TelemetryError::BadArg)?;
14924 side_ref.opts.ingress_enabled = enabled;
14925 if !enabled {
14926 for route in st.reliable_return_routes.values_mut() {
14927 if route.side == side && route.local_admission_token != 0 {
14928 route.admission_cancelled = true;
14929 }
14930 }
14931 }
14932 #[cfg(feature = "discovery")]
14933 if !enabled {
14934 st.cancel_managed_refresh_side(side);
14935 }
14936 #[cfg(feature = "discovery")]
14937 Self::note_discovery_topology_change_locked(&mut st, now_ms);
14938 Ok(())
14939 }
14940
14941 pub fn set_side_egress_enabled(
14945 &self,
14946 side: RouterSideId,
14947 enabled: bool,
14948 ) -> TelemetryResult<()> {
14949 #[cfg(feature = "discovery")]
14950 let now_ms = self.clock.now_ms();
14951 let mut st = self.state.lock();
14952 let changed = st
14953 .sides
14954 .get(side)
14955 .and_then(|side| side.as_ref())
14956 .ok_or(TelemetryError::BadArg)?
14957 .opts
14958 .egress_enabled
14959 != enabled;
14960 if !enabled
14961 && changed
14962 && st.reliable_return_routes.values().any(|route| {
14963 route.side == side && route.local_admission_token != 0 && route.admission_committing
14964 })
14965 {
14966 return Err(TelemetryError::Io("side rx commit busy"));
14967 }
14968
14969 let side_ref = st
14970 .sides
14971 .get_mut(side)
14972 .and_then(|side| side.as_mut())
14973 .ok_or(TelemetryError::BadArg)?;
14974 side_ref.opts.egress_enabled = enabled;
14975 if !enabled {
14976 #[cfg(feature = "discovery")]
14977 st.cancel_managed_refresh_side(side);
14978 if changed {
14981 for route in st.reliable_return_routes.values_mut() {
14982 if route.side == side && route.local_admission_token != 0 {
14983 route.admission_cancelled = true;
14984 }
14985 }
14986 }
14987 }
14988 #[cfg(feature = "latest-value-egress")]
14989 if changed && st.latest_side() == Some(side) {
14990 if let Some(next) = st.latest_service_seq.checked_add(1) {
14991 st.latest_service_seq = next;
14992 } else {
14993 st.tx_policy_flags &= 1;
14994 }
14995 st.latest_turn_phase = 0;
14996 st.latest_turn_credit = 0;
14997 st.clear_canonical_turn_flags();
14998 }
14999 if !enabled {
15000 st.retain_transmit(
15001 |queued| {
15002 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
15003 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
15004 },
15005 );
15006 }
15007 #[cfg(feature = "discovery")]
15008 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15009 Ok(())
15010 }
15011
15012 pub fn set_source_route_mode(
15018 &self,
15019 src: Option<RouterSideId>,
15020 mode: RouteSelectionMode,
15021 ) -> TelemetryResult<()> {
15022 #[cfg(feature = "discovery")]
15023 let now_ms = self.clock.now_ms();
15024 let mut st = self.state.lock();
15025 if let Some(src) = src {
15026 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15027 }
15028 st.source_route_modes.insert(src, mode);
15032 st.route_selection_cursors.remove(&src);
15033 #[cfg(feature = "discovery")]
15034 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15035 Ok(())
15036 }
15037
15038 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
15040 #[cfg(feature = "discovery")]
15041 let now_ms = self.clock.now_ms();
15042 let mut st = self.state.lock();
15043 if let Some(src) = src {
15044 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15045 }
15046 st.source_route_modes.remove(&src);
15047 st.route_selection_cursors.remove(&src);
15048 #[cfg(feature = "discovery")]
15049 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15050 Ok(())
15051 }
15052
15053 pub fn set_route_weight(
15058 &self,
15059 src: Option<RouterSideId>,
15060 dst: RouterSideId,
15061 weight: u32,
15062 ) -> TelemetryResult<()> {
15063 #[cfg(feature = "discovery")]
15064 let now_ms = self.clock.now_ms();
15065 let mut st = self.state.lock();
15066 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15067 if let Some(src) = src {
15068 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15069 }
15070 st.route_weights.insert((src, dst), weight);
15071 st.route_selection_cursors.remove(&src);
15072 #[cfg(feature = "discovery")]
15073 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15074 Ok(())
15075 }
15076
15077 pub fn clear_route_weight(
15079 &self,
15080 src: Option<RouterSideId>,
15081 dst: RouterSideId,
15082 ) -> TelemetryResult<()> {
15083 #[cfg(feature = "discovery")]
15084 let now_ms = self.clock.now_ms();
15085 let mut st = self.state.lock();
15086 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15087 if let Some(src) = src {
15088 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15089 }
15090 st.route_weights.remove(&(src, dst));
15091 st.route_selection_cursors.remove(&src);
15092 #[cfg(feature = "discovery")]
15093 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15094 Ok(())
15095 }
15096
15097 pub fn set_route_priority(
15102 &self,
15103 src: Option<RouterSideId>,
15104 dst: RouterSideId,
15105 priority: u32,
15106 ) -> TelemetryResult<()> {
15107 #[cfg(feature = "discovery")]
15108 let now_ms = self.clock.now_ms();
15109 let mut st = self.state.lock();
15110 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15111 if let Some(src) = src {
15112 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15113 }
15114 st.route_priorities.insert((src, dst), priority);
15115 #[cfg(feature = "discovery")]
15116 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15117 Ok(())
15118 }
15119
15120 pub fn clear_route_priority(
15122 &self,
15123 src: Option<RouterSideId>,
15124 dst: RouterSideId,
15125 ) -> TelemetryResult<()> {
15126 #[cfg(feature = "discovery")]
15127 let now_ms = self.clock.now_ms();
15128 let mut st = self.state.lock();
15129 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15130 if let Some(src) = src {
15131 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15132 }
15133 st.route_priorities.remove(&(src, dst));
15134 #[cfg(feature = "discovery")]
15135 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15136 Ok(())
15137 }
15138
15139 pub fn set_route(
15144 &self,
15145 src: Option<RouterSideId>,
15146 dst: RouterSideId,
15147 enabled: bool,
15148 ) -> TelemetryResult<()> {
15149 #[cfg(feature = "discovery")]
15150 let now_ms = self.clock.now_ms();
15151 let mut st = self.state.lock();
15152 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15153 if let Some(src) = src {
15154 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15155 }
15156 st.route_overrides.insert((src, dst), enabled);
15157 #[cfg(feature = "discovery")]
15158 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15159 Ok(())
15160 }
15161
15162 pub fn set_typed_route(
15167 &self,
15168 src: Option<RouterSideId>,
15169 ty: DataType,
15170 dst: RouterSideId,
15171 enabled: bool,
15172 ) -> TelemetryResult<()> {
15173 #[cfg(feature = "discovery")]
15174 let now_ms = self.clock.now_ms();
15175 let mut st = self.state.lock();
15176 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15177 if let Some(src) = src {
15178 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15179 }
15180 st.typed_route_overrides
15181 .insert((src, ty.as_u32(), dst), enabled);
15182 #[cfg(feature = "discovery")]
15183 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15184 Ok(())
15185 }
15186
15187 pub fn clear_typed_route(
15189 &self,
15190 src: Option<RouterSideId>,
15191 ty: DataType,
15192 dst: RouterSideId,
15193 ) -> TelemetryResult<()> {
15194 #[cfg(feature = "discovery")]
15195 let now_ms = self.clock.now_ms();
15196 let mut st = self.state.lock();
15197 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15198 if let Some(src) = src {
15199 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15200 }
15201 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
15202 #[cfg(feature = "discovery")]
15203 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15204 Ok(())
15205 }
15206
15207 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
15209 #[cfg(feature = "discovery")]
15210 let now_ms = self.clock.now_ms();
15211 let mut st = self.state.lock();
15212 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15213 if let Some(src) = src {
15214 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15215 }
15216 st.route_overrides.remove(&(src, dst));
15217 #[cfg(feature = "discovery")]
15218 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15219 Ok(())
15220 }
15221
15222 #[cfg(feature = "discovery")]
15224 pub fn announce_discovery(&self) -> TelemetryResult<()> {
15225 self.queue_discovery_announce(true, false)
15226 }
15227
15228 #[cfg(feature = "discovery")]
15230 pub fn announce_leave(&self) -> TelemetryResult<()> {
15231 let sender = self.sender_arc();
15232 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
15233 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15234 }
15235
15236 #[cfg(feature = "discovery")]
15238 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
15239 let mut emitted = self.poll_discovery_announce()?;
15241 let side_count = self.state.lock().sides.len();
15242 for side in 0..side_count {
15243 if self.state.lock().sides[side].is_some() {
15244 emitted |= self.poll_schema_response_on_side(side)?;
15245 }
15246 }
15247 Ok(emitted)
15248 }
15249
15250 #[cfg(feature = "discovery")]
15251 pub fn request_topology(&self) -> TelemetryResult<()> {
15252 let sender = self.sender_arc();
15253 let pkt =
15254 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
15255 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15256 }
15257
15258 #[cfg(feature = "discovery")]
15259 pub fn request_schema(&self) -> TelemetryResult<()> {
15260 let sender = self.sender_arc();
15261 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
15262 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15263 }
15264
15265 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
15272 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
15273 }
15274
15275 pub fn enable_network_variable(
15280 &self,
15281 ty: DataType,
15282 permissions: NetworkVariablePermissions,
15283 ) -> TelemetryResult<()> {
15284 if is_internal_control_type(ty) {
15285 return Err(TelemetryError::InvalidType);
15286 }
15287 let mut st = self.state.lock();
15288 #[cfg(feature = "discovery")]
15289 let newly_advertised = st
15290 .local_services
15291 .as_mut()
15292 .expect("local services enabled")
15293 .managed_variable_types
15294 .insert(ty.as_u32());
15295 #[cfg(not(feature = "discovery"))]
15296 st.local_services
15297 .as_mut()
15298 .expect("local services enabled")
15299 .managed_variable_types
15300 .insert(ty.as_u32());
15301 st.local_services
15302 .as_mut()
15303 .expect("local services enabled")
15304 .managed_variable_permissions
15305 .insert(ty.as_u32(), permissions);
15306 #[cfg(feature = "discovery")]
15307 if newly_advertised {
15308 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15309 }
15310 Ok(())
15311 }
15312
15313 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
15319 where
15320 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
15321 {
15322 if is_internal_control_type(ty) {
15323 return Err(TelemetryError::InvalidType);
15324 }
15325 let mut st = self.state.lock();
15326 #[cfg(feature = "discovery")]
15327 let newly_advertised = st
15328 .local_services
15329 .as_mut()
15330 .expect("local services enabled")
15331 .managed_variable_types
15332 .insert(ty.as_u32());
15333 #[cfg(not(feature = "discovery"))]
15334 st.local_services
15335 .as_mut()
15336 .expect("local services enabled")
15337 .managed_variable_types
15338 .insert(ty.as_u32());
15339 st.local_services
15340 .as_mut()
15341 .expect("local services enabled")
15342 .network_variable_update_handlers
15343 .entry(ty.as_u32())
15344 .or_default()
15345 .push(NetworkVariableUpdateHandler {
15346 handler: Arc::new(f),
15347 });
15348 #[cfg(feature = "discovery")]
15349 if newly_advertised {
15350 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15351 }
15352 Ok(())
15353 }
15354
15355 pub fn disable_managed_variable(&self, ty: DataType) {
15356 let mut st = self.state.lock();
15357 #[cfg(feature = "discovery")]
15358 let was_advertised = st
15359 .local_services
15360 .as_mut()
15361 .expect("local services enabled")
15362 .managed_variable_types
15363 .remove(&ty.as_u32());
15364 #[cfg(not(feature = "discovery"))]
15365 st.local_services
15366 .as_mut()
15367 .expect("local services enabled")
15368 .managed_variable_types
15369 .remove(&ty.as_u32());
15370 st.local_services
15371 .as_mut()
15372 .expect("local services enabled")
15373 .managed_variable_permissions
15374 .remove(&ty.as_u32());
15375 st.local_services
15376 .as_mut()
15377 .expect("local services enabled")
15378 .managed_variable_latest
15379 .remove(&ty.as_u32());
15380 st.local_services
15381 .as_mut()
15382 .expect("local services enabled")
15383 .network_variable_update_handlers
15384 .remove(&ty.as_u32());
15385 #[cfg(feature = "discovery")]
15386 if was_advertised {
15387 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15388 }
15389 }
15390
15391 pub fn invalidate_network_variable_cache(&self, ty: DataType) -> bool {
15400 self.state
15401 .lock()
15402 .local_services
15403 .as_mut()
15404 .is_some_and(|local| local.managed_variable_latest.remove(&ty.as_u32()).is_some())
15405 }
15406
15407 pub fn clear_network_variable_caches(&self) -> usize {
15415 let mut state = self.state.lock();
15416 let Some(local) = state.local_services.as_mut() else {
15417 return 0;
15418 };
15419 let count = local.managed_variable_latest.len();
15420 local.managed_variable_latest.clear();
15421 count
15422 }
15423
15424 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
15425 if is_internal_control_type(pkt.data_type()) {
15426 return Err(TelemetryError::InvalidType);
15427 }
15428 pkt.validate()?;
15429 {
15430 let mut st = self.state.lock();
15431 #[cfg(feature = "discovery")]
15432 let newly_advertised = st
15433 .local_services
15434 .as_mut()
15435 .expect("local services enabled")
15436 .managed_variable_types
15437 .insert(pkt.data_type().as_u32());
15438 #[cfg(not(feature = "discovery"))]
15439 st.local_services
15440 .as_mut()
15441 .expect("local services enabled")
15442 .managed_variable_types
15443 .insert(pkt.data_type().as_u32());
15444 #[cfg(feature = "discovery")]
15445 if newly_advertised {
15446 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15447 }
15448 }
15449 self.cache_managed_variable_packet(&pkt, false, None)
15450 }
15451
15452 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
15453 self.managed_variable_latest(ty)
15454 }
15455
15456 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
15462 if is_internal_control_type(pkt.data_type()) {
15463 return Err(TelemetryError::InvalidType);
15464 }
15465 pkt.validate()?;
15466 let ty = pkt.data_type();
15467 {
15468 let mut st = self.state.lock();
15469 st.local_services
15470 .as_mut()
15471 .expect("local services enabled")
15472 .managed_variable_types
15473 .insert(ty.as_u32());
15474 let perms = Self::managed_variable_permissions_locked(&st, ty);
15475 if !perms.write {
15476 return Err(TelemetryError::PermissionDenied);
15477 }
15478 }
15479 self.cache_managed_variable_packet(&pkt, false, None)?;
15480 #[cfg(feature = "discovery")]
15481 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15482 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
15483 if self.side_tx_active() {
15484 return self.tx_queue_item_with_priority(
15485 item,
15486 false,
15487 crate::transport_priority(DataType::ManagedVariableValue),
15488 );
15489 }
15490 self.tx_item(item)
15491 }
15492
15493 #[cfg(feature = "discovery")]
15504 pub fn get_network_variable(
15505 &self,
15506 ty: DataType,
15507 stale_after_ms: Option<u64>,
15508 ) -> TelemetryResult<Option<Packet>> {
15509 if is_internal_control_type(ty) {
15510 return Err(TelemetryError::InvalidType);
15511 }
15512 {
15513 let mut st = self.state.lock();
15514 st.local_services
15515 .as_mut()
15516 .expect("local services enabled")
15517 .managed_variable_types
15518 .insert(ty.as_u32());
15519 let perms = Self::managed_variable_permissions_locked(&st, ty);
15520 if !perms.read {
15521 return Err(TelemetryError::PermissionDenied);
15522 }
15523 }
15524 let cached = self.managed_variable_latest_with_age(ty);
15525 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
15526 (None, _) => true,
15527 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
15528 (Some(_), None) => false,
15529 };
15530 if needs_refresh {
15531 self.request_managed_variable(ty)?;
15532 }
15533 Ok(cached.map(|(pkt, _age_ms)| pkt))
15534 }
15535
15536 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
15538 if is_internal_control_type(ty) {
15539 return Err(TelemetryError::InvalidType);
15540 }
15541 if !self.can_read_managed_variable(ty) {
15542 return Err(TelemetryError::PermissionDenied);
15543 }
15544 Ok(self.managed_variable_latest(ty))
15545 }
15546
15547 #[cfg(feature = "discovery")]
15548 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
15549 if is_internal_control_type(ty) {
15550 return Err(TelemetryError::InvalidType);
15551 }
15552 if !self.can_read_managed_variable(ty) {
15553 return Err(TelemetryError::PermissionDenied);
15554 }
15555 let sender = self.sender_arc();
15556 let pkt =
15557 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
15558 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15559 }
15560
15561 #[cfg(feature = "discovery")]
15562 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
15563 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
15564 self.request_managed_variable(ty)
15565 }
15566
15567 #[cfg(feature = "discovery")]
15569 pub fn export_topology(&self) -> TopologySnapshot {
15570 let now_ms = self.clock.now_ms();
15571 let mut st = self.state.lock();
15572 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
15573 Self::note_discovery_topology_change_locked(&mut st, now_ms);
15574 }
15575 let routes = st
15576 .discovery_routes
15577 .iter()
15578 .filter_map(|(&side_id, route)| {
15579 let side = st.sides.get(side_id)?.as_ref()?;
15580 let announcers = route
15581 .announcers
15582 .iter()
15583 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
15584 sender_id: sender_id.clone(),
15585 reachable_endpoints: sender_state
15586 .reachable
15587 .iter()
15588 .copied()
15589 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
15590 .collect(),
15591 reachable_network_variables: sender_state
15592 .reachable_network_variables
15593 .clone(),
15594 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
15595 routers: sender_state.topology_boards.as_ref().clone(),
15596 last_seen_ms: sender_state.last_seen_ms,
15597 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
15598 })
15599 .collect();
15600 Some(TopologySideRoute {
15601 side_id,
15602 side_name: side.name.to_string(),
15603 reachable_endpoints: route
15604 .reachable
15605 .iter()
15606 .copied()
15607 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
15608 .collect(),
15609 reachable_network_variables: route.reachable_network_variables.clone(),
15610 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
15611 announcers,
15612 last_seen_ms: route.last_seen_ms,
15613 age_ms: now_ms.saturating_sub(route.last_seen_ms),
15614 })
15615 })
15616 .collect();
15617 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
15618 let advertised_endpoints =
15619 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
15620 let advertised_timesync_sources =
15621 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
15622 let links = discovery::topology_links_from_boards(&routers);
15623 TopologySnapshot {
15624 advertised_endpoints,
15625 advertised_timesync_sources,
15626 routers,
15627 links,
15628 routes,
15629 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
15630 next_announce_ms: st.discovery_cadence.next_announce_ms,
15631 }
15632 }
15633
15634 #[cfg(feature = "discovery")]
15635 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
15636 let now_ms = self.clock.now_ms();
15637 let st = self.state.lock();
15638 let mut side_ids = Vec::new();
15639 let mut side_names = Vec::new();
15640 let mut last_seen_ms = None::<u64>;
15641 let mut reachable_endpoints = Vec::new();
15642 let mut reachable_timesync_sources = Vec::new();
15643 let mut packets_sent = 0u64;
15644 let mut packets_received = 0u64;
15645 let mut bytes_sent = 0u64;
15646 let mut bytes_received = 0u64;
15647
15648 for (side_id, route) in &st.discovery_routes {
15649 let Some(sender_state) = route.announcers.get(sender_id) else {
15650 continue;
15651 };
15652 side_ids.push(*side_id);
15653 if let Some(side_name) = st
15654 .sides
15655 .get(*side_id)
15656 .and_then(|side| side.as_ref())
15657 .map(|side| side.name.clone())
15658 {
15659 side_names.push(side_name.to_string());
15660 }
15661 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
15662 reachable_endpoints.extend(sender_state.reachable.iter().copied());
15663 reachable_timesync_sources
15664 .extend(sender_state.reachable_timesync_sources.iter().cloned());
15665 if let Some(stats) = st.side_runtime_stats.get(side_id) {
15666 packets_sent = packets_sent.saturating_add(stats.tx_packets);
15667 packets_received = packets_received.saturating_add(stats.rx_packets);
15668 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
15669 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
15670 }
15671 }
15672
15673 if side_ids.is_empty() {
15674 return None;
15675 }
15676 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
15677 crate::small_sort::sort(&mut reachable_endpoints);
15678 reachable_endpoints.dedup();
15679 reachable_timesync_sources.sort_unstable();
15680 reachable_timesync_sources.dedup();
15681 crate::small_sort::sort(&mut side_ids);
15682 side_ids.dedup();
15683 side_names.sort_unstable();
15684 side_names.dedup();
15685 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
15686 Some(ClientStatsSnapshot {
15687 sender_id: sender_id.to_string(),
15688 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
15689 side_ids,
15690 side_names,
15691 last_seen_ms,
15692 age_ms,
15693 reachable_endpoints,
15694 reachable_timesync_sources,
15695 packets_sent,
15696 packets_received,
15697 bytes_sent,
15698 bytes_received,
15699 })
15700 }
15701
15702 pub fn copy_reliable_pending_stats(
15706 &self,
15707 first: usize,
15708 rows: &mut [crate::diagnostics::ReliablePendingRow],
15709 ) -> usize {
15710 rows.fill(crate::diagnostics::ReliablePendingRow::default());
15711 let st = self.state.lock();
15712 for (row, (&id, sent)) in rows
15713 .iter_mut()
15714 .zip(st.end_to_end_reliable_tx.iter().skip(first))
15715 {
15716 row.packet_id = id;
15717 row.last_send_ms = sent.last_send_ms;
15718 row.data_type = Self::item_data_type(&sent.data).map_or(0, DataType::as_u32);
15719 row.pending_count = sent.pending_destinations.len().min(u32::MAX as usize) as u32;
15720 row.retries = sent.retries;
15721 row.flags = u32::from(sent.queued)
15722 | (u32::from(sent.waiting_for_predecessor) << 1)
15723 | (u32::from(
15724 st.reliable_return_routes
15725 .get(&id)
15726 .is_some_and(|route| route.upstream_contract.is_some()),
15727 ) << 2)
15728 | (u32::from(sent.tracked_destinations) << 3);
15729 for (out, owner) in row
15730 .pending_owners
15731 .iter_mut()
15732 .zip(sent.pending_destinations.keys())
15733 {
15734 *out = *owner;
15735 }
15736 }
15737 st.end_to_end_reliable_tx.len()
15738 }
15739
15740 pub fn export_reliable_pending_stats(&self) -> Vec<crate::diagnostics::ReliablePendingStats> {
15746 let st = self.state.lock();
15747 st.end_to_end_reliable_tx
15748 .iter()
15749 .filter_map(|(&packet_id, sent)| {
15750 Some(crate::diagnostics::ReliablePendingStats {
15751 packet_id,
15752 data_type: {
15753 #[cfg(feature = "std")]
15754 {
15755 if sent.pending_type != 0 {
15756 DataType(sent.pending_type)
15757 } else {
15758 Self::item_data_type(&sent.data).ok()?
15759 }
15760 }
15761 #[cfg(not(feature = "std"))]
15762 {
15763 Self::item_data_type(&sent.data).ok()?
15764 }
15765 },
15766 pending_destinations: sent.pending_destinations.keys().copied().collect(),
15767 retries: sent.retries,
15768 queued: sent.queued,
15769 waiting_for_predecessor: sent.waiting_for_predecessor,
15770 last_send_ms: sent.last_send_ms,
15771 })
15772 })
15773 .collect()
15774 }
15775
15776 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
15777 let now_ms = self.clock.now_ms();
15778 let isr_rx = self
15779 .isr_rx_queue
15780 .as_ref()
15781 .and_then(|queue| queue.snapshot())
15782 .unwrap_or((0, 0));
15783 let st = self.state.lock();
15784
15785 let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
15786 (0, 0)
15787 } else {
15788 st.replay_counters()
15789 };
15790 let mut sides = Vec::new();
15791 for (side_id, side) in st.sides.iter().enumerate() {
15792 let Some(side) = side.as_ref() else { continue };
15793 let empty_stats = SideRuntimeStatsInner::default();
15794 let stats = st
15795 .side_runtime_stats
15796 .get(&side_id)
15797 .map(Box::as_ref)
15798 .unwrap_or(&empty_stats);
15799 let adaptive = st
15800 .adaptive_route_stats
15801 .get(&side_id)
15802 .cloned()
15803 .unwrap_or_default()
15804 .snapshot(now_ms, true);
15805 let (tx_template_count, rx_template_count) = st
15806 .side_transport
15807 .get(&side_id)
15808 .map(|state| (state.tx_template_count(), state.rx_template_count()))
15809 .unwrap_or((0, 0));
15810 let mut data_types: Vec<RuntimeTypeStats> = stats
15811 .data_types
15812 .iter()
15813 .map(|(&ty, item)| {
15814 let item = item.counters();
15815 RuntimeTypeStats {
15816 data_type: DataType(ty),
15817 tx_packets: item.tx_packets,
15818 tx_bytes: item.tx_bytes,
15819 rx_packets: item.rx_packets,
15820 rx_bytes: item.rx_bytes,
15821 relayed_tx_packets: item.relayed_tx_packets,
15822 relayed_tx_bytes: item.relayed_tx_bytes,
15823 relayed_rx_packets: item.relayed_rx_packets,
15824 relayed_rx_bytes: item.relayed_rx_bytes,
15825 tx_retries: item.tx_retries,
15826 handler_failures: item.handler_failures,
15827 }
15828 })
15829 .collect();
15830 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
15831 sides.push(RuntimeSideStats {
15832 side_id,
15833 side_name: side.name.to_string(),
15834 reliable_enabled: side.opts.reliable_enabled,
15835 link_local_enabled: side.opts.link_local_enabled,
15836 header_template_enabled: side.opts.header_template_enabled,
15837 max_frame_bytes: side.opts.max_frame_bytes,
15838 compact_header_target_bytes: side.opts.compact_header_target_bytes,
15839 side_transport_profile: side.opts.effective_transport_profile().as_str(),
15840 ingress_enabled: side.opts.ingress_enabled,
15841 egress_enabled: side.opts.egress_enabled,
15842 tx_packets: stats.tx_packets,
15843 tx_bytes: stats.tx_bytes,
15844 rx_packets: stats.rx_packets,
15845 rx_bytes: stats.rx_bytes,
15846 relayed_tx_packets: stats.relayed_tx_packets,
15847 relayed_tx_bytes: stats.relayed_tx_bytes,
15848 relayed_rx_packets: stats.relayed_rx_packets,
15849 relayed_rx_bytes: stats.relayed_rx_bytes,
15850 local_delivery_packets: stats.local_delivery_packets,
15851 tx_retries: stats.tx_retries,
15852 tx_handler_failures: stats.tx_handler_failures,
15853 local_handler_failures: stats.local_handler_failures,
15854 total_handler_retries: stats.total_handler_retries,
15855 side_transport_full_frames: stats.side_transport_full_frames,
15856 side_transport_compact_frames: stats.side_transport_compact_frames,
15857 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
15858 side_transport_compact_omitted_timestamp_frames: stats
15859 .side_transport_compact_omitted_timestamp_frames,
15860 side_transport_chunk_frames: stats.side_transport_chunk_frames,
15861 side_transport_chunk_reclaims: st
15862 .side_transport
15863 .get(&side_id)
15864 .map_or(0, |state| u64::from(state.rx_chunk_reclaims)),
15865 side_transport_raw_bytes: stats.side_transport_raw_bytes,
15866 side_transport_wire_bytes: stats.side_transport_wire_bytes,
15867 side_transport_bytes_saved: stats.side_transport_bytes_saved,
15868 side_transport_min_compact_overhead_bytes: stats
15869 .side_transport_min_compact_overhead_bytes,
15870 side_transport_max_compact_overhead_bytes: stats
15871 .side_transport_max_compact_overhead_bytes,
15872 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
15873 side_transport_template_evictions: stats.side_transport_template_evictions,
15874 side_transport_tx_template_count: tx_template_count,
15875 side_transport_rx_template_count: rx_template_count,
15876 max_side_transport_templates: side.opts.max_side_transport_templates,
15877 adaptive,
15878 data_types,
15879 });
15880 }
15881
15882 let mut route_modes: Vec<RouteModeStats> = st
15883 .route_selection_cursors
15884 .iter()
15885 .map(|(src, cursor)| RouteModeStats {
15886 src_side_id: *src,
15887 selection_mode: st.source_route_modes.get(src).copied(),
15888 cursor: *cursor,
15889 })
15890 .collect();
15891 for src in st.source_route_modes.keys() {
15892 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
15893 route_modes.push(RouteModeStats {
15894 src_side_id: *src,
15895 selection_mode: st.source_route_modes.get(src).copied(),
15896 cursor: 0,
15897 });
15898 }
15899 }
15900 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
15901
15902 let mut route_overrides: Vec<RouteOverrideStats> = st
15903 .route_overrides
15904 .iter()
15905 .map(|((src, dst), enabled)| RouteOverrideStats {
15906 src_side_id: *src,
15907 dst_side_id: *dst,
15908 enabled: *enabled,
15909 })
15910 .collect();
15911 route_overrides.sort_unstable_by_key(|item| {
15912 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15913 });
15914
15915 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
15916 .typed_route_overrides
15917 .iter()
15918 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
15919 src_side_id: *src,
15920 data_type: DataType(*ty),
15921 dst_side_id: *dst,
15922 enabled: *enabled,
15923 })
15924 .collect();
15925 typed_route_overrides.sort_unstable_by_key(|item| {
15926 (
15927 item.src_side_id.unwrap_or(usize::MAX),
15928 item.data_type.as_u32(),
15929 item.dst_side_id,
15930 )
15931 });
15932
15933 let mut route_weights: Vec<RouteWeightStats> = st
15934 .route_weights
15935 .iter()
15936 .map(|((src, dst), weight)| RouteWeightStats {
15937 src_side_id: *src,
15938 dst_side_id: *dst,
15939 weight: *weight,
15940 })
15941 .collect();
15942 route_weights.sort_unstable_by_key(|item| {
15943 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15944 });
15945
15946 let mut route_priorities: Vec<RoutePriorityStats> = st
15947 .route_priorities
15948 .iter()
15949 .map(|((src, dst), priority)| RoutePriorityStats {
15950 src_side_id: *src,
15951 dst_side_id: *dst,
15952 priority: *priority,
15953 })
15954 .collect();
15955 route_priorities.sort_unstable_by_key(|item| {
15956 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15957 });
15958
15959 #[cfg(feature = "discovery")]
15960 let discovery = DiscoveryRuntimeStats {
15961 route_count: st.discovery_routes.len(),
15962 announcer_count: st
15963 .discovery_routes
15964 .values()
15965 .map(|route| route.announcers.len())
15966 .sum(),
15967 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
15968 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
15969 };
15970 #[cfg(not(feature = "discovery"))]
15971 let discovery = DiscoveryRuntimeStats {
15972 route_count: 0,
15973 announcer_count: 0,
15974 current_announce_interval_ms: None,
15975 next_announce_ms: None,
15976 };
15977
15978 RuntimeStatsSnapshot {
15979 sides,
15980 route_modes,
15981 route_overrides,
15982 typed_route_overrides,
15983 route_weights,
15984 route_priorities,
15985 queues: QueueRuntimeStats {
15986 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
15987 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
15988 tx_len: st.transmit_queue.len().saturating_sub(replay_len),
15989 tx_bytes: st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
15990 replay_len,
15991 replay_bytes,
15992 recent_rx_len: st.recent_rx.len(),
15993 recent_rx_bytes: st.recent_rx.bytes_used(),
15994 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
15995 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
15996 shared_queue_bytes_used: st.shared_queue_bytes_used(),
15997 },
15998 reliable: ReliableRuntimeStats {
15999 reliable_return_route_count: st.reliable_return_routes.len(),
16000 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
16001 end_to_end_pending_destination_count: st
16002 .end_to_end_reliable_tx
16003 .values()
16004 .map(|sent| sent.pending_destinations.len())
16005 .sum(),
16006 end_to_end_acked_cache_count: st
16007 .forwarded_receipts
16008 .as_ref()
16009 .map_or(0, |cache| cache.destinations.len()),
16010 },
16011 discovery,
16012 total_handler_failures: st.total_handler_failures,
16013 total_handler_retries: st.total_handler_retries,
16014 }
16015 }
16016
16017 #[cfg(feature = "std")]
16030 pub fn has_pending_reliable_tx_type(&self, ty: DataType) -> bool {
16031 let st = self.state.lock();
16032 let total = self
16033 .active_queued_tx
16034 .load(core::sync::atomic::Ordering::Acquire) as u64;
16035 let known = st
16036 .active_queued_types
16037 .iter()
16038 .try_fold(0u64, |sum, &(kind, count)| {
16039 if kind == 0 && count != 0 {
16040 return None;
16041 }
16042 sum.checked_add(u64::from(count))
16043 });
16044 if known != Some(total)
16047 || st
16048 .active_queued_types
16049 .iter()
16050 .any(|&(kind, count)| count != 0 && kind == ty.as_u32())
16051 {
16052 return true;
16053 }
16054 let matches_data = |data: &RouterItem| match data {
16055 RouterItem::Packet(packet) => packet.data_type() == ty,
16056 RouterItem::Packed(_) => true,
16057 };
16058 let matches_owner = |owner: &EndToEndReliableSent| {
16059 owner.pending_type == 0 || owner.pending_type == ty.as_u32()
16060 };
16061 st.transmit_queue.iter().any(|queued| {
16062 if queued.accepted_type != 0 {
16063 return queued.accepted_type == ty.as_u32();
16064 }
16065 match &queued.item {
16066 RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
16067 matches_data(data)
16068 }
16069 RouterTxItem::ReliableReplay { ty: found, .. } => *found == ty,
16070 RouterTxItem::EndToEndReplay { packet_id } => st
16071 .end_to_end_reliable_tx
16072 .get(packet_id)
16073 .is_some_and(|owner| matches_owner(owner)),
16074 }
16075 }) || st
16076 .end_to_end_reliable_tx
16077 .values()
16078 .any(|owner| matches_owner(owner))
16079 || st
16080 .reliable_tx
16081 .iter()
16082 .any(|((_, found), owner)| *found == ty.as_u32() && !owner.sent.is_empty())
16083 }
16084
16085 #[cfg(feature = "std")]
16088 fn track_queued_tx<'a>(
16089 &'a self,
16090 state: &mut RouterInner,
16091 work: Option<TxWork>,
16092 ) -> Option<(TxWork, QueuedTxActivityGuard<'a>)> {
16093 work.map(|work| {
16094 let total = self
16095 .active_queued_tx
16096 .load(core::sync::atomic::Ordering::Acquire);
16097 let next = total.checked_add(1).expect("active queued TX overflow");
16100 let kind = if work.queued.accepted_type != 0 {
16101 Some(work.queued.accepted_type)
16102 } else {
16103 match &work.queued.item {
16104 RouterTxItem::Broadcast(RouterItem::Packet(packet))
16105 | RouterTxItem::ToSide {
16106 data: RouterItem::Packet(packet),
16107 ..
16108 } => Some(packet.data_type().as_u32()),
16109 RouterTxItem::ReliableReplay { ty, .. } => Some(ty.as_u32()),
16110 RouterTxItem::EndToEndReplay { packet_id } => state
16111 .end_to_end_reliable_tx
16112 .get(packet_id)
16113 .map(|owner| owner.pending_type),
16114 _ => None,
16115 }
16116 }
16117 .filter(|kind| *kind != 0);
16118 let key = kind.and_then(|kind| {
16119 let index = state
16120 .active_queued_types
16121 .iter()
16122 .position(|slot| slot.0 == kind)
16123 .or_else(|| {
16124 state
16125 .active_queued_types
16126 .iter()
16127 .position(|slot| slot.1 == 0)
16128 })?;
16129 let slot = &mut state.active_queued_types[index];
16130 let count = slot.1.checked_add(1)?;
16131 *slot = (kind, count);
16132 Some(ActiveQueuedTypeKey {
16133 slot: core::num::NonZeroU8::new((index + 1) as u8).expect("inline type slot"),
16134 kind,
16135 })
16136 });
16137 self.active_queued_tx
16138 .store(next, core::sync::atomic::Ordering::Release);
16139 (work, QueuedTxActivityGuard { router: self, key })
16140 })
16141 }
16142
16143 pub fn has_queued_work(&self) -> bool {
16150 if self
16151 .isr_rx_queue
16152 .as_ref()
16153 .is_some_and(|queue| queue.try_lock().map_or(true, |queue| !queue.is_empty()))
16154 {
16155 return true;
16156 }
16157 let state = self.state.lock();
16158 !state.received_queue.is_empty() || !state.transmit_queue.is_empty()
16159 }
16160
16161 pub fn queue_counters(&self) -> [u32; 16] {
16167 let now = self.clock.now_ms();
16168 let st = self.state.lock();
16169 let ty_of = |item: &RouterTxItem| match item {
16170 RouterTxItem::Broadcast(RouterItem::Packet(packet))
16171 | RouterTxItem::ToSide {
16172 data: RouterItem::Packet(packet),
16173 ..
16174 } => Some(packet.data_type()),
16175 RouterTxItem::Broadcast(RouterItem::Packed(_))
16176 | RouterTxItem::ToSide {
16177 data: RouterItem::Packed(_),
16178 ..
16179 }
16180 | RouterTxItem::ReliableReplay { .. } => None,
16181 RouterTxItem::EndToEndReplay { packet_id } => st
16182 .end_to_end_reliable_tx
16183 .get(packet_id)
16184 .and_then(|pending| match &pending.data {
16185 RouterItem::Packet(packet) => Some(packet.data_type()),
16186 RouterItem::Packed(_) => None,
16187 }),
16188 };
16189 let sum_cost = |total: usize, cost: usize| total.saturating_add(cost);
16190 let front = st.transmit_queue.iter().next();
16191 let ack_count = st
16192 .transmit_queue
16193 .iter()
16194 .filter(|q| {
16195 matches!(
16196 ty_of(&q.item),
16197 Some(
16198 DataType::ReliableAck
16199 | DataType::ReliablePartialAck
16200 | DataType::ReliablePacketRequest
16201 )
16202 )
16203 })
16204 .count();
16205 let schema_count = st
16206 .transmit_queue
16207 .iter()
16208 .filter(|q| ty_of(&q.item) == Some(DataType::DiscoverySchema))
16209 .count();
16210 [
16211 1,
16212 st.received_queue.len() as u32,
16213 st.transmit_queue.len() as u32,
16214 st.received_queue
16215 .iter()
16216 .map(ByteCost::byte_cost)
16217 .fold(0, sum_cost) as u32,
16218 st.transmit_queue
16219 .iter()
16220 .map(ByteCost::byte_cost)
16221 .fold(0, sum_cost) as u32,
16222 front
16223 .and_then(|q| ty_of(&q.item))
16224 .map_or(0, DataType::as_u32),
16225 front.map_or(0, |q| q.priority.into()),
16226 front.map_or(0, |q| {
16227 q.retry_after_ms.saturating_sub(now).min(u32::MAX.into()) as u32
16228 }),
16229 ack_count as u32,
16230 schema_count as u32,
16231 st.transmit_queue
16232 .iter()
16233 .map(|q| u32::from(q.priority))
16234 .min()
16235 .unwrap_or(0),
16236 st.end_to_end_reliable_tx.len() as u32,
16237 st.end_to_end_reliable_tx
16238 .values()
16239 .map(|entry| match &entry.data {
16240 RouterItem::Packet(packet) => packet.byte_cost(),
16241 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
16242 })
16243 .fold(0, sum_cost) as u32,
16244 front.map_or(0, |q| match q.item {
16245 RouterTxItem::Broadcast(_) => 1,
16246 RouterTxItem::ToSide { .. } => 2,
16247 RouterTxItem::ReliableReplay { .. } => 3,
16248 RouterTxItem::EndToEndReplay { .. } => 4,
16249 }),
16250 now as u32,
16251 (now >> 32) as u32,
16252 ]
16253 }
16254
16255 pub fn memory_counters(&self) -> [u32; 16] {
16256 let st = self.state.lock();
16257 #[cfg(feature = "discovery")]
16258 let discovery = st.discovery_bytes_used();
16259 #[cfg(not(feature = "discovery"))]
16260 let discovery = 0;
16261 let templates = st
16262 .side_transport
16263 .values()
16264 .map(|side| side.tx_templates.len() + side.rx_templates_by_id.len())
16265 .sum::<usize>();
16266 let template_bytes = st
16267 .side_transport
16268 .values()
16269 .flat_map(|side| {
16270 side.tx_templates
16271 .values()
16272 .chain(side.rx_templates_by_id.values())
16273 })
16274 .map(|t| size_of::<SideHeaderTemplate>() + t.prefix().len() + t.between().len())
16275 .sum::<usize>();
16276 let chunks = st
16277 .side_transport
16278 .values()
16279 .flat_map(|side| side.rx_chunks.values())
16280 .flat_map(|assembly| assembly.received.iter().flatten())
16281 .map(|bytes| bytes.len())
16282 .sum::<usize>();
16283 let type_stats = st
16284 .side_runtime_stats
16285 .values()
16286 .map(|side| side.data_types.len())
16287 .sum::<usize>();
16288 let managed = st.local_cache_bytes();
16289 let pending = st
16290 .end_to_end_reliable_tx
16291 .values()
16292 .map(|entry| match &entry.data {
16293 RouterItem::Packet(packet) => packet.byte_cost(),
16294 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
16295 })
16296 .sum::<usize>();
16297 [
16298 1,
16299 st.memory.max_queue_budget,
16300 st.shared_queue_bytes_used(),
16301 st.received_queue.bytes_used(),
16302 st.transmit_queue.bytes_used(),
16303 st.received_queue.capacity() * size_of::<RouterRxItem>()
16304 + st.transmit_queue.capacity() * size_of::<TxQueued>()
16305 + st.recent_rx.capacity() * size_of::<u64>(),
16306 st.reliable_rx_buffered_bytes(),
16307 discovery,
16308 crate::config::schema_bytes_used(),
16309 managed,
16310 pending,
16311 templates,
16312 template_bytes,
16313 chunks,
16314 type_stats,
16315 st.side_runtime_stats
16316 .values()
16317 .flat_map(|side| side.data_types.values())
16318 .map(|stats| size_of::<TypeRuntimeStatsInner>() + stats.heap_bytes())
16319 .sum::<usize>(),
16320 ]
16321 .map(|value| u32::try_from(value).unwrap_or(u32::MAX))
16322 }
16323
16324 pub fn export_memory_layout_json(&self) -> String {
16326 let isr_rx = self
16327 .isr_rx_queue
16328 .as_ref()
16329 .and_then(|queue| queue.snapshot())
16330 .unwrap_or((0, 0));
16331 let st = self.state.lock();
16332 #[cfg(feature = "discovery")]
16333 let discovery_bytes = st.discovery_bytes_used();
16334 #[cfg(not(feature = "discovery"))]
16335 let discovery_bytes = 0usize;
16336 let schema_bytes = crate::config::schema_bytes_used();
16337 let network_variable_cache_bytes = st.local_cache_bytes();
16338 let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
16339 (0, 0)
16340 } else {
16341 st.replay_counters()
16342 };
16343 let mut out = String::new();
16344 let memory = st.memory;
16345 let _ = fmt::Write::write_fmt(
16346 &mut out,
16347 format_args!(
16348 "{{\"kind\":\"{}\",\
16349 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
16350 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
16351 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
16352 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
16353 \"replay_queue_bytes_used\":{},\"replay_queue_bytes_allocated\":{},\"replay_queue_len\":{},\
16354 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
16355 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
16356 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
16357 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
16358 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
16359 if self.cfg.role.local_services() {
16360 "router"
16361 } else {
16362 "relay"
16363 },
16364 st.shared_queue_bytes_used(),
16365 memory.max_queue_budget,
16366 st.received_queue.bytes_used(),
16367 st.received_queue.max_bytes(),
16368 st.received_queue.len(),
16369 isr_rx.1,
16370 memory.max_queue_budget,
16371 isr_rx.0,
16372 st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
16373 st.transmit_queue.max_bytes(),
16374 st.transmit_queue.len().saturating_sub(replay_len),
16375 replay_bytes,
16376 memory.max_queue_budget,
16377 replay_len,
16378 st.recent_rx.bytes_used(),
16379 st.recent_rx.max_bytes(),
16380 st.recent_rx.len(),
16381 st.reliable_rx_buffered_bytes(),
16382 memory.max_queue_budget,
16383 st.reliable_rx_buffer_len(),
16384 discovery_bytes,
16385 memory.max_queue_budget,
16386 schema_bytes,
16387 memory.max_queue_budget,
16388 network_variable_cache_bytes,
16389 memory.max_queue_budget,
16390 st.local_cache_len(),
16391 ),
16392 );
16393 out
16394 }
16395
16396 #[cfg(test)]
16397 #[cfg(feature = "discovery")]
16398 pub(crate) fn debug_end_to_end_pending_destination_count(
16399 &self,
16400 packet_id: u64,
16401 ) -> Option<usize> {
16402 let st = self.state.lock();
16403 st.end_to_end_reliable_tx
16404 .get(&packet_id)
16405 .map(|sent| sent.pending_destinations.len())
16406 }
16407
16408 #[cfg(test)]
16409 #[cfg(feature = "discovery")]
16410 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
16411 let st = self.state.lock();
16412 st.end_to_end_reliable_tx.len()
16413 }
16414
16415 #[cfg(test)]
16416 #[cfg(feature = "discovery")]
16417 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
16418 let st = self.state.lock();
16419 st.reliable_return_routes.len()
16420 }
16421
16422 #[cfg(test)]
16423 pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
16424 let st = self.state.lock();
16425 (st.sides.len(), st.sides.capacity())
16426 }
16427
16428 #[cfg(test)]
16429 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
16430 let st = self.state.lock();
16431 (
16432 st.received_queue.len(),
16433 st.transmit_queue.len(),
16434 st.recent_rx.len(),
16435 )
16436 }
16437
16438 #[cfg(test)]
16439 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
16440 let st = self.state.lock();
16441 st.shared_queue_bytes_used()
16442 }
16443
16444 #[cfg(test)]
16445 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
16446 let st = self.state.lock();
16447 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
16448 }
16449
16450 fn get_hash(item: &RouterItem) -> u64 {
16454 match item {
16455 RouterItem::Packet(pkt) => pkt.packet_id(),
16456 RouterItem::Packed(bytes) => {
16457 match wire_format::packet_id_from_wire(bytes.as_ref()) {
16458 Ok(id) => id,
16459 Err(_e) => {
16460 let h: u64 = 0x9E37_79B9_7F4A_7C15;
16463 hash_bytes_u64(h, bytes.as_ref())
16464 }
16465 }
16466 }
16467 }
16468 }
16469
16470 fn remove_pkt_id(&self, item: &RouterItem) {
16472 let hash = Self::get_hash(item);
16473 let mut st = self.state.lock();
16474 st.recent_rx.remove_value(&hash);
16475 }
16476
16477 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
16480 self.is_duplicate_packet_id(Self::get_hash(item))
16481 }
16482
16483 fn is_duplicate_packet_id(&self, id: u64) -> TelemetryResult<bool> {
16484 let mut st = self.state.lock();
16485 if st.recent_rx.contains(&id)
16486 || st
16487 .reliable_return_routes
16488 .get(&id)
16489 .is_some_and(|route| route.delivery_completed)
16490 {
16491 Ok(true)
16492 } else {
16493 st.push_recent_rx(id)?;
16494 Ok(false)
16495 }
16496 }
16497
16498 fn handle_callback_error(
16503 &self,
16504 pkt: &Packet,
16505 dest: Option<DataEndpoint>,
16506 e: TelemetryError,
16507 called_from_queue: bool,
16508 ) -> TelemetryResult<()> {
16509 if !self.cfg.role.local_services() {
16510 return Err(e);
16511 }
16512
16513 if matches!(&e, TelemetryError::Io("memory pressure")) {
16516 return Err(e);
16517 }
16518
16519 let device = self.sender_arc();
16520 let error_msg = match dest {
16521 Some(failed_local) => format!(
16522 "Handler for endpoint {:?} failed on device {:?}: {:?}",
16523 failed_local, device, e
16524 ),
16525 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
16526 };
16527
16528 let mut recipients: Vec<DataEndpoint> = pkt
16529 .endpoints()
16530 .iter()
16531 .copied()
16532 .filter(|&ep| self.cfg.is_local_endpoint(ep))
16533 .collect();
16534 crate::small_sort::sort(&mut recipients);
16535 recipients.dedup();
16536
16537 if let Some(failed_local) = dest {
16538 recipients.retain(|&ep| ep != failed_local);
16539 }
16540
16541 if recipients.is_empty() && dest.is_none() {
16545 fallback_stdout(&error_msg);
16546 return Ok(());
16547 }
16548
16549 if recipients.is_empty() {
16552 recipients = pkt.endpoints().to_vec();
16553 crate::small_sort::sort(&mut recipients);
16554 recipients.dedup();
16555 if let Some(failed_local) = dest {
16556 recipients.retain(|&ep| ep != failed_local);
16557 }
16558 }
16559
16560 if recipients.is_empty() {
16561 fallback_stdout(&error_msg);
16562 return Ok(());
16563 }
16564
16565 let payload = make_error_payload(&error_msg);
16566
16567 let sender = self.sender_arc();
16568 let error_pkt = Packet::new(
16569 DataType::TelemetryError,
16570 &recipients,
16571 sender.as_ref(),
16572 self.packet_timestamp_ms(),
16573 crate::shared_bytes::convert(payload),
16574 )?;
16575
16576 self.emit_internal_tx(
16577 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
16578 false,
16579 called_from_queue,
16580 )
16581 }
16582
16583 #[inline]
16587 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
16588 self.process_tx_queue_with_timeout(0)
16589 }
16590
16591 #[inline]
16593 pub fn process_all_queues(&self) -> TelemetryResult<()> {
16594 self.process_all_queues_with_timeout(0)
16595 }
16596
16597 #[inline]
16599 pub fn clear_queues(&self) {
16600 let mut st = self.state.lock();
16601 st.retain_transmit(|_| false);
16602 st.transmit_queue.cancel_deferred_turn();
16603 #[cfg(feature = "latest-value-egress")]
16604 if st.latest_turn_phase < 3 {
16605 st.latest_turn_phase = 0;
16606 st.latest_turn_credit = 0;
16607 }
16608 #[cfg(feature = "discovery")]
16609 for response in st.discovery_side_throttle.values_mut() {
16610 if response.schema_response_active() {
16611 response.set_schema_response_active(false);
16612 response.schema_response_phase = SchemaResponsePhase::Idle;
16613 response.set_schema_response_pending(true);
16614 }
16615 }
16616 st.tx_policy_flags &= !1;
16617 st.received_queue.clear();
16618 st.rx_priority_burst = 0;
16619 drop(st);
16620 let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
16621 }
16622
16623 #[inline]
16625 pub fn clear_rx_queue(&self) {
16626 let mut st = self.state.lock();
16627 st.received_queue.clear();
16628 st.rx_priority_burst = 0;
16629 drop(st);
16630 let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
16631 }
16632
16633 #[inline]
16635 pub fn clear_tx_queue(&self) {
16636 let mut st = self.state.lock();
16637 st.retain_transmit(|_| false);
16638 st.transmit_queue.cancel_deferred_turn();
16639 #[cfg(feature = "latest-value-egress")]
16640 if st.latest_turn_phase < 3 {
16641 st.latest_turn_phase = 0;
16642 st.latest_turn_credit = 0;
16643 }
16644 #[cfg(feature = "discovery")]
16645 for response in st.discovery_side_throttle.values_mut() {
16646 if response.schema_response_active() {
16647 response.set_schema_response_active(false);
16648 response.schema_response_phase = SchemaResponsePhase::Idle;
16649 response.set_schema_response_pending(true);
16650 }
16651 }
16652 st.tx_priority_burst = 0;
16653 st.tx_policy_flags &= !1;
16654 }
16655
16656 #[inline(never)]
16657 fn pop_transmit_queue_locked(
16658 st: &mut RouterInner,
16659 now_ms: u64,
16660 control_only: bool,
16661 ) -> Option<TxWork> {
16662 let slow = st.tx_policy_flags & 1 != 0;
16663 let mut next_token = st.ordered_claim_seq.wrapping_add(1).max(1);
16664 if slow {
16665 while st.claim_index(next_token).is_some() {
16666 next_token = next_token.wrapping_add(1).max(1);
16667 }
16668 }
16669 #[cfg(feature = "latest-value-egress")]
16670 let latest_blocked = st.latest_blocked_side(now_ms);
16671 #[cfg(feature = "latest-value-egress")]
16672 let latest_turn = st.latest_turn_selection(latest_blocked.is_none(), now_ms);
16673 #[cfg(feature = "latest-value-egress")]
16674 let latest_side = st.latest_side();
16675 #[cfg(feature = "latest-value-egress")]
16676 let latest_frame = latest_side
16677 .and_then(|side| st.sides.get(side)?.as_ref())
16678 .map_or(0, |side| side.opts.max_frame_bytes);
16679 let (selected, owed_turn) = st.transmit_queue.select_front_fair_deferred(
16680 &mut st.tx_priority_burst,
16681 MAX_PRIORITY_BURST,
16682 |item| item.priority,
16683 |_queue, item| {
16684 !matches!(
16685 item.delivery_state,
16686 QueuedDelivery::ClaimedOrdered
16687 | QueuedDelivery::ClaimedUnordered
16688 | QueuedDelivery::OrderedTail
16689 ) && item.retry_after_ms <= now_ms
16690 && (!control_only || item.priority >= 200)
16691 && {
16692 #[cfg(feature = "latest-value-egress")]
16693 {
16694 let turn_ready = latest_turn.is_none_or(|index| {
16695 !latest::managed_turn_item(item, latest_side, latest_frame)
16696 || _queue
16697 .get(index)
16698 .is_some_and(|chosen| core::ptr::eq(chosen, item))
16699 });
16700 turn_ready
16701 && !(item.disposable & TX_LATEST_PENDING != 0
16702 && matches!(&item.item,
16703 RouterTxItem::ToSide { dst, .. } if latest_blocked == Some(*dst)))
16704 }
16705 #[cfg(not(feature = "latest-value-egress"))]
16706 {
16707 true
16708 }
16709 }
16710 },
16711 )?;
16712 #[cfg(feature = "latest-value-egress")]
16713 if latest_turn == Some(selected) {
16714 if let Some(next) = st.latest_service_seq.checked_add(1) {
16715 st.latest_service_seq = next;
16716 } else {
16717 st.tx_policy_flags &= 1;
16718 st.latest_turn_phase = 0;
16719 st.latest_turn_credit = 0;
16720 st.clear_canonical_turn_flags();
16721 }
16722 if st.latest_side().is_some()
16723 && st.canonical_catalog_owner(st.transmit_queue.get(selected)?)
16724 {
16725 st.latest_turn_phase = 3;
16726 } else if let Some(side) = st.latest_side() {
16727 if st
16728 .transmit_queue
16729 .get(selected)
16730 .is_some_and(|queued| queued.disposable & TX_LATEST_TURN == 0)
16731 && let Some(progress) = st
16732 .side_transport
16733 .get_mut(&side)
16734 .and_then(|side| side.tx_chunk_progress.as_mut())
16735 {
16736 progress.clear_latest_owner();
16739 }
16740 st.latest_turn_phase = 4;
16741 st.transmit_queue
16742 .update_metadata(selected, |q| q.disposable |= TX_LATEST_TURN);
16743 }
16744 }
16745 let current = st.transmit_queue.get(selected)?;
16746 let reliable = RouterInner::reliable_original_type(current).is_some();
16747 let ordered = reliable && RouterInner::ordered_original_type(current).is_some();
16748 let child = current.disposable & TX_E2E_CHILD != 0;
16749 let (queued, claim) = if child {
16750 let queued = current.clone();
16751 st.transmit_queue.update_metadata(selected, |item| {
16752 item.delivery_state = QueuedDelivery::ClaimedUnordered;
16753 if let RouterTxItem::ToSide { src, .. } = &mut item.item {
16754 *src = Some(next_token as usize);
16755 }
16756 });
16757 st.ordered_claim_seq = next_token;
16758 (
16759 queued,
16760 Some(TxClaim {
16761 token: next_token,
16762 ordered: false,
16763 side_retry: true,
16764 }),
16765 )
16766 } else if reliable {
16767 let queued = current.clone();
16768 st.transmit_queue.update_metadata(selected, |item| {
16769 item.delivery_state = if ordered {
16770 QueuedDelivery::ClaimedOrdered
16771 } else {
16772 QueuedDelivery::ClaimedUnordered
16773 };
16774 item.deferred_since_ms = u64::from(next_token);
16775 });
16776 st.ordered_claim_seq = next_token;
16777 (
16778 queued,
16779 Some(TxClaim {
16780 token: next_token,
16781 ordered,
16782 side_retry: false,
16783 }),
16784 )
16785 } else {
16786 (st.transmit_queue.remove_pos(selected)?, None)
16787 };
16788 if claim.is_some() {
16789 st.tx_policy_flags |= 1;
16790 } else if slow {
16791 st.refresh_retry_hint();
16792 }
16793 #[cfg(feature = "latest-value-egress")]
16796 let queued = {
16797 let mut selected = queued;
16798 if selected.disposable & TX_LATEST_PENDING != 0 {
16799 selected.disposable = TX_HISTORY_OWNED | (selected.disposable & TX_LATEST_TURN);
16800 }
16801 selected
16802 };
16803 #[cfg(feature = "latest-value-egress")]
16804 let service = {
16805 #[cfg(feature = "latest-value-egress")]
16806 {
16807 EgressService {
16808 token: st.latest_service_seq,
16809 side: st.latest_dispatch_tag(&queued),
16810 kind: if queued.disposable & TX_CANONICAL_OWED != 0 {
16811 1
16812 } else if queued.disposable & TX_LATEST_TURN != 0 {
16813 2
16814 } else {
16815 0
16816 },
16817 }
16818 }
16819 };
16820 Some(TxWork {
16821 queued,
16822 claim,
16823 owed_turn,
16824 #[cfg(feature = "latest-value-egress")]
16825 service,
16826 })
16827 }
16828
16829 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16832 self.expire_side_chunks();
16833 let start = self.clock.now_ms();
16834 loop {
16835 let received_control = crate::memory_admission::allow_drain(
16836 self.drain_queued_control_rx_before_tx(start, timeout_ms),
16837 )?;
16838 let control_budget_used = received_control
16839 && timeout_ms != 0
16840 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
16841 if !control_budget_used {
16844 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
16845 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
16846 }
16847
16848 let pkt_opt = {
16849 let mut st = self.state.lock();
16850 #[cfg(feature = "std")]
16851 let work = if self
16852 .active_queued_tx
16853 .load(core::sync::atomic::Ordering::Acquire)
16854 == usize::MAX
16855 {
16856 None } else {
16858 Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false)
16859 };
16860 #[cfg(not(feature = "std"))]
16861 let work = Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false);
16862 #[cfg(feature = "std")]
16863 let work = self.track_queued_tx(&mut st, work);
16864 work
16865 };
16866 let Some(pkt) = pkt_opt else { break };
16867 #[cfg(feature = "std")]
16868 let (pkt, _activity) = pkt;
16869 self.dispatch_queued_item(pkt)?;
16870 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
16871 break;
16872 }
16873 }
16874 Ok(())
16875 }
16876
16877 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16880 #[cfg(feature = "timesync")]
16881 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
16882 #[cfg(feature = "discovery")]
16883 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16884 self.process_tx_queue_with_timeout_impl(timeout_ms)
16885 }
16886
16887 pub fn dispatch_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16893 self.process_tx_queue_with_timeout_impl(timeout_ms)
16894 }
16895
16896 #[inline]
16898 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
16899 self.rx_item(&item, true)
16900 }
16901
16902 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16905 self.expire_side_chunks();
16906 let start = self.clock.now_ms();
16907 loop {
16908 let item_opt = self
16909 .isr_rx_queue
16910 .as_ref()
16911 .and_then(|queue| queue.pop_front().unwrap_or(None))
16912 .or_else(|| {
16913 let mut st = self.state.lock();
16914 st.pop_received_fair()
16915 });
16916 let Some(item) = item_opt else { break };
16917 self.process_rx_queue_item(item)?;
16918 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
16919 break;
16920 }
16921 }
16922 Ok(())
16923 }
16924
16925 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16928 #[cfg(feature = "timesync")]
16929 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
16930 #[cfg(feature = "discovery")]
16931 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16932 self.process_rx_queue_with_timeout_impl(timeout_ms)
16933 }
16934
16935 fn process_one_queued_rx(&self) -> TelemetryResult<bool> {
16938 let item = self
16939 .isr_rx_queue
16940 .as_ref()
16941 .and_then(|queue| queue.pop_front().unwrap_or(None))
16942 .or_else(|| self.state.lock().pop_received_fair());
16943 let Some(item) = item else {
16944 return Ok(false);
16945 };
16946 self.process_rx_queue_item(item)?;
16947 Ok(true)
16948 }
16949
16950 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16951 self.expire_side_chunks();
16952 let receive_first = !self.cfg.role.local_services();
16957 let start = self.clock.now_ms();
16958 loop {
16959 let received_control = crate::memory_admission::allow_drain(
16960 self.drain_queued_control_rx_before_tx(start, timeout_ms),
16961 )?;
16962 let mut did_any = received_control;
16963 let control_budget_used = received_control
16964 && timeout_ms != 0
16965 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
16966 if !control_budget_used {
16967 crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
16968 crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
16969 }
16970
16971 if receive_first && !control_budget_used {
16972 did_any |= self.process_one_queued_rx()?;
16973 }
16974
16975 if let Some(pkt) = {
16976 let mut st = self.state.lock();
16977 let now = self.clock.now_ms();
16978 #[cfg(feature = "std")]
16981 let work = if self
16982 .active_queued_tx
16983 .load(core::sync::atomic::Ordering::Acquire)
16984 == usize::MAX
16985 {
16986 None } else {
16988 Self::pop_transmit_queue_locked(&mut st, now, false)
16989 };
16990 #[cfg(not(feature = "std"))]
16991 let work = Self::pop_transmit_queue_locked(&mut st, now, false);
16992 #[cfg(feature = "std")]
16993 let work = self.track_queued_tx(&mut st, work);
16994 work
16995 } {
16996 #[cfg(feature = "std")]
16997 let (pkt, _activity) = pkt;
16998 self.dispatch_queued_item(pkt)?;
16999 did_any = true;
17000 }
17001 if control_budget_used {
17003 return Ok(());
17004 }
17005 if !receive_first {
17006 did_any |= self.process_one_queued_rx()?;
17007 }
17008 if !did_any
17009 || (timeout_ms != 0
17010 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms))
17011 {
17012 break;
17013 }
17014 }
17015
17016 Ok(())
17017 }
17018
17019 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
17022 #[cfg(feature = "timesync")]
17023 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
17024 #[cfg(feature = "discovery")]
17025 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17026 self.process_all_queues_with_timeout_impl(timeout_ms)
17027 }
17028
17029 pub fn drain_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
17034 self.process_all_queues_with_timeout_impl(timeout_ms)
17035 }
17036
17037 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
17042 #[cfg(feature = "timesync")]
17043 let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
17044
17045 #[cfg(feature = "discovery")]
17046 {
17047 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17048 }
17049
17050 self.process_all_queues_with_timeout_impl(timeout_ms)
17051 }
17052
17053 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
17058 #[cfg(feature = "discovery")]
17059 {
17060 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17061 }
17062
17063 self.process_all_queues_with_timeout_impl(timeout_ms)
17064 }
17065
17066 #[inline]
17068 fn tx_queue_item_with_flags(
17069 &self,
17070 item: RouterTxItem,
17071 ignore_local: bool,
17072 ) -> TelemetryResult<()> {
17073 self.tx_queue_item_with_flags_at(item, ignore_local, 0)
17074 }
17075
17076 #[cfg(feature = "compression")]
17077 fn retry_codec_owner(
17078 &self,
17079 item: RouterTxItem,
17080 ignore_local: bool,
17081 reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
17082 ) -> TelemetryResult<()> {
17083 self.retry_codec_owner_at_priority(item, ignore_local, reservation, 0, 0, false)
17084 }
17085
17086 #[cfg(feature = "compression")]
17087 fn retry_codec_owner_at_priority(
17088 &self,
17089 item: RouterTxItem,
17090 ignore_local: bool,
17091 reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
17092 priority_floor: u8,
17093 _turn_flags: u8,
17094 immediate: bool,
17095 ) -> TelemetryResult<()> {
17096 let priority = priority_floor.max(match &item {
17097 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
17098 _ => return Err(TelemetryError::BadArg),
17099 });
17100 #[cfg(feature = "discovery")]
17101 let schema_owner = RouterInner::schema_response_owner(&item);
17102 let mut state = self.state.lock();
17103 if let Some(reservation) = reservation {
17104 reservation.release_locked(&mut state);
17105 }
17106 let result = state.push_transmit(TxQueued {
17107 disposable: TX_DISPOSABLE | _turn_flags,
17108 item,
17109 ignore_local,
17110 priority,
17111 delivery_state: QueuedDelivery::Untracked,
17112 accepted_type: 0,
17113 deferred_since_ms: 0,
17114 retry_after_ms: self.clock.now_ms().saturating_add(u64::from(!immediate)),
17115 });
17116 #[cfg(feature = "discovery")]
17117 if result.is_err() {
17118 RouterInner::schema_response_owner_lost(
17119 &mut state.discovery_side_throttle,
17120 schema_owner,
17121 );
17122 }
17123 result
17124 }
17125
17126 fn claim_schema_is_current(
17127 &self,
17128 ty: DataType,
17129 claim: Option<TxClaim>,
17130 ) -> TelemetryResult<bool> {
17131 let Some(claim) = claim else {
17132 return Ok(true);
17133 };
17134 let expected = if claim.ordered {
17135 crate::ReliableMode::Ordered
17136 } else {
17137 crate::ReliableMode::Unordered
17138 };
17139 let current = reliable_mode(ty);
17140 let mut st = self.state.lock();
17141 let owned = st.claim_index(claim.token).is_some();
17142 if !owned {
17143 return Ok(false);
17144 }
17145 if current == expected {
17146 return Ok(true);
17147 }
17148 st.cancel_accepted_type(ty.as_u32());
17149 st.refresh_retry_hint();
17150 Err(TelemetryError::HandlerError(
17151 "accepted reliable schema changed before dispatch",
17152 ))
17153 }
17154
17155 fn mark_claim_local_delivered(&self, claim: Option<TxClaim>) -> bool {
17156 let Some(claim) = claim else {
17157 return true;
17158 };
17159 let mut st = self.state.lock();
17160 let index = st.claim_index(claim.token);
17161 let Some(index) = index else {
17162 return false;
17163 };
17164 st.transmit_queue
17165 .update_metadata(index, |item| item.ignore_local = true);
17166 true
17167 }
17168
17169 fn dispatch_queued_item(&self, work: TxWork) -> TelemetryResult<()> {
17170 let owed_turn = work.owed_turn;
17171 let queued = work.queued;
17172 let parent = (queued.disposable & TX_E2E_CHILD != 0).then_some(queued.deferred_since_ms);
17173 let since = matches!(
17174 queued.delivery_state,
17175 QueuedDelivery::DeferredOrdered | QueuedDelivery::DeferredUnordered
17176 )
17177 .then_some(queued.deferred_since_ms);
17178 let mut context = DispatchContext {
17179 priority: queued.priority,
17180 claim: work.claim,
17181 since,
17182 completion_flags: queued.disposable & {
17183 #[cfg(feature = "latest-value-egress")]
17184 {
17185 TX_CANONICAL_TURN | TX_LATEST_TURN
17186 }
17187 #[cfg(not(feature = "latest-value-egress"))]
17188 {
17189 0
17190 }
17191 },
17192 admitted: false,
17193 #[cfg(feature = "latest-value-egress")]
17194 latest_service: work.service,
17195 };
17196 let schema = if work.claim.is_some_and(|claim| claim.side_retry) {
17197 Ok(true)
17198 } else {
17199 self.claim_schema_is_current(DataType(queued.accepted_type), work.claim)
17200 };
17201 if !matches!(schema, Ok(true)) {
17202 self.finish_dispatch_turn(owed_turn, &context);
17203 return schema.map(|_| ());
17204 }
17205 let mut child_unwind = work.claim.filter(|claim| claim.side_retry).map(|claim| {
17206 e2e_backpressure::ChildUnwind {
17207 router: self,
17208 claim,
17209 armed: true,
17210 }
17211 });
17212 let mut result =
17213 self.tx_item_impl_with_retry(queued.item, queued.ignore_local, true, &mut context);
17214 if let Some(claim) = work.claim {
17215 if claim.side_retry {
17216 self.finish_e2e_child_claim(
17217 claim,
17218 parent.expect("child parent"),
17219 &context,
17220 result.is_ok(),
17221 )?;
17222 if let Some(guard) = &mut child_unwind {
17223 guard.armed = false;
17224 }
17225 self.finish_dispatch_turn(owed_turn, &context);
17226 return result;
17227 }
17228 let mut st = self.state.lock();
17229 if result.is_err() && !context.is_terminal() && !context.owner_transferred() {
17230 let Some(index) = st.claim_index(claim.token) else {
17231 drop(st);
17232 self.finish_dispatch_turn(owed_turn, &context);
17233 return result;
17234 };
17235 let original = st.transmit_queue.get(index).expect("claimed owner");
17236 if !original.ignore_local
17237 && let RouterTxItem::Broadcast(data) = &original.item
17238 {
17239 let id = Self::get_hash(data);
17240 st.recent_rx.remove_value(&id);
17241 }
17242 drop(st);
17243 result = self.defer_accepted_original(&mut context);
17244 if result.is_ok() {
17245 self.finish_dispatch_turn(owed_turn, &context);
17246 return Ok(());
17247 }
17248 st = self.state.lock();
17249 }
17250 if let Some(index) = st.claim_index(claim.token) {
17251 if context.is_terminal() && claim.ordered {
17252 let ty = st
17253 .transmit_queue
17254 .get(index)
17255 .expect("claimed owner")
17256 .accepted_type;
17257 st.cancel_accepted_type(ty);
17258 } else {
17259 let ty = st
17260 .transmit_queue
17261 .get(index)
17262 .expect("claimed owner")
17263 .accepted_type;
17264 let _ = st.transmit_queue.remove_pos(index);
17265 if claim.ordered {
17266 let next = st.transmit_queue.iter().position(|item| {
17269 item.delivery_state == QueuedDelivery::OrderedTail
17270 && item.accepted_type == ty
17271 });
17272 if let Some(next) = next {
17273 st.transmit_queue.update_metadata(next, |item| {
17274 item.delivery_state = QueuedDelivery::Ordered
17275 });
17276 }
17277 }
17278 }
17279 }
17280 st.refresh_retry_hint();
17281 }
17282 self.finish_dispatch_turn(owed_turn, &context);
17283 result
17284 }
17285
17286 fn current_egress_requeue(&self, context: &DispatchContext) -> (u8, bool) {
17287 #[cfg(feature = "latest-value-egress")]
17288 {
17289 let service = context.egress_service();
17290 let st = self.state.lock();
17291 if service.side == 0
17292 || service.token != st.latest_service_seq
17293 || st.latest_side() != Some(usize::from(service.side - 1))
17294 {
17295 return (0, false);
17296 }
17297 }
17298 (
17299 context.canonical_turn_flags(),
17300 context.yielded_canonical_turn(),
17301 )
17302 }
17303
17304 fn finish_dispatch_turn(&self, owed_turn: bool, context: &DispatchContext) {
17305 let mut guard = self.state.lock();
17306 let st = &mut *guard;
17307 #[cfg(feature = "latest-value-egress")]
17308 if context.latest_service.side != 0
17309 && context.latest_service.token == st.latest_service_seq
17310 && st.latest_side() == Some(usize::from(context.latest_service.side - 1))
17311 {
17312 let kind = if context.completion_flags & TX_CANONICAL_OWED != 0 {
17313 1
17314 } else if context.completion_flags & TX_LATEST_TURN != 0 {
17315 2
17316 } else {
17317 0
17318 };
17319 st.latest_turn_phase = match kind {
17320 1 if context.admitted => 2,
17321 1 if context.refused() => 1,
17322 1 => 0,
17323 2 if context.admitted => {
17324 if st.latest_turn_credit > 0 {
17325 2
17326 } else {
17327 0
17328 }
17329 }
17330 2 if context.refused() => 2,
17331 2 => 0,
17332 _ => st.latest_turn_phase,
17333 };
17334 }
17335 st.transmit_queue.finish_deferred_turn(
17336 context.priority,
17337 owed_turn,
17338 context.admitted,
17339 context.refused(),
17340 &mut st.tx_priority_burst,
17341 );
17342 }
17343
17344 fn expire_deferred_original(
17345 &self,
17346 ty: DataType,
17347 since: u64,
17348 ordered: bool,
17349 ) -> TelemetryResult<()> {
17350 let lifetime = runtime_reliable_retransmit_ms()
17351 .saturating_mul(u64::from(runtime_reliable_max_retries()) + 1);
17352 let now = self.clock.now_ms();
17353 if now >= since && now - since < lifetime {
17354 return Ok(());
17355 }
17356 if ordered {
17357 let mut st = self.state.lock();
17358 st.cancel_accepted_type(ty.as_u32());
17359 st.refresh_retry_hint();
17360 }
17361 Err(TelemetryError::HandlerError(
17362 "accepted reliable delivery expired before dispatch",
17363 ))
17364 }
17365
17366 fn defer_accepted_original(&self, context: &mut DispatchContext) -> TelemetryResult<()> {
17367 context.mark_refused();
17368 let Some(claim) = context.claim else {
17369 return Err(TelemetryError::Io("reliable owner unavailable"));
17370 };
17371 let now = self.clock.now_ms();
17372 let Some(retry_after_ms) = now.checked_add(1) else {
17373 context.mark_terminal();
17374 return Err(TelemetryError::HandlerError(
17375 "accepted reliable clock exhausted",
17376 ));
17377 };
17378 let mut st = self.state.lock();
17379 let Some(index) = st.claim_index(claim.token) else {
17380 return Ok(());
17381 };
17382 st.transmit_queue.update_metadata(index, |item| {
17383 item.delivery_state = if claim.ordered {
17384 QueuedDelivery::DeferredOrdered
17385 } else {
17386 QueuedDelivery::DeferredUnordered
17387 };
17388 item.deferred_since_ms = context.since.unwrap_or(now);
17389 item.retry_after_ms = retry_after_ms;
17390 });
17391 st.tx_policy_flags |= 1;
17392 Ok(())
17393 }
17394
17395 #[cfg(feature = "discovery")]
17396 #[allow(unused_mut)]
17397 fn replace_claim_payload(&self, claim: TxClaim, mut data: RouterItem) -> TelemetryResult<()> {
17398 #[cfg(feature = "compact-packet-store")]
17399 data.park_for_queue()?;
17400 let payload_cost = match &data {
17401 RouterItem::Packet(packet) => packet.byte_cost(),
17402 RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
17403 };
17404 let mut st = self.state.lock();
17405 let Some(index) = st.claim_index(claim.token) else {
17406 return Ok(());
17407 };
17408 let original = st.transmit_queue.get(index).expect("claimed owner");
17409 let old_cost = original.byte_cost();
17410 let new_cost = old_cost
17411 .saturating_sub(original.item.byte_cost())
17412 .saturating_add(payload_cost);
17413 let priority = original.priority;
17414 let growth = new_cost.saturating_sub(old_cost);
17415 if growth != 0 {
17416 st.make_shared_queue_room(growth, priority, None)?;
17417 }
17418 let Some(index) = st.claim_index(claim.token) else {
17419 return Ok(());
17420 };
17421 st.transmit_queue.update_admitted(index, new_cost, |item| {
17422 item.item = RouterTxItem::Broadcast(data)
17423 })
17424 }
17425
17426 fn tx_queue_item_for_retry(
17427 &self,
17428 item: RouterTxItem,
17429 ignore_local: bool,
17430 ) -> TelemetryResult<()> {
17431 self.tx_queue_item_with_flags_at(item, ignore_local, self.clock.now_ms().saturating_add(1))
17432 }
17433
17434 #[cfg(not(feature = "compression"))]
17435 fn tx_queue_item_for_retry_at_priority_with_turn(
17436 &self,
17437 item: RouterTxItem,
17438 ignore_local: bool,
17439 priority: u8,
17440 turn_flags: u8,
17441 immediate: bool,
17442 ) -> TelemetryResult<()> {
17443 #[cfg(feature = "discovery")]
17444 let schema_owner = RouterInner::schema_response_owner(&item);
17445 let result = self.tx_queue_item_with_priority_and_turn_at(
17446 item,
17447 ignore_local,
17448 priority,
17449 self.clock.now_ms().saturating_add(u64::from(!immediate)),
17450 turn_flags,
17451 );
17452 #[cfg(feature = "discovery")]
17453 if result.is_err() {
17454 RouterInner::schema_response_owner_lost(
17455 &mut self.state.lock().discovery_side_throttle,
17456 schema_owner,
17457 );
17458 }
17459 result
17460 }
17461
17462 fn tx_queue_item_with_flags_at(
17463 &self,
17464 item: RouterTxItem,
17465 ignore_local: bool,
17466 retry_after_ms: u64,
17467 ) -> TelemetryResult<()> {
17468 let priority = match &item {
17469 RouterTxItem::Broadcast(data) => self.item_service_priority(data)?,
17470 RouterTxItem::EndToEndReplay { .. } => {
17471 Self::router_item_priority_bumped(DataType::ReliableAck)
17472 }
17473 RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
17474 RouterTxItem::ReliableReplay { ty, .. } => Self::router_item_priority_bumped(*ty),
17475 };
17476 self.tx_queue_item_with_priority_at(item, ignore_local, priority, retry_after_ms)
17477 }
17478
17479 #[inline]
17480 fn tx_queue_item_with_priority(
17481 &self,
17482 item: RouterTxItem,
17483 ignore_local: bool,
17484 priority: u8,
17485 ) -> TelemetryResult<()> {
17486 self.tx_queue_item_with_priority_at(item, ignore_local, priority, 0)
17487 }
17488
17489 fn tx_queue_item_with_priority_at(
17490 &self,
17491 item: RouterTxItem,
17492 ignore_local: bool,
17493 priority: u8,
17494 retry_after_ms: u64,
17495 ) -> TelemetryResult<()> {
17496 self.tx_queue_item_with_priority_and_turn_at(
17497 item,
17498 ignore_local,
17499 priority,
17500 retry_after_ms,
17501 0,
17502 )
17503 }
17504 fn tx_queue_item_with_priority_and_turn_at(
17505 &self,
17506 item: RouterTxItem,
17507 ignore_local: bool,
17508 priority: u8,
17509 retry_after_ms: u64,
17510 turn_flags: u8,
17511 ) -> TelemetryResult<()> {
17512 let mut st = self.state.lock();
17513 st.push_transmit(TxQueued {
17514 disposable: TX_DISPOSABLE | turn_flags,
17515 item,
17516 ignore_local,
17517 priority,
17518 delivery_state: QueuedDelivery::Untracked,
17519 accepted_type: 0,
17520 deferred_since_ms: 0,
17521 retry_after_ms,
17522 })?;
17523 Ok(())
17524 }
17525
17526 #[inline]
17528 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
17529 self.tx_queue_item_with_flags(item, false)
17530 }
17531
17532 #[cfg(feature = "std")]
17533 #[inline]
17534 fn try_enter_side_tx(&self, side: usize) -> Option<crate::lock::ActiveSideTxGuard<'_>> {
17535 let gate = self
17536 .state
17537 .lock()
17538 .side_transport
17539 .get(&side)?
17540 .tx_callback_gate
17541 .clone();
17542 self.side_tx_activity.try_enter(gate)
17543 }
17544 #[cfg(not(feature = "std"))]
17545 #[inline]
17546 fn try_enter_side_tx(&self, _side: usize) -> Option<crate::lock::ReentryGuard<'_>> {
17547 self.side_tx_gate.try_enter()
17548 }
17549 #[inline]
17550 fn side_tx_active(&self) -> bool {
17551 #[cfg(feature = "std")]
17552 {
17553 self.side_tx_activity.is_active()
17554 }
17555 #[cfg(not(feature = "std"))]
17556 {
17557 self.side_tx_gate.is_active()
17558 }
17559 }
17560
17561 #[inline]
17565 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
17566 self.process_rx_queue_with_timeout(0)
17567 }
17568
17569 #[inline]
17571 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
17572 crate::memory_admission::check_receive(bytes)?;
17573 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17574 let priority = self.item_service_priority(&data)?;
17575 let mut st = self.state.lock();
17576 st.push_received(RouterRxItem {
17577 src: None,
17578 data,
17579 priority,
17580 })?;
17581 Ok(())
17582 }
17583
17584 #[inline]
17589 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
17590 crate::memory_admission::check_receive(bytes)?;
17591 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17592 let priority = Self::router_item_priority(&data)?;
17593 self.isr_rx_queue
17594 .as_ref()
17595 .ok_or(TelemetryError::PermissionDenied)?
17596 .push_back_prioritized(RouterRxItem {
17597 src: None,
17598 data,
17599 priority,
17600 })
17601 }
17602
17603 #[inline]
17605 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
17606 pkt.validate()?;
17607 let data = RouterItem::Packet(pkt);
17608 let priority = self.item_service_priority(&data)?;
17609 let mut st = self.state.lock();
17610 st.push_received(RouterRxItem {
17611 src: None,
17612 data,
17613 priority,
17614 })?;
17615 Ok(())
17616 }
17617
17618 #[inline]
17623 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
17624 pkt.validate()?;
17625 let data = RouterItem::Packet(pkt);
17626 let priority = Self::router_item_priority(&data)?;
17627 self.isr_rx_queue
17628 .as_ref()
17629 .ok_or(TelemetryError::PermissionDenied)?
17630 .push_back_prioritized(RouterRxItem {
17631 src: None,
17632 data,
17633 priority,
17634 })
17635 }
17636
17637 #[inline]
17639 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
17640 self.ensure_side_ingress_enabled(side)?;
17641 pkt.validate()?;
17642 let data = RouterItem::Packet(pkt);
17643 let priority = self.item_service_priority(&data)?;
17644 let mut st = self.state.lock();
17645 st.push_received(RouterRxItem {
17646 src: Some(side),
17647 data,
17648 priority,
17649 })?;
17650 Ok(())
17651 }
17652
17653 #[inline]
17658 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
17659 self.ensure_side_ingress_enabled(side)?;
17660 pkt.validate()?;
17661 let data = RouterItem::Packet(pkt);
17662 let priority = Self::router_item_priority(&data)?;
17663 self.isr_rx_queue
17664 .as_ref()
17665 .ok_or(TelemetryError::PermissionDenied)?
17666 .push_back_prioritized(RouterRxItem {
17667 src: Some(side),
17668 data,
17669 priority,
17670 })
17671 }
17672
17673 #[inline]
17675 pub fn rx_packed_queue_from_side(
17676 &self,
17677 bytes: &[u8],
17678 side: RouterSideId,
17679 ) -> TelemetryResult<()> {
17680 self.ensure_side_ingress_enabled(side)?;
17681 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
17682 return Ok(());
17683 };
17684 let data = RouterItem::Packed(decoded);
17685 let priority = self.item_service_priority(&data)?;
17686 let mut st = self.state.lock();
17687 st.push_received(RouterRxItem {
17688 src: Some(side),
17689 data,
17690 priority,
17691 })?;
17692 Ok(())
17693 }
17694
17695 #[inline]
17700 pub fn rx_packed_queue_from_side_isr(
17701 &self,
17702 bytes: &[u8],
17703 side: RouterSideId,
17704 ) -> TelemetryResult<()> {
17705 self.ensure_side_ingress_enabled(side)?;
17706 crate::memory_admission::check_receive(bytes)?;
17707 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17708 let priority = Self::router_item_priority(&data)?;
17709 self.isr_rx_queue
17710 .as_ref()
17711 .ok_or(TelemetryError::PermissionDenied)?
17712 .push_back_prioritized(RouterRxItem {
17713 src: Some(side),
17714 data,
17715 priority,
17716 })
17717 }
17718
17719 fn retry_with_attempts<F, T>(
17720 &self,
17721 times: usize,
17722 f: F,
17723 ) -> Result<(T, usize), (TelemetryError, usize)>
17724 where
17725 F: Fn() -> TelemetryResult<T>,
17726 {
17727 let times = if self.cfg.role.local_services() {
17728 times
17729 } else {
17730 times.min(1)
17731 };
17732 let mut last_err = None;
17733 for attempt in 0..times {
17734 match f() {
17735 Ok(v) => return Ok((v, attempt + 1)),
17736 Err(e) if Self::is_transient_tx_error(&e) => return Err((e, attempt + 1)),
17739 Err(e) => last_err = Some((e, attempt + 1)),
17740 }
17741 }
17742 Err(last_err.expect("times > 0"))
17743 }
17744
17745 #[inline]
17747 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
17748 self.cfg
17749 .handlers()
17750 .iter()
17751 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
17752 }
17753
17754 #[inline]
17756 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
17757 self.cfg
17758 .handlers()
17759 .iter()
17760 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
17761 }
17762
17763 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
17764 pkt.endpoints()
17765 .iter()
17766 .copied()
17767 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
17768 }
17769
17770 fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
17777 self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
17778 }
17779
17780 fn call_handler_with_retries(
17786 &self,
17787 dest: DataEndpoint,
17788 handler: &EndpointHandler,
17789 data: Option<&[u8]>,
17790 pkt_for_ctx: Option<&Packet>,
17791 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
17792 called_from_queue: bool,
17793 ) -> TelemetryResult<()> {
17794 let owned_tmp: Option<RouterItem>;
17795
17796 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
17797 (Some(d), _) => {
17798 owned_tmp = Some(RouterItem::Packed(crate::shared_bytes::convert(
17799 Arc::<[u8]>::from(d),
17800 )));
17801 owned_tmp.as_ref().unwrap()
17802 }
17803 (None, Some(pkt)) => {
17804 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
17805 owned_tmp.as_ref().unwrap()
17806 }
17807 (None, None) => {
17808 debug_assert!(
17809 false,
17810 "call_handler_with_retries called without data or packet context"
17811 );
17812 return Ok(());
17813 }
17814 };
17815
17816 match (&handler.handler, data) {
17817 (EndpointHandlerFn::Packet(f), _) => {
17818 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
17819 with_retries(
17820 self,
17821 dest,
17822 item_for_ctx,
17823 pkt_for_ctx,
17824 env_for_ctx,
17825 called_from_queue,
17826 || f(pkt),
17827 )
17828 }
17829
17830 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
17831 self,
17832 dest,
17833 item_for_ctx,
17834 pkt_for_ctx,
17835 env_for_ctx,
17836 called_from_queue,
17837 || f(bytes),
17838 ),
17839
17840 (EndpointHandlerFn::Packed(_), None) => Ok(()),
17841 }
17842 }
17843
17844 fn handle_callback_error_from_env(
17849 &self,
17850 env: &wire_format::TelemetryEnvelope,
17851 dest: Option<DataEndpoint>,
17852 e: TelemetryError,
17853 called_from_queue: bool,
17854 ) -> TelemetryResult<()> {
17855 if !self.cfg.role.local_services() {
17856 return Err(e);
17857 }
17858
17859 if matches!(&e, TelemetryError::Io("memory pressure")) {
17862 return Err(e);
17863 }
17864
17865 let mut recipients: Vec<DataEndpoint> = env
17866 .endpoints
17867 .iter()
17868 .copied()
17869 .filter(|&ep| self.cfg.is_local_endpoint(ep))
17870 .collect();
17871 crate::small_sort::sort(&mut recipients);
17872 recipients.dedup();
17873 if let Some(failed) = dest {
17874 recipients.retain(|&ep| ep != failed);
17875 }
17876
17877 let device = self.sender_arc();
17878 let error_msg = format!(
17879 "Handler for endpoint {:?} failed on device {:?}: {:?}",
17880 dest, device, e
17881 );
17882
17883 if recipients.is_empty() && dest.is_none() {
17884 fallback_stdout(&error_msg);
17885 return Ok(());
17886 }
17887
17888 if recipients.is_empty() {
17889 recipients = env.endpoints.to_vec();
17890 crate::small_sort::sort(&mut recipients);
17891 recipients.dedup();
17892 if let Some(failed) = dest {
17893 recipients.retain(|&ep| ep != failed);
17894 }
17895 }
17896
17897 if recipients.is_empty() {
17898 fallback_stdout(&error_msg);
17899 return Ok(());
17900 }
17901
17902 let payload = make_error_payload(&error_msg);
17903
17904 let error_pkt = Packet::new(
17905 DataType::TelemetryError,
17906 &recipients,
17907 &env.sender.clone(),
17908 env.timestamp_ms,
17909 crate::shared_bytes::convert(payload),
17910 )?;
17911 self.emit_internal_tx(
17912 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
17913 false,
17914 called_from_queue,
17915 )
17916 }
17917
17918 fn handle_internal_reliable_packet(
17919 &self,
17920 pkt: &Packet,
17921 src: Option<RouterSideId>,
17922 called_from_queue: bool,
17923 ) -> TelemetryResult<bool> {
17924 if !matches!(
17925 pkt.data_type(),
17926 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
17927 ) {
17928 return Ok(false);
17929 }
17930
17931 let Some(src) = src else {
17932 return Ok(false);
17933 };
17934
17935 if Self::is_end_to_end_ack_packet(pkt)
17936 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
17937 {
17938 #[cfg(feature = "std")]
17939 let explicit_completion_receipt = pkt.wire_target_senders().len() == 2
17940 && pkt.wire_target_senders()[1] == Self::sender_hash(self.sender_arc().as_ref());
17941 let targets_local_sender = self.cfg.role.local_services()
17942 && (pkt.wire_target_senders().is_empty()
17943 || pkt
17944 .wire_target_senders()
17945 .contains(&Self::sender_hash(self.sender_arc().as_ref())));
17946 if !targets_local_sender && let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
17947 Self::note_end_to_end_acked_destination_locked(
17948 &mut self.state.lock(),
17949 packet_id,
17950 owner,
17951 );
17952 }
17953 let alias = {
17954 let st = self.state.lock();
17955 st.reliable_return_routes
17956 .get(&packet_id)
17957 .filter(|route| route.upstream_contract.is_some())
17958 .cloned()
17959 };
17960 if let Some(alias) = alias {
17961 let (upstream_id, publisher_hash) = alias.upstream_contract.expect("checked alias");
17962 #[cfg(feature = "std")]
17963 if reliable_trace_enabled() {
17964 eprintln!(
17965 "[sedsnet-reliable] proxy receipt id={packet_id:#x} upstream={upstream_id:#x} owner={:?}",
17966 Self::end_to_end_ack_sender_hash(pkt)
17967 );
17968 }
17969 let completed = {
17970 let mut st = self.state.lock();
17971 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
17972 if let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
17973 sent.pending_destinations.remove(&owner);
17974 }
17975 if !sent.pending_destinations.is_empty() {
17976 return Ok(true);
17977 }
17978 st.end_to_end_reliable_tx.remove(&packet_id);
17979 true
17980 } else {
17981 return Ok(true);
17984 }
17985 };
17986 if completed {
17987 let mut st = self.state.lock();
17988 if Self::remember_reliable_return_route_locked(&mut st, upstream_id) {
17989 st.reliable_return_routes
17990 .entry(upstream_id)
17991 .and_modify(|route| {
17992 route.delivery_completed = true;
17993 route.upstream_contract = Some((upstream_id, publisher_hash));
17994 })
17995 .or_insert(ReliableReturnRouteState {
17996 side: alias.side,
17997 delivery_completed: true,
17998 local_admission_completed: true,
17999 local_admission_token: 0,
18000 admission_cancelled: false,
18001 admission_committing: false,
18002 last_replay_ms: alias.last_replay_ms,
18003 upstream_contract: Some((upstream_id, publisher_hash)),
18004 });
18005 }
18006 }
18007 self.queue_end_to_end_reliable_ack_for_sender_hash(
18008 upstream_id,
18009 publisher_hash,
18010 called_from_queue,
18011 )?;
18012 self.release_ordered_successors()?;
18013 return Ok(true);
18014 }
18015 let mut st = self.state.lock();
18022 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
18023 if sent.waiting_for_predecessor {
18024 return Ok(targets_local_sender);
18025 }
18026 if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
18027 let _removed = sent.pending_destinations.remove(&sender_hash);
18028 #[cfg(feature = "std")]
18029 if reliable_trace_enabled() {
18030 eprintln!(
18031 "[sedsnet-reliable] receipt time={} id={packet_id:#x} sender={sender_hash:#x} matched={} remaining={:?}",
18032 self.clock.now_ms(),
18033 _removed.is_some(),
18034 sent.pending_destinations
18035 );
18036 }
18037 #[cfg(feature = "std")]
18038 let completion = if _removed.is_some()
18039 && explicit_completion_receipt
18040 && targets_local_sender
18041 && sent.tracked_destinations
18042 && sent.pending_destinations.is_empty()
18043 {
18044 st.end_to_end_reliable_tx.remove(&packet_id)
18045 } else {
18046 None
18047 };
18048 #[cfg(not(feature = "std"))]
18049 if sent.pending_destinations.is_empty() {
18050 st.end_to_end_reliable_tx.remove(&packet_id);
18051 }
18052 #[cfg(feature = "std")]
18053 if completion.is_none()
18054 && st
18055 .end_to_end_reliable_tx
18056 .get(&packet_id)
18057 .is_some_and(|sent| sent.pending_destinations.is_empty())
18058 {
18059 st.end_to_end_reliable_tx.remove(&packet_id);
18060 }
18061 #[cfg(feature = "std")]
18062 let completion_cutoff = completion.as_ref().map_or(0, |_| completion::serial());
18063 drop(st);
18064 #[cfg(feature = "std")]
18065 let release_result = self.release_ordered_successors();
18066 #[cfg(not(feature = "std"))]
18067 self.release_ordered_successors()?;
18068 #[cfg(feature = "std")]
18069 {
18070 if let Some(completed) = completion {
18071 self.retire_completed_writes(packet_id, &completed, completion_cutoff);
18072 }
18073 release_result?;
18074 }
18075 return Ok(targets_local_sender);
18076 }
18077 st.end_to_end_reliable_tx.remove(&packet_id);
18078 drop(st);
18079 self.release_ordered_successors()?;
18080 return Ok(targets_local_sender);
18081 }
18082 return Ok(targets_local_sender);
18085 }
18086
18087 let vals = pkt.data_as_u32()?;
18088 if vals.len() != 2 {
18089 return Err(TelemetryError::Unpack("bad reliable control payload"));
18090 }
18091 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
18092 let seq = vals[1];
18093
18094 match pkt.data_type() {
18095 DataType::ReliableAck => {
18096 self.handle_reliable_ack(src, ty, seq);
18097 Ok(true)
18098 }
18099 DataType::ReliablePartialAck => {
18100 self.handle_reliable_partial_ack(src, ty, seq);
18101 Ok(true)
18102 }
18103 DataType::ReliablePacketRequest => {
18104 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
18105 Ok(true)
18106 }
18107 _ => Ok(false),
18108 }
18109 }
18110
18111 #[cfg(feature = "discovery")]
18116 fn forward_requested_managed_observation(
18117 &self,
18118 item: &RouterRxItem,
18119 duplicate: bool,
18120 ty: DataType,
18121 ) -> TelemetryResult<()> {
18122 let Some(src) = item.src else {
18123 return Ok(());
18124 };
18125 if !self.is_managed_variable_type(ty) || is_internal_control_type(ty) {
18126 return Ok(());
18127 }
18128 let id = Self::get_hash(&item.data);
18129 let untargeted = self.item_target_senders(&item.data)?.is_empty();
18130 let now = self.clock.now_ms();
18131 let priority = self.item_service_priority(&item.data)?;
18132 let mut state = self.state.lock();
18133 let current = state
18134 .local_services
18135 .as_ref()
18136 .and_then(|local| local.managed_variable_latest.get(&ty.as_u32()))
18137 .map(|entry| (entry.packet.packet_id(), entry.sender_address));
18138 for route in state.discovery_routes.values_mut() {
18141 for request in &mut route.requested_network_variables {
18142 if request.ty == ty
18143 && request.refresh_pending
18144 && (now < request.last_seen_ms
18145 || now - request.last_seen_ms > DISCOVERY_ROUTE_TTL_MS
18146 || current != Some((request.value_id, request.writer)))
18147 {
18148 request.refresh_pending = false;
18149 }
18150 }
18151 }
18152 if !duplicate || !untargeted {
18153 return Ok(());
18154 }
18155 loop {
18156 let candidate = state.discovery_routes.iter().find_map(|(&side, route)| {
18157 if side == src
18158 || !Self::side_ref(&state, side).is_ok_and(|link| link.opts.egress_enabled)
18159 {
18160 return None;
18161 }
18162 route
18163 .requested_network_variables
18164 .iter()
18165 .find(|request| {
18166 request.ty == ty
18167 && request.refresh_pending
18168 && request.upstream == src
18169 && request.value_id == id
18170 })
18171 .map(|request| (side, *request))
18172 });
18173 let Some((side, owner)) = candidate else {
18174 break;
18175 };
18176 let owned = state.transmit_queue.iter().any(|queued| {
18180 queued.ignore_local
18181 && queued.delivery_state == QueuedDelivery::Untracked
18182 && queued.disposable & TX_E2E_CHILD == 0
18183 && matches!(&queued.item,
18184 RouterTxItem::ToSide { src: queued_src, dst, data }
18185 if *queued_src == Some(src) && *dst == side
18186 && Self::get_hash(data) == id)
18187 });
18188 if !owned {
18189 state.push_transmit(TxQueued {
18193 disposable: TX_FRESH,
18194 item: RouterTxItem::ToSide {
18195 src: Some(src),
18196 dst: side,
18197 data: item.data.clone(),
18198 },
18199 ignore_local: true,
18200 priority,
18201 delivery_state: QueuedDelivery::Untracked,
18202 accepted_type: 0,
18203 deferred_since_ms: 0,
18204 retry_after_ms: 0,
18205 })?;
18206 }
18207 if let Some(request) = state.discovery_routes.get_mut(&side).and_then(|route| {
18208 route
18209 .requested_network_variables
18210 .iter_mut()
18211 .find(|request| {
18212 request.ty == ty
18213 && request.request_id == owner.request_id
18214 && request.requester == owner.requester
18215 })
18216 }) {
18217 request.refresh_pending = false;
18218 request.next_refresh_ms = now.saturating_add(runtime_reliable_retransmit_ms());
18219 }
18220 }
18221 Ok(())
18222 }
18223
18224 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
18230 let mut _local_admission_guard = None;
18234 let packed_frame = match &item.data {
18235 RouterItem::Packed(bytes) => Some(wire_format::peek_routing_frame_info(bytes.as_ref())),
18236 RouterItem::Packet(_) => None,
18237 };
18238 if crate::memory_admission::supports_work_kinds()
18239 && let (Some(side), RouterItem::Packed(bytes)) = (item.src, &item.data)
18240 {
18241 self.admit_received_frame_with_view(
18242 side,
18243 bytes,
18244 packed_frame.as_ref().and_then(|frame| frame.as_ref().ok()),
18245 )?;
18246 }
18247 let packed_routing = packed_frame
18248 .as_ref()
18249 .and_then(|frame| frame.as_ref().ok())
18250 .map(|frame| &frame.envelope);
18251 if let Some(src) = item.src {
18252 self.ensure_side_ingress_enabled(src)?;
18253 match &item.data {
18254 RouterItem::Packet(pkt) => {
18255 let bytes = wire_format::packet_wire_size(pkt);
18256 self.note_side_rx(src, pkt.data_type(), bytes, true);
18257 }
18258 RouterItem::Packed(bytes) => {
18259 if let Some(env) = packed_routing.as_ref() {
18260 self.note_side_rx(src, env.ty, bytes.len(), true);
18261 }
18262 }
18263 }
18264 match &item.data {
18265 RouterItem::Packet(pkt) => {
18266 if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
18267 && !is_internal_control_type(pkt.data_type())
18268 {
18269 match self.begin_reliable_local_admission(src, pkt.packet_id())? {
18270 Some(guard) => _local_admission_guard = Some(guard),
18271 None => {
18272 self.queue_duplicate_proxy_edge_receipt(item, pkt.packet_id())?;
18273 return Ok(());
18274 }
18275 }
18276 }
18277 }
18278 RouterItem::Packed(bytes) => {
18279 if let Some(env) = packed_routing.as_ref()
18280 && (is_reliable_type(env.ty) || env.target_senders().len() != 0)
18281 && !is_internal_control_type(env.ty)
18282 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
18283 {
18284 match self.begin_reliable_local_admission(src, packet_id)? {
18285 Some(guard) => _local_admission_guard = Some(guard),
18286 None => {
18287 self.queue_duplicate_proxy_edge_receipt(item, packet_id)?;
18288 return Ok(());
18289 }
18290 }
18291 }
18292 }
18293 }
18294 }
18295 match &item.data {
18296 RouterItem::Packet(pkt) => {
18297 if !is_internal_control_type(pkt.data_type()) {
18298 self.cache_managed_variable_packet(pkt, true, item.src)?;
18299 }
18300 }
18301 RouterItem::Packed(bytes) => {
18302 if let Some(env) = packed_routing.as_ref()
18303 && !is_internal_control_type(env.ty)
18304 && self.is_managed_variable_type(env.ty)
18305 {
18306 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18307 pkt.validate()?;
18308 self.cache_managed_variable_packet(&pkt, true, item.src)?;
18309 }
18310 }
18311 }
18312 if let Some(guard) = _local_admission_guard.as_ref() {
18313 guard.check()?;
18314 }
18315 let mut released_buffered: Vec<crate::SharedBytes> = Vec::new();
18316 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
18317 let (_opts, handler_is_packed, hop_reliable_enabled) = {
18318 let st = self.state.lock();
18319 let side_ref = Self::side_ref(&st, src)?;
18320 let opts = side_ref.opts.clone();
18321 (
18322 opts.clone(),
18323 matches!(
18324 side_ref.tx_handler,
18325 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
18326 ),
18327 opts.reliable_enabled
18328 && self.cfg.reliable_enabled()
18329 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
18330 )
18331 };
18332
18333 if hop_reliable_enabled && handler_is_packed {
18334 let frame = match packed_frame
18335 .as_ref()
18336 .expect("packed frame")
18337 .as_ref()
18338 .map_err(Clone::clone)
18339 {
18340 Ok(frame) => frame,
18341 Err(e) => {
18342 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
18343 if let Ok(frame) =
18344 wire_format::peek_frame_info_unchecked(bytes.as_ref())
18345 && is_reliable_type(frame.envelope.ty)
18346 && let Some(hdr) = frame.reliable
18347 {
18348 let unordered =
18349 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
18350 let unsequenced =
18351 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
18352
18353 if !unsequenced {
18354 let requested = if unordered {
18355 hdr.seq
18356 } else {
18357 let mut st = self.state.lock();
18358 let rx_state = self.reliable_rx_state_mut(
18359 &mut st,
18360 src,
18361 frame.envelope.ty,
18362 );
18363 rx_state.expected_seq.min(hdr.seq)
18364 };
18365 self.queue_reliable_packet_request(
18366 src,
18367 frame.envelope.ty,
18368 requested,
18369 called_from_queue,
18370 )?;
18371 }
18372 }
18373 return Ok(());
18374 }
18375 return Err(e);
18376 }
18377 };
18378 if is_reliable_type(frame.envelope.ty)
18379 && let Some(hdr) = frame.reliable
18380 {
18381 if frame.ack_only() {
18382 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
18383 return Ok(());
18384 }
18385 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
18386 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
18387
18388 if !unsequenced {
18389 if unordered {
18390 self.queue_reliable_ack(
18391 src,
18392 frame.envelope.ty,
18393 hdr.seq,
18394 called_from_queue,
18395 )?;
18396 } else {
18397 let mut release: Vec<crate::SharedBytes> = Vec::new();
18398 let mut last_delivered = None;
18399 let mut ack_old = None;
18400 let mut request_missing = None;
18401 let mut partial_ack = None;
18402 {
18403 let mut st = self.state.lock();
18404 let rx_state =
18405 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
18406 let expected_seq = rx_state.expected_seq;
18407 if hdr.seq < expected_seq {
18408 ack_old = Some(expected_seq.saturating_sub(1));
18409 } else if hdr.seq > expected_seq {
18410 request_missing = Some(expected_seq);
18411 partial_ack = Some(hdr.seq);
18412 st.buffer_reliable_rx(
18413 src,
18414 frame.envelope.ty,
18415 hdr.seq,
18416 bytes.clone(),
18417 )?;
18418 } else {
18419 release.push(bytes.clone());
18420 last_delivered = Some(hdr.seq);
18421 let mut next_expected = hdr.seq.wrapping_add(1);
18422 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
18423 release.push(buf);
18424 last_delivered = Some(next_expected);
18425 let next = next_expected.wrapping_add(1);
18426 next_expected = if next == 0 { 1 } else { next };
18427 }
18428 rx_state.expected_seq = next_expected;
18429 }
18430 }
18431
18432 if let Some(ack_seq) = ack_old {
18433 self.queue_reliable_ack(
18434 src,
18435 frame.envelope.ty,
18436 ack_seq,
18437 called_from_queue,
18438 )?;
18439 return Ok(());
18440 }
18441 if let Some(request_seq) = request_missing {
18442 if let Some(partial_seq) = partial_ack {
18443 self.queue_reliable_partial_ack(
18444 src,
18445 frame.envelope.ty,
18446 partial_seq,
18447 called_from_queue,
18448 )?;
18449 }
18450 self.queue_reliable_packet_request(
18451 src,
18452 frame.envelope.ty,
18453 request_seq,
18454 called_from_queue,
18455 )?;
18456 return Ok(());
18457 }
18458
18459 if let Some(ack_seq) = last_delivered {
18460 self.queue_reliable_ack(
18461 src,
18462 frame.envelope.ty,
18463 ack_seq,
18464 called_from_queue,
18465 )?;
18466 }
18467
18468 released_buffered.extend(release.into_iter().skip(1));
18469 }
18470 }
18471 }
18472 } else {
18473 match packed_frame
18474 .as_ref()
18475 .expect("packed frame")
18476 .as_ref()
18477 .map_err(Clone::clone)
18478 {
18479 Ok(frame) => {
18480 if frame.ack_only() {
18481 return Ok(());
18482 }
18483 }
18484 Err(e) => {
18485 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
18486 return Ok(());
18487 }
18488 return Err(e);
18489 }
18490 }
18491 }
18492 }
18493
18494 let duplicate = self.is_duplicate_pkt(&item.data)?;
18495 if duplicate
18496 && let Some(guard) = _local_admission_guard.as_ref()
18497 {
18498 guard.begin_commit(true)?;
18499 }
18500 #[cfg(feature = "discovery")]
18501 {
18502 let ty = match &item.data {
18505 RouterItem::Packet(packet) => packet.data_type(),
18506 RouterItem::Packed(_) => packed_frame
18507 .as_ref()
18508 .expect("packed frame")
18509 .as_ref()
18510 .map_err(Clone::clone)?
18511 .envelope
18512 .ty,
18513 };
18514 self.forward_requested_managed_observation(item, duplicate, ty)?;
18515 }
18516 if duplicate {
18517 #[cfg(feature = "discovery")]
18518 let shared_pending = if item.src.is_some()
18519 && Self::item_data_type(&item.data).is_ok_and(|ty| {
18524 is_reliable_type(ty) && !is_internal_control_type(ty)
18525 })
18526 && self.item_target_senders(&item.data)?.is_empty()
18527 {
18528 let RemoteSidePlan::Target(sides) =
18529 self.remote_side_plan_with_selection(&item.data, item.src, false)?;
18530 let st = self.state.lock();
18531 sides.into_iter().any(|side| {
18532 self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
18533 })
18534 } else {
18535 false
18536 };
18537 #[cfg(not(feature = "discovery"))]
18538 let shared_pending = false;
18539 if shared_pending {
18540 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
18541 }
18542
18543 if !shared_pending
18546 && ((item.src.is_some() && self.item_targets_another_sender(&item.data)?)
18547 || (item.src.is_some() && self.should_route_remote(&item.data, item.src)? && {
18548 let (endpoints, _) = self.item_route_info(&item.data)?;
18549 self.item_targets_router_as_intermediate_hop(&item.data, &endpoints)?
18550 }))
18551 {
18552 let id = Self::get_hash(&item.data);
18553 let replay = {
18554 let mut st = self.state.lock();
18555 st.reliable_return_routes.get_mut(&id).is_some_and(|route| {
18556 let now = self.clock.now_ms();
18557 if now.saturating_sub(route.last_replay_ms)
18558 < runtime_reliable_retransmit_ms()
18559 {
18560 return false;
18561 }
18562 route.last_replay_ms = now;
18563 true
18564 })
18565 };
18566 if replay {
18567 self.relay_send(item.data.clone(), item.src, called_from_queue)?;
18568 }
18569 }
18570 if item.src.is_some() {
18571 let local_sender = self.sender_arc();
18572 match &item.data {
18573 RouterItem::Packet(pkt)
18574 if (is_reliable_type(pkt.data_type())
18575 || !pkt.wire_target_senders().is_empty())
18576 && pkt.sender() != local_sender.as_ref()
18577 && self.item_targets_local_sender(&item.data)?
18578 && self.packet_has_local_delivery(pkt)
18579 && !(self.should_route_remote(&item.data, item.src)?
18580 && self.item_targets_router_as_intermediate_hop(
18581 &item.data,
18582 pkt.endpoints(),
18583 )?) =>
18584 {
18585 self.queue_admitted_end_to_end_ack(
18586 pkt.packet_id(),
18587 pkt.sender(),
18588 called_from_queue,
18589 _local_admission_guard.as_ref(),
18590 )?;
18591 }
18592 RouterItem::Packed(bytes) => {
18593 if let Ok(pkt) = self.unpack_packet_for_role(bytes.as_ref())
18594 && (is_reliable_type(pkt.data_type())
18595 || !pkt.wire_target_senders().is_empty())
18596 && pkt.sender() != local_sender.as_ref()
18597 && self.item_targets_local_sender(&item.data)?
18598 && self.packet_has_local_delivery(&pkt)
18599 && !(self.should_route_remote(&item.data, item.src)?
18600 && self.item_targets_router_as_intermediate_hop(
18601 &item.data,
18602 pkt.endpoints(),
18603 )?)
18604 {
18605 let packet_id = _local_admission_guard
18606 .as_ref()
18607 .map(|guard| guard.packet_id)
18608 .unwrap_or_else(|| {
18609 wire_format::packet_id_from_wire(bytes.as_ref())
18610 .unwrap_or_else(|_| pkt.packet_id())
18611 });
18612 self.queue_admitted_end_to_end_ack(
18613 packet_id,
18614 pkt.sender(),
18615 called_from_queue,
18616 _local_admission_guard.as_ref(),
18617 )?;
18618 }
18619 }
18620 _ => {}
18621 }
18622 }
18623 return Ok(());
18624 }
18625
18626 self.dispatch_rx_data(item, called_from_queue, _local_admission_guard.as_ref())?;
18627 if let Some(guard) = _local_admission_guard.as_ref() {
18628 guard.end_commit();
18629 }
18630
18631 for release_bytes in released_buffered {
18632 let release_data = RouterItem::Packed(release_bytes.clone());
18633 if let Some(guard) = _local_admission_guard.as_ref() {
18634 guard.check()?;
18635 }
18636 let release_id = Self::get_hash(&release_data);
18637 let release_type = wire_format::scheduling_type(release_bytes.as_ref())
18638 .ok_or(TelemetryError::InvalidType)?;
18639 let release_guard = if !is_internal_control_type(release_type)
18640 && let Some(src) = item.src
18641 {
18642 self.begin_reliable_local_admission_with_parent(
18643 src,
18644 release_id,
18645 _local_admission_guard.as_ref(),
18646 )?
18647 } else {
18648 None
18649 };
18650 if self.is_duplicate_packet_id(release_id)? {
18651 continue;
18652 }
18653 let release_item = RouterRxItem {
18654 src: item.src,
18655 priority: Self::router_item_priority(&release_data)?,
18656 data: release_data,
18657 };
18658 self.dispatch_rx_data(&release_item, called_from_queue, release_guard.as_ref())?;
18659 }
18660
18661 Ok(())
18662 }
18663
18664 fn dispatch_rx_data(
18665 &self,
18666 item: &RouterRxItem,
18667 called_from_queue: bool,
18668 admission: Option<&ReliableLocalAdmissionGuard<'_>>,
18669 ) -> TelemetryResult<()> {
18670 match &item.data {
18671 RouterItem::Packet(pkt) => {
18672 pkt.validate()?;
18673 #[cfg(feature = "discovery")]
18674 if let Some(side) = item.src {
18675 self.refresh_peer_liveness(side, sender_address_u32(pkt.sender()));
18676 }
18677
18678 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
18679 return Ok(());
18680 }
18681
18682 if pkt.data_type() == DataType::P2pMessage {
18683 if self.item_targets_local_sender(&item.data)? {
18684 self.dispatch_p2p_packet(pkt)?;
18685 if item.src.is_some() {
18686 self.queue_admitted_end_to_end_ack(
18687 pkt.packet_id(),
18688 pkt.sender(),
18689 called_from_queue,
18690 admission,
18691 )?;
18692 }
18693 }
18694 if self.should_route_remote(&item.data, item.src)? {
18695 self.relay_send(
18696 RouterItem::Packet(pkt.to_owned()),
18697 item.src,
18698 called_from_queue,
18699 )?;
18700 }
18701 return Ok(());
18702 }
18703
18704 #[cfg(feature = "timesync")]
18705 if matches!(
18706 pkt.data_type(),
18707 DataType::TimeSyncAnnounce
18708 | DataType::TimeSyncRequest
18709 | DataType::TimeSyncResponse
18710 ) {
18711 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
18712 return Ok(());
18713 }
18714
18715 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
18716 if !discovery_is_hop_local_advertisement(
18721 pkt.data_type(),
18722 !self.cfg.role.local_services(),
18723 ) && self.should_route_remote(&item.data, item.src)?
18724 {
18725 self.relay_send(
18726 RouterItem::Packet(pkt.to_owned()),
18727 item.src,
18728 called_from_queue,
18729 )?;
18730 }
18731 return Ok(());
18732 }
18733
18734 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
18735 crate::small_sort::sort(&mut eps);
18736 eps.dedup();
18737 let had_local_handler = eps.iter().copied().any(|ep| {
18738 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18739 });
18740 let had_local_delivery =
18741 had_local_handler || self.is_managed_variable_type(pkt.data_type());
18742
18743 let has_remote = self.should_route_remote(&item.data, item.src)?;
18744 let targets_local = self.item_targets_local_sender(&item.data)?;
18745
18746 let has_packed_local = eps
18747 .iter()
18748 .copied()
18749 .any(|ep| self.endpoint_has_packed_handler(ep));
18750 let bytes_opt = if has_packed_local {
18751 Some(wire_format::pack_packet(pkt))
18752 } else {
18753 None
18754 };
18755
18756 let mut local_delivery_error = None;
18757 if targets_local {
18758 for dest in eps {
18759 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
18760 let result = match (&h.handler, &bytes_opt) {
18761 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
18762 .call_handler_with_retries(
18763 dest,
18764 h,
18765 Some(bytes.as_ref()),
18766 Some(pkt),
18767 None,
18768 called_from_queue,
18769 ),
18770 (EndpointHandlerFn::Packed(_), None) => {
18771 let bytes = wire_format::pack_packet(pkt);
18772 self.call_handler_with_retries(
18773 dest,
18774 h,
18775 Some(bytes.as_ref()),
18776 Some(pkt),
18777 None,
18778 called_from_queue,
18779 )
18780 }
18781 (EndpointHandlerFn::Packet(_), _) => self
18782 .call_handler_with_retries(
18783 dest,
18784 h,
18785 None,
18786 Some(pkt),
18787 None,
18788 called_from_queue,
18789 ),
18790 };
18791 if let Err(error) = result {
18792 if let Some(src) = item.src {
18793 self.note_side_local_handler_failure(
18794 src,
18795 pkt.data_type(),
18796 runtime_max_handler_retries(),
18797 );
18798 }
18799 local_delivery_error.get_or_insert(error);
18800 }
18801 }
18802 }
18803 }
18804
18805 if let Some(guard) = admission {
18806 guard.begin_commit(local_delivery_error.is_none())?;
18807 }
18808
18809 if let Some(src) = item.src
18810 && had_local_delivery
18811 && local_delivery_error.is_none()
18812 && targets_local
18813 {
18814 self.note_side_local_delivery(src, pkt.data_type());
18815 }
18816
18817 if item.src.is_some()
18818 && had_local_delivery
18819 && local_delivery_error.is_none()
18820 && targets_local
18821 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
18822 && !(has_remote
18823 && self
18824 .item_targets_router_as_intermediate_hop(&item.data, pkt.endpoints())?)
18825 {
18826 self.queue_admitted_end_to_end_ack(
18827 pkt.packet_id(),
18828 pkt.sender(),
18829 called_from_queue,
18830 admission,
18831 )?;
18832 }
18833
18834 if let Some(guard) = admission {
18835 guard.check()?;
18836 }
18837 if has_remote {
18838 let relay_item = RouterItem::Packet(pkt.to_owned());
18839 self.relay_send(relay_item, item.src, called_from_queue)?;
18840 }
18841
18842 local_delivery_error.map_or(Ok(()), Err)
18846 }
18847 RouterItem::Packed(bytes) => {
18848 let routing = wire_format::peek_routing_envelope(bytes.as_ref())?;
18852 if !is_internal_control_type(routing.ty)
18853 && !routing.is_reliable
18854 && routing.target_senders().len() == 0
18855 && !self.is_managed_variable_type(routing.ty)
18856 && !routing.endpoints().any(|ep| {
18857 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18858 })
18859 {
18860 return self.relay_send(item.data.clone(), item.src, called_from_queue);
18864 }
18865
18866 let env = wire_format::peek_envelope(bytes.as_ref())?;
18867
18868 if matches!(
18869 env.ty,
18870 DataType::ReliableAck
18871 | DataType::ReliablePartialAck
18872 | DataType::ReliablePacketRequest
18873 ) {
18874 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18875 pkt.validate()?;
18876 let handled =
18877 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
18878 if !handled && self.should_route_remote(&item.data, item.src)? {
18881 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18882 }
18883 return Ok(());
18884 }
18885
18886 if env.ty == DataType::P2pMessage {
18887 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18888 pkt.validate()?;
18889 if self.item_targets_local_sender(&item.data)? {
18890 self.dispatch_p2p_packet(&pkt)?;
18891 if item.src.is_some() {
18892 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
18893 .unwrap_or_else(|_| pkt.packet_id());
18894 self.queue_end_to_end_reliable_ack_for_packet_id(
18895 packet_id,
18896 pkt.sender(),
18897 called_from_queue,
18898 )?;
18899 }
18900 }
18901 if self.should_route_remote(&item.data, item.src)? {
18902 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18903 }
18904 return Ok(());
18905 }
18906
18907 #[cfg(feature = "timesync")]
18908 if matches!(
18909 env.ty,
18910 DataType::TimeSyncAnnounce
18911 | DataType::TimeSyncRequest
18912 | DataType::TimeSyncResponse
18913 ) {
18914 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18915 pkt.validate()?;
18916 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
18917 return Ok(());
18918 }
18919
18920 #[cfg(feature = "discovery")]
18921 if discovery::is_discovery_type(env.ty) {
18922 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18923 pkt.validate()?;
18924 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
18925 if !discovery_is_hop_local_advertisement(
18926 env.ty,
18927 !self.cfg.role.local_services(),
18928 ) && self.should_route_remote(&item.data, item.src)?
18929 {
18930 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18931 }
18932 return Ok(());
18933 }
18934
18935 let any_packet_needed = env
18936 .endpoints
18937 .iter()
18938 .copied()
18939 .any(|ep| self.endpoint_has_packet_handler(ep));
18940
18941 let mut pkt_opt = if any_packet_needed {
18942 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18943 pkt.validate()?;
18944 Some(pkt)
18945 } else {
18946 None
18947 };
18948
18949 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
18950 crate::small_sort::sort(&mut eps);
18951 eps.dedup();
18952 let had_local_handler = eps.iter().copied().any(|ep| {
18953 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18954 });
18955 let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
18956
18957 let has_remote = self.should_route_remote(&item.data, item.src)?;
18958 let targets_local = self.item_targets_local_sender(&item.data)?;
18959
18960 let mut local_delivery_error = None;
18961 if targets_local {
18962 for dest in eps {
18963 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
18964 let result = match &h.handler {
18965 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
18966 dest,
18967 h,
18968 Some(bytes.as_ref()),
18969 pkt_opt.as_ref(),
18970 Some(&env),
18971 called_from_queue,
18972 ),
18973 EndpointHandlerFn::Packet(_) => {
18974 if pkt_opt.is_none() {
18975 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18976 pkt.validate()?;
18977 pkt_opt = Some(pkt);
18978 }
18979 let pkt_ref = pkt_opt.as_ref().expect("just set");
18980 self.call_handler_with_retries(
18981 dest,
18982 h,
18983 None,
18984 Some(pkt_ref),
18985 Some(&env),
18986 called_from_queue,
18987 )
18988 }
18989 };
18990 if let Err(error) = result {
18991 if let Some(src) = item.src {
18992 self.note_side_local_handler_failure(
18993 src,
18994 env.ty,
18995 runtime_max_handler_retries(),
18996 );
18997 }
18998 local_delivery_error.get_or_insert(error);
18999 }
19000 }
19001 }
19002 }
19003
19004 if let Some(guard) = admission {
19005 guard.begin_commit(local_delivery_error.is_none())?;
19006 }
19007
19008 if item.src.is_some()
19009 && had_local_delivery
19010 && local_delivery_error.is_none()
19011 && targets_local
19012 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
19013 && !(has_remote
19014 && self
19015 .item_targets_router_as_intermediate_hop(&item.data, &env.endpoints)?)
19016 {
19017 let packet_id = if let Some(guard) = admission {
19018 guard.packet_id
19019 } else {
19020 wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
19021 pkt_opt
19022 .as_ref()
19023 .map(Packet::packet_id)
19024 .ok_or(TelemetryError::Unpack("missing packet id"))
19025 })?
19026 };
19027 self.queue_admitted_end_to_end_ack(
19028 packet_id,
19029 &env.sender,
19030 called_from_queue,
19031 admission,
19032 )?;
19033 }
19034
19035 if let Some(guard) = admission {
19036 guard.check()?;
19037 }
19038 if has_remote {
19039 let relay_item = match pkt_opt {
19040 Some(ref p) => RouterItem::Packet(p.clone()),
19041 None => RouterItem::Packed(bytes.clone()),
19042 };
19043 self.relay_send(relay_item, item.src, called_from_queue)?;
19044 }
19045
19046 local_delivery_error.map_or(Ok(()), Err)
19050 }
19051 }
19052 }
19053
19054 fn dispatch_local_for_item(
19055 &self,
19056 item: &RouterItem,
19057 called_from_queue: bool,
19058 ) -> TelemetryResult<()> {
19059 match item {
19060 RouterItem::Packet(pkt) => {
19061 pkt.validate()?;
19062 if is_internal_control_type(pkt.data_type()) {
19063 return Ok(());
19064 }
19065 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
19066 if !self.item_targets_local_sender(item)? {
19067 return Ok(());
19068 }
19069
19070 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
19071 crate::small_sort::sort(&mut eps);
19072 eps.dedup();
19073
19074 let has_packed_local = eps
19075 .iter()
19076 .copied()
19077 .any(|ep| self.endpoint_has_packed_handler(ep));
19078 let bytes_opt = if has_packed_local {
19079 Some(wire_format::pack_packet(pkt))
19080 } else {
19081 None
19082 };
19083
19084 for dest in eps {
19085 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
19086 match (&h.handler, &bytes_opt) {
19087 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
19088 self.call_handler_with_retries(
19089 dest,
19090 h,
19091 Some(bytes.as_ref()),
19092 Some(pkt),
19093 None,
19094 called_from_queue,
19095 )?;
19096 }
19097 (EndpointHandlerFn::Packed(_), None) => {
19098 let bytes = wire_format::pack_packet(pkt);
19099 self.call_handler_with_retries(
19100 dest,
19101 h,
19102 Some(bytes.as_ref()),
19103 Some(pkt),
19104 None,
19105 called_from_queue,
19106 )?;
19107 }
19108 (EndpointHandlerFn::Packet(_), _) => {
19109 self.call_handler_with_retries(
19110 dest,
19111 h,
19112 None,
19113 Some(pkt),
19114 None,
19115 called_from_queue,
19116 )?;
19117 }
19118 }
19119 }
19120 }
19121 }
19122 RouterItem::Packed(bytes) => {
19123 let env = wire_format::peek_envelope(bytes.as_ref())?;
19124 if is_internal_control_type(env.ty) {
19125 return Ok(());
19126 }
19127 self.ensure_e2e_policy_supported_for_type(env.ty)?;
19128 if !self.item_targets_local_sender(item)? {
19129 return Ok(());
19130 }
19131
19132 let any_packet_needed = env
19133 .endpoints
19134 .iter()
19135 .copied()
19136 .any(|ep| self.endpoint_has_packet_handler(ep));
19137
19138 let mut pkt_opt = if any_packet_needed {
19139 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
19140 pkt.validate()?;
19141 Some(pkt)
19142 } else {
19143 None
19144 };
19145
19146 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
19147 crate::small_sort::sort(&mut eps);
19148 eps.dedup();
19149
19150 for dest in eps {
19151 for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
19152 match &h.handler {
19153 EndpointHandlerFn::Packed(_) => {
19154 self.call_handler_with_retries(
19155 dest,
19156 h,
19157 Some(bytes.as_ref()),
19158 pkt_opt.as_ref(),
19159 Some(&env),
19160 called_from_queue,
19161 )?;
19162 }
19163 EndpointHandlerFn::Packet(_) => {
19164 if pkt_opt.is_none() {
19165 let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
19166 pkt.validate()?;
19167 pkt_opt = Some(pkt);
19168 }
19169 let pkt_ref = pkt_opt.as_ref().expect("just set");
19170 self.call_handler_with_retries(
19171 dest,
19172 h,
19173 None,
19174 Some(pkt_ref),
19175 Some(&env),
19176 called_from_queue,
19177 )?;
19178 }
19179 }
19180 }
19181 }
19182 }
19183 }
19184
19185 Ok(())
19186 }
19187
19188 fn tx_item_impl(
19194 &self,
19195 item: RouterTxItem,
19196 ignore_local: bool,
19197 called_from_queue: bool,
19198 ) -> TelemetryResult<()> {
19199 self.tx_item_impl_with_retry(
19200 item,
19201 ignore_local,
19202 called_from_queue,
19203 &mut DispatchContext::default(),
19204 )
19205 }
19206
19207 fn tx_item_impl_with_retry(
19208 &self,
19209 item: RouterTxItem,
19210 ignore_local: bool,
19211 called_from_queue: bool,
19212 context: &mut DispatchContext,
19213 ) -> TelemetryResult<()> {
19214 #[cfg(feature = "std")]
19215 let _clear_retry_intent = self.retry_queued_intent(context.claim);
19216 match item {
19217 RouterTxItem::Broadcast(data) => {
19218 let claim = context.claim;
19219 let deferred_since = context.since;
19220 let ty = Self::item_data_type(&data)?;
19221
19222 if let Some(since) = deferred_since
19223 && let Err(error) = self.expire_deferred_original(
19224 ty,
19225 since,
19226 claim.is_some_and(|claim| claim.ordered),
19227 )
19228 {
19229 context.mark_terminal();
19230 return Err(error);
19231 }
19232 if !called_from_queue
19233 && !ignore_local
19234 && matches!(reliable_mode(ty), crate::ReliableMode::Ordered)
19235 && self
19236 .state
19237 .lock()
19238 .transmit_queue
19239 .iter()
19240 .any(|queued| RouterInner::ordered_original_type(queued) == Some(ty))
19241 {
19242 return self.tx_queue_item(RouterTxItem::Broadcast(data));
19243 }
19244
19245 self.ensure_e2e_policy_supported_for_type(ty)?;
19246 if let RouterItem::Packet(pkt) = &data
19247 && !is_internal_control_type(pkt.data_type())
19248 {
19249 self.remember_managed_variable_packet(pkt)?;
19250 }
19251 #[cfg(feature = "discovery")]
19252 let is_discovery = discovery::is_discovery_type(ty);
19253 if !ignore_local {
19254 if self.is_duplicate_pkt(&data)? {
19255 return Ok(());
19256 }
19257 #[cfg(feature = "discovery")]
19258 if !is_discovery
19259 && !is_internal_control_type(ty)
19260 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
19261 {
19262 if claim.is_some() {
19264 context.mark_terminal();
19265 return Err(TelemetryError::HandlerError(
19266 "accepted local delivery failed",
19267 ));
19268 }
19269 return Err(error);
19270 }
19271 #[cfg(not(feature = "discovery"))]
19272 if !is_internal_control_type(ty)
19273 && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
19274 {
19275 if claim.is_some() {
19277 context.mark_terminal();
19278 return Err(TelemetryError::HandlerError(
19279 "accepted local delivery failed",
19280 ));
19281 }
19282 return Err(error);
19283 }
19284 }
19285
19286 if !ignore_local && !self.mark_claim_local_delivered(claim) {
19287 return Ok(());
19289 }
19290
19291 if !self.claim_schema_is_current(ty, claim)? {
19292 return Ok(());
19293 }
19294
19295 match &data {
19296 RouterItem::Packet(packet) => packet.validate()?,
19297 RouterItem::Packed(_) => {}
19299 }
19300 let freezes_contract = !is_internal_control_type(ty)
19303 && self.item_requires_end_to_end_tracking(&data, ty)?;
19304 #[cfg(feature = "discovery")]
19310 let cached_plan = if freezes_contract {
19311 None
19312 } else {
19313 Some(self.remote_side_plan(&data, None)?)
19314 };
19315 #[cfg(feature = "discovery")]
19316 let send_remote = match &cached_plan {
19317 Some(RemoteSidePlan::Target(sides)) => !sides.is_empty(),
19318 None => self.should_route_remote(&data, None)?,
19319 };
19320 #[cfg(not(feature = "discovery"))]
19321 let send_remote = self.should_route_remote(&data, None)?;
19322
19323 if !send_remote {
19324 let (endpoints, ty) = self.item_route_info(&data)?;
19325 if !is_internal_control_type(ty)
19326 && is_reliable_type(ty)
19327 && has_nonlocal_endpoint(&endpoints, &self.cfg)
19328 && !self.state.lock().is_managed_variable(ty.as_u32())
19329 {
19330 self.remove_pkt_id(&data);
19334 if called_from_queue {
19335 return self.defer_accepted_original(context);
19336 }
19337 return Err(TelemetryError::Io("no discovered route"));
19338 }
19339 return Ok(());
19340 }
19341 #[cfg(feature = "discovery")]
19342 let mut data = data;
19343 if freezes_contract {
19344 #[cfg(feature = "discovery")]
19345 {
19346 let pending = {
19347 let st = self.state.lock();
19348 let mut pending = self.expected_end_to_end_destinations_locked(
19349 &st, &data, None, false, false,
19350 )?;
19351 self.filter_trackable_end_to_end_destinations_locked(
19352 &st,
19353 ty,
19354 &mut pending,
19355 false,
19356 );
19357 pending
19358 };
19359 if !pending.is_empty() {
19360 let targets: Vec<u64> = pending.keys().copied().collect();
19362 data = self.attach_wire_contract_to_item(data, &targets)?;
19363 match self.register_end_to_end_reliable_tx(&data, Some(pending)) {
19364 Ok(ready) => {
19365 context.mark_owner_transferred();
19366 if !ready {
19367 return Ok(());
19368 }
19369 }
19370 Err(error) => {
19371 self.remove_pkt_id(&data);
19374 if called_from_queue
19375 && matches!(
19376 error,
19377 TelemetryError::Io(
19378 "reliable window full" | "memory pressure"
19379 )
19380 )
19381 {
19382 if let Some(claim) = claim {
19386 self.replace_claim_payload(claim, data)?;
19387 }
19388 self.defer_accepted_original(context)?;
19389 return Ok(());
19390 }
19391 return Err(error);
19392 }
19393 }
19394 }
19395 }
19396 }
19397 #[cfg(feature = "discovery")]
19398 let plan = match cached_plan {
19399 Some(plan) => plan,
19400 None => self.remote_side_plan(&data, None)?,
19401 };
19402 #[cfg(not(feature = "discovery"))]
19403 let plan = self.remote_side_plan(&data, None)?;
19404 let RemoteSidePlan::Target(sides) = plan;
19405 let e2e_parent = context.owner_transferred().then(|| Self::get_hash(&data));
19406 #[cfg(feature = "std")]
19407 let _retry_intent = self.retry_intent(e2e_parent, false);
19408 let _parent_lease = self.lease_e2e_send(e2e_parent);
19409 for side in sides.iter().copied() {
19410 #[cfg(feature = "latest-value-egress")]
19411 if self.queue_latest_egress(side, None, &data)? {
19412 continue;
19413 }
19414 #[cfg(feature = "compression")]
19415 let mut side_data = data.clone();
19416 #[cfg(feature = "compression")]
19417 let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
19418 side,
19419 &mut side_data,
19420 false,
19421 &mut context.admitted,
19422 );
19423 #[cfg(not(feature = "compression"))]
19424 let send_result = self.send_reliable_to_side_with_progress(
19425 side,
19426 data.clone(),
19427 false,
19428 &mut context.admitted,
19429 );
19430 if let Err(e) = send_result {
19431 if Self::is_transient_tx_error(&e) {
19432 context.mark_refused();
19433 if let Some(parent) = e2e_parent {
19434 self.enqueue_e2e_child(
19435 RouterTxItem::ToSide {
19436 src: None,
19437 dst: side,
19438 #[cfg(feature = "compression")]
19439 data: side_data,
19440 #[cfg(not(feature = "compression"))]
19441 data: data.clone(),
19442 },
19443 Self::router_item_priority(&data)?,
19444 parent,
19445 #[cfg(feature = "compression")]
19446 &mut reservation,
19447 )?;
19448 } else {
19449 #[cfg(feature = "compression")]
19450 self.retry_codec_owner(
19451 RouterTxItem::ToSide {
19452 src: None,
19453 dst: side,
19454 data: side_data,
19455 },
19456 true,
19457 &mut reservation,
19458 )?;
19459 #[cfg(not(feature = "compression"))]
19460 self.tx_queue_item_for_retry(
19461 RouterTxItem::ToSide {
19462 src: None,
19463 dst: side,
19464 data: data.clone(),
19465 },
19466 true,
19467 )?;
19468 }
19469 continue;
19470 }
19471 match &data {
19472 RouterItem::Packet(pkt) => {
19473 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19474 }
19475 RouterItem::Packed(bytes) => {
19476 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19477 let _ = self.handle_callback_error_from_env(
19478 &env,
19479 None,
19480 e,
19481 called_from_queue,
19482 );
19483 }
19484 }
19485 }
19486 return Err(TelemetryError::HandlerError("tx handler failed"));
19487 }
19488 }
19489 }
19490 RouterTxItem::ToSide { src, dst, data } => {
19491 let egress_service = context.egress_service();
19492 #[cfg(feature = "compression")]
19493 let mut data = data;
19494 let ty = Self::item_data_type(&data)?;
19495 self.ensure_e2e_policy_supported_for_type(ty)?;
19496 if let RouterItem::Packet(pkt) = &data
19497 && !is_internal_control_type(pkt.data_type())
19498 {
19499 self.remember_managed_variable_packet(pkt)?;
19500 }
19501 if !ignore_local {
19502 if self.is_duplicate_pkt(&data)? {
19503 return Ok(());
19504 }
19505 let suppress_local = is_internal_control_type(ty);
19506 if !suppress_local {
19507 self.dispatch_local_for_item(&data, called_from_queue)?;
19508 }
19509 }
19510 let allowed = {
19511 let st = self.state.lock();
19512 let ty = Some(ty);
19513 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
19514 #[cfg(all(feature = "discovery", feature = "timesync"))]
19515 let timesync_allowed = ty
19516 .map(|ty| {
19517 Self::timesync_allowed_for_side_locked(
19518 &st,
19519 dst,
19520 ty,
19521 self.clock.now_ms(),
19522 )
19523 })
19524 .unwrap_or(true);
19525 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
19526 let timesync_allowed = true;
19527 route_allowed && timesync_allowed
19528 };
19529 if !allowed {
19530 #[cfg(feature = "discovery")]
19531 if ty == DataType::DiscoverySchema {
19532 RouterInner::schema_response_owner_lost(
19533 &mut self.state.lock().discovery_side_throttle,
19534 Some((dst, Self::get_hash(&data))),
19535 );
19536 }
19537 return Ok(());
19538 }
19539 #[cfg(feature = "compression")]
19540 let (send_result, mut reservation) =
19541 if let Some(claim) = context.claim.filter(|claim| claim.side_retry) {
19542 let prepared = self.prepare_e2e_child_for_send(claim, dst, &mut data);
19543 (
19544 prepared.and_then(|()| {
19545 self.send_reliable_to_side_with_service(
19546 dst,
19547 data.clone(),
19548 src.is_some(),
19549 &mut context.admitted,
19550 egress_service,
19551 )
19552 }),
19553 None,
19554 )
19555 } else {
19556 self.send_queued_to_side_with_service(
19557 dst,
19558 &mut data,
19559 src.is_some(),
19560 &mut context.admitted,
19561 egress_service,
19562 )
19563 };
19564 #[cfg(not(feature = "compression"))]
19565 let send_result = self.send_reliable_to_side_with_service(
19566 dst,
19567 data.clone(),
19568 src.is_some(),
19569 &mut context.admitted,
19570 egress_service,
19571 );
19572 if let Err(e) = send_result {
19573 if Self::is_transient_tx_error(&e) {
19574 context.mark_refused();
19575 if let Some(claim) = context.claim.filter(|claim| claim.side_retry) {
19576 self.restore_e2e_child_claim(claim)?;
19577 return Ok(());
19578 }
19579 let retry_priority =
19580 context.priority.max(self.item_service_priority(&data)?);
19581 let (turn_flags, immediate) = self.current_egress_requeue(context);
19582 #[cfg(feature = "compression")]
19583 self.retry_codec_owner_at_priority(
19584 RouterTxItem::ToSide { src, dst, data },
19585 true,
19586 &mut reservation,
19587 retry_priority,
19588 turn_flags,
19589 immediate,
19590 )?;
19591 #[cfg(not(feature = "compression"))]
19592 self.tx_queue_item_for_retry_at_priority_with_turn(
19593 RouterTxItem::ToSide { src, dst, data },
19594 true,
19595 retry_priority,
19596 turn_flags,
19597 immediate,
19598 )?;
19599 return Ok(());
19600 }
19601 if !self.cfg.role.local_services() {
19602 return Err(e);
19603 }
19604 match &data {
19605 RouterItem::Packet(pkt) => {
19606 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19607 }
19608 RouterItem::Packed(bytes) => {
19609 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19610 let _ = self.handle_callback_error_from_env(
19611 &env,
19612 None,
19613 e,
19614 called_from_queue,
19615 );
19616 }
19617 }
19618 }
19619 return Err(TelemetryError::HandlerError("tx handler failed"));
19620 }
19621 }
19622 RouterTxItem::EndToEndReplay { packet_id } => {
19623 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
19624 return Ok(());
19625 };
19626 if sides.is_empty() {
19627 let RemoteSidePlan::Target(fallback_sides) =
19628 self.remote_side_plan(&data, None)?;
19629 sides = fallback_sides;
19630 }
19631 #[cfg(feature = "std")]
19632 let _retry_intent = self.retry_intent(Some(packet_id), true);
19633 let _parent_lease = self.lease_e2e_send(Some(packet_id));
19634 {
19635 let st = self.state.lock();
19636 sides.retain(|side| {
19637 !Self::e2e_child_owned(&st.transmit_queue, *side, packet_id)
19638 });
19639 }
19640 for side in sides.iter().copied() {
19641 #[cfg(feature = "compression")]
19642 let mut side_data = data.clone();
19643 #[cfg(feature = "compression")]
19644 let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
19645 side,
19646 &mut side_data,
19647 false,
19648 &mut context.admitted,
19649 );
19650 #[cfg(not(feature = "compression"))]
19651 let send_result = self.send_reliable_to_side_with_progress(
19652 side,
19653 data.clone(),
19654 false,
19655 &mut context.admitted,
19656 );
19657 if let Err(e) = send_result {
19658 if Self::is_transient_tx_error(&e) {
19659 context.mark_refused();
19660 let retry_priority =
19661 Self::router_item_priority_bumped(DataType::ReliableAck);
19662 self.enqueue_e2e_child(
19663 RouterTxItem::ToSide {
19664 src: None,
19665 dst: side,
19666 #[cfg(feature = "compression")]
19667 data: side_data,
19668 #[cfg(not(feature = "compression"))]
19669 data: data.clone(),
19670 },
19671 retry_priority,
19672 packet_id,
19673 #[cfg(feature = "compression")]
19674 &mut reservation,
19675 )?;
19676 continue;
19677 }
19678 match &data {
19679 RouterItem::Packet(pkt) => {
19680 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19681 }
19682 RouterItem::Packed(bytes) => {
19683 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19684 let _ = self.handle_callback_error_from_env(
19685 &env,
19686 None,
19687 e,
19688 called_from_queue,
19689 );
19690 }
19691 }
19692 }
19693 return Err(TelemetryError::HandlerError("tx handler failed"));
19694 }
19695 }
19696 }
19697 RouterTxItem::ReliableReplay {
19698 dst,
19699 ty,
19700 seq,
19701 bytes,
19702 } => {
19703 {
19704 let mut st = self.state.lock();
19705 let Some(sent) = st
19706 .reliable_tx
19707 .get_mut(&Self::reliable_key(dst, ty))
19708 .and_then(|tx| tx.sent.get_mut(&seq))
19709 else {
19710 return Ok(());
19711 };
19712 sent.queued = false;
19713 sent.last_send_ms = self.clock.now_ms();
19714 }
19715 if let Err(e) =
19716 self.send_reliable_raw_to_side(dst, bytes.clone(), false, &mut context.admitted)
19717 {
19718 if Self::is_transient_tx_error(&e) {
19719 context.mark_refused();
19720 self.tx_queue_item_for_retry(
19721 RouterTxItem::ReliableReplay {
19722 dst,
19723 ty,
19724 seq,
19725 bytes,
19726 },
19727 true,
19728 )?;
19729 return Ok(());
19730 }
19731 return Err(e);
19732 }
19733 let mut st = self.state.lock();
19734 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
19735 if let Some(sent) = tx_state.sent.get_mut(&seq) {
19736 sent.last_send_ms = self.clock.now_ms();
19737 }
19738 }
19739 }
19740
19741 Ok(())
19742 }
19743
19744 #[inline]
19746 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
19747 self.tx_item_impl(item, false, false)
19748 }
19749
19750 #[inline]
19757 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
19758 if self.side_tx_active() {
19759 return self.rx_packed_queue(bytes);
19760 }
19761 crate::memory_admission::check_receive(bytes)?;
19762 let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
19763 let item = RouterRxItem {
19764 src: None,
19765 priority: Self::router_item_priority(&data)?,
19766 data,
19767 };
19768 self.rx_item(&item, false)
19769 }
19770
19771 #[inline]
19776 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
19777 if self.side_tx_active() {
19778 return self.rx_queue(pkt.clone());
19779 }
19780 let data = RouterItem::Packet(pkt.clone());
19781 let item = RouterRxItem {
19782 src: None,
19783 priority: Self::router_item_priority(&data)?,
19784 data,
19785 };
19786 self.rx_item(&item, false)
19787 }
19788
19789 #[inline]
19794 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
19795 if self.side_tx_active() {
19796 return self.rx_queue_from_side(pkt.clone(), side);
19797 }
19798 self.ensure_side_ingress_enabled(side)?;
19799 let data = RouterItem::Packet(pkt.clone());
19800 let item = RouterRxItem {
19801 src: Some(side),
19802 priority: Self::router_item_priority(&data)?,
19803 data,
19804 };
19805 self.rx_item(&item, false)
19806 }
19807
19808 #[inline]
19813 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
19814 if self.side_tx_active() {
19815 return self.rx_packed_queue_from_side(bytes, side);
19816 }
19817 self.ensure_side_ingress_enabled(side)?;
19818 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
19819 return Ok(());
19820 };
19821 let data = RouterItem::Packed(decoded);
19822 let item = RouterRxItem {
19823 src: Some(side),
19824 priority: Self::router_item_priority(&data)?,
19825 data,
19826 };
19827 self.rx_item(&item, false)
19828 }
19829
19830 fn ensure_application_origin_allowed(&self) -> TelemetryResult<()> {
19831 if self.cfg.role.local_services() {
19832 Ok(())
19833 } else {
19834 Err(TelemetryError::PermissionDenied)
19835 }
19836 }
19837
19838 #[cfg(all(test, feature = "discovery"))]
19839 pub(crate) fn debug_end_to_end_acked_destination_count(&self, id: u64) -> Option<usize> {
19840 self.state
19841 .lock()
19842 .forwarded_receipts
19843 .as_ref()?
19844 .destinations
19845 .get(&id)
19846 .map(BTreeSet::len)
19847 }
19848 #[cfg(all(test, feature = "discovery"))]
19849 pub(crate) fn debug_end_to_end_acked_packet_count(&self) -> usize {
19850 self.state
19851 .lock()
19852 .forwarded_receipts
19853 .as_ref()
19854 .map_or(0, |cache| cache.destinations.len())
19855 }
19856
19857 #[inline]
19864 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
19865 self.ensure_application_origin_allowed()?;
19866 #[cfg(feature = "discovery")]
19867 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19868 if self.side_tx_active() {
19869 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
19870 }
19871 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
19872 }
19873
19874 #[inline]
19879 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
19880 self.ensure_application_origin_allowed()?;
19881 #[cfg(feature = "discovery")]
19882 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19883 if self.side_tx_active() {
19884 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
19885 crate::shared_bytes::convert(pkt),
19886 )));
19887 }
19888 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(
19889 crate::shared_bytes::convert(pkt),
19890 )))
19891 }
19892
19893 #[inline]
19899 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
19900 self.ensure_application_origin_allowed()?;
19901 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
19902 }
19903
19904 #[inline]
19906 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
19907 self.ensure_application_origin_allowed()?;
19908 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
19909 crate::shared_bytes::convert(data),
19910 )))
19911 }
19912
19913 #[inline(never)]
19919 fn log_dispatch<T: LeBytes>(
19920 &self,
19921 ty: DataType,
19922 data: &[T],
19923 timestamp: Option<u64>,
19924 mut queued: bool,
19925 ) -> TelemetryResult<()> {
19926 self.ensure_application_origin_allowed()?;
19927 if !queued {
19928 #[cfg(feature = "discovery")]
19929 let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19930 queued = self.side_tx_active();
19931 }
19932 let timestamp = timestamp.unwrap_or_else(|| self.packet_timestamp_ms());
19933 log_raw(self.sender_arc(), ty, data, timestamp, |pkt| {
19934 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
19935 if queued {
19936 self.tx_queue_item(item)
19937 } else {
19938 self.tx_item(item)
19939 }
19940 })
19941 }
19942
19943 #[inline]
19946 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
19947 self.log_dispatch(ty, data, None, false)
19948 }
19949
19950 #[inline]
19952 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
19953 self.log_dispatch(ty, data, None, true)
19954 }
19955
19956 #[inline]
19958 pub fn log_ts<T: LeBytes>(
19959 &self,
19960 ty: DataType,
19961 timestamp: u64,
19962 data: &[T],
19963 ) -> TelemetryResult<()> {
19964 self.log_dispatch(ty, data, Some(timestamp), false)
19965 }
19966
19967 #[inline]
19969 pub fn log_queue_ts<T: LeBytes>(
19970 &self,
19971 ty: DataType,
19972 timestamp: u64,
19973 data: &[T],
19974 ) -> TelemetryResult<()> {
19975 self.log_dispatch(ty, data, Some(timestamp), true)
19976 }
19977}
19978
19979#[cfg(all(test, feature = "std", feature = "discovery"))]
19980mod queued_schema_identity_tests {
19981 use super::*;
19982
19983 fn catalog(timestamp: u64, byte: u8) -> Packet {
19984 Packet::new(
19985 crate::DataType::DiscoverySchema,
19986 &[DataEndpoint::Discovery],
19987 "SCHEMA_PEER",
19988 timestamp,
19989 crate::shared_bytes::convert(Arc::<[u8]>::from(vec![byte; 1024])),
19990 )
19991 .unwrap()
19992 }
19993
19994 #[test]
19995 fn inline_and_large_typed_publications_share_headers_and_keep_le_bytes() {
19996 crate::tests::ensure_common_test_schema();
19997 let endpoint = DataEndpoint::named("SD_CARD");
19998 let ty = crate::config::register_data_type_with_description(
19999 "INLINE_TYPED_BOUNDARY",
20000 "",
20001 MessageElement::Dynamic(crate::MessageDataType::UInt32, crate::MessageClass::Data),
20002 &[endpoint],
20003 crate::ReliableMode::None,
20004 16,
20005 )
20006 .unwrap();
20007 let sender: Arc<str> = Arc::from("DAQ_SOURCE");
20008 let endpoints = message_meta(ty);
20009 for count in [
20010 0,
20011 1,
20012 crate::config::STACK_PAYLOAD_SIZE / 4,
20013 crate::config::STACK_PAYLOAD_SIZE / 4 + 1,
20014 64,
20015 ] {
20016 let values = (0..count)
20017 .map(|i| 0x1234_5678u32.wrapping_add(i as u32))
20018 .collect::<Vec<_>>();
20019 log_raw(sender.clone(), ty, &values, 42, |packet| {
20020 let bytes = values
20021 .iter()
20022 .flat_map(|v| v.to_le_bytes())
20023 .collect::<Vec<_>>();
20024 assert_eq!(packet.payload(), bytes);
20025 assert_eq!(packet.sender().as_ptr(), sender.as_ptr());
20026 assert_eq!(
20027 packet.endpoints().as_ptr(),
20028 endpoints.endpoints_ref().as_ptr()
20029 );
20030 packet.validate().unwrap();
20031 Ok(())
20032 })
20033 .unwrap();
20034 }
20035 #[derive(Clone, Copy)]
20036 struct BadWidth;
20037 impl LeBytes for BadWidth {
20038 const WIDTH: usize = 0;
20039 fn write_le(self, _: &mut [u8]) {
20040 unreachable!()
20041 }
20042 fn from_le_slice(_: &[u8]) -> Self {
20043 Self
20044 }
20045 }
20046 assert!(matches!(
20047 log_raw(sender.clone(), ty, &[BadWidth], 42, |_| Ok(())),
20048 Err(TelemetryError::BadArg)
20049 ));
20050 #[derive(Clone, Copy)]
20051 struct HugeWidth;
20052 impl LeBytes for HugeWidth {
20053 const WIDTH: usize = usize::MAX;
20054 fn write_le(self, _: &mut [u8]) {
20055 unreachable!()
20056 }
20057 fn from_le_slice(_: &[u8]) -> Self {
20058 Self
20059 }
20060 }
20061 assert!(matches!(
20062 log_raw(sender, ty, &[HugeWidth, HugeWidth], 42, |_| Ok(())),
20063 Err(TelemetryError::PacketTooLarge(_))
20064 ));
20065 }
20066
20067 #[test]
20068 fn queue_accounting_reports_live_cost_without_draining_or_pinning() {
20069 crate::tests::ensure_common_test_schema();
20070 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 42u64));
20071 let packet = Packet::new(
20072 crate::DataType::named("IMU_DATA"),
20073 &[DataEndpoint::named("SD_CARD")],
20074 "SOURCE",
20075 1,
20076 crate::shared_bytes::convert(Arc::<[u8]>::from([0u8; 24])),
20077 )
20078 .unwrap();
20079 router.tx_queue(packet).unwrap();
20080 assert!(router.has_queued_work());
20081 let expected = router.memory_counters();
20082 let a = router.queue_counters();
20083 assert_eq!(a[0], 1);
20084 assert_eq!(a[1], 0);
20085 assert_eq!(a[2], 1);
20086 assert_eq!(a[3], expected[3]);
20087 assert_eq!(a[4], expected[4]);
20088 assert_eq!(a[5], crate::DataType::named("IMU_DATA").as_u32());
20089 assert_eq!(a[14], 42);
20090 assert_eq!(a[15], 0);
20091 assert_eq!(router.queue_counters(), a);
20092 assert_eq!(router.memory_counters(), expected);
20093 router.drain_queues_with_timeout(0).unwrap();
20094 let empty = router.queue_counters();
20095 assert_eq!(empty[2], 0);
20096 assert_eq!(empty[4], 0);
20097 assert_eq!(router.memory_counters()[4], 0);
20098 assert!(!router.has_queued_work());
20099 }
20100
20101 #[test]
20102 fn repeated_schema_preserves_partial_transfer_but_updates_and_other_sides_do_not_coalesce() {
20103 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20104 let side = router.add_side_packed("A", |_| Ok(()));
20105 let other_side = router.add_side_packed("B", |_| Ok(()));
20106 let original = catalog(1, 42);
20107 let duplicate = catalog(2, 42);
20108 assert_ne!(original.packet_id(), duplicate.packet_id());
20109 let mut st = router.state.lock();
20110 st.transmit_queue.clear();
20111 let queued = |dst, pkt| TxQueued {
20112 disposable: TX_DISPOSABLE,
20113 item: RouterTxItem::ToSide {
20114 src: None,
20115 dst,
20116 data: RouterItem::Packet(pkt),
20117 },
20118 ignore_local: true,
20119 priority: 255,
20120 delivery_state: QueuedDelivery::Untracked,
20121 accepted_type: 0,
20122 deferred_since_ms: 0,
20123 retry_after_ms: 123,
20124 };
20125 st.push_transmit(queued(side, original.clone())).unwrap();
20126 st.push_transmit(queued(side, duplicate.clone())).unwrap();
20127 assert_eq!(st.transmit_queue.len(), 1);
20128 let pending = st.transmit_queue.iter().next().unwrap();
20129 assert_eq!(pending.retry_after_ms, 123);
20130 assert!(
20131 matches!(&pending.item, RouterTxItem::ToSide { data: RouterItem::Packet(pkt), .. }
20132 if pkt.packet_id() == original.packet_id())
20133 );
20134 st.push_transmit(queued(other_side, duplicate)).unwrap();
20135 assert_eq!(st.transmit_queue.len(), 2);
20136 let changed = catalog(3, 43);
20137 st.push_transmit(queued(side, changed.clone())).unwrap();
20138 assert_eq!(st.transmit_queue.len(), 2);
20139 assert!(st.transmit_queue.iter().any(|queued| matches!(&queued.item,
20140 RouterTxItem::ToSide { dst, data: RouterItem::Packet(pkt), .. }
20141 if *dst == side && pkt.packet_id() == changed.packet_id())));
20142 }
20143
20144 #[test]
20145 fn repeated_announcements_complete_one_fragment_per_dispatch() {
20146 use core::sync::atomic::{AtomicUsize, Ordering};
20147 let allowance = Arc::new(AtomicUsize::new(0));
20148 let accepted = Arc::new(std::sync::Mutex::new(Vec::<Arc<[u8]>>::new()));
20149 let available = allowance.clone();
20150 let frames = accepted.clone();
20151 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 1u64));
20152 let side = node.add_side_packed_with_options(
20153 "bounded CAN",
20154 move |bytes| {
20155 if available
20156 .try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
20157 count.checked_sub(1)
20158 })
20159 .is_err()
20160 {
20161 return Err(TelemetryError::Io("side tx busy"));
20162 }
20163 frames.lock().unwrap().push(Arc::<[u8]>::from(bytes));
20164 Ok(())
20165 },
20166 RouterSideOptions::default().with_small_packet_transport(128),
20167 );
20168 let handler = node.state.lock().sides[side]
20169 .as_ref()
20170 .unwrap()
20171 .tx_handler
20172 .clone();
20173 let mut complete = false;
20174 for timestamp in 1..100 {
20175 let mut state = 0x517cc1b7u32;
20177 let payload: Vec<u8> = (0..3600)
20178 .map(|_| {
20179 state ^= state << 13;
20180 state ^= state >> 17;
20181 state ^= state << 5;
20182 state as u8
20183 })
20184 .collect();
20185 let packet = Packet::new(
20186 crate::DataType::DiscoverySchema,
20187 &[crate::DataEndpoint::Discovery],
20188 "SCHEMA_PEER",
20189 timestamp,
20190 crate::shared_bytes::convert(Arc::<[u8]>::from(payload)),
20191 )
20192 .unwrap();
20193 node.state
20194 .lock()
20195 .push_transmit(TxQueued {
20196 disposable: TX_DISPOSABLE,
20197 item: RouterTxItem::ToSide {
20198 src: None,
20199 dst: side,
20200 data: RouterItem::Packet(packet),
20201 },
20202 ignore_local: true,
20203 priority: 255,
20204 delivery_state: QueuedDelivery::Untracked,
20205 accepted_type: 0,
20206 deferred_since_ms: 0,
20207 retry_after_ms: 0,
20208 })
20209 .unwrap();
20210 let data = match &node.state.lock().transmit_queue.iter().next().unwrap().item {
20211 RouterTxItem::ToSide { data, .. } => data.clone(),
20212 _ => panic!("unexpected item"),
20213 };
20214 allowance.store(1, Ordering::Relaxed);
20215 match node.call_side_tx_handler(side, &handler, &data, false) {
20216 Ok(()) => {
20217 complete = true;
20218 break;
20219 }
20220 Err(TelemetryError::Io(_)) => {}
20221 other => panic!("unexpected result: {other:?}"),
20222 }
20223 }
20224 assert!(
20225 complete,
20226 "new announcement timestamps must not replay the prefix forever"
20227 );
20228 let received = accepted.lock().unwrap();
20229 assert!(
20232 received.len() > 1,
20233 "fixture must exercise fragmentation, not a compressed single frame"
20234 );
20235 let first = &received[0];
20236 assert_eq!(&first[..4], &[83, 68, 84, 3]);
20237 assert_eq!(
20238 u16::from_le_bytes([first[10], first[11]]) as usize,
20239 received.len()
20240 );
20241 let transfer_id = &first[4..8];
20242 for (index, frame) in received.iter().enumerate() {
20243 assert_eq!(&frame[4..8], transfer_id);
20244 assert_eq!(u16::from_le_bytes([frame[8], frame[9]]) as usize, index);
20245 }
20246 }
20247 #[test]
20248 fn schema_receipt_resolves_assigned_numeric_address_without_adding_peers() {
20249 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20250 let side = router.add_side_packed("A", |_| Ok(()));
20251 let assigned = 0x1234u32;
20252 assert_ne!(sender_address_u32("SCHEMA_PEER"), assigned);
20253 {
20254 let mut st = router.state.lock();
20255 st.discovery_routes
20256 .entry(side)
20257 .or_default()
20258 .announcers
20259 .insert(
20260 "SCHEMA_PEER".into(),
20261 Box::new(DiscoverySenderState {
20262 advertised_address: Some(assigned),
20263 ..DiscoverySenderState::default()
20264 }),
20265 );
20266 }
20267 router.note_schema_received(side, "@addr:4660");
20268 router.note_schema_received(side, "@addr:4661");
20269 let st = router.state.lock();
20270 let peers = &st.discovery_routes.get(&side).unwrap().announcers;
20271 assert_eq!(peers.len(), 1);
20272 assert!(peers["SCHEMA_PEER"].has_schema);
20273 }
20274 #[test]
20275 fn busy_handler_returns_after_one_attempt_and_other_errors_keep_retry_budget() {
20276 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20277 let attempts = core::cell::Cell::new(0);
20278 let result: Result<((), usize), _> = router.retry_with_attempts(8, || {
20279 attempts.set(attempts.get() + 1);
20280 Err(TelemetryError::Io("side tx busy"))
20281 });
20282 assert!(matches!(
20283 result,
20284 Err((TelemetryError::Io("side tx busy"), 1))
20285 ));
20286 assert_eq!(attempts.get(), 1);
20287 attempts.set(0);
20288 let result: Result<((), usize), _> = router.retry_with_attempts(3, || {
20289 attempts.set(attempts.get() + 1);
20290 Err(TelemetryError::Io("permanent handler failure"))
20291 });
20292 assert!(matches!(
20293 result,
20294 Err((TelemetryError::Io("permanent handler failure"), 3))
20295 ));
20296 assert_eq!(attempts.get(), 3);
20297 }
20298 #[test]
20299 fn rejected_first_template_frame_is_retried_self_describing() {
20300 crate::tests::ensure_common_test_schema();
20301 let ty = crate::config::register_data_type_with_description(
20302 "ROUTER_BUSY_FIRST_TEMPLATE",
20303 "",
20304 crate::MessageElement::Static(
20305 1,
20306 crate::MessageDataType::UInt8,
20307 crate::MessageClass::Data,
20308 ),
20309 &[crate::DataEndpoint::named("RADIO")],
20310 crate::ReliableMode::None,
20311 1,
20312 )
20313 .unwrap();
20314 let tx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20315 let rx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20316 let opts = RouterSideOptions::default().with_small_packet_transport(0);
20317 let a = tx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
20318 let b = rx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
20319 let packet = Packet::new(
20320 ty,
20321 &[crate::DataEndpoint::named("RADIO")],
20322 "BUSY_PEER",
20323 1,
20324 crate::shared_bytes::convert(Arc::<[u8]>::from([42u8])),
20325 )
20326 .unwrap();
20327 let raw = wire_format::pack_packet(&packet);
20328 let first = tx
20330 .encode_side_transport_frames(
20331 a,
20332 opts.clone(),
20333 crate::shared_bytes::convert(raw.clone()),
20334 )
20335 .unwrap();
20336 assert!(
20337 tx.send_side_frames_resumable(a, first, |_| Err(TelemetryError::Io("side tx busy")))
20338 .is_err()
20339 );
20340 let retry = tx
20341 .encode_side_transport_frames(
20342 a,
20343 opts.clone(),
20344 crate::shared_bytes::convert(raw.clone()),
20345 )
20346 .unwrap();
20347 let mut received = None;
20348 tx.send_side_frames_resumable(a, retry, |frame| {
20349 assert_eq!(
20350 frame[3], SIDE_TRANSPORT_KIND_FULL,
20351 "retry requires the unaccepted dictionary bootstrap"
20352 );
20353 received = rx.decode_side_transport_frame(b, frame)?;
20354 Ok(())
20355 })
20356 .unwrap();
20357 assert_eq!(received, Some(crate::shared_bytes::convert(raw.clone())));
20358 let next = tx
20360 .encode_side_transport_frames(a, opts, crate::shared_bytes::convert(raw.clone()))
20361 .unwrap();
20362 tx.send_side_frames_resumable(a, next, |frame| {
20363 assert_eq!(frame[3], SIDE_TRANSPORT_KIND_COMPACT);
20364 assert_eq!(
20365 rx.decode_side_transport_frame(b, frame)?,
20366 Some(crate::shared_bytes::convert(raw.clone()))
20367 );
20368 Ok(())
20369 })
20370 .unwrap();
20371 }
20372 #[test]
20373 fn already_queued_receipt_precedes_ordered_retry_exhaustion() {
20374 crate::tests::ensure_common_test_schema();
20375 let ep = DataEndpoint::named("RADIO");
20376 let ty = crate::config::register_data_type_with_description(
20377 "QUEUED_RECEIPT_DEADLINE",
20378 "",
20379 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20380 &[ep],
20381 crate::ReliableMode::Ordered,
20382 1,
20383 )
20384 .unwrap();
20385 for isr in [false, true] {
20386 let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
20387 let clock = now.clone();
20388 let router = Router::new_with_clock(
20389 RouterConfig::new([]).with_hostname("FC"),
20390 Box::new(move || clock.load(Ordering::Relaxed)),
20391 );
20392 let side = router.add_side_packed("CAN", |_| Ok(()));
20393 let packet = Packet::new(ty, &[ep], "FC", 1, Arc::<[u8]>::from([42u8])).unwrap();
20394 let id = packet.packet_id();
20395 let owner = Router::sender_hash("GS");
20396 let mut destinations = BTreeMap::new();
20397 destinations.insert(owner, side);
20398 router.state.lock().end_to_end_reliable_tx.insert(
20399 id,
20400 EndToEndReliableSent {
20401 #[cfg(feature = "std")]
20402 completion_epoch: 0,
20403 #[cfg(feature = "std")]
20404 pending_type: ty.as_u32(),
20405 ordered_type: Some(ty.as_u32()),
20406 waiting_for_predecessor: false,
20407 data: RouterItem::Packet(packet),
20408 pending_destinations: destinations,
20409 tracked_destinations: true,
20410 last_send_ms: 0,
20411 retries: runtime_reliable_max_retries(),
20412 queued: false,
20413 }
20414 .try_box()
20415 .unwrap(),
20416 );
20417 let snapshot = router.export_reliable_pending_stats();
20418 assert_eq!(snapshot.len(), 1);
20419 assert_eq!(snapshot[0].packet_id, id);
20420 assert_eq!(snapshot[0].data_type, ty);
20421 assert_eq!(snapshot[0].pending_destinations, vec![owner]);
20422 assert_eq!(snapshot[0].retries, runtime_reliable_max_retries());
20423 assert!(!snapshot[0].queued);
20424 assert!(!snapshot[0].waiting_for_predecessor);
20425 let ack = Packet::new_with_wire_contract(
20426 DataType::ReliableAck,
20427 message_meta(DataType::ReliableAck).endpoints_ref(),
20428 "E2EACK:GS",
20429 1,
20430 1,
20431 Arc::<[u8]>::from(id.to_le_bytes()),
20432 None,
20433 Arc::from([owner, Router::sender_hash("FC")]),
20434 )
20435 .unwrap();
20436 let packed = wire_format::pack_packet(&ack);
20437 if isr {
20438 router.rx_packed_queue_from_side_isr(&packed, side).unwrap();
20439 } else {
20440 router.rx_packed_queue_from_side(&packed, side).unwrap();
20441 }
20442 now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
20443 router
20444 .drain_queues_with_timeout(1)
20445 .expect("a received ACK must be processed before cancelling its ordered delivery");
20446 assert!(!router.state.lock().end_to_end_reliable_tx.contains_key(&id));
20447 assert!(router.export_reliable_pending_stats().is_empty());
20448 }
20449 }
20450 #[test]
20451 fn embedded_gateway_receipt_with_numeric_sender_clears_host_pending_owner() {
20452 crate::tests::ensure_common_test_schema();
20453 let bytes = [
20456 0x0c, 0x01, 0x01, 0x08, 0xd4, 0xe5, 0xea, 0xb6, 0xb7, 0x33, 0xff, 0x90, 0x02, 0xb4,
20457 0xe6, 0xdd, 0x94, 0x07, 0x14, 0x03, 0x44, 0x02, 0x02, 0x06, 0x25, 0xd2, 0x59, 0x1b,
20458 0xbb, 0xfc, 0xa7, 0x5f, 0x43, 0x92, 0x9e, 0x12, 0xbb, 0xfc, 0xa7, 0x26, 0xa2, 0xa0,
20459 0xbe, 0x39, 0x65, 0xe2, 0xd8, 0x61, 0x68, 0x22, 0xfd,
20460 ];
20461 let ack = wire_format::unpack_packet(&bytes).unwrap();
20462 let owner = Router::sender_hash("GB");
20463 assert_eq!(Router::end_to_end_ack_sender_hash(&ack), Some(owner));
20464 let id = Router::decode_end_to_end_reliable_ack(ack.payload()).unwrap();
20465 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20466 let side = router.add_side_packed("Pico", |_| Ok(()));
20467 let data = ack.clone();
20468 router.state.lock().end_to_end_reliable_tx.insert(
20469 id,
20470 EndToEndReliableSent {
20471 #[cfg(feature = "std")]
20472 completion_epoch: 0,
20473 #[cfg(feature = "std")]
20474 pending_type: data.data_type().as_u32(),
20475 ordered_type: None,
20476 waiting_for_predecessor: false,
20477 data: RouterItem::Packet(data),
20478 pending_destinations: BTreeMap::from([(owner, side)]),
20479 tracked_destinations: true,
20480 last_send_ms: 0,
20481 retries: 0,
20482 queued: false,
20483 }
20484 .try_box()
20485 .unwrap(),
20486 );
20487 router.rx_packed_queue_from_side(&bytes, side).unwrap();
20488 router.drain_queues_with_timeout(1).unwrap();
20489 assert!(router.export_reliable_pending_stats().is_empty());
20490 }
20491
20492 #[test]
20493 fn frozen_delivery_contract_bypasses_dictionary_even_for_best_effort_schema() {
20494 crate::tests::ensure_common_test_schema();
20495 let ep = DataEndpoint::named("RADIO");
20496 let ty = crate::config::register_data_type_with_description(
20497 "MANAGED_CONTROL_WITH_FROZEN_OWNERS",
20498 "",
20499 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20500 &[ep],
20501 crate::ReliableMode::None,
20502 16,
20503 )
20504 .unwrap();
20505 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20506 let opts = RouterSideOptions::default().with_template_transport();
20507 let side = router.add_side_packed_with_options("radio", |_| Ok(()), opts.clone());
20508 let packet = Packet::new_with_wire_contract(
20509 ty,
20510 &[ep],
20511 "GS",
20512 1,
20513 1,
20514 Arc::<[u8]>::from([1u8]),
20515 None,
20516 Arc::from([Router::sender_hash("RF")]),
20517 )
20518 .unwrap();
20519 let raw = wire_format::pack_packet(&packet);
20520 assert_eq!(
20521 router
20522 .item_service_priority(&RouterItem::Packet(packet.clone()))
20523 .unwrap(),
20524 254
20525 );
20526 assert_eq!(
20527 router
20528 .item_service_priority(&RouterItem::Packed(crate::shared_bytes::convert(
20529 raw.clone()
20530 )))
20531 .unwrap(),
20532 254
20533 );
20534 assert!(
20535 !wire_format::peek_routing_frame_info(&raw)
20536 .unwrap()
20537 .envelope
20538 .is_reliable
20539 );
20540 for _ in 0..3 {
20541 let frames = router
20542 .encode_side_transport_frames(
20543 side,
20544 opts.clone(),
20545 crate::shared_bytes::convert(raw.clone()),
20546 )
20547 .unwrap();
20548 let mut count = 0;
20549 router
20550 .send_side_frames_resumable(side, frames, |bytes| {
20551 assert_eq!(
20552 bytes,
20553 raw.as_ref(),
20554 "delivery contract must remain independently decodable"
20555 );
20556 count += 1;
20557 Ok(())
20558 })
20559 .unwrap();
20560 assert_eq!(count, 1);
20561 }
20562 }
20563
20564 #[test]
20565 fn managed_receive_queues_keep_service_priority_with_low_schema_priority() {
20566 crate::tests::ensure_common_test_schema();
20567 let ep = DataEndpoint::named("RADIO");
20568 let ty = crate::config::register_data_type_with_description(
20569 "MANAGED_RX_SERVICE_PRIORITY",
20570 "",
20571 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20572 &[ep],
20573 crate::ReliableMode::None,
20574 16,
20575 )
20576 .unwrap();
20577 let router = Router::new(RouterConfig::new([]).with_hostname("RF"));
20578 router
20579 .enable_network_variable(ty, NetworkVariablePermissions::READ_ONLY)
20580 .unwrap();
20581 let side = router.add_side_packed("radio", |_| Ok(()));
20582 router.clear_queues();
20583 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
20584 let wire = wire_format::pack_packet(&packet);
20585 router.rx_queue(packet.clone()).unwrap();
20586 router.rx_queue_from_side(packet.clone(), side).unwrap();
20587 router.rx_packed_queue(&wire).unwrap();
20588 router.rx_packed_queue_from_side(&wire, side).unwrap();
20589 for _ in 0..4 {
20590 let queued = router
20591 .pop_next_queued_control_rx_item()
20592 .unwrap()
20593 .expect("registered managed controls must precede queued bulk work");
20594 assert_eq!(
20595 queued.priority,
20596 crate::transport_priority(DataType::ManagedVariableValue)
20597 );
20598 }
20599 assert!(router.pop_next_queued_control_rx_item().unwrap().is_none());
20600 }
20601
20602 #[test]
20603 fn managed_value_keeps_service_priority_after_side_backpressure() {
20604 crate::tests::ensure_common_test_schema();
20605 let ep = DataEndpoint::named("RADIO");
20606 let ty = crate::config::register_data_type_with_description(
20607 "MANAGED_RETRY_SERVICE_PRIORITY",
20608 "",
20609 MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20610 &[ep],
20611 crate::ReliableMode::None,
20612 16,
20613 )
20614 .unwrap();
20615 let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20616 router
20617 .enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
20618 .unwrap();
20619 let side = router.add_side_packed("radio", |_| Err(TelemetryError::Io("side tx busy")));
20620 let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
20621 let item = RouterItem::Packet(packet);
20622 for queued in [
20623 RouterTxItem::Broadcast(item.clone()),
20624 RouterTxItem::ToSide {
20625 src: None,
20626 dst: side,
20627 data: item.clone(),
20628 },
20629 ] {
20630 router.tx_queue_item_for_retry(queued, true).unwrap();
20631 let queued = router.state.lock().transmit_queue.pop_front().unwrap();
20632 assert_eq!(
20633 queued.priority,
20634 crate::transport_priority(DataType::ManagedVariableValue),
20635 "a busy side must not demote managed controls to bulk application priority"
20636 );
20637 assert!(queued.retry_after_ms > 0);
20638 }
20639 }
20640 #[test]
20641 fn receipt_budget_does_not_starve_transmit() {
20642 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
20643 crate::tests::ensure_common_test_schema();
20644 let now = Arc::new(AtomicU64::new(0));
20645 let clock = now.clone();
20646 let count = Arc::new(AtomicUsize::new(0));
20647 let accepted = count.clone();
20648 let node = Router::new_with_clock(
20649 RouterConfig::default(),
20650 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
20651 );
20652 let side = node.add_side_packed("wire", move |_| {
20653 accepted.fetch_add(1, Ordering::Relaxed);
20654 Ok(())
20655 });
20656 node.clear_queues();
20657 let packet = Packet::new(
20658 crate::DataType::ReliableAck,
20659 &[crate::DataEndpoint::TelemetryError],
20660 "E2EACK:PEER",
20661 1,
20662 Arc::<[u8]>::from([0u8; 8]),
20663 )
20664 .unwrap();
20665 node.state
20666 .lock()
20667 .push_transmit(TxQueued {
20668 disposable: TX_DISPOSABLE,
20669 item: RouterTxItem::ToSide {
20670 src: None,
20671 dst: side,
20672 data: RouterItem::Packet(packet.clone()),
20673 },
20674 ignore_local: true,
20675 priority: 250,
20676 delivery_state: QueuedDelivery::Untracked,
20677 accepted_type: 0,
20678 deferred_since_ms: 0,
20679 retry_after_ms: 0,
20680 })
20681 .unwrap();
20682 node.rx_packed_queue_from_side(
20683 &wire_format::pack_packet(
20684 &Packet::new(
20685 crate::DataType::ReliableAck,
20686 &[crate::DataEndpoint::TelemetryError],
20687 "RX_PEER",
20688 2,
20689 Arc::<[u8]>::from([0u8; 8]),
20690 )
20691 .unwrap(),
20692 ),
20693 side,
20694 )
20695 .unwrap();
20696 node.drain_queues_with_timeout(1).unwrap();
20697 assert_eq!(
20698 count.load(Ordering::Relaxed),
20699 1,
20700 "control RX exhausting its cooperative budget must still allow one queued TX"
20701 );
20702 }
20703 #[test]
20704 fn receipt_budget_does_not_starve_bulk_transmit() {
20705 use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
20706 crate::tests::ensure_common_test_schema();
20707 let now = Arc::new(AtomicU64::new(0));
20708 let clock = now.clone();
20709 let count = Arc::new(AtomicUsize::new(0));
20710 let accepted = count.clone();
20711 let node = Router::new_with_clock(
20712 RouterConfig::default(),
20713 Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
20714 );
20715 let side = node.add_side_packed("wire", move |_| {
20716 accepted.fetch_add(1, Ordering::Relaxed);
20717 Ok(())
20718 });
20719 node.clear_queues();
20720 let packet = Packet::new(
20721 crate::DataType::ReliableAck,
20722 &[crate::DataEndpoint::TelemetryError],
20723 "E2EACK:PEER",
20724 1,
20725 Arc::<[u8]>::from([0u8; 8]),
20726 )
20727 .unwrap();
20728 node.state
20729 .lock()
20730 .push_transmit(TxQueued {
20731 disposable: TX_DISPOSABLE,
20732 item: RouterTxItem::ToSide {
20733 src: None,
20734 dst: side,
20735 data: RouterItem::Packet(packet.clone()),
20736 },
20737 ignore_local: true,
20738 priority: 1,
20739 delivery_state: QueuedDelivery::Untracked,
20740 accepted_type: 0,
20741 deferred_since_ms: 0,
20742 retry_after_ms: 0,
20743 })
20744 .unwrap();
20745 node.rx_packed_queue_from_side(
20746 &wire_format::pack_packet(
20747 &Packet::new(
20748 crate::DataType::ReliableAck,
20749 &[crate::DataEndpoint::TelemetryError],
20750 "RX_PEER",
20751 2,
20752 Arc::<[u8]>::from([0u8; 8]),
20753 )
20754 .unwrap(),
20755 ),
20756 side,
20757 )
20758 .unwrap();
20759 node.process_all_queues_with_timeout_impl(1).unwrap();
20760 assert_eq!(
20761 count.load(Ordering::Relaxed),
20762 1,
20763 "control RX exhausting its cooperative budget must still allow one queued bulk TX"
20764 );
20765 }
20766}
20767
20768#[cfg(all(test, feature = "discovery", feature = "std"))]
20769fn untemplated_test_node() -> Router {
20770 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0))
20771}
20772#[cfg(all(test, feature = "discovery", feature = "std"))]
20773fn untemplated_test_options() -> RouterSideOptions {
20774 RouterSideOptions::default().with_small_packet_transport(64)
20775}
20776#[cfg(all(test, feature = "discovery", feature = "std"))]
20777#[allow(clippy::duplicate_mod)]
20778#[path = "tests/untemplated_transport.rs"]
20779mod untemplated_transport_tests;
20780
20781#[cfg(all(test, feature = "std", feature = "discovery"))]
20782mod queued_producer_maintenance_tests {
20783 use super::*;
20784 #[test]
20785 fn queued_publication_defers_discovery_to_worker_maintenance() {
20786 crate::tests::ensure_common_test_schema();
20787 let ty = DataType::named("GPS_DATA");
20788 for variant in 0..4 {
20789 let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
20790 let output = sent.clone();
20791 let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20792 router.add_side_packed("wire", move |bytes| {
20793 output
20794 .lock()
20795 .unwrap()
20796 .push(wire_format::unpack_packet(bytes)?.data_type());
20797 Ok(())
20798 });
20799 router.clear_queues();
20800 let packet =
20801 Packet::from_f32_slice(ty, &[1.0, 2.0, 3.0], message_meta(ty).endpoints_ref(), 42)
20802 .unwrap();
20803 match variant {
20804 0 => router.tx_queue(packet).unwrap(),
20805 1 => router
20806 .tx_packed_queue(wire_format::pack_packet(&packet))
20807 .unwrap(),
20808 2 => router.log_queue(ty, &[1.0f32, 2.0, 3.0]).unwrap(),
20809 _ => router.log_queue_ts(ty, 42, &[1.0f32, 2.0, 3.0]).unwrap(),
20810 }
20811 assert!(
20812 sent.lock().unwrap().is_empty(),
20813 "producer invoked a transport"
20814 );
20815 assert_eq!(
20816 router.state.lock().transmit_queue.len(),
20817 1,
20818 "queued API performed discovery on the producer stack (variant {variant})"
20819 );
20820 router.periodic(0).unwrap();
20821 let sent = sent.lock().unwrap();
20822 assert!(sent.contains(&ty));
20823 assert!(
20824 sent.iter().copied().any(discovery::is_discovery_type),
20825 "worker maintenance must still announce the board"
20826 );
20827 }
20828 }
20829}
20830
20831#[cfg(all(test, feature = "std"))]
20832mod bounded_fair_queue_tests {
20833 use super::*;
20834 #[test]
20835 fn small_canonical_reports_survive_fast_senders_fixed_ring_bursts() {
20836 let ep = crate::config::register_endpoint("ROUTER_SMALL_QUEUE_DEST", false).unwrap();
20837 let ty = crate::config::register_data_type_with_description(
20838 "ROUTER_SMALL_QUEUE_REPORT",
20839 "",
20840 crate::MessageElement::Static(
20841 4,
20842 crate::MessageDataType::UInt8,
20843 crate::MessageClass::Data,
20844 ),
20845 &[ep],
20846 crate::ReliableMode::None,
20847 0,
20848 )
20849 .unwrap();
20850 for tx in [false, true] {
20851 for slow_last in [false, true] {
20852 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20853 let make = |sender| {
20854 RouterItem::Packed(crate::shared_bytes::convert(wire_format::pack_packet(
20855 &Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 4])).unwrap(),
20856 )))
20857 };
20858 let rx = |data| RouterRxItem {
20859 src: None,
20860 data,
20861 priority: 0,
20862 };
20863 let tx_item = |data| TxQueued {
20864 disposable: TX_DISPOSABLE,
20865 item: RouterTxItem::Broadcast(data),
20866 ignore_local: true,
20867 priority: 0,
20868 delivery_state: QueuedDelivery::Untracked,
20869 accepted_type: 0,
20870 deferred_since_ms: 0,
20871 retry_after_ms: 0,
20872 };
20873 let mut st = node.state.lock();
20874 st.memory.max_queue_budget = usize::MAX;
20875 if tx {
20876 st.transmit_queue = BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
20877 } else {
20878 st.received_queue = BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
20879 }
20880 let order = if slow_last {
20881 ["FAST", "FAST", "SLOW"]
20882 } else {
20883 ["SLOW", "FAST", "FAST"]
20884 };
20885 for sender in order.into_iter().chain(core::iter::repeat_n("FAST", 32)) {
20886 if tx {
20887 st.push_transmit(tx_item(make(sender))).unwrap();
20888 } else {
20889 st.push_received(rx(make(sender))).unwrap();
20890 }
20891 }
20892 let slow = wire_format::source_address_for_sender("SLOW");
20893 let retained = if tx {
20894 st.transmit_queue
20895 .iter()
20896 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
20897 } else {
20898 st.received_queue.iter().any(|item| {
20899 RouterInner::fair_bulk_owner(&item.data, item.priority) == Some(slow)
20900 })
20901 };
20902 assert!(
20903 retained,
20904 "small report was displaced: tx={tx}, slow_last={slow_last}"
20905 );
20906 }
20907 }
20908 }
20909
20910 #[test]
20911 fn slow_stream_survives_same_board_fast_stream_bursts() {
20912 let ep = crate::config::register_endpoint("ROUTER_STREAM_QUEUE_DEST", false).unwrap();
20913 let ty = crate::config::register_data_type_with_description(
20914 "ROUTER_STREAM_QUEUE_FAST",
20915 "",
20916 crate::MessageElement::Static(
20917 64,
20918 crate::MessageDataType::UInt8,
20919 crate::MessageClass::Data,
20920 ),
20921 &[ep],
20922 crate::ReliableMode::None,
20923 0,
20924 )
20925 .unwrap();
20926 let slow_ty = crate::config::register_data_type_with_description(
20927 "ROUTER_STREAM_QUEUE_SLOW",
20928 "",
20929 crate::MessageElement::Static(
20930 64,
20931 crate::MessageDataType::UInt8,
20932 crate::MessageClass::Data,
20933 ),
20934 &[ep],
20935 crate::ReliableMode::None,
20936 0,
20937 )
20938 .unwrap();
20939 for mode in 0..3 {
20940 for tx in [false, true] {
20941 for slow_last in [false, true] {
20942 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20943 let make = |sender| {
20944 let packet = Packet::new(
20945 if sender == "SLOW" { slow_ty } else { ty },
20946 &[ep],
20947 "SAME_BOARD",
20948 1,
20949 Arc::<[u8]>::from([1; 64]),
20950 )
20951 .unwrap();
20952 if mode == 1 {
20953 RouterItem::Packet(packet)
20954 } else {
20955 RouterItem::Packed(crate::shared_bytes::convert(
20956 wire_format::pack_packet(&packet),
20957 ))
20958 }
20959 };
20960 let rx = |data| RouterRxItem {
20961 src: None,
20962 data,
20963 priority: 0,
20964 };
20965 let tx_item = |data| TxQueued {
20966 disposable: TX_DISPOSABLE,
20967 item: RouterTxItem::Broadcast(data),
20968 ignore_local: true,
20969 priority: 0,
20970 delivery_state: QueuedDelivery::Untracked,
20971 accepted_type: 0,
20972 deferred_since_ms: 0,
20973 retry_after_ms: 0,
20974 };
20975 let mut st = node.state.lock();
20976 st.memory.max_queue_budget = usize::MAX;
20977 if mode == 0 {
20978 if tx {
20979 st.transmit_queue =
20980 BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
20981 } else {
20982 st.received_queue =
20983 BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
20984 }
20985 } else if mode == 1 {
20986 let cost = if tx {
20987 tx_item(make("SLOW")).byte_cost()
20988 + 2 * tx_item(make("FAST")).byte_cost()
20989 } else {
20990 rx(make("SLOW")).byte_cost() + 2 * rx(make("FAST")).byte_cost()
20991 };
20992 if tx {
20993 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
20994 } else {
20995 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
20996 }
20997 }
20998 let order = if slow_last {
20999 ["FAST", "FAST", "SLOW"]
21000 } else {
21001 ["SLOW", "FAST", "FAST"]
21002 };
21003 for (index, sender) in order
21004 .into_iter()
21005 .chain(core::iter::repeat_n("FAST", 32))
21006 .enumerate()
21007 {
21008 if mode == 2 && index == 3 {
21009 st.memory.max_queue_budget = st.shared_queue_bytes_used();
21010 }
21011 if tx {
21012 st.push_transmit(tx_item(make(sender))).unwrap();
21013 } else {
21014 st.push_received(rx(make(sender))).unwrap();
21015 }
21016 }
21017 let is_slow = |data: &RouterItem| match data {
21018 RouterItem::Packet(p) => p.data_type() == slow_ty,
21019 RouterItem::Packed(b) => {
21020 wire_format::scheduling_type(b.as_ref()) == Some(slow_ty)
21021 }
21022 };
21023 let retained = if tx {
21024 st.transmit_queue.iter().any(|item| {
21025 let data = match &item.item {
21026 RouterTxItem::Broadcast(data)
21027 | RouterTxItem::ToSide { data, .. } => data,
21028 _ => return false,
21029 };
21030 is_slow(data)
21031 })
21032 } else {
21033 st.received_queue.iter().any(|item| is_slow(&item.data))
21034 };
21035 assert!(
21036 retained,
21037 "slow stream displaced: mode={mode}, tx={tx}, slow_last={slow_last}"
21038 );
21039 }
21040 }
21041 }
21042 }
21043
21044 #[test]
21045 fn fast_producer_preserves_slow_reports_at_each_queue_boundary() {
21046 let ep = crate::config::register_endpoint("QUEUE_FAIR_DEST", false).unwrap();
21047 let ty = crate::config::register_data_type_with_description(
21048 "QUEUE_FAIR_REPORT",
21049 "",
21050 MessageElement::Static(64, crate::MessageDataType::UInt8, crate::MessageClass::Data),
21051 &[ep],
21052 crate::ReliableMode::None,
21053 0,
21054 )
21055 .unwrap();
21056 for mode in 0..3 {
21057 for tx in [false, true] {
21058 for slow_last in [false, true] {
21059 for next_sender in ["FAST", "NEW_SLOW"] {
21060 let router =
21061 Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
21062 let make = |sender| {
21063 RouterItem::Packet(
21064 Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 64]))
21065 .unwrap(),
21066 )
21067 };
21068 let rx = |data| RouterRxItem {
21069 src: None,
21070 data,
21071 priority: 0,
21072 };
21073 let tx_item = |data| TxQueued {
21074 disposable: TX_DISPOSABLE,
21075 item: RouterTxItem::Broadcast(data),
21076 ignore_local: true,
21077 priority: 0,
21078 delivery_state: QueuedDelivery::Untracked,
21079 accepted_type: 0,
21080 deferred_since_ms: 0,
21081 retry_after_ms: 0,
21082 };
21083 let order = if slow_last {
21084 ["FAST", "FAST", "SLOW"]
21085 } else {
21086 ["SLOW", "FAST", "FAST"]
21087 };
21088 let mut st = router.state.lock();
21089 st.memory.max_queue_budget = usize::MAX;
21090 let cost = if tx {
21091 order.iter().map(|s| tx_item(make(s)).byte_cost()).sum()
21092 } else {
21093 order.iter().map(|s| rx(make(s)).byte_cost()).sum()
21094 };
21095 if mode == 0 {
21096 if tx {
21097 st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
21098 } else {
21099 st.received_queue = BoundedDeque::new(cost, cost, 2.0);
21100 }
21101 } else if mode == 1 {
21102 if tx {
21103 st.transmit_queue =
21104 BoundedDeque::new(cost * 10, 3 * size_of::<TxQueued>(), 1.0);
21105 } else {
21106 st.received_queue = BoundedDeque::new(
21107 cost * 10,
21108 3 * size_of::<RouterRxItem>(),
21109 1.0,
21110 );
21111 }
21112 }
21113 for sender in order {
21114 if tx {
21115 st.push_transmit(tx_item(make(sender))).unwrap();
21116 } else {
21117 st.push_received(rx(make(sender))).unwrap();
21118 }
21119 }
21120 if mode == 2 {
21121 st.memory.max_queue_budget = st.shared_queue_bytes_used();
21122 }
21123 if tx {
21124 st.push_transmit(tx_item(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
21125 } else {
21126 st.push_received(rx(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
21127 }
21128 let slow = wire_format::source_address_for_sender("SLOW");
21129 let retained = if tx {
21130 st.transmit_queue
21131 .iter()
21132 .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
21133 } else {
21134 st.received_queue.iter().any(|item| {
21135 RouterInner::fair_bulk_owner(&item.data, item.priority)
21136 == Some(slow)
21137 })
21138 };
21139 assert!(
21140 retained,
21141 "slow report evicted: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"
21142 );
21143 }
21144 }
21145 }
21146 }
21147 }
21148}
21149
21150#[cfg(all(test, feature = "discovery"))]
21151#[test]
21152fn cow_discovery_rejected_growth_keeps_original_graph_and_routes() {
21153 crate::tests::ensure_common_test_schema();
21154 let node = Router::new_with_clock(RouterConfig::default().with_sender("GB"), Box::new(|| 0));
21155 let side = node.add_side_packed("can", |_| Ok(()));
21156 let baseline = discovery::build_discovery_topology(
21157 "PEER",
21158 0,
21159 &[TopologyBoardNode {
21160 sender_id: "PEER".into(),
21161 reachable_endpoints: Vec::new(),
21162 reachable_timesync_sources: Vec::new(),
21163 connections: Vec::new(),
21164 }],
21165 )
21166 .unwrap();
21167 node.rx_from_side(&baseline, side).unwrap();
21168 node.process_all_queues().unwrap();
21169 node.clear_queues();
21170 let before = node.export_topology();
21171 let original = {
21172 let mut st = node.state.lock();
21173 let graph = st.discovery_routes[&side].announcers["PEER"]
21174 .topology_boards
21175 .clone();
21176 st.memory.max_queue_budget = st.shared_queue_bytes_used();
21177 graph
21178 };
21179 let update = discovery::build_discovery_topology_delta(
21180 "PEER",
21181 1,
21182 &[TopologyBoardNode {
21183 sender_id: "PEER".into(),
21184 reachable_endpoints: Vec::new(),
21185 reachable_timesync_sources: vec!["NEW_CLOCK".into()],
21186 connections: vec!["NEW_NEIGHBOR".into()],
21187 }],
21188 &[],
21189 )
21190 .unwrap();
21191 assert!(
21192 node.rx_from_side(&update, side)
21193 .and_then(|_| node.process_all_queues())
21194 .is_err(),
21195 "growth must fail when no queue budget can be reclaimed"
21196 );
21197 assert_eq!(node.export_topology().routes, before.routes);
21198 let st = node.state.lock();
21199 assert!(Arc::ptr_eq(
21200 &original,
21201 &st.discovery_routes[&side].announcers["PEER"].topology_boards
21202 ));
21203 assert!(original[0].connections.is_empty());
21204 assert!(original[0].reachable_timesync_sources.is_empty());
21205}
21206
21207#[cfg(all(test, feature = "std"))]
21208mod bounded_rx_priority_tests {
21209 use super::*;
21210 #[test]
21211 fn retained_slow_report_cannot_starve_behind_continuous_fast_data() {
21212 let ep =
21213 crate::config::register_endpoint("RouterRxItem_FAIR_EP_RouterInner", false).unwrap();
21214 let ty = crate::config::register_data_type_with_description(
21215 "RouterRxItem_FAIR_DATA_RouterInner",
21216 "",
21217 crate::MessageElement::Static(
21218 1,
21219 crate::MessageDataType::Float32,
21220 crate::MessageClass::Data,
21221 ),
21222 &[ep],
21223 crate::ReliableMode::None,
21224 0,
21225 )
21226 .unwrap();
21227 let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
21228 let mut st = node.state.lock();
21229 st.memory.max_queue_budget = 65536;
21230 st.received_queue = BoundedDeque::new(32768, 16 * size_of::<RouterRxItem>(), 1.0);
21231 let make = |sender: &str, stamp, priority| {
21232 let packet = Packet::new(ty, &[ep], sender, stamp, Arc::<[u8]>::from([0; 4])).unwrap();
21233 RouterRxItem {
21234 src: None,
21235 data: RouterItem::Packet(packet),
21236 priority,
21237 }
21238 };
21239 st.push_received(make("VALVE", 0, 0)).unwrap();
21240 let mut delivered = false;
21241 for tick in 1..=32u64 {
21242 for index in 0..3 {
21243 st.push_received(make("DAQ", tick * 3 + index, 16)).unwrap();
21244 }
21245 let item = st.pop_received_fair().unwrap();
21246 let sender = match &item.data {
21247 RouterItem::Packet(packet) => packet.sender(),
21248 _ => panic!("expected packet"),
21249 };
21250 if sender == "VALVE" {
21251 delivered = true;
21252 assert!(tick <= u64::from(MAX_PRIORITY_BURST) + 1);
21253 break;
21254 }
21255 }
21256 assert!(
21257 delivered,
21258 "retention alone does not deliver low-rate reports under a continuous higher-priority source"
21259 );
21260 st.received_queue.clear();
21261 st.rx_priority_burst = MAX_PRIORITY_BURST;
21262 st.push_received(make("VALVE", 100, 0)).unwrap();
21263 st.push_received(make("VALVE", 101, 0)).unwrap();
21264 st.push_received(make("DAQ", 102, 16)).unwrap();
21265 st.push_received(make("COMMAND", 103, 254)).unwrap();
21266 assert_eq!(
21267 st.pop_received_fair().unwrap().priority,
21268 254,
21269 "a fairness turn must never delay protected control"
21270 );
21271 let mut first_slow = None;
21272 for tick in 0..=MAX_PRIORITY_BURST {
21273 st.push_received(make("DAQ", 200 + u64::from(tick), 16))
21274 .unwrap();
21275 let item = st.pop_received_fair().unwrap();
21276 if let RouterItem::Packet(packet) = &item.data
21277 && packet.sender() == "VALVE"
21278 {
21279 first_slow = Some(packet.timestamp());
21280 break;
21281 }
21282 }
21283 assert_eq!(
21284 first_slow,
21285 Some(100),
21286 "fairness preserves same-priority FIFO order"
21287 );
21288 st.rx_priority_burst = MAX_PRIORITY_BURST;
21289 drop(st);
21290 node.clear_rx_queue();
21291 assert_eq!(node.state.lock().rx_priority_burst, 0);
21292 }
21293}
21294
21295#[cfg(all(test, feature = "std", feature = "discovery"))]
21296#[path = "tests/accepted_tx_ownership.rs"]
21297mod accepted_tx_ownership;
21298
21299#[cfg(feature = "latest-value-egress")]
21300#[path = "router_latest.rs"]
21301mod latest;
21302
21303#[cfg(all(test, feature = "std", feature = "discovery"))]
21304#[path = "tests/retry_backpressure_native.rs"]
21305mod retry_backpressure_native;
21306
21307#[path = "router_e2e_backpressure.rs"]
21308mod e2e_backpressure;
21309#[cfg(all(test, feature = "std", feature = "discovery", feature = "timesync"))]
21310#[path = "tests/peer_record_footprint.rs"]
21311mod peer_record_footprint;
21312
21313#[cfg(feature = "std")]
21314#[path = "router_completion.rs"]
21315mod completion;
21316#[cfg(feature = "std")]
21317pub use completion::{PackedWriteLease, RetryTracker, RetryTrackerLimits, WriteStart};
21318
21319#[cfg(all(test, feature = "std", feature = "discovery"))]
21320#[path = "tests/actual_bootstrap_arbitration.rs"]
21321mod actual_bootstrap_arbitration;
21322
21323#[cfg(all(test, feature = "discovery", feature = "std"))]
21324#[path = "tests/router_managed_refresh.rs"]
21325mod managed_refresh_tests;
21326
21327#[cfg(all(test, feature = "std", feature = "discovery"))]
21328#[path = "tests/pending_type_metadata.rs"]
21329mod pending_type_metadata;
21330#[cfg(all(test, feature = "std", feature = "discovery"))]
21331#[path = "tests/sender_identity_match.rs"]
21332mod sender_identity_match;
21333
21334#[cfg(all(test, feature = "discovery", feature = "std"))]
21335#[path = "tests/router_cold_cycle_trace.rs"]
21336mod router_cold_cycle_trace;
21337
21338#[cfg(all(test, feature = "std", feature = "discovery"))]
21339#[path = "tests/toside_inspection_metadata.rs"]
21340mod toside_inspection_metadata;
21341
21342#[cfg(all(test, feature = "std", feature = "discovery"))]
21343#[path = "tests/active_queued_types.rs"]
21344mod active_queued_types;