1use crate::config::RuntimeMemoryConfig;
12use crate::diagnostics::{
13 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
14 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
15 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
16};
17#[cfg(feature = "discovery")]
18use crate::discovery::{
19 self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
20 DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS, DISCOVERY_SLOW_LINK_PING_INTERVAL_MS,
21 DiscoveryCadenceState, TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS, TopologyAnnouncerRoute,
22 TopologyBoardNode, TopologySideRoute, TopologySnapshot,
23};
24use crate::packet::{hash_bytes_u64, sender_address_u32};
25use crate::queue::{BoundedDeque, ByteCost};
26#[cfg(all(not(feature = "std"), target_os = "none"))]
27use crate::seds_error_msg;
28#[cfg(feature = "timesync")]
29use crate::timesync::{
30 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
31 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
32 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
33 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
34 decode_timesync_request, decode_timesync_response,
35};
36use crate::{
37 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
38 config::{
39 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
40 runtime_reliable_max_end_to_end_pending, runtime_reliable_max_pending,
41 runtime_reliable_max_retries, runtime_reliable_max_return_routes,
42 runtime_reliable_retransmit_ms,
43 },
44 get_needed_message_size, impl_letype_num, is_reliable_type,
45 lock::{ReentryGate, RouterMutex},
46 message_e2e_encryption_policy, message_meta, message_priority,
47 packet::Packet,
48 reliable_mode, wire_format,
49};
50use alloc::string::{String, ToString};
51use alloc::{
52 borrow::ToOwned,
53 boxed::Box,
54 collections::{BTreeMap, BTreeSet, VecDeque},
55 format,
56 sync::Arc,
57 vec,
58 vec::Vec,
59};
60use core::cell::UnsafeCell;
61use core::fmt;
62use core::fmt::{Debug, Formatter};
63use core::mem::size_of;
64use core::ops::{Deref, DerefMut};
65use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
66use crc32fast::Hasher as Crc32Hasher;
67#[cfg(feature = "std")]
68use std::time::Instant;
69
70pub type RouterSideId = usize;
72
73const SIDE_TRANSPORT_MAGIC: &[u8; 3] = b"SDT";
74const SIDE_TRANSPORT_KIND_FULL: u8 = 0x01;
75const SIDE_TRANSPORT_KIND_COMPACT: u8 = 0x02;
76const SIDE_TRANSPORT_KIND_CHUNK: u8 = 0x03;
77const SIDE_TRANSPORT_KIND_COMPACT_DELTA: u8 = 0x04;
78const SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP: u8 = 0x05;
79const SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED: u8 = 0x01;
80const SIDE_TRANSPORT_FLAG_WIRE_CONTRACT: u8 = 0x04;
81const SIDE_TRANSPORT_FLAG_PACKET_NONCE: u8 = 0x08;
82const SIDE_TRANSPORT_FLAG_ENDPOINT_BITMAP_PRESENT: u8 = 0x20;
83const SIDE_TRANSPORT_FLAG_COMPACT_RELIABLE_HEADER: u8 = 0x40;
84const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
85const SIDE_TRANSPORT_CHUNK_OVERHEAD: usize = 3 + 1 + 4 + 2 + 2 + wire_format::CRC32_BYTES;
86const SIDE_TIMESTAMP_POLICY_WORDS: usize = ((crate::MAX_VALUE_DATA_TYPE as usize) + 1).div_ceil(64);
87const SIDE_TRANSPORT_EP_BITMAP_BITS: usize = (crate::MAX_VALUE_DATA_ENDPOINT as usize) + 1;
88const SIDE_TRANSPORT_EP_BITMAP_BYTES: usize = SIDE_TRANSPORT_EP_BITMAP_BITS.div_ceil(8);
89pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
90pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
91pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
92static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
93
94#[derive(Clone, Copy, Debug, PartialEq, Eq)]
95pub enum SideTransportProfile {
96 Canonical,
97 Template,
98 Ipv6Like,
99 Ipv4Like,
100}
101
102impl SideTransportProfile {
103 #[inline]
104 pub const fn as_str(self) -> &'static str {
105 match self {
106 Self::Canonical => "canonical",
107 Self::Template => "template",
108 Self::Ipv6Like => "ipv6_like",
109 Self::Ipv4Like => "ipv4_like",
110 }
111 }
112
113 #[cfg(feature = "discovery")]
114 #[inline]
115 pub const fn discovery_code(self) -> u8 {
116 match self {
117 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
118 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
119 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
120 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
121 }
122 }
123}
124
125#[derive(Clone, Debug, PartialEq, Eq)]
126struct SideHeaderTemplate {
127 hash: u64,
128 base_flags: u8,
129 prefix: Arc<[u8]>,
130 between: Arc<[u8]>,
131 reliable_flags: Option<u8>,
132 reliable_compact: bool,
133}
134
135#[derive(Clone, Debug, Default, PartialEq, Eq)]
136struct SideChunkAssembly {
137 total: u16,
138 received: BTreeMap<u16, Arc<[u8]>>,
139}
140
141#[derive(Clone, Debug, Default)]
142struct SideTransportState {
143 tx_template_ids: BTreeMap<u64, u32>,
144 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
145 tx_last_timestamps: BTreeMap<u32, u64>,
146 tx_compact_uses: BTreeMap<u32, u8>,
147 rx_templates: BTreeMap<u64, SideHeaderTemplate>,
148 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
149 rx_last_timestamps: BTreeMap<u32, u64>,
150 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
151 next_chunk_id: u32,
152 next_template_id: u32,
153}
154
155impl SideTransportState {
156 fn clear_header_templates(&mut self) {
157 self.tx_template_ids.clear();
158 self.tx_templates.clear();
159 self.tx_last_timestamps.clear();
160 self.tx_compact_uses.clear();
161 self.rx_templates.clear();
162 self.rx_templates_by_id.clear();
163 self.rx_last_timestamps.clear();
164 }
165
166 fn tx_template_count(&self) -> usize {
167 self.tx_template_ids.len()
168 }
169
170 fn rx_template_count(&self) -> usize {
171 self.rx_templates_by_id.len()
172 }
173
174 fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
175 let mut removed = 0usize;
176 while self.tx_template_ids.len() > limit {
177 let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
178 break;
179 };
180 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
181 self.tx_last_timestamps.remove(&old_id);
182 self.tx_compact_uses.remove(&old_id);
183 }
184 self.tx_templates.remove(&old_hash);
185 removed += 1;
186 }
187 removed
188 }
189
190 fn insert_tx_template(
191 &mut self,
192 template: SideHeaderTemplate,
193 template_id: u32,
194 max_templates: usize,
195 ) -> bool {
196 if max_templates == 0 {
197 return false;
198 }
199 let evicted = !self.tx_template_ids.contains_key(&template.hash)
200 && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
201 self.tx_template_ids.insert(template.hash, template_id);
202 self.tx_templates.insert(template.hash, template);
203 self.tx_compact_uses.insert(template_id, 0);
204 evicted
205 }
206
207 fn insert_rx_template(
208 &mut self,
209 template_id: u32,
210 template: SideHeaderTemplate,
211 max_templates: usize,
212 ) -> bool {
213 if max_templates == 0 {
214 return false;
215 }
216 let mut evicted = false;
217 if self.rx_templates_by_id.len() >= max_templates
218 && !self.rx_templates_by_id.contains_key(&template_id)
219 && let Some(old_hash) = self.rx_templates.keys().next().copied()
224 && let Some(old_id) = self.rx_templates_by_id.iter().find_map(|(id, existing)| {
225 (existing.hash == old_hash).then_some(*id)
226 })
227 && let Some(old_template) = self.rx_templates_by_id.remove(&old_id)
228 {
229 self.rx_templates.remove(&old_template.hash);
230 self.rx_last_timestamps.remove(&old_id);
231 evicted = true;
232 }
233 self.rx_templates_by_id
234 .insert(template_id, template.clone());
235 self.rx_templates.insert(template.hash, template);
236 evicted
237 }
238}
239
240type SideTemplateExtract<'a> = (
241 SideHeaderTemplate,
242 DataType,
243 u8,
244 u64,
245 u64,
246 u16,
247 Option<(u32, u32)>,
248 &'a [u8],
249);
250
251#[derive(Clone, Copy, Debug, PartialEq, Eq)]
252pub struct CompactTimestampOmissionPolicy {
253 all: bool,
254 words: [u64; SIDE_TIMESTAMP_POLICY_WORDS],
255}
256
257impl CompactTimestampOmissionPolicy {
258 #[inline]
259 pub const fn none() -> Self {
260 Self {
261 all: false,
262 words: [0; SIDE_TIMESTAMP_POLICY_WORDS],
263 }
264 }
265
266 #[inline]
267 pub const fn all() -> Self {
268 Self {
269 all: true,
270 words: [0; SIDE_TIMESTAMP_POLICY_WORDS],
271 }
272 }
273
274 #[inline]
275 pub fn with_data_type(mut self, ty: DataType) -> Self {
276 self.insert(ty);
277 self
278 }
279
280 #[inline]
281 pub fn insert(&mut self, ty: DataType) {
282 let id = ty.as_u32() as usize;
283 if id <= crate::MAX_VALUE_DATA_TYPE as usize {
284 self.words[id / 64] |= 1u64 << (id % 64);
285 }
286 }
287
288 #[inline]
289 pub fn contains(self, ty: DataType) -> bool {
290 if self.all {
291 return true;
292 }
293 let id = ty.as_u32() as usize;
294 id <= crate::MAX_VALUE_DATA_TYPE as usize
295 && (self.words[id / 64] & (1u64 << (id % 64))) != 0
296 }
297
298 #[inline]
299 pub fn is_empty(self) -> bool {
300 !self.all && self.words.iter().all(|word| *word == 0)
301 }
302}
303
304impl Default for CompactTimestampOmissionPolicy {
305 #[inline]
306 fn default() -> Self {
307 Self::none()
308 }
309}
310
311#[derive(Clone, Copy, Debug, PartialEq, Eq)]
312enum SideCompactTimestampMode {
313 Absolute,
314 Delta,
315 Omitted,
316}
317
318#[derive(Clone, Debug, PartialEq, Eq)]
319pub enum RouterItem {
320 Packet(Packet),
321 Packed(Arc<[u8]>),
322}
323
324#[derive(Clone, Debug, PartialEq, Eq)]
325struct RouterRxItem {
326 src: Option<RouterSideId>,
327 data: RouterItem,
328 priority: u8,
329}
330
331#[derive(Clone, Debug, PartialEq, Eq)]
332enum RouterTxItem {
333 Broadcast(RouterItem),
334 EndToEndReplay {
335 packet_id: u64,
336 },
337 ToSide {
338 src: Option<RouterSideId>,
339 dst: RouterSideId,
340 data: RouterItem,
341 },
342 ReliableReplay {
343 dst: RouterSideId,
344 bytes: Arc<[u8]>,
345 },
346}
347
348impl ByteCost for RouterRxItem {
349 #[inline]
350 fn byte_cost(&self) -> usize {
351 match &self.data {
352 RouterItem::Packet(pkt) => pkt.byte_cost(),
353 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
354 }
355 }
356}
357
358impl ByteCost for RouterTxItem {
359 #[inline]
360 fn byte_cost(&self) -> usize {
361 match self {
362 RouterTxItem::Broadcast(data) => match data {
363 RouterItem::Packet(pkt) => pkt.byte_cost(),
364 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
365 },
366 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
367 RouterTxItem::ToSide { data, .. } => match data {
368 RouterItem::Packet(pkt) => pkt.byte_cost(),
369 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
370 },
371 RouterTxItem::ReliableReplay { bytes, .. } => size_of::<Arc<[u8]>>() + bytes.len(),
372 }
373 }
374}
375
376#[derive(Clone, Debug, PartialEq, Eq)]
380struct TxQueued {
381 item: RouterTxItem,
382 ignore_local: bool,
383 priority: u8,
384}
385
386const MAX_PRIORITY_BURST: u8 = 8;
391
392impl ByteCost for TxQueued {
394 #[inline]
396 fn byte_cost(&self) -> usize {
397 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>()
398 }
399}
400
401impl ByteCost for u64 {
403 #[inline]
405 fn byte_cost(&self) -> usize {
406 size_of::<u64>()
407 }
408}
409
410#[derive(Debug, Clone)]
413struct ReliableTxState {
414 next_seq: u32,
415 sent_order: VecDeque<u32>,
416 sent: BTreeMap<u32, ReliableSent>,
417}
418
419#[derive(Debug, Clone)]
420struct ReliableSent {
421 bytes: Arc<[u8]>,
422 last_send_ms: u64,
423 retries: u32,
424 queued: bool,
425 partial_acked: bool,
426}
427
428#[derive(Debug, Clone)]
429struct EndToEndReliableSent {
430 data: RouterItem,
431 pending_destinations: BTreeMap<u64, RouterSideId>,
432 tracked_destinations: bool,
433 last_send_ms: u64,
434 retries: u32,
435 queued: bool,
436}
437
438#[derive(Debug, Clone)]
439struct ReliableRxState {
440 expected_seq: u32,
441 buffered: BTreeMap<u32, Arc<[u8]>>,
442}
443
444#[derive(Debug, Clone)]
445struct ReliableReturnRouteState {
446 side: RouterSideId,
447}
448
449#[cfg(feature = "discovery")]
450#[derive(Debug, Clone, Default, PartialEq, Eq)]
451struct DiscoverySenderState {
452 advertised_address: Option<u32>,
455 reachable: Vec<DataEndpoint>,
456 advertised_reachable: Vec<DataEndpoint>,
457 reachable_network_variables: Vec<DataType>,
458 reachable_timesync_sources: Vec<String>,
459 advertised_reachable_timesync_sources: Vec<String>,
460 link_capabilities: Option<discovery::LinkCapabilities>,
461 topology_boards: Vec<TopologyBoardNode>,
462 last_seen_ms: u64,
463}
464
465#[cfg(feature = "discovery")]
466#[derive(Debug, Clone, Default, PartialEq, Eq)]
467struct DiscoverySideState {
468 reachable: Vec<DataEndpoint>,
469 reachable_network_variables: Vec<DataType>,
470 requested_network_variables: Vec<(DataType, u64)>,
471 reachable_timesync_sources: Vec<String>,
472 last_seen_ms: u64,
473 announcers: BTreeMap<String, DiscoverySenderState>,
474}
475
476#[cfg(feature = "discovery")]
477#[derive(Debug, Clone, Default)]
478struct DiscoverySideThrottleState {
479 next_ping_ms: u64,
480 next_full_ms: u64,
481 pending_incremental: bool,
482 incremental_retries_remaining: u8,
483 has_sent_full: bool,
484 last_topology: Vec<TopologyBoardNode>,
485}
486
487#[cfg(all(feature = "discovery", feature = "timesync"))]
488#[derive(Debug, Clone, Default)]
489struct TimeSyncSideThrottleState {
490 next_allowed_ms: u64,
491}
492
493#[cfg(feature = "discovery")]
494#[derive(Debug, Clone, Copy, PartialEq, Eq)]
495enum DiscoveryAdvertiseLevel {
496 MinimalPing,
497 Incremental,
498 Full,
499}
500
501#[derive(Debug, Clone, Default)]
502struct AdaptiveRouteStats {
503 estimated_bandwidth_bps: u64,
504 peak_bandwidth_bps: u64,
505 last_observed_ms: u64,
506 last_slow_observed_ms: u64,
507 sample_count: u64,
508 window_started_ms: u64,
509 window_bytes: u64,
510 peak_usage_bps: u64,
511}
512
513impl AdaptiveRouteStats {
514 #[inline]
515 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
516 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
517 sample_bps
518 } else if sample_bps >= self.estimated_bandwidth_bps {
519 self.estimated_bandwidth_bps
520 .saturating_mul(3)
521 .saturating_add(sample_bps.saturating_mul(5))
522 / 8
523 } else {
524 self.estimated_bandwidth_bps
525 .saturating_mul(7)
526 .saturating_add(sample_bps)
527 / 8
528 };
529 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
530 self.last_observed_ms = now_ms;
531 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
532 self.last_slow_observed_ms = now_ms;
533 }
534 self.sample_count = self.sample_count.saturating_add(1);
535 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
536 self.window_started_ms = now_ms;
537 self.window_bytes = 0;
538 }
539 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
540 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
541 }
542
543 #[inline]
544 fn current_usage_bps(&self, now_ms: u64) -> u64 {
545 if self.window_started_ms == 0 {
546 return 0;
547 }
548 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
549 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
550 .min(u128::from(u64::MAX)) as u64
551 }
552
553 #[inline]
554 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
555 let capacity = self
556 .estimated_bandwidth_bps
557 .max(self.peak_bandwidth_bps)
558 .max(1);
559 capacity.saturating_sub(self.current_usage_bps(now_ms))
560 }
561
562 #[inline]
563 fn weight(&self, now_ms: u64) -> u64 {
564 self.available_headroom_bps(now_ms).max(1)
565 }
566
567 #[inline]
568 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
569 let current_usage_bps = self.current_usage_bps(now_ms);
570 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
571 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
572 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
573 AdaptiveLinkStats {
574 auto_balancing_enabled,
575 estimated_capacity_bps,
576 peak_capacity_bps,
577 current_usage_bps,
578 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
579 available_headroom_bps,
580 effective_weight: available_headroom_bps.max(1),
581 last_observed_ms: self.last_observed_ms,
582 sample_count: self.sample_count,
583 }
584 }
585}
586
587#[derive(Debug, Clone, Default)]
588struct TypeRuntimeStatsInner {
589 tx_packets: u64,
590 tx_bytes: u64,
591 rx_packets: u64,
592 rx_bytes: u64,
593 relayed_tx_packets: u64,
594 relayed_tx_bytes: u64,
595 relayed_rx_packets: u64,
596 relayed_rx_bytes: u64,
597 tx_retries: u64,
598 handler_failures: u64,
599}
600
601#[derive(Debug, Clone, Default)]
602struct SideRuntimeStatsInner {
603 tx_packets: u64,
604 tx_bytes: u64,
605 rx_packets: u64,
606 rx_bytes: u64,
607 relayed_tx_packets: u64,
608 relayed_tx_bytes: u64,
609 relayed_rx_packets: u64,
610 relayed_rx_bytes: u64,
611 local_delivery_packets: u64,
612 tx_retries: u64,
613 tx_handler_failures: u64,
614 local_handler_failures: u64,
615 total_handler_retries: u64,
616 side_transport_full_frames: u64,
617 side_transport_compact_frames: u64,
618 side_transport_compact_delta_frames: u64,
619 side_transport_compact_omitted_timestamp_frames: u64,
620 side_transport_chunk_frames: u64,
621 side_transport_raw_bytes: u64,
622 side_transport_wire_bytes: u64,
623 side_transport_bytes_saved: u64,
624 side_transport_min_compact_overhead_bytes: Option<usize>,
625 side_transport_max_compact_overhead_bytes: Option<usize>,
626 side_transport_compact_target_misses: u64,
627 side_transport_template_evictions: u64,
628 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
629}
630
631impl SideRuntimeStatsInner {
632 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
633 self.data_types.entry(ty.as_u32()).or_default()
634 }
635
636 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
637 self.tx_packets = self.tx_packets.saturating_add(1);
638 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
639 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
640 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
641 if relayed {
642 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
643 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
644 }
645 let stats = self.type_stats_mut(ty);
646 stats.tx_packets = stats.tx_packets.saturating_add(1);
647 stats.tx_bytes = stats.tx_bytes.saturating_add(bytes as u64);
648 stats.tx_retries = stats.tx_retries.saturating_add(retries as u64);
649 if relayed {
650 stats.relayed_tx_packets = stats.relayed_tx_packets.saturating_add(1);
651 stats.relayed_tx_bytes = stats.relayed_tx_bytes.saturating_add(bytes as u64);
652 }
653 }
654
655 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
656 self.rx_packets = self.rx_packets.saturating_add(1);
657 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
658 if relayed {
659 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
660 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
661 }
662 let stats = self.type_stats_mut(ty);
663 stats.rx_packets = stats.rx_packets.saturating_add(1);
664 stats.rx_bytes = stats.rx_bytes.saturating_add(bytes as u64);
665 if relayed {
666 stats.relayed_rx_packets = stats.relayed_rx_packets.saturating_add(1);
667 stats.relayed_rx_bytes = stats.relayed_rx_bytes.saturating_add(bytes as u64);
668 }
669 }
670
671 fn note_local_delivery(&mut self, ty: DataType) {
672 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
673 let stats = self.type_stats_mut(ty);
674 stats.rx_packets = stats.rx_packets.saturating_add(1);
675 }
676
677 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
678 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
679 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
680 let stats = self.type_stats_mut(ty);
681 stats.handler_failures = stats.handler_failures.saturating_add(1);
682 }
683
684 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
685 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
686 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
687 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
688 let stats = self.type_stats_mut(ty);
689 stats.handler_failures = stats.handler_failures.saturating_add(1);
690 stats.tx_retries = stats.tx_retries.saturating_add(retries as u64);
691 }
692
693 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
694 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
695 self.note_side_transport_bytes(raw_bytes, wire_bytes);
696 }
697
698 fn note_side_transport_compact(
699 &mut self,
700 raw_bytes: usize,
701 wire_bytes: usize,
702 compact_overhead_bytes: usize,
703 used_timestamp_delta: bool,
704 omitted_timestamp: bool,
705 ) {
706 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
707 if used_timestamp_delta {
708 self.side_transport_compact_delta_frames =
709 self.side_transport_compact_delta_frames.saturating_add(1);
710 }
711 if omitted_timestamp {
712 self.side_transport_compact_omitted_timestamp_frames = self
713 .side_transport_compact_omitted_timestamp_frames
714 .saturating_add(1);
715 }
716 self.note_side_transport_bytes(raw_bytes, wire_bytes);
717 self.side_transport_min_compact_overhead_bytes = Some(
718 self.side_transport_min_compact_overhead_bytes
719 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
720 );
721 self.side_transport_max_compact_overhead_bytes = Some(
722 self.side_transport_max_compact_overhead_bytes
723 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
724 );
725 }
726
727 fn note_side_transport_chunks(&mut self, chunks: usize) {
728 self.side_transport_chunk_frames = self
729 .side_transport_chunk_frames
730 .saturating_add(chunks as u64);
731 }
732
733 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
734 self.side_transport_raw_bytes = self
735 .side_transport_raw_bytes
736 .saturating_add(raw_bytes as u64);
737 self.side_transport_wire_bytes = self
738 .side_transport_wire_bytes
739 .saturating_add(wire_bytes as u64);
740 if raw_bytes > wire_bytes {
741 self.side_transport_bytes_saved = self
742 .side_transport_bytes_saved
743 .saturating_add((raw_bytes - wire_bytes) as u64);
744 }
745 }
746
747 fn note_side_transport_compact_target_miss(&mut self) {
748 self.side_transport_compact_target_misses =
749 self.side_transport_compact_target_misses.saturating_add(1);
750 }
751
752 fn note_side_transport_template_eviction(&mut self) {
753 self.side_transport_template_evictions =
754 self.side_transport_template_evictions.saturating_add(1);
755 }
756}
757
758#[derive(Clone, Copy, Debug, PartialEq, Eq)]
759enum RouteSelectionOrigin {
760 Flood,
761 Discovered,
762}
763
764type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
767
768type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
770type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
771
772#[derive(Clone)]
778pub enum EndpointHandlerFn {
779 Packet(Arc<PacketHandlerFn>),
780 Packed(Arc<PackedHandlerFn>),
781}
782
783pub struct EndpointHandler {
785 endpoint: DataEndpoint,
786 handler: EndpointHandlerFn,
787}
788
789impl Debug for EndpointHandlerFn {
791 #[inline]
792 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
793 match self {
794 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
795 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
796 }
797 }
798}
799
800#[derive(Clone)]
802pub enum RouterTxHandlerFn {
803 Packed(Arc<PackedHandlerFn>),
804 PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
805 Packet(Arc<PacketHandlerFn>),
806}
807
808impl Debug for RouterTxHandlerFn {
809 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
810 match self {
811 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
812 RouterTxHandlerFn::PackedWithPriority(_) => f
813 .debug_tuple("PackedWithPriority")
814 .field(&"<handler>")
815 .finish(),
816 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
817 }
818 }
819}
820
821#[derive(Clone, Copy, Debug)]
822pub struct RouterSideOptions {
823 pub reliable_enabled: bool,
834 pub link_local_enabled: bool,
836 pub header_template_enabled: bool,
843 pub max_frame_bytes: usize,
849 pub compact_header_target_bytes: usize,
856 pub max_side_transport_templates: usize,
862 pub omit_unchanged_compact_timestamps: bool,
868 pub compact_timestamp_omission_types: CompactTimestampOmissionPolicy,
873 pub side_transport_profile: SideTransportProfile,
875 pub ingress_enabled: bool,
877 pub egress_enabled: bool,
879}
880
881impl Default for RouterSideOptions {
882 fn default() -> Self {
883 Self {
884 reliable_enabled: false,
885 link_local_enabled: false,
886 header_template_enabled: false,
887 max_frame_bytes: 0,
888 compact_header_target_bytes: 0,
889 max_side_transport_templates: DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT,
890 omit_unchanged_compact_timestamps: false,
891 compact_timestamp_omission_types: CompactTimestampOmissionPolicy::none(),
892 side_transport_profile: SideTransportProfile::Canonical,
893 ingress_enabled: true,
894 egress_enabled: true,
895 }
896 }
897}
898
899impl RouterSideOptions {
900 #[inline]
905 pub fn with_small_packet_transport(mut self, max_frame_bytes: usize) -> Self {
906 self.header_template_enabled = true;
907 self.max_frame_bytes = max_frame_bytes;
908 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
909 self.side_transport_profile = SideTransportProfile::Ipv6Like;
910 self
911 }
912
913 #[inline]
914 pub fn with_ipv4_like_compact_header_target(mut self) -> Self {
915 self.header_template_enabled = true;
916 self.compact_header_target_bytes = IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES;
917 self.omit_unchanged_compact_timestamps = true;
918 self.side_transport_profile = SideTransportProfile::Ipv4Like;
919 self
920 }
921
922 #[inline]
923 pub fn with_ipv6_like_compact_header_target(mut self) -> Self {
924 self.header_template_enabled = true;
925 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
926 self.side_transport_profile = SideTransportProfile::Ipv6Like;
927 self
928 }
929
930 #[inline]
931 pub fn with_template_transport(mut self) -> Self {
932 self.header_template_enabled = true;
933 self.side_transport_profile = SideTransportProfile::Template;
934 self
935 }
936
937 #[inline]
938 pub fn with_omitted_unchanged_compact_timestamps(mut self) -> Self {
939 self.header_template_enabled = true;
940 self.omit_unchanged_compact_timestamps = true;
941 self
942 }
943
944 #[inline]
945 pub fn with_omitted_unchanged_compact_timestamps_for_type(mut self, ty: DataType) -> Self {
946 self.header_template_enabled = true;
947 self.compact_timestamp_omission_types.insert(ty);
948 self
949 }
950
951 #[inline]
952 pub fn effective_transport_profile(self) -> SideTransportProfile {
953 if !self.header_template_enabled && self.max_frame_bytes == 0 {
954 SideTransportProfile::Canonical
955 } else if self.side_transport_profile == SideTransportProfile::Canonical {
956 SideTransportProfile::Template
957 } else {
958 self.side_transport_profile
959 }
960 }
961
962 #[cfg(feature = "discovery")]
963 #[inline]
964 pub fn link_capabilities(self) -> discovery::LinkCapabilities {
965 let mut flags = discovery::LINK_CAPABILITY_END_TO_END_RELIABILITY;
966 if self.header_template_enabled {
967 flags |= discovery::LINK_CAPABILITY_HEADER_TEMPLATES;
968 }
969 if self.max_frame_bytes != 0 {
970 flags |= discovery::LINK_CAPABILITY_CHUNKING;
971 }
972 if self.reliable_enabled {
973 flags |= discovery::LINK_CAPABILITY_RELIABILITY;
974 }
975 if self.omit_unchanged_compact_timestamps
976 || !self.compact_timestamp_omission_types.is_empty()
977 {
978 flags |= discovery::LINK_CAPABILITY_OMIT_UNCHANGED_TIMESTAMPS;
979 }
980 #[cfg(feature = "cryptography")]
981 {
982 flags |= discovery::LINK_CAPABILITY_CRYPTO;
983 }
984 discovery::LinkCapabilities {
985 version: 1,
986 flags,
987 profile: self.effective_transport_profile().discovery_code(),
988 max_frame_bytes: self.max_frame_bytes.min(u32::MAX as usize) as u32,
989 compact_header_target_bytes: self.compact_header_target_bytes.min(u32::MAX as usize)
990 as u32,
991 max_side_transport_templates: self.max_side_transport_templates.min(u32::MAX as usize)
992 as u32,
993 }
994 }
995}
996
997#[derive(Clone, Debug)]
999pub struct RouterSide {
1000 pub name: Arc<str>,
1001 pub tx_handler: RouterTxHandlerFn,
1002 pub opts: RouterSideOptions,
1003}
1004
1005impl Debug for EndpointHandler {
1007 #[inline]
1008 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1009 f.debug_struct("EndpointHandler")
1010 .field("endpoint", &self.endpoint)
1011 .field("handler", &self.handler)
1012 .finish()
1013 }
1014}
1015
1016#[inline]
1017pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1018 #[cfg(feature = "timesync")]
1019 if matches!(endpoint, DataEndpoint::TimeSync) {
1020 return true;
1021 }
1022 discovery::is_discovery_endpoint(endpoint)
1023}
1024
1025impl EndpointHandler {
1026 #[inline]
1030 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1031 where
1032 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1033 {
1034 assert!(
1035 !endpoint_is_router_internal(endpoint),
1036 "reserved internal endpoint handlers must not be user-registered"
1037 );
1038 #[cfg(feature = "std")]
1039 crate::config::ensure_endpoint_id(endpoint, false)
1040 .expect("endpoint handler endpoint registration failed");
1041 Self {
1042 endpoint,
1043 handler: EndpointHandlerFn::Packet(Arc::new(f)),
1044 }
1045 }
1046
1047 #[inline]
1049 #[cfg(feature = "std")]
1050 pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1051 where
1052 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1053 {
1054 Self::new_packet_handler(endpoint.id, f)
1055 }
1056
1057 #[cfg(not(feature = "std"))]
1059 #[inline]
1060 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1061 where
1062 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1063 {
1064 Self::new_packet_handler(endpoint.id, f)
1065 }
1066
1067 #[cfg(feature = "std")]
1069 #[inline]
1070 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1071 where
1072 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1073 {
1074 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1075 .ok_or(TelemetryError::BadArg)?;
1076 Ok(Self::new_packet_handler(endpoint.id, f))
1077 }
1078
1079 #[inline]
1083 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1084 where
1085 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1086 {
1087 assert!(
1088 !endpoint_is_router_internal(endpoint),
1089 "reserved internal endpoint handlers must not be user-registered"
1090 );
1091 #[cfg(feature = "std")]
1092 crate::config::ensure_endpoint_id(endpoint, false)
1093 .expect("endpoint handler endpoint registration failed");
1094 Self {
1095 endpoint,
1096 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1097 }
1098 }
1099
1100 #[inline]
1102 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1103 where
1104 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1105 {
1106 Self::new_packed_handler(endpoint.id, f)
1107 }
1108
1109 #[cfg(feature = "std")]
1111 #[inline]
1112 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1113 where
1114 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1115 {
1116 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1117 .ok_or(TelemetryError::BadArg)?;
1118 Ok(Self::new_packed_handler(endpoint.id, f))
1119 }
1120
1121 #[inline]
1123 pub fn get_endpoint(&self) -> DataEndpoint {
1124 self.endpoint
1125 }
1126
1127 #[inline]
1129 pub fn get_handler(&self) -> &EndpointHandlerFn {
1130 &self.handler
1131 }
1132}
1133
1134pub trait Clock {
1135 fn now_ms(&self) -> u64;
1137
1138 fn now_ns(&self) -> u64 {
1142 self.now_ms().saturating_mul(1_000_000)
1143 }
1144}
1145
1146impl<T: Fn() -> u64> Clock for T {
1147 #[inline]
1148 fn now_ms(&self) -> u64 {
1149 self()
1150 }
1151}
1152
1153#[cfg(feature = "std")]
1154#[derive(Debug)]
1155struct StdMonotonicClock {
1156 start: Instant,
1157}
1158
1159#[cfg(feature = "std")]
1160impl Default for StdMonotonicClock {
1161 fn default() -> Self {
1162 Self {
1163 start: Instant::now(),
1164 }
1165 }
1166}
1167
1168#[cfg(feature = "std")]
1169impl Clock for StdMonotonicClock {
1170 fn now_ms(&self) -> u64 {
1171 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1172 }
1173
1174 fn now_ns(&self) -> u64 {
1175 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1176 }
1177}
1178
1179#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1181pub enum RouterE2eEncryptionMode {
1182 Disabled,
1184 RequiredOnly,
1186 Preferred,
1188 ForceAll,
1190}
1191
1192pub type NodeAddress = u32;
1193pub type P2pPort = u16;
1194pub type P2pStreamId = u32;
1195
1196const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1197const P2P_STREAM_VERSION: u8 = 1;
1198const P2P_STREAM_SYN: u8 = 0x01;
1199const P2P_STREAM_ACK: u8 = 0x02;
1200const P2P_STREAM_FIN: u8 = 0x04;
1201const P2P_STREAM_RST: u8 = 0x08;
1202const P2P_STREAM_DATA: u8 = 0x10;
1203
1204#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1205pub enum AddressAssignmentMode {
1206 Dynamic,
1207 Requested(NodeAddress),
1208 Static(NodeAddress),
1209}
1210
1211impl AddressAssignmentMode {
1212 #[inline]
1213 fn mode_code(self) -> u8 {
1214 match self {
1215 Self::Dynamic => 0,
1216 Self::Requested(_) => 1,
1217 Self::Static(_) => 2,
1218 }
1219 }
1220
1221 #[inline]
1222 fn requested_address(self) -> NodeAddress {
1223 match self {
1224 Self::Dynamic => 0,
1225 Self::Requested(addr) | Self::Static(addr) => addr,
1226 }
1227 }
1228}
1229
1230#[derive(Debug, Clone, PartialEq, Eq)]
1231pub enum AddressChangeReason {
1232 Configured,
1233 DynamicConflict,
1234 RequestedConflict,
1235 StaticConflict,
1236 HostnameConflict,
1237}
1238
1239#[derive(Debug, Clone, PartialEq, Eq)]
1240pub struct AddressChange {
1241 pub old_hostname: Arc<str>,
1242 pub new_hostname: Arc<str>,
1243 pub old_address: NodeAddress,
1244 pub new_address: NodeAddress,
1245 pub reason: AddressChangeReason,
1246}
1247
1248#[derive(Debug, Clone, PartialEq, Eq)]
1249pub struct AddressBookEntry {
1250 pub hostname: Arc<str>,
1251 pub address: NodeAddress,
1252 pub requested_address: NodeAddress,
1253 pub mode: AddressAssignmentMode,
1254 pub birth_ms: u64,
1255 pub owner_hash: u64,
1256 pub last_seen_ms: u64,
1257}
1258
1259pub struct P2pMessage<'a> {
1260 pub source_hostname: &'a str,
1261 pub source_address: NodeAddress,
1262 pub source_port: P2pPort,
1263 pub destination_port: P2pPort,
1264 pub payload: &'a [u8],
1265}
1266
1267#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1268pub enum P2pStreamEventKind {
1269 Accepted,
1270 Connected,
1271 Data,
1272 Closed,
1273 Reset,
1274}
1275
1276pub struct P2pStreamEvent<'a> {
1277 pub kind: P2pStreamEventKind,
1278 pub stream_id: P2pStreamId,
1279 pub peer_stream_id: P2pStreamId,
1280 pub sequence: u32,
1281 pub peer_hostname: &'a str,
1282 pub peer_address: NodeAddress,
1283 pub local_port: P2pPort,
1284 pub peer_port: P2pPort,
1285 pub payload: &'a [u8],
1286}
1287
1288struct P2pDecoded<'a> {
1289 source_hostname: &'a str,
1290 source_address: NodeAddress,
1291 source_port: P2pPort,
1292 destination_port: P2pPort,
1293 payload: &'a [u8],
1294}
1295
1296#[derive(Debug, Clone)]
1297struct P2pStreamDecoded<'a> {
1298 flags: u8,
1299 source_stream_id: P2pStreamId,
1300 destination_stream_id: P2pStreamId,
1301 sequence: u32,
1302 payload: &'a [u8],
1303}
1304
1305type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1306
1307#[derive(Clone)]
1308struct AddressChangeHandler {
1309 handler: Arc<AddressChangeFn>,
1310}
1311
1312impl Debug for AddressChangeHandler {
1313 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1314 f.write_str("AddressChangeHandler(<handler>)")
1315 }
1316}
1317
1318type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1319
1320#[derive(Clone)]
1321struct P2pPortHandler {
1322 handler: Arc<P2pPortHandlerFn>,
1323}
1324
1325impl Debug for P2pPortHandler {
1326 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1327 f.write_str("P2pPortHandler(<handler>)")
1328 }
1329}
1330
1331type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1332
1333#[derive(Clone)]
1334struct P2pStreamHandler {
1335 handler: Arc<P2pStreamHandlerFn>,
1336}
1337
1338impl Debug for P2pStreamHandler {
1339 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1340 f.write_str("P2pStreamHandler(<handler>)")
1341 }
1342}
1343
1344#[derive(Debug, Clone)]
1345struct P2pStreamSession {
1346 peer_hostname: Arc<str>,
1347 peer_address: NodeAddress,
1348 local_port: P2pPort,
1349 peer_port: P2pPort,
1350 peer_stream_id: P2pStreamId,
1351 next_sequence: u32,
1352 connected: bool,
1353}
1354
1355#[derive(Debug, Clone)]
1356struct PendingP2pStreamEvent {
1357 handlers: Vec<P2pStreamHandler>,
1358 kind: P2pStreamEventKind,
1359 stream_id: P2pStreamId,
1360 peer_stream_id: P2pStreamId,
1361 peer_hostname: Arc<str>,
1362 peer_address: NodeAddress,
1363 local_port: P2pPort,
1364 peer_port: P2pPort,
1365}
1366
1367#[derive(Debug, Clone)]
1368pub struct RouterConfig {
1369 handlers: Arc<[EndpointHandler]>,
1371 reliable_enabled: bool,
1373 sender: Option<Arc<str>>,
1375 address_mode: AddressAssignmentMode,
1377 address_change_handlers: Arc<[AddressChangeHandler]>,
1379 e2e_encryption: RouterE2eEncryptionMode,
1381 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
1383 e2e_key_id: u32,
1384 memory: RuntimeMemoryConfig,
1385 #[cfg(feature = "timesync")]
1386 timesync: Option<TimeSyncConfig>,
1387}
1388
1389impl RouterConfig {
1390 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
1395 #[cfg(feature = "cryptography")]
1396 {
1397 RouterE2eEncryptionMode::Preferred
1398 }
1399 #[cfg(not(feature = "cryptography"))]
1400 {
1401 RouterE2eEncryptionMode::Disabled
1402 }
1403 }
1404
1405 pub fn new<H>(handlers: H) -> Self
1407 where
1408 H: Into<Arc<[EndpointHandler]>>,
1409 {
1410 let handlers: Arc<[EndpointHandler]> = handlers.into();
1411 assert!(
1412 handlers
1413 .iter()
1414 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
1415 "reserved internal endpoint handlers must not be user-registered"
1416 );
1417 Self {
1418 handlers,
1419 reliable_enabled: true,
1420 sender: None,
1421 address_mode: AddressAssignmentMode::Dynamic,
1422 address_change_handlers: Arc::from([]),
1423 e2e_encryption: Self::default_e2e_encryption_mode(),
1424 e2e_key_id: 0,
1425 memory: RuntimeMemoryConfig::default(),
1426 #[cfg(feature = "timesync")]
1427 timesync: None,
1428 }
1429 }
1430
1431 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
1433 self.reliable_enabled = enabled;
1434 self
1435 }
1436
1437 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
1439 self.sender = Some(Arc::from(sender.as_ref()));
1440 self
1441 }
1442
1443 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
1445 self.with_sender(hostname)
1446 }
1447
1448 pub fn with_dynamic_address(mut self) -> Self {
1450 self.address_mode = AddressAssignmentMode::Dynamic;
1451 self
1452 }
1453
1454 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
1456 self.address_mode = AddressAssignmentMode::Requested(address);
1457 self
1458 }
1459
1460 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
1462 self.address_mode = AddressAssignmentMode::Static(address);
1463 self
1464 }
1465
1466 pub fn on_address_change<F>(mut self, f: F) -> Self
1468 where
1469 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
1470 {
1471 let mut handlers = self.address_change_handlers.to_vec();
1472 handlers.push(AddressChangeHandler {
1473 handler: Arc::new(f),
1474 });
1475 self.address_change_handlers = Arc::from(handlers);
1476 self
1477 }
1478
1479 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
1481 self.e2e_encryption = mode;
1482 self
1483 }
1484
1485 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
1487 self.e2e_key_id = key_id;
1488 self
1489 }
1490
1491 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
1493 memory.validate()?;
1494 self.memory = memory;
1495 Ok(self)
1496 }
1497
1498 #[cfg(feature = "timesync")]
1499 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
1501 self.timesync = Some(cfg);
1502 self
1503 }
1504
1505 #[inline]
1506 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1508 if endpoint_is_router_internal(ep) {
1509 return false;
1510 }
1511 self.handlers.iter().any(|h| h.endpoint == ep)
1512 }
1513
1514 #[inline]
1515 fn reliable_enabled(&self) -> bool {
1516 self.reliable_enabled
1517 }
1518
1519 #[inline]
1520 fn sender(&self) -> String {
1521 self.sender
1522 .as_deref()
1523 .map(ToString::to_string)
1524 .unwrap_or_else(runtime_device_identifier)
1525 }
1526
1527 #[inline]
1528 fn address_mode(&self) -> AddressAssignmentMode {
1529 self.address_mode
1530 }
1531
1532 #[inline]
1533 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1534 self.e2e_encryption
1535 }
1536
1537 #[cfg(feature = "cryptography")]
1538 #[inline]
1539 fn e2e_key_id(&self) -> u32 {
1540 self.e2e_key_id
1541 }
1542
1543 #[cfg(feature = "timesync")]
1544 #[inline]
1545 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1546 self.timesync
1547 }
1548
1549 #[inline]
1550 fn memory_config(&self) -> RuntimeMemoryConfig {
1551 self.memory
1552 }
1553}
1554
1555impl Default for RouterConfig {
1556 fn default() -> Self {
1557 Self {
1558 handlers: Arc::from([]),
1559 reliable_enabled: true,
1560 sender: None,
1561 address_mode: AddressAssignmentMode::Dynamic,
1562 address_change_handlers: Arc::from([]),
1563 e2e_encryption: Self::default_e2e_encryption_mode(),
1564 e2e_key_id: 0,
1565 memory: RuntimeMemoryConfig::default(),
1566 #[cfg(feature = "timesync")]
1567 timesync: None,
1568 }
1569 }
1570}
1571
1572pub trait LeBytes: Copy + Sized {
1575 const WIDTH: usize;
1576 fn write_le(self, out: &mut [u8]);
1577 fn from_le_slice(bytes: &[u8]) -> Self;
1578}
1579
1580impl_letype_num!(u8, 1);
1581impl_letype_num!(u16, 2);
1582impl_letype_num!(u32, 4);
1583impl_letype_num!(u64, 8);
1584impl_letype_num!(u128, 16);
1585impl_letype_num!(i8, 1);
1586impl_letype_num!(i16, 2);
1587impl_letype_num!(i32, 4);
1588impl_letype_num!(i64, 8);
1589impl_letype_num!(i128, 16);
1590impl_letype_num!(f32, 4);
1591impl_letype_num!(f64, 8);
1592
1593pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
1595 let total = data.len() * T::WIDTH;
1596 let mut buf = vec![0u8; total];
1597
1598 for (i, v) in data.iter().copied().enumerate() {
1599 let start = i * T::WIDTH;
1600 v.write_le(&mut buf[start..start + T::WIDTH]);
1601 }
1602
1603 Arc::from(buf)
1604}
1605
1606fn make_error_payload(msg: &str) -> Arc<[u8]> {
1609 let meta = message_meta(DataType::TelemetryError);
1610 match meta.element {
1611 MessageElement::Static(_, _, _) => {
1612 let max = get_needed_message_size(DataType::TelemetryError);
1613 let bytes = msg.as_bytes();
1614 let n = core::cmp::min(max, bytes.len());
1615 let mut buf = vec![0u8; max];
1616 if n > 0 {
1617 buf[..n].copy_from_slice(&bytes[..n]);
1618 }
1619 Arc::from(buf)
1620 }
1621 MessageElement::Dynamic(_, _) => Arc::from(msg.as_bytes()),
1622 }
1623}
1624
1625fn log_raw<T, F>(
1629 sender: &str,
1630 ty: DataType,
1631 data: &[T],
1632 timestamp: u64,
1633 mut tx_function: F,
1634) -> TelemetryResult<()>
1635where
1636 T: LeBytes,
1637 F: FnMut(Packet) -> TelemetryResult<()>,
1638{
1639 let meta = message_meta(ty);
1640 let got = data.len() * T::WIDTH;
1641
1642 match meta.element {
1643 MessageElement::Static(_, _, _) => {
1644 if got != get_needed_message_size(ty) {
1645 return Err(TelemetryError::SizeMismatch {
1646 expected: get_needed_message_size(ty),
1647 got,
1648 });
1649 }
1650 }
1651 MessageElement::Dynamic(_, _) => {
1652 if !got.is_multiple_of(T::WIDTH) {
1654 return Err(TelemetryError::SizeMismatch {
1655 expected: T::WIDTH,
1656 got,
1657 });
1658 }
1659 }
1660 }
1661
1662 let payload = encode_slice_le(data);
1663 let pkt = Packet::new(ty, meta.endpoints_ref(), sender, timestamp, payload)?;
1664 tx_function(pkt)
1665}
1666
1667fn fallback_stdout(_msg: &str) {
1671 #[cfg(feature = "std")]
1672 {
1673 eprintln!("{}", _msg);
1674 }
1675 #[cfg(all(not(feature = "std"), target_os = "none"))]
1676 {
1677 let message = format!("{}\n", _msg);
1678 unsafe {
1679 seds_error_msg(message.as_ptr(), message.len());
1680 }
1681 }
1682}
1683
1684#[derive(Debug, Clone)]
1689struct RouterInner {
1690 memory: RuntimeMemoryConfig,
1691 sides: Vec<Option<RouterSide>>,
1692 route_overrides: BTreeMap<(Option<RouterSideId>, RouterSideId), bool>,
1693 typed_route_overrides: BTreeMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
1694 route_weights: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
1695 route_priorities: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
1696 source_route_modes: BTreeMap<Option<RouterSideId>, RouteSelectionMode>,
1697 route_selection_cursors: BTreeMap<Option<RouterSideId>, u64>,
1698 adaptive_route_stats: BTreeMap<RouterSideId, AdaptiveRouteStats>,
1699 side_runtime_stats: BTreeMap<RouterSideId, SideRuntimeStatsInner>,
1700 side_transport: BTreeMap<RouterSideId, SideTransportState>,
1701 managed_variable_types: BTreeSet<u32>,
1702 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
1703 managed_variable_latest: BTreeMap<u32, ManagedVariableCacheEntry>,
1704 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
1705 local_address: AddressBookEntry,
1706 address_book: BTreeMap<String, AddressBookEntry>,
1707 address_by_value: BTreeMap<NodeAddress, String>,
1708 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
1709 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
1710 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
1711 next_p2p_stream_id: P2pStreamId,
1712 received_queue: BoundedDeque<RouterRxItem>,
1713 transmit_queue: BoundedDeque<TxQueued>,
1714 tx_priority_burst: u8,
1715 recent_rx: BoundedDeque<u64>,
1716 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
1717 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
1718 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
1719 reliable_return_route_order: VecDeque<u64>,
1720 end_to_end_reliable_tx: BTreeMap<u64, EndToEndReliableSent>,
1721 end_to_end_reliable_tx_order: VecDeque<u64>,
1722 total_handler_failures: u64,
1723 total_handler_retries: u64,
1724 #[cfg(feature = "discovery")]
1725 discovery_routes: BTreeMap<RouterSideId, DiscoverySideState>,
1726 #[cfg(feature = "discovery")]
1727 discovery_cadence: DiscoveryCadenceState,
1728 #[cfg(feature = "discovery")]
1729 discovery_side_throttle: BTreeMap<RouterSideId, DiscoverySideThrottleState>,
1730 #[cfg(all(feature = "discovery", feature = "timesync"))]
1731 timesync_side_throttle: BTreeMap<RouterSideId, TimeSyncSideThrottleState>,
1732}
1733
1734#[derive(Debug, Clone)]
1735struct ManagedVariableCacheEntry {
1736 packet: Packet,
1737 sender_address: u32,
1738 cached_at_ms: u64,
1739}
1740
1741type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1742
1743#[derive(Clone)]
1744struct NetworkVariableUpdateHandler {
1745 handler: Arc<NetworkVariableUpdateFn>,
1746}
1747
1748impl Debug for NetworkVariableUpdateHandler {
1749 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1750 f.write_str("NetworkVariableUpdateHandler(<handler>)")
1751 }
1752}
1753
1754#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1756pub struct NetworkVariablePermissions {
1757 pub read: bool,
1758 pub write: bool,
1759}
1760
1761impl NetworkVariablePermissions {
1762 pub const NONE: Self = Self {
1763 read: false,
1764 write: false,
1765 };
1766 pub const READ_ONLY: Self = Self {
1767 read: true,
1768 write: false,
1769 };
1770 pub const WRITE_ONLY: Self = Self {
1771 read: false,
1772 write: true,
1773 };
1774 pub const READ_WRITE: Self = Self {
1775 read: true,
1776 write: true,
1777 };
1778}
1779
1780#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1781enum RouterQueueKind {
1782 Received,
1783 Transmit,
1784}
1785
1786impl RouterInner {
1787 #[cfg(feature = "discovery")]
1788 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
1789 board
1790 .sender_id
1791 .len()
1792 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
1793 .saturating_add(
1794 board
1795 .reachable_timesync_sources
1796 .iter()
1797 .map(|s| s.len())
1798 .sum::<usize>(),
1799 )
1800 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
1801 }
1802
1803 #[cfg(feature = "discovery")]
1804 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
1805 sender
1806 .len()
1807 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
1808 .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
1809 .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
1810 .saturating_add(
1811 state
1812 .reachable_timesync_sources
1813 .iter()
1814 .map(|s| s.len())
1815 .sum::<usize>(),
1816 )
1817 .saturating_add(
1818 state
1819 .advertised_reachable_timesync_sources
1820 .iter()
1821 .map(|s| s.len())
1822 .sum::<usize>(),
1823 )
1824 .saturating_add(
1825 state
1826 .topology_boards
1827 .iter()
1828 .map(Self::topology_board_byte_cost)
1829 .sum::<usize>(),
1830 )
1831 .saturating_add(size_of::<DiscoverySenderState>())
1832 }
1833
1834 #[cfg(feature = "discovery")]
1835 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
1836 size_of::<RouterSideId>()
1837 .saturating_add(size_of::<DiscoverySideState>())
1838 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
1839 .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
1840 .saturating_add(
1841 route.requested_network_variables.len()
1842 * (size_of::<DataType>() + size_of::<u64>()),
1843 )
1844 .saturating_add(
1845 route
1846 .reachable_timesync_sources
1847 .iter()
1848 .map(|s| s.len())
1849 .sum::<usize>(),
1850 )
1851 .saturating_add(
1852 route
1853 .announcers
1854 .iter()
1855 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
1856 .sum::<usize>(),
1857 )
1858 .saturating_add(side.saturating_sub(side))
1859 }
1860
1861 #[cfg(feature = "discovery")]
1862 fn discovery_bytes_used(&self) -> usize {
1863 self.discovery_routes
1864 .iter()
1865 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
1866 .sum()
1867 }
1868
1869 #[inline]
1870 fn reliable_rx_buffered_bytes(&self) -> usize {
1871 self.reliable_rx
1872 .values()
1873 .flat_map(|state| state.buffered.values())
1874 .map(|bytes| size_of::<Arc<[u8]>>() + bytes.len())
1875 .sum()
1876 }
1877
1878 fn reliable_rx_buffer_len(&self) -> usize {
1879 self.reliable_rx
1880 .values()
1881 .map(|state| state.buffered.len())
1882 .sum()
1883 }
1884
1885 #[inline]
1886 fn shared_queue_bytes_used(&self) -> usize {
1887 self.received_queue
1888 .bytes_used()
1889 .saturating_add(self.transmit_queue.bytes_used())
1890 .saturating_add(self.recent_rx.max_bytes())
1891 .saturating_add(self.reliable_rx_buffered_bytes())
1892 .saturating_add(crate::config::schema_bytes_used())
1893 .saturating_add({
1894 #[cfg(feature = "discovery")]
1895 {
1896 self.discovery_bytes_used()
1897 }
1898 #[cfg(not(feature = "discovery"))]
1899 {
1900 0
1901 }
1902 })
1903 }
1904
1905 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
1906 match preferred {
1907 RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
1912 RouterQueueKind::Transmit => self.transmit_queue.pop_lowest_priority().is_some(),
1913 }
1914 }
1915
1916 #[cfg(feature = "discovery")]
1917 fn pop_discovery_route(&mut self) -> bool {
1918 let Some((&side, _)) = self
1919 .discovery_routes
1920 .iter()
1921 .min_by_key(|(_, route)| route.last_seen_ms)
1922 else {
1923 return false;
1924 };
1925 self.discovery_routes.remove(&side);
1926 Self::queue_budget_warning("topology route evicted because shared queue budget is full");
1927 true
1928 }
1929
1930 fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
1931 let received = self
1932 .received_queue
1933 .lowest_priority(|queued| queued.priority)
1934 .map(|priority| (RouterQueueKind::Received, priority));
1935 let transmit = self
1936 .transmit_queue
1937 .lowest_priority(|queued| queued.priority)
1938 .map(|priority| (RouterQueueKind::Transmit, priority));
1939 [received, transmit]
1940 .into_iter()
1941 .flatten()
1942 .filter(|(_, priority)| {
1943 *priority < incoming_priority
1944 || (incoming_priority != 254 && *priority == incoming_priority)
1945 })
1946 .min_by_key(|(kind, priority)| {
1947 (
1948 *priority,
1949 if *kind == RouterQueueKind::Received {
1950 0
1951 } else {
1952 1
1953 },
1954 )
1955 })
1956 .map(|(kind, _)| kind)
1957 }
1958
1959 fn make_shared_queue_room(
1960 &mut self,
1961 incoming_cost: usize,
1962 incoming_priority: u8,
1963 ) -> TelemetryResult<()> {
1964 if incoming_cost > self.memory.max_queue_budget {
1965 return Err(TelemetryError::PacketTooLarge(
1966 "Item exceeds maximum shared queue budget",
1967 ));
1968 }
1969
1970 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
1971 > self.memory.max_queue_budget
1972 {
1973 let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
1974 return Err(TelemetryError::Io("shared priority queue saturated"));
1975 };
1976 if !self.pop_shared_queue_item(victim) {
1977 return Err(TelemetryError::PacketTooLarge(
1978 "Item exceeds maximum shared queue budget",
1979 ));
1980 }
1981 }
1982
1983 Ok(())
1984 }
1985
1986 #[inline]
1987 fn queue_budget_warning(msg: &str) {
1988 #[cfg(feature = "std")]
1989 eprintln!("sedsnet queue budget warning: {msg}");
1990 let _ = msg;
1991 }
1992
1993 #[cfg(feature = "discovery")]
1994 fn fit_discovery_budget(&mut self) {
1995 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
1996 if !self.pop_discovery_route() {
1997 break;
1998 }
1999 }
2000 }
2001
2002 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
2003 self.make_shared_queue_room(item.byte_cost(), item.priority)?;
2004 self.received_queue
2005 .push_back_prioritized(item, |queued| queued.priority)
2006 }
2007
2008 fn push_transmit(&mut self, item: TxQueued) -> TelemetryResult<()> {
2009 #[cfg(feature = "discovery")]
2010 if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
2011 let incoming = match data {
2012 RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
2013 Some((pkt.data_type(), pkt.sender().to_owned()))
2014 }
2015 RouterItem::Packet(_) | RouterItem::Packed(_) => None,
2016 };
2017 if let Some((incoming_ty, incoming_sender)) = incoming {
2018 self.transmit_queue.retain(|queued| {
2019 let RouterTxItem::ToSide {
2020 dst: queued_dst,
2021 data: queued_data,
2022 ..
2023 } = &queued.item
2024 else {
2025 return true;
2026 };
2027 let same_snapshot = match queued_data {
2028 RouterItem::Packet(pkt) => {
2029 pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
2030 }
2031 RouterItem::Packed(_) => false,
2032 };
2033 *queued_dst != *dst || !same_snapshot
2034 });
2035 }
2036 }
2037 self.make_shared_queue_room(item.byte_cost(), item.priority)?;
2038 self.transmit_queue
2039 .push_back_prioritized(item, |queued| queued.priority)
2040 }
2041
2042 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
2043 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
2044 let _ = self.recent_rx.pop_front();
2045 }
2046 self.make_shared_queue_room(0, 0)?;
2047 self.recent_rx.push_back(id)
2048 }
2049
2050 fn buffer_reliable_rx(
2051 &mut self,
2052 side: RouterSideId,
2053 ty: DataType,
2054 seq: u32,
2055 bytes: Arc<[u8]>,
2056 ) -> TelemetryResult<()> {
2057 let key = Router::reliable_key(side, ty);
2058 if self
2059 .reliable_rx
2060 .get(&key)
2061 .is_some_and(|state| state.buffered.contains_key(&seq))
2062 {
2063 return Ok(());
2064 }
2065 let cost = size_of::<Arc<[u8]>>() + bytes.len();
2066 self.make_shared_queue_room(cost, crate::transport_priority(ty))?;
2067 let rx_state = self
2068 .reliable_rx
2069 .entry(key)
2070 .or_insert_with(|| ReliableRxState {
2071 expected_seq: 1,
2072 buffered: BTreeMap::new(),
2073 });
2074 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
2075 let _ = rx_state.buffered.pop_first();
2076 }
2077 rx_state.buffered.insert(seq, bytes);
2078 Ok(())
2079 }
2080}
2081
2082struct IsrRxQueue {
2087 busy: AtomicBool,
2088 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
2089}
2090
2091unsafe impl Send for IsrRxQueue {}
2092unsafe impl Sync for IsrRxQueue {}
2093
2094struct IsrRxQueueGuard<'a> {
2095 owner: &'a IsrRxQueue,
2096}
2097
2098impl Deref for IsrRxQueueGuard<'_> {
2099 type Target = BoundedDeque<RouterRxItem>;
2100
2101 #[inline]
2102 fn deref(&self) -> &Self::Target {
2103 unsafe { &*self.owner.q.get() }
2104 }
2105}
2106
2107impl DerefMut for IsrRxQueueGuard<'_> {
2108 #[inline]
2109 fn deref_mut(&mut self) -> &mut Self::Target {
2110 unsafe { &mut *self.owner.q.get() }
2111 }
2112}
2113
2114impl Drop for IsrRxQueueGuard<'_> {
2115 #[inline]
2116 fn drop(&mut self) {
2117 self.owner.busy.store(false, Ordering::Release);
2118 }
2119}
2120
2121impl IsrRxQueue {
2122 #[inline]
2123 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
2124 Self {
2125 busy: AtomicBool::new(false),
2126 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
2127 }
2128 }
2129
2130 #[inline]
2131 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
2132 match self
2133 .busy
2134 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
2135 {
2136 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
2137 Err(_) => Err(TelemetryError::Io("rx queue busy")),
2138 }
2139 }
2140
2141 #[allow(dead_code)]
2142 #[inline]
2143 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
2144 let mut g = self.try_lock()?;
2145 g.push_back(item)
2146 }
2147
2148 #[inline]
2149 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
2150 let mut g = self.try_lock()?;
2151 g.push_back_prioritized(item, |queued| queued.priority)
2152 }
2153
2154 #[inline]
2155 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
2156 let mut g = self.try_lock()?;
2157 Ok(g.pop_front())
2158 }
2159
2160 #[inline]
2161 fn clear(&self) -> TelemetryResult<()> {
2162 let mut g = self.try_lock()?;
2163 g.clear();
2164 Ok(())
2165 }
2166
2167 #[inline]
2168 fn snapshot(&self) -> Option<(usize, usize)> {
2169 let g = self.try_lock().ok()?;
2170 Some((g.len(), g.bytes_used()))
2171 }
2172}
2173
2174pub struct Router {
2178 sender: RouterMutex<Arc<str>>,
2179 cfg: RouterConfig,
2180 state: RouterMutex<RouterInner>,
2181 isr_rx_queue: IsrRxQueue,
2182 side_tx_gate: ReentryGate,
2183 clock: Box<dyn Clock + Send + Sync>,
2184 #[cfg(feature = "timesync")]
2185 timesync: RouterMutex<TimeSyncRuntime>,
2186}
2187
2188#[cfg(feature = "timesync")]
2189#[derive(Debug, Clone)]
2190struct PendingTimeSyncRequest {
2191 seq: u64,
2192 t1_mono_ms: u64,
2193 source: String,
2194}
2195
2196#[cfg(feature = "timesync")]
2197#[derive(Debug, Clone)]
2198struct RemoteTimeSyncSource {
2199 priority: u64,
2200 last_sample_mono_ms: u64,
2201 sample_unix_ms: u64,
2202}
2203
2204#[cfg(feature = "timesync")]
2205#[derive(Debug, Clone)]
2206struct TimeSyncRuntime {
2207 cfg: Option<TimeSyncConfig>,
2208 tracker: Option<TimeSyncTracker>,
2209 clock: NetworkClock,
2210 disciplined_clock: SlewedNetworkClock,
2211 remote_sources: BTreeMap<String, RemoteTimeSyncSource>,
2212 next_seq: u64,
2213 next_announce_mono_ms: u64,
2214 next_request_mono_ms: u64,
2215 pending_request: Option<PendingTimeSyncRequest>,
2216}
2217
2218#[cfg(feature = "timesync")]
2219impl TimeSyncRuntime {
2220 fn new(cfg: Option<TimeSyncConfig>) -> Self {
2221 Self {
2222 tracker: cfg.map(TimeSyncTracker::new),
2223 cfg,
2224 clock: NetworkClock::default(),
2225 disciplined_clock: SlewedNetworkClock::new(
2226 cfg.map(|c| c.max_slew_ppm)
2227 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
2228 ),
2229 remote_sources: BTreeMap::new(),
2230 next_seq: 1,
2231 next_announce_mono_ms: 0,
2232 next_request_mono_ms: 0,
2233 pending_request: None,
2234 }
2235 }
2236}
2237
2238enum RemoteSidePlan {
2239 Target(Vec<RouterSideId>),
2240}
2241
2242#[cfg(feature = "discovery")]
2243#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
2244struct DiscoveryCandidateMatch {
2245 side: RouterSideId,
2246 overlap: usize,
2247}
2248
2249#[cfg(feature = "discovery")]
2250#[inline]
2251const fn discovery_is_hop_local_advertisement(ty: DataType) -> bool {
2252 matches!(
2253 ty,
2254 DataType::DiscoveryAnnounce
2255 | DataType::DiscoveryTimeSyncSources
2256 | DataType::DiscoveryTopology
2257 | DataType::DiscoveryAddress
2258 | DataType::DiscoveryLinkCapabilities
2259 )
2260}
2261
2262impl Debug for Router {
2263 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2264 let sender = self.sender();
2265 f.debug_struct("Router")
2266 .field("sender", &sender)
2267 .field("cfg", &self.cfg)
2268 .field("state", &"<mutex>")
2269 .field("clock", &"Clock")
2270 .finish()
2271 }
2272}
2273
2274#[inline]
2277fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &RouterConfig) -> bool {
2278 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
2279}
2280
2281#[inline]
2282fn force_remote_for_type(ty: DataType) -> bool {
2283 matches!(
2284 ty,
2285 DataType::ReliableAck
2286 | DataType::ReliablePartialAck
2287 | DataType::ReliablePacketRequest
2288 | DataType::P2pMessage
2289 ) || {
2290 #[cfg(feature = "timesync")]
2291 {
2292 matches!(
2293 ty,
2294 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2295 )
2296 }
2297 #[cfg(not(feature = "timesync"))]
2298 {
2299 false
2300 }
2301 }
2302}
2303
2304#[inline]
2305fn is_internal_control_type(ty: DataType) -> bool {
2306 if matches!(
2307 ty,
2308 DataType::ReliableAck
2309 | DataType::ReliablePartialAck
2310 | DataType::ReliablePacketRequest
2311 | DataType::P2pMessage
2312 ) {
2313 return true;
2314 }
2315
2316 #[cfg(feature = "timesync")]
2317 if matches!(
2318 ty,
2319 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2320 ) {
2321 return true;
2322 }
2323
2324 #[cfg(feature = "discovery")]
2325 if discovery::is_discovery_type(ty) {
2326 return true;
2327 }
2328
2329 let _ = ty;
2330 false
2331}
2332
2333fn with_retries<F>(
2335 this: &Router,
2336 dest: DataEndpoint,
2337 data: &RouterItem,
2338 pkt_for_ctx: Option<&Packet>,
2339 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
2340 called_from_queue: bool,
2341 run: F,
2342) -> TelemetryResult<()>
2343where
2344 F: Fn() -> TelemetryResult<()>,
2345{
2346 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
2347 Ok(((), attempts)) => {
2348 if attempts > 1 {
2349 let mut st = this.state.lock();
2350 st.total_handler_retries = st
2351 .total_handler_retries
2352 .saturating_add(attempts.saturating_sub(1) as u64);
2353 }
2354 Ok(())
2355 }
2356 Err((e, attempts)) => {
2357 {
2358 let mut st = this.state.lock();
2359 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
2360 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
2361 }
2362
2363 this.remove_pkt_id(data);
2365
2366 if let Some(pkt) = pkt_for_ctx {
2368 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
2369 } else if let Some(env) = env_for_ctx {
2370 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
2371 }
2372
2373 Err(TelemetryError::HandlerError("local handler failed"))
2374 }
2375 }
2376}
2377impl Router {
2379 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
2380 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
2381
2382 #[inline]
2383 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
2384 st.sides
2385 .get(side)
2386 .and_then(|side| side.as_ref())
2387 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
2388 }
2389
2390 fn note_side_tx_success(
2391 &self,
2392 side: RouterSideId,
2393 ty: DataType,
2394 bytes: usize,
2395 relayed: bool,
2396 attempts: usize,
2397 ) {
2398 let mut st = self.state.lock();
2399 let entry = st.side_runtime_stats.entry(side).or_default();
2400 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
2401 }
2402
2403 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
2404 let mut st = self.state.lock();
2405 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
2406 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
2407 let entry = st.side_runtime_stats.entry(side).or_default();
2408 entry.note_tx_failure(ty, attempts);
2409 }
2410
2411 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
2412 let mut st = self.state.lock();
2413 let entry = st.side_runtime_stats.entry(side).or_default();
2414 entry.note_rx(ty, bytes, relayed);
2415 }
2416
2417 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
2418 let mut st = self.state.lock();
2419 let entry = st.side_runtime_stats.entry(side).or_default();
2420 entry.note_local_delivery(ty);
2421 }
2422
2423 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
2424 let mut st = self.state.lock();
2425 let entry = st.side_runtime_stats.entry(side).or_default();
2426 entry.note_local_handler_failure(ty, retries);
2427 }
2428
2429 fn cache_managed_variable_packet(
2430 &self,
2431 pkt: &Packet,
2432 notify_handlers: bool,
2433 ) -> TelemetryResult<()> {
2434 let sender_address = pkt
2435 .sender()
2436 .strip_prefix("@addr:")
2437 .and_then(|value| value.parse::<u32>().ok())
2438 .unwrap_or_else(|| sender_address_u32(pkt.sender()));
2439 let handlers = {
2440 let mut st = self.state.lock();
2441 if !st
2442 .managed_variable_types
2443 .contains(&pkt.data_type().as_u32())
2444 {
2445 return Ok(());
2446 }
2447 let existing = st.managed_variable_latest.get(&pkt.data_type().as_u32());
2448 if existing.is_some_and(|entry| {
2455 let current = &entry.packet;
2456 sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
2457 }) {
2458 return Ok(());
2459 }
2460 let changed = existing.is_none_or(|entry| entry.packet != *pkt);
2461 st.managed_variable_latest.insert(
2462 pkt.data_type().as_u32(),
2463 ManagedVariableCacheEntry {
2464 packet: pkt.clone(),
2465 sender_address,
2466 cached_at_ms: self.clock.now_ms(),
2467 },
2468 );
2469 if notify_handlers && changed {
2470 st.network_variable_update_handlers
2471 .get(&pkt.data_type().as_u32())
2472 .cloned()
2473 .unwrap_or_default()
2474 } else {
2475 Vec::new()
2476 }
2477 };
2478 for handler in handlers {
2479 (handler.handler)(pkt)?;
2480 }
2481 Ok(())
2482 }
2483
2484 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
2485 self.cache_managed_variable_packet(pkt, true)
2486 }
2487
2488 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
2489 let st = self.state.lock();
2490 st.managed_variable_latest
2491 .get(&ty.as_u32())
2492 .map(|entry| entry.packet.clone())
2493 }
2494
2495 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
2496 let now_ms = self.clock.now_ms();
2497 let st = self.state.lock();
2498 st.managed_variable_latest.get(&ty.as_u32()).map(|entry| {
2499 (
2500 entry.packet.clone(),
2501 now_ms.saturating_sub(entry.cached_at_ms),
2502 )
2503 })
2504 }
2505
2506 fn is_managed_variable_type(&self, ty: DataType) -> bool {
2507 let st = self.state.lock();
2508 st.managed_variable_types.contains(&ty.as_u32())
2509 }
2510
2511 fn managed_variable_permissions_locked(
2512 st: &RouterInner,
2513 ty: DataType,
2514 ) -> NetworkVariablePermissions {
2515 st.managed_variable_permissions
2516 .get(&ty.as_u32())
2517 .copied()
2518 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
2519 }
2520
2521 fn can_read_managed_variable(&self, ty: DataType) -> bool {
2522 let st = self.state.lock();
2523 Self::managed_variable_permissions_locked(&st, ty).read
2524 }
2525
2526 fn can_write_managed_variable(&self, ty: DataType) -> bool {
2527 let st = self.state.lock();
2528 Self::managed_variable_permissions_locked(&st, ty).write
2529 }
2530
2531 #[inline]
2532 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
2533 let st = self.state.lock();
2534 let side_ref = Self::side_ref(&st, side)?;
2535 if side_ref.opts.ingress_enabled {
2536 Ok(())
2537 } else {
2538 Err(TelemetryError::HandlerError(
2539 "router: ingress disabled for side id",
2540 ))
2541 }
2542 }
2543
2544 #[inline]
2545 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
2546 src != Some(dst)
2547 }
2548
2549 #[inline]
2550 fn route_allowed_locked(
2551 &self,
2552 st: &RouterInner,
2553 src: Option<RouterSideId>,
2554 ty: Option<DataType>,
2555 dst: RouterSideId,
2556 ) -> bool {
2557 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
2558 return false;
2559 };
2560 if !dst_side.opts.egress_enabled {
2561 return false;
2562 }
2563 if let Some(src_id) = src {
2564 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
2565 return false;
2566 };
2567 if !src_side.opts.ingress_enabled {
2568 return false;
2569 }
2570 }
2571 let base_allowed = st
2572 .route_overrides
2573 .get(&(src, dst))
2574 .copied()
2575 .unwrap_or_else(|| self.default_route_enabled(src, dst));
2576 if !base_allowed {
2577 return false;
2578 }
2579
2580 let Some(ty) = ty else {
2581 return true;
2582 };
2583 if st
2584 .typed_route_overrides
2585 .keys()
2586 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
2587 {
2588 return st
2589 .typed_route_overrides
2590 .get(&(src, ty.as_u32(), dst))
2591 .copied()
2592 .unwrap_or(false);
2593 }
2594 true
2595 }
2596
2597 fn has_typed_route_overrides_locked(
2598 st: &RouterInner,
2599 src: Option<RouterSideId>,
2600 ty: DataType,
2601 ) -> bool {
2602 st.typed_route_overrides
2603 .keys()
2604 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
2605 }
2606
2607 #[cfg(feature = "discovery")]
2608 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
2609 where
2610 I: IntoIterator<Item = DataEndpoint>,
2611 {
2612 let mut overlap = 0usize;
2613 for ep in reachable {
2614 if eps.contains(&ep) {
2615 overlap = overlap.saturating_add(1);
2616 }
2617 }
2618 overlap
2619 }
2620
2621 #[inline]
2622 fn preferred_scoring_endpoints(
2623 &self,
2624 eps: &[DataEndpoint],
2625 prefer_nonlocal: bool,
2626 ) -> Vec<DataEndpoint> {
2627 if !prefer_nonlocal {
2628 return eps.to_vec();
2629 }
2630 let nonlocal: Vec<DataEndpoint> = eps
2631 .iter()
2632 .copied()
2633 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
2634 .collect();
2635 if nonlocal.is_empty() {
2636 eps.to_vec()
2637 } else {
2638 nonlocal
2639 }
2640 }
2641
2642 #[cfg(feature = "discovery")]
2643 fn pop_next_queued_discovery_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
2644 {
2645 let mut isr_rx = self.isr_rx_queue.try_lock()?;
2646 let idx = isr_rx.iter().position(Self::queued_rx_item_is_discovery);
2647 if let Some(idx) = idx {
2648 return Ok(isr_rx.remove_pos(idx));
2649 }
2650 }
2651
2652 let mut st = self.state.lock();
2653 let idx = st
2654 .received_queue
2655 .iter()
2656 .position(Self::queued_rx_item_is_discovery);
2657 if let Some(idx) = idx {
2658 return Ok(st.received_queue.remove_pos(idx));
2659 }
2660 Ok(None)
2661 }
2662
2663 #[cfg(feature = "discovery")]
2664 fn queued_rx_item_is_discovery(item: &RouterRxItem) -> bool {
2665 match &item.data {
2666 RouterItem::Packet(pkt) => discovery::is_discovery_type(pkt.data_type()),
2667 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())
2668 .map(|env| discovery::is_discovery_type(env.ty))
2669 .unwrap_or(false),
2670 }
2671 }
2672
2673 #[cfg(feature = "discovery")]
2674 fn drain_queued_discovery_rx_before_tx(&self) -> TelemetryResult<bool> {
2675 let mut did_any = false;
2676 while let Some(item) = self.pop_next_queued_discovery_rx_item()? {
2677 self.process_rx_queue_item(item)?;
2678 did_any = true;
2679 }
2680 Ok(did_any)
2681 }
2682
2683 fn eligible_side_ids_locked(
2684 &self,
2685 st: &RouterInner,
2686 src: Option<RouterSideId>,
2687 ty: Option<DataType>,
2688 restrict_link_local: bool,
2689 ) -> Vec<RouterSideId> {
2690 st.sides
2691 .iter()
2692 .enumerate()
2693 .filter_map(|(side_id, side)| {
2694 let side = side.as_ref()?;
2695 if restrict_link_local && !side.opts.link_local_enabled {
2696 return None;
2697 }
2698 if self.route_allowed_locked(st, src, ty, side_id) {
2699 Some(side_id)
2700 } else {
2701 None
2702 }
2703 })
2704 .collect()
2705 }
2706
2707 fn apply_route_selection_locked(
2708 &self,
2709 st: &mut RouterInner,
2710 src: Option<RouterSideId>,
2711 mut sides: Vec<RouterSideId>,
2712 origin: RouteSelectionOrigin,
2713 ) -> Vec<RouterSideId> {
2714 if sides.len() <= 1 {
2715 return sides;
2716 }
2717
2718 let selection_mode = st.source_route_modes.get(&src).copied();
2719 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
2720 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
2721 }
2722
2723 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
2724 RouteSelectionMode::Fanout => sides,
2725 RouteSelectionMode::Weighted => {
2726 sides.sort_unstable();
2727 let total_weight = sides.iter().fold(0_u64, |acc, side| {
2728 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
2729 });
2730 if total_weight == 0 {
2731 return Vec::new();
2732 }
2733 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2734 let pick = *cursor % total_weight;
2735 *cursor = cursor.wrapping_add(1);
2736 let mut remaining = pick;
2737 for side in sides {
2738 let weight =
2739 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
2740 if remaining < weight {
2741 return vec![side];
2742 }
2743 remaining -= weight;
2744 }
2745 Vec::new()
2746 }
2747 RouteSelectionMode::Failover => {
2748 sides.sort_by_key(|side| {
2749 (
2750 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
2751 *side,
2752 )
2753 });
2754 sides.truncate(1);
2755 sides
2756 }
2757 }
2758 }
2759
2760 fn apply_adaptive_discovery_selection_locked(
2761 &self,
2762 st: &mut RouterInner,
2763 src: Option<RouterSideId>,
2764 mut sides: Vec<RouterSideId>,
2765 ) -> Vec<RouterSideId> {
2766 sides.sort_unstable();
2767 let mut unmeasured: Vec<_> = sides
2768 .iter()
2769 .copied()
2770 .filter(|side| !st.adaptive_route_stats.contains_key(side))
2771 .collect();
2772 if !unmeasured.is_empty() {
2773 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2774 let pick = (*cursor as usize) % unmeasured.len();
2775 *cursor = cursor.wrapping_add(1);
2776 return vec![unmeasured.swap_remove(pick)];
2777 }
2778
2779 let now_ms = self.clock.now_ms();
2780 let total_weight = sides.iter().fold(0_u64, |acc, side| {
2781 acc + st
2782 .adaptive_route_stats
2783 .get(side)
2784 .map(|stats| stats.weight(now_ms))
2785 .unwrap_or(1)
2786 });
2787 if total_weight == 0 {
2788 sides.truncate(1);
2789 return sides;
2790 }
2791
2792 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2793 let pick = *cursor % total_weight;
2794 *cursor = cursor.wrapping_add(1);
2795 let mut remaining = pick;
2796 for side in sides {
2797 let weight = st
2798 .adaptive_route_stats
2799 .get(&side)
2800 .map(|stats| stats.weight(now_ms))
2801 .unwrap_or(1);
2802 if remaining < weight {
2803 return vec![side];
2804 }
2805 remaining -= weight;
2806 }
2807 Vec::new()
2808 }
2809
2810 fn record_side_tx_sample(
2811 &self,
2812 side: RouterSideId,
2813 bytes: usize,
2814 started_ms: u64,
2815 ended_ms: u64,
2816 ) {
2817 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
2818 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
2819 .min(u128::from(u64::MAX)) as u64;
2820 let mut st = self.state.lock();
2821 st.adaptive_route_stats
2822 .entry(side)
2823 .or_default()
2824 .observe(bytes, sample_bps, ended_ms);
2825 }
2826
2827 pub fn note_side_link_probe_sample(
2832 &self,
2833 side: RouterSideId,
2834 bytes: usize,
2835 duration_ms: u64,
2836 ) -> TelemetryResult<()> {
2837 {
2838 let st = self.state.lock();
2839 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
2840 }
2841 let ended_ms = self.clock.now_ms();
2842 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
2843 Ok(())
2844 }
2845
2846 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
2847 match data {
2848 RouterItem::Packet(pkt) => Ok(wire_format::pack_packet(pkt).len()),
2849 RouterItem::Packed(bytes) => Ok(bytes.len()),
2850 }
2851 }
2852
2853 #[inline]
2863 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
2864 match data {
2865 RouterItem::Packet(pkt) => {
2866 if !Self::is_end_to_end_ack_packet(pkt) {
2867 return Ok(None);
2868 }
2869 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
2870 }
2871 RouterItem::Packed(bytes) => {
2872 if wire_format::peek_frame_info(bytes.as_ref())
2873 .ok()
2874 .is_some_and(|frame| frame.ack_only())
2875 {
2876 return Ok(None);
2877 }
2878 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
2879 if !Self::is_end_to_end_ack_packet(&pkt) {
2880 return Ok(None);
2881 }
2882 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
2883 }
2884 }
2885 }
2886
2887 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
2888 if payload.len() != 8 {
2889 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
2890 }
2891 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
2892 }
2893
2894 #[inline]
2895 fn sender_hash(sender: &str) -> u64 {
2896 if let Some(address) = sender
2897 .strip_prefix("@addr:")
2898 .and_then(|v| v.parse::<u32>().ok())
2899 {
2900 return u64::from(address);
2901 }
2902 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
2903 }
2904
2905 #[inline]
2906 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
2907 let hash = Self::sender_hash(hostname);
2908 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
2909 if address == 0 {
2910 address = 1;
2911 }
2912 address
2913 }
2914
2915 fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
2920 let Some(address) = sender
2921 .strip_prefix("@addr:")
2922 .and_then(|value| value.parse::<u32>().ok())
2923 else {
2924 return sender.to_string();
2925 };
2926 if let Some(hostname) = st.address_by_value.get(&address) {
2927 return hostname.clone();
2928 }
2929 st.address_book
2930 .values()
2931 .find(|entry| sender_address_u32(entry.hostname.as_ref()) == address)
2932 .map(|entry| entry.hostname.to_string())
2933 .unwrap_or_else(|| sender.to_string())
2934 }
2935
2936 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
2937 match mode {
2938 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
2939 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
2940 _ => AddressAssignmentMode::Dynamic,
2941 }
2942 }
2943
2944 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
2945 let rank = match entry.mode {
2946 AddressAssignmentMode::Static(_) => 2,
2947 AddressAssignmentMode::Requested(_) => 1,
2948 AddressAssignmentMode::Dynamic => 0,
2949 };
2950 (
2951 rank,
2952 core::cmp::Reverse(entry.birth_ms),
2953 u64::MAX - entry.owner_hash,
2954 )
2955 }
2956
2957 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
2958 (
2959 core::cmp::Reverse(entry.birth_ms),
2960 u64::MAX - entry.owner_hash,
2961 )
2962 }
2963
2964 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
2965 let mut candidate = if seed == 0 { 1 } else { seed };
2966 for _ in 0..u32::MAX {
2967 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
2968 return candidate;
2969 }
2970 candidate = candidate.wrapping_add(1);
2971 if candidate == 0 {
2972 candidate = 1;
2973 }
2974 }
2975 1
2976 }
2977
2978 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
2979 if !st.address_book.contains_key(base) {
2980 return base.to_string();
2981 }
2982 let stem = if base.is_empty() { "node" } else { base };
2983 let mut candidate = format!("{stem}-{owner_hash:08x}");
2984 let mut suffix = 1u32;
2985 while st.address_book.contains_key(&candidate) {
2986 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
2987 suffix = suffix.saturating_add(1);
2988 }
2989 candidate
2990 }
2991
2992 fn update_local_identity_locked(
2993 &self,
2994 st: &mut RouterInner,
2995 mut local: AddressBookEntry,
2996 reason: AddressChangeReason,
2997 ) -> AddressChange {
2998 let old = st.local_address.clone();
2999 st.address_book.remove(old.hostname.as_ref());
3000 st.address_by_value.remove(&old.address);
3001 local.last_seen_ms = self.clock.now_ms();
3002 st.address_by_value
3003 .insert(local.address, local.hostname.to_string());
3004 st.address_book
3005 .insert(local.hostname.to_string(), local.clone());
3006 st.local_address = local.clone();
3007 *self.sender.lock() = local.hostname.clone();
3008 #[cfg(feature = "discovery")]
3009 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
3010 AddressChange {
3011 old_hostname: old.hostname,
3012 new_hostname: local.hostname,
3013 old_address: old.address,
3014 new_address: local.address,
3015 reason,
3016 }
3017 }
3018
3019 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
3020 for handler in self.cfg.address_change_handlers.iter() {
3021 (handler.handler)(change.clone())?;
3022 }
3023 Ok(())
3024 }
3025
3026 #[cfg(feature = "discovery")]
3027 fn local_address_advertisement(
3028 &self,
3029 reachable_endpoints: Vec<DataEndpoint>,
3030 reachable_network_variables: Vec<DataType>,
3031 reachable_timesync_sources: Vec<String>,
3032 link_capabilities: discovery::LinkCapabilities,
3033 state: u8,
3034 ) -> discovery::AddressAdvertisement {
3035 let st = self.state.lock();
3036 discovery::AddressAdvertisement {
3037 hostname: st.local_address.hostname.to_string(),
3038 address: st.local_address.address,
3039 requested_address: st.local_address.requested_address,
3040 mode: st.local_address.mode.mode_code(),
3041 state,
3042 birth_ms: st.local_address.birth_ms,
3043 owner_hash: st.local_address.owner_hash,
3044 reachable_endpoints,
3045 reachable_network_variables,
3046 reachable_timesync_sources,
3047 link_capabilities,
3048 }
3049 }
3050
3051 #[cfg(feature = "discovery")]
3052 fn ingest_address_advertisement(
3053 &self,
3054 ad: discovery::AddressAdvertisement,
3055 ) -> TelemetryResult<bool> {
3056 let now_ms = self.clock.now_ms();
3057 let mut remote = AddressBookEntry {
3058 hostname: Arc::from(ad.hostname.as_str()),
3059 address: ad.address,
3060 requested_address: ad.requested_address,
3061 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
3062 birth_ms: ad.birth_ms,
3063 owner_hash: ad.owner_hash,
3064 last_seen_ms: now_ms,
3065 };
3066 let mut change = None;
3067 let mut changed = false;
3068 {
3069 let mut st = self.state.lock();
3070 let local = st.local_address.clone();
3071 let hostname_conflict =
3072 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
3073 let address_conflict =
3074 remote.address == local.address && remote.owner_hash != local.owner_hash;
3075
3076 if hostname_conflict
3077 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
3078 {
3079 let mut next = local.clone();
3080 next.hostname = Arc::from(Self::unique_hostname_locked(
3081 &st,
3082 local.hostname.as_ref(),
3083 local.owner_hash,
3084 ));
3085 change = Some(self.update_local_identity_locked(
3086 &mut st,
3087 next,
3088 AddressChangeReason::HostnameConflict,
3089 ));
3090 changed = true;
3091 } else if address_conflict
3092 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
3093 {
3094 let mut next = local.clone();
3095 let seed = next
3096 .requested_address
3097 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
3098 next.address = Self::allocate_free_address_locked(&st, seed);
3099 let reason = match next.mode {
3100 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
3101 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
3102 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
3103 };
3104 change = Some(self.update_local_identity_locked(&mut st, next, reason));
3105 changed = true;
3106 } else if hostname_conflict {
3107 remote.hostname = Arc::from(Self::unique_hostname_locked(
3108 &st,
3109 remote.hostname.as_ref(),
3110 remote.owner_hash,
3111 ));
3112 changed = true;
3113 } else if address_conflict {
3114 let seed = remote
3115 .requested_address
3116 .max(Self::fallback_address_for_hostname(
3117 remote.hostname.as_ref(),
3118 ));
3119 remote.address = Self::allocate_free_address_locked(&st, seed);
3120 changed = true;
3121 }
3122
3123 st.address_book.remove(remote.hostname.as_ref());
3124 st.address_by_value.remove(&remote.address);
3125 st.address_by_value
3126 .insert(remote.address, remote.hostname.to_string());
3127 st.address_book
3128 .insert(remote.hostname.to_string(), remote.clone());
3129 }
3130 if let Some(change) = change {
3131 self.notify_address_change(change)?;
3132 }
3133 Ok(changed)
3134 }
3135
3136 #[inline]
3137 fn is_end_to_end_ack_sender(sender: &str) -> bool {
3138 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
3139 }
3140
3141 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
3142 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
3143 && !ack_sender.is_empty()
3144 {
3145 return Some(Self::sender_hash(ack_sender));
3146 }
3147 None
3148 }
3149
3150 fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
3156 if pkt.data_type() != DataType::ReliableAck {
3157 return None;
3158 }
3159 pkt.wire_target_senders()
3160 .first()
3161 .copied()
3162 .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
3163 }
3164
3165 fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
3166 Self::end_to_end_ack_sender_hash(pkt).is_some()
3167 && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
3168 }
3169
3170 fn encode_end_to_end_ack_sender(&self) -> String {
3171 let sender = self.sender_arc();
3172 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
3173 }
3174
3175 #[cfg(feature = "discovery")]
3176 fn is_end_to_end_destination_sender(sender: &str) -> bool {
3177 sender != "RELAY" && !Self::is_end_to_end_ack_sender(sender)
3178 }
3179
3180 fn encode_end_to_end_reliable_ack(packet_id: u64) -> Arc<[u8]> {
3181 let mut payload = Vec::with_capacity(8);
3182 payload.extend_from_slice(&packet_id.to_le_bytes());
3183 Arc::from(payload)
3184 }
3185
3186 fn encode_p2p_payload(
3187 src_hostname: &str,
3188 src_address: NodeAddress,
3189 src_port: P2pPort,
3190 dst_port: P2pPort,
3191 payload: &[u8],
3192 ) -> TelemetryResult<Arc<[u8]>> {
3193 let host_len = u16::try_from(src_hostname.len())
3194 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
3195 let payload_len = u32::try_from(payload.len())
3196 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
3197 let mut out = Vec::with_capacity(
3198 15usize
3199 .saturating_add(src_hostname.len())
3200 .saturating_add(payload.len()),
3201 );
3202 out.push(1);
3203 out.extend_from_slice(&dst_port.to_le_bytes());
3204 out.extend_from_slice(&src_port.to_le_bytes());
3205 out.extend_from_slice(&src_address.to_le_bytes());
3206 out.extend_from_slice(&host_len.to_le_bytes());
3207 out.extend_from_slice(&payload_len.to_le_bytes());
3208 out.extend_from_slice(src_hostname.as_bytes());
3209 out.extend_from_slice(payload);
3210 Ok(out.into())
3211 }
3212
3213 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
3214 if payload.len() < 15 {
3215 return Err(TelemetryError::Unpack("p2p frame short"));
3216 }
3217 if payload[0] != 1 {
3218 return Err(TelemetryError::Unpack("p2p frame version"));
3219 }
3220 let destination_port =
3221 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
3222 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
3223 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
3224 let host_len =
3225 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
3226 let body_len =
3227 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
3228 let host_start = 15usize;
3229 let host_end = host_start.saturating_add(host_len);
3230 let body_end = host_end.saturating_add(body_len);
3231 if host_end > payload.len() || body_end != payload.len() {
3232 return Err(TelemetryError::Unpack("p2p frame length"));
3233 }
3234 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
3235 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
3236 Ok(P2pDecoded {
3237 source_hostname,
3238 source_address,
3239 source_port,
3240 destination_port,
3241 payload: &payload[host_end..body_end],
3242 })
3243 }
3244
3245 fn encode_p2p_stream_payload(
3246 flags: u8,
3247 source_stream_id: P2pStreamId,
3248 destination_stream_id: P2pStreamId,
3249 sequence: u32,
3250 payload: &[u8],
3251 ) -> TelemetryResult<Arc<[u8]>> {
3252 let payload_len = u32::try_from(payload.len())
3253 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
3254 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
3255 out.extend_from_slice(&P2P_STREAM_MAGIC);
3256 out.push(P2P_STREAM_VERSION);
3257 out.push(flags);
3258 out.extend_from_slice(&source_stream_id.to_le_bytes());
3259 out.extend_from_slice(&destination_stream_id.to_le_bytes());
3260 out.extend_from_slice(&sequence.to_le_bytes());
3261 out.extend_from_slice(&payload_len.to_le_bytes());
3262 out.extend_from_slice(payload);
3263 Ok(out.into())
3264 }
3265
3266 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
3267 if !payload.starts_with(&P2P_STREAM_MAGIC) {
3268 return Ok(None);
3269 }
3270 if payload.len() < 22 {
3271 return Err(TelemetryError::Unpack("p2p stream frame short"));
3272 }
3273 if payload[4] != P2P_STREAM_VERSION {
3274 return Err(TelemetryError::Unpack("p2p stream frame version"));
3275 }
3276 let flags = payload[5];
3277 let source_stream_id =
3278 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
3279 let destination_stream_id =
3280 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
3281 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
3282 let body_len =
3283 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
3284 let body_end = 22usize.saturating_add(body_len);
3285 if body_end != payload.len() {
3286 return Err(TelemetryError::Unpack("p2p stream frame length"));
3287 }
3288 Ok(Some(P2pStreamDecoded {
3289 flags,
3290 source_stream_id,
3291 destination_stream_id,
3292 sequence,
3293 payload: &payload[22..],
3294 }))
3295 }
3296
3297 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
3298 for _ in 0..u32::MAX {
3299 let id = st.next_p2p_stream_id.max(1);
3300 st.next_p2p_stream_id = st.next_p2p_stream_id.wrapping_add(1).max(1);
3301 if !st.p2p_stream_sessions.contains_key(&id) {
3302 return id;
3303 }
3304 }
3305 0
3306 }
3307
3308 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
3309 if pkt.data_type() != DataType::P2pMessage {
3310 return Ok(());
3311 }
3312 let decoded = Self::decode_p2p_payload(pkt.payload())?;
3313 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
3314 return self.dispatch_p2p_stream_frame(&decoded, &stream);
3315 }
3316 let handlers = {
3317 let st = self.state.lock();
3318 st.p2p_port_handlers
3319 .get(&decoded.destination_port)
3320 .cloned()
3321 .unwrap_or_default()
3322 };
3323 for handler in handlers {
3324 (handler.handler)(P2pMessage {
3325 source_hostname: decoded.source_hostname,
3326 source_address: decoded.source_address,
3327 source_port: decoded.source_port,
3328 destination_port: decoded.destination_port,
3329 payload: decoded.payload,
3330 })?;
3331 }
3332 Ok(())
3333 }
3334
3335 fn dispatch_p2p_stream_frame(
3336 &self,
3337 msg: &P2pDecoded<'_>,
3338 frame: &P2pStreamDecoded<'_>,
3339 ) -> TelemetryResult<()> {
3340 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
3341 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, Arc<[u8]>)> = None;
3342 {
3343 let mut st = self.state.lock();
3344 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
3345 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
3346 let existing_id = st.p2p_stream_sessions.iter().find_map(|(id, session)| {
3347 (session.peer_stream_id == frame.source_stream_id
3348 && session.local_port == msg.destination_port
3349 && session.peer_port == msg.source_port
3350 && session.peer_address == msg.source_address
3351 && session.peer_hostname.as_ref() == msg.source_hostname)
3352 .then_some(*id)
3353 });
3354 let local_id = if let Some(local_id) = existing_id {
3355 local_id
3356 } else {
3357 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
3358 if local_id == 0 {
3359 return Err(TelemetryError::Io("p2p stream id exhausted"));
3360 }
3361 st.p2p_stream_sessions.insert(
3362 local_id,
3363 P2pStreamSession {
3364 peer_hostname: peer_hostname.clone(),
3365 peer_address: msg.source_address,
3366 local_port: msg.destination_port,
3367 peer_port: msg.source_port,
3368 peer_stream_id: frame.source_stream_id,
3369 next_sequence: 1,
3370 connected: true,
3371 },
3372 );
3373 let handlers = st
3374 .p2p_stream_handlers
3375 .get(&msg.destination_port)
3376 .cloned()
3377 .unwrap_or_default();
3378 events.push(PendingP2pStreamEvent {
3379 handlers,
3380 kind: P2pStreamEventKind::Accepted,
3381 stream_id: local_id,
3382 peer_stream_id: frame.source_stream_id,
3383 peer_hostname: peer_hostname.clone(),
3384 peer_address: msg.source_address,
3385 local_port: msg.destination_port,
3386 peer_port: msg.source_port,
3387 });
3388 local_id
3389 };
3390 let dst =
3391 st.address_book
3392 .get(msg.source_hostname)
3393 .cloned()
3394 .unwrap_or(AddressBookEntry {
3395 hostname: peer_hostname,
3396 address: msg.source_address,
3397 requested_address: msg.source_address,
3398 mode: AddressAssignmentMode::Dynamic,
3399 birth_ms: self.clock.now_ms(),
3400 owner_hash: Self::sender_hash(msg.source_hostname),
3401 last_seen_ms: self.clock.now_ms(),
3402 });
3403 let payload = Self::encode_p2p_stream_payload(
3404 P2P_STREAM_SYN | P2P_STREAM_ACK,
3405 local_id,
3406 frame.source_stream_id,
3407 0,
3408 &[],
3409 )?;
3410 reply = Some((dst, msg.source_port, msg.destination_port, payload));
3411 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
3412 if let Some(session) = st.p2p_stream_sessions.get_mut(&frame.destination_stream_id)
3413 {
3414 session.peer_stream_id = frame.source_stream_id;
3415 session.connected = true;
3416 let peer_hostname = session.peer_hostname.clone();
3417 let peer_address = session.peer_address;
3418 let local_port = session.local_port;
3419 let peer_port = session.peer_port;
3420 let handlers = st
3421 .p2p_stream_handlers
3422 .get(&local_port)
3423 .cloned()
3424 .unwrap_or_default();
3425 events.push(PendingP2pStreamEvent {
3426 handlers,
3427 kind: P2pStreamEventKind::Connected,
3428 stream_id: frame.destination_stream_id,
3429 peer_stream_id: frame.source_stream_id,
3430 peer_hostname,
3431 peer_address,
3432 local_port,
3433 peer_port,
3434 });
3435 }
3436 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
3437 let kind = if frame.flags & P2P_STREAM_RST != 0 {
3438 P2pStreamEventKind::Reset
3439 } else if frame.flags & P2P_STREAM_FIN != 0 {
3440 P2pStreamEventKind::Closed
3441 } else {
3442 P2pStreamEventKind::Data
3443 };
3444 let session_id = if frame.destination_stream_id != 0 {
3445 Some(frame.destination_stream_id)
3446 } else {
3447 st.p2p_stream_sessions.iter().find_map(|(id, session)| {
3448 (session.peer_stream_id == frame.source_stream_id
3449 && session.local_port == msg.destination_port
3450 && session.peer_port == msg.source_port
3451 && session.peer_address == msg.source_address
3452 && session.peer_hostname.as_ref() == msg.source_hostname)
3453 .then_some(*id)
3454 })
3455 };
3456 if let Some(session_id) = session_id
3457 && let Some(session) = st.p2p_stream_sessions.get(&session_id)
3458 {
3459 let handlers = st
3460 .p2p_stream_handlers
3461 .get(&session.local_port)
3462 .cloned()
3463 .unwrap_or_default();
3464 events.push(PendingP2pStreamEvent {
3465 handlers,
3466 kind,
3467 stream_id: session_id,
3468 peer_stream_id: frame.source_stream_id,
3469 peer_hostname: session.peer_hostname.clone(),
3470 peer_address: session.peer_address,
3471 local_port: session.local_port,
3472 peer_port: session.peer_port,
3473 });
3474 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
3475 st.p2p_stream_sessions.remove(&session_id);
3476 }
3477 }
3478 }
3479 }
3480 if let Some((dst, dst_port, src_port, payload)) = reply {
3481 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
3482 }
3483 for pending in events {
3484 for handler in pending.handlers {
3485 (handler.handler)(P2pStreamEvent {
3486 kind: pending.kind,
3487 stream_id: pending.stream_id,
3488 peer_stream_id: pending.peer_stream_id,
3489 sequence: frame.sequence,
3490 peer_hostname: pending.peer_hostname.as_ref(),
3491 peer_address: pending.peer_address,
3492 local_port: pending.local_port,
3493 peer_port: pending.peer_port,
3494 payload: frame.payload,
3495 })?;
3496 }
3497 }
3498 Ok(())
3499 }
3500
3501 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) {
3507 let mut st = self.state.lock();
3508 Self::remember_reliable_return_route_locked(&mut st, packet_id);
3509 st.reliable_return_routes
3510 .insert(packet_id, ReliableReturnRouteState { side });
3511 }
3512
3513 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) {
3519 let cap = runtime_reliable_max_return_routes().max(1);
3520 st.reliable_return_route_order
3521 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
3522 while st.reliable_return_route_order.len() >= cap {
3523 if let Some(oldest) = st.reliable_return_route_order.pop_front() {
3524 st.reliable_return_routes.remove(&oldest);
3525 } else {
3526 break;
3527 }
3528 }
3529 st.reliable_return_route_order.push_back(packet_id);
3530 }
3531
3532 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
3533 let cap = runtime_reliable_max_end_to_end_pending().max(1);
3534 st.end_to_end_reliable_tx_order
3535 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
3536 while st.end_to_end_reliable_tx_order.len() >= cap {
3537 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
3538 st.end_to_end_reliable_tx.remove(&oldest);
3539 } else {
3540 break;
3541 }
3542 }
3543 st.end_to_end_reliable_tx_order.push_back(packet_id);
3544 }
3545
3546 #[cfg(feature = "discovery")]
3547 fn expected_end_to_end_destinations_locked(
3548 &self,
3549 st: &RouterInner,
3550 data: &RouterItem,
3551 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
3552 let (eps, ty) = self.item_route_info(data)?;
3553 let now_ms = self.clock.now_ms();
3554 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
3555 let prefer_best_overlap =
3556 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
3557 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
3558 let mut candidates: Vec<(u64, RouterSideId, usize)> = Vec::new();
3559 let mut best_overlap = 0usize;
3560 let mut out = BTreeMap::new();
3561 for (&side, route) in st.discovery_routes.iter() {
3562 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3563 continue;
3564 }
3565 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
3566 continue;
3567 };
3568 if restrict_link_local && !side_ref.opts.link_local_enabled {
3569 continue;
3570 }
3571 if !self.route_allowed_locked(st, None, Some(ty), side) {
3572 continue;
3573 }
3574 for sender_state in route.announcers.values() {
3575 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3576 continue;
3577 }
3578 for board in sender_state.topology_boards.iter() {
3579 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
3580 continue;
3581 }
3582 let overlap = Self::endpoint_overlap_count(
3583 board.reachable_endpoints.iter().copied(),
3584 &scoring_eps,
3585 );
3586 if overlap > 0 {
3587 if prefer_best_overlap {
3588 best_overlap = best_overlap.max(overlap);
3589 candidates.push((Self::sender_hash(&board.sender_id), side, overlap));
3590 } else {
3591 out.insert(Self::sender_hash(&board.sender_id), side);
3592 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3593 return Ok(out);
3594 }
3595 }
3596 }
3597 }
3598 }
3599 }
3600
3601 if prefer_best_overlap {
3602 for (sender_hash, side, overlap) in candidates {
3603 if overlap == best_overlap {
3604 out.insert(sender_hash, side);
3605 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3606 return Ok(out);
3607 }
3608 }
3609 }
3610 }
3611 Ok(out)
3612 }
3613
3614 #[cfg(feature = "discovery")]
3615 #[allow(clippy::too_many_arguments)]
3616 fn discovered_route_candidates_locked(
3617 &self,
3618 st: &RouterInner,
3619 exclude: Option<RouterSideId>,
3620 ty: DataType,
3621 eps: &[DataEndpoint],
3622 target_senders: &[u64],
3623 prefer_nonlocal: bool,
3624 preferred_timesync_source: Option<&str>,
3625 ) -> Vec<DiscoveryCandidateMatch> {
3626 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
3627 let now_ms = self.clock.now_ms();
3628 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
3629 let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
3630 exclude != Some(side)
3631 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
3632 && route.reachable_network_variables.contains(&ty)
3633 });
3634 let mut out = Vec::new();
3635 let mut unique_endpoint_fallbacks = Vec::new();
3636
3637 for (&side, route) in st.discovery_routes.iter() {
3638 if exclude == Some(side)
3639 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
3640 {
3641 continue;
3642 }
3643 if restrict_link_local
3644 && st
3645 .sides
3646 .get(side)
3647 .and_then(|s| s.as_ref())
3648 .map(|s| !s.opts.link_local_enabled)
3649 .unwrap_or(true)
3650 {
3651 continue;
3652 }
3653 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
3654 continue;
3655 }
3656 if !target_senders.is_empty()
3657 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
3658 {
3659 if scoring_eps
3660 .iter()
3661 .copied()
3662 .any(|ep| route.reachable.contains(&ep))
3663 {
3664 unique_endpoint_fallbacks.push(side);
3665 }
3666 continue;
3667 }
3668 if !target_senders.is_empty() {
3669 out.push(DiscoveryCandidateMatch {
3670 side,
3671 overlap: usize::MAX,
3672 });
3673 continue;
3674 }
3675 if has_variable_owner {
3676 if route.reachable_network_variables.contains(&ty) {
3677 out.push(DiscoveryCandidateMatch {
3678 side,
3679 overlap: usize::MAX,
3680 });
3681 }
3682 continue;
3683 }
3684 if preferred_timesync_source
3685 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
3686 {
3687 out.push(DiscoveryCandidateMatch {
3688 side,
3689 overlap: usize::MAX,
3690 });
3691 continue;
3692 }
3693 let overlap =
3694 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
3695 if overlap > 0 {
3696 out.push(DiscoveryCandidateMatch { side, overlap });
3697 }
3698 }
3699 if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
3700 out.push(DiscoveryCandidateMatch {
3701 side: unique_endpoint_fallbacks[0],
3702 overlap: usize::MAX,
3703 });
3704 }
3705 out
3706 }
3707
3708 #[cfg(feature = "discovery")]
3714 fn retain_shortest_discovery_candidates_locked(
3715 st: &RouterInner,
3716 matches: &mut Vec<DiscoveryCandidateMatch>,
3717 endpoints: &[DataEndpoint],
3718 target_senders: &[u64],
3719 now_ms: u64,
3720 ) {
3721 if matches.len() <= 1 || endpoints.is_empty() {
3722 return;
3723 }
3724
3725 let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
3726 let route = st.discovery_routes.get(&side)?;
3727 route
3728 .announcers
3729 .iter()
3730 .filter(|(_, state)| {
3731 now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
3732 })
3733 .filter_map(|(announcer, state)| {
3734 let targets: BTreeSet<&str> = state
3735 .topology_boards
3736 .iter()
3737 .filter(|board| {
3738 target_sender
3739 .map(|target| Self::sender_hash(&board.sender_id) == target)
3740 .unwrap_or_else(|| {
3741 endpoints.iter().any(|endpoint| {
3742 board.reachable_endpoints.contains(endpoint)
3743 })
3744 })
3745 })
3746 .map(|board| board.sender_id.as_str())
3747 .collect();
3748 if targets.is_empty() {
3749 return if let Some(target) = target_sender {
3750 state
3751 .topology_boards
3752 .iter()
3753 .any(|board| Self::sender_hash(&board.sender_id) == target)
3754 .then_some(usize::MAX / 2)
3755 } else {
3756 state
3757 .reachable
3758 .iter()
3759 .any(|endpoint| endpoints.contains(endpoint))
3760 .then_some(usize::MAX / 2)
3761 };
3762 }
3763
3764 let mut distances: BTreeMap<&str, usize> = BTreeMap::new();
3765 let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
3766 while let Some((sender, distance)) = pending.pop_front() {
3767 if distances.contains_key(sender) {
3768 continue;
3769 }
3770 distances.insert(sender, distance);
3771 if targets.contains(sender) {
3772 return Some(distance);
3773 }
3774 for board in &state.topology_boards {
3775 if board.sender_id == sender {
3776 for peer in &board.connections {
3777 if !distances.contains_key(peer.as_str()) {
3778 pending.push_back((peer.as_str(), distance + 1));
3779 }
3780 }
3781 } else if board.connections.iter().any(|peer| peer == sender)
3782 && !distances.contains_key(board.sender_id.as_str())
3783 {
3784 pending.push_back((board.sender_id.as_str(), distance + 1));
3785 }
3786 }
3787 }
3788 None
3789 })
3790 .min()
3791 };
3792
3793 let route_targets: Vec<Option<u64>> = if target_senders.is_empty() {
3794 vec![None]
3795 } else {
3796 target_senders.iter().copied().map(Some).collect()
3797 };
3798 let mut selected = BTreeSet::new();
3799 for target in route_targets {
3800 let distances: Vec<(RouterSideId, usize)> = matches
3801 .iter()
3802 .filter_map(|candidate| {
3803 route_distance(candidate.side, target)
3804 .map(|distance| (candidate.side, distance))
3805 })
3806 .collect();
3807 let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
3808 continue;
3809 };
3810 selected.extend(
3811 distances
3812 .into_iter()
3813 .filter(|(_, distance)| *distance == best)
3814 .map(|(side, _)| side),
3815 );
3816 }
3817 if !selected.is_empty() {
3818 matches.retain(|candidate| selected.contains(&candidate.side));
3819 }
3820 }
3821
3822 #[cfg(feature = "discovery")]
3823 fn select_discovered_candidate_sides_locked(
3824 &self,
3825 st: &mut RouterInner,
3826 exclude: Option<RouterSideId>,
3827 ty: DataType,
3828 target_senders: &[u64],
3829 prefer_best_overlap: bool,
3830 matches: Vec<DiscoveryCandidateMatch>,
3831 ) -> Vec<RouterSideId> {
3832 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
3833 || !target_senders.is_empty()
3834 {
3835 RouteSelectionOrigin::Flood
3836 } else {
3837 RouteSelectionOrigin::Discovered
3838 };
3839
3840 let mut matches = matches;
3841 if matches.iter().any(|m| m.overlap == usize::MAX) {
3842 matches.retain(|m| m.overlap == usize::MAX);
3843 }
3844
3845 let selected: Vec<RouterSideId> = if prefer_best_overlap {
3846 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
3847 matches
3848 .into_iter()
3849 .filter(|m| m.overlap == best_overlap)
3850 .map(|m| m.side)
3851 .collect()
3852 } else {
3853 matches.into_iter().map(|m| m.side).collect()
3854 };
3855
3856 let has_distributed_variable_owners = selected.iter().any(|side| {
3863 st.discovery_routes
3864 .get(side)
3865 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
3866 });
3867 if DataType::try_named("HEARTBEAT") == Some(ty) || has_distributed_variable_owners {
3868 selected
3869 } else {
3870 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
3871 }
3872 }
3873
3874 fn register_end_to_end_reliable_tx(&self, data: &RouterItem) -> TelemetryResult<()> {
3875 let packet_id = Self::get_hash(data);
3876 let now_ms = self.clock.now_ms();
3877 let ty = match data {
3878 RouterItem::Packet(pkt) => pkt.data_type(),
3879 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
3880 };
3881 let mut st = self.state.lock();
3882 #[cfg(feature = "discovery")]
3883 let mut pending_destinations = self.expected_end_to_end_destinations_locked(&st, data)?;
3884 #[cfg(not(feature = "discovery"))]
3885 let mut pending_destinations = BTreeMap::new();
3886 self.filter_trackable_end_to_end_destinations_locked(&st, ty, &mut pending_destinations);
3887 let tracked_destinations = !pending_destinations.is_empty();
3888 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
3889 st.end_to_end_reliable_tx.insert(
3890 packet_id,
3891 EndToEndReliableSent {
3892 data: data.clone(),
3893 pending_destinations,
3894 tracked_destinations,
3895 last_send_ms: now_ms,
3896 retries: 0,
3897 queued: false,
3898 },
3899 );
3900 Ok(())
3901 }
3902
3903 #[cfg(feature = "discovery")]
3904 fn reconcile_end_to_end_reliable_destinations_locked(
3905 &self,
3906 st: &mut RouterInner,
3907 ) -> TelemetryResult<()> {
3908 let active_destinations = self.active_end_to_end_destinations_locked(st);
3909 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
3910 let mut completed = Vec::new();
3911
3912 for packet_id in packet_ids {
3913 let Some(data) = st
3914 .end_to_end_reliable_tx
3915 .get(&packet_id)
3916 .map(|sent| sent.data.clone())
3917 else {
3918 continue;
3919 };
3920 let expected = self.expected_end_to_end_destinations_locked(st, &data)?;
3921 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
3922 continue;
3923 };
3924 if !sent.tracked_destinations {
3925 continue;
3926 }
3927 sent.pending_destinations.retain(|sender_hash, side| {
3928 match (
3929 expected.get(sender_hash),
3930 active_destinations.get(sender_hash),
3931 ) {
3932 (Some(next_side), _) | (None, Some(next_side)) => {
3933 *side = *next_side;
3934 true
3935 }
3936 (None, None) => false,
3937 }
3938 });
3939 if sent.pending_destinations.is_empty() {
3940 completed.push(packet_id);
3941 }
3942 }
3943
3944 for packet_id in completed {
3945 st.end_to_end_reliable_tx.remove(&packet_id);
3946 }
3947
3948 Ok(())
3949 }
3950
3951 #[cfg(feature = "discovery")]
3952 fn active_end_to_end_destinations_locked(
3953 &self,
3954 st: &RouterInner,
3955 ) -> BTreeMap<u64, RouterSideId> {
3956 let now_ms = self.clock.now_ms();
3957 let mut out = BTreeMap::new();
3958 for (&side, route) in st.discovery_routes.iter() {
3959 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3960 continue;
3961 }
3962 for sender_state in route.announcers.values() {
3963 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3964 continue;
3965 }
3966 for board in sender_state.topology_boards.iter() {
3967 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
3968 continue;
3969 }
3970 out.insert(Self::sender_hash(&board.sender_id), side);
3971 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3972 return out;
3973 }
3974 }
3975 }
3976 }
3977 out
3978 }
3979
3980 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
3981 matches!(
3982 st.sides
3983 .get(side)
3984 .and_then(Option::as_ref)
3985 .map(|side| &side.tx_handler),
3986 Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
3987 )
3988 }
3989
3990 fn filter_trackable_end_to_end_destinations_locked(
3991 &self,
3992 st: &RouterInner,
3993 ty: DataType,
3994 pending: &mut BTreeMap<u64, RouterSideId>,
3995 ) {
3996 if DataType::try_named("HEARTBEAT") == Some(ty) {
4002 pending.clear();
4003 return;
4004 }
4005 let now_ms = self.clock.now_ms();
4006 pending.retain(|_, side| {
4007 Self::side_supports_end_to_end_tracking_locked(st, *side)
4008 && (is_reliable_type(ty)
4009 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
4010 });
4011 }
4012
4013 #[cfg(feature = "discovery")]
4014 fn side_has_multiple_announcers_locked(
4015 &self,
4016 st: &RouterInner,
4017 side: RouterSideId,
4018 now_ms: u64,
4019 ) -> bool {
4020 st.discovery_routes
4021 .get(&side)
4022 .map(|route| {
4023 route
4024 .announcers
4025 .values()
4026 .filter(|sender| {
4027 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
4028 })
4029 .take(2)
4030 .count()
4031 > 1
4032 })
4033 .unwrap_or(false)
4034 }
4035
4036 #[cfg(not(feature = "discovery"))]
4037 fn side_has_multiple_announcers_locked(
4038 &self,
4039 _st: &RouterInner,
4040 _side: RouterSideId,
4041 _now_ms: u64,
4042 ) -> bool {
4043 false
4044 }
4045
4046 fn queue_end_to_end_reliable_ack(
4047 &self,
4048 pkt: &Packet,
4049 called_from_queue: bool,
4050 ) -> TelemetryResult<()> {
4051 self.queue_end_to_end_reliable_ack_for_packet_id(pkt.packet_id(), called_from_queue)
4052 }
4053
4054 fn queue_end_to_end_reliable_ack_for_packet_id(
4055 &self,
4056 packet_id: u64,
4057 called_from_queue: bool,
4058 ) -> TelemetryResult<()> {
4059 if !self.cfg.reliable_enabled() {
4060 return Ok(());
4061 }
4062 let ack_sender = self.encode_end_to_end_ack_sender();
4063 let ack = Packet::new(
4064 DataType::ReliableAck,
4065 message_meta(DataType::ReliableAck).endpoints_ref(),
4066 ack_sender.as_str(),
4067 self.packet_timestamp_ms(),
4068 Self::encode_end_to_end_reliable_ack(packet_id),
4069 )?;
4070 let local_sender = self.sender_arc();
4071 let ack = self.attach_wire_contract_to_item(
4072 RouterItem::Packet(ack),
4073 &[Self::sender_hash(local_sender.as_ref())],
4074 )?;
4075 self.emit_internal_tx(RouterTxItem::Broadcast(ack), true, called_from_queue)
4076 }
4077
4078 fn emit_internal_tx(
4079 &self,
4080 item: RouterTxItem,
4081 ignore_local: bool,
4082 called_from_queue: bool,
4083 ) -> TelemetryResult<()> {
4084 if called_from_queue {
4085 self.tx_queue_item_with_flags(item, ignore_local)
4086 } else {
4087 self.tx_item_impl(item, ignore_local, false)
4088 }
4089 }
4090
4091 fn emit_internal_tx_with_priority(
4092 &self,
4093 item: RouterTxItem,
4094 ignore_local: bool,
4095 priority: u8,
4096 called_from_queue: bool,
4097 ) -> TelemetryResult<()> {
4098 if called_from_queue {
4099 self.tx_queue_item_with_priority(item, ignore_local, priority)
4100 } else {
4101 self.tx_item_impl(item, ignore_local, false)
4102 }
4103 }
4104
4105 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
4106 {
4107 let mut st = self.state.lock();
4108 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
4109 return Ok(());
4110 };
4111 if sent.queued {
4112 return Ok(());
4113 }
4114 sent.queued = true;
4115 }
4116 self.tx_queue_item_with_priority(
4117 RouterTxItem::EndToEndReplay { packet_id },
4118 true,
4119 Self::router_item_priority_bumped(DataType::ReliableAck),
4120 )
4121 }
4122
4123 fn end_to_end_retransmit_sides(
4124 &self,
4125 packet_id: u64,
4126 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
4127 let mut st = self.state.lock();
4128 let (data, tracked_destinations, mut sides) = {
4129 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
4130 sent.queued = false;
4131 sent.last_send_ms = self.clock.now_ms();
4132 let data = sent.data.clone();
4133 let tracked_destinations = sent.tracked_destinations;
4134 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
4135 (data, tracked_destinations, sides)
4136 };
4137 if tracked_destinations && sides.is_empty() {
4138 st.end_to_end_reliable_tx.remove(&packet_id);
4139 return None;
4140 }
4141 sides.sort_unstable();
4142 sides.dedup();
4143 Some((data, sides))
4144 }
4145
4146 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
4147 let ty = match data {
4148 RouterItem::Packet(pkt) => pkt.data_type(),
4149 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
4150 };
4151 Ok(crate::scheduler_priority(ty))
4152 }
4153
4154 #[inline]
4155 fn router_item_priority_bumped(ty: DataType) -> u8 {
4156 crate::scheduler_priority(ty).saturating_add(16)
4157 }
4158
4159 #[inline]
4160 fn is_side_tx_busy(err: &TelemetryError) -> bool {
4161 matches!(err, TelemetryError::Io("side tx busy"))
4162 }
4163
4164 #[cfg(feature = "timesync")]
4165 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
4166 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
4167 || st
4168 .clock
4169 .current_time(now_mono_ns)
4170 .and_then(|reading| reading.unix_time_ms)
4171 .is_some()
4172 }
4173
4174 #[cfg(feature = "timesync")]
4175 fn reconcile_pending_timesync_request_locked(
4176 st: &mut TimeSyncRuntime,
4177 leader: &Option<TimeSyncLeader>,
4178 now_ms: u64,
4179 ) {
4180 let active_remote = match leader {
4181 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
4182 _ => None,
4183 };
4184 let should_clear = st
4185 .pending_request
4186 .as_ref()
4187 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
4188 if should_clear {
4189 st.pending_request = None;
4190 st.next_request_mono_ms = now_ms;
4191 }
4192 }
4193
4194 #[inline]
4196 fn enqueue_to_sides(
4197 &self,
4198 data: RouterItem,
4199 exclude: Option<RouterSideId>,
4200 ignore_local: bool,
4201 ) -> TelemetryResult<()> {
4202 let data = self.clear_intermediate_hop_targets(data)?;
4203 let plan = self.remote_side_plan(&data, exclude)?;
4204 let mut st = self.state.lock();
4205 let ty = Self::item_data_type(&data)?;
4206 let priority = if st.managed_variable_types.contains(&ty.as_u32()) {
4207 crate::transport_priority(DataType::ManagedVariableValue)
4208 } else {
4209 Self::router_item_priority(&data)?
4210 };
4211
4212 let RemoteSidePlan::Target(sides) = plan;
4213 for idx in sides {
4214 st.push_transmit(TxQueued {
4215 item: RouterTxItem::ToSide {
4216 src: exclude,
4217 dst: idx,
4218 data: data.clone(),
4219 },
4220 ignore_local,
4221 priority,
4222 })?;
4223 }
4224
4225 Ok(())
4226 }
4227
4228 fn relay_send(
4229 &self,
4230 data: RouterItem,
4231 src: Option<RouterSideId>,
4232 called_from_queue: bool,
4233 ) -> TelemetryResult<()> {
4234 if called_from_queue {
4235 return self.enqueue_to_sides(data, src, true);
4236 }
4237
4238 let data = self.clear_intermediate_hop_targets(data)?;
4239 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
4240 for side in sides {
4241 self.tx_item_impl(
4242 RouterTxItem::ToSide {
4243 src,
4244 dst: side,
4245 data: data.clone(),
4246 },
4247 true,
4248 false,
4249 )?;
4250 }
4251
4252 Ok(())
4253 }
4254
4255 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
4256 match data {
4257 RouterItem::Packet(pkt) => {
4258 pkt.validate()?;
4259 let mut eps = pkt.endpoints().to_vec();
4260 eps.sort_unstable();
4261 eps.dedup();
4262 Ok((eps, pkt.data_type()))
4263 }
4264 RouterItem::Packed(bytes) => {
4265 let env = wire_format::peek_envelope(bytes.as_ref())?;
4266 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
4267 eps.sort_unstable();
4268 eps.dedup();
4269 Ok((eps, env.ty))
4270 }
4271 }
4272 }
4273
4274 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
4275 match data {
4276 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
4277 RouterItem::Packed(bytes) => Ok(wire_format::peek_envelope(bytes.as_ref())?.ty),
4278 }
4279 }
4280
4281 fn e2e_crypto_supported(&self) -> bool {
4282 #[cfg(feature = "cryptography")]
4283 {
4284 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
4285 && crate::crypto::registered_crypto_available()
4286 }
4287 #[cfg(not(feature = "cryptography"))]
4288 {
4289 false
4290 }
4291 }
4292
4293 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
4294 if is_internal_control_type(ty) {
4295 return false;
4296 }
4297 match self.cfg.e2e_encryption() {
4298 RouterE2eEncryptionMode::Disabled => {
4299 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
4300 }
4301 RouterE2eEncryptionMode::RequiredOnly => {
4302 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
4303 }
4304 RouterE2eEncryptionMode::Preferred => matches!(
4305 message_e2e_encryption_policy(ty),
4306 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
4307 ),
4308 RouterE2eEncryptionMode::ForceAll => true,
4309 }
4310 }
4311
4312 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
4313 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
4314 return Err(TelemetryError::BadArg);
4315 }
4316 Ok(())
4317 }
4318
4319 #[cfg(feature = "cryptography")]
4320 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
4321 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
4322 Some(wire_format::E2eSealConfig {
4323 key_id: self.cfg.e2e_key_id(),
4324 })
4325 } else {
4326 None
4327 }
4328 }
4329
4330 #[inline]
4331 fn pack_packet_for_router(
4332 &self,
4333 pkt: &Packet,
4334 reliable: Option<wire_format::ReliableHeader>,
4335 ) -> TelemetryResult<Arc<[u8]>> {
4336 #[cfg(feature = "cryptography")]
4337 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
4338 return wire_format::pack_packet_with_wire_contract_e2e(
4339 pkt,
4340 reliable,
4341 pkt.wire_shape(),
4342 pkt.wire_target_senders(),
4343 e2e,
4344 );
4345 }
4346 Ok(match reliable {
4347 Some(hdr) => wire_format::pack_packet_with_reliable(pkt, hdr),
4348 None => wire_format::pack_packet(pkt),
4349 })
4350 }
4351
4352 #[inline]
4353 fn pack_packet_for_contract(
4354 &self,
4355 pkt: &Packet,
4356 reliable: Option<wire_format::ReliableHeader>,
4357 shape: Option<MessageElement>,
4358 target_senders: &[u64],
4359 ) -> TelemetryResult<Arc<[u8]>> {
4360 #[cfg(feature = "cryptography")]
4361 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
4362 return wire_format::pack_packet_with_wire_contract_e2e(
4363 pkt,
4364 reliable,
4365 shape,
4366 target_senders,
4367 e2e,
4368 );
4369 }
4370 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
4371 }
4372
4373 #[cfg(feature = "cryptography")]
4374 #[inline]
4375 fn prepare_packed_for_remote(
4376 &self,
4377 bytes: Arc<[u8]>,
4378 reliable_override: Option<Option<wire_format::ReliableHeader>>,
4379 ) -> TelemetryResult<Arc<[u8]>> {
4380 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
4381 if frame.ack_only() || self.e2e_seal_config_for_type(frame.envelope.ty).is_none() {
4382 return Ok(bytes);
4383 }
4384 let reliable = reliable_override.unwrap_or(frame.reliable);
4385 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4386 self.pack_packet_for_contract(
4387 &pkt,
4388 reliable,
4389 frame.envelope.wire_shape,
4390 &frame.envelope.target_senders,
4391 )
4392 }
4393
4394 fn item_target_senders(&self, data: &RouterItem) -> TelemetryResult<Arc<[u64]>> {
4395 match data {
4396 RouterItem::Packet(pkt) => Ok(Arc::from(pkt.wire_target_senders())),
4397 RouterItem::Packed(bytes) => {
4398 Ok(wire_format::peek_envelope(bytes.as_ref())?.target_senders)
4399 }
4400 }
4401 }
4402
4403 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
4404 let targets = self.item_target_senders(data)?;
4405 if targets.is_empty() {
4406 return Ok(true);
4407 }
4408 let local_sender = self.sender_arc();
4409 let local_hash = Self::sender_hash(local_sender.as_ref());
4410 Ok(targets.contains(&local_hash))
4411 }
4412
4413 fn item_targets_router_as_intermediate_hop(
4414 &self,
4415 data: &RouterItem,
4416 endpoints: &[DataEndpoint],
4417 ) -> TelemetryResult<bool> {
4418 let targets = self.item_target_senders(data)?;
4419 if targets.is_empty()
4420 || targets.len() != 1
4425 || endpoints.iter().copied().any(|endpoint| {
4426 self.endpoint_has_packet_handler(endpoint)
4427 || self.endpoint_has_packed_handler(endpoint)
4428 })
4429 {
4430 return Ok(false);
4431 }
4432 let local_sender = self.sender_arc();
4433 Ok(targets.contains(&Self::sender_hash(local_sender.as_ref())))
4434 }
4435
4436 fn clear_intermediate_hop_targets(&self, data: RouterItem) -> TelemetryResult<RouterItem> {
4437 let (endpoints, _) = self.item_route_info(&data)?;
4438 if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
4439 return Ok(data);
4440 }
4441 let pkt = match &data {
4442 RouterItem::Packet(pkt) => pkt.clone(),
4443 RouterItem::Packed(bytes) => wire_format::unpack_packet(bytes.as_ref())?,
4444 };
4445 let reliable = match &data {
4446 RouterItem::Packet(_) => None,
4447 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
4448 };
4449 let shape = match &data {
4450 RouterItem::Packet(pkt) => pkt.wire_shape(),
4451 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.wire_shape,
4452 };
4453 Ok(RouterItem::Packed(self.pack_packet_for_contract(
4454 &pkt,
4455 reliable,
4456 shape,
4457 &[],
4458 )?))
4459 }
4460
4461 #[cfg(feature = "discovery")]
4462 fn side_matches_target_senders_locked(
4463 st: &RouterInner,
4464 side: RouterSideId,
4465 target_senders: &[u64],
4466 now_ms: u64,
4467 ) -> bool {
4468 st.discovery_routes
4469 .get(&side)
4470 .map(|route| {
4471 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4472 return false;
4473 }
4474 route.announcers.iter().any(|(announcer, sender_state)| {
4475 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4476 return false;
4477 }
4478 target_senders.contains(&Self::sender_hash(announcer))
4479 || sender_state
4480 .advertised_address
4481 .is_some_and(|address| target_senders.contains(&u64::from(address)))
4482 || sender_state.topology_boards.iter().any(|board| {
4483 target_senders.contains(&Self::sender_hash(&board.sender_id))
4484 })
4485 })
4486 })
4487 .unwrap_or(false)
4488 }
4489
4490 fn attach_wire_contract_to_item(
4491 &self,
4492 data: RouterItem,
4493 target_senders: &[u64],
4494 ) -> TelemetryResult<RouterItem> {
4495 match data {
4496 RouterItem::Packet(pkt) => {
4497 let reliable = if is_reliable_type(pkt.data_type()) {
4498 Some(wire_format::ReliableHeader {
4499 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
4500 seq: 0,
4501 ack: 0,
4502 })
4503 } else {
4504 None
4505 };
4506 let bytes = self.pack_packet_for_contract(
4507 &pkt,
4508 reliable,
4509 Some(message_meta(pkt.data_type()).element),
4510 target_senders,
4511 )?;
4512 Ok(RouterItem::Packed(bytes))
4513 }
4514 RouterItem::Packed(bytes) => Ok(RouterItem::Packed(bytes)),
4515 }
4516 }
4517
4518 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
4519 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
4520 }
4521
4522 fn should_route_remote(
4523 &self,
4524 data: &RouterItem,
4525 exclude: Option<RouterSideId>,
4526 ) -> TelemetryResult<bool> {
4527 #[cfg(feature = "discovery")]
4528 {
4529 let RemoteSidePlan::Target(sides) = self.remote_side_plan(data, exclude)?;
4530 Ok(!sides.is_empty())
4531 }
4532
4533 #[cfg(not(feature = "discovery"))]
4534 {
4535 let (eps, ty) = self.item_route_info(data)?;
4536 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
4537 return Ok(false);
4538 }
4539 let st = self.state.lock();
4540 Ok(!self
4541 .eligible_side_ids_locked(
4542 &st,
4543 exclude,
4544 Some(ty),
4545 Self::endpoints_are_link_local_only(&eps),
4546 )
4547 .is_empty())
4548 }
4549 }
4550
4551 #[cfg(feature = "discovery")]
4552 fn has_explicit_route_policy_locked(
4553 st: &RouterInner,
4554 src: Option<RouterSideId>,
4555 ty: DataType,
4556 ) -> bool {
4557 st.route_overrides
4558 .keys()
4559 .any(|(route_src, _)| *route_src == src)
4560 || Self::has_typed_route_overrides_locked(st, src, ty)
4561 }
4562
4563 fn remote_side_plan(
4564 &self,
4565 data: &RouterItem,
4566 exclude: Option<RouterSideId>,
4567 ) -> TelemetryResult<RemoteSidePlan> {
4568 #[cfg(feature = "discovery")]
4569 {
4570 let (eps, ty) = self.item_route_info(data)?;
4571 let target_senders = self.item_target_senders(data)?;
4572 let resolve_beyond_local_hop =
4573 self.item_targets_router_as_intermediate_hop(data, &eps)?;
4574 let routing_target_senders: &[u64] = if resolve_beyond_local_hop {
4575 &[]
4576 } else {
4577 target_senders.as_ref()
4578 };
4579 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
4580 if discovery::is_discovery_type(ty) {
4581 let mut st = self.state.lock();
4582 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
4583 return Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4584 &mut st,
4585 exclude,
4586 sides,
4587 RouteSelectionOrigin::Flood,
4588 )));
4589 }
4590 #[cfg(feature = "timesync")]
4591 if ty == DataType::TimeSyncAnnounce {
4592 let mut st = self.state.lock();
4599 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
4600 let sides =
4601 Self::filter_timesync_sides_locked(&mut st, ty, self.clock.now_ms(), sides);
4602 return Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4603 &mut st,
4604 exclude,
4605 sides,
4606 RouteSelectionOrigin::Flood,
4607 )));
4608 }
4609 let mut st = self.state.lock();
4610 let now_ms = self.clock.now_ms();
4615 let has_remote_variable_owner = st.managed_variable_types.contains(&ty.as_u32())
4616 && st.discovery_routes.iter().any(|(&side, route)| {
4617 exclude != Some(side)
4618 && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
4619 && route.reachable_network_variables.contains(&ty)
4620 });
4621 if !(has_nonlocal_endpoint(&eps, &self.cfg)
4622 || force_remote_for_type(ty)
4623 || !target_senders.is_empty()
4624 || has_remote_variable_owner)
4625 {
4626 return Ok(RemoteSidePlan::Target(Vec::new()));
4627 }
4628
4629 #[cfg(feature = "timesync")]
4630 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
4631 #[cfg(not(feature = "timesync"))]
4632 let preferred_timesync_source: Option<String> = None;
4633
4634 if let Some(packet_id) = preferred_packet_id {
4635 let target_side = st
4636 .reliable_return_routes
4637 .get(&packet_id)
4638 .map(|route| route.side);
4639 if let Some(side) = target_side
4640 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
4641 {
4642 #[cfg(feature = "timesync")]
4643 if !Self::timesync_allowed_for_side_locked(
4644 &mut st,
4645 side,
4646 ty,
4647 self.clock.now_ms(),
4648 ) {
4649 return Ok(RemoteSidePlan::Target(Vec::new()));
4650 }
4651 return Ok(RemoteSidePlan::Target(vec![side]));
4652 }
4653 }
4659 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
4660 let prefer_best_overlap = is_reliable_type(ty)
4661 && routing_target_senders.is_empty()
4662 && preferred_packet_id.is_none();
4663 if st.discovery_routes.is_empty() {
4664 let mut fallback =
4665 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
4666 #[cfg(feature = "timesync")]
4667 {
4668 fallback = Self::filter_timesync_sides_locked(
4669 &mut st,
4670 ty,
4671 self.clock.now_ms(),
4672 fallback,
4673 );
4674 }
4675 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
4676 fallback
4677 } else {
4678 Vec::new()
4679 }));
4680 }
4681 let mut matches = self.discovered_route_candidates_locked(
4682 &st,
4683 exclude,
4684 ty,
4685 &eps,
4686 routing_target_senders,
4687 prefer_best_overlap,
4688 preferred_timesync_source.as_deref(),
4689 );
4690 #[cfg(feature = "timesync")]
4691 {
4692 matches =
4693 Self::filter_timesync_matches_locked(&mut st, ty, self.clock.now_ms(), matches);
4694 }
4695 let has_distributed_variable_owners = matches.iter().any(|candidate| {
4701 st.discovery_routes
4702 .get(&candidate.side)
4703 .is_some_and(|route| route.reachable_network_variables.contains(&ty))
4704 });
4705 if DataType::try_named("HEARTBEAT") != Some(ty) && !has_distributed_variable_owners {
4706 Self::retain_shortest_discovery_candidates_locked(
4707 &st,
4708 &mut matches,
4709 &eps,
4710 routing_target_senders,
4711 self.clock.now_ms(),
4712 );
4713 }
4714
4715 if !matches.is_empty() {
4716 Ok(RemoteSidePlan::Target(
4717 self.select_discovered_candidate_sides_locked(
4718 &mut st,
4719 exclude,
4720 ty,
4721 routing_target_senders,
4722 prefer_best_overlap,
4723 matches,
4724 ),
4725 ))
4726 } else if exclude.is_some() {
4727 let fallback =
4728 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
4729 Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
4730 fallback
4731 } else {
4732 Vec::new()
4733 }))
4734 } else if prefer_best_overlap {
4735 Ok(RemoteSidePlan::Target(Vec::new()))
4736 } else {
4737 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
4738 let mut sides =
4739 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
4740 #[cfg(feature = "timesync")]
4741 {
4742 sides = Self::filter_timesync_sides_locked(
4743 &mut st,
4744 ty,
4745 self.clock.now_ms(),
4746 sides,
4747 );
4748 }
4749 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4750 &mut st,
4751 exclude,
4752 sides,
4753 RouteSelectionOrigin::Flood,
4754 )))
4755 } else {
4756 Ok(RemoteSidePlan::Target(Vec::new()))
4757 }
4758 }
4759 }
4760 #[cfg(not(feature = "discovery"))]
4761 {
4762 let (_, ty) = self.item_route_info(data)?;
4763 let mut st = self.state.lock();
4764 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
4765 let target_side = st
4766 .reliable_return_routes
4767 .get(&packet_id)
4768 .map(|route| route.side);
4769 if let Some(side) = target_side
4770 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
4771 {
4772 return Ok(RemoteSidePlan::Target(vec![side]));
4773 }
4774 return Ok(RemoteSidePlan::Target(Vec::new()));
4775 }
4776 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
4777 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4778 &mut st,
4779 exclude,
4780 sides,
4781 RouteSelectionOrigin::Flood,
4782 )))
4783 }
4784 }
4785
4786 #[cfg(feature = "discovery")]
4787 fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
4788 let mut eps: Vec<DataEndpoint> = self.cfg.handlers.iter().map(|h| h.endpoint).collect();
4789 let _ = st;
4795 #[cfg(feature = "timesync")]
4796 if self.cfg.timesync_config().is_some() {
4797 eps.push(DataEndpoint::TimeSync);
4798 }
4799 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
4800 eps.sort_unstable();
4801 eps.dedup();
4802 eps
4803 }
4804
4805 #[cfg(feature = "discovery")]
4806 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
4807 #[cfg(feature = "timesync")]
4808 {
4809 let st = self.timesync.lock();
4810 if let Some(tracker) = st.tracker.as_ref()
4811 && tracker.should_serve(
4812 now_ms,
4813 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
4814 )
4815 {
4816 return vec![self.sender_arc().to_string()];
4817 }
4818 }
4819 Vec::new()
4820 }
4821
4822 #[cfg(all(feature = "discovery", feature = "timesync"))]
4823 fn preferred_timesync_route_source(
4824 &self,
4825 data: &RouterItem,
4826 ty: DataType,
4827 ) -> TelemetryResult<Option<String>> {
4828 if !matches!(
4829 ty,
4830 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4831 ) {
4832 return Ok(None);
4833 }
4834
4835 match data {
4836 RouterItem::Packet(pkt) => match ty {
4837 DataType::TimeSyncRequest => {
4838 let local_sender = self.sender_arc();
4839 if pkt.sender() == local_sender.as_ref() {
4840 Ok(self.timesync.lock().tracker.as_ref().and_then(|tracker| {
4841 tracker.current_source().map(|src| src.sender.clone())
4842 }))
4843 } else {
4844 Ok(None)
4845 }
4846 }
4847 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
4848 Ok(Some(pkt.sender().to_owned()))
4849 }
4850 _ => Ok(None),
4851 },
4852 RouterItem::Packed(bytes) => {
4853 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4854 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
4855 }
4856 }
4857 }
4858
4859 #[cfg(feature = "discovery")]
4860 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
4861 st.discovery_cadence.on_topology_change(now_ms);
4862 for side_id in st
4867 .sides
4868 .iter()
4869 .enumerate()
4870 .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
4871 {
4872 st.discovery_side_throttle
4873 .entry(side_id)
4874 .or_default()
4875 .pending_incremental = true;
4876 st.discovery_side_throttle
4877 .entry(side_id)
4878 .or_default()
4879 .incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
4880 }
4881 for transport in st.side_transport.values_mut() {
4885 transport.clear_header_templates();
4886 }
4887 }
4888
4889 #[cfg(feature = "discovery")]
4890 fn sender_topology_board_mut<'a>(
4891 sender_state: &'a mut DiscoverySenderState,
4892 sender_id: &str,
4893 ) -> &'a mut TopologyBoardNode {
4894 if let Some(idx) = sender_state
4895 .topology_boards
4896 .iter()
4897 .position(|board| board.sender_id == sender_id)
4898 {
4899 return &mut sender_state.topology_boards[idx];
4900 }
4901 sender_state.topology_boards.push(TopologyBoardNode {
4902 sender_id: sender_id.to_string(),
4903 reachable_endpoints: Vec::new(),
4904 reachable_timesync_sources: Vec::new(),
4905 connections: Vec::new(),
4906 });
4907 sender_state
4908 .topology_boards
4909 .last_mut()
4910 .expect("board inserted above")
4911 }
4912
4913 #[cfg(feature = "discovery")]
4914 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
4915 discovery::normalize_topology_boards(&mut sender_state.topology_boards);
4916 let (mut reachable, mut reachable_timesync_sources) =
4917 discovery::summarize_topology_boards(&sender_state.topology_boards);
4918 reachable.extend(sender_state.advertised_reachable.iter().copied());
4919 reachable.sort_unstable();
4920 reachable.dedup();
4921 reachable_timesync_sources.extend(
4922 sender_state
4923 .advertised_reachable_timesync_sources
4924 .iter()
4925 .cloned(),
4926 );
4927 reachable_timesync_sources.sort_unstable();
4928 reachable_timesync_sources.dedup();
4929 sender_state.reachable = reachable;
4930 sender_state.reachable_timesync_sources = reachable_timesync_sources;
4931 }
4932
4933 #[cfg(feature = "discovery")]
4934 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
4935 let mut reachable = Vec::new();
4936 let mut reachable_network_variables = Vec::new();
4937 let mut reachable_timesync_sources = Vec::new();
4938 let mut last_seen_ms = 0u64;
4939 for sender in route.announcers.values() {
4940 reachable.extend(sender.reachable.iter().copied());
4941 reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
4942 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
4943 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
4944 }
4945 reachable_network_variables
4946 .extend(route.requested_network_variables.iter().map(|(ty, _)| *ty));
4947 last_seen_ms = last_seen_ms.max(
4948 route
4949 .requested_network_variables
4950 .iter()
4951 .map(|(_, last_seen_ms)| *last_seen_ms)
4952 .max()
4953 .unwrap_or(0),
4954 );
4955 reachable.sort_unstable();
4956 reachable.dedup();
4957 reachable_network_variables.sort_unstable();
4958 reachable_network_variables.dedup();
4959 reachable_timesync_sources.sort_unstable();
4960 reachable_timesync_sources.dedup();
4961 route.reachable = reachable;
4962 route.reachable_network_variables = reachable_network_variables;
4963 route.reachable_timesync_sources = reachable_timesync_sources;
4964 route.last_seen_ms = last_seen_ms;
4965 }
4966
4967 #[cfg(feature = "discovery")]
4968 fn local_discovery_topology_board(
4969 &self,
4970 st: &RouterInner,
4971 now_ms: u64,
4972 link_local_enabled: bool,
4973 ) -> TopologyBoardNode {
4974 let mut reachable_endpoints = self.local_discovery_endpoints(st);
4975 if !link_local_enabled {
4976 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
4977 }
4978 let mut connections = Vec::new();
4979 for route in st.discovery_routes.values() {
4980 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4981 continue;
4982 }
4983 for (sender, sender_state) in route.announcers.iter() {
4984 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
4985 connections.push(sender.clone());
4986 }
4987 }
4988 }
4989 connections.sort_unstable();
4990 connections.dedup();
4991 let sender = self.sender_arc();
4992 TopologyBoardNode {
4993 sender_id: sender.to_string(),
4994 reachable_endpoints,
4995 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
4996 connections,
4997 }
4998 }
4999
5000 #[cfg(feature = "discovery")]
5001 fn advertised_discovery_topology_for_link_locked(
5002 &self,
5003 st: &RouterInner,
5004 now_ms: u64,
5005 link_local_enabled: bool,
5006 exclude_side: Option<RouterSideId>,
5007 ) -> Vec<TopologyBoardNode> {
5008 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
5009 for (&route_side, route) in st.discovery_routes.iter() {
5010 if exclude_side == Some(route_side) {
5011 continue;
5012 }
5013 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5014 continue;
5015 }
5016 for (announcer, sender_state) in route.announcers.iter() {
5017 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5018 continue;
5019 }
5020 let mut sender_boards = sender_state.topology_boards.clone();
5021 if sender_boards.is_empty() {
5022 let sender = self.sender_arc();
5023 sender_boards.push(TopologyBoardNode {
5024 sender_id: announcer.clone(),
5025 reachable_endpoints: sender_state.reachable.clone(),
5026 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
5027 connections: vec![sender.to_string()],
5028 });
5029 } else if let Some(board) = sender_boards
5030 .iter_mut()
5031 .find(|board| board.sender_id == *announcer)
5032 {
5033 board.connections.push(self.sender_arc().to_string());
5034 }
5035 if !link_local_enabled {
5036 for board in sender_boards.iter_mut() {
5037 board
5038 .reachable_endpoints
5039 .retain(|ep| !ep.is_link_local_only());
5040 }
5041 }
5042 discovery::merge_topology_boards(&mut boards, &sender_boards);
5043 }
5044 }
5045 discovery::normalize_topology_boards(&mut boards);
5046 boards
5047 }
5048
5049 #[cfg(feature = "discovery")]
5050 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
5051 let before = st.discovery_routes.clone();
5052 st.discovery_routes.retain(|_, route| {
5053 route.announcers.retain(|_, sender| {
5054 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5055 });
5056 route
5057 .requested_network_variables
5058 .retain(|(_, last_seen_ms)| {
5059 now_ms.saturating_sub(*last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
5060 });
5061 Self::recompute_discovery_side_state(route);
5062 !route.announcers.is_empty() || !route.requested_network_variables.is_empty()
5063 });
5064 st.discovery_routes != before
5065 }
5066
5067 #[cfg(feature = "discovery")]
5068 fn advertised_discovery_endpoints_for_link_locked(
5069 &self,
5070 st: &RouterInner,
5071 now_ms: u64,
5072 link_local_enabled: bool,
5073 exclude_side: Option<RouterSideId>,
5074 ) -> Vec<DataEndpoint> {
5075 let (reachable_endpoints, _) = discovery::summarize_topology_boards(
5076 &self.advertised_discovery_topology_for_link_locked(
5077 st,
5078 now_ms,
5079 link_local_enabled,
5080 exclude_side,
5081 ),
5082 );
5083 reachable_endpoints
5084 .into_iter()
5085 .filter(|ep| {
5086 !discovery::is_discovery_endpoint(*ep)
5087 && (link_local_enabled || !ep.is_link_local_only())
5088 })
5089 .collect()
5090 }
5091
5092 #[cfg(feature = "discovery")]
5093 fn advertised_discovery_timesync_sources_for_link_locked(
5094 &self,
5095 st: &RouterInner,
5096 now_ms: u64,
5097 exclude_side: Option<RouterSideId>,
5098 ) -> Vec<String> {
5099 let (_, sources) = discovery::summarize_topology_boards(
5100 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
5101 );
5102 sources
5103 }
5104
5105 #[cfg(feature = "discovery")]
5106 fn advertised_network_variables_for_link_locked(
5107 &self,
5108 st: &RouterInner,
5109 now_ms: u64,
5110 exclude_side: Option<RouterSideId>,
5111 ) -> Vec<DataType> {
5112 let mut types: Vec<DataType> = st
5113 .managed_variable_types
5114 .iter()
5115 .copied()
5116 .map(DataType)
5117 .collect();
5118 for (&route_side, route) in st.discovery_routes.iter() {
5119 if exclude_side == Some(route_side) {
5120 continue;
5121 }
5122 if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
5123 types.extend(route.reachable_network_variables.iter().copied());
5124 }
5125 }
5126 types.sort_unstable();
5127 types.dedup();
5128 types
5129 }
5130
5131 #[cfg(feature = "discovery")]
5132 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
5133 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
5134 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
5135 }
5136
5137 #[cfg(feature = "discovery")]
5138 fn should_answer_discovery_request_locked(
5139 &self,
5140 st: &RouterInner,
5141 requester: &str,
5142 now_ms: u64,
5143 ) -> bool {
5144 if requester == self.sender_arc().as_ref() {
5145 return false;
5146 }
5147 self.discovery_master_sender_locked(st, now_ms) == self.sender_arc().as_ref()
5148 }
5149
5150 #[cfg(feature = "discovery")]
5151 #[inline]
5152 fn side_is_slow_control_link_locked(
5153 st: &RouterInner,
5154 side_id: RouterSideId,
5155 now_ms: u64,
5156 ) -> bool {
5157 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
5158 let recent_slow = stats.last_slow_observed_ms > 0
5159 && now_ms.saturating_sub(stats.last_slow_observed_ms)
5160 <= DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
5161 stats.sample_count > 0
5162 && ((stats.estimated_bandwidth_bps > 0
5163 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
5164 || recent_slow)
5165 })
5166 }
5167
5168 #[cfg(feature = "discovery")]
5169 fn discovery_level_for_side_locked(
5170 st: &mut RouterInner,
5171 side_id: RouterSideId,
5172 now_ms: u64,
5173 ) -> Option<DiscoveryAdvertiseLevel> {
5174 let slow = Self::side_is_slow_control_link_locked(st, side_id, now_ms);
5175 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
5176 if !throttle.has_sent_full || now_ms >= throttle.next_full_ms {
5177 throttle.has_sent_full = true;
5178 throttle.pending_incremental = false;
5179 throttle.incremental_retries_remaining = 0;
5180 throttle.next_full_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS);
5181 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
5182 return Some(DiscoveryAdvertiseLevel::Full);
5183 }
5184 if throttle.pending_incremental {
5185 throttle.incremental_retries_remaining =
5186 throttle.incremental_retries_remaining.saturating_sub(1);
5187 throttle.pending_incremental = throttle.incremental_retries_remaining > 0;
5188 return Some(DiscoveryAdvertiseLevel::Incremental);
5189 }
5190 if slow && now_ms >= throttle.next_ping_ms {
5191 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
5192 return Some(DiscoveryAdvertiseLevel::MinimalPing);
5193 }
5194 if slow {
5195 None
5196 } else {
5197 Some(DiscoveryAdvertiseLevel::Incremental)
5200 }
5201 }
5202
5203 #[cfg(all(feature = "discovery", feature = "timesync"))]
5204 #[inline]
5205 fn is_timesync_type(ty: DataType) -> bool {
5206 matches!(
5207 ty,
5208 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
5209 )
5210 }
5211
5212 #[cfg(all(feature = "discovery", feature = "timesync"))]
5213 fn timesync_allowed_for_side_locked(
5214 st: &mut RouterInner,
5215 side_id: RouterSideId,
5216 ty: DataType,
5217 now_ms: u64,
5218 ) -> bool {
5219 if !Self::is_timesync_type(ty) {
5220 return true;
5221 }
5222 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
5223 st.timesync_side_throttle.remove(&side_id);
5224 return true;
5225 }
5226
5227 let throttle = st.timesync_side_throttle.entry(side_id).or_default();
5228 if now_ms >= throttle.next_allowed_ms {
5229 throttle.next_allowed_ms = now_ms.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
5230 return true;
5231 }
5232 false
5233 }
5234
5235 #[cfg(all(feature = "discovery", feature = "timesync"))]
5236 fn filter_timesync_sides_locked(
5237 st: &mut RouterInner,
5238 ty: DataType,
5239 now_ms: u64,
5240 sides: Vec<RouterSideId>,
5241 ) -> Vec<RouterSideId> {
5242 sides
5243 .into_iter()
5244 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
5245 .collect()
5246 }
5247
5248 #[cfg(all(feature = "discovery", feature = "timesync"))]
5249 fn filter_timesync_matches_locked(
5250 st: &mut RouterInner,
5251 ty: DataType,
5252 now_ms: u64,
5253 matches: Vec<DiscoveryCandidateMatch>,
5254 ) -> Vec<DiscoveryCandidateMatch> {
5255 matches
5256 .into_iter()
5257 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
5258 .collect()
5259 }
5260
5261 #[cfg(feature = "discovery")]
5262 fn emit_discovery_snapshot(
5263 &self,
5264 called_from_queue: bool,
5265 include_schema: bool,
5266 include_topology: bool,
5267 skip_in_flight_sides: bool,
5268 ) -> TelemetryResult<()> {
5269 let now_ms = self.clock.now_ms();
5270 let per_side = {
5271 let mut st = self.state.lock();
5272 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
5273 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5274 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5275 }
5276 st.fit_discovery_budget();
5277 let side_entries = st
5278 .sides
5279 .iter()
5280 .enumerate()
5281 .filter_map(|(side_id, side)| {
5282 side.as_ref()
5283 .map(|side| (side_id, side.opts.link_local_enabled, side.opts))
5284 })
5285 .collect::<Vec<_>>();
5286 let local_is_master =
5287 self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
5288 let mut per_side = Vec::new();
5289 for (side_id, link_local_enabled, opts) in side_entries {
5290 if skip_in_flight_sides
5291 && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
5292 *pending_side == side_id
5293 && !tx_state.sent.is_empty()
5294 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
5295 })
5296 {
5297 continue;
5298 }
5299 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
5300 {
5301 continue;
5302 }
5303 let Some(level) = Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
5304 else {
5305 continue;
5306 };
5307 let include_side_topology = include_topology
5308 && self.route_allowed_locked(
5309 &st,
5310 None,
5311 Some(DataType::DiscoveryTopology),
5312 side_id,
5313 );
5314 let capabilities = opts.link_capabilities();
5315 if level == DiscoveryAdvertiseLevel::MinimalPing {
5316 per_side.push((
5317 side_id,
5318 level,
5319 Vec::new(),
5320 Vec::new(),
5321 Vec::new(),
5322 Vec::new(),
5323 Vec::new(),
5324 capabilities,
5325 local_is_master,
5326 ));
5327 continue;
5328 }
5329 let current_topology = if include_side_topology {
5330 self.advertised_discovery_topology_for_link_locked(
5331 &st,
5332 now_ms,
5333 link_local_enabled,
5334 Some(side_id),
5335 )
5336 } else {
5337 Vec::new()
5338 };
5339 let (topology, removed_topology) = match level {
5340 DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
5341 DiscoveryAdvertiseLevel::Incremental => {
5342 let previous = &st
5343 .discovery_side_throttle
5344 .get(&side_id)
5345 .expect("discovery throttle exists")
5346 .last_topology;
5347 discovery::topology_delta(previous, ¤t_topology)
5348 }
5349 DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
5350 };
5351 let throttle = st
5352 .discovery_side_throttle
5353 .get_mut(&side_id)
5354 .expect("discovery throttle exists");
5355 if level == DiscoveryAdvertiseLevel::Full
5356 || (level == DiscoveryAdvertiseLevel::Incremental
5357 && throttle.incremental_retries_remaining == 0)
5358 {
5359 throttle.last_topology = current_topology;
5360 }
5361 per_side.push((
5362 side_id,
5363 level,
5364 self.advertised_discovery_endpoints_for_link_locked(
5365 &st,
5366 now_ms,
5367 link_local_enabled,
5368 Some(side_id),
5369 ),
5370 self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
5371 self.advertised_discovery_timesync_sources_for_link_locked(
5372 &st,
5373 now_ms,
5374 Some(side_id),
5375 ),
5376 topology,
5377 removed_topology,
5378 capabilities,
5379 local_is_master,
5380 ));
5381 }
5382 per_side
5383 };
5384 for (
5385 side_id,
5386 level,
5387 endpoints,
5388 network_variables,
5389 timesync_sources,
5390 topology,
5391 removed_topology,
5392 capabilities,
5393 local_is_master,
5394 ) in per_side
5395 {
5396 let sender = self.sender_arc();
5397 if matches!(
5398 level,
5399 DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
5400 ) {
5401 let address = self.local_address_advertisement(
5402 endpoints.clone(),
5403 network_variables.clone(),
5404 timesync_sources.clone(),
5405 capabilities,
5406 if local_is_master {
5407 discovery::ADDRESS_STATE_APPROVED
5408 } else {
5409 discovery::ADDRESS_STATE_REQUEST
5410 },
5411 );
5412 let pkt = discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?;
5413 self.emit_internal_tx(
5414 RouterTxItem::ToSide {
5415 src: None,
5416 dst: side_id,
5417 data: RouterItem::Packet(pkt),
5418 },
5419 true,
5420 called_from_queue,
5421 )?;
5422 }
5423 if level == DiscoveryAdvertiseLevel::MinimalPing {
5424 let pkt = discovery::build_discovery_announce(
5425 sender.as_ref(),
5426 now_ms,
5427 endpoints.as_slice(),
5428 )?;
5429 self.emit_internal_tx(
5430 RouterTxItem::ToSide {
5431 src: None,
5432 dst: side_id,
5433 data: RouterItem::Packet(pkt),
5434 },
5435 true,
5436 called_from_queue,
5437 )?;
5438 }
5439 if include_topology
5440 && (!topology.is_empty() || !removed_topology.is_empty())
5441 && level != DiscoveryAdvertiseLevel::MinimalPing
5442 {
5443 let pkt = if level == DiscoveryAdvertiseLevel::Full {
5444 discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
5445 } else {
5446 discovery::build_discovery_topology_delta(
5447 sender.as_ref(),
5448 now_ms,
5449 &topology,
5450 &removed_topology,
5451 )?
5452 };
5453 self.emit_internal_tx(
5454 RouterTxItem::ToSide {
5455 src: None,
5456 dst: side_id,
5457 data: RouterItem::Packet(pkt),
5458 },
5459 true,
5460 called_from_queue,
5461 )?;
5462 }
5463 if include_schema && level == DiscoveryAdvertiseLevel::Full {
5468 let pkt = discovery::build_discovery_schema(sender.as_ref(), now_ms)?;
5469 self.emit_internal_tx(
5470 RouterTxItem::ToSide {
5471 src: None,
5472 dst: side_id,
5473 data: RouterItem::Packet(pkt),
5474 },
5475 true,
5476 called_from_queue,
5477 )?;
5478 }
5479 }
5480 Ok(())
5481 }
5482
5483 #[cfg(feature = "discovery")]
5484 fn queue_discovery_announce(
5485 &self,
5486 include_schema: bool,
5487 skip_in_flight_sides: bool,
5488 ) -> TelemetryResult<()> {
5489 let now_ms = self.clock.now_ms();
5490 {
5491 let mut st = self.state.lock();
5492 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
5493 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5494 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5495 }
5496 st.fit_discovery_budget();
5497 if st.sides.iter().all(|side| side.is_none()) {
5498 return Ok(());
5499 }
5500 if include_schema {
5503 for throttle in st.discovery_side_throttle.values_mut() {
5504 throttle.has_sent_full = false;
5505 }
5506 }
5507 st.discovery_cadence.on_announce_sent(now_ms);
5508 }
5509 self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
5510 }
5511
5512 #[cfg(feature = "discovery")]
5513 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
5514 let now_ms = self.clock.now_ms();
5515 let due = {
5516 let mut st = self.state.lock();
5517 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
5518 if removed {
5519 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5520 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5521 }
5522 st.fit_discovery_budget();
5523 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
5524 let Some(side) = side.as_ref() else {
5525 return false;
5526 };
5527 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
5528 {
5529 return false;
5530 }
5531 let _ = side;
5532 !st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
5533 *pending_side == side_id
5534 && !tx_state.sent.is_empty()
5535 && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
5536 })
5537 });
5538 if st.sides.is_empty() || !has_any {
5539 return Ok(false);
5540 }
5541 st.discovery_cadence.due(now_ms)
5542 };
5543 if !due {
5544 return Ok(false);
5545 }
5546 self.queue_discovery_announce(false, true)?;
5554 Ok(true)
5555 }
5556
5557 #[cfg(feature = "discovery")]
5558 fn learn_discovery_packet(
5559 &self,
5560 pkt: &Packet,
5561 src: Option<RouterSideId>,
5562 called_from_queue: bool,
5563 ) -> TelemetryResult<bool> {
5564 if !discovery::is_discovery_type(pkt.data_type()) {
5565 return Ok(false);
5566 }
5567 let Some(side) = src else {
5568 return Ok(true);
5569 };
5570 if pkt.data_type() == DataType::DiscoveryAddress {
5571 let mut ad = discovery::decode_discovery_address(pkt)?;
5572 let mut changed = self.ingest_address_advertisement(ad.clone())?;
5573 let sender_id = if ad.hostname.is_empty() {
5579 pkt.sender()
5580 } else {
5581 ad.hostname.as_str()
5582 };
5583 let now_ms = self.clock.now_ms();
5584 let mut st = self.state.lock();
5585 let side_link_local_enabled = st
5586 .sides
5587 .get(side)
5588 .and_then(|entry| entry.as_ref())
5589 .map(|side_ref| side_ref.opts.link_local_enabled)
5590 .unwrap_or(false);
5591 if !side_link_local_enabled {
5592 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
5593 }
5594 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
5595 let previously_reachable_network_variables = route.reachable_network_variables.clone();
5596 if pkt.sender() != sender_id {
5597 route.announcers.remove(pkt.sender());
5598 }
5599 let mut sender_state = route.announcers.get(sender_id).cloned().unwrap_or_default();
5600 if sender_state.advertised_address != Some(ad.address) {
5606 sender_state.advertised_address = Some(ad.address);
5607 changed = true;
5608 }
5609 if sender_state.advertised_reachable != ad.reachable_endpoints {
5610 sender_state.advertised_reachable = ad.reachable_endpoints;
5611 changed = true;
5612 }
5613 if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
5614 sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
5615 changed = true;
5616 }
5617 if sender_state.reachable_network_variables != ad.reachable_network_variables {
5618 sender_state.reachable_network_variables = ad.reachable_network_variables;
5619 changed = true;
5620 }
5621 if sender_state.link_capabilities != Some(ad.link_capabilities) {
5622 sender_state.link_capabilities = Some(ad.link_capabilities);
5623 changed = true;
5624 }
5625 Self::refresh_sender_topology_state(&mut sender_state);
5626 sender_state.last_seen_ms = now_ms;
5627 route.announcers.insert(sender_id.to_string(), sender_state);
5628 Self::recompute_discovery_side_state(&mut route);
5629 let newly_reachable_network_variables: Vec<DataType> = route
5630 .reachable_network_variables
5631 .iter()
5632 .copied()
5633 .filter(|ty| !previously_reachable_network_variables.contains(ty))
5634 .collect();
5635 st.discovery_routes.insert(side, route);
5636 st.fit_discovery_budget();
5637 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5638 if changed {
5639 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
5640 }
5641 let replay: Vec<Packet> = newly_reachable_network_variables
5647 .iter()
5648 .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
5649 .filter_map(|ty| {
5650 st.managed_variable_latest
5651 .get(&ty.as_u32())
5652 .map(|entry| entry.packet.clone())
5653 })
5654 .collect();
5655 drop(st);
5656 for value in replay {
5657 self.emit_internal_tx_with_priority(
5658 RouterTxItem::ToSide {
5659 src: None,
5660 dst: side,
5661 data: RouterItem::Packet(value),
5662 },
5663 true,
5664 crate::transport_priority(DataType::ManagedVariableValue),
5665 called_from_queue,
5666 )?;
5667 }
5668 return Ok(true);
5669 }
5670 let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
5671 Some(discovery::decode_discovery_topology_update(pkt)?)
5672 } else {
5673 None
5674 };
5675 let packet_sender = {
5676 let st = self.state.lock();
5677 let canonical = Self::canonical_sender_locked(&st, pkt.sender());
5678 if canonical != pkt.sender() {
5679 canonical
5680 } else if let Some(address) = pkt
5681 .sender()
5682 .strip_prefix("@addr:")
5683 .and_then(|value| value.parse::<u32>().ok())
5684 {
5685 decoded_topology
5691 .as_ref()
5692 .and_then(|update| {
5693 update
5694 .boards
5695 .iter()
5696 .find(|board| sender_address_u32(&board.sender_id) == address)
5697 })
5698 .map(|board| board.sender_id.clone())
5699 .unwrap_or(canonical)
5700 } else {
5701 canonical
5702 }
5703 };
5704 let local_sender = self.sender_arc();
5705 if packet_sender == local_sender.as_ref() {
5706 return Ok(true);
5707 }
5708 if pkt.data_type() == DataType::DiscoveryTopologyRequest {
5709 let now_ms = self.clock.now_ms();
5710 let should_answer = {
5711 let mut st = self.state.lock();
5712 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
5713 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5714 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5715 }
5716 st.discovery_side_throttle
5717 .entry(side)
5718 .or_default()
5719 .has_sent_full = false;
5720 self.should_answer_discovery_request_locked(&st, &packet_sender, now_ms)
5721 };
5722 if should_answer {
5723 self.emit_discovery_snapshot(called_from_queue, false, true, false)?;
5724 }
5725 return Ok(true);
5726 }
5727 if pkt.data_type() == DataType::DiscoverySchemaRequest {
5728 let now_ms = self.clock.now_ms();
5729 let should_answer = {
5730 let mut st = self.state.lock();
5731 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
5732 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5733 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5734 }
5735 st.discovery_side_throttle
5736 .entry(side)
5737 .or_default()
5738 .has_sent_full = false;
5739 self.should_answer_discovery_request_locked(&st, &packet_sender, now_ms)
5740 };
5741 if should_answer {
5742 self.emit_discovery_snapshot(called_from_queue, true, true, false)?;
5743 }
5744 return Ok(true);
5745 }
5746 if pkt.data_type() == DataType::ManagedVariableRequest {
5747 let ty = discovery::decode_managed_variable_request(pkt)?;
5748 {
5754 let now_ms = self.clock.now_ms();
5755 let mut st = self.state.lock();
5756 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
5757 let newly_reachable = if let Some((_, last_seen_ms)) = route
5758 .requested_network_variables
5759 .iter_mut()
5760 .find(|(requested, _)| *requested == ty)
5761 {
5762 *last_seen_ms = now_ms;
5763 false
5764 } else {
5765 route.requested_network_variables.push((ty, now_ms));
5766 true
5767 };
5768 Self::recompute_discovery_side_state(&mut route);
5769 st.discovery_routes.insert(side, route);
5770 st.fit_discovery_budget();
5771 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5772 if newly_reachable {
5773 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5774 }
5775 }
5776 if !self.can_write_managed_variable(ty) {
5777 return Ok(false);
5782 }
5783 if let Some(value) = self.managed_variable_latest(ty) {
5784 self.emit_internal_tx(
5785 RouterTxItem::ToSide {
5786 src: None,
5787 dst: side,
5788 data: RouterItem::Packet(value),
5789 },
5790 true,
5791 called_from_queue,
5792 )?;
5793 return Ok(true);
5794 }
5795 return Ok(false);
5798 }
5799 if pkt.data_type() == DataType::DiscoverySchema {
5800 let snapshot = discovery::decode_discovery_schema(pkt)?;
5801 let budget = self.state.lock().memory.max_queue_budget;
5810 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
5811 if report.changed() {
5812 let mut st = self.state.lock();
5813 st.make_shared_queue_room(0, crate::transport_priority(DataType::DiscoverySchema))?;
5814 st.fit_discovery_budget();
5815 }
5816 return Ok(true);
5817 }
5818 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
5819 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
5820 return Ok(true);
5821 }
5822 let mut st = self.state.lock();
5823 let now_ms = self.clock.now_ms();
5824 if pkt.data_type() == DataType::DiscoveryLeave {
5825 let leaving = packet_sender.as_str();
5826 let before = st.discovery_routes.clone();
5827 for route in st.discovery_routes.values_mut() {
5828 route.announcers.remove(leaving);
5829 for sender_state in route.announcers.values_mut() {
5830 sender_state
5831 .topology_boards
5832 .retain(|board| board.sender_id != leaving);
5833 for board in sender_state.topology_boards.iter_mut() {
5834 board.connections.retain(|peer| peer != leaving);
5835 }
5836 Self::refresh_sender_topology_state(sender_state);
5837 }
5838 Self::recompute_discovery_side_state(route);
5839 }
5840 st.discovery_routes
5841 .retain(|_, route| !route.announcers.is_empty());
5842 if st.discovery_routes != before {
5843 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5844 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5845 }
5846 return Ok(true);
5847 }
5848 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
5849 let side_link_local_enabled = st
5850 .sides
5851 .get(side)
5852 .and_then(|entry| entry.as_ref())
5853 .map(|side_ref| side_ref.opts.link_local_enabled)
5854 .unwrap_or(false);
5855 let mut sender_state = route
5856 .announcers
5857 .get(&packet_sender)
5858 .cloned()
5859 .unwrap_or_default();
5860 let changed = match pkt.data_type() {
5861 DataType::DiscoveryAnnounce => {
5862 let mut reachable = discovery::decode_discovery_announce(pkt)?;
5863 if !side_link_local_enabled {
5864 reachable.retain(|ep| !ep.is_link_local_only());
5865 }
5866 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
5867 let changed = board.reachable_endpoints != reachable;
5868 board.reachable_endpoints = reachable;
5869 Self::refresh_sender_topology_state(&mut sender_state);
5870 changed
5871 }
5872 DataType::DiscoveryTimeSyncSources => {
5873 let sources = discovery::decode_discovery_timesync_sources(pkt)?;
5874 let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
5875 let changed = board.reachable_timesync_sources != sources;
5876 board.reachable_timesync_sources = sources;
5877 Self::refresh_sender_topology_state(&mut sender_state);
5878 changed
5879 }
5880 DataType::DiscoveryTopology => {
5881 let mut update = decoded_topology
5882 .take()
5883 .expect("topology packet was decoded before route selection");
5884 for board in update.boards.iter_mut() {
5885 board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
5886 for peer in board.connections.iter_mut() {
5887 *peer = Self::canonical_sender_locked(&st, peer);
5888 }
5889 }
5890 if !side_link_local_enabled {
5891 for board in update.boards.iter_mut() {
5892 board
5893 .reachable_endpoints
5894 .retain(|ep| !ep.is_link_local_only());
5895 }
5896 }
5897 let before = sender_state.topology_boards.clone();
5898 if update.incremental {
5899 sender_state.topology_boards.retain(|board| {
5900 !update
5901 .removed
5902 .iter()
5903 .any(|removed| removed == &board.sender_id)
5904 && !update
5905 .boards
5906 .iter()
5907 .any(|replacement| replacement.sender_id == board.sender_id)
5908 });
5909 discovery::merge_topology_boards(
5910 &mut sender_state.topology_boards,
5911 &update.boards,
5912 );
5913 } else {
5914 sender_state.topology_boards = update.boards;
5915 }
5916 let changed = sender_state.topology_boards != before;
5917 Self::refresh_sender_topology_state(&mut sender_state);
5918 changed
5919 }
5920 DataType::DiscoverySchema => false,
5921 _ => false,
5922 };
5923 sender_state.last_seen_ms = now_ms;
5924 route.announcers.insert(packet_sender, sender_state);
5925 Self::recompute_discovery_side_state(&mut route);
5926 st.discovery_routes.insert(side, route);
5927 st.fit_discovery_budget();
5928 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5929 if changed {
5930 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5931 }
5932 Ok(true)
5933 }
5934
5935 #[cfg(not(feature = "discovery"))]
5936 fn queue_discovery_announce(&self) -> TelemetryResult<()> {
5937 Ok(())
5938 }
5939
5940 #[cfg(not(feature = "discovery"))]
5941 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
5942 Ok(false)
5943 }
5944
5945 #[cfg(not(feature = "discovery"))]
5946 fn learn_discovery_packet(
5947 &self,
5948 _pkt: &Packet,
5949 _src: Option<RouterSideId>,
5950 _called_from_queue: bool,
5951 ) -> TelemetryResult<bool> {
5952 Ok(false)
5953 }
5954
5955 #[inline]
5956 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
5957 (side, ty.as_u32())
5958 }
5959
5960 fn reliable_tx_state_mut<'a>(
5961 &'a self,
5962 st: &'a mut RouterInner,
5963 side: RouterSideId,
5964 ty: DataType,
5965 ) -> &'a mut ReliableTxState {
5966 let key = Self::reliable_key(side, ty);
5967 st.reliable_tx
5968 .entry(key)
5969 .or_insert_with(|| ReliableTxState {
5970 next_seq: 1,
5971 sent_order: VecDeque::new(),
5972 sent: BTreeMap::new(),
5973 })
5974 }
5975
5976 fn reliable_rx_state_mut<'a>(
5977 &'a self,
5978 st: &'a mut RouterInner,
5979 side: RouterSideId,
5980 ty: DataType,
5981 ) -> &'a mut ReliableRxState {
5982 let key = Self::reliable_key(side, ty);
5983 st.reliable_rx
5984 .entry(key)
5985 .or_insert_with(|| ReliableRxState {
5986 expected_seq: 1,
5987 buffered: BTreeMap::new(),
5988 })
5989 }
5990
5991 fn reliable_control_packet(
5992 &self,
5993 control_ty: DataType,
5994 ty: DataType,
5995 seq: u32,
5996 ) -> TelemetryResult<Packet> {
5997 let sender = self.sender_arc();
5998 Packet::new(
5999 control_ty,
6000 message_meta(control_ty).endpoints_ref(),
6001 sender.as_ref(),
6002 self.packet_timestamp_ms(),
6003 encode_slice_le(&[ty.as_u32(), seq]),
6004 )
6005 }
6006
6007 fn queue_reliable_ack(
6008 &self,
6009 side: RouterSideId,
6010 ty: DataType,
6011 seq: u32,
6012 called_from_queue: bool,
6013 ) -> TelemetryResult<()> {
6014 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
6015 self.emit_internal_tx_with_priority(
6016 RouterTxItem::ToSide {
6017 src: None,
6018 dst: side,
6019 data: RouterItem::Packet(pkt),
6020 },
6021 true,
6022 message_priority(DataType::ReliableAck),
6023 called_from_queue,
6024 )
6025 }
6026
6027 fn queue_reliable_packet_request(
6028 &self,
6029 side: RouterSideId,
6030 ty: DataType,
6031 seq: u32,
6032 called_from_queue: bool,
6033 ) -> TelemetryResult<()> {
6034 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
6035 self.emit_internal_tx_with_priority(
6036 RouterTxItem::ToSide {
6037 src: None,
6038 dst: side,
6039 data: RouterItem::Packet(pkt),
6040 },
6041 true,
6042 message_priority(DataType::ReliablePacketRequest),
6043 called_from_queue,
6044 )
6045 }
6046
6047 fn queue_reliable_partial_ack(
6048 &self,
6049 side: RouterSideId,
6050 ty: DataType,
6051 seq: u32,
6052 called_from_queue: bool,
6053 ) -> TelemetryResult<()> {
6054 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
6055 self.emit_internal_tx_with_priority(
6056 RouterTxItem::ToSide {
6057 src: None,
6058 dst: side,
6059 data: RouterItem::Packet(pkt),
6060 },
6061 true,
6062 message_priority(DataType::ReliablePartialAck),
6063 called_from_queue,
6064 )
6065 }
6066
6067 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
6068 let mut st = self.state.lock();
6069 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
6070 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
6071 tx_state.sent.remove(&ack);
6072 tx_state.sent_order.retain(|seq| *seq != ack);
6073 return;
6074 }
6075
6076 while let Some(seq) = tx_state.sent_order.front().copied() {
6077 if seq > ack {
6078 break;
6079 }
6080 tx_state.sent_order.pop_front();
6081 tx_state.sent.remove(&seq);
6082 }
6083 }
6084
6085 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
6086 let mut st = self.state.lock();
6087 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
6088 if let Some(sent) = tx_state.sent.get_mut(&seq) {
6089 sent.partial_acked = true;
6090 }
6091 }
6092
6093 fn queue_reliable_retransmit(
6094 &self,
6095 side: RouterSideId,
6096 ty: DataType,
6097 seq: u32,
6098 called_from_queue: bool,
6099 ) -> TelemetryResult<()> {
6100 let mut queued = None;
6101 {
6102 let mut st = self.state.lock();
6103 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
6104 if let Some(sent) = tx_state.sent.get_mut(&seq)
6105 && !sent.queued
6106 {
6107 sent.queued = true;
6108 sent.partial_acked = false;
6109 queued = Some(sent.bytes.clone());
6110 }
6111 }
6112
6113 if let Some(bytes) = queued {
6114 if called_from_queue {
6115 self.tx_queue_item_with_priority(
6116 RouterTxItem::ReliableReplay { dst: side, bytes },
6117 true,
6118 Self::router_item_priority_bumped(ty),
6119 )?;
6120 } else {
6121 self.tx_item_impl(
6122 RouterTxItem::ReliableReplay { dst: side, bytes },
6123 true,
6124 false,
6125 )?;
6126 }
6127 }
6128
6129 Ok(())
6130 }
6131
6132 fn send_reliable_raw_to_side(
6133 &self,
6134 side: RouterSideId,
6135 bytes: Arc<[u8]>,
6136 relayed: bool,
6137 ) -> TelemetryResult<()> {
6138 let handler = {
6139 let st = self.state.lock();
6140 let side_ref = Self::side_ref(&st, side)?;
6141 if !side_ref.opts.egress_enabled {
6142 return Ok(());
6143 }
6144 (side_ref.tx_handler.clone(), side_ref.opts)
6145 };
6146
6147 let (handler, opts) = handler;
6148
6149 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
6150 return Err(TelemetryError::Io("side tx busy"));
6151 };
6152 let started_ms = self.clock.now_ms();
6153 let ty = wire_format::peek_envelope(bytes.as_ref())
6154 .map(|env| env.ty)
6155 .unwrap_or(DataType::ReliableAck);
6156 let priority = self.effective_transport_priority(ty);
6157 let result = match handler {
6158 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
6159 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
6160 let mut attempts_total = 0usize;
6161 let mut sent_bytes = 0usize;
6162 for frame in frames {
6163 match self.retry_with_attempts(runtime_max_handler_retries(), || {
6164 Self::packed_handler_call(&packed, frame.as_ref(), priority)
6165 }) {
6166 Ok((_, attempts)) => {
6167 attempts_total = attempts_total.saturating_add(attempts);
6168 sent_bytes = sent_bytes.saturating_add(frame.len());
6169 }
6170 Err((err, attempts)) => {
6171 self.note_side_tx_failure(
6172 side,
6173 ty,
6174 attempts_total.saturating_add(attempts),
6175 );
6176 return Err(err);
6177 }
6178 }
6179 }
6180 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
6181 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6182 return Ok(());
6183 }
6184 RouterTxHandlerFn::Packet(f) => {
6185 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
6186 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
6187 }
6188 };
6189 match result {
6190 Ok((_, attempts)) => {
6191 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
6192 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
6193 Ok(())
6194 }
6195 Err((err, attempts)) => {
6196 self.note_side_tx_failure(side, ty, attempts);
6197 Err(err)
6198 }
6199 }
6200 }
6201
6202 fn send_reliable_to_side(
6203 &self,
6204 side: RouterSideId,
6205 data: RouterItem,
6206 relayed: bool,
6207 ) -> TelemetryResult<()> {
6208 let (handler, opts, hop_reliable_enabled) = {
6209 let st = self.state.lock();
6210 let side_ref = Self::side_ref(&st, side)?;
6211 let opts = side_ref.opts;
6212 let hop_reliable_enabled = opts.reliable_enabled
6213 && self.cfg.reliable_enabled()
6214 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
6215 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
6216 };
6217
6218 if !matches!(
6219 &handler,
6220 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
6221 ) {
6222 return self.call_side_tx_handler(side, &handler, &data, relayed);
6223 }
6224
6225 if !hop_reliable_enabled {
6226 let mut adjusted_opts = opts;
6227 adjusted_opts.reliable_enabled = false;
6228 let preserve_end_to_end_ack = true;
6233 if let Some(adjusted) =
6234 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
6235 {
6236 return self.call_side_tx_handler(side, &handler, &adjusted, relayed);
6237 }
6238 return Ok(());
6239 }
6240
6241 let ty = match &data {
6242 RouterItem::Packet(pkt) => pkt.data_type(),
6243 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.envelope.ty,
6244 };
6245 let priority = self.effective_transport_priority(ty);
6246
6247 if !is_reliable_type(ty) {
6248 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
6249 self.call_side_tx_handler(side, &handler, &adjusted, relayed)?;
6250 }
6251 return Ok(());
6252 }
6253
6254 let (seq, flags) = {
6255 let mut st = self.state.lock();
6256 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
6257 if tx_state.sent.len() >= runtime_reliable_max_pending() {
6258 return Err(TelemetryError::PacketTooLarge(
6259 "router reliable history full",
6260 ));
6261 }
6262 let seq = tx_state.next_seq;
6263 let next = tx_state.next_seq.wrapping_add(1);
6264 tx_state.next_seq = if next == 0 { 1 } else { next };
6265 let flags = match reliable_mode(ty) {
6266 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
6267 _ => 0,
6268 };
6269 (seq, flags)
6270 };
6271
6272 let bytes: Arc<[u8]> = match data {
6273 RouterItem::Packet(pkt) => self.pack_packet_for_router(
6274 &pkt,
6275 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
6276 )?,
6277 RouterItem::Packed(bytes) => {
6278 #[cfg(feature = "cryptography")]
6279 if self.e2e_seal_config_for_type(ty).is_some() {
6280 self.prepare_packed_for_remote(
6281 bytes,
6282 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
6283 )?
6284 } else {
6285 let Some(rewritten) =
6286 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
6287 else {
6288 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
6289 return Err(TelemetryError::Io("side tx busy"));
6290 };
6291 let started_ms = self.clock.now_ms();
6292 let frames =
6293 self.encode_side_transport_frames(side, opts, bytes.clone())?;
6294 let mut attempts_total = 0usize;
6295 let mut sent_bytes = 0usize;
6296 for frame in frames {
6297 match self.retry_with_attempts(runtime_max_handler_retries(), || {
6298 Self::packed_handler_call(&handler, frame.as_ref(), priority)
6299 }) {
6300 Ok((_, attempts)) => {
6301 attempts_total = attempts_total.saturating_add(attempts);
6302 sent_bytes = sent_bytes.saturating_add(frame.len());
6303 }
6304 Err((err, attempts)) => {
6305 self.note_side_tx_failure(
6306 side,
6307 ty,
6308 attempts_total.saturating_add(attempts),
6309 );
6310 return Err(err);
6311 }
6312 }
6313 }
6314 self.record_side_tx_sample(
6315 side,
6316 sent_bytes,
6317 started_ms,
6318 self.clock.now_ms(),
6319 );
6320 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6321 return Ok(());
6322 };
6323 rewritten
6324 }
6325 #[cfg(not(feature = "cryptography"))]
6326 {
6327 let Some(rewritten) =
6328 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
6329 else {
6330 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
6331 return Err(TelemetryError::Io("side tx busy"));
6332 };
6333 let started_ms = self.clock.now_ms();
6334 let frames =
6335 self.encode_side_transport_frames(side, opts, bytes.clone())?;
6336 let mut attempts_total = 0usize;
6337 let mut sent_bytes = 0usize;
6338 for frame in frames {
6339 match self.retry_with_attempts(runtime_max_handler_retries(), || {
6340 Self::packed_handler_call(&handler, frame.as_ref(), priority)
6341 }) {
6342 Ok((_, attempts)) => {
6343 attempts_total = attempts_total.saturating_add(attempts);
6344 sent_bytes = sent_bytes.saturating_add(frame.len());
6345 }
6346 Err((err, attempts)) => {
6347 self.note_side_tx_failure(
6348 side,
6349 ty,
6350 attempts_total.saturating_add(attempts),
6351 );
6352 return Err(err);
6353 }
6354 }
6355 }
6356 self.record_side_tx_sample(
6357 side,
6358 sent_bytes,
6359 started_ms,
6360 self.clock.now_ms(),
6361 );
6362 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6363 return Ok(());
6364 };
6365 rewritten
6366 }
6367 }
6368 };
6369
6370 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
6371 return Err(TelemetryError::Io("side tx busy"));
6372 };
6373 let started_ms = self.clock.now_ms();
6374 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
6375 let mut attempts_total = 0usize;
6376 let mut sent_bytes = 0usize;
6377 for frame in frames {
6378 match self.retry_with_attempts(runtime_max_handler_retries(), || {
6379 Self::packed_handler_call(&handler, frame.as_ref(), priority)
6380 }) {
6381 Ok((_, attempts)) => {
6382 attempts_total = attempts_total.saturating_add(attempts);
6383 sent_bytes = sent_bytes.saturating_add(frame.len());
6384 }
6385 Err((err, attempts)) => {
6386 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
6387 return Err(err);
6388 }
6389 }
6390 }
6391 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
6392 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6393
6394 {
6395 let mut st = self.state.lock();
6396 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
6397 tx_state.sent_order.push_back(seq);
6398 tx_state.sent.insert(
6399 seq,
6400 ReliableSent {
6401 bytes: bytes.clone(),
6402 last_send_ms: self.clock.now_ms(),
6403 retries: 0,
6404 queued: false,
6405 partial_acked: false,
6406 },
6407 );
6408 }
6409
6410 Ok(())
6411 }
6412
6413 #[inline]
6414 fn crc32_bytes(data: &[u8]) -> u32 {
6415 let mut hasher = Crc32Hasher::new();
6416 hasher.update(data);
6417 hasher.finalize()
6418 }
6419
6420 fn read_uleb128_local(buf: &[u8], off: &mut usize) -> TelemetryResult<u64> {
6421 let mut result = 0u64;
6422 let mut shift = 0u32;
6423 for _ in 0..10 {
6424 let byte = *buf.get(*off).ok_or(TelemetryError::Unpack("short read"))?;
6425 *off += 1;
6426 result |= u64::from(byte & 0x7F) << shift;
6427 if (byte & 0x80) == 0 {
6428 return Ok(result);
6429 }
6430 shift += 7;
6431 }
6432 Err(TelemetryError::Unpack("uleb128 too long"))
6433 }
6434
6435 fn write_uleb128_local(mut value: u64, out: &mut Vec<u8>) {
6436 loop {
6437 let mut byte = (value & 0x7F) as u8;
6438 value >>= 7;
6439 if value != 0 {
6440 byte |= 0x80;
6441 }
6442 out.push(byte);
6443 if value == 0 {
6444 break;
6445 }
6446 }
6447 }
6448
6449 fn uleb128_len_local(mut value: u64) -> usize {
6450 let mut len = 1;
6451 while value >= 0x80 {
6452 value >>= 7;
6453 len += 1;
6454 }
6455 len
6456 }
6457
6458 fn wrap_side_transport_frame(kind: u8, body: &[u8]) -> Arc<[u8]> {
6459 let mut out = Vec::with_capacity(
6460 SIDE_TRANSPORT_MAGIC.len() + 1 + body.len() + wire_format::CRC32_BYTES,
6461 );
6462 out.extend_from_slice(SIDE_TRANSPORT_MAGIC);
6463 out.push(kind);
6464 out.extend_from_slice(body);
6465 let crc = Self::crc32_bytes(&out);
6466 out.extend_from_slice(&crc.to_le_bytes());
6467 Arc::from(out)
6468 }
6469
6470 fn parse_side_transport_wrapper(bytes: &[u8]) -> TelemetryResult<Option<(u8, &[u8])>> {
6471 if bytes.len() < SIDE_TRANSPORT_MAGIC.len() + 1 + wire_format::CRC32_BYTES {
6472 return Ok(None);
6473 }
6474 if &bytes[..SIDE_TRANSPORT_MAGIC.len()] != SIDE_TRANSPORT_MAGIC {
6475 return Ok(None);
6476 }
6477 let data_len = bytes.len() - wire_format::CRC32_BYTES;
6478 let expected = u32::from_le_bytes([
6479 bytes[data_len],
6480 bytes[data_len + 1],
6481 bytes[data_len + 2],
6482 bytes[data_len + 3],
6483 ]);
6484 let data = &bytes[..data_len];
6485 if Self::crc32_bytes(data) != expected {
6486 return Err(TelemetryError::Unpack("side transport crc32 mismatch"));
6487 }
6488 let kind = data[SIDE_TRANSPORT_MAGIC.len()];
6489 Ok(Some((kind, &data[SIDE_TRANSPORT_MAGIC.len() + 1..])))
6490 }
6491
6492 fn extract_side_header_template(bytes: &[u8]) -> TelemetryResult<SideTemplateExtract<'_>> {
6493 if bytes.len() < wire_format::CRC32_BYTES + 4 {
6494 return Err(TelemetryError::Unpack("short buffer"));
6495 }
6496 let data_len = bytes.len() - wire_format::CRC32_BYTES;
6497 let data = &bytes[..data_len];
6498 let mut off = 0usize;
6499 let flags = *data
6500 .get(off)
6501 .ok_or(TelemetryError::Unpack("short prelude"))?;
6502 off += 1;
6503 off += 1; let ty_end_start = off;
6505 let ty_u64 = Self::read_uleb128_local(data, &mut off)?;
6506 let ty_u32 = u32::try_from(ty_u64).map_err(|_| TelemetryError::Unpack("bad data type"))?;
6507 if ty_u32 > crate::MAX_VALUE_DATA_TYPE {
6508 return Err(TelemetryError::Unpack("bad data type"));
6509 }
6510 let ty = DataType(ty_u32);
6511 let data_size_off = off;
6512 let data_size = Self::read_uleb128_local(data, &mut off)?;
6513 let _timestamp_off = off;
6514 let timestamp = Self::read_uleb128_local(data, &mut off)?;
6515 let nonce = if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
6516 u16::try_from(Self::read_uleb128_local(data, &mut off)?)
6517 .map_err(|_| TelemetryError::Unpack("packet nonce too large"))?
6518 } else {
6519 0
6520 };
6521 let between_start = off;
6522 let _source_address = u32::try_from(Self::read_uleb128_local(data, &mut off)?)
6523 .map_err(|_| TelemetryError::Unpack("source address too large"))?;
6524 let endpoint_bitmap_bytes = if (flags & SIDE_TRANSPORT_FLAG_ENDPOINT_BITMAP_PRESENT) != 0 {
6525 SIDE_TRANSPORT_EP_BITMAP_BYTES
6526 } else {
6527 0
6528 };
6529 if data.len() < off + endpoint_bitmap_bytes {
6530 return Err(TelemetryError::Unpack("short buffer"));
6531 }
6532 off += endpoint_bitmap_bytes;
6533 if (flags & SIDE_TRANSPORT_FLAG_WIRE_CONTRACT) != 0 {
6534 let contract_len = usize::try_from(Self::read_uleb128_local(data, &mut off)?)
6535 .map_err(|_| TelemetryError::Unpack("wire contract length"))?;
6536 if data.len() < off + contract_len {
6537 return Err(TelemetryError::Unpack("short buffer"));
6538 }
6539 off += contract_len;
6540 }
6541 let reliable_span = wire_format::reliable_header_span(bytes)?;
6542 let (reliable_flags, reliable_seq_ack, reliable_compact, payload_off) =
6543 if let Some((rel_off, rel_len, hdr)) = reliable_span {
6544 if data.len() < rel_off + rel_len {
6545 return Err(TelemetryError::Unpack("short buffer"));
6546 }
6547 (
6548 Some(hdr.flags),
6549 Some((hdr.seq, hdr.ack)),
6550 (flags & SIDE_TRANSPORT_FLAG_COMPACT_RELIABLE_HEADER) != 0,
6551 rel_off + rel_len,
6552 )
6553 } else {
6554 (None, None, false, off)
6555 };
6556 if payload_off > data.len() {
6557 return Err(TelemetryError::Unpack("short buffer"));
6558 }
6559 let payload = &data[payload_off..];
6560 let prefix = Arc::<[u8]>::from(&data[1..data_size_off]);
6561 let between_end = reliable_span
6562 .map(|(rel_off, _, _)| rel_off)
6563 .unwrap_or(payload_off);
6564 let between = Arc::<[u8]>::from(&data[between_start..between_end]);
6565 let base_flags =
6566 flags & !(SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED | SIDE_TRANSPORT_FLAG_PACKET_NONCE);
6567 let mut hash = 0xD1B5_4A32_9C7E_01F3u64;
6568 hash = hash_bytes_u64(hash, &[base_flags]);
6569 hash = hash_bytes_u64(hash, &prefix);
6570 hash = hash_bytes_u64(hash, &between);
6571 if let Some(rel_flags) = reliable_flags {
6572 hash = hash_bytes_u64(hash, &[rel_flags]);
6573 }
6574 let template = SideHeaderTemplate {
6575 hash,
6576 base_flags,
6577 prefix,
6578 between,
6579 reliable_flags,
6580 reliable_compact,
6581 };
6582 let _ = ty_end_start;
6583 Ok((
6584 template,
6585 ty,
6586 flags,
6587 data_size,
6588 timestamp,
6589 nonce,
6590 reliable_seq_ack,
6591 payload,
6592 ))
6593 }
6594
6595 fn reconstruct_side_compact_frame(
6596 template: &SideHeaderTemplate,
6597 body: &[u8],
6598 timestamp_mode: SideCompactTimestampMode,
6599 timestamp_base: Option<u64>,
6600 ) -> TelemetryResult<(Arc<[u8]>, u64)> {
6601 if body.is_empty() {
6602 return Err(TelemetryError::Unpack("short side compact frame"));
6603 }
6604 let mut off = 0usize;
6605 let flags = body[off];
6606 off += 1;
6607 if (flags & !(SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED | SIDE_TRANSPORT_FLAG_PACKET_NONCE))
6608 != template.base_flags
6609 {
6610 return Err(TelemetryError::Unpack("side compact flags mismatch"));
6611 }
6612 let data_size = Self::read_uleb128_local(body, &mut off)?;
6613 let timestamp = match timestamp_mode {
6614 SideCompactTimestampMode::Absolute => Self::read_uleb128_local(body, &mut off)?,
6615 SideCompactTimestampMode::Delta => {
6616 let timestamp_field = Self::read_uleb128_local(body, &mut off)?;
6617 let base = timestamp_base.ok_or(TelemetryError::Unpack(
6618 "missing side compact timestamp context",
6619 ))?;
6620 base.checked_add(timestamp_field)
6621 .ok_or(TelemetryError::Unpack(
6622 "side compact timestamp delta overflow",
6623 ))?
6624 }
6625 SideCompactTimestampMode::Omitted => timestamp_base.ok_or(TelemetryError::Unpack(
6626 "missing side compact timestamp context",
6627 ))?,
6628 };
6629 let nonce = if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
6630 Some(Self::read_uleb128_local(body, &mut off)?)
6631 } else {
6632 None
6633 };
6634 let reliable_seq_ack = if template.reliable_flags.is_some() {
6635 let seq = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6636 .map_err(|_| TelemetryError::Unpack("side compact reliable seq too large"))?;
6637 let ack = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6638 .map_err(|_| TelemetryError::Unpack("side compact reliable ack too large"))?;
6639 Some((seq, ack))
6640 } else {
6641 None
6642 };
6643 let payload = &body[off..];
6644 let mut raw = Vec::with_capacity(
6645 1 + template.prefix.len() + template.between.len() + payload.len() + 32,
6646 );
6647 raw.push(flags);
6648 raw.extend_from_slice(&template.prefix);
6649 Self::write_uleb128_local(data_size, &mut raw);
6650 Self::write_uleb128_local(timestamp, &mut raw);
6651 if let Some(nonce) = nonce {
6652 Self::write_uleb128_local(nonce, &mut raw);
6653 }
6654 raw.extend_from_slice(&template.between);
6655 if let Some(rel_flags) = template.reliable_flags {
6656 let (seq, ack) =
6657 reliable_seq_ack.ok_or(TelemetryError::Unpack("missing side compact reliable"))?;
6658 wire_format::write_reliable_header_encoded(
6659 wire_format::ReliableHeader {
6660 flags: rel_flags,
6661 seq,
6662 ack,
6663 },
6664 template.reliable_compact,
6665 &mut raw,
6666 );
6667 }
6668 raw.extend_from_slice(payload);
6669 let crc = Self::crc32_bytes(&raw);
6670 raw.extend_from_slice(&crc.to_le_bytes());
6671 Ok((Arc::from(raw), timestamp))
6672 }
6673 fn split_side_transport_frame(
6674 &self,
6675 side: RouterSideId,
6676 frame: Arc<[u8]>,
6677 max_frame_bytes: usize,
6678 ) -> TelemetryResult<Vec<Arc<[u8]>>> {
6679 if max_frame_bytes <= SIDE_TRANSPORT_CHUNK_OVERHEAD {
6680 return Err(TelemetryError::BadArg);
6681 }
6682 let payload_budget = max_frame_bytes - SIDE_TRANSPORT_CHUNK_OVERHEAD;
6683 let mut st = self.state.lock();
6684 let side_state = st
6685 .side_transport
6686 .get_mut(&side)
6687 .ok_or(TelemetryError::BadArg)?;
6688 let transfer_id = side_state.next_chunk_id.wrapping_add(1).max(1);
6689 side_state.next_chunk_id = transfer_id;
6690 drop(st);
6691
6692 let total = frame.len().div_ceil(payload_budget);
6693 let total_u16 =
6694 u16::try_from(total).map_err(|_| TelemetryError::PacketTooLarge("too many chunks"))?;
6695 let mut frames = Vec::with_capacity(total);
6696 for (idx, chunk) in frame.chunks(payload_budget).enumerate() {
6697 let mut body = Vec::with_capacity(8 + chunk.len());
6698 body.extend_from_slice(&transfer_id.to_le_bytes());
6699 body.extend_from_slice(&(idx as u16).to_le_bytes());
6700 body.extend_from_slice(&total_u16.to_le_bytes());
6701 body.extend_from_slice(chunk);
6702 frames.push(Self::wrap_side_transport_frame(
6703 SIDE_TRANSPORT_KIND_CHUNK,
6704 &body,
6705 ));
6706 }
6707 Ok(frames)
6708 }
6709
6710 fn encode_side_transport_frames(
6711 &self,
6712 side: RouterSideId,
6713 opts: RouterSideOptions,
6714 raw: Arc<[u8]>,
6715 ) -> TelemetryResult<Vec<Arc<[u8]>>> {
6716 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
6717 return Ok(vec![raw]);
6718 }
6719
6720 let raw_len = raw.len();
6721 let mut compact_payload_len = None;
6722 let mut used_compact = false;
6723 let mut used_timestamp_delta = false;
6724 let mut omitted_timestamp = false;
6725 let wrapped = if opts.header_template_enabled {
6726 let (template, ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
6727 Self::extract_side_header_template(raw.as_ref())?;
6728 let (template_id, use_compact, previous_timestamp) = {
6729 let mut st = self.state.lock();
6730 let mut effective_template_limit = opts.max_side_transport_templates;
6735 #[cfg(feature = "discovery")]
6736 {
6737 effective_template_limit = st
6738 .discovery_routes
6739 .get(&side)
6740 .into_iter()
6741 .flat_map(|route| route.announcers.values())
6742 .filter_map(|sender| sender.link_capabilities)
6743 .filter(|caps| {
6744 caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
6745 && caps.max_side_transport_templates > 0
6746 })
6747 .map(|caps| caps.max_side_transport_templates as usize)
6748 .fold(effective_template_limit, core::cmp::min);
6749 }
6750 let trimmed = st
6751 .side_transport
6752 .get_mut(&side)
6753 .ok_or(TelemetryError::BadArg)?
6754 .trim_tx_templates_to(effective_template_limit);
6755 for _ in 0..trimmed {
6756 st.side_runtime_stats
6757 .entry(side)
6758 .or_default()
6759 .note_side_transport_template_eviction();
6760 }
6761 let side_state = st
6762 .side_transport
6763 .get_mut(&side)
6764 .ok_or(TelemetryError::BadArg)?;
6765 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
6766 let previous = side_state.tx_last_timestamps.get(&id).copied();
6767 const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
6770 let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
6771 let use_compact = *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES;
6772 if use_compact {
6773 *compact_uses = compact_uses.saturating_add(1);
6774 } else {
6775 *compact_uses = 0;
6776 }
6777 (id, use_compact, previous)
6778 } else {
6779 let next = side_state.next_template_id.wrapping_add(1).max(1);
6780 side_state.next_template_id = next;
6781 let evicted = side_state.insert_tx_template(
6782 template.clone(),
6783 next,
6784 effective_template_limit,
6785 );
6786 if evicted {
6787 st.side_runtime_stats
6788 .entry(side)
6789 .or_default()
6790 .note_side_transport_template_eviction();
6791 }
6792 if let Some(side_state) = st.side_transport.get_mut(&side) {
6793 side_state.tx_last_timestamps.insert(next, timestamp);
6794 }
6795 (next, false, None)
6796 }
6797 };
6798 if use_compact {
6799 used_compact = true;
6800 compact_payload_len = Some(payload.len());
6801 let timestamp_field = if let Some(previous) = previous_timestamp {
6802 let delta = timestamp.saturating_sub(previous);
6803 let omit_timestamp = opts.omit_unchanged_compact_timestamps
6804 || opts.compact_timestamp_omission_types.contains(ty);
6805 if omit_timestamp && timestamp == previous {
6806 omitted_timestamp = true;
6807 None
6808 } else if timestamp >= previous
6809 && Self::uleb128_len_local(delta) < Self::uleb128_len_local(timestamp)
6810 {
6811 used_timestamp_delta = true;
6812 Some(delta)
6813 } else {
6814 Some(timestamp)
6815 }
6816 } else {
6817 Some(timestamp)
6818 };
6819 let mut body = Vec::with_capacity(payload.len() + 32);
6820 body.push(flags);
6821 Self::write_uleb128_local(u64::from(template_id), &mut body);
6822 Self::write_uleb128_local(data_size, &mut body);
6823 if let Some(timestamp_field) = timestamp_field {
6824 Self::write_uleb128_local(timestamp_field, &mut body);
6825 }
6826 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
6827 Self::write_uleb128_local(u64::from(nonce), &mut body);
6828 }
6829 if let Some((seq, ack)) = reliable_seq_ack {
6830 Self::write_uleb128_local(u64::from(seq), &mut body);
6831 Self::write_uleb128_local(u64::from(ack), &mut body);
6832 }
6833 body.extend_from_slice(payload);
6834 {
6835 let mut st = self.state.lock();
6836 if let Some(side_state) = st.side_transport.get_mut(&side) {
6837 side_state.tx_last_timestamps.insert(template_id, timestamp);
6838 }
6839 }
6840 let kind = if omitted_timestamp {
6841 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
6842 } else if used_timestamp_delta {
6843 SIDE_TRANSPORT_KIND_COMPACT_DELTA
6844 } else {
6845 SIDE_TRANSPORT_KIND_COMPACT
6846 };
6847 Self::wrap_side_transport_frame(kind, &body)
6848 } else {
6849 let mut body = Vec::with_capacity(raw.len() + 4);
6850 Self::write_uleb128_local(u64::from(template_id), &mut body);
6851 body.extend_from_slice(raw.as_ref());
6852 Self::wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
6853 }
6854 } else {
6855 Self::wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
6856 };
6857
6858 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
6859 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
6860 } else {
6861 Ok(vec![wrapped])
6862 }?;
6863 let wire_len = frames.iter().map(|frame| frame.len()).sum::<usize>();
6864 let mut st = self.state.lock();
6865 let stats = st.side_runtime_stats.entry(side).or_default();
6866 if used_compact {
6867 let overhead = compact_payload_len
6868 .map(|payload_len| wire_len.saturating_sub(payload_len))
6869 .unwrap_or(wire_len);
6870 stats.note_side_transport_compact(
6871 raw_len,
6872 wire_len,
6873 overhead,
6874 used_timestamp_delta,
6875 omitted_timestamp,
6876 );
6877 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
6878 {
6879 stats.note_side_transport_compact_target_miss();
6880 }
6881 } else {
6882 stats.note_side_transport_full(raw_len, wire_len);
6883 }
6884 if frames.len() > 1 {
6885 stats.note_side_transport_chunks(frames.len());
6886 }
6887 Ok(frames)
6888 }
6889
6890 fn decode_side_transport_frame(
6891 &self,
6892 side: RouterSideId,
6893 bytes: &[u8],
6894 ) -> TelemetryResult<Option<Arc<[u8]>>> {
6895 let Some((kind, body)) = Self::parse_side_transport_wrapper(bytes)? else {
6896 return Ok(Some(Arc::from(bytes)));
6897 };
6898 match kind {
6899 SIDE_TRANSPORT_KIND_FULL => {
6900 let mut off = 0usize;
6901 let template_id = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6902 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
6903 let raw = Arc::<[u8]>::from(&body[off..]);
6904 if let Ok((template, _, _, _, timestamp, _, _, _)) =
6905 Self::extract_side_header_template(raw.as_ref())
6906 {
6907 let mut st = self.state.lock();
6908 let max_templates = st
6909 .sides
6910 .get(side)
6911 .and_then(|side| side.as_ref())
6912 .map(|side| side.opts.max_side_transport_templates)
6913 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
6914 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
6915 let evicted =
6916 side_state.insert_rx_template(template_id, template, max_templates);
6917 side_state.rx_last_timestamps.insert(template_id, timestamp);
6918 evicted
6919 });
6920 if evicted {
6921 st.side_runtime_stats
6922 .entry(side)
6923 .or_default()
6924 .note_side_transport_template_eviction();
6925 }
6926 }
6927 Ok(Some(raw))
6928 }
6929 SIDE_TRANSPORT_KIND_COMPACT
6930 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
6931 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
6932 if body.is_empty() {
6933 return Err(TelemetryError::Unpack("short side compact frame"));
6934 }
6935 let mut off = 1usize;
6936 let template_id = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6937 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
6938 let mut compact_body = Vec::with_capacity(1 + body.len().saturating_sub(off));
6939 compact_body.push(body[0]);
6940 compact_body.extend_from_slice(&body[off..]);
6941 let (template, timestamp_base) = {
6942 let st = self.state.lock();
6943 let state = st.side_transport.get(&side);
6944 let template = state
6945 .and_then(|state| state.rx_templates_by_id.get(&template_id))
6946 .cloned();
6947 let timestamp_base = if matches!(
6948 kind,
6949 SIDE_TRANSPORT_KIND_COMPACT_DELTA
6950 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
6951 ) {
6952 state
6953 .and_then(|state| state.rx_last_timestamps.get(&template_id))
6954 .copied()
6955 } else {
6956 None
6957 };
6958 (template, timestamp_base)
6959 };
6960 let Some(template) = template else {
6964 return Ok(None);
6965 };
6966 let timestamp_mode = match kind {
6967 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
6968 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
6969 _ => SideCompactTimestampMode::Absolute,
6970 };
6971 let (frame, timestamp) = Self::reconstruct_side_compact_frame(
6972 &template,
6973 &compact_body,
6974 timestamp_mode,
6975 timestamp_base,
6976 )?;
6977 let mut st = self.state.lock();
6978 if let Some(side_state) = st.side_transport.get_mut(&side) {
6979 side_state.rx_last_timestamps.insert(template_id, timestamp);
6980 }
6981 Ok(Some(frame))
6982 }
6983 SIDE_TRANSPORT_KIND_CHUNK => {
6984 if body.len() < 8 {
6985 return Err(TelemetryError::Unpack("short side chunk frame"));
6986 }
6987 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
6988 let index = u16::from_le_bytes([body[4], body[5]]);
6989 let total = u16::from_le_bytes([body[6], body[7]]);
6990 let payload = Arc::<[u8]>::from(&body[8..]);
6991 let assembled = {
6992 let mut st = self.state.lock();
6993 let side_state = st
6994 .side_transport
6995 .get_mut(&side)
6996 .ok_or(TelemetryError::BadArg)?;
6997 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
6998 if entry.total == 0 {
6999 entry.total = total;
7000 } else if entry.total != total {
7001 side_state.rx_chunks.remove(&transfer_id);
7002 return Err(TelemetryError::Unpack("side chunk total mismatch"));
7003 }
7004 entry.received.entry(index).or_insert(payload);
7005 if entry.received.len() == usize::from(total) {
7006 let entry = side_state
7007 .rx_chunks
7008 .remove(&transfer_id)
7009 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
7010 let mut out = Vec::new();
7011 for idx in 0..entry.total {
7012 let chunk = entry
7013 .received
7014 .get(&idx)
7015 .ok_or(TelemetryError::Unpack("side chunk gap"))?;
7016 out.extend_from_slice(chunk);
7017 }
7018 Some(Arc::<[u8]>::from(out))
7019 } else {
7020 None
7021 }
7022 };
7023 match assembled {
7024 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
7025 None => Ok(None),
7026 }
7027 }
7028 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
7029 }
7030 }
7031
7032 fn packed_handler_call(
7033 handler: &RouterTxHandlerFn,
7034 bytes: &[u8],
7035 priority: u8,
7036 ) -> TelemetryResult<()> {
7037 match handler {
7038 RouterTxHandlerFn::Packed(handler) => handler(bytes),
7039 RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
7040 RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
7041 }
7042 }
7043
7044 fn effective_transport_priority(&self, ty: DataType) -> u8 {
7045 if self.is_managed_variable_type(ty) {
7046 crate::transport_priority(DataType::ManagedVariableValue)
7047 } else {
7048 crate::transport_priority(ty)
7049 }
7050 }
7051
7052 fn call_side_tx_handler(
7053 &self,
7054 side: RouterSideId,
7055 handler: &RouterTxHandlerFn,
7056 data: &RouterItem,
7057 relayed: bool,
7058 ) -> TelemetryResult<()> {
7059 let opts = {
7060 let st = self.state.lock();
7061 Self::side_ref(&st, side)?.opts
7062 };
7063 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
7064 return Err(TelemetryError::Io("side tx busy"));
7065 };
7066 let started_ms = self.clock.now_ms();
7067 let ty = match data {
7068 RouterItem::Packet(pkt) => pkt.data_type(),
7069 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
7070 };
7071 let priority = self.effective_transport_priority(ty);
7072 let result = match (handler, data) {
7073 (
7074 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
7075 RouterItem::Packed(bytes),
7076 ) => {
7077 #[cfg(feature = "cryptography")]
7078 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
7079 #[cfg(not(feature = "cryptography"))]
7080 let send_bytes = bytes.clone();
7081 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
7082 let mut attempts_total = 0usize;
7083 let mut sent_bytes = 0usize;
7084 for frame in frames {
7085 match self.retry_with_attempts(runtime_max_handler_retries(), || {
7086 Self::packed_handler_call(packed, frame.as_ref(), priority)
7087 }) {
7088 Ok((_, attempts)) => {
7089 attempts_total = attempts_total.saturating_add(attempts);
7090 sent_bytes = sent_bytes.saturating_add(frame.len());
7091 }
7092 Err((err, attempts)) => {
7093 self.note_side_tx_failure(
7094 side,
7095 ty,
7096 attempts_total.saturating_add(attempts),
7097 );
7098 return Err(err);
7099 }
7100 }
7101 }
7102 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
7103 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
7104 return Ok(());
7105 }
7106 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
7107 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
7108 }
7109 (
7110 packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
7111 RouterItem::Packet(pkt),
7112 ) => {
7113 let owned = self.pack_packet_for_router(pkt, None)?;
7114 let frames = self.encode_side_transport_frames(side, opts, owned)?;
7115 let mut attempts_total = 0usize;
7116 let mut sent_bytes = 0usize;
7117 for frame in frames {
7118 match self.retry_with_attempts(runtime_max_handler_retries(), || {
7119 Self::packed_handler_call(packed, frame.as_ref(), priority)
7120 }) {
7121 Ok((_, attempts)) => {
7122 attempts_total = attempts_total.saturating_add(attempts);
7123 sent_bytes = sent_bytes.saturating_add(frame.len());
7124 }
7125 Err((err, attempts)) => {
7126 self.note_side_tx_failure(
7127 side,
7128 ty,
7129 attempts_total.saturating_add(attempts),
7130 );
7131 return Err(err);
7132 }
7133 }
7134 }
7135 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
7136 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
7137 return Ok(());
7138 }
7139 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
7140 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
7141 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
7142 }
7143 };
7144 match result {
7145 Ok((_, attempts)) => {
7146 if let Ok(bytes) = Self::router_item_wire_len(data) {
7147 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
7148 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
7149 }
7150 Ok(())
7151 }
7152 Err((err, attempts)) => {
7153 self.note_side_tx_failure(side, ty, attempts);
7154 Err(err)
7155 }
7156 }
7157 }
7158
7159 fn adjust_reliable_for_side(
7160 &self,
7161 opts: RouterSideOptions,
7162 data: RouterItem,
7163 preserve_end_to_end_ack: bool,
7164 ) -> TelemetryResult<Option<RouterItem>> {
7165 if opts.reliable_enabled {
7166 return Ok(Some(data));
7167 }
7168
7169 match data {
7170 RouterItem::Packed(bytes) => {
7171 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
7172 if is_reliable_type(frame.envelope.ty)
7173 && let Some(hdr) = frame.reliable
7174 {
7175 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
7176 return Ok(None);
7177 }
7178 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
7179 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
7180 bytes.as_ref(),
7181 wire_format::RELIABLE_FLAG_UNSEQUENCED,
7182 hdr.seq,
7183 0,
7184 )?
7185 else {
7186 return Ok(Some(RouterItem::Packed(bytes)));
7187 };
7188 return Ok(Some(RouterItem::Packed(rewritten)));
7189 }
7190 }
7191 Ok(Some(RouterItem::Packed(bytes)))
7192 }
7193 RouterItem::Packet(pkt) => {
7194 if matches!(
7195 pkt.data_type(),
7196 DataType::ReliableAck
7197 | DataType::ReliablePartialAck
7198 | DataType::ReliablePacketRequest
7199 ) {
7200 if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
7201 return Ok(Some(RouterItem::Packet(pkt)));
7202 }
7203 return Ok(None);
7204 }
7205 Ok(Some(RouterItem::Packet(pkt)))
7206 }
7207 }
7208 }
7209
7210 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
7211 let now = self.clock.now_ms();
7212 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
7213
7214 {
7215 let mut st = self.state.lock();
7216 if st.reliable_tx.is_empty() {
7217 return Ok(());
7218 }
7219
7220 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
7221 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
7222 continue;
7223 };
7224 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
7225 for seq in sent_order {
7226 let Some(sent) = tx_state.sent.get_mut(&seq) else {
7227 continue;
7228 };
7229 if sent.queued
7230 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
7231 {
7232 continue;
7233 }
7234 if sent.partial_acked {
7235 sent.partial_acked = false;
7242 sent.last_send_ms = now;
7243 continue;
7244 }
7245 if sent.retries >= runtime_reliable_max_retries() {
7246 tx_state.sent.remove(&seq);
7247 tx_state.sent_order.retain(|existing| *existing != seq);
7248 continue;
7249 }
7250 sent.retries += 1;
7251 requeue.push((*side, ty, seq));
7252 }
7253 }
7254 }
7255
7256 for (side, ty, seq) in requeue {
7257 self.queue_reliable_retransmit(side, ty, seq, true)?;
7258 }
7259
7260 Ok(())
7261 }
7262
7263 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
7264 let now = self.clock.now_ms();
7265 let mut requeue = Vec::new();
7266
7267 {
7268 let mut st = self.state.lock();
7269 #[cfg(feature = "discovery")]
7270 {
7271 if Self::prune_discovery_routes_locked(&mut st, now) {
7272 Self::note_discovery_topology_change_locked(&mut st, now);
7273 }
7274 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
7275 }
7276 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
7277 for packet_id in packet_ids {
7278 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
7279 continue;
7280 };
7281 if sent.queued
7282 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
7283 {
7284 continue;
7285 }
7286 if sent.retries >= runtime_reliable_max_retries() {
7287 st.end_to_end_reliable_tx.remove(&packet_id);
7288 continue;
7289 }
7290 sent.retries += 1;
7291 requeue.push(packet_id);
7292 }
7293 }
7294
7295 for packet_id in requeue {
7296 self.queue_end_to_end_reliable_retransmit(packet_id)?;
7297 }
7298
7299 Ok(())
7300 }
7301
7302 #[cfg(feature = "timesync")]
7303 #[inline]
7304 fn monotonic_now_ns(&self) -> u64 {
7305 self.clock.now_ns()
7306 }
7307
7308 #[cfg(feature = "timesync")]
7309 #[inline]
7310 fn monotonic_now_ms(&self) -> u64 {
7311 self.clock.now_ms()
7312 }
7313
7314 #[cfg(feature = "timesync")]
7315 fn refresh_timesync_state(&self, now_mono_ms: u64) {
7316 let now_mono_ns = self.monotonic_now_ns();
7317 let mut st = self.timesync.lock();
7318 st.clock.prune_expired(now_mono_ms);
7319 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
7320 st.remote_sources
7321 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
7322 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
7323 let leader = if let Some(tracker) = st.tracker.as_mut() {
7324 let _ = tracker.refresh(now_mono_ms);
7325 tracker.leader(now_mono_ms, has_usable_time)
7326 } else {
7327 None
7328 };
7329 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
7330 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
7331 let target_ms = st
7332 .remote_sources
7333 .get(remote.sender.as_str())
7334 .map(|src| src.sample_unix_ms);
7335 if let Some(target_ms) = target_ms {
7336 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
7337 }
7338 }
7339 }
7340
7341 #[cfg(feature = "timesync")]
7342 pub fn update_network_time_source(
7344 &self,
7345 source: &str,
7346 priority: u64,
7347 time: PartialNetworkTime,
7348 ttl_ms: Option<u64>,
7349 ) {
7350 let now_ms = self.monotonic_now_ms();
7351 let now_ns = self.monotonic_now_ns();
7352 let mut st = self.timesync.lock();
7353 st.clock
7354 .update_source(source, priority, time, now_ms, now_ns, ttl_ms);
7355 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
7356 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
7357 }
7358 }
7359
7360 #[cfg(feature = "timesync")]
7361 fn set_network_time_source_impl(
7362 &self,
7363 source: &str,
7364 priority: u64,
7365 time: PartialNetworkTime,
7366 ttl_ms: Option<u64>,
7367 ) {
7368 let observed_mono_ms = self.monotonic_now_ms();
7369 let observed_mono_ns = self.monotonic_now_ns();
7370 let mut st = self.timesync.lock();
7371 let commit_mono_ms = self.monotonic_now_ms();
7372 let commit_mono_ns = self.monotonic_now_ns();
7373 let adjusted = if let Some(base) = time.to_network_time() {
7374 let elapsed_ns = commit_mono_ns.saturating_sub(observed_mono_ns);
7375 advance_network_time(base, elapsed_ns)
7376 .map(PartialNetworkTime::from)
7377 .unwrap_or(time)
7378 } else {
7379 time
7380 };
7381 let adjusted_mono_ms =
7382 observed_mono_ms.saturating_add(commit_mono_ms.saturating_sub(observed_mono_ms));
7383 st.clock.update_source(
7384 source,
7385 priority,
7386 adjusted,
7387 commit_mono_ms.max(adjusted_mono_ms),
7388 commit_mono_ns,
7389 ttl_ms,
7390 );
7391 if let Some(unix_ms) = adjusted.to_network_time().and_then(|t| t.as_unix_ms()) {
7392 st.disciplined_clock.steer_unix_ms(commit_mono_ns, unix_ms);
7393 }
7394 }
7395
7396 #[cfg(feature = "timesync")]
7397 fn local_network_time_priority(&self) -> u64 {
7398 let st = self.timesync.lock();
7399 st.cfg.map(|cfg| cfg.priority).unwrap_or(0)
7400 }
7401
7402 #[cfg(feature = "timesync")]
7403 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
7405 let priority = self.local_network_time_priority();
7406 if time.is_complete_date() && time.is_complete_time() {
7407 self.set_network_time_source_impl(LOCAL_TIMESYNC_FULL_SOURCE_ID, priority, time, None);
7408 let mut st = self.timesync.lock();
7409 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
7410 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
7411 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
7412 return;
7413 }
7414
7415 {
7416 let mut st = self.timesync.lock();
7417 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
7418 }
7419
7420 if time.year.is_some() || time.month.is_some() || time.day.is_some() {
7421 self.set_network_time_source_impl(
7422 LOCAL_TIMESYNC_DATE_SOURCE_ID,
7423 priority,
7424 PartialNetworkTime {
7425 year: time.year,
7426 month: time.month,
7427 day: time.day,
7428 ..Default::default()
7429 },
7430 None,
7431 );
7432 }
7433
7434 if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
7435 self.set_network_time_source_impl(
7436 LOCAL_TIMESYNC_TOD_SOURCE_ID,
7437 priority,
7438 PartialNetworkTime {
7439 hour: time.hour,
7440 minute: time.minute,
7441 second: time.second,
7442 nanosecond: time.nanosecond,
7443 ..Default::default()
7444 },
7445 None,
7446 );
7447 }
7448
7449 if time.nanosecond.is_some() {
7450 self.set_network_time_source_impl(
7451 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
7452 priority,
7453 PartialNetworkTime {
7454 nanosecond: time.nanosecond,
7455 ..Default::default()
7456 },
7457 None,
7458 );
7459 }
7460 }
7461
7462 #[cfg(feature = "timesync")]
7463 pub fn clear_local_network_time(&self) {
7465 let mut st = self.timesync.lock();
7466 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
7467 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
7468 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
7469 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
7470 }
7471
7472 #[cfg(feature = "timesync")]
7473 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
7475 self.set_local_network_time(PartialNetworkTime {
7476 year: Some(year),
7477 month: Some(month),
7478 day: Some(day),
7479 ..Default::default()
7480 });
7481 }
7482
7483 #[cfg(feature = "timesync")]
7484 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
7486 self.set_local_network_time(PartialNetworkTime {
7487 hour: Some(hour),
7488 minute: Some(minute),
7489 ..Default::default()
7490 });
7491 }
7492
7493 #[cfg(feature = "timesync")]
7494 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
7496 self.set_local_network_time(PartialNetworkTime {
7497 hour: Some(hour),
7498 minute: Some(minute),
7499 second: Some(second),
7500 ..Default::default()
7501 });
7502 }
7503
7504 #[cfg(feature = "timesync")]
7505 pub fn set_local_network_time_hms_millis(
7507 &self,
7508 hour: u8,
7509 minute: u8,
7510 second: u8,
7511 millisecond: u16,
7512 ) {
7513 self.set_local_network_time(PartialNetworkTime {
7514 hour: Some(hour),
7515 minute: Some(minute),
7516 second: Some(second),
7517 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
7518 ..Default::default()
7519 });
7520 }
7521
7522 #[cfg(feature = "timesync")]
7523 pub fn set_local_network_time_hms_nanos(
7525 &self,
7526 hour: u8,
7527 minute: u8,
7528 second: u8,
7529 nanosecond: u32,
7530 ) {
7531 self.set_local_network_time(PartialNetworkTime {
7532 hour: Some(hour),
7533 minute: Some(minute),
7534 second: Some(second),
7535 nanosecond: Some(nanosecond),
7536 ..Default::default()
7537 });
7538 }
7539
7540 #[cfg(feature = "timesync")]
7541 pub fn set_local_network_datetime(
7543 &self,
7544 year: i32,
7545 month: u8,
7546 day: u8,
7547 hour: u8,
7548 minute: u8,
7549 second: u8,
7550 ) {
7551 self.set_local_network_time(PartialNetworkTime {
7552 year: Some(year),
7553 month: Some(month),
7554 day: Some(day),
7555 hour: Some(hour),
7556 minute: Some(minute),
7557 second: Some(second),
7558 ..Default::default()
7559 });
7560 }
7561
7562 #[cfg(feature = "timesync")]
7563 #[allow(clippy::too_many_arguments)]
7564 pub fn set_local_network_datetime_millis(
7566 &self,
7567 year: i32,
7568 month: u8,
7569 day: u8,
7570 hour: u8,
7571 minute: u8,
7572 second: u8,
7573 millisecond: u16,
7574 ) {
7575 self.set_local_network_time(PartialNetworkTime {
7576 year: Some(year),
7577 month: Some(month),
7578 day: Some(day),
7579 hour: Some(hour),
7580 minute: Some(minute),
7581 second: Some(second),
7582 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
7583 });
7584 }
7585
7586 #[cfg(feature = "timesync")]
7587 #[allow(clippy::too_many_arguments)]
7588 pub fn set_local_network_datetime_nanos(
7590 &self,
7591 year: i32,
7592 month: u8,
7593 day: u8,
7594 hour: u8,
7595 minute: u8,
7596 second: u8,
7597 nanosecond: u32,
7598 ) {
7599 self.set_local_network_time(PartialNetworkTime {
7600 year: Some(year),
7601 month: Some(month),
7602 day: Some(day),
7603 hour: Some(hour),
7604 minute: Some(minute),
7605 second: Some(second),
7606 nanosecond: Some(nanosecond),
7607 });
7608 }
7609
7610 #[cfg(feature = "timesync")]
7611 pub fn clear_network_time_source(&self, source: &str) {
7613 let mut st = self.timesync.lock();
7614 st.clock.remove_source(source);
7615 }
7616
7617 #[cfg(feature = "timesync")]
7618 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
7620 let mut st = self.timesync.lock();
7621 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
7622 st.cfg = cfg;
7623 st.tracker = cfg.map(TimeSyncTracker::new);
7624 st.disciplined_clock = SlewedNetworkClock::new(
7625 cfg.map(|c| c.max_slew_ppm)
7626 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
7627 );
7628 st.remote_sources.clear();
7629 st.next_seq = 1;
7630 st.next_announce_mono_ms = 0;
7631 st.next_request_mono_ms = 0;
7632 st.pending_request = None;
7633 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
7634 for source in stale_remote_sources {
7635 st.clock.remove_source(&source);
7636 }
7637 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
7638 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
7639 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
7640 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
7641 }
7642
7643 #[cfg(feature = "timesync")]
7644 pub fn network_time(&self) -> Option<NetworkTimeReading> {
7646 let now_ms = self.monotonic_now_ms();
7647 let now_ns = self.monotonic_now_ns();
7648 self.refresh_timesync_state(now_ms);
7649 let st = self.timesync.lock();
7650 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
7651 return Some(NetworkTimeReading {
7652 time: PartialNetworkTime::from_unix_ms(unix_ms),
7653 unix_time_ms: Some(unix_ms),
7654 });
7655 }
7656 st.clock.current_time(now_ns)
7657 }
7658
7659 #[cfg(feature = "timesync")]
7660 pub fn network_time_ms(&self) -> Option<u64> {
7662 self.network_time().and_then(|t| t.unix_time_ms)
7663 }
7664
7665 #[cfg(feature = "timesync")]
7666 fn packet_timestamp_ms(&self) -> u64 {
7667 self.network_time_ms()
7668 .unwrap_or_else(|| self.monotonic_now_ms())
7669 }
7670
7671 #[cfg(not(feature = "timesync"))]
7672 fn packet_timestamp_ms(&self) -> u64 {
7673 self.clock.now_ms()
7674 }
7675
7676 #[cfg(feature = "timesync")]
7677 fn queue_internal_timesync_request(
7678 &self,
7679 seq: u64,
7680 t1_mono_ms: u64,
7681 called_from_queue: bool,
7682 ) -> TelemetryResult<()> {
7683 let pkt_ts = self.packet_timestamp_ms();
7684 if called_from_queue {
7685 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
7686 } else {
7687 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
7688 }
7689 }
7690
7691 #[cfg(feature = "timesync")]
7692 fn queue_internal_timesync_response(
7693 &self,
7694 seq: u64,
7695 t1_mono_ms: u64,
7696 t2_network_ms: u64,
7697 t3_network_ms: u64,
7698 dst: Option<RouterSideId>,
7699 called_from_queue: bool,
7700 ) -> TelemetryResult<()> {
7701 let pkt_ts = self.packet_timestamp_ms();
7702 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
7703 let sender = self.sender_arc();
7704 let pkt = Packet::new(
7705 DataType::TimeSyncResponse,
7706 &[DataEndpoint::TimeSync],
7707 sender.as_ref(),
7708 pkt_ts,
7709 payload,
7710 )?;
7711 match dst {
7712 Some(dst) => self.emit_internal_tx(
7713 RouterTxItem::ToSide {
7714 src: None,
7715 dst,
7716 data: RouterItem::Packet(pkt),
7717 },
7718 true,
7719 called_from_queue,
7720 ),
7721 None => self.emit_internal_tx(
7722 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
7723 true,
7724 called_from_queue,
7725 ),
7726 }
7727 }
7728
7729 #[cfg(feature = "timesync")]
7730 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
7732 let now_ms = self.monotonic_now_ms();
7733 let now_ns = self.monotonic_now_ns();
7734 let mut queued_any = false;
7735 let mut announce_priority = None;
7736 let mut request = None;
7737
7738 {
7739 let mut st = self.timesync.lock();
7740 st.clock.prune_expired(now_ms);
7741 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
7742 st.remote_sources
7743 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
7744 let Some(cfg) = st.cfg else {
7745 return Ok(false);
7746 };
7747
7748 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
7749 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
7750 let _ = tracker.refresh(now_ms);
7751 (
7752 tracker.leader(now_ms, has_usable_time),
7753 tracker.local_announce_priority(now_ms, has_usable_time),
7754 )
7755 } else {
7756 (None, None)
7757 };
7758 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
7759
7760 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
7761 let target_ms = st
7762 .remote_sources
7763 .get(&remote.sender)
7764 .map(|src| src.sample_unix_ms);
7765 if let Some(target_ms) = target_ms {
7766 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
7767 }
7768 }
7769
7770 if let Some(priority) = announce_prio
7771 && now_ms >= st.next_announce_mono_ms
7772 {
7773 announce_priority = Some(priority);
7774 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
7775 }
7776
7777 if let Some(TimeSyncLeader::Remote(remote)) = leader
7778 && now_ms >= st.next_request_mono_ms
7779 && st.pending_request.is_none()
7780 {
7781 let seq = st.next_seq;
7782 let next = st.next_seq.wrapping_add(1);
7783 st.next_seq = if next == 0 { 1 } else { next };
7784 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
7785 st.pending_request = Some(PendingTimeSyncRequest {
7786 seq,
7787 t1_mono_ms: now_ms,
7788 source: remote.sender,
7789 });
7790 request = Some((seq, now_ms));
7791 }
7792 }
7793
7794 if let Some(priority) = announce_priority {
7795 let time_ms = self.packet_timestamp_ms();
7796 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
7797 queued_any = true;
7798 }
7799 if let Some((seq, t1_mono_ms)) = request {
7800 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
7801 queued_any = true;
7802 }
7803
7804 Ok(queued_any)
7805 }
7806
7807 #[cfg(feature = "timesync")]
7808 fn handle_internal_timesync_packet(
7809 &self,
7810 pkt: &Packet,
7811 src: Option<RouterSideId>,
7812 called_from_queue: bool,
7813 ) -> TelemetryResult<bool> {
7814 let Some(cfg) = self.cfg.timesync_config() else {
7815 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
7816 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
7817 }
7818 return Ok(true);
7819 };
7820
7821 let now_mono_ms = self.monotonic_now_ms();
7822 let now_mono_ns = self.monotonic_now_ns();
7823 let mut response = None;
7824 let mut poll_after = false;
7825
7826 {
7827 let mut st = self.timesync.lock();
7828 st.clock.prune_expired(now_mono_ms);
7829 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
7830 st.remote_sources
7831 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
7832 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
7833 if st.tracker.is_none() {
7834 return Ok(true);
7835 }
7836
7837 match pkt.data_type() {
7838 DataType::TimeSyncAnnounce => {
7839 let ann = decode_timesync_announce(pkt)?;
7840 let should_steer = {
7841 let tracker = st.tracker.as_mut().expect("tracker checked above");
7842 let _ = tracker.handle_announce(pkt, now_mono_ms)?;
7843 matches!(
7844 tracker.leader(now_mono_ms, has_usable_time),
7845 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender()
7846 )
7847 };
7848 st.remote_sources.insert(
7849 pkt.sender().to_owned(),
7850 RemoteTimeSyncSource {
7851 priority: ann.priority,
7852 last_sample_mono_ms: now_mono_ms,
7853 sample_unix_ms: ann.time_ms,
7854 },
7855 );
7856 st.clock.update_source(
7857 pkt.sender(),
7858 ann.priority,
7859 PartialNetworkTime::from_unix_ms(ann.time_ms),
7860 now_mono_ms,
7861 now_mono_ns,
7862 Some(cfg.source_timeout_ms),
7863 );
7864 if should_steer {
7865 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
7866 }
7867 poll_after = true;
7868 }
7869 DataType::TimeSyncRequest => {
7870 let should_serve = {
7871 let tracker = st.tracker.as_ref().expect("tracker checked above");
7872 tracker.should_serve(now_mono_ms, has_usable_time)
7873 };
7874 if should_serve {
7875 let req = decode_timesync_request(pkt)?;
7876 let network_now = st
7877 .disciplined_clock
7878 .read_unix_ms(now_mono_ns)
7879 .or_else(|| {
7880 st.clock
7881 .current_time(now_mono_ns)
7882 .and_then(|t| t.unix_time_ms)
7883 })
7884 .unwrap_or(now_mono_ms);
7885 let t2 = network_now;
7886 let t3 = network_now;
7887 response = Some((req.seq, req.t1_ms, t2, t3, src));
7888 }
7889 }
7890 DataType::TimeSyncResponse => {
7891 let resp = decode_timesync_response(pkt)?;
7892 let pending = st.pending_request.clone();
7893 if let Some(pending) = pending
7894 && pending.seq == resp.seq
7895 && pending.source == pkt.sender()
7896 {
7897 let source_priority = {
7898 let tracker = st.tracker.as_ref().expect("tracker checked above");
7899 tracker
7900 .best_active_source(now_mono_ms)
7901 .map(|s| s.priority)
7902 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
7903 .unwrap_or(cfg.priority)
7904 };
7905 let (estimate_ms, _delay_ms) = compute_network_time_sample(
7906 pending.t1_mono_ms,
7907 resp.t2_ms,
7908 resp.t3_ms,
7909 now_mono_ms,
7910 );
7911 st.remote_sources.insert(
7912 pkt.sender().to_owned(),
7913 RemoteTimeSyncSource {
7914 priority: source_priority,
7915 last_sample_mono_ms: now_mono_ms,
7916 sample_unix_ms: estimate_ms,
7917 },
7918 );
7919 st.clock.update_source(
7920 pkt.sender(),
7921 source_priority,
7922 PartialNetworkTime::from_unix_ms(estimate_ms),
7923 now_mono_ms,
7924 now_mono_ns,
7925 Some(cfg.source_timeout_ms),
7926 );
7927 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
7928 st.pending_request = None;
7929 }
7930 }
7931 _ => {}
7932 }
7933 }
7934
7935 if let Some((seq, t1, t2, t3, dst)) = response {
7936 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
7937 }
7938 if poll_after {
7939 let _ = self.poll_timesync()?;
7940 }
7941
7942 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
7943 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
7944 }
7945
7946 Ok(true)
7947 }
7948
7949 #[cfg(feature = "std")]
7951 pub fn new(cfg: RouterConfig) -> Self {
7952 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
7953 }
7954
7955 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
7957 #[cfg(feature = "timesync")]
7958 let timesync_cfg = cfg.timesync_config();
7959 let memory = cfg.memory_config();
7960 let hostname: Arc<str> = Arc::from(cfg.sender());
7961 let address_mode = cfg.address_mode();
7962 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
7963 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
7964 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
7965 let requested = address_mode.requested_address();
7966 let address = if requested == 0 { fallback } else { requested };
7967 let local_address = AddressBookEntry {
7968 hostname: hostname.clone(),
7969 address,
7970 requested_address: requested,
7971 mode: address_mode,
7972 birth_ms: clock
7973 .now_ms()
7974 .saturating_mul(1_000_000)
7975 .saturating_add(instance_seq),
7976 owner_hash,
7977 last_seen_ms: clock.now_ms(),
7978 };
7979 let mut address_book = BTreeMap::new();
7980 address_book.insert(hostname.to_string(), local_address.clone());
7981 let mut address_by_value = BTreeMap::new();
7982 address_by_value.insert(address, hostname.to_string());
7983 Self {
7984 sender: RouterMutex::new(hostname),
7985 cfg,
7986 state: RouterMutex::new(RouterInner {
7987 memory,
7988 sides: Vec::new(),
7989 route_overrides: BTreeMap::new(),
7990 typed_route_overrides: BTreeMap::new(),
7991 route_weights: BTreeMap::new(),
7992 route_priorities: BTreeMap::new(),
7993 source_route_modes: BTreeMap::new(),
7994 route_selection_cursors: BTreeMap::new(),
7995 adaptive_route_stats: BTreeMap::new(),
7996 side_runtime_stats: BTreeMap::new(),
7997 side_transport: BTreeMap::new(),
7998 managed_variable_types: BTreeSet::new(),
7999 managed_variable_permissions: BTreeMap::new(),
8000 managed_variable_latest: BTreeMap::new(),
8001 network_variable_update_handlers: BTreeMap::new(),
8002 local_address,
8003 address_book,
8004 address_by_value,
8005 p2p_port_handlers: BTreeMap::new(),
8006 p2p_stream_handlers: BTreeMap::new(),
8007 p2p_stream_sessions: BTreeMap::new(),
8008 next_p2p_stream_id: 1,
8009 received_queue: BoundedDeque::new(
8010 memory.max_queue_budget,
8011 memory.starting_queue_size,
8012 memory.queue_grow_step,
8013 ),
8014 transmit_queue: BoundedDeque::new(
8015 memory.max_queue_budget,
8016 memory.starting_queue_size,
8017 memory.queue_grow_step,
8018 ),
8019 tx_priority_burst: 0,
8020 recent_rx: BoundedDeque::new(
8021 memory.recent_rx_queue_bytes(),
8022 memory.recent_rx_queue_bytes(),
8023 memory.queue_grow_step,
8024 ),
8025 reliable_tx: BTreeMap::new(),
8026 reliable_rx: BTreeMap::new(),
8027 reliable_return_routes: BTreeMap::new(),
8028 reliable_return_route_order: VecDeque::new(),
8029 end_to_end_reliable_tx: BTreeMap::new(),
8030 end_to_end_reliable_tx_order: VecDeque::new(),
8031 total_handler_failures: 0,
8032 total_handler_retries: 0,
8033 #[cfg(feature = "discovery")]
8034 discovery_routes: BTreeMap::new(),
8035 #[cfg(feature = "discovery")]
8036 discovery_cadence: DiscoveryCadenceState::default(),
8037 #[cfg(feature = "discovery")]
8038 discovery_side_throttle: BTreeMap::new(),
8039 #[cfg(all(feature = "discovery", feature = "timesync"))]
8040 timesync_side_throttle: BTreeMap::new(),
8041 }),
8042 isr_rx_queue: IsrRxQueue::new(
8043 memory.max_queue_budget,
8044 memory.starting_queue_size,
8045 memory.queue_grow_step,
8046 ),
8047 side_tx_gate: ReentryGate::new(),
8048 clock,
8049 #[cfg(feature = "timesync")]
8050 timesync: RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)),
8051 }
8052 }
8053
8054 #[inline]
8055 fn sender_arc(&self) -> Arc<str> {
8056 self.sender.lock().clone()
8057 }
8058
8059 #[inline]
8060 pub fn sender(&self) -> Arc<str> {
8061 self.sender_arc()
8062 }
8063
8064 pub fn current_address(&self) -> NodeAddress {
8065 self.state.lock().local_address.address
8066 }
8067
8068 pub fn hostname(&self) -> Arc<str> {
8069 self.sender_arc()
8070 }
8071
8072 pub fn address_book(&self) -> Vec<AddressBookEntry> {
8073 self.state.lock().address_book.values().cloned().collect()
8074 }
8075
8076 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
8077 self.state.lock().address_book.get(hostname).cloned()
8078 }
8079
8080 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
8081 let st = self.state.lock();
8082 st.address_by_value
8083 .get(&address)
8084 .and_then(|hostname| st.address_book.get(hostname))
8085 .cloned()
8086 }
8087
8088 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
8089 where
8090 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
8091 {
8092 if port == 0 {
8093 return Err(TelemetryError::BadArg);
8094 }
8095 let mut st = self.state.lock();
8096 st.p2p_port_handlers
8097 .entry(port)
8098 .or_default()
8099 .push(P2pPortHandler {
8100 handler: Arc::new(f),
8101 });
8102 Ok(())
8103 }
8104
8105 pub fn clear_p2p_port(&self, port: P2pPort) {
8106 self.state.lock().p2p_port_handlers.remove(&port);
8107 }
8108
8109 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
8110 where
8111 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
8112 {
8113 if port == 0 {
8114 return Err(TelemetryError::BadArg);
8115 }
8116 let mut st = self.state.lock();
8117 st.p2p_stream_handlers
8118 .entry(port)
8119 .or_default()
8120 .push(P2pStreamHandler {
8121 handler: Arc::new(f),
8122 });
8123 Ok(())
8124 }
8125
8126 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
8127 self.state.lock().p2p_stream_handlers.remove(&port);
8128 }
8129
8130 fn send_p2p_to_entry(
8131 &self,
8132 dst: AddressBookEntry,
8133 dst_port: P2pPort,
8134 src_port: P2pPort,
8135 payload: &[u8],
8136 ) -> TelemetryResult<()> {
8137 if dst_port == 0 {
8138 return Err(TelemetryError::BadArg);
8139 }
8140 let local = self.state.lock().local_address.clone();
8141 let payload = Self::encode_p2p_payload(
8142 local.hostname.as_ref(),
8143 local.address,
8144 src_port,
8145 dst_port,
8146 payload,
8147 )?;
8148 let pkt = Packet::new(
8149 DataType::P2pMessage,
8150 &[DataEndpoint::Discovery],
8151 local.hostname.as_ref(),
8152 self.packet_timestamp_ms(),
8153 payload,
8154 )?;
8155 let target = Self::sender_hash(dst.hostname.as_ref());
8156 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
8157 if target == Self::sender_hash(self.sender_arc().as_ref()) {
8158 self.dispatch_p2p_packet(&pkt)?;
8159 return Ok(());
8160 }
8161 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
8162 }
8163
8164 pub fn send_p2p_to_hostname(
8165 &self,
8166 hostname: &str,
8167 dst_port: P2pPort,
8168 src_port: P2pPort,
8169 payload: &[u8],
8170 ) -> TelemetryResult<()> {
8171 let dst = self
8172 .resolve_hostname(hostname)
8173 .ok_or(TelemetryError::BadArg)?;
8174 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
8175 }
8176
8177 pub fn send_p2p_to_address(
8178 &self,
8179 address: NodeAddress,
8180 dst_port: P2pPort,
8181 src_port: P2pPort,
8182 payload: &[u8],
8183 ) -> TelemetryResult<()> {
8184 let dst = self
8185 .resolve_address(address)
8186 .ok_or(TelemetryError::BadArg)?;
8187 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
8188 }
8189
8190 fn open_p2p_stream_to_entry(
8191 &self,
8192 dst: AddressBookEntry,
8193 dst_port: P2pPort,
8194 src_port: P2pPort,
8195 ) -> TelemetryResult<P2pStreamId> {
8196 if dst_port == 0 || src_port == 0 {
8197 return Err(TelemetryError::BadArg);
8198 }
8199 let stream_id = {
8200 let mut st = self.state.lock();
8201 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
8202 if stream_id == 0 {
8203 return Err(TelemetryError::Io("p2p stream id exhausted"));
8204 }
8205 st.p2p_stream_sessions.insert(
8206 stream_id,
8207 P2pStreamSession {
8208 peer_hostname: dst.hostname.clone(),
8209 peer_address: dst.address,
8210 local_port: src_port,
8211 peer_port: dst_port,
8212 peer_stream_id: 0,
8213 next_sequence: 1,
8214 connected: false,
8215 },
8216 );
8217 stream_id
8218 };
8219 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
8220 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
8221 Ok(stream_id)
8222 }
8223
8224 pub fn open_p2p_stream_to_hostname(
8225 &self,
8226 hostname: &str,
8227 dst_port: P2pPort,
8228 src_port: P2pPort,
8229 ) -> TelemetryResult<P2pStreamId> {
8230 let dst = self
8231 .resolve_hostname(hostname)
8232 .ok_or(TelemetryError::BadArg)?;
8233 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
8234 }
8235
8236 pub fn open_p2p_stream_to_address(
8237 &self,
8238 address: NodeAddress,
8239 dst_port: P2pPort,
8240 src_port: P2pPort,
8241 ) -> TelemetryResult<P2pStreamId> {
8242 let dst = self
8243 .resolve_address(address)
8244 .ok_or(TelemetryError::BadArg)?;
8245 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
8246 }
8247
8248 fn send_p2p_stream_control(
8249 &self,
8250 stream_id: P2pStreamId,
8251 flags: u8,
8252 payload: &[u8],
8253 ) -> TelemetryResult<()> {
8254 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
8255 let mut st = self.state.lock();
8256 let Some(session) = st.p2p_stream_sessions.get_mut(&stream_id) else {
8257 return Err(TelemetryError::BadArg);
8258 };
8259 let peer_hostname = session.peer_hostname.clone();
8260 let peer_address = session.peer_address;
8261 let peer_port = session.peer_port;
8262 let local_port = session.local_port;
8263 let peer_stream_id = session.peer_stream_id;
8264 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
8265 let seq = session.next_sequence;
8266 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
8267 let dst = st
8268 .address_book
8269 .get(peer_hostname.as_ref())
8270 .cloned()
8271 .unwrap_or(AddressBookEntry {
8272 hostname: peer_hostname.clone(),
8273 address: peer_address,
8274 requested_address: peer_address,
8275 mode: AddressAssignmentMode::Dynamic,
8276 birth_ms: self.clock.now_ms(),
8277 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
8278 last_seen_ms: self.clock.now_ms(),
8279 });
8280 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
8281 };
8282 if refresh_syn {
8283 let syn_payload =
8284 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
8285 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
8286 }
8287 let stream_payload =
8288 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
8289 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
8290 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
8291 self.state.lock().p2p_stream_sessions.remove(&stream_id);
8292 }
8293 Ok(())
8294 }
8295
8296 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
8297 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
8298 }
8299
8300 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
8301 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
8302 }
8303
8304 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
8305 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
8306 }
8307
8308 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
8309 let hostname: Arc<str> = Arc::from(sender.as_ref());
8310 let mut st = self.state.lock();
8311 let mut local = st.local_address.clone();
8312 local.hostname = hostname.clone();
8313 local.owner_hash = Self::sender_hash(hostname.as_ref());
8314 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
8315 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
8316 local.requested_address = 0;
8317 }
8318 let change =
8319 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
8320 drop(st);
8321 let _ = self.notify_address_change(change);
8322 }
8323
8324 pub fn set_address_assignment(
8325 &self,
8326 mode: AddressAssignmentMode,
8327 ) -> TelemetryResult<AddressChange> {
8328 let mut st = self.state.lock();
8329 let mut local = st.local_address.clone();
8330 local.mode = mode;
8331 local.requested_address = mode.requested_address();
8332 local.address = match mode {
8333 AddressAssignmentMode::Dynamic => {
8334 Self::fallback_address_for_hostname(local.hostname.as_ref())
8335 }
8336 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
8337 if address == 0 {
8338 1
8339 } else {
8340 address
8341 }
8342 }
8343 };
8344 let change =
8345 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
8346 drop(st);
8347 self.notify_address_change(change.clone())?;
8348 Ok(change)
8349 }
8350
8351 pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
8360 where
8361 N: AsRef<str>,
8362 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
8363 {
8364 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
8365 }
8366
8367 pub fn add_side_packed_small_packets<N, F>(
8371 &self,
8372 name: N,
8373 tx: F,
8374 max_frame_bytes: usize,
8375 ) -> RouterSideId
8376 where
8377 N: AsRef<str>,
8378 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
8379 {
8380 self.add_side_packed_with_options(
8381 name,
8382 tx,
8383 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
8384 )
8385 }
8386
8387 pub fn add_side_packed_with_options<N, F>(
8396 &self,
8397 name: N,
8398 tx: F,
8399 opts: RouterSideOptions,
8400 ) -> RouterSideId
8401 where
8402 N: AsRef<str>,
8403 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
8404 {
8405 let mut st = self.state.lock();
8406 let side = Some(RouterSide {
8407 name: Arc::from(name.as_ref()),
8408 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
8409 opts,
8410 });
8411 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
8412 st.sides[id] = side;
8413 id
8414 } else {
8415 let id = st.sides.len();
8416 st.sides.push(side);
8417 id
8418 };
8419 st.side_runtime_stats
8420 .insert(id, SideRuntimeStatsInner::default());
8421 st.side_transport.insert(id, SideTransportState::default());
8422 #[cfg(feature = "discovery")]
8423 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8424 id
8425 }
8426
8427 pub fn add_side_packed_with_priority_and_options<N, F>(
8436 &self,
8437 name: N,
8438 tx: F,
8439 opts: RouterSideOptions,
8440 ) -> RouterSideId
8441 where
8442 N: AsRef<str>,
8443 F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
8444 {
8445 let mut st = self.state.lock();
8446 let side = Some(RouterSide {
8447 name: Arc::from(name.as_ref()),
8448 tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
8449 opts,
8450 });
8451 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
8452 st.sides[id] = side;
8453 id
8454 } else {
8455 let id = st.sides.len();
8456 st.sides.push(side);
8457 id
8458 };
8459 st.side_runtime_stats
8460 .insert(id, SideRuntimeStatsInner::default());
8461 st.side_transport.insert(id, SideTransportState::default());
8462 #[cfg(feature = "discovery")]
8463 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8464 id
8465 }
8466
8467 pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
8472 where
8473 N: AsRef<str>,
8474 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
8475 {
8476 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
8477 }
8478
8479 pub fn add_side_packet_with_options<N, F>(
8485 &self,
8486 name: N,
8487 tx: F,
8488 opts: RouterSideOptions,
8489 ) -> RouterSideId
8490 where
8491 N: AsRef<str>,
8492 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
8493 {
8494 let mut st = self.state.lock();
8495 let side = Some(RouterSide {
8496 name: Arc::from(name.as_ref()),
8497 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
8498 opts,
8499 });
8500 let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
8501 st.sides[id] = side;
8502 id
8503 } else {
8504 let id = st.sides.len();
8505 st.sides.push(side);
8506 id
8507 };
8508 st.side_runtime_stats
8509 .insert(id, SideRuntimeStatsInner::default());
8510 st.side_transport.insert(id, SideTransportState::default());
8511 #[cfg(feature = "discovery")]
8512 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8513 id
8514 }
8515
8516 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
8521 let now_ms = self.clock.now_ms();
8522 {
8523 let mut st = self.state.lock();
8524 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
8525 if slot.is_none() {
8526 return Err(TelemetryError::BadArg);
8527 }
8528 *slot = None;
8529 while st.sides.last().is_some_and(Option::is_none) {
8530 st.sides.pop();
8531 }
8532 st.route_overrides
8537 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
8538 st.typed_route_overrides
8539 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
8540 st.route_weights
8541 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
8542 st.route_priorities
8543 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
8544 st.source_route_modes.remove(&Some(side));
8545 st.route_selection_cursors.remove(&Some(side));
8546 st.adaptive_route_stats.remove(&side);
8547 #[cfg(feature = "discovery")]
8548 st.discovery_side_throttle.remove(&side);
8549 #[cfg(all(feature = "discovery", feature = "timesync"))]
8550 st.timesync_side_throttle.remove(&side);
8551 st.side_runtime_stats.remove(&side);
8552 st.side_transport.remove(&side);
8553 st.reliable_return_routes
8554 .retain(|_, route| route.side != side);
8555 st.transmit_queue.retain(
8556 |queued| {
8557 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
8558 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
8559 },
8560 );
8561 st.received_queue.retain(|queued| queued.src != Some(side));
8562 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
8563 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
8564 #[cfg(feature = "discovery")]
8565 {
8566 st.discovery_routes.remove(&side);
8567 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8568 }
8569 }
8570 let mut isr_rx = self.isr_rx_queue.try_lock()?;
8571 isr_rx.retain(|queued| queued.src != Some(side));
8572 Ok(())
8573 }
8574
8575 pub fn set_side_ingress_enabled(
8580 &self,
8581 side: RouterSideId,
8582 enabled: bool,
8583 ) -> TelemetryResult<()> {
8584 let now_ms = self.clock.now_ms();
8585 let mut st = self.state.lock();
8586 let side_ref = st
8587 .sides
8588 .get_mut(side)
8589 .and_then(|side| side.as_mut())
8590 .ok_or(TelemetryError::BadArg)?;
8591 side_ref.opts.ingress_enabled = enabled;
8592 #[cfg(feature = "discovery")]
8593 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8594 Ok(())
8595 }
8596
8597 pub fn set_side_egress_enabled(
8601 &self,
8602 side: RouterSideId,
8603 enabled: bool,
8604 ) -> TelemetryResult<()> {
8605 let now_ms = self.clock.now_ms();
8606 let mut st = self.state.lock();
8607 let side_ref = st
8608 .sides
8609 .get_mut(side)
8610 .and_then(|side| side.as_mut())
8611 .ok_or(TelemetryError::BadArg)?;
8612 side_ref.opts.egress_enabled = enabled;
8613 if !enabled {
8614 st.transmit_queue.retain(
8615 |queued| {
8616 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
8617 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
8618 },
8619 );
8620 }
8621 #[cfg(feature = "discovery")]
8622 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8623 Ok(())
8624 }
8625
8626 pub fn set_source_route_mode(
8632 &self,
8633 src: Option<RouterSideId>,
8634 mode: RouteSelectionMode,
8635 ) -> TelemetryResult<()> {
8636 let now_ms = self.clock.now_ms();
8637 let mut st = self.state.lock();
8638 if let Some(src) = src {
8639 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8640 }
8641 st.source_route_modes.insert(src, mode);
8645 st.route_selection_cursors.remove(&src);
8646 #[cfg(feature = "discovery")]
8647 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8648 Ok(())
8649 }
8650
8651 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
8653 let now_ms = self.clock.now_ms();
8654 let mut st = self.state.lock();
8655 if let Some(src) = src {
8656 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8657 }
8658 st.source_route_modes.remove(&src);
8659 st.route_selection_cursors.remove(&src);
8660 #[cfg(feature = "discovery")]
8661 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8662 Ok(())
8663 }
8664
8665 pub fn set_route_weight(
8670 &self,
8671 src: Option<RouterSideId>,
8672 dst: RouterSideId,
8673 weight: u32,
8674 ) -> TelemetryResult<()> {
8675 let now_ms = self.clock.now_ms();
8676 let mut st = self.state.lock();
8677 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8678 if let Some(src) = src {
8679 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8680 }
8681 st.route_weights.insert((src, dst), weight);
8682 st.route_selection_cursors.remove(&src);
8683 #[cfg(feature = "discovery")]
8684 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8685 Ok(())
8686 }
8687
8688 pub fn clear_route_weight(
8690 &self,
8691 src: Option<RouterSideId>,
8692 dst: RouterSideId,
8693 ) -> TelemetryResult<()> {
8694 let now_ms = self.clock.now_ms();
8695 let mut st = self.state.lock();
8696 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8697 if let Some(src) = src {
8698 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8699 }
8700 st.route_weights.remove(&(src, dst));
8701 st.route_selection_cursors.remove(&src);
8702 #[cfg(feature = "discovery")]
8703 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8704 Ok(())
8705 }
8706
8707 pub fn set_route_priority(
8712 &self,
8713 src: Option<RouterSideId>,
8714 dst: RouterSideId,
8715 priority: u32,
8716 ) -> TelemetryResult<()> {
8717 let now_ms = self.clock.now_ms();
8718 let mut st = self.state.lock();
8719 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8720 if let Some(src) = src {
8721 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8722 }
8723 st.route_priorities.insert((src, dst), priority);
8724 #[cfg(feature = "discovery")]
8725 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8726 Ok(())
8727 }
8728
8729 pub fn clear_route_priority(
8731 &self,
8732 src: Option<RouterSideId>,
8733 dst: RouterSideId,
8734 ) -> TelemetryResult<()> {
8735 let now_ms = self.clock.now_ms();
8736 let mut st = self.state.lock();
8737 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8738 if let Some(src) = src {
8739 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8740 }
8741 st.route_priorities.remove(&(src, dst));
8742 #[cfg(feature = "discovery")]
8743 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8744 Ok(())
8745 }
8746
8747 pub fn set_route(
8752 &self,
8753 src: Option<RouterSideId>,
8754 dst: RouterSideId,
8755 enabled: bool,
8756 ) -> TelemetryResult<()> {
8757 let now_ms = self.clock.now_ms();
8758 let mut st = self.state.lock();
8759 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8760 if let Some(src) = src {
8761 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8762 }
8763 st.route_overrides.insert((src, dst), enabled);
8764 #[cfg(feature = "discovery")]
8765 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8766 Ok(())
8767 }
8768
8769 pub fn set_typed_route(
8774 &self,
8775 src: Option<RouterSideId>,
8776 ty: DataType,
8777 dst: RouterSideId,
8778 enabled: bool,
8779 ) -> TelemetryResult<()> {
8780 let now_ms = self.clock.now_ms();
8781 let mut st = self.state.lock();
8782 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8783 if let Some(src) = src {
8784 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8785 }
8786 st.typed_route_overrides
8787 .insert((src, ty.as_u32(), dst), enabled);
8788 #[cfg(feature = "discovery")]
8789 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8790 Ok(())
8791 }
8792
8793 pub fn clear_typed_route(
8795 &self,
8796 src: Option<RouterSideId>,
8797 ty: DataType,
8798 dst: RouterSideId,
8799 ) -> TelemetryResult<()> {
8800 let now_ms = self.clock.now_ms();
8801 let mut st = self.state.lock();
8802 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8803 if let Some(src) = src {
8804 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8805 }
8806 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
8807 #[cfg(feature = "discovery")]
8808 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8809 Ok(())
8810 }
8811
8812 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
8814 let now_ms = self.clock.now_ms();
8815 let mut st = self.state.lock();
8816 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
8817 if let Some(src) = src {
8818 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
8819 }
8820 st.route_overrides.remove(&(src, dst));
8821 #[cfg(feature = "discovery")]
8822 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8823 Ok(())
8824 }
8825
8826 #[cfg(feature = "discovery")]
8828 pub fn announce_discovery(&self) -> TelemetryResult<()> {
8829 self.queue_discovery_announce(true, false)
8830 }
8831
8832 #[cfg(feature = "discovery")]
8834 pub fn announce_leave(&self) -> TelemetryResult<()> {
8835 let sender = self.sender_arc();
8836 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
8837 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
8838 }
8839
8840 #[cfg(feature = "discovery")]
8842 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
8843 self.poll_discovery_announce()
8844 }
8845
8846 #[cfg(feature = "discovery")]
8847 pub fn request_topology(&self) -> TelemetryResult<()> {
8848 let sender = self.sender_arc();
8849 let pkt =
8850 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
8851 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
8852 }
8853
8854 #[cfg(feature = "discovery")]
8855 pub fn request_schema(&self) -> TelemetryResult<()> {
8856 let sender = self.sender_arc();
8857 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
8858 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
8859 }
8860
8861 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
8868 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
8869 }
8870
8871 pub fn enable_network_variable(
8876 &self,
8877 ty: DataType,
8878 permissions: NetworkVariablePermissions,
8879 ) -> TelemetryResult<()> {
8880 if is_internal_control_type(ty) {
8881 return Err(TelemetryError::InvalidType);
8882 }
8883 let mut st = self.state.lock();
8884 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
8885 st.managed_variable_permissions
8886 .insert(ty.as_u32(), permissions);
8887 #[cfg(feature = "discovery")]
8888 if newly_advertised {
8889 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8890 }
8891 Ok(())
8892 }
8893
8894 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
8900 where
8901 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
8902 {
8903 if is_internal_control_type(ty) {
8904 return Err(TelemetryError::InvalidType);
8905 }
8906 let mut st = self.state.lock();
8907 let newly_advertised = st.managed_variable_types.insert(ty.as_u32());
8908 st.network_variable_update_handlers
8909 .entry(ty.as_u32())
8910 .or_default()
8911 .push(NetworkVariableUpdateHandler {
8912 handler: Arc::new(f),
8913 });
8914 #[cfg(feature = "discovery")]
8915 if newly_advertised {
8916 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8917 }
8918 Ok(())
8919 }
8920
8921 pub fn disable_managed_variable(&self, ty: DataType) {
8922 let mut st = self.state.lock();
8923 let was_advertised = st.managed_variable_types.remove(&ty.as_u32());
8924 st.managed_variable_permissions.remove(&ty.as_u32());
8925 st.managed_variable_latest.remove(&ty.as_u32());
8926 st.network_variable_update_handlers.remove(&ty.as_u32());
8927 #[cfg(feature = "discovery")]
8928 if was_advertised {
8929 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8930 }
8931 }
8932
8933 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
8934 if is_internal_control_type(pkt.data_type()) {
8935 return Err(TelemetryError::InvalidType);
8936 }
8937 pkt.validate()?;
8938 {
8939 let mut st = self.state.lock();
8940 let newly_advertised = st.managed_variable_types.insert(pkt.data_type().as_u32());
8941 #[cfg(feature = "discovery")]
8942 if newly_advertised {
8943 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
8944 }
8945 }
8946 self.cache_managed_variable_packet(&pkt, false)
8947 }
8948
8949 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
8950 self.managed_variable_latest(ty)
8951 }
8952
8953 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
8959 if is_internal_control_type(pkt.data_type()) {
8960 return Err(TelemetryError::InvalidType);
8961 }
8962 pkt.validate()?;
8963 let ty = pkt.data_type();
8964 {
8965 let mut st = self.state.lock();
8966 st.managed_variable_types.insert(ty.as_u32());
8967 let perms = Self::managed_variable_permissions_locked(&st, ty);
8968 if !perms.write {
8969 return Err(TelemetryError::PermissionDenied);
8970 }
8971 }
8972 self.cache_managed_variable_packet(&pkt, false)?;
8973 #[cfg(feature = "discovery")]
8974 let _ = self.poll_discovery()?;
8975 let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
8976 if self.side_tx_active() {
8977 return self.tx_queue_item_with_priority(
8978 item,
8979 false,
8980 crate::transport_priority(DataType::ManagedVariableValue),
8981 );
8982 }
8983 self.tx_item(item)
8984 }
8985
8986 #[cfg(feature = "discovery")]
8993 pub fn get_network_variable(
8994 &self,
8995 ty: DataType,
8996 stale_after_ms: Option<u64>,
8997 ) -> TelemetryResult<Option<Packet>> {
8998 if is_internal_control_type(ty) {
8999 return Err(TelemetryError::InvalidType);
9000 }
9001 {
9002 let mut st = self.state.lock();
9003 st.managed_variable_types.insert(ty.as_u32());
9004 let perms = Self::managed_variable_permissions_locked(&st, ty);
9005 if !perms.read {
9006 return Err(TelemetryError::PermissionDenied);
9007 }
9008 }
9009 let cached = self.managed_variable_latest_with_age(ty);
9010 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
9011 (None, _) => true,
9012 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
9013 (Some(_), None) => false,
9014 };
9015 if needs_refresh {
9016 self.request_managed_variable(ty)?;
9017 }
9018 Ok(cached.map(|(pkt, _age_ms)| pkt))
9019 }
9020
9021 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
9023 if is_internal_control_type(ty) {
9024 return Err(TelemetryError::InvalidType);
9025 }
9026 if !self.can_read_managed_variable(ty) {
9027 return Err(TelemetryError::PermissionDenied);
9028 }
9029 Ok(self.managed_variable_latest(ty))
9030 }
9031
9032 #[cfg(feature = "discovery")]
9033 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
9034 if is_internal_control_type(ty) {
9035 return Err(TelemetryError::InvalidType);
9036 }
9037 if !self.can_read_managed_variable(ty) {
9038 return Err(TelemetryError::PermissionDenied);
9039 }
9040 let sender = self.sender_arc();
9041 let pkt =
9042 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
9043 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
9044 }
9045
9046 #[cfg(feature = "discovery")]
9047 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
9048 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
9049 self.request_managed_variable(ty)
9050 }
9051
9052 #[cfg(feature = "discovery")]
9054 pub fn export_topology(&self) -> TopologySnapshot {
9055 let now_ms = self.clock.now_ms();
9056 let mut st = self.state.lock();
9057 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9058 Self::note_discovery_topology_change_locked(&mut st, now_ms);
9059 }
9060 let routes = st
9061 .discovery_routes
9062 .iter()
9063 .filter_map(|(&side_id, route)| {
9064 let side = st.sides.get(side_id)?.as_ref()?;
9065 let announcers = route
9066 .announcers
9067 .iter()
9068 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
9069 sender_id: sender_id.clone(),
9070 reachable_endpoints: sender_state
9071 .reachable
9072 .iter()
9073 .copied()
9074 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
9075 .collect(),
9076 reachable_network_variables: sender_state
9077 .reachable_network_variables
9078 .clone(),
9079 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
9080 routers: sender_state.topology_boards.clone(),
9081 last_seen_ms: sender_state.last_seen_ms,
9082 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
9083 })
9084 .collect();
9085 Some(TopologySideRoute {
9086 side_id,
9087 side_name: side.name.to_string(),
9088 reachable_endpoints: route
9089 .reachable
9090 .iter()
9091 .copied()
9092 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
9093 .collect(),
9094 reachable_network_variables: route.reachable_network_variables.clone(),
9095 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
9096 announcers,
9097 last_seen_ms: route.last_seen_ms,
9098 age_ms: now_ms.saturating_sub(route.last_seen_ms),
9099 })
9100 })
9101 .collect();
9102 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
9103 let advertised_endpoints =
9104 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
9105 let advertised_timesync_sources =
9106 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
9107 let links = discovery::topology_links_from_boards(&routers);
9108 TopologySnapshot {
9109 advertised_endpoints,
9110 advertised_timesync_sources,
9111 routers,
9112 links,
9113 routes,
9114 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
9115 next_announce_ms: st.discovery_cadence.next_announce_ms,
9116 }
9117 }
9118
9119 #[cfg(feature = "discovery")]
9120 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
9121 let now_ms = self.clock.now_ms();
9122 let st = self.state.lock();
9123 let mut side_ids = Vec::new();
9124 let mut side_names = Vec::new();
9125 let mut last_seen_ms = None::<u64>;
9126 let mut reachable_endpoints = Vec::new();
9127 let mut reachable_timesync_sources = Vec::new();
9128 let mut packets_sent = 0u64;
9129 let mut packets_received = 0u64;
9130 let mut bytes_sent = 0u64;
9131 let mut bytes_received = 0u64;
9132
9133 for (side_id, route) in &st.discovery_routes {
9134 let Some(sender_state) = route.announcers.get(sender_id) else {
9135 continue;
9136 };
9137 side_ids.push(*side_id);
9138 if let Some(side_name) = st
9139 .sides
9140 .get(*side_id)
9141 .and_then(|side| side.as_ref())
9142 .map(|side| side.name.clone())
9143 {
9144 side_names.push(side_name.to_string());
9145 }
9146 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
9147 reachable_endpoints.extend(sender_state.reachable.iter().copied());
9148 reachable_timesync_sources
9149 .extend(sender_state.reachable_timesync_sources.iter().cloned());
9150 if let Some(stats) = st.side_runtime_stats.get(side_id) {
9151 packets_sent = packets_sent.saturating_add(stats.tx_packets);
9152 packets_received = packets_received.saturating_add(stats.rx_packets);
9153 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
9154 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
9155 }
9156 }
9157
9158 if side_ids.is_empty() {
9159 return None;
9160 }
9161 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
9162 reachable_endpoints.sort_unstable();
9163 reachable_endpoints.dedup();
9164 reachable_timesync_sources.sort_unstable();
9165 reachable_timesync_sources.dedup();
9166 side_ids.sort_unstable();
9167 side_ids.dedup();
9168 side_names.sort_unstable();
9169 side_names.dedup();
9170 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
9171 Some(ClientStatsSnapshot {
9172 sender_id: sender_id.to_string(),
9173 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
9174 side_ids,
9175 side_names,
9176 last_seen_ms,
9177 age_ms,
9178 reachable_endpoints,
9179 reachable_timesync_sources,
9180 packets_sent,
9181 packets_received,
9182 bytes_sent,
9183 bytes_received,
9184 })
9185 }
9186
9187 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
9188 let now_ms = self.clock.now_ms();
9189 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
9190 let st = self.state.lock();
9191
9192 let mut sides = Vec::new();
9193 for (side_id, side) in st.sides.iter().enumerate() {
9194 let Some(side) = side.as_ref() else { continue };
9195 let stats = st
9196 .side_runtime_stats
9197 .get(&side_id)
9198 .cloned()
9199 .unwrap_or_default();
9200 let adaptive = st
9201 .adaptive_route_stats
9202 .get(&side_id)
9203 .cloned()
9204 .unwrap_or_default()
9205 .snapshot(now_ms, true);
9206 let (tx_template_count, rx_template_count) = st
9207 .side_transport
9208 .get(&side_id)
9209 .map(|state| (state.tx_template_count(), state.rx_template_count()))
9210 .unwrap_or((0, 0));
9211 let mut data_types: Vec<RuntimeTypeStats> = stats
9212 .data_types
9213 .into_iter()
9214 .map(|(ty, item)| RuntimeTypeStats {
9215 data_type: DataType(ty),
9216 tx_packets: item.tx_packets,
9217 tx_bytes: item.tx_bytes,
9218 rx_packets: item.rx_packets,
9219 rx_bytes: item.rx_bytes,
9220 relayed_tx_packets: item.relayed_tx_packets,
9221 relayed_tx_bytes: item.relayed_tx_bytes,
9222 relayed_rx_packets: item.relayed_rx_packets,
9223 relayed_rx_bytes: item.relayed_rx_bytes,
9224 tx_retries: item.tx_retries,
9225 handler_failures: item.handler_failures,
9226 })
9227 .collect();
9228 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
9229 sides.push(RuntimeSideStats {
9230 side_id,
9231 side_name: side.name.to_string(),
9232 reliable_enabled: side.opts.reliable_enabled,
9233 link_local_enabled: side.opts.link_local_enabled,
9234 header_template_enabled: side.opts.header_template_enabled,
9235 max_frame_bytes: side.opts.max_frame_bytes,
9236 compact_header_target_bytes: side.opts.compact_header_target_bytes,
9237 side_transport_profile: side.opts.effective_transport_profile().as_str(),
9238 ingress_enabled: side.opts.ingress_enabled,
9239 egress_enabled: side.opts.egress_enabled,
9240 tx_packets: stats.tx_packets,
9241 tx_bytes: stats.tx_bytes,
9242 rx_packets: stats.rx_packets,
9243 rx_bytes: stats.rx_bytes,
9244 relayed_tx_packets: stats.relayed_tx_packets,
9245 relayed_tx_bytes: stats.relayed_tx_bytes,
9246 relayed_rx_packets: stats.relayed_rx_packets,
9247 relayed_rx_bytes: stats.relayed_rx_bytes,
9248 local_delivery_packets: stats.local_delivery_packets,
9249 tx_retries: stats.tx_retries,
9250 tx_handler_failures: stats.tx_handler_failures,
9251 local_handler_failures: stats.local_handler_failures,
9252 total_handler_retries: stats.total_handler_retries,
9253 side_transport_full_frames: stats.side_transport_full_frames,
9254 side_transport_compact_frames: stats.side_transport_compact_frames,
9255 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
9256 side_transport_compact_omitted_timestamp_frames: stats
9257 .side_transport_compact_omitted_timestamp_frames,
9258 side_transport_chunk_frames: stats.side_transport_chunk_frames,
9259 side_transport_raw_bytes: stats.side_transport_raw_bytes,
9260 side_transport_wire_bytes: stats.side_transport_wire_bytes,
9261 side_transport_bytes_saved: stats.side_transport_bytes_saved,
9262 side_transport_min_compact_overhead_bytes: stats
9263 .side_transport_min_compact_overhead_bytes,
9264 side_transport_max_compact_overhead_bytes: stats
9265 .side_transport_max_compact_overhead_bytes,
9266 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
9267 side_transport_template_evictions: stats.side_transport_template_evictions,
9268 side_transport_tx_template_count: tx_template_count,
9269 side_transport_rx_template_count: rx_template_count,
9270 max_side_transport_templates: side.opts.max_side_transport_templates,
9271 adaptive,
9272 data_types,
9273 });
9274 }
9275
9276 let mut route_modes: Vec<RouteModeStats> = st
9277 .route_selection_cursors
9278 .iter()
9279 .map(|(src, cursor)| RouteModeStats {
9280 src_side_id: *src,
9281 selection_mode: st.source_route_modes.get(src).copied(),
9282 cursor: *cursor,
9283 })
9284 .collect();
9285 for src in st.source_route_modes.keys() {
9286 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
9287 route_modes.push(RouteModeStats {
9288 src_side_id: *src,
9289 selection_mode: st.source_route_modes.get(src).copied(),
9290 cursor: 0,
9291 });
9292 }
9293 }
9294 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
9295
9296 let mut route_overrides: Vec<RouteOverrideStats> = st
9297 .route_overrides
9298 .iter()
9299 .map(|((src, dst), enabled)| RouteOverrideStats {
9300 src_side_id: *src,
9301 dst_side_id: *dst,
9302 enabled: *enabled,
9303 })
9304 .collect();
9305 route_overrides.sort_unstable_by_key(|item| {
9306 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
9307 });
9308
9309 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
9310 .typed_route_overrides
9311 .iter()
9312 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
9313 src_side_id: *src,
9314 data_type: DataType(*ty),
9315 dst_side_id: *dst,
9316 enabled: *enabled,
9317 })
9318 .collect();
9319 typed_route_overrides.sort_unstable_by_key(|item| {
9320 (
9321 item.src_side_id.unwrap_or(usize::MAX),
9322 item.data_type.as_u32(),
9323 item.dst_side_id,
9324 )
9325 });
9326
9327 let mut route_weights: Vec<RouteWeightStats> = st
9328 .route_weights
9329 .iter()
9330 .map(|((src, dst), weight)| RouteWeightStats {
9331 src_side_id: *src,
9332 dst_side_id: *dst,
9333 weight: *weight,
9334 })
9335 .collect();
9336 route_weights.sort_unstable_by_key(|item| {
9337 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
9338 });
9339
9340 let mut route_priorities: Vec<RoutePriorityStats> = st
9341 .route_priorities
9342 .iter()
9343 .map(|((src, dst), priority)| RoutePriorityStats {
9344 src_side_id: *src,
9345 dst_side_id: *dst,
9346 priority: *priority,
9347 })
9348 .collect();
9349 route_priorities.sort_unstable_by_key(|item| {
9350 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
9351 });
9352
9353 #[cfg(feature = "discovery")]
9354 let discovery = DiscoveryRuntimeStats {
9355 route_count: st.discovery_routes.len(),
9356 announcer_count: st
9357 .discovery_routes
9358 .values()
9359 .map(|route| route.announcers.len())
9360 .sum(),
9361 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
9362 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
9363 };
9364 #[cfg(not(feature = "discovery"))]
9365 let discovery = DiscoveryRuntimeStats {
9366 route_count: 0,
9367 announcer_count: 0,
9368 current_announce_interval_ms: None,
9369 next_announce_ms: None,
9370 };
9371
9372 RuntimeStatsSnapshot {
9373 sides,
9374 route_modes,
9375 route_overrides,
9376 typed_route_overrides,
9377 route_weights,
9378 route_priorities,
9379 queues: QueueRuntimeStats {
9380 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
9381 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
9382 tx_len: st.transmit_queue.len(),
9383 tx_bytes: st.transmit_queue.bytes_used(),
9384 replay_len: 0,
9385 replay_bytes: 0,
9386 recent_rx_len: st.recent_rx.len(),
9387 recent_rx_bytes: st.recent_rx.bytes_used(),
9388 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
9389 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
9390 shared_queue_bytes_used: st.shared_queue_bytes_used(),
9391 },
9392 reliable: ReliableRuntimeStats {
9393 reliable_return_route_count: st.reliable_return_routes.len(),
9394 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
9395 end_to_end_pending_destination_count: st
9396 .end_to_end_reliable_tx
9397 .values()
9398 .map(|sent| sent.pending_destinations.len())
9399 .sum(),
9400 end_to_end_acked_cache_count: 0,
9401 },
9402 discovery,
9403 total_handler_failures: st.total_handler_failures,
9404 total_handler_retries: st.total_handler_retries,
9405 }
9406 }
9407
9408 pub fn export_memory_layout_json(&self) -> String {
9410 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
9411 let st = self.state.lock();
9412 #[cfg(feature = "discovery")]
9413 let discovery_bytes = st.discovery_bytes_used();
9414 #[cfg(not(feature = "discovery"))]
9415 let discovery_bytes = 0usize;
9416 let schema_bytes = crate::config::schema_bytes_used();
9417 let network_variable_cache_bytes = st
9418 .managed_variable_latest
9419 .values()
9420 .map(|entry| entry.packet.byte_cost())
9421 .sum::<usize>();
9422 let mut out = String::new();
9423 let memory = st.memory;
9424 let _ = fmt::Write::write_fmt(
9425 &mut out,
9426 format_args!(
9427 "{{\"kind\":\"router\",\
9428 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
9429 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
9430 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
9431 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
9432 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
9433 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
9434 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
9435 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
9436 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
9437 st.shared_queue_bytes_used(),
9438 memory.max_queue_budget,
9439 st.received_queue.bytes_used(),
9440 st.received_queue.max_bytes(),
9441 st.received_queue.len(),
9442 isr_rx.1,
9443 memory.max_queue_budget,
9444 isr_rx.0,
9445 st.transmit_queue.bytes_used(),
9446 st.transmit_queue.max_bytes(),
9447 st.transmit_queue.len(),
9448 st.recent_rx.bytes_used(),
9449 st.recent_rx.max_bytes(),
9450 st.recent_rx.len(),
9451 st.reliable_rx_buffered_bytes(),
9452 memory.max_queue_budget,
9453 st.reliable_rx_buffer_len(),
9454 discovery_bytes,
9455 memory.max_queue_budget,
9456 schema_bytes,
9457 memory.max_queue_budget,
9458 network_variable_cache_bytes,
9459 memory.max_queue_budget,
9460 st.managed_variable_latest.len(),
9461 ),
9462 );
9463 out
9464 }
9465
9466 #[cfg(test)]
9467 pub(crate) fn debug_end_to_end_pending_destination_count(
9468 &self,
9469 packet_id: u64,
9470 ) -> Option<usize> {
9471 let st = self.state.lock();
9472 st.end_to_end_reliable_tx
9473 .get(&packet_id)
9474 .map(|sent| sent.pending_destinations.len())
9475 }
9476
9477 #[cfg(test)]
9478 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
9479 let st = self.state.lock();
9480 st.end_to_end_reliable_tx.len()
9481 }
9482
9483 #[cfg(test)]
9484 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
9485 let st = self.state.lock();
9486 st.reliable_return_routes.len()
9487 }
9488
9489 #[cfg(test)]
9490 pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
9491 let st = self.state.lock();
9492 (st.sides.len(), st.sides.capacity())
9493 }
9494
9495 #[cfg(test)]
9496 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
9497 let st = self.state.lock();
9498 (
9499 st.received_queue.len(),
9500 st.transmit_queue.len(),
9501 st.recent_rx.len(),
9502 )
9503 }
9504
9505 #[cfg(test)]
9506 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
9507 let st = self.state.lock();
9508 st.shared_queue_bytes_used()
9509 }
9510
9511 #[cfg(test)]
9512 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
9513 let st = self.state.lock();
9514 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
9515 }
9516
9517 fn get_hash(item: &RouterItem) -> u64 {
9521 match item {
9522 RouterItem::Packet(pkt) => pkt.packet_id(),
9523 RouterItem::Packed(bytes) => {
9524 match wire_format::packet_id_from_wire(bytes.as_ref()) {
9525 Ok(id) => id,
9526 Err(_e) => {
9527 let h: u64 = 0x9E37_79B9_7F4A_7C15;
9530 hash_bytes_u64(h, bytes.as_ref())
9531 }
9532 }
9533 }
9534 }
9535 }
9536
9537 fn remove_pkt_id(&self, item: &RouterItem) {
9539 let hash = Self::get_hash(item);
9540 let mut st = self.state.lock();
9541 st.recent_rx.remove_value(&hash);
9542 }
9543
9544 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
9547 let id = Self::get_hash(item);
9548 let mut st = self.state.lock();
9549 if st.recent_rx.contains(&id) {
9550 Ok(true)
9551 } else {
9552 st.push_recent_rx(id)?;
9553 Ok(false)
9554 }
9555 }
9556
9557 fn handle_callback_error(
9562 &self,
9563 pkt: &Packet,
9564 dest: Option<DataEndpoint>,
9565 e: TelemetryError,
9566 called_from_queue: bool,
9567 ) -> TelemetryResult<()> {
9568 let device = self.sender_arc();
9569 let error_msg = match dest {
9570 Some(failed_local) => format!(
9571 "Handler for endpoint {:?} failed on device {:?}: {:?}",
9572 failed_local, device, e
9573 ),
9574 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
9575 };
9576
9577 let mut recipients: Vec<DataEndpoint> = pkt
9578 .endpoints()
9579 .iter()
9580 .copied()
9581 .filter(|&ep| self.cfg.is_local_endpoint(ep))
9582 .collect();
9583 recipients.sort_unstable();
9584 recipients.dedup();
9585
9586 if let Some(failed_local) = dest {
9587 recipients.retain(|&ep| ep != failed_local);
9588 }
9589
9590 if recipients.is_empty() && dest.is_none() {
9594 fallback_stdout(&error_msg);
9595 return Ok(());
9596 }
9597
9598 if recipients.is_empty() {
9601 recipients = pkt.endpoints().to_vec();
9602 recipients.sort_unstable();
9603 recipients.dedup();
9604 if let Some(failed_local) = dest {
9605 recipients.retain(|&ep| ep != failed_local);
9606 }
9607 }
9608
9609 if recipients.is_empty() {
9610 fallback_stdout(&error_msg);
9611 return Ok(());
9612 }
9613
9614 let payload = make_error_payload(&error_msg);
9615
9616 let sender = self.sender_arc();
9617 let error_pkt = Packet::new(
9618 DataType::TelemetryError,
9619 &recipients,
9620 sender.as_ref(),
9621 self.packet_timestamp_ms(),
9622 payload,
9623 )?;
9624
9625 self.emit_internal_tx(
9626 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
9627 false,
9628 called_from_queue,
9629 )
9630 }
9631
9632 #[inline]
9636 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
9637 self.process_tx_queue_with_timeout(0)
9638 }
9639
9640 #[inline]
9642 pub fn process_all_queues(&self) -> TelemetryResult<()> {
9643 self.process_all_queues_with_timeout(0)
9644 }
9645
9646 #[inline]
9648 pub fn clear_queues(&self) {
9649 let mut st = self.state.lock();
9650 st.transmit_queue.clear();
9651 st.received_queue.clear();
9652 drop(st);
9653 let _ = self.isr_rx_queue.clear();
9654 }
9655
9656 #[inline]
9658 pub fn clear_rx_queue(&self) {
9659 let mut st = self.state.lock();
9660 st.received_queue.clear();
9661 drop(st);
9662 let _ = self.isr_rx_queue.clear();
9663 }
9664
9665 #[inline]
9667 pub fn clear_tx_queue(&self) {
9668 let mut st = self.state.lock();
9669 st.transmit_queue.clear();
9670 st.tx_priority_burst = 0;
9671 }
9672
9673 fn pop_transmit_queue_locked(st: &mut RouterInner) -> Option<TxQueued> {
9674 st.transmit_queue
9675 .pop_front_fair(&mut st.tx_priority_burst, MAX_PRIORITY_BURST, |item| {
9676 item.priority
9677 })
9678 }
9679
9680 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
9683 let start = self.clock.now_ms();
9684 loop {
9685 self.process_reliable_timeouts()?;
9686 self.process_end_to_end_reliable_timeouts()?;
9687 #[cfg(feature = "discovery")]
9688 let _ = self.drain_queued_discovery_rx_before_tx()?;
9689 let pkt_opt = {
9690 let mut st = self.state.lock();
9691 Self::pop_transmit_queue_locked(&mut st)
9692 };
9693 let Some(pkt) = pkt_opt else { break };
9694 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
9695 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
9696 break;
9697 }
9698 }
9699 Ok(())
9700 }
9701
9702 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
9705 #[cfg(feature = "timesync")]
9706 let _ = self.poll_timesync()?;
9707 #[cfg(feature = "discovery")]
9708 let _ = self.poll_discovery()?;
9709 self.process_tx_queue_with_timeout_impl(timeout_ms)
9710 }
9711
9712 #[inline]
9714 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
9715 self.rx_item(&item, true)
9716 }
9717
9718 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
9721 let start = self.clock.now_ms();
9722 loop {
9723 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
9724 let mut st = self.state.lock();
9725 st.received_queue.pop_front()
9726 });
9727 let Some(item) = item_opt else { break };
9728 self.process_rx_queue_item(item)?;
9729 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
9730 break;
9731 }
9732 }
9733 Ok(())
9734 }
9735
9736 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
9739 #[cfg(feature = "timesync")]
9740 let _ = self.poll_timesync()?;
9741 #[cfg(feature = "discovery")]
9742 let _ = self.poll_discovery()?;
9743 self.process_rx_queue_with_timeout_impl(timeout_ms)
9744 }
9745
9746 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
9749 if timeout_ms == 0 {
9750 loop {
9751 let mut did_any = false;
9752 self.process_reliable_timeouts()?;
9753 self.process_end_to_end_reliable_timeouts()?;
9754 #[cfg(feature = "discovery")]
9755 if self.drain_queued_discovery_rx_before_tx()? {
9756 did_any = true;
9757 }
9758
9759 if let Some(pkt) = {
9760 let mut st = self.state.lock();
9761 Self::pop_transmit_queue_locked(&mut st)
9762 } {
9763 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
9764 did_any = true;
9765 }
9766
9767 if let Some(item) = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
9768 let mut st = self.state.lock();
9769 st.received_queue.pop_front()
9770 }) {
9771 self.process_rx_queue_item(item)?;
9772 did_any = true;
9773 }
9774
9775 if !did_any {
9776 break;
9777 }
9778 }
9779 return Ok(());
9780 }
9781
9782 let tx_budget_ms = u64::from(timeout_ms / 2);
9783 let rx_budget_ms = u64::from(timeout_ms) - tx_budget_ms;
9784
9785 let tx_start = self.clock.now_ms();
9786 loop {
9787 self.process_reliable_timeouts()?;
9788 self.process_end_to_end_reliable_timeouts()?;
9789 #[cfg(feature = "discovery")]
9790 let _ = self.drain_queued_discovery_rx_before_tx()?;
9791 let pkt_opt = {
9792 let mut st = self.state.lock();
9793 Self::pop_transmit_queue_locked(&mut st)
9794 };
9795 let Some(pkt) = pkt_opt else { break };
9796 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
9797 if self.clock.now_ms().wrapping_sub(tx_start) >= tx_budget_ms {
9798 break;
9799 }
9800 }
9801
9802 let rx_start = self.clock.now_ms();
9803 loop {
9804 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
9805 let mut st = self.state.lock();
9806 st.received_queue.pop_front()
9807 });
9808 let Some(item) = item_opt else { break };
9809 self.process_rx_queue_item(item)?;
9810 if self.clock.now_ms().wrapping_sub(rx_start) >= rx_budget_ms {
9811 break;
9812 }
9813 }
9814
9815 Ok(())
9816 }
9817
9818 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
9821 #[cfg(feature = "timesync")]
9822 let _ = self.poll_timesync()?;
9823 #[cfg(feature = "discovery")]
9824 let _ = self.poll_discovery()?;
9825 self.process_all_queues_with_timeout_impl(timeout_ms)
9826 }
9827
9828 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
9833 #[cfg(feature = "timesync")]
9834 let _ = self.poll_timesync()?;
9835
9836 #[cfg(feature = "discovery")]
9837 {
9838 let _ = self.poll_discovery()?;
9839 }
9840
9841 self.process_all_queues_with_timeout_impl(timeout_ms)
9842 }
9843
9844 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
9849 #[cfg(feature = "discovery")]
9850 {
9851 let _ = self.poll_discovery()?;
9852 }
9853
9854 self.process_all_queues_with_timeout_impl(timeout_ms)
9855 }
9856
9857 #[inline]
9859 fn tx_queue_item_with_flags(
9860 &self,
9861 item: RouterTxItem,
9862 ignore_local: bool,
9863 ) -> TelemetryResult<()> {
9864 let priority = match &item {
9865 RouterTxItem::Broadcast(data) => Self::router_item_priority(data)?,
9866 RouterTxItem::EndToEndReplay { .. } => {
9867 Self::router_item_priority_bumped(DataType::ReliableAck)
9868 }
9869 RouterTxItem::ToSide { data, .. } => Self::router_item_priority(data)?,
9870 RouterTxItem::ReliableReplay { bytes, .. } => {
9871 let ty = wire_format::peek_envelope(bytes.as_ref())?.ty;
9872 Self::router_item_priority_bumped(ty)
9873 }
9874 };
9875 self.tx_queue_item_with_priority(item, ignore_local, priority)
9876 }
9877
9878 #[inline]
9879 fn tx_queue_item_with_priority(
9880 &self,
9881 item: RouterTxItem,
9882 ignore_local: bool,
9883 priority: u8,
9884 ) -> TelemetryResult<()> {
9885 let mut st = self.state.lock();
9886 st.push_transmit(TxQueued {
9887 item,
9888 ignore_local,
9889 priority,
9890 })?;
9891 Ok(())
9892 }
9893
9894 #[inline]
9896 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
9897 self.tx_queue_item_with_flags(item, false)
9898 }
9899
9900 #[inline]
9901 fn try_enter_side_tx(&self) -> Option<crate::lock::ReentryGuard<'_>> {
9902 self.side_tx_gate.try_enter()
9903 }
9904
9905 #[inline]
9906 fn side_tx_active(&self) -> bool {
9907 self.side_tx_gate.is_active()
9908 }
9909
9910 #[inline]
9914 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
9915 self.process_rx_queue_with_timeout(0)
9916 }
9917
9918 #[inline]
9920 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
9921 let data = RouterItem::Packed(Arc::from(bytes));
9922 let priority = Self::router_item_priority(&data)?;
9923 let mut st = self.state.lock();
9924 st.push_received(RouterRxItem {
9925 src: None,
9926 data,
9927 priority,
9928 })?;
9929 Ok(())
9930 }
9931
9932 #[inline]
9937 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
9938 let data = RouterItem::Packed(Arc::from(bytes));
9939 let priority = Self::router_item_priority(&data)?;
9940 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
9941 src: None,
9942 data,
9943 priority,
9944 })
9945 }
9946
9947 #[inline]
9949 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
9950 pkt.validate()?;
9951 let data = RouterItem::Packet(pkt);
9952 let priority = Self::router_item_priority(&data)?;
9953 let mut st = self.state.lock();
9954 st.push_received(RouterRxItem {
9955 src: None,
9956 data,
9957 priority,
9958 })?;
9959 Ok(())
9960 }
9961
9962 #[inline]
9967 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
9968 pkt.validate()?;
9969 let data = RouterItem::Packet(pkt);
9970 let priority = Self::router_item_priority(&data)?;
9971 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
9972 src: None,
9973 data,
9974 priority,
9975 })
9976 }
9977
9978 #[inline]
9980 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
9981 self.ensure_side_ingress_enabled(side)?;
9982 pkt.validate()?;
9983 let data = RouterItem::Packet(pkt);
9984 let priority = Self::router_item_priority(&data)?;
9985 let mut st = self.state.lock();
9986 st.push_received(RouterRxItem {
9987 src: Some(side),
9988 data,
9989 priority,
9990 })?;
9991 Ok(())
9992 }
9993
9994 #[inline]
9999 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
10000 self.ensure_side_ingress_enabled(side)?;
10001 pkt.validate()?;
10002 let data = RouterItem::Packet(pkt);
10003 let priority = Self::router_item_priority(&data)?;
10004 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
10005 src: Some(side),
10006 data,
10007 priority,
10008 })
10009 }
10010
10011 #[inline]
10013 pub fn rx_packed_queue_from_side(
10014 &self,
10015 bytes: &[u8],
10016 side: RouterSideId,
10017 ) -> TelemetryResult<()> {
10018 self.ensure_side_ingress_enabled(side)?;
10019 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
10020 return Ok(());
10021 };
10022 let data = RouterItem::Packed(decoded);
10023 let priority = Self::router_item_priority(&data)?;
10024 let mut st = self.state.lock();
10025 st.push_received(RouterRxItem {
10026 src: Some(side),
10027 data,
10028 priority,
10029 })?;
10030 Ok(())
10031 }
10032
10033 #[inline]
10038 pub fn rx_packed_queue_from_side_isr(
10039 &self,
10040 bytes: &[u8],
10041 side: RouterSideId,
10042 ) -> TelemetryResult<()> {
10043 self.ensure_side_ingress_enabled(side)?;
10044 let data = RouterItem::Packed(Arc::from(bytes));
10045 let priority = Self::router_item_priority(&data)?;
10046 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
10047 src: Some(side),
10048 data,
10049 priority,
10050 })
10051 }
10052
10053 fn retry_with_attempts<F, T, E>(&self, times: usize, f: F) -> Result<(T, usize), (E, usize)>
10054 where
10055 F: Fn() -> Result<T, E>,
10056 {
10057 let mut last_err = None;
10058 for attempt in 0..times {
10059 match f() {
10060 Ok(v) => return Ok((v, attempt + 1)),
10061 Err(e) => last_err = Some((e, attempt + 1)),
10062 }
10063 }
10064 Err(last_err.expect("times > 0"))
10065 }
10066
10067 #[inline]
10069 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
10070 self.cfg
10071 .handlers
10072 .iter()
10073 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
10074 }
10075
10076 #[inline]
10078 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
10079 self.cfg
10080 .handlers
10081 .iter()
10082 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
10083 }
10084
10085 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
10086 pkt.endpoints()
10087 .iter()
10088 .copied()
10089 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
10090 }
10091
10092 fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
10099 self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
10100 }
10101
10102 fn call_handler_with_retries(
10108 &self,
10109 dest: DataEndpoint,
10110 handler: &EndpointHandler,
10111 data: Option<&[u8]>,
10112 pkt_for_ctx: Option<&Packet>,
10113 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
10114 called_from_queue: bool,
10115 ) -> TelemetryResult<()> {
10116 let owned_tmp: Option<RouterItem>;
10117
10118 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
10119 (Some(d), _) => {
10120 owned_tmp = Some(RouterItem::Packed(Arc::from(d)));
10121 owned_tmp.as_ref().unwrap()
10122 }
10123 (None, Some(pkt)) => {
10124 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
10125 owned_tmp.as_ref().unwrap()
10126 }
10127 (None, None) => {
10128 debug_assert!(
10129 false,
10130 "call_handler_with_retries called without data or packet context"
10131 );
10132 return Ok(());
10133 }
10134 };
10135
10136 match (&handler.handler, data) {
10137 (EndpointHandlerFn::Packet(f), _) => {
10138 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
10139 with_retries(
10140 self,
10141 dest,
10142 item_for_ctx,
10143 pkt_for_ctx,
10144 env_for_ctx,
10145 called_from_queue,
10146 || f(pkt),
10147 )
10148 }
10149
10150 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
10151 self,
10152 dest,
10153 item_for_ctx,
10154 pkt_for_ctx,
10155 env_for_ctx,
10156 called_from_queue,
10157 || f(bytes),
10158 ),
10159
10160 (EndpointHandlerFn::Packed(_), None) => Ok(()),
10161 }
10162 }
10163
10164 fn handle_callback_error_from_env(
10169 &self,
10170 env: &wire_format::TelemetryEnvelope,
10171 dest: Option<DataEndpoint>,
10172 e: TelemetryError,
10173 called_from_queue: bool,
10174 ) -> TelemetryResult<()> {
10175 let mut recipients: Vec<DataEndpoint> = env
10176 .endpoints
10177 .iter()
10178 .copied()
10179 .filter(|&ep| self.cfg.is_local_endpoint(ep))
10180 .collect();
10181 recipients.sort_unstable();
10182 recipients.dedup();
10183 if let Some(failed) = dest {
10184 recipients.retain(|&ep| ep != failed);
10185 }
10186
10187 let device = self.sender_arc();
10188 let error_msg = format!(
10189 "Handler for endpoint {:?} failed on device {:?}: {:?}",
10190 dest, device, e
10191 );
10192
10193 if recipients.is_empty() && dest.is_none() {
10194 fallback_stdout(&error_msg);
10195 return Ok(());
10196 }
10197
10198 if recipients.is_empty() {
10199 recipients = env.endpoints.to_vec();
10200 recipients.sort_unstable();
10201 recipients.dedup();
10202 if let Some(failed) = dest {
10203 recipients.retain(|&ep| ep != failed);
10204 }
10205 }
10206
10207 if recipients.is_empty() {
10208 fallback_stdout(&error_msg);
10209 return Ok(());
10210 }
10211
10212 let payload = make_error_payload(&error_msg);
10213
10214 let error_pkt = Packet::new(
10215 DataType::TelemetryError,
10216 &recipients,
10217 &env.sender.clone(),
10218 env.timestamp_ms,
10219 payload,
10220 )?;
10221 self.emit_internal_tx(
10222 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
10223 false,
10224 called_from_queue,
10225 )
10226 }
10227
10228 fn handle_internal_reliable_packet(
10229 &self,
10230 pkt: &Packet,
10231 src: Option<RouterSideId>,
10232 called_from_queue: bool,
10233 ) -> TelemetryResult<bool> {
10234 if !matches!(
10235 pkt.data_type(),
10236 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
10237 ) {
10238 return Ok(false);
10239 }
10240
10241 let Some(src) = src else {
10242 return Ok(false);
10243 };
10244
10245 if Self::is_end_to_end_ack_packet(pkt)
10246 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
10247 {
10248 let targets_local_sender = pkt.wire_target_senders().is_empty()
10255 || pkt
10256 .wire_target_senders()
10257 .contains(&Self::sender_hash(self.sender_arc().as_ref()));
10258 let mut st = self.state.lock();
10259 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
10260 if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
10261 sent.pending_destinations.remove(&sender_hash);
10262 if sent.pending_destinations.is_empty() {
10263 st.end_to_end_reliable_tx.remove(&packet_id);
10264 }
10265 return Ok(targets_local_sender);
10266 }
10267 st.end_to_end_reliable_tx.remove(&packet_id);
10268 return Ok(targets_local_sender);
10269 }
10270 return Ok(false);
10271 }
10272
10273 let vals = pkt.data_as_u32()?;
10274 if vals.len() != 2 {
10275 return Err(TelemetryError::Unpack("bad reliable control payload"));
10276 }
10277 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
10278 let seq = vals[1];
10279
10280 match pkt.data_type() {
10281 DataType::ReliableAck => {
10282 self.handle_reliable_ack(src, ty, seq);
10283 Ok(true)
10284 }
10285 DataType::ReliablePartialAck => {
10286 self.handle_reliable_partial_ack(src, ty, seq);
10287 Ok(true)
10288 }
10289 DataType::ReliablePacketRequest => {
10290 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
10291 Ok(true)
10292 }
10293 _ => Ok(false),
10294 }
10295 }
10296
10297 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
10303 if let Some(src) = item.src {
10304 self.ensure_side_ingress_enabled(src)?;
10305 match &item.data {
10306 RouterItem::Packet(pkt) => {
10307 let bytes = wire_format::pack_packet(pkt).len();
10308 self.note_side_rx(src, pkt.data_type(), bytes, true);
10309 }
10310 RouterItem::Packed(bytes) => {
10311 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
10312 self.note_side_rx(src, env.ty, bytes.len(), true);
10313 }
10314 }
10315 }
10316 match &item.data {
10317 RouterItem::Packet(pkt) => {
10318 if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
10319 && !is_internal_control_type(pkt.data_type())
10320 {
10321 self.note_reliable_return_route(src, pkt.packet_id());
10322 }
10323 }
10324 RouterItem::Packed(bytes) => {
10325 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref())
10326 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
10327 && !is_internal_control_type(env.ty)
10328 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
10329 {
10330 self.note_reliable_return_route(src, packet_id);
10331 }
10332 }
10333 }
10334 }
10335 match &item.data {
10336 RouterItem::Packet(pkt) => {
10337 if !is_internal_control_type(pkt.data_type()) {
10338 self.remember_managed_variable_packet(pkt)?;
10339 }
10340 }
10341 RouterItem::Packed(bytes) => {
10342 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref())
10343 && !is_internal_control_type(env.ty)
10344 && self.is_managed_variable_type(env.ty)
10345 {
10346 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10347 pkt.validate()?;
10348 self.remember_managed_variable_packet(&pkt)?;
10349 }
10350 }
10351 }
10352 let mut released_buffered: Vec<Arc<[u8]>> = Vec::new();
10353 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
10354 let (_opts, handler_is_packed, hop_reliable_enabled) = {
10355 let st = self.state.lock();
10356 let side_ref = Self::side_ref(&st, src)?;
10357 let opts = side_ref.opts;
10358 (
10359 opts,
10360 matches!(
10361 side_ref.tx_handler,
10362 RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
10363 ),
10364 opts.reliable_enabled
10365 && self.cfg.reliable_enabled()
10366 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
10367 )
10368 };
10369
10370 if hop_reliable_enabled && handler_is_packed {
10371 let frame = match wire_format::peek_frame_info(bytes.as_ref()) {
10372 Ok(frame) => frame,
10373 Err(e) => {
10374 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
10375 if let Ok(frame) =
10376 wire_format::peek_frame_info_unchecked(bytes.as_ref())
10377 && is_reliable_type(frame.envelope.ty)
10378 && let Some(hdr) = frame.reliable
10379 {
10380 let unordered =
10381 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
10382 let unsequenced =
10383 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
10384
10385 if !unsequenced {
10386 let requested = if unordered {
10387 hdr.seq
10388 } else {
10389 let mut st = self.state.lock();
10390 let rx_state = self.reliable_rx_state_mut(
10391 &mut st,
10392 src,
10393 frame.envelope.ty,
10394 );
10395 rx_state.expected_seq.min(hdr.seq)
10396 };
10397 self.queue_reliable_packet_request(
10398 src,
10399 frame.envelope.ty,
10400 requested,
10401 called_from_queue,
10402 )?;
10403 }
10404 }
10405 return Ok(());
10406 }
10407 return Err(e);
10408 }
10409 };
10410 if is_reliable_type(frame.envelope.ty)
10411 && let Some(hdr) = frame.reliable
10412 {
10413 if frame.ack_only() {
10414 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
10415 return Ok(());
10416 }
10417 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
10418 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
10419
10420 if !unsequenced {
10421 if unordered {
10422 self.queue_reliable_ack(
10423 src,
10424 frame.envelope.ty,
10425 hdr.seq,
10426 called_from_queue,
10427 )?;
10428 } else {
10429 let mut release: Vec<Arc<[u8]>> = Vec::new();
10430 let mut last_delivered = None;
10431 let mut ack_old = None;
10432 let mut request_missing = None;
10433 let mut partial_ack = None;
10434 {
10435 let mut st = self.state.lock();
10436 let rx_state =
10437 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
10438 let expected_seq = rx_state.expected_seq;
10439 if hdr.seq < expected_seq {
10440 ack_old = Some(expected_seq.saturating_sub(1));
10441 } else if hdr.seq > expected_seq {
10442 request_missing = Some(expected_seq);
10443 partial_ack = Some(hdr.seq);
10444 st.buffer_reliable_rx(
10445 src,
10446 frame.envelope.ty,
10447 hdr.seq,
10448 bytes.clone(),
10449 )?;
10450 } else {
10451 release.push(bytes.clone());
10452 last_delivered = Some(hdr.seq);
10453 let mut next_expected = hdr.seq.wrapping_add(1);
10454 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
10455 release.push(buf);
10456 last_delivered = Some(next_expected);
10457 let next = next_expected.wrapping_add(1);
10458 next_expected = if next == 0 { 1 } else { next };
10459 }
10460 rx_state.expected_seq = next_expected;
10461 }
10462 }
10463
10464 if let Some(ack_seq) = ack_old {
10465 self.queue_reliable_ack(
10466 src,
10467 frame.envelope.ty,
10468 ack_seq,
10469 called_from_queue,
10470 )?;
10471 return Ok(());
10472 }
10473 if let Some(request_seq) = request_missing {
10474 if let Some(partial_seq) = partial_ack {
10475 self.queue_reliable_partial_ack(
10476 src,
10477 frame.envelope.ty,
10478 partial_seq,
10479 called_from_queue,
10480 )?;
10481 }
10482 self.queue_reliable_packet_request(
10483 src,
10484 frame.envelope.ty,
10485 request_seq,
10486 called_from_queue,
10487 )?;
10488 return Ok(());
10489 }
10490
10491 if let Some(ack_seq) = last_delivered {
10492 self.queue_reliable_ack(
10493 src,
10494 frame.envelope.ty,
10495 ack_seq,
10496 called_from_queue,
10497 )?;
10498 }
10499
10500 released_buffered.extend(release.into_iter().skip(1));
10501 }
10502 }
10503 }
10504 } else {
10505 match wire_format::peek_frame_info(bytes.as_ref()) {
10506 Ok(frame) => {
10507 if frame.ack_only() {
10508 return Ok(());
10509 }
10510 }
10511 Err(e) => {
10512 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
10513 return Ok(());
10514 }
10515 return Err(e);
10516 }
10517 }
10518 }
10519 }
10520
10521 if self.is_duplicate_pkt(&item.data)? {
10522 if item.src.is_some() {
10523 let local_sender = self.sender_arc();
10524 match &item.data {
10525 RouterItem::Packet(pkt)
10526 if (is_reliable_type(pkt.data_type())
10527 || !pkt.wire_target_senders().is_empty())
10528 && pkt.sender() != local_sender.as_ref()
10529 && self.item_targets_local_sender(&item.data)?
10530 && self.packet_has_local_delivery(pkt) =>
10531 {
10532 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
10533 }
10534 RouterItem::Packed(bytes) => {
10535 if let Ok(pkt) = wire_format::unpack_packet(bytes.as_ref())
10536 && (is_reliable_type(pkt.data_type())
10537 || !pkt.wire_target_senders().is_empty())
10538 && pkt.sender() != local_sender.as_ref()
10539 && self.item_targets_local_sender(&item.data)?
10540 && self.packet_has_local_delivery(&pkt)
10541 {
10542 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
10543 .unwrap_or_else(|_| pkt.packet_id());
10544 self.queue_end_to_end_reliable_ack_for_packet_id(
10545 packet_id,
10546 called_from_queue,
10547 )?;
10548 }
10549 }
10550 _ => {}
10551 }
10552 }
10553 return Ok(());
10554 }
10555
10556 self.dispatch_rx_data(item, called_from_queue)?;
10557
10558 for release_bytes in released_buffered {
10559 let release_data = RouterItem::Packed(release_bytes.clone());
10560 if self.is_duplicate_pkt(&release_data)? {
10561 continue;
10562 }
10563 let release_item = RouterRxItem {
10564 src: item.src,
10565 priority: Self::router_item_priority(&release_data)?,
10566 data: release_data,
10567 };
10568 self.dispatch_rx_data(&release_item, called_from_queue)?;
10569 }
10570
10571 Ok(())
10572 }
10573
10574 fn dispatch_rx_data(
10575 &self,
10576 item: &RouterRxItem,
10577 called_from_queue: bool,
10578 ) -> TelemetryResult<()> {
10579 match &item.data {
10580 RouterItem::Packet(pkt) => {
10581 pkt.validate()?;
10582
10583 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
10584 return Ok(());
10585 }
10586
10587 if pkt.data_type() == DataType::P2pMessage {
10588 if self.item_targets_local_sender(&item.data)? {
10589 self.dispatch_p2p_packet(pkt)?;
10590 if item.src.is_some() {
10591 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
10592 }
10593 }
10594 if self.should_route_remote(&item.data, item.src)? {
10595 self.relay_send(
10596 RouterItem::Packet(pkt.to_owned()),
10597 item.src,
10598 called_from_queue,
10599 )?;
10600 }
10601 return Ok(());
10602 }
10603
10604 #[cfg(feature = "timesync")]
10605 if matches!(
10606 pkt.data_type(),
10607 DataType::TimeSyncAnnounce
10608 | DataType::TimeSyncRequest
10609 | DataType::TimeSyncResponse
10610 ) {
10611 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
10612 return Ok(());
10613 }
10614
10615 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
10616 if !discovery_is_hop_local_advertisement(pkt.data_type())
10621 && self.should_route_remote(&item.data, item.src)?
10622 {
10623 self.relay_send(
10624 RouterItem::Packet(pkt.to_owned()),
10625 item.src,
10626 called_from_queue,
10627 )?;
10628 }
10629 return Ok(());
10630 }
10631
10632 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
10633 eps.sort_unstable();
10634 eps.dedup();
10635 let had_local_handler = eps.iter().copied().any(|ep| {
10636 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
10637 });
10638 let had_local_delivery =
10639 had_local_handler || self.is_managed_variable_type(pkt.data_type());
10640
10641 let has_remote = self.should_route_remote(&item.data, item.src)?;
10642 let targets_local = self.item_targets_local_sender(&item.data)?;
10643
10644 let has_packed_local = eps
10645 .iter()
10646 .copied()
10647 .any(|ep| self.endpoint_has_packed_handler(ep));
10648 let bytes_opt = if has_packed_local {
10649 Some(wire_format::pack_packet(pkt))
10650 } else {
10651 None
10652 };
10653
10654 if targets_local {
10655 for dest in eps {
10656 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
10657 let result = match (&h.handler, &bytes_opt) {
10658 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
10659 .call_handler_with_retries(
10660 dest,
10661 h,
10662 Some(bytes.as_ref()),
10663 Some(pkt),
10664 None,
10665 called_from_queue,
10666 ),
10667 (EndpointHandlerFn::Packed(_), None) => {
10668 let bytes = wire_format::pack_packet(pkt);
10669 self.call_handler_with_retries(
10670 dest,
10671 h,
10672 Some(bytes.as_ref()),
10673 Some(pkt),
10674 None,
10675 called_from_queue,
10676 )
10677 }
10678 (EndpointHandlerFn::Packet(_), _) => self
10679 .call_handler_with_retries(
10680 dest,
10681 h,
10682 None,
10683 Some(pkt),
10684 None,
10685 called_from_queue,
10686 ),
10687 };
10688 if result.is_err()
10689 && let Some(src) = item.src
10690 {
10691 self.note_side_local_handler_failure(
10692 src,
10693 pkt.data_type(),
10694 runtime_max_handler_retries(),
10695 );
10696 }
10697 }
10698 }
10699 }
10700
10701 if let Some(src) = item.src
10702 && had_local_delivery
10703 && targets_local
10704 {
10705 self.note_side_local_delivery(src, pkt.data_type());
10706 }
10707
10708 if item.src.is_some()
10709 && had_local_delivery
10710 && targets_local
10711 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
10712 {
10713 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
10714 }
10715
10716 if has_remote {
10717 let relay_item = RouterItem::Packet(pkt.to_owned());
10718 self.relay_send(relay_item, item.src, called_from_queue)?;
10719 }
10720
10721 Ok(())
10722 }
10723 RouterItem::Packed(bytes) => {
10724 let env = wire_format::peek_envelope(bytes.as_ref())?;
10725
10726 if matches!(
10727 env.ty,
10728 DataType::ReliableAck
10729 | DataType::ReliablePartialAck
10730 | DataType::ReliablePacketRequest
10731 ) {
10732 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10733 pkt.validate()?;
10734 let handled =
10735 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
10736 if !handled && self.should_route_remote(&item.data, item.src)? {
10739 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
10740 }
10741 return Ok(());
10742 }
10743
10744 if env.ty == DataType::P2pMessage {
10745 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10746 pkt.validate()?;
10747 if self.item_targets_local_sender(&item.data)? {
10748 self.dispatch_p2p_packet(&pkt)?;
10749 if item.src.is_some() {
10750 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
10751 .unwrap_or_else(|_| pkt.packet_id());
10752 self.queue_end_to_end_reliable_ack_for_packet_id(
10753 packet_id,
10754 called_from_queue,
10755 )?;
10756 }
10757 }
10758 if self.should_route_remote(&item.data, item.src)? {
10759 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
10760 }
10761 return Ok(());
10762 }
10763
10764 #[cfg(feature = "timesync")]
10765 if matches!(
10766 env.ty,
10767 DataType::TimeSyncAnnounce
10768 | DataType::TimeSyncRequest
10769 | DataType::TimeSyncResponse
10770 ) {
10771 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10772 pkt.validate()?;
10773 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
10774 return Ok(());
10775 }
10776
10777 #[cfg(feature = "discovery")]
10778 if discovery::is_discovery_type(env.ty) {
10779 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10780 pkt.validate()?;
10781 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
10782 if !discovery_is_hop_local_advertisement(env.ty)
10783 && self.should_route_remote(&item.data, item.src)?
10784 {
10785 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
10786 }
10787 return Ok(());
10788 }
10789
10790 let any_packet_needed = env
10791 .endpoints
10792 .iter()
10793 .copied()
10794 .any(|ep| self.endpoint_has_packet_handler(ep));
10795
10796 let mut pkt_opt = if any_packet_needed {
10797 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10798 pkt.validate()?;
10799 Some(pkt)
10800 } else {
10801 None
10802 };
10803
10804 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
10805 eps.sort_unstable();
10806 eps.dedup();
10807 let had_local_handler = eps.iter().copied().any(|ep| {
10808 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
10809 });
10810 let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
10811
10812 let has_remote = self.should_route_remote(&item.data, item.src)?;
10813 let targets_local = self.item_targets_local_sender(&item.data)?;
10814
10815 if targets_local {
10816 for dest in eps {
10817 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
10818 let result = match &h.handler {
10819 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
10820 dest,
10821 h,
10822 Some(bytes.as_ref()),
10823 pkt_opt.as_ref(),
10824 Some(&env),
10825 called_from_queue,
10826 ),
10827 EndpointHandlerFn::Packet(_) => {
10828 if pkt_opt.is_none() {
10829 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10830 pkt.validate()?;
10831 pkt_opt = Some(pkt);
10832 }
10833 let pkt_ref = pkt_opt.as_ref().expect("just set");
10834 self.call_handler_with_retries(
10835 dest,
10836 h,
10837 None,
10838 Some(pkt_ref),
10839 Some(&env),
10840 called_from_queue,
10841 )
10842 }
10843 };
10844 if result.is_err()
10845 && let Some(src) = item.src
10846 {
10847 self.note_side_local_handler_failure(
10848 src,
10849 env.ty,
10850 runtime_max_handler_retries(),
10851 );
10852 }
10853 }
10854 }
10855 }
10856
10857 if item.src.is_some()
10858 && had_local_delivery
10859 && targets_local
10860 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
10861 {
10862 let packet_id =
10863 wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
10864 pkt_opt
10865 .as_ref()
10866 .map(Packet::packet_id)
10867 .ok_or(TelemetryError::Unpack("missing packet id"))
10868 })?;
10869 self.queue_end_to_end_reliable_ack_for_packet_id(packet_id, called_from_queue)?;
10870 }
10871
10872 if has_remote {
10873 let relay_item = match pkt_opt {
10874 Some(ref p) => RouterItem::Packet(p.clone()),
10875 None => RouterItem::Packed(bytes.clone()),
10876 };
10877 self.relay_send(relay_item, item.src, called_from_queue)?;
10878 }
10879
10880 Ok(())
10881 }
10882 }
10883 }
10884
10885 fn dispatch_local_for_item(
10886 &self,
10887 item: &RouterItem,
10888 called_from_queue: bool,
10889 ) -> TelemetryResult<()> {
10890 match item {
10891 RouterItem::Packet(pkt) => {
10892 pkt.validate()?;
10893 if is_internal_control_type(pkt.data_type()) {
10894 return Ok(());
10895 }
10896 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
10897 if !self.item_targets_local_sender(item)? {
10898 return Ok(());
10899 }
10900
10901 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
10902 eps.sort_unstable();
10903 eps.dedup();
10904
10905 let has_packed_local = eps
10906 .iter()
10907 .copied()
10908 .any(|ep| self.endpoint_has_packed_handler(ep));
10909 let bytes_opt = if has_packed_local {
10910 Some(wire_format::pack_packet(pkt))
10911 } else {
10912 None
10913 };
10914
10915 for dest in eps {
10916 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
10917 match (&h.handler, &bytes_opt) {
10918 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
10919 self.call_handler_with_retries(
10920 dest,
10921 h,
10922 Some(bytes.as_ref()),
10923 Some(pkt),
10924 None,
10925 called_from_queue,
10926 )?;
10927 }
10928 (EndpointHandlerFn::Packed(_), None) => {
10929 let bytes = wire_format::pack_packet(pkt);
10930 self.call_handler_with_retries(
10931 dest,
10932 h,
10933 Some(bytes.as_ref()),
10934 Some(pkt),
10935 None,
10936 called_from_queue,
10937 )?;
10938 }
10939 (EndpointHandlerFn::Packet(_), _) => {
10940 self.call_handler_with_retries(
10941 dest,
10942 h,
10943 None,
10944 Some(pkt),
10945 None,
10946 called_from_queue,
10947 )?;
10948 }
10949 }
10950 }
10951 }
10952 }
10953 RouterItem::Packed(bytes) => {
10954 let env = wire_format::peek_envelope(bytes.as_ref())?;
10955 if is_internal_control_type(env.ty) {
10956 return Ok(());
10957 }
10958 self.ensure_e2e_policy_supported_for_type(env.ty)?;
10959 if !self.item_targets_local_sender(item)? {
10960 return Ok(());
10961 }
10962
10963 let any_packet_needed = env
10964 .endpoints
10965 .iter()
10966 .copied()
10967 .any(|ep| self.endpoint_has_packet_handler(ep));
10968
10969 let mut pkt_opt = if any_packet_needed {
10970 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10971 pkt.validate()?;
10972 Some(pkt)
10973 } else {
10974 None
10975 };
10976
10977 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
10978 eps.sort_unstable();
10979 eps.dedup();
10980
10981 for dest in eps {
10982 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
10983 match &h.handler {
10984 EndpointHandlerFn::Packed(_) => {
10985 self.call_handler_with_retries(
10986 dest,
10987 h,
10988 Some(bytes.as_ref()),
10989 pkt_opt.as_ref(),
10990 Some(&env),
10991 called_from_queue,
10992 )?;
10993 }
10994 EndpointHandlerFn::Packet(_) => {
10995 if pkt_opt.is_none() {
10996 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
10997 pkt.validate()?;
10998 pkt_opt = Some(pkt);
10999 }
11000 let pkt_ref = pkt_opt.as_ref().expect("just set");
11001 self.call_handler_with_retries(
11002 dest,
11003 h,
11004 None,
11005 Some(pkt_ref),
11006 Some(&env),
11007 called_from_queue,
11008 )?;
11009 }
11010 }
11011 }
11012 }
11013 }
11014 }
11015
11016 Ok(())
11017 }
11018
11019 fn tx_item_impl(
11025 &self,
11026 item: RouterTxItem,
11027 ignore_local: bool,
11028 called_from_queue: bool,
11029 ) -> TelemetryResult<()> {
11030 match item {
11031 RouterTxItem::Broadcast(data) => {
11032 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
11033 if let RouterItem::Packet(pkt) = &data
11034 && !is_internal_control_type(pkt.data_type())
11035 {
11036 self.remember_managed_variable_packet(pkt)?;
11037 }
11038 #[cfg(feature = "discovery")]
11039 let is_discovery = matches!(&data, RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()))
11040 || matches!(&data, RouterItem::Packed(bytes)
11041 if wire_format::peek_envelope(bytes.as_ref())
11042 .map(|env| discovery::is_discovery_type(env.ty))
11043 .unwrap_or(false));
11044 if !ignore_local {
11045 if self.is_duplicate_pkt(&data)? {
11046 return Ok(());
11047 }
11048 #[cfg(feature = "discovery")]
11049 if !is_discovery
11050 && !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
11051 && !matches!(&data, RouterItem::Packed(bytes)
11052 if wire_format::peek_envelope(bytes.as_ref())
11053 .map(|env| is_internal_control_type(env.ty))
11054 .unwrap_or(false))
11055 {
11056 self.dispatch_local_for_item(&data, called_from_queue)?;
11057 }
11058 #[cfg(not(feature = "discovery"))]
11059 if !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
11060 && !matches!(&data, RouterItem::Packed(bytes)
11061 if wire_format::peek_envelope(bytes.as_ref())
11062 .map(|env| is_internal_control_type(env.ty))
11063 .unwrap_or(false))
11064 {
11065 self.dispatch_local_for_item(&data, called_from_queue)?;
11066 }
11067 }
11068
11069 let send_remote = match &data {
11070 RouterItem::Packet(pkt) => {
11071 pkt.validate()?;
11072 self.should_route_remote(&data, None)?
11073 }
11074 RouterItem::Packed(bytes) => {
11075 let _ = wire_format::peek_envelope(bytes.as_ref())?;
11076 self.should_route_remote(&data, None)?
11077 }
11078 };
11079
11080 if !send_remote {
11081 return Ok(());
11082 }
11083 let mut data = data;
11084 let ty = match &data {
11085 RouterItem::Packet(pkt) => pkt.data_type(),
11086 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
11087 };
11088 if !ignore_local && !is_internal_control_type(ty) {
11089 #[cfg(feature = "discovery")]
11090 {
11091 let pending = {
11092 let st = self.state.lock();
11093 let mut pending =
11094 self.expected_end_to_end_destinations_locked(&st, &data)?;
11095 self.filter_trackable_end_to_end_destinations_locked(
11096 &st,
11097 ty,
11098 &mut pending,
11099 );
11100 pending
11101 };
11102 if !pending.is_empty() {
11103 let mut targets: Vec<u64> = pending.keys().copied().collect();
11104 targets.sort_unstable();
11105 targets.dedup();
11106 data = self.attach_wire_contract_to_item(data, &targets)?;
11107 self.register_end_to_end_reliable_tx(&data)?;
11108 }
11109 }
11110 }
11111 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, None)?;
11112 for (idx, side) in sides.iter().copied().enumerate() {
11113 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
11114 if Self::is_side_tx_busy(&e) {
11115 for retry_side in sides[idx..].iter().copied() {
11116 self.tx_queue_item_with_flags(
11117 RouterTxItem::ToSide {
11118 src: None,
11119 dst: retry_side,
11120 data: data.clone(),
11121 },
11122 true,
11123 )?;
11124 }
11125 return Ok(());
11126 }
11127 match &data {
11128 RouterItem::Packet(pkt) => {
11129 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
11130 }
11131 RouterItem::Packed(bytes) => {
11132 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
11133 let _ = self.handle_callback_error_from_env(
11134 &env,
11135 None,
11136 e,
11137 called_from_queue,
11138 );
11139 }
11140 }
11141 }
11142 return Err(TelemetryError::HandlerError("tx handler failed"));
11143 }
11144 }
11145 }
11146 RouterTxItem::ToSide { src, dst, data } => {
11147 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
11148 if let RouterItem::Packet(pkt) = &data
11149 && !is_internal_control_type(pkt.data_type())
11150 {
11151 self.remember_managed_variable_packet(pkt)?;
11152 }
11153 if !ignore_local {
11154 if self.is_duplicate_pkt(&data)? {
11155 return Ok(());
11156 }
11157 let suppress_local = matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
11158 || matches!(&data, RouterItem::Packed(bytes)
11159 if wire_format::peek_envelope(bytes.as_ref())
11160 .map(|env| is_internal_control_type(env.ty))
11161 .unwrap_or(false));
11162 if !suppress_local {
11163 self.dispatch_local_for_item(&data, called_from_queue)?;
11164 }
11165 }
11166 let allowed = {
11167 let mut st = self.state.lock();
11168 let ty = match &data {
11169 RouterItem::Packet(pkt) => Some(pkt.data_type()),
11170 RouterItem::Packed(bytes) => {
11171 Some(wire_format::peek_envelope(bytes.as_ref())?.ty)
11172 }
11173 };
11174 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
11175 #[cfg(all(feature = "discovery", feature = "timesync"))]
11176 let timesync_allowed = ty
11177 .map(|ty| {
11178 Self::timesync_allowed_for_side_locked(
11179 &mut st,
11180 dst,
11181 ty,
11182 self.clock.now_ms(),
11183 )
11184 })
11185 .unwrap_or(true);
11186 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
11187 let timesync_allowed = true;
11188 route_allowed && timesync_allowed
11189 };
11190 if !allowed {
11191 return Ok(());
11192 }
11193 if let Err(e) = self.send_reliable_to_side(dst, data.clone(), src.is_some()) {
11194 if Self::is_side_tx_busy(&e) {
11195 self.tx_queue_item_with_flags(
11196 RouterTxItem::ToSide { src, dst, data },
11197 true,
11198 )?;
11199 return Ok(());
11200 }
11201 match &data {
11202 RouterItem::Packet(pkt) => {
11203 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
11204 }
11205 RouterItem::Packed(bytes) => {
11206 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
11207 let _ = self.handle_callback_error_from_env(
11208 &env,
11209 None,
11210 e,
11211 called_from_queue,
11212 );
11213 }
11214 }
11215 }
11216 return Err(TelemetryError::HandlerError("tx handler failed"));
11217 }
11218 }
11219 RouterTxItem::EndToEndReplay { packet_id } => {
11220 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
11221 return Ok(());
11222 };
11223 if sides.is_empty() {
11224 let RemoteSidePlan::Target(fallback_sides) =
11225 self.remote_side_plan(&data, None)?;
11226 sides = fallback_sides;
11227 }
11228 for (idx, side) in sides.iter().copied().enumerate() {
11229 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
11230 if Self::is_side_tx_busy(&e) {
11231 for retry_side in sides[idx..].iter().copied() {
11232 self.tx_queue_item_with_flags(
11233 RouterTxItem::ToSide {
11234 src: None,
11235 dst: retry_side,
11236 data: data.clone(),
11237 },
11238 true,
11239 )?;
11240 }
11241 return Ok(());
11242 }
11243 match &data {
11244 RouterItem::Packet(pkt) => {
11245 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
11246 }
11247 RouterItem::Packed(bytes) => {
11248 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
11249 let _ = self.handle_callback_error_from_env(
11250 &env,
11251 None,
11252 e,
11253 called_from_queue,
11254 );
11255 }
11256 }
11257 }
11258 return Err(TelemetryError::HandlerError("tx handler failed"));
11259 }
11260 }
11261 }
11262 RouterTxItem::ReliableReplay { dst, bytes } => {
11263 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
11264 let ty = frame.envelope.ty;
11265 let Some(hdr) = frame.reliable else {
11266 return Ok(());
11267 };
11268 {
11269 let mut st = self.state.lock();
11270 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
11271 if !tx_state.sent.contains_key(&hdr.seq) {
11272 return Ok(());
11273 }
11274 }
11275 if let Err(e) = self.send_reliable_raw_to_side(dst, bytes.clone(), false) {
11276 if Self::is_side_tx_busy(&e) {
11277 self.tx_queue_item_with_flags(
11278 RouterTxItem::ReliableReplay { dst, bytes },
11279 true,
11280 )?;
11281 return Ok(());
11282 }
11283 return Err(e);
11284 }
11285 let mut st = self.state.lock();
11286 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
11287 if let Some(sent) = tx_state.sent.get_mut(&hdr.seq) {
11288 sent.last_send_ms = self.clock.now_ms();
11289 sent.queued = false;
11290 }
11291 }
11292 }
11293
11294 Ok(())
11295 }
11296
11297 #[inline]
11299 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
11300 self.tx_item_impl(item, false, false)
11301 }
11302
11303 #[inline]
11310 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
11311 if self.side_tx_active() {
11312 return self.rx_packed_queue(bytes);
11313 }
11314 let data = RouterItem::Packed(Arc::from(bytes));
11315 let item = RouterRxItem {
11316 src: None,
11317 priority: Self::router_item_priority(&data)?,
11318 data,
11319 };
11320 self.rx_item(&item, false)
11321 }
11322
11323 #[inline]
11328 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
11329 if self.side_tx_active() {
11330 return self.rx_queue(pkt.clone());
11331 }
11332 let data = RouterItem::Packet(pkt.clone());
11333 let item = RouterRxItem {
11334 src: None,
11335 priority: Self::router_item_priority(&data)?,
11336 data,
11337 };
11338 self.rx_item(&item, false)
11339 }
11340
11341 #[inline]
11346 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
11347 if self.side_tx_active() {
11348 return self.rx_queue_from_side(pkt.clone(), side);
11349 }
11350 self.ensure_side_ingress_enabled(side)?;
11351 let data = RouterItem::Packet(pkt.clone());
11352 let item = RouterRxItem {
11353 src: Some(side),
11354 priority: Self::router_item_priority(&data)?,
11355 data,
11356 };
11357 self.rx_item(&item, false)
11358 }
11359
11360 #[inline]
11365 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
11366 if self.side_tx_active() {
11367 return self.rx_packed_queue_from_side(bytes, side);
11368 }
11369 self.ensure_side_ingress_enabled(side)?;
11370 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
11371 return Ok(());
11372 };
11373 let data = RouterItem::Packed(decoded);
11374 let item = RouterRxItem {
11375 src: Some(side),
11376 priority: Self::router_item_priority(&data)?,
11377 data,
11378 };
11379 self.rx_item(&item, false)
11380 }
11381
11382 #[inline]
11389 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
11390 #[cfg(feature = "discovery")]
11391 let _ = self.poll_discovery()?;
11392 if self.side_tx_active() {
11393 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
11394 }
11395 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11396 }
11397
11398 #[inline]
11403 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
11404 #[cfg(feature = "discovery")]
11405 let _ = self.poll_discovery()?;
11406 if self.side_tx_active() {
11407 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(pkt)));
11408 }
11409 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(pkt)))
11410 }
11411
11412 #[inline]
11416 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
11417 #[cfg(feature = "discovery")]
11418 let _ = self.poll_discovery()?;
11419 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11420 }
11421
11422 #[inline]
11424 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
11425 #[cfg(feature = "discovery")]
11426 let _ = self.poll_discovery()?;
11427 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(data)))
11428 }
11429
11430 #[inline]
11437 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
11438 #[cfg(feature = "discovery")]
11439 let _ = self.poll_discovery()?;
11440 if self.side_tx_active() {
11441 return self.log_queue(ty, data);
11442 }
11443 let sender = self.sender_arc();
11444 log_raw(
11445 sender.as_ref(),
11446 ty,
11447 data,
11448 self.packet_timestamp_ms(),
11449 |pkt| self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
11450 )
11451 }
11452
11453 #[inline]
11455 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
11456 #[cfg(feature = "discovery")]
11457 let _ = self.poll_discovery()?;
11458 let sender = self.sender_arc();
11459 log_raw(
11460 sender.as_ref(),
11461 ty,
11462 data,
11463 self.packet_timestamp_ms(),
11464 |pkt| self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
11465 )
11466 }
11467
11468 #[inline]
11470 pub fn log_ts<T: LeBytes>(
11471 &self,
11472 ty: DataType,
11473 timestamp: u64,
11474 data: &[T],
11475 ) -> TelemetryResult<()> {
11476 #[cfg(feature = "discovery")]
11477 let _ = self.poll_discovery()?;
11478 if self.side_tx_active() {
11479 return self.log_queue_ts(ty, timestamp, data);
11480 }
11481 let sender = self.sender_arc();
11482 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
11483 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11484 })
11485 }
11486
11487 #[inline]
11489 pub fn log_queue_ts<T: LeBytes>(
11490 &self,
11491 ty: DataType,
11492 timestamp: u64,
11493 data: &[T],
11494 ) -> TelemetryResult<()> {
11495 #[cfg(feature = "discovery")]
11496 let _ = self.poll_discovery()?;
11497 let sender = self.sender_arc();
11498 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
11499 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
11500 })
11501 }
11502}