1use crate::config::RuntimeMemoryConfig;
12use crate::diagnostics::{
13 AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
14 RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
15 RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
16};
17#[cfg(feature = "discovery")]
18use crate::discovery::{
19 self, ClientStatsSnapshot, DISCOVERY_ROUTE_TTL_MS, DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS,
20 DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState,
21 TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS, TopologyAnnouncerRoute, TopologyBoardNode,
22 TopologySideRoute, TopologySnapshot,
23};
24use crate::packet::hash_bytes_u64;
25use crate::queue::{BoundedDeque, ByteCost};
26#[cfg(all(not(feature = "std"), target_os = "none"))]
27use crate::seds_error_msg;
28#[cfg(feature = "timesync")]
29use crate::timesync::{
30 INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
31 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
32 NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
33 TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
34 decode_timesync_request, decode_timesync_response,
35};
36use crate::{
37 E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
38 config::{
39 DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
40 runtime_reliable_max_end_to_end_pending, runtime_reliable_max_pending,
41 runtime_reliable_max_retries, runtime_reliable_max_return_routes,
42 runtime_reliable_retransmit_ms,
43 },
44 get_needed_message_size, impl_letype_num, is_reliable_type,
45 lock::{ReentryGate, RouterMutex},
46 message_e2e_encryption_policy, message_meta, message_priority,
47 packet::Packet,
48 reliable_mode, wire_format,
49};
50use alloc::string::{String, ToString};
51use alloc::{
52 borrow::ToOwned,
53 boxed::Box,
54 collections::{BTreeMap, BTreeSet, VecDeque},
55 format,
56 sync::Arc,
57 vec,
58 vec::Vec,
59};
60use core::cell::UnsafeCell;
61use core::fmt;
62use core::fmt::{Debug, Formatter};
63use core::mem::size_of;
64use core::ops::{Deref, DerefMut};
65use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
66use crc32fast::Hasher as Crc32Hasher;
67#[cfg(feature = "std")]
68use std::time::Instant;
69
70pub type RouterSideId = usize;
72
73const SIDE_TRANSPORT_MAGIC: &[u8; 3] = b"SDT";
74const SIDE_TRANSPORT_KIND_FULL: u8 = 0x01;
75const SIDE_TRANSPORT_KIND_COMPACT: u8 = 0x02;
76const SIDE_TRANSPORT_KIND_CHUNK: u8 = 0x03;
77const SIDE_TRANSPORT_KIND_COMPACT_DELTA: u8 = 0x04;
78const SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP: u8 = 0x05;
79const SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED: u8 = 0x01;
80const SIDE_TRANSPORT_FLAG_WIRE_CONTRACT: u8 = 0x04;
81const SIDE_TRANSPORT_FLAG_PACKET_NONCE: u8 = 0x08;
82const SIDE_TRANSPORT_FLAG_ENDPOINT_BITMAP_PRESENT: u8 = 0x20;
83const SIDE_TRANSPORT_FLAG_COMPACT_RELIABLE_HEADER: u8 = 0x40;
84const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
85const SIDE_TRANSPORT_CHUNK_OVERHEAD: usize = 3 + 1 + 4 + 2 + 2 + wire_format::CRC32_BYTES;
86const SIDE_TIMESTAMP_POLICY_WORDS: usize = ((crate::MAX_VALUE_DATA_TYPE as usize) + 1).div_ceil(64);
87const SIDE_TRANSPORT_EP_BITMAP_BITS: usize = (crate::MAX_VALUE_DATA_ENDPOINT as usize) + 1;
88const SIDE_TRANSPORT_EP_BITMAP_BYTES: usize = SIDE_TRANSPORT_EP_BITMAP_BITS.div_ceil(8);
89pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
90pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
91pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
92static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
93
94#[derive(Clone, Copy, Debug, PartialEq, Eq)]
95pub enum SideTransportProfile {
96 Canonical,
97 Template,
98 Ipv6Like,
99 Ipv4Like,
100}
101
102impl SideTransportProfile {
103 #[inline]
104 pub const fn as_str(self) -> &'static str {
105 match self {
106 Self::Canonical => "canonical",
107 Self::Template => "template",
108 Self::Ipv6Like => "ipv6_like",
109 Self::Ipv4Like => "ipv4_like",
110 }
111 }
112
113 #[cfg(feature = "discovery")]
114 #[inline]
115 pub const fn discovery_code(self) -> u8 {
116 match self {
117 Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
118 Self::Template => discovery::LINK_PROFILE_TEMPLATE,
119 Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
120 Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
121 }
122 }
123}
124
125#[derive(Clone, Debug, PartialEq, Eq)]
126struct SideHeaderTemplate {
127 hash: u64,
128 base_flags: u8,
129 prefix: Arc<[u8]>,
130 between: Arc<[u8]>,
131 reliable_flags: Option<u8>,
132 reliable_compact: bool,
133}
134
135#[derive(Clone, Debug, Default, PartialEq, Eq)]
136struct SideChunkAssembly {
137 total: u16,
138 received: BTreeMap<u16, Arc<[u8]>>,
139}
140
141#[derive(Clone, Debug, Default)]
142struct SideTransportState {
143 tx_template_ids: BTreeMap<u64, u32>,
144 tx_templates: BTreeMap<u64, SideHeaderTemplate>,
145 tx_last_timestamps: BTreeMap<u32, u64>,
146 rx_templates: BTreeMap<u64, SideHeaderTemplate>,
147 rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
148 rx_last_timestamps: BTreeMap<u32, u64>,
149 rx_chunks: BTreeMap<u32, SideChunkAssembly>,
150 next_chunk_id: u32,
151 next_template_id: u32,
152}
153
154impl SideTransportState {
155 fn tx_template_count(&self) -> usize {
156 self.tx_template_ids.len()
157 }
158
159 fn rx_template_count(&self) -> usize {
160 self.rx_templates_by_id.len()
161 }
162
163 fn insert_tx_template(
164 &mut self,
165 template: SideHeaderTemplate,
166 template_id: u32,
167 max_templates: usize,
168 ) -> bool {
169 if max_templates == 0 {
170 return false;
171 }
172 let mut evicted = false;
173 if self.tx_template_ids.len() >= max_templates
174 && !self.tx_template_ids.contains_key(&template.hash)
175 && let Some(old_hash) = self.tx_template_ids.keys().next().copied()
176 {
177 if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
178 self.tx_last_timestamps.remove(&old_id);
179 }
180 self.tx_templates.remove(&old_hash);
181 evicted = true;
182 }
183 self.tx_template_ids.insert(template.hash, template_id);
184 self.tx_templates.insert(template.hash, template);
185 evicted
186 }
187
188 fn insert_rx_template(
189 &mut self,
190 template_id: u32,
191 template: SideHeaderTemplate,
192 max_templates: usize,
193 ) -> bool {
194 if max_templates == 0 {
195 return false;
196 }
197 let mut evicted = false;
198 if self.rx_templates_by_id.len() >= max_templates
199 && !self.rx_templates_by_id.contains_key(&template_id)
200 && let Some(old_id) = self.rx_templates_by_id.keys().next().copied()
201 && let Some(old_template) = self.rx_templates_by_id.remove(&old_id)
202 {
203 self.rx_templates.remove(&old_template.hash);
204 self.rx_last_timestamps.remove(&old_id);
205 evicted = true;
206 }
207 self.rx_templates_by_id
208 .insert(template_id, template.clone());
209 self.rx_templates.insert(template.hash, template);
210 evicted
211 }
212}
213
214type SideTemplateExtract<'a> = (
215 SideHeaderTemplate,
216 DataType,
217 u8,
218 u64,
219 u64,
220 u16,
221 Option<(u32, u32)>,
222 &'a [u8],
223);
224
225#[derive(Clone, Copy, Debug, PartialEq, Eq)]
226pub struct CompactTimestampOmissionPolicy {
227 all: bool,
228 words: [u64; SIDE_TIMESTAMP_POLICY_WORDS],
229}
230
231impl CompactTimestampOmissionPolicy {
232 #[inline]
233 pub const fn none() -> Self {
234 Self {
235 all: false,
236 words: [0; SIDE_TIMESTAMP_POLICY_WORDS],
237 }
238 }
239
240 #[inline]
241 pub const fn all() -> Self {
242 Self {
243 all: true,
244 words: [0; SIDE_TIMESTAMP_POLICY_WORDS],
245 }
246 }
247
248 #[inline]
249 pub fn with_data_type(mut self, ty: DataType) -> Self {
250 self.insert(ty);
251 self
252 }
253
254 #[inline]
255 pub fn insert(&mut self, ty: DataType) {
256 let id = ty.as_u32() as usize;
257 if id <= crate::MAX_VALUE_DATA_TYPE as usize {
258 self.words[id / 64] |= 1u64 << (id % 64);
259 }
260 }
261
262 #[inline]
263 pub fn contains(self, ty: DataType) -> bool {
264 if self.all {
265 return true;
266 }
267 let id = ty.as_u32() as usize;
268 id <= crate::MAX_VALUE_DATA_TYPE as usize
269 && (self.words[id / 64] & (1u64 << (id % 64))) != 0
270 }
271
272 #[inline]
273 pub fn is_empty(self) -> bool {
274 !self.all && self.words.iter().all(|word| *word == 0)
275 }
276}
277
278impl Default for CompactTimestampOmissionPolicy {
279 #[inline]
280 fn default() -> Self {
281 Self::none()
282 }
283}
284
285#[derive(Clone, Copy, Debug, PartialEq, Eq)]
286enum SideCompactTimestampMode {
287 Absolute,
288 Delta,
289 Omitted,
290}
291
292#[derive(Clone, Debug, PartialEq, Eq)]
293pub enum RouterItem {
294 Packet(Packet),
295 Packed(Arc<[u8]>),
296}
297
298#[derive(Clone, Debug, PartialEq, Eq)]
299struct RouterRxItem {
300 src: Option<RouterSideId>,
301 data: RouterItem,
302 priority: u8,
303}
304
305#[derive(Clone, Debug, PartialEq, Eq)]
306enum RouterTxItem {
307 Broadcast(RouterItem),
308 EndToEndReplay {
309 packet_id: u64,
310 },
311 ToSide {
312 src: Option<RouterSideId>,
313 dst: RouterSideId,
314 data: RouterItem,
315 },
316 ReliableReplay {
317 dst: RouterSideId,
318 bytes: Arc<[u8]>,
319 },
320}
321
322impl ByteCost for RouterRxItem {
323 #[inline]
324 fn byte_cost(&self) -> usize {
325 match &self.data {
326 RouterItem::Packet(pkt) => pkt.byte_cost(),
327 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
328 }
329 }
330}
331
332impl ByteCost for RouterTxItem {
333 #[inline]
334 fn byte_cost(&self) -> usize {
335 match self {
336 RouterTxItem::Broadcast(data) => match data {
337 RouterItem::Packet(pkt) => pkt.byte_cost(),
338 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
339 },
340 RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
341 RouterTxItem::ToSide { data, .. } => match data {
342 RouterItem::Packet(pkt) => pkt.byte_cost(),
343 RouterItem::Packed(bytes) => size_of::<Arc<[u8]>>() + bytes.len(),
344 },
345 RouterTxItem::ReliableReplay { bytes, .. } => size_of::<Arc<[u8]>>() + bytes.len(),
346 }
347 }
348}
349
350#[derive(Clone, Debug, PartialEq, Eq)]
354struct TxQueued {
355 item: RouterTxItem,
356 ignore_local: bool,
357 priority: u8,
358}
359
360impl ByteCost for TxQueued {
362 #[inline]
364 fn byte_cost(&self) -> usize {
365 self.item.byte_cost() + size_of::<bool>() + size_of::<u8>()
366 }
367}
368
369impl ByteCost for u64 {
371 #[inline]
373 fn byte_cost(&self) -> usize {
374 size_of::<u64>()
375 }
376}
377
378#[derive(Debug, Clone)]
381struct ReliableTxState {
382 next_seq: u32,
383 sent_order: VecDeque<u32>,
384 sent: BTreeMap<u32, ReliableSent>,
385}
386
387#[derive(Debug, Clone)]
388struct ReliableSent {
389 bytes: Arc<[u8]>,
390 last_send_ms: u64,
391 retries: u32,
392 queued: bool,
393 partial_acked: bool,
394}
395
396#[derive(Debug, Clone)]
397struct EndToEndReliableSent {
398 data: RouterItem,
399 pending_destinations: BTreeMap<u64, RouterSideId>,
400 tracked_destinations: bool,
401 last_send_ms: u64,
402 retries: u32,
403 queued: bool,
404}
405
406#[derive(Debug, Clone)]
407struct ReliableRxState {
408 expected_seq: u32,
409 buffered: BTreeMap<u32, Arc<[u8]>>,
410}
411
412#[derive(Debug, Clone)]
413struct ReliableReturnRouteState {
414 side: RouterSideId,
415}
416
417#[cfg(feature = "discovery")]
418#[derive(Debug, Clone, Default, PartialEq, Eq)]
419struct DiscoverySenderState {
420 reachable: Vec<DataEndpoint>,
421 reachable_timesync_sources: Vec<String>,
422 topology_boards: Vec<TopologyBoardNode>,
423 last_seen_ms: u64,
424}
425
426#[cfg(feature = "discovery")]
427#[derive(Debug, Clone, Default, PartialEq, Eq)]
428struct DiscoverySideState {
429 reachable: Vec<DataEndpoint>,
430 reachable_timesync_sources: Vec<String>,
431 last_seen_ms: u64,
432 announcers: BTreeMap<String, DiscoverySenderState>,
433}
434
435#[cfg(feature = "discovery")]
436#[derive(Debug, Clone, Default)]
437struct DiscoverySideThrottleState {
438 next_ping_ms: u64,
439 next_full_ms: u64,
440}
441
442#[cfg(all(feature = "discovery", feature = "timesync"))]
443#[derive(Debug, Clone, Default)]
444struct TimeSyncSideThrottleState {
445 next_allowed_ms: u64,
446}
447
448#[cfg(feature = "discovery")]
449#[derive(Debug, Clone, Copy, PartialEq, Eq)]
450enum DiscoveryAdvertiseLevel {
451 MinimalPing,
452 Full,
453}
454
455#[derive(Debug, Clone, Default)]
456struct AdaptiveRouteStats {
457 estimated_bandwidth_bps: u64,
458 peak_bandwidth_bps: u64,
459 last_observed_ms: u64,
460 last_slow_observed_ms: u64,
461 sample_count: u64,
462 window_started_ms: u64,
463 window_bytes: u64,
464 peak_usage_bps: u64,
465}
466
467impl AdaptiveRouteStats {
468 #[inline]
469 fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
470 self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
471 sample_bps
472 } else if sample_bps >= self.estimated_bandwidth_bps {
473 self.estimated_bandwidth_bps
474 .saturating_mul(3)
475 .saturating_add(sample_bps.saturating_mul(5))
476 / 8
477 } else {
478 self.estimated_bandwidth_bps
479 .saturating_mul(7)
480 .saturating_add(sample_bps)
481 / 8
482 };
483 self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
484 self.last_observed_ms = now_ms;
485 if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
486 self.last_slow_observed_ms = now_ms;
487 }
488 self.sample_count = self.sample_count.saturating_add(1);
489 if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
490 self.window_started_ms = now_ms;
491 self.window_bytes = 0;
492 }
493 self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
494 self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
495 }
496
497 #[inline]
498 fn current_usage_bps(&self, now_ms: u64) -> u64 {
499 if self.window_started_ms == 0 {
500 return 0;
501 }
502 let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
503 (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
504 .min(u128::from(u64::MAX)) as u64
505 }
506
507 #[inline]
508 fn available_headroom_bps(&self, now_ms: u64) -> u64 {
509 let capacity = self
510 .estimated_bandwidth_bps
511 .max(self.peak_bandwidth_bps)
512 .max(1);
513 capacity.saturating_sub(self.current_usage_bps(now_ms))
514 }
515
516 #[inline]
517 fn weight(&self, now_ms: u64) -> u64 {
518 self.available_headroom_bps(now_ms).max(1)
519 }
520
521 #[inline]
522 fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
523 let current_usage_bps = self.current_usage_bps(now_ms);
524 let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
525 let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
526 let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
527 AdaptiveLinkStats {
528 auto_balancing_enabled,
529 estimated_capacity_bps,
530 peak_capacity_bps,
531 current_usage_bps,
532 peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
533 available_headroom_bps,
534 effective_weight: available_headroom_bps.max(1),
535 last_observed_ms: self.last_observed_ms,
536 sample_count: self.sample_count,
537 }
538 }
539}
540
541#[derive(Debug, Clone, Default)]
542struct TypeRuntimeStatsInner {
543 tx_packets: u64,
544 tx_bytes: u64,
545 rx_packets: u64,
546 rx_bytes: u64,
547 relayed_tx_packets: u64,
548 relayed_tx_bytes: u64,
549 relayed_rx_packets: u64,
550 relayed_rx_bytes: u64,
551 tx_retries: u64,
552 handler_failures: u64,
553}
554
555#[derive(Debug, Clone, Default)]
556struct SideRuntimeStatsInner {
557 tx_packets: u64,
558 tx_bytes: u64,
559 rx_packets: u64,
560 rx_bytes: u64,
561 relayed_tx_packets: u64,
562 relayed_tx_bytes: u64,
563 relayed_rx_packets: u64,
564 relayed_rx_bytes: u64,
565 local_delivery_packets: u64,
566 tx_retries: u64,
567 tx_handler_failures: u64,
568 local_handler_failures: u64,
569 total_handler_retries: u64,
570 side_transport_full_frames: u64,
571 side_transport_compact_frames: u64,
572 side_transport_compact_delta_frames: u64,
573 side_transport_compact_omitted_timestamp_frames: u64,
574 side_transport_chunk_frames: u64,
575 side_transport_raw_bytes: u64,
576 side_transport_wire_bytes: u64,
577 side_transport_bytes_saved: u64,
578 side_transport_min_compact_overhead_bytes: Option<usize>,
579 side_transport_max_compact_overhead_bytes: Option<usize>,
580 side_transport_compact_target_misses: u64,
581 side_transport_template_evictions: u64,
582 data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
583}
584
585impl SideRuntimeStatsInner {
586 fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
587 self.data_types.entry(ty.as_u32()).or_default()
588 }
589
590 fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
591 self.tx_packets = self.tx_packets.saturating_add(1);
592 self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
593 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
594 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
595 if relayed {
596 self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
597 self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
598 }
599 let stats = self.type_stats_mut(ty);
600 stats.tx_packets = stats.tx_packets.saturating_add(1);
601 stats.tx_bytes = stats.tx_bytes.saturating_add(bytes as u64);
602 stats.tx_retries = stats.tx_retries.saturating_add(retries as u64);
603 if relayed {
604 stats.relayed_tx_packets = stats.relayed_tx_packets.saturating_add(1);
605 stats.relayed_tx_bytes = stats.relayed_tx_bytes.saturating_add(bytes as u64);
606 }
607 }
608
609 fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
610 self.rx_packets = self.rx_packets.saturating_add(1);
611 self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
612 if relayed {
613 self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
614 self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
615 }
616 let stats = self.type_stats_mut(ty);
617 stats.rx_packets = stats.rx_packets.saturating_add(1);
618 stats.rx_bytes = stats.rx_bytes.saturating_add(bytes as u64);
619 if relayed {
620 stats.relayed_rx_packets = stats.relayed_rx_packets.saturating_add(1);
621 stats.relayed_rx_bytes = stats.relayed_rx_bytes.saturating_add(bytes as u64);
622 }
623 }
624
625 fn note_local_delivery(&mut self, ty: DataType) {
626 self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
627 let stats = self.type_stats_mut(ty);
628 stats.rx_packets = stats.rx_packets.saturating_add(1);
629 }
630
631 fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
632 self.local_handler_failures = self.local_handler_failures.saturating_add(1);
633 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
634 let stats = self.type_stats_mut(ty);
635 stats.handler_failures = stats.handler_failures.saturating_add(1);
636 }
637
638 fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
639 self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
640 self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
641 self.tx_retries = self.tx_retries.saturating_add(retries as u64);
642 let stats = self.type_stats_mut(ty);
643 stats.handler_failures = stats.handler_failures.saturating_add(1);
644 stats.tx_retries = stats.tx_retries.saturating_add(retries as u64);
645 }
646
647 fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
648 self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
649 self.note_side_transport_bytes(raw_bytes, wire_bytes);
650 }
651
652 fn note_side_transport_compact(
653 &mut self,
654 raw_bytes: usize,
655 wire_bytes: usize,
656 compact_overhead_bytes: usize,
657 used_timestamp_delta: bool,
658 omitted_timestamp: bool,
659 ) {
660 self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
661 if used_timestamp_delta {
662 self.side_transport_compact_delta_frames =
663 self.side_transport_compact_delta_frames.saturating_add(1);
664 }
665 if omitted_timestamp {
666 self.side_transport_compact_omitted_timestamp_frames = self
667 .side_transport_compact_omitted_timestamp_frames
668 .saturating_add(1);
669 }
670 self.note_side_transport_bytes(raw_bytes, wire_bytes);
671 self.side_transport_min_compact_overhead_bytes = Some(
672 self.side_transport_min_compact_overhead_bytes
673 .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
674 );
675 self.side_transport_max_compact_overhead_bytes = Some(
676 self.side_transport_max_compact_overhead_bytes
677 .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
678 );
679 }
680
681 fn note_side_transport_chunks(&mut self, chunks: usize) {
682 self.side_transport_chunk_frames = self
683 .side_transport_chunk_frames
684 .saturating_add(chunks as u64);
685 }
686
687 fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
688 self.side_transport_raw_bytes = self
689 .side_transport_raw_bytes
690 .saturating_add(raw_bytes as u64);
691 self.side_transport_wire_bytes = self
692 .side_transport_wire_bytes
693 .saturating_add(wire_bytes as u64);
694 if raw_bytes > wire_bytes {
695 self.side_transport_bytes_saved = self
696 .side_transport_bytes_saved
697 .saturating_add((raw_bytes - wire_bytes) as u64);
698 }
699 }
700
701 fn note_side_transport_compact_target_miss(&mut self) {
702 self.side_transport_compact_target_misses =
703 self.side_transport_compact_target_misses.saturating_add(1);
704 }
705
706 fn note_side_transport_template_eviction(&mut self) {
707 self.side_transport_template_evictions =
708 self.side_transport_template_evictions.saturating_add(1);
709 }
710}
711
712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713enum RouteSelectionOrigin {
714 Flood,
715 Discovered,
716}
717
718type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
721
722type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
724
725#[derive(Clone)]
731pub enum EndpointHandlerFn {
732 Packet(Arc<PacketHandlerFn>),
733 Packed(Arc<PackedHandlerFn>),
734}
735
736pub struct EndpointHandler {
738 endpoint: DataEndpoint,
739 handler: EndpointHandlerFn,
740}
741
742impl Debug for EndpointHandlerFn {
744 #[inline]
745 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
746 match self {
747 EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
748 EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
749 }
750 }
751}
752
753#[derive(Clone)]
755pub enum RouterTxHandlerFn {
756 Packed(Arc<PackedHandlerFn>),
757 Packet(Arc<PacketHandlerFn>),
758}
759
760impl Debug for RouterTxHandlerFn {
761 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
762 match self {
763 RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
764 RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
765 }
766 }
767}
768
769#[derive(Clone, Copy, Debug)]
770pub struct RouterSideOptions {
771 pub reliable_enabled: bool,
782 pub link_local_enabled: bool,
784 pub header_template_enabled: bool,
791 pub max_frame_bytes: usize,
797 pub compact_header_target_bytes: usize,
804 pub max_side_transport_templates: usize,
810 pub omit_unchanged_compact_timestamps: bool,
816 pub compact_timestamp_omission_types: CompactTimestampOmissionPolicy,
821 pub side_transport_profile: SideTransportProfile,
823 pub ingress_enabled: bool,
825 pub egress_enabled: bool,
827}
828
829impl Default for RouterSideOptions {
830 fn default() -> Self {
831 Self {
832 reliable_enabled: false,
833 link_local_enabled: false,
834 header_template_enabled: false,
835 max_frame_bytes: 0,
836 compact_header_target_bytes: 0,
837 max_side_transport_templates: DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT,
838 omit_unchanged_compact_timestamps: false,
839 compact_timestamp_omission_types: CompactTimestampOmissionPolicy::none(),
840 side_transport_profile: SideTransportProfile::Canonical,
841 ingress_enabled: true,
842 egress_enabled: true,
843 }
844 }
845}
846
847impl RouterSideOptions {
848 #[inline]
853 pub fn with_small_packet_transport(mut self, max_frame_bytes: usize) -> Self {
854 self.header_template_enabled = true;
855 self.max_frame_bytes = max_frame_bytes;
856 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
857 self.side_transport_profile = SideTransportProfile::Ipv6Like;
858 self
859 }
860
861 #[inline]
862 pub fn with_ipv4_like_compact_header_target(mut self) -> Self {
863 self.header_template_enabled = true;
864 self.compact_header_target_bytes = IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES;
865 self.omit_unchanged_compact_timestamps = true;
866 self.side_transport_profile = SideTransportProfile::Ipv4Like;
867 self
868 }
869
870 #[inline]
871 pub fn with_ipv6_like_compact_header_target(mut self) -> Self {
872 self.header_template_enabled = true;
873 self.compact_header_target_bytes = IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES;
874 self.side_transport_profile = SideTransportProfile::Ipv6Like;
875 self
876 }
877
878 #[inline]
879 pub fn with_template_transport(mut self) -> Self {
880 self.header_template_enabled = true;
881 self.side_transport_profile = SideTransportProfile::Template;
882 self
883 }
884
885 #[inline]
886 pub fn with_omitted_unchanged_compact_timestamps(mut self) -> Self {
887 self.header_template_enabled = true;
888 self.omit_unchanged_compact_timestamps = true;
889 self
890 }
891
892 #[inline]
893 pub fn with_omitted_unchanged_compact_timestamps_for_type(mut self, ty: DataType) -> Self {
894 self.header_template_enabled = true;
895 self.compact_timestamp_omission_types.insert(ty);
896 self
897 }
898
899 #[inline]
900 pub fn effective_transport_profile(self) -> SideTransportProfile {
901 if !self.header_template_enabled && self.max_frame_bytes == 0 {
902 SideTransportProfile::Canonical
903 } else if self.side_transport_profile == SideTransportProfile::Canonical {
904 SideTransportProfile::Template
905 } else {
906 self.side_transport_profile
907 }
908 }
909
910 #[cfg(feature = "discovery")]
911 #[inline]
912 pub fn link_capabilities(self) -> discovery::LinkCapabilities {
913 let mut flags = discovery::LINK_CAPABILITY_END_TO_END_RELIABILITY;
914 if self.header_template_enabled {
915 flags |= discovery::LINK_CAPABILITY_HEADER_TEMPLATES;
916 }
917 if self.max_frame_bytes != 0 {
918 flags |= discovery::LINK_CAPABILITY_CHUNKING;
919 }
920 if self.reliable_enabled {
921 flags |= discovery::LINK_CAPABILITY_RELIABILITY;
922 }
923 if self.omit_unchanged_compact_timestamps
924 || !self.compact_timestamp_omission_types.is_empty()
925 {
926 flags |= discovery::LINK_CAPABILITY_OMIT_UNCHANGED_TIMESTAMPS;
927 }
928 #[cfg(feature = "cryptography")]
929 {
930 flags |= discovery::LINK_CAPABILITY_CRYPTO;
931 }
932 discovery::LinkCapabilities {
933 version: 1,
934 flags,
935 profile: self.effective_transport_profile().discovery_code(),
936 max_frame_bytes: self.max_frame_bytes.min(u32::MAX as usize) as u32,
937 compact_header_target_bytes: self.compact_header_target_bytes.min(u32::MAX as usize)
938 as u32,
939 max_side_transport_templates: self.max_side_transport_templates.min(u32::MAX as usize)
940 as u32,
941 }
942 }
943}
944
945#[derive(Clone, Debug)]
947pub struct RouterSide {
948 pub name: &'static str,
949 pub tx_handler: RouterTxHandlerFn,
950 pub opts: RouterSideOptions,
951}
952
953impl Debug for EndpointHandler {
955 #[inline]
956 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
957 f.debug_struct("EndpointHandler")
958 .field("endpoint", &self.endpoint)
959 .field("handler", &self.handler)
960 .finish()
961 }
962}
963
964#[inline]
965pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
966 #[cfg(feature = "timesync")]
967 if matches!(endpoint, DataEndpoint::TimeSync) {
968 return true;
969 }
970 discovery::is_discovery_endpoint(endpoint)
971}
972
973impl EndpointHandler {
974 #[inline]
978 pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
979 where
980 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
981 {
982 assert!(
983 !endpoint_is_router_internal(endpoint),
984 "reserved internal endpoint handlers must not be user-registered"
985 );
986 #[cfg(feature = "std")]
987 crate::config::ensure_endpoint_id(endpoint, false)
988 .expect("endpoint handler endpoint registration failed");
989 Self {
990 endpoint,
991 handler: EndpointHandlerFn::Packet(Arc::new(f)),
992 }
993 }
994
995 #[inline]
997 pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
998 where
999 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1000 {
1001 Self::new_packet_handler(endpoint.id, f)
1002 }
1003
1004 #[cfg(feature = "std")]
1006 #[inline]
1007 pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1008 where
1009 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1010 {
1011 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1012 .ok_or(TelemetryError::BadArg)?;
1013 Ok(Self::new_packet_handler(endpoint.id, f))
1014 }
1015
1016 #[inline]
1020 pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1021 where
1022 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1023 {
1024 assert!(
1025 !endpoint_is_router_internal(endpoint),
1026 "reserved internal endpoint handlers must not be user-registered"
1027 );
1028 #[cfg(feature = "std")]
1029 crate::config::ensure_endpoint_id(endpoint, false)
1030 .expect("endpoint handler endpoint registration failed");
1031 Self {
1032 endpoint,
1033 handler: EndpointHandlerFn::Packed(Arc::new(f)),
1034 }
1035 }
1036
1037 #[inline]
1039 pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1040 where
1041 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1042 {
1043 Self::new_packed_handler(endpoint.id, f)
1044 }
1045
1046 #[cfg(feature = "std")]
1048 #[inline]
1049 pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1050 where
1051 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1052 {
1053 let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1054 .ok_or(TelemetryError::BadArg)?;
1055 Ok(Self::new_packed_handler(endpoint.id, f))
1056 }
1057
1058 #[inline]
1060 pub fn get_endpoint(&self) -> DataEndpoint {
1061 self.endpoint
1062 }
1063
1064 #[inline]
1066 pub fn get_handler(&self) -> &EndpointHandlerFn {
1067 &self.handler
1068 }
1069}
1070
1071pub trait Clock {
1072 fn now_ms(&self) -> u64;
1074
1075 fn now_ns(&self) -> u64 {
1079 self.now_ms().saturating_mul(1_000_000)
1080 }
1081}
1082
1083impl<T: Fn() -> u64> Clock for T {
1084 #[inline]
1085 fn now_ms(&self) -> u64 {
1086 self()
1087 }
1088}
1089
1090#[cfg(feature = "std")]
1091#[derive(Debug)]
1092struct StdMonotonicClock {
1093 start: Instant,
1094}
1095
1096#[cfg(feature = "std")]
1097impl Default for StdMonotonicClock {
1098 fn default() -> Self {
1099 Self {
1100 start: Instant::now(),
1101 }
1102 }
1103}
1104
1105#[cfg(feature = "std")]
1106impl Clock for StdMonotonicClock {
1107 fn now_ms(&self) -> u64 {
1108 u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1109 }
1110
1111 fn now_ns(&self) -> u64 {
1112 u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1113 }
1114}
1115
1116#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1118pub enum RouterE2eEncryptionMode {
1119 Disabled,
1121 RequiredOnly,
1123 Preferred,
1125 ForceAll,
1127}
1128
1129pub type NodeAddress = u32;
1130pub type P2pPort = u16;
1131pub type P2pStreamId = u32;
1132
1133const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1134const P2P_STREAM_VERSION: u8 = 1;
1135const P2P_STREAM_SYN: u8 = 0x01;
1136const P2P_STREAM_ACK: u8 = 0x02;
1137const P2P_STREAM_FIN: u8 = 0x04;
1138const P2P_STREAM_RST: u8 = 0x08;
1139const P2P_STREAM_DATA: u8 = 0x10;
1140
1141#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1142pub enum AddressAssignmentMode {
1143 Dynamic,
1144 Requested(NodeAddress),
1145 Static(NodeAddress),
1146}
1147
1148impl AddressAssignmentMode {
1149 #[inline]
1150 fn mode_code(self) -> u8 {
1151 match self {
1152 Self::Dynamic => 0,
1153 Self::Requested(_) => 1,
1154 Self::Static(_) => 2,
1155 }
1156 }
1157
1158 #[inline]
1159 fn requested_address(self) -> NodeAddress {
1160 match self {
1161 Self::Dynamic => 0,
1162 Self::Requested(addr) | Self::Static(addr) => addr,
1163 }
1164 }
1165}
1166
1167#[derive(Debug, Clone, PartialEq, Eq)]
1168pub enum AddressChangeReason {
1169 Configured,
1170 DynamicConflict,
1171 RequestedConflict,
1172 StaticConflict,
1173 HostnameConflict,
1174}
1175
1176#[derive(Debug, Clone, PartialEq, Eq)]
1177pub struct AddressChange {
1178 pub old_hostname: Arc<str>,
1179 pub new_hostname: Arc<str>,
1180 pub old_address: NodeAddress,
1181 pub new_address: NodeAddress,
1182 pub reason: AddressChangeReason,
1183}
1184
1185#[derive(Debug, Clone, PartialEq, Eq)]
1186pub struct AddressBookEntry {
1187 pub hostname: Arc<str>,
1188 pub address: NodeAddress,
1189 pub requested_address: NodeAddress,
1190 pub mode: AddressAssignmentMode,
1191 pub birth_ms: u64,
1192 pub owner_hash: u64,
1193 pub last_seen_ms: u64,
1194}
1195
1196pub struct P2pMessage<'a> {
1197 pub source_hostname: &'a str,
1198 pub source_address: NodeAddress,
1199 pub source_port: P2pPort,
1200 pub destination_port: P2pPort,
1201 pub payload: &'a [u8],
1202}
1203
1204#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1205pub enum P2pStreamEventKind {
1206 Accepted,
1207 Connected,
1208 Data,
1209 Closed,
1210 Reset,
1211}
1212
1213pub struct P2pStreamEvent<'a> {
1214 pub kind: P2pStreamEventKind,
1215 pub stream_id: P2pStreamId,
1216 pub peer_stream_id: P2pStreamId,
1217 pub sequence: u32,
1218 pub peer_hostname: &'a str,
1219 pub peer_address: NodeAddress,
1220 pub local_port: P2pPort,
1221 pub peer_port: P2pPort,
1222 pub payload: &'a [u8],
1223}
1224
1225struct P2pDecoded<'a> {
1226 source_hostname: &'a str,
1227 source_address: NodeAddress,
1228 source_port: P2pPort,
1229 destination_port: P2pPort,
1230 payload: &'a [u8],
1231}
1232
1233#[derive(Debug, Clone)]
1234struct P2pStreamDecoded<'a> {
1235 flags: u8,
1236 source_stream_id: P2pStreamId,
1237 destination_stream_id: P2pStreamId,
1238 sequence: u32,
1239 payload: &'a [u8],
1240}
1241
1242type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1243
1244#[derive(Clone)]
1245struct AddressChangeHandler {
1246 handler: Arc<AddressChangeFn>,
1247}
1248
1249impl Debug for AddressChangeHandler {
1250 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1251 f.write_str("AddressChangeHandler(<handler>)")
1252 }
1253}
1254
1255type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1256
1257#[derive(Clone)]
1258struct P2pPortHandler {
1259 handler: Arc<P2pPortHandlerFn>,
1260}
1261
1262impl Debug for P2pPortHandler {
1263 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1264 f.write_str("P2pPortHandler(<handler>)")
1265 }
1266}
1267
1268type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1269
1270#[derive(Clone)]
1271struct P2pStreamHandler {
1272 handler: Arc<P2pStreamHandlerFn>,
1273}
1274
1275impl Debug for P2pStreamHandler {
1276 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1277 f.write_str("P2pStreamHandler(<handler>)")
1278 }
1279}
1280
1281#[derive(Debug, Clone)]
1282struct P2pStreamSession {
1283 peer_hostname: Arc<str>,
1284 peer_address: NodeAddress,
1285 local_port: P2pPort,
1286 peer_port: P2pPort,
1287 peer_stream_id: P2pStreamId,
1288 next_sequence: u32,
1289 connected: bool,
1290}
1291
1292#[derive(Debug, Clone)]
1293struct PendingP2pStreamEvent {
1294 handlers: Vec<P2pStreamHandler>,
1295 kind: P2pStreamEventKind,
1296 stream_id: P2pStreamId,
1297 peer_stream_id: P2pStreamId,
1298 peer_hostname: Arc<str>,
1299 peer_address: NodeAddress,
1300 local_port: P2pPort,
1301 peer_port: P2pPort,
1302}
1303
1304#[derive(Debug, Clone)]
1305pub struct RouterConfig {
1306 handlers: Arc<[EndpointHandler]>,
1308 reliable_enabled: bool,
1310 sender: Option<Arc<str>>,
1312 address_mode: AddressAssignmentMode,
1314 address_change_handlers: Arc<[AddressChangeHandler]>,
1316 e2e_encryption: RouterE2eEncryptionMode,
1318 #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
1320 e2e_key_id: u32,
1321 memory: RuntimeMemoryConfig,
1322 #[cfg(feature = "timesync")]
1323 timesync: Option<TimeSyncConfig>,
1324}
1325
1326impl RouterConfig {
1327 pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
1332 #[cfg(feature = "cryptography")]
1333 {
1334 RouterE2eEncryptionMode::Preferred
1335 }
1336 #[cfg(not(feature = "cryptography"))]
1337 {
1338 RouterE2eEncryptionMode::Disabled
1339 }
1340 }
1341
1342 pub fn new<H>(handlers: H) -> Self
1344 where
1345 H: Into<Arc<[EndpointHandler]>>,
1346 {
1347 let handlers: Arc<[EndpointHandler]> = handlers.into();
1348 assert!(
1349 handlers
1350 .iter()
1351 .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
1352 "reserved internal endpoint handlers must not be user-registered"
1353 );
1354 Self {
1355 handlers,
1356 reliable_enabled: true,
1357 sender: None,
1358 address_mode: AddressAssignmentMode::Dynamic,
1359 address_change_handlers: Arc::from([]),
1360 e2e_encryption: Self::default_e2e_encryption_mode(),
1361 e2e_key_id: 0,
1362 memory: RuntimeMemoryConfig::default(),
1363 #[cfg(feature = "timesync")]
1364 timesync: None,
1365 }
1366 }
1367
1368 pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
1370 self.reliable_enabled = enabled;
1371 self
1372 }
1373
1374 pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
1376 self.sender = Some(Arc::from(sender.as_ref()));
1377 self
1378 }
1379
1380 pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
1382 self.with_sender(hostname)
1383 }
1384
1385 pub fn with_dynamic_address(mut self) -> Self {
1387 self.address_mode = AddressAssignmentMode::Dynamic;
1388 self
1389 }
1390
1391 pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
1393 self.address_mode = AddressAssignmentMode::Requested(address);
1394 self
1395 }
1396
1397 pub fn with_static_address(mut self, address: NodeAddress) -> Self {
1399 self.address_mode = AddressAssignmentMode::Static(address);
1400 self
1401 }
1402
1403 pub fn on_address_change<F>(mut self, f: F) -> Self
1405 where
1406 F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
1407 {
1408 let mut handlers = self.address_change_handlers.to_vec();
1409 handlers.push(AddressChangeHandler {
1410 handler: Arc::new(f),
1411 });
1412 self.address_change_handlers = Arc::from(handlers);
1413 self
1414 }
1415
1416 pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
1418 self.e2e_encryption = mode;
1419 self
1420 }
1421
1422 pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
1424 self.e2e_key_id = key_id;
1425 self
1426 }
1427
1428 pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
1430 memory.validate()?;
1431 self.memory = memory;
1432 Ok(self)
1433 }
1434
1435 #[cfg(feature = "timesync")]
1436 pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
1438 self.timesync = Some(cfg);
1439 self
1440 }
1441
1442 #[inline]
1443 fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
1445 if endpoint_is_router_internal(ep) {
1446 return false;
1447 }
1448 self.handlers.iter().any(|h| h.endpoint == ep)
1449 }
1450
1451 #[inline]
1452 fn reliable_enabled(&self) -> bool {
1453 self.reliable_enabled
1454 }
1455
1456 #[inline]
1457 fn sender(&self) -> String {
1458 self.sender
1459 .as_deref()
1460 .map(ToString::to_string)
1461 .unwrap_or_else(runtime_device_identifier)
1462 }
1463
1464 #[inline]
1465 fn address_mode(&self) -> AddressAssignmentMode {
1466 self.address_mode
1467 }
1468
1469 #[inline]
1470 fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
1471 self.e2e_encryption
1472 }
1473
1474 #[cfg(feature = "cryptography")]
1475 #[inline]
1476 fn e2e_key_id(&self) -> u32 {
1477 self.e2e_key_id
1478 }
1479
1480 #[cfg(feature = "timesync")]
1481 #[inline]
1482 fn timesync_config(&self) -> Option<TimeSyncConfig> {
1483 self.timesync
1484 }
1485
1486 #[inline]
1487 fn memory_config(&self) -> RuntimeMemoryConfig {
1488 self.memory
1489 }
1490}
1491
1492impl Default for RouterConfig {
1493 fn default() -> Self {
1494 Self {
1495 handlers: Arc::from([]),
1496 reliable_enabled: true,
1497 sender: None,
1498 address_mode: AddressAssignmentMode::Dynamic,
1499 address_change_handlers: Arc::from([]),
1500 e2e_encryption: Self::default_e2e_encryption_mode(),
1501 e2e_key_id: 0,
1502 memory: RuntimeMemoryConfig::default(),
1503 #[cfg(feature = "timesync")]
1504 timesync: None,
1505 }
1506 }
1507}
1508
1509pub trait LeBytes: Copy + Sized {
1512 const WIDTH: usize;
1513 fn write_le(self, out: &mut [u8]);
1514 fn from_le_slice(bytes: &[u8]) -> Self;
1515}
1516
1517impl_letype_num!(u8, 1);
1518impl_letype_num!(u16, 2);
1519impl_letype_num!(u32, 4);
1520impl_letype_num!(u64, 8);
1521impl_letype_num!(u128, 16);
1522impl_letype_num!(i8, 1);
1523impl_letype_num!(i16, 2);
1524impl_letype_num!(i32, 4);
1525impl_letype_num!(i64, 8);
1526impl_letype_num!(i128, 16);
1527impl_letype_num!(f32, 4);
1528impl_letype_num!(f64, 8);
1529
1530pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
1532 let total = data.len() * T::WIDTH;
1533 let mut buf = vec![0u8; total];
1534
1535 for (i, v) in data.iter().copied().enumerate() {
1536 let start = i * T::WIDTH;
1537 v.write_le(&mut buf[start..start + T::WIDTH]);
1538 }
1539
1540 Arc::from(buf)
1541}
1542
1543fn make_error_payload(msg: &str) -> Arc<[u8]> {
1546 let meta = message_meta(DataType::TelemetryError);
1547 match meta.element {
1548 MessageElement::Static(_, _, _) => {
1549 let max = get_needed_message_size(DataType::TelemetryError);
1550 let bytes = msg.as_bytes();
1551 let n = core::cmp::min(max, bytes.len());
1552 let mut buf = vec![0u8; max];
1553 if n > 0 {
1554 buf[..n].copy_from_slice(&bytes[..n]);
1555 }
1556 Arc::from(buf)
1557 }
1558 MessageElement::Dynamic(_, _) => Arc::from(msg.as_bytes()),
1559 }
1560}
1561
1562fn log_raw<T, F>(
1566 sender: &str,
1567 ty: DataType,
1568 data: &[T],
1569 timestamp: u64,
1570 mut tx_function: F,
1571) -> TelemetryResult<()>
1572where
1573 T: LeBytes,
1574 F: FnMut(Packet) -> TelemetryResult<()>,
1575{
1576 let meta = message_meta(ty);
1577 let got = data.len() * T::WIDTH;
1578
1579 match meta.element {
1580 MessageElement::Static(_, _, _) => {
1581 if got != get_needed_message_size(ty) {
1582 return Err(TelemetryError::SizeMismatch {
1583 expected: get_needed_message_size(ty),
1584 got,
1585 });
1586 }
1587 }
1588 MessageElement::Dynamic(_, _) => {
1589 if !got.is_multiple_of(T::WIDTH) {
1591 return Err(TelemetryError::SizeMismatch {
1592 expected: T::WIDTH,
1593 got,
1594 });
1595 }
1596 }
1597 }
1598
1599 let payload = encode_slice_le(data);
1600 let pkt = Packet::new(ty, meta.endpoints, sender, timestamp, payload)?;
1601 tx_function(pkt)
1602}
1603
1604fn fallback_stdout(msg: &str) {
1608 #[cfg(feature = "std")]
1609 {
1610 eprintln!("{}", msg);
1611 }
1612 #[cfg(all(not(feature = "std"), target_os = "none"))]
1613 {
1614 let message = format!("{}\n", msg);
1615 unsafe {
1616 seds_error_msg(message.as_ptr(), message.len());
1617 }
1618 }
1619}
1620
1621#[derive(Debug, Clone)]
1626struct RouterInner {
1627 memory: RuntimeMemoryConfig,
1628 sides: Vec<Option<RouterSide>>,
1629 route_overrides: BTreeMap<(Option<RouterSideId>, RouterSideId), bool>,
1630 typed_route_overrides: BTreeMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
1631 route_weights: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
1632 route_priorities: BTreeMap<(Option<RouterSideId>, RouterSideId), u32>,
1633 source_route_modes: BTreeMap<Option<RouterSideId>, RouteSelectionMode>,
1634 route_selection_cursors: BTreeMap<Option<RouterSideId>, u64>,
1635 adaptive_route_stats: BTreeMap<RouterSideId, AdaptiveRouteStats>,
1636 side_runtime_stats: BTreeMap<RouterSideId, SideRuntimeStatsInner>,
1637 side_transport: BTreeMap<RouterSideId, SideTransportState>,
1638 managed_variable_types: BTreeSet<u32>,
1639 managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
1640 managed_variable_latest: BTreeMap<u32, ManagedVariableCacheEntry>,
1641 network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
1642 local_address: AddressBookEntry,
1643 address_book: BTreeMap<String, AddressBookEntry>,
1644 address_by_value: BTreeMap<NodeAddress, String>,
1645 p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
1646 p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
1647 p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
1648 next_p2p_stream_id: P2pStreamId,
1649 received_queue: BoundedDeque<RouterRxItem>,
1650 transmit_queue: BoundedDeque<TxQueued>,
1651 recent_rx: BoundedDeque<u64>,
1652 reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
1653 reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
1654 reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
1655 reliable_return_route_order: VecDeque<u64>,
1656 end_to_end_reliable_tx: BTreeMap<u64, EndToEndReliableSent>,
1657 end_to_end_reliable_tx_order: VecDeque<u64>,
1658 total_handler_failures: u64,
1659 total_handler_retries: u64,
1660 #[cfg(feature = "discovery")]
1661 discovery_routes: BTreeMap<RouterSideId, DiscoverySideState>,
1662 #[cfg(feature = "discovery")]
1663 discovery_cadence: DiscoveryCadenceState,
1664 #[cfg(feature = "discovery")]
1665 discovery_side_throttle: BTreeMap<RouterSideId, DiscoverySideThrottleState>,
1666 #[cfg(all(feature = "discovery", feature = "timesync"))]
1667 timesync_side_throttle: BTreeMap<RouterSideId, TimeSyncSideThrottleState>,
1668}
1669
1670#[derive(Debug, Clone)]
1671struct ManagedVariableCacheEntry {
1672 packet: Packet,
1673 cached_at_ms: u64,
1674}
1675
1676type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1677
1678#[derive(Clone)]
1679struct NetworkVariableUpdateHandler {
1680 handler: Arc<NetworkVariableUpdateFn>,
1681}
1682
1683impl Debug for NetworkVariableUpdateHandler {
1684 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1685 f.write_str("NetworkVariableUpdateHandler(<handler>)")
1686 }
1687}
1688
1689#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1691pub struct NetworkVariablePermissions {
1692 pub read: bool,
1693 pub write: bool,
1694}
1695
1696impl NetworkVariablePermissions {
1697 pub const NONE: Self = Self {
1698 read: false,
1699 write: false,
1700 };
1701 pub const READ_ONLY: Self = Self {
1702 read: true,
1703 write: false,
1704 };
1705 pub const WRITE_ONLY: Self = Self {
1706 read: false,
1707 write: true,
1708 };
1709 pub const READ_WRITE: Self = Self {
1710 read: true,
1711 write: true,
1712 };
1713}
1714
1715#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1716enum RouterQueueKind {
1717 Received,
1718 Transmit,
1719 Recent,
1720 ReliableRxBuffer,
1721 #[cfg(feature = "discovery")]
1722 Discovery,
1723}
1724
1725impl RouterInner {
1726 #[cfg(feature = "discovery")]
1727 fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
1728 board
1729 .sender_id
1730 .len()
1731 .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
1732 .saturating_add(
1733 board
1734 .reachable_timesync_sources
1735 .iter()
1736 .map(|s| s.len())
1737 .sum::<usize>(),
1738 )
1739 .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
1740 }
1741
1742 #[cfg(feature = "discovery")]
1743 fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
1744 sender
1745 .len()
1746 .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
1747 .saturating_add(
1748 state
1749 .reachable_timesync_sources
1750 .iter()
1751 .map(|s| s.len())
1752 .sum::<usize>(),
1753 )
1754 .saturating_add(
1755 state
1756 .topology_boards
1757 .iter()
1758 .map(Self::topology_board_byte_cost)
1759 .sum::<usize>(),
1760 )
1761 .saturating_add(size_of::<DiscoverySenderState>())
1762 }
1763
1764 #[cfg(feature = "discovery")]
1765 fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
1766 size_of::<RouterSideId>()
1767 .saturating_add(size_of::<DiscoverySideState>())
1768 .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
1769 .saturating_add(
1770 route
1771 .reachable_timesync_sources
1772 .iter()
1773 .map(|s| s.len())
1774 .sum::<usize>(),
1775 )
1776 .saturating_add(
1777 route
1778 .announcers
1779 .iter()
1780 .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
1781 .sum::<usize>(),
1782 )
1783 .saturating_add(side.saturating_sub(side))
1784 }
1785
1786 #[cfg(feature = "discovery")]
1787 fn discovery_bytes_used(&self) -> usize {
1788 self.discovery_routes
1789 .iter()
1790 .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
1791 .sum()
1792 }
1793
1794 #[inline]
1795 fn reliable_rx_buffered_bytes(&self) -> usize {
1796 self.reliable_rx
1797 .values()
1798 .flat_map(|state| state.buffered.values())
1799 .map(|bytes| size_of::<Arc<[u8]>>() + bytes.len())
1800 .sum()
1801 }
1802
1803 #[inline]
1804 fn shared_queue_bytes_used(&self) -> usize {
1805 self.received_queue
1806 .bytes_used()
1807 .saturating_add(self.transmit_queue.bytes_used())
1808 .saturating_add(self.recent_rx.max_bytes())
1809 .saturating_add(self.reliable_rx_buffered_bytes())
1810 .saturating_add(crate::config::schema_bytes_used())
1811 .saturating_add({
1812 #[cfg(feature = "discovery")]
1813 {
1814 self.discovery_bytes_used()
1815 }
1816 #[cfg(not(feature = "discovery"))]
1817 {
1818 0
1819 }
1820 })
1821 }
1822
1823 fn reliable_rx_buffer_len(&self) -> usize {
1824 self.reliable_rx
1825 .values()
1826 .map(|state| state.buffered.len())
1827 .sum()
1828 }
1829
1830 fn pop_reliable_rx_buffered(&mut self) -> Option<Arc<[u8]>> {
1831 let key = self
1832 .reliable_rx
1833 .iter()
1834 .find_map(|(key, state)| (!state.buffered.is_empty()).then_some(*key))?;
1835 self.reliable_rx
1836 .get_mut(&key)?
1837 .buffered
1838 .pop_first()
1839 .map(|(_, v)| v)
1840 }
1841
1842 fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
1843 match preferred {
1844 RouterQueueKind::Received => self.received_queue.pop_front().is_some(),
1845 RouterQueueKind::Transmit => self.transmit_queue.pop_front().is_some(),
1846 RouterQueueKind::Recent => self.recent_rx.pop_front().is_some(),
1847 RouterQueueKind::ReliableRxBuffer => self.pop_reliable_rx_buffered().is_some(),
1848 #[cfg(feature = "discovery")]
1849 RouterQueueKind::Discovery => self.pop_discovery_route(),
1850 }
1851 }
1852
1853 #[cfg(feature = "discovery")]
1854 fn pop_discovery_route(&mut self) -> bool {
1855 let Some((&side, _)) = self
1856 .discovery_routes
1857 .iter()
1858 .min_by_key(|(_, route)| route.last_seen_ms)
1859 else {
1860 return false;
1861 };
1862 self.discovery_routes.remove(&side);
1863 Self::queue_budget_warning("topology route evicted because shared queue budget is full");
1864 true
1865 }
1866
1867 fn largest_shared_queue(&self) -> Option<RouterQueueKind> {
1868 let candidates = [
1869 (
1870 RouterQueueKind::Received,
1871 self.received_queue.bytes_used(),
1872 self.received_queue.len(),
1873 ),
1874 (
1875 RouterQueueKind::Transmit,
1876 self.transmit_queue.bytes_used(),
1877 self.transmit_queue.len(),
1878 ),
1879 (RouterQueueKind::Recent, 0, 0),
1880 (
1881 RouterQueueKind::ReliableRxBuffer,
1882 self.reliable_rx_buffered_bytes(),
1883 self.reliable_rx_buffer_len(),
1884 ),
1885 #[cfg(feature = "discovery")]
1886 (
1887 RouterQueueKind::Discovery,
1888 self.discovery_bytes_used(),
1889 self.discovery_routes.len(),
1890 ),
1891 ];
1892 candidates
1893 .into_iter()
1894 .filter(|(_, bytes, len)| *bytes > 0 && *len > 0)
1895 .max_by_key(|(kind, bytes, _)| {
1896 (
1897 *bytes,
1898 if *kind == RouterQueueKind::ReliableRxBuffer {
1899 0
1900 } else {
1901 1
1902 },
1903 )
1904 })
1905 .map(|(kind, _, _)| kind)
1906 }
1907
1908 fn make_shared_queue_room(
1909 &mut self,
1910 incoming_cost: usize,
1911 preferred: RouterQueueKind,
1912 ) -> TelemetryResult<()> {
1913 if incoming_cost > self.memory.max_queue_budget {
1914 return Err(TelemetryError::PacketTooLarge(
1915 "Item exceeds maximum shared queue budget",
1916 ));
1917 }
1918
1919 while self.shared_queue_bytes_used().saturating_add(incoming_cost)
1920 > self.memory.max_queue_budget
1921 {
1922 let victim = self.largest_shared_queue().unwrap_or(preferred);
1923 if victim == RouterQueueKind::Discovery {
1924 Self::queue_budget_warning("topology data is using the largest queue budget share");
1925 }
1926 if !self.pop_shared_queue_item(victim) && !self.pop_shared_queue_item(preferred) {
1927 return Err(TelemetryError::PacketTooLarge(
1928 "Item exceeds maximum shared queue budget",
1929 ));
1930 }
1931 }
1932
1933 Ok(())
1934 }
1935
1936 #[inline]
1937 fn queue_budget_warning(msg: &str) {
1938 #[cfg(feature = "std")]
1939 eprintln!("sedsnet queue budget warning: {msg}");
1940 let _ = msg;
1941 }
1942
1943 #[cfg(feature = "discovery")]
1944 fn fit_discovery_budget(&mut self) {
1945 while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
1946 if !self.pop_discovery_route() {
1947 break;
1948 }
1949 }
1950 }
1951
1952 fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
1953 self.make_shared_queue_room(item.byte_cost(), RouterQueueKind::Received)?;
1954 self.received_queue
1955 .push_back_prioritized(item, |queued| queued.priority)
1956 }
1957
1958 fn push_transmit(&mut self, item: TxQueued) -> TelemetryResult<()> {
1959 self.make_shared_queue_room(item.byte_cost(), RouterQueueKind::Transmit)?;
1960 self.transmit_queue
1961 .push_back_prioritized(item, |queued| queued.priority)
1962 }
1963
1964 fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
1965 while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
1966 let _ = self.recent_rx.pop_front();
1967 }
1968 self.make_shared_queue_room(0, RouterQueueKind::Recent)?;
1969 self.recent_rx.push_back(id)
1970 }
1971
1972 fn buffer_reliable_rx(
1973 &mut self,
1974 side: RouterSideId,
1975 ty: DataType,
1976 seq: u32,
1977 bytes: Arc<[u8]>,
1978 ) -> TelemetryResult<()> {
1979 let key = Router::reliable_key(side, ty);
1980 if self
1981 .reliable_rx
1982 .get(&key)
1983 .is_some_and(|state| state.buffered.contains_key(&seq))
1984 {
1985 return Ok(());
1986 }
1987 let cost = size_of::<Arc<[u8]>>() + bytes.len();
1988 self.make_shared_queue_room(cost, RouterQueueKind::ReliableRxBuffer)?;
1989 let rx_state = self
1990 .reliable_rx
1991 .entry(key)
1992 .or_insert_with(|| ReliableRxState {
1993 expected_seq: 1,
1994 buffered: BTreeMap::new(),
1995 });
1996 if rx_state.buffered.len() >= runtime_reliable_max_pending() {
1997 let _ = rx_state.buffered.pop_first();
1998 }
1999 rx_state.buffered.insert(seq, bytes);
2000 Ok(())
2001 }
2002}
2003
2004struct IsrRxQueue {
2009 busy: AtomicBool,
2010 q: UnsafeCell<BoundedDeque<RouterRxItem>>,
2011}
2012
2013unsafe impl Send for IsrRxQueue {}
2014unsafe impl Sync for IsrRxQueue {}
2015
2016struct IsrRxQueueGuard<'a> {
2017 owner: &'a IsrRxQueue,
2018}
2019
2020impl Deref for IsrRxQueueGuard<'_> {
2021 type Target = BoundedDeque<RouterRxItem>;
2022
2023 #[inline]
2024 fn deref(&self) -> &Self::Target {
2025 unsafe { &*self.owner.q.get() }
2026 }
2027}
2028
2029impl DerefMut for IsrRxQueueGuard<'_> {
2030 #[inline]
2031 fn deref_mut(&mut self) -> &mut Self::Target {
2032 unsafe { &mut *self.owner.q.get() }
2033 }
2034}
2035
2036impl Drop for IsrRxQueueGuard<'_> {
2037 #[inline]
2038 fn drop(&mut self) {
2039 self.owner.busy.store(false, Ordering::Release);
2040 }
2041}
2042
2043impl IsrRxQueue {
2044 #[inline]
2045 fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
2046 Self {
2047 busy: AtomicBool::new(false),
2048 q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
2049 }
2050 }
2051
2052 #[inline]
2053 fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
2054 match self
2055 .busy
2056 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
2057 {
2058 Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
2059 Err(_) => Err(TelemetryError::Io("rx queue busy")),
2060 }
2061 }
2062
2063 #[allow(dead_code)]
2064 #[inline]
2065 fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
2066 let mut g = self.try_lock()?;
2067 g.push_back(item)
2068 }
2069
2070 #[inline]
2071 fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
2072 let mut g = self.try_lock()?;
2073 g.push_back_prioritized(item, |queued| queued.priority)
2074 }
2075
2076 #[inline]
2077 fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
2078 let mut g = self.try_lock()?;
2079 Ok(g.pop_front())
2080 }
2081
2082 #[inline]
2083 fn clear(&self) -> TelemetryResult<()> {
2084 let mut g = self.try_lock()?;
2085 g.clear();
2086 Ok(())
2087 }
2088
2089 #[inline]
2090 fn snapshot(&self) -> Option<(usize, usize)> {
2091 let g = self.try_lock().ok()?;
2092 Some((g.len(), g.bytes_used()))
2093 }
2094}
2095
2096pub struct Router {
2100 sender: RouterMutex<Arc<str>>,
2101 cfg: RouterConfig,
2102 state: RouterMutex<RouterInner>,
2103 isr_rx_queue: IsrRxQueue,
2104 side_tx_gate: ReentryGate,
2105 clock: Box<dyn Clock + Send + Sync>,
2106 #[cfg(feature = "timesync")]
2107 timesync: RouterMutex<TimeSyncRuntime>,
2108}
2109
2110#[cfg(feature = "timesync")]
2111#[derive(Debug, Clone)]
2112struct PendingTimeSyncRequest {
2113 seq: u64,
2114 t1_mono_ms: u64,
2115 source: String,
2116}
2117
2118#[cfg(feature = "timesync")]
2119#[derive(Debug, Clone)]
2120struct RemoteTimeSyncSource {
2121 priority: u64,
2122 last_sample_mono_ms: u64,
2123 sample_unix_ms: u64,
2124}
2125
2126#[cfg(feature = "timesync")]
2127#[derive(Debug, Clone)]
2128struct TimeSyncRuntime {
2129 cfg: Option<TimeSyncConfig>,
2130 tracker: Option<TimeSyncTracker>,
2131 clock: NetworkClock,
2132 disciplined_clock: SlewedNetworkClock,
2133 remote_sources: BTreeMap<String, RemoteTimeSyncSource>,
2134 next_seq: u64,
2135 next_announce_mono_ms: u64,
2136 next_request_mono_ms: u64,
2137 pending_request: Option<PendingTimeSyncRequest>,
2138}
2139
2140#[cfg(feature = "timesync")]
2141impl TimeSyncRuntime {
2142 fn new(cfg: Option<TimeSyncConfig>) -> Self {
2143 Self {
2144 tracker: cfg.map(TimeSyncTracker::new),
2145 cfg,
2146 clock: NetworkClock::default(),
2147 disciplined_clock: SlewedNetworkClock::new(
2148 cfg.map(|c| c.max_slew_ppm)
2149 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
2150 ),
2151 remote_sources: BTreeMap::new(),
2152 next_seq: 1,
2153 next_announce_mono_ms: 0,
2154 next_request_mono_ms: 0,
2155 pending_request: None,
2156 }
2157 }
2158}
2159
2160enum RemoteSidePlan {
2161 Target(Vec<RouterSideId>),
2162}
2163
2164#[cfg(feature = "discovery")]
2165#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
2166struct DiscoveryCandidateMatch {
2167 side: RouterSideId,
2168 overlap: usize,
2169}
2170
2171impl Debug for Router {
2172 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2173 let sender = self.sender();
2174 f.debug_struct("Router")
2175 .field("sender", &sender)
2176 .field("cfg", &self.cfg)
2177 .field("state", &"<mutex>")
2178 .field("clock", &"Clock")
2179 .finish()
2180 }
2181}
2182
2183#[inline]
2186fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &RouterConfig) -> bool {
2187 eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
2188}
2189
2190#[inline]
2191fn force_remote_for_type(ty: DataType) -> bool {
2192 matches!(
2193 ty,
2194 DataType::ReliableAck
2195 | DataType::ReliablePartialAck
2196 | DataType::ReliablePacketRequest
2197 | DataType::P2pMessage
2198 ) || {
2199 #[cfg(feature = "timesync")]
2200 {
2201 matches!(
2202 ty,
2203 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2204 )
2205 }
2206 #[cfg(not(feature = "timesync"))]
2207 {
2208 false
2209 }
2210 }
2211}
2212
2213#[inline]
2214fn is_internal_control_type(ty: DataType) -> bool {
2215 if matches!(
2216 ty,
2217 DataType::ReliableAck
2218 | DataType::ReliablePartialAck
2219 | DataType::ReliablePacketRequest
2220 | DataType::P2pMessage
2221 ) {
2222 return true;
2223 }
2224
2225 #[cfg(feature = "timesync")]
2226 if matches!(
2227 ty,
2228 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
2229 ) {
2230 return true;
2231 }
2232
2233 #[cfg(feature = "discovery")]
2234 if discovery::is_discovery_type(ty) {
2235 return true;
2236 }
2237
2238 let _ = ty;
2239 false
2240}
2241
2242fn with_retries<F>(
2244 this: &Router,
2245 dest: DataEndpoint,
2246 data: &RouterItem,
2247 pkt_for_ctx: Option<&Packet>,
2248 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
2249 called_from_queue: bool,
2250 run: F,
2251) -> TelemetryResult<()>
2252where
2253 F: Fn() -> TelemetryResult<()>,
2254{
2255 match this.retry_with_attempts(runtime_max_handler_retries(), run) {
2256 Ok(((), attempts)) => {
2257 if attempts > 1 {
2258 let mut st = this.state.lock();
2259 st.total_handler_retries = st
2260 .total_handler_retries
2261 .saturating_add(attempts.saturating_sub(1) as u64);
2262 }
2263 Ok(())
2264 }
2265 Err((e, attempts)) => {
2266 {
2267 let mut st = this.state.lock();
2268 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
2269 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
2270 }
2271
2272 this.remove_pkt_id(data);
2274
2275 if let Some(pkt) = pkt_for_ctx {
2277 let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
2278 } else if let Some(env) = env_for_ctx {
2279 let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
2280 }
2281
2282 Err(TelemetryError::HandlerError("local handler failed"))
2283 }
2284 }
2285}
2286impl Router {
2288 const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
2289 const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
2290
2291 #[inline]
2292 fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
2293 st.sides
2294 .get(side)
2295 .and_then(|side| side.as_ref())
2296 .ok_or(TelemetryError::HandlerError("router: invalid side id"))
2297 }
2298
2299 fn note_side_tx_success(
2300 &self,
2301 side: RouterSideId,
2302 ty: DataType,
2303 bytes: usize,
2304 relayed: bool,
2305 attempts: usize,
2306 ) {
2307 let mut st = self.state.lock();
2308 let entry = st.side_runtime_stats.entry(side).or_default();
2309 entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
2310 }
2311
2312 fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
2313 let mut st = self.state.lock();
2314 st.total_handler_failures = st.total_handler_failures.saturating_add(1);
2315 st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
2316 let entry = st.side_runtime_stats.entry(side).or_default();
2317 entry.note_tx_failure(ty, attempts);
2318 }
2319
2320 fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
2321 let mut st = self.state.lock();
2322 let entry = st.side_runtime_stats.entry(side).or_default();
2323 entry.note_rx(ty, bytes, relayed);
2324 }
2325
2326 fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
2327 let mut st = self.state.lock();
2328 let entry = st.side_runtime_stats.entry(side).or_default();
2329 entry.note_local_delivery(ty);
2330 }
2331
2332 fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
2333 let mut st = self.state.lock();
2334 let entry = st.side_runtime_stats.entry(side).or_default();
2335 entry.note_local_handler_failure(ty, retries);
2336 }
2337
2338 fn cache_managed_variable_packet(
2339 &self,
2340 pkt: &Packet,
2341 notify_handlers: bool,
2342 ) -> TelemetryResult<()> {
2343 let handlers = {
2344 let mut st = self.state.lock();
2345 if !st
2346 .managed_variable_types
2347 .contains(&pkt.data_type().as_u32())
2348 {
2349 return Ok(());
2350 }
2351 let changed = st
2352 .managed_variable_latest
2353 .get(&pkt.data_type().as_u32())
2354 .is_none_or(|entry| entry.packet != *pkt);
2355 st.managed_variable_latest.insert(
2356 pkt.data_type().as_u32(),
2357 ManagedVariableCacheEntry {
2358 packet: pkt.clone(),
2359 cached_at_ms: self.clock.now_ms(),
2360 },
2361 );
2362 if notify_handlers && changed {
2363 st.network_variable_update_handlers
2364 .get(&pkt.data_type().as_u32())
2365 .cloned()
2366 .unwrap_or_default()
2367 } else {
2368 Vec::new()
2369 }
2370 };
2371 for handler in handlers {
2372 (handler.handler)(pkt)?;
2373 }
2374 Ok(())
2375 }
2376
2377 fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
2378 self.cache_managed_variable_packet(pkt, true)
2379 }
2380
2381 fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
2382 let st = self.state.lock();
2383 st.managed_variable_latest
2384 .get(&ty.as_u32())
2385 .map(|entry| entry.packet.clone())
2386 }
2387
2388 fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
2389 let now_ms = self.clock.now_ms();
2390 let st = self.state.lock();
2391 st.managed_variable_latest.get(&ty.as_u32()).map(|entry| {
2392 (
2393 entry.packet.clone(),
2394 now_ms.saturating_sub(entry.cached_at_ms),
2395 )
2396 })
2397 }
2398
2399 fn is_managed_variable_type(&self, ty: DataType) -> bool {
2400 let st = self.state.lock();
2401 st.managed_variable_types.contains(&ty.as_u32())
2402 }
2403
2404 fn managed_variable_permissions_locked(
2405 st: &RouterInner,
2406 ty: DataType,
2407 ) -> NetworkVariablePermissions {
2408 st.managed_variable_permissions
2409 .get(&ty.as_u32())
2410 .copied()
2411 .unwrap_or(NetworkVariablePermissions::READ_WRITE)
2412 }
2413
2414 fn can_read_managed_variable(&self, ty: DataType) -> bool {
2415 let st = self.state.lock();
2416 Self::managed_variable_permissions_locked(&st, ty).read
2417 }
2418
2419 #[inline]
2420 fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
2421 let st = self.state.lock();
2422 let side_ref = Self::side_ref(&st, side)?;
2423 if side_ref.opts.ingress_enabled {
2424 Ok(())
2425 } else {
2426 Err(TelemetryError::HandlerError(
2427 "router: ingress disabled for side id",
2428 ))
2429 }
2430 }
2431
2432 #[inline]
2433 fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
2434 src != Some(dst)
2435 }
2436
2437 #[inline]
2438 fn route_allowed_locked(
2439 &self,
2440 st: &RouterInner,
2441 src: Option<RouterSideId>,
2442 ty: Option<DataType>,
2443 dst: RouterSideId,
2444 ) -> bool {
2445 let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
2446 return false;
2447 };
2448 if !dst_side.opts.egress_enabled {
2449 return false;
2450 }
2451 if let Some(src_id) = src {
2452 let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
2453 return false;
2454 };
2455 if !src_side.opts.ingress_enabled {
2456 return false;
2457 }
2458 }
2459 let base_allowed = st
2460 .route_overrides
2461 .get(&(src, dst))
2462 .copied()
2463 .unwrap_or_else(|| self.default_route_enabled(src, dst));
2464 if !base_allowed {
2465 return false;
2466 }
2467
2468 let Some(ty) = ty else {
2469 return true;
2470 };
2471 if st
2472 .typed_route_overrides
2473 .keys()
2474 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
2475 {
2476 return st
2477 .typed_route_overrides
2478 .get(&(src, ty.as_u32(), dst))
2479 .copied()
2480 .unwrap_or(false);
2481 }
2482 true
2483 }
2484
2485 fn has_typed_route_overrides_locked(
2486 st: &RouterInner,
2487 src: Option<RouterSideId>,
2488 ty: DataType,
2489 ) -> bool {
2490 st.typed_route_overrides
2491 .keys()
2492 .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
2493 }
2494
2495 #[cfg(feature = "discovery")]
2496 fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
2497 where
2498 I: IntoIterator<Item = DataEndpoint>,
2499 {
2500 let mut overlap = 0usize;
2501 for ep in reachable {
2502 if eps.contains(&ep) {
2503 overlap = overlap.saturating_add(1);
2504 }
2505 }
2506 overlap
2507 }
2508
2509 #[inline]
2510 fn preferred_scoring_endpoints(
2511 &self,
2512 eps: &[DataEndpoint],
2513 prefer_nonlocal: bool,
2514 ) -> Vec<DataEndpoint> {
2515 if !prefer_nonlocal {
2516 return eps.to_vec();
2517 }
2518 let nonlocal: Vec<DataEndpoint> = eps
2519 .iter()
2520 .copied()
2521 .filter(|&ep| !self.cfg.is_local_endpoint(ep))
2522 .collect();
2523 if nonlocal.is_empty() {
2524 eps.to_vec()
2525 } else {
2526 nonlocal
2527 }
2528 }
2529
2530 #[cfg(feature = "discovery")]
2531 fn pop_next_queued_discovery_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
2532 {
2533 let mut isr_rx = self.isr_rx_queue.try_lock()?;
2534 let idx = isr_rx.iter().position(Self::queued_rx_item_is_discovery);
2535 if let Some(idx) = idx {
2536 return Ok(isr_rx.remove_pos(idx));
2537 }
2538 }
2539
2540 let mut st = self.state.lock();
2541 let idx = st
2542 .received_queue
2543 .iter()
2544 .position(Self::queued_rx_item_is_discovery);
2545 if let Some(idx) = idx {
2546 return Ok(st.received_queue.remove_pos(idx));
2547 }
2548 Ok(None)
2549 }
2550
2551 #[cfg(feature = "discovery")]
2552 fn queued_rx_item_is_discovery(item: &RouterRxItem) -> bool {
2553 match &item.data {
2554 RouterItem::Packet(pkt) => discovery::is_discovery_type(pkt.data_type()),
2555 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())
2556 .map(|env| discovery::is_discovery_type(env.ty))
2557 .unwrap_or(false),
2558 }
2559 }
2560
2561 #[cfg(feature = "discovery")]
2562 fn drain_queued_discovery_rx_before_tx(&self) -> TelemetryResult<bool> {
2563 let mut did_any = false;
2564 while let Some(item) = self.pop_next_queued_discovery_rx_item()? {
2565 self.process_rx_queue_item(item)?;
2566 did_any = true;
2567 }
2568 Ok(did_any)
2569 }
2570
2571 fn eligible_side_ids_locked(
2572 &self,
2573 st: &RouterInner,
2574 src: Option<RouterSideId>,
2575 ty: Option<DataType>,
2576 restrict_link_local: bool,
2577 ) -> Vec<RouterSideId> {
2578 st.sides
2579 .iter()
2580 .enumerate()
2581 .filter_map(|(side_id, side)| {
2582 let side = side.as_ref()?;
2583 if restrict_link_local && !side.opts.link_local_enabled {
2584 return None;
2585 }
2586 if self.route_allowed_locked(st, src, ty, side_id) {
2587 Some(side_id)
2588 } else {
2589 None
2590 }
2591 })
2592 .collect()
2593 }
2594
2595 fn apply_route_selection_locked(
2596 &self,
2597 st: &mut RouterInner,
2598 src: Option<RouterSideId>,
2599 mut sides: Vec<RouterSideId>,
2600 origin: RouteSelectionOrigin,
2601 ) -> Vec<RouterSideId> {
2602 if sides.len() <= 1 {
2603 return sides;
2604 }
2605
2606 let selection_mode = st.source_route_modes.get(&src).copied();
2607 if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
2608 return self.apply_adaptive_discovery_selection_locked(st, src, sides);
2609 }
2610
2611 match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
2612 RouteSelectionMode::Fanout => sides,
2613 RouteSelectionMode::Weighted => {
2614 sides.sort_unstable();
2615 let total_weight = sides.iter().fold(0_u64, |acc, side| {
2616 acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
2617 });
2618 if total_weight == 0 {
2619 return Vec::new();
2620 }
2621 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2622 let pick = *cursor % total_weight;
2623 *cursor = cursor.wrapping_add(1);
2624 let mut remaining = pick;
2625 for side in sides {
2626 let weight =
2627 u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
2628 if remaining < weight {
2629 return vec![side];
2630 }
2631 remaining -= weight;
2632 }
2633 Vec::new()
2634 }
2635 RouteSelectionMode::Failover => {
2636 sides.sort_by_key(|side| {
2637 (
2638 st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
2639 *side,
2640 )
2641 });
2642 sides.truncate(1);
2643 sides
2644 }
2645 }
2646 }
2647
2648 fn apply_adaptive_discovery_selection_locked(
2649 &self,
2650 st: &mut RouterInner,
2651 src: Option<RouterSideId>,
2652 mut sides: Vec<RouterSideId>,
2653 ) -> Vec<RouterSideId> {
2654 sides.sort_unstable();
2655 let mut unmeasured: Vec<_> = sides
2656 .iter()
2657 .copied()
2658 .filter(|side| !st.adaptive_route_stats.contains_key(side))
2659 .collect();
2660 if !unmeasured.is_empty() {
2661 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2662 let pick = (*cursor as usize) % unmeasured.len();
2663 *cursor = cursor.wrapping_add(1);
2664 return vec![unmeasured.swap_remove(pick)];
2665 }
2666
2667 let now_ms = self.clock.now_ms();
2668 let total_weight = sides.iter().fold(0_u64, |acc, side| {
2669 acc + st
2670 .adaptive_route_stats
2671 .get(side)
2672 .map(|stats| stats.weight(now_ms))
2673 .unwrap_or(1)
2674 });
2675 if total_weight == 0 {
2676 sides.truncate(1);
2677 return sides;
2678 }
2679
2680 let cursor = st.route_selection_cursors.entry(src).or_insert(0);
2681 let pick = *cursor % total_weight;
2682 *cursor = cursor.wrapping_add(1);
2683 let mut remaining = pick;
2684 for side in sides {
2685 let weight = st
2686 .adaptive_route_stats
2687 .get(&side)
2688 .map(|stats| stats.weight(now_ms))
2689 .unwrap_or(1);
2690 if remaining < weight {
2691 return vec![side];
2692 }
2693 remaining -= weight;
2694 }
2695 Vec::new()
2696 }
2697
2698 fn record_side_tx_sample(
2699 &self,
2700 side: RouterSideId,
2701 bytes: usize,
2702 started_ms: u64,
2703 ended_ms: u64,
2704 ) {
2705 let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
2706 let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
2707 .min(u128::from(u64::MAX)) as u64;
2708 let mut st = self.state.lock();
2709 st.adaptive_route_stats
2710 .entry(side)
2711 .or_default()
2712 .observe(bytes, sample_bps, ended_ms);
2713 }
2714
2715 pub fn note_side_link_probe_sample(
2720 &self,
2721 side: RouterSideId,
2722 bytes: usize,
2723 duration_ms: u64,
2724 ) -> TelemetryResult<()> {
2725 {
2726 let st = self.state.lock();
2727 let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
2728 }
2729 let ended_ms = self.clock.now_ms();
2730 self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
2731 Ok(())
2732 }
2733
2734 fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
2735 match data {
2736 RouterItem::Packet(pkt) => Ok(wire_format::pack_packet(pkt).len()),
2737 RouterItem::Packed(bytes) => Ok(bytes.len()),
2738 }
2739 }
2740
2741 #[inline]
2751 fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
2752 match data {
2753 RouterItem::Packet(pkt) => {
2754 if pkt.data_type() != DataType::ReliableAck
2755 || !Self::is_end_to_end_ack_sender(pkt.sender())
2756 {
2757 return Ok(None);
2758 }
2759 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
2760 }
2761 RouterItem::Packed(bytes) => {
2762 if wire_format::peek_frame_info(bytes.as_ref())
2763 .ok()
2764 .is_some_and(|frame| frame.ack_only())
2765 {
2766 return Ok(None);
2767 }
2768 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
2769 if pkt.data_type() != DataType::ReliableAck
2770 || !Self::is_end_to_end_ack_sender(pkt.sender())
2771 {
2772 return Ok(None);
2773 }
2774 Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
2775 }
2776 }
2777 }
2778
2779 fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
2780 if payload.len() != 8 {
2781 return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
2782 }
2783 Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
2784 }
2785
2786 #[inline]
2787 fn sender_hash(sender: &str) -> u64 {
2788 hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
2789 }
2790
2791 #[inline]
2792 fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
2793 let hash = Self::sender_hash(hostname);
2794 let mut address = (hash as u32) ^ ((hash >> 32) as u32);
2795 if address == 0 {
2796 address = 1;
2797 }
2798 address
2799 }
2800
2801 fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
2802 match mode {
2803 2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
2804 1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
2805 _ => AddressAssignmentMode::Dynamic,
2806 }
2807 }
2808
2809 fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
2810 let rank = match entry.mode {
2811 AddressAssignmentMode::Static(_) => 2,
2812 AddressAssignmentMode::Requested(_) => 1,
2813 AddressAssignmentMode::Dynamic => 0,
2814 };
2815 (
2816 rank,
2817 core::cmp::Reverse(entry.birth_ms),
2818 u64::MAX - entry.owner_hash,
2819 )
2820 }
2821
2822 fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
2823 (
2824 core::cmp::Reverse(entry.birth_ms),
2825 u64::MAX - entry.owner_hash,
2826 )
2827 }
2828
2829 fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
2830 let mut candidate = if seed == 0 { 1 } else { seed };
2831 for _ in 0..u32::MAX {
2832 if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
2833 return candidate;
2834 }
2835 candidate = candidate.wrapping_add(1);
2836 if candidate == 0 {
2837 candidate = 1;
2838 }
2839 }
2840 1
2841 }
2842
2843 fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
2844 if !st.address_book.contains_key(base) {
2845 return base.to_string();
2846 }
2847 let stem = if base.is_empty() { "node" } else { base };
2848 let mut candidate = format!("{stem}-{owner_hash:08x}");
2849 let mut suffix = 1u32;
2850 while st.address_book.contains_key(&candidate) {
2851 candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
2852 suffix = suffix.saturating_add(1);
2853 }
2854 candidate
2855 }
2856
2857 fn update_local_identity_locked(
2858 &self,
2859 st: &mut RouterInner,
2860 mut local: AddressBookEntry,
2861 reason: AddressChangeReason,
2862 ) -> AddressChange {
2863 let old = st.local_address.clone();
2864 st.address_book.remove(old.hostname.as_ref());
2865 st.address_by_value.remove(&old.address);
2866 local.last_seen_ms = self.clock.now_ms();
2867 st.address_by_value
2868 .insert(local.address, local.hostname.to_string());
2869 st.address_book
2870 .insert(local.hostname.to_string(), local.clone());
2871 st.local_address = local.clone();
2872 *self.sender.lock() = local.hostname.clone();
2873 #[cfg(feature = "discovery")]
2874 Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
2875 AddressChange {
2876 old_hostname: old.hostname,
2877 new_hostname: local.hostname,
2878 old_address: old.address,
2879 new_address: local.address,
2880 reason,
2881 }
2882 }
2883
2884 fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
2885 for handler in self.cfg.address_change_handlers.iter() {
2886 (handler.handler)(change.clone())?;
2887 }
2888 Ok(())
2889 }
2890
2891 #[cfg(feature = "discovery")]
2892 fn local_address_advertisement(
2893 &self,
2894 reachable_endpoints: Vec<DataEndpoint>,
2895 reachable_timesync_sources: Vec<String>,
2896 link_capabilities: discovery::LinkCapabilities,
2897 state: u8,
2898 ) -> discovery::AddressAdvertisement {
2899 let st = self.state.lock();
2900 discovery::AddressAdvertisement {
2901 hostname: st.local_address.hostname.to_string(),
2902 address: st.local_address.address,
2903 requested_address: st.local_address.requested_address,
2904 mode: st.local_address.mode.mode_code(),
2905 state,
2906 birth_ms: st.local_address.birth_ms,
2907 owner_hash: st.local_address.owner_hash,
2908 reachable_endpoints,
2909 reachable_timesync_sources,
2910 link_capabilities,
2911 }
2912 }
2913
2914 #[cfg(feature = "discovery")]
2915 fn ingest_address_advertisement(
2916 &self,
2917 ad: discovery::AddressAdvertisement,
2918 ) -> TelemetryResult<bool> {
2919 let now_ms = self.clock.now_ms();
2920 let mut remote = AddressBookEntry {
2921 hostname: Arc::from(ad.hostname.as_str()),
2922 address: ad.address,
2923 requested_address: ad.requested_address,
2924 mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
2925 birth_ms: ad.birth_ms,
2926 owner_hash: ad.owner_hash,
2927 last_seen_ms: now_ms,
2928 };
2929 let mut change = None;
2930 let mut changed = false;
2931 {
2932 let mut st = self.state.lock();
2933 let local = st.local_address.clone();
2934 let hostname_conflict =
2935 remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
2936 let address_conflict =
2937 remote.address == local.address && remote.owner_hash != local.owner_hash;
2938
2939 if hostname_conflict
2940 && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
2941 {
2942 let mut next = local.clone();
2943 next.hostname = Arc::from(Self::unique_hostname_locked(
2944 &st,
2945 local.hostname.as_ref(),
2946 local.owner_hash,
2947 ));
2948 change = Some(self.update_local_identity_locked(
2949 &mut st,
2950 next,
2951 AddressChangeReason::HostnameConflict,
2952 ));
2953 changed = true;
2954 } else if address_conflict
2955 && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
2956 {
2957 let mut next = local.clone();
2958 let seed = next
2959 .requested_address
2960 .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
2961 next.address = Self::allocate_free_address_locked(&st, seed);
2962 let reason = match next.mode {
2963 AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
2964 AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
2965 AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
2966 };
2967 change = Some(self.update_local_identity_locked(&mut st, next, reason));
2968 changed = true;
2969 } else if hostname_conflict {
2970 remote.hostname = Arc::from(Self::unique_hostname_locked(
2971 &st,
2972 remote.hostname.as_ref(),
2973 remote.owner_hash,
2974 ));
2975 changed = true;
2976 } else if address_conflict {
2977 let seed = remote
2978 .requested_address
2979 .max(Self::fallback_address_for_hostname(
2980 remote.hostname.as_ref(),
2981 ));
2982 remote.address = Self::allocate_free_address_locked(&st, seed);
2983 changed = true;
2984 }
2985
2986 st.address_book.remove(remote.hostname.as_ref());
2987 st.address_by_value.remove(&remote.address);
2988 st.address_by_value
2989 .insert(remote.address, remote.hostname.to_string());
2990 st.address_book
2991 .insert(remote.hostname.to_string(), remote.clone());
2992 }
2993 if let Some(change) = change {
2994 self.notify_address_change(change)?;
2995 }
2996 Ok(changed)
2997 }
2998
2999 #[inline]
3000 fn is_end_to_end_ack_sender(sender: &str) -> bool {
3001 sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
3002 }
3003
3004 fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
3005 if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
3006 && !ack_sender.is_empty()
3007 {
3008 return Some(Self::sender_hash(ack_sender));
3009 }
3010 None
3011 }
3012
3013 fn encode_end_to_end_ack_sender(&self) -> String {
3014 let sender = self.sender_arc();
3015 format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
3016 }
3017
3018 #[cfg(feature = "discovery")]
3019 fn is_end_to_end_destination_sender(sender: &str) -> bool {
3020 sender != "RELAY" && !Self::is_end_to_end_ack_sender(sender)
3021 }
3022
3023 fn encode_end_to_end_reliable_ack(packet_id: u64) -> Arc<[u8]> {
3024 let mut payload = Vec::with_capacity(8);
3025 payload.extend_from_slice(&packet_id.to_le_bytes());
3026 Arc::from(payload)
3027 }
3028
3029 fn encode_p2p_payload(
3030 src_hostname: &str,
3031 src_address: NodeAddress,
3032 src_port: P2pPort,
3033 dst_port: P2pPort,
3034 payload: &[u8],
3035 ) -> TelemetryResult<Arc<[u8]>> {
3036 let host_len = u16::try_from(src_hostname.len())
3037 .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
3038 let payload_len = u32::try_from(payload.len())
3039 .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
3040 let mut out = Vec::with_capacity(
3041 15usize
3042 .saturating_add(src_hostname.len())
3043 .saturating_add(payload.len()),
3044 );
3045 out.push(1);
3046 out.extend_from_slice(&dst_port.to_le_bytes());
3047 out.extend_from_slice(&src_port.to_le_bytes());
3048 out.extend_from_slice(&src_address.to_le_bytes());
3049 out.extend_from_slice(&host_len.to_le_bytes());
3050 out.extend_from_slice(&payload_len.to_le_bytes());
3051 out.extend_from_slice(src_hostname.as_bytes());
3052 out.extend_from_slice(payload);
3053 Ok(out.into())
3054 }
3055
3056 fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
3057 if payload.len() < 15 {
3058 return Err(TelemetryError::Unpack("p2p frame short"));
3059 }
3060 if payload[0] != 1 {
3061 return Err(TelemetryError::Unpack("p2p frame version"));
3062 }
3063 let destination_port =
3064 u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
3065 let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
3066 let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
3067 let host_len =
3068 u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
3069 let body_len =
3070 u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
3071 let host_start = 15usize;
3072 let host_end = host_start.saturating_add(host_len);
3073 let body_end = host_end.saturating_add(body_len);
3074 if host_end > payload.len() || body_end != payload.len() {
3075 return Err(TelemetryError::Unpack("p2p frame length"));
3076 }
3077 let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
3078 .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
3079 Ok(P2pDecoded {
3080 source_hostname,
3081 source_address,
3082 source_port,
3083 destination_port,
3084 payload: &payload[host_end..body_end],
3085 })
3086 }
3087
3088 fn encode_p2p_stream_payload(
3089 flags: u8,
3090 source_stream_id: P2pStreamId,
3091 destination_stream_id: P2pStreamId,
3092 sequence: u32,
3093 payload: &[u8],
3094 ) -> TelemetryResult<Arc<[u8]>> {
3095 let payload_len = u32::try_from(payload.len())
3096 .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
3097 let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
3098 out.extend_from_slice(&P2P_STREAM_MAGIC);
3099 out.push(P2P_STREAM_VERSION);
3100 out.push(flags);
3101 out.extend_from_slice(&source_stream_id.to_le_bytes());
3102 out.extend_from_slice(&destination_stream_id.to_le_bytes());
3103 out.extend_from_slice(&sequence.to_le_bytes());
3104 out.extend_from_slice(&payload_len.to_le_bytes());
3105 out.extend_from_slice(payload);
3106 Ok(out.into())
3107 }
3108
3109 fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
3110 if !payload.starts_with(&P2P_STREAM_MAGIC) {
3111 return Ok(None);
3112 }
3113 if payload.len() < 22 {
3114 return Err(TelemetryError::Unpack("p2p stream frame short"));
3115 }
3116 if payload[4] != P2P_STREAM_VERSION {
3117 return Err(TelemetryError::Unpack("p2p stream frame version"));
3118 }
3119 let flags = payload[5];
3120 let source_stream_id =
3121 u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
3122 let destination_stream_id =
3123 u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
3124 let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
3125 let body_len =
3126 u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
3127 let body_end = 22usize.saturating_add(body_len);
3128 if body_end != payload.len() {
3129 return Err(TelemetryError::Unpack("p2p stream frame length"));
3130 }
3131 Ok(Some(P2pStreamDecoded {
3132 flags,
3133 source_stream_id,
3134 destination_stream_id,
3135 sequence,
3136 payload: &payload[22..],
3137 }))
3138 }
3139
3140 fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
3141 for _ in 0..u32::MAX {
3142 let id = st.next_p2p_stream_id.max(1);
3143 st.next_p2p_stream_id = st.next_p2p_stream_id.wrapping_add(1).max(1);
3144 if !st.p2p_stream_sessions.contains_key(&id) {
3145 return id;
3146 }
3147 }
3148 0
3149 }
3150
3151 fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
3152 if pkt.data_type() != DataType::P2pMessage {
3153 return Ok(());
3154 }
3155 let decoded = Self::decode_p2p_payload(pkt.payload())?;
3156 if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
3157 return self.dispatch_p2p_stream_frame(&decoded, &stream);
3158 }
3159 let handlers = {
3160 let st = self.state.lock();
3161 st.p2p_port_handlers
3162 .get(&decoded.destination_port)
3163 .cloned()
3164 .unwrap_or_default()
3165 };
3166 for handler in handlers {
3167 (handler.handler)(P2pMessage {
3168 source_hostname: decoded.source_hostname,
3169 source_address: decoded.source_address,
3170 source_port: decoded.source_port,
3171 destination_port: decoded.destination_port,
3172 payload: decoded.payload,
3173 })?;
3174 }
3175 Ok(())
3176 }
3177
3178 fn dispatch_p2p_stream_frame(
3179 &self,
3180 msg: &P2pDecoded<'_>,
3181 frame: &P2pStreamDecoded<'_>,
3182 ) -> TelemetryResult<()> {
3183 let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
3184 let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, Arc<[u8]>)> = None;
3185 {
3186 let mut st = self.state.lock();
3187 if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
3188 let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
3189 let existing_id = st.p2p_stream_sessions.iter().find_map(|(id, session)| {
3190 (session.peer_stream_id == frame.source_stream_id
3191 && session.local_port == msg.destination_port
3192 && session.peer_port == msg.source_port
3193 && session.peer_address == msg.source_address
3194 && session.peer_hostname.as_ref() == msg.source_hostname)
3195 .then_some(*id)
3196 });
3197 let local_id = if let Some(local_id) = existing_id {
3198 local_id
3199 } else {
3200 let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
3201 if local_id == 0 {
3202 return Err(TelemetryError::Io("p2p stream id exhausted"));
3203 }
3204 st.p2p_stream_sessions.insert(
3205 local_id,
3206 P2pStreamSession {
3207 peer_hostname: peer_hostname.clone(),
3208 peer_address: msg.source_address,
3209 local_port: msg.destination_port,
3210 peer_port: msg.source_port,
3211 peer_stream_id: frame.source_stream_id,
3212 next_sequence: 1,
3213 connected: true,
3214 },
3215 );
3216 let handlers = st
3217 .p2p_stream_handlers
3218 .get(&msg.destination_port)
3219 .cloned()
3220 .unwrap_or_default();
3221 events.push(PendingP2pStreamEvent {
3222 handlers,
3223 kind: P2pStreamEventKind::Accepted,
3224 stream_id: local_id,
3225 peer_stream_id: frame.source_stream_id,
3226 peer_hostname: peer_hostname.clone(),
3227 peer_address: msg.source_address,
3228 local_port: msg.destination_port,
3229 peer_port: msg.source_port,
3230 });
3231 local_id
3232 };
3233 let dst =
3234 st.address_book
3235 .get(msg.source_hostname)
3236 .cloned()
3237 .unwrap_or(AddressBookEntry {
3238 hostname: peer_hostname,
3239 address: msg.source_address,
3240 requested_address: msg.source_address,
3241 mode: AddressAssignmentMode::Dynamic,
3242 birth_ms: self.clock.now_ms(),
3243 owner_hash: Self::sender_hash(msg.source_hostname),
3244 last_seen_ms: self.clock.now_ms(),
3245 });
3246 let payload = Self::encode_p2p_stream_payload(
3247 P2P_STREAM_SYN | P2P_STREAM_ACK,
3248 local_id,
3249 frame.source_stream_id,
3250 0,
3251 &[],
3252 )?;
3253 reply = Some((dst, msg.source_port, msg.destination_port, payload));
3254 } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
3255 if let Some(session) = st.p2p_stream_sessions.get_mut(&frame.destination_stream_id)
3256 {
3257 session.peer_stream_id = frame.source_stream_id;
3258 session.connected = true;
3259 let peer_hostname = session.peer_hostname.clone();
3260 let peer_address = session.peer_address;
3261 let local_port = session.local_port;
3262 let peer_port = session.peer_port;
3263 let handlers = st
3264 .p2p_stream_handlers
3265 .get(&local_port)
3266 .cloned()
3267 .unwrap_or_default();
3268 events.push(PendingP2pStreamEvent {
3269 handlers,
3270 kind: P2pStreamEventKind::Connected,
3271 stream_id: frame.destination_stream_id,
3272 peer_stream_id: frame.source_stream_id,
3273 peer_hostname,
3274 peer_address,
3275 local_port,
3276 peer_port,
3277 });
3278 }
3279 } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
3280 let kind = if frame.flags & P2P_STREAM_RST != 0 {
3281 P2pStreamEventKind::Reset
3282 } else if frame.flags & P2P_STREAM_FIN != 0 {
3283 P2pStreamEventKind::Closed
3284 } else {
3285 P2pStreamEventKind::Data
3286 };
3287 let session_id = if frame.destination_stream_id != 0 {
3288 Some(frame.destination_stream_id)
3289 } else {
3290 st.p2p_stream_sessions.iter().find_map(|(id, session)| {
3291 (session.peer_stream_id == frame.source_stream_id
3292 && session.local_port == msg.destination_port
3293 && session.peer_port == msg.source_port
3294 && session.peer_address == msg.source_address
3295 && session.peer_hostname.as_ref() == msg.source_hostname)
3296 .then_some(*id)
3297 })
3298 };
3299 if let Some(session_id) = session_id
3300 && let Some(session) = st.p2p_stream_sessions.get(&session_id)
3301 {
3302 let handlers = st
3303 .p2p_stream_handlers
3304 .get(&session.local_port)
3305 .cloned()
3306 .unwrap_or_default();
3307 events.push(PendingP2pStreamEvent {
3308 handlers,
3309 kind,
3310 stream_id: session_id,
3311 peer_stream_id: frame.source_stream_id,
3312 peer_hostname: session.peer_hostname.clone(),
3313 peer_address: session.peer_address,
3314 local_port: session.local_port,
3315 peer_port: session.peer_port,
3316 });
3317 if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
3318 st.p2p_stream_sessions.remove(&session_id);
3319 }
3320 }
3321 }
3322 }
3323 if let Some((dst, dst_port, src_port, payload)) = reply {
3324 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
3325 }
3326 for pending in events {
3327 for handler in pending.handlers {
3328 (handler.handler)(P2pStreamEvent {
3329 kind: pending.kind,
3330 stream_id: pending.stream_id,
3331 peer_stream_id: pending.peer_stream_id,
3332 sequence: frame.sequence,
3333 peer_hostname: pending.peer_hostname.as_ref(),
3334 peer_address: pending.peer_address,
3335 local_port: pending.local_port,
3336 peer_port: pending.peer_port,
3337 payload: frame.payload,
3338 })?;
3339 }
3340 }
3341 Ok(())
3342 }
3343
3344 fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) {
3350 let mut st = self.state.lock();
3351 Self::remember_reliable_return_route_locked(&mut st, packet_id);
3352 st.reliable_return_routes
3353 .insert(packet_id, ReliableReturnRouteState { side });
3354 }
3355
3356 fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) {
3362 let cap = runtime_reliable_max_return_routes().max(1);
3363 st.reliable_return_route_order
3364 .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
3365 while st.reliable_return_route_order.len() >= cap {
3366 if let Some(oldest) = st.reliable_return_route_order.pop_front() {
3367 st.reliable_return_routes.remove(&oldest);
3368 } else {
3369 break;
3370 }
3371 }
3372 st.reliable_return_route_order.push_back(packet_id);
3373 }
3374
3375 fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
3376 let cap = runtime_reliable_max_end_to_end_pending().max(1);
3377 st.end_to_end_reliable_tx_order
3378 .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
3379 while st.end_to_end_reliable_tx_order.len() >= cap {
3380 if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
3381 st.end_to_end_reliable_tx.remove(&oldest);
3382 } else {
3383 break;
3384 }
3385 }
3386 st.end_to_end_reliable_tx_order.push_back(packet_id);
3387 }
3388
3389 #[cfg(feature = "discovery")]
3390 fn expected_end_to_end_destinations_locked(
3391 &self,
3392 st: &RouterInner,
3393 data: &RouterItem,
3394 ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
3395 let (eps, ty) = self.item_route_info(data)?;
3396 let now_ms = self.clock.now_ms();
3397 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
3398 let prefer_best_overlap =
3399 is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
3400 let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
3401 let mut candidates: Vec<(u64, RouterSideId, usize)> = Vec::new();
3402 let mut best_overlap = 0usize;
3403 let mut out = BTreeMap::new();
3404 for (&side, route) in st.discovery_routes.iter() {
3405 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3406 continue;
3407 }
3408 let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
3409 continue;
3410 };
3411 if restrict_link_local && !side_ref.opts.link_local_enabled {
3412 continue;
3413 }
3414 if !self.route_allowed_locked(st, None, Some(ty), side) {
3415 continue;
3416 }
3417 for sender_state in route.announcers.values() {
3418 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3419 continue;
3420 }
3421 for board in sender_state.topology_boards.iter() {
3422 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
3423 continue;
3424 }
3425 let overlap = Self::endpoint_overlap_count(
3426 board.reachable_endpoints.iter().copied(),
3427 &scoring_eps,
3428 );
3429 if overlap > 0 {
3430 if prefer_best_overlap {
3431 best_overlap = best_overlap.max(overlap);
3432 candidates.push((Self::sender_hash(&board.sender_id), side, overlap));
3433 } else {
3434 out.insert(Self::sender_hash(&board.sender_id), side);
3435 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3436 return Ok(out);
3437 }
3438 }
3439 }
3440 }
3441 }
3442 }
3443
3444 if prefer_best_overlap {
3445 for (sender_hash, side, overlap) in candidates {
3446 if overlap == best_overlap {
3447 out.insert(sender_hash, side);
3448 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3449 return Ok(out);
3450 }
3451 }
3452 }
3453 }
3454 Ok(out)
3455 }
3456
3457 #[cfg(feature = "discovery")]
3458 #[allow(clippy::too_many_arguments)]
3459 fn discovered_route_candidates_locked(
3460 &self,
3461 st: &RouterInner,
3462 exclude: Option<RouterSideId>,
3463 ty: DataType,
3464 eps: &[DataEndpoint],
3465 target_senders: &[u64],
3466 prefer_nonlocal: bool,
3467 preferred_timesync_source: Option<&str>,
3468 ) -> Vec<DiscoveryCandidateMatch> {
3469 let restrict_link_local = Self::endpoints_are_link_local_only(eps);
3470 let now_ms = self.clock.now_ms();
3471 let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
3472 let mut out = Vec::new();
3473
3474 for (&side, route) in st.discovery_routes.iter() {
3475 if exclude == Some(side)
3476 || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
3477 {
3478 continue;
3479 }
3480 if restrict_link_local
3481 && st
3482 .sides
3483 .get(side)
3484 .and_then(|s| s.as_ref())
3485 .map(|s| !s.opts.link_local_enabled)
3486 .unwrap_or(true)
3487 {
3488 continue;
3489 }
3490 if !self.route_allowed_locked(st, exclude, Some(ty), side) {
3491 continue;
3492 }
3493 if !target_senders.is_empty()
3494 && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
3495 {
3496 continue;
3497 }
3498 if !target_senders.is_empty() {
3499 out.push(DiscoveryCandidateMatch {
3500 side,
3501 overlap: usize::MAX,
3502 });
3503 continue;
3504 }
3505 if preferred_timesync_source
3506 .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
3507 {
3508 out.push(DiscoveryCandidateMatch {
3509 side,
3510 overlap: usize::MAX,
3511 });
3512 continue;
3513 }
3514 let overlap =
3515 Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
3516 if overlap > 0 {
3517 out.push(DiscoveryCandidateMatch { side, overlap });
3518 }
3519 }
3520 out
3521 }
3522
3523 #[cfg(feature = "discovery")]
3524 fn select_discovered_candidate_sides_locked(
3525 &self,
3526 st: &mut RouterInner,
3527 exclude: Option<RouterSideId>,
3528 ty: DataType,
3529 target_senders: &[u64],
3530 prefer_best_overlap: bool,
3531 matches: Vec<DiscoveryCandidateMatch>,
3532 ) -> Vec<RouterSideId> {
3533 let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
3534 || !target_senders.is_empty()
3535 {
3536 RouteSelectionOrigin::Flood
3537 } else {
3538 RouteSelectionOrigin::Discovered
3539 };
3540
3541 let mut matches = matches;
3542 if matches.iter().any(|m| m.overlap == usize::MAX) {
3543 matches.retain(|m| m.overlap == usize::MAX);
3544 }
3545
3546 let selected: Vec<RouterSideId> = if prefer_best_overlap {
3547 let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
3548 matches
3549 .into_iter()
3550 .filter(|m| m.overlap == best_overlap)
3551 .map(|m| m.side)
3552 .collect()
3553 } else {
3554 matches.into_iter().map(|m| m.side).collect()
3555 };
3556
3557 self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
3558 }
3559
3560 fn register_end_to_end_reliable_tx(&self, data: &RouterItem) -> TelemetryResult<()> {
3561 let packet_id = Self::get_hash(data);
3562 let now_ms = self.clock.now_ms();
3563 let ty = match data {
3564 RouterItem::Packet(pkt) => pkt.data_type(),
3565 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
3566 };
3567 let mut st = self.state.lock();
3568 #[cfg(feature = "discovery")]
3569 let mut pending_destinations = self.expected_end_to_end_destinations_locked(&st, data)?;
3570 #[cfg(not(feature = "discovery"))]
3571 let mut pending_destinations = BTreeMap::new();
3572 self.filter_trackable_end_to_end_destinations_locked(&st, ty, &mut pending_destinations);
3573 let tracked_destinations = !pending_destinations.is_empty();
3574 Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
3575 st.end_to_end_reliable_tx.insert(
3576 packet_id,
3577 EndToEndReliableSent {
3578 data: data.clone(),
3579 pending_destinations,
3580 tracked_destinations,
3581 last_send_ms: now_ms,
3582 retries: 0,
3583 queued: false,
3584 },
3585 );
3586 Ok(())
3587 }
3588
3589 #[cfg(feature = "discovery")]
3590 fn reconcile_end_to_end_reliable_destinations_locked(
3591 &self,
3592 st: &mut RouterInner,
3593 ) -> TelemetryResult<()> {
3594 let active_destinations = self.active_end_to_end_destinations_locked(st);
3595 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
3596 let mut completed = Vec::new();
3597
3598 for packet_id in packet_ids {
3599 let Some(data) = st
3600 .end_to_end_reliable_tx
3601 .get(&packet_id)
3602 .map(|sent| sent.data.clone())
3603 else {
3604 continue;
3605 };
3606 let expected = self.expected_end_to_end_destinations_locked(st, &data)?;
3607 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
3608 continue;
3609 };
3610 if !sent.tracked_destinations {
3611 continue;
3612 }
3613 sent.pending_destinations.retain(|sender_hash, side| {
3614 match (
3615 expected.get(sender_hash),
3616 active_destinations.get(sender_hash),
3617 ) {
3618 (Some(next_side), _) | (None, Some(next_side)) => {
3619 *side = *next_side;
3620 true
3621 }
3622 (None, None) => false,
3623 }
3624 });
3625 if sent.pending_destinations.is_empty() {
3626 completed.push(packet_id);
3627 }
3628 }
3629
3630 for packet_id in completed {
3631 st.end_to_end_reliable_tx.remove(&packet_id);
3632 }
3633
3634 Ok(())
3635 }
3636
3637 #[cfg(feature = "discovery")]
3638 fn active_end_to_end_destinations_locked(
3639 &self,
3640 st: &RouterInner,
3641 ) -> BTreeMap<u64, RouterSideId> {
3642 let now_ms = self.clock.now_ms();
3643 let mut out = BTreeMap::new();
3644 for (&side, route) in st.discovery_routes.iter() {
3645 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3646 continue;
3647 }
3648 for sender_state in route.announcers.values() {
3649 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
3650 continue;
3651 }
3652 for board in sender_state.topology_boards.iter() {
3653 if !Self::is_end_to_end_destination_sender(&board.sender_id) {
3654 continue;
3655 }
3656 out.insert(Self::sender_hash(&board.sender_id), side);
3657 if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
3658 return out;
3659 }
3660 }
3661 }
3662 }
3663 out
3664 }
3665
3666 fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
3667 matches!(
3668 st.sides
3669 .get(side)
3670 .and_then(Option::as_ref)
3671 .map(|side| &side.tx_handler),
3672 Some(RouterTxHandlerFn::Packed(_))
3673 )
3674 }
3675
3676 fn filter_trackable_end_to_end_destinations_locked(
3677 &self,
3678 st: &RouterInner,
3679 ty: DataType,
3680 pending: &mut BTreeMap<u64, RouterSideId>,
3681 ) {
3682 let now_ms = self.clock.now_ms();
3683 pending.retain(|_, side| {
3684 Self::side_supports_end_to_end_tracking_locked(st, *side)
3685 && (is_reliable_type(ty)
3686 || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
3687 });
3688 }
3689
3690 #[cfg(feature = "discovery")]
3691 fn side_has_multiple_announcers_locked(
3692 &self,
3693 st: &RouterInner,
3694 side: RouterSideId,
3695 now_ms: u64,
3696 ) -> bool {
3697 st.discovery_routes
3698 .get(&side)
3699 .map(|route| {
3700 route
3701 .announcers
3702 .values()
3703 .filter(|sender| {
3704 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
3705 })
3706 .take(2)
3707 .count()
3708 > 1
3709 })
3710 .unwrap_or(false)
3711 }
3712
3713 #[cfg(not(feature = "discovery"))]
3714 fn side_has_multiple_announcers_locked(
3715 &self,
3716 _st: &RouterInner,
3717 _side: RouterSideId,
3718 _now_ms: u64,
3719 ) -> bool {
3720 false
3721 }
3722
3723 fn queue_end_to_end_reliable_ack(
3724 &self,
3725 pkt: &Packet,
3726 called_from_queue: bool,
3727 ) -> TelemetryResult<()> {
3728 self.queue_end_to_end_reliable_ack_for_packet_id(pkt.packet_id(), called_from_queue)
3729 }
3730
3731 fn queue_end_to_end_reliable_ack_for_packet_id(
3732 &self,
3733 packet_id: u64,
3734 called_from_queue: bool,
3735 ) -> TelemetryResult<()> {
3736 let ack_sender = self.encode_end_to_end_ack_sender();
3737 let ack = Packet::new(
3738 DataType::ReliableAck,
3739 message_meta(DataType::ReliableAck).endpoints,
3740 ack_sender.as_str(),
3741 self.packet_timestamp_ms(),
3742 Self::encode_end_to_end_reliable_ack(packet_id),
3743 )?;
3744 self.emit_internal_tx(
3745 RouterTxItem::Broadcast(RouterItem::Packet(ack)),
3746 true,
3747 called_from_queue,
3748 )
3749 }
3750
3751 fn emit_internal_tx(
3752 &self,
3753 item: RouterTxItem,
3754 ignore_local: bool,
3755 called_from_queue: bool,
3756 ) -> TelemetryResult<()> {
3757 if called_from_queue {
3758 self.tx_queue_item_with_flags(item, ignore_local)
3759 } else {
3760 self.tx_item_impl(item, ignore_local, false)
3761 }
3762 }
3763
3764 fn emit_internal_tx_with_priority(
3765 &self,
3766 item: RouterTxItem,
3767 ignore_local: bool,
3768 priority: u8,
3769 called_from_queue: bool,
3770 ) -> TelemetryResult<()> {
3771 if called_from_queue {
3772 self.tx_queue_item_with_priority(item, ignore_local, priority)
3773 } else {
3774 self.tx_item_impl(item, ignore_local, false)
3775 }
3776 }
3777
3778 fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
3779 {
3780 let mut st = self.state.lock();
3781 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
3782 return Ok(());
3783 };
3784 if sent.queued {
3785 return Ok(());
3786 }
3787 sent.queued = true;
3788 }
3789 self.tx_queue_item_with_priority(
3790 RouterTxItem::EndToEndReplay { packet_id },
3791 true,
3792 Self::router_item_priority_bumped(DataType::ReliableAck),
3793 )
3794 }
3795
3796 fn end_to_end_retransmit_sides(
3797 &self,
3798 packet_id: u64,
3799 ) -> Option<(RouterItem, Vec<RouterSideId>)> {
3800 let mut st = self.state.lock();
3801 let (data, tracked_destinations, mut sides) = {
3802 let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
3803 sent.queued = false;
3804 sent.last_send_ms = self.clock.now_ms();
3805 let data = sent.data.clone();
3806 let tracked_destinations = sent.tracked_destinations;
3807 let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
3808 (data, tracked_destinations, sides)
3809 };
3810 if tracked_destinations && sides.is_empty() {
3811 st.end_to_end_reliable_tx.remove(&packet_id);
3812 return None;
3813 }
3814 sides.sort_unstable();
3815 sides.dedup();
3816 Some((data, sides))
3817 }
3818
3819 fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
3820 let ty = match data {
3821 RouterItem::Packet(pkt) => pkt.data_type(),
3822 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
3823 };
3824 Ok(message_priority(ty))
3825 }
3826
3827 #[inline]
3828 fn router_item_priority_bumped(ty: DataType) -> u8 {
3829 message_priority(ty).saturating_add(16)
3830 }
3831
3832 #[inline]
3833 fn is_side_tx_busy(err: &TelemetryError) -> bool {
3834 matches!(err, TelemetryError::Io("side tx busy"))
3835 }
3836
3837 #[cfg(feature = "timesync")]
3838 fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
3839 st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
3840 || st
3841 .clock
3842 .current_time(now_mono_ns)
3843 .and_then(|reading| reading.unix_time_ms)
3844 .is_some()
3845 }
3846
3847 #[cfg(feature = "timesync")]
3848 fn reconcile_pending_timesync_request_locked(
3849 st: &mut TimeSyncRuntime,
3850 leader: &Option<TimeSyncLeader>,
3851 now_ms: u64,
3852 ) {
3853 let active_remote = match leader {
3854 Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
3855 _ => None,
3856 };
3857 let should_clear = st
3858 .pending_request
3859 .as_ref()
3860 .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
3861 if should_clear {
3862 st.pending_request = None;
3863 st.next_request_mono_ms = now_ms;
3864 }
3865 }
3866
3867 #[inline]
3869 fn enqueue_to_sides(
3870 &self,
3871 data: RouterItem,
3872 exclude: Option<RouterSideId>,
3873 ignore_local: bool,
3874 ) -> TelemetryResult<()> {
3875 let plan = self.remote_side_plan(&data, exclude)?;
3876 let mut st = self.state.lock();
3877 let priority = Self::router_item_priority(&data)?;
3878
3879 let RemoteSidePlan::Target(sides) = plan;
3880 for idx in sides {
3881 st.push_transmit(TxQueued {
3882 item: RouterTxItem::ToSide {
3883 src: exclude,
3884 dst: idx,
3885 data: data.clone(),
3886 },
3887 ignore_local,
3888 priority,
3889 })?;
3890 }
3891
3892 Ok(())
3893 }
3894
3895 fn relay_send(
3896 &self,
3897 data: RouterItem,
3898 src: Option<RouterSideId>,
3899 called_from_queue: bool,
3900 ) -> TelemetryResult<()> {
3901 if called_from_queue {
3902 return self.enqueue_to_sides(data, src, true);
3903 }
3904
3905 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
3906 for side in sides {
3907 self.tx_item_impl(
3908 RouterTxItem::ToSide {
3909 src,
3910 dst: side,
3911 data: data.clone(),
3912 },
3913 true,
3914 false,
3915 )?;
3916 }
3917
3918 Ok(())
3919 }
3920
3921 fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
3922 match data {
3923 RouterItem::Packet(pkt) => {
3924 pkt.validate()?;
3925 let mut eps = pkt.endpoints().to_vec();
3926 eps.sort_unstable();
3927 eps.dedup();
3928 Ok((eps, pkt.data_type()))
3929 }
3930 RouterItem::Packed(bytes) => {
3931 let env = wire_format::peek_envelope(bytes.as_ref())?;
3932 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
3933 eps.sort_unstable();
3934 eps.dedup();
3935 Ok((eps, env.ty))
3936 }
3937 }
3938 }
3939
3940 fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
3941 match data {
3942 RouterItem::Packet(pkt) => Ok(pkt.data_type()),
3943 RouterItem::Packed(bytes) => Ok(wire_format::peek_envelope(bytes.as_ref())?.ty),
3944 }
3945 }
3946
3947 fn e2e_crypto_supported(&self) -> bool {
3948 #[cfg(feature = "cryptography")]
3949 {
3950 self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
3951 && crate::crypto::registered_crypto_available()
3952 }
3953 #[cfg(not(feature = "cryptography"))]
3954 {
3955 false
3956 }
3957 }
3958
3959 fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
3960 if is_internal_control_type(ty) {
3961 return false;
3962 }
3963 match self.cfg.e2e_encryption() {
3964 RouterE2eEncryptionMode::Disabled => {
3965 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
3966 }
3967 RouterE2eEncryptionMode::RequiredOnly => {
3968 message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
3969 }
3970 RouterE2eEncryptionMode::Preferred => matches!(
3971 message_e2e_encryption_policy(ty),
3972 E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
3973 ),
3974 RouterE2eEncryptionMode::ForceAll => true,
3975 }
3976 }
3977
3978 fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
3979 if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
3980 return Err(TelemetryError::BadArg);
3981 }
3982 Ok(())
3983 }
3984
3985 #[cfg(feature = "cryptography")]
3986 fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
3987 if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
3988 Some(wire_format::E2eSealConfig {
3989 key_id: self.cfg.e2e_key_id(),
3990 })
3991 } else {
3992 None
3993 }
3994 }
3995
3996 #[inline]
3997 fn pack_packet_for_router(
3998 &self,
3999 pkt: &Packet,
4000 reliable: Option<wire_format::ReliableHeader>,
4001 ) -> TelemetryResult<Arc<[u8]>> {
4002 #[cfg(feature = "cryptography")]
4003 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
4004 return wire_format::pack_packet_with_wire_contract_e2e(
4005 pkt,
4006 reliable,
4007 pkt.wire_shape(),
4008 pkt.wire_target_senders(),
4009 e2e,
4010 );
4011 }
4012 Ok(match reliable {
4013 Some(hdr) => wire_format::pack_packet_with_reliable(pkt, hdr),
4014 None => wire_format::pack_packet(pkt),
4015 })
4016 }
4017
4018 #[inline]
4019 fn pack_packet_for_contract(
4020 &self,
4021 pkt: &Packet,
4022 reliable: Option<wire_format::ReliableHeader>,
4023 shape: Option<MessageElement>,
4024 target_senders: &[u64],
4025 ) -> TelemetryResult<Arc<[u8]>> {
4026 #[cfg(feature = "cryptography")]
4027 if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
4028 return wire_format::pack_packet_with_wire_contract_e2e(
4029 pkt,
4030 reliable,
4031 shape,
4032 target_senders,
4033 e2e,
4034 );
4035 }
4036 wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
4037 }
4038
4039 #[cfg(feature = "cryptography")]
4040 #[inline]
4041 fn prepare_packed_for_remote(
4042 &self,
4043 bytes: Arc<[u8]>,
4044 reliable_override: Option<Option<wire_format::ReliableHeader>>,
4045 ) -> TelemetryResult<Arc<[u8]>> {
4046 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
4047 if frame.ack_only() || self.e2e_seal_config_for_type(frame.envelope.ty).is_none() {
4048 return Ok(bytes);
4049 }
4050 let reliable = reliable_override.unwrap_or(frame.reliable);
4051 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4052 self.pack_packet_for_contract(
4053 &pkt,
4054 reliable,
4055 frame.envelope.wire_shape,
4056 &frame.envelope.target_senders,
4057 )
4058 }
4059
4060 fn item_target_senders(&self, data: &RouterItem) -> TelemetryResult<Arc<[u64]>> {
4061 match data {
4062 RouterItem::Packet(pkt) => Ok(Arc::from(pkt.wire_target_senders())),
4063 RouterItem::Packed(bytes) => {
4064 Ok(wire_format::peek_envelope(bytes.as_ref())?.target_senders)
4065 }
4066 }
4067 }
4068
4069 fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
4070 let targets = self.item_target_senders(data)?;
4071 if targets.is_empty() {
4072 return Ok(true);
4073 }
4074 let local_sender = self.sender_arc();
4075 let local_hash = Self::sender_hash(local_sender.as_ref());
4076 Ok(targets.contains(&local_hash))
4077 }
4078
4079 #[cfg(feature = "discovery")]
4080 fn side_matches_target_senders_locked(
4081 st: &RouterInner,
4082 side: RouterSideId,
4083 target_senders: &[u64],
4084 now_ms: u64,
4085 ) -> bool {
4086 st.discovery_routes
4087 .get(&side)
4088 .map(|route| {
4089 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4090 return false;
4091 }
4092 route.announcers.values().any(|sender_state| {
4093 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4094 return false;
4095 }
4096 sender_state
4097 .topology_boards
4098 .iter()
4099 .any(|board| target_senders.contains(&Self::sender_hash(&board.sender_id)))
4100 })
4101 })
4102 .unwrap_or(false)
4103 }
4104
4105 fn attach_wire_contract_to_item(
4106 &self,
4107 data: RouterItem,
4108 target_senders: &[u64],
4109 ) -> TelemetryResult<RouterItem> {
4110 match data {
4111 RouterItem::Packet(pkt) => {
4112 let reliable = if is_reliable_type(pkt.data_type()) {
4113 Some(wire_format::ReliableHeader {
4114 flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
4115 seq: 0,
4116 ack: 0,
4117 })
4118 } else {
4119 None
4120 };
4121 let bytes = self.pack_packet_for_contract(
4122 &pkt,
4123 reliable,
4124 Some(message_meta(pkt.data_type()).element),
4125 target_senders,
4126 )?;
4127 Ok(RouterItem::Packed(bytes))
4128 }
4129 RouterItem::Packed(bytes) => Ok(RouterItem::Packed(bytes)),
4130 }
4131 }
4132
4133 fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
4134 !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
4135 }
4136
4137 fn should_route_remote(
4138 &self,
4139 data: &RouterItem,
4140 exclude: Option<RouterSideId>,
4141 ) -> TelemetryResult<bool> {
4142 #[cfg(feature = "discovery")]
4143 {
4144 let RemoteSidePlan::Target(sides) = self.remote_side_plan(data, exclude)?;
4145 Ok(!sides.is_empty())
4146 }
4147
4148 #[cfg(not(feature = "discovery"))]
4149 {
4150 let (eps, ty) = self.item_route_info(data)?;
4151 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
4152 return Ok(false);
4153 }
4154 let st = self.state.lock();
4155 Ok(!self
4156 .eligible_side_ids_locked(
4157 &st,
4158 exclude,
4159 Some(ty),
4160 Self::endpoints_are_link_local_only(&eps),
4161 )
4162 .is_empty())
4163 }
4164 }
4165
4166 #[cfg(feature = "discovery")]
4167 fn has_explicit_route_policy_locked(
4168 st: &RouterInner,
4169 src: Option<RouterSideId>,
4170 ty: DataType,
4171 ) -> bool {
4172 st.route_overrides
4173 .keys()
4174 .any(|(route_src, _)| *route_src == src)
4175 || Self::has_typed_route_overrides_locked(st, src, ty)
4176 }
4177
4178 fn remote_side_plan(
4179 &self,
4180 data: &RouterItem,
4181 exclude: Option<RouterSideId>,
4182 ) -> TelemetryResult<RemoteSidePlan> {
4183 #[cfg(feature = "discovery")]
4184 {
4185 let (eps, ty) = self.item_route_info(data)?;
4186 let target_senders = self.item_target_senders(data)?;
4187 let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
4188 if discovery::is_discovery_type(ty) {
4189 let mut st = self.state.lock();
4190 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
4191 return Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4192 &mut st,
4193 exclude,
4194 sides,
4195 RouteSelectionOrigin::Flood,
4196 )));
4197 }
4198 if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
4199 return Ok(RemoteSidePlan::Target(Vec::new()));
4200 }
4201
4202 #[cfg(feature = "timesync")]
4203 let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
4204 #[cfg(not(feature = "timesync"))]
4205 let preferred_timesync_source: Option<String> = None;
4206
4207 let mut st = self.state.lock();
4208 if let Some(packet_id) = preferred_packet_id {
4209 let target_side = st
4210 .reliable_return_routes
4211 .get(&packet_id)
4212 .map(|route| route.side);
4213 if let Some(side) = target_side
4214 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
4215 {
4216 #[cfg(feature = "timesync")]
4217 if !Self::timesync_allowed_for_side_locked(
4218 &mut st,
4219 side,
4220 ty,
4221 self.clock.now_ms(),
4222 ) {
4223 return Ok(RemoteSidePlan::Target(Vec::new()));
4224 }
4225 return Ok(RemoteSidePlan::Target(vec![side]));
4226 }
4227 return Ok(RemoteSidePlan::Target(Vec::new()));
4228 }
4229 let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
4230 let prefer_best_overlap =
4231 is_reliable_type(ty) && target_senders.is_empty() && preferred_packet_id.is_none();
4232 if st.discovery_routes.is_empty() {
4233 let mut fallback =
4234 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
4235 #[cfg(feature = "timesync")]
4236 {
4237 fallback = Self::filter_timesync_sides_locked(
4238 &mut st,
4239 ty,
4240 self.clock.now_ms(),
4241 fallback,
4242 );
4243 }
4244 return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
4245 fallback
4246 } else {
4247 Vec::new()
4248 }));
4249 }
4250 let mut matches = self.discovered_route_candidates_locked(
4251 &st,
4252 exclude,
4253 ty,
4254 &eps,
4255 &target_senders,
4256 prefer_best_overlap,
4257 preferred_timesync_source.as_deref(),
4258 );
4259 #[cfg(feature = "timesync")]
4260 {
4261 matches =
4262 Self::filter_timesync_matches_locked(&mut st, ty, self.clock.now_ms(), matches);
4263 }
4264
4265 if !matches.is_empty() {
4266 Ok(RemoteSidePlan::Target(
4267 self.select_discovered_candidate_sides_locked(
4268 &mut st,
4269 exclude,
4270 ty,
4271 &target_senders,
4272 prefer_best_overlap,
4273 matches,
4274 ),
4275 ))
4276 } else if prefer_best_overlap {
4277 Ok(RemoteSidePlan::Target(Vec::new()))
4278 } else {
4279 if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
4280 let mut sides =
4281 self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
4282 #[cfg(feature = "timesync")]
4283 {
4284 sides = Self::filter_timesync_sides_locked(
4285 &mut st,
4286 ty,
4287 self.clock.now_ms(),
4288 sides,
4289 );
4290 }
4291 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4292 &mut st,
4293 exclude,
4294 sides,
4295 RouteSelectionOrigin::Flood,
4296 )))
4297 } else {
4298 Ok(RemoteSidePlan::Target(Vec::new()))
4299 }
4300 }
4301 }
4302 #[cfg(not(feature = "discovery"))]
4303 {
4304 let (_, ty) = self.item_route_info(data)?;
4305 let mut st = self.state.lock();
4306 if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
4307 let target_side = st
4308 .reliable_return_routes
4309 .get(&packet_id)
4310 .map(|route| route.side);
4311 if let Some(side) = target_side
4312 .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
4313 {
4314 return Ok(RemoteSidePlan::Target(vec![side]));
4315 }
4316 return Ok(RemoteSidePlan::Target(Vec::new()));
4317 }
4318 let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
4319 Ok(RemoteSidePlan::Target(self.apply_route_selection_locked(
4320 &mut st,
4321 exclude,
4322 sides,
4323 RouteSelectionOrigin::Flood,
4324 )))
4325 }
4326 }
4327
4328 #[cfg(feature = "discovery")]
4329 fn local_discovery_endpoints(&self) -> Vec<DataEndpoint> {
4330 let mut eps: Vec<DataEndpoint> = self.cfg.handlers.iter().map(|h| h.endpoint).collect();
4331 #[cfg(feature = "timesync")]
4332 if self.cfg.timesync_config().is_some() {
4333 eps.push(DataEndpoint::TimeSync);
4334 }
4335 eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
4336 eps.sort_unstable();
4337 eps.dedup();
4338 eps
4339 }
4340
4341 #[cfg(feature = "discovery")]
4342 fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
4343 #[cfg(feature = "timesync")]
4344 {
4345 let st = self.timesync.lock();
4346 if let Some(tracker) = st.tracker.as_ref()
4347 && tracker.should_serve(
4348 now_ms,
4349 Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
4350 )
4351 {
4352 return vec![self.sender_arc().to_string()];
4353 }
4354 }
4355 Vec::new()
4356 }
4357
4358 #[cfg(all(feature = "discovery", feature = "timesync"))]
4359 fn preferred_timesync_route_source(
4360 &self,
4361 data: &RouterItem,
4362 ty: DataType,
4363 ) -> TelemetryResult<Option<String>> {
4364 if !matches!(
4365 ty,
4366 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4367 ) {
4368 return Ok(None);
4369 }
4370
4371 match data {
4372 RouterItem::Packet(pkt) => match ty {
4373 DataType::TimeSyncRequest => {
4374 let local_sender = self.sender_arc();
4375 if pkt.sender() == local_sender.as_ref() {
4376 Ok(self.timesync.lock().tracker.as_ref().and_then(|tracker| {
4377 tracker.current_source().map(|src| src.sender.clone())
4378 }))
4379 } else {
4380 Ok(None)
4381 }
4382 }
4383 DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
4384 Ok(Some(pkt.sender().to_owned()))
4385 }
4386 _ => Ok(None),
4387 },
4388 RouterItem::Packed(bytes) => {
4389 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4390 self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
4391 }
4392 }
4393 }
4394
4395 #[cfg(feature = "discovery")]
4396 fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
4397 st.discovery_cadence.on_topology_change(now_ms);
4398 }
4399
4400 #[cfg(feature = "discovery")]
4401 fn sender_topology_board_mut<'a>(
4402 sender_state: &'a mut DiscoverySenderState,
4403 sender_id: &str,
4404 ) -> &'a mut TopologyBoardNode {
4405 if let Some(idx) = sender_state
4406 .topology_boards
4407 .iter()
4408 .position(|board| board.sender_id == sender_id)
4409 {
4410 return &mut sender_state.topology_boards[idx];
4411 }
4412 sender_state.topology_boards.push(TopologyBoardNode {
4413 sender_id: sender_id.to_string(),
4414 reachable_endpoints: Vec::new(),
4415 reachable_timesync_sources: Vec::new(),
4416 connections: Vec::new(),
4417 });
4418 sender_state
4419 .topology_boards
4420 .last_mut()
4421 .expect("board inserted above")
4422 }
4423
4424 #[cfg(feature = "discovery")]
4425 fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
4426 discovery::normalize_topology_boards(&mut sender_state.topology_boards);
4427 let (reachable, reachable_timesync_sources) =
4428 discovery::summarize_topology_boards(&sender_state.topology_boards);
4429 sender_state.reachable = reachable;
4430 sender_state.reachable_timesync_sources = reachable_timesync_sources;
4431 }
4432
4433 #[cfg(feature = "discovery")]
4434 fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
4435 let mut reachable = Vec::new();
4436 let mut reachable_timesync_sources = Vec::new();
4437 let mut last_seen_ms = 0u64;
4438 for sender in route.announcers.values() {
4439 reachable.extend(sender.reachable.iter().copied());
4440 reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
4441 last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
4442 }
4443 reachable.sort_unstable();
4444 reachable.dedup();
4445 reachable_timesync_sources.sort_unstable();
4446 reachable_timesync_sources.dedup();
4447 route.reachable = reachable;
4448 route.reachable_timesync_sources = reachable_timesync_sources;
4449 route.last_seen_ms = last_seen_ms;
4450 }
4451
4452 #[cfg(feature = "discovery")]
4453 fn local_discovery_topology_board(
4454 &self,
4455 st: &RouterInner,
4456 now_ms: u64,
4457 link_local_enabled: bool,
4458 ) -> TopologyBoardNode {
4459 let mut reachable_endpoints = self.local_discovery_endpoints();
4460 if !link_local_enabled {
4461 reachable_endpoints.retain(|ep| !ep.is_link_local_only());
4462 }
4463 let mut connections = Vec::new();
4464 for route in st.discovery_routes.values() {
4465 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4466 continue;
4467 }
4468 for (sender, sender_state) in route.announcers.iter() {
4469 if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
4470 connections.push(sender.clone());
4471 }
4472 }
4473 }
4474 connections.sort_unstable();
4475 connections.dedup();
4476 let sender = self.sender_arc();
4477 TopologyBoardNode {
4478 sender_id: sender.to_string(),
4479 reachable_endpoints,
4480 reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
4481 connections,
4482 }
4483 }
4484
4485 #[cfg(feature = "discovery")]
4486 fn advertised_discovery_topology_for_link_locked(
4487 &self,
4488 st: &RouterInner,
4489 now_ms: u64,
4490 link_local_enabled: bool,
4491 ) -> Vec<TopologyBoardNode> {
4492 let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
4493 for route in st.discovery_routes.values() {
4494 if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4495 continue;
4496 }
4497 for (announcer, sender_state) in route.announcers.iter() {
4498 if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
4499 continue;
4500 }
4501 let mut sender_boards = sender_state.topology_boards.clone();
4502 if sender_boards.is_empty() {
4503 let sender = self.sender_arc();
4504 sender_boards.push(TopologyBoardNode {
4505 sender_id: announcer.clone(),
4506 reachable_endpoints: sender_state.reachable.clone(),
4507 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
4508 connections: vec![sender.to_string()],
4509 });
4510 } else if let Some(board) = sender_boards
4511 .iter_mut()
4512 .find(|board| board.sender_id == *announcer)
4513 {
4514 board.connections.push(self.sender_arc().to_string());
4515 }
4516 if !link_local_enabled {
4517 for board in sender_boards.iter_mut() {
4518 board
4519 .reachable_endpoints
4520 .retain(|ep| !ep.is_link_local_only());
4521 }
4522 }
4523 discovery::merge_topology_boards(&mut boards, &sender_boards);
4524 }
4525 }
4526 discovery::normalize_topology_boards(&mut boards);
4527 boards
4528 }
4529
4530 #[cfg(feature = "discovery")]
4531 fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
4532 let before = st.discovery_routes.clone();
4533 st.discovery_routes.retain(|_, route| {
4534 route.announcers.retain(|_, sender| {
4535 now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
4536 });
4537 Self::recompute_discovery_side_state(route);
4538 !route.announcers.is_empty()
4539 });
4540 st.discovery_routes != before
4541 }
4542
4543 #[cfg(feature = "discovery")]
4544 fn advertised_discovery_endpoints_for_link_locked(
4545 &self,
4546 st: &RouterInner,
4547 now_ms: u64,
4548 link_local_enabled: bool,
4549 ) -> Vec<DataEndpoint> {
4550 let (reachable_endpoints, _) = discovery::summarize_topology_boards(
4551 &self.advertised_discovery_topology_for_link_locked(st, now_ms, link_local_enabled),
4552 );
4553 reachable_endpoints
4554 .into_iter()
4555 .filter(|ep| {
4556 !discovery::is_discovery_endpoint(*ep)
4557 && (link_local_enabled || !ep.is_link_local_only())
4558 })
4559 .collect()
4560 }
4561
4562 #[cfg(feature = "discovery")]
4563 fn advertised_discovery_timesync_sources_for_link_locked(
4564 &self,
4565 st: &RouterInner,
4566 now_ms: u64,
4567 ) -> Vec<String> {
4568 let (_, sources) = discovery::summarize_topology_boards(
4569 &self.advertised_discovery_topology_for_link_locked(st, now_ms, true),
4570 );
4571 sources
4572 }
4573
4574 #[cfg(feature = "discovery")]
4575 fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
4576 let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true);
4577 discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
4578 }
4579
4580 #[cfg(feature = "discovery")]
4581 fn should_answer_discovery_request_locked(
4582 &self,
4583 st: &RouterInner,
4584 requester: &str,
4585 now_ms: u64,
4586 ) -> bool {
4587 if requester == self.sender_arc().as_ref() {
4588 return false;
4589 }
4590 self.discovery_master_sender_locked(st, now_ms) == self.sender_arc().as_ref()
4591 }
4592
4593 #[cfg(feature = "discovery")]
4594 #[inline]
4595 fn side_is_slow_control_link_locked(
4596 st: &RouterInner,
4597 side_id: RouterSideId,
4598 now_ms: u64,
4599 ) -> bool {
4600 st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
4601 let recent_slow = stats.last_slow_observed_ms > 0
4602 && now_ms.saturating_sub(stats.last_slow_observed_ms)
4603 <= DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
4604 stats.sample_count > 0
4605 && ((stats.estimated_bandwidth_bps > 0
4606 && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
4607 || recent_slow)
4608 })
4609 }
4610
4611 #[cfg(feature = "discovery")]
4612 fn discovery_level_for_side_locked(
4613 st: &mut RouterInner,
4614 side_id: RouterSideId,
4615 now_ms: u64,
4616 ) -> Option<DiscoveryAdvertiseLevel> {
4617 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
4618 st.discovery_side_throttle.remove(&side_id);
4619 return Some(DiscoveryAdvertiseLevel::Full);
4620 }
4621
4622 let throttle = st.discovery_side_throttle.entry(side_id).or_default();
4623 if now_ms >= throttle.next_full_ms {
4624 throttle.next_full_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS);
4625 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
4626 return Some(DiscoveryAdvertiseLevel::Full);
4627 }
4628 if now_ms >= throttle.next_ping_ms {
4629 throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
4630 return Some(DiscoveryAdvertiseLevel::MinimalPing);
4631 }
4632 None
4633 }
4634
4635 #[cfg(all(feature = "discovery", feature = "timesync"))]
4636 #[inline]
4637 fn is_timesync_type(ty: DataType) -> bool {
4638 matches!(
4639 ty,
4640 DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4641 )
4642 }
4643
4644 #[cfg(all(feature = "discovery", feature = "timesync"))]
4645 fn timesync_allowed_for_side_locked(
4646 st: &mut RouterInner,
4647 side_id: RouterSideId,
4648 ty: DataType,
4649 now_ms: u64,
4650 ) -> bool {
4651 if !Self::is_timesync_type(ty) {
4652 return true;
4653 }
4654 if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
4655 st.timesync_side_throttle.remove(&side_id);
4656 return true;
4657 }
4658
4659 let throttle = st.timesync_side_throttle.entry(side_id).or_default();
4660 if now_ms >= throttle.next_allowed_ms {
4661 throttle.next_allowed_ms = now_ms.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
4662 return true;
4663 }
4664 false
4665 }
4666
4667 #[cfg(all(feature = "discovery", feature = "timesync"))]
4668 fn filter_timesync_sides_locked(
4669 st: &mut RouterInner,
4670 ty: DataType,
4671 now_ms: u64,
4672 sides: Vec<RouterSideId>,
4673 ) -> Vec<RouterSideId> {
4674 sides
4675 .into_iter()
4676 .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
4677 .collect()
4678 }
4679
4680 #[cfg(all(feature = "discovery", feature = "timesync"))]
4681 fn filter_timesync_matches_locked(
4682 st: &mut RouterInner,
4683 ty: DataType,
4684 now_ms: u64,
4685 matches: Vec<DiscoveryCandidateMatch>,
4686 ) -> Vec<DiscoveryCandidateMatch> {
4687 matches
4688 .into_iter()
4689 .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
4690 .collect()
4691 }
4692
4693 #[cfg(feature = "discovery")]
4694 fn emit_discovery_snapshot(
4695 &self,
4696 called_from_queue: bool,
4697 include_schema: bool,
4698 include_topology: bool,
4699 ) -> TelemetryResult<()> {
4700 let now_ms = self.clock.now_ms();
4701 let per_side = {
4702 let mut st = self.state.lock();
4703 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
4704 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4705 Self::note_discovery_topology_change_locked(&mut st, now_ms);
4706 }
4707 st.fit_discovery_budget();
4708 let side_entries = st
4709 .sides
4710 .iter()
4711 .enumerate()
4712 .filter_map(|(side_id, side)| {
4713 side.as_ref()
4714 .map(|side| (side_id, side.opts.link_local_enabled, side.opts))
4715 })
4716 .collect::<Vec<_>>();
4717 let local_is_master =
4718 self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
4719 let mut per_side = Vec::new();
4720 for (side_id, link_local_enabled, opts) in side_entries {
4721 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
4722 {
4723 continue;
4724 }
4725 let Some(level) = Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
4726 else {
4727 continue;
4728 };
4729 let capabilities = opts.link_capabilities();
4730 if level == DiscoveryAdvertiseLevel::MinimalPing {
4731 per_side.push((
4732 side_id,
4733 level,
4734 Vec::new(),
4735 Vec::new(),
4736 Vec::new(),
4737 capabilities,
4738 local_is_master,
4739 ));
4740 continue;
4741 }
4742 per_side.push((
4743 side_id,
4744 level,
4745 self.advertised_discovery_endpoints_for_link_locked(
4746 &st,
4747 now_ms,
4748 link_local_enabled,
4749 ),
4750 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms),
4751 self.advertised_discovery_topology_for_link_locked(
4752 &st,
4753 now_ms,
4754 link_local_enabled,
4755 ),
4756 capabilities,
4757 local_is_master,
4758 ));
4759 }
4760 per_side
4761 };
4762 for (
4763 side_id,
4764 level,
4765 endpoints,
4766 timesync_sources,
4767 topology,
4768 capabilities,
4769 local_is_master,
4770 ) in per_side
4771 {
4772 let sender = self.sender_arc();
4773 if include_schema && level == DiscoveryAdvertiseLevel::Full {
4774 let pkt = discovery::build_discovery_schema(sender.as_ref(), now_ms)?;
4775 self.emit_internal_tx(
4776 RouterTxItem::ToSide {
4777 src: None,
4778 dst: side_id,
4779 data: RouterItem::Packet(pkt),
4780 },
4781 true,
4782 called_from_queue,
4783 )?;
4784 }
4785 if level == DiscoveryAdvertiseLevel::Full {
4786 let address = self.local_address_advertisement(
4787 endpoints.clone(),
4788 timesync_sources.clone(),
4789 capabilities,
4790 if local_is_master {
4791 discovery::ADDRESS_STATE_APPROVED
4792 } else {
4793 discovery::ADDRESS_STATE_REQUEST
4794 },
4795 );
4796 let pkt = discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?;
4797 self.emit_internal_tx(
4798 RouterTxItem::ToSide {
4799 src: None,
4800 dst: side_id,
4801 data: RouterItem::Packet(pkt),
4802 },
4803 true,
4804 called_from_queue,
4805 )?;
4806 }
4807 if level == DiscoveryAdvertiseLevel::MinimalPing {
4808 let pkt = discovery::build_discovery_announce(
4809 sender.as_ref(),
4810 now_ms,
4811 endpoints.as_slice(),
4812 )?;
4813 self.emit_internal_tx(
4814 RouterTxItem::ToSide {
4815 src: None,
4816 dst: side_id,
4817 data: RouterItem::Packet(pkt),
4818 },
4819 true,
4820 called_from_queue,
4821 )?;
4822 }
4823 if include_topology && level == DiscoveryAdvertiseLevel::Full && !topology.is_empty() {
4824 let pkt = discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?;
4825 self.emit_internal_tx(
4826 RouterTxItem::ToSide {
4827 src: None,
4828 dst: side_id,
4829 data: RouterItem::Packet(pkt),
4830 },
4831 true,
4832 called_from_queue,
4833 )?;
4834 }
4835 }
4836 Ok(())
4837 }
4838
4839 #[cfg(feature = "discovery")]
4840 fn queue_discovery_announce(&self) -> TelemetryResult<()> {
4841 let now_ms = self.clock.now_ms();
4842 {
4843 let mut st = self.state.lock();
4844 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
4845 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4846 Self::note_discovery_topology_change_locked(&mut st, now_ms);
4847 }
4848 st.fit_discovery_budget();
4849 if st.sides.iter().all(|side| side.is_none()) {
4850 return Ok(());
4851 }
4852 st.discovery_cadence.on_announce_sent(now_ms);
4853 }
4854 self.emit_discovery_snapshot(true, true, true)
4855 }
4856
4857 #[cfg(feature = "discovery")]
4858 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
4859 let now_ms = self.clock.now_ms();
4860 let due = {
4861 let mut st = self.state.lock();
4862 let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
4863 if removed {
4864 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4865 Self::note_discovery_topology_change_locked(&mut st, now_ms);
4866 }
4867 st.fit_discovery_budget();
4868 let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
4869 let Some(side) = side.as_ref() else {
4870 return false;
4871 };
4872 if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
4873 {
4874 return false;
4875 }
4876 let _ = side;
4877 true
4878 });
4879 if st.sides.is_empty() || !has_any {
4880 return Ok(false);
4881 }
4882 st.discovery_cadence.due(now_ms)
4883 };
4884 if !due {
4885 return Ok(false);
4886 }
4887 self.queue_discovery_announce()?;
4888 Ok(true)
4889 }
4890
4891 #[cfg(feature = "discovery")]
4892 fn learn_discovery_packet(
4893 &self,
4894 pkt: &Packet,
4895 src: Option<RouterSideId>,
4896 called_from_queue: bool,
4897 ) -> TelemetryResult<bool> {
4898 if !discovery::is_discovery_type(pkt.data_type()) {
4899 return Ok(false);
4900 }
4901 let Some(side) = src else {
4902 return Ok(true);
4903 };
4904 if pkt.data_type() == DataType::DiscoveryAddress {
4905 let mut ad = discovery::decode_discovery_address(pkt)?;
4906 let mut changed = self.ingest_address_advertisement(ad.clone())?;
4907 let now_ms = self.clock.now_ms();
4908 let mut st = self.state.lock();
4909 let side_link_local_enabled = st
4910 .sides
4911 .get(side)
4912 .and_then(|entry| entry.as_ref())
4913 .map(|side_ref| side_ref.opts.link_local_enabled)
4914 .unwrap_or(false);
4915 if !side_link_local_enabled {
4916 ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
4917 }
4918 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
4919 let mut sender_state = route
4920 .announcers
4921 .get(pkt.sender())
4922 .cloned()
4923 .unwrap_or_default();
4924 let board = Self::sender_topology_board_mut(&mut sender_state, pkt.sender());
4925 if board.reachable_endpoints != ad.reachable_endpoints {
4926 board.reachable_endpoints = ad.reachable_endpoints;
4927 changed = true;
4928 }
4929 if board.reachable_timesync_sources != ad.reachable_timesync_sources {
4930 board.reachable_timesync_sources = ad.reachable_timesync_sources;
4931 changed = true;
4932 }
4933 Self::refresh_sender_topology_state(&mut sender_state);
4934 sender_state.last_seen_ms = now_ms;
4935 route
4936 .announcers
4937 .insert(pkt.sender().to_string(), sender_state);
4938 Self::recompute_discovery_side_state(&mut route);
4939 st.discovery_routes.insert(side, route);
4940 st.fit_discovery_budget();
4941 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4942 if changed {
4943 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
4944 }
4945 return Ok(true);
4946 }
4947 let local_sender = self.sender_arc();
4948 if pkt.sender() == local_sender.as_ref() {
4949 return Ok(true);
4950 }
4951 if pkt.data_type() == DataType::DiscoveryTopologyRequest {
4952 let now_ms = self.clock.now_ms();
4953 let should_answer = {
4954 let mut st = self.state.lock();
4955 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
4956 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4957 Self::note_discovery_topology_change_locked(&mut st, now_ms);
4958 }
4959 self.should_answer_discovery_request_locked(&st, pkt.sender(), now_ms)
4960 };
4961 if should_answer {
4962 self.emit_discovery_snapshot(called_from_queue, false, true)?;
4963 }
4964 return Ok(true);
4965 }
4966 if pkt.data_type() == DataType::DiscoverySchemaRequest {
4967 let now_ms = self.clock.now_ms();
4968 let should_answer = {
4969 let mut st = self.state.lock();
4970 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
4971 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
4972 Self::note_discovery_topology_change_locked(&mut st, now_ms);
4973 }
4974 self.should_answer_discovery_request_locked(&st, pkt.sender(), now_ms)
4975 };
4976 if should_answer {
4977 self.emit_discovery_snapshot(called_from_queue, true, true)?;
4978 }
4979 return Ok(true);
4980 }
4981 if pkt.data_type() == DataType::ManagedVariableRequest {
4982 let ty = discovery::decode_managed_variable_request(pkt)?;
4983 if !self.can_read_managed_variable(ty) {
4984 let payload = make_error_payload("managed variable permission denied");
4985 let err = Packet::new(
4986 DataType::TelemetryError,
4987 message_meta(DataType::TelemetryError).endpoints,
4988 self.sender_arc().as_ref(),
4989 self.clock.now_ms(),
4990 payload,
4991 )?;
4992 self.emit_internal_tx(
4993 RouterTxItem::ToSide {
4994 src: None,
4995 dst: side,
4996 data: RouterItem::Packet(err),
4997 },
4998 true,
4999 called_from_queue,
5000 )?;
5001 return Ok(true);
5002 }
5003 if let Some(value) = self.managed_variable_latest(ty) {
5004 self.emit_internal_tx(
5005 RouterTxItem::ToSide {
5006 src: None,
5007 dst: side,
5008 data: RouterItem::Packet(value),
5009 },
5010 true,
5011 called_from_queue,
5012 )?;
5013 }
5014 return Ok(true);
5015 }
5016 if pkt.data_type() == DataType::DiscoverySchema {
5017 let snapshot = discovery::decode_discovery_schema(pkt)?;
5018 let incoming_cost = crate::config::owned_schema_byte_cost(&snapshot);
5019 let mut st = self.state.lock();
5020 st.make_shared_queue_room(incoming_cost, RouterQueueKind::Discovery)?;
5021 let budget = st.memory.max_queue_budget;
5022 drop(st);
5023 let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
5024 if report.changed() {
5025 let mut st = self.state.lock();
5026 st.fit_discovery_budget();
5027 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
5028 }
5029 return Ok(true);
5030 }
5031 if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
5032 let _ = discovery::decode_discovery_link_capabilities(pkt)?;
5033 return Ok(true);
5034 }
5035 let mut st = self.state.lock();
5036 let now_ms = self.clock.now_ms();
5037 if pkt.data_type() == DataType::DiscoveryLeave {
5038 let leaving = pkt.sender();
5039 let before = st.discovery_routes.clone();
5040 for route in st.discovery_routes.values_mut() {
5041 route.announcers.remove(leaving);
5042 for sender_state in route.announcers.values_mut() {
5043 sender_state
5044 .topology_boards
5045 .retain(|board| board.sender_id != leaving);
5046 for board in sender_state.topology_boards.iter_mut() {
5047 board.connections.retain(|peer| peer != leaving);
5048 }
5049 Self::refresh_sender_topology_state(sender_state);
5050 }
5051 Self::recompute_discovery_side_state(route);
5052 }
5053 st.discovery_routes
5054 .retain(|_, route| !route.announcers.is_empty());
5055 if st.discovery_routes != before {
5056 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5057 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5058 }
5059 return Ok(true);
5060 }
5061 let mut route = st.discovery_routes.get(&side).cloned().unwrap_or_default();
5062 let side_link_local_enabled = st
5063 .sides
5064 .get(side)
5065 .and_then(|entry| entry.as_ref())
5066 .map(|side_ref| side_ref.opts.link_local_enabled)
5067 .unwrap_or(false);
5068 let mut sender_state = route
5069 .announcers
5070 .get(pkt.sender())
5071 .cloned()
5072 .unwrap_or_default();
5073 let changed = match pkt.data_type() {
5074 DataType::DiscoveryAnnounce => {
5075 let mut reachable = discovery::decode_discovery_announce(pkt)?;
5076 if !side_link_local_enabled {
5077 reachable.retain(|ep| !ep.is_link_local_only());
5078 }
5079 let board = Self::sender_topology_board_mut(&mut sender_state, pkt.sender());
5080 let changed = board.reachable_endpoints != reachable;
5081 board.reachable_endpoints = reachable;
5082 Self::refresh_sender_topology_state(&mut sender_state);
5083 changed
5084 }
5085 DataType::DiscoveryTimeSyncSources => {
5086 let sources = discovery::decode_discovery_timesync_sources(pkt)?;
5087 let board = Self::sender_topology_board_mut(&mut sender_state, pkt.sender());
5088 let changed = board.reachable_timesync_sources != sources;
5089 board.reachable_timesync_sources = sources;
5090 Self::refresh_sender_topology_state(&mut sender_state);
5091 changed
5092 }
5093 DataType::DiscoveryTopology => {
5094 let mut boards = discovery::decode_discovery_topology(pkt)?;
5095 if !side_link_local_enabled {
5096 for board in boards.iter_mut() {
5097 board
5098 .reachable_endpoints
5099 .retain(|ep| !ep.is_link_local_only());
5100 }
5101 }
5102 let changed = sender_state.topology_boards != boards;
5103 sender_state.topology_boards = boards;
5104 Self::refresh_sender_topology_state(&mut sender_state);
5105 changed
5106 }
5107 DataType::DiscoverySchema => false,
5108 _ => false,
5109 };
5110 sender_state.last_seen_ms = now_ms;
5111 route
5112 .announcers
5113 .insert(pkt.sender().to_string(), sender_state);
5114 Self::recompute_discovery_side_state(&mut route);
5115 st.discovery_routes.insert(side, route);
5116 st.fit_discovery_budget();
5117 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
5118 if changed {
5119 Self::note_discovery_topology_change_locked(&mut st, now_ms);
5120 }
5121 Ok(true)
5122 }
5123
5124 #[cfg(not(feature = "discovery"))]
5125 fn queue_discovery_announce(&self) -> TelemetryResult<()> {
5126 Ok(())
5127 }
5128
5129 #[cfg(not(feature = "discovery"))]
5130 fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
5131 Ok(false)
5132 }
5133
5134 #[cfg(not(feature = "discovery"))]
5135 fn learn_discovery_packet(
5136 &self,
5137 _pkt: &Packet,
5138 _src: Option<RouterSideId>,
5139 _called_from_queue: bool,
5140 ) -> TelemetryResult<bool> {
5141 Ok(false)
5142 }
5143
5144 #[inline]
5145 fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
5146 (side, ty.as_u32())
5147 }
5148
5149 fn reliable_tx_state_mut<'a>(
5150 &'a self,
5151 st: &'a mut RouterInner,
5152 side: RouterSideId,
5153 ty: DataType,
5154 ) -> &'a mut ReliableTxState {
5155 let key = Self::reliable_key(side, ty);
5156 st.reliable_tx
5157 .entry(key)
5158 .or_insert_with(|| ReliableTxState {
5159 next_seq: 1,
5160 sent_order: VecDeque::new(),
5161 sent: BTreeMap::new(),
5162 })
5163 }
5164
5165 fn reliable_rx_state_mut<'a>(
5166 &'a self,
5167 st: &'a mut RouterInner,
5168 side: RouterSideId,
5169 ty: DataType,
5170 ) -> &'a mut ReliableRxState {
5171 let key = Self::reliable_key(side, ty);
5172 st.reliable_rx
5173 .entry(key)
5174 .or_insert_with(|| ReliableRxState {
5175 expected_seq: 1,
5176 buffered: BTreeMap::new(),
5177 })
5178 }
5179
5180 fn reliable_control_packet(
5181 &self,
5182 control_ty: DataType,
5183 ty: DataType,
5184 seq: u32,
5185 ) -> TelemetryResult<Packet> {
5186 let sender = self.sender_arc();
5187 Packet::new(
5188 control_ty,
5189 message_meta(control_ty).endpoints,
5190 sender.as_ref(),
5191 self.packet_timestamp_ms(),
5192 encode_slice_le(&[ty.as_u32(), seq]),
5193 )
5194 }
5195
5196 fn queue_reliable_ack(
5197 &self,
5198 side: RouterSideId,
5199 ty: DataType,
5200 seq: u32,
5201 called_from_queue: bool,
5202 ) -> TelemetryResult<()> {
5203 let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
5204 self.emit_internal_tx_with_priority(
5205 RouterTxItem::ToSide {
5206 src: None,
5207 dst: side,
5208 data: RouterItem::Packet(pkt),
5209 },
5210 true,
5211 message_priority(DataType::ReliableAck),
5212 called_from_queue,
5213 )
5214 }
5215
5216 fn queue_reliable_packet_request(
5217 &self,
5218 side: RouterSideId,
5219 ty: DataType,
5220 seq: u32,
5221 called_from_queue: bool,
5222 ) -> TelemetryResult<()> {
5223 let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
5224 self.emit_internal_tx_with_priority(
5225 RouterTxItem::ToSide {
5226 src: None,
5227 dst: side,
5228 data: RouterItem::Packet(pkt),
5229 },
5230 true,
5231 message_priority(DataType::ReliablePacketRequest),
5232 called_from_queue,
5233 )
5234 }
5235
5236 fn queue_reliable_partial_ack(
5237 &self,
5238 side: RouterSideId,
5239 ty: DataType,
5240 seq: u32,
5241 called_from_queue: bool,
5242 ) -> TelemetryResult<()> {
5243 let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
5244 self.emit_internal_tx_with_priority(
5245 RouterTxItem::ToSide {
5246 src: None,
5247 dst: side,
5248 data: RouterItem::Packet(pkt),
5249 },
5250 true,
5251 message_priority(DataType::ReliablePartialAck),
5252 called_from_queue,
5253 )
5254 }
5255
5256 fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
5257 let mut st = self.state.lock();
5258 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
5259 if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
5260 tx_state.sent.remove(&ack);
5261 tx_state.sent_order.retain(|seq| *seq != ack);
5262 return;
5263 }
5264
5265 while let Some(seq) = tx_state.sent_order.front().copied() {
5266 if seq > ack {
5267 break;
5268 }
5269 tx_state.sent_order.pop_front();
5270 tx_state.sent.remove(&seq);
5271 }
5272 }
5273
5274 fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
5275 let mut st = self.state.lock();
5276 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
5277 if let Some(sent) = tx_state.sent.get_mut(&seq) {
5278 sent.partial_acked = true;
5279 }
5280 }
5281
5282 fn queue_reliable_retransmit(
5283 &self,
5284 side: RouterSideId,
5285 ty: DataType,
5286 seq: u32,
5287 called_from_queue: bool,
5288 ) -> TelemetryResult<()> {
5289 let mut queued = None;
5290 {
5291 let mut st = self.state.lock();
5292 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
5293 if let Some(sent) = tx_state.sent.get_mut(&seq)
5294 && !sent.queued
5295 {
5296 sent.queued = true;
5297 sent.partial_acked = false;
5298 queued = Some(sent.bytes.clone());
5299 }
5300 }
5301
5302 if let Some(bytes) = queued {
5303 if called_from_queue {
5304 self.tx_queue_item_with_priority(
5305 RouterTxItem::ReliableReplay { dst: side, bytes },
5306 true,
5307 Self::router_item_priority_bumped(ty),
5308 )?;
5309 } else {
5310 self.tx_item_impl(
5311 RouterTxItem::ReliableReplay { dst: side, bytes },
5312 true,
5313 false,
5314 )?;
5315 }
5316 }
5317
5318 Ok(())
5319 }
5320
5321 fn send_reliable_raw_to_side(
5322 &self,
5323 side: RouterSideId,
5324 bytes: Arc<[u8]>,
5325 relayed: bool,
5326 ) -> TelemetryResult<()> {
5327 let handler = {
5328 let st = self.state.lock();
5329 let side_ref = Self::side_ref(&st, side)?;
5330 if !side_ref.opts.egress_enabled {
5331 return Ok(());
5332 }
5333 (side_ref.tx_handler.clone(), side_ref.opts)
5334 };
5335
5336 let (handler, opts) = handler;
5337
5338 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
5339 return Err(TelemetryError::Io("side tx busy"));
5340 };
5341 let started_ms = self.clock.now_ms();
5342 let ty = wire_format::peek_envelope(bytes.as_ref())
5343 .map(|env| env.ty)
5344 .unwrap_or(DataType::ReliableAck);
5345 let result = match handler {
5346 RouterTxHandlerFn::Packed(f) => {
5347 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
5348 let mut attempts_total = 0usize;
5349 let mut sent_bytes = 0usize;
5350 for frame in frames {
5351 match self
5352 .retry_with_attempts(runtime_max_handler_retries(), || f(frame.as_ref()))
5353 {
5354 Ok((_, attempts)) => {
5355 attempts_total = attempts_total.saturating_add(attempts);
5356 sent_bytes = sent_bytes.saturating_add(frame.len());
5357 }
5358 Err((err, attempts)) => {
5359 self.note_side_tx_failure(
5360 side,
5361 ty,
5362 attempts_total.saturating_add(attempts),
5363 );
5364 return Err(err);
5365 }
5366 }
5367 }
5368 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
5369 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
5370 return Ok(());
5371 }
5372 RouterTxHandlerFn::Packet(f) => {
5373 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
5374 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
5375 }
5376 };
5377 match result {
5378 Ok((_, attempts)) => {
5379 self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
5380 self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
5381 Ok(())
5382 }
5383 Err((err, attempts)) => {
5384 self.note_side_tx_failure(side, ty, attempts);
5385 Err(err)
5386 }
5387 }
5388 }
5389
5390 fn send_reliable_to_side(
5391 &self,
5392 side: RouterSideId,
5393 data: RouterItem,
5394 relayed: bool,
5395 ) -> TelemetryResult<()> {
5396 let (handler, opts, hop_reliable_enabled) = {
5397 let st = self.state.lock();
5398 let side_ref = Self::side_ref(&st, side)?;
5399 let opts = side_ref.opts;
5400 let hop_reliable_enabled = opts.reliable_enabled
5401 && self.cfg.reliable_enabled()
5402 && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
5403 (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
5404 };
5405
5406 let RouterTxHandlerFn::Packed(f) = &handler else {
5407 return self.call_side_tx_handler(side, &handler, &data, relayed);
5408 };
5409
5410 if !hop_reliable_enabled {
5411 let mut adjusted_opts = opts;
5412 adjusted_opts.reliable_enabled = false;
5413 let preserve_end_to_end_ack = opts.reliable_enabled && self.cfg.reliable_enabled();
5414 if let Some(adjusted) =
5415 self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
5416 {
5417 return self.call_side_tx_handler(side, &handler, &adjusted, relayed);
5418 }
5419 return Ok(());
5420 }
5421
5422 let ty = match &data {
5423 RouterItem::Packet(pkt) => pkt.data_type(),
5424 RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.envelope.ty,
5425 };
5426
5427 if !is_reliable_type(ty) {
5428 if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
5429 self.call_side_tx_handler(side, &handler, &adjusted, relayed)?;
5430 }
5431 return Ok(());
5432 }
5433
5434 let (seq, flags) = {
5435 let mut st = self.state.lock();
5436 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
5437 if tx_state.sent.len() >= runtime_reliable_max_pending() {
5438 return Err(TelemetryError::PacketTooLarge(
5439 "router reliable history full",
5440 ));
5441 }
5442 let seq = tx_state.next_seq;
5443 let next = tx_state.next_seq.wrapping_add(1);
5444 tx_state.next_seq = if next == 0 { 1 } else { next };
5445 let flags = match reliable_mode(ty) {
5446 crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
5447 _ => 0,
5448 };
5449 (seq, flags)
5450 };
5451
5452 let bytes: Arc<[u8]> = match data {
5453 RouterItem::Packet(pkt) => self.pack_packet_for_router(
5454 &pkt,
5455 Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
5456 )?,
5457 RouterItem::Packed(bytes) => {
5458 #[cfg(feature = "cryptography")]
5459 if self.e2e_seal_config_for_type(ty).is_some() {
5460 self.prepare_packed_for_remote(
5461 bytes,
5462 Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
5463 )?
5464 } else {
5465 let Some(rewritten) =
5466 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
5467 else {
5468 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
5469 return Err(TelemetryError::Io("side tx busy"));
5470 };
5471 let started_ms = self.clock.now_ms();
5472 let frames =
5473 self.encode_side_transport_frames(side, opts, bytes.clone())?;
5474 let mut attempts_total = 0usize;
5475 let mut sent_bytes = 0usize;
5476 for frame in frames {
5477 match self.retry_with_attempts(runtime_max_handler_retries(), || {
5478 f(frame.as_ref())
5479 }) {
5480 Ok((_, attempts)) => {
5481 attempts_total = attempts_total.saturating_add(attempts);
5482 sent_bytes = sent_bytes.saturating_add(frame.len());
5483 }
5484 Err((err, attempts)) => {
5485 self.note_side_tx_failure(
5486 side,
5487 ty,
5488 attempts_total.saturating_add(attempts),
5489 );
5490 return Err(err);
5491 }
5492 }
5493 }
5494 self.record_side_tx_sample(
5495 side,
5496 sent_bytes,
5497 started_ms,
5498 self.clock.now_ms(),
5499 );
5500 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
5501 return Ok(());
5502 };
5503 rewritten
5504 }
5505 #[cfg(not(feature = "cryptography"))]
5506 {
5507 let Some(rewritten) =
5508 wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
5509 else {
5510 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
5511 return Err(TelemetryError::Io("side tx busy"));
5512 };
5513 let started_ms = self.clock.now_ms();
5514 let frames =
5515 self.encode_side_transport_frames(side, opts, bytes.clone())?;
5516 let mut attempts_total = 0usize;
5517 let mut sent_bytes = 0usize;
5518 for frame in frames {
5519 match self.retry_with_attempts(runtime_max_handler_retries(), || {
5520 f(frame.as_ref())
5521 }) {
5522 Ok((_, attempts)) => {
5523 attempts_total = attempts_total.saturating_add(attempts);
5524 sent_bytes = sent_bytes.saturating_add(frame.len());
5525 }
5526 Err((err, attempts)) => {
5527 self.note_side_tx_failure(
5528 side,
5529 ty,
5530 attempts_total.saturating_add(attempts),
5531 );
5532 return Err(err);
5533 }
5534 }
5535 }
5536 self.record_side_tx_sample(
5537 side,
5538 sent_bytes,
5539 started_ms,
5540 self.clock.now_ms(),
5541 );
5542 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
5543 return Ok(());
5544 };
5545 rewritten
5546 }
5547 }
5548 };
5549
5550 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
5551 return Err(TelemetryError::Io("side tx busy"));
5552 };
5553 let started_ms = self.clock.now_ms();
5554 let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
5555 let mut attempts_total = 0usize;
5556 let mut sent_bytes = 0usize;
5557 for frame in frames {
5558 match self.retry_with_attempts(runtime_max_handler_retries(), || f(frame.as_ref())) {
5559 Ok((_, attempts)) => {
5560 attempts_total = attempts_total.saturating_add(attempts);
5561 sent_bytes = sent_bytes.saturating_add(frame.len());
5562 }
5563 Err((err, attempts)) => {
5564 self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
5565 return Err(err);
5566 }
5567 }
5568 }
5569 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
5570 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
5571
5572 {
5573 let mut st = self.state.lock();
5574 let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
5575 tx_state.sent_order.push_back(seq);
5576 tx_state.sent.insert(
5577 seq,
5578 ReliableSent {
5579 bytes: bytes.clone(),
5580 last_send_ms: self.clock.now_ms(),
5581 retries: 0,
5582 queued: false,
5583 partial_acked: false,
5584 },
5585 );
5586 }
5587
5588 Ok(())
5589 }
5590
5591 #[inline]
5592 fn crc32_bytes(data: &[u8]) -> u32 {
5593 let mut hasher = Crc32Hasher::new();
5594 hasher.update(data);
5595 hasher.finalize()
5596 }
5597
5598 fn read_uleb128_local(buf: &[u8], off: &mut usize) -> TelemetryResult<u64> {
5599 let mut result = 0u64;
5600 let mut shift = 0u32;
5601 for _ in 0..10 {
5602 let byte = *buf.get(*off).ok_or(TelemetryError::Unpack("short read"))?;
5603 *off += 1;
5604 result |= u64::from(byte & 0x7F) << shift;
5605 if (byte & 0x80) == 0 {
5606 return Ok(result);
5607 }
5608 shift += 7;
5609 }
5610 Err(TelemetryError::Unpack("uleb128 too long"))
5611 }
5612
5613 fn write_uleb128_local(mut value: u64, out: &mut Vec<u8>) {
5614 loop {
5615 let mut byte = (value & 0x7F) as u8;
5616 value >>= 7;
5617 if value != 0 {
5618 byte |= 0x80;
5619 }
5620 out.push(byte);
5621 if value == 0 {
5622 break;
5623 }
5624 }
5625 }
5626
5627 fn uleb128_len_local(mut value: u64) -> usize {
5628 let mut len = 1;
5629 while value >= 0x80 {
5630 value >>= 7;
5631 len += 1;
5632 }
5633 len
5634 }
5635
5636 fn wrap_side_transport_frame(kind: u8, body: &[u8]) -> Arc<[u8]> {
5637 let mut out = Vec::with_capacity(
5638 SIDE_TRANSPORT_MAGIC.len() + 1 + body.len() + wire_format::CRC32_BYTES,
5639 );
5640 out.extend_from_slice(SIDE_TRANSPORT_MAGIC);
5641 out.push(kind);
5642 out.extend_from_slice(body);
5643 let crc = Self::crc32_bytes(&out);
5644 out.extend_from_slice(&crc.to_le_bytes());
5645 Arc::from(out)
5646 }
5647
5648 fn parse_side_transport_wrapper(bytes: &[u8]) -> TelemetryResult<Option<(u8, &[u8])>> {
5649 if bytes.len() < SIDE_TRANSPORT_MAGIC.len() + 1 + wire_format::CRC32_BYTES {
5650 return Ok(None);
5651 }
5652 if &bytes[..SIDE_TRANSPORT_MAGIC.len()] != SIDE_TRANSPORT_MAGIC {
5653 return Ok(None);
5654 }
5655 let data_len = bytes.len() - wire_format::CRC32_BYTES;
5656 let expected = u32::from_le_bytes([
5657 bytes[data_len],
5658 bytes[data_len + 1],
5659 bytes[data_len + 2],
5660 bytes[data_len + 3],
5661 ]);
5662 let data = &bytes[..data_len];
5663 if Self::crc32_bytes(data) != expected {
5664 return Err(TelemetryError::Unpack("side transport crc32 mismatch"));
5665 }
5666 let kind = data[SIDE_TRANSPORT_MAGIC.len()];
5667 Ok(Some((kind, &data[SIDE_TRANSPORT_MAGIC.len() + 1..])))
5668 }
5669
5670 fn extract_side_header_template(bytes: &[u8]) -> TelemetryResult<SideTemplateExtract<'_>> {
5671 if bytes.len() < wire_format::CRC32_BYTES + 4 {
5672 return Err(TelemetryError::Unpack("short buffer"));
5673 }
5674 let data_len = bytes.len() - wire_format::CRC32_BYTES;
5675 let data = &bytes[..data_len];
5676 let mut off = 0usize;
5677 let flags = *data
5678 .get(off)
5679 .ok_or(TelemetryError::Unpack("short prelude"))?;
5680 off += 1;
5681 off += 1; let ty_end_start = off;
5683 let ty_u64 = Self::read_uleb128_local(data, &mut off)?;
5684 let ty_u32 = u32::try_from(ty_u64).map_err(|_| TelemetryError::Unpack("bad data type"))?;
5685 if ty_u32 > crate::MAX_VALUE_DATA_TYPE {
5686 return Err(TelemetryError::Unpack("bad data type"));
5687 }
5688 let ty = DataType(ty_u32);
5689 let data_size_off = off;
5690 let data_size = Self::read_uleb128_local(data, &mut off)?;
5691 let _timestamp_off = off;
5692 let timestamp = Self::read_uleb128_local(data, &mut off)?;
5693 let nonce = if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
5694 u16::try_from(Self::read_uleb128_local(data, &mut off)?)
5695 .map_err(|_| TelemetryError::Unpack("packet nonce too large"))?
5696 } else {
5697 0
5698 };
5699 let between_start = off;
5700 let _source_address = u32::try_from(Self::read_uleb128_local(data, &mut off)?)
5701 .map_err(|_| TelemetryError::Unpack("source address too large"))?;
5702 let endpoint_bitmap_bytes = if (flags & SIDE_TRANSPORT_FLAG_ENDPOINT_BITMAP_PRESENT) != 0 {
5703 SIDE_TRANSPORT_EP_BITMAP_BYTES
5704 } else {
5705 0
5706 };
5707 if data.len() < off + endpoint_bitmap_bytes {
5708 return Err(TelemetryError::Unpack("short buffer"));
5709 }
5710 off += endpoint_bitmap_bytes;
5711 if (flags & SIDE_TRANSPORT_FLAG_WIRE_CONTRACT) != 0 {
5712 let contract_len = usize::try_from(Self::read_uleb128_local(data, &mut off)?)
5713 .map_err(|_| TelemetryError::Unpack("wire contract length"))?;
5714 if data.len() < off + contract_len {
5715 return Err(TelemetryError::Unpack("short buffer"));
5716 }
5717 off += contract_len;
5718 }
5719 let reliable_span = wire_format::reliable_header_span(bytes)?;
5720 let (reliable_flags, reliable_seq_ack, reliable_compact, payload_off) =
5721 if let Some((rel_off, rel_len, hdr)) = reliable_span {
5722 if data.len() < rel_off + rel_len {
5723 return Err(TelemetryError::Unpack("short buffer"));
5724 }
5725 (
5726 Some(hdr.flags),
5727 Some((hdr.seq, hdr.ack)),
5728 (flags & SIDE_TRANSPORT_FLAG_COMPACT_RELIABLE_HEADER) != 0,
5729 rel_off + rel_len,
5730 )
5731 } else {
5732 (None, None, false, off)
5733 };
5734 if payload_off > data.len() {
5735 return Err(TelemetryError::Unpack("short buffer"));
5736 }
5737 let payload = &data[payload_off..];
5738 let prefix = Arc::<[u8]>::from(&data[1..data_size_off]);
5739 let between_end = reliable_span
5740 .map(|(rel_off, _, _)| rel_off)
5741 .unwrap_or(payload_off);
5742 let between = Arc::<[u8]>::from(&data[between_start..between_end]);
5743 let base_flags =
5744 flags & !(SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED | SIDE_TRANSPORT_FLAG_PACKET_NONCE);
5745 let mut hash = 0xD1B5_4A32_9C7E_01F3u64;
5746 hash = hash_bytes_u64(hash, &[base_flags]);
5747 hash = hash_bytes_u64(hash, &prefix);
5748 hash = hash_bytes_u64(hash, &between);
5749 if let Some(rel_flags) = reliable_flags {
5750 hash = hash_bytes_u64(hash, &[rel_flags]);
5751 }
5752 let template = SideHeaderTemplate {
5753 hash,
5754 base_flags,
5755 prefix,
5756 between,
5757 reliable_flags,
5758 reliable_compact,
5759 };
5760 let _ = ty_end_start;
5761 Ok((
5762 template,
5763 ty,
5764 flags,
5765 data_size,
5766 timestamp,
5767 nonce,
5768 reliable_seq_ack,
5769 payload,
5770 ))
5771 }
5772
5773 fn reconstruct_side_compact_frame(
5774 template: &SideHeaderTemplate,
5775 body: &[u8],
5776 timestamp_mode: SideCompactTimestampMode,
5777 timestamp_base: Option<u64>,
5778 ) -> TelemetryResult<(Arc<[u8]>, u64)> {
5779 if body.is_empty() {
5780 return Err(TelemetryError::Unpack("short side compact frame"));
5781 }
5782 let mut off = 0usize;
5783 let flags = body[off];
5784 off += 1;
5785 if (flags & !(SIDE_TRANSPORT_FLAG_PAYLOAD_COMPRESSED | SIDE_TRANSPORT_FLAG_PACKET_NONCE))
5786 != template.base_flags
5787 {
5788 return Err(TelemetryError::Unpack("side compact flags mismatch"));
5789 }
5790 let data_size = Self::read_uleb128_local(body, &mut off)?;
5791 let timestamp = match timestamp_mode {
5792 SideCompactTimestampMode::Absolute => Self::read_uleb128_local(body, &mut off)?,
5793 SideCompactTimestampMode::Delta => {
5794 let timestamp_field = Self::read_uleb128_local(body, &mut off)?;
5795 let base = timestamp_base.ok_or(TelemetryError::Unpack(
5796 "missing side compact timestamp context",
5797 ))?;
5798 base.checked_add(timestamp_field)
5799 .ok_or(TelemetryError::Unpack(
5800 "side compact timestamp delta overflow",
5801 ))?
5802 }
5803 SideCompactTimestampMode::Omitted => timestamp_base.ok_or(TelemetryError::Unpack(
5804 "missing side compact timestamp context",
5805 ))?,
5806 };
5807 let nonce = if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
5808 Some(Self::read_uleb128_local(body, &mut off)?)
5809 } else {
5810 None
5811 };
5812 let reliable_seq_ack = if template.reliable_flags.is_some() {
5813 let seq = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
5814 .map_err(|_| TelemetryError::Unpack("side compact reliable seq too large"))?;
5815 let ack = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
5816 .map_err(|_| TelemetryError::Unpack("side compact reliable ack too large"))?;
5817 Some((seq, ack))
5818 } else {
5819 None
5820 };
5821 let payload = &body[off..];
5822 let mut raw = Vec::with_capacity(
5823 1 + template.prefix.len() + template.between.len() + payload.len() + 32,
5824 );
5825 raw.push(flags);
5826 raw.extend_from_slice(&template.prefix);
5827 Self::write_uleb128_local(data_size, &mut raw);
5828 Self::write_uleb128_local(timestamp, &mut raw);
5829 if let Some(nonce) = nonce {
5830 Self::write_uleb128_local(nonce, &mut raw);
5831 }
5832 raw.extend_from_slice(&template.between);
5833 if let Some(rel_flags) = template.reliable_flags {
5834 let (seq, ack) =
5835 reliable_seq_ack.ok_or(TelemetryError::Unpack("missing side compact reliable"))?;
5836 wire_format::write_reliable_header_encoded(
5837 wire_format::ReliableHeader {
5838 flags: rel_flags,
5839 seq,
5840 ack,
5841 },
5842 template.reliable_compact,
5843 &mut raw,
5844 );
5845 }
5846 raw.extend_from_slice(payload);
5847 let crc = Self::crc32_bytes(&raw);
5848 raw.extend_from_slice(&crc.to_le_bytes());
5849 Ok((Arc::from(raw), timestamp))
5850 }
5851 fn split_side_transport_frame(
5852 &self,
5853 side: RouterSideId,
5854 frame: Arc<[u8]>,
5855 max_frame_bytes: usize,
5856 ) -> TelemetryResult<Vec<Arc<[u8]>>> {
5857 if max_frame_bytes <= SIDE_TRANSPORT_CHUNK_OVERHEAD {
5858 return Err(TelemetryError::BadArg);
5859 }
5860 let payload_budget = max_frame_bytes - SIDE_TRANSPORT_CHUNK_OVERHEAD;
5861 let mut st = self.state.lock();
5862 let side_state = st
5863 .side_transport
5864 .get_mut(&side)
5865 .ok_or(TelemetryError::BadArg)?;
5866 let transfer_id = side_state.next_chunk_id.wrapping_add(1).max(1);
5867 side_state.next_chunk_id = transfer_id;
5868 drop(st);
5869
5870 let total = frame.len().div_ceil(payload_budget);
5871 let total_u16 =
5872 u16::try_from(total).map_err(|_| TelemetryError::PacketTooLarge("too many chunks"))?;
5873 let mut frames = Vec::with_capacity(total);
5874 for (idx, chunk) in frame.chunks(payload_budget).enumerate() {
5875 let mut body = Vec::with_capacity(8 + chunk.len());
5876 body.extend_from_slice(&transfer_id.to_le_bytes());
5877 body.extend_from_slice(&(idx as u16).to_le_bytes());
5878 body.extend_from_slice(&total_u16.to_le_bytes());
5879 body.extend_from_slice(chunk);
5880 frames.push(Self::wrap_side_transport_frame(
5881 SIDE_TRANSPORT_KIND_CHUNK,
5882 &body,
5883 ));
5884 }
5885 Ok(frames)
5886 }
5887
5888 fn encode_side_transport_frames(
5889 &self,
5890 side: RouterSideId,
5891 opts: RouterSideOptions,
5892 raw: Arc<[u8]>,
5893 ) -> TelemetryResult<Vec<Arc<[u8]>>> {
5894 if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
5895 return Ok(vec![raw]);
5896 }
5897
5898 let raw_len = raw.len();
5899 let mut compact_payload_len = None;
5900 let mut used_compact = false;
5901 let mut used_timestamp_delta = false;
5902 let mut omitted_timestamp = false;
5903 let wrapped = if opts.header_template_enabled {
5904 let (template, ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
5905 Self::extract_side_header_template(raw.as_ref())?;
5906 let (template_id, use_compact, previous_timestamp) = {
5907 let mut st = self.state.lock();
5908 let side_state = st
5909 .side_transport
5910 .get_mut(&side)
5911 .ok_or(TelemetryError::BadArg)?;
5912 if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
5913 let previous = side_state.tx_last_timestamps.get(&id).copied();
5914 (id, true, previous)
5915 } else {
5916 let next = side_state.next_template_id.wrapping_add(1).max(1);
5917 side_state.next_template_id = next;
5918 let evicted = side_state.insert_tx_template(
5919 template.clone(),
5920 next,
5921 opts.max_side_transport_templates,
5922 );
5923 if evicted {
5924 st.side_runtime_stats
5925 .entry(side)
5926 .or_default()
5927 .note_side_transport_template_eviction();
5928 }
5929 if let Some(side_state) = st.side_transport.get_mut(&side) {
5930 side_state.tx_last_timestamps.insert(next, timestamp);
5931 }
5932 (next, false, None)
5933 }
5934 };
5935 if use_compact {
5936 used_compact = true;
5937 compact_payload_len = Some(payload.len());
5938 let timestamp_field = if let Some(previous) = previous_timestamp {
5939 let delta = timestamp.saturating_sub(previous);
5940 let omit_timestamp = opts.omit_unchanged_compact_timestamps
5941 || opts.compact_timestamp_omission_types.contains(ty);
5942 if omit_timestamp && timestamp == previous {
5943 omitted_timestamp = true;
5944 None
5945 } else if timestamp >= previous
5946 && Self::uleb128_len_local(delta) < Self::uleb128_len_local(timestamp)
5947 {
5948 used_timestamp_delta = true;
5949 Some(delta)
5950 } else {
5951 Some(timestamp)
5952 }
5953 } else {
5954 Some(timestamp)
5955 };
5956 let mut body = Vec::with_capacity(payload.len() + 32);
5957 body.push(flags);
5958 Self::write_uleb128_local(u64::from(template_id), &mut body);
5959 Self::write_uleb128_local(data_size, &mut body);
5960 if let Some(timestamp_field) = timestamp_field {
5961 Self::write_uleb128_local(timestamp_field, &mut body);
5962 }
5963 if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
5964 Self::write_uleb128_local(u64::from(nonce), &mut body);
5965 }
5966 if let Some((seq, ack)) = reliable_seq_ack {
5967 Self::write_uleb128_local(u64::from(seq), &mut body);
5968 Self::write_uleb128_local(u64::from(ack), &mut body);
5969 }
5970 body.extend_from_slice(payload);
5971 {
5972 let mut st = self.state.lock();
5973 if let Some(side_state) = st.side_transport.get_mut(&side) {
5974 side_state.tx_last_timestamps.insert(template_id, timestamp);
5975 }
5976 }
5977 let kind = if omitted_timestamp {
5978 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
5979 } else if used_timestamp_delta {
5980 SIDE_TRANSPORT_KIND_COMPACT_DELTA
5981 } else {
5982 SIDE_TRANSPORT_KIND_COMPACT
5983 };
5984 Self::wrap_side_transport_frame(kind, &body)
5985 } else {
5986 let mut body = Vec::with_capacity(raw.len() + 4);
5987 Self::write_uleb128_local(u64::from(template_id), &mut body);
5988 body.extend_from_slice(raw.as_ref());
5989 Self::wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
5990 }
5991 } else {
5992 Self::wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
5993 };
5994
5995 let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
5996 self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
5997 } else {
5998 Ok(vec![wrapped])
5999 }?;
6000 let wire_len = frames.iter().map(|frame| frame.len()).sum::<usize>();
6001 let mut st = self.state.lock();
6002 let stats = st.side_runtime_stats.entry(side).or_default();
6003 if used_compact {
6004 let overhead = compact_payload_len
6005 .map(|payload_len| wire_len.saturating_sub(payload_len))
6006 .unwrap_or(wire_len);
6007 stats.note_side_transport_compact(
6008 raw_len,
6009 wire_len,
6010 overhead,
6011 used_timestamp_delta,
6012 omitted_timestamp,
6013 );
6014 if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
6015 {
6016 stats.note_side_transport_compact_target_miss();
6017 }
6018 } else {
6019 stats.note_side_transport_full(raw_len, wire_len);
6020 }
6021 if frames.len() > 1 {
6022 stats.note_side_transport_chunks(frames.len());
6023 }
6024 Ok(frames)
6025 }
6026
6027 fn decode_side_transport_frame(
6028 &self,
6029 side: RouterSideId,
6030 bytes: &[u8],
6031 ) -> TelemetryResult<Option<Arc<[u8]>>> {
6032 let Some((kind, body)) = Self::parse_side_transport_wrapper(bytes)? else {
6033 return Ok(Some(Arc::from(bytes)));
6034 };
6035 match kind {
6036 SIDE_TRANSPORT_KIND_FULL => {
6037 let mut off = 0usize;
6038 let template_id = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6039 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
6040 let raw = Arc::<[u8]>::from(&body[off..]);
6041 if let Ok((template, _, _, _, timestamp, _, _, _)) =
6042 Self::extract_side_header_template(raw.as_ref())
6043 {
6044 let mut st = self.state.lock();
6045 let max_templates = st
6046 .sides
6047 .get(side)
6048 .and_then(|side| side.as_ref())
6049 .map(|side| side.opts.max_side_transport_templates)
6050 .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
6051 let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
6052 let evicted =
6053 side_state.insert_rx_template(template_id, template, max_templates);
6054 side_state.rx_last_timestamps.insert(template_id, timestamp);
6055 evicted
6056 });
6057 if evicted {
6058 st.side_runtime_stats
6059 .entry(side)
6060 .or_default()
6061 .note_side_transport_template_eviction();
6062 }
6063 }
6064 Ok(Some(raw))
6065 }
6066 SIDE_TRANSPORT_KIND_COMPACT
6067 | SIDE_TRANSPORT_KIND_COMPACT_DELTA
6068 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
6069 if body.is_empty() {
6070 return Err(TelemetryError::Unpack("short side compact frame"));
6071 }
6072 let mut off = 1usize;
6073 let template_id = u32::try_from(Self::read_uleb128_local(body, &mut off)?)
6074 .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
6075 let mut compact_body = Vec::with_capacity(1 + body.len().saturating_sub(off));
6076 compact_body.push(body[0]);
6077 compact_body.extend_from_slice(&body[off..]);
6078 let (template, timestamp_base) = {
6079 let st = self.state.lock();
6080 let state = st.side_transport.get(&side);
6081 let template = state
6082 .and_then(|state| state.rx_templates_by_id.get(&template_id))
6083 .cloned();
6084 let timestamp_base = if matches!(
6085 kind,
6086 SIDE_TRANSPORT_KIND_COMPACT_DELTA
6087 | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
6088 ) {
6089 state
6090 .and_then(|state| state.rx_last_timestamps.get(&template_id))
6091 .copied()
6092 } else {
6093 None
6094 };
6095 (template, timestamp_base)
6096 };
6097 let template =
6098 template.ok_or(TelemetryError::Unpack("unknown side compact template"))?;
6099 let timestamp_mode = match kind {
6100 SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
6101 SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
6102 _ => SideCompactTimestampMode::Absolute,
6103 };
6104 let (frame, timestamp) = Self::reconstruct_side_compact_frame(
6105 &template,
6106 &compact_body,
6107 timestamp_mode,
6108 timestamp_base,
6109 )?;
6110 let mut st = self.state.lock();
6111 if let Some(side_state) = st.side_transport.get_mut(&side) {
6112 side_state.rx_last_timestamps.insert(template_id, timestamp);
6113 }
6114 Ok(Some(frame))
6115 }
6116 SIDE_TRANSPORT_KIND_CHUNK => {
6117 if body.len() < 8 {
6118 return Err(TelemetryError::Unpack("short side chunk frame"));
6119 }
6120 let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
6121 let index = u16::from_le_bytes([body[4], body[5]]);
6122 let total = u16::from_le_bytes([body[6], body[7]]);
6123 let payload = Arc::<[u8]>::from(&body[8..]);
6124 let assembled = {
6125 let mut st = self.state.lock();
6126 let side_state = st
6127 .side_transport
6128 .get_mut(&side)
6129 .ok_or(TelemetryError::BadArg)?;
6130 let entry = side_state.rx_chunks.entry(transfer_id).or_default();
6131 if entry.total == 0 {
6132 entry.total = total;
6133 } else if entry.total != total {
6134 side_state.rx_chunks.remove(&transfer_id);
6135 return Err(TelemetryError::Unpack("side chunk total mismatch"));
6136 }
6137 entry.received.entry(index).or_insert(payload);
6138 if entry.received.len() == usize::from(total) {
6139 let entry = side_state
6140 .rx_chunks
6141 .remove(&transfer_id)
6142 .ok_or(TelemetryError::Unpack("side chunk missing"))?;
6143 let mut out = Vec::new();
6144 for idx in 0..entry.total {
6145 let chunk = entry
6146 .received
6147 .get(&idx)
6148 .ok_or(TelemetryError::Unpack("side chunk gap"))?;
6149 out.extend_from_slice(chunk);
6150 }
6151 Some(Arc::<[u8]>::from(out))
6152 } else {
6153 None
6154 }
6155 };
6156 match assembled {
6157 Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
6158 None => Ok(None),
6159 }
6160 }
6161 _ => Err(TelemetryError::Unpack("unknown side transport frame")),
6162 }
6163 }
6164
6165 fn call_side_tx_handler(
6166 &self,
6167 side: RouterSideId,
6168 handler: &RouterTxHandlerFn,
6169 data: &RouterItem,
6170 relayed: bool,
6171 ) -> TelemetryResult<()> {
6172 let opts = {
6173 let st = self.state.lock();
6174 Self::side_ref(&st, side)?.opts
6175 };
6176 let Some(_side_tx_guard) = self.try_enter_side_tx() else {
6177 return Err(TelemetryError::Io("side tx busy"));
6178 };
6179 let started_ms = self.clock.now_ms();
6180 let ty = match data {
6181 RouterItem::Packet(pkt) => pkt.data_type(),
6182 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
6183 };
6184 let result = match (handler, data) {
6185 (RouterTxHandlerFn::Packed(f), RouterItem::Packed(bytes)) => {
6186 #[cfg(feature = "cryptography")]
6187 let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
6188 #[cfg(not(feature = "cryptography"))]
6189 let send_bytes = bytes.clone();
6190 let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
6191 let mut attempts_total = 0usize;
6192 let mut sent_bytes = 0usize;
6193 for frame in frames {
6194 match self
6195 .retry_with_attempts(runtime_max_handler_retries(), || f(frame.as_ref()))
6196 {
6197 Ok((_, attempts)) => {
6198 attempts_total = attempts_total.saturating_add(attempts);
6199 sent_bytes = sent_bytes.saturating_add(frame.len());
6200 }
6201 Err((err, attempts)) => {
6202 self.note_side_tx_failure(
6203 side,
6204 ty,
6205 attempts_total.saturating_add(attempts),
6206 );
6207 return Err(err);
6208 }
6209 }
6210 }
6211 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
6212 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6213 return Ok(());
6214 }
6215 (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
6216 self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
6217 }
6218 (RouterTxHandlerFn::Packed(f), RouterItem::Packet(pkt)) => {
6219 let owned = self.pack_packet_for_router(pkt, None)?;
6220 let frames = self.encode_side_transport_frames(side, opts, owned)?;
6221 let mut attempts_total = 0usize;
6222 let mut sent_bytes = 0usize;
6223 for frame in frames {
6224 match self
6225 .retry_with_attempts(runtime_max_handler_retries(), || f(frame.as_ref()))
6226 {
6227 Ok((_, attempts)) => {
6228 attempts_total = attempts_total.saturating_add(attempts);
6229 sent_bytes = sent_bytes.saturating_add(frame.len());
6230 }
6231 Err((err, attempts)) => {
6232 self.note_side_tx_failure(
6233 side,
6234 ty,
6235 attempts_total.saturating_add(attempts),
6236 );
6237 return Err(err);
6238 }
6239 }
6240 }
6241 self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
6242 self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
6243 return Ok(());
6244 }
6245 (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
6246 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
6247 self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
6248 }
6249 };
6250 match result {
6251 Ok((_, attempts)) => {
6252 if let Ok(bytes) = Self::router_item_wire_len(data) {
6253 self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
6254 self.note_side_tx_success(side, ty, bytes, relayed, attempts);
6255 }
6256 Ok(())
6257 }
6258 Err((err, attempts)) => {
6259 self.note_side_tx_failure(side, ty, attempts);
6260 Err(err)
6261 }
6262 }
6263 }
6264
6265 fn adjust_reliable_for_side(
6266 &self,
6267 opts: RouterSideOptions,
6268 data: RouterItem,
6269 preserve_end_to_end_ack: bool,
6270 ) -> TelemetryResult<Option<RouterItem>> {
6271 if opts.reliable_enabled {
6272 return Ok(Some(data));
6273 }
6274
6275 match data {
6276 RouterItem::Packed(bytes) => {
6277 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
6278 if is_reliable_type(frame.envelope.ty)
6279 && let Some(hdr) = frame.reliable
6280 {
6281 if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
6282 return Ok(None);
6283 }
6284 if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
6285 let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
6286 bytes.as_ref(),
6287 wire_format::RELIABLE_FLAG_UNSEQUENCED,
6288 hdr.seq,
6289 0,
6290 )?
6291 else {
6292 return Ok(Some(RouterItem::Packed(bytes)));
6293 };
6294 return Ok(Some(RouterItem::Packed(rewritten)));
6295 }
6296 }
6297 Ok(Some(RouterItem::Packed(bytes)))
6298 }
6299 RouterItem::Packet(pkt) => {
6300 if matches!(
6301 pkt.data_type(),
6302 DataType::ReliableAck
6303 | DataType::ReliablePartialAck
6304 | DataType::ReliablePacketRequest
6305 ) {
6306 if preserve_end_to_end_ack
6307 && pkt.data_type() == DataType::ReliableAck
6308 && Self::is_end_to_end_ack_sender(pkt.sender())
6309 {
6310 return Ok(Some(RouterItem::Packet(pkt)));
6311 }
6312 return Ok(None);
6313 }
6314 Ok(Some(RouterItem::Packet(pkt)))
6315 }
6316 }
6317 }
6318
6319 fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
6320 let now = self.clock.now_ms();
6321 let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
6322
6323 {
6324 let mut st = self.state.lock();
6325 if st.reliable_tx.is_empty() {
6326 return Ok(());
6327 }
6328
6329 for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
6330 let Some(ty) = DataType::try_from_u32(*ty_u32) else {
6331 continue;
6332 };
6333 let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
6334 for seq in sent_order {
6335 let Some(sent) = tx_state.sent.get_mut(&seq) else {
6336 continue;
6337 };
6338 if sent.queued
6339 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
6340 {
6341 continue;
6342 }
6343 if sent.partial_acked {
6344 continue;
6345 }
6346 if sent.retries >= runtime_reliable_max_retries() {
6347 tx_state.sent.remove(&seq);
6348 tx_state.sent_order.retain(|existing| *existing != seq);
6349 continue;
6350 }
6351 sent.retries += 1;
6352 requeue.push((*side, ty, seq));
6353 }
6354 }
6355 }
6356
6357 for (side, ty, seq) in requeue {
6358 self.queue_reliable_retransmit(side, ty, seq, true)?;
6359 }
6360
6361 Ok(())
6362 }
6363
6364 fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
6365 let now = self.clock.now_ms();
6366 let mut requeue = Vec::new();
6367
6368 {
6369 let mut st = self.state.lock();
6370 #[cfg(feature = "discovery")]
6371 {
6372 if Self::prune_discovery_routes_locked(&mut st, now) {
6373 Self::note_discovery_topology_change_locked(&mut st, now);
6374 }
6375 self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
6376 }
6377 let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
6378 for packet_id in packet_ids {
6379 let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
6380 continue;
6381 };
6382 if sent.queued
6383 || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
6384 {
6385 continue;
6386 }
6387 if sent.retries >= runtime_reliable_max_retries() {
6388 st.end_to_end_reliable_tx.remove(&packet_id);
6389 continue;
6390 }
6391 sent.retries += 1;
6392 requeue.push(packet_id);
6393 }
6394 }
6395
6396 for packet_id in requeue {
6397 self.queue_end_to_end_reliable_retransmit(packet_id)?;
6398 }
6399
6400 Ok(())
6401 }
6402
6403 #[cfg(feature = "timesync")]
6404 #[inline]
6405 fn monotonic_now_ns(&self) -> u64 {
6406 self.clock.now_ns()
6407 }
6408
6409 #[cfg(feature = "timesync")]
6410 #[inline]
6411 fn monotonic_now_ms(&self) -> u64 {
6412 self.clock.now_ms()
6413 }
6414
6415 #[cfg(feature = "timesync")]
6416 fn refresh_timesync_state(&self, now_mono_ms: u64) {
6417 let now_mono_ns = self.monotonic_now_ns();
6418 let mut st = self.timesync.lock();
6419 st.clock.prune_expired(now_mono_ms);
6420 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
6421 st.remote_sources
6422 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
6423 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
6424 let leader = if let Some(tracker) = st.tracker.as_mut() {
6425 let _ = tracker.refresh(now_mono_ms);
6426 tracker.leader(now_mono_ms, has_usable_time)
6427 } else {
6428 None
6429 };
6430 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
6431 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
6432 let target_ms = st
6433 .remote_sources
6434 .get(remote.sender.as_str())
6435 .map(|src| src.sample_unix_ms);
6436 if let Some(target_ms) = target_ms {
6437 st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
6438 }
6439 }
6440 }
6441
6442 #[cfg(feature = "timesync")]
6443 pub fn update_network_time_source(
6445 &self,
6446 source: &str,
6447 priority: u64,
6448 time: PartialNetworkTime,
6449 ttl_ms: Option<u64>,
6450 ) {
6451 let now_ms = self.monotonic_now_ms();
6452 let now_ns = self.monotonic_now_ns();
6453 let mut st = self.timesync.lock();
6454 st.clock
6455 .update_source(source, priority, time, now_ms, now_ns, ttl_ms);
6456 if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
6457 st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
6458 }
6459 }
6460
6461 #[cfg(feature = "timesync")]
6462 fn set_network_time_source_impl(
6463 &self,
6464 source: &str,
6465 priority: u64,
6466 time: PartialNetworkTime,
6467 ttl_ms: Option<u64>,
6468 ) {
6469 let observed_mono_ms = self.monotonic_now_ms();
6470 let observed_mono_ns = self.monotonic_now_ns();
6471 let mut st = self.timesync.lock();
6472 let commit_mono_ms = self.monotonic_now_ms();
6473 let commit_mono_ns = self.monotonic_now_ns();
6474 let adjusted = if let Some(base) = time.to_network_time() {
6475 let elapsed_ns = commit_mono_ns.saturating_sub(observed_mono_ns);
6476 advance_network_time(base, elapsed_ns)
6477 .map(PartialNetworkTime::from)
6478 .unwrap_or(time)
6479 } else {
6480 time
6481 };
6482 let adjusted_mono_ms =
6483 observed_mono_ms.saturating_add(commit_mono_ms.saturating_sub(observed_mono_ms));
6484 st.clock.update_source(
6485 source,
6486 priority,
6487 adjusted,
6488 commit_mono_ms.max(adjusted_mono_ms),
6489 commit_mono_ns,
6490 ttl_ms,
6491 );
6492 if let Some(unix_ms) = adjusted.to_network_time().and_then(|t| t.as_unix_ms()) {
6493 st.disciplined_clock.steer_unix_ms(commit_mono_ns, unix_ms);
6494 }
6495 }
6496
6497 #[cfg(feature = "timesync")]
6498 fn local_network_time_priority(&self) -> u64 {
6499 let st = self.timesync.lock();
6500 st.cfg.map(|cfg| cfg.priority).unwrap_or(0)
6501 }
6502
6503 #[cfg(feature = "timesync")]
6504 pub fn set_local_network_time(&self, time: PartialNetworkTime) {
6506 let priority = self.local_network_time_priority();
6507 if time.is_complete_date() && time.is_complete_time() {
6508 self.set_network_time_source_impl(LOCAL_TIMESYNC_FULL_SOURCE_ID, priority, time, None);
6509 let mut st = self.timesync.lock();
6510 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
6511 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
6512 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
6513 return;
6514 }
6515
6516 {
6517 let mut st = self.timesync.lock();
6518 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
6519 }
6520
6521 if time.year.is_some() || time.month.is_some() || time.day.is_some() {
6522 self.set_network_time_source_impl(
6523 LOCAL_TIMESYNC_DATE_SOURCE_ID,
6524 priority,
6525 PartialNetworkTime {
6526 year: time.year,
6527 month: time.month,
6528 day: time.day,
6529 ..Default::default()
6530 },
6531 None,
6532 );
6533 }
6534
6535 if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
6536 self.set_network_time_source_impl(
6537 LOCAL_TIMESYNC_TOD_SOURCE_ID,
6538 priority,
6539 PartialNetworkTime {
6540 hour: time.hour,
6541 minute: time.minute,
6542 second: time.second,
6543 nanosecond: time.nanosecond,
6544 ..Default::default()
6545 },
6546 None,
6547 );
6548 }
6549
6550 if time.nanosecond.is_some() {
6551 self.set_network_time_source_impl(
6552 LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
6553 priority,
6554 PartialNetworkTime {
6555 nanosecond: time.nanosecond,
6556 ..Default::default()
6557 },
6558 None,
6559 );
6560 }
6561 }
6562
6563 #[cfg(feature = "timesync")]
6564 pub fn clear_local_network_time(&self) {
6566 let mut st = self.timesync.lock();
6567 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
6568 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
6569 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
6570 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
6571 }
6572
6573 #[cfg(feature = "timesync")]
6574 pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
6576 self.set_local_network_time(PartialNetworkTime {
6577 year: Some(year),
6578 month: Some(month),
6579 day: Some(day),
6580 ..Default::default()
6581 });
6582 }
6583
6584 #[cfg(feature = "timesync")]
6585 pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
6587 self.set_local_network_time(PartialNetworkTime {
6588 hour: Some(hour),
6589 minute: Some(minute),
6590 ..Default::default()
6591 });
6592 }
6593
6594 #[cfg(feature = "timesync")]
6595 pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
6597 self.set_local_network_time(PartialNetworkTime {
6598 hour: Some(hour),
6599 minute: Some(minute),
6600 second: Some(second),
6601 ..Default::default()
6602 });
6603 }
6604
6605 #[cfg(feature = "timesync")]
6606 pub fn set_local_network_time_hms_millis(
6608 &self,
6609 hour: u8,
6610 minute: u8,
6611 second: u8,
6612 millisecond: u16,
6613 ) {
6614 self.set_local_network_time(PartialNetworkTime {
6615 hour: Some(hour),
6616 minute: Some(minute),
6617 second: Some(second),
6618 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
6619 ..Default::default()
6620 });
6621 }
6622
6623 #[cfg(feature = "timesync")]
6624 pub fn set_local_network_time_hms_nanos(
6626 &self,
6627 hour: u8,
6628 minute: u8,
6629 second: u8,
6630 nanosecond: u32,
6631 ) {
6632 self.set_local_network_time(PartialNetworkTime {
6633 hour: Some(hour),
6634 minute: Some(minute),
6635 second: Some(second),
6636 nanosecond: Some(nanosecond),
6637 ..Default::default()
6638 });
6639 }
6640
6641 #[cfg(feature = "timesync")]
6642 pub fn set_local_network_datetime(
6644 &self,
6645 year: i32,
6646 month: u8,
6647 day: u8,
6648 hour: u8,
6649 minute: u8,
6650 second: u8,
6651 ) {
6652 self.set_local_network_time(PartialNetworkTime {
6653 year: Some(year),
6654 month: Some(month),
6655 day: Some(day),
6656 hour: Some(hour),
6657 minute: Some(minute),
6658 second: Some(second),
6659 ..Default::default()
6660 });
6661 }
6662
6663 #[cfg(feature = "timesync")]
6664 #[allow(clippy::too_many_arguments)]
6665 pub fn set_local_network_datetime_millis(
6667 &self,
6668 year: i32,
6669 month: u8,
6670 day: u8,
6671 hour: u8,
6672 minute: u8,
6673 second: u8,
6674 millisecond: u16,
6675 ) {
6676 self.set_local_network_time(PartialNetworkTime {
6677 year: Some(year),
6678 month: Some(month),
6679 day: Some(day),
6680 hour: Some(hour),
6681 minute: Some(minute),
6682 second: Some(second),
6683 nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
6684 });
6685 }
6686
6687 #[cfg(feature = "timesync")]
6688 #[allow(clippy::too_many_arguments)]
6689 pub fn set_local_network_datetime_nanos(
6691 &self,
6692 year: i32,
6693 month: u8,
6694 day: u8,
6695 hour: u8,
6696 minute: u8,
6697 second: u8,
6698 nanosecond: u32,
6699 ) {
6700 self.set_local_network_time(PartialNetworkTime {
6701 year: Some(year),
6702 month: Some(month),
6703 day: Some(day),
6704 hour: Some(hour),
6705 minute: Some(minute),
6706 second: Some(second),
6707 nanosecond: Some(nanosecond),
6708 });
6709 }
6710
6711 #[cfg(feature = "timesync")]
6712 pub fn clear_network_time_source(&self, source: &str) {
6714 let mut st = self.timesync.lock();
6715 st.clock.remove_source(source);
6716 }
6717
6718 #[cfg(feature = "timesync")]
6719 pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
6721 let mut st = self.timesync.lock();
6722 let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
6723 st.cfg = cfg;
6724 st.tracker = cfg.map(TimeSyncTracker::new);
6725 st.disciplined_clock = SlewedNetworkClock::new(
6726 cfg.map(|c| c.max_slew_ppm)
6727 .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
6728 );
6729 st.remote_sources.clear();
6730 st.next_seq = 1;
6731 st.next_announce_mono_ms = 0;
6732 st.next_request_mono_ms = 0;
6733 st.pending_request = None;
6734 st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
6735 for source in stale_remote_sources {
6736 st.clock.remove_source(&source);
6737 }
6738 st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
6739 st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
6740 st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
6741 st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
6742 }
6743
6744 #[cfg(feature = "timesync")]
6745 pub fn network_time(&self) -> Option<NetworkTimeReading> {
6747 let now_ms = self.monotonic_now_ms();
6748 let now_ns = self.monotonic_now_ns();
6749 self.refresh_timesync_state(now_ms);
6750 let st = self.timesync.lock();
6751 if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
6752 return Some(NetworkTimeReading {
6753 time: PartialNetworkTime::from_unix_ms(unix_ms),
6754 unix_time_ms: Some(unix_ms),
6755 });
6756 }
6757 st.clock.current_time(now_ns)
6758 }
6759
6760 #[cfg(feature = "timesync")]
6761 pub fn network_time_ms(&self) -> Option<u64> {
6763 self.network_time().and_then(|t| t.unix_time_ms)
6764 }
6765
6766 #[cfg(feature = "timesync")]
6767 fn packet_timestamp_ms(&self) -> u64 {
6768 self.network_time_ms()
6769 .unwrap_or_else(|| self.monotonic_now_ms())
6770 }
6771
6772 #[cfg(not(feature = "timesync"))]
6773 fn packet_timestamp_ms(&self) -> u64 {
6774 self.clock.now_ms()
6775 }
6776
6777 #[cfg(feature = "timesync")]
6778 fn queue_internal_timesync_request(
6779 &self,
6780 seq: u64,
6781 t1_mono_ms: u64,
6782 called_from_queue: bool,
6783 ) -> TelemetryResult<()> {
6784 let pkt_ts = self.packet_timestamp_ms();
6785 if called_from_queue {
6786 self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
6787 } else {
6788 self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
6789 }
6790 }
6791
6792 #[cfg(feature = "timesync")]
6793 fn queue_internal_timesync_response(
6794 &self,
6795 seq: u64,
6796 t1_mono_ms: u64,
6797 t2_network_ms: u64,
6798 t3_network_ms: u64,
6799 dst: Option<RouterSideId>,
6800 called_from_queue: bool,
6801 ) -> TelemetryResult<()> {
6802 let pkt_ts = self.packet_timestamp_ms();
6803 let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
6804 let sender = self.sender_arc();
6805 let pkt = Packet::new(
6806 DataType::TimeSyncResponse,
6807 &[DataEndpoint::TimeSync],
6808 sender.as_ref(),
6809 pkt_ts,
6810 payload,
6811 )?;
6812 match dst {
6813 Some(dst) => self.emit_internal_tx(
6814 RouterTxItem::ToSide {
6815 src: None,
6816 dst,
6817 data: RouterItem::Packet(pkt),
6818 },
6819 true,
6820 called_from_queue,
6821 ),
6822 None => self.emit_internal_tx(
6823 RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
6824 true,
6825 called_from_queue,
6826 ),
6827 }
6828 }
6829
6830 #[cfg(feature = "timesync")]
6831 pub fn poll_timesync(&self) -> TelemetryResult<bool> {
6833 let now_ms = self.monotonic_now_ms();
6834 let now_ns = self.monotonic_now_ns();
6835 let mut queued_any = false;
6836 let mut announce_priority = None;
6837 let mut request = None;
6838
6839 {
6840 let mut st = self.timesync.lock();
6841 st.clock.prune_expired(now_ms);
6842 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
6843 st.remote_sources
6844 .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
6845 let Some(cfg) = st.cfg else {
6846 return Ok(false);
6847 };
6848
6849 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
6850 let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
6851 let _ = tracker.refresh(now_ms);
6852 (
6853 tracker.leader(now_ms, has_usable_time),
6854 tracker.local_announce_priority(now_ms, has_usable_time),
6855 )
6856 } else {
6857 (None, None)
6858 };
6859 Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
6860
6861 if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
6862 let target_ms = st
6863 .remote_sources
6864 .get(&remote.sender)
6865 .map(|src| src.sample_unix_ms);
6866 if let Some(target_ms) = target_ms {
6867 st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
6868 }
6869 }
6870
6871 if let Some(priority) = announce_prio
6872 && now_ms >= st.next_announce_mono_ms
6873 {
6874 announce_priority = Some(priority);
6875 st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
6876 }
6877
6878 if let Some(TimeSyncLeader::Remote(remote)) = leader
6879 && now_ms >= st.next_request_mono_ms
6880 && st.pending_request.is_none()
6881 {
6882 let seq = st.next_seq;
6883 let next = st.next_seq.wrapping_add(1);
6884 st.next_seq = if next == 0 { 1 } else { next };
6885 st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
6886 st.pending_request = Some(PendingTimeSyncRequest {
6887 seq,
6888 t1_mono_ms: now_ms,
6889 source: remote.sender,
6890 });
6891 request = Some((seq, now_ms));
6892 }
6893 }
6894
6895 if let Some(priority) = announce_priority {
6896 let time_ms = self.packet_timestamp_ms();
6897 self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
6898 queued_any = true;
6899 }
6900 if let Some((seq, t1_mono_ms)) = request {
6901 self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
6902 queued_any = true;
6903 }
6904
6905 Ok(queued_any)
6906 }
6907
6908 #[cfg(feature = "timesync")]
6909 fn handle_internal_timesync_packet(
6910 &self,
6911 pkt: &Packet,
6912 src: Option<RouterSideId>,
6913 called_from_queue: bool,
6914 ) -> TelemetryResult<bool> {
6915 let Some(cfg) = self.cfg.timesync_config() else {
6916 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
6917 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
6918 }
6919 return Ok(true);
6920 };
6921
6922 let now_mono_ms = self.monotonic_now_ms();
6923 let now_mono_ns = self.monotonic_now_ns();
6924 let mut response = None;
6925 let mut poll_after = false;
6926
6927 {
6928 let mut st = self.timesync.lock();
6929 st.clock.prune_expired(now_mono_ms);
6930 let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
6931 st.remote_sources
6932 .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
6933 let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
6934 if st.tracker.is_none() {
6935 return Ok(true);
6936 }
6937
6938 match pkt.data_type() {
6939 DataType::TimeSyncAnnounce => {
6940 let ann = decode_timesync_announce(pkt)?;
6941 let should_steer = {
6942 let tracker = st.tracker.as_mut().expect("tracker checked above");
6943 let _ = tracker.handle_announce(pkt, now_mono_ms)?;
6944 matches!(
6945 tracker.leader(now_mono_ms, has_usable_time),
6946 Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender()
6947 )
6948 };
6949 st.remote_sources.insert(
6950 pkt.sender().to_owned(),
6951 RemoteTimeSyncSource {
6952 priority: ann.priority,
6953 last_sample_mono_ms: now_mono_ms,
6954 sample_unix_ms: ann.time_ms,
6955 },
6956 );
6957 st.clock.update_source(
6958 pkt.sender(),
6959 ann.priority,
6960 PartialNetworkTime::from_unix_ms(ann.time_ms),
6961 now_mono_ms,
6962 now_mono_ns,
6963 Some(cfg.source_timeout_ms),
6964 );
6965 if should_steer {
6966 st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
6967 }
6968 poll_after = true;
6969 }
6970 DataType::TimeSyncRequest => {
6971 let should_serve = {
6972 let tracker = st.tracker.as_ref().expect("tracker checked above");
6973 tracker.should_serve(now_mono_ms, has_usable_time)
6974 };
6975 if should_serve {
6976 let req = decode_timesync_request(pkt)?;
6977 let network_now = st
6978 .disciplined_clock
6979 .read_unix_ms(now_mono_ns)
6980 .or_else(|| {
6981 st.clock
6982 .current_time(now_mono_ns)
6983 .and_then(|t| t.unix_time_ms)
6984 })
6985 .unwrap_or(now_mono_ms);
6986 let t2 = network_now;
6987 let t3 = network_now;
6988 response = Some((req.seq, req.t1_ms, t2, t3, src));
6989 }
6990 }
6991 DataType::TimeSyncResponse => {
6992 let resp = decode_timesync_response(pkt)?;
6993 let pending = st.pending_request.clone();
6994 if let Some(pending) = pending
6995 && pending.seq == resp.seq
6996 && pending.source == pkt.sender()
6997 {
6998 let source_priority = {
6999 let tracker = st.tracker.as_ref().expect("tracker checked above");
7000 tracker
7001 .best_active_source(now_mono_ms)
7002 .map(|s| s.priority)
7003 .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
7004 .unwrap_or(cfg.priority)
7005 };
7006 let (estimate_ms, _delay_ms) = compute_network_time_sample(
7007 pending.t1_mono_ms,
7008 resp.t2_ms,
7009 resp.t3_ms,
7010 now_mono_ms,
7011 );
7012 st.remote_sources.insert(
7013 pkt.sender().to_owned(),
7014 RemoteTimeSyncSource {
7015 priority: source_priority,
7016 last_sample_mono_ms: now_mono_ms,
7017 sample_unix_ms: estimate_ms,
7018 },
7019 );
7020 st.clock.update_source(
7021 pkt.sender(),
7022 source_priority,
7023 PartialNetworkTime::from_unix_ms(estimate_ms),
7024 now_mono_ms,
7025 now_mono_ns,
7026 Some(cfg.source_timeout_ms),
7027 );
7028 st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
7029 st.pending_request = None;
7030 }
7031 }
7032 _ => {}
7033 }
7034 }
7035
7036 if let Some((seq, t1, t2, t3, dst)) = response {
7037 self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
7038 }
7039 if poll_after {
7040 let _ = self.poll_timesync()?;
7041 }
7042
7043 if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
7044 self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
7045 }
7046
7047 Ok(true)
7048 }
7049
7050 #[cfg(feature = "std")]
7052 pub fn new(cfg: RouterConfig) -> Self {
7053 Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
7054 }
7055
7056 pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
7058 #[cfg(feature = "timesync")]
7059 let timesync_cfg = cfg.timesync_config();
7060 let memory = cfg.memory_config();
7061 let hostname: Arc<str> = Arc::from(cfg.sender());
7062 let address_mode = cfg.address_mode();
7063 let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
7064 let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
7065 let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
7066 let requested = address_mode.requested_address();
7067 let address = if requested == 0 { fallback } else { requested };
7068 let local_address = AddressBookEntry {
7069 hostname: hostname.clone(),
7070 address,
7071 requested_address: requested,
7072 mode: address_mode,
7073 birth_ms: clock
7074 .now_ms()
7075 .saturating_mul(1_000_000)
7076 .saturating_add(instance_seq),
7077 owner_hash,
7078 last_seen_ms: clock.now_ms(),
7079 };
7080 let mut address_book = BTreeMap::new();
7081 address_book.insert(hostname.to_string(), local_address.clone());
7082 let mut address_by_value = BTreeMap::new();
7083 address_by_value.insert(address, hostname.to_string());
7084 Self {
7085 sender: RouterMutex::new(hostname),
7086 cfg,
7087 state: RouterMutex::new(RouterInner {
7088 memory,
7089 sides: Vec::new(),
7090 route_overrides: BTreeMap::new(),
7091 typed_route_overrides: BTreeMap::new(),
7092 route_weights: BTreeMap::new(),
7093 route_priorities: BTreeMap::new(),
7094 source_route_modes: BTreeMap::new(),
7095 route_selection_cursors: BTreeMap::new(),
7096 adaptive_route_stats: BTreeMap::new(),
7097 side_runtime_stats: BTreeMap::new(),
7098 side_transport: BTreeMap::new(),
7099 managed_variable_types: BTreeSet::new(),
7100 managed_variable_permissions: BTreeMap::new(),
7101 managed_variable_latest: BTreeMap::new(),
7102 network_variable_update_handlers: BTreeMap::new(),
7103 local_address,
7104 address_book,
7105 address_by_value,
7106 p2p_port_handlers: BTreeMap::new(),
7107 p2p_stream_handlers: BTreeMap::new(),
7108 p2p_stream_sessions: BTreeMap::new(),
7109 next_p2p_stream_id: 1,
7110 received_queue: BoundedDeque::new(
7111 memory.max_queue_budget,
7112 memory.starting_queue_size,
7113 memory.queue_grow_step,
7114 ),
7115 transmit_queue: BoundedDeque::new(
7116 memory.max_queue_budget,
7117 memory.starting_queue_size,
7118 memory.queue_grow_step,
7119 ),
7120 recent_rx: BoundedDeque::new(
7121 memory.recent_rx_queue_bytes(),
7122 memory.recent_rx_queue_bytes(),
7123 memory.queue_grow_step,
7124 ),
7125 reliable_tx: BTreeMap::new(),
7126 reliable_rx: BTreeMap::new(),
7127 reliable_return_routes: BTreeMap::new(),
7128 reliable_return_route_order: VecDeque::new(),
7129 end_to_end_reliable_tx: BTreeMap::new(),
7130 end_to_end_reliable_tx_order: VecDeque::new(),
7131 total_handler_failures: 0,
7132 total_handler_retries: 0,
7133 #[cfg(feature = "discovery")]
7134 discovery_routes: BTreeMap::new(),
7135 #[cfg(feature = "discovery")]
7136 discovery_cadence: DiscoveryCadenceState::default(),
7137 #[cfg(feature = "discovery")]
7138 discovery_side_throttle: BTreeMap::new(),
7139 #[cfg(all(feature = "discovery", feature = "timesync"))]
7140 timesync_side_throttle: BTreeMap::new(),
7141 }),
7142 isr_rx_queue: IsrRxQueue::new(
7143 memory.max_queue_budget,
7144 memory.starting_queue_size,
7145 memory.queue_grow_step,
7146 ),
7147 side_tx_gate: ReentryGate::new(),
7148 clock,
7149 #[cfg(feature = "timesync")]
7150 timesync: RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)),
7151 }
7152 }
7153
7154 #[inline]
7155 fn sender_arc(&self) -> Arc<str> {
7156 self.sender.lock().clone()
7157 }
7158
7159 #[inline]
7160 pub fn sender(&self) -> Arc<str> {
7161 self.sender_arc()
7162 }
7163
7164 pub fn current_address(&self) -> NodeAddress {
7165 self.state.lock().local_address.address
7166 }
7167
7168 pub fn hostname(&self) -> Arc<str> {
7169 self.sender_arc()
7170 }
7171
7172 pub fn address_book(&self) -> Vec<AddressBookEntry> {
7173 self.state.lock().address_book.values().cloned().collect()
7174 }
7175
7176 pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
7177 self.state.lock().address_book.get(hostname).cloned()
7178 }
7179
7180 pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
7181 let st = self.state.lock();
7182 st.address_by_value
7183 .get(&address)
7184 .and_then(|hostname| st.address_book.get(hostname))
7185 .cloned()
7186 }
7187
7188 pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
7189 where
7190 F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
7191 {
7192 if port == 0 {
7193 return Err(TelemetryError::BadArg);
7194 }
7195 let mut st = self.state.lock();
7196 st.p2p_port_handlers
7197 .entry(port)
7198 .or_default()
7199 .push(P2pPortHandler {
7200 handler: Arc::new(f),
7201 });
7202 Ok(())
7203 }
7204
7205 pub fn clear_p2p_port(&self, port: P2pPort) {
7206 self.state.lock().p2p_port_handlers.remove(&port);
7207 }
7208
7209 pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
7210 where
7211 F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
7212 {
7213 if port == 0 {
7214 return Err(TelemetryError::BadArg);
7215 }
7216 let mut st = self.state.lock();
7217 st.p2p_stream_handlers
7218 .entry(port)
7219 .or_default()
7220 .push(P2pStreamHandler {
7221 handler: Arc::new(f),
7222 });
7223 Ok(())
7224 }
7225
7226 pub fn clear_p2p_stream_port(&self, port: P2pPort) {
7227 self.state.lock().p2p_stream_handlers.remove(&port);
7228 }
7229
7230 fn send_p2p_to_entry(
7231 &self,
7232 dst: AddressBookEntry,
7233 dst_port: P2pPort,
7234 src_port: P2pPort,
7235 payload: &[u8],
7236 ) -> TelemetryResult<()> {
7237 if dst_port == 0 {
7238 return Err(TelemetryError::BadArg);
7239 }
7240 let local = self.state.lock().local_address.clone();
7241 let payload = Self::encode_p2p_payload(
7242 local.hostname.as_ref(),
7243 local.address,
7244 src_port,
7245 dst_port,
7246 payload,
7247 )?;
7248 let pkt = Packet::new(
7249 DataType::P2pMessage,
7250 &[DataEndpoint::Discovery],
7251 local.hostname.as_ref(),
7252 self.packet_timestamp_ms(),
7253 payload,
7254 )?;
7255 let target = Self::sender_hash(dst.hostname.as_ref());
7256 let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
7257 if target == Self::sender_hash(self.sender_arc().as_ref()) {
7258 self.dispatch_p2p_packet(&pkt)?;
7259 return Ok(());
7260 }
7261 self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
7262 }
7263
7264 pub fn send_p2p_to_hostname(
7265 &self,
7266 hostname: &str,
7267 dst_port: P2pPort,
7268 src_port: P2pPort,
7269 payload: &[u8],
7270 ) -> TelemetryResult<()> {
7271 let dst = self
7272 .resolve_hostname(hostname)
7273 .ok_or(TelemetryError::BadArg)?;
7274 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
7275 }
7276
7277 pub fn send_p2p_to_address(
7278 &self,
7279 address: NodeAddress,
7280 dst_port: P2pPort,
7281 src_port: P2pPort,
7282 payload: &[u8],
7283 ) -> TelemetryResult<()> {
7284 let dst = self
7285 .resolve_address(address)
7286 .ok_or(TelemetryError::BadArg)?;
7287 self.send_p2p_to_entry(dst, dst_port, src_port, payload)
7288 }
7289
7290 fn open_p2p_stream_to_entry(
7291 &self,
7292 dst: AddressBookEntry,
7293 dst_port: P2pPort,
7294 src_port: P2pPort,
7295 ) -> TelemetryResult<P2pStreamId> {
7296 if dst_port == 0 || src_port == 0 {
7297 return Err(TelemetryError::BadArg);
7298 }
7299 let stream_id = {
7300 let mut st = self.state.lock();
7301 let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
7302 if stream_id == 0 {
7303 return Err(TelemetryError::Io("p2p stream id exhausted"));
7304 }
7305 st.p2p_stream_sessions.insert(
7306 stream_id,
7307 P2pStreamSession {
7308 peer_hostname: dst.hostname.clone(),
7309 peer_address: dst.address,
7310 local_port: src_port,
7311 peer_port: dst_port,
7312 peer_stream_id: 0,
7313 next_sequence: 1,
7314 connected: false,
7315 },
7316 );
7317 stream_id
7318 };
7319 let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
7320 self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
7321 Ok(stream_id)
7322 }
7323
7324 pub fn open_p2p_stream_to_hostname(
7325 &self,
7326 hostname: &str,
7327 dst_port: P2pPort,
7328 src_port: P2pPort,
7329 ) -> TelemetryResult<P2pStreamId> {
7330 let dst = self
7331 .resolve_hostname(hostname)
7332 .ok_or(TelemetryError::BadArg)?;
7333 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
7334 }
7335
7336 pub fn open_p2p_stream_to_address(
7337 &self,
7338 address: NodeAddress,
7339 dst_port: P2pPort,
7340 src_port: P2pPort,
7341 ) -> TelemetryResult<P2pStreamId> {
7342 let dst = self
7343 .resolve_address(address)
7344 .ok_or(TelemetryError::BadArg)?;
7345 self.open_p2p_stream_to_entry(dst, dst_port, src_port)
7346 }
7347
7348 fn send_p2p_stream_control(
7349 &self,
7350 stream_id: P2pStreamId,
7351 flags: u8,
7352 payload: &[u8],
7353 ) -> TelemetryResult<()> {
7354 let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
7355 let mut st = self.state.lock();
7356 let Some(session) = st.p2p_stream_sessions.get_mut(&stream_id) else {
7357 return Err(TelemetryError::BadArg);
7358 };
7359 let peer_hostname = session.peer_hostname.clone();
7360 let peer_address = session.peer_address;
7361 let peer_port = session.peer_port;
7362 let local_port = session.local_port;
7363 let peer_stream_id = session.peer_stream_id;
7364 let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
7365 let seq = session.next_sequence;
7366 session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
7367 let dst = st
7368 .address_book
7369 .get(peer_hostname.as_ref())
7370 .cloned()
7371 .unwrap_or(AddressBookEntry {
7372 hostname: peer_hostname.clone(),
7373 address: peer_address,
7374 requested_address: peer_address,
7375 mode: AddressAssignmentMode::Dynamic,
7376 birth_ms: self.clock.now_ms(),
7377 owner_hash: Self::sender_hash(peer_hostname.as_ref()),
7378 last_seen_ms: self.clock.now_ms(),
7379 });
7380 (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
7381 };
7382 if refresh_syn {
7383 let syn_payload =
7384 Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
7385 self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
7386 }
7387 let stream_payload =
7388 Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
7389 self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
7390 if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
7391 self.state.lock().p2p_stream_sessions.remove(&stream_id);
7392 }
7393 Ok(())
7394 }
7395
7396 pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
7397 self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
7398 }
7399
7400 pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
7401 self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
7402 }
7403
7404 pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
7405 self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
7406 }
7407
7408 pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
7409 let hostname: Arc<str> = Arc::from(sender.as_ref());
7410 let mut st = self.state.lock();
7411 let mut local = st.local_address.clone();
7412 local.hostname = hostname.clone();
7413 local.owner_hash = Self::sender_hash(hostname.as_ref());
7414 if matches!(local.mode, AddressAssignmentMode::Dynamic) {
7415 local.address = Self::fallback_address_for_hostname(hostname.as_ref());
7416 local.requested_address = 0;
7417 }
7418 let change =
7419 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
7420 drop(st);
7421 let _ = self.notify_address_change(change);
7422 }
7423
7424 pub fn set_address_assignment(
7425 &self,
7426 mode: AddressAssignmentMode,
7427 ) -> TelemetryResult<AddressChange> {
7428 let mut st = self.state.lock();
7429 let mut local = st.local_address.clone();
7430 local.mode = mode;
7431 local.requested_address = mode.requested_address();
7432 local.address = match mode {
7433 AddressAssignmentMode::Dynamic => {
7434 Self::fallback_address_for_hostname(local.hostname.as_ref())
7435 }
7436 AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
7437 if address == 0 {
7438 1
7439 } else {
7440 address
7441 }
7442 }
7443 };
7444 let change =
7445 self.update_local_identity_locked(&mut st, local, AddressChangeReason::Configured);
7446 drop(st);
7447 self.notify_address_change(change.clone())?;
7448 Ok(change)
7449 }
7450
7451 pub fn add_side_packed<F>(&self, name: &'static str, tx: F) -> RouterSideId
7460 where
7461 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
7462 {
7463 self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
7464 }
7465
7466 pub fn add_side_packed_small_packets<F>(
7470 &self,
7471 name: &'static str,
7472 tx: F,
7473 max_frame_bytes: usize,
7474 ) -> RouterSideId
7475 where
7476 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
7477 {
7478 self.add_side_packed_with_options(
7479 name,
7480 tx,
7481 RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
7482 )
7483 }
7484
7485 pub fn add_side_packed_with_options<F>(
7494 &self,
7495 name: &'static str,
7496 tx: F,
7497 opts: RouterSideOptions,
7498 ) -> RouterSideId
7499 where
7500 F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
7501 {
7502 let mut st = self.state.lock();
7503 let id = st.sides.len();
7504 st.sides.push(Some(RouterSide {
7505 name,
7506 tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
7507 opts,
7508 }));
7509 st.side_runtime_stats
7510 .insert(id, SideRuntimeStatsInner::default());
7511 st.side_transport.insert(id, SideTransportState::default());
7512 #[cfg(feature = "discovery")]
7513 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
7514 id
7515 }
7516
7517 pub fn add_side_packet<F>(&self, name: &'static str, tx: F) -> RouterSideId
7522 where
7523 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
7524 {
7525 self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
7526 }
7527
7528 pub fn add_side_packet_with_options<F>(
7534 &self,
7535 name: &'static str,
7536 tx: F,
7537 opts: RouterSideOptions,
7538 ) -> RouterSideId
7539 where
7540 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
7541 {
7542 let mut st = self.state.lock();
7543 let id = st.sides.len();
7544 st.sides.push(Some(RouterSide {
7545 name,
7546 tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
7547 opts,
7548 }));
7549 st.side_runtime_stats
7550 .insert(id, SideRuntimeStatsInner::default());
7551 st.side_transport.insert(id, SideTransportState::default());
7552 #[cfg(feature = "discovery")]
7553 Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
7554 id
7555 }
7556
7557 pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
7562 let now_ms = self.clock.now_ms();
7563 {
7564 let mut st = self.state.lock();
7565 let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
7566 if slot.is_none() {
7567 return Err(TelemetryError::BadArg);
7568 }
7569 *slot = None;
7570 st.route_overrides
7571 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
7572 st.typed_route_overrides
7573 .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
7574 st.route_weights
7575 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
7576 st.route_priorities
7577 .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
7578 st.source_route_modes.remove(&Some(side));
7579 st.route_selection_cursors.remove(&Some(side));
7580 st.adaptive_route_stats.remove(&side);
7581 #[cfg(feature = "discovery")]
7582 st.discovery_side_throttle.remove(&side);
7583 #[cfg(all(feature = "discovery", feature = "timesync"))]
7584 st.timesync_side_throttle.remove(&side);
7585 st.side_runtime_stats.remove(&side);
7586 st.side_transport.remove(&side);
7587 st.reliable_return_routes
7588 .retain(|_, route| route.side != side);
7589 st.transmit_queue.retain(
7590 |queued| {
7591 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
7592 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
7593 },
7594 );
7595 st.received_queue.retain(|queued| queued.src != Some(side));
7596 st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
7597 st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
7598 #[cfg(feature = "discovery")]
7599 {
7600 st.discovery_routes.remove(&side);
7601 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7602 }
7603 }
7604 let mut isr_rx = self.isr_rx_queue.try_lock()?;
7605 isr_rx.retain(|queued| queued.src != Some(side));
7606 Ok(())
7607 }
7608
7609 pub fn set_side_ingress_enabled(
7614 &self,
7615 side: RouterSideId,
7616 enabled: bool,
7617 ) -> TelemetryResult<()> {
7618 let now_ms = self.clock.now_ms();
7619 let mut st = self.state.lock();
7620 let side_ref = st
7621 .sides
7622 .get_mut(side)
7623 .and_then(|side| side.as_mut())
7624 .ok_or(TelemetryError::BadArg)?;
7625 side_ref.opts.ingress_enabled = enabled;
7626 #[cfg(feature = "discovery")]
7627 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7628 Ok(())
7629 }
7630
7631 pub fn set_side_egress_enabled(
7635 &self,
7636 side: RouterSideId,
7637 enabled: bool,
7638 ) -> TelemetryResult<()> {
7639 let now_ms = self.clock.now_ms();
7640 let mut st = self.state.lock();
7641 let side_ref = st
7642 .sides
7643 .get_mut(side)
7644 .and_then(|side| side.as_mut())
7645 .ok_or(TelemetryError::BadArg)?;
7646 side_ref.opts.egress_enabled = enabled;
7647 if !enabled {
7648 st.transmit_queue.retain(
7649 |queued| {
7650 !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
7651 && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
7652 },
7653 );
7654 }
7655 #[cfg(feature = "discovery")]
7656 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7657 Ok(())
7658 }
7659
7660 pub fn set_source_route_mode(
7666 &self,
7667 src: Option<RouterSideId>,
7668 mode: RouteSelectionMode,
7669 ) -> TelemetryResult<()> {
7670 let now_ms = self.clock.now_ms();
7671 let mut st = self.state.lock();
7672 if let Some(src) = src {
7673 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7674 }
7675 if mode == RouteSelectionMode::Fanout {
7676 st.source_route_modes.remove(&src);
7677 } else {
7678 st.source_route_modes.insert(src, mode);
7679 }
7680 st.route_selection_cursors.remove(&src);
7681 #[cfg(feature = "discovery")]
7682 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7683 Ok(())
7684 }
7685
7686 pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
7688 self.set_source_route_mode(src, RouteSelectionMode::Fanout)
7689 }
7690
7691 pub fn set_route_weight(
7696 &self,
7697 src: Option<RouterSideId>,
7698 dst: RouterSideId,
7699 weight: u32,
7700 ) -> TelemetryResult<()> {
7701 let now_ms = self.clock.now_ms();
7702 let mut st = self.state.lock();
7703 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7704 if let Some(src) = src {
7705 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7706 }
7707 st.route_weights.insert((src, dst), weight);
7708 st.route_selection_cursors.remove(&src);
7709 #[cfg(feature = "discovery")]
7710 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7711 Ok(())
7712 }
7713
7714 pub fn clear_route_weight(
7716 &self,
7717 src: Option<RouterSideId>,
7718 dst: RouterSideId,
7719 ) -> TelemetryResult<()> {
7720 let now_ms = self.clock.now_ms();
7721 let mut st = self.state.lock();
7722 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7723 if let Some(src) = src {
7724 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7725 }
7726 st.route_weights.remove(&(src, dst));
7727 st.route_selection_cursors.remove(&src);
7728 #[cfg(feature = "discovery")]
7729 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7730 Ok(())
7731 }
7732
7733 pub fn set_route_priority(
7738 &self,
7739 src: Option<RouterSideId>,
7740 dst: RouterSideId,
7741 priority: u32,
7742 ) -> TelemetryResult<()> {
7743 let now_ms = self.clock.now_ms();
7744 let mut st = self.state.lock();
7745 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7746 if let Some(src) = src {
7747 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7748 }
7749 st.route_priorities.insert((src, dst), priority);
7750 #[cfg(feature = "discovery")]
7751 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7752 Ok(())
7753 }
7754
7755 pub fn clear_route_priority(
7757 &self,
7758 src: Option<RouterSideId>,
7759 dst: RouterSideId,
7760 ) -> TelemetryResult<()> {
7761 let now_ms = self.clock.now_ms();
7762 let mut st = self.state.lock();
7763 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7764 if let Some(src) = src {
7765 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7766 }
7767 st.route_priorities.remove(&(src, dst));
7768 #[cfg(feature = "discovery")]
7769 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7770 Ok(())
7771 }
7772
7773 pub fn set_route(
7778 &self,
7779 src: Option<RouterSideId>,
7780 dst: RouterSideId,
7781 enabled: bool,
7782 ) -> TelemetryResult<()> {
7783 let now_ms = self.clock.now_ms();
7784 let mut st = self.state.lock();
7785 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7786 if let Some(src) = src {
7787 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7788 }
7789 st.route_overrides.insert((src, dst), enabled);
7790 #[cfg(feature = "discovery")]
7791 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7792 Ok(())
7793 }
7794
7795 pub fn set_typed_route(
7800 &self,
7801 src: Option<RouterSideId>,
7802 ty: DataType,
7803 dst: RouterSideId,
7804 enabled: bool,
7805 ) -> TelemetryResult<()> {
7806 let now_ms = self.clock.now_ms();
7807 let mut st = self.state.lock();
7808 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7809 if let Some(src) = src {
7810 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7811 }
7812 st.typed_route_overrides
7813 .insert((src, ty.as_u32(), dst), enabled);
7814 #[cfg(feature = "discovery")]
7815 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7816 Ok(())
7817 }
7818
7819 pub fn clear_typed_route(
7821 &self,
7822 src: Option<RouterSideId>,
7823 ty: DataType,
7824 dst: RouterSideId,
7825 ) -> TelemetryResult<()> {
7826 let now_ms = self.clock.now_ms();
7827 let mut st = self.state.lock();
7828 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7829 if let Some(src) = src {
7830 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7831 }
7832 st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
7833 #[cfg(feature = "discovery")]
7834 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7835 Ok(())
7836 }
7837
7838 pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
7840 let now_ms = self.clock.now_ms();
7841 let mut st = self.state.lock();
7842 let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
7843 if let Some(src) = src {
7844 let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
7845 }
7846 st.route_overrides.remove(&(src, dst));
7847 #[cfg(feature = "discovery")]
7848 Self::note_discovery_topology_change_locked(&mut st, now_ms);
7849 Ok(())
7850 }
7851
7852 #[cfg(feature = "discovery")]
7854 pub fn announce_discovery(&self) -> TelemetryResult<()> {
7855 self.queue_discovery_announce()
7856 }
7857
7858 #[cfg(feature = "discovery")]
7860 pub fn announce_leave(&self) -> TelemetryResult<()> {
7861 let sender = self.sender_arc();
7862 let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
7863 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
7864 }
7865
7866 #[cfg(feature = "discovery")]
7868 pub fn poll_discovery(&self) -> TelemetryResult<bool> {
7869 self.poll_discovery_announce()
7870 }
7871
7872 #[cfg(feature = "discovery")]
7873 pub fn request_topology(&self) -> TelemetryResult<()> {
7874 let sender = self.sender_arc();
7875 let pkt =
7876 discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
7877 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
7878 }
7879
7880 #[cfg(feature = "discovery")]
7881 pub fn request_schema(&self) -> TelemetryResult<()> {
7882 let sender = self.sender_arc();
7883 let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
7884 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
7885 }
7886
7887 pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
7894 self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
7895 }
7896
7897 pub fn enable_network_variable(
7902 &self,
7903 ty: DataType,
7904 permissions: NetworkVariablePermissions,
7905 ) -> TelemetryResult<()> {
7906 if is_internal_control_type(ty) {
7907 return Err(TelemetryError::InvalidType);
7908 }
7909 let mut st = self.state.lock();
7910 st.managed_variable_types.insert(ty.as_u32());
7911 st.managed_variable_permissions
7912 .insert(ty.as_u32(), permissions);
7913 Ok(())
7914 }
7915
7916 pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
7922 where
7923 F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
7924 {
7925 if is_internal_control_type(ty) {
7926 return Err(TelemetryError::InvalidType);
7927 }
7928 let mut st = self.state.lock();
7929 st.managed_variable_types.insert(ty.as_u32());
7930 st.network_variable_update_handlers
7931 .entry(ty.as_u32())
7932 .or_default()
7933 .push(NetworkVariableUpdateHandler {
7934 handler: Arc::new(f),
7935 });
7936 Ok(())
7937 }
7938
7939 pub fn disable_managed_variable(&self, ty: DataType) {
7940 let mut st = self.state.lock();
7941 st.managed_variable_types.remove(&ty.as_u32());
7942 st.managed_variable_permissions.remove(&ty.as_u32());
7943 st.managed_variable_latest.remove(&ty.as_u32());
7944 st.network_variable_update_handlers.remove(&ty.as_u32());
7945 }
7946
7947 pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
7948 if is_internal_control_type(pkt.data_type()) {
7949 return Err(TelemetryError::InvalidType);
7950 }
7951 pkt.validate()?;
7952 {
7953 let mut st = self.state.lock();
7954 st.managed_variable_types.insert(pkt.data_type().as_u32());
7955 }
7956 self.cache_managed_variable_packet(&pkt, false)
7957 }
7958
7959 pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
7960 self.managed_variable_latest(ty)
7961 }
7962
7963 pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
7969 if is_internal_control_type(pkt.data_type()) {
7970 return Err(TelemetryError::InvalidType);
7971 }
7972 pkt.validate()?;
7973 let ty = pkt.data_type();
7974 {
7975 let mut st = self.state.lock();
7976 st.managed_variable_types.insert(ty.as_u32());
7977 let perms = Self::managed_variable_permissions_locked(&st, ty);
7978 if !perms.write {
7979 return Err(TelemetryError::PermissionDenied);
7980 }
7981 }
7982 self.cache_managed_variable_packet(&pkt, false)?;
7983 self.tx(pkt)
7984 }
7985
7986 #[cfg(feature = "discovery")]
7993 pub fn get_network_variable(
7994 &self,
7995 ty: DataType,
7996 stale_after_ms: Option<u64>,
7997 ) -> TelemetryResult<Option<Packet>> {
7998 if is_internal_control_type(ty) {
7999 return Err(TelemetryError::InvalidType);
8000 }
8001 {
8002 let mut st = self.state.lock();
8003 st.managed_variable_types.insert(ty.as_u32());
8004 let perms = Self::managed_variable_permissions_locked(&st, ty);
8005 if !perms.read {
8006 return Err(TelemetryError::PermissionDenied);
8007 }
8008 }
8009 let cached = self.managed_variable_latest_with_age(ty);
8010 let needs_refresh = match (cached.as_ref(), stale_after_ms) {
8011 (None, _) => true,
8012 (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
8013 (Some(_), None) => false,
8014 };
8015 if needs_refresh {
8016 self.request_managed_variable(ty)?;
8017 }
8018 Ok(cached.map(|(pkt, _age_ms)| pkt))
8019 }
8020
8021 pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
8023 if is_internal_control_type(ty) {
8024 return Err(TelemetryError::InvalidType);
8025 }
8026 if !self.can_read_managed_variable(ty) {
8027 return Err(TelemetryError::PermissionDenied);
8028 }
8029 Ok(self.managed_variable_latest(ty))
8030 }
8031
8032 #[cfg(feature = "discovery")]
8033 pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
8034 if is_internal_control_type(ty) {
8035 return Err(TelemetryError::InvalidType);
8036 }
8037 if !self.can_read_managed_variable(ty) {
8038 return Err(TelemetryError::PermissionDenied);
8039 }
8040 let sender = self.sender_arc();
8041 let pkt =
8042 discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
8043 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
8044 }
8045
8046 #[cfg(feature = "discovery")]
8047 pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
8048 let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
8049 self.request_managed_variable(ty)
8050 }
8051
8052 #[cfg(feature = "discovery")]
8054 pub fn export_topology(&self) -> TopologySnapshot {
8055 let now_ms = self.clock.now_ms();
8056 let mut st = self.state.lock();
8057 if Self::prune_discovery_routes_locked(&mut st, now_ms) {
8058 Self::note_discovery_topology_change_locked(&mut st, now_ms);
8059 }
8060 let routes = st
8061 .discovery_routes
8062 .iter()
8063 .filter_map(|(&side_id, route)| {
8064 let side = st.sides.get(side_id)?.as_ref()?;
8065 let announcers = route
8066 .announcers
8067 .iter()
8068 .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
8069 sender_id: sender_id.clone(),
8070 reachable_endpoints: sender_state
8071 .reachable
8072 .iter()
8073 .copied()
8074 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
8075 .collect(),
8076 reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
8077 routers: sender_state.topology_boards.clone(),
8078 last_seen_ms: sender_state.last_seen_ms,
8079 age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
8080 })
8081 .collect();
8082 Some(TopologySideRoute {
8083 side_id,
8084 side_name: side.name,
8085 reachable_endpoints: route
8086 .reachable
8087 .iter()
8088 .copied()
8089 .filter(|ep| !discovery::is_router_control_endpoint(*ep))
8090 .collect(),
8091 reachable_timesync_sources: route.reachable_timesync_sources.clone(),
8092 announcers,
8093 last_seen_ms: route.last_seen_ms,
8094 age_ms: now_ms.saturating_sub(route.last_seen_ms),
8095 })
8096 })
8097 .collect();
8098 let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true);
8099 let advertised_endpoints =
8100 self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true);
8101 let advertised_timesync_sources =
8102 self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms);
8103 let links = discovery::topology_links_from_boards(&routers);
8104 TopologySnapshot {
8105 advertised_endpoints,
8106 advertised_timesync_sources,
8107 routers,
8108 links,
8109 routes,
8110 current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
8111 next_announce_ms: st.discovery_cadence.next_announce_ms,
8112 }
8113 }
8114
8115 #[cfg(feature = "discovery")]
8116 pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
8117 let now_ms = self.clock.now_ms();
8118 let st = self.state.lock();
8119 let mut side_ids = Vec::new();
8120 let mut side_names = Vec::new();
8121 let mut last_seen_ms = None::<u64>;
8122 let mut reachable_endpoints = Vec::new();
8123 let mut reachable_timesync_sources = Vec::new();
8124 let mut packets_sent = 0u64;
8125 let mut packets_received = 0u64;
8126 let mut bytes_sent = 0u64;
8127 let mut bytes_received = 0u64;
8128
8129 for (side_id, route) in &st.discovery_routes {
8130 let Some(sender_state) = route.announcers.get(sender_id) else {
8131 continue;
8132 };
8133 side_ids.push(*side_id);
8134 if let Some(side_name) = st
8135 .sides
8136 .get(*side_id)
8137 .and_then(|side| side.as_ref())
8138 .map(|side| side.name)
8139 {
8140 side_names.push(side_name);
8141 }
8142 last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
8143 reachable_endpoints.extend(sender_state.reachable.iter().copied());
8144 reachable_timesync_sources
8145 .extend(sender_state.reachable_timesync_sources.iter().cloned());
8146 if let Some(stats) = st.side_runtime_stats.get(side_id) {
8147 packets_sent = packets_sent.saturating_add(stats.tx_packets);
8148 packets_received = packets_received.saturating_add(stats.rx_packets);
8149 bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
8150 bytes_received = bytes_received.saturating_add(stats.rx_bytes);
8151 }
8152 }
8153
8154 if side_ids.is_empty() {
8155 return None;
8156 }
8157 reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
8158 reachable_endpoints.sort_unstable();
8159 reachable_endpoints.dedup();
8160 reachable_timesync_sources.sort_unstable();
8161 reachable_timesync_sources.dedup();
8162 side_ids.sort_unstable();
8163 side_ids.dedup();
8164 side_names.sort_unstable();
8165 side_names.dedup();
8166 let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
8167 Some(ClientStatsSnapshot {
8168 sender_id: sender_id.to_string(),
8169 connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
8170 side_ids,
8171 side_names,
8172 last_seen_ms,
8173 age_ms,
8174 reachable_endpoints,
8175 reachable_timesync_sources,
8176 packets_sent,
8177 packets_received,
8178 bytes_sent,
8179 bytes_received,
8180 })
8181 }
8182
8183 pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
8184 let now_ms = self.clock.now_ms();
8185 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
8186 let st = self.state.lock();
8187
8188 let mut sides = Vec::new();
8189 for (side_id, side) in st.sides.iter().enumerate() {
8190 let Some(side) = side.as_ref() else { continue };
8191 let stats = st
8192 .side_runtime_stats
8193 .get(&side_id)
8194 .cloned()
8195 .unwrap_or_default();
8196 let adaptive = st
8197 .adaptive_route_stats
8198 .get(&side_id)
8199 .cloned()
8200 .unwrap_or_default()
8201 .snapshot(now_ms, true);
8202 let (tx_template_count, rx_template_count) = st
8203 .side_transport
8204 .get(&side_id)
8205 .map(|state| (state.tx_template_count(), state.rx_template_count()))
8206 .unwrap_or((0, 0));
8207 let mut data_types: Vec<RuntimeTypeStats> = stats
8208 .data_types
8209 .into_iter()
8210 .map(|(ty, item)| RuntimeTypeStats {
8211 data_type: DataType(ty),
8212 tx_packets: item.tx_packets,
8213 tx_bytes: item.tx_bytes,
8214 rx_packets: item.rx_packets,
8215 rx_bytes: item.rx_bytes,
8216 relayed_tx_packets: item.relayed_tx_packets,
8217 relayed_tx_bytes: item.relayed_tx_bytes,
8218 relayed_rx_packets: item.relayed_rx_packets,
8219 relayed_rx_bytes: item.relayed_rx_bytes,
8220 tx_retries: item.tx_retries,
8221 handler_failures: item.handler_failures,
8222 })
8223 .collect();
8224 data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
8225 sides.push(RuntimeSideStats {
8226 side_id,
8227 side_name: side.name,
8228 reliable_enabled: side.opts.reliable_enabled,
8229 link_local_enabled: side.opts.link_local_enabled,
8230 header_template_enabled: side.opts.header_template_enabled,
8231 max_frame_bytes: side.opts.max_frame_bytes,
8232 compact_header_target_bytes: side.opts.compact_header_target_bytes,
8233 side_transport_profile: side.opts.effective_transport_profile().as_str(),
8234 ingress_enabled: side.opts.ingress_enabled,
8235 egress_enabled: side.opts.egress_enabled,
8236 tx_packets: stats.tx_packets,
8237 tx_bytes: stats.tx_bytes,
8238 rx_packets: stats.rx_packets,
8239 rx_bytes: stats.rx_bytes,
8240 relayed_tx_packets: stats.relayed_tx_packets,
8241 relayed_tx_bytes: stats.relayed_tx_bytes,
8242 relayed_rx_packets: stats.relayed_rx_packets,
8243 relayed_rx_bytes: stats.relayed_rx_bytes,
8244 local_delivery_packets: stats.local_delivery_packets,
8245 tx_retries: stats.tx_retries,
8246 tx_handler_failures: stats.tx_handler_failures,
8247 local_handler_failures: stats.local_handler_failures,
8248 total_handler_retries: stats.total_handler_retries,
8249 side_transport_full_frames: stats.side_transport_full_frames,
8250 side_transport_compact_frames: stats.side_transport_compact_frames,
8251 side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
8252 side_transport_compact_omitted_timestamp_frames: stats
8253 .side_transport_compact_omitted_timestamp_frames,
8254 side_transport_chunk_frames: stats.side_transport_chunk_frames,
8255 side_transport_raw_bytes: stats.side_transport_raw_bytes,
8256 side_transport_wire_bytes: stats.side_transport_wire_bytes,
8257 side_transport_bytes_saved: stats.side_transport_bytes_saved,
8258 side_transport_min_compact_overhead_bytes: stats
8259 .side_transport_min_compact_overhead_bytes,
8260 side_transport_max_compact_overhead_bytes: stats
8261 .side_transport_max_compact_overhead_bytes,
8262 side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
8263 side_transport_template_evictions: stats.side_transport_template_evictions,
8264 side_transport_tx_template_count: tx_template_count,
8265 side_transport_rx_template_count: rx_template_count,
8266 max_side_transport_templates: side.opts.max_side_transport_templates,
8267 adaptive,
8268 data_types,
8269 });
8270 }
8271
8272 let mut route_modes: Vec<RouteModeStats> = st
8273 .route_selection_cursors
8274 .iter()
8275 .map(|(src, cursor)| RouteModeStats {
8276 src_side_id: *src,
8277 selection_mode: st.source_route_modes.get(src).copied(),
8278 cursor: *cursor,
8279 })
8280 .collect();
8281 for src in st.source_route_modes.keys() {
8282 if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
8283 route_modes.push(RouteModeStats {
8284 src_side_id: *src,
8285 selection_mode: st.source_route_modes.get(src).copied(),
8286 cursor: 0,
8287 });
8288 }
8289 }
8290 route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
8291
8292 let mut route_overrides: Vec<RouteOverrideStats> = st
8293 .route_overrides
8294 .iter()
8295 .map(|((src, dst), enabled)| RouteOverrideStats {
8296 src_side_id: *src,
8297 dst_side_id: *dst,
8298 enabled: *enabled,
8299 })
8300 .collect();
8301 route_overrides.sort_unstable_by_key(|item| {
8302 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
8303 });
8304
8305 let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
8306 .typed_route_overrides
8307 .iter()
8308 .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
8309 src_side_id: *src,
8310 data_type: DataType(*ty),
8311 dst_side_id: *dst,
8312 enabled: *enabled,
8313 })
8314 .collect();
8315 typed_route_overrides.sort_unstable_by_key(|item| {
8316 (
8317 item.src_side_id.unwrap_or(usize::MAX),
8318 item.data_type.as_u32(),
8319 item.dst_side_id,
8320 )
8321 });
8322
8323 let mut route_weights: Vec<RouteWeightStats> = st
8324 .route_weights
8325 .iter()
8326 .map(|((src, dst), weight)| RouteWeightStats {
8327 src_side_id: *src,
8328 dst_side_id: *dst,
8329 weight: *weight,
8330 })
8331 .collect();
8332 route_weights.sort_unstable_by_key(|item| {
8333 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
8334 });
8335
8336 let mut route_priorities: Vec<RoutePriorityStats> = st
8337 .route_priorities
8338 .iter()
8339 .map(|((src, dst), priority)| RoutePriorityStats {
8340 src_side_id: *src,
8341 dst_side_id: *dst,
8342 priority: *priority,
8343 })
8344 .collect();
8345 route_priorities.sort_unstable_by_key(|item| {
8346 (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
8347 });
8348
8349 #[cfg(feature = "discovery")]
8350 let discovery = DiscoveryRuntimeStats {
8351 route_count: st.discovery_routes.len(),
8352 announcer_count: st
8353 .discovery_routes
8354 .values()
8355 .map(|route| route.announcers.len())
8356 .sum(),
8357 current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
8358 next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
8359 };
8360 #[cfg(not(feature = "discovery"))]
8361 let discovery = DiscoveryRuntimeStats {
8362 route_count: 0,
8363 announcer_count: 0,
8364 current_announce_interval_ms: None,
8365 next_announce_ms: None,
8366 };
8367
8368 RuntimeStatsSnapshot {
8369 sides,
8370 route_modes,
8371 route_overrides,
8372 typed_route_overrides,
8373 route_weights,
8374 route_priorities,
8375 queues: QueueRuntimeStats {
8376 rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
8377 rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
8378 tx_len: st.transmit_queue.len(),
8379 tx_bytes: st.transmit_queue.bytes_used(),
8380 replay_len: 0,
8381 replay_bytes: 0,
8382 recent_rx_len: st.recent_rx.len(),
8383 recent_rx_bytes: st.recent_rx.bytes_used(),
8384 reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
8385 reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
8386 shared_queue_bytes_used: st.shared_queue_bytes_used(),
8387 },
8388 reliable: ReliableRuntimeStats {
8389 reliable_return_route_count: st.reliable_return_routes.len(),
8390 end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
8391 end_to_end_pending_destination_count: st
8392 .end_to_end_reliable_tx
8393 .values()
8394 .map(|sent| sent.pending_destinations.len())
8395 .sum(),
8396 end_to_end_acked_cache_count: 0,
8397 },
8398 discovery,
8399 total_handler_failures: st.total_handler_failures,
8400 total_handler_retries: st.total_handler_retries,
8401 }
8402 }
8403
8404 pub fn export_memory_layout_json(&self) -> String {
8406 let isr_rx = self.isr_rx_queue.snapshot().unwrap_or((0, 0));
8407 let st = self.state.lock();
8408 #[cfg(feature = "discovery")]
8409 let discovery_bytes = st.discovery_bytes_used();
8410 #[cfg(not(feature = "discovery"))]
8411 let discovery_bytes = 0usize;
8412 let schema_bytes = crate::config::schema_bytes_used();
8413 let network_variable_cache_bytes = st
8414 .managed_variable_latest
8415 .values()
8416 .map(|entry| entry.packet.byte_cost())
8417 .sum::<usize>();
8418 let mut out = String::new();
8419 let memory = st.memory;
8420 let _ = fmt::Write::write_fmt(
8421 &mut out,
8422 format_args!(
8423 "{{\"kind\":\"router\",\
8424 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
8425 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
8426 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
8427 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
8428 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
8429 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
8430 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
8431 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
8432 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
8433 st.shared_queue_bytes_used(),
8434 memory.max_queue_budget,
8435 st.received_queue.bytes_used(),
8436 st.received_queue.max_bytes(),
8437 st.received_queue.len(),
8438 isr_rx.1,
8439 memory.max_queue_budget,
8440 isr_rx.0,
8441 st.transmit_queue.bytes_used(),
8442 st.transmit_queue.max_bytes(),
8443 st.transmit_queue.len(),
8444 st.recent_rx.bytes_used(),
8445 st.recent_rx.max_bytes(),
8446 st.recent_rx.len(),
8447 st.reliable_rx_buffered_bytes(),
8448 memory.max_queue_budget,
8449 st.reliable_rx_buffer_len(),
8450 discovery_bytes,
8451 memory.max_queue_budget,
8452 schema_bytes,
8453 memory.max_queue_budget,
8454 network_variable_cache_bytes,
8455 memory.max_queue_budget,
8456 st.managed_variable_latest.len(),
8457 ),
8458 );
8459 out
8460 }
8461
8462 #[cfg(test)]
8463 pub(crate) fn debug_end_to_end_pending_destination_count(
8464 &self,
8465 packet_id: u64,
8466 ) -> Option<usize> {
8467 let st = self.state.lock();
8468 st.end_to_end_reliable_tx
8469 .get(&packet_id)
8470 .map(|sent| sent.pending_destinations.len())
8471 }
8472
8473 #[cfg(test)]
8474 pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
8475 let st = self.state.lock();
8476 st.end_to_end_reliable_tx.len()
8477 }
8478
8479 #[cfg(test)]
8480 pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
8481 let st = self.state.lock();
8482 st.reliable_return_routes.len()
8483 }
8484
8485 #[cfg(test)]
8486 pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
8487 let st = self.state.lock();
8488 (
8489 st.received_queue.len(),
8490 st.transmit_queue.len(),
8491 st.recent_rx.len(),
8492 )
8493 }
8494
8495 #[cfg(test)]
8496 pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
8497 let st = self.state.lock();
8498 st.shared_queue_bytes_used()
8499 }
8500
8501 #[cfg(test)]
8502 pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
8503 let st = self.state.lock();
8504 (st.recent_rx.capacity(), st.recent_rx.max_bytes())
8505 }
8506
8507 fn get_hash(item: &RouterItem) -> u64 {
8511 match item {
8512 RouterItem::Packet(pkt) => pkt.packet_id(),
8513 RouterItem::Packed(bytes) => {
8514 match wire_format::packet_id_from_wire(bytes.as_ref()) {
8515 Ok(id) => id,
8516 Err(_e) => {
8517 let h: u64 = 0x9E37_79B9_7F4A_7C15;
8520 hash_bytes_u64(h, bytes.as_ref())
8521 }
8522 }
8523 }
8524 }
8525 }
8526
8527 fn remove_pkt_id(&self, item: &RouterItem) {
8529 let hash = Self::get_hash(item);
8530 let mut st = self.state.lock();
8531 st.recent_rx.remove_value(&hash);
8532 }
8533
8534 fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
8537 let id = Self::get_hash(item);
8538 let mut st = self.state.lock();
8539 if st.recent_rx.contains(&id) {
8540 Ok(true)
8541 } else {
8542 st.push_recent_rx(id)?;
8543 Ok(false)
8544 }
8545 }
8546
8547 fn handle_callback_error(
8552 &self,
8553 pkt: &Packet,
8554 dest: Option<DataEndpoint>,
8555 e: TelemetryError,
8556 called_from_queue: bool,
8557 ) -> TelemetryResult<()> {
8558 let device = self.sender_arc();
8559 let error_msg = match dest {
8560 Some(failed_local) => format!(
8561 "Handler for endpoint {:?} failed on device {:?}: {:?}",
8562 failed_local, device, e
8563 ),
8564 None => format!("TX Handler failed on device {:?}: {:?}", device, e),
8565 };
8566
8567 let mut recipients: Vec<DataEndpoint> = pkt
8568 .endpoints()
8569 .iter()
8570 .copied()
8571 .filter(|&ep| self.cfg.is_local_endpoint(ep))
8572 .collect();
8573 recipients.sort_unstable();
8574 recipients.dedup();
8575
8576 if let Some(failed_local) = dest {
8577 recipients.retain(|&ep| ep != failed_local);
8578 }
8579
8580 if recipients.is_empty() {
8583 recipients = pkt.endpoints().to_vec();
8584 recipients.sort_unstable();
8585 recipients.dedup();
8586 if let Some(failed_local) = dest {
8587 recipients.retain(|&ep| ep != failed_local);
8588 }
8589 }
8590
8591 if recipients.is_empty() {
8592 fallback_stdout(&error_msg);
8593 return Ok(());
8594 }
8595
8596 let payload = make_error_payload(&error_msg);
8597
8598 let sender = self.sender_arc();
8599 let error_pkt = Packet::new(
8600 DataType::TelemetryError,
8601 &recipients,
8602 sender.as_ref(),
8603 self.packet_timestamp_ms(),
8604 payload,
8605 )?;
8606
8607 self.emit_internal_tx(
8608 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
8609 false,
8610 called_from_queue,
8611 )
8612 }
8613
8614 #[inline]
8618 pub fn process_tx_queue(&self) -> TelemetryResult<()> {
8619 self.process_tx_queue_with_timeout(0)
8620 }
8621
8622 #[inline]
8624 pub fn process_all_queues(&self) -> TelemetryResult<()> {
8625 self.process_all_queues_with_timeout(0)
8626 }
8627
8628 #[inline]
8630 pub fn clear_queues(&self) {
8631 let mut st = self.state.lock();
8632 st.transmit_queue.clear();
8633 st.received_queue.clear();
8634 drop(st);
8635 let _ = self.isr_rx_queue.clear();
8636 }
8637
8638 #[inline]
8640 pub fn clear_rx_queue(&self) {
8641 let mut st = self.state.lock();
8642 st.received_queue.clear();
8643 drop(st);
8644 let _ = self.isr_rx_queue.clear();
8645 }
8646
8647 #[inline]
8649 pub fn clear_tx_queue(&self) {
8650 let mut st = self.state.lock();
8651 st.transmit_queue.clear();
8652 }
8653
8654 fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
8657 let start = self.clock.now_ms();
8658 loop {
8659 self.process_reliable_timeouts()?;
8660 self.process_end_to_end_reliable_timeouts()?;
8661 #[cfg(feature = "discovery")]
8662 let _ = self.drain_queued_discovery_rx_before_tx()?;
8663 let pkt_opt = {
8664 let mut st = self.state.lock();
8665 st.transmit_queue.pop_front()
8666 };
8667 let Some(pkt) = pkt_opt else { break };
8668 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
8669 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
8670 break;
8671 }
8672 }
8673 Ok(())
8674 }
8675
8676 pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
8679 #[cfg(feature = "timesync")]
8680 let _ = self.poll_timesync()?;
8681 #[cfg(feature = "discovery")]
8682 let _ = self.poll_discovery()?;
8683 self.process_tx_queue_with_timeout_impl(timeout_ms)
8684 }
8685
8686 #[inline]
8688 fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
8689 self.rx_item(&item, true)
8690 }
8691
8692 fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
8695 let start = self.clock.now_ms();
8696 loop {
8697 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
8698 let mut st = self.state.lock();
8699 st.received_queue.pop_front()
8700 });
8701 let Some(item) = item_opt else { break };
8702 self.process_rx_queue_item(item)?;
8703 if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
8704 break;
8705 }
8706 }
8707 Ok(())
8708 }
8709
8710 pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
8713 #[cfg(feature = "timesync")]
8714 let _ = self.poll_timesync()?;
8715 #[cfg(feature = "discovery")]
8716 let _ = self.poll_discovery()?;
8717 self.process_rx_queue_with_timeout_impl(timeout_ms)
8718 }
8719
8720 fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
8723 if timeout_ms == 0 {
8724 loop {
8725 let mut did_any = false;
8726 self.process_reliable_timeouts()?;
8727 self.process_end_to_end_reliable_timeouts()?;
8728 #[cfg(feature = "discovery")]
8729 if self.drain_queued_discovery_rx_before_tx()? {
8730 did_any = true;
8731 }
8732
8733 if let Some(pkt) = {
8734 let mut st = self.state.lock();
8735 st.transmit_queue.pop_front()
8736 } {
8737 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
8738 did_any = true;
8739 }
8740
8741 if let Some(item) = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
8742 let mut st = self.state.lock();
8743 st.received_queue.pop_front()
8744 }) {
8745 self.process_rx_queue_item(item)?;
8746 did_any = true;
8747 }
8748
8749 if !did_any {
8750 break;
8751 }
8752 }
8753 return Ok(());
8754 }
8755
8756 let tx_budget_ms = u64::from(timeout_ms / 2);
8757 let rx_budget_ms = u64::from(timeout_ms) - tx_budget_ms;
8758
8759 let tx_start = self.clock.now_ms();
8760 loop {
8761 self.process_reliable_timeouts()?;
8762 self.process_end_to_end_reliable_timeouts()?;
8763 #[cfg(feature = "discovery")]
8764 let _ = self.drain_queued_discovery_rx_before_tx()?;
8765 let pkt_opt = {
8766 let mut st = self.state.lock();
8767 st.transmit_queue.pop_front()
8768 };
8769 let Some(pkt) = pkt_opt else { break };
8770 self.tx_item_impl(pkt.item, pkt.ignore_local, true)?;
8771 if self.clock.now_ms().wrapping_sub(tx_start) >= tx_budget_ms {
8772 break;
8773 }
8774 }
8775
8776 let rx_start = self.clock.now_ms();
8777 loop {
8778 let item_opt = self.isr_rx_queue.pop_front().unwrap_or(None).or_else(|| {
8779 let mut st = self.state.lock();
8780 st.received_queue.pop_front()
8781 });
8782 let Some(item) = item_opt else { break };
8783 self.process_rx_queue_item(item)?;
8784 if self.clock.now_ms().wrapping_sub(rx_start) >= rx_budget_ms {
8785 break;
8786 }
8787 }
8788
8789 Ok(())
8790 }
8791
8792 pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
8795 #[cfg(feature = "timesync")]
8796 let _ = self.poll_timesync()?;
8797 #[cfg(feature = "discovery")]
8798 let _ = self.poll_discovery()?;
8799 self.process_all_queues_with_timeout_impl(timeout_ms)
8800 }
8801
8802 pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
8807 #[cfg(feature = "timesync")]
8808 let _ = self.poll_timesync()?;
8809
8810 #[cfg(feature = "discovery")]
8811 {
8812 let _ = self.poll_discovery()?;
8813 }
8814
8815 self.process_all_queues_with_timeout_impl(timeout_ms)
8816 }
8817
8818 pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
8823 #[cfg(feature = "discovery")]
8824 {
8825 let _ = self.poll_discovery()?;
8826 }
8827
8828 self.process_all_queues_with_timeout_impl(timeout_ms)
8829 }
8830
8831 #[inline]
8833 fn tx_queue_item_with_flags(
8834 &self,
8835 item: RouterTxItem,
8836 ignore_local: bool,
8837 ) -> TelemetryResult<()> {
8838 let priority = match &item {
8839 RouterTxItem::Broadcast(data) => Self::router_item_priority(data)?,
8840 RouterTxItem::EndToEndReplay { .. } => {
8841 Self::router_item_priority_bumped(DataType::ReliableAck)
8842 }
8843 RouterTxItem::ToSide { data, .. } => Self::router_item_priority(data)?,
8844 RouterTxItem::ReliableReplay { bytes, .. } => {
8845 let ty = wire_format::peek_envelope(bytes.as_ref())?.ty;
8846 Self::router_item_priority_bumped(ty)
8847 }
8848 };
8849 self.tx_queue_item_with_priority(item, ignore_local, priority)
8850 }
8851
8852 #[inline]
8853 fn tx_queue_item_with_priority(
8854 &self,
8855 item: RouterTxItem,
8856 ignore_local: bool,
8857 priority: u8,
8858 ) -> TelemetryResult<()> {
8859 let mut st = self.state.lock();
8860 st.push_transmit(TxQueued {
8861 item,
8862 ignore_local,
8863 priority,
8864 })?;
8865 Ok(())
8866 }
8867
8868 #[inline]
8870 fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
8871 self.tx_queue_item_with_flags(item, false)
8872 }
8873
8874 #[inline]
8875 fn try_enter_side_tx(&self) -> Option<crate::lock::ReentryGuard<'_>> {
8876 self.side_tx_gate.try_enter()
8877 }
8878
8879 #[inline]
8880 fn side_tx_active(&self) -> bool {
8881 self.side_tx_gate.is_active()
8882 }
8883
8884 #[inline]
8888 pub fn process_rx_queue(&self) -> TelemetryResult<()> {
8889 self.process_rx_queue_with_timeout(0)
8890 }
8891
8892 #[inline]
8894 pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
8895 let data = RouterItem::Packed(Arc::from(bytes));
8896 let priority = Self::router_item_priority(&data)?;
8897 let mut st = self.state.lock();
8898 st.push_received(RouterRxItem {
8899 src: None,
8900 data,
8901 priority,
8902 })?;
8903 Ok(())
8904 }
8905
8906 #[inline]
8911 pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
8912 let data = RouterItem::Packed(Arc::from(bytes));
8913 let priority = Self::router_item_priority(&data)?;
8914 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
8915 src: None,
8916 data,
8917 priority,
8918 })
8919 }
8920
8921 #[inline]
8923 pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
8924 pkt.validate()?;
8925 let data = RouterItem::Packet(pkt);
8926 let priority = Self::router_item_priority(&data)?;
8927 let mut st = self.state.lock();
8928 st.push_received(RouterRxItem {
8929 src: None,
8930 data,
8931 priority,
8932 })?;
8933 Ok(())
8934 }
8935
8936 #[inline]
8941 pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
8942 pkt.validate()?;
8943 let data = RouterItem::Packet(pkt);
8944 let priority = Self::router_item_priority(&data)?;
8945 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
8946 src: None,
8947 data,
8948 priority,
8949 })
8950 }
8951
8952 #[inline]
8954 pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
8955 self.ensure_side_ingress_enabled(side)?;
8956 pkt.validate()?;
8957 let data = RouterItem::Packet(pkt);
8958 let priority = Self::router_item_priority(&data)?;
8959 let mut st = self.state.lock();
8960 st.push_received(RouterRxItem {
8961 src: Some(side),
8962 data,
8963 priority,
8964 })?;
8965 Ok(())
8966 }
8967
8968 #[inline]
8973 pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
8974 self.ensure_side_ingress_enabled(side)?;
8975 pkt.validate()?;
8976 let data = RouterItem::Packet(pkt);
8977 let priority = Self::router_item_priority(&data)?;
8978 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
8979 src: Some(side),
8980 data,
8981 priority,
8982 })
8983 }
8984
8985 #[inline]
8987 pub fn rx_packed_queue_from_side(
8988 &self,
8989 bytes: &[u8],
8990 side: RouterSideId,
8991 ) -> TelemetryResult<()> {
8992 self.ensure_side_ingress_enabled(side)?;
8993 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
8994 return Ok(());
8995 };
8996 let data = RouterItem::Packed(decoded);
8997 let priority = Self::router_item_priority(&data)?;
8998 let mut st = self.state.lock();
8999 st.push_received(RouterRxItem {
9000 src: Some(side),
9001 data,
9002 priority,
9003 })?;
9004 Ok(())
9005 }
9006
9007 #[inline]
9012 pub fn rx_packed_queue_from_side_isr(
9013 &self,
9014 bytes: &[u8],
9015 side: RouterSideId,
9016 ) -> TelemetryResult<()> {
9017 self.ensure_side_ingress_enabled(side)?;
9018 let data = RouterItem::Packed(Arc::from(bytes));
9019 let priority = Self::router_item_priority(&data)?;
9020 self.isr_rx_queue.push_back_prioritized(RouterRxItem {
9021 src: Some(side),
9022 data,
9023 priority,
9024 })
9025 }
9026
9027 fn retry_with_attempts<F, T, E>(&self, times: usize, f: F) -> Result<(T, usize), (E, usize)>
9028 where
9029 F: Fn() -> Result<T, E>,
9030 {
9031 let mut last_err = None;
9032 for attempt in 0..times {
9033 match f() {
9034 Ok(v) => return Ok((v, attempt + 1)),
9035 Err(e) => last_err = Some((e, attempt + 1)),
9036 }
9037 }
9038 Err(last_err.expect("times > 0"))
9039 }
9040
9041 #[inline]
9043 fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
9044 self.cfg
9045 .handlers
9046 .iter()
9047 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
9048 }
9049
9050 #[inline]
9052 fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
9053 self.cfg
9054 .handlers
9055 .iter()
9056 .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
9057 }
9058
9059 fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
9060 pkt.endpoints()
9061 .iter()
9062 .copied()
9063 .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
9064 }
9065
9066 fn call_handler_with_retries(
9072 &self,
9073 dest: DataEndpoint,
9074 handler: &EndpointHandler,
9075 data: Option<&[u8]>,
9076 pkt_for_ctx: Option<&Packet>,
9077 env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
9078 called_from_queue: bool,
9079 ) -> TelemetryResult<()> {
9080 let owned_tmp: Option<RouterItem>;
9081
9082 let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
9083 (Some(d), _) => {
9084 owned_tmp = Some(RouterItem::Packed(Arc::from(d)));
9085 owned_tmp.as_ref().unwrap()
9086 }
9087 (None, Some(pkt)) => {
9088 owned_tmp = Some(RouterItem::Packet(pkt.clone()));
9089 owned_tmp.as_ref().unwrap()
9090 }
9091 (None, None) => {
9092 debug_assert!(
9093 false,
9094 "call_handler_with_retries called without data or packet context"
9095 );
9096 return Ok(());
9097 }
9098 };
9099
9100 match (&handler.handler, data) {
9101 (EndpointHandlerFn::Packet(f), _) => {
9102 let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
9103 with_retries(
9104 self,
9105 dest,
9106 item_for_ctx,
9107 pkt_for_ctx,
9108 env_for_ctx,
9109 called_from_queue,
9110 || f(pkt),
9111 )
9112 }
9113
9114 (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
9115 self,
9116 dest,
9117 item_for_ctx,
9118 pkt_for_ctx,
9119 env_for_ctx,
9120 called_from_queue,
9121 || f(bytes),
9122 ),
9123
9124 (EndpointHandlerFn::Packed(_), None) => Ok(()),
9125 }
9126 }
9127
9128 fn handle_callback_error_from_env(
9133 &self,
9134 env: &wire_format::TelemetryEnvelope,
9135 dest: Option<DataEndpoint>,
9136 e: TelemetryError,
9137 called_from_queue: bool,
9138 ) -> TelemetryResult<()> {
9139 let mut recipients: Vec<DataEndpoint> = env
9140 .endpoints
9141 .iter()
9142 .copied()
9143 .filter(|&ep| self.cfg.is_local_endpoint(ep))
9144 .collect();
9145 recipients.sort_unstable();
9146 recipients.dedup();
9147 if let Some(failed) = dest {
9148 recipients.retain(|&ep| ep != failed);
9149 }
9150
9151 if recipients.is_empty() {
9152 recipients = env.endpoints.to_vec();
9153 recipients.sort_unstable();
9154 recipients.dedup();
9155 if let Some(failed) = dest {
9156 recipients.retain(|&ep| ep != failed);
9157 }
9158 }
9159
9160 let device = self.sender_arc();
9161 let error_msg = format!(
9162 "Handler for endpoint {:?} failed on device {:?}: {:?}",
9163 dest, device, e
9164 );
9165 if recipients.is_empty() {
9166 fallback_stdout(&error_msg);
9167 return Ok(());
9168 }
9169
9170 let payload = make_error_payload(&error_msg);
9171
9172 let error_pkt = Packet::new(
9173 DataType::TelemetryError,
9174 &recipients,
9175 &env.sender.clone(),
9176 env.timestamp_ms,
9177 payload,
9178 )?;
9179 self.emit_internal_tx(
9180 RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
9181 false,
9182 called_from_queue,
9183 )
9184 }
9185
9186 fn handle_internal_reliable_packet(
9187 &self,
9188 pkt: &Packet,
9189 src: Option<RouterSideId>,
9190 called_from_queue: bool,
9191 ) -> TelemetryResult<bool> {
9192 if !matches!(
9193 pkt.data_type(),
9194 DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
9195 ) {
9196 return Ok(false);
9197 }
9198
9199 let Some(src) = src else {
9200 return Ok(false);
9201 };
9202
9203 if pkt.data_type() == DataType::ReliableAck
9204 && Self::is_end_to_end_ack_sender(pkt.sender())
9205 && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
9206 {
9207 let mut st = self.state.lock();
9208 if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
9209 if let Some(sender_hash) = Self::decode_end_to_end_ack_sender_hash(pkt.sender()) {
9210 sent.pending_destinations.remove(&sender_hash);
9211 if sent.pending_destinations.is_empty() {
9212 st.end_to_end_reliable_tx.remove(&packet_id);
9213 }
9214 return Ok(true);
9215 }
9216 st.end_to_end_reliable_tx.remove(&packet_id);
9217 return Ok(true);
9218 }
9219 return Ok(false);
9220 }
9221
9222 let vals = pkt.data_as_u32()?;
9223 if vals.len() != 2 {
9224 return Err(TelemetryError::Unpack("bad reliable control payload"));
9225 }
9226 let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
9227 let seq = vals[1];
9228
9229 match pkt.data_type() {
9230 DataType::ReliableAck => {
9231 self.handle_reliable_ack(src, ty, seq);
9232 Ok(true)
9233 }
9234 DataType::ReliablePartialAck => {
9235 self.handle_reliable_partial_ack(src, ty, seq);
9236 Ok(true)
9237 }
9238 DataType::ReliablePacketRequest => {
9239 self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
9240 Ok(true)
9241 }
9242 _ => Ok(false),
9243 }
9244 }
9245
9246 fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
9252 if let Some(src) = item.src {
9253 self.ensure_side_ingress_enabled(src)?;
9254 match &item.data {
9255 RouterItem::Packet(pkt) => {
9256 let bytes = wire_format::pack_packet(pkt).len();
9257 self.note_side_rx(src, pkt.data_type(), bytes, true);
9258 }
9259 RouterItem::Packed(bytes) => {
9260 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
9261 self.note_side_rx(src, env.ty, bytes.len(), true);
9262 }
9263 }
9264 }
9265 match &item.data {
9266 RouterItem::Packet(pkt) => {
9267 if is_reliable_type(pkt.data_type())
9268 && !is_internal_control_type(pkt.data_type())
9269 {
9270 self.note_reliable_return_route(src, pkt.packet_id());
9271 }
9272 }
9273 RouterItem::Packed(bytes) => {
9274 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref())
9275 && is_reliable_type(env.ty)
9276 && !is_internal_control_type(env.ty)
9277 && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
9278 {
9279 self.note_reliable_return_route(src, packet_id);
9280 }
9281 }
9282 }
9283 }
9284 match &item.data {
9285 RouterItem::Packet(pkt) => {
9286 if !is_internal_control_type(pkt.data_type()) {
9287 self.remember_managed_variable_packet(pkt)?;
9288 }
9289 }
9290 RouterItem::Packed(bytes) => {
9291 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref())
9292 && !is_internal_control_type(env.ty)
9293 && self.is_managed_variable_type(env.ty)
9294 {
9295 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9296 pkt.validate()?;
9297 self.remember_managed_variable_packet(&pkt)?;
9298 }
9299 }
9300 }
9301 let mut released_buffered: Vec<Arc<[u8]>> = Vec::new();
9302 if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
9303 let (_opts, handler_is_packed, hop_reliable_enabled) = {
9304 let st = self.state.lock();
9305 let side_ref = Self::side_ref(&st, src)?;
9306 let opts = side_ref.opts;
9307 (
9308 opts,
9309 matches!(side_ref.tx_handler, RouterTxHandlerFn::Packed(_)),
9310 opts.reliable_enabled
9311 && self.cfg.reliable_enabled()
9312 && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
9313 )
9314 };
9315
9316 if hop_reliable_enabled && handler_is_packed {
9317 let frame = match wire_format::peek_frame_info(bytes.as_ref()) {
9318 Ok(frame) => frame,
9319 Err(e) => {
9320 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
9321 if let Ok(frame) =
9322 wire_format::peek_frame_info_unchecked(bytes.as_ref())
9323 && is_reliable_type(frame.envelope.ty)
9324 && let Some(hdr) = frame.reliable
9325 {
9326 let unordered =
9327 (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
9328 let unsequenced =
9329 (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
9330
9331 if !unsequenced {
9332 let requested = if unordered {
9333 hdr.seq
9334 } else {
9335 let mut st = self.state.lock();
9336 let rx_state = self.reliable_rx_state_mut(
9337 &mut st,
9338 src,
9339 frame.envelope.ty,
9340 );
9341 rx_state.expected_seq.min(hdr.seq)
9342 };
9343 self.queue_reliable_packet_request(
9344 src,
9345 frame.envelope.ty,
9346 requested,
9347 called_from_queue,
9348 )?;
9349 }
9350 }
9351 return Ok(());
9352 }
9353 return Err(e);
9354 }
9355 };
9356 if is_reliable_type(frame.envelope.ty)
9357 && let Some(hdr) = frame.reliable
9358 {
9359 if frame.ack_only() {
9360 self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
9361 return Ok(());
9362 }
9363 let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
9364 let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
9365
9366 if !unsequenced {
9367 if unordered {
9368 self.queue_reliable_ack(
9369 src,
9370 frame.envelope.ty,
9371 hdr.seq,
9372 called_from_queue,
9373 )?;
9374 } else {
9375 let mut release: Vec<Arc<[u8]>> = Vec::new();
9376 let mut last_delivered = None;
9377 let mut ack_old = None;
9378 let mut request_missing = None;
9379 let mut partial_ack = None;
9380 {
9381 let mut st = self.state.lock();
9382 let rx_state =
9383 self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
9384 let expected_seq = rx_state.expected_seq;
9385 if hdr.seq < expected_seq {
9386 ack_old = Some(expected_seq.saturating_sub(1));
9387 } else if hdr.seq > expected_seq {
9388 request_missing = Some(expected_seq);
9389 partial_ack = Some(hdr.seq);
9390 st.buffer_reliable_rx(
9391 src,
9392 frame.envelope.ty,
9393 hdr.seq,
9394 bytes.clone(),
9395 )?;
9396 } else {
9397 release.push(bytes.clone());
9398 last_delivered = Some(hdr.seq);
9399 let mut next_expected = hdr.seq.wrapping_add(1);
9400 while let Some(buf) = rx_state.buffered.remove(&next_expected) {
9401 release.push(buf);
9402 last_delivered = Some(next_expected);
9403 let next = next_expected.wrapping_add(1);
9404 next_expected = if next == 0 { 1 } else { next };
9405 }
9406 rx_state.expected_seq = next_expected;
9407 }
9408 }
9409
9410 if let Some(ack_seq) = ack_old {
9411 self.queue_reliable_ack(
9412 src,
9413 frame.envelope.ty,
9414 ack_seq,
9415 called_from_queue,
9416 )?;
9417 return Ok(());
9418 }
9419 if let Some(request_seq) = request_missing {
9420 if let Some(partial_seq) = partial_ack {
9421 self.queue_reliable_partial_ack(
9422 src,
9423 frame.envelope.ty,
9424 partial_seq,
9425 called_from_queue,
9426 )?;
9427 }
9428 self.queue_reliable_packet_request(
9429 src,
9430 frame.envelope.ty,
9431 request_seq,
9432 called_from_queue,
9433 )?;
9434 return Ok(());
9435 }
9436
9437 if let Some(ack_seq) = last_delivered {
9438 self.queue_reliable_ack(
9439 src,
9440 frame.envelope.ty,
9441 ack_seq,
9442 called_from_queue,
9443 )?;
9444 }
9445
9446 released_buffered.extend(release.into_iter().skip(1));
9447 }
9448 }
9449 }
9450 } else {
9451 match wire_format::peek_frame_info(bytes.as_ref()) {
9452 Ok(frame) => {
9453 if frame.ack_only() {
9454 return Ok(());
9455 }
9456 }
9457 Err(e) => {
9458 if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
9459 return Ok(());
9460 }
9461 return Err(e);
9462 }
9463 }
9464 }
9465 }
9466
9467 if self.is_duplicate_pkt(&item.data)? {
9468 if item.src.is_some() {
9469 let local_sender = self.sender_arc();
9470 match &item.data {
9471 RouterItem::Packet(pkt)
9472 if (is_reliable_type(pkt.data_type())
9473 || !pkt.wire_target_senders().is_empty())
9474 && pkt.sender() != local_sender.as_ref()
9475 && self.item_targets_local_sender(&item.data)?
9476 && self.packet_has_local_handler(pkt) =>
9477 {
9478 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
9479 }
9480 RouterItem::Packed(bytes) => {
9481 if let Ok(pkt) = wire_format::unpack_packet(bytes.as_ref())
9482 && (is_reliable_type(pkt.data_type())
9483 || !pkt.wire_target_senders().is_empty())
9484 && pkt.sender() != local_sender.as_ref()
9485 && self.item_targets_local_sender(&item.data)?
9486 && self.packet_has_local_handler(&pkt)
9487 {
9488 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
9489 .unwrap_or_else(|_| pkt.packet_id());
9490 self.queue_end_to_end_reliable_ack_for_packet_id(
9491 packet_id,
9492 called_from_queue,
9493 )?;
9494 }
9495 }
9496 _ => {}
9497 }
9498 }
9499 return Ok(());
9500 }
9501
9502 self.dispatch_rx_data(item, called_from_queue)?;
9503
9504 for release_bytes in released_buffered {
9505 let release_data = RouterItem::Packed(release_bytes.clone());
9506 if self.is_duplicate_pkt(&release_data)? {
9507 continue;
9508 }
9509 let release_item = RouterRxItem {
9510 src: item.src,
9511 priority: Self::router_item_priority(&release_data)?,
9512 data: release_data,
9513 };
9514 self.dispatch_rx_data(&release_item, called_from_queue)?;
9515 }
9516
9517 Ok(())
9518 }
9519
9520 fn dispatch_rx_data(
9521 &self,
9522 item: &RouterRxItem,
9523 called_from_queue: bool,
9524 ) -> TelemetryResult<()> {
9525 match &item.data {
9526 RouterItem::Packet(pkt) => {
9527 pkt.validate()?;
9528
9529 if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
9530 return Ok(());
9531 }
9532
9533 if pkt.data_type() == DataType::P2pMessage {
9534 if self.item_targets_local_sender(&item.data)? {
9535 self.dispatch_p2p_packet(pkt)?;
9536 if item.src.is_some() {
9537 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
9538 }
9539 }
9540 if self.should_route_remote(&item.data, item.src)? {
9541 self.relay_send(
9542 RouterItem::Packet(pkt.to_owned()),
9543 item.src,
9544 called_from_queue,
9545 )?;
9546 }
9547 return Ok(());
9548 }
9549
9550 #[cfg(feature = "timesync")]
9551 if matches!(
9552 pkt.data_type(),
9553 DataType::TimeSyncAnnounce
9554 | DataType::TimeSyncRequest
9555 | DataType::TimeSyncResponse
9556 ) {
9557 self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
9558 return Ok(());
9559 }
9560
9561 if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
9562 if self.should_route_remote(&item.data, item.src)? {
9563 self.relay_send(
9564 RouterItem::Packet(pkt.to_owned()),
9565 item.src,
9566 called_from_queue,
9567 )?;
9568 }
9569 return Ok(());
9570 }
9571
9572 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
9573 eps.sort_unstable();
9574 eps.dedup();
9575 let had_local_handler = eps.iter().copied().any(|ep| {
9576 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
9577 });
9578
9579 let has_remote = self.should_route_remote(&item.data, item.src)?;
9580 let targets_local = self.item_targets_local_sender(&item.data)?;
9581
9582 let has_packed_local = eps
9583 .iter()
9584 .copied()
9585 .any(|ep| self.endpoint_has_packed_handler(ep));
9586 let bytes_opt = if has_packed_local {
9587 Some(wire_format::pack_packet(pkt))
9588 } else {
9589 None
9590 };
9591
9592 if targets_local {
9593 for dest in eps {
9594 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
9595 let result = match (&h.handler, &bytes_opt) {
9596 (EndpointHandlerFn::Packed(_), Some(bytes)) => self
9597 .call_handler_with_retries(
9598 dest,
9599 h,
9600 Some(bytes.as_ref()),
9601 Some(pkt),
9602 None,
9603 called_from_queue,
9604 ),
9605 (EndpointHandlerFn::Packed(_), None) => {
9606 let bytes = wire_format::pack_packet(pkt);
9607 self.call_handler_with_retries(
9608 dest,
9609 h,
9610 Some(bytes.as_ref()),
9611 Some(pkt),
9612 None,
9613 called_from_queue,
9614 )
9615 }
9616 (EndpointHandlerFn::Packet(_), _) => self
9617 .call_handler_with_retries(
9618 dest,
9619 h,
9620 None,
9621 Some(pkt),
9622 None,
9623 called_from_queue,
9624 ),
9625 };
9626 if result.is_err()
9627 && let Some(src) = item.src
9628 {
9629 self.note_side_local_handler_failure(
9630 src,
9631 pkt.data_type(),
9632 runtime_max_handler_retries(),
9633 );
9634 }
9635 }
9636 }
9637 }
9638
9639 if let Some(src) = item.src
9640 && had_local_handler
9641 && targets_local
9642 {
9643 self.note_side_local_delivery(src, pkt.data_type());
9644 }
9645
9646 if item.src.is_some()
9647 && had_local_handler
9648 && targets_local
9649 && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
9650 {
9651 self.queue_end_to_end_reliable_ack(pkt, called_from_queue)?;
9652 }
9653
9654 if has_remote {
9655 let relay_item = RouterItem::Packet(pkt.to_owned());
9656 self.relay_send(relay_item, item.src, called_from_queue)?;
9657 }
9658
9659 Ok(())
9660 }
9661 RouterItem::Packed(bytes) => {
9662 let env = wire_format::peek_envelope(bytes.as_ref())?;
9663
9664 if matches!(
9665 env.ty,
9666 DataType::ReliableAck
9667 | DataType::ReliablePartialAck
9668 | DataType::ReliablePacketRequest
9669 ) {
9670 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9671 pkt.validate()?;
9672 let _ =
9673 self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
9674 return Ok(());
9675 }
9676
9677 if env.ty == DataType::P2pMessage {
9678 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9679 pkt.validate()?;
9680 if self.item_targets_local_sender(&item.data)? {
9681 self.dispatch_p2p_packet(&pkt)?;
9682 if item.src.is_some() {
9683 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
9684 .unwrap_or_else(|_| pkt.packet_id());
9685 self.queue_end_to_end_reliable_ack_for_packet_id(
9686 packet_id,
9687 called_from_queue,
9688 )?;
9689 }
9690 }
9691 if self.should_route_remote(&item.data, item.src)? {
9692 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
9693 }
9694 return Ok(());
9695 }
9696
9697 #[cfg(feature = "timesync")]
9698 if matches!(
9699 env.ty,
9700 DataType::TimeSyncAnnounce
9701 | DataType::TimeSyncRequest
9702 | DataType::TimeSyncResponse
9703 ) {
9704 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9705 pkt.validate()?;
9706 self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
9707 return Ok(());
9708 }
9709
9710 #[cfg(feature = "discovery")]
9711 if discovery::is_discovery_type(env.ty) {
9712 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9713 pkt.validate()?;
9714 let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
9715 if self.should_route_remote(&item.data, item.src)? {
9716 self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
9717 }
9718 return Ok(());
9719 }
9720
9721 let any_packet_needed = env
9722 .endpoints
9723 .iter()
9724 .copied()
9725 .any(|ep| self.endpoint_has_packet_handler(ep));
9726
9727 let mut pkt_opt = if any_packet_needed {
9728 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9729 pkt.validate()?;
9730 Some(pkt)
9731 } else {
9732 None
9733 };
9734
9735 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
9736 eps.sort_unstable();
9737 eps.dedup();
9738 let had_local_handler = eps.iter().copied().any(|ep| {
9739 self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
9740 });
9741
9742 let has_remote = self.should_route_remote(&item.data, item.src)?;
9743 let targets_local = self.item_targets_local_sender(&item.data)?;
9744
9745 if targets_local {
9746 for dest in eps {
9747 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
9748 let result = match &h.handler {
9749 EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
9750 dest,
9751 h,
9752 Some(bytes.as_ref()),
9753 pkt_opt.as_ref(),
9754 Some(&env),
9755 called_from_queue,
9756 ),
9757 EndpointHandlerFn::Packet(_) => {
9758 if pkt_opt.is_none() {
9759 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9760 pkt.validate()?;
9761 pkt_opt = Some(pkt);
9762 }
9763 let pkt_ref = pkt_opt.as_ref().expect("just set");
9764 self.call_handler_with_retries(
9765 dest,
9766 h,
9767 None,
9768 Some(pkt_ref),
9769 Some(&env),
9770 called_from_queue,
9771 )
9772 }
9773 };
9774 if result.is_err()
9775 && let Some(src) = item.src
9776 {
9777 self.note_side_local_handler_failure(
9778 src,
9779 env.ty,
9780 runtime_max_handler_retries(),
9781 );
9782 }
9783 }
9784 }
9785 }
9786
9787 if item.src.is_some()
9788 && had_local_handler
9789 && targets_local
9790 && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
9791 && let Some(pkt) = pkt_opt.as_ref()
9792 {
9793 let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
9794 .unwrap_or_else(|_| pkt.packet_id());
9795 self.queue_end_to_end_reliable_ack_for_packet_id(packet_id, called_from_queue)?;
9796 }
9797
9798 if has_remote {
9799 let relay_item = match pkt_opt {
9800 Some(ref p) => RouterItem::Packet(p.clone()),
9801 None => RouterItem::Packed(bytes.clone()),
9802 };
9803 self.relay_send(relay_item, item.src, called_from_queue)?;
9804 }
9805
9806 Ok(())
9807 }
9808 }
9809 }
9810
9811 fn dispatch_local_for_item(
9812 &self,
9813 item: &RouterItem,
9814 called_from_queue: bool,
9815 ) -> TelemetryResult<()> {
9816 match item {
9817 RouterItem::Packet(pkt) => {
9818 pkt.validate()?;
9819 if is_internal_control_type(pkt.data_type()) {
9820 return Ok(());
9821 }
9822 self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
9823 if !self.item_targets_local_sender(item)? {
9824 return Ok(());
9825 }
9826
9827 let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
9828 eps.sort_unstable();
9829 eps.dedup();
9830
9831 let has_packed_local = eps
9832 .iter()
9833 .copied()
9834 .any(|ep| self.endpoint_has_packed_handler(ep));
9835 let bytes_opt = if has_packed_local {
9836 Some(wire_format::pack_packet(pkt))
9837 } else {
9838 None
9839 };
9840
9841 for dest in eps {
9842 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
9843 match (&h.handler, &bytes_opt) {
9844 (EndpointHandlerFn::Packed(_), Some(bytes)) => {
9845 self.call_handler_with_retries(
9846 dest,
9847 h,
9848 Some(bytes.as_ref()),
9849 Some(pkt),
9850 None,
9851 called_from_queue,
9852 )?;
9853 }
9854 (EndpointHandlerFn::Packed(_), None) => {
9855 let bytes = wire_format::pack_packet(pkt);
9856 self.call_handler_with_retries(
9857 dest,
9858 h,
9859 Some(bytes.as_ref()),
9860 Some(pkt),
9861 None,
9862 called_from_queue,
9863 )?;
9864 }
9865 (EndpointHandlerFn::Packet(_), _) => {
9866 self.call_handler_with_retries(
9867 dest,
9868 h,
9869 None,
9870 Some(pkt),
9871 None,
9872 called_from_queue,
9873 )?;
9874 }
9875 }
9876 }
9877 }
9878 }
9879 RouterItem::Packed(bytes) => {
9880 let env = wire_format::peek_envelope(bytes.as_ref())?;
9881 if is_internal_control_type(env.ty) {
9882 return Ok(());
9883 }
9884 self.ensure_e2e_policy_supported_for_type(env.ty)?;
9885 if !self.item_targets_local_sender(item)? {
9886 return Ok(());
9887 }
9888
9889 let any_packet_needed = env
9890 .endpoints
9891 .iter()
9892 .copied()
9893 .any(|ep| self.endpoint_has_packet_handler(ep));
9894
9895 let mut pkt_opt = if any_packet_needed {
9896 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9897 pkt.validate()?;
9898 Some(pkt)
9899 } else {
9900 None
9901 };
9902
9903 let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
9904 eps.sort_unstable();
9905 eps.dedup();
9906
9907 for dest in eps {
9908 for h in self.cfg.handlers.iter().filter(|h| h.endpoint == dest) {
9909 match &h.handler {
9910 EndpointHandlerFn::Packed(_) => {
9911 self.call_handler_with_retries(
9912 dest,
9913 h,
9914 Some(bytes.as_ref()),
9915 pkt_opt.as_ref(),
9916 Some(&env),
9917 called_from_queue,
9918 )?;
9919 }
9920 EndpointHandlerFn::Packet(_) => {
9921 if pkt_opt.is_none() {
9922 let pkt = wire_format::unpack_packet(bytes.as_ref())?;
9923 pkt.validate()?;
9924 pkt_opt = Some(pkt);
9925 }
9926 let pkt_ref = pkt_opt.as_ref().expect("just set");
9927 self.call_handler_with_retries(
9928 dest,
9929 h,
9930 None,
9931 Some(pkt_ref),
9932 Some(&env),
9933 called_from_queue,
9934 )?;
9935 }
9936 }
9937 }
9938 }
9939 }
9940 }
9941
9942 Ok(())
9943 }
9944
9945 fn tx_item_impl(
9951 &self,
9952 item: RouterTxItem,
9953 ignore_local: bool,
9954 called_from_queue: bool,
9955 ) -> TelemetryResult<()> {
9956 match item {
9957 RouterTxItem::Broadcast(data) => {
9958 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
9959 if let RouterItem::Packet(pkt) = &data
9960 && !is_internal_control_type(pkt.data_type())
9961 {
9962 self.remember_managed_variable_packet(pkt)?;
9963 }
9964 #[cfg(feature = "discovery")]
9965 let is_discovery = matches!(&data, RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()))
9966 || matches!(&data, RouterItem::Packed(bytes)
9967 if wire_format::peek_envelope(bytes.as_ref())
9968 .map(|env| discovery::is_discovery_type(env.ty))
9969 .unwrap_or(false));
9970 if !ignore_local {
9971 if self.is_duplicate_pkt(&data)? {
9972 return Ok(());
9973 }
9974 #[cfg(feature = "discovery")]
9975 if !is_discovery
9976 && !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
9977 && !matches!(&data, RouterItem::Packed(bytes)
9978 if wire_format::peek_envelope(bytes.as_ref())
9979 .map(|env| is_internal_control_type(env.ty))
9980 .unwrap_or(false))
9981 {
9982 self.dispatch_local_for_item(&data, called_from_queue)?;
9983 }
9984 #[cfg(not(feature = "discovery"))]
9985 if !matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
9986 && !matches!(&data, RouterItem::Packed(bytes)
9987 if wire_format::peek_envelope(bytes.as_ref())
9988 .map(|env| is_internal_control_type(env.ty))
9989 .unwrap_or(false))
9990 {
9991 self.dispatch_local_for_item(&data, called_from_queue)?;
9992 }
9993 }
9994
9995 let send_remote = match &data {
9996 RouterItem::Packet(pkt) => {
9997 pkt.validate()?;
9998 self.should_route_remote(&data, None)?
9999 }
10000 RouterItem::Packed(bytes) => {
10001 let _ = wire_format::peek_envelope(bytes.as_ref())?;
10002 self.should_route_remote(&data, None)?
10003 }
10004 };
10005
10006 if !send_remote {
10007 return Ok(());
10008 }
10009 let mut data = data;
10010 let ty = match &data {
10011 RouterItem::Packet(pkt) => pkt.data_type(),
10012 RouterItem::Packed(bytes) => wire_format::peek_envelope(bytes.as_ref())?.ty,
10013 };
10014 if !ignore_local && !is_internal_control_type(ty) {
10015 #[cfg(feature = "discovery")]
10016 {
10017 let pending = {
10018 let st = self.state.lock();
10019 let mut pending =
10020 self.expected_end_to_end_destinations_locked(&st, &data)?;
10021 self.filter_trackable_end_to_end_destinations_locked(
10022 &st,
10023 ty,
10024 &mut pending,
10025 );
10026 pending
10027 };
10028 if !pending.is_empty() {
10029 let mut targets: Vec<u64> = pending.keys().copied().collect();
10030 targets.sort_unstable();
10031 targets.dedup();
10032 data = self.attach_wire_contract_to_item(data, &targets)?;
10033 self.register_end_to_end_reliable_tx(&data)?;
10034 }
10035 }
10036 }
10037 let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, None)?;
10038 for (idx, side) in sides.iter().copied().enumerate() {
10039 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
10040 if Self::is_side_tx_busy(&e) {
10041 for retry_side in sides[idx..].iter().copied() {
10042 self.tx_queue_item_with_flags(
10043 RouterTxItem::ToSide {
10044 src: None,
10045 dst: retry_side,
10046 data: data.clone(),
10047 },
10048 true,
10049 )?;
10050 }
10051 return Ok(());
10052 }
10053 match &data {
10054 RouterItem::Packet(pkt) => {
10055 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
10056 }
10057 RouterItem::Packed(bytes) => {
10058 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
10059 let _ = self.handle_callback_error_from_env(
10060 &env,
10061 None,
10062 e,
10063 called_from_queue,
10064 );
10065 }
10066 }
10067 }
10068 return Err(TelemetryError::HandlerError("tx handler failed"));
10069 }
10070 }
10071 }
10072 RouterTxItem::ToSide { src, dst, data } => {
10073 self.ensure_e2e_policy_supported_for_type(Self::item_data_type(&data)?)?;
10074 if let RouterItem::Packet(pkt) = &data
10075 && !is_internal_control_type(pkt.data_type())
10076 {
10077 self.remember_managed_variable_packet(pkt)?;
10078 }
10079 if !ignore_local {
10080 if self.is_duplicate_pkt(&data)? {
10081 return Ok(());
10082 }
10083 let suppress_local = matches!(&data, RouterItem::Packet(pkt) if is_internal_control_type(pkt.data_type()))
10084 || matches!(&data, RouterItem::Packed(bytes)
10085 if wire_format::peek_envelope(bytes.as_ref())
10086 .map(|env| is_internal_control_type(env.ty))
10087 .unwrap_or(false));
10088 if !suppress_local {
10089 self.dispatch_local_for_item(&data, called_from_queue)?;
10090 }
10091 }
10092 let allowed = {
10093 let mut st = self.state.lock();
10094 let ty = match &data {
10095 RouterItem::Packet(pkt) => Some(pkt.data_type()),
10096 RouterItem::Packed(bytes) => {
10097 Some(wire_format::peek_envelope(bytes.as_ref())?.ty)
10098 }
10099 };
10100 let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
10101 #[cfg(all(feature = "discovery", feature = "timesync"))]
10102 let timesync_allowed = ty
10103 .map(|ty| {
10104 Self::timesync_allowed_for_side_locked(
10105 &mut st,
10106 dst,
10107 ty,
10108 self.clock.now_ms(),
10109 )
10110 })
10111 .unwrap_or(true);
10112 #[cfg(not(all(feature = "discovery", feature = "timesync")))]
10113 let timesync_allowed = true;
10114 route_allowed && timesync_allowed
10115 };
10116 if !allowed {
10117 return Ok(());
10118 }
10119 if let Err(e) = self.send_reliable_to_side(dst, data.clone(), src.is_some()) {
10120 if Self::is_side_tx_busy(&e) {
10121 self.tx_queue_item_with_flags(
10122 RouterTxItem::ToSide { src, dst, data },
10123 true,
10124 )?;
10125 return Ok(());
10126 }
10127 match &data {
10128 RouterItem::Packet(pkt) => {
10129 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
10130 }
10131 RouterItem::Packed(bytes) => {
10132 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
10133 let _ = self.handle_callback_error_from_env(
10134 &env,
10135 None,
10136 e,
10137 called_from_queue,
10138 );
10139 }
10140 }
10141 }
10142 return Err(TelemetryError::HandlerError("tx handler failed"));
10143 }
10144 }
10145 RouterTxItem::EndToEndReplay { packet_id } => {
10146 let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
10147 return Ok(());
10148 };
10149 if sides.is_empty() {
10150 let RemoteSidePlan::Target(fallback_sides) =
10151 self.remote_side_plan(&data, None)?;
10152 sides = fallback_sides;
10153 }
10154 for (idx, side) in sides.iter().copied().enumerate() {
10155 if let Err(e) = self.send_reliable_to_side(side, data.clone(), false) {
10156 if Self::is_side_tx_busy(&e) {
10157 for retry_side in sides[idx..].iter().copied() {
10158 self.tx_queue_item_with_flags(
10159 RouterTxItem::ToSide {
10160 src: None,
10161 dst: retry_side,
10162 data: data.clone(),
10163 },
10164 true,
10165 )?;
10166 }
10167 return Ok(());
10168 }
10169 match &data {
10170 RouterItem::Packet(pkt) => {
10171 let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
10172 }
10173 RouterItem::Packed(bytes) => {
10174 if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
10175 let _ = self.handle_callback_error_from_env(
10176 &env,
10177 None,
10178 e,
10179 called_from_queue,
10180 );
10181 }
10182 }
10183 }
10184 return Err(TelemetryError::HandlerError("tx handler failed"));
10185 }
10186 }
10187 }
10188 RouterTxItem::ReliableReplay { dst, bytes } => {
10189 let frame = wire_format::peek_frame_info(bytes.as_ref())?;
10190 let ty = frame.envelope.ty;
10191 let Some(hdr) = frame.reliable else {
10192 return Ok(());
10193 };
10194 {
10195 let mut st = self.state.lock();
10196 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
10197 if !tx_state.sent.contains_key(&hdr.seq) {
10198 return Ok(());
10199 }
10200 }
10201 if let Err(e) = self.send_reliable_raw_to_side(dst, bytes.clone(), false) {
10202 if Self::is_side_tx_busy(&e) {
10203 self.tx_queue_item_with_flags(
10204 RouterTxItem::ReliableReplay { dst, bytes },
10205 true,
10206 )?;
10207 return Ok(());
10208 }
10209 return Err(e);
10210 }
10211 let mut st = self.state.lock();
10212 let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
10213 if let Some(sent) = tx_state.sent.get_mut(&hdr.seq) {
10214 sent.last_send_ms = self.clock.now_ms();
10215 sent.queued = false;
10216 }
10217 }
10218 }
10219
10220 Ok(())
10221 }
10222
10223 #[inline]
10225 fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
10226 self.tx_item_impl(item, false, false)
10227 }
10228
10229 #[inline]
10236 pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
10237 if self.side_tx_active() {
10238 return self.rx_packed_queue(bytes);
10239 }
10240 let data = RouterItem::Packed(Arc::from(bytes));
10241 let item = RouterRxItem {
10242 src: None,
10243 priority: Self::router_item_priority(&data)?,
10244 data,
10245 };
10246 self.rx_item(&item, false)
10247 }
10248
10249 #[inline]
10254 pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
10255 if self.side_tx_active() {
10256 return self.rx_queue(pkt.clone());
10257 }
10258 let data = RouterItem::Packet(pkt.clone());
10259 let item = RouterRxItem {
10260 src: None,
10261 priority: Self::router_item_priority(&data)?,
10262 data,
10263 };
10264 self.rx_item(&item, false)
10265 }
10266
10267 #[inline]
10272 pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
10273 if self.side_tx_active() {
10274 return self.rx_queue_from_side(pkt.clone(), side);
10275 }
10276 self.ensure_side_ingress_enabled(side)?;
10277 let data = RouterItem::Packet(pkt.clone());
10278 let item = RouterRxItem {
10279 src: Some(side),
10280 priority: Self::router_item_priority(&data)?,
10281 data,
10282 };
10283 self.rx_item(&item, false)
10284 }
10285
10286 #[inline]
10291 pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
10292 if self.side_tx_active() {
10293 return self.rx_packed_queue_from_side(bytes, side);
10294 }
10295 self.ensure_side_ingress_enabled(side)?;
10296 let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
10297 return Ok(());
10298 };
10299 let data = RouterItem::Packed(decoded);
10300 let item = RouterRxItem {
10301 src: Some(side),
10302 priority: Self::router_item_priority(&data)?,
10303 data,
10304 };
10305 self.rx_item(&item, false)
10306 }
10307
10308 #[inline]
10315 pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
10316 #[cfg(feature = "discovery")]
10317 let _ = self.poll_discovery()?;
10318 if self.side_tx_active() {
10319 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
10320 }
10321 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
10322 }
10323
10324 #[inline]
10329 pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
10330 #[cfg(feature = "discovery")]
10331 let _ = self.poll_discovery()?;
10332 if self.side_tx_active() {
10333 return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(pkt)));
10334 }
10335 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(pkt)))
10336 }
10337
10338 #[inline]
10342 pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
10343 #[cfg(feature = "discovery")]
10344 let _ = self.poll_discovery()?;
10345 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
10346 }
10347
10348 #[inline]
10350 pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
10351 #[cfg(feature = "discovery")]
10352 let _ = self.poll_discovery()?;
10353 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(data)))
10354 }
10355
10356 #[inline]
10363 pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
10364 #[cfg(feature = "discovery")]
10365 let _ = self.poll_discovery()?;
10366 if self.side_tx_active() {
10367 return self.log_queue(ty, data);
10368 }
10369 let sender = self.sender_arc();
10370 log_raw(
10371 sender.as_ref(),
10372 ty,
10373 data,
10374 self.packet_timestamp_ms(),
10375 |pkt| self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
10376 )
10377 }
10378
10379 #[inline]
10381 pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
10382 #[cfg(feature = "discovery")]
10383 let _ = self.poll_discovery()?;
10384 let sender = self.sender_arc();
10385 log_raw(
10386 sender.as_ref(),
10387 ty,
10388 data,
10389 self.packet_timestamp_ms(),
10390 |pkt| self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt))),
10391 )
10392 }
10393
10394 #[inline]
10396 pub fn log_ts<T: LeBytes>(
10397 &self,
10398 ty: DataType,
10399 timestamp: u64,
10400 data: &[T],
10401 ) -> TelemetryResult<()> {
10402 #[cfg(feature = "discovery")]
10403 let _ = self.poll_discovery()?;
10404 if self.side_tx_active() {
10405 return self.log_queue_ts(ty, timestamp, data);
10406 }
10407 let sender = self.sender_arc();
10408 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
10409 self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
10410 })
10411 }
10412
10413 #[inline]
10415 pub fn log_queue_ts<T: LeBytes>(
10416 &self,
10417 ty: DataType,
10418 timestamp: u64,
10419 data: &[T],
10420 ) -> TelemetryResult<()> {
10421 #[cfg(feature = "discovery")]
10422 let _ = self.poll_discovery()?;
10423 let sender = self.sender_arc();
10424 log_raw(sender.as_ref(), ty, data, timestamp, |pkt| {
10425 self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
10426 })
10427 }
10428}