Skip to main content

sedsnet/
router.rs

1//! Telemetry Router
2//!
3//! Router with internal named sides (like Relay), plus local processing.
4//!
5//! Design:
6//! - Sides are registered with per-side TX handlers (packed or packet).
7//! - RX is tagged by source side; route rules decide whether it is forwarded.
8//! - Local endpoint handlers process packets as before (no side parameter).
9//! - De-duplication remains packet-id based and side-agnostic.
10
11#[cfg(all(test, feature = "std", feature = "discovery"))]
12#[path = "tests/scoped_schema_recovery_audit.rs"]
13mod scoped_schema_recovery_audit;
14
15#[cfg(test)]
16#[path = "tests/relay_legacy_contracts.rs"]
17mod relay_legacy_contracts;
18
19#[cfg(all(test, feature = "std", feature = "discovery"))]
20#[path = "tests/shared_routing_roles.rs"]
21mod shared_routing_roles;
22
23#[cfg(all(test, feature = "std", feature = "discovery"))]
24#[path = "tests/schema_prefix_admission.rs"]
25mod schema_prefix_admission;
26
27#[cfg(all(test, feature = "std", feature = "discovery"))]
28#[path = "tests/schema_receive_progress.rs"]
29mod schema_receive_progress;
30
31#[cfg(all(test, feature = "std", feature = "discovery"))]
32#[path = "tests/radio_trace_decoder.rs"]
33mod radio_trace_decoder;
34
35#[cfg(all(test, feature = "std", feature = "discovery"))]
36#[path = "tests/schema_response_admission.rs"]
37mod schema_response_admission_tests;
38
39#[cfg(all(test, feature = "std", feature = "discovery"))]
40#[path = "tests/accepted_route_ownership.rs"]
41mod accepted_route_ownership;
42
43#[cfg(all(test, feature = "std"))]
44#[path = "tests/independent_side_tx_router.rs"]
45mod independent_side_tx;
46#[cfg(all(test, feature = "std", feature = "discovery"))]
47#[path = "tests/ingress_target_filter.rs"]
48mod ingress_target_filter_tests;
49#[cfg(all(test, feature = "std", feature = "discovery", feature = "compression"))]
50#[path = "tests/payload_codec_negotiation.rs"]
51mod payload_codec_negotiation;
52#[cfg(all(test, feature = "std", feature = "discovery"))]
53#[path = "tests/reliable_tx_window_backpressure.rs"]
54mod reliable_tx_window_backpressure_tests;
55#[cfg(all(test, feature = "std", feature = "discovery"))]
56#[path = "tests/shared_chunk_budget_router.rs"]
57mod shared_chunk_budget_tests;
58// Apply the same cache contract independently to router and relay state.
59#[cfg(test)]
60#[allow(clippy::duplicate_mod)]
61#[path = "tests/side_template_index.rs"]
62mod side_template_index_tests;
63#[cfg(feature = "discovery")]
64use crate::config::runtime_reliable_max_end_to_end_pending;
65#[cfg(all(test, feature = "std", feature = "discovery"))]
66#[path = "tests/replay_ownership_router.rs"]
67mod replay_ownership_tests;
68use crate::side_transport::{
69    SIDE_TRANSPORT_FLAG_PACKET_NONCE, SIDE_TRANSPORT_KIND_CHUNK, SIDE_TRANSPORT_KIND_COMPACT,
70    SIDE_TRANSPORT_KIND_COMPACT_DELTA, SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP,
71    SIDE_TRANSPORT_KIND_FULL, SideChunkAssembly, SideCompactTimestampMode, SideHeaderTemplate,
72    SideTransportFrames, extract_side_header_template, extract_side_header_template_view,
73    parse_side_transport_wrapper, read_uleb128_local, reconstruct_side_compact_frame,
74    wrap_side_transport_frame, write_uleb128_local,
75};
76#[cfg(all(test, feature = "discovery"))]
77#[path = "tests/ack_return_identity.rs"]
78mod ack_return_identity;
79#[cfg(all(test, feature = "discovery"))]
80#[path = "tests/ack_split_horizon.rs"]
81mod ack_split_horizon;
82
83#[cfg(all(test, feature = "discovery"))]
84#[path = "tests/discovery_liveness.rs"]
85mod discovery_liveness;
86
87#[cfg(all(test, feature = "discovery"))]
88mod compact_summary_tests {
89    use super::*;
90
91    #[test]
92    fn ordered_pending_window_is_bounded_and_exhaustion_cancels_the_tail() {
93        crate::tests::ensure_common_test_schema();
94        let ep = DataEndpoint::named("RADIO");
95        let ty = crate::config::register_data_type_with_description(
96            "ORDERED_WINDOW_REGRESSION",
97            "bounded ordered window",
98            MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
99            &[ep],
100            crate::ReliableMode::Ordered,
101            200,
102        )
103        .unwrap();
104        let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
105        let clock = now.clone();
106        let router = Router::new_with_clock(
107            RouterConfig::new([]),
108            Box::new(move || clock.load(Ordering::Relaxed)),
109        );
110        let capacity = runtime_reliable_max_end_to_end_pending().max(1);
111        for index in 0..capacity {
112            let item = RouterItem::Packet(
113                Packet::new(
114                    ty,
115                    &[ep],
116                    "SOURCE",
117                    index as u64 + 1,
118                    Arc::<[u8]>::from([index as u8]),
119                )
120                .unwrap(),
121            );
122            assert_eq!(
123                router.register_end_to_end_reliable_tx(&item, None).unwrap(),
124                index == 0
125            );
126        }
127        let rejected = RouterItem::Packet(
128            Packet::new(
129                ty,
130                &[ep],
131                "SOURCE",
132                capacity as u64 + 1,
133                Arc::<[u8]>::from([0]),
134            )
135            .unwrap(),
136        );
137        assert!(
138            router
139                .register_end_to_end_reliable_tx(&rejected, None)
140                .is_err()
141        );
142        {
143            let mut state = router.state.lock();
144            assert_eq!(state.end_to_end_reliable_tx.len(), capacity);
145            let first = state.end_to_end_reliable_tx_order[0];
146            state
147                .end_to_end_reliable_tx
148                .get_mut(&first)
149                .unwrap()
150                .retries = runtime_reliable_max_retries();
151        }
152        now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
153        assert!(router.process_end_to_end_reliable_timeouts().is_err());
154        assert!(router.state.lock().end_to_end_reliable_tx.is_empty());
155    }
156    #[test]
157    fn compact_reachability_survives_incomplete_detailed_topology() {
158        crate::tests::ensure_common_test_schema();
159        let router = Router::new(RouterConfig::new([]).with_sender("GB"));
160        let endpoint = DataEndpoint::try_named("GROUND_STATION")
161            .or_else(|| DataEndpoint::try_named("RADIO"))
162            .unwrap();
163        router.compact_summary_fixture(endpoint);
164        let state = router.state.lock();
165        let now = router.clock.now_ms();
166        let advertised =
167            router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(1));
168        assert!(
169            advertised.contains(&endpoint),
170            "compact subscriber route was lost at the bridge"
171        );
172        let reflected =
173            router.advertised_discovery_endpoints_for_link_locked(&state, now, true, Some(0));
174        assert!(
175            !reflected.contains(&endpoint),
176            "split horizon must still exclude ingress"
177        );
178        assert_eq!(
179            router.advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(1)),
180            vec!["clock-behind-bridge".to_string()]
181        );
182        assert!(
183            router
184                .advertised_discovery_timesync_sources_for_link_locked(&state, now, Some(0))
185                .is_empty()
186        );
187        assert!(
188            !router
189                .advertised_discovery_endpoints_for_link_locked(
190                    &state,
191                    now + DISCOVERY_ROUTE_TTL_MS + 1,
192                    true,
193                    Some(1)
194                )
195                .contains(&endpoint)
196        );
197        assert!(
198            router
199                .advertised_discovery_timesync_sources_for_link_locked(
200                    &state,
201                    now + DISCOVERY_ROUTE_TTL_MS + 1,
202                    Some(1)
203                )
204                .is_empty()
205        );
206    }
207}
208
209use crate::config::RuntimeMemoryConfig;
210use crate::diagnostics::{
211    AdaptiveLinkStats, DiscoveryRuntimeStats, QueueRuntimeStats, ReliableRuntimeStats,
212    RouteModeStats, RouteOverrideStats, RoutePriorityStats, RouteWeightStats, RuntimeSideStats,
213    RuntimeStatsSnapshot, RuntimeTypeStats, TypedRouteOverrideStats,
214};
215#[cfg(all(feature = "discovery", feature = "timesync"))]
216use crate::discovery::TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS;
217#[cfg(feature = "discovery")]
218use crate::discovery::{
219    self, ClientStatsSnapshot, DISCOVERY_INCREMENTAL_RETRY_COUNT, DISCOVERY_ROUTE_TTL_MS,
220    DISCOVERY_SLOW_LINK_PING_INTERVAL_MS, DiscoveryCadenceState, TopologyAnnouncerRoute,
221    TopologyBoardNode, TopologySideRoute, TopologySnapshot,
222};
223use crate::packet::{hash_bytes_u64, sender_address_u32};
224use crate::queue::{BoundedDeque, ByteCost};
225#[cfg(all(not(feature = "std"), target_os = "none"))]
226use crate::seds_error_msg;
227#[cfg(feature = "timesync")]
228use crate::timesync::{
229    INTERNAL_TIMESYNC_SOURCE_ID, LOCAL_TIMESYNC_DATE_SOURCE_ID, LOCAL_TIMESYNC_FULL_SOURCE_ID,
230    LOCAL_TIMESYNC_SUBSEC_SOURCE_ID, LOCAL_TIMESYNC_TOD_SOURCE_ID, NetworkClock,
231    NetworkTimeReading, PartialNetworkTime, SlewedNetworkClock, TimeSyncConfig, TimeSyncLeader,
232    TimeSyncTracker, advance_network_time, compute_network_time_sample, decode_timesync_announce,
233    decode_timesync_request, decode_timesync_response,
234};
235use crate::{
236    E2eEncryptionPolicy, MessageElement, RouteSelectionMode, TelemetryError, TelemetryResult,
237    config::{
238        DataEndpoint, DataType, runtime_device_identifier, runtime_max_handler_retries,
239        runtime_reliable_max_pending, runtime_reliable_max_retries,
240        runtime_reliable_max_return_routes, runtime_reliable_retransmit_ms,
241    },
242    get_needed_message_size, impl_letype_num, is_reliable_type,
243    lock::{ReentryGate, RouterMutex},
244    message_e2e_encryption_policy, message_meta, message_priority,
245    packet::Packet,
246    reliable_mode, wire_format,
247};
248use alloc::string::{String, ToString};
249
250#[cfg(any(test, all(feature = "embedded", not(feature = "std"))))]
251use crate::small_map::SmallMap as RouterMap;
252#[cfg(not(any(test, all(feature = "embedded", not(feature = "std")))))]
253use alloc::collections::BTreeMap as RouterMap;
254
255use alloc::collections::BTreeMap;
256
257use alloc::{
258    borrow::ToOwned,
259    boxed::Box,
260    collections::{BTreeSet, VecDeque},
261    format,
262    sync::Arc,
263    vec,
264    vec::Vec,
265};
266use core::cell::UnsafeCell;
267use core::fmt;
268use core::fmt::{Debug, Formatter};
269use core::mem::size_of;
270use core::ops::{Deref, DerefMut};
271use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
272#[cfg(feature = "std")]
273use std::time::Instant;
274
275/// Logical side index (CAN, UART, RADIO, etc.)
276pub type RouterSideId = usize;
277
278const CONTROL_SLOW_LINK_CAPACITY_BPS: u64 = 512;
279pub const IPV4_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 20;
280pub const IPV6_LIKE_COMPACT_HEADER_TARGET_BYTES: usize = 40;
281pub const DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT: usize = 64;
282static ROUTER_INSTANCE_SEQ: AtomicU32 = AtomicU32::new(1);
283
284#[derive(Clone, Copy, Debug, PartialEq, Eq)]
285pub enum SideTransportProfile {
286    Canonical,
287    Template,
288    Ipv6Like,
289    Ipv4Like,
290}
291
292impl SideTransportProfile {
293    #[inline]
294    pub const fn as_str(self) -> &'static str {
295        match self {
296            Self::Canonical => "canonical",
297            Self::Template => "template",
298            Self::Ipv6Like => "ipv6_like",
299            Self::Ipv4Like => "ipv4_like",
300        }
301    }
302
303    #[cfg(feature = "discovery")]
304    #[inline]
305    pub const fn discovery_code(self) -> u8 {
306        match self {
307            Self::Canonical => discovery::LINK_PROFILE_CANONICAL,
308            Self::Template => discovery::LINK_PROFILE_TEMPLATE,
309            Self::Ipv6Like => discovery::LINK_PROFILE_IPV6_LIKE,
310            Self::Ipv4Like => discovery::LINK_PROFILE_IPV4_LIKE,
311        }
312    }
313}
314
315#[cfg(test)]
316mod chunk_assembly_memory_tests {
317    use super::*;
318    #[test]
319    fn full_chunk_budget_reclaims_other_partial_transfers_before_active_schema() {
320        let mut state = SideTransportState::default();
321        for id in 1..=4 {
322            let mut assembly = SideChunkAssembly {
323                total: 28,
324                last_seen_ms: id as u64,
325                ..Default::default()
326            };
327            for index in 0..23 {
328                assembly.insert(
329                    index,
330                    crate::shared_bytes::convert(Arc::<[u8]>::from([id as u8; 112])),
331                );
332            }
333            state.rx_chunks.insert(id, assembly);
334        }
335        state.rx_chunks.get_mut(&1).unwrap().insert(
336            23,
337            crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 56])),
338        );
339        // 16192 + (56 + 64) = 16312 bytes, as in the Gateway capture.
340        state.refresh_rx_chunk_bytes();
341        SideChunkAssembly::make_room(
342            &mut state.rx_chunks,
343            &mut state.rx_chunk_bytes,
344            4,
345            23,
346            28,
347            112,
348            16384,
349        )
350        .unwrap();
351        assert!(!state.rx_chunks.contains_key(&1));
352        assert_eq!(state.rx_chunks[&4].received_len(), 23);
353        for index in 23..28 {
354            state.refresh_rx_chunk_bytes();
355            SideChunkAssembly::make_room(
356                &mut state.rx_chunks,
357                &mut state.rx_chunk_bytes,
358                4,
359                index,
360                28,
361                112,
362                16384,
363            )
364            .unwrap();
365            state.rx_chunks.get_mut(&4).unwrap().insert(
366                index,
367                crate::shared_bytes::convert(Arc::<[u8]>::from([4u8; 112])),
368            );
369        }
370        assert_eq!(
371            state.rx_chunks.remove(&4).unwrap().assemble().unwrap(),
372            vec![4u8; 28 * 112]
373        );
374        assert!(
375            state
376                .rx_chunks
377                .values()
378                .flat_map(|a| a.received.iter().flatten())
379                .map(|p| p.len() + 64)
380                .sum::<usize>()
381                <= 16384
382        );
383    }
384
385    #[test]
386    fn duplicate_chunk_keeps_partial_transfers_but_oversize_transfer_is_rejected() {
387        let mut state = SideTransportState::default();
388        let mut assembly = SideChunkAssembly {
389            total: 2,
390            ..Default::default()
391        };
392        assembly.insert(
393            0,
394            crate::shared_bytes::convert(Arc::<[u8]>::from([1u8; 112])),
395        );
396        state.rx_chunks.insert(1, assembly.clone());
397        state.rx_chunks.insert(2, assembly);
398        state.refresh_rx_chunk_bytes();
399        SideChunkAssembly::make_room(
400            &mut state.rx_chunks,
401            &mut state.rx_chunk_bytes,
402            1,
403            0,
404            2,
405            112,
406            352,
407        )
408        .unwrap();
409        assert_eq!(state.rx_chunks.len(), 2);
410        state.refresh_rx_chunk_bytes();
411        assert!(
412            SideChunkAssembly::make_room(
413                &mut state.rx_chunks,
414                &mut state.rx_chunk_bytes,
415                1,
416                1,
417                2,
418                400,
419                352
420            )
421            .is_err()
422        );
423        assert!(!state.rx_chunks.contains_key(&1));
424        assert!(state.rx_chunks.contains_key(&2));
425    }
426
427    #[test]
428    fn schema_sized_reassembly_allocates_exactly_and_preserves_order() {
429        let payload: Vec<u8> = (0..3652).map(|i| (i % 251) as u8).collect();
430        let mut assembly = SideChunkAssembly {
431            total: payload.len().div_ceil(56) as u16,
432            ..Default::default()
433        };
434        for (index, chunk) in payload.chunks(56).enumerate().rev() {
435            assembly.insert(
436                index as u16,
437                crate::shared_bytes::convert(Arc::<[u8]>::from(chunk)),
438            );
439        }
440        assert_eq!(assembly.received_len(), usize::from(assembly.total));
441        let out = assembly.assemble().unwrap();
442        assert_eq!(out, payload);
443        assert_eq!(
444            out.capacity(),
445            payload.len(),
446            "do not double discovery reassembly capacity"
447        );
448    }
449    #[test]
450    fn incomplete_or_noncontiguous_chunks_are_rejected() {
451        let mut assembly = SideChunkAssembly {
452            total: 2,
453            ..Default::default()
454        };
455        assembly.insert(0, crate::shared_bytes::convert(Arc::<[u8]>::from([1u8])));
456        assert!(assembly.clone().assemble().is_err());
457        assert!(
458            assembly
459                .insert_with(2, 0, || Arc::<[u8]>::from([2u8]))
460                .is_err()
461        );
462        assert!(assembly.assemble().is_err());
463    }
464}
465
466#[derive(Clone, Debug, Default)]
467struct SideTransportState {
468    #[cfg(feature = "std")]
469    tx_callback_gate: Arc<ReentryGate>,
470    tx_chunk_progress: Option<Box<crate::side_transport::SideTxProgress>>,
471    #[cfg(all(feature = "compression", feature = "discovery"))]
472    payload_codec: crate::payload_codec::Ledger,
473    #[cfg(all(feature = "compression", feature = "discovery"))]
474    payload_codec_choice: u64,
475    tx_template_ids: RouterMap<u64, u32>,
476    tx_templates: BTreeMap<u64, SideHeaderTemplate>,
477    tx_last_timestamps: RouterMap<u32, u64>,
478    tx_compact_uses: RouterMap<u32, u8>,
479    rx_templates: RouterMap<u64, u32>,
480    rx_templates_by_id: BTreeMap<u32, SideHeaderTemplate>,
481    rx_last_timestamps: RouterMap<u32, u64>,
482    rx_chunks: BTreeMap<u32, SideChunkAssembly>,
483    rx_chunk_reclaims: u32,
484    rx_chunk_bytes: usize,
485}
486
487impl SideTransportState {
488    #[inline(never)]
489    fn take_chunk(&mut self, id: u32, reclaimed: bool) -> Option<SideChunkAssembly> {
490        let value = SideChunkAssembly::remove(&mut self.rx_chunks, &mut self.rx_chunk_bytes, id)?;
491        if reclaimed {
492            self.rx_chunk_reclaims = self.rx_chunk_reclaims.saturating_add(1);
493        }
494        Some(value)
495    }
496    /// Seed fixtures that construct raw maps directly.
497    #[cfg(test)]
498    #[inline(never)]
499    fn refresh_rx_chunk_bytes(&mut self) {
500        self.rx_chunk_bytes = self
501            .rx_chunks
502            .values()
503            .map(SideChunkAssembly::byte_cost)
504            .sum();
505    }
506    fn refresh_transmit_header_templates(&mut self) {
507        // Refresh on the wire, not by forgetting the eviction history. RX
508        // retains its dictionary; clearing only TX makes bounded caches evict
509        // different entries after the next insertion/capacity negotiation.
510        for uses in self.tx_compact_uses.values_mut() {
511            *uses = u8::MAX;
512        }
513        // Local topology changes do not invalidate dictionaries received from
514        // peers. Dropping RX state here loses already-in-flight compact ACKs.
515    }
516
517    fn tx_template_count(&self) -> usize {
518        self.tx_template_ids.len()
519    }
520
521    fn rx_template_count(&self) -> usize {
522        self.rx_templates_by_id.len()
523    }
524
525    fn trim_tx_templates_to(&mut self, limit: usize) -> usize {
526        let mut removed = 0usize;
527        while self.tx_template_ids.len() > limit {
528            let Some(old_hash) = self.tx_template_ids.keys().next().copied() else {
529                break;
530            };
531            if let Some(old_id) = self.tx_template_ids.remove(&old_hash) {
532                self.tx_last_timestamps.remove(&old_id);
533                self.tx_compact_uses.remove(&old_id);
534            }
535            self.tx_templates.remove(&old_hash);
536            removed += 1;
537        }
538        removed
539    }
540
541    fn insert_tx_template(
542        &mut self,
543        template: SideHeaderTemplate,
544        template_id: u32,
545        max_templates: usize,
546    ) -> bool {
547        if max_templates == 0 {
548            return false;
549        }
550        let evicted = !self.tx_template_ids.contains_key(&template.hash)
551            && self.trim_tx_templates_to(max_templates.saturating_sub(1)) > 0;
552        self.tx_template_ids.insert(template.hash, template_id);
553        self.tx_templates.insert(template.hash, template);
554        self.tx_compact_uses.insert(template_id, 0);
555        evicted
556    }
557
558    fn insert_rx_template(
559        &mut self,
560        template_id: u32,
561        template: SideHeaderTemplate,
562        max_templates: usize,
563    ) -> bool {
564        if max_templates == 0 {
565            return false;
566        }
567        // Keep one owned header. The hash index needs only its wire ID, not
568        // another copy of the header and its shared-buffer owners.
569        if let Some(old_id) = self.rx_templates.get(&template.hash).copied()
570            && old_id != template_id
571        {
572            self.rx_templates_by_id.remove(&old_id);
573            self.rx_last_timestamps.remove(&old_id);
574            self.rx_templates.remove(&template.hash);
575        }
576        if let Some(previous) = self.rx_templates_by_id.get(&template_id)
577            && previous.hash != template.hash
578        {
579            self.rx_templates.remove(&previous.hash);
580            self.rx_last_timestamps.remove(&template_id);
581        }
582        let mut evicted = false;
583        if self.rx_templates_by_id.len() >= max_templates
584            && !self.rx_templates_by_id.contains_key(&template_id)
585            // Match TX's deterministic lowest-hash eviction on both routers
586            // and relays; lowest wire ID can select a different live header.
587            && let Some((&old_hash, &old_id)) = self.rx_templates.first_key_value()
588        {
589            self.rx_templates.remove(&old_hash);
590            self.rx_templates_by_id.remove(&old_id);
591            self.rx_last_timestamps.remove(&old_id);
592            evicted = true;
593        }
594        self.rx_templates.insert(template.hash, template_id);
595        self.rx_templates_by_id.insert(template_id, template);
596        evicted
597    }
598}
599
600#[derive(Clone, Debug, PartialEq, Eq)]
601pub struct CompactTimestampOmissionPolicy {
602    all: bool,
603    types: Option<Arc<[DataType]>>,
604}
605impl CompactTimestampOmissionPolicy {
606    pub const fn none() -> Self {
607        Self {
608            all: false,
609            types: None,
610        }
611    }
612    pub const fn all() -> Self {
613        Self {
614            all: true,
615            types: None,
616        }
617    }
618    pub fn with_data_type(mut self, ty: DataType) -> Self {
619        self.insert(ty);
620        self
621    }
622    pub fn insert(&mut self, ty: DataType) {
623        if self.contains(ty) {
624            return;
625        }
626        let current = self.types.as_deref().unwrap_or(&[]);
627        let mut types = Vec::with_capacity(current.len() + 1);
628        types.extend_from_slice(current);
629        let index = types.binary_search(&ty).unwrap_err();
630        types.insert(index, ty);
631        self.types = Some(Arc::from(types));
632    }
633    pub fn contains(&self, ty: DataType) -> bool {
634        self.all
635            || self
636                .types
637                .as_deref()
638                .is_some_and(|types| types.binary_search(&ty).is_ok())
639    }
640    pub fn is_empty(&self) -> bool {
641        !self.all && self.types.is_none()
642    }
643}
644
645impl Default for CompactTimestampOmissionPolicy {
646    #[inline]
647    fn default() -> Self {
648        Self::none()
649    }
650}
651
652#[derive(Clone, Debug, PartialEq, Eq)]
653pub enum RouterItem {
654    Packet(Packet),
655    Packed(crate::SharedBytes),
656}
657
658#[cfg(feature = "compact-packet-store")]
659impl RouterItem {
660    fn park_for_queue(&mut self) -> TelemetryResult<()> {
661        match self {
662            Self::Packet(packet) => packet.park_for_queue()?,
663            Self::Packed(bytes) => bytes.park_for_queue()?,
664        }
665        Ok(())
666    }
667}
668
669#[derive(Clone, Debug, PartialEq, Eq)]
670struct RouterRxItem {
671    src: Option<RouterSideId>,
672    data: RouterItem,
673    priority: u8,
674}
675
676#[derive(Clone, Debug, PartialEq, Eq)]
677enum RouterTxItem {
678    Broadcast(RouterItem),
679    EndToEndReplay {
680        packet_id: u64,
681    },
682    ToSide {
683        src: Option<RouterSideId>,
684        dst: RouterSideId,
685        data: RouterItem,
686    },
687    ReliableReplay {
688        dst: RouterSideId,
689        ty: DataType,
690        seq: u32,
691        bytes: crate::SharedBytes,
692    },
693}
694
695impl ByteCost for RouterRxItem {
696    #[cfg(feature = "compact-packet-store")]
697    fn park_for_queue(&mut self) -> TelemetryResult<()> {
698        self.data.park_for_queue()
699    }
700    #[inline]
701    fn byte_cost(&self) -> usize {
702        match &self.data {
703            RouterItem::Packet(pkt) => pkt.byte_cost(),
704            RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
705        }
706    }
707}
708
709impl ByteCost for RouterTxItem {
710    #[cfg(feature = "compact-packet-store")]
711    fn park_for_queue(&mut self) -> TelemetryResult<()> {
712        match self {
713            Self::Broadcast(item) | Self::ToSide { data: item, .. } => item.park_for_queue(),
714            Self::ReliableReplay { bytes, .. } => bytes.park_for_queue(),
715            _ => Ok(()),
716        }
717    }
718    #[inline]
719    fn byte_cost(&self) -> usize {
720        match self {
721            RouterTxItem::Broadcast(data) => match data {
722                RouterItem::Packet(pkt) => pkt.byte_cost(),
723                RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
724            },
725            RouterTxItem::EndToEndReplay { .. } => size_of::<u64>(),
726            RouterTxItem::ToSide { data, .. } => match data {
727                RouterItem::Packet(pkt) => pkt.byte_cost(),
728                RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
729            },
730            RouterTxItem::ReliableReplay { bytes, .. } => {
731                size_of::<crate::SharedBytes>() + bytes.len()
732            }
733        }
734    }
735}
736
737#[derive(Clone, Copy)]
738enum ReplayMarkerKey {
739    EndToEnd(u64),
740    Hop(RouterSideId, DataType, u32),
741}
742impl RouterTxItem {
743    fn replay_marker_key(&self) -> Option<ReplayMarkerKey> {
744        match self {
745            Self::EndToEndReplay { packet_id } => Some(ReplayMarkerKey::EndToEnd(*packet_id)),
746            Self::ReliableReplay { dst, ty, seq, .. } => {
747                Some(ReplayMarkerKey::Hop(*dst, *ty, *seq))
748            }
749            _ => None,
750        }
751    }
752}
753
754#[repr(u8)]
755#[derive(Clone, Copy, Debug, PartialEq, Eq)]
756enum QueuedDelivery {
757    Untracked,
758    OrderedTail,
759    Ordered,
760    DeferredOrdered,
761    DeferredUnordered,
762    Unordered,
763    ClaimedOrdered,
764    ClaimedUnordered,
765}
766
767const TX_DISPOSABLE: u8 = 1;
768const TX_E2E_CHILD: u8 = 8;
769#[cfg(feature = "std")]
770const TX_E2E_RETRY: u8 = 16;
771#[cfg(feature = "latest-value-egress")]
772const TX_LATEST_PENDING: u8 = 2;
773#[cfg(feature = "latest-value-egress")]
774const TX_HISTORY_OWNED: u8 = 4;
775#[cfg(feature = "latest-value-egress")]
776const TX_CANONICAL_OWED: u8 = 32;
777#[cfg(feature = "latest-value-egress")]
778const TX_CANONICAL_SERVED: u8 = 64;
779#[cfg(feature = "latest-value-egress")]
780const TX_CANONICAL_TURN: u8 = TX_CANONICAL_OWED | TX_CANONICAL_SERVED;
781#[cfg(feature = "latest-value-egress")]
782const TX_LATEST_TURN: u8 = 128;
783#[cfg(feature = "latest-value-egress")]
784const TX_FRESH: u8 = TX_DISPOSABLE | TX_LATEST_PENDING;
785#[cfg(not(feature = "latest-value-egress"))]
786const TX_FRESH: u8 = TX_DISPOSABLE;
787
788/// Transmit queue item with flags.
789/// Holds a RouterTxItem and a flag to ignore local dispatch.
790/// Used internally by the Router transmit queue.
791#[derive(Clone, Debug, PartialEq, Eq)]
792struct TxQueued {
793    // Cached before queue parking; padding keeps the metadata footprint stable.
794    disposable: u8,
795    item: RouterTxItem,
796    ignore_local: bool,
797    priority: u8,
798    delivery_state: QueuedDelivery,
799    accepted_type: u32,
800    deferred_since_ms: u64,
801    retry_after_ms: u64,
802}
803
804#[derive(Clone, Copy)]
805struct TxClaim {
806    token: u32,
807    ordered: bool,
808    side_retry: bool,
809}
810const _: () = {
811    assert!(size_of::<TxClaim>() == size_of::<(u32, bool)>());
812    assert!(size_of::<Option<TxClaim>>() == size_of::<Option<(u32, bool)>>());
813};
814#[derive(Clone, Copy, Default)]
815struct EgressService {
816    #[cfg(feature = "latest-value-egress")]
817    token: u32,
818    #[cfg(feature = "latest-value-egress")]
819    side: u8,
820    #[cfg(feature = "latest-value-egress")]
821    kind: u8,
822}
823#[derive(Default)]
824struct DispatchContext {
825    // Retain a denied queued replay's service band across side-owned retries.
826    priority: u8,
827    claim: Option<TxClaim>,
828    since: Option<u64>,
829    completion_flags: u8,
830    admitted: bool,
831    #[cfg(feature = "latest-value-egress")]
832    latest_service: EgressService,
833}
834impl DispatchContext {
835    const TERMINAL: u8 = 1;
836    const OWNER_TRANSFERRED: u8 = 2;
837    const REFUSED: u8 = 4;
838    fn refused(&self) -> bool {
839        self.completion_flags & Self::REFUSED != 0
840    }
841    fn mark_refused(&mut self) {
842        self.completion_flags |= Self::REFUSED;
843    }
844
845    fn egress_service(&self) -> EgressService {
846        #[cfg(feature = "latest-value-egress")]
847        {
848            self.latest_service
849        }
850        #[cfg(not(feature = "latest-value-egress"))]
851        {
852            EgressService::default()
853        }
854    }
855    fn yielded_canonical_turn(&self) -> bool {
856        #[cfg(feature = "latest-value-egress")]
857        {
858            self.latest_service.side != 0
859                && self.completion_flags & TX_CANONICAL_OWED != 0
860                && self.admitted
861        }
862        #[cfg(not(feature = "latest-value-egress"))]
863        {
864            false
865        }
866    }
867    fn canonical_turn_flags(&self) -> u8 {
868        #[cfg(feature = "latest-value-egress")]
869        if self.completion_flags & TX_LATEST_TURN != 0 {
870            return TX_LATEST_TURN;
871        }
872        #[cfg(feature = "latest-value-egress")]
873        if self.completion_flags & TX_CANONICAL_OWED != 0 {
874            return if self.admitted {
875                TX_CANONICAL_SERVED
876            } else {
877                self.completion_flags & TX_CANONICAL_TURN
878            };
879        }
880        0
881    }
882
883    fn is_terminal(&self) -> bool {
884        self.completion_flags & Self::TERMINAL != 0
885    }
886    fn owner_transferred(&self) -> bool {
887        self.completion_flags & Self::OWNER_TRANSFERRED != 0
888    }
889    fn mark_terminal(&mut self) {
890        self.completion_flags |= Self::TERMINAL;
891    }
892    #[cfg(feature = "discovery")]
893    fn mark_owner_transferred(&mut self) {
894        self.completion_flags |= Self::OWNER_TRANSFERRED;
895    }
896}
897// Pack completion flags so priority fits existing padding on i686 too.
898// The dispatch-only priority uses existing stack padding; retained queue,
899// reliable maps and packet ownership metadata do not grow.
900const _: () = assert!(
901    size_of::<DispatchContext>()
902        <= size_of::<(Option<TxClaim>, Option<u64>, bool, bool)>() + size_of::<u64>()
903);
904
905struct TxWork {
906    queued: TxQueued,
907    claim: Option<TxClaim>,
908    owed_turn: bool,
909    #[cfg(feature = "latest-value-egress")]
910    service: EgressService,
911}
912
913/// Do not allow a continuously replenished stream of control traffic to keep
914/// an already-queued application packet from ever reaching a transport.
915/// Priorities still determine normal ordering; this only services the oldest
916/// lowest-priority item after a bounded high-priority burst.
917const MAX_PRIORITY_BURST: u8 = 8;
918
919/// ByteCost implementation for TxQueued.
920// Compare against the complete ownership metadata before the cached bit was
921// added. Its padding absorbs that bit on supported ABIs. The older four-field
922// tuple has different padding on i686 (u64 alignment 4) and ARM/64-bit targets
923// (alignment 8), so its growth is not universally eight bytes.
924const _: () = {
925    assert!(
926        size_of::<TxQueued>()
927            == size_of::<(RouterTxItem, bool, u8, QueuedDelivery, u32, u64, u64)>()
928    );
929    let metadata_bytes = 2 * size_of::<u64>()
930        + size_of::<u32>()
931        + size_of::<QueuedDelivery>()
932        + 2 * size_of::<bool>()
933        + size_of::<u8>();
934    let alignment = core::mem::align_of::<TxQueued>();
935    let expected = (size_of::<RouterTxItem>() + metadata_bytes).div_ceil(alignment) * alignment;
936    assert!(size_of::<TxQueued>() == expected);
937};
938
939impl ByteCost for TxQueued {
940    #[cfg(feature = "compact-packet-store")]
941    fn park_for_queue(&mut self) -> TelemetryResult<()> {
942        self.item.park_for_queue()
943    }
944    /// Byte cost is the cost of the inner item plus one bool.
945    #[inline]
946    fn byte_cost(&self) -> usize {
947        self.item.byte_cost() + size_of::<bool>() + size_of::<u8>() + 2 * size_of::<u64>()
948    }
949}
950
951/// ByteCost implementation for `u64` (used by `recent_rx`).
952impl ByteCost for u64 {
953    /// Byte cost is size of u64.
954    #[inline]
955    fn byte_cost(&self) -> usize {
956        size_of::<u64>()
957    }
958}
959
960// -------------------- Reliable delivery state --------------------
961
962#[derive(Debug, Clone)]
963struct ReliableTxState {
964    next_seq: u32,
965    sent_order: VecDeque<u32>,
966    sent: BTreeMap<u32, ReliableSent>,
967}
968
969#[derive(Debug, Clone)]
970struct ReliableSent {
971    bytes: crate::SharedBytes,
972    last_send_ms: u64,
973    retries: u32,
974    queued: bool,
975    partial_acked: bool,
976}
977
978// The std-only cached type grows the native boxed owner by up to one u64
979// alignment unit. Reserve/charge that bounded metadata, not just its u32 width.
980#[cfg(feature = "std")]
981const PENDING_TYPE_METADATA_BYTES: usize = size_of::<u64>();
982
983#[derive(Debug, Clone)]
984struct EndToEndReliableSent {
985    #[cfg(feature = "std")]
986    completion_epoch: u64,
987    // std-only ownership inspection must not materialize a retained Packed
988    // payload. Zero is unknown provenance and remains conservatively pending.
989    #[cfg(feature = "std")]
990    pending_type: u32,
991    ordered_type: Option<u32>,
992    waiting_for_predecessor: bool,
993    data: RouterItem,
994    pending_destinations: BTreeMap<u64, RouterSideId>,
995    tracked_destinations: bool,
996    last_send_ms: u64,
997    retries: u32,
998    queued: bool,
999}
1000
1001impl EndToEndReliableSent {
1002    // Keeping the retained packet inline makes each B-tree leaf several KiB.
1003    // Small independent records avoid that contiguous allocation on fragmented
1004    // embedded heaps. A failed record allocation remains retryable pressure.
1005    #[cfg(feature = "discovery")]
1006    fn try_box(self) -> TelemetryResult<Box<Self>> {
1007        let layout = core::alloc::Layout::new::<Self>();
1008        // SAFETY: layout is nonzero and valid for Self. On failure self drops
1009        // normally. On success initialize exactly once and transfer ownership
1010        // to Box, which uses the same global allocator and layout on drop.
1011        let ptr = unsafe { alloc::alloc::alloc(layout).cast::<Self>() };
1012        if ptr.is_null() {
1013            return Err(TelemetryError::Io("memory pressure"));
1014        }
1015        unsafe {
1016            ptr.write(self);
1017            Ok(Box::from_raw(ptr))
1018        }
1019    }
1020}
1021
1022#[derive(Debug, Clone)]
1023struct ReliableRxState {
1024    expected_seq: u32,
1025    buffered: RouterMap<u32, crate::SharedBytes>,
1026}
1027
1028// Logical incremental record growth relative to the prior return-route value.
1029// This is not the full BTree allocation cost; allocator node deltas are measured
1030// separately. The fixed RouterInner sequence is part of object footprint.
1031const RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES: usize = size_of::<u64>();
1032
1033#[derive(Debug, Clone)]
1034struct ReliableReturnRouteState {
1035    side: RouterSideId,
1036    // A final receipt can be lost after local delivery or completion of every
1037    // child owner. Retain that fact for the lifetime of this bounded route.
1038    delivery_completed: bool,
1039    local_admission_completed: bool,
1040    local_admission_token: u64,
1041    admission_cancelled: bool,
1042    admission_committing: bool,
1043    last_replay_ms: u64,
1044    // A legacy contract names this router, not its downstream owner.
1045    // Translate the child receipt back to the router identity and original
1046    // packet ID that the upstream publisher is awaiting.
1047    upstream_contract: Option<(u64, u64)>,
1048}
1049
1050// Same-ID direct RX must not certify a cached observation while its required
1051// callback is still deciding. This lease never holds state across callbacks.
1052struct ReliableLocalAdmissionGuard<'a> {
1053    router: &'a Router,
1054    packet_id: u64,
1055    token: u64,
1056}
1057#[cfg(all(test, feature = "std"))]
1058std::thread_local! {
1059    static LOCAL_ADMISSION_CHECKS: core::cell::Cell<usize> = const {core::cell::Cell::new(0)};
1060}
1061impl ReliableLocalAdmissionGuard<'_> {
1062    fn check_locked(&self, state: &RouterInner) -> TelemetryResult<()> {
1063        #[cfg(all(test, feature = "std"))]
1064        LOCAL_ADMISSION_CHECKS.with(|count| count.set(count.get() + 1));
1065        if state
1066            .reliable_return_routes
1067            .get(&self.packet_id)
1068            .is_some_and(|route| {
1069                route.local_admission_token == self.token && !route.admission_cancelled
1070            })
1071        {
1072            Ok(())
1073        } else {
1074            Err(TelemetryError::Io("side changed during rx"))
1075        }
1076    }
1077    fn check(&self) -> TelemetryResult<()> {
1078        self.check_locked(&self.router.state.lock())
1079    }
1080    fn begin_commit(&self, local_accepted: bool) -> TelemetryResult<()> {
1081        let mut state = self.router.state.lock();
1082        self.check_locked(&state)?;
1083        let route = state
1084            .reliable_return_routes
1085            .get_mut(&self.packet_id)
1086            .expect("checked lease");
1087        route.admission_committing = true;
1088        route.local_admission_completed |= local_accepted;
1089        Ok(())
1090    }
1091    fn end_commit(&self) {
1092        let mut state = self.router.state.lock();
1093        if let Some(route) = state.reliable_return_routes.get_mut(&self.packet_id)
1094            && route.local_admission_token == self.token
1095        {
1096            route.admission_committing = false;
1097        }
1098    }
1099}
1100
1101impl Drop for ReliableLocalAdmissionGuard<'_> {
1102    fn drop(&mut self) {
1103        let mut state = self.router.state.lock();
1104        let cancelled = if let Some(route) = state.reliable_return_routes.get_mut(&self.packet_id)
1105            && route.local_admission_token == self.token
1106        {
1107            route.local_admission_token = 0;
1108            route.admission_committing = false;
1109            route.admission_cancelled
1110        } else {
1111            false
1112        };
1113        if cancelled {
1114            state.reliable_return_routes.remove(&self.packet_id);
1115            state
1116                .reliable_return_route_order
1117                .retain(|id| *id != self.packet_id);
1118        }
1119    }
1120}
1121
1122#[cfg(feature = "discovery")]
1123#[derive(Debug, Clone, Default, PartialEq, Eq)]
1124struct DiscoverySenderState {
1125    /// Numeric identity advertised for this hostname. Compact frames may carry
1126    /// this address instead of the hostname, so routing must accept either.
1127    advertised_address: Option<u32>,
1128    reachable: Vec<DataEndpoint>,
1129    advertised_reachable: Vec<DataEndpoint>,
1130    reachable_network_variables: Vec<DataType>,
1131    reachable_timesync_sources: Vec<String>,
1132    advertised_reachable_timesync_sources: Vec<String>,
1133    link_capabilities: Option<discovery::LinkCapabilities>,
1134    topology_boards: Arc<Vec<TopologyBoardNode>>,
1135    has_full_topology: bool,
1136    has_schema: bool,
1137    last_seen_ms: u64,
1138}
1139
1140/// One bounded hop-send grant per requested variable and ingress segment.
1141/// It does not create a new value or a new end-to-end reader cohort.
1142#[cfg(feature = "discovery")]
1143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144struct ManagedVariableSubscription {
1145    last_seen_ms: u64,
1146    next_refresh_ms: u64,
1147    request_id: u64,
1148    value_id: u64,
1149    upstream: RouterSideId,
1150    ty: DataType,
1151    requester: u32,
1152    writer: u32,
1153    refresh_pending: bool,
1154}
1155
1156#[cfg(feature = "discovery")]
1157fn reserve_managed_subscriptions(
1158    requests: &mut Vec<ManagedVariableSubscription>,
1159    additional: usize,
1160) -> TelemetryResult<()> {
1161    let needed = requests
1162        .len()
1163        .checked_add(additional)
1164        .ok_or(TelemetryError::Io("memory pressure"))?;
1165    if needed <= requests.capacity() {
1166        return Ok(());
1167    }
1168    let bytes = needed
1169        .checked_mul(size_of::<ManagedVariableSubscription>())
1170        .ok_or(TelemetryError::Io("memory pressure"))?;
1171    crate::memory_admission::check(bytes, bytes)?;
1172    requests
1173        .try_reserve_exact(additional)
1174        .map_err(|_| TelemetryError::Io("memory pressure"))
1175}
1176
1177#[cfg(feature = "discovery")]
1178#[derive(Debug, Clone, Default, PartialEq, Eq)]
1179struct DiscoverySideState {
1180    reachable: Vec<DataEndpoint>,
1181    reachable_network_variables: Vec<DataType>,
1182    requested_network_variables: Vec<ManagedVariableSubscription>,
1183    reachable_timesync_sources: Vec<String>,
1184    last_seen_ms: u64,
1185    announcers: BTreeMap<String, Box<DiscoverySenderState>>,
1186}
1187
1188#[cfg(feature = "discovery")]
1189impl DiscoverySideState {
1190    fn try_clone(&self) -> TelemetryResult<Self> {
1191        // A refused new record leaves the authoritative route unchanged. Clone
1192        // pointer values explicitly instead of adding infallible Box::clone
1193        // allocations to the existing admitted snapshot path.
1194        let announcers = crate::peer_record::try_clone_map(&self.announcers)?;
1195        let mut requests = Vec::new();
1196        reserve_managed_subscriptions(&mut requests, self.requested_network_variables.len())?;
1197        requests.extend_from_slice(&self.requested_network_variables);
1198        Ok(Self {
1199            reachable: self.reachable.clone(),
1200            reachable_network_variables: self.reachable_network_variables.clone(),
1201            requested_network_variables: requests,
1202            reachable_timesync_sources: self.reachable_timesync_sources.clone(),
1203            last_seen_ms: self.last_seen_ms,
1204            announcers,
1205        })
1206    }
1207}
1208
1209#[cfg(feature = "discovery")]
1210#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1211#[repr(u8)]
1212enum SchemaResponsePhase {
1213    #[default]
1214    Idle,
1215    PrefixBuilding,
1216    CatalogWaiting,
1217    CatalogBuilding,
1218    Owned,
1219}
1220
1221#[cfg(feature = "discovery")]
1222#[derive(Debug, Clone, Default)]
1223struct DiscoverySideThrottleState {
1224    next_topology_request_ms: u64,
1225    next_schema_request_ms: u64,
1226    // Bounded per-side rotation; advance only after query admission.
1227    schema_request_cursor: u32,
1228    // Request bursts share one active catalog and one fresh pending generation.
1229    // No payload is retained here: the existing TX queue owns partial transfers.
1230    next_schema_response_ms: u64,
1231    schema_response_packet_id: u64,
1232    schema_response_phase: SchemaResponsePhase,
1233    flags: u8,
1234    next_ping_ms: u64,
1235    // Scheduled and accepted sends differ when a constrained link is backed up.
1236    last_liveness_tx_ms: u64,
1237    incremental_retries_remaining: u8,
1238    last_topology: Vec<TopologyBoardNode>,
1239}
1240
1241#[cfg(feature = "discovery")]
1242impl DiscoverySideThrottleState {
1243    #[inline]
1244    fn flag(&self, mask: u8) -> bool {
1245        self.flags & mask != 0
1246    }
1247    #[inline]
1248    fn set_flag(&mut self, mask: u8, value: bool) {
1249        if value {
1250            self.flags |= mask;
1251        } else {
1252            self.flags &= !mask;
1253        }
1254    }
1255    #[inline]
1256    fn schema_response_active(&self) -> bool {
1257        self.flag(1 << 0)
1258    }
1259    #[inline]
1260    fn set_schema_response_active(&mut self, value: bool) {
1261        self.set_flag(1 << 0, value)
1262    }
1263    #[inline]
1264    fn schema_response_started(&self) -> bool {
1265        self.flag(1 << 1)
1266    }
1267    #[inline]
1268    fn set_schema_response_started(&mut self, value: bool) {
1269        self.set_flag(1 << 1, value)
1270    }
1271    #[inline]
1272    fn schema_response_pending(&self) -> bool {
1273        self.flag(1 << 2)
1274    }
1275    #[inline]
1276    fn set_schema_response_pending(&mut self, value: bool) {
1277        self.set_flag(1 << 2, value)
1278    }
1279    #[inline]
1280    fn pending_incremental(&self) -> bool {
1281        self.flag(1 << 3)
1282    }
1283    #[inline]
1284    fn set_pending_incremental(&mut self, value: bool) {
1285        self.set_flag(1 << 3, value)
1286    }
1287    #[inline]
1288    fn has_sent_full(&self) -> bool {
1289        self.flag(1 << 4)
1290    }
1291    #[inline]
1292    fn set_has_sent_full(&mut self, value: bool) {
1293        self.set_flag(1 << 4, value)
1294    }
1295
1296    fn liveness_due(&self, now_ms: u64) -> bool {
1297        now_ms >= self.next_ping_ms
1298            && now_ms.saturating_sub(self.last_liveness_tx_ms)
1299                >= DISCOVERY_SLOW_LINK_PING_INTERVAL_MS
1300    }
1301}
1302
1303#[cfg(all(feature = "discovery", feature = "timesync"))]
1304#[derive(Debug, Clone, Default)]
1305struct TimeSyncSideThrottleState {
1306    next_allowed_ms: u64,
1307}
1308
1309#[cfg(feature = "discovery")]
1310#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1311enum DiscoveryAdvertiseLevel {
1312    MinimalPing,
1313    Incremental,
1314    Full,
1315}
1316
1317#[derive(Debug, Clone, Default)]
1318struct AdaptiveRouteStats {
1319    estimated_bandwidth_bps: u64,
1320    peak_bandwidth_bps: u64,
1321    last_observed_ms: u64,
1322    last_slow_observed_ms: u64,
1323    sample_count: u64,
1324    window_started_ms: u64,
1325    window_bytes: u64,
1326    peak_usage_bps: u64,
1327}
1328
1329impl AdaptiveRouteStats {
1330    #[inline]
1331    fn observe(&mut self, bytes: usize, sample_bps: u64, now_ms: u64) {
1332        self.estimated_bandwidth_bps = if self.estimated_bandwidth_bps == 0 {
1333            sample_bps
1334        } else if sample_bps >= self.estimated_bandwidth_bps {
1335            self.estimated_bandwidth_bps
1336                .saturating_mul(3)
1337                .saturating_add(sample_bps.saturating_mul(5))
1338                / 8
1339        } else {
1340            self.estimated_bandwidth_bps
1341                .saturating_mul(7)
1342                .saturating_add(sample_bps)
1343                / 8
1344        };
1345        self.peak_bandwidth_bps = self.peak_bandwidth_bps.max(sample_bps);
1346        self.last_observed_ms = now_ms;
1347        if sample_bps > 0 && sample_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS {
1348            self.last_slow_observed_ms = now_ms;
1349        }
1350        self.sample_count = self.sample_count.saturating_add(1);
1351        if self.window_started_ms == 0 || now_ms.saturating_sub(self.window_started_ms) > 1_000 {
1352            self.window_started_ms = now_ms;
1353            self.window_bytes = 0;
1354        }
1355        self.window_bytes = self.window_bytes.saturating_add(bytes as u64);
1356        self.peak_usage_bps = self.peak_usage_bps.max(self.current_usage_bps(now_ms));
1357    }
1358
1359    #[inline]
1360    fn current_usage_bps(&self, now_ms: u64) -> u64 {
1361        if self.window_started_ms == 0 {
1362            return 0;
1363        }
1364        let elapsed_ms = now_ms.saturating_sub(self.window_started_ms).max(1);
1365        (u128::from(self.window_bytes).saturating_mul(1000) / u128::from(elapsed_ms))
1366            .min(u128::from(u64::MAX)) as u64
1367    }
1368
1369    #[inline]
1370    fn available_headroom_bps(&self, now_ms: u64) -> u64 {
1371        let capacity = self
1372            .estimated_bandwidth_bps
1373            .max(self.peak_bandwidth_bps)
1374            .max(1);
1375        capacity.saturating_sub(self.current_usage_bps(now_ms))
1376    }
1377
1378    #[inline]
1379    fn weight(&self, now_ms: u64) -> u64 {
1380        self.available_headroom_bps(now_ms).max(1)
1381    }
1382
1383    #[inline]
1384    fn snapshot(&self, now_ms: u64, auto_balancing_enabled: bool) -> AdaptiveLinkStats {
1385        let current_usage_bps = self.current_usage_bps(now_ms);
1386        let estimated_capacity_bps = self.estimated_bandwidth_bps.max(1);
1387        let peak_capacity_bps = self.peak_bandwidth_bps.max(estimated_capacity_bps);
1388        let available_headroom_bps = peak_capacity_bps.saturating_sub(current_usage_bps);
1389        AdaptiveLinkStats {
1390            auto_balancing_enabled,
1391            estimated_capacity_bps,
1392            peak_capacity_bps,
1393            current_usage_bps,
1394            peak_usage_bps: self.peak_usage_bps.max(current_usage_bps),
1395            available_headroom_bps,
1396            effective_weight: available_headroom_bps.max(1),
1397            last_observed_ms: self.last_observed_ms,
1398            sample_count: self.sample_count,
1399        }
1400    }
1401}
1402
1403use crate::type_stats::TypeRuntimeStatsInner;
1404
1405#[derive(Debug, Clone, Default)]
1406struct SideRuntimeStatsInner {
1407    tx_packets: u64,
1408    tx_bytes: u64,
1409    rx_packets: u64,
1410    rx_bytes: u64,
1411    relayed_tx_packets: u64,
1412    relayed_tx_bytes: u64,
1413    relayed_rx_packets: u64,
1414    relayed_rx_bytes: u64,
1415    local_delivery_packets: u64,
1416    tx_retries: u64,
1417    tx_handler_failures: u64,
1418    local_handler_failures: u64,
1419    total_handler_retries: u64,
1420    side_transport_full_frames: u64,
1421    side_transport_compact_frames: u64,
1422    side_transport_compact_delta_frames: u64,
1423    side_transport_compact_omitted_timestamp_frames: u64,
1424    side_transport_chunk_frames: u64,
1425    side_transport_raw_bytes: u64,
1426    side_transport_wire_bytes: u64,
1427    side_transport_bytes_saved: u64,
1428    side_transport_min_compact_overhead_bytes: Option<usize>,
1429    side_transport_max_compact_overhead_bytes: Option<usize>,
1430    side_transport_compact_target_misses: u64,
1431    side_transport_template_evictions: u64,
1432    data_types: BTreeMap<u32, TypeRuntimeStatsInner>,
1433}
1434
1435impl SideRuntimeStatsInner {
1436    fn type_stats_mut(&mut self, ty: DataType) -> &mut TypeRuntimeStatsInner {
1437        self.data_types.entry(ty.as_u32()).or_default()
1438    }
1439
1440    fn note_tx(&mut self, ty: DataType, bytes: usize, relayed: bool, retries: usize) {
1441        self.tx_packets = self.tx_packets.saturating_add(1);
1442        self.tx_bytes = self.tx_bytes.saturating_add(bytes as u64);
1443        self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1444        self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1445        if relayed {
1446            self.relayed_tx_packets = self.relayed_tx_packets.saturating_add(1);
1447            self.relayed_tx_bytes = self.relayed_tx_bytes.saturating_add(bytes as u64);
1448        }
1449        self.type_stats_mut(ty)
1450            .tx(bytes as u64, relayed, retries as u64);
1451    }
1452
1453    fn note_rx(&mut self, ty: DataType, bytes: usize, relayed: bool) {
1454        self.rx_packets = self.rx_packets.saturating_add(1);
1455        self.rx_bytes = self.rx_bytes.saturating_add(bytes as u64);
1456        if relayed {
1457            self.relayed_rx_packets = self.relayed_rx_packets.saturating_add(1);
1458            self.relayed_rx_bytes = self.relayed_rx_bytes.saturating_add(bytes as u64);
1459        }
1460        self.type_stats_mut(ty).rx(bytes as u64, relayed);
1461    }
1462
1463    fn note_local_delivery(&mut self, ty: DataType) {
1464        self.local_delivery_packets = self.local_delivery_packets.saturating_add(1);
1465        self.type_stats_mut(ty).rx(0, false);
1466    }
1467
1468    fn note_local_handler_failure(&mut self, ty: DataType, retries: usize) {
1469        self.local_handler_failures = self.local_handler_failures.saturating_add(1);
1470        self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1471        self.type_stats_mut(ty).failure(0);
1472    }
1473
1474    fn note_tx_failure(&mut self, ty: DataType, retries: usize) {
1475        self.tx_handler_failures = self.tx_handler_failures.saturating_add(1);
1476        self.total_handler_retries = self.total_handler_retries.saturating_add(retries as u64);
1477        self.tx_retries = self.tx_retries.saturating_add(retries as u64);
1478        self.type_stats_mut(ty).failure(retries as u64);
1479    }
1480
1481    fn note_side_transport_full(&mut self, raw_bytes: usize, wire_bytes: usize) {
1482        self.side_transport_full_frames = self.side_transport_full_frames.saturating_add(1);
1483        self.note_side_transport_bytes(raw_bytes, wire_bytes);
1484    }
1485
1486    fn note_side_transport_compact(
1487        &mut self,
1488        raw_bytes: usize,
1489        wire_bytes: usize,
1490        compact_overhead_bytes: usize,
1491        used_timestamp_delta: bool,
1492        omitted_timestamp: bool,
1493    ) {
1494        self.side_transport_compact_frames = self.side_transport_compact_frames.saturating_add(1);
1495        if used_timestamp_delta {
1496            self.side_transport_compact_delta_frames =
1497                self.side_transport_compact_delta_frames.saturating_add(1);
1498        }
1499        if omitted_timestamp {
1500            self.side_transport_compact_omitted_timestamp_frames = self
1501                .side_transport_compact_omitted_timestamp_frames
1502                .saturating_add(1);
1503        }
1504        self.note_side_transport_bytes(raw_bytes, wire_bytes);
1505        self.side_transport_min_compact_overhead_bytes = Some(
1506            self.side_transport_min_compact_overhead_bytes
1507                .map_or(compact_overhead_bytes, |v| v.min(compact_overhead_bytes)),
1508        );
1509        self.side_transport_max_compact_overhead_bytes = Some(
1510            self.side_transport_max_compact_overhead_bytes
1511                .map_or(compact_overhead_bytes, |v| v.max(compact_overhead_bytes)),
1512        );
1513    }
1514
1515    fn note_side_transport_chunks(&mut self, chunks: usize) {
1516        self.side_transport_chunk_frames = self
1517            .side_transport_chunk_frames
1518            .saturating_add(chunks as u64);
1519    }
1520
1521    fn note_side_transport_bytes(&mut self, raw_bytes: usize, wire_bytes: usize) {
1522        self.side_transport_raw_bytes = self
1523            .side_transport_raw_bytes
1524            .saturating_add(raw_bytes as u64);
1525        self.side_transport_wire_bytes = self
1526            .side_transport_wire_bytes
1527            .saturating_add(wire_bytes as u64);
1528        if raw_bytes > wire_bytes {
1529            self.side_transport_bytes_saved = self
1530                .side_transport_bytes_saved
1531                .saturating_add((raw_bytes - wire_bytes) as u64);
1532        }
1533    }
1534
1535    fn note_side_transport_compact_target_miss(&mut self) {
1536        self.side_transport_compact_target_misses =
1537            self.side_transport_compact_target_misses.saturating_add(1);
1538    }
1539
1540    fn note_side_transport_template_eviction(&mut self) {
1541        self.side_transport_template_evictions =
1542            self.side_transport_template_evictions.saturating_add(1);
1543    }
1544}
1545
1546#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1547enum RouteSelectionOrigin {
1548    Flood,
1549    Discovered,
1550}
1551
1552// -------------------- endpoint + board config --------------------
1553/// Packet Handler function type
1554type PacketHandlerFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
1555
1556/// Packed Handler function type
1557type PackedHandlerFn = dyn Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static;
1558type PrioritizedPackedHandlerFn = dyn Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static;
1559
1560// Make handlers usable across tasks
1561/// Endpoint handler function enum.
1562/// Holds either a `Packet` handler or a packed byte-slice handler.
1563/// /// - Packet handler signature: `Fn(&Packet) -> TelemetryResult<()>`
1564/// /// - Packed handler signature: `Fn(&[u8]) -> TelemetryResult<()>`
1565#[derive(Clone)]
1566pub enum EndpointHandlerFn {
1567    Packet(Arc<PacketHandlerFn>),
1568    Packed(Arc<PackedHandlerFn>),
1569}
1570
1571/// Endpoint handler for a specific data endpoint.
1572pub struct EndpointHandler {
1573    endpoint: DataEndpoint,
1574    handler: EndpointHandlerFn,
1575}
1576
1577/// Debug implementation for EndpointHandlerFn.
1578impl Debug for EndpointHandlerFn {
1579    #[inline]
1580    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1581        match self {
1582            EndpointHandlerFn::Packet(_) => f.write_str("EndpointHandlerFn::Packet(<handler>)"),
1583            EndpointHandlerFn::Packed(_) => f.write_str("EndpointHandlerFn::Packed(<handler>)"),
1584        }
1585    }
1586}
1587
1588/// TX handler for a router side: either packed or packet-based.
1589#[derive(Clone)]
1590pub enum RouterTxHandlerFn {
1591    Packed(Arc<PackedHandlerFn>),
1592    PackedWithPriority(Arc<PrioritizedPackedHandlerFn>),
1593    Packet(Arc<PacketHandlerFn>),
1594}
1595
1596impl Debug for RouterTxHandlerFn {
1597    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1598        match self {
1599            RouterTxHandlerFn::Packed(_) => f.debug_tuple("Packed").field(&"<handler>").finish(),
1600            RouterTxHandlerFn::PackedWithPriority(_) => f
1601                .debug_tuple("PackedWithPriority")
1602                .field(&"<handler>")
1603                .finish(),
1604            RouterTxHandlerFn::Packet(_) => f.debug_tuple("Packet").field(&"<handler>").finish(),
1605        }
1606    }
1607}
1608
1609pub use crate::routing_config::SideOptions as RouterSideOptions;
1610
1611/// One side of the router – a name + TX handler.
1612#[derive(Clone, Debug)]
1613pub struct RouterSide {
1614    pub name: Arc<str>,
1615    pub tx_handler: RouterTxHandlerFn,
1616    pub opts: RouterSideOptions,
1617}
1618
1619/// Debug implementation for EndpointHandler.
1620impl Debug for EndpointHandler {
1621    #[inline]
1622    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1623        f.debug_struct("EndpointHandler")
1624            .field("endpoint", &self.endpoint)
1625            .field("handler", &self.handler)
1626            .finish()
1627    }
1628}
1629
1630#[inline]
1631pub(crate) const fn endpoint_is_router_internal(endpoint: DataEndpoint) -> bool {
1632    #[cfg(feature = "timesync")]
1633    if matches!(endpoint, DataEndpoint::TimeSync) {
1634        return true;
1635    }
1636    #[cfg(feature = "discovery")]
1637    {
1638        discovery::is_discovery_endpoint(endpoint)
1639    }
1640    #[cfg(not(feature = "discovery"))]
1641    {
1642        let _ = endpoint;
1643        false
1644    }
1645}
1646
1647impl EndpointHandler {
1648    /// Create a new endpoint handler for `Packet` callbacks.
1649    ///
1650    /// Handler signature is `Fn(&Packet) -> TelemetryResult<()>`.
1651    #[inline]
1652    pub fn new_packet_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1653    where
1654        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1655    {
1656        assert!(
1657            !endpoint_is_router_internal(endpoint),
1658            "reserved internal endpoint handlers must not be user-registered"
1659        );
1660        #[cfg(feature = "std")]
1661        crate::config::ensure_endpoint_id(endpoint, false)
1662            .expect("endpoint handler endpoint registration failed");
1663        Self {
1664            endpoint,
1665            handler: EndpointHandlerFn::Packet(Arc::new(f)),
1666        }
1667    }
1668
1669    /// Create a new packet handler from a runtime endpoint definition.
1670    #[inline]
1671    #[cfg(feature = "std")]
1672    pub fn new_packet_handler_for<F>(endpoint: crate::config::OwnedEndpointDefinition, f: F) -> Self
1673    where
1674        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1675    {
1676        Self::new_packet_handler(endpoint.id, f)
1677    }
1678
1679    /// Create a new packet handler from a static endpoint definition.
1680    #[cfg(not(feature = "std"))]
1681    #[inline]
1682    pub fn new_packet_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1683    where
1684        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1685    {
1686        Self::new_packet_handler(endpoint.id, f)
1687    }
1688
1689    /// Create a new packet handler by endpoint name.
1690    #[cfg(feature = "std")]
1691    #[inline]
1692    pub fn new_packet_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1693    where
1694        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
1695    {
1696        let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1697            .ok_or(TelemetryError::BadArg)?;
1698        Ok(Self::new_packet_handler(endpoint.id, f))
1699    }
1700
1701    /// Create a new endpoint handler for packed byte-slice callbacks.
1702    ///
1703    /// Handler signature is `Fn(&[u8]) -> TelemetryResult<()>`.
1704    #[inline]
1705    pub fn new_packed_handler<F>(endpoint: DataEndpoint, f: F) -> Self
1706    where
1707        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1708    {
1709        assert!(
1710            !endpoint_is_router_internal(endpoint),
1711            "reserved internal endpoint handlers must not be user-registered"
1712        );
1713        #[cfg(feature = "std")]
1714        crate::config::ensure_endpoint_id(endpoint, false)
1715            .expect("endpoint handler endpoint registration failed");
1716        Self {
1717            endpoint,
1718            handler: EndpointHandlerFn::Packed(Arc::new(f)),
1719        }
1720    }
1721
1722    /// Create a new packed handler from a runtime endpoint definition.
1723    #[inline]
1724    pub fn new_packed_handler_for<F>(endpoint: crate::config::EndpointDefinition, f: F) -> Self
1725    where
1726        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1727    {
1728        Self::new_packed_handler(endpoint.id, f)
1729    }
1730
1731    /// Create a new packed handler by endpoint name.
1732    #[cfg(feature = "std")]
1733    #[inline]
1734    pub fn new_packed_handler_by_name<F>(endpoint_name: &str, f: F) -> TelemetryResult<Self>
1735    where
1736        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
1737    {
1738        let endpoint = crate::config::endpoint_definition_by_name(endpoint_name)
1739            .ok_or(TelemetryError::BadArg)?;
1740        Ok(Self::new_packed_handler(endpoint.id, f))
1741    }
1742
1743    /// Return the endpoint that the handler is registered for.
1744    #[inline]
1745    pub fn get_endpoint(&self) -> DataEndpoint {
1746        self.endpoint
1747    }
1748
1749    /// Return a reference to the handler function.
1750    #[inline]
1751    pub fn get_handler(&self) -> &EndpointHandlerFn {
1752        &self.handler
1753    }
1754}
1755
1756pub trait Clock {
1757    /// Return the current time in milliseconds.
1758    fn now_ms(&self) -> u64;
1759
1760    /// Return the current time in nanoseconds.
1761    ///
1762    /// The default implementation derives this from [`Clock::now_ms`].
1763    fn now_ns(&self) -> u64 {
1764        self.now_ms().saturating_mul(1_000_000)
1765    }
1766}
1767
1768impl<T: Fn() -> u64> Clock for T {
1769    #[inline]
1770    fn now_ms(&self) -> u64 {
1771        self()
1772    }
1773}
1774
1775#[cfg(feature = "std")]
1776#[derive(Debug)]
1777struct StdMonotonicClock {
1778    start: Instant,
1779}
1780
1781#[cfg(feature = "std")]
1782impl Default for StdMonotonicClock {
1783    fn default() -> Self {
1784        Self {
1785            start: Instant::now(),
1786        }
1787    }
1788}
1789
1790#[cfg(feature = "std")]
1791impl Clock for StdMonotonicClock {
1792    fn now_ms(&self) -> u64 {
1793        u64::try_from(self.start.elapsed().as_millis()).unwrap_or(u64::MAX)
1794    }
1795
1796    fn now_ns(&self) -> u64 {
1797        u64::try_from(self.start.elapsed().as_nanos()).unwrap_or(u64::MAX)
1798    }
1799}
1800
1801/// Router-level E2E cryptography behavior.
1802#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1803pub enum RouterE2eEncryptionMode {
1804    /// Do not use E2E cryptography. Data types marked `RequireOn` are rejected.
1805    Disabled,
1806    /// Use E2E cryptography only for data types that require it.
1807    RequiredOnly,
1808    /// Use E2E cryptography for required and preferred data types.
1809    Preferred,
1810    /// Require E2E cryptography for every non-control data type.
1811    ForceAll,
1812}
1813
1814pub type NodeAddress = u32;
1815pub type P2pPort = u16;
1816pub type P2pStreamId = u32;
1817
1818const P2P_STREAM_MAGIC: [u8; 4] = *b"SDSP";
1819const P2P_STREAM_VERSION: u8 = 1;
1820const P2P_STREAM_SYN: u8 = 0x01;
1821const P2P_STREAM_ACK: u8 = 0x02;
1822const P2P_STREAM_FIN: u8 = 0x04;
1823const P2P_STREAM_RST: u8 = 0x08;
1824const P2P_STREAM_DATA: u8 = 0x10;
1825
1826#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1827pub enum AddressAssignmentMode {
1828    Dynamic,
1829    Requested(NodeAddress),
1830    Static(NodeAddress),
1831}
1832
1833impl AddressAssignmentMode {
1834    #[inline]
1835    #[cfg(feature = "discovery")]
1836    fn mode_code(self) -> u8 {
1837        match self {
1838            Self::Dynamic => 0,
1839            Self::Requested(_) => 1,
1840            Self::Static(_) => 2,
1841        }
1842    }
1843
1844    #[inline]
1845    fn requested_address(self) -> NodeAddress {
1846        match self {
1847            Self::Dynamic => 0,
1848            Self::Requested(addr) | Self::Static(addr) => addr,
1849        }
1850    }
1851}
1852
1853#[derive(Debug, Clone, PartialEq, Eq)]
1854pub enum AddressChangeReason {
1855    Configured,
1856    DynamicConflict,
1857    RequestedConflict,
1858    StaticConflict,
1859    HostnameConflict,
1860}
1861
1862#[derive(Debug, Clone, PartialEq, Eq)]
1863pub struct AddressChange {
1864    pub old_hostname: Arc<str>,
1865    pub new_hostname: Arc<str>,
1866    pub old_address: NodeAddress,
1867    pub new_address: NodeAddress,
1868    pub reason: AddressChangeReason,
1869}
1870
1871#[derive(Debug, Clone, PartialEq, Eq)]
1872pub struct AddressBookEntry {
1873    pub hostname: Arc<str>,
1874    pub address: NodeAddress,
1875    pub requested_address: NodeAddress,
1876    pub mode: AddressAssignmentMode,
1877    pub birth_ms: u64,
1878    pub owner_hash: u64,
1879    pub last_seen_ms: u64,
1880}
1881
1882#[cfg(feature = "discovery")]
1883struct PreparedAddressRecords {
1884    peer: Option<Box<AddressBookEntry>>,
1885    local: Option<Box<AddressBookEntry>>,
1886}
1887
1888pub struct P2pMessage<'a> {
1889    pub source_hostname: &'a str,
1890    pub source_address: NodeAddress,
1891    pub source_port: P2pPort,
1892    pub destination_port: P2pPort,
1893    pub payload: &'a [u8],
1894}
1895
1896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1897pub enum P2pStreamEventKind {
1898    Accepted,
1899    Connected,
1900    Data,
1901    Closed,
1902    Reset,
1903}
1904
1905pub struct P2pStreamEvent<'a> {
1906    pub kind: P2pStreamEventKind,
1907    pub stream_id: P2pStreamId,
1908    pub peer_stream_id: P2pStreamId,
1909    pub sequence: u32,
1910    pub peer_hostname: &'a str,
1911    pub peer_address: NodeAddress,
1912    pub local_port: P2pPort,
1913    pub peer_port: P2pPort,
1914    pub payload: &'a [u8],
1915}
1916
1917struct P2pDecoded<'a> {
1918    source_hostname: &'a str,
1919    source_address: NodeAddress,
1920    source_port: P2pPort,
1921    destination_port: P2pPort,
1922    payload: &'a [u8],
1923}
1924
1925#[derive(Debug, Clone)]
1926struct P2pStreamDecoded<'a> {
1927    flags: u8,
1928    source_stream_id: P2pStreamId,
1929    destination_stream_id: P2pStreamId,
1930    sequence: u32,
1931    payload: &'a [u8],
1932}
1933
1934type AddressChangeFn = dyn Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static;
1935
1936#[derive(Clone)]
1937struct AddressChangeHandler {
1938    handler: Arc<AddressChangeFn>,
1939}
1940
1941impl Debug for AddressChangeHandler {
1942    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1943        f.write_str("AddressChangeHandler(<handler>)")
1944    }
1945}
1946
1947type P2pPortHandlerFn = dyn Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1948
1949#[derive(Clone)]
1950struct P2pPortHandler {
1951    handler: Arc<P2pPortHandlerFn>,
1952}
1953
1954impl Debug for P2pPortHandler {
1955    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1956        f.write_str("P2pPortHandler(<handler>)")
1957    }
1958}
1959
1960type P2pStreamHandlerFn = dyn Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static;
1961
1962#[derive(Clone)]
1963struct P2pStreamHandler {
1964    handler: Arc<P2pStreamHandlerFn>,
1965}
1966
1967impl Debug for P2pStreamHandler {
1968    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
1969        f.write_str("P2pStreamHandler(<handler>)")
1970    }
1971}
1972
1973#[derive(Debug, Clone)]
1974struct P2pStreamSession {
1975    peer_hostname: Arc<str>,
1976    peer_address: NodeAddress,
1977    local_port: P2pPort,
1978    peer_port: P2pPort,
1979    peer_stream_id: P2pStreamId,
1980    next_sequence: u32,
1981    connected: bool,
1982}
1983
1984#[derive(Debug, Clone)]
1985struct PendingP2pStreamEvent {
1986    handlers: Vec<P2pStreamHandler>,
1987    kind: P2pStreamEventKind,
1988    stream_id: P2pStreamId,
1989    peer_stream_id: P2pStreamId,
1990    peer_hostname: Arc<str>,
1991    peer_address: NodeAddress,
1992    local_port: P2pPort,
1993    peer_port: P2pPort,
1994}
1995
1996#[derive(Debug, Clone)]
1997pub struct RouterConfig {
1998    role: crate::routing_config::RoutingRole,
1999    /// Handlers for local endpoints.
2000    handlers: Arc<[EndpointHandler]>,
2001    /// Whether to enable reliable ordering/ACKs for reliable data types.
2002    reliable_enabled: bool,
2003    /// Optional per-router sender override.
2004    sender: Option<Arc<str>>,
2005    /// Address assignment policy used for P2P routing.
2006    address_mode: AddressAssignmentMode,
2007    /// Callbacks invoked when conflict resolution changes this router's identity.
2008    address_change_handlers: Arc<[AddressChangeHandler]>,
2009    /// End-to-end cryptography behavior for user data.
2010    e2e_encryption: RouterE2eEncryptionMode,
2011    /// Application-defined key id passed to the cryptography provider.
2012    #[cfg_attr(not(feature = "cryptography"), allow(dead_code))]
2013    e2e_key_id: u32,
2014    memory: RuntimeMemoryConfig,
2015    #[cfg(feature = "discovery")]
2016    preferred_discovery_master: Option<Arc<str>>,
2017    #[cfg(feature = "timesync")]
2018    timesync: Option<TimeSyncConfig>,
2019}
2020
2021impl RouterConfig {
2022    pub(crate) fn relay() -> Self {
2023        let mut cfg = Self::default().with_sender("RELAY");
2024        cfg.role = crate::routing_config::RoutingRole::Relay;
2025        cfg
2026    }
2027
2028    pub(crate) fn configured_sender(&self) -> Arc<str> {
2029        self.sender
2030            .clone()
2031            .unwrap_or_else(|| Arc::from(runtime_device_identifier()))
2032    }
2033
2034    /// Default router E2E mode for this build.
2035    ///
2036    /// Builds with cryptography prefer encrypted payloads automatically for data types that request
2037    /// it. Minimal builds without cryptography stay disabled and reject `RequireOn` traffic.
2038    pub fn default_e2e_encryption_mode() -> RouterE2eEncryptionMode {
2039        #[cfg(feature = "cryptography")]
2040        {
2041            RouterE2eEncryptionMode::Preferred
2042        }
2043        #[cfg(not(feature = "cryptography"))]
2044        {
2045            RouterE2eEncryptionMode::Disabled
2046        }
2047    }
2048
2049    /// Create a new router configuration with the specified local endpoint handlers.
2050    pub fn new<H>(handlers: H) -> Self
2051    where
2052        H: Into<Arc<[EndpointHandler]>>,
2053    {
2054        let handlers: Arc<[EndpointHandler]> = handlers.into();
2055        assert!(
2056            handlers
2057                .iter()
2058                .all(|handler| !endpoint_is_router_internal(handler.endpoint)),
2059            "reserved internal endpoint handlers must not be user-registered"
2060        );
2061        Self {
2062            role: crate::routing_config::RoutingRole::Router,
2063            handlers,
2064            reliable_enabled: true,
2065            sender: None,
2066            address_mode: AddressAssignmentMode::Dynamic,
2067            address_change_handlers: Arc::from([]),
2068            e2e_encryption: Self::default_e2e_encryption_mode(),
2069            e2e_key_id: 0,
2070            memory: RuntimeMemoryConfig::default(),
2071            #[cfg(feature = "discovery")]
2072            preferred_discovery_master: None,
2073            #[cfg(feature = "timesync")]
2074            timesync: None,
2075        }
2076    }
2077
2078    /// Enable or disable reliable delivery for this router instance.
2079    pub fn with_reliable_enabled(mut self, enabled: bool) -> Self {
2080        self.reliable_enabled = enabled;
2081        self
2082    }
2083
2084    /// Override the sender identifier for this router instance.
2085    pub fn with_sender<S: AsRef<str>>(mut self, sender: S) -> Self {
2086        self.sender = Some(Arc::from(sender.as_ref()));
2087        self
2088    }
2089
2090    /// Alias for `with_sender`; discovery uses this name for P2P service resolution.
2091    pub fn with_hostname<S: AsRef<str>>(self, hostname: S) -> Self {
2092        self.with_sender(hostname)
2093    }
2094
2095    /// Request a dynamic address assigned by the discovered network authority.
2096    pub fn with_dynamic_address(mut self) -> Self {
2097        self.address_mode = AddressAssignmentMode::Dynamic;
2098        self
2099    }
2100
2101    /// Request a preferred address. It is kept when unique and reassigned on conflict.
2102    pub fn with_requested_address(mut self, address: NodeAddress) -> Self {
2103        self.address_mode = AddressAssignmentMode::Requested(address);
2104        self
2105    }
2106
2107    /// Require a static address. If two static nodes conflict, the oldest keeps it.
2108    pub fn with_static_address(mut self, address: NodeAddress) -> Self {
2109        self.address_mode = AddressAssignmentMode::Static(address);
2110        self
2111    }
2112
2113    /// Register a callback for local address or hostname changes after conflict resolution.
2114    pub fn on_address_change<F>(mut self, f: F) -> Self
2115    where
2116        F: Fn(AddressChange) -> TelemetryResult<()> + Send + Sync + 'static,
2117    {
2118        let mut handlers = self.address_change_handlers.to_vec();
2119        handlers.push(AddressChangeHandler {
2120            handler: Arc::new(f),
2121        });
2122        self.address_change_handlers = Arc::from(handlers);
2123        self
2124    }
2125
2126    /// Configure this router's end-to-end cryptography policy.
2127    pub fn with_e2e_encryption(mut self, mode: RouterE2eEncryptionMode) -> Self {
2128        self.e2e_encryption = mode;
2129        self
2130    }
2131
2132    /// Configure the application-defined key id used for E2E encrypted payloads.
2133    pub fn with_e2e_key_id(mut self, key_id: u32) -> Self {
2134        self.e2e_key_id = key_id;
2135        self
2136    }
2137
2138    /// Override per-router queue and dedupe memory limits.
2139    pub fn with_memory_config(mut self, memory: RuntimeMemoryConfig) -> TelemetryResult<Self> {
2140        memory.validate()?;
2141        self.memory = memory;
2142        Ok(self)
2143    }
2144
2145    /// Prefer a named router as discovery master whenever it is visible.
2146    ///
2147    /// All routers in a deployment should use the same preference. If that
2148    /// router is absent, normal topology-based election is used. Discovery
2149    /// leadership remains independent from time-sync source priority.
2150    #[cfg(feature = "discovery")]
2151    pub fn with_preferred_discovery_master<S: AsRef<str>>(mut self, hostname: S) -> Self {
2152        let hostname = hostname.as_ref();
2153        self.preferred_discovery_master = if hostname.is_empty() {
2154            None
2155        } else {
2156            Some(Arc::from(hostname))
2157        };
2158        self
2159    }
2160
2161    #[cfg(feature = "timesync")]
2162    /// Enables and configures built-in time synchronization for this router.
2163    pub fn with_timesync(mut self, cfg: TimeSyncConfig) -> Self {
2164        self.timesync = Some(cfg);
2165        self
2166    }
2167
2168    #[inline]
2169    /// Check if the specified endpoint is local to this router.
2170    fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
2171        if endpoint_is_router_internal(ep) {
2172            return false;
2173        }
2174        self.handlers.iter().any(|h| h.endpoint == ep)
2175    }
2176
2177    #[inline]
2178    fn reliable_enabled(&self) -> bool {
2179        self.reliable_enabled
2180    }
2181
2182    #[inline]
2183    fn address_mode(&self) -> AddressAssignmentMode {
2184        self.address_mode
2185    }
2186
2187    #[inline]
2188    fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
2189        self.e2e_encryption
2190    }
2191
2192    #[cfg(feature = "cryptography")]
2193    #[inline]
2194    fn e2e_key_id(&self) -> u32 {
2195        self.e2e_key_id
2196    }
2197
2198    #[cfg(feature = "timesync")]
2199    #[inline]
2200    fn timesync_config(&self) -> Option<TimeSyncConfig> {
2201        self.timesync
2202    }
2203
2204    #[inline]
2205    pub(crate) fn memory_config(&self) -> RuntimeMemoryConfig {
2206        self.memory
2207    }
2208}
2209
2210impl Default for RouterConfig {
2211    fn default() -> Self {
2212        Self {
2213            role: crate::routing_config::RoutingRole::Router,
2214            handlers: Arc::from([]),
2215            reliable_enabled: true,
2216            sender: None,
2217            address_mode: AddressAssignmentMode::Dynamic,
2218            address_change_handlers: Arc::from([]),
2219            e2e_encryption: Self::default_e2e_encryption_mode(),
2220            e2e_key_id: 0,
2221            memory: RuntimeMemoryConfig::default(),
2222            #[cfg(feature = "discovery")]
2223            preferred_discovery_master: None,
2224            #[cfg(feature = "timesync")]
2225            timesync: None,
2226        }
2227    }
2228}
2229
2230/// Retain application configuration only for a participating router. Both
2231/// roles consume the same builder; a forwarding-only core keeps no dormant
2232/// handler, address-assignment, encryption-origin or local-clock configuration.
2233#[derive(Debug)]
2234struct ActiveRoutingConfig {
2235    role: crate::routing_config::RoutingRole,
2236    reliable_enabled: bool,
2237    local: Option<Box<RouterConfig>>,
2238}
2239impl ActiveRoutingConfig {
2240    fn new(cfg: RouterConfig) -> Self {
2241        let role = cfg.role;
2242        Self {
2243            role,
2244            reliable_enabled: cfg.reliable_enabled(),
2245            local: role.local_services().then(|| Box::new(cfg)),
2246        }
2247    }
2248    fn handlers(&self) -> &[EndpointHandler] {
2249        self.local.as_ref().map_or(&[], |cfg| cfg.handlers.as_ref())
2250    }
2251    fn address_change_handlers(&self) -> &[AddressChangeHandler] {
2252        self.local
2253            .as_ref()
2254            .map_or(&[], |cfg| cfg.address_change_handlers.as_ref())
2255    }
2256    fn is_local_endpoint(&self, ep: DataEndpoint) -> bool {
2257        self.local
2258            .as_ref()
2259            .is_some_and(|cfg| cfg.is_local_endpoint(ep))
2260    }
2261    fn reliable_enabled(&self) -> bool {
2262        self.reliable_enabled
2263    }
2264    fn e2e_encryption(&self) -> RouterE2eEncryptionMode {
2265        self.local
2266            .as_ref()
2267            .map_or(RouterE2eEncryptionMode::Disabled, |cfg| {
2268                cfg.e2e_encryption()
2269            })
2270    }
2271    #[cfg(feature = "cryptography")]
2272    fn e2e_key_id(&self) -> u32 {
2273        self.local.as_ref().map_or(0, |cfg| cfg.e2e_key_id())
2274    }
2275    #[cfg(feature = "timesync")]
2276    fn timesync_config(&self) -> Option<TimeSyncConfig> {
2277        self.local.as_ref().and_then(|cfg| cfg.timesync_config())
2278    }
2279}
2280
2281// -------------------- generic little-endian serialization --------------------
2282
2283pub trait LeBytes: Copy + Sized {
2284    const WIDTH: usize;
2285    fn write_le(self, out: &mut [u8]);
2286    fn from_le_slice(bytes: &[u8]) -> Self;
2287}
2288
2289impl_letype_num!(u8, 1);
2290impl_letype_num!(u16, 2);
2291impl_letype_num!(u32, 4);
2292impl_letype_num!(u64, 8);
2293impl_letype_num!(u128, 16);
2294impl_letype_num!(i8, 1);
2295impl_letype_num!(i16, 2);
2296impl_letype_num!(i32, 4);
2297impl_letype_num!(i64, 8);
2298impl_letype_num!(i128, 16);
2299impl_letype_num!(f32, 4);
2300impl_letype_num!(f64, 8);
2301
2302/// Encode a slice of `LeBytes` into a contiguous little-endian byte array.
2303pub(crate) fn encode_slice_le<T: LeBytes>(data: &[T]) -> Arc<[u8]> {
2304    let total = data.len() * T::WIDTH;
2305    let mut buf = vec![0u8; total];
2306
2307    for (i, v) in data.iter().copied().enumerate() {
2308        let start = i * T::WIDTH;
2309        v.write_le(&mut buf[start..start + T::WIDTH]);
2310    }
2311
2312    Arc::from(buf)
2313}
2314
2315/// Build an error payload for `TelemetryError` packets, respecting the
2316/// static/dynamic sizing rules from `message_meta`.
2317fn make_error_payload(msg: &str) -> crate::SharedBytes {
2318    let meta = message_meta(DataType::TelemetryError);
2319    match meta.element {
2320        MessageElement::Static(_, _, _) => {
2321            let max = get_needed_message_size(DataType::TelemetryError);
2322            let bytes = msg.as_bytes();
2323            let n = core::cmp::min(max, bytes.len());
2324            let mut buf = vec![0u8; max];
2325            if n > 0 {
2326                buf[..n].copy_from_slice(&bytes[..n]);
2327            }
2328            crate::shared_bytes::convert(Arc::<[u8]>::from(buf))
2329        }
2330        MessageElement::Dynamic(_, _) => {
2331            crate::shared_bytes::convert(Arc::<[u8]>::from(msg.as_bytes()))
2332        }
2333    }
2334}
2335
2336/// Generic raw logger function used by Router::log and Router::log_queue.
2337/// Builds a Packet from the provided data slice and passes it to the
2338/// provided transmission function.
2339fn log_raw<T, F>(
2340    sender: Arc<str>,
2341    ty: DataType,
2342    data: &[T],
2343    timestamp: u64,
2344    mut tx_function: F,
2345) -> TelemetryResult<()>
2346where
2347    T: LeBytes,
2348    F: FnMut(Packet) -> TelemetryResult<()>,
2349{
2350    let meta = message_meta(ty);
2351    let got = data
2352        .len()
2353        .checked_mul(T::WIDTH)
2354        .ok_or(TelemetryError::PacketTooLarge(
2355            "typed payload size overflow",
2356        ))?;
2357    if T::WIDTH == 0 {
2358        return Err(TelemetryError::BadArg);
2359    }
2360
2361    match meta.element {
2362        MessageElement::Static(_, _, _) => {
2363            let expected = crate::data_type_size(meta.element.data_type()) * meta.element;
2364            if got != expected {
2365                return Err(TelemetryError::SizeMismatch { expected, got });
2366            }
2367        }
2368        MessageElement::Dynamic(_, _) => {
2369            // checked_mul above already formed a whole number of typed values.
2370            // Packet construction still validates the actual schema encoding.
2371        }
2372    }
2373
2374    const INLINE_ENCODE_BYTES: usize = if crate::config::STACK_PAYLOAD_SIZE < u8::MAX as usize {
2375        crate::config::STACK_PAYLOAD_SIZE
2376    } else {
2377        u8::MAX as usize
2378    };
2379    let payload = if got <= INLINE_ENCODE_BYTES {
2380        // Bounded by the same configured inline capacity as Packet. Tiny C/Rust
2381        // publications need neither a temporary Vec nor a payload Arc.
2382        let mut bytes = [0u8; INLINE_ENCODE_BYTES];
2383        for (value, destination) in data
2384            .iter()
2385            .copied()
2386            .zip(bytes[..got].chunks_exact_mut(T::WIDTH))
2387        {
2388            value.write_le(destination);
2389        }
2390        crate::config::StandardSmallPayload::new(&bytes[..got])
2391    } else {
2392        crate::config::StandardSmallPayload::from_arc(encode_slice_le(data))
2393    };
2394    let pkt = Packet::new_local(ty, meta.endpoints_owned(), sender, timestamp, payload)?;
2395    tx_function(pkt)
2396}
2397
2398// Hosted diagnostics only: no trace branches, formatting or storage on MCUs.
2399#[cfg(feature = "std")]
2400fn reliable_trace_enabled() -> bool {
2401    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
2402    *ENABLED.get_or_init(|| {
2403        matches!(
2404            std::env::var("SEDSNET_TRACE_RELIABLE").as_deref(),
2405            Ok("1" | "true")
2406        )
2407    })
2408}
2409
2410/// Fallback printing for error messages when no local endpoints exist.
2411/// - With `std`: prints to stderr.
2412/// - Without `std`: forwards to `seds_error_msg` (platform-provided).
2413fn fallback_stdout(_msg: &str) {
2414    #[cfg(feature = "std")]
2415    {
2416        eprintln!("{}", _msg);
2417    }
2418    #[cfg(all(not(feature = "std"), target_os = "none"))]
2419    {
2420        // The length-delimited platform callback owns presentation. Forward
2421        // the existing message without allocating while reporting an error.
2422        unsafe {
2423            seds_error_msg(_msg.as_ptr(), _msg.len());
2424        }
2425    }
2426}
2427
2428// -------------------- Router --------------------
2429
2430/// Internal mutable state of the Router, protected by `RouterMutex`.
2431/// Holds the RX/TX queues and the recent-RX de-duplication set.
2432#[derive(Debug, Clone)]
2433/// Application services are allocated only for participating router roles.
2434/// Forwarding state and all wire ownership remain in the shared engine.
2435struct LocalServicesState {
2436    managed_variable_types: BTreeSet<u32>,
2437    managed_variable_permissions: BTreeMap<u32, NetworkVariablePermissions>,
2438    managed_variable_latest: BTreeMap<u32, Box<ManagedVariableCacheEntry>>,
2439    network_variable_update_handlers: BTreeMap<u32, Vec<NetworkVariableUpdateHandler>>,
2440    p2p_port_handlers: BTreeMap<P2pPort, Vec<P2pPortHandler>>,
2441    p2p_stream_handlers: BTreeMap<P2pPort, Vec<P2pStreamHandler>>,
2442    p2p_stream_sessions: BTreeMap<P2pStreamId, P2pStreamSession>,
2443    next_p2p_stream_id: P2pStreamId,
2444}
2445
2446#[derive(Default)]
2447struct ForwardedReceiptCache {
2448    destinations: BTreeMap<u64, BTreeSet<u64>>,
2449    order: VecDeque<u64>,
2450}
2451
2452struct RouterInner {
2453    #[cfg(feature = "compression")]
2454    codec_reserved_bytes: usize,
2455    memory: RuntimeMemoryConfig,
2456    sides: Vec<Option<RouterSide>>,
2457    route_overrides: RouterMap<(Option<RouterSideId>, RouterSideId), bool>,
2458    typed_route_overrides: RouterMap<(Option<RouterSideId>, u32, RouterSideId), bool>,
2459    route_weights: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2460    route_priorities: RouterMap<(Option<RouterSideId>, RouterSideId), u32>,
2461    source_route_modes: RouterMap<Option<RouterSideId>, RouteSelectionMode>,
2462    route_selection_cursors: RouterMap<Option<RouterSideId>, u64>,
2463    adaptive_route_stats: RouterMap<RouterSideId, AdaptiveRouteStats>,
2464    // Own each large side value once and keep sparse index entries pointer-sized.
2465    // Embedded side indexes use compact sorted vectors; desktop indexes use B-trees.
2466    side_runtime_stats: RouterMap<RouterSideId, Box<SideRuntimeStatsInner>>,
2467    side_transport: RouterMap<RouterSideId, Box<SideTransportState>>,
2468    local_services: Option<Box<LocalServicesState>>,
2469    local_address: AddressBookEntry,
2470    address_book: BTreeMap<String, Box<AddressBookEntry>>,
2471    address_by_value: BTreeMap<NodeAddress, String>,
2472    received_queue: BoundedDeque<RouterRxItem>,
2473    transmit_queue: BoundedDeque<TxQueued>,
2474    #[cfg(feature = "std")]
2475    active_queued_types: [(u32, u32); ACTIVE_QUEUED_TYPE_SLOTS],
2476    tx_priority_burst: u8,
2477    tx_policy_flags: u8,
2478    #[cfg(feature = "latest-value-egress")]
2479    latest_turn_phase: u8,
2480    #[cfg(feature = "latest-value-egress")]
2481    latest_turn_credit: i32,
2482    #[cfg(feature = "latest-value-egress")]
2483    latest_service_seq: u32,
2484    ordered_claim_seq: u32,
2485    rx_priority_burst: u8,
2486    recent_rx: BoundedDeque<u64>,
2487    reliable_tx: BTreeMap<(RouterSideId, u32), ReliableTxState>,
2488    reliable_rx: BTreeMap<(RouterSideId, u32), ReliableRxState>,
2489    reliable_return_routes: BTreeMap<u64, ReliableReturnRouteState>,
2490    local_admission_sequence: u64,
2491    reliable_return_route_order: VecDeque<u64>,
2492    forwarded_receipts: Option<Box<ForwardedReceiptCache>>,
2493    end_to_end_reliable_tx: BTreeMap<u64, Box<EndToEndReliableSent>>,
2494    end_to_end_reliable_tx_order: VecDeque<u64>,
2495    total_handler_failures: u64,
2496    total_handler_retries: u64,
2497    #[cfg(feature = "discovery")]
2498    discovery_routes: RouterMap<RouterSideId, DiscoverySideState>,
2499    #[cfg(feature = "discovery")]
2500    discovery_cadence: DiscoveryCadenceState,
2501    #[cfg(feature = "discovery")]
2502    discovery_side_throttle: RouterMap<RouterSideId, DiscoverySideThrottleState>,
2503    #[cfg(feature = "discovery")]
2504    schema_response_sequence: u64,
2505    #[cfg(feature = "discovery")]
2506    preferred_discovery_master: Option<Arc<str>>,
2507    #[cfg(all(feature = "discovery", feature = "timesync"))]
2508    timesync_side_throttle: RouterMap<RouterSideId, TimeSyncSideThrottleState>,
2509}
2510#[derive(Debug, Clone)]
2511struct ManagedVariableCacheEntry {
2512    packet: Packet,
2513    sender_address: u32,
2514    #[cfg(feature = "discovery")]
2515    cached_at_ms: u64,
2516    // First actual ingress for this logical observation. Unlike the bounded
2517    // ACK return cache, this remains with a quiet stable value. Zero is only
2518    // the Option niche; a side index is represented by checked index + 1.
2519    #[cfg(feature = "discovery")]
2520    upstream: Option<core::num::NonZeroUsize>,
2521}
2522
2523type NetworkVariableUpdateFn = dyn Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static;
2524
2525#[derive(Clone)]
2526struct NetworkVariableUpdateHandler {
2527    handler: Arc<NetworkVariableUpdateFn>,
2528}
2529
2530impl Debug for NetworkVariableUpdateHandler {
2531    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
2532        f.write_str("NetworkVariableUpdateHandler(<handler>)")
2533    }
2534}
2535
2536/// Local permission policy for a network-managed variable.
2537#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2538pub struct NetworkVariablePermissions {
2539    pub read: bool,
2540    pub write: bool,
2541}
2542
2543impl NetworkVariablePermissions {
2544    pub const NONE: Self = Self {
2545        read: false,
2546        write: false,
2547    };
2548    pub const READ_ONLY: Self = Self {
2549        read: true,
2550        write: false,
2551    };
2552    pub const WRITE_ONLY: Self = Self {
2553        read: false,
2554        write: true,
2555    };
2556    pub const READ_WRITE: Self = Self {
2557        read: true,
2558        write: true,
2559    };
2560}
2561
2562#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2563enum RouterQueueKind {
2564    Received,
2565    Transmit,
2566}
2567
2568#[cfg(feature = "compression")]
2569impl crate::payload_codec::QueueBudget for RouterInner {
2570    fn codec_reserved(&mut self) -> &mut usize {
2571        &mut self.codec_reserved_bytes
2572    }
2573    fn codec_used(&self) -> usize {
2574        self.shared_queue_bytes_used()
2575    }
2576    fn codec_limit(&self) -> usize {
2577        self.memory.max_queue_budget
2578    }
2579}
2580
2581impl RouterInner {
2582    fn is_managed_variable(&self, ty: u32) -> bool {
2583        self.local_services
2584            .as_ref()
2585            .is_some_and(|local| local.managed_variable_types.contains(&ty))
2586    }
2587
2588    fn local_cache_bytes(&self) -> usize {
2589        self.local_services
2590            .iter()
2591            .flat_map(|local| local.managed_variable_latest.values())
2592            .map(|entry| entry.packet.byte_cost())
2593            .sum()
2594    }
2595
2596    fn replay_counters(&self) -> (usize, usize) {
2597        self.transmit_queue
2598            .iter()
2599            .filter(|q| q.item.replay_marker_key().is_some())
2600            .fold((0usize, 0usize), |(n, bytes), q| {
2601                (n + 1, bytes.saturating_add(q.byte_cost()))
2602            })
2603    }
2604
2605    fn local_cache_len(&self) -> usize {
2606        self.local_services
2607            .as_ref()
2608            .map_or(0, |local| local.managed_variable_latest.len())
2609    }
2610
2611    #[cfg(all(feature = "compression", feature = "discovery"))]
2612    fn reclaim_codec_metadata(&mut self) -> bool {
2613        self.side_transport.values_mut().any(|transport| {
2614            transport
2615                .payload_codec
2616                .evict_for_budget(DISCOVERY_ROUTE_TTL_MS)
2617        })
2618    }
2619
2620    /// Prepare required plaintext before publishing its queue owner. The state
2621    /// lock prevents another producer taking the checked expanded byte credit.
2622    /// A refused optional preparation leaves the original accepted work owned;
2623    /// dispatch will backpressure it until the required decoder scratch fits.
2624    #[cfg(feature = "compression")]
2625    fn prepare_codec_queue_owner(&mut self, side: Option<usize>, data: &mut RouterItem) {
2626        let RouterItem::Packed(bytes) = data else {
2627            return;
2628        };
2629        if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
2630            return;
2631        }
2632        let capable = {
2633            #[cfg(feature = "discovery")]
2634            {
2635                side.and_then(|side| self.side_transport.get(&side))
2636                    .is_some_and(|transport| {
2637                        transport.payload_codec.retention_can_stay_compressed()
2638                    })
2639            }
2640            #[cfg(not(feature = "discovery"))]
2641            {
2642                let _ = side;
2643                false
2644            }
2645        };
2646        if capable {
2647            return;
2648        }
2649        let Ok(required) = wire_format::decoded_frame_upper_bound(bytes.as_ref()) else {
2650            return;
2651        };
2652        if self
2653            .shared_queue_bytes_used()
2654            .checked_add(required)
2655            .and_then(|used| {
2656                used.checked_add(
2657                    size_of::<crate::SharedBytes>()
2658                        + size_of::<bool>()
2659                        + size_of::<u8>()
2660                        + size_of::<u64>(),
2661                )
2662            })
2663            .is_none_or(|used| used > self.memory.max_queue_budget)
2664        {
2665            return;
2666        }
2667        if let Ok(Some(raw)) = wire_format::payload_encoding_for_retention(
2668            bytes.as_ref(),
2669            self.memory.max_queue_budget,
2670        ) {
2671            *bytes = crate::shared_bytes::convert(raw);
2672        }
2673    }
2674
2675    #[cfg(feature = "discovery")]
2676    fn schema_response_owner(item: &RouterTxItem) -> Option<(RouterSideId, u64)> {
2677        match item {
2678            RouterTxItem::ToSide { dst, data, .. }
2679                if Router::item_data_type(data).ok() == Some(DataType::DiscoverySchema) =>
2680            {
2681                Some((*dst, Router::get_hash(data)))
2682            }
2683            _ => None,
2684        }
2685    }
2686
2687    #[cfg(feature = "discovery")]
2688    fn schema_response_owner_lost(
2689        responses: &mut RouterMap<RouterSideId, DiscoverySideThrottleState>,
2690        owner: Option<(RouterSideId, u64)>,
2691    ) {
2692        if let Some((side, id)) = owner
2693            && let Some(response) = responses.get_mut(&side)
2694            && response.schema_response_active()
2695            && response.schema_response_phase == SchemaResponsePhase::Owned
2696            && response.schema_response_packet_id == id
2697        {
2698            // The route prefix was accepted for this generation. Quota loss
2699            // may restart its self-describing catalog without replaying that
2700            // prefix and preempting another side's queued catalog forever.
2701            response.schema_response_packet_id = 0;
2702            response.set_schema_response_started(false);
2703            response.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
2704        }
2705    }
2706
2707    fn retain_transmit<F>(&mut self, keep: F)
2708    where
2709        F: FnMut(&TxQueued) -> bool,
2710    {
2711        let hop = &mut self.reliable_tx;
2712        let end = &mut self.end_to_end_reliable_tx;
2713        #[cfg(feature = "discovery")]
2714        let responses = &mut self.discovery_side_throttle;
2715        self.transmit_queue.retain_observed(keep, |evicted| {
2716            #[cfg(feature = "discovery")]
2717            Self::schema_response_owner_lost(responses, Self::schema_response_owner(&evicted.item));
2718            if let Some(key) = evicted.item.replay_marker_key() {
2719                Self::set_replay_queued(hop, end, key, false);
2720            }
2721        });
2722    }
2723
2724    #[inline(never)]
2725    fn claim_index(&self, token: u32) -> Option<usize> {
2726        self.transmit_queue.iter().position(|item| {
2727            matches!(
2728                item.delivery_state,
2729                QueuedDelivery::ClaimedOrdered | QueuedDelivery::ClaimedUnordered
2730            ) && if item.disposable & TX_E2E_CHILD != 0 {
2731                matches!(&item.item, RouterTxItem::ToSide { src: Some(value), .. } if *value == token as usize)
2732            } else { item.deferred_since_ms == u64::from(token) }
2733        })
2734    }
2735
2736    #[inline(never)]
2737    fn refresh_retry_hint(&mut self) {
2738        let pending = self.transmit_queue.iter().any(|item| {
2739            matches!(
2740                item.delivery_state,
2741                QueuedDelivery::ClaimedOrdered
2742                    | QueuedDelivery::ClaimedUnordered
2743                    | QueuedDelivery::DeferredOrdered
2744            )
2745        });
2746        self.tx_policy_flags = (self.tx_policy_flags & !1) | u8::from(pending);
2747    }
2748
2749    fn cancel_accepted_type(&mut self, ty: u32) {
2750        self.retain_transmit(|item| {
2751            !matches!(&item.item,
2752            RouterTxItem::Broadcast(_) if item.accepted_type == ty)
2753        });
2754    }
2755
2756    fn replay_is_queued(&self, key: ReplayMarkerKey) -> Option<bool> {
2757        match key {
2758            ReplayMarkerKey::EndToEnd(id) => self.end_to_end_reliable_tx.get(&id).map(|s| s.queued),
2759            ReplayMarkerKey::Hop(side, ty, seq) => self
2760                .reliable_tx
2761                .get(&(side, ty.as_u32()))
2762                .and_then(|tx| tx.sent.get(&seq))
2763                .map(|s| s.queued),
2764        }
2765    }
2766    fn set_replay_queued(
2767        hop: &mut BTreeMap<(RouterSideId, u32), ReliableTxState>,
2768        end: &mut BTreeMap<u64, Box<EndToEndReliableSent>>,
2769        key: ReplayMarkerKey,
2770        queued: bool,
2771    ) {
2772        let flag = match key {
2773            ReplayMarkerKey::EndToEnd(id) => end.get_mut(&id).map(|s| &mut s.queued),
2774            ReplayMarkerKey::Hop(side, ty, seq) => hop
2775                .get_mut(&(side, ty.as_u32()))
2776                .and_then(|tx| tx.sent.get_mut(&seq))
2777                .map(|s| &mut s.queued),
2778        };
2779        if let Some(flag) = flag {
2780            *flag = queued;
2781        }
2782    }
2783
2784    #[cfg(any(feature = "std", feature = "fair-admission"))]
2785    fn fair_bulk_owner(data: &RouterItem, priority: u8) -> Option<u32> {
2786        match data {
2787            RouterItem::Packet(packet) => crate::admission_fairness::packet_owner(packet, priority),
2788            RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2789                crate::admission_fairness::packed_owner(view, priority)
2790            })
2791            .ok()
2792            .flatten(),
2793        }
2794    }
2795    #[cfg(any(feature = "std", feature = "fair-admission"))]
2796    fn tx_bulk_owner(item: &TxQueued) -> Option<u32> {
2797        if item.disposable & TX_E2E_CHILD != 0 {
2798            return None;
2799        }
2800        #[cfg(feature = "latest-value-egress")]
2801        if item.disposable & TX_HISTORY_OWNED != 0 {
2802            return None;
2803        }
2804        match &item.item {
2805            RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2806                Self::fair_bulk_owner(data, item.priority)
2807            }
2808            _ => None,
2809        }
2810    }
2811    #[cfg(any(feature = "std", feature = "fair-admission"))]
2812    fn bulk_stream(data: &RouterItem) -> Option<u32> {
2813        match data {
2814            RouterItem::Packet(packet) => Some(packet.data_type().as_u32()),
2815            RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
2816                wire_format::scheduling_type(view).map(|ty| ty.as_u32())
2817            })
2818            .ok()
2819            .flatten(),
2820        }
2821    }
2822    #[cfg(any(feature = "std", feature = "fair-admission"))]
2823    fn tx_bulk_stream(item: &TxQueued) -> Option<u32> {
2824        match &item.item {
2825            RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
2826                Self::bulk_stream(data)
2827            }
2828            _ => None,
2829        }
2830    }
2831    #[cfg(any(feature = "std", feature = "fair-admission"))]
2832    fn bulk_loads(&self, priority: u8) -> crate::admission_fairness::Loads {
2833        let mut loads = crate::admission_fairness::Loads::new();
2834        for item in self.received_queue.iter() {
2835            if item.priority <= priority
2836                && let Some(owner) = Self::fair_bulk_owner(&item.data, item.priority)
2837            {
2838                loads.record(owner, item.byte_cost());
2839            }
2840        }
2841        for item in self.transmit_queue.iter() {
2842            if item.priority <= priority
2843                && let Some(owner) = Self::tx_bulk_owner(item)
2844            {
2845                loads.record(owner, item.byte_cost());
2846            }
2847        }
2848        loads
2849    }
2850    #[cfg(any(feature = "std", feature = "fair-admission"))]
2851    fn competing_bulk_owner(&self, incoming: u32, priority: u8) -> Option<u32> {
2852        self.bulk_loads(priority).dominant_other(incoming)
2853    }
2854    #[cfg(any(feature = "std", feature = "fair-admission"))]
2855    fn reclaim_bulk_from(
2856        &mut self,
2857        owner: u32,
2858        priority: u8,
2859        incoming_stream: Option<u32>,
2860    ) -> bool {
2861        let mut streams = crate::admission_fairness::Loads::new();
2862        for item in self.received_queue.iter() {
2863            if item.priority <= priority
2864                && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2865                && let Some(ty) = Self::bulk_stream(&item.data)
2866            {
2867                streams.record(ty, item.byte_cost());
2868            }
2869        }
2870        for item in self.transmit_queue.iter() {
2871            if item.priority <= priority
2872                && Self::tx_bulk_owner(item) == Some(owner)
2873                && let Some(ty) = Self::tx_bulk_stream(item)
2874            {
2875                streams.record(ty, item.byte_cost());
2876            }
2877        }
2878        let stream = streams.queue_victim(incoming_stream);
2879        let rx = self
2880            .received_queue
2881            .iter()
2882            .enumerate()
2883            .filter(|(_, item)| {
2884                item.priority <= priority
2885                    && Self::fair_bulk_owner(&item.data, item.priority) == Some(owner)
2886                    && stream.is_none_or(|ty| Self::bulk_stream(&item.data) == Some(ty))
2887            })
2888            .min_by_key(|(index, item)| (item.priority, *index))
2889            .map(|(index, item)| (index, item.priority));
2890        let tx = self
2891            .transmit_queue
2892            .iter()
2893            .enumerate()
2894            .filter(|(_, item)| {
2895                item.priority <= priority
2896                    && Self::tx_bulk_owner(item) == Some(owner)
2897                    && stream.is_none_or(|ty| Self::tx_bulk_stream(item) == Some(ty))
2898            })
2899            .min_by_key(|(index, item)| (item.priority, *index))
2900            .map(|(index, item)| (index, item.priority));
2901        match (rx, tx) {
2902            (Some((index, rp)), Some((_, tp))) if rp <= tp => {
2903                self.received_queue.remove_pos(index).is_some()
2904            }
2905            (Some((index, _)), None) => self.received_queue.remove_pos(index).is_some(),
2906            (_, Some((index, _))) => self.transmit_queue.remove_pos(index).is_some(),
2907            _ => false,
2908        }
2909    }
2910
2911    #[cfg(feature = "discovery")]
2912    fn topology_board_byte_cost(board: &TopologyBoardNode) -> usize {
2913        board
2914            .sender_id
2915            .len()
2916            .saturating_add(board.reachable_endpoints.len() * size_of::<DataEndpoint>())
2917            .saturating_add(
2918                board
2919                    .reachable_timesync_sources
2920                    .iter()
2921                    .map(|s| s.len())
2922                    .sum::<usize>(),
2923            )
2924            .saturating_add(board.connections.iter().map(|s| s.len()).sum::<usize>())
2925    }
2926
2927    #[cfg(feature = "discovery")]
2928    fn discovery_sender_byte_cost(sender: &str, state: &DiscoverySenderState) -> usize {
2929        sender
2930            .len()
2931            .saturating_add(state.reachable.len() * size_of::<DataEndpoint>())
2932            .saturating_add(state.advertised_reachable.len() * size_of::<DataEndpoint>())
2933            .saturating_add(state.reachable_network_variables.len() * size_of::<DataType>())
2934            .saturating_add(
2935                state
2936                    .reachable_timesync_sources
2937                    .iter()
2938                    .map(|s| s.len())
2939                    .sum::<usize>(),
2940            )
2941            .saturating_add(
2942                state
2943                    .advertised_reachable_timesync_sources
2944                    .iter()
2945                    .map(|s| s.len())
2946                    .sum::<usize>(),
2947            )
2948            .saturating_add(
2949                state
2950                    .topology_boards
2951                    .iter()
2952                    .map(Self::topology_board_byte_cost)
2953                    .sum::<usize>(),
2954            )
2955            .saturating_add(crate::peer_record::allocation_bytes::<DiscoverySenderState>())
2956    }
2957
2958    #[cfg(feature = "discovery")]
2959    fn discovery_route_byte_cost(side: RouterSideId, route: &DiscoverySideState) -> usize {
2960        size_of::<RouterSideId>()
2961            .saturating_add(size_of::<DiscoverySideState>())
2962            .saturating_add(route.reachable.len() * size_of::<DataEndpoint>())
2963            .saturating_add(route.reachable_network_variables.len() * size_of::<DataType>())
2964            .saturating_add(
2965                route.requested_network_variables.len() * size_of::<ManagedVariableSubscription>(),
2966            )
2967            .saturating_add(
2968                route
2969                    .reachable_timesync_sources
2970                    .iter()
2971                    .map(|s| s.len())
2972                    .sum::<usize>(),
2973            )
2974            .saturating_add(
2975                route
2976                    .announcers
2977                    .iter()
2978                    .map(|(sender, state)| Self::discovery_sender_byte_cost(sender, state))
2979                    .sum::<usize>(),
2980            )
2981            .saturating_add(side.saturating_sub(side))
2982    }
2983
2984    #[cfg(feature = "discovery")]
2985    fn discovery_bytes_used(&self) -> usize {
2986        self.discovery_routes
2987            .iter()
2988            .map(|(side, route)| Self::discovery_route_byte_cost(*side, route))
2989            .sum()
2990    }
2991
2992    #[inline]
2993    fn reliable_rx_buffered_bytes(&self) -> usize {
2994        self.reliable_rx
2995            .values()
2996            .flat_map(|state| state.buffered.values())
2997            .map(|bytes| size_of::<crate::SharedBytes>() + bytes.len())
2998            .sum()
2999    }
3000
3001    fn reliable_rx_buffer_len(&self) -> usize {
3002        self.reliable_rx
3003            .values()
3004            .map(|state| state.buffered.len())
3005            .sum()
3006    }
3007
3008    #[inline(never)]
3009    fn side_chunk_bytes_used(&self) -> usize {
3010        self.side_transport
3011            .values()
3012            .map(|side| side.rx_chunk_bytes)
3013            .sum()
3014    }
3015    fn discard_side_chunk(&mut self, side: RouterSideId, id: u32) -> bool {
3016        self.side_transport
3017            .get_mut(&side)
3018            .is_some_and(|state| state.take_chunk(id, true).is_some())
3019    }
3020    fn reclaim_oldest_side_chunk(&mut self, keep: Option<(RouterSideId, u32)>) -> bool {
3021        let mut victim = None;
3022        let mut time = u64::MAX;
3023        for (&side, state) in &self.side_transport {
3024            let preserved = match keep {
3025                Some((owner, id)) if owner == side => Some(id),
3026                _ => None,
3027            };
3028            if let Some((id, seen)) = SideChunkAssembly::oldest(&state.rx_chunks, preserved)
3029                && (victim.is_none() || seen < time)
3030            {
3031                time = seen;
3032                victim = Some((side, id));
3033            }
3034        }
3035        match victim {
3036            Some((side, id)) => self.discard_side_chunk(side, id),
3037            None => false,
3038        }
3039    }
3040    fn make_side_chunk_room(
3041        &mut self,
3042        side: RouterSideId,
3043        id: u32,
3044        index: u16,
3045        total: u16,
3046        payload_len: usize,
3047    ) -> TelemetryResult<()> {
3048        let budget = self
3049            .memory
3050            .max_queue_budget
3051            .saturating_sub(self.protected_metadata_bytes_used());
3052        let state = self
3053            .side_transport
3054            .get_mut(&side)
3055            .ok_or(TelemetryError::BadArg)?;
3056        let existing = state.rx_chunks.get(&id).map(|a| a.total);
3057        if existing.is_some_and(|old| old != 0 && old != total) {
3058            state.take_chunk(id, true);
3059            return Err(TelemetryError::Unpack("side chunk total mismatch"));
3060        }
3061        let before = state.rx_chunks.len();
3062        let room = SideChunkAssembly::make_room(
3063            &mut state.rx_chunks,
3064            &mut state.rx_chunk_bytes,
3065            id,
3066            index,
3067            total,
3068            payload_len,
3069            budget,
3070        );
3071        if before != state.rx_chunks.len() {
3072            state.rx_chunk_reclaims = state
3073                .rx_chunk_reclaims
3074                .saturating_add((before - state.rx_chunks.len()) as u32);
3075        }
3076        let cost = room?;
3077        if cost == 0 {
3078            return Ok(());
3079        }
3080        if existing.is_none()
3081            && state.rx_chunks.len() >= 4
3082            && let Some((oldest, _)) = SideChunkAssembly::oldest(&state.rx_chunks, None)
3083        {
3084            state.take_chunk(oldest, true);
3085        }
3086        while self.shared_queue_bytes_used().saturating_add(cost) > self.memory.max_queue_budget {
3087            #[cfg(any(feature = "std", feature = "fair-admission"))]
3088            if let Some(owner) = self.bulk_loads(199).queue_victim(None)
3089                && self.reclaim_bulk_from(owner, 199, None)
3090            {
3091                continue;
3092            }
3093            if self.reclaim_oldest_side_chunk(Some((side, id))) {
3094                continue;
3095            }
3096            // The sender retries this fragment, not the earlier accepted ones.
3097            // Keep the bounded partial assembly while protected queues drain.
3098            return Err(TelemetryError::Io("memory pressure"));
3099        }
3100        Ok(())
3101    }
3102
3103    #[inline(never)]
3104    fn protected_metadata_bytes_used(&self) -> usize {
3105        let codec_bytes = {
3106            #[cfg(all(feature = "compression", feature = "discovery"))]
3107            {
3108                self.side_transport
3109                    .values()
3110                    .map(|side| side.payload_codec.heap_bytes())
3111                    .sum::<usize>()
3112            }
3113            #[cfg(not(all(feature = "compression", feature = "discovery")))]
3114            {
3115                0usize
3116            }
3117        };
3118        let completion_bytes = {
3119            #[cfg(feature = "std")]
3120            {
3121                self.end_to_end_reliable_tx
3122                    .values()
3123                    .map(|sent| {
3124                        PENDING_TYPE_METADATA_BYTES
3125                            + usize::from(sent.completion_epoch != 0) * size_of::<u64>()
3126                    })
3127                    .sum::<usize>()
3128            }
3129            #[cfg(not(feature = "std"))]
3130            {
3131                0usize
3132            }
3133        };
3134        let return_route_growth = self
3135            .reliable_return_routes
3136            .len()
3137            .saturating_mul(RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES);
3138        codec_bytes
3139            .saturating_add(completion_bytes)
3140            .saturating_add(return_route_growth)
3141            .saturating_add(
3142                self.recent_rx
3143                    .max_bytes()
3144                    .saturating_add(self.reliable_rx_buffered_bytes())
3145                    .saturating_add(crate::config::schema_bytes_used())
3146                    .saturating_add({
3147                        #[cfg(feature = "discovery")]
3148                        {
3149                            self.discovery_bytes_used()
3150                        }
3151                        #[cfg(not(feature = "discovery"))]
3152                        {
3153                            0
3154                        }
3155                    }),
3156            )
3157    }
3158    #[inline]
3159    fn shared_queue_bytes_used(&self) -> usize {
3160        self.received_queue
3161            .bytes_used()
3162            .saturating_add(self.transmit_queue.bytes_used())
3163            .saturating_add(self.side_chunk_bytes_used())
3164            .saturating_add(self.protected_metadata_bytes_used())
3165            .saturating_add({
3166                #[cfg(feature = "compression")]
3167                {
3168                    self.codec_reserved_bytes
3169                }
3170                #[cfg(not(feature = "compression"))]
3171                {
3172                    0
3173                }
3174            })
3175    }
3176
3177    // A reliable original can be its payload's only owner before its first
3178    // dispatch. Only replay markers have a separately retained retry record.
3179    // Refuse new admission rather than silently discard an accepted original.
3180    fn classify_transmit_disposable(item: &TxQueued) -> bool {
3181        let data = match &item.item {
3182            RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => data,
3183            _ => return true,
3184        };
3185        match data {
3186            RouterItem::Packet(pkt) => {
3187                is_internal_control_type(pkt.data_type())
3188                    || (!is_reliable_type(pkt.data_type()) && pkt.wire_target_senders().is_empty())
3189            }
3190            RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3191                wire_format::peek_routing_frame_info(view).map_or(true, |frame| {
3192                    is_internal_control_type(frame.envelope.ty)
3193                        || (!is_reliable_type(frame.envelope.ty)
3194                            && frame.reliable.is_none()
3195                            && frame.envelope.target_senders().len() == 0)
3196                })
3197            })
3198            .unwrap_or(false),
3199        }
3200    }
3201
3202    fn transmit_eviction_candidate(queue: &BoundedDeque<TxQueued>) -> Option<usize> {
3203        queue
3204            .iter()
3205            .enumerate()
3206            .filter(|(_, queued)| queued.disposable & TX_DISPOSABLE != 0)
3207            .min_by_key(|(index, queued)| (queued.priority, core::cmp::Reverse(*index)))
3208            .map(|(index, _)| index)
3209    }
3210
3211    fn pop_shared_queue_item(&mut self, preferred: RouterQueueKind) -> bool {
3212        match preferred {
3213            // RX and TX queues are stored in descending priority order.  When
3214            // the shared byte budget is exhausted, evict from the tail so a
3215            // low-priority telemetry packet cannot displace discovery or a
3216            // managed-variable update at the head of the queue.
3217            RouterQueueKind::Received => self.received_queue.pop_lowest_priority().is_some(),
3218            RouterQueueKind::Transmit => {
3219                let Some(index) = Self::transmit_eviction_candidate(&self.transmit_queue) else {
3220                    return false;
3221                };
3222                let Some(item) = self.transmit_queue.remove_pos(index) else {
3223                    return false;
3224                };
3225                #[cfg(feature = "discovery")]
3226                Self::schema_response_owner_lost(
3227                    &mut self.discovery_side_throttle,
3228                    Self::schema_response_owner(&item.item),
3229                );
3230                if let Some(key) = item.item.replay_marker_key() {
3231                    Self::set_replay_queued(
3232                        &mut self.reliable_tx,
3233                        &mut self.end_to_end_reliable_tx,
3234                        key,
3235                        false,
3236                    );
3237                }
3238                true
3239            }
3240        }
3241    }
3242
3243    fn lower_priority_shared_queue(&self, incoming_priority: u8) -> Option<RouterQueueKind> {
3244        let received = self
3245            .received_queue
3246            .lowest_priority(|queued| queued.priority)
3247            .map(|priority| (RouterQueueKind::Received, priority));
3248        let transmit = self
3249            .transmit_queue
3250            .iter()
3251            .filter(|queued| queued.disposable & TX_DISPOSABLE != 0)
3252            .map(|queued| queued.priority)
3253            .min()
3254            .map(|priority| (RouterQueueKind::Transmit, priority));
3255        [received, transmit]
3256            .into_iter()
3257            .flatten()
3258            .filter(|(_, priority)| {
3259                *priority < incoming_priority
3260                    || (incoming_priority != 254 && *priority == incoming_priority)
3261            })
3262            .min_by_key(|(kind, priority)| {
3263                (
3264                    *priority,
3265                    if *kind == RouterQueueKind::Received {
3266                        0
3267                    } else {
3268                        1
3269                    },
3270                )
3271            })
3272            .map(|(kind, _)| kind)
3273    }
3274
3275    fn make_shared_queue_room(
3276        &mut self,
3277        incoming_cost: usize,
3278        incoming_priority: u8,
3279        _incoming: Option<&RouterItem>,
3280    ) -> TelemetryResult<()> {
3281        if incoming_cost > self.memory.max_queue_budget {
3282            return Err(TelemetryError::PacketTooLarge(
3283                "Item exceeds maximum shared queue budget",
3284            ));
3285        }
3286
3287        #[cfg(any(feature = "std", feature = "fair-admission"))]
3288        let mut fair_victim = None;
3289        while self.shared_queue_bytes_used().saturating_add(incoming_cost)
3290            > self.memory.max_queue_budget
3291        {
3292            #[cfg(all(feature = "compression", feature = "discovery"))]
3293            if self.reclaim_codec_metadata() {
3294                continue;
3295            }
3296
3297            #[cfg(any(feature = "std", feature = "fair-admission"))]
3298            {
3299                let incoming =
3300                    _incoming.and_then(|data| Self::fair_bulk_owner(data, incoming_priority));
3301                let victim = *fair_victim.get_or_insert_with(|| {
3302                    self.bulk_loads(incoming_priority.min(199))
3303                        .queue_victim(incoming)
3304                });
3305                if let Some(victim) = victim
3306                    && self.reclaim_bulk_from(
3307                        victim,
3308                        incoming_priority.min(199),
3309                        (incoming == Some(victim))
3310                            .then(|| _incoming.and_then(Self::bulk_stream))
3311                            .flatten(),
3312                    )
3313                {
3314                    continue;
3315                }
3316                if incoming.is_some() {
3317                    return Err(TelemetryError::Io("shared fair queue saturated"));
3318                }
3319            }
3320            let Some(victim) = self.lower_priority_shared_queue(incoming_priority) else {
3321                if incoming_priority >= 200 && self.reclaim_oldest_side_chunk(None) {
3322                    continue;
3323                }
3324                return Err(TelemetryError::Io("shared priority queue saturated"));
3325            };
3326            if !self.pop_shared_queue_item(victim) {
3327                return Err(TelemetryError::PacketTooLarge(
3328                    "Item exceeds maximum shared queue budget",
3329                ));
3330            }
3331        }
3332
3333        Ok(())
3334    }
3335
3336    #[cfg(feature = "discovery")]
3337    fn schema_growth_budget(&self) -> usize {
3338        // Queued packets are reclaimable. Existing reachability and active
3339        // reliable reassembly remain protected when admitting a new schema.
3340        let protected = self
3341            .recent_rx
3342            .max_bytes()
3343            .saturating_add(self.discovery_bytes_used())
3344            .saturating_add(self.reliable_rx_buffered_bytes())
3345            .saturating_add({
3346                #[cfg(feature = "compression")]
3347                {
3348                    self.codec_reserved_bytes
3349                }
3350                #[cfg(not(feature = "compression"))]
3351                {
3352                    0
3353                }
3354            });
3355        self.memory.max_queue_budget.saturating_sub(protected)
3356    }
3357
3358    #[cfg(feature = "discovery")]
3359    fn cancel_managed_refresh_side(&mut self, side: RouterSideId) {
3360        for (&ingress, route) in &mut self.discovery_routes {
3361            for request in &mut route.requested_network_variables {
3362                if ingress == side || request.upstream == side {
3363                    request.refresh_pending = false;
3364                }
3365            }
3366        }
3367        if let Some(local) = self.local_services.as_mut() {
3368            for entry in local.managed_variable_latest.values_mut() {
3369                if entry
3370                    .upstream
3371                    .is_some_and(|upstream| upstream.get() - 1 == side)
3372                {
3373                    entry.upstream = None;
3374                }
3375            }
3376        }
3377    }
3378
3379    #[cfg(feature = "discovery")]
3380    fn commit_discovery_route(
3381        &mut self,
3382        side: RouterSideId,
3383        route: DiscoverySideState,
3384    ) -> TelemetryResult<()> {
3385        let previous = self
3386            .discovery_routes
3387            .get(&side)
3388            .map_or(0, |route| Self::discovery_route_byte_cost(side, route));
3389        let incoming = Self::discovery_route_byte_cost(side, &route);
3390        let growth = incoming.saturating_sub(previous);
3391        self.make_shared_queue_room(growth, 255, None)?;
3392        self.discovery_routes.insert(side, route);
3393        Ok(())
3394    }
3395
3396    #[cfg(feature = "discovery")]
3397    fn fit_discovery_budget(&mut self) {
3398        while self.shared_queue_bytes_used() > self.memory.max_queue_budget {
3399            #[cfg(all(feature = "compression", feature = "discovery"))]
3400            if self.reclaim_codec_metadata() {
3401                continue;
3402            }
3403
3404            // Discovery growth must discard queued work before reachability.
3405            // Evicting a whole side here forgets every downstream command route.
3406            if let Some(victim) = self.lower_priority_shared_queue(255)
3407                && self.pop_shared_queue_item(victim)
3408            {
3409                continue;
3410            }
3411            // Live routes are protected. Growth is admitted transactionally;
3412            // external registry/config changes cannot justify erasing routes.
3413            break;
3414        }
3415    }
3416
3417    fn reliable_original_type(item: &TxQueued) -> Option<DataType> {
3418        let RouterTxItem::Broadcast(data) = &item.item else {
3419            return None;
3420        };
3421        let _ = data;
3422        let ty = DataType(item.accepted_type);
3423        (item.delivery_state != QueuedDelivery::Untracked).then_some(ty)
3424    }
3425
3426    fn ordered_original_type(item: &TxQueued) -> Option<DataType> {
3427        let RouterTxItem::Broadcast(data) = &item.item else {
3428            return None;
3429        };
3430        let _ = data;
3431        let ty = DataType(item.accepted_type);
3432        matches!(
3433            item.delivery_state,
3434            QueuedDelivery::Ordered
3435                | QueuedDelivery::OrderedTail
3436                | QueuedDelivery::DeferredOrdered
3437                | QueuedDelivery::ClaimedOrdered
3438        )
3439        .then_some(ty)
3440    }
3441
3442    fn pop_received_fair(&mut self) -> Option<RouterRxItem> {
3443        self.received_queue.pop_front_bulk_fair(
3444            &mut self.rx_priority_burst,
3445            MAX_PRIORITY_BURST,
3446            |item| item.priority,
3447        )
3448    }
3449
3450    fn push_received(&mut self, item: RouterRxItem) -> TelemetryResult<()> {
3451        self.make_shared_queue_room(item.byte_cost(), item.priority, Some(&item.data))?;
3452        #[cfg(any(feature = "std", feature = "fair-admission"))]
3453        {
3454            #[cfg(any(feature = "std", feature = "fair-admission"))]
3455            let mut fair_victim = None;
3456            self.received_queue.push_back_prioritized_select(
3457                item,
3458                |queued| queued.priority,
3459                |queue, _incoming| {
3460                    #[cfg(any(feature = "std", feature = "fair-admission"))]
3461                    {
3462                        crate::admission_fairness::queue_victim(
3463                            queue,
3464                            _incoming,
3465                            |item| Self::fair_bulk_owner(&item.data, item.priority),
3466                            |item| item.priority,
3467                            |item| Self::bulk_stream(&item.data),
3468                            &mut fair_victim,
3469                        )
3470                    }
3471                },
3472                core::mem::drop,
3473            )
3474        }
3475        #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3476        {
3477            self.received_queue
3478                .push_back_prioritized(item, |queued| queued.priority)
3479        }
3480    }
3481
3482    fn push_transmit(&mut self, mut item: TxQueued) -> TelemetryResult<()> {
3483        let child = item.disposable & TX_E2E_CHILD != 0;
3484        #[cfg(feature = "latest-value-egress")]
3485        let turn_flags = if !child
3486            && item.delivery_state == QueuedDelivery::Untracked
3487            && self.canonical_catalog_owner(&item)
3488        {
3489            item.disposable & TX_CANONICAL_TURN
3490        } else if !child
3491            && item.delivery_state == QueuedDelivery::Untracked
3492            && matches!(&item.item, RouterTxItem::ToSide { dst, .. }
3493                if self.latest_side() == Some(*dst))
3494        {
3495            item.disposable & TX_LATEST_TURN
3496        } else {
3497            0
3498        };
3499        #[cfg(feature = "std")]
3500        let retry_origin = item.disposable & TX_E2E_RETRY;
3501        #[cfg(feature = "latest-value-egress")]
3502        if item.disposable & TX_LATEST_PENDING != 0 && !self.latest_entry_eligible(&item) {
3503            item.disposable &= !TX_LATEST_PENDING;
3504        }
3505        #[cfg(feature = "compression")]
3506        match &mut item.item {
3507            RouterTxItem::Broadcast(data) => self.prepare_codec_queue_owner(None, data),
3508            RouterTxItem::ToSide { dst, data, .. } => {
3509                self.prepare_codec_queue_owner(Some(*dst), data)
3510            }
3511            _ => {}
3512        }
3513        let type_source = match &item.item {
3514            RouterTxItem::Broadcast(data) => Some(data),
3515            // This cache does not grant application acceptance: ToSide stays
3516            // Untracked. Preserve explicit validated producer metadata (ACKs).
3517            #[cfg(feature = "std")]
3518            RouterTxItem::ToSide { data, .. } if item.accepted_type == 0 => Some(data),
3519            _ => None,
3520        };
3521        if let Some(data) = type_source {
3522            item.accepted_type = match data {
3523                RouterItem::Packet(packet) => packet.data_type().as_u32(),
3524                RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
3525                    wire_format::scheduling_type(view)
3526                })
3527                .ok()
3528                .flatten()
3529                .map_or(0, |ty| ty.as_u32()),
3530            };
3531        }
3532        if item.delivery_state == QueuedDelivery::Untracked
3533            && matches!(&item.item, RouterTxItem::Broadcast(_))
3534        {
3535            item.delivery_state = match reliable_mode(DataType(item.accepted_type)) {
3536                crate::ReliableMode::Ordered => QueuedDelivery::Ordered,
3537                crate::ReliableMode::Unordered => QueuedDelivery::Unordered,
3538                crate::ReliableMode::None => QueuedDelivery::Untracked,
3539            };
3540        }
3541        // A partial schema merge may raise priority while an Ordered stream
3542        // is queued. Preserve its original service band until its tail drains.
3543        if matches!(
3544            item.delivery_state,
3545            QueuedDelivery::Ordered | QueuedDelivery::DeferredOrdered
3546        ) && let Some(priority) = self.transmit_queue.iter().find_map(|queued| {
3547            (Self::ordered_original_type(queued).is_some()
3548                && queued.accepted_type == item.accepted_type)
3549                .then_some(queued.priority)
3550        }) {
3551            item.priority = priority;
3552            item.delivery_state = QueuedDelivery::OrderedTail;
3553        }
3554        #[cfg(not(feature = "latest-value-egress"))]
3555        {
3556            item.disposable = if item.delivery_state != QueuedDelivery::Untracked {
3557                0
3558            } else {
3559                u8::from(Self::classify_transmit_disposable(&item))
3560            };
3561        }
3562        #[cfg(feature = "latest-value-egress")]
3563        {
3564            let history_owned = item.disposable & TX_HISTORY_OWNED != 0
3565                || (item.retry_after_ms != 0
3566                    && matches!(&item.item,
3567                    RouterTxItem::ToSide { dst, .. } if self.latest_side() == Some(*dst)));
3568            item.disposable = if history_owned {
3569                TX_HISTORY_OWNED
3570            } else if item.delivery_state != QueuedDelivery::Untracked {
3571                0
3572            } else {
3573                u8::from(Self::classify_transmit_disposable(&item))
3574                    | (item.disposable & TX_LATEST_PENDING)
3575            };
3576            item.disposable |= turn_flags;
3577            if item.disposable & TX_LATEST_PENDING != 0 && self.replace_latest_pending(&mut item)? {
3578                return Ok(());
3579            }
3580        }
3581        if child {
3582            item.disposable = (item.disposable & !TX_DISPOSABLE) | TX_E2E_CHILD;
3583            #[cfg(feature = "std")]
3584            {
3585                item.disposable |= retry_origin;
3586            }
3587        }
3588        let ordered_retry = Self::ordered_original_type(&item).is_some()
3589            && item.delivery_state == QueuedDelivery::DeferredOrdered;
3590        let replay = item.item.replay_marker_key();
3591        // Admission, eviction and the reservation change share this state lock.
3592        // An acknowledged/orphan marker has no payload left to retransmit.
3593        if replay.is_some_and(|key| self.replay_is_queued(key).is_none_or(|queued| queued)) {
3594            return Ok(());
3595        }
3596        #[cfg(feature = "discovery")]
3597        if let RouterTxItem::ToSide { dst, data, .. } = &item.item {
3598            // A new timestamp for the same catalog must not replace the
3599            // queued packet: a chunked side may already have sent its prefix.
3600            if let RouterItem::Packet(incoming) = data
3601                && self.transmit_queue.iter().any(|queued| {
3602                    matches!(&queued.item,
3603                        RouterTxItem::ToSide { dst: queued_dst, data: RouterItem::Packet(existing), .. }
3604                        if queued_dst == dst && discovery::same_schema_snapshot(existing, incoming))
3605                })
3606            {
3607                return Ok(());
3608            }
3609            let incoming = match data {
3610                RouterItem::Packet(pkt) if discovery::is_discovery_type(pkt.data_type()) => {
3611                    Some((pkt.data_type(), pkt.sender().to_owned()))
3612                }
3613                RouterItem::Packet(_) | RouterItem::Packed(_) => None,
3614            };
3615            if let Some((incoming_ty, incoming_sender)) = incoming {
3616                self.retain_transmit(|queued| {
3617                    let RouterTxItem::ToSide {
3618                        dst: queued_dst,
3619                        data: queued_data,
3620                        ..
3621                    } = &queued.item
3622                    else {
3623                        return true;
3624                    };
3625                    let same_snapshot = match queued_data {
3626                        RouterItem::Packet(pkt) => {
3627                            pkt.data_type() == incoming_ty && pkt.sender() == incoming_sender
3628                        }
3629                        RouterItem::Packed(_) => false,
3630                    };
3631                    *queued_dst != *dst || !same_snapshot
3632                });
3633            }
3634        }
3635        self.make_shared_queue_room(
3636            item.byte_cost(),
3637            item.priority,
3638            match &item.item {
3639                RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => Some(data),
3640                _ => None,
3641            },
3642        )?;
3643        let reliable_tx = &mut self.reliable_tx;
3644        let end_to_end = &mut self.end_to_end_reliable_tx;
3645        #[cfg(feature = "discovery")]
3646        let responses = &mut self.discovery_side_throttle;
3647        let result = {
3648            #[cfg(any(feature = "std", feature = "fair-admission"))]
3649            {
3650                #[cfg(any(feature = "std", feature = "fair-admission"))]
3651                let mut fair_victim = None;
3652                self.transmit_queue.push_back_prioritized_select(
3653                    item,
3654                    |queued| queued.priority,
3655                    |queue, _incoming| {
3656                        #[cfg(any(feature = "std", feature = "fair-admission"))]
3657                        {
3658                            if Self::tx_bulk_owner(_incoming).is_some() {
3659                                crate::admission_fairness::queue_victim(
3660                                    queue,
3661                                    _incoming,
3662                                    Self::tx_bulk_owner,
3663                                    |item| item.priority,
3664                                    Self::tx_bulk_stream,
3665                                    &mut fair_victim,
3666                                )
3667                            } else {
3668                                Self::transmit_eviction_candidate(queue)
3669                            }
3670                        }
3671                    },
3672                    |evicted: TxQueued| {
3673                        #[cfg(feature = "discovery")]
3674                        Self::schema_response_owner_lost(
3675                            responses,
3676                            Self::schema_response_owner(&evicted.item),
3677                        );
3678                        if let Some(key) = evicted.item.replay_marker_key() {
3679                            Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3680                        }
3681                    },
3682                )
3683            }
3684            #[cfg(not(any(feature = "std", feature = "fair-admission")))]
3685            {
3686                self.transmit_queue.push_back_prioritized_select(
3687                    item,
3688                    |queued| queued.priority,
3689                    |queue, _| Self::transmit_eviction_candidate(queue),
3690                    |evicted: TxQueued| {
3691                        #[cfg(feature = "discovery")]
3692                        Self::schema_response_owner_lost(
3693                            responses,
3694                            Self::schema_response_owner(&evicted.item),
3695                        );
3696                        if let Some(key) = evicted.item.replay_marker_key() {
3697                            Self::set_replay_queued(reliable_tx, end_to_end, key, false);
3698                        }
3699                    },
3700                )
3701            }
3702        };
3703        if result.is_ok() && ordered_retry {
3704            self.tx_policy_flags |= 1;
3705        }
3706        if result.is_ok()
3707            && let Some(key) = replay
3708        {
3709            Self::set_replay_queued(reliable_tx, end_to_end, key, true);
3710        }
3711        result
3712    }
3713
3714    fn push_recent_rx(&mut self, id: u64) -> TelemetryResult<()> {
3715        while self.recent_rx.len() >= self.memory.max_recent_rx_ids {
3716            let _ = self.recent_rx.pop_front();
3717        }
3718        self.make_shared_queue_room(0, 0, None)?;
3719        self.recent_rx.push_back(id)
3720    }
3721
3722    #[allow(unused_mut)]
3723    fn buffer_reliable_rx(
3724        &mut self,
3725        side: RouterSideId,
3726        ty: DataType,
3727        seq: u32,
3728        mut bytes: crate::SharedBytes,
3729    ) -> TelemetryResult<()> {
3730        let key = Router::reliable_key(side, ty);
3731        if self
3732            .reliable_rx
3733            .get(&key)
3734            .is_some_and(|state| state.buffered.contains_key(&seq))
3735        {
3736            return Ok(());
3737        }
3738        let cost = size_of::<crate::SharedBytes>() + bytes.len();
3739        self.make_shared_queue_room(cost, crate::transport_priority(ty), None)?;
3740        #[cfg(feature = "compact-packet-store")]
3741        bytes.park_for_queue()?;
3742        let rx_state = self
3743            .reliable_rx
3744            .entry(key)
3745            .or_insert_with(|| ReliableRxState {
3746                expected_seq: 1,
3747                buffered: Default::default(),
3748            });
3749        if rx_state.buffered.len() >= runtime_reliable_max_pending() {
3750            let _ = rx_state.buffered.pop_first();
3751        }
3752        rx_state.buffered.insert(seq, bytes);
3753        Ok(())
3754    }
3755}
3756
3757/// Non-blocking RX queue used by ISR-safe `rx_queue*` APIs.
3758///
3759/// Uses a tiny atomic try-lock so enqueue never blocks. If contended, push/pop
3760/// operations return `TelemetryError::Io("rx queue busy")`.
3761struct IsrRxQueue {
3762    busy: AtomicBool,
3763    q: UnsafeCell<BoundedDeque<RouterRxItem>>,
3764}
3765
3766unsafe impl Send for IsrRxQueue {}
3767unsafe impl Sync for IsrRxQueue {}
3768
3769struct IsrRxQueueGuard<'a> {
3770    owner: &'a IsrRxQueue,
3771}
3772
3773impl Deref for IsrRxQueueGuard<'_> {
3774    type Target = BoundedDeque<RouterRxItem>;
3775
3776    #[inline]
3777    fn deref(&self) -> &Self::Target {
3778        unsafe { &*self.owner.q.get() }
3779    }
3780}
3781
3782impl DerefMut for IsrRxQueueGuard<'_> {
3783    #[inline]
3784    fn deref_mut(&mut self) -> &mut Self::Target {
3785        unsafe { &mut *self.owner.q.get() }
3786    }
3787}
3788
3789impl Drop for IsrRxQueueGuard<'_> {
3790    #[inline]
3791    fn drop(&mut self) {
3792        self.owner.busy.store(false, Ordering::Release);
3793    }
3794}
3795
3796impl IsrRxQueue {
3797    #[inline]
3798    fn new(max_bytes: usize, starting_bytes: usize, grow_mult: f64) -> Self {
3799        Self {
3800            busy: AtomicBool::new(false),
3801            q: UnsafeCell::new(BoundedDeque::new(max_bytes, starting_bytes, grow_mult)),
3802        }
3803    }
3804
3805    #[inline]
3806    fn try_lock(&self) -> TelemetryResult<IsrRxQueueGuard<'_>> {
3807        match self
3808            .busy
3809            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
3810        {
3811            Ok(_) => Ok(IsrRxQueueGuard { owner: self }),
3812            Err(_) => Err(TelemetryError::Io("rx queue busy")),
3813        }
3814    }
3815
3816    #[allow(dead_code)]
3817    #[inline]
3818    fn push_back(&self, item: RouterRxItem) -> TelemetryResult<()> {
3819        let mut g = self.try_lock()?;
3820        g.push_back(item)
3821    }
3822
3823    #[inline]
3824    fn push_back_prioritized(&self, item: RouterRxItem) -> TelemetryResult<()> {
3825        let mut g = self.try_lock()?;
3826        g.push_back_prioritized(item, |queued| queued.priority)
3827    }
3828
3829    #[inline]
3830    fn pop_front(&self) -> TelemetryResult<Option<RouterRxItem>> {
3831        let mut g = self.try_lock()?;
3832        Ok(g.pop_front())
3833    }
3834
3835    #[inline]
3836    fn clear(&self) -> TelemetryResult<()> {
3837        let mut g = self.try_lock()?;
3838        g.clear();
3839        Ok(())
3840    }
3841
3842    #[inline]
3843    fn snapshot(&self) -> Option<(usize, usize)> {
3844        let g = self.try_lock().ok()?;
3845        Some((g.len(), g.bytes_used()))
3846    }
3847}
3848
3849/// Telemetry Router for handling incoming and outgoing telemetry packets.
3850/// Supports queuing, processing, and dispatching to local endpoint handlers.
3851/// Thread-safe via internal locking.
3852pub struct Router {
3853    #[cfg(feature = "std")]
3854    retry_trackers: RouterMutex<Vec<std::sync::Weak<completion::TrackerInner>>>,
3855    sender: RouterMutex<Arc<str>>,
3856    cfg: ActiveRoutingConfig,
3857    state: RouterMutex<RouterInner>,
3858    isr_rx_queue: Option<Box<IsrRxQueue>>,
3859    #[cfg(not(feature = "std"))]
3860    side_tx_gate: ReentryGate,
3861    #[cfg(feature = "std")]
3862    side_tx_activity: crate::lock::SideTxActivity,
3863    #[cfg(feature = "std")]
3864    active_queued_tx: core::sync::atomic::AtomicUsize,
3865    clock: Box<dyn Clock + Send + Sync>,
3866    #[cfg(feature = "timesync")]
3867    timesync: Option<Box<RouterMutex<TimeSyncRuntime>>>,
3868}
3869
3870#[cfg(feature = "std")]
3871const ACTIVE_QUEUED_TYPE_SLOTS: usize = 16;
3872#[cfg(feature = "std")]
3873#[derive(Clone, Copy)]
3874struct ActiveQueuedTypeKey {
3875    slot: core::num::NonZeroU8,
3876    kind: u32,
3877}
3878#[cfg(feature = "std")]
3879struct QueuedTxActivityGuard<'a> {
3880    router: &'a Router,
3881    key: Option<ActiveQueuedTypeKey>,
3882}
3883#[cfg(feature = "std")]
3884impl Drop for QueuedTxActivityGuard<'_> {
3885    fn drop(&mut self) {
3886        let mut state = self.router.state.lock();
3887        if let Some(key) = self.key {
3888            let slot = &mut state.active_queued_types[usize::from(key.slot.get() - 1)];
3889            assert_eq!(slot.0, key.kind, "active queued type guard changed owner");
3890            slot.1 = slot.1.checked_sub(1).expect("active queued type underflow");
3891            if slot.1 == 0 {
3892                slot.0 = 0;
3893            }
3894        }
3895        let total = self
3896            .router
3897            .active_queued_tx
3898            .load(core::sync::atomic::Ordering::Acquire);
3899        self.router.active_queued_tx.store(
3900            total.checked_sub(1).expect("active queued TX underflow"),
3901            core::sync::atomic::Ordering::Release,
3902        );
3903    }
3904}
3905
3906#[cfg(feature = "timesync")]
3907#[derive(Debug, Clone)]
3908struct PendingTimeSyncRequest {
3909    seq: u64,
3910    t1_mono_ms: u64,
3911    source: String,
3912}
3913
3914#[cfg(feature = "timesync")]
3915#[derive(Debug, Clone)]
3916struct RemoteTimeSyncSource {
3917    priority: u64,
3918    last_sample_mono_ms: u64,
3919    sample_unix_ms: u64,
3920}
3921
3922#[cfg(feature = "timesync")]
3923#[derive(Debug, Clone)]
3924struct TimeSyncRuntime {
3925    cfg: Option<TimeSyncConfig>,
3926    tracker: Option<TimeSyncTracker>,
3927    clock: NetworkClock,
3928    disciplined_clock: SlewedNetworkClock,
3929    remote_sources: BTreeMap<String, Box<RemoteTimeSyncSource>>,
3930    next_seq: u64,
3931    next_announce_mono_ms: u64,
3932    next_request_mono_ms: u64,
3933    pending_request: Option<PendingTimeSyncRequest>,
3934}
3935
3936#[cfg(feature = "timesync")]
3937impl TimeSyncRuntime {
3938    fn new(cfg: Option<TimeSyncConfig>) -> Self {
3939        Self {
3940            tracker: cfg.map(TimeSyncTracker::new),
3941            cfg,
3942            clock: NetworkClock::default(),
3943            disciplined_clock: SlewedNetworkClock::new(
3944                cfg.map(|c| c.max_slew_ppm)
3945                    .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
3946            ),
3947            remote_sources: Default::default(),
3948            next_seq: 1,
3949            next_announce_mono_ms: 0,
3950            next_request_mono_ms: 0,
3951            pending_request: None,
3952        }
3953    }
3954}
3955
3956enum RemoteSidePlan {
3957    Target(Vec<RouterSideId>),
3958}
3959
3960#[cfg(feature = "discovery")]
3961#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
3962struct DiscoveryCandidateMatch {
3963    side: RouterSideId,
3964    overlap: usize,
3965}
3966
3967#[cfg(feature = "discovery")]
3968#[inline]
3969const fn discovery_is_hop_local_advertisement(ty: DataType, transparent: bool) -> bool {
3970    matches!(
3971        ty,
3972        DataType::DiscoveryAnnounce
3973            | DataType::DiscoveryTimeSyncSources
3974            | DataType::DiscoveryTopology
3975            | DataType::DiscoveryLinkCapabilities
3976    ) || (matches!(ty, DataType::DiscoveryAddress) && !transparent)
3977}
3978
3979#[cfg(not(feature = "discovery"))]
3980const fn discovery_is_hop_local_advertisement(_ty: DataType, _transparent: bool) -> bool {
3981    false
3982}
3983
3984impl Debug for Router {
3985    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
3986        let sender = self.sender();
3987        f.debug_struct("Router")
3988            .field("sender", &sender)
3989            .field("cfg", &self.cfg)
3990            .field("state", &"<mutex>")
3991            .field("clock", &"Clock")
3992            .finish()
3993    }
3994}
3995
3996/// Check if any of the provided endpoints require remote forwarding when
3997/// discovery is unavailable and routing falls back to local-vs-remote schema.
3998#[inline]
3999fn has_nonlocal_endpoint(eps: &[DataEndpoint], cfg: &ActiveRoutingConfig) -> bool {
4000    eps.iter().copied().any(|ep| !cfg.is_local_endpoint(ep))
4001}
4002
4003#[inline]
4004fn force_remote_for_type(ty: DataType) -> bool {
4005    matches!(
4006        ty,
4007        DataType::ReliableAck
4008            | DataType::ReliablePartialAck
4009            | DataType::ReliablePacketRequest
4010            | DataType::P2pMessage
4011    ) || {
4012        #[cfg(feature = "timesync")]
4013        {
4014            matches!(
4015                ty,
4016                DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4017            )
4018        }
4019        #[cfg(not(feature = "timesync"))]
4020        {
4021            false
4022        }
4023    }
4024}
4025
4026#[inline]
4027fn is_internal_control_type(ty: DataType) -> bool {
4028    if matches!(
4029        ty,
4030        DataType::ReliableAck
4031            | DataType::ReliablePartialAck
4032            | DataType::ReliablePacketRequest
4033            | DataType::P2pMessage
4034    ) {
4035        return true;
4036    }
4037
4038    #[cfg(feature = "timesync")]
4039    if matches!(
4040        ty,
4041        DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
4042    ) {
4043        return true;
4044    }
4045
4046    #[cfg(feature = "discovery")]
4047    if discovery::is_discovery_type(ty) {
4048        return true;
4049    }
4050
4051    let _ = ty;
4052    false
4053}
4054
4055/// Helper function to call a handler with retries and error handling.
4056fn with_retries<F>(
4057    this: &Router,
4058    dest: DataEndpoint,
4059    data: &RouterItem,
4060    pkt_for_ctx: Option<&Packet>,
4061    env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
4062    called_from_queue: bool,
4063    run: F,
4064) -> TelemetryResult<()>
4065where
4066    F: Fn() -> TelemetryResult<()>,
4067{
4068    match this.retry_with_attempts(runtime_max_handler_retries(), run) {
4069        Ok(((), attempts)) => {
4070            if attempts > 1 {
4071                let mut st = this.state.lock();
4072                st.total_handler_retries = st
4073                    .total_handler_retries
4074                    .saturating_add(attempts.saturating_sub(1) as u64);
4075            }
4076            Ok(())
4077        }
4078        Err((e, attempts)) => {
4079            {
4080                let mut st = this.state.lock();
4081                st.total_handler_failures = st.total_handler_failures.saturating_add(1);
4082                st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
4083            }
4084
4085            // If handler fails, remove from dedupe so it can be retried later if resent.
4086            this.remove_pkt_id(data);
4087
4088            // Emit error packet (to local endpoints).
4089            if let Some(pkt) = pkt_for_ctx {
4090                let _ = this.handle_callback_error(pkt, Some(dest), e, called_from_queue);
4091            } else if let Some(env) = env_for_ctx {
4092                let _ = this.handle_callback_error_from_env(env, Some(dest), e, called_from_queue);
4093            }
4094
4095            Err(TelemetryError::HandlerError("local handler failed"))
4096        }
4097    }
4098}
4099/// Router implementation
4100impl Router {
4101    #[cfg(feature = "discovery")]
4102    const END_TO_END_ACK_SENDER: &'static str = "E2EACK";
4103    const END_TO_END_ACK_PREFIX: &'static str = "E2EACK:";
4104
4105    #[inline]
4106    fn side_ref(st: &RouterInner, side: RouterSideId) -> TelemetryResult<&RouterSide> {
4107        st.sides
4108            .get(side)
4109            .and_then(|side| side.as_ref())
4110            .ok_or(TelemetryError::HandlerError("router: invalid side id"))
4111    }
4112
4113    fn note_side_tx_success(
4114        &self,
4115        side: RouterSideId,
4116        ty: DataType,
4117        bytes: usize,
4118        relayed: bool,
4119        attempts: usize,
4120    ) {
4121        let mut st = self.state.lock();
4122        #[cfg(feature = "discovery")]
4123        if ty == crate::DataType::DiscoveryAnnounce
4124            && !relayed
4125            && let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
4126        {
4127            throttle.last_liveness_tx_ms = self.clock.now_ms();
4128        }
4129        #[cfg(all(feature = "discovery", feature = "timesync"))]
4130        if Self::is_timesync_type(ty) {
4131            let now = self.clock.now_ms();
4132            if Self::side_is_slow_control_link_locked(&st, side, now) {
4133                st.timesync_side_throttle
4134                    .entry(side)
4135                    .or_default()
4136                    .next_allowed_ms = now.saturating_add(TIMESYNC_SLOW_LINK_MIN_INTERVAL_MS);
4137            } else {
4138                st.timesync_side_throttle.remove(&side);
4139            }
4140        }
4141        let entry = st.side_runtime_stats.entry(side).or_default();
4142        entry.note_tx(ty, bytes, relayed, attempts.saturating_sub(1));
4143    }
4144
4145    fn note_side_tx_failure(&self, side: RouterSideId, ty: DataType, attempts: usize) {
4146        let mut st = self.state.lock();
4147        st.total_handler_failures = st.total_handler_failures.saturating_add(1);
4148        st.total_handler_retries = st.total_handler_retries.saturating_add(attempts as u64);
4149        let entry = st.side_runtime_stats.entry(side).or_default();
4150        entry.note_tx_failure(ty, attempts);
4151    }
4152
4153    fn note_side_rx(&self, side: RouterSideId, ty: DataType, bytes: usize, relayed: bool) {
4154        let mut st = self.state.lock();
4155        let entry = st.side_runtime_stats.entry(side).or_default();
4156        entry.note_rx(ty, bytes, relayed);
4157    }
4158
4159    fn note_side_local_delivery(&self, side: RouterSideId, ty: DataType) {
4160        let mut st = self.state.lock();
4161        let entry = st.side_runtime_stats.entry(side).or_default();
4162        entry.note_local_delivery(ty);
4163    }
4164
4165    fn note_side_local_handler_failure(&self, side: RouterSideId, ty: DataType, retries: usize) {
4166        let mut st = self.state.lock();
4167        let entry = st.side_runtime_stats.entry(side).or_default();
4168        entry.note_local_handler_failure(ty, retries);
4169    }
4170
4171    fn cache_managed_variable_packet(
4172        &self,
4173        pkt: &Packet,
4174        notify_handlers: bool,
4175        source_side: Option<RouterSideId>,
4176    ) -> TelemetryResult<()> {
4177        #[cfg(not(feature = "discovery"))]
4178        let _ = source_side;
4179        let sender_address = pkt
4180            .sender()
4181            .strip_prefix("@addr:")
4182            .and_then(|value| value.parse::<u32>().ok())
4183            .unwrap_or_else(|| sender_address_u32(pkt.sender()));
4184        let handlers = {
4185            let mut st = self.state.lock();
4186            if !st.is_managed_variable(pkt.data_type().as_u32()) {
4187                return Ok(());
4188            }
4189            let existing = st
4190                .local_services
4191                .as_ref()
4192                .expect("local services enabled")
4193                .managed_variable_latest
4194                .get(&pkt.data_type().as_u32());
4195            // Sender-local timestamps can order updates from one authoritative
4196            // writer even before network time is available. They cannot order
4197            // a restored local seed against a remote writer, and packet nonces
4198            // are deliberately pseudo-random deduplication keys rather than
4199            // revisions. Accept equal-timestamp updates in arrival order so
4200            // rapid toggles made during one clock tick are not discarded.
4201            if existing.is_some_and(|entry| {
4202                let current = &entry.packet;
4203                sender_address == entry.sender_address && pkt.timestamp() < current.timestamp()
4204            }) {
4205                return Ok(());
4206            }
4207            // A proxy may change only the frozen transport recipients when
4208            // returning this exact source observation. That is not a new
4209            // application revision and must not replay update side effects.
4210            let changed = existing.is_none_or(|entry| entry.packet.packet_id() != pkt.packet_id());
4211            #[cfg(feature = "discovery")]
4212            let upstream = existing
4213                .filter(|_| !changed)
4214                .and_then(|entry| entry.upstream)
4215                .or_else(|| {
4216                    source_side
4217                        .and_then(|side| side.checked_add(1))
4218                        .and_then(core::num::NonZeroUsize::new)
4219                });
4220            // A B-tree leaf reserves eleven values, even for one variable.
4221            // Keep that sparse root small; reuse each box on later updates so
4222            // frequent state changes introduce no allocator churn.
4223            let updated = ManagedVariableCacheEntry {
4224                packet: pkt.clone(),
4225                sender_address,
4226                #[cfg(feature = "discovery")]
4227                cached_at_ms: self.clock.now_ms(),
4228                #[cfg(feature = "discovery")]
4229                upstream,
4230            };
4231            if let Some(entry) = st
4232                .local_services
4233                .as_mut()
4234                .expect("local services enabled")
4235                .managed_variable_latest
4236                .get_mut(&pkt.data_type().as_u32())
4237            {
4238                **entry = updated;
4239            } else {
4240                st.local_services
4241                    .as_mut()
4242                    .expect("local services enabled")
4243                    .managed_variable_latest
4244                    .insert(pkt.data_type().as_u32(), Box::new(updated));
4245            }
4246            if notify_handlers && changed {
4247                st.local_services
4248                    .as_ref()
4249                    .expect("local services enabled")
4250                    .network_variable_update_handlers
4251                    .get(&pkt.data_type().as_u32())
4252                    .cloned()
4253                    .unwrap_or_default()
4254            } else {
4255                Vec::new()
4256            }
4257        };
4258        for handler in handlers {
4259            (handler.handler)(pkt)?;
4260        }
4261        Ok(())
4262    }
4263
4264    fn remember_managed_variable_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
4265        self.cache_managed_variable_packet(pkt, true, None)
4266    }
4267
4268    fn managed_variable_latest(&self, ty: DataType) -> Option<Packet> {
4269        let st = self.state.lock();
4270        st.local_services
4271            .as_ref()?
4272            .managed_variable_latest
4273            .get(&ty.as_u32())
4274            .map(|entry| entry.packet.clone())
4275    }
4276
4277    #[cfg(feature = "discovery")]
4278    fn managed_variable_latest_with_age(&self, ty: DataType) -> Option<(Packet, u64)> {
4279        let now_ms = self.clock.now_ms();
4280        let st = self.state.lock();
4281        st.local_services
4282            .as_ref()?
4283            .managed_variable_latest
4284            .get(&ty.as_u32())
4285            .map(|entry| {
4286                (
4287                    entry.packet.clone(),
4288                    now_ms.saturating_sub(entry.cached_at_ms),
4289                )
4290            })
4291    }
4292
4293    fn is_managed_variable_type(&self, ty: DataType) -> bool {
4294        let st = self.state.lock();
4295        st.is_managed_variable(ty.as_u32())
4296    }
4297
4298    fn managed_variable_permissions_locked(
4299        st: &RouterInner,
4300        ty: DataType,
4301    ) -> NetworkVariablePermissions {
4302        let Some(local) = st.local_services.as_ref() else {
4303            return NetworkVariablePermissions::NONE;
4304        };
4305        local
4306            .managed_variable_permissions
4307            .get(&ty.as_u32())
4308            .copied()
4309            .unwrap_or(NetworkVariablePermissions::READ_WRITE)
4310    }
4311
4312    fn can_read_managed_variable(&self, ty: DataType) -> bool {
4313        let st = self.state.lock();
4314        Self::managed_variable_permissions_locked(&st, ty).read
4315    }
4316
4317    #[cfg(feature = "discovery")]
4318    fn can_write_managed_variable(&self, ty: DataType) -> bool {
4319        let st = self.state.lock();
4320        Self::managed_variable_permissions_locked(&st, ty).write
4321    }
4322
4323    #[inline]
4324    fn ensure_side_ingress_enabled(&self, side: RouterSideId) -> TelemetryResult<()> {
4325        let st = self.state.lock();
4326        let side_ref = Self::side_ref(&st, side)?;
4327        if side_ref.opts.ingress_enabled {
4328            Ok(())
4329        } else {
4330            Err(TelemetryError::HandlerError(
4331                "router: ingress disabled for side id",
4332            ))
4333        }
4334    }
4335
4336    #[inline]
4337    fn default_route_enabled(&self, src: Option<RouterSideId>, dst: RouterSideId) -> bool {
4338        src != Some(dst)
4339    }
4340
4341    #[inline]
4342    fn route_allowed_locked(
4343        &self,
4344        st: &RouterInner,
4345        src: Option<RouterSideId>,
4346        ty: Option<DataType>,
4347        dst: RouterSideId,
4348    ) -> bool {
4349        let Some(dst_side) = st.sides.get(dst).and_then(|side| side.as_ref()) else {
4350            return false;
4351        };
4352        if !dst_side.opts.egress_enabled {
4353            return false;
4354        }
4355        if let Some(src_id) = src {
4356            let Some(src_side) = st.sides.get(src_id).and_then(|side| side.as_ref()) else {
4357                return false;
4358            };
4359            if !src_side.opts.ingress_enabled {
4360                return false;
4361            }
4362        }
4363        let base_allowed = st
4364            .route_overrides
4365            .get(&(src, dst))
4366            .copied()
4367            .unwrap_or_else(|| self.default_route_enabled(src, dst));
4368        if !base_allowed {
4369            return false;
4370        }
4371
4372        let Some(ty) = ty else {
4373            return true;
4374        };
4375        if st
4376            .typed_route_overrides
4377            .keys()
4378            .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
4379        {
4380            return st
4381                .typed_route_overrides
4382                .get(&(src, ty.as_u32(), dst))
4383                .copied()
4384                .unwrap_or(false);
4385        }
4386        true
4387    }
4388
4389    #[cfg(feature = "discovery")]
4390    fn has_typed_route_overrides_locked(
4391        st: &RouterInner,
4392        src: Option<RouterSideId>,
4393        ty: DataType,
4394    ) -> bool {
4395        st.typed_route_overrides
4396            .keys()
4397            .any(|(typed_src, typed_ty, _)| *typed_src == src && *typed_ty == ty.as_u32())
4398    }
4399
4400    #[cfg(feature = "discovery")]
4401    fn endpoint_overlap_count<I>(reachable: I, eps: &[DataEndpoint]) -> usize
4402    where
4403        I: IntoIterator<Item = DataEndpoint>,
4404    {
4405        let mut overlap = 0usize;
4406        for ep in reachable {
4407            if eps.contains(&ep) {
4408                overlap = overlap.saturating_add(1);
4409            }
4410        }
4411        overlap
4412    }
4413
4414    #[inline]
4415    #[cfg(feature = "discovery")]
4416    fn preferred_scoring_endpoints<'a>(
4417        &self,
4418        eps: &'a [DataEndpoint],
4419        prefer_nonlocal: bool,
4420    ) -> alloc::borrow::Cow<'a, [DataEndpoint]> {
4421        if !prefer_nonlocal || eps.iter().all(|&ep| !self.cfg.is_local_endpoint(ep)) {
4422            return alloc::borrow::Cow::Borrowed(eps);
4423        }
4424        let nonlocal: Vec<DataEndpoint> = eps
4425            .iter()
4426            .copied()
4427            .filter(|&ep| !self.cfg.is_local_endpoint(ep))
4428            .collect();
4429        if nonlocal.is_empty() {
4430            alloc::borrow::Cow::Borrowed(eps)
4431        } else {
4432            alloc::borrow::Cow::Owned(nonlocal)
4433        }
4434    }
4435
4436    fn pop_next_queued_control_rx_item(&self) -> TelemetryResult<Option<RouterRxItem>> {
4437        if let Some(queue) = self.isr_rx_queue.as_ref() {
4438            let mut isr_rx = queue.try_lock()?;
4439            let idx = isr_rx
4440                .iter()
4441                .position(|item| self.queued_rx_item_is_control(item));
4442            if let Some(idx) = idx {
4443                return Ok(isr_rx.remove_pos(idx));
4444            }
4445        }
4446
4447        let mut st = self.state.lock();
4448        let idx = st
4449            .received_queue
4450            .iter()
4451            .position(|item| self.queued_rx_item_is_control(item));
4452        if let Some(idx) = idx {
4453            return Ok(st.received_queue.remove_pos(idx));
4454        }
4455        Ok(None)
4456    }
4457
4458    fn queued_rx_item_is_control(&self, item: &RouterRxItem) -> bool {
4459        // The forwarding facade historically predrains only receipts. Other
4460        // ingress remains ordinary RX work, whose admission refusal must reach
4461        // the caller rather than be treated as background maintenance.
4462        if !self.cfg.role.local_services()
4463            && item.priority < crate::transport_priority(DataType::ReliableAck)
4464        {
4465            return false;
4466        }
4467        // Admission computes and retains the reserved service band once.
4468        // Bulk queues can contain hundreds of entries: do not reparse every
4469        // packed header just to rediscover that none is a control packet.
4470        if item.priority < crate::transport_priority(DataType::ManagedVariableValue) {
4471            return false;
4472        }
4473        // Registered managed values keep their application type on the wire,
4474        // but their queued service class must match their transmission class.
4475        if item.priority == crate::transport_priority(DataType::ManagedVariableValue) {
4476            return true;
4477        }
4478        let ty = match &item.data {
4479            RouterItem::Packet(pkt) => Some(pkt.data_type()),
4480            RouterItem::Packed(bytes) => {
4481                if bytes.len() <= 128
4482                    && wire_format::peek_routing_frame_info(bytes)
4483                        .is_ok_and(|frame| frame.ack_only())
4484                {
4485                    return true;
4486                }
4487                wire_format::scheduling_type(bytes.as_ref())
4488            }
4489        };
4490        match ty {
4491            Some(
4492                DataType::ReliableAck
4493                | DataType::ReliablePartialAck
4494                | DataType::ReliablePacketRequest,
4495            ) => true,
4496            #[cfg(feature = "discovery")]
4497            Some(ty) => self.cfg.role.local_services() && discovery::is_discovery_type(ty),
4498            _ => false,
4499        }
4500    }
4501
4502    fn drain_queued_control_rx_before_tx(
4503        &self,
4504        start: u64,
4505        timeout_ms: u32,
4506    ) -> TelemetryResult<bool> {
4507        let mut did_any = false;
4508        while let Some(item) = self.pop_next_queued_control_rx_item()? {
4509            self.process_rx_queue_item(item)?;
4510            did_any = true;
4511            // Receipts precede retry exhaustion; discovery precedes routing.
4512            // A control burst still shares the caller's bounded deadline.
4513            if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms) {
4514                break;
4515            }
4516        }
4517        Ok(did_any)
4518    }
4519
4520    fn eligible_side_ids_locked(
4521        &self,
4522        st: &RouterInner,
4523        src: Option<RouterSideId>,
4524        ty: Option<DataType>,
4525        restrict_link_local: bool,
4526    ) -> Vec<RouterSideId> {
4527        st.sides
4528            .iter()
4529            .enumerate()
4530            .filter_map(|(side_id, side)| {
4531                let side = side.as_ref()?;
4532                if restrict_link_local && !side.opts.link_local_enabled {
4533                    return None;
4534                }
4535                if self.route_allowed_locked(st, src, ty, side_id) {
4536                    Some(side_id)
4537                } else {
4538                    None
4539                }
4540            })
4541            .collect()
4542    }
4543
4544    fn apply_route_selection_locked(
4545        &self,
4546        st: &mut RouterInner,
4547        src: Option<RouterSideId>,
4548        mut sides: Vec<RouterSideId>,
4549        origin: RouteSelectionOrigin,
4550    ) -> Vec<RouterSideId> {
4551        if sides.len() <= 1 {
4552            return sides;
4553        }
4554
4555        let selection_mode = st.source_route_modes.get(&src).copied();
4556        if selection_mode.is_none() && origin == RouteSelectionOrigin::Discovered {
4557            return self.apply_adaptive_discovery_selection_locked(st, src, sides);
4558        }
4559
4560        match selection_mode.unwrap_or(RouteSelectionMode::Fanout) {
4561            RouteSelectionMode::Fanout => sides,
4562            RouteSelectionMode::Weighted => {
4563                crate::small_sort::sort(&mut sides);
4564                let total_weight = sides.iter().fold(0_u64, |acc, side| {
4565                    acc + u64::from(st.route_weights.get(&(src, *side)).copied().unwrap_or(1))
4566                });
4567                if total_weight == 0 {
4568                    return Vec::new();
4569                }
4570                let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4571                let pick = *cursor % total_weight;
4572                *cursor = cursor.wrapping_add(1);
4573                let mut remaining = pick;
4574                for side in sides {
4575                    let weight =
4576                        u64::from(st.route_weights.get(&(src, side)).copied().unwrap_or(1));
4577                    if remaining < weight {
4578                        return vec![side];
4579                    }
4580                    remaining -= weight;
4581                }
4582                Vec::new()
4583            }
4584            RouteSelectionMode::Failover => {
4585                sides.sort_by_key(|side| {
4586                    (
4587                        st.route_priorities.get(&(src, *side)).copied().unwrap_or(0),
4588                        *side,
4589                    )
4590                });
4591                sides.truncate(1);
4592                sides
4593            }
4594        }
4595    }
4596
4597    fn apply_adaptive_discovery_selection_locked(
4598        &self,
4599        st: &mut RouterInner,
4600        src: Option<RouterSideId>,
4601        mut sides: Vec<RouterSideId>,
4602    ) -> Vec<RouterSideId> {
4603        crate::small_sort::sort(&mut sides);
4604        let mut unmeasured: Vec<_> = sides
4605            .iter()
4606            .copied()
4607            .filter(|side| !st.adaptive_route_stats.contains_key(side))
4608            .collect();
4609        if !unmeasured.is_empty() {
4610            let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4611            let pick = (*cursor as usize) % unmeasured.len();
4612            *cursor = cursor.wrapping_add(1);
4613            return vec![unmeasured.swap_remove(pick)];
4614        }
4615
4616        let now_ms = self.clock.now_ms();
4617        let total_weight = sides.iter().fold(0_u64, |acc, side| {
4618            acc + st
4619                .adaptive_route_stats
4620                .get(side)
4621                .map(|stats| stats.weight(now_ms))
4622                .unwrap_or(1)
4623        });
4624        if total_weight == 0 {
4625            sides.truncate(1);
4626            return sides;
4627        }
4628
4629        let cursor = st.route_selection_cursors.entry(src).or_insert(0);
4630        let pick = *cursor % total_weight;
4631        *cursor = cursor.wrapping_add(1);
4632        let mut remaining = pick;
4633        for side in sides {
4634            let weight = st
4635                .adaptive_route_stats
4636                .get(&side)
4637                .map(|stats| stats.weight(now_ms))
4638                .unwrap_or(1);
4639            if remaining < weight {
4640                return vec![side];
4641            }
4642            remaining -= weight;
4643        }
4644        Vec::new()
4645    }
4646
4647    fn record_side_tx_sample(
4648        &self,
4649        side: RouterSideId,
4650        bytes: usize,
4651        started_ms: u64,
4652        ended_ms: u64,
4653    ) {
4654        let sample_ms = ended_ms.saturating_sub(started_ms).max(1);
4655        let sample_bps = ((bytes as u128).saturating_mul(1000) / u128::from(sample_ms))
4656            .min(u128::from(u64::MAX)) as u64;
4657        let mut st = self.state.lock();
4658        st.adaptive_route_stats
4659            .entry(side)
4660            .or_default()
4661            .observe(bytes, sample_bps, ended_ms);
4662    }
4663
4664    /// Seed adaptive route selection with a transport-measured link probe.
4665    ///
4666    /// Call this after a side-specific bring-up probe, or whenever the transport already knows the
4667    /// duration for a frame. The router does not emit synthetic probe frames by itself.
4668    pub fn note_side_link_probe_sample(
4669        &self,
4670        side: RouterSideId,
4671        bytes: usize,
4672        duration_ms: u64,
4673    ) -> TelemetryResult<()> {
4674        {
4675            let st = self.state.lock();
4676            let _ = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
4677        }
4678        let ended_ms = self.clock.now_ms();
4679        self.record_side_tx_sample(side, bytes, ended_ms.saturating_sub(duration_ms), ended_ms);
4680        Ok(())
4681    }
4682
4683    fn router_item_wire_len(data: &RouterItem) -> TelemetryResult<usize> {
4684        match data {
4685            RouterItem::Packet(pkt) => Ok(wire_format::packet_wire_size(pkt)),
4686            RouterItem::Packed(bytes) => Ok(bytes.len()),
4687        }
4688    }
4689
4690    /// Extract the logical packet ID targeted by an end-to-end reliable ACK item.
4691    ///
4692    /// Router TX and replay queues can carry either decoded `Packet` values or
4693    /// packed frames. This helper normalizes both forms so ACK-routing code
4694    /// can reason about them uniformly.
4695    ///
4696    /// Only router-generated end-to-end `ReliableAck` packets qualify here.
4697    /// Ordinary application traffic and hop-level reliable control frames return
4698    /// `Ok(None)`.
4699    #[inline]
4700    fn reliable_control_target_packet_id(data: &RouterItem) -> TelemetryResult<Option<u64>> {
4701        match data {
4702            RouterItem::Packet(pkt) => {
4703                if !Self::is_end_to_end_ack_packet(pkt) {
4704                    return Ok(None);
4705                }
4706                Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4707            }
4708            RouterItem::Packed(bytes) => {
4709                if wire_format::peek_routing_frame_info(bytes.as_ref())
4710                    .ok()
4711                    .is_some_and(|frame| {
4712                        frame.envelope.ty != DataType::ReliableAck || frame.ack_only()
4713                    })
4714                {
4715                    return Ok(None);
4716                }
4717                let pkt = wire_format::unpack_packet(bytes.as_ref())?;
4718                if !Self::is_end_to_end_ack_packet(&pkt) {
4719                    return Ok(None);
4720                }
4721                Self::decode_end_to_end_reliable_ack(pkt.payload()).map(Some)
4722            }
4723        }
4724    }
4725
4726    fn decode_end_to_end_reliable_ack(payload: &[u8]) -> TelemetryResult<u64> {
4727        if payload.len() != 8 {
4728            return Err(TelemetryError::Unpack("bad reliable e2e ack payload"));
4729        }
4730        Ok(u64::from_le_bytes(payload[0..8].try_into().unwrap()))
4731    }
4732
4733    #[inline]
4734    fn sender_hash(sender: &str) -> u64 {
4735        if let Some(address) = sender
4736            .strip_prefix("@addr:")
4737            .and_then(|v| v.parse::<u32>().ok())
4738        {
4739            return u64::from(address);
4740        }
4741        hash_bytes_u64(0x517C_C1B7_2722_0A95, sender.as_bytes())
4742    }
4743
4744    #[inline]
4745    fn fallback_address_for_hostname(hostname: &str) -> NodeAddress {
4746        let hash = Self::sender_hash(hostname);
4747        let mut address = (hash as u32) ^ ((hash >> 32) as u32);
4748        if address == 0 {
4749            address = 1;
4750        }
4751        address
4752    }
4753
4754    /// Resolve the compact wire header's numeric source back to the stable
4755    /// hostname learned by address discovery. The DHCP-style assigned address
4756    /// and the wire address intentionally occupy separate namespaces, so both
4757    /// forms must be considered here.
4758    fn canonical_sender_locked(st: &RouterInner, sender: &str) -> String {
4759        let Some(address) = sender
4760            .strip_prefix("@addr:")
4761            .and_then(|value| value.parse::<u32>().ok())
4762        else {
4763            return sender.to_string();
4764        };
4765        Self::hostname_for_address_locked(st, address).unwrap_or_else(|| sender.to_string())
4766    }
4767
4768    /// Resolve both assigned node addresses and stable compact wire hashes.
4769    ///
4770    /// Remote boards behind a router appear in incremental topology summaries,
4771    /// but do not send their link-local address advertisement across that
4772    /// router. Include those retained topology names so application packets
4773    /// can still be attributed at the far edge of a multi-hop network.
4774    fn hostname_for_address_locked(st: &RouterInner, address: NodeAddress) -> Option<String> {
4775        Self::hostname_for_address_borrowed_locked(st, address).map(str::to_string)
4776    }
4777
4778    fn hostname_for_address_borrowed_locked<'a>(
4779        st: &'a RouterInner,
4780        address: NodeAddress,
4781    ) -> Option<&'a str> {
4782        if let Some(hostname) = st.address_by_value.get(&address) {
4783            return Some(hostname.as_str());
4784        }
4785
4786        let mut found: Option<&'a str> = None;
4787        let mut collision = false;
4788        let mut consider = |candidate: &'a str| {
4789            if sender_address_u32(candidate) != address {
4790                return;
4791            }
4792            match found {
4793                None => found = Some(candidate),
4794                Some(existing) if existing == candidate => {}
4795                // Do not guess if two discovered names have the same compact
4796                // 32-bit wire address. Address discovery must resolve that
4797                // conflict before callers can safely attribute the packet.
4798                Some(_) => collision = true,
4799            }
4800        };
4801
4802        for entry in st.address_book.values() {
4803            consider(entry.hostname.as_ref());
4804        }
4805        #[cfg(feature = "discovery")]
4806        for route in st.discovery_routes.values() {
4807            for (announcer, state) in &route.announcers {
4808                consider(announcer);
4809                for board in state.topology_boards.iter() {
4810                    consider(&board.sender_id);
4811                    // A constrained bridge can retain a remote only as a
4812                    // named connection on its own topology node.  That name
4813                    // is still authoritative discovery identity and is
4814                    // sufficient to resolve the deterministic compact sender
4815                    // address used by application and ACK packets.
4816                    for peer in &board.connections {
4817                        consider(peer);
4818                    }
4819                }
4820            }
4821        }
4822        if collision { None } else { found }
4823    }
4824
4825    #[cfg(feature = "discovery")]
4826    fn address_mode_from_code(mode: u8, requested: NodeAddress) -> AddressAssignmentMode {
4827        match mode {
4828            2 => AddressAssignmentMode::Static(if requested == 0 { 1 } else { requested }),
4829            1 => AddressAssignmentMode::Requested(if requested == 0 { 1 } else { requested }),
4830            _ => AddressAssignmentMode::Dynamic,
4831        }
4832    }
4833
4834    #[cfg(feature = "discovery")]
4835    fn address_winner_pref(entry: &AddressBookEntry) -> (u8, core::cmp::Reverse<u64>, u64) {
4836        let rank = match entry.mode {
4837            AddressAssignmentMode::Static(_) => 2,
4838            AddressAssignmentMode::Requested(_) => 1,
4839            AddressAssignmentMode::Dynamic => 0,
4840        };
4841        (
4842            rank,
4843            core::cmp::Reverse(entry.birth_ms),
4844            u64::MAX - entry.owner_hash,
4845        )
4846    }
4847
4848    #[cfg(feature = "discovery")]
4849    fn identity_winner_pref(entry: &AddressBookEntry) -> (core::cmp::Reverse<u64>, u64) {
4850        (
4851            core::cmp::Reverse(entry.birth_ms),
4852            u64::MAX - entry.owner_hash,
4853        )
4854    }
4855
4856    #[cfg(feature = "discovery")]
4857    fn allocate_free_address_locked(st: &RouterInner, seed: NodeAddress) -> NodeAddress {
4858        let mut candidate = if seed == 0 { 1 } else { seed };
4859        for _ in 0..u32::MAX {
4860            if candidate != 0 && !st.address_by_value.contains_key(&candidate) {
4861                return candidate;
4862            }
4863            candidate = candidate.wrapping_add(1);
4864            if candidate == 0 {
4865                candidate = 1;
4866            }
4867        }
4868        1
4869    }
4870
4871    #[cfg(feature = "discovery")]
4872    fn unique_hostname_locked(st: &RouterInner, base: &str, owner_hash: u64) -> String {
4873        if !st.address_book.contains_key(base) && st.local_address.hostname.as_ref() != base {
4874            return base.to_string();
4875        }
4876        let stem = if base.is_empty() { "node" } else { base };
4877        let mut candidate = format!("{stem}-{owner_hash:08x}");
4878        let mut suffix = 1u32;
4879        while st.address_book.contains_key(&candidate)
4880            || st.local_address.hostname.as_ref() == candidate
4881        {
4882            candidate = format!("{stem}-{owner_hash:08x}-{suffix}");
4883            suffix = suffix.saturating_add(1);
4884        }
4885        candidate
4886    }
4887
4888    fn update_local_identity_locked(
4889        &self,
4890        st: &mut RouterInner,
4891        mut local: AddressBookEntry,
4892        reason: AddressChangeReason,
4893        spare: Option<Box<AddressBookEntry>>,
4894    ) -> TelemetryResult<AddressChange> {
4895        let old = st.local_address.clone();
4896        // Relays have no application identity record until explicitly configured
4897        // or moved by a conflict. Reserve that storage before changing identity.
4898        let mut record = match st.address_book.remove(old.hostname.as_ref()) {
4899            Some(record) => record,
4900            None => spare.ok_or(TelemetryError::Io("local identity changed"))?,
4901        };
4902        st.address_by_value.remove(&old.address);
4903        local.last_seen_ms = self.clock.now_ms();
4904        st.address_by_value
4905            .insert(local.address, local.hostname.to_string());
4906        *record = local.clone();
4907        st.address_book.insert(local.hostname.to_string(), record);
4908        st.local_address = local.clone();
4909        *self.sender.lock() = local.hostname.clone();
4910        #[cfg(feature = "discovery")]
4911        Self::note_discovery_topology_change_locked(st, self.clock.now_ms());
4912        Ok(AddressChange {
4913            old_hostname: old.hostname,
4914            new_hostname: local.hostname,
4915            old_address: old.address,
4916            new_address: local.address,
4917            reason,
4918        })
4919    }
4920
4921    fn notify_address_change(&self, change: AddressChange) -> TelemetryResult<()> {
4922        for handler in self.cfg.address_change_handlers().iter() {
4923            (handler.handler)(change.clone())?;
4924        }
4925        Ok(())
4926    }
4927
4928    #[cfg(feature = "discovery")]
4929    fn local_address_advertisement(
4930        &self,
4931        reachable_endpoints: Vec<DataEndpoint>,
4932        reachable_network_variables: Vec<DataType>,
4933        reachable_timesync_sources: Vec<String>,
4934        link_capabilities: discovery::LinkCapabilities,
4935        state: u8,
4936    ) -> discovery::AddressAdvertisement {
4937        let st = self.state.lock();
4938        discovery::AddressAdvertisement {
4939            hostname: st.local_address.hostname.to_string(),
4940            address: st.local_address.address,
4941            requested_address: st.local_address.requested_address,
4942            mode: st.local_address.mode.mode_code(),
4943            state,
4944            birth_ms: st.local_address.birth_ms,
4945            owner_hash: st.local_address.owner_hash,
4946            reachable_endpoints,
4947            reachable_network_variables,
4948            reachable_timesync_sources,
4949            link_capabilities,
4950        }
4951    }
4952
4953    #[cfg(feature = "discovery")]
4954    fn prepare_address_record(
4955        st: &RouterInner,
4956        ad: &discovery::AddressAdvertisement,
4957        now_ms: u64,
4958    ) -> TelemetryResult<PreparedAddressRecords> {
4959        let remote = AddressBookEntry {
4960            hostname: Arc::from(ad.hostname.as_str()),
4961            address: ad.address,
4962            requested_address: ad.requested_address,
4963            mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
4964            birth_ms: ad.birth_ms,
4965            owner_hash: ad.owner_hash,
4966            last_seen_ms: now_ms,
4967        };
4968        let peer = if st
4969            .address_book
4970            .get(ad.hostname.as_str())
4971            .is_some_and(|entry| entry.owner_hash == ad.owner_hash)
4972        {
4973            None
4974        } else {
4975            Some(crate::peer_record::try_box(remote.clone())?)
4976        };
4977        let local = &st.local_address;
4978        let moves_local = remote.owner_hash != local.owner_hash
4979            && ((remote.hostname == local.hostname
4980                && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(local))
4981                || (remote.address == local.address
4982                    && Self::address_winner_pref(&remote) > Self::address_winner_pref(local)));
4983        let spare = if moves_local && !st.address_book.contains_key(local.hostname.as_ref()) {
4984            Some(crate::peer_record::try_box(local.clone())?)
4985        } else {
4986            None
4987        };
4988        Ok(PreparedAddressRecords { peer, local: spare })
4989    }
4990
4991    #[cfg(all(feature = "discovery", feature = "timesync", test))]
4992    fn ingest_address_advertisement(
4993        &self,
4994        ad: discovery::AddressAdvertisement,
4995    ) -> TelemetryResult<bool> {
4996        let prepared = Self::prepare_address_record(&self.state.lock(), &ad, self.clock.now_ms())?;
4997        self.ingest_prepared_address(ad, prepared)
4998    }
4999
5000    #[cfg(feature = "discovery")]
5001    fn ingest_prepared_address(
5002        &self,
5003        ad: discovery::AddressAdvertisement,
5004        prepared: PreparedAddressRecords,
5005    ) -> TelemetryResult<bool> {
5006        let PreparedAddressRecords {
5007            peer: mut fresh_peer,
5008            local: mut local_spare,
5009        } = prepared;
5010        let now_ms = self.clock.now_ms();
5011        let mut remote = match fresh_peer.as_ref() {
5012            Some(record) => record.as_ref().clone(),
5013            None => AddressBookEntry {
5014                hostname: Arc::from(ad.hostname.as_str()),
5015                address: ad.address,
5016                requested_address: ad.requested_address,
5017                mode: Self::address_mode_from_code(ad.mode, ad.requested_address),
5018                birth_ms: ad.birth_ms,
5019                owner_hash: ad.owner_hash,
5020                last_seen_ms: now_ms,
5021            },
5022        };
5023        remote.last_seen_ms = now_ms;
5024        let mut change = None;
5025        let mut changed = false;
5026        {
5027            let mut st = self.state.lock();
5028            let local = st.local_address.clone();
5029            // Cold storage was reserved before publishing this packet's route.
5030            // A concurrent identity change must retry instead of allocating
5031            // after route commit or reusing another owner's identity record.
5032            if fresh_peer.is_none()
5033                && !st
5034                    .address_book
5035                    .get(remote.hostname.as_ref())
5036                    .is_some_and(|entry| entry.owner_hash == remote.owner_hash)
5037            {
5038                return Err(TelemetryError::Io("peer identity changed"));
5039            }
5040            let hostname_conflict =
5041                remote.hostname == local.hostname && remote.owner_hash != local.owner_hash;
5042            let address_conflict =
5043                remote.address == local.address && remote.owner_hash != local.owner_hash;
5044
5045            if hostname_conflict
5046                && Self::identity_winner_pref(&remote) > Self::identity_winner_pref(&local)
5047            {
5048                let mut next = local.clone();
5049                next.hostname = Arc::from(Self::unique_hostname_locked(
5050                    &st,
5051                    local.hostname.as_ref(),
5052                    local.owner_hash,
5053                ));
5054                change = Some(self.update_local_identity_locked(
5055                    &mut st,
5056                    next,
5057                    AddressChangeReason::HostnameConflict,
5058                    local_spare.take(),
5059                )?);
5060                changed = true;
5061            } else if address_conflict
5062                && Self::address_winner_pref(&remote) > Self::address_winner_pref(&local)
5063            {
5064                let mut next = local.clone();
5065                let seed = next
5066                    .requested_address
5067                    .max(Self::fallback_address_for_hostname(next.hostname.as_ref()));
5068                next.address = Self::allocate_free_address_locked(&st, seed);
5069                let reason = match next.mode {
5070                    AddressAssignmentMode::Static(_) => AddressChangeReason::StaticConflict,
5071                    AddressAssignmentMode::Requested(_) => AddressChangeReason::RequestedConflict,
5072                    AddressAssignmentMode::Dynamic => AddressChangeReason::DynamicConflict,
5073                };
5074                change = Some(self.update_local_identity_locked(
5075                    &mut st,
5076                    next,
5077                    reason,
5078                    local_spare.take(),
5079                )?);
5080                changed = true;
5081            } else if hostname_conflict {
5082                remote.hostname = Arc::from(Self::unique_hostname_locked(
5083                    &st,
5084                    remote.hostname.as_ref(),
5085                    remote.owner_hash,
5086                ));
5087                changed = true;
5088            } else if address_conflict {
5089                let seed = remote
5090                    .requested_address
5091                    .max(Self::fallback_address_for_hostname(
5092                        remote.hostname.as_ref(),
5093                    ));
5094                remote.address = Self::allocate_free_address_locked(&st, seed);
5095                changed = true;
5096            }
5097
5098            let mut record = fresh_peer.take().unwrap_or_else(|| {
5099                st.address_book
5100                    .remove(ad.hostname.as_str())
5101                    .expect("known peer remains present")
5102            });
5103            *record = remote.clone();
5104            st.address_by_value.remove(&remote.address);
5105            st.address_by_value
5106                .insert(remote.address, remote.hostname.to_string());
5107            st.address_book.insert(remote.hostname.to_string(), record);
5108        }
5109        if let Some(change) = change {
5110            self.notify_address_change(change)?;
5111        }
5112        Ok(changed)
5113    }
5114
5115    #[inline]
5116    #[cfg(feature = "discovery")]
5117    fn is_end_to_end_ack_sender(sender: &str) -> bool {
5118        sender == Self::END_TO_END_ACK_SENDER || sender.starts_with(Self::END_TO_END_ACK_PREFIX)
5119    }
5120
5121    fn decode_end_to_end_ack_sender_hash(sender: &str) -> Option<u64> {
5122        if let Some(ack_sender) = sender.strip_prefix(Self::END_TO_END_ACK_PREFIX)
5123            && !ack_sender.is_empty()
5124        {
5125            return Some(Self::sender_hash(ack_sender));
5126        }
5127        None
5128    }
5129
5130    /// Return the stable destination key carried by an end-to-end ACK.
5131    ///
5132    /// Compact embedded headers encode senders as numeric addresses, so the legacy
5133    /// `E2EACK:<hostname>` marker cannot survive a firmware hop. New ACKs carry the acknowledging
5134    /// owner's hash in their frozen target contract; marker decoding remains for older peers.
5135    fn end_to_end_ack_sender_hash(pkt: &Packet) -> Option<u64> {
5136        if pkt.data_type() != DataType::ReliableAck {
5137            return None;
5138        }
5139        pkt.wire_target_senders()
5140            .first()
5141            .copied()
5142            .or_else(|| Self::decode_end_to_end_ack_sender_hash(pkt.sender()))
5143    }
5144
5145    fn is_end_to_end_ack_packet(pkt: &Packet) -> bool {
5146        Self::end_to_end_ack_sender_hash(pkt).is_some()
5147            && Self::decode_end_to_end_reliable_ack(pkt.payload()).is_ok()
5148    }
5149
5150    fn encode_end_to_end_ack_sender(&self) -> String {
5151        let sender = self.sender_arc();
5152        format!("{}{}", Self::END_TO_END_ACK_PREFIX, sender)
5153    }
5154
5155    #[cfg(feature = "discovery")]
5156    fn is_end_to_end_destination_sender(&self, sender: &str) -> bool {
5157        sender != self.sender_arc().as_ref() && !Self::is_end_to_end_ack_sender(sender)
5158    }
5159
5160    fn encode_p2p_payload(
5161        src_hostname: &str,
5162        src_address: NodeAddress,
5163        src_port: P2pPort,
5164        dst_port: P2pPort,
5165        payload: &[u8],
5166    ) -> TelemetryResult<crate::SharedBytes> {
5167        let host_len = u16::try_from(src_hostname.len())
5168            .map_err(|_| TelemetryError::Pack("p2p hostname too long"))?;
5169        let payload_len = u32::try_from(payload.len())
5170            .map_err(|_| TelemetryError::Pack("p2p payload too long"))?;
5171        let mut out = Vec::with_capacity(
5172            15usize
5173                .saturating_add(src_hostname.len())
5174                .saturating_add(payload.len()),
5175        );
5176        out.push(1);
5177        out.extend_from_slice(&dst_port.to_le_bytes());
5178        out.extend_from_slice(&src_port.to_le_bytes());
5179        out.extend_from_slice(&src_address.to_le_bytes());
5180        out.extend_from_slice(&host_len.to_le_bytes());
5181        out.extend_from_slice(&payload_len.to_le_bytes());
5182        out.extend_from_slice(src_hostname.as_bytes());
5183        out.extend_from_slice(payload);
5184        Ok(out.into())
5185    }
5186
5187    fn decode_p2p_payload(payload: &[u8]) -> TelemetryResult<P2pDecoded<'_>> {
5188        if payload.len() < 15 {
5189            return Err(TelemetryError::Unpack("p2p frame short"));
5190        }
5191        if payload[0] != 1 {
5192            return Err(TelemetryError::Unpack("p2p frame version"));
5193        }
5194        let destination_port =
5195            u16::from_le_bytes(payload[1..3].try_into().expect("2-byte dst port"));
5196        let source_port = u16::from_le_bytes(payload[3..5].try_into().expect("2-byte src port"));
5197        let source_address = u32::from_le_bytes(payload[5..9].try_into().expect("4-byte address"));
5198        let host_len =
5199            u16::from_le_bytes(payload[9..11].try_into().expect("2-byte host len")) as usize;
5200        let body_len =
5201            u32::from_le_bytes(payload[11..15].try_into().expect("4-byte body len")) as usize;
5202        let host_start = 15usize;
5203        let host_end = host_start.saturating_add(host_len);
5204        let body_end = host_end.saturating_add(body_len);
5205        if host_end > payload.len() || body_end != payload.len() {
5206            return Err(TelemetryError::Unpack("p2p frame length"));
5207        }
5208        let source_hostname = core::str::from_utf8(&payload[host_start..host_end])
5209            .map_err(|_| TelemetryError::Unpack("p2p hostname utf8"))?;
5210        Ok(P2pDecoded {
5211            source_hostname,
5212            source_address,
5213            source_port,
5214            destination_port,
5215            payload: &payload[host_end..body_end],
5216        })
5217    }
5218
5219    fn encode_p2p_stream_payload(
5220        flags: u8,
5221        source_stream_id: P2pStreamId,
5222        destination_stream_id: P2pStreamId,
5223        sequence: u32,
5224        payload: &[u8],
5225    ) -> TelemetryResult<crate::SharedBytes> {
5226        let payload_len = u32::try_from(payload.len())
5227            .map_err(|_| TelemetryError::Pack("p2p stream payload too long"))?;
5228        let mut out = Vec::with_capacity(22usize.saturating_add(payload.len()));
5229        out.extend_from_slice(&P2P_STREAM_MAGIC);
5230        out.push(P2P_STREAM_VERSION);
5231        out.push(flags);
5232        out.extend_from_slice(&source_stream_id.to_le_bytes());
5233        out.extend_from_slice(&destination_stream_id.to_le_bytes());
5234        out.extend_from_slice(&sequence.to_le_bytes());
5235        out.extend_from_slice(&payload_len.to_le_bytes());
5236        out.extend_from_slice(payload);
5237        Ok(out.into())
5238    }
5239
5240    fn decode_p2p_stream_payload(payload: &[u8]) -> TelemetryResult<Option<P2pStreamDecoded<'_>>> {
5241        if !payload.starts_with(&P2P_STREAM_MAGIC) {
5242            return Ok(None);
5243        }
5244        if payload.len() < 22 {
5245            return Err(TelemetryError::Unpack("p2p stream frame short"));
5246        }
5247        if payload[4] != P2P_STREAM_VERSION {
5248            return Err(TelemetryError::Unpack("p2p stream frame version"));
5249        }
5250        let flags = payload[5];
5251        let source_stream_id =
5252            u32::from_le_bytes(payload[6..10].try_into().expect("stream source id"));
5253        let destination_stream_id =
5254            u32::from_le_bytes(payload[10..14].try_into().expect("stream destination id"));
5255        let sequence = u32::from_le_bytes(payload[14..18].try_into().expect("stream sequence"));
5256        let body_len =
5257            u32::from_le_bytes(payload[18..22].try_into().expect("stream body len")) as usize;
5258        let body_end = 22usize.saturating_add(body_len);
5259        if body_end != payload.len() {
5260            return Err(TelemetryError::Unpack("p2p stream frame length"));
5261        }
5262        Ok(Some(P2pStreamDecoded {
5263            flags,
5264            source_stream_id,
5265            destination_stream_id,
5266            sequence,
5267            payload: &payload[22..],
5268        }))
5269    }
5270
5271    fn allocate_p2p_stream_id_locked(st: &mut RouterInner) -> P2pStreamId {
5272        for _ in 0..u32::MAX {
5273            let id = st
5274                .local_services
5275                .as_ref()
5276                .expect("local services enabled")
5277                .next_p2p_stream_id
5278                .max(1);
5279            st.local_services
5280                .as_mut()
5281                .expect("local services enabled")
5282                .next_p2p_stream_id = st
5283                .local_services
5284                .as_ref()
5285                .expect("local services enabled")
5286                .next_p2p_stream_id
5287                .wrapping_add(1)
5288                .max(1);
5289            if !st
5290                .local_services
5291                .as_ref()
5292                .expect("local services enabled")
5293                .p2p_stream_sessions
5294                .contains_key(&id)
5295            {
5296                return id;
5297            }
5298        }
5299        0
5300    }
5301
5302    fn dispatch_p2p_packet(&self, pkt: &Packet) -> TelemetryResult<()> {
5303        if pkt.data_type() != DataType::P2pMessage {
5304            return Ok(());
5305        }
5306        let decoded = Self::decode_p2p_payload(pkt.payload())?;
5307        if let Some(stream) = Self::decode_p2p_stream_payload(decoded.payload)? {
5308            return self.dispatch_p2p_stream_frame(&decoded, &stream);
5309        }
5310        let handlers = {
5311            let st = self.state.lock();
5312            st.local_services
5313                .as_ref()
5314                .expect("local services enabled")
5315                .p2p_port_handlers
5316                .get(&decoded.destination_port)
5317                .cloned()
5318                .unwrap_or_default()
5319        };
5320        for handler in handlers {
5321            (handler.handler)(P2pMessage {
5322                source_hostname: decoded.source_hostname,
5323                source_address: decoded.source_address,
5324                source_port: decoded.source_port,
5325                destination_port: decoded.destination_port,
5326                payload: decoded.payload,
5327            })?;
5328        }
5329        Ok(())
5330    }
5331
5332    fn dispatch_p2p_stream_frame(
5333        &self,
5334        msg: &P2pDecoded<'_>,
5335        frame: &P2pStreamDecoded<'_>,
5336    ) -> TelemetryResult<()> {
5337        let mut events: Vec<PendingP2pStreamEvent> = Vec::new();
5338        let mut reply: Option<(AddressBookEntry, P2pPort, P2pPort, crate::SharedBytes)> = None;
5339        {
5340            let mut st = self.state.lock();
5341            if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK == 0 {
5342                let peer_hostname: Arc<str> = Arc::from(msg.source_hostname);
5343                let existing_id = st
5344                    .local_services
5345                    .as_ref()
5346                    .expect("local services enabled")
5347                    .p2p_stream_sessions
5348                    .iter()
5349                    .find_map(|(id, session)| {
5350                        (session.peer_stream_id == frame.source_stream_id
5351                            && session.local_port == msg.destination_port
5352                            && session.peer_port == msg.source_port
5353                            && session.peer_address == msg.source_address
5354                            && session.peer_hostname.as_ref() == msg.source_hostname)
5355                            .then_some(*id)
5356                    });
5357                let local_id = if let Some(local_id) = existing_id {
5358                    local_id
5359                } else {
5360                    let local_id = Self::allocate_p2p_stream_id_locked(&mut st);
5361                    if local_id == 0 {
5362                        return Err(TelemetryError::Io("p2p stream id exhausted"));
5363                    }
5364                    st.local_services
5365                        .as_mut()
5366                        .expect("local services enabled")
5367                        .p2p_stream_sessions
5368                        .insert(
5369                            local_id,
5370                            P2pStreamSession {
5371                                peer_hostname: peer_hostname.clone(),
5372                                peer_address: msg.source_address,
5373                                local_port: msg.destination_port,
5374                                peer_port: msg.source_port,
5375                                peer_stream_id: frame.source_stream_id,
5376                                next_sequence: 1,
5377                                connected: true,
5378                            },
5379                        );
5380                    let handlers = st
5381                        .local_services
5382                        .as_ref()
5383                        .expect("local services enabled")
5384                        .p2p_stream_handlers
5385                        .get(&msg.destination_port)
5386                        .cloned()
5387                        .unwrap_or_default();
5388                    events.push(PendingP2pStreamEvent {
5389                        handlers,
5390                        kind: P2pStreamEventKind::Accepted,
5391                        stream_id: local_id,
5392                        peer_stream_id: frame.source_stream_id,
5393                        peer_hostname: peer_hostname.clone(),
5394                        peer_address: msg.source_address,
5395                        local_port: msg.destination_port,
5396                        peer_port: msg.source_port,
5397                    });
5398                    local_id
5399                };
5400                let dst = st
5401                    .address_book
5402                    .get(msg.source_hostname)
5403                    .map(|entry| entry.as_ref().clone())
5404                    .unwrap_or(AddressBookEntry {
5405                        hostname: peer_hostname,
5406                        address: msg.source_address,
5407                        requested_address: msg.source_address,
5408                        mode: AddressAssignmentMode::Dynamic,
5409                        birth_ms: self.clock.now_ms(),
5410                        owner_hash: Self::sender_hash(msg.source_hostname),
5411                        last_seen_ms: self.clock.now_ms(),
5412                    });
5413                let payload = Self::encode_p2p_stream_payload(
5414                    P2P_STREAM_SYN | P2P_STREAM_ACK,
5415                    local_id,
5416                    frame.source_stream_id,
5417                    0,
5418                    &[],
5419                )?;
5420                reply = Some((dst, msg.source_port, msg.destination_port, payload));
5421            } else if frame.flags & P2P_STREAM_SYN != 0 && frame.flags & P2P_STREAM_ACK != 0 {
5422                if let Some(session) = st
5423                    .local_services
5424                    .as_mut()
5425                    .expect("local services enabled")
5426                    .p2p_stream_sessions
5427                    .get_mut(&frame.destination_stream_id)
5428                {
5429                    session.peer_stream_id = frame.source_stream_id;
5430                    session.connected = true;
5431                    let peer_hostname = session.peer_hostname.clone();
5432                    let peer_address = session.peer_address;
5433                    let local_port = session.local_port;
5434                    let peer_port = session.peer_port;
5435                    let handlers = st
5436                        .local_services
5437                        .as_ref()
5438                        .expect("local services enabled")
5439                        .p2p_stream_handlers
5440                        .get(&local_port)
5441                        .cloned()
5442                        .unwrap_or_default();
5443                    events.push(PendingP2pStreamEvent {
5444                        handlers,
5445                        kind: P2pStreamEventKind::Connected,
5446                        stream_id: frame.destination_stream_id,
5447                        peer_stream_id: frame.source_stream_id,
5448                        peer_hostname,
5449                        peer_address,
5450                        local_port,
5451                        peer_port,
5452                    });
5453                }
5454            } else if frame.flags & (P2P_STREAM_FIN | P2P_STREAM_RST | P2P_STREAM_DATA) != 0 {
5455                let kind = if frame.flags & P2P_STREAM_RST != 0 {
5456                    P2pStreamEventKind::Reset
5457                } else if frame.flags & P2P_STREAM_FIN != 0 {
5458                    P2pStreamEventKind::Closed
5459                } else {
5460                    P2pStreamEventKind::Data
5461                };
5462                let session_id = if frame.destination_stream_id != 0 {
5463                    Some(frame.destination_stream_id)
5464                } else {
5465                    st.local_services
5466                        .as_ref()
5467                        .expect("local services enabled")
5468                        .p2p_stream_sessions
5469                        .iter()
5470                        .find_map(|(id, session)| {
5471                            (session.peer_stream_id == frame.source_stream_id
5472                                && session.local_port == msg.destination_port
5473                                && session.peer_port == msg.source_port
5474                                && session.peer_address == msg.source_address
5475                                && session.peer_hostname.as_ref() == msg.source_hostname)
5476                                .then_some(*id)
5477                        })
5478                };
5479                if let Some(session_id) = session_id
5480                    && let Some(session) = st
5481                        .local_services
5482                        .as_ref()
5483                        .expect("local services enabled")
5484                        .p2p_stream_sessions
5485                        .get(&session_id)
5486                {
5487                    let handlers = st
5488                        .local_services
5489                        .as_ref()
5490                        .expect("local services enabled")
5491                        .p2p_stream_handlers
5492                        .get(&session.local_port)
5493                        .cloned()
5494                        .unwrap_or_default();
5495                    events.push(PendingP2pStreamEvent {
5496                        handlers,
5497                        kind,
5498                        stream_id: session_id,
5499                        peer_stream_id: frame.source_stream_id,
5500                        peer_hostname: session.peer_hostname.clone(),
5501                        peer_address: session.peer_address,
5502                        local_port: session.local_port,
5503                        peer_port: session.peer_port,
5504                    });
5505                    if matches!(kind, P2pStreamEventKind::Closed | P2pStreamEventKind::Reset) {
5506                        st.local_services
5507                            .as_mut()
5508                            .expect("local services enabled")
5509                            .p2p_stream_sessions
5510                            .remove(&session_id);
5511                    }
5512                }
5513            }
5514        }
5515        if let Some((dst, dst_port, src_port, payload)) = reply {
5516            self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
5517        }
5518        for pending in events {
5519            for handler in pending.handlers {
5520                (handler.handler)(P2pStreamEvent {
5521                    kind: pending.kind,
5522                    stream_id: pending.stream_id,
5523                    peer_stream_id: pending.peer_stream_id,
5524                    sequence: frame.sequence,
5525                    peer_hostname: pending.peer_hostname.as_ref(),
5526                    peer_address: pending.peer_address,
5527                    local_port: pending.local_port,
5528                    peer_port: pending.peer_port,
5529                    payload: frame.payload,
5530                })?;
5531            }
5532        }
5533        Ok(())
5534    }
5535
5536    // Close only a duplicate non-parent edge, never this router's aggregate
5537    // receipt. The first ingress continues to own the full downstream subtree.
5538    fn queue_duplicate_proxy_edge_receipt(
5539        &self,
5540        item: &RouterRxItem,
5541        packet_id: u64,
5542    ) -> TelemetryResult<()> {
5543        let Some(src) = item.src else { return Ok(()) };
5544        let eligible = |state: &RouterInner| {
5545            state.end_to_end_reliable_tx.contains_key(&packet_id)
5546                && state
5547                    .reliable_return_routes
5548                    .get(&packet_id)
5549                    .is_some_and(|route| {
5550                        route.side != src
5551                            && route.local_admission_completed
5552                            && !route.admission_cancelled
5553                            && route.local_admission_token == 0
5554                            && route.upstream_contract.is_some()
5555                    })
5556        };
5557        if !eligible(&self.state.lock()) {
5558            return Ok(());
5559        }
5560        let local_sender = self.sender_arc();
5561        let local_hash = Self::sender_hash(local_sender.as_ref());
5562        let source_address = match &item.data {
5563            RouterItem::Packet(packet) => {
5564                packet.validate()?;
5565                if !packet.wire_target_senders().contains(&local_hash) {
5566                    return Ok(());
5567                }
5568                sender_address_u32(packet.sender())
5569            }
5570            RouterItem::Packed(bytes) => {
5571                let address = crate::shared_bytes::with_slice(bytes, |view| {
5572                    let frame = wire_format::peek_routing_frame_info(view)?;
5573                    Ok::<_, TelemetryError>(
5574                        frame
5575                            .envelope
5576                            .target_senders()
5577                            .any(|target| target == local_hash)
5578                            .then_some(frame.envelope.source_address),
5579                    )
5580                })??;
5581                let Some(address) = address else {
5582                    return Ok(());
5583                };
5584                address
5585            }
5586        };
5587        // Bind to the original retained observation, not a mutable discovery
5588        // alias or a newly decompressed payload. Its logical ID and source
5589        // address are unchanged across the proxy's frozen child contracts.
5590        let publisher_hash = {
5591            let state = self.state.lock();
5592            if !eligible(&state) {
5593                return Ok(());
5594            }
5595            let retained = &state
5596                .end_to_end_reliable_tx
5597                .get(&packet_id)
5598                .expect("eligible owner")
5599                .data;
5600            let retained_address = match retained {
5601                RouterItem::Packet(packet) => sender_address_u32(packet.sender()),
5602                RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
5603                    wire_format::peek_routing_envelope(view).map(|envelope| envelope.source_address)
5604                })??,
5605            };
5606            if retained_address != source_address {
5607                return Ok(());
5608            }
5609            let Some((original_id, publisher)) = state
5610                .reliable_return_routes
5611                .get(&packet_id)
5612                .and_then(|route| route.upstream_contract)
5613            else {
5614                return Ok(());
5615            };
5616            if original_id != packet_id {
5617                return Ok(());
5618            };
5619            publisher
5620        };
5621        let ack = Packet::new_local(
5622            DataType::ReliableAck,
5623            message_meta(DataType::ReliableAck).endpoints_owned(),
5624            Arc::from(self.encode_end_to_end_ack_sender()),
5625            self.packet_timestamp_ms(),
5626            crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
5627        )?;
5628        let ack = self
5629            .attach_wire_contract_to_item(RouterItem::Packet(ack), &[local_hash, publisher_hash])?;
5630        let priority = self.item_service_priority(&ack)?;
5631        let mut state = self.state.lock();
5632        if !eligible(&state)
5633            || !Self::side_ref(&state, src).is_ok_and(|side| side.opts.egress_enabled)
5634            || state
5635                .reliable_return_routes
5636                .get(&packet_id)
5637                .and_then(|route| route.upstream_contract)
5638                != Some((packet_id, publisher_hash))
5639        {
5640            return Ok(());
5641        }
5642        let already_queued = state.transmit_queue.iter().any(|queued| {
5643            let RouterTxItem::ToSide { dst, data, .. } = &queued.item else {
5644                return false;
5645            };
5646            if *dst != src {
5647                return false;
5648            }
5649            match data {
5650                RouterItem::Packet(packet) => {
5651                    packet.data_type() == DataType::ReliableAck
5652                        && packet.payload() == packet_id.to_le_bytes()
5653                }
5654                RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
5655                    wire_format::scheduling_type(view) == Some(DataType::ReliableAck)
5656                        && wire_format::unpack_packet(view)
5657                            .is_ok_and(|packet| packet.payload() == packet_id.to_le_bytes())
5658                })
5659                .unwrap_or(false),
5660            }
5661        });
5662        if !already_queued {
5663            state.push_transmit(TxQueued {
5664                disposable: TX_DISPOSABLE,
5665                item: RouterTxItem::ToSide {
5666                    src: None,
5667                    dst: src,
5668                    data: ack,
5669                },
5670                ignore_local: true,
5671                priority,
5672                delivery_state: QueuedDelivery::Untracked,
5673                accepted_type: DataType::ReliableAck.as_u32(),
5674                deferred_since_ms: 0,
5675                retry_after_ms: 0,
5676            })?;
5677        }
5678        Ok(())
5679    }
5680
5681    // Existing private route-fixture helper acquires and immediately retires
5682    // the same admission lease; production retains it across actual RX work.
5683    #[cfg(test)]
5684    fn note_reliable_return_route(&self, side: RouterSideId, packet_id: u64) -> bool {
5685        self.begin_reliable_local_admission(side, packet_id)
5686            .is_ok_and(|guard| guard.is_some())
5687    }
5688
5689    /// Record the original live ingress for `packet_id`, preserving it across echoes.
5690    ///
5691    /// End-to-end reliable acknowledgements should return only toward the
5692    /// source-side that originated the data flow, not be flooded everywhere.
5693    /// This helper stores that learned return path in a bounded LRU-like cache.
5694    fn begin_reliable_local_admission(
5695        &self,
5696        side: RouterSideId,
5697        packet_id: u64,
5698    ) -> TelemetryResult<Option<ReliableLocalAdmissionGuard<'_>>> {
5699        self.begin_reliable_local_admission_with_parent(side, packet_id, None)
5700    }
5701
5702    fn begin_reliable_local_admission_with_parent(
5703        &self,
5704        side: RouterSideId,
5705        packet_id: u64,
5706        parent: Option<&ReliableLocalAdmissionGuard<'_>>,
5707    ) -> TelemetryResult<Option<ReliableLocalAdmissionGuard<'_>>> {
5708        let mut st = self.state.lock();
5709        if let Some(parent) = parent {
5710            parent.check_locked(&st)?;
5711        }
5712
5713        if !Self::side_ref(&st, side)?.opts.ingress_enabled {
5714            return Err(TelemetryError::HandlerError(
5715                "router: ingress disabled for side id",
5716            ));
5717        }
5718
5719        if st
5720            .reliable_return_routes
5721            .get(&packet_id)
5722            .is_some_and(|route| route.local_admission_token != 0)
5723        {
5724            return Err(TelemetryError::Io("rx packet busy"));
5725        }
5726        // The retained proxy record owns retries until its child cohort
5727        // completes. Neither CAN reflections nor repeated upstream copies may
5728        // replace its return route or reapply the managed value meanwhile.
5729        if st.end_to_end_reliable_tx.contains_key(&packet_id)
5730            && st
5731                .reliable_return_routes
5732                .get(&packet_id)
5733                .is_some_and(|route| route.upstream_contract.is_some())
5734        {
5735            return Ok(None);
5736        }
5737        // A retry can remain in RX dedupe after its smaller return-path cache
5738        // entry was evicted. Relearning that path must not restart the replay
5739        // delay on every retry, or sustained cache churn suppresses it forever.
5740        let now = self.clock.now_ms();
5741        let retry_ms = runtime_reliable_retransmit_ms();
5742        let duplicate = st.recent_rx.contains(&packet_id)
5743            || st
5744                .reliable_return_routes
5745                .get(&packet_id)
5746                .is_some_and(|route| route.delivery_completed);
5747        let last_replay_ms = if duplicate {
5748            now.saturating_sub(retry_ms)
5749        } else {
5750            now
5751        };
5752        // A reflected copy is not evidence that the original publisher
5753        // moved. Keep the first live ingress during the current retry
5754        // window, otherwise an echo can redirect an already queued ACK into
5755        // the downstream bus. A retry-window expiry or disabled/removed side
5756        // still permits alternate-path recovery without explicit operator action.
5757        let replace_side = !duplicate
5758            || st
5759                .reliable_return_routes
5760                .get(&packet_id)
5761                .is_none_or(|route| {
5762                    now.saturating_sub(route.last_replay_ms) >= retry_ms
5763                        || !Self::side_ref(&st, route.side)
5764                            .is_ok_and(|side| side.opts.egress_enabled)
5765                });
5766        if !Self::can_admit_reliable_return_route_growth_locked(&st, packet_id) {
5767            return Err(TelemetryError::Io("memory pressure"));
5768        }
5769        let token = st
5770            .local_admission_sequence
5771            .checked_add(1)
5772            .ok_or(TelemetryError::Io("local admission sequence exhausted"))?;
5773        if !Self::remember_reliable_return_route_locked(&mut st, packet_id) {
5774            return Err(TelemetryError::Io("reliable return route window full"));
5775        }
5776        st.local_admission_sequence = token;
5777        st.reliable_return_routes
5778            .entry(packet_id)
5779            .and_modify(|route| {
5780                if replace_side {
5781                    route.side = side;
5782                }
5783            })
5784            .or_insert(ReliableReturnRouteState {
5785                side,
5786                delivery_completed: false,
5787                local_admission_completed: false,
5788                local_admission_token: 0,
5789                admission_cancelled: false,
5790                admission_committing: false,
5791                last_replay_ms,
5792                upstream_contract: None,
5793            });
5794        st.reliable_return_routes
5795            .get_mut(&packet_id)
5796            .expect("reserved route")
5797            .local_admission_token = token;
5798        Ok(Some(ReliableLocalAdmissionGuard {
5799            router: self,
5800            packet_id,
5801            token,
5802        }))
5803    }
5804
5805    /// Ensure `packet_id` is retained in the bounded reliable return-route cache.
5806    ///
5807    /// Existing entries are refreshed to the back of the order list. When the
5808    /// cache is full, evict the oldest route not pinned by active delivery.
5809    /// Refuse a new entry if every retained route is still active.
5810    fn can_admit_reliable_return_route_growth_locked(st: &RouterInner, packet_id: u64) -> bool {
5811        if st.reliable_return_routes.contains_key(&packet_id)
5812            || st.reliable_return_routes.len() >= runtime_reliable_max_return_routes().max(1)
5813        {
5814            // Reusing a live row or evicting one bounded row adds no logical
5815            // record charge. The eventual window/pin checks remain unchanged.
5816            return true;
5817        }
5818        st.shared_queue_bytes_used()
5819            .checked_add(RELIABLE_RETURN_ROUTE_EXTRA_LOGICAL_BYTES)
5820            .is_some_and(|bytes| bytes <= st.memory.max_queue_budget)
5821    }
5822
5823    fn remember_reliable_return_route_locked(st: &mut RouterInner, packet_id: u64) -> bool {
5824        if !Self::can_admit_reliable_return_route_growth_locked(st, packet_id) {
5825            return false;
5826        }
5827        let cap = runtime_reliable_max_return_routes().max(1);
5828        st.reliable_return_route_order
5829            .retain(|id| st.reliable_return_routes.contains_key(id) && *id != packet_id);
5830        while st.reliable_return_route_order.len() >= cap {
5831            let Some(index) = st.reliable_return_route_order.iter().position(|id| {
5832                !st.end_to_end_reliable_tx.contains_key(id)
5833                    && !st
5834                        .reliable_return_routes
5835                        .get(id)
5836                        .is_some_and(|route| route.local_admission_token != 0)
5837            }) else {
5838                return false;
5839            };
5840            if let Some(oldest) = st.reliable_return_route_order.remove(index) {
5841                st.reliable_return_routes.remove(&oldest);
5842            }
5843        }
5844        st.reliable_return_route_order.push_back(packet_id);
5845        true
5846    }
5847
5848    #[cfg(feature = "discovery")]
5849    fn remember_end_to_end_reliable_tx_locked(st: &mut RouterInner, packet_id: u64) {
5850        let cap = runtime_reliable_max_end_to_end_pending().max(1);
5851        st.end_to_end_reliable_tx_order
5852            .retain(|id| st.end_to_end_reliable_tx.contains_key(id) && *id != packet_id);
5853        while st.end_to_end_reliable_tx_order.len() >= cap {
5854            if let Some(oldest) = st.end_to_end_reliable_tx_order.pop_front() {
5855                st.end_to_end_reliable_tx.remove(&oldest);
5856            } else {
5857                break;
5858            }
5859        }
5860        st.end_to_end_reliable_tx_order.push_back(packet_id);
5861    }
5862
5863    #[cfg(feature = "discovery")]
5864    fn expected_end_to_end_destinations_locked(
5865        &self,
5866        st: &RouterInner,
5867        data: &RouterItem,
5868        exclude: Option<RouterSideId>,
5869        managed: bool,
5870        resolve_proxy_segment: bool,
5871    ) -> TelemetryResult<BTreeMap<u64, RouterSideId>> {
5872        let (eps, ty) = self.item_route_info(data)?;
5873        // A newly resolved proxy subtree must not wait for its original
5874        // publisher: self-originated echoes are intentionally not delivered.
5875        // Existing frozen cohorts are never rewritten. Ambiguous/unresolved
5876        // compact senders remove no candidate; do not guess from a 32-bit hash.
5877        let original_publisher = if resolve_proxy_segment {
5878            match data {
5879                RouterItem::Packet(packet) => {
5880                    if let Some(address) = packet
5881                        .sender()
5882                        .strip_prefix("@addr:")
5883                        .and_then(|value| value.parse::<u32>().ok())
5884                    {
5885                        Self::hostname_for_address_borrowed_locked(st, address)
5886                            .map(Self::sender_hash)
5887                    } else {
5888                        Some(Self::sender_hash(packet.sender()))
5889                    }
5890                }
5891                RouterItem::Packed(bytes) => {
5892                    let address = crate::shared_bytes::with_slice(bytes, |view| {
5893                        wire_format::peek_routing_envelope(view)
5894                            .map(|envelope| envelope.source_address)
5895                    })??;
5896                    Self::hostname_for_address_borrowed_locked(st, address).map(Self::sender_hash)
5897                }
5898            }
5899        } else {
5900            None
5901        };
5902
5903        // A targeted proxy creates a separate downstream contract. All other
5904        // dispatch/retry planning must retain the existing frozen owner set.
5905        let frozen_targets = if resolve_proxy_segment {
5906            wire_format::TargetSenderSet::Hashes(&[])
5907        } else {
5908            match data {
5909                RouterItem::Packet(packet) => {
5910                    wire_format::TargetSenderSet::Hashes(packet.wire_target_senders())
5911                }
5912                RouterItem::Packed(bytes) => {
5913                    wire_format::peek_routing_envelope(bytes)?.target_sender_set()
5914                }
5915            }
5916        };
5917        let now_ms = self.clock.now_ms();
5918        let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
5919        /* Managed-variable ownership is advertised independently from normal
5920         * endpoint handlers.  Using topology-board endpoint overlap here can
5921         * silently narrow a variable update to one segment once detailed
5922         * topology arrives (for example FLIGHT_STATE is a conventional
5923         * endpoint on the fill side but only a managed variable on avionics).
5924         * Freeze one destination contract per advertising segment instead;
5925         * its announcer is the segment router that can ACK and replicate the
5926         * value to the owners behind it. */
5927        if managed || st.is_managed_variable(ty.as_u32()) {
5928            let mut out = BTreeMap::new();
5929            for (&side, route) in st.discovery_routes.iter() {
5930                if Some(side) == exclude {
5931                    continue;
5932                }
5933                if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5934                    continue;
5935                }
5936                let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5937                    continue;
5938                };
5939                if restrict_link_local && !side_ref.opts.link_local_enabled {
5940                    continue;
5941                }
5942                if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5943                    continue;
5944                }
5945                for (sender_id, sender_state) in route.announcers.iter() {
5946                    if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
5947                        || !sender_state.reachable_network_variables.contains(&ty)
5948                        || !self.is_end_to_end_destination_sender(sender_id)
5949                        || original_publisher == Some(Self::sender_hash(sender_id))
5950                    {
5951                        continue;
5952                    }
5953                    let owner = Self::sender_hash(sender_id);
5954                    if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5955                        continue;
5956                    }
5957                    out.insert(owner, side);
5958                    if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
5959                        return Ok(out);
5960                    }
5961                }
5962            }
5963            return Ok(out);
5964        }
5965        let prefer_best_overlap =
5966            is_reliable_type(ty) && Self::reliable_control_target_packet_id(data)?.is_none();
5967        let scoring_eps = self.preferred_scoring_endpoints(&eps, prefer_best_overlap);
5968        let mut best_overlap = 0usize;
5969        let limit = runtime_reliable_max_end_to_end_pending().max(1);
5970        let mut out = BTreeMap::new();
5971        for (&side, route) in st.discovery_routes.iter() {
5972            if Some(side) == exclude {
5973                continue;
5974            }
5975            if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5976                continue;
5977            }
5978            let Some(side_ref) = st.sides.get(side).and_then(Option::as_ref) else {
5979                continue;
5980            };
5981            if restrict_link_local && !side_ref.opts.link_local_enabled {
5982                continue;
5983            }
5984            if !self.route_allowed_locked(st, exclude, Some(ty), side) {
5985                continue;
5986            }
5987            for sender_state in route.announcers.values() {
5988                if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
5989                    continue;
5990                }
5991                for board in sender_state.topology_boards.iter() {
5992                    if !self.is_end_to_end_destination_sender(&board.sender_id)
5993                        || original_publisher == Some(Self::sender_hash(&board.sender_id))
5994                    {
5995                        continue;
5996                    }
5997                    let owner = Self::sender_hash(&board.sender_id);
5998                    if !frozen_targets.is_empty() && !frozen_targets.contains(owner) {
5999                        continue;
6000                    }
6001                    let overlap = Self::endpoint_overlap_count(
6002                        board.reachable_endpoints.iter().copied(),
6003                        &scoring_eps,
6004                    );
6005                    if overlap > 0 {
6006                        if prefer_best_overlap {
6007                            if overlap < best_overlap {
6008                                continue;
6009                            }
6010                            if overlap > best_overlap {
6011                                out.clear();
6012                                best_overlap = overlap;
6013                            }
6014                            // Keep only the first bounded cohort at the best
6015                            // score. Later higher scores replace it; no second
6016                            // candidate vector or allocation pass is needed.
6017                            if out.len() >= limit {
6018                                continue;
6019                            }
6020                        }
6021                        out.insert(owner, side);
6022                        if !prefer_best_overlap && out.len() >= limit {
6023                            return Ok(out);
6024                        }
6025                    }
6026                }
6027            }
6028        }
6029
6030        Ok(out)
6031    }
6032
6033    #[cfg(feature = "discovery")]
6034    #[allow(clippy::too_many_arguments)]
6035    fn discovered_route_candidates_locked(
6036        &self,
6037        st: &RouterInner,
6038        exclude: Option<RouterSideId>,
6039        ty: DataType,
6040        eps: &[DataEndpoint],
6041        target_senders: wire_format::TargetSenderSet<'_>,
6042        prefer_nonlocal: bool,
6043        preferred_timesync_source: Option<&str>,
6044    ) -> Vec<DiscoveryCandidateMatch> {
6045        let restrict_link_local = Self::endpoints_are_link_local_only(eps);
6046        let now_ms = self.clock.now_ms();
6047        let scoring_eps = self.preferred_scoring_endpoints(eps, prefer_nonlocal);
6048        let has_variable_owner = st.discovery_routes.iter().any(|(&side, route)| {
6049            exclude != Some(side)
6050                && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6051                && route.reachable_network_variables.contains(&ty)
6052        });
6053        let mut out = Vec::new();
6054        let mut unique_endpoint_fallbacks = Vec::new();
6055
6056        for (&side, route) in st.discovery_routes.iter() {
6057            if exclude == Some(side)
6058                || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
6059            {
6060                continue;
6061            }
6062            if restrict_link_local
6063                && st
6064                    .sides
6065                    .get(side)
6066                    .and_then(|s| s.as_ref())
6067                    .map(|s| !s.opts.link_local_enabled)
6068                    .unwrap_or(true)
6069            {
6070                continue;
6071            }
6072            if !self.route_allowed_locked(st, exclude, Some(ty), side) {
6073                continue;
6074            }
6075            if !target_senders.is_empty()
6076                && !Self::side_matches_target_senders_locked(st, side, target_senders, now_ms)
6077            {
6078                if scoring_eps
6079                    .iter()
6080                    .copied()
6081                    .any(|ep| route.reachable.contains(&ep))
6082                {
6083                    unique_endpoint_fallbacks.push(side);
6084                }
6085                continue;
6086            }
6087            if !target_senders.is_empty() {
6088                out.push(DiscoveryCandidateMatch {
6089                    side,
6090                    overlap: usize::MAX,
6091                });
6092                continue;
6093            }
6094            if has_variable_owner {
6095                if route.reachable_network_variables.contains(&ty) {
6096                    out.push(DiscoveryCandidateMatch {
6097                        side,
6098                        overlap: usize::MAX,
6099                    });
6100                }
6101                continue;
6102            }
6103            if preferred_timesync_source
6104                .is_some_and(|source| route.reachable_timesync_sources.iter().any(|s| s == source))
6105            {
6106                out.push(DiscoveryCandidateMatch {
6107                    side,
6108                    overlap: usize::MAX,
6109                });
6110                continue;
6111            }
6112            let overlap =
6113                Self::endpoint_overlap_count(route.reachable.iter().copied(), &scoring_eps);
6114            if overlap > 0 {
6115                out.push(DiscoveryCandidateMatch { side, overlap });
6116            }
6117        }
6118        if !target_senders.is_empty() && out.is_empty() && unique_endpoint_fallbacks.len() == 1 {
6119            out.push(DiscoveryCandidateMatch {
6120                side: unique_endpoint_fallbacks[0],
6121                overlap: usize::MAX,
6122            });
6123        }
6124        out
6125    }
6126
6127    /// Remove reflected discovery routes when another side advertises a
6128    /// shorter topology path to the destination endpoint. A multi-side Router
6129    /// must make this choice before adaptive weighting; otherwise an endpoint
6130    /// learned directly and through a remote bridge can be sent down either
6131    /// side with equal probability.
6132    #[cfg(feature = "discovery")]
6133    fn retain_shortest_discovery_candidates_locked(
6134        st: &RouterInner,
6135        matches: &mut Vec<DiscoveryCandidateMatch>,
6136        endpoints: &[DataEndpoint],
6137        target_senders: wire_format::TargetSenderSet<'_>,
6138        now_ms: u64,
6139    ) {
6140        if matches.len() <= 1 || endpoints.is_empty() {
6141            return;
6142        }
6143
6144        let route_distance = |side: RouterSideId, target_sender: Option<u64>| -> Option<usize> {
6145            let route = st.discovery_routes.get(&side)?;
6146            route
6147                .announcers
6148                .iter()
6149                .filter(|(_, state)| {
6150                    now_ms.saturating_sub(state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6151                })
6152                .filter_map(|(announcer, state)| {
6153                    let is_target = |board: &TopologyBoardNode| {
6154                        target_sender
6155                            .map(|target| Self::sender_hash(&board.sender_id) == target)
6156                            .unwrap_or_else(|| {
6157                                endpoints
6158                                    .iter()
6159                                    .any(|endpoint| board.reachable_endpoints.contains(endpoint))
6160                            })
6161                    };
6162                    // Targets are already owned by the topology snapshot. Avoid
6163                    // allocating a second set of their borrowed names.
6164                    if !state.topology_boards.iter().any(is_target) {
6165                        return if let Some(target) = target_sender {
6166                            state
6167                                .topology_boards
6168                                .iter()
6169                                .any(|board| Self::sender_hash(&board.sender_id) == target)
6170                                .then_some(usize::MAX / 2)
6171                        } else {
6172                            state
6173                                .reachable
6174                                .iter()
6175                                .any(|endpoint| endpoints.contains(endpoint))
6176                                .then_some(usize::MAX / 2)
6177                        };
6178                    }
6179
6180                    let mut visited: BTreeSet<&str> = BTreeSet::new();
6181                    let mut pending = VecDeque::from([(announcer.as_str(), 0usize)]);
6182                    while let Some((sender, distance)) = pending.pop_front() {
6183                        if !visited.insert(sender) {
6184                            continue;
6185                        }
6186                        for board in state.topology_boards.iter() {
6187                            if board.sender_id == sender {
6188                                if is_target(board) {
6189                                    return Some(distance);
6190                                }
6191                                for peer in &board.connections {
6192                                    if !visited.contains(peer.as_str()) {
6193                                        pending.push_back((peer.as_str(), distance + 1));
6194                                    }
6195                                }
6196                            } else if board.connections.iter().any(|peer| peer == sender)
6197                                && !visited.contains(board.sender_id.as_str())
6198                            {
6199                                pending.push_back((board.sender_id.as_str(), distance + 1));
6200                            }
6201                        }
6202                    }
6203                    None
6204                })
6205                .min()
6206        };
6207
6208        let route_targets = target_senders
6209            .iter()
6210            .map(Some)
6211            .chain(core::iter::once(None).filter(|_| target_senders.is_empty()));
6212        let mut selected = BTreeSet::new();
6213        for target in route_targets {
6214            let distances: Vec<(RouterSideId, usize)> = matches
6215                .iter()
6216                .filter_map(|candidate| {
6217                    route_distance(candidate.side, target)
6218                        .map(|distance| (candidate.side, distance))
6219                })
6220                .collect();
6221            let Some(best) = distances.iter().map(|(_, distance)| *distance).min() else {
6222                continue;
6223            };
6224            selected.extend(
6225                distances
6226                    .into_iter()
6227                    .filter(|(_, distance)| *distance == best)
6228                    .map(|(side, _)| side),
6229            );
6230        }
6231        if !selected.is_empty() {
6232            matches.retain(|candidate| selected.contains(&candidate.side));
6233        }
6234    }
6235
6236    /// Distinct endpoint roles are multicast destinations, not interchangeable
6237    /// paths. Pick one shortest path per role and reuse an already selected side
6238    /// when it covers that role; adaptive choice remains within redundant paths.
6239    #[cfg(feature = "discovery")]
6240    fn select_endpoint_role_sides_locked(
6241        &self,
6242        st: &mut RouterInner,
6243        exclude: Option<RouterSideId>,
6244        endpoints: &[DataEndpoint],
6245        matches: &[DiscoveryCandidateMatch],
6246        now_ms: u64,
6247    ) -> Vec<RouterSideId> {
6248        let mut selected = Vec::new();
6249        for endpoint in endpoints {
6250            let mut candidates: Vec<_> = matches
6251                .iter()
6252                .copied()
6253                .filter(|candidate| {
6254                    st.discovery_routes
6255                        .get(&candidate.side)
6256                        .is_some_and(|route| route.reachable.contains(endpoint))
6257                })
6258                .collect();
6259            Self::retain_shortest_discovery_candidates_locked(
6260                st,
6261                &mut candidates,
6262                core::slice::from_ref(endpoint),
6263                wire_format::TargetSenderSet::Hashes(&[]),
6264                now_ms,
6265            );
6266            if candidates
6267                .iter()
6268                .any(|candidate| selected.contains(&candidate.side))
6269            {
6270                continue;
6271            }
6272            let sides = self.apply_route_selection_locked(
6273                st,
6274                exclude,
6275                candidates
6276                    .into_iter()
6277                    .map(|candidate| candidate.side)
6278                    .collect(),
6279                RouteSelectionOrigin::Discovered,
6280            );
6281            for side in sides {
6282                if !selected.contains(&side) {
6283                    selected.push(side);
6284                }
6285            }
6286        }
6287        selected
6288    }
6289
6290    #[cfg(feature = "discovery")]
6291    fn select_discovered_candidate_sides_locked(
6292        &self,
6293        st: &mut RouterInner,
6294        packet: (&RouterItem, DataType),
6295        exclude: Option<RouterSideId>,
6296        target_senders: wire_format::TargetSenderSet<'_>,
6297        prefer_best_overlap: bool,
6298        matches: Vec<DiscoveryCandidateMatch>,
6299    ) -> Vec<RouterSideId> {
6300        let (data, ty) = packet;
6301        let discovered_origin = if Self::has_typed_route_overrides_locked(st, exclude, ty)
6302            || !target_senders.is_empty()
6303            || (!self.cfg.role.local_services() && is_reliable_type(ty))
6304        {
6305            RouteSelectionOrigin::Flood
6306        } else {
6307            RouteSelectionOrigin::Discovered
6308        };
6309
6310        let mut matches = matches;
6311        if matches.iter().any(|m| m.overlap == usize::MAX) {
6312            matches.retain(|m| m.overlap == usize::MAX);
6313        }
6314
6315        let selected: Vec<RouterSideId> = if prefer_best_overlap {
6316            let best_overlap = matches.iter().map(|m| m.overlap).max().unwrap_or(0);
6317            matches
6318                .into_iter()
6319                .filter(|m| m.overlap == best_overlap)
6320                .map(|m| m.side)
6321                .collect()
6322        } else {
6323            matches.into_iter().map(|m| m.side).collect()
6324        };
6325
6326        let selected = if st.forwarded_receipts.is_some() && is_reliable_type(ty) {
6327            self.filter_end_to_end_satisfied_sides_locked(
6328                st,
6329                data,
6330                selected,
6331                &self
6332                    .item_route_info(data)
6333                    .map(|info| info.0)
6334                    .unwrap_or_default(),
6335                ty,
6336            )
6337        } else {
6338            selected
6339        };
6340
6341        // Replicated managed state follows its advertised owner contract.
6342        // Ordinary application types use the same route selection regardless
6343        // of their configured names.
6344        let has_distributed_variable_owners = selected.iter().any(|side| {
6345            st.discovery_routes
6346                .get(side)
6347                .is_some_and(|route| route.reachable_network_variables.contains(&ty))
6348        });
6349        if has_distributed_variable_owners {
6350            selected
6351        } else {
6352            self.apply_route_selection_locked(st, exclude, selected, discovered_origin)
6353        }
6354    }
6355
6356    fn note_end_to_end_acked_destination_locked(
6357        st: &mut RouterInner,
6358        packet_id: u64,
6359        sender_hash: u64,
6360    ) {
6361        let cache = st
6362            .forwarded_receipts
6363            .get_or_insert_with(|| Box::new(ForwardedReceiptCache::default()));
6364        let entry_cap = crate::config::runtime_reliable_max_end_to_end_ack_cache().max(1);
6365        cache
6366            .order
6367            .retain(|id| cache.destinations.contains_key(id) && *id != packet_id);
6368        while cache.order.len() >= entry_cap {
6369            if let Some(oldest) = cache.order.pop_front() {
6370                cache.destinations.remove(&oldest);
6371            } else {
6372                break;
6373            }
6374        }
6375        cache.order.push_back(packet_id);
6376
6377        let acked = cache.destinations.entry(packet_id).or_default();
6378        let sender_cap = crate::config::runtime_reliable_max_end_to_end_pending().max(1);
6379        if acked.len() < sender_cap || acked.contains(&sender_hash) {
6380            acked.insert(sender_hash);
6381        }
6382    }
6383
6384    #[cfg(feature = "discovery")]
6385    fn filter_end_to_end_satisfied_sides_locked(
6386        &self,
6387        st: &RouterInner,
6388        data: &RouterItem,
6389        sides: Vec<RouterSideId>,
6390        eps: &[crate::DataEndpoint],
6391        ty: crate::DataType,
6392    ) -> Vec<RouterSideId> {
6393        if !is_reliable_type(ty)
6394            || Self::reliable_control_target_packet_id(data)
6395                .ok()
6396                .flatten()
6397                .is_some()
6398        {
6399            return sides;
6400        }
6401        let packet_id = match data {
6402            RouterItem::Packet(pkt) => pkt.packet_id(),
6403            RouterItem::Packed(bytes) => match wire_format::packet_id_from_wire(bytes.as_ref()) {
6404                Ok(packet_id) => packet_id,
6405                Err(TelemetryError::Unpack("reliable control frame")) => return sides,
6406                Err(_) => return sides,
6407            },
6408        };
6409        let Some(acked) = st
6410            .forwarded_receipts
6411            .as_ref()
6412            .and_then(|cache| cache.destinations.get(&packet_id))
6413        else {
6414            return sides;
6415        };
6416        let now_ms = self.clock.now_ms();
6417        let mut filtered = Vec::new();
6418        for side in sides {
6419            let Some(route) = st.discovery_routes.get(&side) else {
6420                filtered.push(side);
6421                continue;
6422            };
6423            let mut still_pending = false;
6424            let mut had_destination_board = false;
6425            for sender_state in route.announcers.values() {
6426                if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6427                    continue;
6428                }
6429                for board in sender_state.topology_boards.iter() {
6430                    if !self.is_end_to_end_destination_sender(&board.sender_id) {
6431                        continue;
6432                    }
6433                    had_destination_board = true;
6434                    let sender_hash = Self::sender_hash(&board.sender_id);
6435                    if acked.contains(&sender_hash) {
6436                        continue;
6437                    }
6438                    if eps
6439                        .iter()
6440                        .copied()
6441                        .any(|ep| board.reachable_endpoints.contains(&ep))
6442                    {
6443                        still_pending = true;
6444                        break;
6445                    }
6446                    // Keep forwarding while any discovered destination sender for this packet
6447                    // remains unacked, even if topology/schema metadata changed for new packets.
6448                    still_pending = true;
6449                    break;
6450                }
6451                if still_pending {
6452                    break;
6453                }
6454            }
6455            if still_pending || !had_destination_board {
6456                filtered.push(side);
6457            }
6458        }
6459        filtered
6460    }
6461
6462    #[cfg(feature = "discovery")]
6463    fn reconcile_forwarded_receipts_locked(&self, st: &mut RouterInner) {
6464        if st.forwarded_receipts.is_none() {
6465            return;
6466        }
6467        let mut active_senders = BTreeSet::new();
6468        for route in st.discovery_routes.values() {
6469            for sender_state in route.announcers.values() {
6470                for board in sender_state.topology_boards.iter() {
6471                    if self.is_end_to_end_destination_sender(&board.sender_id) {
6472                        active_senders.insert(Self::sender_hash(&board.sender_id));
6473                    }
6474                }
6475            }
6476        }
6477        st.forwarded_receipts
6478            .as_mut()
6479            .expect("checked cache")
6480            .destinations
6481            .retain(|_, acked| {
6482                acked.retain(|sender_hash| active_senders.contains(sender_hash));
6483                !acked.is_empty()
6484            });
6485    }
6486
6487    #[cfg(feature = "discovery")]
6488    fn register_end_to_end_reliable_tx(
6489        &self,
6490        data: &RouterItem,
6491        destinations: Option<BTreeMap<u64, RouterSideId>>,
6492    ) -> TelemetryResult<bool> {
6493        let packet_id = Self::get_hash(data);
6494        #[cfg(feature = "std")]
6495        let completion_epoch = self.retry_birth();
6496        let now_ms = self.clock.now_ms();
6497        let ty = match data {
6498            RouterItem::Packet(pkt) => pkt.data_type(),
6499            RouterItem::Packed(bytes) => wire_format::peek_routing_envelope(bytes.as_ref())?.ty,
6500        };
6501        let mut st = self.state.lock();
6502        #[cfg(feature = "discovery")]
6503        let mut pending_destinations = match destinations {
6504            Some(destinations) => destinations,
6505            None => self.expected_end_to_end_destinations_locked(&st, data, None, false, false)?,
6506        };
6507        #[cfg(not(feature = "discovery"))]
6508        let mut pending_destinations = destinations.unwrap_or_default();
6509        // Callers pass the destination map used to construct this frozen
6510        // contract. Re-filtering the same map allocates owner scratch and
6511        // instantiates a second tree-extraction path on small MCUs.
6512        let frozen = match data {
6513            RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
6514            RouterItem::Packed(bytes) => {
6515                wire_format::peek_routing_envelope(bytes)?
6516                    .target_senders()
6517                    .len()
6518                    != 0
6519            }
6520        };
6521        self.filter_trackable_end_to_end_destinations_locked(
6522            &st,
6523            ty,
6524            &mut pending_destinations,
6525            frozen,
6526        );
6527        let tracked_destinations = !pending_destinations.is_empty();
6528        if let Some(existing) = st.end_to_end_reliable_tx.get(&packet_id) {
6529            return Ok(!existing.waiting_for_predecessor);
6530        }
6531        // Never evict an unacknowledged predecessor to admit a newer message.
6532        if st.end_to_end_reliable_tx.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
6533            return Err(TelemetryError::Io("reliable window full"));
6534        }
6535        #[cfg(feature = "std")]
6536        if st
6537            .shared_queue_bytes_used()
6538            .saturating_add(PENDING_TYPE_METADATA_BYTES)
6539            .saturating_add(usize::from(completion_epoch != 0) * size_of::<u64>())
6540            > st.memory.max_queue_budget
6541        {
6542            return Err(TelemetryError::Io("memory pressure"));
6543        }
6544        let ordered_type =
6545            matches!(reliable_mode(ty), crate::ReliableMode::Ordered).then_some(ty.as_u32());
6546        let waiting_for_predecessor = ordered_type.is_some()
6547            && st
6548                .end_to_end_reliable_tx
6549                .values()
6550                .any(|sent| sent.ordered_type == ordered_type);
6551        #[cfg(feature = "std")]
6552        if reliable_trace_enabled() {
6553            eprintln!(
6554                "[sedsnet-reliable] register time={now_ms} type={} id={packet_id:#x} waiting={waiting_for_predecessor} targets={pending_destinations:?}",
6555                ty.as_u32()
6556            );
6557        }
6558        #[cfg(feature = "compression")]
6559        let retained = match data {
6560            RouterItem::Packed(bytes) => RouterItem::Packed(
6561                wire_format::payload_encoding_for_retention(
6562                    bytes.as_ref(),
6563                    st.memory.max_queue_budget,
6564                )?
6565                .map(crate::shared_bytes::convert)
6566                .unwrap_or_else(|| bytes.clone()),
6567            ),
6568            packet => packet.clone(),
6569        };
6570        #[cfg(not(feature = "compression"))]
6571        let retained = data.clone();
6572        #[cfg(feature = "compact-packet-store")]
6573        let mut retained = retained;
6574        #[cfg(feature = "compact-packet-store")]
6575        retained.park_for_queue()?;
6576        let sent = EndToEndReliableSent {
6577            #[cfg(feature = "std")]
6578            completion_epoch,
6579            #[cfg(feature = "std")]
6580            pending_type: ty.as_u32(),
6581            ordered_type,
6582            waiting_for_predecessor,
6583            data: retained,
6584            pending_destinations,
6585            tracked_destinations,
6586            last_send_ms: now_ms,
6587            retries: 0,
6588            queued: false,
6589        }
6590        .try_box()?;
6591        Self::remember_end_to_end_reliable_tx_locked(&mut st, packet_id);
6592        st.end_to_end_reliable_tx.insert(packet_id, sent);
6593        Ok(!waiting_for_predecessor)
6594    }
6595
6596    // Select at most one in-flight packet per ordered type. Waiting messages
6597    // share the existing bounded pending window, not an unbounded extra queue.
6598    fn release_ordered_successors(&self) -> TelemetryResult<()> {
6599        let ready = {
6600            let mut st = self.state.lock();
6601            let RouterInner {
6602                end_to_end_reliable_tx_order: order,
6603                end_to_end_reliable_tx: pending,
6604                ..
6605            } = &mut *st;
6606            let mut ready = Vec::new();
6607            for (index, id) in order.iter().copied().enumerate() {
6608                let Some(sent) = pending.get(&id) else {
6609                    continue;
6610                };
6611                if !sent.waiting_for_predecessor {
6612                    continue;
6613                }
6614                let Some(ty) = sent.ordered_type else {
6615                    continue;
6616                };
6617                // The retained window is bounded. Inspect earlier entries
6618                // instead of rebuilding an allocated active-type set on every
6619                // idle service pass. Only an actual release needs ready storage.
6620                if order.iter().take(index).any(|earlier| {
6621                    pending
6622                        .get(earlier)
6623                        .is_some_and(|sent| sent.ordered_type == Some(ty))
6624                }) {
6625                    continue;
6626                }
6627                let sent = pending.get_mut(&id).expect("retained ordered entry");
6628                sent.waiting_for_predecessor = false;
6629                sent.last_send_ms = self.clock.now_ms();
6630                ready.push(id);
6631            }
6632            ready
6633        };
6634        for id in ready {
6635            self.queue_end_to_end_reliable_retransmit(id)?;
6636        }
6637        Ok(())
6638    }
6639
6640    #[cfg(feature = "discovery")]
6641    fn reconcile_end_to_end_reliable_destinations_locked(
6642        &self,
6643        st: &mut RouterInner,
6644    ) -> TelemetryResult<()> {
6645        self.reconcile_forwarded_receipts_locked(st);
6646        if st.end_to_end_reliable_tx.is_empty() {
6647            return Ok(());
6648        }
6649        let active_destinations = self.active_end_to_end_destinations_locked(st);
6650        let packet_ids: Vec<u64> = st.end_to_end_reliable_tx.keys().copied().collect();
6651
6652        for packet_id in packet_ids {
6653            let Some(data) = st
6654                .end_to_end_reliable_tx
6655                .get(&packet_id)
6656                .map(|sent| sent.data.clone())
6657            else {
6658                continue;
6659            };
6660            let excluded = st
6661                .reliable_return_routes
6662                .get(&packet_id)
6663                .filter(|route| route.upstream_contract.is_some())
6664                .map(|route| route.side);
6665            let expected =
6666                self.expected_end_to_end_destinations_locked(st, &data, excluded, false, false)?;
6667            let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) else {
6668                continue;
6669            };
6670            if !sent.tracked_destinations {
6671                continue;
6672            }
6673            // A topology withdrawal is not a destination receipt, for either
6674            // reliable mode. Keep the frozen cohort until a real ACK or the
6675            // bounded delivery timeout; only update a known owner's next hop.
6676            for (sender, side) in &mut sent.pending_destinations {
6677                if let Some(next_side) = expected.get(sender).or_else(|| {
6678                    active_destinations
6679                        .get(sender)
6680                        .filter(|side| Some(**side) != excluded)
6681                }) {
6682                    *side = *next_side;
6683                }
6684            }
6685        }
6686
6687        Ok(())
6688    }
6689
6690    #[cfg(feature = "discovery")]
6691    fn active_end_to_end_destinations_locked(
6692        &self,
6693        st: &RouterInner,
6694    ) -> BTreeMap<u64, RouterSideId> {
6695        let now_ms = self.clock.now_ms();
6696        let mut out = BTreeMap::new();
6697        for (&side, route) in st.discovery_routes.iter() {
6698            if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6699                continue;
6700            }
6701            for (sender_name, sender_state) in &route.announcers {
6702                if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
6703                    continue;
6704                }
6705                if self.is_end_to_end_destination_sender(sender_name) {
6706                    out.insert(Self::sender_hash(sender_name), side);
6707                }
6708                for board in sender_state.topology_boards.iter() {
6709                    if !self.is_end_to_end_destination_sender(&board.sender_id) {
6710                        continue;
6711                    }
6712                    out.insert(Self::sender_hash(&board.sender_id), side);
6713                    if out.len() >= runtime_reliable_max_end_to_end_pending().max(1) {
6714                        return out;
6715                    }
6716                }
6717            }
6718        }
6719        out
6720    }
6721
6722    #[cfg(feature = "discovery")]
6723    fn side_supports_end_to_end_tracking_locked(st: &RouterInner, side: RouterSideId) -> bool {
6724        matches!(
6725            st.sides
6726                .get(side)
6727                .and_then(Option::as_ref)
6728                .map(|side| &side.tx_handler),
6729            Some(RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_))
6730        )
6731    }
6732
6733    #[cfg(feature = "discovery")]
6734    fn filter_trackable_end_to_end_destinations_locked(
6735        &self,
6736        st: &RouterInner,
6737        ty: DataType,
6738        pending: &mut BTreeMap<u64, RouterSideId>,
6739        frozen: bool,
6740    ) {
6741        if frozen {
6742            return;
6743        }
6744        let now_ms = self.clock.now_ms();
6745        pending.retain(|_, side| {
6746            Self::side_supports_end_to_end_tracking_locked(st, *side)
6747                && (st.is_managed_variable(ty.as_u32())
6748                    || is_reliable_type(ty)
6749                    || !self.side_has_multiple_announcers_locked(st, *side, now_ms))
6750        });
6751    }
6752
6753    #[cfg(feature = "discovery")]
6754    fn side_has_multiple_announcers_locked(
6755        &self,
6756        st: &RouterInner,
6757        side: RouterSideId,
6758        now_ms: u64,
6759    ) -> bool {
6760        st.discovery_routes
6761            .get(&side)
6762            .map(|route| {
6763                route
6764                    .announcers
6765                    .values()
6766                    .filter(|sender| {
6767                        now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
6768                    })
6769                    .take(2)
6770                    .count()
6771                    > 1
6772            })
6773            .unwrap_or(false)
6774    }
6775
6776    #[cfg(not(feature = "discovery"))]
6777    fn side_has_multiple_announcers_locked(
6778        &self,
6779        _st: &RouterInner,
6780        _side: RouterSideId,
6781        _now_ms: u64,
6782    ) -> bool {
6783        false
6784    }
6785
6786    // Preserve the existing private route-fixture helper; production guarded
6787    // receives use the token-bound atomic handoff below.
6788    #[cfg(test)]
6789    fn queue_end_to_end_reliable_ack(
6790        &self,
6791        pkt: &Packet,
6792        called_from_queue: bool,
6793    ) -> TelemetryResult<()> {
6794        self.queue_end_to_end_reliable_ack_for_packet_id(
6795            pkt.packet_id(),
6796            pkt.sender(),
6797            called_from_queue,
6798        )
6799    }
6800
6801    fn queue_end_to_end_reliable_ack_for_packet_id(
6802        &self,
6803        packet_id: u64,
6804        original_sender: &str,
6805        called_from_queue: bool,
6806    ) -> TelemetryResult<()> {
6807        if !self.cfg.reliable_enabled() {
6808            return Ok(());
6809        }
6810        let original_sender_hash = {
6811            let mut st = self.state.lock();
6812            // This helper is reached after local delivery succeeds, before
6813            // attempting its final receipt. Keep that fact in the existing
6814            // bounded reliable return-route cache: best-effort telemetry can
6815            // churn recent_rx thousands of times while this receipt retries.
6816            if let Some(route) = st.reliable_return_routes.get_mut(&packet_id) {
6817                route.delivery_completed = true;
6818            }
6819            Self::sender_hash(&Self::canonical_sender_locked(&st, original_sender))
6820        };
6821        self.queue_end_to_end_reliable_ack_for_sender_hash(
6822            packet_id,
6823            original_sender_hash,
6824            called_from_queue,
6825        )
6826    }
6827
6828    fn build_end_to_end_ack_data(
6829        &self,
6830        packet_id: u64,
6831        original_sender_hash: u64,
6832    ) -> TelemetryResult<RouterItem> {
6833        let ack_sender = self.encode_end_to_end_ack_sender();
6834        let ack = Packet::new_local(
6835            DataType::ReliableAck,
6836            message_meta(DataType::ReliableAck).endpoints_owned(),
6837            Arc::from(ack_sender),
6838            self.packet_timestamp_ms(),
6839            crate::config::StandardSmallPayload::new(&packet_id.to_le_bytes()),
6840        )?;
6841        let local_sender = self.sender_arc();
6842        // Keep the acknowledging owner first for compatibility. The second
6843        // identity is the original publisher: it lets a busy relay recover the
6844        // ACK route from discovery after its bounded packet-id cache expires.
6845        self.attach_wire_contract_to_item(
6846            RouterItem::Packet(ack),
6847            &[
6848                Self::sender_hash(local_sender.as_ref()),
6849                original_sender_hash,
6850            ],
6851        )
6852    }
6853
6854    fn queue_admitted_end_to_end_ack(
6855        &self,
6856        packet_id: u64,
6857        original_sender: &str,
6858        called_from_queue: bool,
6859        admission: Option<&ReliableLocalAdmissionGuard<'_>>,
6860    ) -> TelemetryResult<()> {
6861        let Some(guard) = admission else {
6862            return self.queue_end_to_end_reliable_ack_for_packet_id(
6863                packet_id,
6864                original_sender,
6865                called_from_queue,
6866            );
6867        };
6868        if !self.cfg.reliable_enabled() {
6869            return Ok(());
6870        }
6871        if guard.packet_id != packet_id || !core::ptr::eq(guard.router, self) {
6872            return Err(TelemetryError::BadArg);
6873        }
6874        let publisher_hash = {
6875            let state = self.state.lock();
6876            guard.check_locked(&state)?;
6877            let dst = state
6878                .reliable_return_routes
6879                .get(&packet_id)
6880                .expect("checked lease")
6881                .side;
6882            // Receive-only observers accept local state without originating a
6883            // receipt. A real mid-RX revocation still fails the lease check.
6884            if !Self::side_ref(&state, dst)?.opts.egress_enabled {
6885                return Ok(());
6886            }
6887            Self::sender_hash(&Self::canonical_sender_locked(&state, original_sender))
6888        };
6889        let data = self.build_end_to_end_ack_data(packet_id, publisher_hash)?;
6890        let priority = self.item_service_priority(&data)?;
6891        // Only this small newly built ACK is hashed for eager selection, never
6892        // the original data or unrelated queued payloads.
6893        let ack_id = Self::get_hash(&data);
6894        let mut state = self.state.lock();
6895        guard.check_locked(&state)?;
6896        let dst = state
6897            .reliable_return_routes
6898            .get(&packet_id)
6899            .expect("checked lease")
6900            .side;
6901        if !Self::side_ref(&state, dst)?.opts.egress_enabled {
6902            return Ok(());
6903        }
6904        state.push_transmit(TxQueued {
6905            disposable: TX_DISPOSABLE,
6906            item: RouterTxItem::ToSide {
6907                src: None,
6908                dst,
6909                data,
6910            },
6911            ignore_local: true,
6912            priority,
6913            delivery_state: QueuedDelivery::Untracked,
6914            accepted_type: DataType::ReliableAck.as_u32(),
6915            deferred_since_ms: 0,
6916            retry_after_ms: 0,
6917        })?;
6918        // Admission and completion have one state-locked linearization point.
6919        state
6920            .reliable_return_routes
6921            .get_mut(&packet_id)
6922            .expect("checked lease")
6923            .delivery_completed = true;
6924        // Preserve the immediate-RX API's eager receipt without dispatching an
6925        // unrelated application command. Admission/completion above stays
6926        // atomic; claiming this one ACK uses normal queued-TX tracking/retry.
6927        #[cfg(feature = "std")]
6928        let can_claim = self
6929            .active_queued_tx
6930            .load(core::sync::atomic::Ordering::Acquire)
6931            != usize::MAX;
6932        #[cfg(not(feature = "std"))]
6933        let can_claim = true;
6934        let work = if !called_from_queue && can_claim {
6935            let index = state.transmit_queue.iter().position(|queued| {
6936                if queued.accepted_type != DataType::ReliableAck.as_u32() {
6937                    return false;
6938                }
6939                let RouterTxItem::ToSide {
6940                    dst: side, data, ..
6941                } = &queued.item
6942                else {
6943                    return false;
6944                };
6945                if *side != dst {
6946                    return false;
6947                }
6948                match data {
6949                    RouterItem::Packet(packet) => packet.packet_id() == ack_id,
6950                    RouterItem::Packed(bytes) => crate::shared_bytes::with_slice(bytes, |view| {
6951                        wire_format::packet_id_from_wire(view).is_ok_and(|id| id == ack_id)
6952                    })
6953                    .unwrap_or(false),
6954                }
6955            });
6956            index
6957                .and_then(|index| state.transmit_queue.remove_pos(index))
6958                .map(|queued| {
6959                    #[cfg(feature = "latest-value-egress")]
6960                    let service = EgressService {
6961                        token: state.latest_service_seq,
6962                        side: state.latest_dispatch_tag(&queued),
6963                        kind: 0,
6964                    };
6965                    TxWork {
6966                        queued,
6967                        claim: None,
6968                        owed_turn: false,
6969                        #[cfg(feature = "latest-value-egress")]
6970                        service,
6971                    }
6972                })
6973        } else {
6974            None
6975        };
6976        #[cfg(feature = "std")]
6977        let work = self.track_queued_tx(&mut state, work);
6978        drop(state);
6979        if let Some(work) = work {
6980            #[cfg(feature = "std")]
6981            let (work, _activity) = work;
6982            self.dispatch_queued_item(work)?;
6983        }
6984        Ok(())
6985    }
6986
6987    fn queue_end_to_end_reliable_ack_for_sender_hash(
6988        &self,
6989        packet_id: u64,
6990        original_sender_hash: u64,
6991        called_from_queue: bool,
6992    ) -> TelemetryResult<()> {
6993        if !self.cfg.reliable_enabled() {
6994            return Ok(());
6995        }
6996        let ack = self.build_end_to_end_ack_data(packet_id, original_sender_hash)?;
6997        // Freeze a known return side in the queued receipt itself. Its bounded
6998        // packet-id cache can churn before dispatch; an echoed data frame must
6999        // not redirect an ACK which already has a valid destination.
7000        let return_side = {
7001            let st = self.state.lock();
7002            st.reliable_return_routes
7003                .get(&packet_id)
7004                .map(|route| route.side)
7005        };
7006        let item = match return_side {
7007            Some(dst) => RouterTxItem::ToSide {
7008                src: None,
7009                dst,
7010                data: ack,
7011            },
7012            None => RouterTxItem::Broadcast(ack),
7013        };
7014        self.emit_internal_tx(item, true, called_from_queue)
7015    }
7016
7017    fn emit_internal_tx(
7018        &self,
7019        item: RouterTxItem,
7020        ignore_local: bool,
7021        called_from_queue: bool,
7022    ) -> TelemetryResult<()> {
7023        if called_from_queue {
7024            self.tx_queue_item_with_flags(item, ignore_local)
7025        } else {
7026            self.tx_item_impl(item, ignore_local, false)
7027        }
7028    }
7029
7030    fn emit_internal_tx_with_priority(
7031        &self,
7032        item: RouterTxItem,
7033        ignore_local: bool,
7034        priority: u8,
7035        called_from_queue: bool,
7036    ) -> TelemetryResult<()> {
7037        if called_from_queue {
7038            self.tx_queue_item_with_priority(item, ignore_local, priority)
7039        } else {
7040            self.tx_item_impl(item, ignore_local, false)
7041        }
7042    }
7043
7044    fn queue_end_to_end_reliable_retransmit(&self, packet_id: u64) -> TelemetryResult<()> {
7045        let now = self.clock.now_ms();
7046        let result = self.tx_queue_item_with_priority(
7047            RouterTxItem::EndToEndReplay { packet_id },
7048            true,
7049            Self::router_item_priority_bumped(DataType::ReliableAck),
7050        );
7051        // A reservation is one retry attempt. If control admission later evicts
7052        // its marker, wait a full interval instead of exhausting the budget in
7053        // successive maintenance polls before any transport can run.
7054        if result.is_ok()
7055            && let Some(sent) = self.state.lock().end_to_end_reliable_tx.get_mut(&packet_id)
7056        {
7057            sent.last_send_ms = sent.last_send_ms.max(now);
7058        }
7059        result
7060    }
7061
7062    fn end_to_end_retransmit_sides(
7063        &self,
7064        packet_id: u64,
7065    ) -> Option<(RouterItem, Vec<RouterSideId>)> {
7066        let mut st = self.state.lock();
7067        let (data, tracked_destinations, mut sides) = {
7068            let sent = st.end_to_end_reliable_tx.get_mut(&packet_id)?;
7069            #[cfg(feature = "std")]
7070            if reliable_trace_enabled() {
7071                eprintln!(
7072                    "[sedsnet-reliable] replay time={} id={packet_id:#x} retries={} targets={:?}",
7073                    self.clock.now_ms(),
7074                    sent.retries,
7075                    sent.pending_destinations
7076                );
7077            }
7078            sent.last_send_ms = self.clock.now_ms();
7079            let data = sent.data.clone();
7080            let tracked_destinations = sent.tracked_destinations;
7081            let sides: Vec<RouterSideId> = sent.pending_destinations.values().copied().collect();
7082            (data, tracked_destinations, sides)
7083        };
7084        if tracked_destinations && sides.is_empty() {
7085            st.end_to_end_reliable_tx.remove(&packet_id);
7086            return None;
7087        }
7088        crate::small_sort::sort(&mut sides);
7089        sides.dedup();
7090        Some((data, sides))
7091    }
7092
7093    fn item_type_priority(data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
7094        let (ty, control) = match data {
7095            RouterItem::Packet(pkt) => (
7096                pkt.data_type(),
7097                pkt.payload().len() <= 16
7098                    && pkt.endpoints().len() <= 3
7099                    && crate::small_frozen_control(
7100                        pkt.data_type(),
7101                        pkt.wire_target_senders().len(),
7102                        pkt.wire_shape(),
7103                    ),
7104            ),
7105            RouterItem::Packed(bytes) => {
7106                let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7107                let env = &frame.envelope;
7108                // Hop ACKs carry the original data type. Their validated
7109                // control header, not that type's bulk priority, releases credit.
7110                if bytes.len() <= 128 && frame.ack_only() {
7111                    return Ok((
7112                        env.ty,
7113                        crate::transport_priority(crate::DataType::ReliableAck),
7114                    ));
7115                }
7116                let control = bytes.len() <= 128
7117                    && bytes.first().is_some_and(|flags| flags & 0x11 == 0)
7118                    && env.endpoints().take(4).count() <= 3
7119                    && crate::small_frozen_control(
7120                        env.ty,
7121                        env.target_senders().len(),
7122                        env.wire_shape,
7123                    );
7124                (env.ty, control)
7125            }
7126        };
7127        let priority = crate::scheduler_priority(ty);
7128        Ok((ty, if control { priority.max(254) } else { priority }))
7129    }
7130
7131    fn router_item_priority(data: &RouterItem) -> TelemetryResult<u8> {
7132        Self::item_type_priority(data).map(|(_, priority)| priority)
7133    }
7134
7135    fn item_service_type_priority(&self, data: &RouterItem) -> TelemetryResult<(DataType, u8)> {
7136        let (ty, priority) = Self::item_type_priority(data)?;
7137        let priority = if self.is_managed_variable_type(ty) {
7138            crate::transport_priority(DataType::ManagedVariableValue)
7139        } else {
7140            priority
7141        };
7142        Ok((ty, priority))
7143    }
7144
7145    fn item_service_priority(&self, data: &RouterItem) -> TelemetryResult<u8> {
7146        self.item_service_type_priority(data)
7147            .map(|(_, priority)| priority)
7148    }
7149
7150    #[inline]
7151    fn router_item_priority_bumped(ty: DataType) -> u8 {
7152        crate::scheduler_priority(ty).saturating_add(16)
7153    }
7154
7155    #[inline]
7156    fn is_transient_tx_error(err: &TelemetryError) -> bool {
7157        matches!(
7158            err,
7159            TelemetryError::Io("side tx busy" | "memory pressure" | "reliable window full")
7160        )
7161    }
7162
7163    #[cfg(feature = "timesync")]
7164    fn timesync_has_usable_time_locked(st: &TimeSyncRuntime, now_mono_ns: u64) -> bool {
7165        st.disciplined_clock.read_unix_ms(now_mono_ns).is_some()
7166            || st
7167                .clock
7168                .current_time(now_mono_ns)
7169                .and_then(|reading| reading.unix_time_ms)
7170                .is_some()
7171    }
7172
7173    #[cfg(feature = "timesync")]
7174    fn reconcile_pending_timesync_request_locked(
7175        st: &mut TimeSyncRuntime,
7176        leader: &Option<TimeSyncLeader>,
7177        now_ms: u64,
7178    ) {
7179        let active_remote = match leader {
7180            Some(TimeSyncLeader::Remote(remote)) => Some(remote.sender.as_str()),
7181            _ => None,
7182        };
7183        let should_clear = st
7184            .pending_request
7185            .as_ref()
7186            .is_some_and(|pending| Some(pending.source.as_str()) != active_remote);
7187        if should_clear {
7188            st.pending_request = None;
7189            st.next_request_mono_ms = now_ms;
7190        }
7191    }
7192
7193    ///Helper function for relay_send
7194    #[inline]
7195    fn enqueue_to_sides(
7196        &self,
7197        data: RouterItem,
7198        exclude: Option<RouterSideId>,
7199        ignore_local: bool,
7200    ) -> TelemetryResult<()> {
7201        let Some(data) = self.clear_intermediate_hop_targets(data, true)? else {
7202            return Ok(());
7203        };
7204        let plan = self.remote_side_plan(&data, exclude)?;
7205        let mut st = self.state.lock();
7206        let ty = Self::item_data_type(&data)?;
7207        let priority = if st.is_managed_variable(ty.as_u32()) {
7208            crate::transport_priority(DataType::ManagedVariableValue)
7209        } else {
7210            Self::router_item_priority(&data)?
7211        };
7212
7213        let RemoteSidePlan::Target(sides) = plan;
7214        for idx in sides {
7215            st.push_transmit(TxQueued {
7216                disposable: TX_FRESH,
7217                item: RouterTxItem::ToSide {
7218                    src: exclude,
7219                    dst: idx,
7220                    data: data.clone(),
7221                },
7222                ignore_local,
7223                priority,
7224                delivery_state: QueuedDelivery::Untracked,
7225                accepted_type: 0,
7226                deferred_since_ms: 0,
7227                retry_after_ms: 0,
7228            })?;
7229        }
7230
7231        Ok(())
7232    }
7233
7234    fn relay_send(
7235        &self,
7236        data: RouterItem,
7237        src: Option<RouterSideId>,
7238        called_from_queue: bool,
7239    ) -> TelemetryResult<()> {
7240        if called_from_queue {
7241            return self.enqueue_to_sides(data, src, true);
7242        }
7243
7244        let Some(data) = self.clear_intermediate_hop_targets(data, false)? else {
7245            return Ok(());
7246        };
7247        let RemoteSidePlan::Target(sides) = self.remote_side_plan(&data, src)?;
7248        for side in sides {
7249            #[cfg(feature = "latest-value-egress")]
7250            if self.queue_latest_egress(side, src, &data)? {
7251                continue;
7252            }
7253            self.tx_item_impl(
7254                RouterTxItem::ToSide {
7255                    src,
7256                    dst: side,
7257                    data: data.clone(),
7258                },
7259                true,
7260                false,
7261            )?;
7262        }
7263
7264        Ok(())
7265    }
7266
7267    fn item_route_info(&self, data: &RouterItem) -> TelemetryResult<(Vec<DataEndpoint>, DataType)> {
7268        match data {
7269            RouterItem::Packet(pkt) => {
7270                pkt.validate()?;
7271                let mut eps = pkt.endpoints().to_vec();
7272                crate::small_sort::sort(&mut eps);
7273                eps.dedup();
7274                Ok((eps, pkt.data_type()))
7275            }
7276            RouterItem::Packed(bytes) => {
7277                let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7278                let mut eps: Vec<DataEndpoint> = env.endpoints().collect();
7279                crate::small_sort::sort(&mut eps);
7280                eps.dedup();
7281                Ok((eps, env.ty))
7282            }
7283        }
7284    }
7285
7286    fn unpack_packet_for_role(&self, bytes: &[u8]) -> TelemetryResult<Packet> {
7287        let managed = crate::memory_admission::supports_work_kinds()
7288            && wire_format::scheduling_type(bytes).is_some_and(|ty| {
7289                !is_internal_control_type(ty) && self.is_managed_variable_type(ty)
7290            });
7291        wire_format::unpack_packet_with_role(bytes, managed)
7292    }
7293
7294    fn item_data_type(data: &RouterItem) -> TelemetryResult<DataType> {
7295        match data {
7296            RouterItem::Packet(pkt) => Ok(pkt.data_type()),
7297            RouterItem::Packed(bytes) => Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.ty),
7298        }
7299    }
7300
7301    fn e2e_crypto_supported(&self) -> bool {
7302        #[cfg(feature = "cryptography")]
7303        {
7304            self.cfg.e2e_encryption() != RouterE2eEncryptionMode::Disabled
7305                && crate::crypto::registered_crypto_available()
7306        }
7307        #[cfg(not(feature = "cryptography"))]
7308        {
7309            false
7310        }
7311    }
7312
7313    fn should_require_e2e_for_type(&self, ty: DataType) -> bool {
7314        if is_internal_control_type(ty) {
7315            return false;
7316        }
7317        match self.cfg.e2e_encryption() {
7318            RouterE2eEncryptionMode::Disabled => {
7319                message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
7320            }
7321            RouterE2eEncryptionMode::RequiredOnly => {
7322                message_e2e_encryption_policy(ty) == E2eEncryptionPolicy::RequireOn
7323            }
7324            RouterE2eEncryptionMode::Preferred => matches!(
7325                message_e2e_encryption_policy(ty),
7326                E2eEncryptionPolicy::PreferOn | E2eEncryptionPolicy::RequireOn
7327            ),
7328            RouterE2eEncryptionMode::ForceAll => true,
7329        }
7330    }
7331
7332    fn ensure_e2e_policy_supported_for_type(&self, ty: DataType) -> TelemetryResult<()> {
7333        if self.should_require_e2e_for_type(ty) && !self.e2e_crypto_supported() {
7334            return Err(TelemetryError::BadArg);
7335        }
7336        Ok(())
7337    }
7338
7339    #[cfg(feature = "cryptography")]
7340    fn e2e_seal_config_for_type(&self, ty: DataType) -> Option<wire_format::E2eSealConfig> {
7341        if self.should_require_e2e_for_type(ty) && self.e2e_crypto_supported() {
7342            Some(wire_format::E2eSealConfig {
7343                key_id: self.cfg.e2e_key_id(),
7344            })
7345        } else {
7346            None
7347        }
7348    }
7349
7350    #[inline]
7351    fn pack_packet_for_router(
7352        &self,
7353        pkt: &Packet,
7354        reliable: Option<wire_format::ReliableHeader>,
7355    ) -> TelemetryResult<crate::SharedBytes> {
7356        #[cfg(feature = "compression")]
7357        {
7358            #[cfg(feature = "cryptography")]
7359            let e2e = self.e2e_seal_config_for_type(pkt.data_type());
7360            #[cfg(not(feature = "cryptography"))]
7361            let e2e = None;
7362            let reliable = reliable.or_else(|| {
7363                is_reliable_type(pkt.data_type()).then_some(wire_format::ReliableHeader {
7364                    flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
7365                    seq: 0,
7366                    ack: 0,
7367                })
7368            });
7369            wire_format::pack_routed_packet(
7370                pkt,
7371                reliable,
7372                pkt.wire_shape(),
7373                pkt.wire_target_senders(),
7374                e2e,
7375            )
7376            .map(crate::shared_bytes::convert)
7377        }
7378        #[cfg(not(feature = "compression"))]
7379        {
7380            #[cfg(feature = "cryptography")]
7381            if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
7382                return (wire_format::pack_packet_with_wire_contract_e2e(
7383                    pkt,
7384                    reliable,
7385                    pkt.wire_shape(),
7386                    pkt.wire_target_senders(),
7387                    e2e,
7388                ))
7389                .map(crate::shared_bytes::convert);
7390            }
7391            Ok(match reliable {
7392                Some(hdr) => {
7393                    crate::shared_bytes::convert(wire_format::pack_packet_with_reliable(pkt, hdr))
7394                }
7395                None => crate::shared_bytes::convert(wire_format::pack_packet(pkt)),
7396            })
7397        }
7398    }
7399
7400    #[inline]
7401    fn pack_packet_for_contract(
7402        &self,
7403        pkt: &Packet,
7404        reliable: Option<wire_format::ReliableHeader>,
7405        shape: Option<MessageElement>,
7406        target_senders: &[u64],
7407    ) -> TelemetryResult<crate::SharedBytes> {
7408        #[cfg(feature = "compression")]
7409        {
7410            #[cfg(feature = "cryptography")]
7411            let e2e = self.e2e_seal_config_for_type(pkt.data_type());
7412            #[cfg(not(feature = "cryptography"))]
7413            let e2e = None;
7414            wire_format::pack_routed_packet(pkt, reliable, shape, target_senders, e2e)
7415                .map(crate::shared_bytes::convert)
7416        }
7417        #[cfg(not(feature = "compression"))]
7418        {
7419            #[cfg(feature = "cryptography")]
7420            if let Some(e2e) = self.e2e_seal_config_for_type(pkt.data_type()) {
7421                return (wire_format::pack_packet_with_wire_contract_e2e(
7422                    pkt,
7423                    reliable,
7424                    shape,
7425                    target_senders,
7426                    e2e,
7427                ))
7428                .map(crate::shared_bytes::convert);
7429            }
7430            wire_format::pack_packet_with_wire_contract(pkt, reliable, shape, target_senders)
7431                .map(crate::shared_bytes::convert)
7432        }
7433    }
7434
7435    #[cfg(feature = "cryptography")]
7436    #[inline]
7437    fn prepare_packed_for_remote(
7438        &self,
7439        bytes: crate::SharedBytes,
7440        reliable_override: Option<Option<wire_format::ReliableHeader>>,
7441    ) -> TelemetryResult<crate::SharedBytes> {
7442        // Both peeks verify the CRC and parse the same canonical routing and
7443        // reliable header. Only sealing needs owned endpoint/target arrays.
7444        let routing = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7445        if routing.ack_only() || self.e2e_seal_config_for_type(routing.envelope.ty).is_none() {
7446            return Ok(bytes);
7447        }
7448        let frame = wire_format::peek_frame_info(bytes.as_ref())?;
7449        let reliable = reliable_override.unwrap_or(frame.reliable);
7450        let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
7451        self.pack_packet_for_contract(
7452            &pkt,
7453            reliable,
7454            frame.envelope.wire_shape,
7455            &frame.envelope.target_senders,
7456        )
7457    }
7458
7459    // Best-effort telemetry does not become reliable merely because discovery
7460    // currently sees one peer. Preserve explicit frozen receipt contracts and
7461    // managed-variable delivery even when the current schema is best effort.
7462    fn item_requires_end_to_end_tracking(
7463        &self,
7464        data: &RouterItem,
7465        ty: DataType,
7466    ) -> TelemetryResult<bool> {
7467        if is_reliable_type(ty) || self.is_managed_variable_type(ty) {
7468            return Ok(true);
7469        }
7470        match data {
7471            RouterItem::Packet(packet) => Ok(!packet.wire_target_senders().is_empty()),
7472            RouterItem::Packed(bytes) => {
7473                let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
7474                Ok(frame.reliable.is_some() || frame.envelope.target_senders().len() != 0)
7475            }
7476        }
7477    }
7478
7479    #[cfg(feature = "discovery")]
7480    fn item_target_senders<'a>(
7481        &self,
7482        data: &'a RouterItem,
7483    ) -> TelemetryResult<wire_format::TargetSenderSet<'a>> {
7484        match data {
7485            RouterItem::Packet(pkt) => Ok(wire_format::TargetSenderSet::Hashes(
7486                pkt.wire_target_senders(),
7487            )),
7488            RouterItem::Packed(bytes) => {
7489                Ok(wire_format::peek_routing_envelope(bytes.as_ref())?.target_sender_set())
7490            }
7491        }
7492    }
7493
7494    fn item_targets_another_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
7495        let local_hash = Self::sender_hash(self.sender_arc().as_ref());
7496        match data {
7497            RouterItem::Packet(pkt) => {
7498                Ok(pkt.wire_target_senders().iter().any(|v| *v != local_hash))
7499            }
7500            RouterItem::Packed(bytes) => {
7501                let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7502                Ok(env.target_senders().any(|target| target != local_hash))
7503            }
7504        }
7505    }
7506
7507    fn item_targets_local_sender(&self, data: &RouterItem) -> TelemetryResult<bool> {
7508        let local_hash = Self::sender_hash(self.sender_arc().as_ref());
7509        match data {
7510            RouterItem::Packet(pkt) => Ok(pkt.wire_target_senders().is_empty()
7511                || pkt.wire_target_senders().contains(&local_hash)),
7512            RouterItem::Packed(bytes) => {
7513                let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
7514                Ok(env.target_senders().len() == 0
7515                    || env.target_senders().any(|target| target == local_hash))
7516            }
7517        }
7518    }
7519
7520    fn item_targets_router_as_intermediate_hop(
7521        &self,
7522        data: &RouterItem,
7523        endpoints: &[DataEndpoint],
7524    ) -> TelemetryResult<bool> {
7525        // Inspect borrowed owners once. ACK contracts identify receipt owners,
7526        // not managed-value proxies; unaddressed transit remains byte-identical.
7527        let local_target = match data {
7528            RouterItem::Packet(pkt) => {
7529                if pkt.wire_target_senders().is_empty() || is_internal_control_type(pkt.data_type())
7530                {
7531                    return Ok(false);
7532                }
7533                pkt.wire_target_senders()
7534                    .contains(&Self::sender_hash(self.sender_arc().as_ref()))
7535            }
7536            RouterItem::Packed(bytes) => {
7537                let env = wire_format::peek_routing_envelope(bytes)?;
7538                if env.target_senders().len() == 0 || is_internal_control_type(env.ty) {
7539                    return Ok(false);
7540                }
7541                let local = Self::sender_hash(self.sender_arc().as_ref());
7542                env.target_senders().any(|owner| owner == local)
7543            }
7544        };
7545        Ok(local_target
7546            && !endpoints.iter().copied().any(|endpoint| {
7547                self.endpoint_has_packet_handler(endpoint)
7548                    || self.endpoint_has_packed_handler(endpoint)
7549            }))
7550    }
7551
7552    fn clear_intermediate_hop_targets(
7553        &self,
7554        data: RouterItem,
7555        called_from_queue: bool,
7556    ) -> TelemetryResult<Option<RouterItem>> {
7557        // Every explicitly targeted proxy resolves its own downstream segment,
7558        // including a publisher contract naming several independent bridges.
7559        // Ordinary broadcasts need no owned endpoint or target scratch.
7560        let has_targets = match &data {
7561            RouterItem::Packet(pkt) => !pkt.wire_target_senders().is_empty(),
7562            RouterItem::Packed(bytes) => {
7563                wire_format::peek_routing_envelope(bytes)?
7564                    .target_senders()
7565                    .len()
7566                    != 0
7567            }
7568        };
7569        if !has_targets {
7570            return Ok(Some(data));
7571        }
7572        let (endpoints, _) = self.item_route_info(&data)?;
7573        if !self.item_targets_router_as_intermediate_hop(&data, &endpoints)? {
7574            return Ok(Some(data));
7575        }
7576        let pkt = match &data {
7577            RouterItem::Packet(pkt) => pkt.clone(),
7578            RouterItem::Packed(bytes) => self.unpack_packet_for_role(bytes.as_ref())?,
7579        };
7580        let reliable = match &data {
7581            RouterItem::Packet(_) => None,
7582            RouterItem::Packed(bytes) => wire_format::peek_frame_info(bytes.as_ref())?.reliable,
7583        };
7584        let shape = match &data {
7585            RouterItem::Packet(pkt) => pkt.wire_shape(),
7586            RouterItem::Packed(bytes) => {
7587                wire_format::peek_routing_envelope(bytes.as_ref())?.wire_shape
7588            }
7589        };
7590        let upstream_id = Self::get_hash(&data);
7591        // A completed proxy still owns the aggregate receipt. A delayed parent
7592        // retry must not recreate an ordered child cohort and block successors.
7593        // Keep the alias too, so late child receipts remain consumed locally.
7594        let completed_contract = {
7595            let st = self.state.lock();
7596            st.reliable_return_routes
7597                .get(&upstream_id)
7598                .filter(|route| route.delivery_completed)
7599                .and_then(|route| route.upstream_contract)
7600        };
7601        if let Some((packet_id, publisher_hash)) = completed_contract {
7602            self.queue_end_to_end_reliable_ack_for_sender_hash(
7603                packet_id,
7604                publisher_hash,
7605                called_from_queue,
7606            )?;
7607            return Ok(None);
7608        }
7609        #[cfg(feature = "discovery")]
7610        let destinations = {
7611            let st = self.state.lock();
7612            let upstream = st
7613                .reliable_return_routes
7614                .get(&upstream_id)
7615                .map(|route| route.side);
7616            let mut pending =
7617                self.expected_end_to_end_destinations_locked(&st, &data, upstream, true, true)?;
7618            if pending.is_empty() {
7619                pending = self
7620                    .expected_end_to_end_destinations_locked(&st, &data, upstream, false, true)?;
7621            }
7622            pending
7623        };
7624        #[cfg(feature = "discovery")]
7625        let targets: Vec<u64> = destinations.keys().copied().collect();
7626        #[cfg(not(feature = "discovery"))]
7627        let targets: Vec<u64> = Vec::new();
7628        #[cfg(feature = "discovery")]
7629        if targets.is_empty() {
7630            if self.packet_has_local_delivery(&pkt) {
7631                self.queue_end_to_end_reliable_ack_for_packet_id(
7632                    upstream_id,
7633                    pkt.sender(),
7634                    called_from_queue,
7635                )?;
7636                return Ok(Some(data));
7637            }
7638            if !is_reliable_type(pkt.data_type()) && reliable.is_none() {
7639                return Err(TelemetryError::Io("no discovered route"));
7640            }
7641        }
7642        // Preserve the publisher's packet identity across proxy hops. The
7643        // alias and frozen reader cohort distinguish downstream receipts;
7644        // changing the nonce here would defeat deduplication around cycles.
7645        let forwarded = self.pack_packet_for_contract(&pkt, reliable, shape, &targets)?;
7646        let downstream_id = wire_format::packet_id_from_wire(forwarded.as_ref())?;
7647        let forwarded = RouterItem::Packed(forwarded);
7648        #[cfg(feature = "discovery")]
7649        let already_pending = self
7650            .state
7651            .lock()
7652            .end_to_end_reliable_tx
7653            .contains_key(&downstream_id);
7654        #[cfg(feature = "discovery")]
7655        let ready = self.register_end_to_end_reliable_tx(&forwarded, Some(destinations))?;
7656        #[cfg(not(feature = "discovery"))]
7657        let (ready, already_pending) = {
7658            let _ = called_from_queue;
7659            (true, false)
7660        };
7661        #[cfg(feature = "std")]
7662        if reliable_trace_enabled() {
7663            eprintln!(
7664                "[sedsnet-reliable] proxy retain upstream={upstream_id:#x} downstream={downstream_id:#x} existing={already_pending} ready={ready}"
7665            );
7666        }
7667        {
7668            let mut st = self.state.lock();
7669            let Some(mut route) = st.reliable_return_routes.get(&upstream_id).cloned() else {
7670                st.end_to_end_reliable_tx.remove(&downstream_id);
7671                return Err(TelemetryError::Io("no reliable return route"));
7672            };
7673            route.upstream_contract = Some((
7674                upstream_id,
7675                Self::sender_hash(&Self::canonical_sender_locked(&st, pkt.sender())),
7676            ));
7677            if !Self::remember_reliable_return_route_locked(&mut st, downstream_id) {
7678                st.end_to_end_reliable_tx.remove(&downstream_id);
7679                return Err(TelemetryError::Io("reliable return route window full"));
7680            }
7681            st.reliable_return_routes.insert(downstream_id, route);
7682        }
7683        Ok((ready && !already_pending).then_some(forwarded))
7684    }
7685
7686    #[cfg(feature = "discovery")]
7687    fn side_matches_target_senders_locked(
7688        st: &RouterInner,
7689        side: RouterSideId,
7690        target_senders: wire_format::TargetSenderSet<'_>,
7691        now_ms: u64,
7692    ) -> bool {
7693        st.discovery_routes
7694            .get(&side)
7695            .map(|route| {
7696                if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7697                    return false;
7698                }
7699                route.announcers.iter().any(|(announcer, sender_state)| {
7700                    if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
7701                        return false;
7702                    }
7703                    target_senders.contains(Self::sender_hash(announcer))
7704                        || sender_state
7705                            .advertised_address
7706                            .is_some_and(|address| target_senders.contains(u64::from(address)))
7707                        || sender_state.topology_boards.iter().any(|board| {
7708                            target_senders.contains(Self::sender_hash(&board.sender_id))
7709                        })
7710                })
7711            })
7712            .unwrap_or(false)
7713    }
7714
7715    fn attach_wire_contract_to_item(
7716        &self,
7717        data: RouterItem,
7718        target_senders: &[u64],
7719    ) -> TelemetryResult<RouterItem> {
7720        let pkt = match data {
7721            RouterItem::Packet(packet) => packet,
7722            RouterItem::Packed(bytes) => {
7723                // Preserve an existing frozen cohort. A locally queued plain
7724                // packed frame needs the same contract as a Packet source.
7725                if wire_format::peek_routing_envelope(&bytes)?
7726                    .target_senders()
7727                    .len()
7728                    != 0
7729                {
7730                    return Ok(RouterItem::Packed(bytes));
7731                }
7732                self.unpack_packet_for_role(&bytes)?
7733            }
7734        };
7735        let reliable = if is_reliable_type(pkt.data_type()) {
7736            Some(wire_format::ReliableHeader {
7737                flags: wire_format::RELIABLE_FLAG_UNSEQUENCED,
7738                seq: 0,
7739                ack: 0,
7740            })
7741        } else {
7742            None
7743        };
7744        // A previously frozen Packet has the same immutable cohort as packed
7745        // input. New discovery may change routes, never its receipt owners.
7746        let targets = if pkt.wire_target_senders().is_empty() {
7747            target_senders
7748        } else {
7749            pkt.wire_target_senders()
7750        };
7751        let bytes = self.pack_packet_for_contract(
7752            &pkt,
7753            reliable,
7754            pkt.wire_shape()
7755                .or_else(|| Some(message_meta(pkt.data_type()).element)),
7756            targets,
7757        )?;
7758        Ok(RouterItem::Packed(bytes))
7759    }
7760
7761    fn endpoints_are_link_local_only(eps: &[DataEndpoint]) -> bool {
7762        !eps.is_empty() && eps.iter().all(|ep| ep.is_link_local_only())
7763    }
7764
7765    fn should_route_remote(
7766        &self,
7767        data: &RouterItem,
7768        exclude: Option<RouterSideId>,
7769    ) -> TelemetryResult<bool> {
7770        #[cfg(feature = "discovery")]
7771        {
7772            let RemoteSidePlan::Target(sides) =
7773                self.remote_side_plan_with_selection(data, exclude, false)?;
7774            Ok(!sides.is_empty())
7775        }
7776
7777        #[cfg(not(feature = "discovery"))]
7778        {
7779            let (eps, ty) = self.item_route_info(data)?;
7780            if !(has_nonlocal_endpoint(&eps, &self.cfg) || force_remote_for_type(ty)) {
7781                return Ok(false);
7782            }
7783            let st = self.state.lock();
7784            Ok(!self
7785                .eligible_side_ids_locked(
7786                    &st,
7787                    exclude,
7788                    Some(ty),
7789                    Self::endpoints_are_link_local_only(&eps),
7790                )
7791                .is_empty())
7792        }
7793    }
7794
7795    #[cfg(feature = "discovery")]
7796    fn has_explicit_route_policy_locked(
7797        st: &RouterInner,
7798        src: Option<RouterSideId>,
7799        ty: DataType,
7800    ) -> bool {
7801        st.route_overrides
7802            .keys()
7803            .any(|(route_src, _)| *route_src == src)
7804            || Self::has_typed_route_overrides_locked(st, src, ty)
7805    }
7806
7807    fn remote_side_plan(
7808        &self,
7809        data: &RouterItem,
7810        exclude: Option<RouterSideId>,
7811    ) -> TelemetryResult<RemoteSidePlan> {
7812        self.remote_side_plan_with_selection(data, exclude, true)
7813    }
7814
7815    // Reachability is a query, not a transmission: it must not spend clock
7816    // allowances or advance weighted/adaptive cursors before actual dispatch.
7817    #[inline(never)]
7818    fn remote_side_plan_with_selection(
7819        &self,
7820        data: &RouterItem,
7821        exclude: Option<RouterSideId>,
7822        select_routes: bool,
7823    ) -> TelemetryResult<RemoteSidePlan> {
7824        #[cfg(feature = "discovery")]
7825        let (eps, ty) = self.item_route_info(data)?;
7826        let select_sides = |state: &mut RouterInner, sides: Vec<RouterSideId>, origin| {
7827            #[cfg(feature = "discovery")]
7828            let sides = self.filter_end_to_end_satisfied_sides_locked(state, data, sides, &eps, ty);
7829            if select_routes {
7830                self.apply_route_selection_locked(state, exclude, sides, origin)
7831            } else {
7832                sides
7833            }
7834        };
7835        #[cfg(feature = "discovery")]
7836        {
7837            let target_senders = self.item_target_senders(data)?;
7838            let resolve_beyond_local_hop =
7839                self.item_targets_router_as_intermediate_hop(data, &eps)?;
7840            let preferred_packet_id = Self::reliable_control_target_packet_id(data)?;
7841            let routing_target_senders = if preferred_packet_id.is_some() {
7842                // ACK target[0] identifies the acknowledging owner, not the
7843                // destination. Never route an ACK back toward that owner.
7844                target_senders.skip(1)
7845            } else if resolve_beyond_local_hop {
7846                wire_format::TargetSenderSet::Hashes(&[])
7847            } else {
7848                target_senders
7849            };
7850            if discovery::is_discovery_type(ty) {
7851                let mut st = self.state.lock();
7852                let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7853                return Ok(RemoteSidePlan::Target(select_sides(
7854                    &mut st,
7855                    sides,
7856                    RouteSelectionOrigin::Flood,
7857                )));
7858            }
7859            #[cfg(feature = "timesync")]
7860            if ty == DataType::TimeSyncAnnounce {
7861                // A source announcement describes the source carried in the
7862                // packet; routing it *toward* that discovered source sends it
7863                // back where it came from and leaves downstream clocks blind.
7864                // Propagate it once away from the ingress side, like topology
7865                // discovery. Dedupe and split-horizon exclusion prevent echoes,
7866                // while requests still use the exact discovered source route.
7867                let mut st = self.state.lock();
7868                let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7869                let sides = if select_routes {
7870                    Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
7871                } else {
7872                    sides
7873                };
7874                return Ok(RemoteSidePlan::Target(select_sides(
7875                    &mut st,
7876                    sides,
7877                    RouteSelectionOrigin::Flood,
7878                )));
7879            }
7880            let mut st = self.state.lock();
7881            // Preserve the local-only traffic optimization unless discovery
7882            // has positively identified a remote owner of this network
7883            // variable. A router may own the same schema endpoint locally and
7884            // still need to replicate the value across an IPC/network boundary.
7885            let now_ms = self.clock.now_ms();
7886            let has_remote_variable_owner = st.is_managed_variable(ty.as_u32())
7887                && st.discovery_routes.iter().any(|(&side, route)| {
7888                    exclude != Some(side)
7889                        && now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
7890                        && route.reachable_network_variables.contains(&ty)
7891                });
7892            if !(has_nonlocal_endpoint(&eps, &self.cfg)
7893                || force_remote_for_type(ty)
7894                || !target_senders.is_empty()
7895                || has_remote_variable_owner)
7896            {
7897                return Ok(RemoteSidePlan::Target(Vec::new()));
7898            }
7899
7900            #[cfg(feature = "timesync")]
7901            let preferred_timesync_source = self.preferred_timesync_route_source(data, ty)?;
7902            #[cfg(not(feature = "timesync"))]
7903            let preferred_timesync_source: Option<String> = None;
7904
7905            if let Some(packet_id) = preferred_packet_id {
7906                let target_side = st
7907                    .reliable_return_routes
7908                    .get(&packet_id)
7909                    .map(|route| route.side);
7910                if let Some(side) = target_side
7911                    .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
7912                {
7913                    #[cfg(feature = "timesync")]
7914                    if select_routes
7915                        && !Self::timesync_allowed_for_side_locked(
7916                            &st,
7917                            side,
7918                            ty,
7919                            self.clock.now_ms(),
7920                        )
7921                    {
7922                        return Ok(RemoteSidePlan::Target(Vec::new()));
7923                    }
7924                    return Ok(RemoteSidePlan::Target(vec![side]));
7925                }
7926                // A receipt names its original publisher. Its internal
7927                // endpoint metadata does not restrict that return path.
7928                let eligible = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
7929                let exact: Vec<_> = eligible
7930                    .iter()
7931                    .copied()
7932                    .filter(|&side| {
7933                        !routing_target_senders.is_empty()
7934                            && Self::side_matches_target_senders_locked(
7935                                &st,
7936                                side,
7937                                routing_target_senders,
7938                                now_ms,
7939                            )
7940                    })
7941                    .collect();
7942                if !exact.is_empty() {
7943                    return Ok(RemoteSidePlan::Target(select_sides(
7944                        &mut st,
7945                        exact,
7946                        RouteSelectionOrigin::Discovered,
7947                    )));
7948                }
7949                // Known publisher routes which are excluded or denied remain
7950                // excluded. Sending same-segment CAN receipts onto the other
7951                // link creates an ACK flood. Reserve one-link recovery for
7952                // unknown/legacy publisher identities.
7953                let publisher_known = !routing_target_senders.is_empty()
7954                    && st.discovery_routes.keys().any(|&side| {
7955                        Self::side_matches_target_senders_locked(
7956                            &st,
7957                            side,
7958                            routing_target_senders,
7959                            now_ms,
7960                        )
7961                    });
7962                return Ok(RemoteSidePlan::Target(
7963                    if !publisher_known && exclude.is_some() && eligible.len() == 1 {
7964                        eligible
7965                    } else {
7966                        Vec::new()
7967                    },
7968                ));
7969            }
7970            let restrict_link_local = Self::endpoints_are_link_local_only(&eps);
7971            let prefer_best_overlap = is_reliable_type(ty)
7972                && routing_target_senders.is_empty()
7973                && preferred_packet_id.is_none();
7974            if st.discovery_routes.is_empty() {
7975                let fallback =
7976                    self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
7977                #[cfg(feature = "timesync")]
7978                let fallback = if select_routes {
7979                    Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), fallback)
7980                } else {
7981                    fallback
7982                };
7983                return Ok(RemoteSidePlan::Target(if fallback.len() == 1 {
7984                    fallback
7985                } else {
7986                    Vec::new()
7987                }));
7988            }
7989            let mut matches = self.discovered_route_candidates_locked(
7990                &st,
7991                exclude,
7992                ty,
7993                &eps,
7994                routing_target_senders,
7995                prefer_best_overlap,
7996                preferred_timesync_source.as_deref(),
7997            );
7998            #[cfg(feature = "timesync")]
7999            {
8000                if select_routes {
8001                    matches =
8002                        Self::filter_timesync_matches_locked(&st, ty, self.clock.now_ms(), matches);
8003                }
8004            }
8005            // Replicated managed variables retain independent owner segments.
8006            // All ordinary application types retain shortest-path suppression.
8007            let has_distributed_variable_owners = matches.iter().any(|candidate| {
8008                st.discovery_routes
8009                    .get(&candidate.side)
8010                    .is_some_and(|route| route.reachable_network_variables.contains(&ty))
8011            });
8012            if !has_distributed_variable_owners {
8013                // A single eligible physical side has no role/path ambiguity.
8014                // Keep the allocation-bounded transit path instead of building
8015                // per-role candidate vectors for the same sole destination.
8016                if self.cfg.role.local_services()
8017                    && eps.len() > 1
8018                    && matches.len() > 1
8019                    && !is_reliable_type(ty)
8020                    && !st.is_managed_variable(ty.as_u32())
8021                    && routing_target_senders.is_empty()
8022                    && preferred_packet_id.is_none()
8023                    && preferred_timesync_source.is_none()
8024                    && !Self::has_explicit_route_policy_locked(&st, exclude, ty)
8025                    && !st.source_route_modes.contains_key(&exclude)
8026                {
8027                    if !select_routes {
8028                        return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
8029                    }
8030                    let sides = self.select_endpoint_role_sides_locked(
8031                        &mut st,
8032                        exclude,
8033                        &eps,
8034                        &matches,
8035                        self.clock.now_ms(),
8036                    );
8037                    return Ok(RemoteSidePlan::Target(sides));
8038                }
8039                Self::retain_shortest_discovery_candidates_locked(
8040                    &st,
8041                    &mut matches,
8042                    &eps,
8043                    routing_target_senders,
8044                    self.clock.now_ms(),
8045                );
8046            }
8047
8048            if !matches.is_empty() {
8049                if !select_routes {
8050                    // Only existence is queried. Shortest-path and side-policy
8051                    // validation already ran; do not select a weighted path.
8052                    return Ok(RemoteSidePlan::Target(vec![matches[0].side]));
8053                }
8054                Ok(RemoteSidePlan::Target(
8055                    self.select_discovered_candidate_sides_locked(
8056                        &mut st,
8057                        (data, ty),
8058                        exclude,
8059                        routing_target_senders,
8060                        prefer_best_overlap,
8061                        matches,
8062                    ),
8063                ))
8064            } else if !self.cfg.role.local_services() && exclude.is_some() {
8065                let fallback =
8066                    self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
8067                let known_elsewhere = !routing_target_senders.is_empty()
8068                    && st.discovery_routes.keys().any(|&side| {
8069                        Self::side_matches_target_senders_locked(
8070                            &st,
8071                            side,
8072                            routing_target_senders,
8073                            self.clock.now_ms(),
8074                        )
8075                    });
8076                Ok(RemoteSidePlan::Target(
8077                    if fallback.len() == 1 && !known_elsewhere {
8078                        select_sides(&mut st, fallback, RouteSelectionOrigin::Flood)
8079                    } else {
8080                        Vec::new()
8081                    },
8082                ))
8083            } else if prefer_best_overlap {
8084                Ok(RemoteSidePlan::Target(Vec::new()))
8085            } else {
8086                if Self::has_explicit_route_policy_locked(&st, exclude, ty) {
8087                    let sides =
8088                        self.eligible_side_ids_locked(&st, exclude, Some(ty), restrict_link_local);
8089                    #[cfg(feature = "timesync")]
8090                    let sides = if select_routes {
8091                        Self::filter_timesync_sides_locked(&st, ty, self.clock.now_ms(), sides)
8092                    } else {
8093                        sides
8094                    };
8095                    Ok(RemoteSidePlan::Target(select_sides(
8096                        &mut st,
8097                        sides,
8098                        RouteSelectionOrigin::Flood,
8099                    )))
8100                } else {
8101                    Ok(RemoteSidePlan::Target(Vec::new()))
8102                }
8103            }
8104        }
8105        #[cfg(not(feature = "discovery"))]
8106        {
8107            let ty = Self::item_data_type(data)?;
8108            let mut st = self.state.lock();
8109            if let Some(packet_id) = Self::reliable_control_target_packet_id(data)? {
8110                let target_side = st
8111                    .reliable_return_routes
8112                    .get(&packet_id)
8113                    .map(|route| route.side);
8114                if let Some(side) = target_side
8115                    .filter(|side| self.route_allowed_locked(&st, exclude, Some(ty), *side))
8116                {
8117                    return Ok(RemoteSidePlan::Target(vec![side]));
8118                }
8119                let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
8120                return Ok(RemoteSidePlan::Target(
8121                    if exclude.is_some() && sides.len() == 1 {
8122                        sides
8123                    } else {
8124                        Vec::new()
8125                    },
8126                ));
8127            }
8128            let sides = self.eligible_side_ids_locked(&st, exclude, Some(ty), false);
8129            Ok(RemoteSidePlan::Target(select_sides(
8130                &mut st,
8131                sides,
8132                RouteSelectionOrigin::Flood,
8133            )))
8134        }
8135    }
8136
8137    #[cfg(feature = "discovery")]
8138    fn local_discovery_endpoints(&self, st: &RouterInner) -> Vec<DataEndpoint> {
8139        let mut eps: Vec<DataEndpoint> = self.cfg.handlers().iter().map(|h| h.endpoint).collect();
8140        // Managed-variable ownership has its own discovery field. Its message
8141        // endpoints describe delivery sinks, not ownership: treating them as
8142        // local endpoints makes every flight-state replica claim that it is
8143        // GroundStation and prevents ordinary telemetry from reaching the
8144        // real GroundStation route.
8145        let _ = st;
8146        #[cfg(feature = "timesync")]
8147        if self.cfg.timesync_config().is_some() {
8148            eps.push(DataEndpoint::TimeSync);
8149        }
8150        eps.retain(|ep| !discovery::is_discovery_endpoint(*ep));
8151        crate::small_sort::sort(&mut eps);
8152        eps.dedup();
8153        eps
8154    }
8155
8156    #[cfg(all(feature = "discovery", not(feature = "timesync")))]
8157    fn local_discovery_timesync_sources(&self, _now_ms: u64) -> Vec<String> {
8158        Vec::new()
8159    }
8160
8161    #[cfg(all(feature = "discovery", feature = "timesync"))]
8162    fn local_discovery_timesync_sources(&self, now_ms: u64) -> Vec<String> {
8163        #[cfg(feature = "timesync")]
8164        if self.timesync.is_none() {
8165            return Vec::new();
8166        }
8167
8168        #[cfg(feature = "timesync")]
8169        {
8170            let st = self
8171                .timesync
8172                .as_ref()
8173                .expect("local time service enabled")
8174                .lock();
8175            if let Some(tracker) = st.tracker.as_ref()
8176                && tracker.should_serve(
8177                    now_ms,
8178                    Self::timesync_has_usable_time_locked(&st, self.monotonic_now_ns()),
8179                )
8180            {
8181                return vec![self.sender_arc().to_string()];
8182            }
8183        }
8184        Vec::new()
8185    }
8186
8187    #[cfg(all(feature = "discovery", feature = "timesync"))]
8188    fn preferred_timesync_route_source(
8189        &self,
8190        data: &RouterItem,
8191        ty: DataType,
8192    ) -> TelemetryResult<Option<String>> {
8193        if !matches!(
8194            ty,
8195            DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
8196        ) {
8197            return Ok(None);
8198        }
8199
8200        match data {
8201            RouterItem::Packet(pkt) => match ty {
8202                DataType::TimeSyncRequest => {
8203                    let local_sender = self.sender_arc();
8204                    if pkt.sender() == local_sender.as_ref() {
8205                        Ok(self.timesync.as_ref().and_then(|runtime| {
8206                            runtime.lock().tracker.as_ref().and_then(|tracker| {
8207                                tracker.current_source().map(|src| src.sender.clone())
8208                            })
8209                        }))
8210                    } else {
8211                        Ok(None)
8212                    }
8213                }
8214                DataType::TimeSyncAnnounce | DataType::TimeSyncResponse => {
8215                    Ok(Some(pkt.sender().to_owned()))
8216                }
8217                _ => Ok(None),
8218            },
8219            RouterItem::Packed(bytes) => {
8220                let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
8221                self.preferred_timesync_route_source(&RouterItem::Packet(pkt), ty)
8222            }
8223        }
8224    }
8225
8226    #[cfg(feature = "discovery")]
8227    fn note_discovery_topology_change_locked(st: &mut RouterInner, now_ms: u64) {
8228        st.discovery_cadence.on_topology_change(now_ms);
8229        // Propagate route changes immediately without restarting discovery or
8230        // resending the complete topology graph. DiscoveryAddress replaces the
8231        // prior compact route summary, so additions and removals converge while
8232        // explicit baseline requests remain available as loss recovery.
8233        for side_id in st
8234            .sides
8235            .iter()
8236            .enumerate()
8237            .filter_map(|(side_id, side)| side.as_ref().map(|_| side_id))
8238        {
8239            let throttle = st.discovery_side_throttle.entry(side_id).or_default();
8240            throttle.set_pending_incremental(true);
8241            throttle.incremental_retries_remaining = DISCOVERY_INCREMENTAL_RETRY_COUNT;
8242        }
8243        // Make the first incremental control frame self-describing. This is a
8244        // small header refresh, not a topology reset, and prevents a restarted
8245        // peer from missing the route delta because it lost its RX dictionary.
8246        for transport in st.side_transport.values_mut() {
8247            transport.refresh_transmit_header_templates();
8248        }
8249    }
8250
8251    #[cfg(feature = "discovery")]
8252    fn sender_topology_board_mut<'a>(
8253        sender_state: &'a mut DiscoverySenderState,
8254        sender_id: &str,
8255    ) -> &'a mut TopologyBoardNode {
8256        let boards = Arc::make_mut(&mut sender_state.topology_boards);
8257        if let Some(idx) = boards.iter().position(|board| board.sender_id == sender_id) {
8258            return &mut boards[idx];
8259        }
8260        boards.push(TopologyBoardNode {
8261            sender_id: sender_id.to_string(),
8262            reachable_endpoints: Vec::new(),
8263            reachable_timesync_sources: Vec::new(),
8264            connections: Vec::new(),
8265        });
8266        boards.last_mut().expect("inserted above")
8267    }
8268
8269    #[cfg(feature = "discovery")]
8270    fn refresh_sender_topology_state(sender_state: &mut DiscoverySenderState) {
8271        if let Some(boards) = Arc::get_mut(&mut sender_state.topology_boards) {
8272            discovery::normalize_topology_boards(boards);
8273        }
8274        let (mut reachable, mut reachable_timesync_sources) =
8275            discovery::summarize_topology_boards(&sender_state.topology_boards);
8276        reachable.extend(sender_state.advertised_reachable.iter().copied());
8277        crate::small_sort::sort(&mut reachable);
8278        reachable.dedup();
8279        reachable_timesync_sources.extend(
8280            sender_state
8281                .advertised_reachable_timesync_sources
8282                .iter()
8283                .cloned(),
8284        );
8285        reachable_timesync_sources.sort_unstable();
8286        reachable_timesync_sources.dedup();
8287        sender_state.reachable = reachable;
8288        sender_state.reachable_timesync_sources = reachable_timesync_sources;
8289    }
8290
8291    #[cfg(feature = "discovery")]
8292    fn recompute_discovery_side_state(route: &mut DiscoverySideState) {
8293        let mut reachable = Vec::new();
8294        let mut reachable_network_variables = Vec::new();
8295        let mut reachable_timesync_sources = Vec::new();
8296        let mut last_seen_ms = 0u64;
8297        for sender in route.announcers.values() {
8298            reachable.extend(sender.reachable.iter().copied());
8299            reachable_network_variables.extend(sender.reachable_network_variables.iter().copied());
8300            reachable_timesync_sources.extend(sender.reachable_timesync_sources.iter().cloned());
8301            last_seen_ms = last_seen_ms.max(sender.last_seen_ms);
8302        }
8303        reachable_network_variables.extend(
8304            route
8305                .requested_network_variables
8306                .iter()
8307                .map(|request| request.ty),
8308        );
8309        last_seen_ms = last_seen_ms.max(
8310            route
8311                .requested_network_variables
8312                .iter()
8313                .map(|request| request.last_seen_ms)
8314                .max()
8315                .unwrap_or(0),
8316        );
8317        crate::small_sort::sort(&mut reachable);
8318        reachable.dedup();
8319        crate::small_sort::sort(&mut reachable_network_variables);
8320        reachable_network_variables.dedup();
8321        reachable_timesync_sources.sort_unstable();
8322        reachable_timesync_sources.dedup();
8323        route.reachable = reachable;
8324        route.reachable_network_variables = reachable_network_variables;
8325        route.reachable_timesync_sources = reachable_timesync_sources;
8326        route.last_seen_ms = last_seen_ms;
8327    }
8328
8329    #[cfg(feature = "discovery")]
8330    fn local_discovery_topology_board(
8331        &self,
8332        st: &RouterInner,
8333        now_ms: u64,
8334        link_local_enabled: bool,
8335    ) -> TopologyBoardNode {
8336        let mut reachable_endpoints = self.local_discovery_endpoints(st);
8337        if !link_local_enabled {
8338            reachable_endpoints.retain(|ep| !ep.is_link_local_only());
8339        }
8340        let mut connections = Vec::new();
8341        for route in st.discovery_routes.values() {
8342            if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8343                continue;
8344            }
8345            for (sender, sender_state) in route.announcers.iter() {
8346                if now_ms.saturating_sub(sender_state.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8347                    connections.push(sender.clone());
8348                }
8349            }
8350        }
8351        connections.sort_unstable();
8352        connections.dedup();
8353        let sender = self.sender_arc();
8354        TopologyBoardNode {
8355            sender_id: sender.to_string(),
8356            reachable_endpoints,
8357            reachable_timesync_sources: self.local_discovery_timesync_sources(now_ms),
8358            connections,
8359        }
8360    }
8361
8362    #[cfg(feature = "discovery")]
8363    fn advertised_discovery_topology_for_link_locked(
8364        &self,
8365        st: &RouterInner,
8366        now_ms: u64,
8367        link_local_enabled: bool,
8368        exclude_side: Option<RouterSideId>,
8369    ) -> Vec<TopologyBoardNode> {
8370        let mut boards = vec![self.local_discovery_topology_board(st, now_ms, link_local_enabled)];
8371        for (&route_side, route) in st.discovery_routes.iter() {
8372            if exclude_side == Some(route_side) {
8373                continue;
8374            }
8375            if now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8376                continue;
8377            }
8378            for (announcer, sender_state) in route.announcers.iter() {
8379                if now_ms.saturating_sub(sender_state.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8380                    continue;
8381                }
8382                if sender_state.topology_boards.is_empty() {
8383                    let sender = self.sender_arc();
8384                    let fallback = TopologyBoardNode {
8385                        sender_id: announcer.clone(),
8386                        reachable_endpoints: sender_state.reachable.clone(),
8387                        reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
8388                        connections: vec![sender.to_string()],
8389                    };
8390                    discovery::merge_topology_boards(&mut boards, core::slice::from_ref(&fallback));
8391                } else {
8392                    // Merge the retained view directly. Cloning it first and
8393                    // cloning again into the output doubled temporary strings.
8394                    discovery::merge_topology_boards(&mut boards, &sender_state.topology_boards);
8395                    if let Some(board) = boards
8396                        .iter_mut()
8397                        .find(|board| board.sender_id == *announcer)
8398                    {
8399                        let sender = self.sender_arc();
8400                        if !board.connections.iter().any(|peer| peer == sender.as_ref()) {
8401                            board.connections.push(sender.to_string());
8402                        }
8403                    }
8404                }
8405            }
8406        }
8407        if !link_local_enabled {
8408            for board in &mut boards {
8409                board
8410                    .reachable_endpoints
8411                    .retain(|ep| !ep.is_link_local_only());
8412            }
8413        }
8414        discovery::normalize_topology_boards(&mut boards);
8415        boards
8416    }
8417
8418    #[cfg(feature = "discovery")]
8419    fn prune_discovery_routes_locked(st: &mut RouterInner, now_ms: u64) -> bool {
8420        // Polling runs at the firmware service rate, not the advertisement
8421        // rate. Never clone the topology just to discover nothing expired.
8422        let mut changed = false;
8423        st.discovery_routes.retain(|_, route| {
8424            let before_peers = route.announcers.len();
8425            let before_requests = route.requested_network_variables.len();
8426            route.announcers.retain(|_, sender| {
8427                now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8428            });
8429            route.requested_network_variables.retain(|request| {
8430                now_ms >= request.last_seen_ms
8431                    && now_ms - request.last_seen_ms <= DISCOVERY_ROUTE_TTL_MS
8432            });
8433            if before_peers != route.announcers.len()
8434                || before_requests != route.requested_network_variables.len()
8435            {
8436                Self::recompute_discovery_side_state(route);
8437                changed = true;
8438            }
8439            let keep =
8440                !route.announcers.is_empty() || !route.requested_network_variables.is_empty();
8441            changed |= !keep;
8442            keep
8443        });
8444        changed
8445    }
8446
8447    #[cfg(feature = "discovery")]
8448    fn advertised_summaries_from_topology_locked(
8449        &self,
8450        st: &RouterInner,
8451        now_ms: u64,
8452        link_local_enabled: bool,
8453        exclude_side: Option<RouterSideId>,
8454        topology: &[TopologyBoardNode],
8455    ) -> (Vec<crate::DataEndpoint>, Vec<String>) {
8456        let (mut endpoints, mut sources) = discovery::summarize_topology_boards(topology);
8457        for (&side, route) in &st.discovery_routes {
8458            if exclude_side == Some(side)
8459                || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8460            {
8461                continue;
8462            }
8463            for peer in route.announcers.values() {
8464                if now_ms.saturating_sub(peer.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS {
8465                    continue;
8466                }
8467                for endpoint in &peer.advertised_reachable {
8468                    if !discovery::is_discovery_endpoint(*endpoint)
8469                        && (link_local_enabled || !endpoint.is_link_local_only())
8470                        && !endpoints.contains(endpoint)
8471                    {
8472                        endpoints.push(*endpoint);
8473                    }
8474                }
8475                for source in &peer.advertised_reachable_timesync_sources {
8476                    if !sources.contains(source) {
8477                        sources.push(source.clone());
8478                    }
8479                }
8480            }
8481        }
8482        endpoints.retain(|endpoint| {
8483            !discovery::is_discovery_endpoint(*endpoint)
8484                && (link_local_enabled || !endpoint.is_link_local_only())
8485        });
8486        crate::small_sort::sort(&mut endpoints);
8487        sources.sort_unstable();
8488        (endpoints, sources)
8489    }
8490
8491    #[cfg(feature = "discovery")]
8492    fn advertised_discovery_endpoints_for_link_locked(
8493        &self,
8494        st: &RouterInner,
8495        now_ms: u64,
8496        link_local_enabled: bool,
8497        exclude_side: Option<RouterSideId>,
8498    ) -> Vec<DataEndpoint> {
8499        let (mut reachable_endpoints, _) = discovery::summarize_topology_boards(
8500            &self.advertised_discovery_topology_for_link_locked(
8501                st,
8502                now_ms,
8503                link_local_enabled,
8504                exclude_side,
8505            ),
8506        );
8507        // Compact advertisements can know routes before detailed topology arrives.
8508        // Preserve that reachability without assigning downstream endpoints to the bridge.
8509        for (&side, route) in &st.discovery_routes {
8510            if exclude_side == Some(side)
8511                || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8512            {
8513                continue;
8514            }
8515            for sender in route.announcers.values() {
8516                if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8517                    reachable_endpoints.extend(sender.advertised_reachable.iter().copied());
8518                }
8519            }
8520        }
8521        crate::small_sort::sort(&mut reachable_endpoints);
8522        reachable_endpoints.dedup();
8523        reachable_endpoints
8524            .into_iter()
8525            .filter(|ep| {
8526                !discovery::is_discovery_endpoint(*ep)
8527                    && (link_local_enabled || !ep.is_link_local_only())
8528            })
8529            .collect()
8530    }
8531
8532    #[cfg(all(test, feature = "discovery"))]
8533    fn compact_summary_fixture(&self, endpoint: DataEndpoint) {
8534        let mut sender = DiscoverySenderState {
8535            advertised_reachable: vec![endpoint],
8536            advertised_reachable_timesync_sources: vec!["clock-behind-bridge".to_string()],
8537            last_seen_ms: self.clock.now_ms(),
8538            topology_boards: Arc::new(vec![TopologyBoardNode {
8539                sender_id: "bridge".to_string(),
8540                reachable_endpoints: Vec::new(),
8541                reachable_timesync_sources: Vec::new(),
8542                connections: Vec::new(),
8543            }]),
8544            ..Default::default()
8545        };
8546        Self::refresh_sender_topology_state(&mut sender);
8547        let mut route = DiscoverySideState::default();
8548        route
8549            .announcers
8550            .insert("bridge".to_string(), Box::new(sender));
8551        Self::recompute_discovery_side_state(&mut route);
8552        self.state.lock().discovery_routes.insert(0, route);
8553    }
8554
8555    #[cfg(feature = "discovery")]
8556    fn advertised_discovery_timesync_sources_for_link_locked(
8557        &self,
8558        st: &RouterInner,
8559        now_ms: u64,
8560        exclude_side: Option<RouterSideId>,
8561    ) -> Vec<String> {
8562        let (_, mut sources) = discovery::summarize_topology_boards(
8563            &self.advertised_discovery_topology_for_link_locked(st, now_ms, true, exclude_side),
8564        );
8565        for (&side, route) in &st.discovery_routes {
8566            if exclude_side == Some(side)
8567                || now_ms.saturating_sub(route.last_seen_ms) > DISCOVERY_ROUTE_TTL_MS
8568            {
8569                continue;
8570            }
8571            for sender in route.announcers.values() {
8572                if now_ms.saturating_sub(sender.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8573                    sources.extend(sender.advertised_reachable_timesync_sources.iter().cloned());
8574                }
8575            }
8576        }
8577        sources.sort_unstable();
8578        sources.dedup();
8579        sources
8580    }
8581
8582    #[cfg(feature = "discovery")]
8583    fn advertised_network_variables_for_link_locked(
8584        &self,
8585        st: &RouterInner,
8586        now_ms: u64,
8587        exclude_side: Option<RouterSideId>,
8588    ) -> Vec<DataType> {
8589        let mut types: Vec<DataType> = st
8590            .local_services
8591            .iter()
8592            .flat_map(|local| local.managed_variable_types.iter())
8593            .copied()
8594            .map(DataType)
8595            .collect();
8596        for (&route_side, route) in st.discovery_routes.iter() {
8597            if exclude_side == Some(route_side) {
8598                continue;
8599            }
8600            if now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8601                types.extend(route.reachable_network_variables.iter().copied());
8602            }
8603        }
8604        crate::small_sort::sort(&mut types);
8605        types.dedup();
8606        types
8607    }
8608
8609    #[cfg(feature = "discovery")]
8610    fn preferred_discovery_master_present_locked<'a>(
8611        &self,
8612        st: &'a RouterInner,
8613        now_ms: u64,
8614    ) -> Option<&'a str> {
8615        if let Some(preferred) = st.preferred_discovery_master.as_deref()
8616            && (preferred == self.sender_arc().as_ref()
8617                || st
8618                    .discovery_routes
8619                    .values()
8620                    .filter(|route| {
8621                        now_ms.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8622                    })
8623                    .any(|route| {
8624                        route
8625                            .announcers
8626                            .iter()
8627                            .filter(|(_, peer)| {
8628                                now_ms.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
8629                            })
8630                            .any(|(name, peer)| {
8631                                name == preferred
8632                                    || peer.topology_boards.iter().any(|board| {
8633                                        board.sender_id == preferred
8634                                            || board
8635                                                .connections
8636                                                .iter()
8637                                                .any(|name| name == preferred)
8638                                    })
8639                            })
8640                    }))
8641        {
8642            // Presence of the preferred master needs a name scan, not a
8643            // cloned topology and a graph election on each advertisement.
8644            return Some(preferred);
8645        }
8646        None
8647    }
8648
8649    #[cfg(feature = "discovery")]
8650    fn discovery_master_sender_locked(&self, st: &RouterInner, now_ms: u64) -> String {
8651        if let Some(preferred) = self.preferred_discovery_master_present_locked(st, now_ms) {
8652            return preferred.to_string();
8653        }
8654        let boards = self.advertised_discovery_topology_for_link_locked(st, now_ms, true, None);
8655        discovery::elect_discovery_master(self.sender_arc().as_ref(), &boards)
8656    }
8657
8658    #[cfg(all(feature = "discovery", feature = "timesync"))]
8659    #[inline]
8660    fn side_is_slow_control_link_locked(
8661        st: &RouterInner,
8662        side_id: RouterSideId,
8663        now_ms: u64,
8664    ) -> bool {
8665        st.adaptive_route_stats.get(&side_id).is_some_and(|stats| {
8666            let recent_slow = stats.last_slow_observed_ms > 0
8667                && now_ms.saturating_sub(stats.last_slow_observed_ms)
8668                    <= discovery::DISCOVERY_SLOW_LINK_FULL_INTERVAL_MS;
8669            stats.sample_count > 0
8670                && ((stats.estimated_bandwidth_bps > 0
8671                    && stats.estimated_bandwidth_bps <= CONTROL_SLOW_LINK_CAPACITY_BPS)
8672                    || recent_slow)
8673        })
8674    }
8675
8676    #[cfg(feature = "discovery")]
8677    fn discovery_level_for_side_locked(
8678        st: &mut RouterInner,
8679        side_id: RouterSideId,
8680        now_ms: u64,
8681    ) -> Option<DiscoveryAdvertiseLevel> {
8682        let throttle = st.discovery_side_throttle.entry(side_id).or_default();
8683        if !throttle.has_sent_full() {
8684            throttle.set_has_sent_full(true);
8685            throttle.set_pending_incremental(false);
8686            throttle.incremental_retries_remaining = 0;
8687            throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
8688            return Some(DiscoveryAdvertiseLevel::Full);
8689        }
8690        if throttle.pending_incremental() {
8691            throttle.incremental_retries_remaining =
8692                throttle.incremental_retries_remaining.saturating_sub(1);
8693            throttle.set_pending_incremental(throttle.incremental_retries_remaining > 0);
8694            return Some(DiscoveryAdvertiseLevel::Incremental);
8695        }
8696        // Stable links carry only a small liveness announcement. Route and
8697        // schema snapshots are emitted on changes or explicit recovery requests.
8698        if throttle.liveness_due(now_ms) {
8699            throttle.next_ping_ms = now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
8700            return Some(DiscoveryAdvertiseLevel::MinimalPing);
8701        }
8702        None
8703    }
8704
8705    #[cfg(all(feature = "discovery", feature = "timesync"))]
8706    #[inline]
8707    fn is_timesync_type(ty: DataType) -> bool {
8708        matches!(
8709            ty,
8710            DataType::TimeSyncAnnounce | DataType::TimeSyncRequest | DataType::TimeSyncResponse
8711        )
8712    }
8713
8714    #[cfg(all(feature = "discovery", feature = "timesync"))]
8715    #[inline(never)]
8716    fn timesync_allowed_for_side_locked(
8717        st: &RouterInner,
8718        side_id: RouterSideId,
8719        ty: DataType,
8720        now_ms: u64,
8721    ) -> bool {
8722        if !Self::is_timesync_type(ty) {
8723            return true;
8724        }
8725        if !Self::side_is_slow_control_link_locked(st, side_id, now_ms) {
8726            return true;
8727        }
8728        // A routing/availability check is not a successful transmission. Keep
8729        // the allowance available across retries and commit only on acceptance.
8730        st.timesync_side_throttle
8731            .get(&side_id)
8732            .is_none_or(|throttle| now_ms >= throttle.next_allowed_ms)
8733    }
8734
8735    #[cfg(all(feature = "discovery", feature = "timesync"))]
8736    fn filter_timesync_sides_locked(
8737        st: &RouterInner,
8738        ty: DataType,
8739        now_ms: u64,
8740        sides: Vec<RouterSideId>,
8741    ) -> Vec<RouterSideId> {
8742        sides
8743            .into_iter()
8744            .filter(|side| Self::timesync_allowed_for_side_locked(st, *side, ty, now_ms))
8745            .collect()
8746    }
8747
8748    #[cfg(all(feature = "discovery", feature = "timesync"))]
8749    fn filter_timesync_matches_locked(
8750        st: &RouterInner,
8751        ty: DataType,
8752        now_ms: u64,
8753        matches: Vec<DiscoveryCandidateMatch>,
8754    ) -> Vec<DiscoveryCandidateMatch> {
8755        matches
8756            .into_iter()
8757            .filter(|m| Self::timesync_allowed_for_side_locked(st, m.side, ty, now_ms))
8758            .collect()
8759    }
8760
8761    #[cfg(feature = "discovery")]
8762    fn refresh_known_discovery_liveness(
8763        &self,
8764        pkt: &Packet,
8765        side: RouterSideId,
8766    ) -> TelemetryResult<bool> {
8767        if pkt.data_type() == crate::DataType::DiscoveryAddress {
8768            return self.refresh_known_address_payload(pkt.payload(), side);
8769        }
8770        if pkt.data_type() == crate::DataType::DiscoveryTopology {
8771            // No decoded graph or retained-side clone is needed for a retry
8772            // whose payload already describes precisely the retained state.
8773            // Keep changed topology behind the full admission guard.
8774            crate::memory_admission::check(512, 128)?;
8775            let now = self.clock.now_ms();
8776            let mut st = self.state.lock();
8777            let canonical = Self::canonical_sender_locked(&st, pkt.sender());
8778            let Some(route) = st.discovery_routes.get_mut(&side) else {
8779                return Ok(false);
8780            };
8781            let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
8782                return Ok(false);
8783            };
8784            let Some(full) =
8785                discovery::topology_payload_matches_retained(pkt.payload(), &peer.topology_boards)
8786            else {
8787                return Ok(false);
8788            };
8789            peer.has_full_topology |= full;
8790            peer.last_seen_ms = now;
8791            route.last_seen_ms = route.last_seen_ms.max(now);
8792            return Ok(true);
8793        }
8794        if pkt.data_type() != crate::DataType::DiscoveryAnnounce || !pkt.payload().is_empty() {
8795            return Ok(false);
8796        }
8797        crate::memory_admission::check(512, 128)?;
8798        let now = self.clock.now_ms();
8799        let mut st = self.state.lock();
8800        let canonical = Self::canonical_sender_locked(&st, pkt.sender());
8801        let Some(route) = st.discovery_routes.get_mut(&side) else {
8802            return Ok(false);
8803        };
8804        let Some(peer) = route.announcers.get_mut(canonical.as_str()) else {
8805            return Ok(false);
8806        };
8807        peer.last_seen_ms = now;
8808        route.last_seen_ms = route.last_seen_ms.max(now);
8809        Ok(true)
8810    }
8811
8812    #[cfg(feature = "discovery")]
8813    fn refresh_known_address_payload(
8814        &self,
8815        payload: &[u8],
8816        side: RouterSideId,
8817    ) -> TelemetryResult<bool> {
8818        let ad = discovery::decode_address_view(payload)?;
8819        let now = self.clock.now_ms();
8820        let mut st = self.state.lock();
8821        let link_local = st
8822            .sides
8823            .get(side)
8824            .and_then(|entry| entry.as_ref())
8825            .is_some_and(|entry| entry.opts.link_local_enabled);
8826        {
8827            let Some(entry) = st.address_book.get(ad.hostname) else {
8828                return Ok(false);
8829            };
8830            if entry.address != ad.address
8831                || entry.requested_address != ad.requested_address
8832                || entry.mode != Self::address_mode_from_code(ad.mode, ad.requested_address)
8833                || entry.owner_hash != ad.owner_hash
8834            {
8835                return Ok(false);
8836            }
8837            if entry.birth_ms != ad.birth_ms
8838                && st.address_book.iter().any(|(hostname, other)| {
8839                    hostname.as_str() != ad.hostname
8840                        && [ad.address, ad.requested_address].iter().any(|candidate| {
8841                            *candidate != 0
8842                                && (other.address == *candidate
8843                                    || other.requested_address == *candidate)
8844                        })
8845                })
8846            {
8847                // Birth time participates in collision arbitration. Reuse an
8848                // incarnation only when the unchanged owner has no contender;
8849                // contested addresses retain the transactional admission path.
8850                return Ok(false);
8851            }
8852        }
8853        let sender = ad.hostname;
8854        let Some(route) = st.discovery_routes.get_mut(&side) else {
8855            return Ok(false);
8856        };
8857        let Some(peer) = route.announcers.get_mut(sender) else {
8858            return Ok(false);
8859        };
8860        if peer.advertised_address != Some(ad.address)
8861            || !ad
8862                .endpoints()
8863                .filter(|ep| link_local || !ep.is_link_local_only())
8864                .eq(peer.advertised_reachable.iter().copied())
8865            || !ad.sources().eq(peer
8866                .advertised_reachable_timesync_sources
8867                .iter()
8868                .map(String::as_str))
8869            || !ad
8870                .variables()
8871                .eq(peer.reachable_network_variables.iter().copied())
8872            || peer.link_capabilities != Some(ad.link_capabilities)
8873        {
8874            return Ok(false);
8875        }
8876        // A liveness update cannot add ownership, change identity, or enlarge
8877        // discovery state. Leave new/changed summaries behind the full guard.
8878        peer.last_seen_ms = now;
8879        route.last_seen_ms = route.last_seen_ms.max(now);
8880        if let Some(entry) = st.address_book.get_mut(ad.hostname) {
8881            // A known, uncontested owner can restart without cloning its
8882            // unchanged route/schema snapshot. Record the new incarnation.
8883            entry.birth_ms = ad.birth_ms;
8884            entry.last_seen_ms = now;
8885        }
8886        #[cfg(feature = "compression")]
8887        {
8888            drop(st);
8889            self.note_payload_codec_address(side, payload, true);
8890        }
8891        Ok(true)
8892    }
8893
8894    #[cfg(feature = "discovery")]
8895    fn queue_pressure_keepalives(
8896        &self,
8897        requested_side: Option<RouterSideId>,
8898    ) -> TelemetryResult<()> {
8899        let now = self.clock.now_ms();
8900        let sender = self.sender_arc();
8901        crate::memory_admission::check(
8902            256 + sender.len().saturating_mul(4),
8903            sender.len().max(256),
8904        )?;
8905        let side_count = self.state.lock().sides.len();
8906        for side in 0..side_count {
8907            let due = {
8908                let st = self.state.lock();
8909                requested_side.is_none_or(|wanted| wanted == side)
8910                    && st.sides[side].is_some()
8911                    && self.route_allowed_locked(
8912                        &st,
8913                        None,
8914                        Some(crate::DataType::DiscoveryAnnounce),
8915                        side,
8916                    )
8917                    && st
8918                        .discovery_side_throttle
8919                        .get(&side)
8920                        .is_none_or(|throttle| now >= throttle.next_ping_ms)
8921            };
8922            if !due {
8923                continue;
8924            }
8925            // Propagate newly learned reachability even when the full topology
8926            // cannot fit. Build only a split-horizon summary, never a graph.
8927            let summary = {
8928                let st = self.state.lock();
8929                let extra = st
8930                    .discovery_routes
8931                    .iter()
8932                    .filter(|(id, _)| **id != side)
8933                    .fold(0usize, |total, (_, route)| {
8934                        total
8935                            .saturating_add(route.reachable.len().saturating_mul(32))
8936                            .saturating_add(
8937                                route.reachable_network_variables.len().saturating_mul(32),
8938                            )
8939                            .saturating_add(
8940                                route
8941                                    .reachable_timesync_sources
8942                                    .iter()
8943                                    .map(|source| source.len().saturating_add(32).saturating_mul(4))
8944                                    .sum::<usize>(),
8945                            )
8946                    });
8947                if crate::memory_admission::check(4096usize.saturating_add(extra), 2048).is_ok() {
8948                    let opts = st.sides[side].as_ref().expect("checked above").opts.clone();
8949                    let mut endpoints = self.local_discovery_endpoints(&st);
8950                    let mut variables =
8951                        self.advertised_network_variables_for_link_locked(&st, now, Some(side));
8952                    let mut sources = self.local_discovery_timesync_sources(now);
8953                    for (&route_side, route) in &st.discovery_routes {
8954                        if route_side == side {
8955                            continue;
8956                        }
8957                        for peer in route.announcers.values() {
8958                            if now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS {
8959                                endpoints.extend(peer.reachable.iter().copied());
8960                                sources.extend(peer.reachable_timesync_sources.iter().cloned());
8961                            }
8962                        }
8963                    }
8964                    endpoints.retain(|ep| {
8965                        !discovery::is_discovery_endpoint(*ep)
8966                            && (opts.link_local_enabled || !ep.is_link_local_only())
8967                    });
8968                    crate::small_sort::sort(&mut endpoints);
8969                    endpoints.dedup();
8970                    crate::small_sort::sort(&mut variables);
8971                    variables.dedup();
8972                    sources.sort_unstable();
8973                    sources.dedup();
8974                    Some((endpoints, variables, sources, opts.link_capabilities()))
8975                } else {
8976                    None
8977                }
8978            };
8979            let pkt = if let Some((endpoints, variables, sources, capabilities)) = summary {
8980                let address = self.local_address_advertisement(
8981                    endpoints,
8982                    variables,
8983                    sources,
8984                    capabilities,
8985                    discovery::ADDRESS_STATE_REQUEST,
8986                );
8987                discovery::build_discovery_address(sender.as_ref(), now, &address)?
8988            } else {
8989                discovery::build_discovery_announce(sender.as_ref(), now, &[])?
8990            };
8991            self.emit_internal_tx(
8992                RouterTxItem::ToSide {
8993                    src: None,
8994                    dst: side,
8995                    data: RouterItem::Packet(pkt),
8996                },
8997                true,
8998                true,
8999            )?;
9000            self.state
9001                .lock()
9002                .discovery_side_throttle
9003                .entry(side)
9004                .or_default()
9005                .next_ping_ms = now.saturating_add(discovery::DISCOVERY_SLOW_INTERVAL_MS);
9006        }
9007        Ok(())
9008    }
9009
9010    #[cfg(feature = "discovery")]
9011    fn emit_discovery_snapshot(
9012        &self,
9013        called_from_queue: bool,
9014        include_schema: bool,
9015        include_topology: bool,
9016        skip_in_flight_sides: bool,
9017    ) -> TelemetryResult<()> {
9018        self.emit_discovery_snapshot_on_side(
9019            called_from_queue,
9020            include_schema,
9021            include_topology,
9022            skip_in_flight_sides,
9023            None,
9024        )
9025    }
9026
9027    #[cfg(feature = "discovery")]
9028    fn emit_discovery_snapshot_on_side(
9029        &self,
9030        called_from_queue: bool,
9031        include_schema: bool,
9032        include_topology: bool,
9033        skip_in_flight_sides: bool,
9034        requested_side: Option<RouterSideId>,
9035    ) -> TelemetryResult<()> {
9036        self.emit_discovery_snapshot_on_side_claimed(
9037            called_from_queue,
9038            include_schema,
9039            include_topology,
9040            skip_in_flight_sides,
9041            requested_side,
9042            None,
9043        )
9044    }
9045
9046    #[cfg(feature = "discovery")]
9047    fn emit_discovery_snapshot_on_side_claimed(
9048        &self,
9049        called_from_queue: bool,
9050        include_schema: bool,
9051        include_topology: bool,
9052        skip_in_flight_sides: bool,
9053        requested_side: Option<RouterSideId>,
9054        prefix_claim: Option<u64>,
9055    ) -> TelemetryResult<()> {
9056        if requested_side.is_none() {
9057            let count = self.state.lock().sides.len();
9058            let mut first_error = None;
9059            for side in 0..count {
9060                // Release the state lock before recursively emitting on a side.
9061                if self.state.lock().sides[side].is_none() {
9062                    continue;
9063                }
9064                if let Err(error) = self.emit_discovery_snapshot_on_side(
9065                    called_from_queue,
9066                    include_schema,
9067                    include_topology,
9068                    skip_in_flight_sides,
9069                    Some(side),
9070                ) && first_error.is_none()
9071                {
9072                    first_error = Some(error);
9073                }
9074            }
9075            return first_error.map_or(Ok(()), Err);
9076        }
9077        if include_schema {
9078            return self
9079                .start_schema_response_generation(
9080                    requested_side.expect("single discovery side"),
9081                    true,
9082                    called_from_queue,
9083                    include_topology,
9084                    skip_in_flight_sides,
9085                )
9086                .map(|_| ());
9087        }
9088        let (scratch, largest) = {
9089            let now = self.clock.now_ms();
9090            let sender = self.sender_arc();
9091            let st = self.state.lock();
9092            let election_required = self.cfg.role.local_services()
9093                && self
9094                    .preferred_discovery_master_present_locked(&st, now)
9095                    .is_none();
9096            let snapshot = discovery::topology_snapshot_scratch(
9097                sender.as_ref(),
9098                self.cfg.handlers().len(),
9099                st.local_services
9100                    .as_ref()
9101                    .map_or(0, |local| local.managed_variable_types.len())
9102                    .saturating_add(
9103                        st.discovery_routes
9104                            .iter()
9105                            .filter(|(side, _)| Some(**side) != requested_side)
9106                            .map(|(_, r)| r.reachable_network_variables.len())
9107                            .sum::<usize>(),
9108                    ),
9109                4,
9110                true,
9111                st.discovery_routes
9112                    .iter()
9113                    .filter(|(side, route)| {
9114                        (Some(**side) != requested_side)
9115                            && now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9116                    })
9117                    .flat_map(|(_, route)| route.announcers.iter())
9118                    .filter(|(_, peer)| {
9119                        now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9120                    })
9121                    .map(|(name, peer)| {
9122                        (
9123                            name.as_str(),
9124                            peer.reachable.as_slice(),
9125                            peer.reachable_timesync_sources.as_slice(),
9126                            peer.topology_boards.as_slice(),
9127                        )
9128                    }),
9129            );
9130            if !election_required {
9131                snapshot
9132            } else {
9133                let election = discovery::topology_snapshot_scratch(
9134                    sender.as_ref(),
9135                    self.cfg.handlers().len(),
9136                    0,
9137                    2,
9138                    false,
9139                    st.discovery_routes
9140                        .values()
9141                        .filter(|route| {
9142                            now.saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9143                        })
9144                        .flat_map(|route| route.announcers.iter())
9145                        .filter(|(_, peer)| {
9146                            now.saturating_sub(peer.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
9147                        })
9148                        .map(|(name, peer)| {
9149                            (
9150                                name.as_str(),
9151                                peer.reachable.as_slice(),
9152                                peer.reachable_timesync_sources.as_slice(),
9153                                peer.topology_boards.as_slice(),
9154                            )
9155                        }),
9156                );
9157                (snapshot.0.max(election.0), snapshot.1.max(election.1))
9158            }
9159        };
9160        // Full metadata also carries the learned topology. Its working set
9161        // must never bypass the graph estimate merely because schema is due.
9162        let (scratch, largest) = if include_schema {
9163            (scratch.max(8192), largest.max(2048))
9164        } else {
9165            (scratch, largest)
9166        };
9167        // Snapshot construction may clone several route/schema summaries.
9168        // Reclaim ordinary queued telemetry before refusing discovery scratch.
9169        // Priority >= 200 commands, ACKs and metadata stay protected; do not
9170        // evict routes or reliable receive/reassembly state.
9171        while let Err(err) = crate::memory_admission::check(scratch, largest) {
9172            if matches!(err, TelemetryError::Io("memory pressure"))
9173                && crate::memory_admission::check(512, 256).is_ok()
9174            {
9175                let mut st = self.state.lock();
9176                if let Some(victim) = st.lower_priority_shared_queue(199)
9177                    && st.pop_shared_queue_item(victim)
9178                {
9179                    continue;
9180                }
9181            }
9182            if !include_schema && matches!(err, TelemetryError::Io("memory pressure")) {
9183                return self.queue_pressure_keepalives(requested_side);
9184            }
9185            return Err(err);
9186        }
9187        let now_ms = self.clock.now_ms();
9188        let per_side = {
9189            let mut st = self.state.lock();
9190            if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9191                self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9192                Self::note_discovery_topology_change_locked(&mut st, now_ms);
9193            }
9194            st.fit_discovery_budget();
9195            let local_is_master = self.cfg.role.local_services()
9196                && self.discovery_master_sender_locked(&st, now_ms) == self.sender_arc().as_ref();
9197            // The caller selects one side; keep its plan on the stack.
9198            let mut per_side = None;
9199            for side_id in 0..st.sides.len() {
9200                let Some(side) = st.sides[side_id].as_ref() else {
9201                    continue;
9202                };
9203                let opts = side.opts.clone();
9204                let link_local_enabled = opts.link_local_enabled;
9205                if requested_side.is_some_and(|requested| requested != side_id) {
9206                    continue;
9207                }
9208                if let Some(claim) = prefix_claim
9209                    && !st
9210                        .discovery_side_throttle
9211                        .get(&side_id)
9212                        .is_some_and(|throttle| {
9213                            throttle.schema_response_active()
9214                                && throttle.schema_response_phase
9215                                    == SchemaResponsePhase::PrefixBuilding
9216                                && throttle.schema_response_packet_id == claim
9217                        })
9218                {
9219                    return Ok(());
9220                }
9221                let discovery_in_flight = skip_in_flight_sides
9222                    && st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
9223                        *pending_side == side_id
9224                            && !tx_state.sent.is_empty()
9225                            && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
9226                    });
9227                if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
9228                {
9229                    continue;
9230                }
9231                // A lost hop ACK must not make a live bridge disappear from
9232                // its neighbors. Keep the baseline pending, but permit a
9233                // bounded, unreliable keepalive independently of its ACK.
9234                let level = if discovery_in_flight {
9235                    let throttle = st.discovery_side_throttle.entry(side_id).or_default();
9236                    if !throttle.liveness_due(now_ms) {
9237                        None
9238                    } else {
9239                        throttle.next_ping_ms =
9240                            now_ms.saturating_add(DISCOVERY_SLOW_LINK_PING_INTERVAL_MS);
9241                        Some(DiscoveryAdvertiseLevel::MinimalPing)
9242                    }
9243                } else {
9244                    Self::discovery_level_for_side_locked(&mut st, side_id, now_ms)
9245                };
9246                let Some(level) = level else {
9247                    continue;
9248                };
9249                let include_side_topology = include_topology
9250                    && self.route_allowed_locked(
9251                        &st,
9252                        None,
9253                        Some(DataType::DiscoveryTopology),
9254                        side_id,
9255                    );
9256                let capabilities = opts.link_capabilities();
9257                if level == DiscoveryAdvertiseLevel::MinimalPing {
9258                    per_side = Some((
9259                        side_id,
9260                        level,
9261                        Vec::new(),
9262                        Vec::new(),
9263                        Vec::new(),
9264                        Vec::new(),
9265                        Vec::new(),
9266                        capabilities,
9267                        local_is_master,
9268                    ));
9269                    continue;
9270                }
9271                let current_topology = if include_side_topology {
9272                    self.advertised_discovery_topology_for_link_locked(
9273                        &st,
9274                        now_ms,
9275                        link_local_enabled,
9276                        Some(side_id),
9277                    )
9278                } else {
9279                    Vec::new()
9280                };
9281                let (endpoints, timesync_sources) = if include_side_topology {
9282                    self.advertised_summaries_from_topology_locked(
9283                        &st,
9284                        now_ms,
9285                        link_local_enabled,
9286                        Some(side_id),
9287                        &current_topology,
9288                    )
9289                } else {
9290                    (
9291                        self.advertised_discovery_endpoints_for_link_locked(
9292                            &st,
9293                            now_ms,
9294                            link_local_enabled,
9295                            Some(side_id),
9296                        ),
9297                        self.advertised_discovery_timesync_sources_for_link_locked(
9298                            &st,
9299                            now_ms,
9300                            Some(side_id),
9301                        ),
9302                    )
9303                };
9304                let (topology, removed_topology) = match level {
9305                    DiscoveryAdvertiseLevel::Full => (current_topology.clone(), Vec::new()),
9306                    DiscoveryAdvertiseLevel::Incremental => {
9307                        let previous = &st
9308                            .discovery_side_throttle
9309                            .get(&side_id)
9310                            .expect("discovery throttle exists")
9311                            .last_topology;
9312                        discovery::topology_delta(previous, &current_topology)
9313                    }
9314                    DiscoveryAdvertiseLevel::MinimalPing => (Vec::new(), Vec::new()),
9315                };
9316                let throttle = st
9317                    .discovery_side_throttle
9318                    .get_mut(&side_id)
9319                    .expect("discovery throttle exists");
9320                if level == DiscoveryAdvertiseLevel::Full
9321                    || (level == DiscoveryAdvertiseLevel::Incremental
9322                        && throttle.incremental_retries_remaining == 0)
9323                {
9324                    throttle.last_topology = current_topology;
9325                }
9326                per_side = Some((
9327                    side_id,
9328                    level,
9329                    endpoints,
9330                    self.advertised_network_variables_for_link_locked(&st, now_ms, Some(side_id)),
9331                    timesync_sources,
9332                    topology,
9333                    removed_topology,
9334                    capabilities,
9335                    local_is_master,
9336                ));
9337            }
9338            per_side
9339        };
9340        if let Some((
9341            side_id,
9342            level,
9343            endpoints,
9344            network_variables,
9345            timesync_sources,
9346            topology,
9347            removed_topology,
9348            capabilities,
9349            local_is_master,
9350        )) = per_side
9351        {
9352            let sender = self.sender_arc();
9353            if matches!(
9354                level,
9355                DiscoveryAdvertiseLevel::Full | DiscoveryAdvertiseLevel::Incremental
9356            ) && (self.cfg.role.local_services() || level == DiscoveryAdvertiseLevel::Full)
9357            {
9358                let pkt = if self.cfg.role.local_services() {
9359                    let address = self.local_address_advertisement(
9360                        endpoints.clone(),
9361                        network_variables.clone(),
9362                        timesync_sources.clone(),
9363                        capabilities,
9364                        if local_is_master {
9365                            discovery::ADDRESS_STATE_APPROVED
9366                        } else {
9367                            discovery::ADDRESS_STATE_REQUEST
9368                        },
9369                    );
9370                    discovery::build_discovery_address(sender.as_ref(), now_ms, &address)?
9371                } else {
9372                    discovery::build_discovery_link_capabilities(
9373                        sender.as_ref(),
9374                        now_ms,
9375                        capabilities,
9376                    )?
9377                };
9378                self.emit_discovery_prefix_item(
9379                    RouterTxItem::ToSide {
9380                        src: None,
9381                        dst: side_id,
9382                        data: RouterItem::Packet(pkt),
9383                    },
9384                    true,
9385                    called_from_queue,
9386                    prefix_claim.map(|claim| (side_id, claim)),
9387                )?;
9388            }
9389            if level == DiscoveryAdvertiseLevel::MinimalPing
9390                || (!self.cfg.role.local_services()
9391                    && (!endpoints.is_empty() || level != DiscoveryAdvertiseLevel::Full))
9392            {
9393                let pkt = discovery::build_discovery_announce(
9394                    sender.as_ref(),
9395                    now_ms,
9396                    endpoints.as_slice(),
9397                )?;
9398                self.emit_discovery_prefix_item(
9399                    RouterTxItem::ToSide {
9400                        src: None,
9401                        dst: side_id,
9402                        data: RouterItem::Packet(pkt),
9403                    },
9404                    true,
9405                    called_from_queue,
9406                    prefix_claim.map(|claim| (side_id, claim)),
9407                )?;
9408            }
9409            if !self.cfg.role.local_services()
9410                && level != DiscoveryAdvertiseLevel::MinimalPing
9411                && !timesync_sources.is_empty()
9412            {
9413                let pkt = discovery::build_discovery_timesync_sources(
9414                    sender.as_ref(),
9415                    now_ms,
9416                    &timesync_sources,
9417                )?;
9418                self.emit_discovery_prefix_item(
9419                    RouterTxItem::ToSide {
9420                        src: None,
9421                        dst: side_id,
9422                        data: RouterItem::Packet(pkt),
9423                    },
9424                    true,
9425                    called_from_queue,
9426                    prefix_claim.map(|claim| (side_id, claim)),
9427                )?;
9428            }
9429            if include_topology
9430                && (!topology.is_empty() || !removed_topology.is_empty())
9431                && level != DiscoveryAdvertiseLevel::MinimalPing
9432            {
9433                let pkt = if level == DiscoveryAdvertiseLevel::Full {
9434                    discovery::build_discovery_topology(sender.as_ref(), now_ms, &topology)?
9435                } else {
9436                    discovery::build_discovery_topology_delta(
9437                        sender.as_ref(),
9438                        now_ms,
9439                        &topology,
9440                        &removed_topology,
9441                    )?
9442                };
9443                self.emit_discovery_prefix_item(
9444                    RouterTxItem::ToSide {
9445                        src: None,
9446                        dst: side_id,
9447                        data: RouterItem::Packet(pkt),
9448                    },
9449                    true,
9450                    called_from_queue,
9451                    prefix_claim.map(|claim| (side_id, claim)),
9452                )?;
9453            }
9454            // Release the temporary topology snapshot before constructing and
9455            // packing the larger schema. These buffers are no longer needed.
9456            drop((
9457                endpoints,
9458                network_variables,
9459                timesync_sources,
9460                topology,
9461                removed_topology,
9462            ));
9463        }
9464        Ok(())
9465    }
9466
9467    #[cfg(feature = "discovery")]
9468    fn emit_discovery_prefix_item(
9469        &self,
9470        item: RouterTxItem,
9471        ignore_local: bool,
9472        called_from_queue: bool,
9473        claim: Option<(RouterSideId, u64)>,
9474    ) -> TelemetryResult<()> {
9475        let Some((side, claim)) = claim else {
9476            return self.emit_internal_tx(item, ignore_local, called_from_queue);
9477        };
9478        let priority = match &item {
9479            RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
9480            _ => return Err(TelemetryError::BadArg),
9481        };
9482        let mut st = self.state.lock();
9483        if !st
9484            .discovery_side_throttle
9485            .get(&side)
9486            .is_some_and(|throttle| {
9487                throttle.schema_response_active()
9488                    && throttle.schema_response_phase == SchemaResponsePhase::PrefixBuilding
9489                    && throttle.schema_response_packet_id == claim
9490            })
9491        {
9492            return Err(TelemetryError::Io("schema prefix cancelled"));
9493        }
9494        st.push_transmit(TxQueued {
9495            disposable: TX_DISPOSABLE,
9496            item,
9497            ignore_local,
9498            priority,
9499            delivery_state: QueuedDelivery::Untracked,
9500            accepted_type: 0,
9501            deferred_since_ms: 0,
9502            retry_after_ms: 0,
9503        })
9504    }
9505
9506    #[cfg(feature = "discovery")]
9507    fn queue_discovery_announce(
9508        &self,
9509        include_schema: bool,
9510        skip_in_flight_sides: bool,
9511    ) -> TelemetryResult<()> {
9512        let now_ms = self.clock.now_ms();
9513        {
9514            let mut st = self.state.lock();
9515            if Self::prune_discovery_routes_locked(&mut st, now_ms) {
9516                self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9517                Self::note_discovery_topology_change_locked(&mut st, now_ms);
9518            }
9519            st.fit_discovery_budget();
9520            if st.sides.iter().all(|side| side.is_none()) {
9521                return Ok(());
9522            }
9523            // Actual generation claims reset their own prefix state. A
9524            // coalesced explicit announce must not reset an active prefix.
9525            st.discovery_cadence.on_announce_sent(now_ms);
9526        }
9527        self.emit_discovery_snapshot(true, include_schema, true, skip_in_flight_sides)
9528    }
9529
9530    #[cfg(feature = "discovery")]
9531    fn missing_schema_request_targets(
9532        st: &RouterInner,
9533        side: RouterSideId,
9534        now: u64,
9535    ) -> ([u32; discovery::SCHEMA_REQUEST_MAX_TARGETS], usize, bool) {
9536        let mut targets = [0u32; discovery::SCHEMA_REQUEST_MAX_TARGETS];
9537        let mut count = 0;
9538        let mut legacy_broadcast = false;
9539        let cursor = st
9540            .discovery_side_throttle
9541            .get(&side)
9542            .map_or(0, |v| v.schema_request_cursor);
9543        let order = |address: u32| {
9544            let delta = address.wrapping_sub(cursor);
9545            if delta == 0 {
9546                1u64 << 32
9547            } else {
9548                u64::from(delta)
9549            }
9550        };
9551        if let Some(route) = st.discovery_routes.get(&side) {
9552            for (name, peer) in &route.announcers {
9553                if !peer.has_full_topology
9554                    || peer.has_schema
9555                    || now.saturating_sub(peer.last_seen_ms) >= DISCOVERY_ROUTE_TTL_MS
9556                {
9557                    continue;
9558                }
9559                if st.side_transport.get(&side).is_some_and(|transport| {
9560                    transport.rx_chunks.values().any(|assembly| {
9561                        assembly.schema_progress_for(sender_address_u32(name), now)
9562                            || peer
9563                                .advertised_address
9564                                .is_some_and(|a| assembly.schema_progress_for(a, now))
9565                    })
9566                }) {
9567                    continue;
9568                }
9569                let address = peer
9570                    .advertised_address
9571                    .filter(|a| *a != 0)
9572                    .unwrap_or_else(|| sender_address_u32(name));
9573                if address == 0 {
9574                    // Numeric zero aliases have no safe scope. Preserve legacy
9575                    // recovery for them without poisoning addressable peers.
9576                    legacy_broadcast = true;
9577                    continue;
9578                }
9579                if targets[..count].contains(&address) {
9580                    continue;
9581                }
9582                let position = targets[..count]
9583                    .iter()
9584                    .position(|old| order(address) < order(*old))
9585                    .unwrap_or(count);
9586                if position >= targets.len() {
9587                    continue;
9588                }
9589                let next = (count + 1).min(targets.len());
9590                for i in ((position + 1)..next).rev() {
9591                    targets[i] = targets[i - 1];
9592                }
9593                targets[position] = address;
9594                count = next;
9595            }
9596        }
9597        (targets, count, legacy_broadcast)
9598    }
9599
9600    #[cfg(feature = "discovery")]
9601    fn note_schema_received(&self, side: RouterSideId, sender: &str) {
9602        let mut st = self.state.lock();
9603        // Schema snapshots can be forwarded. They must never introduce a new
9604        // link-local announcer or change shared-bus reliability semantics.
9605        // If schema precedes topology, bounded recovery will request it again.
9606        let address = sender
9607            .strip_prefix("@addr:")
9608            .and_then(|value| value.parse::<u32>().ok());
9609        if let Some(route) = st.discovery_routes.get_mut(&side) {
9610            for (name, peer) in &mut route.announcers {
9611                if name == sender
9612                    || address.is_some_and(|address| {
9613                        peer.advertised_address == Some(address)
9614                            || sender_address_u32(name) == address
9615                    })
9616                {
9617                    peer.has_schema = true;
9618                    break;
9619                }
9620            }
9621        }
9622    }
9623
9624    /// Claim one generation before building either its prefix or catalog.
9625    /// The phase byte fits existing throttle padding; no payload lives here.
9626    #[cfg(feature = "discovery")]
9627    fn start_schema_response_generation(
9628        &self,
9629        side: RouterSideId,
9630        explicit: bool,
9631        called_from_queue: bool,
9632        include_topology: bool,
9633        skip_in_flight_sides: bool,
9634    ) -> TelemetryResult<bool> {
9635        let now = self.clock.now_ms();
9636        let claim = {
9637            let mut st = self.state.lock();
9638            let throttle = st.discovery_side_throttle.entry(side).or_default();
9639            if throttle.schema_response_active() {
9640                if explicit {
9641                    throttle.set_schema_response_pending(true);
9642                }
9643                return Ok(false);
9644            }
9645            if !explicit
9646                && (!throttle.schema_response_pending() || now < throttle.next_schema_response_ms)
9647            {
9648                return Ok(false);
9649            }
9650            let claim = st
9651                .schema_response_sequence
9652                .checked_add(1)
9653                .ok_or(TelemetryError::Io("schema generation exhausted"))?;
9654            st.schema_response_sequence = claim;
9655            let throttle = st
9656                .discovery_side_throttle
9657                .get_mut(&side)
9658                .expect("schema throttle");
9659            throttle.set_schema_response_pending(false);
9660            throttle.set_schema_response_active(true);
9661            throttle.schema_response_phase = SchemaResponsePhase::PrefixBuilding;
9662            throttle.schema_response_packet_id = claim;
9663            throttle.set_schema_response_started(false);
9664            throttle.next_schema_response_ms =
9665                now.saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
9666            throttle.set_has_sent_full(false);
9667            throttle.last_topology = Vec::new();
9668            if let Some(transport) = st.side_transport.get_mut(&side) {
9669                transport.refresh_transmit_header_templates();
9670            }
9671            claim
9672        };
9673        // Admit the route prefix exactly once before waiting for a catalog
9674        // descriptor. Refused prefixes retry as prefixes; catalog refusal must
9675        // never recreate accepted Address/Topology and fill the small ring.
9676        let prefix = self.emit_discovery_snapshot_on_side_claimed(
9677            called_from_queue,
9678            false,
9679            include_topology,
9680            skip_in_flight_sides,
9681            Some(side),
9682            Some(claim),
9683        );
9684        {
9685            let mut st = self.state.lock();
9686            let throttle = st
9687                .discovery_side_throttle
9688                .get_mut(&side)
9689                .ok_or(TelemetryError::BadArg)?;
9690            if !throttle.schema_response_active()
9691                || throttle.schema_response_phase != SchemaResponsePhase::PrefixBuilding
9692                || throttle.schema_response_packet_id != claim
9693            {
9694                return Ok(false);
9695            }
9696            if let Err(error) = prefix {
9697                throttle.set_schema_response_active(false);
9698                throttle.schema_response_phase = SchemaResponsePhase::Idle;
9699                throttle.set_schema_response_pending(true);
9700                return Err(error);
9701            }
9702            if !throttle.has_sent_full() {
9703                // Liveness-only links still do not acquire a schema owner.
9704                throttle.set_schema_response_active(false);
9705                throttle.schema_response_phase = SchemaResponsePhase::Idle;
9706                return Ok(true);
9707            }
9708            throttle.schema_response_packet_id = 0;
9709            throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
9710        }
9711        self.admit_schema_response_catalog(side)?;
9712        Ok(true)
9713    }
9714
9715    #[cfg(feature = "discovery")]
9716    fn admit_schema_response_catalog(&self, side: RouterSideId) -> TelemetryResult<bool> {
9717        self.admit_schema_response_catalog_with(side, || {
9718            discovery::build_discovery_schema(self.sender_arc().as_ref(), self.clock.now_ms())
9719        })
9720    }
9721
9722    #[cfg(feature = "discovery")]
9723    fn admit_schema_response_catalog_with(
9724        &self,
9725        side: RouterSideId,
9726        build: impl FnOnce() -> TelemetryResult<Packet>,
9727    ) -> TelemetryResult<bool> {
9728        let claim = {
9729            let mut st = self.state.lock();
9730            let throttle = st
9731                .discovery_side_throttle
9732                .get(&side)
9733                .ok_or(TelemetryError::BadArg)?;
9734            if !throttle.schema_response_active()
9735                || throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting
9736            {
9737                return Ok(false);
9738            }
9739            let protected_full = st.transmit_queue.len() >= st.transmit_queue.capacity()
9740                && !RouterInner::transmit_eviction_candidate(&st.transmit_queue)
9741                    .and_then(|index| st.transmit_queue.get(index))
9742                    .is_some_and(|queued| {
9743                        queued.priority < crate::transport_priority(DataType::DiscoverySchema)
9744                    });
9745            if protected_full
9746                || !self.route_allowed_locked(&st, None, Some(DataType::DiscoverySchema), side)
9747            {
9748                return Ok(false);
9749            }
9750            let claim = st
9751                .schema_response_sequence
9752                .checked_add(1)
9753                .ok_or(TelemetryError::Io("schema generation exhausted"))?;
9754            st.schema_response_sequence = claim;
9755            let throttle = st
9756                .discovery_side_throttle
9757                .get_mut(&side)
9758                .expect("schema throttle");
9759            throttle.schema_response_phase = SchemaResponsePhase::CatalogBuilding;
9760            throttle.schema_response_packet_id = claim;
9761            claim
9762        };
9763        // Keep the existing full-schema working-set guard; allocation and
9764        // probe callbacks run outside the state lock. Stale builders cannot
9765        // publish over a clear/restart, even if its phase has cycled back.
9766        let built = crate::memory_admission::check(8192, 2048).and_then(|_| build());
9767        let prepared = built.and_then(|packet| {
9768            let item = RouterTxItem::ToSide {
9769                src: None,
9770                dst: side,
9771                data: RouterItem::Packet(packet),
9772            };
9773            let priority = match &item {
9774                RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
9775                _ => unreachable!(),
9776            };
9777            Ok((item, priority))
9778        });
9779        let mut st = self.state.lock();
9780        let throttle = st
9781            .discovery_side_throttle
9782            .get(&side)
9783            .ok_or(TelemetryError::BadArg)?;
9784        if !throttle.schema_response_active()
9785            || throttle.schema_response_phase != SchemaResponsePhase::CatalogBuilding
9786            || throttle.schema_response_packet_id != claim
9787        {
9788            return Ok(false);
9789        }
9790        // Ownership publication and existing bounded queue admission share one
9791        // lock. There is no clear-then-orphan gap between these operations.
9792        let result = prepared.and_then(|(item, priority)| {
9793            let packet_id = Self::get_hash(match &item {
9794                RouterTxItem::ToSide { data, .. } => data,
9795                _ => unreachable!(),
9796            });
9797            let throttle = st
9798                .discovery_side_throttle
9799                .get_mut(&side)
9800                .expect("schema throttle");
9801            throttle.schema_response_packet_id = packet_id;
9802            throttle.schema_response_phase = SchemaResponsePhase::Owned;
9803            st.push_transmit(TxQueued {
9804                disposable: TX_DISPOSABLE,
9805                item,
9806                ignore_local: true,
9807                priority,
9808                delivery_state: QueuedDelivery::Untracked,
9809                accepted_type: 0,
9810                deferred_since_ms: 0,
9811                retry_after_ms: 0,
9812            })
9813        });
9814        let throttle = st
9815            .discovery_side_throttle
9816            .get_mut(&side)
9817            .expect("schema throttle");
9818        match result {
9819            Ok(()) => Ok(true),
9820            Err(TelemetryError::Io(_)) => {
9821                throttle.schema_response_packet_id = 0;
9822                throttle.schema_response_phase = SchemaResponsePhase::CatalogWaiting;
9823                Ok(false)
9824            }
9825            Err(error) => {
9826                throttle.set_schema_response_active(false);
9827                throttle.schema_response_phase = SchemaResponsePhase::Idle;
9828                throttle.set_schema_response_pending(true);
9829                Err(error)
9830            }
9831        }
9832    }
9833
9834    #[cfg(feature = "discovery")]
9835    fn poll_schema_response_on_side(&self, side: RouterSideId) -> TelemetryResult<bool> {
9836        let waiting = {
9837            let st = self.state.lock();
9838            let Some(throttle) = st.discovery_side_throttle.get(&side) else {
9839                return Ok(false);
9840            };
9841            if throttle.schema_response_active() {
9842                if throttle.schema_response_phase != SchemaResponsePhase::CatalogWaiting {
9843                    return Ok(false);
9844                }
9845                true
9846            } else {
9847                if !throttle.schema_response_pending()
9848                    || self.clock.now_ms() < throttle.next_schema_response_ms
9849                {
9850                    return Ok(false);
9851                }
9852                // A pre-existing owner may be an explicit startup catalog.
9853                if st.transmit_queue.iter().any(|entry| {
9854                    matches!(&entry.item,
9855                    RouterTxItem::ToSide {dst, data, ..} if *dst == side
9856                    && Self::item_data_type(data).ok() == Some(DataType::DiscoverySchema))
9857                }) {
9858                    return Ok(false);
9859                }
9860                false
9861            }
9862        };
9863        if waiting {
9864            self.admit_schema_response_catalog(side)
9865        } else {
9866            self.start_schema_response_generation(side, false, true, true, false)
9867        }
9868    }
9869
9870    #[cfg(feature = "discovery")]
9871    fn poll_discovery_announce(&self) -> TelemetryResult<bool> {
9872        let now_ms = self.clock.now_ms();
9873        // An address/keepalive proves liveness, not receipt of the detailed
9874        // baseline. Recover a lost bootstrap without waiting 120 seconds or
9875        // discarding existing routes. Coalesce all missing peers on one side.
9876        let recovery_sides = {
9877            let mut st = self.state.lock();
9878            let mut due = Vec::new();
9879            for side in 0..st.sides.len() {
9880                let missing = st.discovery_routes.get(&side).is_some_and(|route| {
9881                    route.announcers.iter().any(|(name, peer)| {
9882                        !peer.has_full_topology
9883                            && now_ms.saturating_sub(peer.last_seen_ms) < DISCOVERY_ROUTE_TTL_MS
9884                            && !route.announcers.values().any(|bridge| {
9885                                bridge.has_full_topology
9886                                    && now_ms.saturating_sub(bridge.last_seen_ms)
9887                                        < DISCOVERY_ROUTE_TTL_MS
9888                                    && bridge
9889                                        .topology_boards
9890                                        .iter()
9891                                        .any(|board| board.sender_id == *name)
9892                            })
9893                    })
9894                });
9895                if !missing {
9896                    continue;
9897                }
9898                let throttle = st.discovery_side_throttle.entry(side).or_default();
9899                if throttle.next_topology_request_ms == 0 {
9900                    throttle.next_topology_request_ms =
9901                        now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
9902                } else if now_ms >= throttle.next_topology_request_ms {
9903                    throttle.next_topology_request_ms =
9904                        now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
9905                    due.push(side);
9906                }
9907            }
9908            due
9909        };
9910        let recovering = !recovery_sides.is_empty();
9911        for side in recovery_sides {
9912            let pkt =
9913                discovery::build_discovery_topology_request(self.sender_arc().as_ref(), now_ms)?;
9914            self.emit_internal_tx(
9915                RouterTxItem::ToSide {
9916                    src: None,
9917                    dst: side,
9918                    data: RouterItem::Packet(pkt),
9919                },
9920                true,
9921                true,
9922            )?;
9923        }
9924        let schema_recovery_sides = {
9925            let mut st = self.state.lock();
9926            let mut due = Vec::new();
9927            for side in 0..st.sides.len() {
9928                // Eligibility cannot change a future grace/retry deadline. Avoid
9929                // scanning/sorting all announcers on the embedded poll hot path.
9930                if st
9931                    .discovery_side_throttle
9932                    .get(&side)
9933                    .is_some_and(|throttle| {
9934                        throttle.next_schema_request_ms != 0
9935                            && now_ms < throttle.next_schema_request_ms
9936                    })
9937                {
9938                    continue;
9939                }
9940                let (_, count, legacy) = Self::missing_schema_request_targets(&st, side, now_ms);
9941                let missing = count != 0 || legacy;
9942                if !missing {
9943                    continue;
9944                }
9945                let throttle = st.discovery_side_throttle.entry(side).or_default();
9946                if throttle.next_schema_request_ms == 0 {
9947                    throttle.next_schema_request_ms =
9948                        now_ms.saturating_add(discovery::DISCOVERY_BASELINE_GRACE_MS);
9949                } else if now_ms >= throttle.next_schema_request_ms {
9950                    throttle.next_schema_request_ms =
9951                        now_ms.saturating_add(discovery::DISCOVERY_BASELINE_RETRY_MS);
9952                    due.push(side);
9953                }
9954            }
9955            due
9956        };
9957        let recovering = recovering || !schema_recovery_sides.is_empty();
9958        for side in schema_recovery_sides {
9959            let (addresses, count, legacy) = {
9960                let st = self.state.lock();
9961                Self::missing_schema_request_targets(&st, side, now_ms)
9962            };
9963            if count == 0 && !legacy {
9964                continue;
9965            }
9966            let pkt = if legacy {
9967                discovery::build_discovery_schema_request(self.sender_arc().as_ref(), now_ms)?
9968            } else {
9969                discovery::build_scoped_discovery_schema_request(
9970                    self.sender_arc().as_ref(),
9971                    now_ms,
9972                    &addresses[..count],
9973                )?
9974            };
9975            self.emit_internal_tx(
9976                RouterTxItem::ToSide {
9977                    src: None,
9978                    dst: side,
9979                    data: RouterItem::Packet(pkt),
9980                },
9981                true,
9982                true,
9983            )?;
9984            if !legacy
9985                && let Some(throttle) = self.state.lock().discovery_side_throttle.get_mut(&side)
9986            {
9987                throttle.schema_request_cursor = addresses[count - 1];
9988            }
9989        }
9990        let due = {
9991            let mut st = self.state.lock();
9992            let removed = Self::prune_discovery_routes_locked(&mut st, now_ms);
9993            if removed {
9994                self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
9995                Self::note_discovery_topology_change_locked(&mut st, now_ms);
9996            }
9997            st.fit_discovery_budget();
9998            let has_any = st.sides.iter().enumerate().any(|(side_id, side)| {
9999                let Some(side) = side.as_ref() else {
10000                    return false;
10001                };
10002                if !self.route_allowed_locked(&st, None, Some(DataType::DiscoveryAnnounce), side_id)
10003                {
10004                    return false;
10005                }
10006                let _ = side;
10007                let in_flight = st.reliable_tx.iter().any(|((pending_side, ty), tx_state)| {
10008                    *pending_side == side_id
10009                        && !tx_state.sent.is_empty()
10010                        && DataType::try_from_u32(*ty).is_some_and(discovery::is_discovery_type)
10011                });
10012                st.discovery_side_throttle
10013                    .get(&side_id)
10014                    .is_none_or(|throttle| {
10015                        (!in_flight
10016                            && (!throttle.has_sent_full() || throttle.pending_incremental()))
10017                            || throttle.liveness_due(now_ms)
10018                    })
10019            });
10020            if st.sides.is_empty() || !has_any {
10021                return Ok(recovering);
10022            }
10023            st.discovery_cadence.due(now_ms)
10024        };
10025        if !due {
10026            return Ok(recovering);
10027        }
10028        // Runtime liveness/topology refreshes must stay small. A full schema
10029        // is sent by the initial explicit announce and in response to
10030        // DiscoverySchemaRequest; repeating it at every cadence can starve
10031        // commands on constrained serial and radio links.
10032        // A stalled reliable link must not prevent freshly learned routes
10033        // from propagating across another healthy side. Only omit the side
10034        // that already has discovery awaiting its hop ACK, except for bounded
10035        // liveness pings that must not depend on that ACK arriving.
10036        self.queue_discovery_announce(false, true)?;
10037        Ok(true)
10038    }
10039
10040    #[cfg(feature = "discovery")]
10041    fn learn_discovery_packet(
10042        &self,
10043        pkt: &Packet,
10044        src: Option<RouterSideId>,
10045        called_from_queue: bool,
10046    ) -> TelemetryResult<bool> {
10047        if !discovery::is_discovery_type(pkt.data_type()) {
10048            return Ok(false);
10049        }
10050        let Some(side) = src else {
10051            return Ok(true);
10052        };
10053        #[cfg(feature = "compression")]
10054        self.note_payload_codec_packet(side, pkt, false);
10055        if self.refresh_known_discovery_liveness(pkt, side)? {
10056            return Ok(true);
10057        }
10058        #[cfg(not(feature = "std"))]
10059        if pkt.data_type() == crate::DataType::DiscoverySchema {
10060            self.merge_schema_changes(pkt.payload())?;
10061            self.note_schema_received(side, pkt.sender());
10062            return Ok(true);
10063        }
10064        if pkt.data_type() == DataType::DiscoverySchemaRequest {
10065            if let Some(targets) = discovery::schema_request_targets(pkt.payload()) {
10066                let assigned = self.state.lock().local_address.address;
10067                let legacy = sender_address_u32(self.sender_arc().as_ref());
10068                if !(0..targets.len() / 4).any(|index| {
10069                    let bytes = &targets[index * 4..index * 4 + 4];
10070                    let address =
10071                        u32::from_le_bytes(bytes.try_into().expect("validated address width"));
10072                    address == assigned || address == legacy
10073                }) {
10074                    // Exclusion only affects THIS request; an accepted pending/inflight
10075                    // generation and its retained owner are deliberately untouched.
10076                    return Ok(true);
10077                }
10078            }
10079            self.state
10080                .lock()
10081                .discovery_side_throttle
10082                .entry(side)
10083                .or_default()
10084                .set_schema_response_pending(true);
10085            self.poll_schema_response_on_side(side)?;
10086            return Ok(true);
10087        }
10088        // A compact address summary does not clone the complete topology or
10089        // schema. Account for its ingress-side aggregates instead; charging
10090        // every new peer for a full snapshot can prevent restart recovery.
10091        if matches!(
10092            pkt.data_type(),
10093            DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
10094        ) {
10095            crate::memory_admission::check(1024, 256)?;
10096            // A requesting peer may have restarted with an empty RX dictionary.
10097            // Its identity and routes need not change. Refresh only this link
10098            // so the next data/state report is immediately self-describing;
10099            // retain RX dictionaries and all learned topology.
10100            if let Some(transport) = self.state.lock().side_transport.get_mut(&side) {
10101                transport.refresh_transmit_header_templates();
10102            }
10103        }
10104        if matches!(
10105            pkt.data_type(),
10106            DataType::DiscoveryTopologyRequest | DataType::DiscoverySchemaRequest
10107        ) {
10108            let now_ms = self.clock.now_ms();
10109            {
10110                let mut st = self.state.lock();
10111                if Self::prune_discovery_routes_locked(&mut st, now_ms) {
10112                    self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10113                    Self::note_discovery_topology_change_locked(&mut st, now_ms);
10114                }
10115                let throttle = st.discovery_side_throttle.entry(side).or_default();
10116                throttle.set_has_sent_full(false);
10117                // A full reply cannot use the old incremental baseline. Drop
10118                // that derived copy before admitting its replacement scratch;
10119                // keep all learned routes, schemas and other links intact.
10120                throttle.last_topology = Vec::new();
10121            }
10122            // Do not depend on the elected master answering a forwarded request.
10123            // A recovering neighbor must get our retained downstream baseline
10124            // even before its identity is known. Reply only on its ingress link;
10125            // do not reset other links or broadcast application traffic.
10126            self.emit_discovery_snapshot_on_side(
10127                called_from_queue,
10128                pkt.data_type() == DataType::DiscoverySchemaRequest,
10129                true,
10130                false,
10131                Some(side),
10132            )?;
10133            return Ok(true);
10134        }
10135        let small_topology_scratch =
10136            if pkt.data_type() == DataType::DiscoveryTopology && pkt.payload().len() <= 128 {
10137                let st = self.state.lock();
10138                let sender = pkt
10139                    .sender()
10140                    .strip_prefix("@addr:")
10141                    .and_then(|value| value.parse::<u32>().ok())
10142                    .and_then(|address| Self::hostname_for_address_borrowed_locked(&st, address))
10143                    .unwrap_or(pkt.sender());
10144                st.discovery_routes.get(&side).and_then(|route| {
10145                    let peer = route.announcers.get(sender)?;
10146                    let mut shallow = discovery::CloneScratch::default();
10147                    shallow.buffer(route.reachable.len().saturating_mul(4));
10148                    shallow.buffer(route.reachable_network_variables.len().saturating_mul(4));
10149                    shallow.buffer(
10150                        route
10151                            .requested_network_variables
10152                            .len()
10153                            .saturating_mul(core::mem::size_of::<ManagedVariableSubscription>()),
10154                    );
10155                    shallow.strings(&route.reachable_timesync_sources);
10156                    for (name, value) in &route.announcers {
10157                        shallow.buffer(name.len());
10158                        shallow.buffer(value.reachable.len().saturating_mul(4));
10159                        shallow.buffer(value.advertised_reachable.len().saturating_mul(4));
10160                        shallow.strings(&value.reachable_timesync_sources);
10161                        shallow.strings(&value.advertised_reachable_timesync_sources);
10162                        shallow.buffer(value.reachable_network_variables.len().saturating_mul(4));
10163                    }
10164                    let entry = core::mem::size_of::<String>()
10165                        + crate::peer_record::allocation_bytes::<DiscoverySenderState>();
10166                    shallow.bytes = shallow
10167                        .bytes
10168                        .saturating_add(route.announcers.len().saturating_mul(entry));
10169                    shallow.largest = shallow
10170                        .largest
10171                        .max(entry.saturating_mul(11).saturating_add(128));
10172                    let mut deep = discovery::CloneScratch::default();
10173                    if discovery::topology_payload_is_incremental(pkt.payload()) {
10174                        deep.graph(&peer.topology_boards, pkt.payload().len() / 16);
10175                    }
10176                    Some((
10177                        2048usize
10178                            .saturating_add(shallow.bytes.saturating_mul(3))
10179                            .saturating_add(deep.bytes)
10180                            .saturating_add(pkt.payload().len().saturating_mul(4)),
10181                        shallow.largest.max(deep.largest).max(2048),
10182                    ))
10183                })
10184            } else {
10185                None
10186            };
10187        let retained_summary = if small_topology_scratch.is_none()
10188            && matches!(
10189                pkt.data_type(),
10190                DataType::DiscoveryAddress
10191                    | DataType::DiscoveryTopology
10192                    | DataType::DiscoveryAnnounce
10193                    | DataType::DiscoveryTimeSyncSources
10194                    | DataType::ManagedVariableRequest
10195            ) {
10196            self.state
10197                .lock()
10198                .discovery_routes
10199                .get(&side)
10200                .map_or(0, |route| {
10201                    RouterInner::discovery_route_byte_cost(side, route)
10202                })
10203        } else {
10204            0
10205        };
10206        let (decode_bytes, largest_block) = if let Some(scratch) = small_topology_scratch {
10207            scratch
10208        } else if pkt.data_type() == DataType::ManagedVariableRequest {
10209            let ty = discovery::decode_managed_variable_request(pkt)?;
10210            let existing = self
10211                .state
10212                .lock()
10213                .discovery_routes
10214                .get(&side)
10215                .is_some_and(|route| {
10216                    route
10217                        .requested_network_variables
10218                        .iter()
10219                        .any(|request| request.ty == ty)
10220                });
10221            if existing {
10222                (1024, 256)
10223            } else {
10224                // New subscriptions clone only this ingress summary; refreshing
10225                // an existing lease constructs no graph or schema definitions.
10226                (
10227                    2048usize.saturating_add(retained_summary.saturating_mul(2)),
10228                    retained_summary
10229                        .checked_next_power_of_two()
10230                        .unwrap_or(usize::MAX)
10231                        .max(1024),
10232                )
10233            }
10234        } else {
10235            let scratch = if pkt.data_type() == DataType::DiscoveryAddress {
10236                4096usize.saturating_add(retained_summary)
10237            } else {
10238                8192usize.saturating_add(retained_summary)
10239            };
10240            let decode_bytes = pkt
10241                .payload()
10242                .len()
10243                .saturating_mul(4)
10244                .saturating_add(scratch);
10245            let largest_block = pkt.payload().len().saturating_mul(2).max(2048);
10246            (decode_bytes, largest_block)
10247        };
10248        while let Err(err) = crate::memory_admission::check(decode_bytes, largest_block) {
10249            if !matches!(err, TelemetryError::Io("memory pressure")) {
10250                return Err(err);
10251            }
10252            let mut st = self.state.lock();
10253            #[cfg(feature = "compression")]
10254            if st.reclaim_codec_metadata() {
10255                continue;
10256            }
10257
10258            let Some(victim) = st.lower_priority_shared_queue(255) else {
10259                return Err(err);
10260            };
10261            if !st.pop_shared_queue_item(victim) {
10262                return Err(err);
10263            }
10264        }
10265        if pkt.data_type() == DataType::DiscoveryAddress {
10266            let mut ad = discovery::decode_discovery_address(pkt)?;
10267            let identity = ad.clone();
10268            let mut changed = false;
10269            /* Packed no_std frames represent their source as @addr:N until
10270             * discovery resolves the address. DiscoveryAddress carries the
10271             * authoritative hostname in its payload, so key reachability by
10272             * that name immediately instead of permanently treating the
10273             * placeholder as a distinct router. */
10274            let sender_id = if ad.hostname.is_empty() {
10275                pkt.sender()
10276            } else {
10277                ad.hostname.as_str()
10278            };
10279            let now_ms = self.clock.now_ms();
10280            let mut st = self.state.lock();
10281            let side_link_local_enabled = st
10282                .sides
10283                .get(side)
10284                .and_then(|entry| entry.as_ref())
10285                .map(|side_ref| side_ref.opts.link_local_enabled)
10286                .unwrap_or(false);
10287            if !side_link_local_enabled {
10288                ad.reachable_endpoints.retain(|ep| !ep.is_link_local_only());
10289            }
10290            let mut route = st
10291                .discovery_routes
10292                .get(&side)
10293                .map(DiscoverySideState::try_clone)
10294                .transpose()?
10295                .unwrap_or_default();
10296            let previously_reachable_network_variables = route.reachable_network_variables.clone();
10297            if pkt.sender() != sender_id {
10298                route.announcers.remove(pkt.sender());
10299            }
10300            let mut sender_state = match route.announcers.remove(sender_id) {
10301                Some(record) => record,
10302                None => crate::peer_record::try_box(DiscoverySenderState::default())?,
10303            };
10304            // DiscoveryAddress is a compact route summary. A multi-sided
10305            // router advertises endpoints reachable *through* it, so these
10306            // entries must not be attached to the announcer's topology node:
10307            // that would falsely claim ownership and freeze reliable packets
10308            // to the bridge before detailed topology arrives.
10309            if sender_state.advertised_address != Some(ad.address) {
10310                sender_state.advertised_address = Some(ad.address);
10311                changed = true;
10312            }
10313            if sender_state.advertised_reachable != ad.reachable_endpoints {
10314                sender_state.advertised_reachable = ad.reachable_endpoints;
10315                changed = true;
10316            }
10317            if sender_state.advertised_reachable_timesync_sources != ad.reachable_timesync_sources {
10318                sender_state.advertised_reachable_timesync_sources = ad.reachable_timesync_sources;
10319                changed = true;
10320            }
10321            if sender_state.reachable_network_variables != ad.reachable_network_variables {
10322                sender_state.reachable_network_variables = ad.reachable_network_variables;
10323                changed = true;
10324            }
10325            if sender_state.link_capabilities != Some(ad.link_capabilities) {
10326                sender_state.link_capabilities = Some(ad.link_capabilities);
10327                changed = true;
10328            }
10329            Self::refresh_sender_topology_state(&mut sender_state);
10330            sender_state.last_seen_ms = now_ms;
10331            route.announcers.insert(sender_id.to_string(), sender_state);
10332            Self::recompute_discovery_side_state(&mut route);
10333            let newly_reachable_network_variables: Vec<DataType> = route
10334                .reachable_network_variables
10335                .iter()
10336                .copied()
10337                .filter(|ty| !previously_reachable_network_variables.contains(ty))
10338                .collect();
10339            let prepared_address = Self::prepare_address_record(&st, &identity, now_ms)?;
10340            st.commit_discovery_route(side, route)?;
10341            self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10342            if changed {
10343                Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
10344            }
10345            // A writer may have changed a managed variable while no remote
10346            // owner route was present. Local-only suppression correctly kept
10347            // that update off unrelated links, but the new owner still needs
10348            // the latest value. Replay only newly reachable variable types to
10349            // this advertising side; do not fan out or restart discovery.
10350            let replay: Vec<Packet> = newly_reachable_network_variables
10351                .iter()
10352                .filter(|ty| Self::managed_variable_permissions_locked(&st, **ty).write)
10353                .filter_map(|ty| {
10354                    st.local_services
10355                        .as_ref()?
10356                        .managed_variable_latest
10357                        .get(&ty.as_u32())
10358                        .map(|entry| entry.packet.clone())
10359                })
10360                .collect();
10361            drop(st);
10362            self.ingest_prepared_address(identity, prepared_address)?;
10363            #[cfg(feature = "compression")]
10364            self.note_payload_codec_packet(side, pkt, true);
10365            for value in replay {
10366                self.emit_internal_tx_with_priority(
10367                    RouterTxItem::ToSide {
10368                        src: None,
10369                        dst: side,
10370                        data: RouterItem::Packet(value),
10371                    },
10372                    true,
10373                    crate::transport_priority(DataType::ManagedVariableValue),
10374                    called_from_queue,
10375                )?;
10376            }
10377            return Ok(true);
10378        }
10379        let mut decoded_topology = if pkt.data_type() == DataType::DiscoveryTopology {
10380            Some(discovery::decode_discovery_topology_update(pkt)?)
10381        } else {
10382            None
10383        };
10384        let packet_sender = {
10385            let st = self.state.lock();
10386            let canonical = Self::canonical_sender_locked(&st, pkt.sender());
10387            if canonical != pkt.sender() {
10388                canonical
10389            } else if let Some(address) = pkt
10390                .sender()
10391                .strip_prefix("@addr:")
10392                .and_then(|value| value.parse::<u32>().ok())
10393            {
10394                // A compact topology frame may arrive before its reliable
10395                // DiscoveryAddress frame. Topology still carries stable board
10396                // names, so use the entry whose wire hash matches the compact
10397                // source address. This keeps autonomous routers keyed by name
10398                // even when the header omits that hostname.
10399                decoded_topology
10400                    .as_ref()
10401                    .and_then(|update| {
10402                        update
10403                            .boards
10404                            .iter()
10405                            .find(|board| sender_address_u32(&board.sender_id) == address)
10406                    })
10407                    .map(|board| board.sender_id.clone())
10408                    .unwrap_or(canonical)
10409            } else {
10410                canonical
10411            }
10412        };
10413        let local_sender = self.sender_arc();
10414        if packet_sender == local_sender.as_ref() {
10415            return Ok(true);
10416        }
10417
10418        if pkt.data_type() == DataType::ManagedVariableRequest {
10419            let ty = discovery::decode_managed_variable_request(pkt)?;
10420            // A refresh request is direct evidence that this ingress segment
10421            // contains a subscriber for the variable. Track one bounded entry
10422            // per type and side rather than manufacturing a full discovery
10423            // announcer for every requester. This keeps later writes selective
10424            // without growing the topology graph or fragmenting embedded pools.
10425            {
10426                let now_ms = self.clock.now_ms();
10427                let mut st = self.state.lock();
10428                // Read-only replicas still forward GET toward a writer. The
10429                // grant names only the existing cached observation and its
10430                // actual upstream; it cannot manufacture a cached response.
10431                let grant = st.local_services.as_ref().and_then(|local| {
10432                    if Self::managed_variable_permissions_locked(&st, ty).write {
10433                        return None;
10434                    }
10435                    let entry = local.managed_variable_latest.get(&ty.as_u32())?;
10436                    let value_id = entry.packet.packet_id();
10437                    let upstream = entry.upstream?.get() - 1;
10438                    (upstream != side
10439                        && Self::side_ref(&st, upstream)
10440                            .is_ok_and(|link| link.opts.ingress_enabled))
10441                    .then_some((value_id, upstream, entry.sender_address))
10442                });
10443                let mut request = ManagedVariableSubscription {
10444                    last_seen_ms: now_ms,
10445                    next_refresh_ms: 0,
10446                    request_id: pkt.packet_id(),
10447                    value_id: 0,
10448                    upstream: side,
10449                    ty,
10450                    requester: sender_address_u32(pkt.sender()),
10451                    writer: 0,
10452                    refresh_pending: false,
10453                };
10454                if let Some((value_id, upstream, writer)) = grant {
10455                    request.value_id = value_id;
10456                    request.upstream = upstream;
10457                    request.writer = writer;
10458                    request.refresh_pending = true;
10459                }
10460                let existing = st.discovery_routes.get_mut(&side).and_then(|route| {
10461                    let previous = route
10462                        .requested_network_variables
10463                        .iter_mut()
10464                        .find(|previous| previous.ty == ty)?;
10465                    let repeated = previous.request_id == request.request_id
10466                        && previous.requester == request.requester;
10467                    let backstep = now_ms < previous.last_seen_ms;
10468                    let cooling =
10469                        previous.next_refresh_ms == u64::MAX || now_ms < previous.next_refresh_ms;
10470                    if backstep {
10471                        previous.refresh_pending = false;
10472                    } else if !repeated && !cooling && !previous.refresh_pending {
10473                        request.next_refresh_ms = previous.next_refresh_ms;
10474                        *previous = request;
10475                    } else {
10476                        // Same GET stays consumed even after recent-ID churn.
10477                        // New GETs cannot flood hop work during the independent
10478                        // cooldown; their lease renewals never slide it.
10479                        previous.last_seen_ms = now_ms;
10480                    }
10481                    route.last_seen_ms = route.last_seen_ms.max(now_ms);
10482                    Some(())
10483                });
10484                let newly_reachable = existing.is_none();
10485                if newly_reachable {
10486                    // Refuse before allocating the expanded row. Existing-row
10487                    // renewals above neither clone nor grow this vector.
10488                    st.make_shared_queue_room(size_of::<ManagedVariableSubscription>(), 255, None)?;
10489                    let mut route = st
10490                        .discovery_routes
10491                        .get(&side)
10492                        .map(DiscoverySideState::try_clone)
10493                        .transpose()?
10494                        .unwrap_or_default();
10495                    reserve_managed_subscriptions(&mut route.requested_network_variables, 1)?;
10496                    route.requested_network_variables.push(request);
10497                    Self::recompute_discovery_side_state(&mut route);
10498                    st.commit_discovery_route(side, route)?;
10499                }
10500                if newly_reachable {
10501                    self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10502                    Self::note_discovery_topology_change_locked(&mut st, now_ms);
10503                }
10504            }
10505            if !self.can_write_managed_variable(ty) {
10506                // Read-only caches are replicas, not authoritative owners.
10507                // Leave the request unhandled so normal discovery routing can
10508                // carry it toward a writer on another segment even when this
10509                // subscriber restored an older persisted value at boot.
10510                return Ok(false);
10511            }
10512            if let Some(value) = self.managed_variable_latest(ty) {
10513                self.emit_internal_tx(
10514                    RouterTxItem::ToSide {
10515                        src: None,
10516                        dst: side,
10517                        data: RouterItem::Packet(value),
10518                    },
10519                    true,
10520                    called_from_queue,
10521                )?;
10522                return Ok(true);
10523            }
10524            // A writer may not have produced its first value yet. Keep the
10525            // request moving in case another authoritative writer has one.
10526            return Ok(false);
10527        }
10528        #[cfg(feature = "std")]
10529        if pkt.data_type() == DataType::DiscoverySchema {
10530            #[cfg(feature = "compression")]
10531            let snapshot =
10532                self.decode_required_schema(|| discovery::decode_discovery_schema(pkt))?;
10533            #[cfg(not(feature = "compression"))]
10534            let snapshot = discovery::decode_discovery_schema(pkt)?;
10535            // The decoded snapshot contains the sender's complete schema, but
10536            // embedded targets retain only definitions that are not already
10537            // compiled into their local schema. Charging the complete wire
10538            // snapshot here made heterogeneous boards reject small schema
10539            // deltas whenever the sender's full snapshot exceeded the shared
10540            // queue budget. The merge performs an atomic projected-retained-
10541            // bytes check; afterwards, reclaim queue/topology space using the
10542            // actual retained schema size included by shared_queue_bytes_used.
10543            let mut st = self.state.lock();
10544            let budget = st.schema_growth_budget();
10545            let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
10546            if report.changed() {
10547                st.fit_discovery_budget();
10548            }
10549            drop(st);
10550            self.note_schema_received(side, &packet_sender);
10551            return Ok(true);
10552        }
10553        if pkt.data_type() == DataType::DiscoveryLinkCapabilities {
10554            let _ = discovery::decode_discovery_link_capabilities(pkt)?;
10555            #[cfg(feature = "compression")]
10556            self.note_payload_codec_packet(side, pkt, true);
10557            return Ok(true);
10558        }
10559        let mut announce = if pkt.data_type() == DataType::DiscoveryAnnounce {
10560            Some(discovery::decode_discovery_announce(pkt)?)
10561        } else {
10562            None
10563        };
10564        let mut time_sources = if pkt.data_type() == DataType::DiscoveryTimeSyncSources {
10565            Some(discovery::decode_discovery_timesync_sources(pkt)?)
10566        } else {
10567            None
10568        };
10569        let mut st = self.state.lock();
10570        let now_ms = self.clock.now_ms();
10571        if pkt.data_type() == DataType::DiscoveryLeave {
10572            let leaving = packet_sender.as_str();
10573            let before = st
10574                .discovery_routes
10575                .iter()
10576                .map(|(side, route)| Ok((*side, route.try_clone()?)))
10577                .collect::<TelemetryResult<RouterMap<_, _>>>()?;
10578            for route in st.discovery_routes.values_mut() {
10579                route.announcers.remove(leaving);
10580                for sender_state in route.announcers.values_mut() {
10581                    Arc::make_mut(&mut sender_state.topology_boards)
10582                        .retain(|board| board.sender_id != leaving);
10583                    for board in Arc::make_mut(&mut sender_state.topology_boards).iter_mut() {
10584                        board.connections.retain(|peer| peer != leaving);
10585                    }
10586                    Self::refresh_sender_topology_state(sender_state);
10587                }
10588                Self::recompute_discovery_side_state(route);
10589            }
10590            st.discovery_routes
10591                .retain(|_, route| !route.announcers.is_empty());
10592            if st.discovery_routes != before {
10593                Self::note_discovery_topology_change_locked(&mut st, now_ms);
10594                self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10595            }
10596            #[cfg(feature = "compression")]
10597            {
10598                drop(st);
10599                self.note_payload_codec_packet(side, pkt, true);
10600            }
10601            return Ok(true);
10602        }
10603        // Resolve aliases while every retained route is still visible.
10604        if let Some(update) = decoded_topology.as_mut() {
10605            for board in update.boards.iter_mut() {
10606                board.sender_id = Self::canonical_sender_locked(&st, &board.sender_id);
10607                for peer in board.connections.iter_mut() {
10608                    *peer = Self::canonical_sender_locked(&st, peer);
10609                }
10610            }
10611        }
10612        let mut route = st
10613            .discovery_routes
10614            .get(&side)
10615            .map(DiscoverySideState::try_clone)
10616            .transpose()?
10617            .unwrap_or_default();
10618        let alias_state = if packet_sender != pkt.sender() {
10619            route.announcers.remove(pkt.sender())
10620        } else {
10621            None
10622        };
10623        let side_link_local_enabled = st
10624            .sides
10625            .get(side)
10626            .and_then(|entry| entry.as_ref())
10627            .map(|side_ref| side_ref.opts.link_local_enabled)
10628            .unwrap_or(false);
10629        let mut sender_state = match route.announcers.remove(&packet_sender).or(alias_state) {
10630            Some(record) => record,
10631            None => crate::peer_record::try_box(DiscoverySenderState::default())?,
10632        };
10633        let changed = match pkt.data_type() {
10634            DataType::DiscoveryAnnounce => {
10635                let mut reachable = announce.take().expect("decoded above");
10636                // Empty announcements are slow-link keepalives, not endpoint
10637                // withdrawals. Preserve ownership until an explicit topology
10638                // update, leave, or expiry removes it. The common receive tail
10639                // still refreshes last_seen_ms without scheduling rediscovery.
10640                if reachable.is_empty() {
10641                    false
10642                } else {
10643                    if !side_link_local_enabled {
10644                        reachable.retain(|ep| !ep.is_link_local_only());
10645                    }
10646                    let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
10647                    let changed = board.reachable_endpoints != reachable;
10648                    board.reachable_endpoints = reachable;
10649                    Self::refresh_sender_topology_state(&mut sender_state);
10650                    changed
10651                }
10652            }
10653            DataType::DiscoveryTimeSyncSources => {
10654                let sources = time_sources.take().expect("decoded above");
10655                let board = Self::sender_topology_board_mut(&mut sender_state, &packet_sender);
10656                let changed = board.reachable_timesync_sources != sources;
10657                board.reachable_timesync_sources = sources;
10658                Self::refresh_sender_topology_state(&mut sender_state);
10659                changed
10660            }
10661            DataType::DiscoveryTopology => {
10662                let mut update = decoded_topology
10663                    .take()
10664                    .expect("topology packet was decoded before route selection");
10665                if !side_link_local_enabled {
10666                    for board in update.boards.iter_mut() {
10667                        board
10668                            .reachable_endpoints
10669                            .retain(|ep| !ep.is_link_local_only());
10670                    }
10671                }
10672                if !update.incremental {
10673                    sender_state.has_full_topology = true;
10674                }
10675                let changed = discovery::apply_shared_topology_update(
10676                    &mut sender_state.topology_boards,
10677                    update,
10678                );
10679                Self::refresh_sender_topology_state(&mut sender_state);
10680                changed
10681            }
10682            DataType::DiscoverySchema => false,
10683            _ => false,
10684        };
10685        sender_state.last_seen_ms = now_ms;
10686        route.announcers.insert(packet_sender, sender_state);
10687        Self::recompute_discovery_side_state(&mut route);
10688        st.commit_discovery_route(side, route)?;
10689        self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
10690        if changed {
10691            Self::note_discovery_topology_change_locked(&mut st, now_ms);
10692        }
10693        Ok(true)
10694    }
10695
10696    #[cfg(not(feature = "discovery"))]
10697    fn learn_discovery_packet(
10698        &self,
10699        _pkt: &Packet,
10700        _src: Option<RouterSideId>,
10701        _called_from_queue: bool,
10702    ) -> TelemetryResult<bool> {
10703        Ok(false)
10704    }
10705
10706    #[inline]
10707    fn reliable_key(side: RouterSideId, ty: DataType) -> (RouterSideId, u32) {
10708        (side, ty.as_u32())
10709    }
10710
10711    fn reliable_tx_state_mut<'a>(
10712        &'a self,
10713        st: &'a mut RouterInner,
10714        side: RouterSideId,
10715        ty: DataType,
10716    ) -> &'a mut ReliableTxState {
10717        let key = Self::reliable_key(side, ty);
10718        st.reliable_tx
10719            .entry(key)
10720            .or_insert_with(|| ReliableTxState {
10721                next_seq: 1,
10722                sent_order: VecDeque::new(),
10723                sent: Default::default(),
10724            })
10725    }
10726
10727    fn reliable_rx_state_mut<'a>(
10728        &'a self,
10729        st: &'a mut RouterInner,
10730        side: RouterSideId,
10731        ty: DataType,
10732    ) -> &'a mut ReliableRxState {
10733        let key = Self::reliable_key(side, ty);
10734        st.reliable_rx
10735            .entry(key)
10736            .or_insert_with(|| ReliableRxState {
10737                expected_seq: 1,
10738                buffered: Default::default(),
10739            })
10740    }
10741
10742    fn reliable_control_packet(
10743        &self,
10744        control_ty: DataType,
10745        ty: DataType,
10746        seq: u32,
10747    ) -> TelemetryResult<Packet> {
10748        let mut bytes = [0u8; 8];
10749        bytes[..4].copy_from_slice(&ty.as_u32().to_le_bytes());
10750        bytes[4..].copy_from_slice(&seq.to_le_bytes());
10751        Packet::new_local(
10752            control_ty,
10753            message_meta(control_ty).endpoints_owned(),
10754            self.sender_arc(),
10755            self.packet_timestamp_ms(),
10756            crate::config::StandardSmallPayload::new(&bytes),
10757        )
10758    }
10759
10760    fn queue_reliable_ack(
10761        &self,
10762        side: RouterSideId,
10763        ty: DataType,
10764        seq: u32,
10765        called_from_queue: bool,
10766    ) -> TelemetryResult<()> {
10767        let pkt = self.reliable_control_packet(DataType::ReliableAck, ty, seq)?;
10768        self.emit_internal_tx_with_priority(
10769            RouterTxItem::ToSide {
10770                src: None,
10771                dst: side,
10772                data: RouterItem::Packet(pkt),
10773            },
10774            true,
10775            message_priority(DataType::ReliableAck),
10776            called_from_queue,
10777        )
10778    }
10779
10780    fn queue_reliable_packet_request(
10781        &self,
10782        side: RouterSideId,
10783        ty: DataType,
10784        seq: u32,
10785        called_from_queue: bool,
10786    ) -> TelemetryResult<()> {
10787        let pkt = self.reliable_control_packet(DataType::ReliablePacketRequest, ty, seq)?;
10788        self.emit_internal_tx_with_priority(
10789            RouterTxItem::ToSide {
10790                src: None,
10791                dst: side,
10792                data: RouterItem::Packet(pkt),
10793            },
10794            true,
10795            message_priority(DataType::ReliablePacketRequest),
10796            called_from_queue,
10797        )
10798    }
10799
10800    fn queue_reliable_partial_ack(
10801        &self,
10802        side: RouterSideId,
10803        ty: DataType,
10804        seq: u32,
10805        called_from_queue: bool,
10806    ) -> TelemetryResult<()> {
10807        let pkt = self.reliable_control_packet(DataType::ReliablePartialAck, ty, seq)?;
10808        self.emit_internal_tx_with_priority(
10809            RouterTxItem::ToSide {
10810                src: None,
10811                dst: side,
10812                data: RouterItem::Packet(pkt),
10813            },
10814            true,
10815            message_priority(DataType::ReliablePartialAck),
10816            called_from_queue,
10817        )
10818    }
10819
10820    fn handle_reliable_ack(&self, side: RouterSideId, ty: DataType, ack: u32) {
10821        let mut st = self.state.lock();
10822        let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10823        if matches!(reliable_mode(ty), crate::ReliableMode::Unordered) {
10824            tx_state.sent.remove(&ack);
10825            tx_state.sent_order.retain(|seq| *seq != ack);
10826            return;
10827        }
10828
10829        while let Some(seq) = tx_state.sent_order.front().copied() {
10830            if seq > ack {
10831                break;
10832            }
10833            tx_state.sent_order.pop_front();
10834            tx_state.sent.remove(&seq);
10835        }
10836    }
10837
10838    fn handle_reliable_partial_ack(&self, side: RouterSideId, ty: DataType, seq: u32) {
10839        let mut st = self.state.lock();
10840        let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10841        if let Some(sent) = tx_state.sent.get_mut(&seq) {
10842            sent.partial_acked = true;
10843        }
10844    }
10845
10846    fn queue_reliable_retransmit(
10847        &self,
10848        side: RouterSideId,
10849        ty: DataType,
10850        seq: u32,
10851        called_from_queue: bool,
10852    ) -> TelemetryResult<()> {
10853        let mut queued = None;
10854        {
10855            let mut st = self.state.lock();
10856            let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
10857            if let Some(sent) = tx_state.sent.get_mut(&seq)
10858                && !sent.queued
10859            {
10860                sent.partial_acked = false;
10861                queued = Some(sent.bytes.clone());
10862            }
10863        }
10864
10865        if let Some(bytes) = queued {
10866            let now = self.clock.now_ms();
10867            if called_from_queue {
10868                self.tx_queue_item_with_priority(
10869                    RouterTxItem::ReliableReplay {
10870                        dst: side,
10871                        ty,
10872                        seq,
10873                        bytes,
10874                    },
10875                    true,
10876                    Self::router_item_priority_bumped(ty),
10877                )?;
10878            } else {
10879                self.tx_item_impl(
10880                    RouterTxItem::ReliableReplay {
10881                        dst: side,
10882                        ty,
10883                        seq,
10884                        bytes,
10885                    },
10886                    true,
10887                    false,
10888                )?;
10889            }
10890            let mut st = self.state.lock();
10891            if let Some(sent) = st
10892                .reliable_tx
10893                .get_mut(&Self::reliable_key(side, ty))
10894                .and_then(|tx| tx.sent.get_mut(&seq))
10895            {
10896                sent.last_send_ms = sent.last_send_ms.max(now);
10897            }
10898        }
10899
10900        Ok(())
10901    }
10902
10903    fn send_reliable_raw_to_side(
10904        &self,
10905        side: RouterSideId,
10906        bytes: crate::SharedBytes,
10907        relayed: bool,
10908        admitted: &mut bool,
10909    ) -> TelemetryResult<()> {
10910        let handler = {
10911            let st = self.state.lock();
10912            let side_ref = Self::side_ref(&st, side)?;
10913            if !side_ref.opts.egress_enabled {
10914                return Ok(());
10915            }
10916            (side_ref.tx_handler.clone(), side_ref.opts.clone())
10917        };
10918
10919        let (handler, opts) = handler;
10920
10921        let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
10922            return Err(TelemetryError::Io("side tx busy"));
10923        };
10924        let started_ms = self.clock.now_ms();
10925        let (ty, priority) = Self::item_type_priority(&RouterItem::Packed(bytes.clone()))?;
10926        let priority = priority.max(self.effective_transport_priority(ty));
10927        let result = match handler {
10928            packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)) => {
10929                let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
10930                let mut attempts_total = 0usize;
10931                let mut sent_bytes = 0usize;
10932                self.send_side_frames_resumable(side, frames, |frame| {
10933                    match self.retry_with_attempts(runtime_max_handler_retries(), || {
10934                        self.packed_handler_call(
10935                            side,
10936                            &packed,
10937                            frame,
10938                            priority,
10939                            EgressService::default(),
10940                        )
10941                    }) {
10942                        Ok((_, attempts)) => {
10943                            *admitted = true;
10944                            attempts_total = attempts_total.saturating_add(attempts);
10945                            sent_bytes = sent_bytes.saturating_add(frame.len());
10946                        }
10947                        Err((err, attempts)) => {
10948                            self.note_side_tx_failure(
10949                                side,
10950                                ty,
10951                                attempts_total.saturating_add(attempts),
10952                            );
10953                            return Err(err);
10954                        }
10955                    }
10956
10957                    Ok(())
10958                })?;
10959                self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
10960                self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
10961                return Ok(());
10962            }
10963            RouterTxHandlerFn::Packet(f) => {
10964                let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
10965                self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
10966            }
10967        };
10968        match result {
10969            Ok((_, attempts)) => {
10970                *admitted = true;
10971                self.record_side_tx_sample(side, bytes.len(), started_ms, self.clock.now_ms());
10972                self.note_side_tx_success(side, ty, bytes.len(), relayed, attempts);
10973                Ok(())
10974            }
10975            Err((err, attempts)) => {
10976                self.note_side_tx_failure(side, ty, attempts);
10977                Err(err)
10978            }
10979        }
10980    }
10981
10982    #[allow(unused_mut)]
10983    #[cfg(feature = "compression")]
10984    #[cfg(test)]
10985    fn send_queued_to_side(
10986        &self,
10987        side: RouterSideId,
10988        data: &mut RouterItem,
10989        relayed: bool,
10990    ) -> (
10991        TelemetryResult<()>,
10992        Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
10993    ) {
10994        self.send_queued_to_side_with_progress(side, data, relayed, &mut false)
10995    }
10996
10997    #[cfg(feature = "compression")]
10998    fn send_queued_to_side_with_progress(
10999        &self,
11000        side: RouterSideId,
11001        data: &mut RouterItem,
11002        relayed: bool,
11003        admitted: &mut bool,
11004    ) -> (
11005        TelemetryResult<()>,
11006        Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
11007    ) {
11008        self.send_queued_to_side_with_service(
11009            side,
11010            data,
11011            relayed,
11012            admitted,
11013            EgressService::default(),
11014        )
11015    }
11016    #[cfg(feature = "compression")]
11017    fn send_queued_to_side_with_service(
11018        &self,
11019        side: RouterSideId,
11020        data: &mut RouterItem,
11021        relayed: bool,
11022        admitted: &mut bool,
11023        service: EgressService,
11024    ) -> (
11025        TelemetryResult<()>,
11026        Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
11027    ) {
11028        let reservation = match self.prepare_queued_payload(side, data) {
11029            Ok(reservation) => reservation,
11030            Err(error) => return (Err(error), None),
11031        };
11032        (
11033            self.send_reliable_to_side_with_service(side, data.clone(), relayed, admitted, service),
11034            reservation,
11035        )
11036    }
11037
11038    #[cfg(test)]
11039    fn send_reliable_to_side(
11040        &self,
11041        side: RouterSideId,
11042        data: RouterItem,
11043        relayed: bool,
11044    ) -> TelemetryResult<()> {
11045        self.send_reliable_to_side_with_progress(side, data, relayed, &mut false)
11046    }
11047
11048    #[cfg(any(test, not(feature = "compression")))]
11049    fn send_reliable_to_side_with_progress(
11050        &self,
11051        side: RouterSideId,
11052        data: RouterItem,
11053        relayed: bool,
11054        admitted: &mut bool,
11055    ) -> TelemetryResult<()> {
11056        self.send_reliable_to_side_with_service(
11057            side,
11058            data,
11059            relayed,
11060            admitted,
11061            EgressService::default(),
11062        )
11063    }
11064    fn send_reliable_to_side_with_service(
11065        &self,
11066        side: RouterSideId,
11067        data: RouterItem,
11068        relayed: bool,
11069        admitted: &mut bool,
11070        service: EgressService,
11071    ) -> TelemetryResult<()> {
11072        #[cfg(feature = "discovery")]
11073        let schema_id = (Self::item_data_type(&data).ok() == Some(DataType::DiscoverySchema))
11074            .then(|| Self::get_hash(&data));
11075        #[cfg(feature = "discovery")]
11076        if let Some(id) = schema_id {
11077            let mut st = self.state.lock();
11078            if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
11079                && throttle.schema_response_active()
11080                && throttle.schema_response_phase == SchemaResponsePhase::Owned
11081                && throttle.schema_response_packet_id == id
11082                && !throttle.schema_response_started()
11083            {
11084                throttle.set_schema_response_started(true);
11085                throttle.next_schema_response_ms = self
11086                    .clock
11087                    .now_ms()
11088                    .saturating_add(discovery::DISCOVERY_FAST_INTERVAL_MS);
11089                // Another peer can restart after this generation was queued
11090                // and after an application packet consumed its initial full
11091                // header. Cover those coalesced requests at actual dispatch,
11092                // once only; partial retries retain their existing templates.
11093                if let Some(transport) = st.side_transport.get_mut(&side) {
11094                    transport.refresh_transmit_header_templates();
11095                }
11096            }
11097        }
11098        let result = self.send_reliable_to_side_impl(side, data, relayed, admitted, service);
11099        #[cfg(feature = "discovery")]
11100        if let Some(id) = schema_id {
11101            let mut st = self.state.lock();
11102            if let Some(throttle) = st.discovery_side_throttle.get_mut(&side)
11103                && throttle.schema_response_active()
11104                && throttle.schema_response_phase == SchemaResponsePhase::Owned
11105                && throttle.schema_response_packet_id == id
11106            {
11107                // Transient partial IO retains this exact owner in the outer
11108                // queue path. It is not a new request for another catalog.
11109                if !result.as_ref().is_err_and(Self::is_transient_tx_error) {
11110                    throttle.set_schema_response_active(false);
11111                    throttle.schema_response_phase = SchemaResponsePhase::Idle;
11112                    throttle.set_schema_response_pending(
11113                        throttle.schema_response_pending() || result.is_err(),
11114                    );
11115                }
11116            }
11117        }
11118        result
11119    }
11120
11121    fn send_reliable_to_side_impl(
11122        &self,
11123        side: RouterSideId,
11124        data: RouterItem,
11125        relayed: bool,
11126        admitted: &mut bool,
11127        service: EgressService,
11128    ) -> TelemetryResult<()> {
11129        // Reliability needs only the type and frozen-owner presence; endpoint
11130        // normalization and an owned wire-owner array are unnecessary here.
11131        let (ty, addressed) = match &data {
11132            RouterItem::Packet(pkt) => (pkt.data_type(), !pkt.wire_target_senders().is_empty()),
11133            RouterItem::Packed(bytes) => {
11134                let env = wire_format::peek_routing_envelope(bytes.as_ref())?;
11135                (env.ty, env.target_senders().len() != 0)
11136            }
11137        };
11138        let end_to_end_ordered =
11139            addressed && matches!(reliable_mode(ty), crate::ReliableMode::Ordered);
11140        let (handler, opts, hop_reliable_enabled) = {
11141            let st = self.state.lock();
11142            let side_ref = Self::side_ref(&st, side)?;
11143            let opts = side_ref.opts.clone();
11144            let hop_reliable_enabled = opts.reliable_enabled
11145                && !end_to_end_ordered
11146                && self.cfg.reliable_enabled()
11147                && !self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms());
11148            (side_ref.tx_handler.clone(), opts, hop_reliable_enabled)
11149        };
11150
11151        if !matches!(
11152            &handler,
11153            RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11154        ) {
11155            return self.call_side_tx_handler_with_service(
11156                side, &handler, &data, relayed, admitted, service,
11157            );
11158        }
11159
11160        if !hop_reliable_enabled {
11161            let mut adjusted_opts = opts;
11162            adjusted_opts.reliable_enabled = false;
11163            // End-to-end ACKs are ordinary routed control packets and must cross best-effort
11164            // segments. `reliable_enabled` controls only this side's hop sequencing/replay; using
11165            // it to discard an ACK prevents a reliable origin from ever confirming destinations
11166            // behind Pico-Fi, UDP, radio, or another transport with its own reliability layer.
11167            let preserve_end_to_end_ack = true;
11168            if let Some(adjusted) =
11169                self.adjust_reliable_for_side(adjusted_opts, data, preserve_end_to_end_ack)?
11170            {
11171                return self.call_side_tx_handler_with_service(
11172                    side, &handler, &adjusted, relayed, admitted, service,
11173                );
11174            }
11175            return Ok(());
11176        }
11177
11178        if let RouterItem::Packed(bytes) = &data {
11179            wire_format::peek_routing_frame_info(bytes.as_ref())?;
11180        }
11181        let priority = self.item_service_priority(&data)?;
11182
11183        if !is_reliable_type(ty) {
11184            if let Some(adjusted) = self.adjust_reliable_for_side(opts, data, true)? {
11185                self.call_side_tx_handler_with_service(
11186                    side, &handler, &adjusted, relayed, admitted, service,
11187                )?;
11188            }
11189            return Ok(());
11190        }
11191
11192        #[cfg(feature = "compression")]
11193        let data = match data {
11194            RouterItem::Packed(bytes) => {
11195                let budget = self.state.lock().memory.max_queue_budget;
11196                let normalized =
11197                    wire_format::payload_encoding_for_retention(bytes.as_ref(), budget)?;
11198                RouterItem::Packed(
11199                    normalized
11200                        .map(crate::shared_bytes::convert)
11201                        .unwrap_or(bytes),
11202                )
11203            }
11204            packet => packet,
11205        };
11206        let (seq, flags) = {
11207            let mut st = self.state.lock();
11208            let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
11209            if tx_state.sent.len() >= runtime_reliable_max_pending() {
11210                return Err(TelemetryError::Io("reliable window full"));
11211            }
11212            let seq = tx_state.next_seq;
11213            let next = tx_state.next_seq.wrapping_add(1);
11214            tx_state.next_seq = if next == 0 { 1 } else { next };
11215            let flags = match reliable_mode(ty) {
11216                crate::ReliableMode::Unordered => wire_format::RELIABLE_FLAG_UNORDERED,
11217                _ => 0,
11218            };
11219            (seq, flags)
11220        };
11221
11222        let bytes: crate::SharedBytes = match data {
11223            RouterItem::Packet(pkt) => self.pack_packet_for_router(
11224                &pkt,
11225                Some(wire_format::ReliableHeader { flags, seq, ack: 0 }),
11226            )?,
11227            RouterItem::Packed(bytes) => {
11228                #[cfg(feature = "cryptography")]
11229                if self.e2e_seal_config_for_type(ty).is_some() {
11230                    self.prepare_packed_for_remote(
11231                        bytes,
11232                        Some(Some(wire_format::ReliableHeader { flags, seq, ack: 0 })),
11233                    )?
11234                } else {
11235                    let Some(rewritten) =
11236                        wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
11237                    else {
11238                        let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11239                            return Err(TelemetryError::Io("side tx busy"));
11240                        };
11241                        let started_ms = self.clock.now_ms();
11242                        let frames =
11243                            self.encode_side_transport_frames(side, opts, bytes.clone())?;
11244                        let mut attempts_total = 0usize;
11245                        let mut sent_bytes = 0usize;
11246                        self.send_side_frames_resumable_with_service(
11247                            side,
11248                            frames,
11249                            service,
11250                            |frame| {
11251                                match self.retry_with_attempts(
11252                                    runtime_max_handler_retries(),
11253                                    || {
11254                                        self.packed_handler_call(
11255                                            side, &handler, frame, priority, service,
11256                                        )
11257                                    },
11258                                ) {
11259                                    Ok((_, attempts)) => {
11260                                        *admitted = true;
11261                                        attempts_total = attempts_total.saturating_add(attempts);
11262                                        sent_bytes = sent_bytes.saturating_add(frame.len());
11263                                    }
11264                                    Err((err, attempts)) => {
11265                                        self.note_side_tx_failure(
11266                                            side,
11267                                            ty,
11268                                            attempts_total.saturating_add(attempts),
11269                                        );
11270                                        return Err(err);
11271                                    }
11272                                }
11273
11274                                Ok(())
11275                            },
11276                        )?;
11277                        self.record_side_tx_sample(
11278                            side,
11279                            sent_bytes,
11280                            started_ms,
11281                            self.clock.now_ms(),
11282                        );
11283                        self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11284                        return Ok(());
11285                    };
11286                    crate::shared_bytes::convert(rewritten)
11287                }
11288                #[cfg(not(feature = "cryptography"))]
11289                {
11290                    let Some(rewritten) =
11291                        wire_format::rewrite_reliable_header_owned(bytes.as_ref(), flags, seq, 0)?
11292                    else {
11293                        let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11294                            return Err(TelemetryError::Io("side tx busy"));
11295                        };
11296                        let started_ms = self.clock.now_ms();
11297                        let frames =
11298                            self.encode_side_transport_frames(side, opts, bytes.clone())?;
11299                        let mut attempts_total = 0usize;
11300                        let mut sent_bytes = 0usize;
11301                        self.send_side_frames_resumable_with_service(
11302                            side,
11303                            frames,
11304                            service,
11305                            |frame| {
11306                                match self.retry_with_attempts(
11307                                    runtime_max_handler_retries(),
11308                                    || {
11309                                        self.packed_handler_call(
11310                                            side, &handler, frame, priority, service,
11311                                        )
11312                                    },
11313                                ) {
11314                                    Ok((_, attempts)) => {
11315                                        *admitted = true;
11316                                        attempts_total = attempts_total.saturating_add(attempts);
11317                                        sent_bytes = sent_bytes.saturating_add(frame.len());
11318                                    }
11319                                    Err((err, attempts)) => {
11320                                        self.note_side_tx_failure(
11321                                            side,
11322                                            ty,
11323                                            attempts_total.saturating_add(attempts),
11324                                        );
11325                                        return Err(err);
11326                                    }
11327                                }
11328
11329                                Ok(())
11330                            },
11331                        )?;
11332                        self.record_side_tx_sample(
11333                            side,
11334                            sent_bytes,
11335                            started_ms,
11336                            self.clock.now_ms(),
11337                        );
11338                        self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11339                        return Ok(());
11340                    };
11341                    crate::shared_bytes::convert(rewritten)
11342                }
11343            }
11344        };
11345
11346        #[cfg(feature = "compact-packet-store")]
11347        let mut bytes = bytes;
11348
11349        let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
11350            return Err(TelemetryError::Io("side tx busy"));
11351        };
11352        let started_ms = self.clock.now_ms();
11353        let frames = self.encode_side_transport_frames(side, opts, bytes.clone())?;
11354        let mut attempts_total = 0usize;
11355        let mut sent_bytes = 0usize;
11356        self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
11357            match self.retry_with_attempts(runtime_max_handler_retries(), || {
11358                self.packed_handler_call(side, &handler, frame, priority, service)
11359            }) {
11360                Ok((_, attempts)) => {
11361                    *admitted = true;
11362                    attempts_total = attempts_total.saturating_add(attempts);
11363                    sent_bytes = sent_bytes.saturating_add(frame.len());
11364                }
11365                Err((err, attempts)) => {
11366                    self.note_side_tx_failure(side, ty, attempts_total.saturating_add(attempts));
11367                    return Err(err);
11368                }
11369            }
11370
11371            Ok(())
11372        })?;
11373        self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
11374        self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
11375
11376        #[cfg(feature = "compact-packet-store")]
11377        {
11378            let _ = bytes.park_for_queue();
11379        }
11380
11381        {
11382            let mut st = self.state.lock();
11383            let tx_state = self.reliable_tx_state_mut(&mut st, side, ty);
11384            tx_state.sent_order.push_back(seq);
11385            tx_state.sent.insert(
11386                seq,
11387                ReliableSent {
11388                    bytes: bytes.clone(),
11389                    last_send_ms: self.clock.now_ms(),
11390                    retries: 0,
11391                    queued: false,
11392                    partial_acked: false,
11393                },
11394            );
11395        }
11396
11397        Ok(())
11398    }
11399
11400    fn run_side_chunk_transfer<T>(
11401        &self,
11402        side: RouterSideId,
11403        key: u64,
11404        history_dependent: bool,
11405        service: EgressService,
11406        send: impl FnOnce(&mut usize) -> TelemetryResult<T>,
11407    ) -> TelemetryResult<T> {
11408        let mut next = {
11409            let mut st = self.state.lock();
11410            #[cfg(feature = "latest-value-egress")]
11411            let defer_capacity = st.latest_side() == Some(side);
11412            #[cfg(feature = "latest-value-egress")]
11413            let latest_owner = defer_capacity
11414                && service.token != 0
11415                && service.side != 0
11416                && side == usize::from(service.side - 1)
11417                && service.token == st.latest_service_seq
11418                && service.kind == 2
11419                && st.latest_turn_phase == 4;
11420            let transport = st
11421                .side_transport
11422                .get_mut(&side)
11423                .ok_or(TelemetryError::BadArg)?;
11424            if transport.tx_chunk_progress.is_none() {
11425                crate::memory_admission::check(
11426                    512,
11427                    core::mem::size_of::<crate::side_transport::SideTxProgress>(),
11428                )?;
11429                transport.tx_chunk_progress =
11430                    Some(Box::new(crate::side_transport::SideTxProgress::default()));
11431            }
11432            let progress = transport.tx_chunk_progress.as_mut().expect("initialized");
11433            #[cfg(feature = "latest-value-egress")]
11434            if defer_capacity
11435                && !progress.has_available_slot(self.clock.now_ms())
11436                && !progress.has_live_key(key, self.clock.now_ms())
11437            {
11438                // This is a known bounded admission refusal before any bytes
11439                // are sent, not a reinterpretation of an arbitrary IO error.
11440                return Err(TelemetryError::Io("side tx busy"));
11441            }
11442            #[cfg(feature = "latest-value-egress")]
11443            {
11444                progress.begin_classified(
11445                    key,
11446                    self.clock.now_ms(),
11447                    history_dependent,
11448                    latest_owner,
11449                )?
11450            }
11451            #[cfg(not(feature = "latest-value-egress"))]
11452            {
11453                let _ = service;
11454                progress.begin(key, self.clock.now_ms(), history_dependent)?
11455            }
11456        };
11457        let result = send(&mut next);
11458        if let Some(progress) = self
11459            .state
11460            .lock()
11461            .side_transport
11462            .get_mut(&side)
11463            .and_then(|transport| transport.tx_chunk_progress.as_mut())
11464        {
11465            progress.finish(key, next, self.clock.now_ms(), &result);
11466        }
11467        result
11468    }
11469
11470    fn send_side_frames_resumable(
11471        &self,
11472        side: RouterSideId,
11473        frames: SideTransportFrames,
11474        send: impl FnMut(&[u8]) -> TelemetryResult<()>,
11475    ) -> TelemetryResult<()> {
11476        self.send_side_frames_resumable_with_service(side, frames, EgressService::default(), send)
11477    }
11478
11479    fn send_side_frames_resumable_with_service(
11480        &self,
11481        side: RouterSideId,
11482        mut frames: SideTransportFrames,
11483        service: EgressService,
11484        mut send: impl FnMut(&[u8]) -> TelemetryResult<()>,
11485    ) -> TelemetryResult<()> {
11486        let bootstraps_template = frames.bootstraps_template();
11487        let result = if frames.len() == 1 {
11488            frames.send_resuming(&mut 0, send)
11489        } else {
11490            let key = frames.progress_key();
11491            let history_dependent = frames.history_dependent();
11492            self.run_side_chunk_transfer(side, key, history_dependent, service, |next| {
11493                frames.send_resuming(next, &mut send)
11494            })
11495        };
11496        if result.is_err() && bootstraps_template {
11497            // Encoding has already inserted the TX template, but a busy or
11498            // failed handler did not accept its bootstrap. Keep dictionary
11499            // eviction history and force a full refresh on the next attempt.
11500            // Never clear RX templates or allocate during this recovery.
11501            if let Some(state) = self.state.lock().side_transport.get_mut(&side) {
11502                state.refresh_transmit_header_templates();
11503            }
11504        }
11505        result
11506    }
11507
11508    fn split_side_transport_frame(
11509        &self,
11510        _side: RouterSideId,
11511        frame: crate::SharedBytes,
11512        max_frame_bytes: usize,
11513    ) -> TelemetryResult<SideTransportFrames> {
11514        SideTransportFrames::split(frame, max_frame_bytes, self.sender_arc().as_bytes())
11515    }
11516
11517    #[cfg(all(feature = "compression", feature = "discovery"))]
11518    fn note_payload_codec_packet(&self, side: RouterSideId, pkt: &Packet, accepted: bool) {
11519        let now = self.clock.now_ms();
11520        let mut st = self.state.lock();
11521        if !st
11522            .sides
11523            .get(side)
11524            .and_then(|entry| entry.as_ref())
11525            .is_some_and(|entry| {
11526                matches!(
11527                    entry.tx_handler,
11528                    RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11529                )
11530            })
11531        {
11532            return;
11533        }
11534        if accepted && pkt.data_type() == crate::DataType::DiscoveryLeave {
11535            for transport in st.side_transport.values_mut() {
11536                let _ = transport.payload_codec.discovery_packet(
11537                    pkt,
11538                    true,
11539                    now,
11540                    DISCOVERY_ROUTE_TTL_MS,
11541                );
11542            }
11543        } else if let Some(transport) = st.side_transport.get_mut(&side) {
11544            let _ = transport.payload_codec.discovery_packet(
11545                pkt,
11546                accepted,
11547                now,
11548                DISCOVERY_ROUTE_TTL_MS,
11549            );
11550        }
11551        if st.shared_queue_bytes_used() > st.memory.max_queue_budget
11552            && let Some(transport) = st.side_transport.get_mut(&side)
11553        {
11554            transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
11555        }
11556    }
11557    #[cfg(all(feature = "compression", feature = "discovery"))]
11558    fn note_payload_codec_address(&self, side: RouterSideId, payload: &[u8], accepted: bool) {
11559        let now = self.clock.now_ms();
11560        let mut st = self.state.lock();
11561        if !st
11562            .sides
11563            .get(side)
11564            .and_then(|entry| entry.as_ref())
11565            .is_some_and(|entry| {
11566                matches!(
11567                    entry.tx_handler,
11568                    RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
11569                )
11570            })
11571        {
11572            return;
11573        }
11574        if let Some(transport) = st.side_transport.get_mut(&side) {
11575            let _ = transport.payload_codec.address_payload(
11576                payload,
11577                accepted,
11578                now,
11579                DISCOVERY_ROUTE_TTL_MS,
11580            );
11581        }
11582        if st.shared_queue_bytes_used() > st.memory.max_queue_budget
11583            && let Some(transport) = st.side_transport.get_mut(&side)
11584        {
11585            transport.payload_codec.evict(now, DISCOVERY_ROUTE_TTL_MS);
11586        }
11587    }
11588
11589    #[cfg(feature = "compression")]
11590    fn payload_for_transport_side(
11591        &self,
11592        side: RouterSideId,
11593        raw: crate::SharedBytes,
11594    ) -> TelemetryResult<crate::SharedBytes> {
11595        #[cfg(not(feature = "discovery"))]
11596        let _ = side;
11597        let (allowed, budget) = {
11598            #[cfg(feature = "discovery")]
11599            let mut st = self.state.lock();
11600            #[cfg(not(feature = "discovery"))]
11601            let st = self.state.lock();
11602            let budget = st.memory.max_queue_budget;
11603            #[cfg(feature = "discovery")]
11604            let allowed = {
11605                let transport = st
11606                    .side_transport
11607                    .get_mut(&side)
11608                    .ok_or(TelemetryError::BadArg)?;
11609                let generation = transport.payload_codec.generation();
11610                if transport.payload_codec_choice != generation {
11611                    transport.payload_codec_choice = generation;
11612                    transport.tx_chunk_progress = None;
11613                    transport.refresh_transmit_header_templates();
11614                }
11615                transport.payload_codec.allowed(self.clock.now_ms())
11616            };
11617            #[cfg(not(feature = "discovery"))]
11618            let allowed = false;
11619            (allowed, budget)
11620        };
11621        Ok(
11622            match wire_format::payload_encoding_for_side(raw.as_ref(), allowed, budget)? {
11623                Some(frame) => crate::shared_bytes::convert(frame),
11624                None => raw,
11625            },
11626        )
11627    }
11628
11629    /// Persist an admitted downgrade in the queue owner before a handler can
11630    /// backpressure it. Retrying retained traffic must not need fresh decoder
11631    /// scratch merely to reproduce the same already-admitted plaintext frame.
11632    #[cfg(feature = "compression")]
11633    fn prepare_queued_payload(
11634        &self,
11635        side: RouterSideId,
11636        data: &mut RouterItem,
11637    ) -> TelemetryResult<Option<crate::payload_codec::QueueReservation<'_, RouterInner>>> {
11638        let RouterItem::Packed(bytes) = data else {
11639            return Ok(None);
11640        };
11641        if !bytes.first().is_some_and(|flags| flags & 0x11 == 1) {
11642            return Ok(None);
11643        }
11644        let capable = {
11645            #[cfg(feature = "discovery")]
11646            {
11647                self.state
11648                    .lock()
11649                    .side_transport
11650                    .get(&side)
11651                    .is_some_and(|transport| transport.payload_codec.allowed(self.clock.now_ms()))
11652            }
11653            #[cfg(not(feature = "discovery"))]
11654            {
11655                false
11656            }
11657        };
11658        if capable {
11659            return Ok(None);
11660        }
11661        let required = wire_format::decoded_frame_upper_bound(bytes.as_ref())?
11662            .checked_add(
11663                size_of::<crate::SharedBytes>()
11664                    + size_of::<bool>()
11665                    + size_of::<u8>()
11666                    + size_of::<u64>(),
11667            )
11668            .ok_or(TelemetryError::Io("memory pressure"))?;
11669        let reservation = crate::payload_codec::QueueReservation::acquire(&self.state, required)?;
11670        *bytes = self.payload_for_transport_side(side, bytes.clone())?;
11671        Ok(Some(reservation))
11672    }
11673
11674    fn encode_side_transport_frames(
11675        &self,
11676        side: RouterSideId,
11677        opts: RouterSideOptions,
11678        raw: crate::SharedBytes,
11679    ) -> TelemetryResult<SideTransportFrames> {
11680        #[cfg(feature = "compression")]
11681        let raw = self.payload_for_transport_side(side, raw)?;
11682        if !opts.header_template_enabled && opts.max_frame_bytes == 0 {
11683            return Ok(SideTransportFrames::single(raw));
11684        }
11685
11686        let raw_len = raw.len();
11687        if opts.max_frame_bytes != 0 && raw_len > opts.max_frame_bytes {
11688            // Canonical bytes keep a partially accepted transfer independent
11689            // of dictionary churn while other traffic is serviced.
11690            let frames = self.split_side_transport_frame(side, raw, opts.max_frame_bytes)?;
11691            let mut st = self.state.lock();
11692            let stats = st.side_runtime_stats.entry(side).or_default();
11693            stats.note_side_transport_full(raw_len, frames.wire_len());
11694            stats.note_side_transport_chunks(frames.len());
11695            return Ok(frames);
11696        }
11697
11698        if opts.header_template_enabled {
11699            // Shared buses cannot retain one coherent dictionary across
11700            // independent producers. Do not build owned templates or wrap
11701            // canonical bytes when no dictionary can be used.
11702            let frame = wire_format::peek_routing_frame_info(raw.as_ref())?;
11703            let without_dictionary = opts.max_side_transport_templates == 0 || {
11704                let st = self.state.lock();
11705                self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
11706            };
11707            let reliable = is_reliable_type(frame.envelope.ty)
11708                || frame.reliable.is_some()
11709                // Managed-variable delivery freezes destination owners even
11710                // when the application schema itself is best effort. Its
11711                // end-to-end receipt must not depend on a lossy dictionary
11712                // bootstrap, and transports must be able to prioritize it
11713                // independently of queued schema/template frames.
11714                || frame.envelope.target_senders().len() != 0
11715                || matches!(
11716                    frame.envelope.ty,
11717                    crate::DataType::ReliableAck
11718                        | crate::DataType::ReliablePartialAck
11719                        | crate::DataType::ReliablePacketRequest
11720                );
11721            if without_dictionary || reliable {
11722                let mut st = self.state.lock();
11723                if without_dictionary {
11724                    let removed = st
11725                        .side_transport
11726                        .get_mut(&side)
11727                        .ok_or(TelemetryError::BadArg)?
11728                        .trim_tx_templates_to(0);
11729                    for _ in 0..removed {
11730                        st.side_runtime_stats
11731                            .entry(side)
11732                            .or_default()
11733                            .note_side_transport_template_eviction();
11734                    }
11735                }
11736                st.side_runtime_stats
11737                    .entry(side)
11738                    .or_default()
11739                    .note_side_transport_full(raw_len, raw_len);
11740                return Ok(SideTransportFrames::single(raw));
11741            }
11742        }
11743
11744        if opts.header_template_enabled && crate::memory_admission::supports_work_kinds() {
11745            let (template, ..) = extract_side_header_template_view(raw.as_ref())?;
11746            let cached = self
11747                .state
11748                .lock()
11749                .side_transport
11750                .get(&side)
11751                .is_some_and(|transport| {
11752                    transport.tx_template_ids.contains_key(&template.hash)
11753                        && transport.tx_templates.contains_key(&template.hash)
11754                });
11755            if !cached
11756                && matches!(
11757                    crate::memory_admission::check_receive(raw.as_ref()),
11758                    Err(TelemetryError::Io("memory pressure"))
11759                )
11760            {
11761                // Compression is optional. Retain the canonical shared frame
11762                // instead of allocating a dictionary entry under pressure.
11763                // Oversized frames returned through the normal chunker above.
11764                self.state
11765                    .lock()
11766                    .side_runtime_stats
11767                    .entry(side)
11768                    .or_default()
11769                    .note_side_transport_full(raw_len, raw_len);
11770                return Ok(SideTransportFrames::single(raw));
11771            }
11772        }
11773
11774        let mut compact_payload_len = None;
11775        let mut used_compact = false;
11776        let used_timestamp_delta = false;
11777        let omitted_timestamp = false;
11778        let wrapped = if opts.header_template_enabled {
11779            let (template, _ty, flags, data_size, timestamp, nonce, reliable_seq_ack, payload) =
11780                extract_side_header_template_view(raw.as_ref())?;
11781            // Reliable and frozen-owner packets already returned above.
11782            // Classify once per encode rather than repeating a schema lookup.
11783            let (template_id, use_compact, _previous_timestamp) = {
11784                let mut st = self.state.lock();
11785                /* Keep the sender's compact dictionary within the smallest
11786                 * capacity advertised by a live peer on this side. Template
11787                 * IDs otherwise outlive entries on a smaller receiver and
11788                 * later compact values are silently undecodable. */
11789                #[cfg(feature = "discovery")]
11790                let mut effective_template_limit = opts.max_side_transport_templates;
11791                #[cfg(not(feature = "discovery"))]
11792                let effective_template_limit = opts.max_side_transport_templates;
11793                #[cfg(feature = "discovery")]
11794                {
11795                    if let Some(route) = st.discovery_routes.get(&side) {
11796                        let active_peers = route
11797                            .announcers
11798                            .values()
11799                            .filter(|sender| {
11800                                self.clock.now_ms().saturating_sub(sender.last_seen_ms)
11801                                    <= DISCOVERY_ROUTE_TTL_MS
11802                            })
11803                            .count();
11804                        /* Compact template state is intentionally side-local.
11805                         * On a shared bus, independent producers cannot divide
11806                         * a receiver's bounded dictionary between themselves,
11807                         * so their compact frames can evict one another. Keep
11808                         * compression for point-to-point links and
11809                         * automatically use self-describing full frames once
11810                         * discovery proves a side is multi-producer. */
11811                        if active_peers > 1 {
11812                            effective_template_limit = 0;
11813                        } else {
11814                            effective_template_limit = route
11815                                .announcers
11816                                .values()
11817                                .filter_map(|sender| sender.link_capabilities)
11818                                .filter(|caps| {
11819                                    caps.flags & discovery::LINK_CAPABILITY_HEADER_TEMPLATES != 0
11820                                        && caps.max_side_transport_templates > 0
11821                                })
11822                                .map(|caps| caps.max_side_transport_templates as usize)
11823                                .fold(effective_template_limit, core::cmp::min);
11824                        }
11825                    }
11826                }
11827                let trimmed = st
11828                    .side_transport
11829                    .get_mut(&side)
11830                    .ok_or(TelemetryError::BadArg)?
11831                    .trim_tx_templates_to(effective_template_limit);
11832                for _ in 0..trimmed {
11833                    st.side_runtime_stats
11834                        .entry(side)
11835                        .or_default()
11836                        .note_side_transport_template_eviction();
11837                }
11838                let side_state = st
11839                    .side_transport
11840                    .get_mut(&side)
11841                    .ok_or(TelemetryError::BadArg)?;
11842                if let Some(id) = side_state.tx_template_ids.get(&template.hash).copied() {
11843                    let previous = side_state.tx_last_timestamps.get(&id).copied();
11844                    /* A full refresh makes compact transport self-healing when
11845                     * the receiver missed the original template frame. */
11846                    const FULL_REFRESH_AFTER_COMPACT_FRAMES: u8 = 8;
11847                    let compact_uses = side_state.tx_compact_uses.entry(id).or_default();
11848                    // Discovery must bootstrap even when a receiver lost its dictionary.
11849                    #[cfg(feature = "discovery")]
11850                    let discovery_refresh = discovery::is_discovery_type(_ty);
11851                    #[cfg(not(feature = "discovery"))]
11852                    let discovery_refresh = false;
11853                    let use_compact =
11854                        *compact_uses < FULL_REFRESH_AFTER_COMPACT_FRAMES && !discovery_refresh;
11855                    if use_compact {
11856                        *compact_uses = compact_uses.saturating_add(1);
11857                    } else {
11858                        *compact_uses = 0;
11859                    }
11860                    (id, use_compact, previous)
11861                } else {
11862                    /* Sequential template IDs collide when multiple routers
11863                     * transmit on one physical bus. The template hash covers
11864                     * the source address and immutable header fields, making
11865                     * this ID producer-stable without adding bytes to compact
11866                     * frames. */
11867                    let template_id = ((template.hash >> 32) as u32 ^ template.hash as u32).max(1);
11868                    let evicted = side_state.insert_tx_template(
11869                        template.into_owned(),
11870                        template_id,
11871                        effective_template_limit,
11872                    );
11873                    if evicted {
11874                        st.side_runtime_stats
11875                            .entry(side)
11876                            .or_default()
11877                            .note_side_transport_template_eviction();
11878                    }
11879                    if let Some(side_state) = st.side_transport.get_mut(&side)
11880                        && effective_template_limit > 0
11881                    {
11882                        side_state.tx_last_timestamps.insert(template_id, timestamp);
11883                    }
11884                    (template_id, false, None)
11885                }
11886            };
11887            if use_compact {
11888                used_compact = true;
11889                compact_payload_len = Some(payload.len());
11890                // A previous transmitted frame is not a confirmed receive base.
11891                // Absolute timestamps preserve identity after loss or reordering.
11892                let timestamp_field = Some(timestamp);
11893                let mut body = Vec::with_capacity(payload.len() + 32);
11894                body.push(flags);
11895                write_uleb128_local(u64::from(template_id), &mut body);
11896                write_uleb128_local(data_size, &mut body);
11897                if let Some(timestamp_field) = timestamp_field {
11898                    write_uleb128_local(timestamp_field, &mut body);
11899                }
11900                if (flags & SIDE_TRANSPORT_FLAG_PACKET_NONCE) != 0 {
11901                    write_uleb128_local(u64::from(nonce), &mut body);
11902                }
11903                if let Some((seq, ack)) = reliable_seq_ack {
11904                    write_uleb128_local(u64::from(seq), &mut body);
11905                    write_uleb128_local(u64::from(ack), &mut body);
11906                }
11907                body.extend_from_slice(payload);
11908                {
11909                    let mut st = self.state.lock();
11910                    if let Some(side_state) = st.side_transport.get_mut(&side) {
11911                        side_state.tx_last_timestamps.insert(template_id, timestamp);
11912                    }
11913                }
11914                let kind = if omitted_timestamp {
11915                    SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
11916                } else if used_timestamp_delta {
11917                    SIDE_TRANSPORT_KIND_COMPACT_DELTA
11918                } else {
11919                    SIDE_TRANSPORT_KIND_COMPACT
11920                };
11921                wrap_side_transport_frame(kind, &body)
11922            } else {
11923                let mut body = Vec::with_capacity(raw.len() + 4);
11924                write_uleb128_local(u64::from(template_id), &mut body);
11925                body.extend_from_slice(raw.as_ref());
11926                wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, &body)
11927            }
11928        } else {
11929            wrap_side_transport_frame(SIDE_TRANSPORT_KIND_FULL, raw.as_ref())
11930        };
11931
11932        let frames = if opts.max_frame_bytes != 0 && wrapped.len() > opts.max_frame_bytes {
11933            self.split_side_transport_frame(side, wrapped, opts.max_frame_bytes)
11934        } else {
11935            Ok(SideTransportFrames::single(wrapped))
11936        }?;
11937        let wire_len = frames.wire_len();
11938        let mut st = self.state.lock();
11939        let stats = st.side_runtime_stats.entry(side).or_default();
11940        if used_compact {
11941            let overhead = compact_payload_len
11942                .map(|payload_len| wire_len.saturating_sub(payload_len))
11943                .unwrap_or(wire_len);
11944            stats.note_side_transport_compact(
11945                raw_len,
11946                wire_len,
11947                overhead,
11948                used_timestamp_delta,
11949                omitted_timestamp,
11950            );
11951            if opts.compact_header_target_bytes != 0 && overhead > opts.compact_header_target_bytes
11952            {
11953                stats.note_side_transport_compact_target_miss();
11954            }
11955        } else {
11956            stats.note_side_transport_full(raw_len, wire_len);
11957        }
11958        if frames.len() > 1 {
11959            stats.note_side_transport_chunks(frames.len());
11960        }
11961        Ok(frames)
11962    }
11963
11964    fn expire_side_chunks(&self) {
11965        let now = self.clock.now_ms();
11966        let mut st = self.state.lock();
11967        for side in st.side_transport.values_mut() {
11968            let side = side.as_mut();
11969            #[cfg(all(feature = "compression", feature = "discovery"))]
11970            side.payload_codec.expire(now, DISCOVERY_ROUTE_TTL_MS);
11971            let reclaimed =
11972                SideChunkAssembly::expire(&mut side.rx_chunks, &mut side.rx_chunk_bytes, now);
11973            side.rx_chunk_reclaims = side.rx_chunk_reclaims.saturating_add(reclaimed);
11974        }
11975    }
11976
11977    // A validated frame from a previously discovered direct peer proves that
11978    // its link is alive, even when allocation admission refuses its payload.
11979    // Update existing timestamps only: no allocation, schema merge or new route.
11980    #[cfg(feature = "discovery")]
11981    fn refresh_ingress_peer_liveness<'a>(
11982        &self,
11983        side: RouterSideId,
11984        bytes: &'a [u8],
11985    ) -> Option<wire_format::RoutingFrameInfo<'a>> {
11986        let frame = wire_format::peek_routing_frame_info(bytes).ok()?;
11987        self.refresh_peer_liveness(side, frame.envelope.source_address);
11988        #[cfg(feature = "compression")]
11989        if frame.envelope.ty == crate::DataType::DiscoveryAddress
11990            && let Ok(Some((_, payload))) =
11991                wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryAddress)
11992        {
11993            self.note_payload_codec_address(side, payload, false);
11994        }
11995        Some(frame)
11996    }
11997
11998    #[cfg(feature = "discovery")]
11999    #[cfg_attr(not(feature = "std"), inline(never))]
12000    fn refresh_peer_liveness(&self, side: RouterSideId, address: u32) {
12001        let now = self.clock.now_ms();
12002        let mut st = self.state.lock();
12003        #[cfg(feature = "compression")]
12004        if let Some(transport) = st.side_transport.get_mut(&side) {
12005            transport
12006                .payload_codec
12007                .observe(address, now, DISCOVERY_ROUTE_TTL_MS);
12008        }
12009        let Some(route) = st.discovery_routes.get_mut(&side) else {
12010            return;
12011        };
12012        let mut found = None;
12013        for (name, peer) in &mut route.announcers {
12014            if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12015                // Ambiguous numeric identities must be resolved by discovery.
12016                if found.is_some() {
12017                    return;
12018                }
12019                found = Some(peer);
12020            }
12021        }
12022        let Some(peer) = found else {
12023            return;
12024        };
12025        peer.last_seen_ms = now;
12026        route.last_seen_ms = route.last_seen_ms.max(now);
12027    }
12028
12029    /// Consume only an exact unsequenced topology retry on its ingress link.
12030    /// CRC, shape, payload length and reliable-header restrictions are checked
12031    /// before this path; it cannot bypass receipts, update a graph or allocate.
12032    #[cfg(feature = "discovery")]
12033    fn consume_known_packed_topology(
12034        &self,
12035        side: RouterSideId,
12036        bytes: &[u8],
12037    ) -> TelemetryResult<bool> {
12038        let Some((address, payload)) =
12039            wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoveryTopology)?
12040        else {
12041            return Ok(false);
12042        };
12043        let now = self.clock.now_ms();
12044        let mut st = self.state.lock();
12045        let Some(route) = st.discovery_routes.get_mut(&side) else {
12046            return Ok(false);
12047        };
12048        let mut found = None;
12049        for (name, peer) in &mut route.announcers {
12050            if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12051                if found.is_some() {
12052                    return Ok(false);
12053                }
12054                found = Some(peer);
12055            }
12056        }
12057        let Some(peer) = found else {
12058            return Ok(false);
12059        };
12060        let Some(full) =
12061            discovery::topology_payload_matches_retained(payload, &peer.topology_boards)
12062        else {
12063            return Ok(false);
12064        };
12065        peer.has_full_topology |= full;
12066        peer.last_seen_ms = now;
12067        route.last_seen_ms = route.last_seen_ms.max(now);
12068        Ok(true)
12069    }
12070
12071    #[cfg(all(feature = "discovery", not(feature = "std")))]
12072    fn consume_known_packed_schema(
12073        &self,
12074        side: RouterSideId,
12075        bytes: &[u8],
12076    ) -> TelemetryResult<bool> {
12077        let Some((address, payload)) =
12078            wire_format::peek_plain_discovery_payload(bytes, crate::DataType::DiscoverySchema)?
12079        else {
12080            return Ok(false);
12081        };
12082        self.refresh_peer_liveness(side, address);
12083        self.merge_schema_changes(payload)?;
12084        let mut st = self.state.lock();
12085        if let Some(route) = st.discovery_routes.get_mut(&side) {
12086            for (name, peer) in &mut route.announcers {
12087                if peer.advertised_address == Some(address) || sender_address_u32(name) == address {
12088                    peer.has_schema = true;
12089                    break;
12090                }
12091            }
12092        }
12093        Ok(true)
12094    }
12095
12096    #[cfg(all(feature = "compression", feature = "discovery"))]
12097    fn decode_required_schema<T>(
12098        &self,
12099        mut decode: impl FnMut() -> TelemetryResult<T>,
12100    ) -> TelemetryResult<T> {
12101        loop {
12102            match decode() {
12103                Err(error @ TelemetryError::Io("memory pressure")) => {
12104                    if !self.state.lock().reclaim_codec_metadata() {
12105                        return Err(error);
12106                    }
12107                }
12108                result => return result,
12109            }
12110        }
12111    }
12112
12113    #[cfg(all(feature = "discovery", not(feature = "std")))]
12114    #[inline(never)]
12115    fn merge_schema_changes(&self, payload: &[u8]) -> TelemetryResult<()> {
12116        // Validate the entire catalog, then materialize only changed entries.
12117        // Packed and already-decoded ingress share one atomic budgeted merge.
12118        #[cfg(feature = "compression")]
12119        let snapshot = self.decode_required_schema(|| discovery::decode_schema_changes(payload))?;
12120        #[cfg(not(feature = "compression"))]
12121        let snapshot = discovery::decode_schema_changes(payload)?;
12122        let mut st = self.state.lock();
12123        let budget = st.schema_growth_budget();
12124        let report = crate::config::merge_owned_schema_snapshot_with_budget(snapshot, budget)?;
12125        if report.changed() {
12126            st.fit_discovery_budget();
12127        }
12128        Ok(())
12129    }
12130
12131    /// Ignore an overheard canonical frame only when every frozen holder is
12132    /// positively known behind its ingress. Unknown/stale routes, runtime
12133    /// schema changes, managed state, hop receipts and explicit policies keep
12134    /// the ordinary receive path. No payload copy, queue slot or ACK is owned.
12135    #[cfg(all(
12136        feature = "discovery",
12137        any(feature = "std", feature = "ingress-target-filter")
12138    ))]
12139    fn targets_only_ingress(
12140        &self,
12141        side: RouterSideId,
12142        frame: &wire_format::RoutingFrameInfo<'_>,
12143    ) -> bool {
12144        let ty = frame.envelope.ty;
12145        // Most packets are untargeted bulk data: do not add a schema lookup
12146        // or registry lock to that existing hot path.
12147        if is_internal_control_type(ty)
12148            || frame.envelope.target_senders().len() == 0
12149            || frame.reliable.is_some_and(|h| {
12150                h.flags != wire_format::RELIABLE_FLAG_UNSEQUENCED || h.seq != 0 || h.ack != 0
12151            })
12152        {
12153            return false;
12154        }
12155        let Some(meta) = crate::config::try_get_message_meta(ty) else {
12156            return false;
12157        };
12158        if frame
12159            .envelope
12160            .wire_shape
12161            .is_some_and(|shape| shape != meta.element)
12162        {
12163            return false;
12164        }
12165        let local_hash = Self::sender_hash(self.sender_arc().as_ref());
12166        let st = self.state.lock();
12167        if st.is_managed_variable(ty.as_u32())
12168            || Self::has_explicit_route_policy_locked(&st, Some(side), ty)
12169        {
12170            return false;
12171        }
12172        let now = self.clock.now_ms();
12173        frame.envelope.target_senders().all(|target| {
12174            target != local_hash
12175                && target != u64::from(st.local_address.address)
12176                && Self::side_matches_target_senders_locked(
12177                    &st,
12178                    side,
12179                    wire_format::TargetSenderSet::Hashes(&[target]),
12180                    now,
12181                )
12182                && !st.sides.iter().enumerate().any(|(other, entry)| {
12183                    other != side
12184                        && entry.as_ref().is_some_and(|e| e.opts.egress_enabled)
12185                        && Self::side_matches_target_senders_locked(
12186                            &st,
12187                            other,
12188                            wire_format::TargetSenderSet::Hashes(&[target]),
12189                            now,
12190                        )
12191                })
12192        })
12193    }
12194
12195    // A two-sided, fully warmed packed bridge needs no new large metadata.
12196    // Re-check before dispatch: schemas/templates/routes can change while queued.
12197    #[cfg(feature = "discovery")]
12198    fn cached_transit_ready(
12199        &self,
12200        side: RouterSideId,
12201        bytes: &[u8],
12202        view: Option<&wire_format::RoutingFrameInfo<'_>>,
12203    ) -> TelemetryResult<bool> {
12204        // Compressed payloads stay opaque on this fully warmed packed-only
12205        // path. No decompression workspace is needed unless the current schema,
12206        // route or local/managed role changes (all rechecked below).
12207        // Encrypted frames retain their independent conservative admission.
12208        if bytes.len() > 128 || bytes.first().is_none_or(|flags| flags & 0x10 != 0) {
12209            return Ok(false);
12210        }
12211        let parsed;
12212        let frame = match view {
12213            Some(frame) => frame,
12214            None => {
12215                parsed = wire_format::peek_routing_frame_info(bytes)?;
12216                &parsed
12217            }
12218        };
12219        let ty = frame.envelope.ty;
12220        if is_internal_control_type(ty)
12221            || frame.envelope.is_reliable
12222            || frame.reliable.is_some()
12223            || frame.envelope.target_senders().len() != 0
12224            // A matching inline schema is immutable metadata on the existing
12225            // shared frame. It needs no owned decoder state. Unknown or changed
12226            // shapes still require full admission and payload decoding.
12227            || frame.envelope.wire_shape.is_some_and(|shape| {
12228                shape != crate::message_meta(ty).element
12229            })
12230            || !(1..=3).contains(&frame.envelope.endpoints().take(4).count())
12231        {
12232            return Ok(false);
12233        }
12234        if self.is_managed_variable_type(ty)
12235            || frame.envelope.endpoints().any(|ep| {
12236                self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
12237            })
12238        {
12239            return Ok(false);
12240        }
12241
12242        // A known peer's bounded canonical frame needs no optional dictionary
12243        // entry. Avoid parsing/hashing that header merely to rediscover this
12244        // fallback. Recheck peer/route state below before granting admission.
12245        let canonical_possible = {
12246            let st = self.state.lock();
12247            st.discovery_routes.get(&side).is_some_and(|route| {
12248                route.announcers.iter().any(|(name, peer)| {
12249                    peer.advertised_address == Some(frame.envelope.source_address)
12250                        || sender_address_u32(name) == frame.envelope.source_address
12251                })
12252            }) && st.sides.iter().flatten().all(|entry| {
12253                entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes
12254            })
12255        };
12256        let template_hash = if canonical_possible {
12257            None
12258        } else {
12259            Some(extract_side_header_template_view(bytes)?.0.hash)
12260        };
12261        let st = self.state.lock();
12262        let active: usize = st.sides.iter().filter(|entry| entry.is_some()).count();
12263        if active != 2 {
12264            return Ok(false);
12265        }
12266        for (id, entry) in st
12267            .sides
12268            .iter()
12269            .enumerate()
12270            .filter_map(|(id, entry)| entry.as_ref().map(|e| (id, e)))
12271        {
12272            if entry.opts.reliable_enabled
12273                || !matches!(
12274                    entry.tx_handler,
12275                    RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
12276                )
12277                || !st
12278                    .side_runtime_stats
12279                    .get(&id)
12280                    .is_some_and(|stats| stats.data_types.contains_key(&ty.as_u32()))
12281            {
12282                return Ok(false);
12283            }
12284            if id == side {
12285                continue;
12286            }
12287            #[cfg(feature = "discovery")]
12288            if !st.discovery_routes.get(&id).is_some_and(|route| {
12289                self.clock.now_ms().saturating_sub(route.last_seen_ms) <= DISCOVERY_ROUTE_TTL_MS
12290                    && frame
12291                        .envelope
12292                        .endpoints()
12293                        .any(|ep| route.reachable.contains(&ep))
12294            }) {
12295                return Ok(false);
12296            }
12297            if entry.opts.header_template_enabled
12298                && entry.opts.max_frame_bytes != 0
12299                && entry.opts.max_side_transport_templates != 0
12300                && !self.side_has_multiple_announcers_locked(&st, id, self.clock.now_ms())
12301                && !template_hash.is_some_and(|hash| {
12302                    st.side_transport.get(&id).is_some_and(|transport| {
12303                        transport.tx_template_ids.contains_key(&hash)
12304                            && transport.tx_templates.contains_key(&hash)
12305                    })
12306                })
12307            {
12308                // An optional compression entry may have been evicted while
12309                // routing/statistics remain warm. Canonical forwarding needs
12310                // no replacement entry, but must fit the transport and belong
12311                // to an already discovered ingress peer.
12312                if (entry.opts.max_frame_bytes == 0 || bytes.len() <= entry.opts.max_frame_bytes)
12313                    && st.discovery_routes.get(&side).is_some_and(|route| {
12314                        route.announcers.iter().any(|(name, peer)| {
12315                            peer.advertised_address == Some(frame.envelope.source_address)
12316                                || sender_address_u32(name) == frame.envelope.source_address
12317                        })
12318                    })
12319                {
12320                    continue;
12321                }
12322                return Ok(false);
12323            }
12324        }
12325        Ok(true)
12326    }
12327    #[cfg(not(feature = "discovery"))]
12328    fn cached_transit_ready(
12329        &self,
12330        _side: RouterSideId,
12331        _bytes: &[u8],
12332        _view: Option<&wire_format::RoutingFrameInfo<'_>>,
12333    ) -> TelemetryResult<bool> {
12334        Ok(false)
12335    }
12336    #[cfg(any(feature = "std", feature = "fair-admission"))]
12337    fn reclaim_for_receive(&self, bytes: &[u8], victim: &mut Option<(u32, u8)>) -> bool {
12338        let Some((incoming, priority)) = crate::admission_fairness::incoming(bytes) else {
12339            return false;
12340        };
12341        self.reclaim_for_incoming(incoming, priority, victim)
12342    }
12343    #[cfg(any(feature = "std", feature = "fair-admission"))]
12344    fn reclaim_for_incoming(
12345        &self,
12346        incoming: u32,
12347        priority: u8,
12348        victim: &mut Option<(u32, u8)>,
12349    ) -> bool {
12350        let mut st = self.state.lock();
12351        let (owner, priority) = if let Some(selected) = *victim {
12352            selected
12353        } else {
12354            let Some(owner) = st.competing_bulk_owner(incoming, priority) else {
12355                return false;
12356            };
12357            *victim = Some((owner, priority));
12358            (owner, priority)
12359        };
12360        st.reclaim_bulk_from(owner, priority, None)
12361    }
12362    fn admit_received_frame(&self, side: RouterSideId, bytes: &[u8]) -> TelemetryResult<()> {
12363        self.admit_received_frame_with_view(side, bytes, None)
12364    }
12365
12366    // Reuse only the view validated in this ingress call. Queue dispatch still
12367    // rechecks bytes and current route/schema state independently.
12368    fn admit_received_frame_with_view(
12369        &self,
12370        side: RouterSideId,
12371        bytes: &[u8],
12372        view: Option<&wire_format::RoutingFrameInfo<'_>>,
12373    ) -> TelemetryResult<()> {
12374        if !crate::memory_admission::has_probe() {
12375            return Ok(());
12376        }
12377        #[cfg(any(feature = "std", feature = "fair-admission"))]
12378        let mut fair_victim = None;
12379        if crate::memory_admission::supports_work_kinds()
12380            && self.cached_transit_ready(side, bytes, view)?
12381        {
12382            let result = crate::memory_admission::check_cached_transit(bytes.len());
12383            #[cfg(all(feature = "compression", feature = "discovery"))]
12384            let result = {
12385                let mut result = result;
12386                while matches!(result, Err(TelemetryError::Io("memory pressure")))
12387                    && self.state.lock().reclaim_codec_metadata()
12388                {
12389                    result = crate::memory_admission::check_cached_transit(bytes.len());
12390                }
12391                result
12392            };
12393
12394            #[cfg(any(feature = "std", feature = "fair-admission"))]
12395            let result = {
12396                let mut result = result;
12397                while matches!(result, Err(TelemetryError::Io("memory pressure"))) {
12398                    if !self.reclaim_for_receive(bytes, &mut fair_victim) {
12399                        break;
12400                    }
12401                    result = crate::memory_admission::check_cached_transit(bytes.len());
12402                }
12403                result
12404            };
12405            result
12406        } else {
12407            let managed = wire_format::scheduling_type(bytes)
12408                .is_some_and(|ty| self.is_managed_variable_type(ty));
12409            let mut protected = None;
12410            loop {
12411                match crate::memory_admission::check_receive_with_role(bytes, managed) {
12412                    Err(TelemetryError::Io("memory pressure")) => {
12413                        #[cfg(all(feature = "compression", feature = "discovery"))]
12414                        if self.state.lock().reclaim_codec_metadata() {
12415                            continue;
12416                        }
12417
12418                        let protected = *protected.get_or_insert_with(|| {
12419                            wire_format::peek_routing_frame_info(bytes)
12420                                .ok()
12421                                .is_some_and(|frame| {
12422                                    managed
12423                                        || message_priority(frame.envelope.ty) >= 200
12424                                        || crate::small_frozen_control(
12425                                            frame.envelope.ty,
12426                                            frame.envelope.target_senders().len(),
12427                                            frame.envelope.wire_shape,
12428                                        )
12429                                })
12430                        });
12431                        if !protected {
12432                            #[cfg(any(feature = "std", feature = "fair-admission"))]
12433                            if self.reclaim_for_receive(bytes, &mut fair_victim) {
12434                                continue;
12435                            }
12436                            return Err(TelemetryError::Io("memory pressure"));
12437                        }
12438                        // Queue priority cannot help before the wire copy is
12439                        // admitted. Reclaim bulk payloads, preserving control
12440                        // traffic and every learned route/schema.
12441                        let mut st = self.state.lock();
12442                        let reclaimed = st
12443                            .lower_priority_shared_queue(199)
12444                            .is_some_and(|queue| st.pop_shared_queue_item(queue));
12445                        if !reclaimed {
12446                            return Err(TelemetryError::Io("memory pressure"));
12447                        }
12448                    }
12449                    result => return result,
12450                }
12451            }
12452        }
12453    }
12454
12455    #[cfg(feature = "discovery")]
12456    fn consume_known_packed_discovery(
12457        &self,
12458        side: RouterSideId,
12459        bytes: &[u8],
12460    ) -> TelemetryResult<bool> {
12461        // Ordinary telemetry pays one bounded type peek, not a parse for each
12462        // discovery subtype. The selected consumer still checks CRC and shape.
12463        let result = match wire_format::scheduling_type(bytes) {
12464            #[cfg(not(feature = "std"))]
12465            Some(crate::DataType::DiscoverySchema) => self.consume_known_packed_schema(side, bytes),
12466            Some(crate::DataType::DiscoveryTopology) => {
12467                self.consume_known_packed_topology(side, bytes)
12468            }
12469            Some(crate::DataType::DiscoveryAddress) => {
12470                match wire_format::peek_plain_discovery_payload(
12471                    bytes,
12472                    crate::DataType::DiscoveryAddress,
12473                ) {
12474                    Ok(Some((address, payload))) => {
12475                        self.refresh_peer_liveness(side, address);
12476                        self.refresh_known_address_payload(payload, side)
12477                            .map(|known| known && self.cfg.role.local_services())
12478                    }
12479                    Ok(None) => Ok(false),
12480                    Err(error) => Err(error),
12481                }
12482            }
12483            _ => Ok(false),
12484        };
12485        if result.is_err() {
12486            // Preserve the existing header-level liveness behavior on rejected
12487            // payloads. A failed CRC still cannot refresh a peer. Valid known
12488            // discovery avoids this second verification on the normal path.
12489            let _ = self.refresh_ingress_peer_liveness(side, bytes);
12490        }
12491        result
12492    }
12493
12494    fn decode_side_transport_frame(
12495        &self,
12496        side: RouterSideId,
12497        bytes: &[u8],
12498    ) -> TelemetryResult<Option<crate::SharedBytes>> {
12499        // Reclaim abandoned transfers before admission, even when new work is refused.
12500        self.expire_side_chunks();
12501        let Some((kind, body)) = parse_side_transport_wrapper(bytes)? else {
12502            #[cfg(feature = "discovery")]
12503            if self.consume_known_packed_discovery(side, bytes)? {
12504                return Ok(None);
12505            }
12506            #[cfg(feature = "discovery")]
12507            let view = self.refresh_ingress_peer_liveness(side, bytes);
12508            #[cfg(not(feature = "discovery"))]
12509            let view = None;
12510            #[cfg(all(
12511                feature = "discovery",
12512                any(feature = "std", feature = "ingress-target-filter")
12513            ))]
12514            if bytes.first().is_some_and(|flags| flags & 0x10 == 0)
12515                && view
12516                    .as_ref()
12517                    .is_some_and(|frame| self.targets_only_ingress(side, frame))
12518            {
12519                return Ok(None);
12520            }
12521            self.admit_received_frame_with_view(side, bytes, view.as_ref())?;
12522            return Ok(Some(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes))));
12523        };
12524        match kind {
12525            SIDE_TRANSPORT_KIND_FULL => {
12526                let header_templates_enabled = self
12527                    .state
12528                    .lock()
12529                    .sides
12530                    .get(side)
12531                    .and_then(|entry| entry.as_ref())
12532                    .map(|side| side.opts.header_template_enabled)
12533                    .unwrap_or(false);
12534                let mut off = 0usize;
12535                let template_id = if header_templates_enabled {
12536                    Some(
12537                        u32::try_from(read_uleb128_local(body, &mut off)?)
12538                            .map_err(|_| TelemetryError::Unpack("side template id too large"))?,
12539                    )
12540                } else {
12541                    None
12542                };
12543                #[cfg(feature = "discovery")]
12544                if self.consume_known_packed_discovery(side, &body[off..])? {
12545                    return Ok(None);
12546                }
12547                #[cfg(feature = "discovery")]
12548                let view = self.refresh_ingress_peer_liveness(side, &body[off..]);
12549                #[cfg(not(feature = "discovery"))]
12550                let view = None;
12551                self.admit_received_frame_with_view(side, &body[off..], view.as_ref())?;
12552                let raw = Arc::<[u8]>::from(&body[off..]);
12553                if let (Some(template_id), Ok((template, _, _, _, timestamp, _, _, _))) =
12554                    (template_id, extract_side_header_template(raw.as_ref()))
12555                {
12556                    let mut st = self.state.lock();
12557                    let max_templates = st
12558                        .sides
12559                        .get(side)
12560                        .and_then(|side| side.as_ref())
12561                        .map(|side| side.opts.max_side_transport_templates)
12562                        .unwrap_or(DEFAULT_SIDE_TRANSPORT_TEMPLATE_LIMIT);
12563                    let evicted = st.side_transport.get_mut(&side).is_some_and(|side_state| {
12564                        let evicted =
12565                            side_state.insert_rx_template(template_id, template, max_templates);
12566                        if max_templates > 0 {
12567                            side_state.rx_last_timestamps.insert(template_id, timestamp);
12568                        }
12569                        evicted
12570                    });
12571                    if evicted {
12572                        st.side_runtime_stats
12573                            .entry(side)
12574                            .or_default()
12575                            .note_side_transport_template_eviction();
12576                    }
12577                }
12578                Ok(Some(crate::shared_bytes::convert(raw)))
12579            }
12580            SIDE_TRANSPORT_KIND_COMPACT
12581            | SIDE_TRANSPORT_KIND_COMPACT_DELTA
12582            | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => {
12583                if body.is_empty() {
12584                    return Err(TelemetryError::Unpack("short side compact frame"));
12585                }
12586                let mut off = 1usize;
12587                let template_id = u32::try_from(read_uleb128_local(body, &mut off)?)
12588                    .map_err(|_| TelemetryError::Unpack("side template id too large"))?;
12589                let (template, timestamp_base) = {
12590                    let st = self.state.lock();
12591                    let state = st.side_transport.get(&side);
12592                    let template = state
12593                        .and_then(|state| state.rx_templates_by_id.get(&template_id))
12594                        .cloned();
12595                    let timestamp_base = if matches!(
12596                        kind,
12597                        SIDE_TRANSPORT_KIND_COMPACT_DELTA
12598                            | SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP
12599                    ) {
12600                        state
12601                            .and_then(|state| state.rx_last_timestamps.get(&template_id))
12602                            .copied()
12603                    } else {
12604                        None
12605                    };
12606                    (template, timestamp_base)
12607                };
12608                /* A full template frame can be lost on a datagram/radio side.
12609                 * Drop this compact frame without tearing down the transport;
12610                 * the sender periodically emits a full refresh. */
12611                let Some(template) = template else {
12612                    if !self.cfg.role.local_services() {
12613                        return Err(TelemetryError::Unpack("unknown side compact template"));
12614                    }
12615                    return Ok(None);
12616                };
12617                // The wrapper CRC and known template identify this peer even
12618                // if payload allocation is refused. No payload copy is needed.
12619                #[cfg(feature = "discovery")]
12620                if let Some(address) = template.source_address() {
12621                    self.refresh_peer_liveness(side, address);
12622                }
12623                let reconstructed_bound = body
12624                    .len()
12625                    .saturating_add(template.prefix().len())
12626                    .saturating_add(template.between().len())
12627                    .saturating_add(32);
12628                // Wrapper CRC and the retained full template are validated
12629                // before compact reconstruction. Preserve the same control
12630                // priority as canonical ingress while reclaiming only bulk.
12631                let protected = template.data_type().is_some_and(|ty| {
12632                    message_priority(ty) >= 200 || self.is_managed_variable_type(ty)
12633                });
12634                let ack = template.is_ack();
12635                #[cfg(any(feature = "std", feature = "fair-admission"))]
12636                let mut fair_incoming = None;
12637                #[cfg(any(feature = "std", feature = "fair-admission"))]
12638                let mut fair_victim = None;
12639                loop {
12640                    match crate::memory_admission::check_compact_reconstruction(
12641                        reconstructed_bound,
12642                        ack,
12643                    ) {
12644                        Err(TelemetryError::Io("memory pressure")) if protected => {
12645                            let mut st = self.state.lock();
12646                            let reclaimed = st
12647                                .lower_priority_shared_queue(199)
12648                                .is_some_and(|queue| st.pop_shared_queue_item(queue));
12649                            if !reclaimed {
12650                                return Err(TelemetryError::Io("memory pressure"));
12651                            }
12652                        }
12653                        Err(TelemetryError::Io("memory pressure")) => {
12654                            // Compact reports need the same fairness before
12655                            // allocation as canonical ingress. Otherwise they
12656                            // never reach the canonical admission/reclaim path.
12657                            #[cfg(any(feature = "std", feature = "fair-admission"))]
12658                            if let Some((incoming, priority)) =
12659                                *fair_incoming.get_or_insert_with(|| {
12660                                    crate::admission_fairness::compact_incoming(&template)
12661                                })
12662                                && self.reclaim_for_incoming(incoming, priority, &mut fair_victim)
12663                            {
12664                                continue;
12665                            }
12666                            return Err(TelemetryError::Io("memory pressure"));
12667                        }
12668                        result => {
12669                            result?;
12670                            break;
12671                        }
12672                    }
12673                }
12674                let timestamp_mode = match kind {
12675                    SIDE_TRANSPORT_KIND_COMPACT_DELTA => SideCompactTimestampMode::Delta,
12676                    SIDE_TRANSPORT_KIND_COMPACT_SAME_TIMESTAMP => SideCompactTimestampMode::Omitted,
12677                    _ => SideCompactTimestampMode::Absolute,
12678                };
12679                let (frame, timestamp) = reconstruct_side_compact_frame(
12680                    &template,
12681                    body[0],
12682                    &body[off..],
12683                    timestamp_mode,
12684                    timestamp_base,
12685                )?;
12686                {
12687                    let mut st = self.state.lock();
12688                    if let Some(side_state) = st.side_transport.get_mut(&side) {
12689                        side_state.rx_last_timestamps.insert(template_id, timestamp);
12690                    }
12691                }
12692                // Byte reconstruction is not decoder/forwarding admission.
12693                // Check the canonical frame before queueing, and recheck at
12694                // dispatch if routes, schemas or available memory have changed.
12695                if crate::memory_admission::supports_work_kinds() {
12696                    self.admit_received_frame(side, frame.as_ref())?;
12697                }
12698                Ok(Some(frame))
12699            }
12700            SIDE_TRANSPORT_KIND_CHUNK => {
12701                if body.len() < 8 {
12702                    return Err(TelemetryError::Unpack("short side chunk frame"));
12703                }
12704                let transfer_id = u32::from_le_bytes([body[0], body[1], body[2], body[3]]);
12705                let index = u16::from_le_bytes([body[4], body[5]]);
12706                let total = u16::from_le_bytes([body[6], body[7]]);
12707                if total == 0 || index >= total {
12708                    return Err(TelemetryError::Unpack("invalid side chunk index"));
12709                }
12710                let assembled = {
12711                    let mut st = self.state.lock();
12712                    let now_ms = self.clock.now_ms();
12713                    st.make_side_chunk_room(side, transfer_id, index, total, body.len() - 8)?;
12714                    // A CRC-validated retry of a retained fragment needs no
12715                    // allocation. Do not discard other traffic or reject the
12716                    // retry because unrelated work consumed the idle reserve.
12717                    if let Some(entry) = st
12718                        .side_transport
12719                        .get_mut(&side)
12720                        .and_then(|state| state.rx_chunks.get_mut(&transfer_id))
12721                        && entry.contains_index(index)
12722                    {
12723                        // Retries are not progress: only new fragments extend
12724                        // the timeout of a retained, incomplete transfer.
12725                        return Ok(None);
12726                    }
12727                    // Budget only insertion into an existing bounded tree.
12728                    // The full packet has independent assembly/decode guards.
12729                    let retained = st
12730                        .side_transport
12731                        .get(&side)
12732                        .and_then(|state| state.rx_chunks.get(&transfer_id))
12733                        .map(|entry| entry.received_len());
12734                    loop {
12735                        match crate::memory_admission::check_retained_fragment(bytes, retained) {
12736                            Err(TelemetryError::Io("memory pressure")) => {
12737                                if st
12738                                    .lower_priority_shared_queue(199)
12739                                    .is_some_and(|queue| st.pop_shared_queue_item(queue))
12740                                    || st.reclaim_oldest_side_chunk(Some((side, transfer_id)))
12741                                {
12742                                    continue;
12743                                }
12744                                return Err(TelemetryError::Io("memory pressure"));
12745                            }
12746                            result => {
12747                                result?;
12748                                break;
12749                            }
12750                        }
12751                    }
12752                    let side_state = st
12753                        .side_transport
12754                        .get_mut(&side)
12755                        .ok_or(TelemetryError::BadArg)?;
12756                    let entry = side_state.rx_chunks.entry(transfer_id).or_default();
12757                    entry.total = total;
12758                    let added =
12759                        entry.insert_with(index, now_ms, || Arc::<[u8]>::from(&body[8..]))?;
12760                    side_state.rx_chunk_bytes = side_state.rx_chunk_bytes.saturating_add(added);
12761                    if entry.received_len() == usize::from(total) {
12762                        let entry = side_state
12763                            .take_chunk(transfer_id, false)
12764                            .ok_or(TelemetryError::Unpack("side chunk missing"))?;
12765                        Some(entry.assemble().inspect_err(|_| {
12766                            side_state.rx_chunk_reclaims =
12767                                side_state.rx_chunk_reclaims.saturating_add(1);
12768                        })?)
12769                    } else {
12770                        None
12771                    }
12772                };
12773                match assembled {
12774                    Some(frame) => self.decode_side_transport_frame(side, frame.as_ref()),
12775                    None => Ok(None),
12776                }
12777            }
12778            _ => Err(TelemetryError::Unpack("unknown side transport frame")),
12779        }
12780    }
12781
12782    fn packed_handler_call(
12783        &self,
12784        side: RouterSideId,
12785        handler: &RouterTxHandlerFn,
12786        bytes: &[u8],
12787        priority: u8,
12788        _service: EgressService,
12789    ) -> TelemetryResult<()> {
12790        let result = match handler {
12791            RouterTxHandlerFn::Packed(handler) => handler(bytes),
12792            RouterTxHandlerFn::PackedWithPriority(handler) => handler(bytes, priority),
12793            RouterTxHandlerFn::Packet(_) => Err(TelemetryError::BadArg),
12794        };
12795        #[cfg(feature = "latest-value-egress")]
12796        if result.is_ok() && _service.token != 0 && _service.side != 0 {
12797            let mut st = self.state.lock();
12798            if st.latest_service_seq == _service.token
12799                && st.latest_side() == Some(side)
12800                && side == usize::from(_service.side - 1)
12801                && matches!((_service.kind, st.latest_turn_phase), (1, 3) | (2, 4))
12802            {
12803                let bytes = i32::try_from(bytes.len()).unwrap_or(i32::MAX);
12804                match _service.kind {
12805                    1 => {
12806                        let cap = st
12807                            .sides
12808                            .get(side)
12809                            .and_then(Option::as_ref)
12810                            .map_or(0, |side| {
12811                                side.opts
12812                                    .max_frame_bytes
12813                                    .saturating_mul(usize::from(MAX_PRIORITY_BURST))
12814                                    .min(i32::MAX as usize) as i32
12815                            });
12816                        st.latest_turn_credit =
12817                            st.latest_turn_credit.saturating_add(bytes).min(cap);
12818                    }
12819                    2 => {
12820                        st.latest_turn_credit = st
12821                            .latest_turn_credit
12822                            .saturating_sub(bytes.saturating_mul(2))
12823                    }
12824                    _ => {}
12825                }
12826            }
12827        }
12828        #[cfg(not(feature = "latest-value-egress"))]
12829        let _ = side;
12830        result
12831    }
12832
12833    fn effective_transport_priority(&self, ty: DataType) -> u8 {
12834        if self.is_managed_variable_type(ty) {
12835            crate::transport_priority(DataType::ManagedVariableValue)
12836        } else {
12837            crate::transport_priority(ty)
12838        }
12839    }
12840
12841    #[cfg(all(test, feature = "std", feature = "discovery"))]
12842    fn call_side_tx_handler(
12843        &self,
12844        side: RouterSideId,
12845        handler: &RouterTxHandlerFn,
12846        data: &RouterItem,
12847        relayed: bool,
12848    ) -> TelemetryResult<()> {
12849        self.call_side_tx_handler_with_progress(side, handler, data, relayed, &mut false)
12850    }
12851
12852    // This branch streams canonical plaintext parts directly; no payload/header
12853    // dictionary, encryption origin, or side template is advanced by an offer.
12854    fn canonical_catalog_stream(packet: &Packet, max_frame_bytes: usize) -> bool {
12855        packet.data_type() == DataType::DiscoverySchema
12856            && max_frame_bytes != 0
12857            && packet.data_size() > max_frame_bytes
12858    }
12859
12860    #[cfg(all(test, feature = "std", feature = "discovery"))]
12861    fn call_side_tx_handler_with_progress(
12862        &self,
12863        side: RouterSideId,
12864        handler: &RouterTxHandlerFn,
12865        data: &RouterItem,
12866        relayed: bool,
12867        admitted: &mut bool,
12868    ) -> TelemetryResult<()> {
12869        self.call_side_tx_handler_with_service(
12870            side,
12871            handler,
12872            data,
12873            relayed,
12874            admitted,
12875            EgressService::default(),
12876        )
12877    }
12878    fn call_side_tx_handler_with_service(
12879        &self,
12880        side: RouterSideId,
12881        handler: &RouterTxHandlerFn,
12882        data: &RouterItem,
12883        relayed: bool,
12884        admitted: &mut bool,
12885        service: EgressService,
12886    ) -> TelemetryResult<()> {
12887        let opts = {
12888            let st = self.state.lock();
12889            Self::side_ref(&st, side)?.opts.clone()
12890        };
12891        let Some(_side_tx_guard) = self.try_enter_side_tx(side) else {
12892            return Err(TelemetryError::Io("side tx busy"));
12893        };
12894        let started_ms = self.clock.now_ms();
12895        // Type and priority share one CRC-validated envelope in this call.
12896        let (ty, priority) = self.item_service_type_priority(data)?;
12897        #[cfg(all(feature = "discovery", feature = "timesync"))]
12898        if !Self::timesync_allowed_for_side_locked(
12899            &self.state.lock(),
12900            side,
12901            ty,
12902            self.clock.now_ms(),
12903        ) {
12904            return Ok(());
12905        }
12906        let result = match (handler, data) {
12907            (
12908                packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
12909                RouterItem::Packed(bytes),
12910            ) => {
12911                #[cfg(feature = "cryptography")]
12912                let send_bytes = self.prepare_packed_for_remote(bytes.clone(), None)?;
12913                #[cfg(not(feature = "cryptography"))]
12914                let send_bytes = bytes.clone();
12915                let frames = self.encode_side_transport_frames(side, opts, send_bytes)?;
12916                let mut attempts_total = 0usize;
12917                let mut sent_bytes = 0usize;
12918                self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
12919                    match self.retry_with_attempts(runtime_max_handler_retries(), || {
12920                        self.packed_handler_call(side, packed, frame, priority, service)
12921                    }) {
12922                        Ok((_, attempts)) => {
12923                            *admitted = true;
12924                            attempts_total = attempts_total.saturating_add(attempts);
12925                            sent_bytes = sent_bytes.saturating_add(frame.len());
12926                        }
12927                        Err((err, attempts)) => {
12928                            self.note_side_tx_failure(
12929                                side,
12930                                ty,
12931                                attempts_total.saturating_add(attempts),
12932                            );
12933                            return Err(err);
12934                        }
12935                    }
12936
12937                    Ok(())
12938                })?;
12939                self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
12940                self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
12941                return Ok(());
12942            }
12943            (RouterTxHandlerFn::Packet(f), RouterItem::Packet(pkt)) => {
12944                self.retry_with_attempts(runtime_max_handler_retries(), || f(pkt))
12945            }
12946            (
12947                packed @ (RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)),
12948                RouterItem::Packet(pkt),
12949            ) => {
12950                // Internal schemas are plaintext and use self-describing frames.
12951                // This path is reached after hop-reliable handling; no replay
12952                // buffer is needed here. Stream fragments from the payload.
12953                if Self::canonical_catalog_stream(pkt, opts.max_frame_bytes) {
12954                    let mut attempts_total = 0usize;
12955                    let mut sent_bytes = 0usize;
12956                    let key = crate::side_transport::canonical_packet_progress_key(
12957                        pkt.packet_id(),
12958                        opts.max_frame_bytes,
12959                    );
12960                    #[cfg(feature = "latest-value-egress")]
12961                    let bounded_turn = {
12962                        let st = self.state.lock();
12963                        service.kind == 1
12964                            && service.side != 0
12965                            && usize::from(service.side - 1) == side
12966                            && service.token == st.latest_service_seq
12967                            && st.latest_side() == Some(side)
12968                    };
12969                    #[cfg(not(feature = "latest-value-egress"))]
12970                    let bounded_turn = false;
12971                    let mut admitted_chunks = 0usize;
12972                    let mut yielded = false;
12973                    let result = self.run_side_chunk_transfer(side, key, false, service, |next| {
12974                        crate::side_transport::send_packet_chunks(
12975                            pkt,
12976                            opts.max_frame_bytes,
12977                            self.sender_arc().as_bytes(),
12978                            next,
12979                            |frame| {
12980                                #[cfg(feature = "latest-value-egress")]
12981                                let fresh_waiting = if bounded_turn && admitted_chunks != 0 {
12982                                    let st = self.state.lock();
12983                                    st.latest_side() == Some(side) && st.transmit_queue.iter().any(|queued|
12984                                        queued.disposable & (TX_LATEST_PENDING | TX_LATEST_TURN) != 0
12985                                            && matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side))
12986                                } else { false };
12987                                #[cfg(not(feature = "latest-value-egress"))]
12988                                let fresh_waiting = bounded_turn && admitted_chunks != 0;
12989                                if fresh_waiting {
12990                                    yielded = true;
12991                                    return Err(TelemetryError::Io("side tx busy"));
12992                                }
12993                                match self.retry_with_attempts(
12994                                    runtime_max_handler_retries(),
12995                                    || {
12996                                        self.packed_handler_call(
12997                                            side, packed, frame, priority, service,
12998                                        )
12999                                    },
13000                                ) {
13001                                    Ok((_, attempts)) => {
13002                                        *admitted = true;
13003                                        attempts_total += attempts;
13004                                        sent_bytes += frame.len();
13005                                        admitted_chunks += 1;
13006                                        Ok(())
13007                                    }
13008                                    Err((err, attempts)) => {
13009                                        attempts_total += attempts;
13010                                        Err(err)
13011                                    }
13012                                }
13013                            },
13014                        )
13015                    });
13016                    let (raw_len, wire_len, chunks) = match result {
13017                        Ok(stats) => stats,
13018                        Err(err) => {
13019                            if !yielded {
13020                                self.note_side_tx_failure(side, ty, attempts_total);
13021                            }
13022                            return Err(err);
13023                        }
13024                    };
13025                    {
13026                        let mut st = self.state.lock();
13027                        let stats = st.side_runtime_stats.entry(side).or_default();
13028                        stats.note_side_transport_full(raw_len, wire_len);
13029                        if chunks > 1 {
13030                            stats.note_side_transport_chunks(chunks);
13031                        }
13032                    }
13033                    self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
13034                    self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
13035                    return Ok(());
13036                }
13037                let owned = self.pack_packet_for_router(pkt, None)?;
13038                let frames = self.encode_side_transport_frames(side, opts, owned)?;
13039                let mut attempts_total = 0usize;
13040                let mut sent_bytes = 0usize;
13041                self.send_side_frames_resumable_with_service(side, frames, service, |frame| {
13042                    match self.retry_with_attempts(runtime_max_handler_retries(), || {
13043                        self.packed_handler_call(side, packed, frame, priority, service)
13044                    }) {
13045                        Ok((_, attempts)) => {
13046                            *admitted = true;
13047                            attempts_total = attempts_total.saturating_add(attempts);
13048                            sent_bytes = sent_bytes.saturating_add(frame.len());
13049                        }
13050                        Err((err, attempts)) => {
13051                            self.note_side_tx_failure(
13052                                side,
13053                                ty,
13054                                attempts_total.saturating_add(attempts),
13055                            );
13056                            return Err(err);
13057                        }
13058                    }
13059
13060                    Ok(())
13061                })?;
13062                self.record_side_tx_sample(side, sent_bytes, started_ms, self.clock.now_ms());
13063                self.note_side_tx_success(side, ty, sent_bytes, relayed, attempts_total);
13064                return Ok(());
13065            }
13066            (RouterTxHandlerFn::Packet(f), RouterItem::Packed(bytes)) => {
13067                let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
13068                self.retry_with_attempts(runtime_max_handler_retries(), || f(&pkt))
13069            }
13070        };
13071        match result {
13072            Ok((_, attempts)) => {
13073                *admitted = true;
13074                if let Ok(bytes) = Self::router_item_wire_len(data) {
13075                    self.record_side_tx_sample(side, bytes, started_ms, self.clock.now_ms());
13076                    self.note_side_tx_success(side, ty, bytes, relayed, attempts);
13077                }
13078                Ok(())
13079            }
13080            Err((err, attempts)) => {
13081                self.note_side_tx_failure(side, ty, attempts);
13082                Err(err)
13083            }
13084        }
13085    }
13086
13087    fn adjust_reliable_for_side(
13088        &self,
13089        opts: RouterSideOptions,
13090        data: RouterItem,
13091        preserve_end_to_end_ack: bool,
13092    ) -> TelemetryResult<Option<RouterItem>> {
13093        if opts.reliable_enabled {
13094            return Ok(Some(data));
13095        }
13096
13097        match data {
13098            RouterItem::Packed(bytes) => {
13099                let frame = wire_format::peek_routing_frame_info(bytes.as_ref())?;
13100                if is_reliable_type(frame.envelope.ty)
13101                    && let Some(hdr) = frame.reliable
13102                {
13103                    if (hdr.flags & wire_format::RELIABLE_FLAG_ACK_ONLY) != 0 {
13104                        return Ok(None);
13105                    }
13106                    if (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) == 0 {
13107                        let Some(rewritten) = wire_format::rewrite_reliable_header_owned(
13108                            bytes.as_ref(),
13109                            wire_format::RELIABLE_FLAG_UNSEQUENCED,
13110                            hdr.seq,
13111                            0,
13112                        )?
13113                        else {
13114                            return Ok(Some(RouterItem::Packed(bytes)));
13115                        };
13116                        return Ok(Some(RouterItem::Packed(crate::shared_bytes::convert(
13117                            rewritten,
13118                        ))));
13119                    }
13120                }
13121                Ok(Some(RouterItem::Packed(bytes)))
13122            }
13123            RouterItem::Packet(pkt) => {
13124                if matches!(
13125                    pkt.data_type(),
13126                    DataType::ReliableAck
13127                        | DataType::ReliablePartialAck
13128                        | DataType::ReliablePacketRequest
13129                ) {
13130                    if preserve_end_to_end_ack && Self::is_end_to_end_ack_packet(&pkt) {
13131                        return Ok(Some(RouterItem::Packet(pkt)));
13132                    }
13133                    return Ok(None);
13134                }
13135                Ok(Some(RouterItem::Packet(pkt)))
13136            }
13137        }
13138    }
13139
13140    fn process_reliable_timeouts(&self) -> TelemetryResult<()> {
13141        let now = self.clock.now_ms();
13142        let mut requeue: Vec<(RouterSideId, DataType, u32)> = Vec::new();
13143
13144        {
13145            let mut st = self.state.lock();
13146            if st.reliable_tx.is_empty() {
13147                return Ok(());
13148            }
13149
13150            for ((side, ty_u32), tx_state) in st.reliable_tx.iter_mut() {
13151                let Some(ty) = DataType::try_from_u32(*ty_u32) else {
13152                    continue;
13153                };
13154                let sent_order: Vec<u32> = tx_state.sent_order.iter().copied().collect();
13155                for seq in sent_order {
13156                    let Some(sent) = tx_state.sent.get_mut(&seq) else {
13157                        continue;
13158                    };
13159                    if sent.queued
13160                        || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
13161                    {
13162                        continue;
13163                    }
13164                    if sent.partial_acked {
13165                        // A partial ACK normally arrives with a request for
13166                        // the missing sequence. If that request is itself
13167                        // lost, retaining this flag forever pins the history
13168                        // entry and eventually rejects unrelated traffic with
13169                        // `reliable history full`. Give the request one retry
13170                        // interval, then resume normal bounded retransmission.
13171                        sent.partial_acked = false;
13172                        sent.last_send_ms = now;
13173                        continue;
13174                    }
13175                    if sent.retries >= runtime_reliable_max_retries() {
13176                        tx_state.sent.remove(&seq);
13177                        tx_state.sent_order.retain(|existing| *existing != seq);
13178                        continue;
13179                    }
13180                    sent.retries += 1;
13181                    requeue.push((*side, ty, seq));
13182                }
13183            }
13184        }
13185
13186        for (side, ty, seq) in requeue {
13187            self.queue_reliable_retransmit(side, ty, seq, true)?;
13188        }
13189
13190        Ok(())
13191    }
13192
13193    fn process_end_to_end_reliable_timeouts(&self) -> TelemetryResult<()> {
13194        let now = self.clock.now_ms();
13195        let mut requeue = Vec::new();
13196        let mut failed_ordered_types = BTreeSet::new();
13197
13198        {
13199            let mut st = self.state.lock();
13200            #[cfg(feature = "discovery")]
13201            {
13202                if Self::prune_discovery_routes_locked(&mut st, now) {
13203                    Self::note_discovery_topology_change_locked(&mut st, now);
13204                    self.reconcile_end_to_end_reliable_destinations_locked(&mut st)?;
13205                }
13206                // Route updates reconcile their retained deliveries when the
13207                // topology changes. Rebuilding the destination map on every
13208                // queue item adds allocation and graph scans without changing
13209                // the frozen receipt contract.
13210            }
13211            // Retain walks the bounded pending map in place. A queue pass
13212            // before any retry deadline must not allocate a list of keys.
13213            let RouterInner {
13214                transmit_queue,
13215                end_to_end_reliable_tx,
13216                ..
13217            } = &mut *st;
13218            end_to_end_reliable_tx.retain(|&packet_id, sent| {
13219                let covered = !sent.pending_destinations.is_empty()
13220                    && sent
13221                        .pending_destinations
13222                        .values()
13223                        .all(|side| Self::e2e_child_owned(transmit_queue, *side, packet_id));
13224                if covered
13225                    || sent.waiting_for_predecessor
13226                    || sent.queued
13227                    || now.wrapping_sub(sent.last_send_ms) < runtime_reliable_retransmit_ms()
13228                {
13229                    return true;
13230                }
13231                if sent.retries >= runtime_reliable_max_retries() {
13232                    if let Some(ty) = sent.ordered_type {
13233                        failed_ordered_types.insert(ty);
13234                    }
13235                    return false;
13236                }
13237                sent.retries += 1;
13238                requeue.push(packet_id);
13239                true
13240            });
13241            // Fail the queued tail too; never deliver it past an unconfirmed gap.
13242            st.end_to_end_reliable_tx.retain(|_, sent| {
13243                !sent
13244                    .ordered_type
13245                    .is_some_and(|ty| failed_ordered_types.contains(&ty))
13246            });
13247        }
13248
13249        for packet_id in requeue {
13250            self.queue_end_to_end_reliable_retransmit(packet_id)?;
13251        }
13252        self.release_ordered_successors()?;
13253        if !failed_ordered_types.is_empty() {
13254            return Err(TelemetryError::HandlerError(
13255                "ordered end-to-end delivery exhausted retries",
13256            ));
13257        }
13258        Ok(())
13259    }
13260
13261    #[cfg(feature = "timesync")]
13262    #[inline]
13263    fn monotonic_now_ns(&self) -> u64 {
13264        self.clock.now_ns()
13265    }
13266
13267    #[cfg(feature = "timesync")]
13268    #[inline]
13269    fn monotonic_now_ms(&self) -> u64 {
13270        self.clock.now_ms()
13271    }
13272
13273    #[cfg(feature = "timesync")]
13274    fn refresh_timesync_state(&self, now_mono_ms: u64) {
13275        if self.timesync.is_none() {
13276            return;
13277        }
13278
13279        let now_mono_ns = self.monotonic_now_ns();
13280        let mut st = self
13281            .timesync
13282            .as_ref()
13283            .expect("local time service enabled")
13284            .lock();
13285        st.clock.prune_expired(now_mono_ms);
13286        let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13287        st.remote_sources
13288            .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13289        let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
13290        let leader = if let Some(tracker) = st.tracker.as_mut() {
13291            let _ = tracker.refresh(now_mono_ms);
13292            tracker.leader(now_mono_ms, has_usable_time)
13293        } else {
13294            None
13295        };
13296        Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_mono_ms);
13297        if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
13298            // Remote samples are anchored at receipt, just like local sources.
13299            // Steering repeatedly toward the unprojected sample slows the clock.
13300            let target_ms = st.remote_sources.get(remote.sender.as_str()).map(|src| {
13301                src.sample_unix_ms
13302                    .saturating_add(now_mono_ms.saturating_sub(src.last_sample_mono_ms))
13303            });
13304            if let Some(target_ms) = target_ms {
13305                st.disciplined_clock.steer_unix_ms(now_mono_ns, target_ms);
13306            }
13307        }
13308    }
13309
13310    #[cfg(feature = "timesync")]
13311    /// Inserts or updates a named network-time source with an optional expiration TTL.
13312    pub fn update_network_time_source(
13313        &self,
13314        source: &str,
13315        priority: u64,
13316        time: PartialNetworkTime,
13317        ttl_ms: Option<u64>,
13318    ) {
13319        if self.timesync.is_none() {
13320            return;
13321        }
13322
13323        let now_ms = self.monotonic_now_ms();
13324        let now_ns = self.monotonic_now_ns();
13325        let mut st = self
13326            .timesync
13327            .as_ref()
13328            .expect("local time service enabled")
13329            .lock();
13330        if st
13331            .clock
13332            .try_update_source(source, priority, time, now_ms, now_ns, ttl_ms)
13333            .is_err()
13334        {
13335            return;
13336        }
13337        if let Some(unix_ms) = time.to_network_time().and_then(|t| t.as_unix_ms()) {
13338            st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
13339        }
13340    }
13341
13342    #[cfg(feature = "timesync")]
13343    /// Sets local time, preserving the previous reading if storage is refused.
13344    /// Use `try_set_local_network_time` when the caller needs refusal status.
13345    pub fn set_local_network_time(&self, time: PartialNetworkTime) {
13346        let _ = self.try_set_local_network_time(time);
13347    }
13348
13349    #[cfg(feature = "timesync")]
13350    /// Transactionally sets any combination of local date/time fragments.
13351    /// New record/key allocation failure preserves all prior clock sources.
13352    pub fn try_set_local_network_time(&self, time: PartialNetworkTime) -> TelemetryResult<()> {
13353        let Some(runtime) = self.timesync.as_ref() else {
13354            return Ok(());
13355        };
13356        let observed_ns = self.monotonic_now_ns();
13357        let mut st = runtime.lock();
13358        let now_ms = self.monotonic_now_ms();
13359        let now_ns = self.monotonic_now_ns();
13360        let priority = st.cfg.map(|cfg| cfg.priority).unwrap_or(0);
13361        // A complete epoch sample can be advanced to the commit instant.
13362        // Partial fields keep the observation anchor below so their eventual
13363        // assembled reading also includes time spent waiting for this lock.
13364        let time = time
13365            .to_network_time()
13366            .and_then(|base| advance_network_time(base, now_ns.saturating_sub(observed_ns)))
13367            .map(PartialNetworkTime::from)
13368            .unwrap_or(time);
13369        if time.is_complete_date() && time.is_complete_time() {
13370            let adjusted = time;
13371            let update = st.clock.prepare_source(
13372                LOCAL_TIMESYNC_FULL_SOURCE_ID,
13373                priority,
13374                adjusted,
13375                now_ms,
13376                now_ns,
13377                None,
13378            )?;
13379            st.clock.commit_source(update);
13380            st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13381            st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13382            st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13383            if let Some(unix_ms) = adjusted
13384                .to_network_time()
13385                .and_then(|time| time.as_unix_ms())
13386            {
13387                st.disciplined_clock.steer_unix_ms(now_ns, unix_ms);
13388            }
13389        } else {
13390            // Prepare every allocation first. In particular, do not erase an
13391            // old full-date source before a replacement fragment can be saved.
13392            let date = if time.year.is_some() || time.month.is_some() || time.day.is_some() {
13393                Some(st.clock.prepare_source(
13394                    LOCAL_TIMESYNC_DATE_SOURCE_ID,
13395                    priority,
13396                    PartialNetworkTime {
13397                        year: time.year,
13398                        month: time.month,
13399                        day: time.day,
13400                        ..Default::default()
13401                    },
13402                    now_ms,
13403                    observed_ns,
13404                    None,
13405                )?)
13406            } else {
13407                None
13408            };
13409            let tod = if time.hour.is_some() || time.minute.is_some() || time.second.is_some() {
13410                Some(st.clock.prepare_source(
13411                    LOCAL_TIMESYNC_TOD_SOURCE_ID,
13412                    priority,
13413                    PartialNetworkTime {
13414                        hour: time.hour,
13415                        minute: time.minute,
13416                        second: time.second,
13417                        nanosecond: time.nanosecond,
13418                        ..Default::default()
13419                    },
13420                    now_ms,
13421                    observed_ns,
13422                    None,
13423                )?)
13424            } else {
13425                None
13426            };
13427            let subsec = if time.nanosecond.is_some() {
13428                Some(st.clock.prepare_source(
13429                    LOCAL_TIMESYNC_SUBSEC_SOURCE_ID,
13430                    priority,
13431                    PartialNetworkTime {
13432                        nanosecond: time.nanosecond,
13433                        ..Default::default()
13434                    },
13435                    now_ms,
13436                    observed_ns,
13437                    None,
13438                )?)
13439            } else {
13440                None
13441            };
13442            st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13443            for update in [date, tod, subsec].into_iter().flatten() {
13444                st.clock.commit_source(update);
13445            }
13446        }
13447        Ok(())
13448    }
13449
13450    #[cfg(feature = "timesync")]
13451    /// Removes all locally supplied network-time fragments from the assembled clock.
13452    pub fn clear_local_network_time(&self) {
13453        if self.timesync.is_none() {
13454            return;
13455        }
13456
13457        let mut st = self
13458            .timesync
13459            .as_ref()
13460            .expect("local time service enabled")
13461            .lock();
13462        st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13463        st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13464        st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13465        st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13466    }
13467
13468    #[cfg(feature = "timesync")]
13469    /// Sets only the local calendar date portion of network time.
13470    pub fn set_local_network_date(&self, year: i32, month: u8, day: u8) {
13471        self.set_local_network_time(PartialNetworkTime {
13472            year: Some(year),
13473            month: Some(month),
13474            day: Some(day),
13475            ..Default::default()
13476        });
13477    }
13478
13479    #[cfg(feature = "timesync")]
13480    /// Sets the local time of day to hour and minute precision.
13481    pub fn set_local_network_time_hm(&self, hour: u8, minute: u8) {
13482        self.set_local_network_time(PartialNetworkTime {
13483            hour: Some(hour),
13484            minute: Some(minute),
13485            ..Default::default()
13486        });
13487    }
13488
13489    #[cfg(feature = "timesync")]
13490    /// Sets the local time of day to second precision.
13491    pub fn set_local_network_time_hms(&self, hour: u8, minute: u8, second: u8) {
13492        self.set_local_network_time(PartialNetworkTime {
13493            hour: Some(hour),
13494            minute: Some(minute),
13495            second: Some(second),
13496            ..Default::default()
13497        });
13498    }
13499
13500    #[cfg(feature = "timesync")]
13501    /// Sets the local time of day with millisecond precision.
13502    pub fn set_local_network_time_hms_millis(
13503        &self,
13504        hour: u8,
13505        minute: u8,
13506        second: u8,
13507        millisecond: u16,
13508    ) {
13509        self.set_local_network_time(PartialNetworkTime {
13510            hour: Some(hour),
13511            minute: Some(minute),
13512            second: Some(second),
13513            nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
13514            ..Default::default()
13515        });
13516    }
13517
13518    #[cfg(feature = "timesync")]
13519    /// Sets the local time of day with nanosecond precision.
13520    pub fn set_local_network_time_hms_nanos(
13521        &self,
13522        hour: u8,
13523        minute: u8,
13524        second: u8,
13525        nanosecond: u32,
13526    ) {
13527        self.set_local_network_time(PartialNetworkTime {
13528            hour: Some(hour),
13529            minute: Some(minute),
13530            second: Some(second),
13531            nanosecond: Some(nanosecond),
13532            ..Default::default()
13533        });
13534    }
13535
13536    #[cfg(feature = "timesync")]
13537    /// Sets a complete local date and time with second precision.
13538    pub fn set_local_network_datetime(
13539        &self,
13540        year: i32,
13541        month: u8,
13542        day: u8,
13543        hour: u8,
13544        minute: u8,
13545        second: u8,
13546    ) {
13547        self.set_local_network_time(PartialNetworkTime {
13548            year: Some(year),
13549            month: Some(month),
13550            day: Some(day),
13551            hour: Some(hour),
13552            minute: Some(minute),
13553            second: Some(second),
13554            ..Default::default()
13555        });
13556    }
13557
13558    #[cfg(feature = "timesync")]
13559    #[allow(clippy::too_many_arguments)]
13560    /// Sets a complete local date and time with millisecond precision.
13561    pub fn set_local_network_datetime_millis(
13562        &self,
13563        year: i32,
13564        month: u8,
13565        day: u8,
13566        hour: u8,
13567        minute: u8,
13568        second: u8,
13569        millisecond: u16,
13570    ) {
13571        self.set_local_network_time(PartialNetworkTime {
13572            year: Some(year),
13573            month: Some(month),
13574            day: Some(day),
13575            hour: Some(hour),
13576            minute: Some(minute),
13577            second: Some(second),
13578            nanosecond: Some((millisecond as u32).saturating_mul(1_000_000)),
13579        });
13580    }
13581
13582    #[cfg(feature = "timesync")]
13583    #[allow(clippy::too_many_arguments)]
13584    /// Sets a complete local date and time with nanosecond precision.
13585    pub fn set_local_network_datetime_nanos(
13586        &self,
13587        year: i32,
13588        month: u8,
13589        day: u8,
13590        hour: u8,
13591        minute: u8,
13592        second: u8,
13593        nanosecond: u32,
13594    ) {
13595        self.set_local_network_time(PartialNetworkTime {
13596            year: Some(year),
13597            month: Some(month),
13598            day: Some(day),
13599            hour: Some(hour),
13600            minute: Some(minute),
13601            second: Some(second),
13602            nanosecond: Some(nanosecond),
13603        });
13604    }
13605
13606    #[cfg(feature = "timesync")]
13607    /// Removes a previously registered named network-time source.
13608    pub fn clear_network_time_source(&self, source: &str) {
13609        if self.timesync.is_none() {
13610            return;
13611        }
13612
13613        let mut st = self
13614            .timesync
13615            .as_ref()
13616            .expect("local time service enabled")
13617            .lock();
13618        st.clock.remove_source(source);
13619    }
13620
13621    #[cfg(feature = "timesync")]
13622    /// Replaces the active time sync configuration and resets runtime state derived from it.
13623    pub fn set_timesync_config(&self, cfg: Option<TimeSyncConfig>) {
13624        if self.timesync.is_none() {
13625            return;
13626        }
13627
13628        let mut st = self
13629            .timesync
13630            .as_ref()
13631            .expect("local time service enabled")
13632            .lock();
13633        let stale_remote_sources: Vec<String> = st.remote_sources.keys().cloned().collect();
13634        st.cfg = cfg;
13635        st.tracker = cfg.map(TimeSyncTracker::new);
13636        st.disciplined_clock = SlewedNetworkClock::new(
13637            cfg.map(|c| c.max_slew_ppm)
13638                .unwrap_or(TimeSyncConfig::default().max_slew_ppm),
13639        );
13640        st.remote_sources.clear();
13641        st.next_seq = 1;
13642        st.next_announce_mono_ms = 0;
13643        st.next_request_mono_ms = 0;
13644        st.pending_request = None;
13645        st.clock.remove_source(INTERNAL_TIMESYNC_SOURCE_ID);
13646        for source in stale_remote_sources {
13647            st.clock.remove_source(&source);
13648        }
13649        st.clock.remove_source(LOCAL_TIMESYNC_FULL_SOURCE_ID);
13650        st.clock.remove_source(LOCAL_TIMESYNC_DATE_SOURCE_ID);
13651        st.clock.remove_source(LOCAL_TIMESYNC_TOD_SOURCE_ID);
13652        st.clock.remove_source(LOCAL_TIMESYNC_SUBSEC_SOURCE_ID);
13653    }
13654
13655    #[cfg(feature = "timesync")]
13656    /// Returns the best currently known network-time reading, if any.
13657    pub fn network_time(&self) -> Option<NetworkTimeReading> {
13658        let runtime = self.timesync.as_ref()?;
13659
13660        let now_ms = self.monotonic_now_ms();
13661        let now_ns = self.monotonic_now_ns();
13662        self.refresh_timesync_state(now_ms);
13663        let st = runtime.lock();
13664        if let Some(unix_ms) = st.disciplined_clock.read_unix_ms(now_ns) {
13665            return Some(NetworkTimeReading {
13666                time: PartialNetworkTime::from_unix_ms(unix_ms),
13667                unix_time_ms: Some(unix_ms),
13668            });
13669        }
13670        st.clock.current_time(now_ns)
13671    }
13672
13673    #[cfg(feature = "timesync")]
13674    /// Returns the current network time as Unix milliseconds when available.
13675    pub fn network_time_ms(&self) -> Option<u64> {
13676        self.network_time().and_then(|t| t.unix_time_ms)
13677    }
13678
13679    #[cfg(feature = "timesync")]
13680    fn packet_timestamp_ms(&self) -> u64 {
13681        self.network_time_ms()
13682            .unwrap_or_else(|| self.monotonic_now_ms())
13683    }
13684
13685    #[cfg(not(feature = "timesync"))]
13686    fn packet_timestamp_ms(&self) -> u64 {
13687        self.clock.now_ms()
13688    }
13689
13690    #[cfg(feature = "timesync")]
13691    fn queue_internal_timesync_request(
13692        &self,
13693        seq: u64,
13694        t1_mono_ms: u64,
13695        called_from_queue: bool,
13696    ) -> TelemetryResult<()> {
13697        let pkt_ts = self.packet_timestamp_ms();
13698        if called_from_queue {
13699            self.log_queue_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
13700        } else {
13701            self.log_ts(DataType::TimeSyncRequest, pkt_ts, &[seq, t1_mono_ms])
13702        }
13703    }
13704
13705    #[cfg(feature = "timesync")]
13706    fn queue_internal_timesync_response(
13707        &self,
13708        seq: u64,
13709        t1_mono_ms: u64,
13710        t2_network_ms: u64,
13711        t3_network_ms: u64,
13712        dst: Option<RouterSideId>,
13713        called_from_queue: bool,
13714    ) -> TelemetryResult<()> {
13715        let pkt_ts = self.packet_timestamp_ms();
13716        let payload = encode_slice_le(&[seq, t1_mono_ms, t2_network_ms, t3_network_ms]);
13717        let sender = self.sender_arc();
13718        let pkt = Packet::new(
13719            DataType::TimeSyncResponse,
13720            &[DataEndpoint::TimeSync],
13721            sender.as_ref(),
13722            pkt_ts,
13723            payload,
13724        )?;
13725        match dst {
13726            Some(dst) => self.emit_internal_tx(
13727                RouterTxItem::ToSide {
13728                    src: None,
13729                    dst,
13730                    data: RouterItem::Packet(pkt),
13731                },
13732                true,
13733                called_from_queue,
13734            ),
13735            None => self.emit_internal_tx(
13736                RouterTxItem::Broadcast(RouterItem::Packet(pkt)),
13737                true,
13738                called_from_queue,
13739            ),
13740        }
13741    }
13742
13743    #[cfg(feature = "timesync")]
13744    /// Runs one time sync maintenance cycle and queues any required announce or request packets.
13745    pub fn poll_timesync(&self) -> TelemetryResult<bool> {
13746        if self.timesync.is_none() {
13747            return Ok(false);
13748        }
13749
13750        let now_ms = self.monotonic_now_ms();
13751        let now_ns = self.monotonic_now_ns();
13752        let mut queued_any = false;
13753        let mut announce_priority = None;
13754        let mut request = None;
13755
13756        {
13757            let mut st = self
13758                .timesync
13759                .as_ref()
13760                .expect("local time service enabled")
13761                .lock();
13762            st.clock.prune_expired(now_ms);
13763            let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13764            st.remote_sources
13765                .retain(|_, src| now_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13766            let Some(cfg) = st.cfg else {
13767                return Ok(false);
13768            };
13769
13770            let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_ns);
13771            let (leader, announce_prio) = if let Some(tracker) = st.tracker.as_mut() {
13772                let _ = tracker.refresh(now_ms);
13773                (
13774                    tracker.leader(now_ms, has_usable_time),
13775                    tracker.local_announce_priority(now_ms, has_usable_time),
13776                )
13777            } else {
13778                (None, None)
13779            };
13780            Self::reconcile_pending_timesync_request_locked(&mut st, &leader, now_ms);
13781
13782            if let Some(TimeSyncLeader::Remote(remote)) = leader.as_ref() {
13783                let target_ms = st.remote_sources.get(&remote.sender).map(|src| {
13784                    src.sample_unix_ms
13785                        .saturating_add(now_ms.saturating_sub(src.last_sample_mono_ms))
13786                });
13787                if let Some(target_ms) = target_ms {
13788                    st.disciplined_clock.steer_unix_ms(now_ns, target_ms);
13789                }
13790            }
13791
13792            if let Some(priority) = announce_prio
13793                && now_ms >= st.next_announce_mono_ms
13794            {
13795                announce_priority = Some(priority);
13796                st.next_announce_mono_ms = now_ms.saturating_add(cfg.announce_interval_ms);
13797            }
13798
13799            if let Some(TimeSyncLeader::Remote(remote)) = leader
13800                && now_ms >= st.next_request_mono_ms
13801                && st.pending_request.is_none()
13802            {
13803                let seq = st.next_seq;
13804                let next = st.next_seq.wrapping_add(1);
13805                st.next_seq = if next == 0 { 1 } else { next };
13806                st.next_request_mono_ms = now_ms.saturating_add(cfg.request_interval_ms);
13807                st.pending_request = Some(PendingTimeSyncRequest {
13808                    seq,
13809                    t1_mono_ms: now_ms,
13810                    source: remote.sender,
13811                });
13812                request = Some((seq, now_ms));
13813            }
13814        }
13815
13816        if let Some(priority) = announce_priority {
13817            let time_ms = self.packet_timestamp_ms();
13818            self.log_queue_ts(DataType::TimeSyncAnnounce, time_ms, &[priority, time_ms])?;
13819            queued_any = true;
13820        }
13821        if let Some((seq, t1_mono_ms)) = request {
13822            self.queue_internal_timesync_request(seq, t1_mono_ms, true)?;
13823            queued_any = true;
13824        }
13825
13826        Ok(queued_any)
13827    }
13828
13829    #[cfg(feature = "timesync")]
13830    fn handle_internal_timesync_packet(
13831        &self,
13832        pkt: &Packet,
13833        src: Option<RouterSideId>,
13834        called_from_queue: bool,
13835    ) -> TelemetryResult<bool> {
13836        let Some(cfg) = self.cfg.timesync_config() else {
13837            if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
13838                self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
13839            }
13840            return Ok(true);
13841        };
13842
13843        let now_mono_ms = self.monotonic_now_ms();
13844        let now_mono_ns = self.monotonic_now_ns();
13845        let mut response = None;
13846        let mut poll_after = false;
13847
13848        {
13849            let mut st = self
13850                .timesync
13851                .as_ref()
13852                .expect("local time service enabled")
13853                .lock();
13854            st.clock.prune_expired(now_mono_ms);
13855            let timeout_ms = st.cfg.map(|cfg| cfg.source_timeout_ms).unwrap_or(0);
13856            st.remote_sources
13857                .retain(|_, src| now_mono_ms.saturating_sub(src.last_sample_mono_ms) <= timeout_ms);
13858            let has_usable_time = Self::timesync_has_usable_time_locked(&st, now_mono_ns);
13859            if st.tracker.is_none() {
13860                return Ok(true);
13861            }
13862
13863            match pkt.data_type() {
13864                DataType::TimeSyncAnnounce => {
13865                    let ann = decode_timesync_announce(pkt)?;
13866                    // Stage all three new records/keys before changing leader,
13867                    // sample cache or current time. A refused allocation drops
13868                    // only this update, preserving every prior source.
13869                    let tracker_update = st
13870                        .tracker
13871                        .as_ref()
13872                        .expect("tracker checked above")
13873                        .prepare_source(pkt.sender(), ann, now_mono_ms)?;
13874                    let remote_update = crate::peer_record::prepare(
13875                        &st.remote_sources,
13876                        pkt.sender(),
13877                        RemoteTimeSyncSource {
13878                            priority: ann.priority,
13879                            last_sample_mono_ms: now_mono_ms,
13880                            sample_unix_ms: ann.time_ms,
13881                        },
13882                    )?;
13883                    let clock_update = st.clock.prepare_source(
13884                        pkt.sender(),
13885                        ann.priority,
13886                        PartialNetworkTime::from_unix_ms(ann.time_ms),
13887                        now_mono_ms,
13888                        now_mono_ns,
13889                        Some(cfg.source_timeout_ms),
13890                    )?;
13891                    remote_update.commit(&mut st.remote_sources);
13892                    st.clock.commit_source(clock_update);
13893                    let should_steer = {
13894                        let tracker = st.tracker.as_mut().expect("tracker checked above");
13895                        tracker.commit_source(tracker_update);
13896                        matches!(tracker.leader(now_mono_ms, has_usable_time),
13897                            Some(TimeSyncLeader::Remote(ref remote)) if remote.sender == pkt.sender())
13898                    };
13899                    if should_steer {
13900                        st.disciplined_clock.steer_unix_ms(now_mono_ns, ann.time_ms);
13901                    }
13902                    poll_after = true;
13903                }
13904                DataType::TimeSyncRequest => {
13905                    let should_serve = {
13906                        let tracker = st.tracker.as_ref().expect("tracker checked above");
13907                        tracker.should_serve(now_mono_ms, has_usable_time)
13908                    };
13909                    if should_serve {
13910                        let req = decode_timesync_request(pkt)?;
13911                        let network_now = st
13912                            .disciplined_clock
13913                            .read_unix_ms(now_mono_ns)
13914                            .or_else(|| {
13915                                st.clock
13916                                    .current_time(now_mono_ns)
13917                                    .and_then(|t| t.unix_time_ms)
13918                            })
13919                            .unwrap_or(now_mono_ms);
13920                        let t2 = network_now;
13921                        let t3 = network_now;
13922                        response = Some((req.seq, req.t1_ms, t2, t3, src));
13923                    }
13924                }
13925                DataType::TimeSyncResponse => {
13926                    let resp = decode_timesync_response(pkt)?;
13927                    let pending = st.pending_request.clone();
13928                    if let Some(pending) = pending
13929                        && pending.seq == resp.seq
13930                        && pending.source == pkt.sender()
13931                    {
13932                        let source_priority = {
13933                            let tracker = st.tracker.as_ref().expect("tracker checked above");
13934                            tracker
13935                                .best_active_source(now_mono_ms)
13936                                .map(|s| s.priority)
13937                                .or_else(|| st.remote_sources.get(pkt.sender()).map(|s| s.priority))
13938                                .unwrap_or(cfg.priority)
13939                        };
13940                        let (estimate_ms, _delay_ms) = compute_network_time_sample(
13941                            pending.t1_mono_ms,
13942                            resp.t2_ms,
13943                            resp.t3_ms,
13944                            now_mono_ms,
13945                        );
13946                        let remote_update = crate::peer_record::prepare(
13947                            &st.remote_sources,
13948                            pkt.sender(),
13949                            RemoteTimeSyncSource {
13950                                priority: source_priority,
13951                                last_sample_mono_ms: now_mono_ms,
13952                                sample_unix_ms: estimate_ms,
13953                            },
13954                        )?;
13955                        let clock_update = st.clock.prepare_source(
13956                            pkt.sender(),
13957                            source_priority,
13958                            PartialNetworkTime::from_unix_ms(estimate_ms),
13959                            now_mono_ms,
13960                            now_mono_ns,
13961                            Some(cfg.source_timeout_ms),
13962                        )?;
13963                        remote_update.commit(&mut st.remote_sources);
13964                        st.clock.commit_source(clock_update);
13965                        st.disciplined_clock.steer_unix_ms(now_mono_ns, estimate_ms);
13966                        st.pending_request = None;
13967                    }
13968                }
13969                _ => {}
13970            }
13971        }
13972
13973        if let Some((seq, t1, t2, t3, dst)) = response {
13974            self.queue_internal_timesync_response(seq, t1, t2, t3, dst, called_from_queue)?;
13975        }
13976        if poll_after {
13977            let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
13978        }
13979
13980        if self.should_route_remote(&RouterItem::Packet(pkt.clone()), src)? {
13981            self.relay_send(RouterItem::Packet(pkt.clone()), src, called_from_queue)?;
13982        }
13983
13984        Ok(true)
13985    }
13986
13987    /// Create a new Router with an internal monotonic clock.
13988    #[cfg(feature = "std")]
13989    pub fn new(cfg: RouterConfig) -> Self {
13990        Self::new_with_clock(cfg, Box::new(StdMonotonicClock::default()))
13991    }
13992
13993    /// Create a new Router with the specified router configuration and clock.
13994    pub fn new_with_clock(cfg: RouterConfig, clock: Box<dyn Clock + Send + Sync>) -> Self {
13995        #[cfg(feature = "timesync")]
13996        let timesync_cfg = cfg.timesync_config();
13997        #[cfg(feature = "discovery")]
13998        let preferred_discovery_master = cfg.preferred_discovery_master.clone();
13999        let memory = cfg.memory_config();
14000        let local_services_enabled = cfg.role.local_services();
14001        let hostname: Arc<str> = cfg.configured_sender();
14002        let address_mode = cfg.address_mode();
14003        let instance_seq = u64::from(ROUTER_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed));
14004        let owner_hash = Self::sender_hash(hostname.as_ref()) ^ instance_seq;
14005        let fallback = Self::fallback_address_for_hostname(hostname.as_ref());
14006        let requested = address_mode.requested_address();
14007        let address = if requested == 0 { fallback } else { requested };
14008        let local_address = AddressBookEntry {
14009            hostname: hostname.clone(),
14010            address,
14011            requested_address: requested,
14012            mode: address_mode,
14013            birth_ms: clock
14014                .now_ms()
14015                .saturating_mul(1_000_000)
14016                .saturating_add(instance_seq),
14017            owner_hash,
14018            last_seen_ms: clock.now_ms(),
14019        };
14020        let mut address_book = BTreeMap::new();
14021        if local_services_enabled {
14022            address_book.insert(hostname.to_string(), Box::new(local_address.clone()));
14023        }
14024        let mut address_by_value = BTreeMap::new();
14025        if local_services_enabled {
14026            address_by_value.insert(address, hostname.to_string());
14027        }
14028        Self {
14029            #[cfg(feature = "std")]
14030            retry_trackers: RouterMutex::new(Vec::new()),
14031            sender: RouterMutex::new(hostname),
14032            cfg: ActiveRoutingConfig::new(cfg),
14033            state: RouterMutex::new(RouterInner {
14034                #[cfg(feature = "compression")]
14035                codec_reserved_bytes: 0,
14036                memory,
14037                sides: Vec::new(),
14038                route_overrides: Default::default(),
14039                typed_route_overrides: Default::default(),
14040                route_weights: Default::default(),
14041                route_priorities: Default::default(),
14042                source_route_modes: Default::default(),
14043                route_selection_cursors: Default::default(),
14044                adaptive_route_stats: Default::default(),
14045                side_runtime_stats: Default::default(),
14046                side_transport: Default::default(),
14047                local_services: local_services_enabled.then(|| {
14048                    Box::new(LocalServicesState {
14049                        managed_variable_types: BTreeSet::new(),
14050                        managed_variable_permissions: Default::default(),
14051                        managed_variable_latest: Default::default(),
14052                        network_variable_update_handlers: Default::default(),
14053                        p2p_port_handlers: Default::default(),
14054                        p2p_stream_handlers: Default::default(),
14055                        p2p_stream_sessions: Default::default(),
14056                        next_p2p_stream_id: 1,
14057                    })
14058                }),
14059                local_address,
14060                address_book,
14061                address_by_value,
14062                received_queue: BoundedDeque::new(
14063                    memory.max_queue_budget,
14064                    memory.starting_queue_size,
14065                    memory.queue_grow_step,
14066                ),
14067                transmit_queue: BoundedDeque::new(
14068                    memory.max_queue_budget,
14069                    memory.starting_queue_size,
14070                    memory.queue_grow_step,
14071                ),
14072                #[cfg(feature = "std")]
14073                active_queued_types: [(0, 0); ACTIVE_QUEUED_TYPE_SLOTS],
14074                tx_priority_burst: 0,
14075                tx_policy_flags: 0,
14076                #[cfg(feature = "latest-value-egress")]
14077                latest_turn_phase: 0,
14078                #[cfg(feature = "latest-value-egress")]
14079                latest_turn_credit: 0,
14080                #[cfg(feature = "latest-value-egress")]
14081                latest_service_seq: 0,
14082                ordered_claim_seq: 0,
14083                rx_priority_burst: 0,
14084                recent_rx: BoundedDeque::new(
14085                    memory.recent_rx_queue_bytes(),
14086                    memory.recent_rx_queue_bytes(),
14087                    memory.queue_grow_step,
14088                ),
14089                reliable_tx: Default::default(),
14090                reliable_rx: Default::default(),
14091                reliable_return_routes: Default::default(),
14092                local_admission_sequence: 0,
14093                reliable_return_route_order: VecDeque::new(),
14094                forwarded_receipts: None,
14095                end_to_end_reliable_tx: Default::default(),
14096                end_to_end_reliable_tx_order: VecDeque::new(),
14097                total_handler_failures: 0,
14098                total_handler_retries: 0,
14099                #[cfg(feature = "discovery")]
14100                discovery_routes: Default::default(),
14101                #[cfg(feature = "discovery")]
14102                discovery_cadence: DiscoveryCadenceState::default(),
14103                #[cfg(feature = "discovery")]
14104                discovery_side_throttle: Default::default(),
14105                #[cfg(feature = "discovery")]
14106                schema_response_sequence: 0,
14107                #[cfg(feature = "discovery")]
14108                preferred_discovery_master,
14109                #[cfg(all(feature = "discovery", feature = "timesync"))]
14110                timesync_side_throttle: Default::default(),
14111            }),
14112            isr_rx_queue: local_services_enabled.then(|| {
14113                Box::new(IsrRxQueue::new(
14114                    memory.max_queue_budget,
14115                    memory.starting_queue_size,
14116                    memory.queue_grow_step,
14117                ))
14118            }),
14119            #[cfg(not(feature = "std"))]
14120            side_tx_gate: ReentryGate::new(),
14121            #[cfg(feature = "std")]
14122            side_tx_activity: crate::lock::SideTxActivity::default(),
14123            #[cfg(feature = "std")]
14124            active_queued_tx: core::sync::atomic::AtomicUsize::new(0),
14125            clock,
14126            #[cfg(feature = "timesync")]
14127            timesync: local_services_enabled
14128                .then(|| Box::new(RouterMutex::new(TimeSyncRuntime::new(timesync_cfg)))),
14129        }
14130    }
14131
14132    #[inline]
14133    fn sender_arc(&self) -> Arc<str> {
14134        self.sender.lock().clone()
14135    }
14136
14137    #[inline]
14138    pub fn sender(&self) -> Arc<str> {
14139        self.sender_arc()
14140    }
14141
14142    pub fn current_address(&self) -> NodeAddress {
14143        self.state.lock().local_address.address
14144    }
14145
14146    pub fn hostname(&self) -> Arc<str> {
14147        self.sender_arc()
14148    }
14149
14150    /// Change the preferred discovery master at runtime.
14151    ///
14152    /// Passing `None` or an empty hostname restores topology-based election.
14153    /// This does not alter time-sync leadership or source priority.
14154    pub fn set_preferred_discovery_master(&self, hostname: Option<&str>) {
14155        #[cfg(feature = "discovery")]
14156        {
14157            let preferred = hostname
14158                .filter(|value| !value.is_empty())
14159                .map(Arc::<str>::from);
14160            let mut st = self.state.lock();
14161            if st.preferred_discovery_master != preferred {
14162                st.preferred_discovery_master = preferred;
14163                Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14164            }
14165        }
14166        #[cfg(not(feature = "discovery"))]
14167        let _ = hostname;
14168    }
14169
14170    /// Return the currently configured preferred discovery master.
14171    pub fn preferred_discovery_master(&self) -> Option<Arc<str>> {
14172        #[cfg(feature = "discovery")]
14173        {
14174            return self.state.lock().preferred_discovery_master.clone();
14175        }
14176        #[cfg(not(feature = "discovery"))]
14177        None
14178    }
14179
14180    /// Match a packet sender to an expected literal hostname without allocating.
14181    ///
14182    /// Literal names compare exactly. Numeric `@addr:<u32>` aliases must resolve
14183    /// through retained assigned-address or collision-aware compact identity
14184    /// metadata; unknown or ambiguous aliases never match by hash alone.
14185    /// Existing explicit assigned-address precedence is preserved; this does
14186    /// not add stronger cross-namespace ambiguity or authenticity guarantees.
14187    /// Empty names, malformed aliases and embedded NULs fail closed. This is
14188    /// routing identity attribution, not packet authentication.
14189    pub fn sender_matches_hostname(&self, sender: &str, expected_hostname: &str) -> bool {
14190        if sender.is_empty()
14191            || expected_hostname.is_empty()
14192            || sender.contains('\0')
14193            || expected_hostname.contains('\0')
14194            || expected_hostname.starts_with("@addr:")
14195        {
14196            return false;
14197        }
14198        let Some(value) = sender.strip_prefix("@addr:") else {
14199            return sender == expected_hostname;
14200        };
14201        if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
14202            return false;
14203        }
14204        let Ok(address) = value.parse::<u32>() else {
14205            return false;
14206        };
14207        let state = self.state.lock();
14208        Self::hostname_for_address_borrowed_locked(&state, address) == Some(expected_hostname)
14209    }
14210
14211    pub fn address_book(&self) -> Vec<AddressBookEntry> {
14212        self.state
14213            .lock()
14214            .address_book
14215            .values()
14216            .map(|entry| entry.as_ref().clone())
14217            .collect()
14218    }
14219
14220    pub fn resolve_hostname(&self, hostname: &str) -> Option<AddressBookEntry> {
14221        let now_ms = self.clock.now_ms();
14222        #[allow(unused_mut)]
14223        let mut st = self.state.lock();
14224        #[cfg(feature = "discovery")]
14225        if Self::prune_discovery_routes_locked(&mut st, now_ms) {
14226            Self::note_discovery_topology_change_locked(&mut st, now_ms);
14227        }
14228        if let Some(entry) = st.address_book.get(hostname) {
14229            return Some(entry.as_ref().clone());
14230        }
14231        // Address advertisements are link-local, but topology names cross
14232        // routers. Mirror resolve_address's topology-only identity support so
14233        // named P2P messages/OTA streams can reach those downstream devices.
14234        // Never invent a target merely because an application requested it;
14235        // the retained topology must resolve its compact identity uniquely.
14236        let address = sender_address_u32(hostname);
14237        if Self::hostname_for_address_locked(&st, address).as_deref() != Some(hostname) {
14238            return None;
14239        }
14240        Some(AddressBookEntry {
14241            hostname: Arc::from(hostname),
14242            address,
14243            requested_address: 0,
14244            mode: AddressAssignmentMode::Dynamic,
14245            birth_ms: 0,
14246            owner_hash: Self::sender_hash(hostname),
14247            last_seen_ms: now_ms,
14248        })
14249    }
14250
14251    pub fn resolve_address(&self, address: NodeAddress) -> Option<AddressBookEntry> {
14252        let st = self.state.lock();
14253        let direct = st
14254            .address_by_value
14255            .get(&address)
14256            .and_then(|hostname| st.address_book.get(hostname))
14257            .map(|entry| entry.as_ref().clone());
14258        if direct.is_some() {
14259            return direct;
14260        }
14261
14262        let hostname = Self::hostname_for_address_locked(&st, address)?;
14263        if let Some(entry) = st.address_book.get(&hostname) {
14264            return Some(entry.as_ref().clone());
14265        }
14266
14267        // A topology-only remote has no DHCP-style address record on this
14268        // link. Return a compact-address identity entry; callers that need to
14269        // route to it still use discovery, while telemetry consumers can use
14270        // the stable hostname immediately.
14271        let hostname: Arc<str> = Arc::from(hostname);
14272        Some(AddressBookEntry {
14273            hostname: hostname.clone(),
14274            address,
14275            requested_address: 0,
14276            mode: AddressAssignmentMode::Dynamic,
14277            birth_ms: 0,
14278            owner_hash: Self::sender_hash(hostname.as_ref()),
14279            last_seen_ms: self.clock.now_ms(),
14280        })
14281    }
14282
14283    pub fn bind_p2p_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
14284    where
14285        F: Fn(P2pMessage<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
14286    {
14287        if port == 0 {
14288            return Err(TelemetryError::BadArg);
14289        }
14290        let mut st = self.state.lock();
14291        st.local_services
14292            .as_mut()
14293            .expect("local services enabled")
14294            .p2p_port_handlers
14295            .entry(port)
14296            .or_default()
14297            .push(P2pPortHandler {
14298                handler: Arc::new(f),
14299            });
14300        Ok(())
14301    }
14302
14303    pub fn clear_p2p_port(&self, port: P2pPort) {
14304        self.state
14305            .lock()
14306            .local_services
14307            .as_mut()
14308            .expect("local services enabled")
14309            .p2p_port_handlers
14310            .remove(&port);
14311    }
14312
14313    pub fn bind_p2p_stream_port<F>(&self, port: P2pPort, f: F) -> TelemetryResult<()>
14314    where
14315        F: Fn(P2pStreamEvent<'_>) -> TelemetryResult<()> + Send + Sync + 'static,
14316    {
14317        if port == 0 {
14318            return Err(TelemetryError::BadArg);
14319        }
14320        let mut st = self.state.lock();
14321        st.local_services
14322            .as_mut()
14323            .expect("local services enabled")
14324            .p2p_stream_handlers
14325            .entry(port)
14326            .or_default()
14327            .push(P2pStreamHandler {
14328                handler: Arc::new(f),
14329            });
14330        Ok(())
14331    }
14332
14333    pub fn clear_p2p_stream_port(&self, port: P2pPort) {
14334        self.state
14335            .lock()
14336            .local_services
14337            .as_mut()
14338            .expect("local services enabled")
14339            .p2p_stream_handlers
14340            .remove(&port);
14341    }
14342
14343    fn send_p2p_to_entry(
14344        &self,
14345        dst: AddressBookEntry,
14346        dst_port: P2pPort,
14347        src_port: P2pPort,
14348        payload: &[u8],
14349    ) -> TelemetryResult<()> {
14350        if dst_port == 0 {
14351            return Err(TelemetryError::BadArg);
14352        }
14353        let local = self.state.lock().local_address.clone();
14354        let payload = Self::encode_p2p_payload(
14355            local.hostname.as_ref(),
14356            local.address,
14357            src_port,
14358            dst_port,
14359            payload,
14360        )?;
14361        let pkt = Packet::new(
14362            DataType::P2pMessage,
14363            &[DataEndpoint::Discovery],
14364            local.hostname.as_ref(),
14365            self.packet_timestamp_ms(),
14366            crate::shared_bytes::convert(payload),
14367        )?;
14368        let target = Self::sender_hash(dst.hostname.as_ref());
14369        let item = self.attach_wire_contract_to_item(RouterItem::Packet(pkt.clone()), &[target])?;
14370        if target == Self::sender_hash(self.sender_arc().as_ref()) {
14371            self.dispatch_p2p_packet(&pkt)?;
14372            return Ok(());
14373        }
14374        self.tx_item_impl(RouterTxItem::Broadcast(item), true, false)
14375    }
14376
14377    pub fn send_p2p_to_hostname(
14378        &self,
14379        hostname: &str,
14380        dst_port: P2pPort,
14381        src_port: P2pPort,
14382        payload: &[u8],
14383    ) -> TelemetryResult<()> {
14384        let dst = self
14385            .resolve_hostname(hostname)
14386            .ok_or(TelemetryError::BadArg)?;
14387        self.send_p2p_to_entry(dst, dst_port, src_port, payload)
14388    }
14389
14390    pub fn send_p2p_to_address(
14391        &self,
14392        address: NodeAddress,
14393        dst_port: P2pPort,
14394        src_port: P2pPort,
14395        payload: &[u8],
14396    ) -> TelemetryResult<()> {
14397        let dst = self
14398            .resolve_address(address)
14399            .ok_or(TelemetryError::BadArg)?;
14400        self.send_p2p_to_entry(dst, dst_port, src_port, payload)
14401    }
14402
14403    fn open_p2p_stream_to_entry(
14404        &self,
14405        dst: AddressBookEntry,
14406        dst_port: P2pPort,
14407        src_port: P2pPort,
14408    ) -> TelemetryResult<P2pStreamId> {
14409        if dst_port == 0 || src_port == 0 {
14410            return Err(TelemetryError::BadArg);
14411        }
14412        let stream_id = {
14413            let mut st = self.state.lock();
14414            let stream_id = Self::allocate_p2p_stream_id_locked(&mut st);
14415            if stream_id == 0 {
14416                return Err(TelemetryError::Io("p2p stream id exhausted"));
14417            }
14418            st.local_services
14419                .as_mut()
14420                .expect("local services enabled")
14421                .p2p_stream_sessions
14422                .insert(
14423                    stream_id,
14424                    P2pStreamSession {
14425                        peer_hostname: dst.hostname.clone(),
14426                        peer_address: dst.address,
14427                        local_port: src_port,
14428                        peer_port: dst_port,
14429                        peer_stream_id: 0,
14430                        next_sequence: 1,
14431                        connected: false,
14432                    },
14433                );
14434            stream_id
14435        };
14436        let payload = Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
14437        self.send_p2p_to_entry(dst, dst_port, src_port, &payload)?;
14438        Ok(stream_id)
14439    }
14440
14441    pub fn open_p2p_stream_to_hostname(
14442        &self,
14443        hostname: &str,
14444        dst_port: P2pPort,
14445        src_port: P2pPort,
14446    ) -> TelemetryResult<P2pStreamId> {
14447        let dst = self
14448            .resolve_hostname(hostname)
14449            .ok_or(TelemetryError::BadArg)?;
14450        self.open_p2p_stream_to_entry(dst, dst_port, src_port)
14451    }
14452
14453    pub fn open_p2p_stream_to_address(
14454        &self,
14455        address: NodeAddress,
14456        dst_port: P2pPort,
14457        src_port: P2pPort,
14458    ) -> TelemetryResult<P2pStreamId> {
14459        let dst = self
14460            .resolve_address(address)
14461            .ok_or(TelemetryError::BadArg)?;
14462        self.open_p2p_stream_to_entry(dst, dst_port, src_port)
14463    }
14464
14465    fn send_p2p_stream_control(
14466        &self,
14467        stream_id: P2pStreamId,
14468        flags: u8,
14469        payload: &[u8],
14470    ) -> TelemetryResult<()> {
14471        let (dst, dst_port, src_port, peer_stream_id, seq, refresh_syn) = {
14472            let mut st = self.state.lock();
14473            let Some(session) = st
14474                .local_services
14475                .as_mut()
14476                .expect("local services enabled")
14477                .p2p_stream_sessions
14478                .get_mut(&stream_id)
14479            else {
14480                return Err(TelemetryError::BadArg);
14481            };
14482            let peer_hostname = session.peer_hostname.clone();
14483            let peer_address = session.peer_address;
14484            let peer_port = session.peer_port;
14485            let local_port = session.local_port;
14486            let peer_stream_id = session.peer_stream_id;
14487            let refresh_syn = peer_stream_id == 0 && flags & P2P_STREAM_SYN == 0;
14488            let seq = session.next_sequence;
14489            session.next_sequence = session.next_sequence.wrapping_add(1).max(1);
14490            let dst = st
14491                .address_book
14492                .get(peer_hostname.as_ref())
14493                .map(|entry| entry.as_ref().clone())
14494                .unwrap_or(AddressBookEntry {
14495                    hostname: peer_hostname.clone(),
14496                    address: peer_address,
14497                    requested_address: peer_address,
14498                    mode: AddressAssignmentMode::Dynamic,
14499                    birth_ms: self.clock.now_ms(),
14500                    owner_hash: Self::sender_hash(peer_hostname.as_ref()),
14501                    last_seen_ms: self.clock.now_ms(),
14502                });
14503            (dst, peer_port, local_port, peer_stream_id, seq, refresh_syn)
14504        };
14505        if refresh_syn {
14506            let syn_payload =
14507                Self::encode_p2p_stream_payload(P2P_STREAM_SYN, stream_id, 0, 0, &[])?;
14508            self.send_p2p_to_entry(dst.clone(), dst_port, src_port, &syn_payload)?;
14509        }
14510        let stream_payload =
14511            Self::encode_p2p_stream_payload(flags, stream_id, peer_stream_id, seq, payload)?;
14512        self.send_p2p_to_entry(dst, dst_port, src_port, &stream_payload)?;
14513        if flags & (P2P_STREAM_FIN | P2P_STREAM_RST) != 0 {
14514            self.state
14515                .lock()
14516                .local_services
14517                .as_mut()
14518                .expect("local services enabled")
14519                .p2p_stream_sessions
14520                .remove(&stream_id);
14521        }
14522        Ok(())
14523    }
14524
14525    pub fn send_p2p_stream(&self, stream_id: P2pStreamId, payload: &[u8]) -> TelemetryResult<()> {
14526        self.send_p2p_stream_control(stream_id, P2P_STREAM_DATA, payload)
14527    }
14528
14529    pub fn close_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
14530        self.send_p2p_stream_control(stream_id, P2P_STREAM_FIN, &[])
14531    }
14532
14533    pub fn reset_p2p_stream(&self, stream_id: P2pStreamId) -> TelemetryResult<()> {
14534        self.send_p2p_stream_control(stream_id, P2P_STREAM_RST, &[])
14535    }
14536
14537    pub fn set_sender<S: AsRef<str>>(&self, sender: S) {
14538        let hostname: Arc<str> = Arc::from(sender.as_ref());
14539        if !self.cfg.role.local_services() {
14540            // A transparent relay renames its protocol sender without claiming
14541            // a local application address or changing accepted wire owners.
14542            *self.sender.lock() = hostname;
14543            return;
14544        }
14545        let mut st = self.state.lock();
14546        let mut local = st.local_address.clone();
14547        local.hostname = hostname.clone();
14548        local.owner_hash = Self::sender_hash(hostname.as_ref());
14549        if matches!(local.mode, AddressAssignmentMode::Dynamic) {
14550            local.address = Self::fallback_address_for_hostname(hostname.as_ref());
14551            local.requested_address = 0;
14552        }
14553        let Ok(change) = self.update_local_identity_locked(
14554            &mut st,
14555            local,
14556            AddressChangeReason::Configured,
14557            None,
14558        ) else {
14559            return;
14560        };
14561        drop(st);
14562        let _ = self.notify_address_change(change);
14563    }
14564
14565    pub fn set_address_assignment(
14566        &self,
14567        mode: AddressAssignmentMode,
14568    ) -> TelemetryResult<AddressChange> {
14569        let mut st = self.state.lock();
14570        let mut local = st.local_address.clone();
14571        local.mode = mode;
14572        local.requested_address = mode.requested_address();
14573        local.address = match mode {
14574            AddressAssignmentMode::Dynamic => {
14575                Self::fallback_address_for_hostname(local.hostname.as_ref())
14576            }
14577            AddressAssignmentMode::Requested(address) | AddressAssignmentMode::Static(address) => {
14578                if address == 0 {
14579                    1
14580                } else {
14581                    address
14582                }
14583            }
14584        };
14585        let spare = if st
14586            .address_book
14587            .contains_key(st.local_address.hostname.as_ref())
14588        {
14589            None
14590        } else {
14591            #[cfg(any(feature = "discovery", feature = "timesync"))]
14592            let record = crate::peer_record::try_box(st.local_address.clone())?;
14593            #[cfg(not(any(feature = "discovery", feature = "timesync")))]
14594            let record = Box::new(st.local_address.clone());
14595            Some(record)
14596        };
14597        let change = self.update_local_identity_locked(
14598            &mut st,
14599            local,
14600            AddressChangeReason::Configured,
14601            spare,
14602        )?;
14603        drop(st);
14604        self.notify_address_change(change.clone())?;
14605        Ok(change)
14606    }
14607
14608    /// Register a side whose TX callback consumes packed packet bytes.
14609    ///
14610    /// `name` is exported in topology/debug views and does not affect routing semantics.
14611    /// `tx` is called whenever the router decides to send a packet toward this side.
14612    ///
14613    /// The default options disable the router's per-link reliable framing on this side. Use
14614    /// [`Router::add_side_packed_with_options`] when this hop should participate in router
14615    /// reliable ACK/retransmit behavior.
14616    pub fn add_side_packed<N, F>(&self, name: N, tx: F) -> RouterSideId
14617    where
14618        N: AsRef<str>,
14619        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14620    {
14621        self.add_side_packed_with_options(name, tx, RouterSideOptions::default())
14622    }
14623
14624    /// Register a packed side with the compact small-packet transport preset enabled.
14625    ///
14626    /// `max_frame_bytes == 0` keeps header-template reuse enabled without chunking.
14627    pub fn add_side_packed_small_packets<N, F>(
14628        &self,
14629        name: N,
14630        tx: F,
14631        max_frame_bytes: usize,
14632    ) -> RouterSideId
14633    where
14634        N: AsRef<str>,
14635        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14636    {
14637        self.add_side_packed_with_options(
14638            name,
14639            tx,
14640            RouterSideOptions::default().with_small_packet_transport(max_frame_bytes),
14641        )
14642    }
14643
14644    /// Register a packed-output side with explicit side options.
14645    ///
14646    /// `opts.reliable_enabled` enables the router's per-hop reliable framing on this side only.
14647    /// That means reliable schema traffic on this side uses router-managed ACKs, packet requests,
14648    /// and retransmits before the bytes reach `tx`.
14649    ///
14650    /// `opts.link_local_enabled` allows link-local-only endpoints and discovery routes to use this
14651    /// side. `ingress_enabled` and `egress_enabled` set the initial directional policy.
14652    pub fn add_side_packed_with_options<N, F>(
14653        &self,
14654        name: N,
14655        tx: F,
14656        opts: RouterSideOptions,
14657    ) -> RouterSideId
14658    where
14659        N: AsRef<str>,
14660        F: Fn(&[u8]) -> TelemetryResult<()> + Send + Sync + 'static,
14661    {
14662        let mut st = self.state.lock();
14663        let side = Some(RouterSide {
14664            name: Arc::from(name.as_ref()),
14665            tx_handler: RouterTxHandlerFn::Packed(Arc::new(tx)),
14666            opts,
14667        });
14668        let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14669            st.sides[id] = side;
14670            id
14671        } else {
14672            let id = st.sides.len();
14673            st.sides.push(side);
14674            id
14675        };
14676        st.side_runtime_stats
14677            .insert(id, Box::<SideRuntimeStatsInner>::default());
14678        st.side_transport
14679            .insert(id, Box::<SideTransportState>::default());
14680        #[cfg(feature = "discovery")]
14681        Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14682        id
14683    }
14684
14685    /// Register a packed-output side whose callback also receives SEDSNet's
14686    /// logical scheduler priority for the frame.
14687    ///
14688    /// This is intended for asynchronous host transports that must queue the
14689    /// emitted bytes after this callback returns. Every compact/chunked frame
14690    /// belonging to a logical packet receives the same priority, so transport
14691    /// backpressure cannot erase discovery, shared-state, time-sync, or schema
14692    /// ordering.
14693    pub fn add_side_packed_with_priority_and_options<N, F>(
14694        &self,
14695        name: N,
14696        tx: F,
14697        opts: RouterSideOptions,
14698    ) -> RouterSideId
14699    where
14700        N: AsRef<str>,
14701        F: Fn(&[u8], u8) -> TelemetryResult<()> + Send + Sync + 'static,
14702    {
14703        let mut st = self.state.lock();
14704        let side = Some(RouterSide {
14705            name: Arc::from(name.as_ref()),
14706            tx_handler: RouterTxHandlerFn::PackedWithPriority(Arc::new(tx)),
14707            opts,
14708        });
14709        let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14710            st.sides[id] = side;
14711            id
14712        } else {
14713            let id = st.sides.len();
14714            st.sides.push(side);
14715            id
14716        };
14717        st.side_runtime_stats
14718            .insert(id, Box::<SideRuntimeStatsInner>::default());
14719        st.side_transport
14720            .insert(id, Box::<SideTransportState>::default());
14721        #[cfg(feature = "discovery")]
14722        Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14723        id
14724    }
14725
14726    /// Register a side whose TX callback receives decoded [`Packet`] values.
14727    ///
14728    /// Packet-output sides do not preserve the packed reliable hop framing, so
14729    /// `RouterSideOptions::reliable_enabled` only has effect for packed sides.
14730    pub fn add_side_packet<N, F>(&self, name: N, tx: F) -> RouterSideId
14731    where
14732        N: AsRef<str>,
14733        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
14734    {
14735        self.add_side_packet_with_options(name, tx, RouterSideOptions::default())
14736    }
14737
14738    /// Register a packet-output side with explicit side options.
14739    ///
14740    /// `opts.reliable_enabled` still records the operator's intent for this side, but packet-based
14741    /// callbacks receive decoded packets rather than the router's packed reliable hop framing.
14742    /// For router-managed per-link reliable sequencing and ACKs, use a packed side instead.
14743    pub fn add_side_packet_with_options<N, F>(
14744        &self,
14745        name: N,
14746        tx: F,
14747        opts: RouterSideOptions,
14748    ) -> RouterSideId
14749    where
14750        N: AsRef<str>,
14751        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
14752    {
14753        let mut st = self.state.lock();
14754        let side = Some(RouterSide {
14755            name: Arc::from(name.as_ref()),
14756            tx_handler: RouterTxHandlerFn::Packet(Arc::new(tx)),
14757            opts,
14758        });
14759        let id = if let Some(id) = st.sides.iter().position(Option::is_none) {
14760            st.sides[id] = side;
14761            id
14762        } else {
14763            let id = st.sides.len();
14764            st.sides.push(side);
14765            id
14766        };
14767        st.side_runtime_stats
14768            .insert(id, Box::<SideRuntimeStatsInner>::default());
14769        st.side_transport
14770            .insert(id, Box::<SideTransportState>::default());
14771        #[cfg(feature = "discovery")]
14772        Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
14773        id
14774    }
14775
14776    /// Remove a side while keeping existing side IDs stable.
14777    ///
14778    /// `side` must be an id returned earlier by one of the `add_side_*` calls. Removed side IDs
14779    /// are tombstoned rather than renumbered so cached IDs for the remaining sides stay valid.
14780    pub fn remove_side(&self, side: RouterSideId) -> TelemetryResult<()> {
14781        #[cfg(feature = "discovery")]
14782        let now_ms = self.clock.now_ms();
14783        {
14784            let mut st = self.state.lock();
14785            if st.reliable_return_routes.values().any(|route| {
14786                route.side == side && route.local_admission_token != 0 && route.admission_committing
14787            }) {
14788                return Err(TelemetryError::Io("side rx commit busy"));
14789            }
14790
14791            #[cfg(feature = "std")]
14792            if st
14793                .side_transport
14794                .get(&side)
14795                .is_some_and(|state| state.tx_callback_gate.is_active())
14796            {
14797                return Err(TelemetryError::Io("side tx busy"));
14798            }
14799            let slot = st.sides.get_mut(side).ok_or(TelemetryError::BadArg)?;
14800            if slot.is_none() {
14801                return Err(TelemetryError::BadArg);
14802            }
14803            #[cfg(feature = "std")]
14804            self.invalidate_retry_side(side);
14805            *slot = None;
14806            #[cfg(feature = "latest-value-egress")]
14807            if st.latest_side() == Some(side) {
14808                st.latest_turn_phase = 0;
14809                st.latest_turn_credit = 0;
14810                st.tx_policy_flags &= 1;
14811            }
14812            while st.sides.last().is_some_and(Option::is_none) {
14813                st.sides.pop();
14814            }
14815            // Keep the small side table allocation for reuse. Embedded links can
14816            // disappear and return frequently; shrinking here turned every tail
14817            // remove/add cycle into a free/allocation pair. The retained capacity
14818            // remains owned by the router and is released when the router drops.
14819            st.route_overrides
14820                .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14821            st.typed_route_overrides
14822                .retain(|(src_side, _, dst_side), _| *src_side != Some(side) && *dst_side != side);
14823            st.route_weights
14824                .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14825            st.route_priorities
14826                .retain(|(src_side, dst_side), _| *src_side != Some(side) && *dst_side != side);
14827            st.source_route_modes.remove(&Some(side));
14828            st.route_selection_cursors.remove(&Some(side));
14829            st.adaptive_route_stats.remove(&side);
14830            #[cfg(feature = "discovery")]
14831            st.discovery_side_throttle.remove(&side);
14832            #[cfg(all(feature = "discovery", feature = "timesync"))]
14833            st.timesync_side_throttle.remove(&side);
14834            st.side_runtime_stats.remove(&side);
14835            st.side_transport.remove(&side);
14836            st.reliable_return_routes.retain(|_, route| {
14837                if route.side != side {
14838                    return true;
14839                }
14840                if route.local_admission_token != 0 {
14841                    route.admission_cancelled = true;
14842                    return true;
14843                }
14844                false
14845            });
14846            st.retain_transmit(
14847                |queued| {
14848                    !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
14849                        && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
14850                },
14851            );
14852            st.received_queue.retain(|queued| queued.src != Some(side));
14853            st.reliable_tx.retain(|(side_id, _), _| *side_id != side);
14854            st.reliable_rx.retain(|(side_id, _), _| *side_id != side);
14855            #[cfg(feature = "discovery")]
14856            {
14857                st.cancel_managed_refresh_side(side);
14858                st.discovery_routes.remove(&side);
14859                Self::note_discovery_topology_change_locked(&mut st, now_ms);
14860            }
14861        }
14862        if let Some(queue) = self.isr_rx_queue.as_ref() {
14863            let mut isr_rx = queue.try_lock()?;
14864            isr_rx.retain(|queued| queued.src != Some(side));
14865        }
14866        Ok(())
14867    }
14868
14869    /// Opt one packed egress into newest-pending best-effort reports (default disabled).
14870    /// Local delivery and other sides are unchanged. Only never-attempted plaintext,
14871    /// non-contract, non-managed, unreliable data below priority200 may be replaced.
14872    /// One side per router, IDs0..126; None disables. Older source timestamps cannot
14873    /// replace newer pending values. Already attempted/partial owners always finish.
14874    #[cfg(feature = "latest-value-egress")]
14875    pub fn set_latest_value_side(&self, side: Option<RouterSideId>) -> TelemetryResult<()> {
14876        let mut st = self.state.lock();
14877        let encoded = if let Some(side) = side {
14878            if side >= 127 {
14879                return Err(TelemetryError::BadArg);
14880            }
14881            let entry = Self::side_ref(&st, side).map_err(|_| TelemetryError::BadArg)?;
14882            if matches!(entry.tx_handler, RouterTxHandlerFn::Packet(_)) {
14883                return Err(TelemetryError::BadArg);
14884            }
14885            ((side as u8) + 1) << 1
14886        } else {
14887            0
14888        };
14889        if encoded != 0 && st.latest_service_seq == u32::MAX {
14890            return Err(TelemetryError::BadArg);
14891        }
14892        st.latest_service_seq = st.latest_service_seq.saturating_add(1);
14893        st.tx_policy_flags = (st.tx_policy_flags & 1) | encoded;
14894        st.latest_turn_phase = 0;
14895        st.latest_turn_credit = 0;
14896        st.clear_canonical_turn_flags();
14897        Ok(())
14898    }
14899
14900    /// Enable or disable ingress processing for a registered side.
14901    ///
14902    /// When disabled, ingress is rejected before local delivery, discovery learning, or forwarding.
14903    pub fn set_side_ingress_enabled(
14904        &self,
14905        side: RouterSideId,
14906        enabled: bool,
14907    ) -> TelemetryResult<()> {
14908        #[cfg(feature = "discovery")]
14909        let now_ms = self.clock.now_ms();
14910        let mut st = self.state.lock();
14911        if !enabled
14912            && st.reliable_return_routes.values().any(|route| {
14913                route.side == side && route.local_admission_token != 0 && route.admission_committing
14914            })
14915        {
14916            return Err(TelemetryError::Io("side rx commit busy"));
14917        }
14918
14919        let side_ref = st
14920            .sides
14921            .get_mut(side)
14922            .and_then(|side| side.as_mut())
14923            .ok_or(TelemetryError::BadArg)?;
14924        side_ref.opts.ingress_enabled = enabled;
14925        if !enabled {
14926            for route in st.reliable_return_routes.values_mut() {
14927                if route.side == side && route.local_admission_token != 0 {
14928                    route.admission_cancelled = true;
14929                }
14930            }
14931        }
14932        #[cfg(feature = "discovery")]
14933        if !enabled {
14934            st.cancel_managed_refresh_side(side);
14935        }
14936        #[cfg(feature = "discovery")]
14937        Self::note_discovery_topology_change_locked(&mut st, now_ms);
14938        Ok(())
14939    }
14940
14941    /// Enable or disable egress toward a registered side.
14942    ///
14943    /// When `enabled` is `false`, the router keeps the side but stops routing packets toward it.
14944    pub fn set_side_egress_enabled(
14945        &self,
14946        side: RouterSideId,
14947        enabled: bool,
14948    ) -> TelemetryResult<()> {
14949        #[cfg(feature = "discovery")]
14950        let now_ms = self.clock.now_ms();
14951        let mut st = self.state.lock();
14952        let changed = st
14953            .sides
14954            .get(side)
14955            .and_then(|side| side.as_ref())
14956            .ok_or(TelemetryError::BadArg)?
14957            .opts
14958            .egress_enabled
14959            != enabled;
14960        if !enabled
14961            && changed
14962            && st.reliable_return_routes.values().any(|route| {
14963                route.side == side && route.local_admission_token != 0 && route.admission_committing
14964            })
14965        {
14966            return Err(TelemetryError::Io("side rx commit busy"));
14967        }
14968
14969        let side_ref = st
14970            .sides
14971            .get_mut(side)
14972            .and_then(|side| side.as_mut())
14973            .ok_or(TelemetryError::BadArg)?;
14974        side_ref.opts.egress_enabled = enabled;
14975        if !enabled {
14976            #[cfg(feature = "discovery")]
14977            st.cancel_managed_refresh_side(side);
14978            // Reasserting an unchanged receive-only policy is not a side
14979            // lifecycle event. Preserve revocation for an actual transition.
14980            if changed {
14981                for route in st.reliable_return_routes.values_mut() {
14982                    if route.side == side && route.local_admission_token != 0 {
14983                        route.admission_cancelled = true;
14984                    }
14985                }
14986            }
14987        }
14988        #[cfg(feature = "latest-value-egress")]
14989        if changed && st.latest_side() == Some(side) {
14990            if let Some(next) = st.latest_service_seq.checked_add(1) {
14991                st.latest_service_seq = next;
14992            } else {
14993                st.tx_policy_flags &= 1;
14994            }
14995            st.latest_turn_phase = 0;
14996            st.latest_turn_credit = 0;
14997            st.clear_canonical_turn_flags();
14998        }
14999        if !enabled {
15000            st.retain_transmit(
15001                |queued| {
15002                    !matches!(&queued.item, RouterTxItem::ToSide { dst, .. } if *dst == side)
15003                        && !matches!(&queued.item, RouterTxItem::ReliableReplay { dst, .. } if *dst == side)
15004                },
15005            );
15006        }
15007        #[cfg(feature = "discovery")]
15008        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15009        Ok(())
15010    }
15011
15012    /// Set the route-selection policy for traffic originating from `src`.
15013    ///
15014    /// `src == None` targets locally-originated router TX. `src == Some(side)` targets traffic
15015    /// that was received from a specific ingress side. `mode` only matters when more than one
15016    /// destination side is currently eligible.
15017    pub fn set_source_route_mode(
15018        &self,
15019        src: Option<RouterSideId>,
15020        mode: RouteSelectionMode,
15021    ) -> TelemetryResult<()> {
15022        #[cfg(feature = "discovery")]
15023        let now_ms = self.clock.now_ms();
15024        let mut st = self.state.lock();
15025        if let Some(src) = src {
15026            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15027        }
15028        // Keep an explicitly selected Fanout mode distinct from the absence
15029        // of a policy. With discovery enabled, no policy intentionally uses
15030        // adaptive single-path selection; explicit Fanout must override that.
15031        st.source_route_modes.insert(src, mode);
15032        st.route_selection_cursors.remove(&src);
15033        #[cfg(feature = "discovery")]
15034        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15035        Ok(())
15036    }
15037
15038    /// Clear any source-specific route-selection override for `src`.
15039    pub fn clear_source_route_mode(&self, src: Option<RouterSideId>) -> TelemetryResult<()> {
15040        #[cfg(feature = "discovery")]
15041        let now_ms = self.clock.now_ms();
15042        let mut st = self.state.lock();
15043        if let Some(src) = src {
15044            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15045        }
15046        st.source_route_modes.remove(&src);
15047        st.route_selection_cursors.remove(&src);
15048        #[cfg(feature = "discovery")]
15049        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15050        Ok(())
15051    }
15052
15053    /// Set the weighted-routing weight from `src` toward `dst`.
15054    ///
15055    /// Higher `weight` values make `dst` more likely to be chosen when the source route mode is
15056    /// [`RouteSelectionMode::Weighted`]. `src == None` applies to locally-originated TX.
15057    pub fn set_route_weight(
15058        &self,
15059        src: Option<RouterSideId>,
15060        dst: RouterSideId,
15061        weight: u32,
15062    ) -> TelemetryResult<()> {
15063        #[cfg(feature = "discovery")]
15064        let now_ms = self.clock.now_ms();
15065        let mut st = self.state.lock();
15066        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15067        if let Some(src) = src {
15068            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15069        }
15070        st.route_weights.insert((src, dst), weight);
15071        st.route_selection_cursors.remove(&src);
15072        #[cfg(feature = "discovery")]
15073        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15074        Ok(())
15075    }
15076
15077    /// Clear a previously configured weighted-routing weight override.
15078    pub fn clear_route_weight(
15079        &self,
15080        src: Option<RouterSideId>,
15081        dst: RouterSideId,
15082    ) -> TelemetryResult<()> {
15083        #[cfg(feature = "discovery")]
15084        let now_ms = self.clock.now_ms();
15085        let mut st = self.state.lock();
15086        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15087        if let Some(src) = src {
15088            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15089        }
15090        st.route_weights.remove(&(src, dst));
15091        st.route_selection_cursors.remove(&src);
15092        #[cfg(feature = "discovery")]
15093        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15094        Ok(())
15095    }
15096
15097    /// Set the failover priority from `src` toward `dst`.
15098    ///
15099    /// Lower numeric `priority` wins when the source route mode is
15100    /// [`RouteSelectionMode::Failover`]. `src == None` applies to locally-originated TX.
15101    pub fn set_route_priority(
15102        &self,
15103        src: Option<RouterSideId>,
15104        dst: RouterSideId,
15105        priority: u32,
15106    ) -> TelemetryResult<()> {
15107        #[cfg(feature = "discovery")]
15108        let now_ms = self.clock.now_ms();
15109        let mut st = self.state.lock();
15110        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15111        if let Some(src) = src {
15112            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15113        }
15114        st.route_priorities.insert((src, dst), priority);
15115        #[cfg(feature = "discovery")]
15116        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15117        Ok(())
15118    }
15119
15120    /// Clear a previously configured failover priority override.
15121    pub fn clear_route_priority(
15122        &self,
15123        src: Option<RouterSideId>,
15124        dst: RouterSideId,
15125    ) -> TelemetryResult<()> {
15126        #[cfg(feature = "discovery")]
15127        let now_ms = self.clock.now_ms();
15128        let mut st = self.state.lock();
15129        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15130        if let Some(src) = src {
15131            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15132        }
15133        st.route_priorities.remove(&(src, dst));
15134        #[cfg(feature = "discovery")]
15135        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15136        Ok(())
15137    }
15138
15139    /// Allow or block routing from `src` toward `dst`.
15140    ///
15141    /// `src == None` controls locally-originated router TX. `enabled == false` is the sink-like
15142    /// building block for disabling specific directions without changing router construction mode.
15143    pub fn set_route(
15144        &self,
15145        src: Option<RouterSideId>,
15146        dst: RouterSideId,
15147        enabled: bool,
15148    ) -> TelemetryResult<()> {
15149        #[cfg(feature = "discovery")]
15150        let now_ms = self.clock.now_ms();
15151        let mut st = self.state.lock();
15152        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15153        if let Some(src) = src {
15154            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15155        }
15156        st.route_overrides.insert((src, dst), enabled);
15157        #[cfg(feature = "discovery")]
15158        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15159        Ok(())
15160    }
15161
15162    /// Allow or block routing for a specific `DataType` from `src` toward `dst`.
15163    ///
15164    /// Typed route rules act as allowlists once any rule exists for the `(src, ty)` pair.
15165    /// `src == None` targets locally-originated router TX.
15166    pub fn set_typed_route(
15167        &self,
15168        src: Option<RouterSideId>,
15169        ty: DataType,
15170        dst: RouterSideId,
15171        enabled: bool,
15172    ) -> TelemetryResult<()> {
15173        #[cfg(feature = "discovery")]
15174        let now_ms = self.clock.now_ms();
15175        let mut st = self.state.lock();
15176        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15177        if let Some(src) = src {
15178            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15179        }
15180        st.typed_route_overrides
15181            .insert((src, ty.as_u32(), dst), enabled);
15182        #[cfg(feature = "discovery")]
15183        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15184        Ok(())
15185    }
15186
15187    /// Clear a typed route override for the `(src, ty, dst)` triple.
15188    pub fn clear_typed_route(
15189        &self,
15190        src: Option<RouterSideId>,
15191        ty: DataType,
15192        dst: RouterSideId,
15193    ) -> TelemetryResult<()> {
15194        #[cfg(feature = "discovery")]
15195        let now_ms = self.clock.now_ms();
15196        let mut st = self.state.lock();
15197        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15198        if let Some(src) = src {
15199            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15200        }
15201        st.typed_route_overrides.remove(&(src, ty.as_u32(), dst));
15202        #[cfg(feature = "discovery")]
15203        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15204        Ok(())
15205    }
15206
15207    /// Clear a non-typed route override so the router falls back to default behavior.
15208    pub fn clear_route(&self, src: Option<RouterSideId>, dst: RouterSideId) -> TelemetryResult<()> {
15209        #[cfg(feature = "discovery")]
15210        let now_ms = self.clock.now_ms();
15211        let mut st = self.state.lock();
15212        let _ = Self::side_ref(&st, dst).map_err(|_| TelemetryError::BadArg)?;
15213        if let Some(src) = src {
15214            let _ = Self::side_ref(&st, src).map_err(|_| TelemetryError::BadArg)?;
15215        }
15216        st.route_overrides.remove(&(src, dst));
15217        #[cfg(feature = "discovery")]
15218        Self::note_discovery_topology_change_locked(&mut st, now_ms);
15219        Ok(())
15220    }
15221
15222    /// Queue a built-in discovery advertisement describing this router's local endpoints.
15223    #[cfg(feature = "discovery")]
15224    pub fn announce_discovery(&self) -> TelemetryResult<()> {
15225        self.queue_discovery_announce(true, false)
15226    }
15227
15228    /// Broadcast that this router is leaving so peers can prune topology immediately.
15229    #[cfg(feature = "discovery")]
15230    pub fn announce_leave(&self) -> TelemetryResult<()> {
15231        let sender = self.sender_arc();
15232        let pkt = discovery::build_discovery_leave(sender.as_ref(), self.clock.now_ms())?;
15233        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15234    }
15235
15236    /// Queue a discovery advertisement if the adaptive cadence says one is due.
15237    #[cfg(feature = "discovery")]
15238    pub fn poll_discovery(&self) -> TelemetryResult<bool> {
15239        // Keep ordinary route/liveness cadence independent of schema pressure.
15240        let mut emitted = self.poll_discovery_announce()?;
15241        let side_count = self.state.lock().sides.len();
15242        for side in 0..side_count {
15243            if self.state.lock().sides[side].is_some() {
15244                emitted |= self.poll_schema_response_on_side(side)?;
15245            }
15246        }
15247        Ok(emitted)
15248    }
15249
15250    #[cfg(feature = "discovery")]
15251    pub fn request_topology(&self) -> TelemetryResult<()> {
15252        let sender = self.sender_arc();
15253        let pkt =
15254            discovery::build_discovery_topology_request(sender.as_ref(), self.clock.now_ms())?;
15255        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15256    }
15257
15258    #[cfg(feature = "discovery")]
15259    pub fn request_schema(&self) -> TelemetryResult<()> {
15260        let sender = self.sender_arc();
15261        let pkt = discovery::build_discovery_schema_request(sender.as_ref(), self.clock.now_ms())?;
15262        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15263    }
15264
15265    /// Mark a data type as a network-managed variable.
15266    ///
15267    /// The router caches the latest packet for this type when it is locally transmitted or
15268    /// received from the network. Peers can request the latest cached value and the router will
15269    /// replay the original value packet, so user endpoint handlers see the same API shape as a
15270    /// normal update.
15271    pub fn enable_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
15272        self.enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
15273    }
15274
15275    /// Mark a data type as a network variable with local read/write permissions.
15276    ///
15277    /// Users do not register a separate endpoint for network variables. Values are cached by
15278    /// data type and refreshed through SEDSnet's internal managed-variable control packets.
15279    pub fn enable_network_variable(
15280        &self,
15281        ty: DataType,
15282        permissions: NetworkVariablePermissions,
15283    ) -> TelemetryResult<()> {
15284        if is_internal_control_type(ty) {
15285            return Err(TelemetryError::InvalidType);
15286        }
15287        let mut st = self.state.lock();
15288        #[cfg(feature = "discovery")]
15289        let newly_advertised = st
15290            .local_services
15291            .as_mut()
15292            .expect("local services enabled")
15293            .managed_variable_types
15294            .insert(ty.as_u32());
15295        #[cfg(not(feature = "discovery"))]
15296        st.local_services
15297            .as_mut()
15298            .expect("local services enabled")
15299            .managed_variable_types
15300            .insert(ty.as_u32());
15301        st.local_services
15302            .as_mut()
15303            .expect("local services enabled")
15304            .managed_variable_permissions
15305            .insert(ty.as_u32(), permissions);
15306        #[cfg(feature = "discovery")]
15307        if newly_advertised {
15308            Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15309        }
15310        Ok(())
15311    }
15312
15313    /// Register a callback that runs when an inbound network update changes this variable cache.
15314    ///
15315    /// The callback is invoked after the router state lock is released, so it may call back into
15316    /// the router. Local `set_network_variable` and `seed_managed_variable` calls update the cache
15317    /// without invoking this network-update callback.
15318    pub fn on_network_variable_update<F>(&self, ty: DataType, f: F) -> TelemetryResult<()>
15319    where
15320        F: Fn(&Packet) -> TelemetryResult<()> + Send + Sync + 'static,
15321    {
15322        if is_internal_control_type(ty) {
15323            return Err(TelemetryError::InvalidType);
15324        }
15325        let mut st = self.state.lock();
15326        #[cfg(feature = "discovery")]
15327        let newly_advertised = st
15328            .local_services
15329            .as_mut()
15330            .expect("local services enabled")
15331            .managed_variable_types
15332            .insert(ty.as_u32());
15333        #[cfg(not(feature = "discovery"))]
15334        st.local_services
15335            .as_mut()
15336            .expect("local services enabled")
15337            .managed_variable_types
15338            .insert(ty.as_u32());
15339        st.local_services
15340            .as_mut()
15341            .expect("local services enabled")
15342            .network_variable_update_handlers
15343            .entry(ty.as_u32())
15344            .or_default()
15345            .push(NetworkVariableUpdateHandler {
15346                handler: Arc::new(f),
15347            });
15348        #[cfg(feature = "discovery")]
15349        if newly_advertised {
15350            Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15351        }
15352        Ok(())
15353    }
15354
15355    pub fn disable_managed_variable(&self, ty: DataType) {
15356        let mut st = self.state.lock();
15357        #[cfg(feature = "discovery")]
15358        let was_advertised = st
15359            .local_services
15360            .as_mut()
15361            .expect("local services enabled")
15362            .managed_variable_types
15363            .remove(&ty.as_u32());
15364        #[cfg(not(feature = "discovery"))]
15365        st.local_services
15366            .as_mut()
15367            .expect("local services enabled")
15368            .managed_variable_types
15369            .remove(&ty.as_u32());
15370        st.local_services
15371            .as_mut()
15372            .expect("local services enabled")
15373            .managed_variable_permissions
15374            .remove(&ty.as_u32());
15375        st.local_services
15376            .as_mut()
15377            .expect("local services enabled")
15378            .managed_variable_latest
15379            .remove(&ty.as_u32());
15380        st.local_services
15381            .as_mut()
15382            .expect("local services enabled")
15383            .network_variable_update_handlers
15384            .remove(&ty.as_u32());
15385        #[cfg(feature = "discovery")]
15386        if was_advertised {
15387            Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15388        }
15389    }
15390
15391    /// Forget the cached observation of one network variable.
15392    ///
15393    /// This retains its permissions, update callbacks and service advertisement.
15394    /// It neither publishes a replacement value nor disables the variable. A
15395    /// callback that rejects an inbound value may use this before returning an
15396    /// error so that the same value can be considered again on a later update.
15397    /// Returns whether a cached packet was removed. Routers without local
15398    /// services return `false` without creating local state.
15399    pub fn invalidate_network_variable_cache(&self, ty: DataType) -> bool {
15400        self.state
15401            .lock()
15402            .local_services
15403            .as_mut()
15404            .is_some_and(|local| local.managed_variable_latest.remove(&ty.as_u32()).is_some())
15405    }
15406
15407    /// Forget all cached network-variable observations, retaining registrations.
15408    ///
15409    /// All cached types are affected, including types unrelated to the caller's
15410    /// current operation. Permissions, callbacks and advertised variable types
15411    /// are unchanged. No replacement values or refresh requests are sent.
15412    /// Returns the number of cached packets removed. Routers without local
15413    /// services return zero without allocating local state.
15414    pub fn clear_network_variable_caches(&self) -> usize {
15415        let mut state = self.state.lock();
15416        let Some(local) = state.local_services.as_mut() else {
15417            return 0;
15418        };
15419        let count = local.managed_variable_latest.len();
15420        local.managed_variable_latest.clear();
15421        count
15422    }
15423
15424    pub fn seed_managed_variable(&self, pkt: Packet) -> TelemetryResult<()> {
15425        if is_internal_control_type(pkt.data_type()) {
15426            return Err(TelemetryError::InvalidType);
15427        }
15428        pkt.validate()?;
15429        {
15430            let mut st = self.state.lock();
15431            #[cfg(feature = "discovery")]
15432            let newly_advertised = st
15433                .local_services
15434                .as_mut()
15435                .expect("local services enabled")
15436                .managed_variable_types
15437                .insert(pkt.data_type().as_u32());
15438            #[cfg(not(feature = "discovery"))]
15439            st.local_services
15440                .as_mut()
15441                .expect("local services enabled")
15442                .managed_variable_types
15443                .insert(pkt.data_type().as_u32());
15444            #[cfg(feature = "discovery")]
15445            if newly_advertised {
15446                Self::note_discovery_topology_change_locked(&mut st, self.clock.now_ms());
15447            }
15448        }
15449        self.cache_managed_variable_packet(&pkt, false, None)
15450    }
15451
15452    pub fn cached_managed_variable(&self, ty: DataType) -> Option<Packet> {
15453        self.managed_variable_latest(ty)
15454    }
15455
15456    /// Set a network variable for the whole network, permission policy allowing.
15457    ///
15458    /// The packet is cached locally and sent as normal user data. Routers that have seen or enabled
15459    /// this variable update their local cache internally; applications do not need to register a
15460    /// managed-variable endpoint.
15461    pub fn set_network_variable(&self, pkt: Packet) -> TelemetryResult<()> {
15462        if is_internal_control_type(pkt.data_type()) {
15463            return Err(TelemetryError::InvalidType);
15464        }
15465        pkt.validate()?;
15466        let ty = pkt.data_type();
15467        {
15468            let mut st = self.state.lock();
15469            st.local_services
15470                .as_mut()
15471                .expect("local services enabled")
15472                .managed_variable_types
15473                .insert(ty.as_u32());
15474            let perms = Self::managed_variable_permissions_locked(&st, ty);
15475            if !perms.write {
15476                return Err(TelemetryError::PermissionDenied);
15477            }
15478        }
15479        self.cache_managed_variable_packet(&pkt, false, None)?;
15480        #[cfg(feature = "discovery")]
15481        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
15482        let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
15483        if self.side_tx_active() {
15484            return self.tx_queue_item_with_priority(
15485                item,
15486                false,
15487                crate::transport_priority(DataType::ManagedVariableValue),
15488            );
15489        }
15490        self.tx_item(item)
15491    }
15492
15493    /// Read a cached network variable, requesting a refresh if missing or stale.
15494    ///
15495    /// Returns the cached value when present. If the value is missing, this queues an internal
15496    /// managed-variable request and returns `Ok(None)`. If the value is stale, this queues a refresh
15497    /// request and returns the stale cached value so callers can continue operating while the network
15498    /// catches up.
15499    ///
15500    /// Cache age is local observation age. A refresh reply preserves the
15501    /// original source timestamp and packet identity; it is not proof of a new
15502    /// source measurement or a request-correlated application readback.
15503    #[cfg(feature = "discovery")]
15504    pub fn get_network_variable(
15505        &self,
15506        ty: DataType,
15507        stale_after_ms: Option<u64>,
15508    ) -> TelemetryResult<Option<Packet>> {
15509        if is_internal_control_type(ty) {
15510            return Err(TelemetryError::InvalidType);
15511        }
15512        {
15513            let mut st = self.state.lock();
15514            st.local_services
15515                .as_mut()
15516                .expect("local services enabled")
15517                .managed_variable_types
15518                .insert(ty.as_u32());
15519            let perms = Self::managed_variable_permissions_locked(&st, ty);
15520            if !perms.read {
15521                return Err(TelemetryError::PermissionDenied);
15522            }
15523        }
15524        let cached = self.managed_variable_latest_with_age(ty);
15525        let needs_refresh = match (cached.as_ref(), stale_after_ms) {
15526            (None, _) => true,
15527            (Some((_pkt, age_ms)), Some(max_age_ms)) => *age_ms > max_age_ms,
15528            (Some(_), None) => false,
15529        };
15530        if needs_refresh {
15531            self.request_managed_variable(ty)?;
15532        }
15533        Ok(cached.map(|(pkt, _age_ms)| pkt))
15534    }
15535
15536    /// Read a cached network variable without issuing a network refresh.
15537    pub fn get_cached_network_variable(&self, ty: DataType) -> TelemetryResult<Option<Packet>> {
15538        if is_internal_control_type(ty) {
15539            return Err(TelemetryError::InvalidType);
15540        }
15541        if !self.can_read_managed_variable(ty) {
15542            return Err(TelemetryError::PermissionDenied);
15543        }
15544        Ok(self.managed_variable_latest(ty))
15545    }
15546
15547    #[cfg(feature = "discovery")]
15548    pub fn request_managed_variable(&self, ty: DataType) -> TelemetryResult<()> {
15549        if is_internal_control_type(ty) {
15550            return Err(TelemetryError::InvalidType);
15551        }
15552        if !self.can_read_managed_variable(ty) {
15553            return Err(TelemetryError::PermissionDenied);
15554        }
15555        let sender = self.sender_arc();
15556        let pkt =
15557            discovery::build_managed_variable_request(sender.as_ref(), self.clock.now_ms(), ty)?;
15558        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
15559    }
15560
15561    #[cfg(feature = "discovery")]
15562    pub fn request_managed_variable_by_name(&self, ty_name: &str) -> TelemetryResult<()> {
15563        let ty = DataType::try_named(ty_name).ok_or(TelemetryError::InvalidType)?;
15564        self.request_managed_variable(ty)
15565    }
15566
15567    /// Export the current discovery-driven network topology view.
15568    #[cfg(feature = "discovery")]
15569    pub fn export_topology(&self) -> TopologySnapshot {
15570        let now_ms = self.clock.now_ms();
15571        let mut st = self.state.lock();
15572        if Self::prune_discovery_routes_locked(&mut st, now_ms) {
15573            Self::note_discovery_topology_change_locked(&mut st, now_ms);
15574        }
15575        let routes = st
15576            .discovery_routes
15577            .iter()
15578            .filter_map(|(&side_id, route)| {
15579                let side = st.sides.get(side_id)?.as_ref()?;
15580                let announcers = route
15581                    .announcers
15582                    .iter()
15583                    .map(|(sender_id, sender_state)| TopologyAnnouncerRoute {
15584                        sender_id: sender_id.clone(),
15585                        reachable_endpoints: sender_state
15586                            .reachable
15587                            .iter()
15588                            .copied()
15589                            .filter(|ep| !discovery::is_router_control_endpoint(*ep))
15590                            .collect(),
15591                        reachable_network_variables: sender_state
15592                            .reachable_network_variables
15593                            .clone(),
15594                        reachable_timesync_sources: sender_state.reachable_timesync_sources.clone(),
15595                        routers: sender_state.topology_boards.as_ref().clone(),
15596                        last_seen_ms: sender_state.last_seen_ms,
15597                        age_ms: now_ms.saturating_sub(sender_state.last_seen_ms),
15598                    })
15599                    .collect();
15600                Some(TopologySideRoute {
15601                    side_id,
15602                    side_name: side.name.to_string(),
15603                    reachable_endpoints: route
15604                        .reachable
15605                        .iter()
15606                        .copied()
15607                        .filter(|ep| !discovery::is_router_control_endpoint(*ep))
15608                        .collect(),
15609                    reachable_network_variables: route.reachable_network_variables.clone(),
15610                    reachable_timesync_sources: route.reachable_timesync_sources.clone(),
15611                    announcers,
15612                    last_seen_ms: route.last_seen_ms,
15613                    age_ms: now_ms.saturating_sub(route.last_seen_ms),
15614                })
15615            })
15616            .collect();
15617        let routers = self.advertised_discovery_topology_for_link_locked(&st, now_ms, true, None);
15618        let advertised_endpoints =
15619            self.advertised_discovery_endpoints_for_link_locked(&st, now_ms, true, None);
15620        let advertised_timesync_sources =
15621            self.advertised_discovery_timesync_sources_for_link_locked(&st, now_ms, None);
15622        let links = discovery::topology_links_from_boards(&routers);
15623        TopologySnapshot {
15624            advertised_endpoints,
15625            advertised_timesync_sources,
15626            routers,
15627            links,
15628            routes,
15629            current_announce_interval_ms: st.discovery_cadence.current_interval_ms,
15630            next_announce_ms: st.discovery_cadence.next_announce_ms,
15631        }
15632    }
15633
15634    #[cfg(feature = "discovery")]
15635    pub fn client_stats(&self, sender_id: &str) -> Option<ClientStatsSnapshot> {
15636        let now_ms = self.clock.now_ms();
15637        let st = self.state.lock();
15638        let mut side_ids = Vec::new();
15639        let mut side_names = Vec::new();
15640        let mut last_seen_ms = None::<u64>;
15641        let mut reachable_endpoints = Vec::new();
15642        let mut reachable_timesync_sources = Vec::new();
15643        let mut packets_sent = 0u64;
15644        let mut packets_received = 0u64;
15645        let mut bytes_sent = 0u64;
15646        let mut bytes_received = 0u64;
15647
15648        for (side_id, route) in &st.discovery_routes {
15649            let Some(sender_state) = route.announcers.get(sender_id) else {
15650                continue;
15651            };
15652            side_ids.push(*side_id);
15653            if let Some(side_name) = st
15654                .sides
15655                .get(*side_id)
15656                .and_then(|side| side.as_ref())
15657                .map(|side| side.name.clone())
15658            {
15659                side_names.push(side_name.to_string());
15660            }
15661            last_seen_ms = Some(last_seen_ms.unwrap_or(0).max(sender_state.last_seen_ms));
15662            reachable_endpoints.extend(sender_state.reachable.iter().copied());
15663            reachable_timesync_sources
15664                .extend(sender_state.reachable_timesync_sources.iter().cloned());
15665            if let Some(stats) = st.side_runtime_stats.get(side_id) {
15666                packets_sent = packets_sent.saturating_add(stats.tx_packets);
15667                packets_received = packets_received.saturating_add(stats.rx_packets);
15668                bytes_sent = bytes_sent.saturating_add(stats.tx_bytes);
15669                bytes_received = bytes_received.saturating_add(stats.rx_bytes);
15670            }
15671        }
15672
15673        if side_ids.is_empty() {
15674            return None;
15675        }
15676        reachable_endpoints.retain(|ep| !discovery::is_router_control_endpoint(*ep));
15677        crate::small_sort::sort(&mut reachable_endpoints);
15678        reachable_endpoints.dedup();
15679        reachable_timesync_sources.sort_unstable();
15680        reachable_timesync_sources.dedup();
15681        crate::small_sort::sort(&mut side_ids);
15682        side_ids.dedup();
15683        side_names.sort_unstable();
15684        side_names.dedup();
15685        let age_ms = last_seen_ms.map(|seen| now_ms.saturating_sub(seen));
15686        Some(ClientStatsSnapshot {
15687            sender_id: sender_id.to_string(),
15688            connected: age_ms.is_some_and(|age| age <= DISCOVERY_ROUTE_TTL_MS),
15689            side_ids,
15690            side_names,
15691            last_seen_ms,
15692            age_ms,
15693            reachable_endpoints,
15694            reachable_timesync_sources,
15695            packets_sent,
15696            packets_received,
15697            bytes_sent,
15698            bytes_received,
15699        })
15700    }
15701
15702    /// Copy bounded metadata into caller storage without allocating. Returns
15703    /// the total pending count across all pages. Copy begins at `first`; rows
15704    /// outside the pending range and unused owner slots are zeroed.
15705    pub fn copy_reliable_pending_stats(
15706        &self,
15707        first: usize,
15708        rows: &mut [crate::diagnostics::ReliablePendingRow],
15709    ) -> usize {
15710        rows.fill(crate::diagnostics::ReliablePendingRow::default());
15711        let st = self.state.lock();
15712        for (row, (&id, sent)) in rows
15713            .iter_mut()
15714            .zip(st.end_to_end_reliable_tx.iter().skip(first))
15715        {
15716            row.packet_id = id;
15717            row.last_send_ms = sent.last_send_ms;
15718            row.data_type = Self::item_data_type(&sent.data).map_or(0, DataType::as_u32);
15719            row.pending_count = sent.pending_destinations.len().min(u32::MAX as usize) as u32;
15720            row.retries = sent.retries;
15721            row.flags = u32::from(sent.queued)
15722                | (u32::from(sent.waiting_for_predecessor) << 1)
15723                | (u32::from(
15724                    st.reliable_return_routes
15725                        .get(&id)
15726                        .is_some_and(|route| route.upstream_contract.is_some()),
15727                ) << 2)
15728                | (u32::from(sent.tracked_destinations) << 3);
15729            for (out, owner) in row
15730                .pending_owners
15731                .iter_mut()
15732                .zip(sent.pending_destinations.keys())
15733            {
15734                *out = *owner;
15735            }
15736        }
15737        st.end_to_end_reliable_tx.len()
15738    }
15739
15740    /// Snapshot outstanding reliable deliveries for diagnostics. Allocates
15741    /// metadata bounded by the existing pending window; never clones payloads
15742    /// or changes retry timing. Known std owner types use cached metadata;
15743    /// unknown types and non-std builds retain payload decoding as a fallback.
15744    /// Not needed in the normal dispatch hot path.
15745    pub fn export_reliable_pending_stats(&self) -> Vec<crate::diagnostics::ReliablePendingStats> {
15746        let st = self.state.lock();
15747        st.end_to_end_reliable_tx
15748            .iter()
15749            .filter_map(|(&packet_id, sent)| {
15750                Some(crate::diagnostics::ReliablePendingStats {
15751                    packet_id,
15752                    data_type: {
15753                        #[cfg(feature = "std")]
15754                        {
15755                            if sent.pending_type != 0 {
15756                                DataType(sent.pending_type)
15757                            } else {
15758                                Self::item_data_type(&sent.data).ok()?
15759                            }
15760                        }
15761                        #[cfg(not(feature = "std"))]
15762                        {
15763                            Self::item_data_type(&sent.data).ok()?
15764                        }
15765                    },
15766                    pending_destinations: sent.pending_destinations.keys().copied().collect(),
15767                    retries: sent.retries,
15768                    queued: sent.queued,
15769                    waiting_for_predecessor: sent.waiting_for_predecessor,
15770                    last_send_ms: sent.last_send_ms,
15771                })
15772            })
15773            .collect()
15774    }
15775
15776    pub fn export_runtime_stats(&self) -> RuntimeStatsSnapshot {
15777        let now_ms = self.clock.now_ms();
15778        let isr_rx = self
15779            .isr_rx_queue
15780            .as_ref()
15781            .and_then(|queue| queue.snapshot())
15782            .unwrap_or((0, 0));
15783        let st = self.state.lock();
15784
15785        let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
15786            (0, 0)
15787        } else {
15788            st.replay_counters()
15789        };
15790        let mut sides = Vec::new();
15791        for (side_id, side) in st.sides.iter().enumerate() {
15792            let Some(side) = side.as_ref() else { continue };
15793            let empty_stats = SideRuntimeStatsInner::default();
15794            let stats = st
15795                .side_runtime_stats
15796                .get(&side_id)
15797                .map(Box::as_ref)
15798                .unwrap_or(&empty_stats);
15799            let adaptive = st
15800                .adaptive_route_stats
15801                .get(&side_id)
15802                .cloned()
15803                .unwrap_or_default()
15804                .snapshot(now_ms, true);
15805            let (tx_template_count, rx_template_count) = st
15806                .side_transport
15807                .get(&side_id)
15808                .map(|state| (state.tx_template_count(), state.rx_template_count()))
15809                .unwrap_or((0, 0));
15810            let mut data_types: Vec<RuntimeTypeStats> = stats
15811                .data_types
15812                .iter()
15813                .map(|(&ty, item)| {
15814                    let item = item.counters();
15815                    RuntimeTypeStats {
15816                        data_type: DataType(ty),
15817                        tx_packets: item.tx_packets,
15818                        tx_bytes: item.tx_bytes,
15819                        rx_packets: item.rx_packets,
15820                        rx_bytes: item.rx_bytes,
15821                        relayed_tx_packets: item.relayed_tx_packets,
15822                        relayed_tx_bytes: item.relayed_tx_bytes,
15823                        relayed_rx_packets: item.relayed_rx_packets,
15824                        relayed_rx_bytes: item.relayed_rx_bytes,
15825                        tx_retries: item.tx_retries,
15826                        handler_failures: item.handler_failures,
15827                    }
15828                })
15829                .collect();
15830            data_types.sort_unstable_by_key(|item| item.data_type.as_u32());
15831            sides.push(RuntimeSideStats {
15832                side_id,
15833                side_name: side.name.to_string(),
15834                reliable_enabled: side.opts.reliable_enabled,
15835                link_local_enabled: side.opts.link_local_enabled,
15836                header_template_enabled: side.opts.header_template_enabled,
15837                max_frame_bytes: side.opts.max_frame_bytes,
15838                compact_header_target_bytes: side.opts.compact_header_target_bytes,
15839                side_transport_profile: side.opts.effective_transport_profile().as_str(),
15840                ingress_enabled: side.opts.ingress_enabled,
15841                egress_enabled: side.opts.egress_enabled,
15842                tx_packets: stats.tx_packets,
15843                tx_bytes: stats.tx_bytes,
15844                rx_packets: stats.rx_packets,
15845                rx_bytes: stats.rx_bytes,
15846                relayed_tx_packets: stats.relayed_tx_packets,
15847                relayed_tx_bytes: stats.relayed_tx_bytes,
15848                relayed_rx_packets: stats.relayed_rx_packets,
15849                relayed_rx_bytes: stats.relayed_rx_bytes,
15850                local_delivery_packets: stats.local_delivery_packets,
15851                tx_retries: stats.tx_retries,
15852                tx_handler_failures: stats.tx_handler_failures,
15853                local_handler_failures: stats.local_handler_failures,
15854                total_handler_retries: stats.total_handler_retries,
15855                side_transport_full_frames: stats.side_transport_full_frames,
15856                side_transport_compact_frames: stats.side_transport_compact_frames,
15857                side_transport_compact_delta_frames: stats.side_transport_compact_delta_frames,
15858                side_transport_compact_omitted_timestamp_frames: stats
15859                    .side_transport_compact_omitted_timestamp_frames,
15860                side_transport_chunk_frames: stats.side_transport_chunk_frames,
15861                side_transport_chunk_reclaims: st
15862                    .side_transport
15863                    .get(&side_id)
15864                    .map_or(0, |state| u64::from(state.rx_chunk_reclaims)),
15865                side_transport_raw_bytes: stats.side_transport_raw_bytes,
15866                side_transport_wire_bytes: stats.side_transport_wire_bytes,
15867                side_transport_bytes_saved: stats.side_transport_bytes_saved,
15868                side_transport_min_compact_overhead_bytes: stats
15869                    .side_transport_min_compact_overhead_bytes,
15870                side_transport_max_compact_overhead_bytes: stats
15871                    .side_transport_max_compact_overhead_bytes,
15872                side_transport_compact_target_misses: stats.side_transport_compact_target_misses,
15873                side_transport_template_evictions: stats.side_transport_template_evictions,
15874                side_transport_tx_template_count: tx_template_count,
15875                side_transport_rx_template_count: rx_template_count,
15876                max_side_transport_templates: side.opts.max_side_transport_templates,
15877                adaptive,
15878                data_types,
15879            });
15880        }
15881
15882        let mut route_modes: Vec<RouteModeStats> = st
15883            .route_selection_cursors
15884            .iter()
15885            .map(|(src, cursor)| RouteModeStats {
15886                src_side_id: *src,
15887                selection_mode: st.source_route_modes.get(src).copied(),
15888                cursor: *cursor,
15889            })
15890            .collect();
15891        for src in st.source_route_modes.keys() {
15892            if !route_modes.iter().any(|mode| mode.src_side_id == *src) {
15893                route_modes.push(RouteModeStats {
15894                    src_side_id: *src,
15895                    selection_mode: st.source_route_modes.get(src).copied(),
15896                    cursor: 0,
15897                });
15898            }
15899        }
15900        route_modes.sort_unstable_by_key(|mode| mode.src_side_id.unwrap_or(usize::MAX));
15901
15902        let mut route_overrides: Vec<RouteOverrideStats> = st
15903            .route_overrides
15904            .iter()
15905            .map(|((src, dst), enabled)| RouteOverrideStats {
15906                src_side_id: *src,
15907                dst_side_id: *dst,
15908                enabled: *enabled,
15909            })
15910            .collect();
15911        route_overrides.sort_unstable_by_key(|item| {
15912            (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15913        });
15914
15915        let mut typed_route_overrides: Vec<TypedRouteOverrideStats> = st
15916            .typed_route_overrides
15917            .iter()
15918            .map(|((src, ty, dst), enabled)| TypedRouteOverrideStats {
15919                src_side_id: *src,
15920                data_type: DataType(*ty),
15921                dst_side_id: *dst,
15922                enabled: *enabled,
15923            })
15924            .collect();
15925        typed_route_overrides.sort_unstable_by_key(|item| {
15926            (
15927                item.src_side_id.unwrap_or(usize::MAX),
15928                item.data_type.as_u32(),
15929                item.dst_side_id,
15930            )
15931        });
15932
15933        let mut route_weights: Vec<RouteWeightStats> = st
15934            .route_weights
15935            .iter()
15936            .map(|((src, dst), weight)| RouteWeightStats {
15937                src_side_id: *src,
15938                dst_side_id: *dst,
15939                weight: *weight,
15940            })
15941            .collect();
15942        route_weights.sort_unstable_by_key(|item| {
15943            (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15944        });
15945
15946        let mut route_priorities: Vec<RoutePriorityStats> = st
15947            .route_priorities
15948            .iter()
15949            .map(|((src, dst), priority)| RoutePriorityStats {
15950                src_side_id: *src,
15951                dst_side_id: *dst,
15952                priority: *priority,
15953            })
15954            .collect();
15955        route_priorities.sort_unstable_by_key(|item| {
15956            (item.src_side_id.unwrap_or(usize::MAX), item.dst_side_id)
15957        });
15958
15959        #[cfg(feature = "discovery")]
15960        let discovery = DiscoveryRuntimeStats {
15961            route_count: st.discovery_routes.len(),
15962            announcer_count: st
15963                .discovery_routes
15964                .values()
15965                .map(|route| route.announcers.len())
15966                .sum(),
15967            current_announce_interval_ms: Some(st.discovery_cadence.current_interval_ms),
15968            next_announce_ms: Some(st.discovery_cadence.next_announce_ms),
15969        };
15970        #[cfg(not(feature = "discovery"))]
15971        let discovery = DiscoveryRuntimeStats {
15972            route_count: 0,
15973            announcer_count: 0,
15974            current_announce_interval_ms: None,
15975            next_announce_ms: None,
15976        };
15977
15978        RuntimeStatsSnapshot {
15979            sides,
15980            route_modes,
15981            route_overrides,
15982            typed_route_overrides,
15983            route_weights,
15984            route_priorities,
15985            queues: QueueRuntimeStats {
15986                rx_len: isr_rx.0.saturating_add(st.received_queue.len()),
15987                rx_bytes: isr_rx.1.saturating_add(st.received_queue.bytes_used()),
15988                tx_len: st.transmit_queue.len().saturating_sub(replay_len),
15989                tx_bytes: st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
15990                replay_len,
15991                replay_bytes,
15992                recent_rx_len: st.recent_rx.len(),
15993                recent_rx_bytes: st.recent_rx.bytes_used(),
15994                reliable_rx_buffered_len: st.reliable_rx_buffer_len(),
15995                reliable_rx_buffered_bytes: st.reliable_rx_buffered_bytes(),
15996                shared_queue_bytes_used: st.shared_queue_bytes_used(),
15997            },
15998            reliable: ReliableRuntimeStats {
15999                reliable_return_route_count: st.reliable_return_routes.len(),
16000                end_to_end_pending_count: st.end_to_end_reliable_tx.len(),
16001                end_to_end_pending_destination_count: st
16002                    .end_to_end_reliable_tx
16003                    .values()
16004                    .map(|sent| sent.pending_destinations.len())
16005                    .sum(),
16006                end_to_end_acked_cache_count: st
16007                    .forwarded_receipts
16008                    .as_ref()
16009                    .map_or(0, |cache| cache.destinations.len()),
16010            },
16011            discovery,
16012            total_handler_failures: st.total_handler_failures,
16013            total_handler_retries: st.total_handler_retries,
16014        }
16015    }
16016
16017    /// Whether SDK delivery ownership remains for a reliable type submitted
16018    /// through `tx_queue`/`tx_packed_queue`. Includes original/claimed work,
16019    /// transport retries and reliable owners. A concurrent queued TX dispatch
16020    /// is exact while its known logical types fit the bounded inline claim table.
16021    /// Unknown or unrepresented active work stays conservatively pending.
16022    /// This allocates nothing and changes no queues. Retained end-to-end owners and their
16023    /// replay markers use the logical type validated when ownership was created,
16024    /// independent of Packed payload representation. Packed queue work without
16025    /// a cached type, or an owner with unknown type metadata, is conservatively
16026    /// pending, without materializing or pinning its payload.
16027    /// It is NOT an RX-originated ownership or
16028    /// external transport/modem-drain guarantee.
16029    #[cfg(feature = "std")]
16030    pub fn has_pending_reliable_tx_type(&self, ty: DataType) -> bool {
16031        let st = self.state.lock();
16032        let total = self
16033            .active_queued_tx
16034            .load(core::sync::atomic::Ordering::Acquire) as u64;
16035        let known = st
16036            .active_queued_types
16037            .iter()
16038            .try_fold(0u64, |sum, &(kind, count)| {
16039                if kind == 0 && count != 0 {
16040                    return None;
16041                }
16042                sum.checked_add(u64::from(count))
16043            });
16044        // Unknown provenance, exhausted slots, overflow or inconsistent counts
16045        // remain conservative. No payload is parsed or pinned by inspection.
16046        if known != Some(total)
16047            || st
16048                .active_queued_types
16049                .iter()
16050                .any(|&(kind, count)| count != 0 && kind == ty.as_u32())
16051        {
16052            return true;
16053        }
16054        let matches_data = |data: &RouterItem| match data {
16055            RouterItem::Packet(packet) => packet.data_type() == ty,
16056            RouterItem::Packed(_) => true,
16057        };
16058        let matches_owner = |owner: &EndToEndReliableSent| {
16059            owner.pending_type == 0 || owner.pending_type == ty.as_u32()
16060        };
16061        st.transmit_queue.iter().any(|queued| {
16062            if queued.accepted_type != 0 {
16063                return queued.accepted_type == ty.as_u32();
16064            }
16065            match &queued.item {
16066                RouterTxItem::Broadcast(data) | RouterTxItem::ToSide { data, .. } => {
16067                    matches_data(data)
16068                }
16069                RouterTxItem::ReliableReplay { ty: found, .. } => *found == ty,
16070                RouterTxItem::EndToEndReplay { packet_id } => st
16071                    .end_to_end_reliable_tx
16072                    .get(packet_id)
16073                    .is_some_and(|owner| matches_owner(owner)),
16074            }
16075        }) || st
16076            .end_to_end_reliable_tx
16077            .values()
16078            .any(|owner| matches_owner(owner))
16079            || st
16080                .reliable_tx
16081                .iter()
16082                .any(|((_, found), owner)| *found == ty.as_u32() && !owner.sent.is_empty())
16083    }
16084
16085    // Called only while the same state lock that pops the work is still held.
16086    // Thus inspection cannot observe a popped retry before this count exists.
16087    #[cfg(feature = "std")]
16088    fn track_queued_tx<'a>(
16089        &'a self,
16090        state: &mut RouterInner,
16091        work: Option<TxWork>,
16092    ) -> Option<(TxWork, QueuedTxActivityGuard<'a>)> {
16093        work.map(|work| {
16094            let total = self
16095                .active_queued_tx
16096                .load(core::sync::atomic::Ordering::Acquire);
16097            // All three callers check before removing queue ownership. Both
16098            // registration and release hold state, so this cannot race Drop.
16099            let next = total.checked_add(1).expect("active queued TX overflow");
16100            let kind = if work.queued.accepted_type != 0 {
16101                Some(work.queued.accepted_type)
16102            } else {
16103                match &work.queued.item {
16104                    RouterTxItem::Broadcast(RouterItem::Packet(packet))
16105                    | RouterTxItem::ToSide {
16106                        data: RouterItem::Packet(packet),
16107                        ..
16108                    } => Some(packet.data_type().as_u32()),
16109                    RouterTxItem::ReliableReplay { ty, .. } => Some(ty.as_u32()),
16110                    RouterTxItem::EndToEndReplay { packet_id } => state
16111                        .end_to_end_reliable_tx
16112                        .get(packet_id)
16113                        .map(|owner| owner.pending_type),
16114                    _ => None,
16115                }
16116            }
16117            .filter(|kind| *kind != 0);
16118            let key = kind.and_then(|kind| {
16119                let index = state
16120                    .active_queued_types
16121                    .iter()
16122                    .position(|slot| slot.0 == kind)
16123                    .or_else(|| {
16124                        state
16125                            .active_queued_types
16126                            .iter()
16127                            .position(|slot| slot.1 == 0)
16128                    })?;
16129                let slot = &mut state.active_queued_types[index];
16130                let count = slot.1.checked_add(1)?;
16131                *slot = (kind, count);
16132                Some(ActiveQueuedTypeKey {
16133                    slot: core::num::NonZeroU8::new((index + 1) as u8).expect("inline type slot"),
16134                    kind,
16135                })
16136            });
16137            self.active_queued_tx
16138                .store(next, core::sync::atomic::Ordering::Release);
16139            (work, QueuedTxActivityGuard { router: self, key })
16140        })
16141    }
16142
16143    /// Allocation-free memory counters, version 1. Values saturate at u32::MAX.
16144    /// Approximate retained costs exclude allocator headers and shared ownership;
16145    /// they are component diagnostics, not an independent heap-size estimate.
16146    /// Whether queued work remains. Bounded workers may yield to their peers
16147    /// instead of sleeping while a backlog waits. A busy ISR guard is treated
16148    /// as pending; callers must still enforce their own CPU-service bound.
16149    pub fn has_queued_work(&self) -> bool {
16150        if self
16151            .isr_rx_queue
16152            .as_ref()
16153            .is_some_and(|queue| queue.try_lock().map_or(true, |queue| !queue.is_empty()))
16154        {
16155            return true;
16156        }
16157        let state = self.state.lock();
16158        !state.received_queue.is_empty() || !state.transmit_queue.is_empty()
16159    }
16160
16161    /// Allocation-free queue accounting for a caller-owned diagnostic buffer.
16162    /// Version, RX/TX counts, independently summed costs, front type/priority/
16163    /// wait, ACK/schema counts, minimum priority, reliable pending count/bytes,
16164    /// front variant and the monotonic clock words. Packed types remain unknown
16165    /// (0) so inspection never allocates/pins payloads, parses bytes or drains.
16166    pub fn queue_counters(&self) -> [u32; 16] {
16167        let now = self.clock.now_ms();
16168        let st = self.state.lock();
16169        let ty_of = |item: &RouterTxItem| match item {
16170            RouterTxItem::Broadcast(RouterItem::Packet(packet))
16171            | RouterTxItem::ToSide {
16172                data: RouterItem::Packet(packet),
16173                ..
16174            } => Some(packet.data_type()),
16175            RouterTxItem::Broadcast(RouterItem::Packed(_))
16176            | RouterTxItem::ToSide {
16177                data: RouterItem::Packed(_),
16178                ..
16179            }
16180            | RouterTxItem::ReliableReplay { .. } => None,
16181            RouterTxItem::EndToEndReplay { packet_id } => st
16182                .end_to_end_reliable_tx
16183                .get(packet_id)
16184                .and_then(|pending| match &pending.data {
16185                    RouterItem::Packet(packet) => Some(packet.data_type()),
16186                    RouterItem::Packed(_) => None,
16187                }),
16188        };
16189        let sum_cost = |total: usize, cost: usize| total.saturating_add(cost);
16190        let front = st.transmit_queue.iter().next();
16191        let ack_count = st
16192            .transmit_queue
16193            .iter()
16194            .filter(|q| {
16195                matches!(
16196                    ty_of(&q.item),
16197                    Some(
16198                        DataType::ReliableAck
16199                            | DataType::ReliablePartialAck
16200                            | DataType::ReliablePacketRequest
16201                    )
16202                )
16203            })
16204            .count();
16205        let schema_count = st
16206            .transmit_queue
16207            .iter()
16208            .filter(|q| ty_of(&q.item) == Some(DataType::DiscoverySchema))
16209            .count();
16210        [
16211            1,
16212            st.received_queue.len() as u32,
16213            st.transmit_queue.len() as u32,
16214            st.received_queue
16215                .iter()
16216                .map(ByteCost::byte_cost)
16217                .fold(0, sum_cost) as u32,
16218            st.transmit_queue
16219                .iter()
16220                .map(ByteCost::byte_cost)
16221                .fold(0, sum_cost) as u32,
16222            front
16223                .and_then(|q| ty_of(&q.item))
16224                .map_or(0, DataType::as_u32),
16225            front.map_or(0, |q| q.priority.into()),
16226            front.map_or(0, |q| {
16227                q.retry_after_ms.saturating_sub(now).min(u32::MAX.into()) as u32
16228            }),
16229            ack_count as u32,
16230            schema_count as u32,
16231            st.transmit_queue
16232                .iter()
16233                .map(|q| u32::from(q.priority))
16234                .min()
16235                .unwrap_or(0),
16236            st.end_to_end_reliable_tx.len() as u32,
16237            st.end_to_end_reliable_tx
16238                .values()
16239                .map(|entry| match &entry.data {
16240                    RouterItem::Packet(packet) => packet.byte_cost(),
16241                    RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
16242                })
16243                .fold(0, sum_cost) as u32,
16244            front.map_or(0, |q| match q.item {
16245                RouterTxItem::Broadcast(_) => 1,
16246                RouterTxItem::ToSide { .. } => 2,
16247                RouterTxItem::ReliableReplay { .. } => 3,
16248                RouterTxItem::EndToEndReplay { .. } => 4,
16249            }),
16250            now as u32,
16251            (now >> 32) as u32,
16252        ]
16253    }
16254
16255    pub fn memory_counters(&self) -> [u32; 16] {
16256        let st = self.state.lock();
16257        #[cfg(feature = "discovery")]
16258        let discovery = st.discovery_bytes_used();
16259        #[cfg(not(feature = "discovery"))]
16260        let discovery = 0;
16261        let templates = st
16262            .side_transport
16263            .values()
16264            .map(|side| side.tx_templates.len() + side.rx_templates_by_id.len())
16265            .sum::<usize>();
16266        let template_bytes = st
16267            .side_transport
16268            .values()
16269            .flat_map(|side| {
16270                side.tx_templates
16271                    .values()
16272                    .chain(side.rx_templates_by_id.values())
16273            })
16274            .map(|t| size_of::<SideHeaderTemplate>() + t.prefix().len() + t.between().len())
16275            .sum::<usize>();
16276        let chunks = st
16277            .side_transport
16278            .values()
16279            .flat_map(|side| side.rx_chunks.values())
16280            .flat_map(|assembly| assembly.received.iter().flatten())
16281            .map(|bytes| bytes.len())
16282            .sum::<usize>();
16283        let type_stats = st
16284            .side_runtime_stats
16285            .values()
16286            .map(|side| side.data_types.len())
16287            .sum::<usize>();
16288        let managed = st.local_cache_bytes();
16289        let pending = st
16290            .end_to_end_reliable_tx
16291            .values()
16292            .map(|entry| match &entry.data {
16293                RouterItem::Packet(packet) => packet.byte_cost(),
16294                RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
16295            })
16296            .sum::<usize>();
16297        [
16298            1,
16299            st.memory.max_queue_budget,
16300            st.shared_queue_bytes_used(),
16301            st.received_queue.bytes_used(),
16302            st.transmit_queue.bytes_used(),
16303            st.received_queue.capacity() * size_of::<RouterRxItem>()
16304                + st.transmit_queue.capacity() * size_of::<TxQueued>()
16305                + st.recent_rx.capacity() * size_of::<u64>(),
16306            st.reliable_rx_buffered_bytes(),
16307            discovery,
16308            crate::config::schema_bytes_used(),
16309            managed,
16310            pending,
16311            templates,
16312            template_bytes,
16313            chunks,
16314            type_stats,
16315            st.side_runtime_stats
16316                .values()
16317                .flat_map(|side| side.data_types.values())
16318                .map(|stats| size_of::<TypeRuntimeStatsInner>() + stats.heap_bytes())
16319                .sum::<usize>(),
16320        ]
16321        .map(|value| u32::try_from(value).unwrap_or(u32::MAX))
16322    }
16323
16324    /// Export current router memory usage/layout as JSON for profiling.
16325    pub fn export_memory_layout_json(&self) -> String {
16326        let isr_rx = self
16327            .isr_rx_queue
16328            .as_ref()
16329            .and_then(|queue| queue.snapshot())
16330            .unwrap_or((0, 0));
16331        let st = self.state.lock();
16332        #[cfg(feature = "discovery")]
16333        let discovery_bytes = st.discovery_bytes_used();
16334        #[cfg(not(feature = "discovery"))]
16335        let discovery_bytes = 0usize;
16336        let schema_bytes = crate::config::schema_bytes_used();
16337        let network_variable_cache_bytes = st.local_cache_bytes();
16338        let (replay_len, replay_bytes) = if self.cfg.role.local_services() {
16339            (0, 0)
16340        } else {
16341            st.replay_counters()
16342        };
16343        let mut out = String::new();
16344        let memory = st.memory;
16345        let _ = fmt::Write::write_fmt(
16346            &mut out,
16347            format_args!(
16348                "{{\"kind\":\"{}\",\
16349                 \"shared_queue_bytes_used\":{},\"shared_queue_bytes_allocated\":{},\
16350                 \"rx_queue_bytes_used\":{},\"rx_queue_bytes_allocated\":{},\"rx_queue_len\":{},\
16351                 \"isr_rx_queue_bytes_used\":{},\"isr_rx_queue_bytes_allocated\":{},\"isr_rx_queue_len\":{},\
16352                 \"tx_queue_bytes_used\":{},\"tx_queue_bytes_allocated\":{},\"tx_queue_len\":{},\
16353                 \"replay_queue_bytes_used\":{},\"replay_queue_bytes_allocated\":{},\"replay_queue_len\":{},\
16354                 \"recent_rx_bytes_used\":{},\"recent_rx_bytes_allocated\":{},\"recent_rx_len\":{},\
16355                 \"reliable_rx_buffer_bytes_used\":{},\"reliable_rx_buffer_bytes_allocated\":{},\"reliable_rx_buffer_len\":{},\
16356                 \"discovery_bytes_used\":{},\"discovery_bytes_allocated\":{},\
16357                 \"schema_bytes_used\":{},\"schema_bytes_allocated\":{},\
16358                 \"network_variable_cache_bytes_used\":{},\"network_variable_cache_bytes_allocated\":{},\"network_variable_cache_len\":{}}}",
16359                if self.cfg.role.local_services() {
16360                    "router"
16361                } else {
16362                    "relay"
16363                },
16364                st.shared_queue_bytes_used(),
16365                memory.max_queue_budget,
16366                st.received_queue.bytes_used(),
16367                st.received_queue.max_bytes(),
16368                st.received_queue.len(),
16369                isr_rx.1,
16370                memory.max_queue_budget,
16371                isr_rx.0,
16372                st.transmit_queue.bytes_used().saturating_sub(replay_bytes),
16373                st.transmit_queue.max_bytes(),
16374                st.transmit_queue.len().saturating_sub(replay_len),
16375                replay_bytes,
16376                memory.max_queue_budget,
16377                replay_len,
16378                st.recent_rx.bytes_used(),
16379                st.recent_rx.max_bytes(),
16380                st.recent_rx.len(),
16381                st.reliable_rx_buffered_bytes(),
16382                memory.max_queue_budget,
16383                st.reliable_rx_buffer_len(),
16384                discovery_bytes,
16385                memory.max_queue_budget,
16386                schema_bytes,
16387                memory.max_queue_budget,
16388                network_variable_cache_bytes,
16389                memory.max_queue_budget,
16390                st.local_cache_len(),
16391            ),
16392        );
16393        out
16394    }
16395
16396    #[cfg(test)]
16397    #[cfg(feature = "discovery")]
16398    pub(crate) fn debug_end_to_end_pending_destination_count(
16399        &self,
16400        packet_id: u64,
16401    ) -> Option<usize> {
16402        let st = self.state.lock();
16403        st.end_to_end_reliable_tx
16404            .get(&packet_id)
16405            .map(|sent| sent.pending_destinations.len())
16406    }
16407
16408    #[cfg(test)]
16409    #[cfg(feature = "discovery")]
16410    pub(crate) fn debug_end_to_end_tracked_count(&self) -> usize {
16411        let st = self.state.lock();
16412        st.end_to_end_reliable_tx.len()
16413    }
16414
16415    #[cfg(test)]
16416    #[cfg(feature = "discovery")]
16417    pub(crate) fn debug_reliable_return_route_count(&self) -> usize {
16418        let st = self.state.lock();
16419        st.reliable_return_routes.len()
16420    }
16421
16422    #[cfg(test)]
16423    pub(crate) fn debug_side_storage(&self) -> (usize, usize) {
16424        let st = self.state.lock();
16425        (st.sides.len(), st.sides.capacity())
16426    }
16427
16428    #[cfg(test)]
16429    pub(crate) fn debug_queue_lengths(&self) -> (usize, usize, usize) {
16430        let st = self.state.lock();
16431        (
16432            st.received_queue.len(),
16433            st.transmit_queue.len(),
16434            st.recent_rx.len(),
16435        )
16436    }
16437
16438    #[cfg(test)]
16439    pub(crate) fn debug_shared_queue_bytes_used(&self) -> usize {
16440        let st = self.state.lock();
16441        st.shared_queue_bytes_used()
16442    }
16443
16444    #[cfg(test)]
16445    pub(crate) fn debug_recent_rx_capacity(&self) -> (usize, usize) {
16446        let st = self.state.lock();
16447        (st.recent_rx.capacity(), st.recent_rx.max_bytes())
16448    }
16449
16450    /// Compute a de-dupe hash for a RouterItem.
16451    /// Uses packet ID for Packet items, and attempts to extract packet ID from
16452    /// packed bytes. If extraction fails, hashes raw bytes as a fallback.
16453    fn get_hash(item: &RouterItem) -> u64 {
16454        match item {
16455            RouterItem::Packet(pkt) => pkt.packet_id(),
16456            RouterItem::Packed(bytes) => {
16457                match wire_format::packet_id_from_wire(bytes.as_ref()) {
16458                    Ok(id) => id,
16459                    Err(_e) => {
16460                        // Fallback: if bytes are malformed (or compression feature mismatch),
16461                        // hash raw bytes so we can still dedupe identical network duplicates.
16462                        let h: u64 = 0x9E37_79B9_7F4A_7C15;
16463                        hash_bytes_u64(h, bytes.as_ref())
16464                    }
16465                }
16466            }
16467        }
16468    }
16469
16470    /// Remove a hash from the ring buffer of recent RX IDs.
16471    fn remove_pkt_id(&self, item: &RouterItem) {
16472        let hash = Self::get_hash(item);
16473        let mut st = self.state.lock();
16474        st.recent_rx.remove_value(&hash);
16475    }
16476
16477    /// Compute a de-dupe ID for a RouterItem and record it.
16478    /// Returns true if this item was seen recently (and should be skipped).
16479    fn is_duplicate_pkt(&self, item: &RouterItem) -> TelemetryResult<bool> {
16480        self.is_duplicate_packet_id(Self::get_hash(item))
16481    }
16482
16483    fn is_duplicate_packet_id(&self, id: u64) -> TelemetryResult<bool> {
16484        let mut st = self.state.lock();
16485        if st.recent_rx.contains(&id)
16486            || st
16487                .reliable_return_routes
16488                .get(&id)
16489                .is_some_and(|route| route.delivery_completed)
16490        {
16491            Ok(true)
16492        } else {
16493            st.push_recent_rx(id)?;
16494            Ok(false)
16495        }
16496    }
16497
16498    /// Error helper when we have a full Packet.
16499    ///
16500    /// Sends a TelemetryError packet to all local endpoints except the failed one (if any).
16501    /// If no local endpoints remain, falls back to `fallback_stdout`.
16502    fn handle_callback_error(
16503        &self,
16504        pkt: &Packet,
16505        dest: Option<DataEndpoint>,
16506        e: TelemetryError,
16507        called_from_queue: bool,
16508    ) -> TelemetryResult<()> {
16509        if !self.cfg.role.local_services() {
16510            return Err(e);
16511        }
16512
16513        // Allocator refusal is backpressure, not a diagnostic packet. Even
16514        // formatting an error string can exhaust the remaining small blocks.
16515        if matches!(&e, TelemetryError::Io("memory pressure")) {
16516            return Err(e);
16517        }
16518
16519        let device = self.sender_arc();
16520        let error_msg = match dest {
16521            Some(failed_local) => format!(
16522                "Handler for endpoint {:?} failed on device {:?}: {:?}",
16523                failed_local, device, e
16524            ),
16525            None => format!("TX Handler failed on device {:?}: {:?}", device, e),
16526        };
16527
16528        let mut recipients: Vec<DataEndpoint> = pkt
16529            .endpoints()
16530            .iter()
16531            .copied()
16532            .filter(|&ep| self.cfg.is_local_endpoint(ep))
16533            .collect();
16534        crate::small_sort::sort(&mut recipients);
16535        recipients.dedup();
16536
16537        if let Some(failed_local) = dest {
16538            recipients.retain(|&ep| ep != failed_local);
16539        }
16540
16541        // A side TX failure cannot be reported through the same failed
16542        // transport. Doing so recursively creates TelemetryError packets until
16543        // an embedded caller exhausts its stack.
16544        if recipients.is_empty() && dest.is_none() {
16545            fallback_stdout(&error_msg);
16546            return Ok(());
16547        }
16548
16549        // Local endpoint failures may still be reported to the packet's other
16550        // endpoints when no local error recipient exists.
16551        if recipients.is_empty() {
16552            recipients = pkt.endpoints().to_vec();
16553            crate::small_sort::sort(&mut recipients);
16554            recipients.dedup();
16555            if let Some(failed_local) = dest {
16556                recipients.retain(|&ep| ep != failed_local);
16557            }
16558        }
16559
16560        if recipients.is_empty() {
16561            fallback_stdout(&error_msg);
16562            return Ok(());
16563        }
16564
16565        let payload = make_error_payload(&error_msg);
16566
16567        let sender = self.sender_arc();
16568        let error_pkt = Packet::new(
16569            DataType::TelemetryError,
16570            &recipients,
16571            sender.as_ref(),
16572            self.packet_timestamp_ms(),
16573            crate::shared_bytes::convert(payload),
16574        )?;
16575
16576        self.emit_internal_tx(
16577            RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
16578            false,
16579            called_from_queue,
16580        )
16581    }
16582
16583    // ---------- PUBLIC API: queues ----------
16584
16585    /// Drain the transmit queue fully.
16586    #[inline]
16587    pub fn process_tx_queue(&self) -> TelemetryResult<()> {
16588        self.process_tx_queue_with_timeout(0)
16589    }
16590
16591    /// Drain both TX and RX queues fully (same semantics as `*_with_timeout(0)`).
16592    #[inline]
16593    pub fn process_all_queues(&self) -> TelemetryResult<()> {
16594        self.process_all_queues_with_timeout(0)
16595    }
16596
16597    /// Clear both the transmit and receive queues without processing.
16598    #[inline]
16599    pub fn clear_queues(&self) {
16600        let mut st = self.state.lock();
16601        st.retain_transmit(|_| false);
16602        st.transmit_queue.cancel_deferred_turn();
16603        #[cfg(feature = "latest-value-egress")]
16604        if st.latest_turn_phase < 3 {
16605            st.latest_turn_phase = 0;
16606            st.latest_turn_credit = 0;
16607        }
16608        #[cfg(feature = "discovery")]
16609        for response in st.discovery_side_throttle.values_mut() {
16610            if response.schema_response_active() {
16611                response.set_schema_response_active(false);
16612                response.schema_response_phase = SchemaResponsePhase::Idle;
16613                response.set_schema_response_pending(true);
16614            }
16615        }
16616        st.tx_policy_flags &= !1;
16617        st.received_queue.clear();
16618        st.rx_priority_burst = 0;
16619        drop(st);
16620        let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
16621    }
16622
16623    /// Clear only the receive queue without processing.
16624    #[inline]
16625    pub fn clear_rx_queue(&self) {
16626        let mut st = self.state.lock();
16627        st.received_queue.clear();
16628        st.rx_priority_burst = 0;
16629        drop(st);
16630        let _ = self.isr_rx_queue.as_ref().map(|queue| queue.clear());
16631    }
16632
16633    /// Clear queued TX without processing; retained reliable delivery can retry later.
16634    #[inline]
16635    pub fn clear_tx_queue(&self) {
16636        let mut st = self.state.lock();
16637        st.retain_transmit(|_| false);
16638        st.transmit_queue.cancel_deferred_turn();
16639        #[cfg(feature = "latest-value-egress")]
16640        if st.latest_turn_phase < 3 {
16641            st.latest_turn_phase = 0;
16642            st.latest_turn_credit = 0;
16643        }
16644        #[cfg(feature = "discovery")]
16645        for response in st.discovery_side_throttle.values_mut() {
16646            if response.schema_response_active() {
16647                response.set_schema_response_active(false);
16648                response.schema_response_phase = SchemaResponsePhase::Idle;
16649                response.set_schema_response_pending(true);
16650            }
16651        }
16652        st.tx_priority_burst = 0;
16653        st.tx_policy_flags &= !1;
16654    }
16655
16656    #[inline(never)]
16657    fn pop_transmit_queue_locked(
16658        st: &mut RouterInner,
16659        now_ms: u64,
16660        control_only: bool,
16661    ) -> Option<TxWork> {
16662        let slow = st.tx_policy_flags & 1 != 0;
16663        let mut next_token = st.ordered_claim_seq.wrapping_add(1).max(1);
16664        if slow {
16665            while st.claim_index(next_token).is_some() {
16666                next_token = next_token.wrapping_add(1).max(1);
16667            }
16668        }
16669        #[cfg(feature = "latest-value-egress")]
16670        let latest_blocked = st.latest_blocked_side(now_ms);
16671        #[cfg(feature = "latest-value-egress")]
16672        let latest_turn = st.latest_turn_selection(latest_blocked.is_none(), now_ms);
16673        #[cfg(feature = "latest-value-egress")]
16674        let latest_side = st.latest_side();
16675        #[cfg(feature = "latest-value-egress")]
16676        let latest_frame = latest_side
16677            .and_then(|side| st.sides.get(side)?.as_ref())
16678            .map_or(0, |side| side.opts.max_frame_bytes);
16679        let (selected, owed_turn) = st.transmit_queue.select_front_fair_deferred(
16680            &mut st.tx_priority_burst,
16681            MAX_PRIORITY_BURST,
16682            |item| item.priority,
16683            |_queue, item| {
16684                !matches!(
16685                    item.delivery_state,
16686                    QueuedDelivery::ClaimedOrdered
16687                        | QueuedDelivery::ClaimedUnordered
16688                        | QueuedDelivery::OrderedTail
16689                ) && item.retry_after_ms <= now_ms
16690                    && (!control_only || item.priority >= 200)
16691                    && {
16692                        #[cfg(feature = "latest-value-egress")]
16693                        {
16694                            let turn_ready = latest_turn.is_none_or(|index| {
16695                                !latest::managed_turn_item(item, latest_side, latest_frame)
16696                                    || _queue
16697                                        .get(index)
16698                                        .is_some_and(|chosen| core::ptr::eq(chosen, item))
16699                            });
16700                            turn_ready
16701                                && !(item.disposable & TX_LATEST_PENDING != 0
16702                                    && matches!(&item.item,
16703                            RouterTxItem::ToSide { dst, .. } if latest_blocked == Some(*dst)))
16704                        }
16705                        #[cfg(not(feature = "latest-value-egress"))]
16706                        {
16707                            true
16708                        }
16709                    }
16710            },
16711        )?;
16712        #[cfg(feature = "latest-value-egress")]
16713        if latest_turn == Some(selected) {
16714            if let Some(next) = st.latest_service_seq.checked_add(1) {
16715                st.latest_service_seq = next;
16716            } else {
16717                st.tx_policy_flags &= 1;
16718                st.latest_turn_phase = 0;
16719                st.latest_turn_credit = 0;
16720                st.clear_canonical_turn_flags();
16721            }
16722            if st.latest_side().is_some()
16723                && st.canonical_catalog_owner(st.transmit_queue.get(selected)?)
16724            {
16725                st.latest_turn_phase = 3;
16726            } else if let Some(side) = st.latest_side() {
16727                if st
16728                    .transmit_queue
16729                    .get(selected)
16730                    .is_some_and(|queued| queued.disposable & TX_LATEST_TURN == 0)
16731                    && let Some(progress) = st
16732                        .side_transport
16733                        .get_mut(&side)
16734                        .and_then(|side| side.tx_chunk_progress.as_mut())
16735                {
16736                    // A genuinely new logical report cannot inherit an old
16737                    // cursor classification left by cancellation or rebind.
16738                    progress.clear_latest_owner();
16739                }
16740                st.latest_turn_phase = 4;
16741                st.transmit_queue
16742                    .update_metadata(selected, |q| q.disposable |= TX_LATEST_TURN);
16743            }
16744        }
16745        let current = st.transmit_queue.get(selected)?;
16746        let reliable = RouterInner::reliable_original_type(current).is_some();
16747        let ordered = reliable && RouterInner::ordered_original_type(current).is_some();
16748        let child = current.disposable & TX_E2E_CHILD != 0;
16749        let (queued, claim) = if child {
16750            let queued = current.clone();
16751            st.transmit_queue.update_metadata(selected, |item| {
16752                item.delivery_state = QueuedDelivery::ClaimedUnordered;
16753                if let RouterTxItem::ToSide { src, .. } = &mut item.item {
16754                    *src = Some(next_token as usize);
16755                }
16756            });
16757            st.ordered_claim_seq = next_token;
16758            (
16759                queued,
16760                Some(TxClaim {
16761                    token: next_token,
16762                    ordered: false,
16763                    side_retry: true,
16764                }),
16765            )
16766        } else if reliable {
16767            let queued = current.clone();
16768            st.transmit_queue.update_metadata(selected, |item| {
16769                item.delivery_state = if ordered {
16770                    QueuedDelivery::ClaimedOrdered
16771                } else {
16772                    QueuedDelivery::ClaimedUnordered
16773                };
16774                item.deferred_since_ms = u64::from(next_token);
16775            });
16776            st.ordered_claim_seq = next_token;
16777            (
16778                queued,
16779                Some(TxClaim {
16780                    token: next_token,
16781                    ordered,
16782                    side_retry: false,
16783                }),
16784            )
16785        } else {
16786            (st.transmit_queue.remove_pos(selected)?, None)
16787        };
16788        if claim.is_some() {
16789            st.tx_policy_flags |= 1;
16790        } else if slow {
16791            st.refresh_retry_hint();
16792        }
16793        // Selecting work transfers it to encoding/transport ownership. Even a
16794        // refused first callback may have advanced codec history. Never coalesce it.
16795        #[cfg(feature = "latest-value-egress")]
16796        let queued = {
16797            let mut selected = queued;
16798            if selected.disposable & TX_LATEST_PENDING != 0 {
16799                selected.disposable = TX_HISTORY_OWNED | (selected.disposable & TX_LATEST_TURN);
16800            }
16801            selected
16802        };
16803        #[cfg(feature = "latest-value-egress")]
16804        let service = {
16805            #[cfg(feature = "latest-value-egress")]
16806            {
16807                EgressService {
16808                    token: st.latest_service_seq,
16809                    side: st.latest_dispatch_tag(&queued),
16810                    kind: if queued.disposable & TX_CANONICAL_OWED != 0 {
16811                        1
16812                    } else if queued.disposable & TX_LATEST_TURN != 0 {
16813                        2
16814                    } else {
16815                        0
16816                    },
16817                }
16818            }
16819        };
16820        Some(TxWork {
16821            queued,
16822            claim,
16823            owed_turn,
16824            #[cfg(feature = "latest-value-egress")]
16825            service,
16826        })
16827    }
16828
16829    /// Process packets in the transmit queue for up to `timeout_ms` milliseconds.
16830    /// If `timeout_ms == 0`, drains the queue fully.
16831    fn process_tx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16832        self.expire_side_chunks();
16833        let start = self.clock.now_ms();
16834        loop {
16835            let received_control = crate::memory_admission::allow_drain(
16836                self.drain_queued_control_rx_before_tx(start, timeout_ms),
16837            )?;
16838            let control_budget_used = received_control
16839                && timeout_ms != 0
16840                && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
16841            // Complete one RX/TX pair even after a cooperative RX overrun.
16842            // Defer timeout cancellation until queued receipts are serviced.
16843            if !control_budget_used {
16844                crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
16845                crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
16846            }
16847
16848            let pkt_opt = {
16849                let mut st = self.state.lock();
16850                #[cfg(feature = "std")]
16851                let work = if self
16852                    .active_queued_tx
16853                    .load(core::sync::atomic::Ordering::Acquire)
16854                    == usize::MAX
16855                {
16856                    None // Leave ownership queued; never wrap the active total.
16857                } else {
16858                    Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false)
16859                };
16860                #[cfg(not(feature = "std"))]
16861                let work = Self::pop_transmit_queue_locked(&mut st, self.clock.now_ms(), false);
16862                #[cfg(feature = "std")]
16863                let work = self.track_queued_tx(&mut st, work);
16864                work
16865            };
16866            let Some(pkt) = pkt_opt else { break };
16867            #[cfg(feature = "std")]
16868            let (pkt, _activity) = pkt;
16869            self.dispatch_queued_item(pkt)?;
16870            if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
16871                break;
16872            }
16873        }
16874        Ok(())
16875    }
16876
16877    /// Process packets in the transmit queue for up to `timeout_ms` milliseconds.
16878    /// If `timeout_ms == 0`, drains the queue fully.
16879    pub fn process_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16880        #[cfg(feature = "timesync")]
16881        let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
16882        #[cfg(feature = "discovery")]
16883        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16884        self.process_tx_queue_with_timeout_impl(timeout_ms)
16885    }
16886
16887    /// Drain queued transmit work without generating periodic discovery/time-sync
16888    /// announcements. Reliable retries and queued discovery reception are still
16889    /// serviced. Applications using this between ingress bursts must also call
16890    /// `periodic` or the normal queue methods regularly for maintenance.
16891    /// A zero timeout drains the queue fully.
16892    pub fn dispatch_tx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16893        self.process_tx_queue_with_timeout_impl(timeout_ms)
16894    }
16895
16896    /// Process a single queued receive item.
16897    #[inline]
16898    fn process_rx_queue_item(&self, item: RouterRxItem) -> TelemetryResult<()> {
16899        self.rx_item(&item, true)
16900    }
16901
16902    /// Process packets in the receive queue for up to `timeout_ms` milliseconds.
16903    /// If `timeout_ms == 0`, drains the queue fully.
16904    fn process_rx_queue_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16905        self.expire_side_chunks();
16906        let start = self.clock.now_ms();
16907        loop {
16908            let item_opt = self
16909                .isr_rx_queue
16910                .as_ref()
16911                .and_then(|queue| queue.pop_front().unwrap_or(None))
16912                .or_else(|| {
16913                    let mut st = self.state.lock();
16914                    st.pop_received_fair()
16915                });
16916            let Some(item) = item_opt else { break };
16917            self.process_rx_queue_item(item)?;
16918            if timeout_ms != 0 && self.clock.now_ms().wrapping_sub(start) >= timeout_ms as u64 {
16919                break;
16920            }
16921        }
16922        Ok(())
16923    }
16924
16925    /// Process packets in the receive queue for up to `timeout_ms` milliseconds.
16926    /// If `timeout_ms == 0`, drains the queue fully.
16927    pub fn process_rx_queue_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
16928        #[cfg(feature = "timesync")]
16929        let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
16930        #[cfg(feature = "discovery")]
16931        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
16932        self.process_rx_queue_with_timeout_impl(timeout_ms)
16933    }
16934
16935    /// Process both transmit and receive queues for up to `timeout_ms` milliseconds.
16936    /// If `timeout_ms == 0`, drains both queues fully.
16937    fn process_one_queued_rx(&self) -> TelemetryResult<bool> {
16938        let item = self
16939            .isr_rx_queue
16940            .as_ref()
16941            .and_then(|queue| queue.pop_front().unwrap_or(None))
16942            .or_else(|| self.state.lock().pop_received_fair());
16943        let Some(item) = item else {
16944            return Ok(false);
16945        };
16946        self.process_rx_queue_item(item)?;
16947        Ok(true)
16948    }
16949
16950    fn process_all_queues_with_timeout_impl(&self, timeout_ms: u32) -> TelemetryResult<()> {
16951        self.expire_side_chunks();
16952        // Share the deadline between directions. Splitting a 1 ms budget in
16953        // half rounded TX down to zero and serviced only one transmit item,
16954        // even when the remainder of the pass was idle. One TX/RX pair is
16955        // atomic for fairness; a slow handler may overrun the deadline.
16956        let receive_first = !self.cfg.role.local_services();
16957        let start = self.clock.now_ms();
16958        loop {
16959            let received_control = crate::memory_admission::allow_drain(
16960                self.drain_queued_control_rx_before_tx(start, timeout_ms),
16961            )?;
16962            let mut did_any = received_control;
16963            let control_budget_used = received_control
16964                && timeout_ms != 0
16965                && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms);
16966            if !control_budget_used {
16967                crate::memory_admission::allow_drain(self.process_reliable_timeouts())?;
16968                crate::memory_admission::allow_drain(self.process_end_to_end_reliable_timeouts())?;
16969            }
16970
16971            if receive_first && !control_budget_used {
16972                did_any |= self.process_one_queued_rx()?;
16973            }
16974
16975            if let Some(pkt) = {
16976                let mut st = self.state.lock();
16977                let now = self.clock.now_ms();
16978                // Complete one fair TX even after RX uses the deadline.
16979                // Priority burst limits still prefer and protect queued ACKs.
16980                #[cfg(feature = "std")]
16981                let work = if self
16982                    .active_queued_tx
16983                    .load(core::sync::atomic::Ordering::Acquire)
16984                    == usize::MAX
16985                {
16986                    None // Leave ownership queued; never wrap the active total.
16987                } else {
16988                    Self::pop_transmit_queue_locked(&mut st, now, false)
16989                };
16990                #[cfg(not(feature = "std"))]
16991                let work = Self::pop_transmit_queue_locked(&mut st, now, false);
16992                #[cfg(feature = "std")]
16993                let work = self.track_queued_tx(&mut st, work);
16994                work
16995            } {
16996                #[cfg(feature = "std")]
16997                let (pkt, _activity) = pkt;
16998                self.dispatch_queued_item(pkt)?;
16999                did_any = true;
17000            }
17001            // Control reception must not starve queued ACKs/transmissions.
17002            if control_budget_used {
17003                return Ok(());
17004            }
17005            if !receive_first {
17006                did_any |= self.process_one_queued_rx()?;
17007            }
17008            if !did_any
17009                || (timeout_ms != 0
17010                    && self.clock.now_ms().wrapping_sub(start) >= u64::from(timeout_ms))
17011            {
17012                break;
17013            }
17014        }
17015
17016        Ok(())
17017    }
17018
17019    /// Process both transmit and receive queues for up to `timeout_ms` milliseconds.
17020    /// If `timeout_ms == 0`, drains both queues fully.
17021    pub fn process_all_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
17022        #[cfg(feature = "timesync")]
17023        let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
17024        #[cfg(feature = "discovery")]
17025        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17026        self.process_all_queues_with_timeout_impl(timeout_ms)
17027    }
17028
17029    /// Drain queued RX/TX work without generating periodic announcements.
17030    /// Reliable retries and received discovery are still processed. Call
17031    /// `periodic` or poll discovery/time sync separately on a regular cadence.
17032    /// The deadline is cooperative: a single callback may exceed it.
17033    pub fn drain_queues_with_timeout(&self, timeout_ms: u32) -> TelemetryResult<()> {
17034        self.process_all_queues_with_timeout_impl(timeout_ms)
17035    }
17036
17037    /// Runs one application-loop maintenance cycle.
17038    ///
17039    /// This polls built-in time sync and discovery when those features are compiled in, then
17040    /// drains queued TX/RX work for up to `timeout_ms` milliseconds.
17041    pub fn periodic(&self, timeout_ms: u32) -> TelemetryResult<()> {
17042        #[cfg(feature = "timesync")]
17043        let _ = crate::memory_admission::allow_drain(self.poll_timesync())?;
17044
17045        #[cfg(feature = "discovery")]
17046        {
17047            let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17048        }
17049
17050        self.process_all_queues_with_timeout_impl(timeout_ms)
17051    }
17052
17053    /// Runs one application-loop maintenance cycle without polling built-in time sync.
17054    ///
17055    /// Discovery is still polled when that feature is compiled in, then queued TX/RX work is
17056    /// drained for up to `timeout_ms` milliseconds.
17057    pub fn periodic_no_timesync(&self, timeout_ms: u32) -> TelemetryResult<()> {
17058        #[cfg(feature = "discovery")]
17059        {
17060            let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
17061        }
17062
17063        self.process_all_queues_with_timeout_impl(timeout_ms)
17064    }
17065
17066    /// Enqueue an item for later transmission with flags.
17067    #[inline]
17068    fn tx_queue_item_with_flags(
17069        &self,
17070        item: RouterTxItem,
17071        ignore_local: bool,
17072    ) -> TelemetryResult<()> {
17073        self.tx_queue_item_with_flags_at(item, ignore_local, 0)
17074    }
17075
17076    #[cfg(feature = "compression")]
17077    fn retry_codec_owner(
17078        &self,
17079        item: RouterTxItem,
17080        ignore_local: bool,
17081        reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
17082    ) -> TelemetryResult<()> {
17083        self.retry_codec_owner_at_priority(item, ignore_local, reservation, 0, 0, false)
17084    }
17085
17086    #[cfg(feature = "compression")]
17087    fn retry_codec_owner_at_priority(
17088        &self,
17089        item: RouterTxItem,
17090        ignore_local: bool,
17091        reservation: &mut Option<crate::payload_codec::QueueReservation<'_, RouterInner>>,
17092        priority_floor: u8,
17093        _turn_flags: u8,
17094        immediate: bool,
17095    ) -> TelemetryResult<()> {
17096        let priority = priority_floor.max(match &item {
17097            RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
17098            _ => return Err(TelemetryError::BadArg),
17099        });
17100        #[cfg(feature = "discovery")]
17101        let schema_owner = RouterInner::schema_response_owner(&item);
17102        let mut state = self.state.lock();
17103        if let Some(reservation) = reservation {
17104            reservation.release_locked(&mut state);
17105        }
17106        let result = state.push_transmit(TxQueued {
17107            disposable: TX_DISPOSABLE | _turn_flags,
17108            item,
17109            ignore_local,
17110            priority,
17111            delivery_state: QueuedDelivery::Untracked,
17112            accepted_type: 0,
17113            deferred_since_ms: 0,
17114            retry_after_ms: self.clock.now_ms().saturating_add(u64::from(!immediate)),
17115        });
17116        #[cfg(feature = "discovery")]
17117        if result.is_err() {
17118            RouterInner::schema_response_owner_lost(
17119                &mut state.discovery_side_throttle,
17120                schema_owner,
17121            );
17122        }
17123        result
17124    }
17125
17126    fn claim_schema_is_current(
17127        &self,
17128        ty: DataType,
17129        claim: Option<TxClaim>,
17130    ) -> TelemetryResult<bool> {
17131        let Some(claim) = claim else {
17132            return Ok(true);
17133        };
17134        let expected = if claim.ordered {
17135            crate::ReliableMode::Ordered
17136        } else {
17137            crate::ReliableMode::Unordered
17138        };
17139        let current = reliable_mode(ty);
17140        let mut st = self.state.lock();
17141        let owned = st.claim_index(claim.token).is_some();
17142        if !owned {
17143            return Ok(false);
17144        }
17145        if current == expected {
17146            return Ok(true);
17147        }
17148        st.cancel_accepted_type(ty.as_u32());
17149        st.refresh_retry_hint();
17150        Err(TelemetryError::HandlerError(
17151            "accepted reliable schema changed before dispatch",
17152        ))
17153    }
17154
17155    fn mark_claim_local_delivered(&self, claim: Option<TxClaim>) -> bool {
17156        let Some(claim) = claim else {
17157            return true;
17158        };
17159        let mut st = self.state.lock();
17160        let index = st.claim_index(claim.token);
17161        let Some(index) = index else {
17162            return false;
17163        };
17164        st.transmit_queue
17165            .update_metadata(index, |item| item.ignore_local = true);
17166        true
17167    }
17168
17169    fn dispatch_queued_item(&self, work: TxWork) -> TelemetryResult<()> {
17170        let owed_turn = work.owed_turn;
17171        let queued = work.queued;
17172        let parent = (queued.disposable & TX_E2E_CHILD != 0).then_some(queued.deferred_since_ms);
17173        let since = matches!(
17174            queued.delivery_state,
17175            QueuedDelivery::DeferredOrdered | QueuedDelivery::DeferredUnordered
17176        )
17177        .then_some(queued.deferred_since_ms);
17178        let mut context = DispatchContext {
17179            priority: queued.priority,
17180            claim: work.claim,
17181            since,
17182            completion_flags: queued.disposable & {
17183                #[cfg(feature = "latest-value-egress")]
17184                {
17185                    TX_CANONICAL_TURN | TX_LATEST_TURN
17186                }
17187                #[cfg(not(feature = "latest-value-egress"))]
17188                {
17189                    0
17190                }
17191            },
17192            admitted: false,
17193            #[cfg(feature = "latest-value-egress")]
17194            latest_service: work.service,
17195        };
17196        let schema = if work.claim.is_some_and(|claim| claim.side_retry) {
17197            Ok(true)
17198        } else {
17199            self.claim_schema_is_current(DataType(queued.accepted_type), work.claim)
17200        };
17201        if !matches!(schema, Ok(true)) {
17202            self.finish_dispatch_turn(owed_turn, &context);
17203            return schema.map(|_| ());
17204        }
17205        let mut child_unwind = work.claim.filter(|claim| claim.side_retry).map(|claim| {
17206            e2e_backpressure::ChildUnwind {
17207                router: self,
17208                claim,
17209                armed: true,
17210            }
17211        });
17212        let mut result =
17213            self.tx_item_impl_with_retry(queued.item, queued.ignore_local, true, &mut context);
17214        if let Some(claim) = work.claim {
17215            if claim.side_retry {
17216                self.finish_e2e_child_claim(
17217                    claim,
17218                    parent.expect("child parent"),
17219                    &context,
17220                    result.is_ok(),
17221                )?;
17222                if let Some(guard) = &mut child_unwind {
17223                    guard.armed = false;
17224                }
17225                self.finish_dispatch_turn(owed_turn, &context);
17226                return result;
17227            }
17228            let mut st = self.state.lock();
17229            if result.is_err() && !context.is_terminal() && !context.owner_transferred() {
17230                let Some(index) = st.claim_index(claim.token) else {
17231                    drop(st);
17232                    self.finish_dispatch_turn(owed_turn, &context);
17233                    return result;
17234                };
17235                let original = st.transmit_queue.get(index).expect("claimed owner");
17236                if !original.ignore_local
17237                    && let RouterTxItem::Broadcast(data) = &original.item
17238                {
17239                    let id = Self::get_hash(data);
17240                    st.recent_rx.remove_value(&id);
17241                }
17242                drop(st);
17243                result = self.defer_accepted_original(&mut context);
17244                if result.is_ok() {
17245                    self.finish_dispatch_turn(owed_turn, &context);
17246                    return Ok(());
17247                }
17248                st = self.state.lock();
17249            }
17250            if let Some(index) = st.claim_index(claim.token) {
17251                if context.is_terminal() && claim.ordered {
17252                    let ty = st
17253                        .transmit_queue
17254                        .get(index)
17255                        .expect("claimed owner")
17256                        .accepted_type;
17257                    st.cancel_accepted_type(ty);
17258                } else {
17259                    let ty = st
17260                        .transmit_queue
17261                        .get(index)
17262                        .expect("claimed owner")
17263                        .accepted_type;
17264                    let _ = st.transmit_queue.remove_pos(index);
17265                    if claim.ordered {
17266                        // Exactly one FIFO head per Ordered type can be eligible.
17267                        // Promotion changes metadata only, retaining the admitted payload.
17268                        let next = st.transmit_queue.iter().position(|item| {
17269                            item.delivery_state == QueuedDelivery::OrderedTail
17270                                && item.accepted_type == ty
17271                        });
17272                        if let Some(next) = next {
17273                            st.transmit_queue.update_metadata(next, |item| {
17274                                item.delivery_state = QueuedDelivery::Ordered
17275                            });
17276                        }
17277                    }
17278                }
17279            }
17280            st.refresh_retry_hint();
17281        }
17282        self.finish_dispatch_turn(owed_turn, &context);
17283        result
17284    }
17285
17286    fn current_egress_requeue(&self, context: &DispatchContext) -> (u8, bool) {
17287        #[cfg(feature = "latest-value-egress")]
17288        {
17289            let service = context.egress_service();
17290            let st = self.state.lock();
17291            if service.side == 0
17292                || service.token != st.latest_service_seq
17293                || st.latest_side() != Some(usize::from(service.side - 1))
17294            {
17295                return (0, false);
17296            }
17297        }
17298        (
17299            context.canonical_turn_flags(),
17300            context.yielded_canonical_turn(),
17301        )
17302    }
17303
17304    fn finish_dispatch_turn(&self, owed_turn: bool, context: &DispatchContext) {
17305        let mut guard = self.state.lock();
17306        let st = &mut *guard;
17307        #[cfg(feature = "latest-value-egress")]
17308        if context.latest_service.side != 0
17309            && context.latest_service.token == st.latest_service_seq
17310            && st.latest_side() == Some(usize::from(context.latest_service.side - 1))
17311        {
17312            let kind = if context.completion_flags & TX_CANONICAL_OWED != 0 {
17313                1
17314            } else if context.completion_flags & TX_LATEST_TURN != 0 {
17315                2
17316            } else {
17317                0
17318            };
17319            st.latest_turn_phase = match kind {
17320                1 if context.admitted => 2,
17321                1 if context.refused() => 1,
17322                1 => 0,
17323                2 if context.admitted => {
17324                    if st.latest_turn_credit > 0 {
17325                        2
17326                    } else {
17327                        0
17328                    }
17329                }
17330                2 if context.refused() => 2,
17331                2 => 0,
17332                _ => st.latest_turn_phase,
17333            };
17334        }
17335        st.transmit_queue.finish_deferred_turn(
17336            context.priority,
17337            owed_turn,
17338            context.admitted,
17339            context.refused(),
17340            &mut st.tx_priority_burst,
17341        );
17342    }
17343
17344    fn expire_deferred_original(
17345        &self,
17346        ty: DataType,
17347        since: u64,
17348        ordered: bool,
17349    ) -> TelemetryResult<()> {
17350        let lifetime = runtime_reliable_retransmit_ms()
17351            .saturating_mul(u64::from(runtime_reliable_max_retries()) + 1);
17352        let now = self.clock.now_ms();
17353        if now >= since && now - since < lifetime {
17354            return Ok(());
17355        }
17356        if ordered {
17357            let mut st = self.state.lock();
17358            st.cancel_accepted_type(ty.as_u32());
17359            st.refresh_retry_hint();
17360        }
17361        Err(TelemetryError::HandlerError(
17362            "accepted reliable delivery expired before dispatch",
17363        ))
17364    }
17365
17366    fn defer_accepted_original(&self, context: &mut DispatchContext) -> TelemetryResult<()> {
17367        context.mark_refused();
17368        let Some(claim) = context.claim else {
17369            return Err(TelemetryError::Io("reliable owner unavailable"));
17370        };
17371        let now = self.clock.now_ms();
17372        let Some(retry_after_ms) = now.checked_add(1) else {
17373            context.mark_terminal();
17374            return Err(TelemetryError::HandlerError(
17375                "accepted reliable clock exhausted",
17376            ));
17377        };
17378        let mut st = self.state.lock();
17379        let Some(index) = st.claim_index(claim.token) else {
17380            return Ok(());
17381        };
17382        st.transmit_queue.update_metadata(index, |item| {
17383            item.delivery_state = if claim.ordered {
17384                QueuedDelivery::DeferredOrdered
17385            } else {
17386                QueuedDelivery::DeferredUnordered
17387            };
17388            item.deferred_since_ms = context.since.unwrap_or(now);
17389            item.retry_after_ms = retry_after_ms;
17390        });
17391        st.tx_policy_flags |= 1;
17392        Ok(())
17393    }
17394
17395    #[cfg(feature = "discovery")]
17396    #[allow(unused_mut)]
17397    fn replace_claim_payload(&self, claim: TxClaim, mut data: RouterItem) -> TelemetryResult<()> {
17398        #[cfg(feature = "compact-packet-store")]
17399        data.park_for_queue()?;
17400        let payload_cost = match &data {
17401            RouterItem::Packet(packet) => packet.byte_cost(),
17402            RouterItem::Packed(bytes) => size_of::<crate::SharedBytes>() + bytes.len(),
17403        };
17404        let mut st = self.state.lock();
17405        let Some(index) = st.claim_index(claim.token) else {
17406            return Ok(());
17407        };
17408        let original = st.transmit_queue.get(index).expect("claimed owner");
17409        let old_cost = original.byte_cost();
17410        let new_cost = old_cost
17411            .saturating_sub(original.item.byte_cost())
17412            .saturating_add(payload_cost);
17413        let priority = original.priority;
17414        let growth = new_cost.saturating_sub(old_cost);
17415        if growth != 0 {
17416            st.make_shared_queue_room(growth, priority, None)?;
17417        }
17418        let Some(index) = st.claim_index(claim.token) else {
17419            return Ok(());
17420        };
17421        st.transmit_queue.update_admitted(index, new_cost, |item| {
17422            item.item = RouterTxItem::Broadcast(data)
17423        })
17424    }
17425
17426    fn tx_queue_item_for_retry(
17427        &self,
17428        item: RouterTxItem,
17429        ignore_local: bool,
17430    ) -> TelemetryResult<()> {
17431        self.tx_queue_item_with_flags_at(item, ignore_local, self.clock.now_ms().saturating_add(1))
17432    }
17433
17434    #[cfg(not(feature = "compression"))]
17435    fn tx_queue_item_for_retry_at_priority_with_turn(
17436        &self,
17437        item: RouterTxItem,
17438        ignore_local: bool,
17439        priority: u8,
17440        turn_flags: u8,
17441        immediate: bool,
17442    ) -> TelemetryResult<()> {
17443        #[cfg(feature = "discovery")]
17444        let schema_owner = RouterInner::schema_response_owner(&item);
17445        let result = self.tx_queue_item_with_priority_and_turn_at(
17446            item,
17447            ignore_local,
17448            priority,
17449            self.clock.now_ms().saturating_add(u64::from(!immediate)),
17450            turn_flags,
17451        );
17452        #[cfg(feature = "discovery")]
17453        if result.is_err() {
17454            RouterInner::schema_response_owner_lost(
17455                &mut self.state.lock().discovery_side_throttle,
17456                schema_owner,
17457            );
17458        }
17459        result
17460    }
17461
17462    fn tx_queue_item_with_flags_at(
17463        &self,
17464        item: RouterTxItem,
17465        ignore_local: bool,
17466        retry_after_ms: u64,
17467    ) -> TelemetryResult<()> {
17468        let priority = match &item {
17469            RouterTxItem::Broadcast(data) => self.item_service_priority(data)?,
17470            RouterTxItem::EndToEndReplay { .. } => {
17471                Self::router_item_priority_bumped(DataType::ReliableAck)
17472            }
17473            RouterTxItem::ToSide { data, .. } => self.item_service_priority(data)?,
17474            RouterTxItem::ReliableReplay { ty, .. } => Self::router_item_priority_bumped(*ty),
17475        };
17476        self.tx_queue_item_with_priority_at(item, ignore_local, priority, retry_after_ms)
17477    }
17478
17479    #[inline]
17480    fn tx_queue_item_with_priority(
17481        &self,
17482        item: RouterTxItem,
17483        ignore_local: bool,
17484        priority: u8,
17485    ) -> TelemetryResult<()> {
17486        self.tx_queue_item_with_priority_at(item, ignore_local, priority, 0)
17487    }
17488
17489    fn tx_queue_item_with_priority_at(
17490        &self,
17491        item: RouterTxItem,
17492        ignore_local: bool,
17493        priority: u8,
17494        retry_after_ms: u64,
17495    ) -> TelemetryResult<()> {
17496        self.tx_queue_item_with_priority_and_turn_at(
17497            item,
17498            ignore_local,
17499            priority,
17500            retry_after_ms,
17501            0,
17502        )
17503    }
17504    fn tx_queue_item_with_priority_and_turn_at(
17505        &self,
17506        item: RouterTxItem,
17507        ignore_local: bool,
17508        priority: u8,
17509        retry_after_ms: u64,
17510        turn_flags: u8,
17511    ) -> TelemetryResult<()> {
17512        let mut st = self.state.lock();
17513        st.push_transmit(TxQueued {
17514            disposable: TX_DISPOSABLE | turn_flags,
17515            item,
17516            ignore_local,
17517            priority,
17518            delivery_state: QueuedDelivery::Untracked,
17519            accepted_type: 0,
17520            deferred_since_ms: 0,
17521            retry_after_ms,
17522        })?;
17523        Ok(())
17524    }
17525
17526    /// Enqueue an item for later transmission (default: local dispatch enabled).
17527    #[inline]
17528    fn tx_queue_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
17529        self.tx_queue_item_with_flags(item, false)
17530    }
17531
17532    #[cfg(feature = "std")]
17533    #[inline]
17534    fn try_enter_side_tx(&self, side: usize) -> Option<crate::lock::ActiveSideTxGuard<'_>> {
17535        let gate = self
17536            .state
17537            .lock()
17538            .side_transport
17539            .get(&side)?
17540            .tx_callback_gate
17541            .clone();
17542        self.side_tx_activity.try_enter(gate)
17543    }
17544    #[cfg(not(feature = "std"))]
17545    #[inline]
17546    fn try_enter_side_tx(&self, _side: usize) -> Option<crate::lock::ReentryGuard<'_>> {
17547        self.side_tx_gate.try_enter()
17548    }
17549    #[inline]
17550    fn side_tx_active(&self) -> bool {
17551        #[cfg(feature = "std")]
17552        {
17553            self.side_tx_activity.is_active()
17554        }
17555        #[cfg(not(feature = "std"))]
17556        {
17557            self.side_tx_gate.is_active()
17558        }
17559    }
17560
17561    // ---------- PUBLIC API: RX queue ----------
17562
17563    /// Drain the receive queue fully.
17564    #[inline]
17565    pub fn process_rx_queue(&self) -> TelemetryResult<()> {
17566        self.process_rx_queue_with_timeout(0)
17567    }
17568
17569    /// Enqueue packed bytes for RX processing as locally-originated input.
17570    #[inline]
17571    pub fn rx_packed_queue(&self, bytes: &[u8]) -> TelemetryResult<()> {
17572        crate::memory_admission::check_receive(bytes)?;
17573        let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17574        let priority = self.item_service_priority(&data)?;
17575        let mut st = self.state.lock();
17576        st.push_received(RouterRxItem {
17577            src: None,
17578            data,
17579            priority,
17580        })?;
17581        Ok(())
17582    }
17583
17584    /// ISR-safe, non-blocking enqueue of packed bytes for RX processing.
17585    ///
17586    /// Returns `TelemetryError::Io("rx queue busy")` if another context is
17587    /// currently mutating the ISR RX queue.
17588    #[inline]
17589    pub fn rx_packed_queue_isr(&self, bytes: &[u8]) -> TelemetryResult<()> {
17590        crate::memory_admission::check_receive(bytes)?;
17591        let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17592        let priority = Self::router_item_priority(&data)?;
17593        self.isr_rx_queue
17594            .as_ref()
17595            .ok_or(TelemetryError::PermissionDenied)?
17596            .push_back_prioritized(RouterRxItem {
17597                src: None,
17598                data,
17599                priority,
17600            })
17601    }
17602
17603    /// Enqueue a decoded packet for RX processing as locally-originated input.
17604    #[inline]
17605    pub fn rx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
17606        pkt.validate()?;
17607        let data = RouterItem::Packet(pkt);
17608        let priority = self.item_service_priority(&data)?;
17609        let mut st = self.state.lock();
17610        st.push_received(RouterRxItem {
17611            src: None,
17612            data,
17613            priority,
17614        })?;
17615        Ok(())
17616    }
17617
17618    /// ISR-safe, non-blocking enqueue of a packet for RX processing.
17619    ///
17620    /// Returns `TelemetryError::Io("rx queue busy")` if another context is
17621    /// currently mutating the ISR RX queue.
17622    #[inline]
17623    pub fn rx_queue_isr(&self, pkt: Packet) -> TelemetryResult<()> {
17624        pkt.validate()?;
17625        let data = RouterItem::Packet(pkt);
17626        let priority = Self::router_item_priority(&data)?;
17627        self.isr_rx_queue
17628            .as_ref()
17629            .ok_or(TelemetryError::PermissionDenied)?
17630            .push_back_prioritized(RouterRxItem {
17631                src: None,
17632                data,
17633                priority,
17634            })
17635    }
17636
17637    /// Enqueue a decoded packet for RX processing with an explicit ingress side.
17638    #[inline]
17639    pub fn rx_queue_from_side(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
17640        self.ensure_side_ingress_enabled(side)?;
17641        pkt.validate()?;
17642        let data = RouterItem::Packet(pkt);
17643        let priority = self.item_service_priority(&data)?;
17644        let mut st = self.state.lock();
17645        st.push_received(RouterRxItem {
17646            src: Some(side),
17647            data,
17648            priority,
17649        })?;
17650        Ok(())
17651    }
17652
17653    /// ISR-safe, non-blocking enqueue of a packet with explicit source side.
17654    ///
17655    /// Returns `TelemetryError::Io("rx queue busy")` if another context is
17656    /// currently mutating the ISR RX queue.
17657    #[inline]
17658    pub fn rx_queue_from_side_isr(&self, pkt: Packet, side: RouterSideId) -> TelemetryResult<()> {
17659        self.ensure_side_ingress_enabled(side)?;
17660        pkt.validate()?;
17661        let data = RouterItem::Packet(pkt);
17662        let priority = Self::router_item_priority(&data)?;
17663        self.isr_rx_queue
17664            .as_ref()
17665            .ok_or(TelemetryError::PermissionDenied)?
17666            .push_back_prioritized(RouterRxItem {
17667                src: Some(side),
17668                data,
17669                priority,
17670            })
17671    }
17672
17673    /// Enqueue packed bytes for RX processing with an explicit ingress side.
17674    #[inline]
17675    pub fn rx_packed_queue_from_side(
17676        &self,
17677        bytes: &[u8],
17678        side: RouterSideId,
17679    ) -> TelemetryResult<()> {
17680        self.ensure_side_ingress_enabled(side)?;
17681        let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
17682            return Ok(());
17683        };
17684        let data = RouterItem::Packed(decoded);
17685        let priority = self.item_service_priority(&data)?;
17686        let mut st = self.state.lock();
17687        st.push_received(RouterRxItem {
17688            src: Some(side),
17689            data,
17690            priority,
17691        })?;
17692        Ok(())
17693    }
17694
17695    /// ISR-safe, non-blocking enqueue of packed bytes with source side.
17696    ///
17697    /// Returns `TelemetryError::Io("rx queue busy")` if another context is
17698    /// currently mutating the ISR RX queue.
17699    #[inline]
17700    pub fn rx_packed_queue_from_side_isr(
17701        &self,
17702        bytes: &[u8],
17703        side: RouterSideId,
17704    ) -> TelemetryResult<()> {
17705        self.ensure_side_ingress_enabled(side)?;
17706        crate::memory_admission::check_receive(bytes)?;
17707        let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
17708        let priority = Self::router_item_priority(&data)?;
17709        self.isr_rx_queue
17710            .as_ref()
17711            .ok_or(TelemetryError::PermissionDenied)?
17712            .push_back_prioritized(RouterRxItem {
17713                src: Some(side),
17714                data,
17715                priority,
17716            })
17717    }
17718
17719    fn retry_with_attempts<F, T>(
17720        &self,
17721        times: usize,
17722        f: F,
17723    ) -> Result<(T, usize), (TelemetryError, usize)>
17724    where
17725        F: Fn() -> TelemetryResult<T>,
17726    {
17727        let times = if self.cfg.role.local_services() {
17728            times
17729        } else {
17730            times.min(1)
17731        };
17732        let mut last_err = None;
17733        for attempt in 0..times {
17734            match f() {
17735                Ok(v) => return Ok((v, attempt + 1)),
17736                // Transport or allocator pressure needs another service pass;
17737                // repeating the same callback synchronously cannot clear it.
17738                Err(e) if Self::is_transient_tx_error(&e) => return Err((e, attempt + 1)),
17739                Err(e) => last_err = Some((e, attempt + 1)),
17740            }
17741        }
17742        Err(last_err.expect("times > 0"))
17743    }
17744
17745    /// Check if the specified endpoint has a packet handler registered.
17746    #[inline]
17747    fn endpoint_has_packet_handler(&self, ep: DataEndpoint) -> bool {
17748        self.cfg
17749            .handlers()
17750            .iter()
17751            .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packet(_)))
17752    }
17753
17754    /// Check if the specified endpoint has a packed handler registered.
17755    #[inline]
17756    fn endpoint_has_packed_handler(&self, ep: DataEndpoint) -> bool {
17757        self.cfg
17758            .handlers()
17759            .iter()
17760            .any(|h| h.endpoint == ep && matches!(h.handler, EndpointHandlerFn::Packed(_)))
17761    }
17762
17763    fn packet_has_local_handler(&self, pkt: &Packet) -> bool {
17764        pkt.endpoints()
17765            .iter()
17766            .copied()
17767            .any(|ep| self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep))
17768    }
17769
17770    /// Whether receiving this packet constitutes delivery at this router.
17771    ///
17772    /// Network variables deliberately use their cache/update callback instead of a conventional
17773    /// endpoint handler. Treat that successful cache delivery as local for end-to-end ACKs;
17774    /// otherwise a reliable writer waits forever for every variable owner that does not also
17775    /// happen to register an unrelated packet handler for the variable's endpoint.
17776    fn packet_has_local_delivery(&self, pkt: &Packet) -> bool {
17777        self.packet_has_local_handler(pkt) || self.is_managed_variable_type(pkt.data_type())
17778    }
17779
17780    /// Call the specified endpoint handler with retries on failure.
17781    ///
17782    /// - `data` is present when called from RX processing (queue or immediate).
17783    /// - `pkt_for_ctx` is required for Packet handlers.
17784    /// - `env_for_ctx` provides header-only context when we haven't unpacked.
17785    fn call_handler_with_retries(
17786        &self,
17787        dest: DataEndpoint,
17788        handler: &EndpointHandler,
17789        data: Option<&[u8]>,
17790        pkt_for_ctx: Option<&Packet>,
17791        env_for_ctx: Option<&wire_format::TelemetryEnvelope>,
17792        called_from_queue: bool,
17793    ) -> TelemetryResult<()> {
17794        let owned_tmp: Option<RouterItem>;
17795
17796        let item_for_ctx: &RouterItem = match (data, pkt_for_ctx) {
17797            (Some(d), _) => {
17798                owned_tmp = Some(RouterItem::Packed(crate::shared_bytes::convert(
17799                    Arc::<[u8]>::from(d),
17800                )));
17801                owned_tmp.as_ref().unwrap()
17802            }
17803            (None, Some(pkt)) => {
17804                owned_tmp = Some(RouterItem::Packet(pkt.clone()));
17805                owned_tmp.as_ref().unwrap()
17806            }
17807            (None, None) => {
17808                debug_assert!(
17809                    false,
17810                    "call_handler_with_retries called without data or packet context"
17811                );
17812                return Ok(());
17813            }
17814        };
17815
17816        match (&handler.handler, data) {
17817            (EndpointHandlerFn::Packet(f), _) => {
17818                let pkt = pkt_for_ctx.expect("Packet handler requires Packet context");
17819                with_retries(
17820                    self,
17821                    dest,
17822                    item_for_ctx,
17823                    pkt_for_ctx,
17824                    env_for_ctx,
17825                    called_from_queue,
17826                    || f(pkt),
17827                )
17828            }
17829
17830            (EndpointHandlerFn::Packed(f), Some(bytes)) => with_retries(
17831                self,
17832                dest,
17833                item_for_ctx,
17834                pkt_for_ctx,
17835                env_for_ctx,
17836                called_from_queue,
17837                || f(bytes),
17838            ),
17839
17840            (EndpointHandlerFn::Packed(_), None) => Ok(()),
17841        }
17842    }
17843
17844    /// Error helper when we only have an envelope (no full packet).
17845    ///
17846    /// Sends a TelemetryError packet to all local endpoints except the failed one (if any).
17847    /// If no local endpoints remain, falls back to `fallback_stdout`.
17848    fn handle_callback_error_from_env(
17849        &self,
17850        env: &wire_format::TelemetryEnvelope,
17851        dest: Option<DataEndpoint>,
17852        e: TelemetryError,
17853        called_from_queue: bool,
17854    ) -> TelemetryResult<()> {
17855        if !self.cfg.role.local_services() {
17856            return Err(e);
17857        }
17858
17859        // Allocator refusal is backpressure, not a diagnostic packet. Even
17860        // formatting an error string can exhaust the remaining small blocks.
17861        if matches!(&e, TelemetryError::Io("memory pressure")) {
17862            return Err(e);
17863        }
17864
17865        let mut recipients: Vec<DataEndpoint> = env
17866            .endpoints
17867            .iter()
17868            .copied()
17869            .filter(|&ep| self.cfg.is_local_endpoint(ep))
17870            .collect();
17871        crate::small_sort::sort(&mut recipients);
17872        recipients.dedup();
17873        if let Some(failed) = dest {
17874            recipients.retain(|&ep| ep != failed);
17875        }
17876
17877        let device = self.sender_arc();
17878        let error_msg = format!(
17879            "Handler for endpoint {:?} failed on device {:?}: {:?}",
17880            dest, device, e
17881        );
17882
17883        if recipients.is_empty() && dest.is_none() {
17884            fallback_stdout(&error_msg);
17885            return Ok(());
17886        }
17887
17888        if recipients.is_empty() {
17889            recipients = env.endpoints.to_vec();
17890            crate::small_sort::sort(&mut recipients);
17891            recipients.dedup();
17892            if let Some(failed) = dest {
17893                recipients.retain(|&ep| ep != failed);
17894            }
17895        }
17896
17897        if recipients.is_empty() {
17898            fallback_stdout(&error_msg);
17899            return Ok(());
17900        }
17901
17902        let payload = make_error_payload(&error_msg);
17903
17904        let error_pkt = Packet::new(
17905            DataType::TelemetryError,
17906            &recipients,
17907            &env.sender.clone(),
17908            env.timestamp_ms,
17909            crate::shared_bytes::convert(payload),
17910        )?;
17911        self.emit_internal_tx(
17912            RouterTxItem::Broadcast(RouterItem::Packet(error_pkt)),
17913            false,
17914            called_from_queue,
17915        )
17916    }
17917
17918    fn handle_internal_reliable_packet(
17919        &self,
17920        pkt: &Packet,
17921        src: Option<RouterSideId>,
17922        called_from_queue: bool,
17923    ) -> TelemetryResult<bool> {
17924        if !matches!(
17925            pkt.data_type(),
17926            DataType::ReliableAck | DataType::ReliablePartialAck | DataType::ReliablePacketRequest
17927        ) {
17928            return Ok(false);
17929        }
17930
17931        let Some(src) = src else {
17932            return Ok(false);
17933        };
17934
17935        if Self::is_end_to_end_ack_packet(pkt)
17936            && let Ok(packet_id) = Self::decode_end_to_end_reliable_ack(pkt.payload())
17937        {
17938            #[cfg(feature = "std")]
17939            let explicit_completion_receipt = pkt.wire_target_senders().len() == 2
17940                && pkt.wire_target_senders()[1] == Self::sender_hash(self.sender_arc().as_ref());
17941            let targets_local_sender = self.cfg.role.local_services()
17942                && (pkt.wire_target_senders().is_empty()
17943                    || pkt
17944                        .wire_target_senders()
17945                        .contains(&Self::sender_hash(self.sender_arc().as_ref())));
17946            if !targets_local_sender && let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
17947                Self::note_end_to_end_acked_destination_locked(
17948                    &mut self.state.lock(),
17949                    packet_id,
17950                    owner,
17951                );
17952            }
17953            let alias = {
17954                let st = self.state.lock();
17955                st.reliable_return_routes
17956                    .get(&packet_id)
17957                    .filter(|route| route.upstream_contract.is_some())
17958                    .cloned()
17959            };
17960            if let Some(alias) = alias {
17961                let (upstream_id, publisher_hash) = alias.upstream_contract.expect("checked alias");
17962                #[cfg(feature = "std")]
17963                if reliable_trace_enabled() {
17964                    eprintln!(
17965                        "[sedsnet-reliable] proxy receipt id={packet_id:#x} upstream={upstream_id:#x} owner={:?}",
17966                        Self::end_to_end_ack_sender_hash(pkt)
17967                    );
17968                }
17969                let completed = {
17970                    let mut st = self.state.lock();
17971                    if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
17972                        if let Some(owner) = Self::end_to_end_ack_sender_hash(pkt) {
17973                            sent.pending_destinations.remove(&owner);
17974                        }
17975                        if !sent.pending_destinations.is_empty() {
17976                            return Ok(true);
17977                        }
17978                        st.end_to_end_reliable_tx.remove(&packet_id);
17979                        true
17980                    } else {
17981                        // An expired replay record is not evidence that the
17982                        // whole child cohort completed. Let the parent retry.
17983                        return Ok(true);
17984                    }
17985                };
17986                if completed {
17987                    let mut st = self.state.lock();
17988                    if Self::remember_reliable_return_route_locked(&mut st, upstream_id) {
17989                        st.reliable_return_routes
17990                            .entry(upstream_id)
17991                            .and_modify(|route| {
17992                                route.delivery_completed = true;
17993                                route.upstream_contract = Some((upstream_id, publisher_hash));
17994                            })
17995                            .or_insert(ReliableReturnRouteState {
17996                                side: alias.side,
17997                                delivery_completed: true,
17998                                local_admission_completed: true,
17999                                local_admission_token: 0,
18000                                admission_cancelled: false,
18001                                admission_committing: false,
18002                                last_replay_ms: alias.last_replay_ms,
18003                                upstream_contract: Some((upstream_id, publisher_hash)),
18004                            });
18005                    }
18006                }
18007                self.queue_end_to_end_reliable_ack_for_sender_hash(
18008                    upstream_id,
18009                    publisher_hash,
18010                    called_from_queue,
18011                )?;
18012                self.release_ordered_successors()?;
18013                return Ok(true);
18014            }
18015            // Relays may also keep a retransmission record for the forwarded
18016            // packet. Clearing that local record must not consume an ACK whose
18017            // frozen wire contract targets the original sender beyond this
18018            // router; it still has to travel back along the learned return
18019            // route. This occurs when a router is itself one managed-variable
18020            // owner while additional owners sit behind another side.
18021            let mut st = self.state.lock();
18022            if let Some(sent) = st.end_to_end_reliable_tx.get_mut(&packet_id) {
18023                if sent.waiting_for_predecessor {
18024                    return Ok(targets_local_sender);
18025                }
18026                if let Some(sender_hash) = Self::end_to_end_ack_sender_hash(pkt) {
18027                    let _removed = sent.pending_destinations.remove(&sender_hash);
18028                    #[cfg(feature = "std")]
18029                    if reliable_trace_enabled() {
18030                        eprintln!(
18031                            "[sedsnet-reliable] receipt time={} id={packet_id:#x} sender={sender_hash:#x} matched={} remaining={:?}",
18032                            self.clock.now_ms(),
18033                            _removed.is_some(),
18034                            sent.pending_destinations
18035                        );
18036                    }
18037                    #[cfg(feature = "std")]
18038                    let completion = if _removed.is_some()
18039                        && explicit_completion_receipt
18040                        && targets_local_sender
18041                        && sent.tracked_destinations
18042                        && sent.pending_destinations.is_empty()
18043                    {
18044                        st.end_to_end_reliable_tx.remove(&packet_id)
18045                    } else {
18046                        None
18047                    };
18048                    #[cfg(not(feature = "std"))]
18049                    if sent.pending_destinations.is_empty() {
18050                        st.end_to_end_reliable_tx.remove(&packet_id);
18051                    }
18052                    #[cfg(feature = "std")]
18053                    if completion.is_none()
18054                        && st
18055                            .end_to_end_reliable_tx
18056                            .get(&packet_id)
18057                            .is_some_and(|sent| sent.pending_destinations.is_empty())
18058                    {
18059                        st.end_to_end_reliable_tx.remove(&packet_id);
18060                    }
18061                    #[cfg(feature = "std")]
18062                    let completion_cutoff = completion.as_ref().map_or(0, |_| completion::serial());
18063                    drop(st);
18064                    #[cfg(feature = "std")]
18065                    let release_result = self.release_ordered_successors();
18066                    #[cfg(not(feature = "std"))]
18067                    self.release_ordered_successors()?;
18068                    #[cfg(feature = "std")]
18069                    {
18070                        if let Some(completed) = completion {
18071                            self.retire_completed_writes(packet_id, &completed, completion_cutoff);
18072                        }
18073                        release_result?;
18074                    }
18075                    return Ok(targets_local_sender);
18076                }
18077                st.end_to_end_reliable_tx.remove(&packet_id);
18078                drop(st);
18079                self.release_ordered_successors()?;
18080                return Ok(targets_local_sender);
18081            }
18082            // A late or duplicate receipt addressed to this publisher is
18083            // complete locally; it must not leak onto another network side.
18084            return Ok(targets_local_sender);
18085        }
18086
18087        let vals = pkt.data_as_u32()?;
18088        if vals.len() != 2 {
18089            return Err(TelemetryError::Unpack("bad reliable control payload"));
18090        }
18091        let ty = DataType::try_from_u32(vals[0]).ok_or(TelemetryError::InvalidType)?;
18092        let seq = vals[1];
18093
18094        match pkt.data_type() {
18095            DataType::ReliableAck => {
18096                self.handle_reliable_ack(src, ty, seq);
18097                Ok(true)
18098            }
18099            DataType::ReliablePartialAck => {
18100                self.handle_reliable_partial_ack(src, ty, seq);
18101                Ok(true)
18102            }
18103            DataType::ReliablePacketRequest => {
18104                self.queue_reliable_retransmit(src, ty, seq, called_from_queue)?;
18105                Ok(true)
18106            }
18107            _ => Ok(false),
18108        }
18109    }
18110
18111    /// Explicit GET permits one unchanged observation to cross a completed
18112    /// proxy again. Queue admission owns only this hop, independently of old
18113    /// end-to-end ACKs. Source time and identity never change; this is not proof
18114    /// of a post-request writer revision. A lost response needs another GET.
18115    #[cfg(feature = "discovery")]
18116    fn forward_requested_managed_observation(
18117        &self,
18118        item: &RouterRxItem,
18119        duplicate: bool,
18120        ty: DataType,
18121    ) -> TelemetryResult<()> {
18122        let Some(src) = item.src else {
18123            return Ok(());
18124        };
18125        if !self.is_managed_variable_type(ty) || is_internal_control_type(ty) {
18126            return Ok(());
18127        }
18128        let id = Self::get_hash(&item.data);
18129        let untargeted = self.item_target_senders(&item.data)?.is_empty();
18130        let now = self.clock.now_ms();
18131        let priority = self.item_service_priority(&item.data)?;
18132        let mut state = self.state.lock();
18133        let current = state
18134            .local_services
18135            .as_ref()
18136            .and_then(|local| local.managed_variable_latest.get(&ty.as_u32()))
18137            .map(|entry| (entry.packet.packet_id(), entry.sender_address));
18138        // A different revision cancels the grant, not completes a response.
18139        // Normal routing owns fresh updates; a backstep cannot extend a grant.
18140        for route in state.discovery_routes.values_mut() {
18141            for request in &mut route.requested_network_variables {
18142                if request.ty == ty
18143                    && request.refresh_pending
18144                    && (now < request.last_seen_ms
18145                        || now - request.last_seen_ms > DISCOVERY_ROUTE_TTL_MS
18146                        || current != Some((request.value_id, request.writer)))
18147                {
18148                    request.refresh_pending = false;
18149                }
18150            }
18151        }
18152        if !duplicate || !untargeted {
18153            return Ok(());
18154        }
18155        loop {
18156            let candidate = state.discovery_routes.iter().find_map(|(&side, route)| {
18157                if side == src
18158                    || !Self::side_ref(&state, side).is_ok_and(|link| link.opts.egress_enabled)
18159                {
18160                    return None;
18161                }
18162                route
18163                    .requested_network_variables
18164                    .iter()
18165                    .find(|request| {
18166                        request.ty == ty
18167                            && request.refresh_pending
18168                            && request.upstream == src
18169                            && request.value_id == id
18170                    })
18171                    .map(|request| (side, *request))
18172            });
18173            let Some((side, owner)) = candidate else {
18174                break;
18175            };
18176            // Reuse an already admitted untracked hop for this exact source
18177            // observation. It cannot be retired by an old reader ACK. Never
18178            // alias a frozen child/replay owner or rewrite an active payload.
18179            let owned = state.transmit_queue.iter().any(|queued| {
18180                queued.ignore_local
18181                    && queued.delivery_state == QueuedDelivery::Untracked
18182                    && queued.disposable & TX_E2E_CHILD == 0
18183                    && matches!(&queued.item,
18184                        RouterTxItem::ToSide { src: queued_src, dst, data }
18185                        if *queued_src == Some(src) && *dst == side
18186                            && Self::get_hash(data) == id)
18187            });
18188            if !owned {
18189                // Ordinary ToSide work, not a new end-to-end reader cohort.
18190                // Refusal leaves the grant pending; admission and grant
18191                // consumption share this lock. No new priority or credit.
18192                state.push_transmit(TxQueued {
18193                    disposable: TX_FRESH,
18194                    item: RouterTxItem::ToSide {
18195                        src: Some(src),
18196                        dst: side,
18197                        data: item.data.clone(),
18198                    },
18199                    ignore_local: true,
18200                    priority,
18201                    delivery_state: QueuedDelivery::Untracked,
18202                    accepted_type: 0,
18203                    deferred_since_ms: 0,
18204                    retry_after_ms: 0,
18205                })?;
18206            }
18207            if let Some(request) = state.discovery_routes.get_mut(&side).and_then(|route| {
18208                route
18209                    .requested_network_variables
18210                    .iter_mut()
18211                    .find(|request| {
18212                        request.ty == ty
18213                            && request.request_id == owner.request_id
18214                            && request.requester == owner.requester
18215                    })
18216            }) {
18217                request.refresh_pending = false;
18218                request.next_refresh_ms = now.saturating_add(runtime_reliable_retransmit_ms());
18219            }
18220        }
18221        Ok(())
18222    }
18223
18224    /// Core receive function handling both Packet and Packed items.
18225    ///
18226    /// Relay mode: if a destination endpoint has no matching local handler and the packet has
18227    /// any remotely-forwardable endpoints, the router will rebroadcast the packet ONCE, excluding
18228    /// the ingress side.
18229    fn rx_item(&self, item: &RouterRxItem, called_from_queue: bool) -> TelemetryResult<()> {
18230        // Reuse this CRC-validated borrowed envelope for accounting and
18231        // role/return-route checks in this invocation. Dispatch still rechecks
18232        // current routing metadata; no validation result survives queueing.
18233        let mut _local_admission_guard = None;
18234        let packed_frame = match &item.data {
18235            RouterItem::Packed(bytes) => Some(wire_format::peek_routing_frame_info(bytes.as_ref())),
18236            RouterItem::Packet(_) => None,
18237        };
18238        if crate::memory_admission::supports_work_kinds()
18239            && let (Some(side), RouterItem::Packed(bytes)) = (item.src, &item.data)
18240        {
18241            self.admit_received_frame_with_view(
18242                side,
18243                bytes,
18244                packed_frame.as_ref().and_then(|frame| frame.as_ref().ok()),
18245            )?;
18246        }
18247        let packed_routing = packed_frame
18248            .as_ref()
18249            .and_then(|frame| frame.as_ref().ok())
18250            .map(|frame| &frame.envelope);
18251        if let Some(src) = item.src {
18252            self.ensure_side_ingress_enabled(src)?;
18253            match &item.data {
18254                RouterItem::Packet(pkt) => {
18255                    let bytes = wire_format::packet_wire_size(pkt);
18256                    self.note_side_rx(src, pkt.data_type(), bytes, true);
18257                }
18258                RouterItem::Packed(bytes) => {
18259                    if let Some(env) = packed_routing.as_ref() {
18260                        self.note_side_rx(src, env.ty, bytes.len(), true);
18261                    }
18262                }
18263            }
18264            match &item.data {
18265                RouterItem::Packet(pkt) => {
18266                    if (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
18267                        && !is_internal_control_type(pkt.data_type())
18268                    {
18269                        match self.begin_reliable_local_admission(src, pkt.packet_id())? {
18270                            Some(guard) => _local_admission_guard = Some(guard),
18271                            None => {
18272                                self.queue_duplicate_proxy_edge_receipt(item, pkt.packet_id())?;
18273                                return Ok(());
18274                            }
18275                        }
18276                    }
18277                }
18278                RouterItem::Packed(bytes) => {
18279                    if let Some(env) = packed_routing.as_ref()
18280                        && (is_reliable_type(env.ty) || env.target_senders().len() != 0)
18281                        && !is_internal_control_type(env.ty)
18282                        && let Ok(packet_id) = wire_format::packet_id_from_wire(bytes.as_ref())
18283                    {
18284                        match self.begin_reliable_local_admission(src, packet_id)? {
18285                            Some(guard) => _local_admission_guard = Some(guard),
18286                            None => {
18287                                self.queue_duplicate_proxy_edge_receipt(item, packet_id)?;
18288                                return Ok(());
18289                            }
18290                        }
18291                    }
18292                }
18293            }
18294        }
18295        match &item.data {
18296            RouterItem::Packet(pkt) => {
18297                if !is_internal_control_type(pkt.data_type()) {
18298                    self.cache_managed_variable_packet(pkt, true, item.src)?;
18299                }
18300            }
18301            RouterItem::Packed(bytes) => {
18302                if let Some(env) = packed_routing.as_ref()
18303                    && !is_internal_control_type(env.ty)
18304                    && self.is_managed_variable_type(env.ty)
18305                {
18306                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18307                    pkt.validate()?;
18308                    self.cache_managed_variable_packet(&pkt, true, item.src)?;
18309                }
18310            }
18311        }
18312        if let Some(guard) = _local_admission_guard.as_ref() {
18313            guard.check()?;
18314        }
18315        let mut released_buffered: Vec<crate::SharedBytes> = Vec::new();
18316        if let (Some(src), RouterItem::Packed(bytes)) = (item.src, &item.data) {
18317            let (_opts, handler_is_packed, hop_reliable_enabled) = {
18318                let st = self.state.lock();
18319                let side_ref = Self::side_ref(&st, src)?;
18320                let opts = side_ref.opts.clone();
18321                (
18322                    opts.clone(),
18323                    matches!(
18324                        side_ref.tx_handler,
18325                        RouterTxHandlerFn::Packed(_) | RouterTxHandlerFn::PackedWithPriority(_)
18326                    ),
18327                    opts.reliable_enabled
18328                        && self.cfg.reliable_enabled()
18329                        && !self.side_has_multiple_announcers_locked(&st, src, self.clock.now_ms()),
18330                )
18331            };
18332
18333            if hop_reliable_enabled && handler_is_packed {
18334                let frame = match packed_frame
18335                    .as_ref()
18336                    .expect("packed frame")
18337                    .as_ref()
18338                    .map_err(Clone::clone)
18339                {
18340                    Ok(frame) => frame,
18341                    Err(e) => {
18342                        if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
18343                            if let Ok(frame) =
18344                                wire_format::peek_frame_info_unchecked(bytes.as_ref())
18345                                && is_reliable_type(frame.envelope.ty)
18346                                && let Some(hdr) = frame.reliable
18347                            {
18348                                let unordered =
18349                                    (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
18350                                let unsequenced =
18351                                    (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
18352
18353                                if !unsequenced {
18354                                    let requested = if unordered {
18355                                        hdr.seq
18356                                    } else {
18357                                        let mut st = self.state.lock();
18358                                        let rx_state = self.reliable_rx_state_mut(
18359                                            &mut st,
18360                                            src,
18361                                            frame.envelope.ty,
18362                                        );
18363                                        rx_state.expected_seq.min(hdr.seq)
18364                                    };
18365                                    self.queue_reliable_packet_request(
18366                                        src,
18367                                        frame.envelope.ty,
18368                                        requested,
18369                                        called_from_queue,
18370                                    )?;
18371                                }
18372                            }
18373                            return Ok(());
18374                        }
18375                        return Err(e);
18376                    }
18377                };
18378                if is_reliable_type(frame.envelope.ty)
18379                    && let Some(hdr) = frame.reliable
18380                {
18381                    if frame.ack_only() {
18382                        self.handle_reliable_ack(src, frame.envelope.ty, hdr.ack);
18383                        return Ok(());
18384                    }
18385                    let unordered = (hdr.flags & wire_format::RELIABLE_FLAG_UNORDERED) != 0;
18386                    let unsequenced = (hdr.flags & wire_format::RELIABLE_FLAG_UNSEQUENCED) != 0;
18387
18388                    if !unsequenced {
18389                        if unordered {
18390                            self.queue_reliable_ack(
18391                                src,
18392                                frame.envelope.ty,
18393                                hdr.seq,
18394                                called_from_queue,
18395                            )?;
18396                        } else {
18397                            let mut release: Vec<crate::SharedBytes> = Vec::new();
18398                            let mut last_delivered = None;
18399                            let mut ack_old = None;
18400                            let mut request_missing = None;
18401                            let mut partial_ack = None;
18402                            {
18403                                let mut st = self.state.lock();
18404                                let rx_state =
18405                                    self.reliable_rx_state_mut(&mut st, src, frame.envelope.ty);
18406                                let expected_seq = rx_state.expected_seq;
18407                                if hdr.seq < expected_seq {
18408                                    ack_old = Some(expected_seq.saturating_sub(1));
18409                                } else if hdr.seq > expected_seq {
18410                                    request_missing = Some(expected_seq);
18411                                    partial_ack = Some(hdr.seq);
18412                                    st.buffer_reliable_rx(
18413                                        src,
18414                                        frame.envelope.ty,
18415                                        hdr.seq,
18416                                        bytes.clone(),
18417                                    )?;
18418                                } else {
18419                                    release.push(bytes.clone());
18420                                    last_delivered = Some(hdr.seq);
18421                                    let mut next_expected = hdr.seq.wrapping_add(1);
18422                                    while let Some(buf) = rx_state.buffered.remove(&next_expected) {
18423                                        release.push(buf);
18424                                        last_delivered = Some(next_expected);
18425                                        let next = next_expected.wrapping_add(1);
18426                                        next_expected = if next == 0 { 1 } else { next };
18427                                    }
18428                                    rx_state.expected_seq = next_expected;
18429                                }
18430                            }
18431
18432                            if let Some(ack_seq) = ack_old {
18433                                self.queue_reliable_ack(
18434                                    src,
18435                                    frame.envelope.ty,
18436                                    ack_seq,
18437                                    called_from_queue,
18438                                )?;
18439                                return Ok(());
18440                            }
18441                            if let Some(request_seq) = request_missing {
18442                                if let Some(partial_seq) = partial_ack {
18443                                    self.queue_reliable_partial_ack(
18444                                        src,
18445                                        frame.envelope.ty,
18446                                        partial_seq,
18447                                        called_from_queue,
18448                                    )?;
18449                                }
18450                                self.queue_reliable_packet_request(
18451                                    src,
18452                                    frame.envelope.ty,
18453                                    request_seq,
18454                                    called_from_queue,
18455                                )?;
18456                                return Ok(());
18457                            }
18458
18459                            if let Some(ack_seq) = last_delivered {
18460                                self.queue_reliable_ack(
18461                                    src,
18462                                    frame.envelope.ty,
18463                                    ack_seq,
18464                                    called_from_queue,
18465                                )?;
18466                            }
18467
18468                            released_buffered.extend(release.into_iter().skip(1));
18469                        }
18470                    }
18471                }
18472            } else {
18473                match packed_frame
18474                    .as_ref()
18475                    .expect("packed frame")
18476                    .as_ref()
18477                    .map_err(Clone::clone)
18478                {
18479                    Ok(frame) => {
18480                        if frame.ack_only() {
18481                            return Ok(());
18482                        }
18483                    }
18484                    Err(e) => {
18485                        if matches!(e, TelemetryError::Unpack(msg) if msg == "crc32 mismatch") {
18486                            return Ok(());
18487                        }
18488                        return Err(e);
18489                    }
18490                }
18491            }
18492        }
18493
18494        let duplicate = self.is_duplicate_pkt(&item.data)?;
18495        if duplicate
18496            && let Some(guard) = _local_admission_guard.as_ref()
18497        {
18498            guard.begin_commit(true)?;
18499        }
18500        #[cfg(feature = "discovery")]
18501        {
18502            // Reuse the already CRC-validated header. Ordinary telemetry must
18503            // not pay another full-frame CRC pass for the managed GET path.
18504            let ty = match &item.data {
18505                RouterItem::Packet(packet) => packet.data_type(),
18506                RouterItem::Packed(_) => packed_frame
18507                    .as_ref()
18508                    .expect("packed frame")
18509                    .as_ref()
18510                    .map_err(Clone::clone)?
18511                    .envelope
18512                    .ty,
18513            };
18514            self.forward_requested_managed_observation(item, duplicate, ty)?;
18515        }
18516        if duplicate {
18517            #[cfg(feature = "discovery")]
18518            let shared_pending = if item.src.is_some()
18519                // SDK protocols use reliable transport too, but their echoed
18520                // identities are not outstanding application-owner retries.
18521                // Re-forwarding catalogs here lets parallel bridges amplify
18522                // the same packet without bound.
18523                && Self::item_data_type(&item.data).is_ok_and(|ty| {
18524                    is_reliable_type(ty) && !is_internal_control_type(ty)
18525                })
18526                && self.item_target_senders(&item.data)?.is_empty()
18527            {
18528                let RemoteSidePlan::Target(sides) =
18529                    self.remote_side_plan_with_selection(&item.data, item.src, false)?;
18530                let st = self.state.lock();
18531                sides.into_iter().any(|side| {
18532                    self.side_has_multiple_announcers_locked(&st, side, self.clock.now_ms())
18533                })
18534            } else {
18535                false
18536            };
18537            #[cfg(not(feature = "discovery"))]
18538            let shared_pending = false;
18539            if shared_pending {
18540                self.relay_send(item.data.clone(), item.src, called_from_queue)?;
18541            }
18542
18543            // A final-ACK retry must cross a router again. Do not redeliver it
18544            // locally, and rate-limit repeated forwarding to avoid echo loops.
18545            if !shared_pending
18546                && ((item.src.is_some() && self.item_targets_another_sender(&item.data)?)
18547                    || (item.src.is_some() && self.should_route_remote(&item.data, item.src)? && {
18548                        let (endpoints, _) = self.item_route_info(&item.data)?;
18549                        self.item_targets_router_as_intermediate_hop(&item.data, &endpoints)?
18550                    }))
18551            {
18552                let id = Self::get_hash(&item.data);
18553                let replay = {
18554                    let mut st = self.state.lock();
18555                    st.reliable_return_routes.get_mut(&id).is_some_and(|route| {
18556                        let now = self.clock.now_ms();
18557                        if now.saturating_sub(route.last_replay_ms)
18558                            < runtime_reliable_retransmit_ms()
18559                        {
18560                            return false;
18561                        }
18562                        route.last_replay_ms = now;
18563                        true
18564                    })
18565                };
18566                if replay {
18567                    self.relay_send(item.data.clone(), item.src, called_from_queue)?;
18568                }
18569            }
18570            if item.src.is_some() {
18571                let local_sender = self.sender_arc();
18572                match &item.data {
18573                    RouterItem::Packet(pkt)
18574                        if (is_reliable_type(pkt.data_type())
18575                            || !pkt.wire_target_senders().is_empty())
18576                            && pkt.sender() != local_sender.as_ref()
18577                            && self.item_targets_local_sender(&item.data)?
18578                            && self.packet_has_local_delivery(pkt)
18579                            && !(self.should_route_remote(&item.data, item.src)?
18580                                && self.item_targets_router_as_intermediate_hop(
18581                                    &item.data,
18582                                    pkt.endpoints(),
18583                                )?) =>
18584                    {
18585                        self.queue_admitted_end_to_end_ack(
18586                            pkt.packet_id(),
18587                            pkt.sender(),
18588                            called_from_queue,
18589                            _local_admission_guard.as_ref(),
18590                        )?;
18591                    }
18592                    RouterItem::Packed(bytes) => {
18593                        if let Ok(pkt) = self.unpack_packet_for_role(bytes.as_ref())
18594                            && (is_reliable_type(pkt.data_type())
18595                                || !pkt.wire_target_senders().is_empty())
18596                            && pkt.sender() != local_sender.as_ref()
18597                            && self.item_targets_local_sender(&item.data)?
18598                            && self.packet_has_local_delivery(&pkt)
18599                            && !(self.should_route_remote(&item.data, item.src)?
18600                                && self.item_targets_router_as_intermediate_hop(
18601                                    &item.data,
18602                                    pkt.endpoints(),
18603                                )?)
18604                        {
18605                            let packet_id = _local_admission_guard
18606                                .as_ref()
18607                                .map(|guard| guard.packet_id)
18608                                .unwrap_or_else(|| {
18609                                    wire_format::packet_id_from_wire(bytes.as_ref())
18610                                        .unwrap_or_else(|_| pkt.packet_id())
18611                                });
18612                            self.queue_admitted_end_to_end_ack(
18613                                packet_id,
18614                                pkt.sender(),
18615                                called_from_queue,
18616                                _local_admission_guard.as_ref(),
18617                            )?;
18618                        }
18619                    }
18620                    _ => {}
18621                }
18622            }
18623            return Ok(());
18624        }
18625
18626        self.dispatch_rx_data(item, called_from_queue, _local_admission_guard.as_ref())?;
18627        if let Some(guard) = _local_admission_guard.as_ref() {
18628            guard.end_commit();
18629        }
18630
18631        for release_bytes in released_buffered {
18632            let release_data = RouterItem::Packed(release_bytes.clone());
18633            if let Some(guard) = _local_admission_guard.as_ref() {
18634                guard.check()?;
18635            }
18636            let release_id = Self::get_hash(&release_data);
18637            let release_type = wire_format::scheduling_type(release_bytes.as_ref())
18638                .ok_or(TelemetryError::InvalidType)?;
18639            let release_guard = if !is_internal_control_type(release_type)
18640                && let Some(src) = item.src
18641            {
18642                self.begin_reliable_local_admission_with_parent(
18643                    src,
18644                    release_id,
18645                    _local_admission_guard.as_ref(),
18646                )?
18647            } else {
18648                None
18649            };
18650            if self.is_duplicate_packet_id(release_id)? {
18651                continue;
18652            }
18653            let release_item = RouterRxItem {
18654                src: item.src,
18655                priority: Self::router_item_priority(&release_data)?,
18656                data: release_data,
18657            };
18658            self.dispatch_rx_data(&release_item, called_from_queue, release_guard.as_ref())?;
18659        }
18660
18661        Ok(())
18662    }
18663
18664    fn dispatch_rx_data(
18665        &self,
18666        item: &RouterRxItem,
18667        called_from_queue: bool,
18668        admission: Option<&ReliableLocalAdmissionGuard<'_>>,
18669    ) -> TelemetryResult<()> {
18670        match &item.data {
18671            RouterItem::Packet(pkt) => {
18672                pkt.validate()?;
18673                #[cfg(feature = "discovery")]
18674                if let Some(side) = item.src {
18675                    self.refresh_peer_liveness(side, sender_address_u32(pkt.sender()));
18676                }
18677
18678                if self.handle_internal_reliable_packet(pkt, item.src, called_from_queue)? {
18679                    return Ok(());
18680                }
18681
18682                if pkt.data_type() == DataType::P2pMessage {
18683                    if self.item_targets_local_sender(&item.data)? {
18684                        self.dispatch_p2p_packet(pkt)?;
18685                        if item.src.is_some() {
18686                            self.queue_admitted_end_to_end_ack(
18687                                pkt.packet_id(),
18688                                pkt.sender(),
18689                                called_from_queue,
18690                                admission,
18691                            )?;
18692                        }
18693                    }
18694                    if self.should_route_remote(&item.data, item.src)? {
18695                        self.relay_send(
18696                            RouterItem::Packet(pkt.to_owned()),
18697                            item.src,
18698                            called_from_queue,
18699                        )?;
18700                    }
18701                    return Ok(());
18702                }
18703
18704                #[cfg(feature = "timesync")]
18705                if matches!(
18706                    pkt.data_type(),
18707                    DataType::TimeSyncAnnounce
18708                        | DataType::TimeSyncRequest
18709                        | DataType::TimeSyncResponse
18710                ) {
18711                    self.handle_internal_timesync_packet(pkt, item.src, called_from_queue)?;
18712                    return Ok(());
18713                }
18714
18715                if self.learn_discovery_packet(pkt, item.src, called_from_queue)? {
18716                    // Reachability advertisements are hop-local. Learning one
18717                    // schedules this Router's own aggregated, split-horizon
18718                    // snapshot; transparently relaying the original packet
18719                    // would make a distant sender look physically adjacent.
18720                    if !discovery_is_hop_local_advertisement(
18721                        pkt.data_type(),
18722                        !self.cfg.role.local_services(),
18723                    ) && self.should_route_remote(&item.data, item.src)?
18724                    {
18725                        self.relay_send(
18726                            RouterItem::Packet(pkt.to_owned()),
18727                            item.src,
18728                            called_from_queue,
18729                        )?;
18730                    }
18731                    return Ok(());
18732                }
18733
18734                let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
18735                crate::small_sort::sort(&mut eps);
18736                eps.dedup();
18737                let had_local_handler = eps.iter().copied().any(|ep| {
18738                    self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18739                });
18740                let had_local_delivery =
18741                    had_local_handler || self.is_managed_variable_type(pkt.data_type());
18742
18743                let has_remote = self.should_route_remote(&item.data, item.src)?;
18744                let targets_local = self.item_targets_local_sender(&item.data)?;
18745
18746                let has_packed_local = eps
18747                    .iter()
18748                    .copied()
18749                    .any(|ep| self.endpoint_has_packed_handler(ep));
18750                let bytes_opt = if has_packed_local {
18751                    Some(wire_format::pack_packet(pkt))
18752                } else {
18753                    None
18754                };
18755
18756                let mut local_delivery_error = None;
18757                if targets_local {
18758                    for dest in eps {
18759                        for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
18760                            let result = match (&h.handler, &bytes_opt) {
18761                                (EndpointHandlerFn::Packed(_), Some(bytes)) => self
18762                                    .call_handler_with_retries(
18763                                        dest,
18764                                        h,
18765                                        Some(bytes.as_ref()),
18766                                        Some(pkt),
18767                                        None,
18768                                        called_from_queue,
18769                                    ),
18770                                (EndpointHandlerFn::Packed(_), None) => {
18771                                    let bytes = wire_format::pack_packet(pkt);
18772                                    self.call_handler_with_retries(
18773                                        dest,
18774                                        h,
18775                                        Some(bytes.as_ref()),
18776                                        Some(pkt),
18777                                        None,
18778                                        called_from_queue,
18779                                    )
18780                                }
18781                                (EndpointHandlerFn::Packet(_), _) => self
18782                                    .call_handler_with_retries(
18783                                        dest,
18784                                        h,
18785                                        None,
18786                                        Some(pkt),
18787                                        None,
18788                                        called_from_queue,
18789                                    ),
18790                            };
18791                            if let Err(error) = result {
18792                                if let Some(src) = item.src {
18793                                    self.note_side_local_handler_failure(
18794                                        src,
18795                                        pkt.data_type(),
18796                                        runtime_max_handler_retries(),
18797                                    );
18798                                }
18799                                local_delivery_error.get_or_insert(error);
18800                            }
18801                        }
18802                    }
18803                }
18804
18805                if let Some(guard) = admission {
18806                    guard.begin_commit(local_delivery_error.is_none())?;
18807                }
18808
18809                if let Some(src) = item.src
18810                    && had_local_delivery
18811                    && local_delivery_error.is_none()
18812                    && targets_local
18813                {
18814                    self.note_side_local_delivery(src, pkt.data_type());
18815                }
18816
18817                if item.src.is_some()
18818                    && had_local_delivery
18819                    && local_delivery_error.is_none()
18820                    && targets_local
18821                    && (is_reliable_type(pkt.data_type()) || !pkt.wire_target_senders().is_empty())
18822                    && !(has_remote
18823                        && self
18824                            .item_targets_router_as_intermediate_hop(&item.data, pkt.endpoints())?)
18825                {
18826                    self.queue_admitted_end_to_end_ack(
18827                        pkt.packet_id(),
18828                        pkt.sender(),
18829                        called_from_queue,
18830                        admission,
18831                    )?;
18832                }
18833
18834                if let Some(guard) = admission {
18835                    guard.check()?;
18836                }
18837                if has_remote {
18838                    let relay_item = RouterItem::Packet(pkt.to_owned());
18839                    self.relay_send(relay_item, item.src, called_from_queue)?;
18840                }
18841
18842                // Do not ACK or retain successful-delivery dedupe after a
18843                // refused local handoff. Other destinations still receive their
18844                // forwarding attempt; the origin can retry this same identity.
18845                local_delivery_error.map_or(Ok(()), Err)
18846            }
18847            RouterItem::Packed(bytes) => {
18848                // Pure best-effort transit needs routing metadata, not an owned
18849                // sender string or endpoint array. Commands, local handlers,
18850                // managed variables and internal protocols retain their normal path.
18851                let routing = wire_format::peek_routing_envelope(bytes.as_ref())?;
18852                if !is_internal_control_type(routing.ty)
18853                    && !routing.is_reliable
18854                    && routing.target_senders().len() == 0
18855                    && !self.is_managed_variable_type(routing.ty)
18856                    && !routing.endpoints().any(|ep| {
18857                        self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18858                    })
18859                {
18860                    // relay_send already resolves the eligible sides, including
18861                    // an empty plan. A preliminary selection duplicates scratch
18862                    // allocations and advances weighted cursors twice.
18863                    return self.relay_send(item.data.clone(), item.src, called_from_queue);
18864                }
18865
18866                let env = wire_format::peek_envelope(bytes.as_ref())?;
18867
18868                if matches!(
18869                    env.ty,
18870                    DataType::ReliableAck
18871                        | DataType::ReliablePartialAck
18872                        | DataType::ReliablePacketRequest
18873                ) {
18874                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18875                    pkt.validate()?;
18876                    let handled =
18877                        self.handle_internal_reliable_packet(&pkt, item.src, called_from_queue)?;
18878                    // Hop-level ACKs terminate here. End-to-end ACKs for a packet originated
18879                    // farther upstream continue along the learned packet return route.
18880                    if !handled && self.should_route_remote(&item.data, item.src)? {
18881                        self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18882                    }
18883                    return Ok(());
18884                }
18885
18886                if env.ty == DataType::P2pMessage {
18887                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18888                    pkt.validate()?;
18889                    if self.item_targets_local_sender(&item.data)? {
18890                        self.dispatch_p2p_packet(&pkt)?;
18891                        if item.src.is_some() {
18892                            let packet_id = wire_format::packet_id_from_wire(bytes.as_ref())
18893                                .unwrap_or_else(|_| pkt.packet_id());
18894                            self.queue_end_to_end_reliable_ack_for_packet_id(
18895                                packet_id,
18896                                pkt.sender(),
18897                                called_from_queue,
18898                            )?;
18899                        }
18900                    }
18901                    if self.should_route_remote(&item.data, item.src)? {
18902                        self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18903                    }
18904                    return Ok(());
18905                }
18906
18907                #[cfg(feature = "timesync")]
18908                if matches!(
18909                    env.ty,
18910                    DataType::TimeSyncAnnounce
18911                        | DataType::TimeSyncRequest
18912                        | DataType::TimeSyncResponse
18913                ) {
18914                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18915                    pkt.validate()?;
18916                    self.handle_internal_timesync_packet(&pkt, item.src, called_from_queue)?;
18917                    return Ok(());
18918                }
18919
18920                #[cfg(feature = "discovery")]
18921                if discovery::is_discovery_type(env.ty) {
18922                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18923                    pkt.validate()?;
18924                    let _ = self.learn_discovery_packet(&pkt, item.src, called_from_queue)?;
18925                    if !discovery_is_hop_local_advertisement(
18926                        env.ty,
18927                        !self.cfg.role.local_services(),
18928                    ) && self.should_route_remote(&item.data, item.src)?
18929                    {
18930                        self.relay_send(RouterItem::Packet(pkt), item.src, called_from_queue)?;
18931                    }
18932                    return Ok(());
18933                }
18934
18935                let any_packet_needed = env
18936                    .endpoints
18937                    .iter()
18938                    .copied()
18939                    .any(|ep| self.endpoint_has_packet_handler(ep));
18940
18941                let mut pkt_opt = if any_packet_needed {
18942                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18943                    pkt.validate()?;
18944                    Some(pkt)
18945                } else {
18946                    None
18947                };
18948
18949                let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
18950                crate::small_sort::sort(&mut eps);
18951                eps.dedup();
18952                let had_local_handler = eps.iter().copied().any(|ep| {
18953                    self.endpoint_has_packet_handler(ep) || self.endpoint_has_packed_handler(ep)
18954                });
18955                let had_local_delivery = had_local_handler || self.is_managed_variable_type(env.ty);
18956
18957                let has_remote = self.should_route_remote(&item.data, item.src)?;
18958                let targets_local = self.item_targets_local_sender(&item.data)?;
18959
18960                let mut local_delivery_error = None;
18961                if targets_local {
18962                    for dest in eps {
18963                        for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
18964                            let result = match &h.handler {
18965                                EndpointHandlerFn::Packed(_) => self.call_handler_with_retries(
18966                                    dest,
18967                                    h,
18968                                    Some(bytes.as_ref()),
18969                                    pkt_opt.as_ref(),
18970                                    Some(&env),
18971                                    called_from_queue,
18972                                ),
18973                                EndpointHandlerFn::Packet(_) => {
18974                                    if pkt_opt.is_none() {
18975                                        let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
18976                                        pkt.validate()?;
18977                                        pkt_opt = Some(pkt);
18978                                    }
18979                                    let pkt_ref = pkt_opt.as_ref().expect("just set");
18980                                    self.call_handler_with_retries(
18981                                        dest,
18982                                        h,
18983                                        None,
18984                                        Some(pkt_ref),
18985                                        Some(&env),
18986                                        called_from_queue,
18987                                    )
18988                                }
18989                            };
18990                            if let Err(error) = result {
18991                                if let Some(src) = item.src {
18992                                    self.note_side_local_handler_failure(
18993                                        src,
18994                                        env.ty,
18995                                        runtime_max_handler_retries(),
18996                                    );
18997                                }
18998                                local_delivery_error.get_or_insert(error);
18999                            }
19000                        }
19001                    }
19002                }
19003
19004                if let Some(guard) = admission {
19005                    guard.begin_commit(local_delivery_error.is_none())?;
19006                }
19007
19008                if item.src.is_some()
19009                    && had_local_delivery
19010                    && local_delivery_error.is_none()
19011                    && targets_local
19012                    && (is_reliable_type(env.ty) || !env.target_senders.is_empty())
19013                    && !(has_remote
19014                        && self
19015                            .item_targets_router_as_intermediate_hop(&item.data, &env.endpoints)?)
19016                {
19017                    let packet_id = if let Some(guard) = admission {
19018                        guard.packet_id
19019                    } else {
19020                        wire_format::packet_id_from_wire(bytes.as_ref()).or_else(|_| {
19021                            pkt_opt
19022                                .as_ref()
19023                                .map(Packet::packet_id)
19024                                .ok_or(TelemetryError::Unpack("missing packet id"))
19025                        })?
19026                    };
19027                    self.queue_admitted_end_to_end_ack(
19028                        packet_id,
19029                        &env.sender,
19030                        called_from_queue,
19031                        admission,
19032                    )?;
19033                }
19034
19035                if let Some(guard) = admission {
19036                    guard.check()?;
19037                }
19038                if has_remote {
19039                    let relay_item = match pkt_opt {
19040                        Some(ref p) => RouterItem::Packet(p.clone()),
19041                        None => RouterItem::Packed(bytes.clone()),
19042                    };
19043                    self.relay_send(relay_item, item.src, called_from_queue)?;
19044                }
19045
19046                // Do not ACK or retain successful-delivery dedupe after a
19047                // refused local handoff. Other destinations still receive their
19048                // forwarding attempt; the origin can retry this same identity.
19049                local_delivery_error.map_or(Ok(()), Err)
19050            }
19051        }
19052    }
19053
19054    fn dispatch_local_for_item(
19055        &self,
19056        item: &RouterItem,
19057        called_from_queue: bool,
19058    ) -> TelemetryResult<()> {
19059        match item {
19060            RouterItem::Packet(pkt) => {
19061                pkt.validate()?;
19062                if is_internal_control_type(pkt.data_type()) {
19063                    return Ok(());
19064                }
19065                self.ensure_e2e_policy_supported_for_type(pkt.data_type())?;
19066                if !self.item_targets_local_sender(item)? {
19067                    return Ok(());
19068                }
19069
19070                let mut eps: Vec<DataEndpoint> = pkt.endpoints().to_vec();
19071                crate::small_sort::sort(&mut eps);
19072                eps.dedup();
19073
19074                let has_packed_local = eps
19075                    .iter()
19076                    .copied()
19077                    .any(|ep| self.endpoint_has_packed_handler(ep));
19078                let bytes_opt = if has_packed_local {
19079                    Some(wire_format::pack_packet(pkt))
19080                } else {
19081                    None
19082                };
19083
19084                for dest in eps {
19085                    for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
19086                        match (&h.handler, &bytes_opt) {
19087                            (EndpointHandlerFn::Packed(_), Some(bytes)) => {
19088                                self.call_handler_with_retries(
19089                                    dest,
19090                                    h,
19091                                    Some(bytes.as_ref()),
19092                                    Some(pkt),
19093                                    None,
19094                                    called_from_queue,
19095                                )?;
19096                            }
19097                            (EndpointHandlerFn::Packed(_), None) => {
19098                                let bytes = wire_format::pack_packet(pkt);
19099                                self.call_handler_with_retries(
19100                                    dest,
19101                                    h,
19102                                    Some(bytes.as_ref()),
19103                                    Some(pkt),
19104                                    None,
19105                                    called_from_queue,
19106                                )?;
19107                            }
19108                            (EndpointHandlerFn::Packet(_), _) => {
19109                                self.call_handler_with_retries(
19110                                    dest,
19111                                    h,
19112                                    None,
19113                                    Some(pkt),
19114                                    None,
19115                                    called_from_queue,
19116                                )?;
19117                            }
19118                        }
19119                    }
19120                }
19121            }
19122            RouterItem::Packed(bytes) => {
19123                let env = wire_format::peek_envelope(bytes.as_ref())?;
19124                if is_internal_control_type(env.ty) {
19125                    return Ok(());
19126                }
19127                self.ensure_e2e_policy_supported_for_type(env.ty)?;
19128                if !self.item_targets_local_sender(item)? {
19129                    return Ok(());
19130                }
19131
19132                let any_packet_needed = env
19133                    .endpoints
19134                    .iter()
19135                    .copied()
19136                    .any(|ep| self.endpoint_has_packet_handler(ep));
19137
19138                let mut pkt_opt = if any_packet_needed {
19139                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
19140                    pkt.validate()?;
19141                    Some(pkt)
19142                } else {
19143                    None
19144                };
19145
19146                let mut eps: Vec<DataEndpoint> = env.endpoints.iter().copied().collect();
19147                crate::small_sort::sort(&mut eps);
19148                eps.dedup();
19149
19150                for dest in eps {
19151                    for h in self.cfg.handlers().iter().filter(|h| h.endpoint == dest) {
19152                        match &h.handler {
19153                            EndpointHandlerFn::Packed(_) => {
19154                                self.call_handler_with_retries(
19155                                    dest,
19156                                    h,
19157                                    Some(bytes.as_ref()),
19158                                    pkt_opt.as_ref(),
19159                                    Some(&env),
19160                                    called_from_queue,
19161                                )?;
19162                            }
19163                            EndpointHandlerFn::Packet(_) => {
19164                                if pkt_opt.is_none() {
19165                                    let pkt = self.unpack_packet_for_role(bytes.as_ref())?;
19166                                    pkt.validate()?;
19167                                    pkt_opt = Some(pkt);
19168                                }
19169                                let pkt_ref = pkt_opt.as_ref().expect("just set");
19170                                self.call_handler_with_retries(
19171                                    dest,
19172                                    h,
19173                                    None,
19174                                    Some(pkt_ref),
19175                                    Some(&env),
19176                                    called_from_queue,
19177                                )?;
19178                            }
19179                        }
19180                    }
19181                }
19182            }
19183        }
19184
19185        Ok(())
19186    }
19187
19188    /// Internal TX implementation used by `tx*()`, `tx_queue*()`, and relay-mode rebroadcast.
19189    ///
19190    /// - Broadcast items are sent to all sides when remote forwarding is required.
19191    /// - ToSide items are sent only to the specified side.
19192    /// - If `ignore_local` is false, local handlers are invoked once.
19193    fn tx_item_impl(
19194        &self,
19195        item: RouterTxItem,
19196        ignore_local: bool,
19197        called_from_queue: bool,
19198    ) -> TelemetryResult<()> {
19199        self.tx_item_impl_with_retry(
19200            item,
19201            ignore_local,
19202            called_from_queue,
19203            &mut DispatchContext::default(),
19204        )
19205    }
19206
19207    fn tx_item_impl_with_retry(
19208        &self,
19209        item: RouterTxItem,
19210        ignore_local: bool,
19211        called_from_queue: bool,
19212        context: &mut DispatchContext,
19213    ) -> TelemetryResult<()> {
19214        #[cfg(feature = "std")]
19215        let _clear_retry_intent = self.retry_queued_intent(context.claim);
19216        match item {
19217            RouterTxItem::Broadcast(data) => {
19218                let claim = context.claim;
19219                let deferred_since = context.since;
19220                let ty = Self::item_data_type(&data)?;
19221
19222                if let Some(since) = deferred_since
19223                    && let Err(error) = self.expire_deferred_original(
19224                        ty,
19225                        since,
19226                        claim.is_some_and(|claim| claim.ordered),
19227                    )
19228                {
19229                    context.mark_terminal();
19230                    return Err(error);
19231                }
19232                if !called_from_queue
19233                    && !ignore_local
19234                    && matches!(reliable_mode(ty), crate::ReliableMode::Ordered)
19235                    && self
19236                        .state
19237                        .lock()
19238                        .transmit_queue
19239                        .iter()
19240                        .any(|queued| RouterInner::ordered_original_type(queued) == Some(ty))
19241                {
19242                    return self.tx_queue_item(RouterTxItem::Broadcast(data));
19243                }
19244
19245                self.ensure_e2e_policy_supported_for_type(ty)?;
19246                if let RouterItem::Packet(pkt) = &data
19247                    && !is_internal_control_type(pkt.data_type())
19248                {
19249                    self.remember_managed_variable_packet(pkt)?;
19250                }
19251                #[cfg(feature = "discovery")]
19252                let is_discovery = discovery::is_discovery_type(ty);
19253                if !ignore_local {
19254                    if self.is_duplicate_pkt(&data)? {
19255                        return Ok(());
19256                    }
19257                    #[cfg(feature = "discovery")]
19258                    if !is_discovery
19259                        && !is_internal_control_type(ty)
19260                        && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
19261                    {
19262                        // Never repeat a partially applied local side effect.
19263                        if claim.is_some() {
19264                            context.mark_terminal();
19265                            return Err(TelemetryError::HandlerError(
19266                                "accepted local delivery failed",
19267                            ));
19268                        }
19269                        return Err(error);
19270                    }
19271                    #[cfg(not(feature = "discovery"))]
19272                    if !is_internal_control_type(ty)
19273                        && let Err(error) = self.dispatch_local_for_item(&data, called_from_queue)
19274                    {
19275                        // Never repeat a partially applied local side effect.
19276                        if claim.is_some() {
19277                            context.mark_terminal();
19278                            return Err(TelemetryError::HandlerError(
19279                                "accepted local delivery failed",
19280                            ));
19281                        }
19282                        return Err(error);
19283                    }
19284                }
19285
19286                if !ignore_local && !self.mark_claim_local_delivered(claim) {
19287                    // An explicit queue clear cancelled the publication during its callback.
19288                    return Ok(());
19289                }
19290
19291                if !self.claim_schema_is_current(ty, claim)? {
19292                    return Ok(());
19293                }
19294
19295                match &data {
19296                    RouterItem::Packet(packet) => packet.validate()?,
19297                    // item_data_type already validated the same immutable packed bytes.
19298                    RouterItem::Packed(_) => {}
19299                }
19300                // Skipping local delivery does not waive a remote receipt
19301                // contract: a queued credit retry must still be registered.
19302                let freezes_contract = !is_internal_control_type(ty)
19303                    && self.item_requires_end_to_end_tracking(&data, ty)?;
19304                // The preliminary reachability check used to select a route,
19305                // allocate scratch, and advance weighted cursors, then repeat
19306                // all of it for dispatch. Reuse this per-dispatch plan when no
19307                // delivery contract can change its destination set. It is not
19308                // a persistent topology/schema cache.
19309                #[cfg(feature = "discovery")]
19310                let cached_plan = if freezes_contract {
19311                    None
19312                } else {
19313                    Some(self.remote_side_plan(&data, None)?)
19314                };
19315                #[cfg(feature = "discovery")]
19316                let send_remote = match &cached_plan {
19317                    Some(RemoteSidePlan::Target(sides)) => !sides.is_empty(),
19318                    None => self.should_route_remote(&data, None)?,
19319                };
19320                #[cfg(not(feature = "discovery"))]
19321                let send_remote = self.should_route_remote(&data, None)?;
19322
19323                if !send_remote {
19324                    let (endpoints, ty) = self.item_route_info(&data)?;
19325                    if !is_internal_control_type(ty)
19326                        && is_reliable_type(ty)
19327                        && has_nonlocal_endpoint(&endpoints, &self.cfg)
19328                        && !self.state.lock().is_managed_variable(ty.as_u32())
19329                    {
19330                        // Nothing was queued or transmitted. Reporting success
19331                        // here loses application state during route expiry.
19332                        // Permit retry of the same packet after discovery heals.
19333                        self.remove_pkt_id(&data);
19334                        if called_from_queue {
19335                            return self.defer_accepted_original(context);
19336                        }
19337                        return Err(TelemetryError::Io("no discovered route"));
19338                    }
19339                    return Ok(());
19340                }
19341                #[cfg(feature = "discovery")]
19342                let mut data = data;
19343                if freezes_contract {
19344                    #[cfg(feature = "discovery")]
19345                    {
19346                        let pending = {
19347                            let st = self.state.lock();
19348                            let mut pending = self.expected_end_to_end_destinations_locked(
19349                                &st, &data, None, false, false,
19350                            )?;
19351                            self.filter_trackable_end_to_end_destinations_locked(
19352                                &st,
19353                                ty,
19354                                &mut pending,
19355                                false,
19356                            );
19357                            pending
19358                        };
19359                        if !pending.is_empty() {
19360                            // RouterMap keys already form a sorted, unique frozen cohort.
19361                            let targets: Vec<u64> = pending.keys().copied().collect();
19362                            data = self.attach_wire_contract_to_item(data, &targets)?;
19363                            match self.register_end_to_end_reliable_tx(&data, Some(pending)) {
19364                                Ok(ready) => {
19365                                    context.mark_owner_transferred();
19366                                    if !ready {
19367                                        return Ok(());
19368                                    }
19369                                }
19370                                Err(error) => {
19371                                    // Freezing destinations preserves the logical packet ID.
19372                                    // Clear its dedupe mark without retaining/cloning the old item.
19373                                    self.remove_pkt_id(&data);
19374                                    if called_from_queue
19375                                        && matches!(
19376                                            error,
19377                                            TelemetryError::Io(
19378                                                "reliable window full" | "memory pressure"
19379                                            )
19380                                        )
19381                                    {
19382                                        // Acceptance transferred ownership to this queue. A full
19383                                        // receipt window is temporary credit pressure, not loss.
19384                                        // Retain the frozen identity and skip local delivery on retry.
19385                                        if let Some(claim) = claim {
19386                                            self.replace_claim_payload(claim, data)?;
19387                                        }
19388                                        self.defer_accepted_original(context)?;
19389                                        return Ok(());
19390                                    }
19391                                    return Err(error);
19392                                }
19393                            }
19394                        }
19395                    }
19396                }
19397                #[cfg(feature = "discovery")]
19398                let plan = match cached_plan {
19399                    Some(plan) => plan,
19400                    None => self.remote_side_plan(&data, None)?,
19401                };
19402                #[cfg(not(feature = "discovery"))]
19403                let plan = self.remote_side_plan(&data, None)?;
19404                let RemoteSidePlan::Target(sides) = plan;
19405                let e2e_parent = context.owner_transferred().then(|| Self::get_hash(&data));
19406                #[cfg(feature = "std")]
19407                let _retry_intent = self.retry_intent(e2e_parent, false);
19408                let _parent_lease = self.lease_e2e_send(e2e_parent);
19409                for side in sides.iter().copied() {
19410                    #[cfg(feature = "latest-value-egress")]
19411                    if self.queue_latest_egress(side, None, &data)? {
19412                        continue;
19413                    }
19414                    #[cfg(feature = "compression")]
19415                    let mut side_data = data.clone();
19416                    #[cfg(feature = "compression")]
19417                    let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
19418                        side,
19419                        &mut side_data,
19420                        false,
19421                        &mut context.admitted,
19422                    );
19423                    #[cfg(not(feature = "compression"))]
19424                    let send_result = self.send_reliable_to_side_with_progress(
19425                        side,
19426                        data.clone(),
19427                        false,
19428                        &mut context.admitted,
19429                    );
19430                    if let Err(e) = send_result {
19431                        if Self::is_transient_tx_error(&e) {
19432                            context.mark_refused();
19433                            if let Some(parent) = e2e_parent {
19434                                self.enqueue_e2e_child(
19435                                    RouterTxItem::ToSide {
19436                                        src: None,
19437                                        dst: side,
19438                                        #[cfg(feature = "compression")]
19439                                        data: side_data,
19440                                        #[cfg(not(feature = "compression"))]
19441                                        data: data.clone(),
19442                                    },
19443                                    Self::router_item_priority(&data)?,
19444                                    parent,
19445                                    #[cfg(feature = "compression")]
19446                                    &mut reservation,
19447                                )?;
19448                            } else {
19449                                #[cfg(feature = "compression")]
19450                                self.retry_codec_owner(
19451                                    RouterTxItem::ToSide {
19452                                        src: None,
19453                                        dst: side,
19454                                        data: side_data,
19455                                    },
19456                                    true,
19457                                    &mut reservation,
19458                                )?;
19459                                #[cfg(not(feature = "compression"))]
19460                                self.tx_queue_item_for_retry(
19461                                    RouterTxItem::ToSide {
19462                                        src: None,
19463                                        dst: side,
19464                                        data: data.clone(),
19465                                    },
19466                                    true,
19467                                )?;
19468                            }
19469                            continue;
19470                        }
19471                        match &data {
19472                            RouterItem::Packet(pkt) => {
19473                                let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19474                            }
19475                            RouterItem::Packed(bytes) => {
19476                                if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19477                                    let _ = self.handle_callback_error_from_env(
19478                                        &env,
19479                                        None,
19480                                        e,
19481                                        called_from_queue,
19482                                    );
19483                                }
19484                            }
19485                        }
19486                        return Err(TelemetryError::HandlerError("tx handler failed"));
19487                    }
19488                }
19489            }
19490            RouterTxItem::ToSide { src, dst, data } => {
19491                let egress_service = context.egress_service();
19492                #[cfg(feature = "compression")]
19493                let mut data = data;
19494                let ty = Self::item_data_type(&data)?;
19495                self.ensure_e2e_policy_supported_for_type(ty)?;
19496                if let RouterItem::Packet(pkt) = &data
19497                    && !is_internal_control_type(pkt.data_type())
19498                {
19499                    self.remember_managed_variable_packet(pkt)?;
19500                }
19501                if !ignore_local {
19502                    if self.is_duplicate_pkt(&data)? {
19503                        return Ok(());
19504                    }
19505                    let suppress_local = is_internal_control_type(ty);
19506                    if !suppress_local {
19507                        self.dispatch_local_for_item(&data, called_from_queue)?;
19508                    }
19509                }
19510                let allowed = {
19511                    let st = self.state.lock();
19512                    let ty = Some(ty);
19513                    let route_allowed = self.route_allowed_locked(&st, src, ty, dst);
19514                    #[cfg(all(feature = "discovery", feature = "timesync"))]
19515                    let timesync_allowed = ty
19516                        .map(|ty| {
19517                            Self::timesync_allowed_for_side_locked(
19518                                &st,
19519                                dst,
19520                                ty,
19521                                self.clock.now_ms(),
19522                            )
19523                        })
19524                        .unwrap_or(true);
19525                    #[cfg(not(all(feature = "discovery", feature = "timesync")))]
19526                    let timesync_allowed = true;
19527                    route_allowed && timesync_allowed
19528                };
19529                if !allowed {
19530                    #[cfg(feature = "discovery")]
19531                    if ty == DataType::DiscoverySchema {
19532                        RouterInner::schema_response_owner_lost(
19533                            &mut self.state.lock().discovery_side_throttle,
19534                            Some((dst, Self::get_hash(&data))),
19535                        );
19536                    }
19537                    return Ok(());
19538                }
19539                #[cfg(feature = "compression")]
19540                let (send_result, mut reservation) =
19541                    if let Some(claim) = context.claim.filter(|claim| claim.side_retry) {
19542                        let prepared = self.prepare_e2e_child_for_send(claim, dst, &mut data);
19543                        (
19544                            prepared.and_then(|()| {
19545                                self.send_reliable_to_side_with_service(
19546                                    dst,
19547                                    data.clone(),
19548                                    src.is_some(),
19549                                    &mut context.admitted,
19550                                    egress_service,
19551                                )
19552                            }),
19553                            None,
19554                        )
19555                    } else {
19556                        self.send_queued_to_side_with_service(
19557                            dst,
19558                            &mut data,
19559                            src.is_some(),
19560                            &mut context.admitted,
19561                            egress_service,
19562                        )
19563                    };
19564                #[cfg(not(feature = "compression"))]
19565                let send_result = self.send_reliable_to_side_with_service(
19566                    dst,
19567                    data.clone(),
19568                    src.is_some(),
19569                    &mut context.admitted,
19570                    egress_service,
19571                );
19572                if let Err(e) = send_result {
19573                    if Self::is_transient_tx_error(&e) {
19574                        context.mark_refused();
19575                        if let Some(claim) = context.claim.filter(|claim| claim.side_retry) {
19576                            self.restore_e2e_child_claim(claim)?;
19577                            return Ok(());
19578                        }
19579                        let retry_priority =
19580                            context.priority.max(self.item_service_priority(&data)?);
19581                        let (turn_flags, immediate) = self.current_egress_requeue(context);
19582                        #[cfg(feature = "compression")]
19583                        self.retry_codec_owner_at_priority(
19584                            RouterTxItem::ToSide { src, dst, data },
19585                            true,
19586                            &mut reservation,
19587                            retry_priority,
19588                            turn_flags,
19589                            immediate,
19590                        )?;
19591                        #[cfg(not(feature = "compression"))]
19592                        self.tx_queue_item_for_retry_at_priority_with_turn(
19593                            RouterTxItem::ToSide { src, dst, data },
19594                            true,
19595                            retry_priority,
19596                            turn_flags,
19597                            immediate,
19598                        )?;
19599                        return Ok(());
19600                    }
19601                    if !self.cfg.role.local_services() {
19602                        return Err(e);
19603                    }
19604                    match &data {
19605                        RouterItem::Packet(pkt) => {
19606                            let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19607                        }
19608                        RouterItem::Packed(bytes) => {
19609                            if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19610                                let _ = self.handle_callback_error_from_env(
19611                                    &env,
19612                                    None,
19613                                    e,
19614                                    called_from_queue,
19615                                );
19616                            }
19617                        }
19618                    }
19619                    return Err(TelemetryError::HandlerError("tx handler failed"));
19620                }
19621            }
19622            RouterTxItem::EndToEndReplay { packet_id } => {
19623                let Some((data, mut sides)) = self.end_to_end_retransmit_sides(packet_id) else {
19624                    return Ok(());
19625                };
19626                if sides.is_empty() {
19627                    let RemoteSidePlan::Target(fallback_sides) =
19628                        self.remote_side_plan(&data, None)?;
19629                    sides = fallback_sides;
19630                }
19631                #[cfg(feature = "std")]
19632                let _retry_intent = self.retry_intent(Some(packet_id), true);
19633                let _parent_lease = self.lease_e2e_send(Some(packet_id));
19634                {
19635                    let st = self.state.lock();
19636                    sides.retain(|side| {
19637                        !Self::e2e_child_owned(&st.transmit_queue, *side, packet_id)
19638                    });
19639                }
19640                for side in sides.iter().copied() {
19641                    #[cfg(feature = "compression")]
19642                    let mut side_data = data.clone();
19643                    #[cfg(feature = "compression")]
19644                    let (send_result, mut reservation) = self.send_queued_to_side_with_progress(
19645                        side,
19646                        &mut side_data,
19647                        false,
19648                        &mut context.admitted,
19649                    );
19650                    #[cfg(not(feature = "compression"))]
19651                    let send_result = self.send_reliable_to_side_with_progress(
19652                        side,
19653                        data.clone(),
19654                        false,
19655                        &mut context.admitted,
19656                    );
19657                    if let Err(e) = send_result {
19658                        if Self::is_transient_tx_error(&e) {
19659                            context.mark_refused();
19660                            let retry_priority =
19661                                Self::router_item_priority_bumped(DataType::ReliableAck);
19662                            self.enqueue_e2e_child(
19663                                RouterTxItem::ToSide {
19664                                    src: None,
19665                                    dst: side,
19666                                    #[cfg(feature = "compression")]
19667                                    data: side_data,
19668                                    #[cfg(not(feature = "compression"))]
19669                                    data: data.clone(),
19670                                },
19671                                retry_priority,
19672                                packet_id,
19673                                #[cfg(feature = "compression")]
19674                                &mut reservation,
19675                            )?;
19676                            continue;
19677                        }
19678                        match &data {
19679                            RouterItem::Packet(pkt) => {
19680                                let _ = self.handle_callback_error(pkt, None, e, called_from_queue);
19681                            }
19682                            RouterItem::Packed(bytes) => {
19683                                if let Ok(env) = wire_format::peek_envelope(bytes.as_ref()) {
19684                                    let _ = self.handle_callback_error_from_env(
19685                                        &env,
19686                                        None,
19687                                        e,
19688                                        called_from_queue,
19689                                    );
19690                                }
19691                            }
19692                        }
19693                        return Err(TelemetryError::HandlerError("tx handler failed"));
19694                    }
19695                }
19696            }
19697            RouterTxItem::ReliableReplay {
19698                dst,
19699                ty,
19700                seq,
19701                bytes,
19702            } => {
19703                {
19704                    let mut st = self.state.lock();
19705                    let Some(sent) = st
19706                        .reliable_tx
19707                        .get_mut(&Self::reliable_key(dst, ty))
19708                        .and_then(|tx| tx.sent.get_mut(&seq))
19709                    else {
19710                        return Ok(());
19711                    };
19712                    sent.queued = false;
19713                    sent.last_send_ms = self.clock.now_ms();
19714                }
19715                if let Err(e) =
19716                    self.send_reliable_raw_to_side(dst, bytes.clone(), false, &mut context.admitted)
19717                {
19718                    if Self::is_transient_tx_error(&e) {
19719                        context.mark_refused();
19720                        self.tx_queue_item_for_retry(
19721                            RouterTxItem::ReliableReplay {
19722                                dst,
19723                                ty,
19724                                seq,
19725                                bytes,
19726                            },
19727                            true,
19728                        )?;
19729                        return Ok(());
19730                    }
19731                    return Err(e);
19732                }
19733                let mut st = self.state.lock();
19734                let tx_state = self.reliable_tx_state_mut(&mut st, dst, ty);
19735                if let Some(sent) = tx_state.sent.get_mut(&seq) {
19736                    sent.last_send_ms = self.clock.now_ms();
19737                }
19738            }
19739        }
19740
19741        Ok(())
19742    }
19743
19744    /// Transmit a telemetry item immediately (remote + local).
19745    #[inline]
19746    fn tx_item(&self, item: RouterTxItem) -> TelemetryResult<()> {
19747        self.tx_item_impl(item, false, false)
19748    }
19749
19750    // ---------- PUBLIC API: RX immediate ----------
19751
19752    /// Process packed bytes immediately as locally-originated input.
19753    ///
19754    /// If this call occurs while a side TX callback is already on the stack, the bytes are queued
19755    /// instead of being processed re-entrantly.
19756    #[inline]
19757    pub fn rx_packed(&self, bytes: &[u8]) -> TelemetryResult<()> {
19758        if self.side_tx_active() {
19759            return self.rx_packed_queue(bytes);
19760        }
19761        crate::memory_admission::check_receive(bytes)?;
19762        let data = RouterItem::Packed(crate::shared_bytes::convert(Arc::<[u8]>::from(bytes)));
19763        let item = RouterRxItem {
19764            src: None,
19765            priority: Self::router_item_priority(&data)?,
19766            data,
19767        };
19768        self.rx_item(&item, false)
19769    }
19770
19771    /// Process a decoded packet immediately as locally-originated input.
19772    ///
19773    /// If this call occurs while a side TX callback is already on the stack, the packet is queued
19774    /// instead of being processed re-entrantly.
19775    #[inline]
19776    pub fn rx(&self, pkt: &Packet) -> TelemetryResult<()> {
19777        if self.side_tx_active() {
19778            return self.rx_queue(pkt.clone());
19779        }
19780        let data = RouterItem::Packet(pkt.clone());
19781        let item = RouterRxItem {
19782            src: None,
19783            priority: Self::router_item_priority(&data)?,
19784            data,
19785        };
19786        self.rx_item(&item, false)
19787    }
19788
19789    /// Process a decoded packet immediately with an explicit ingress side id.
19790    ///
19791    /// If this call occurs while a side TX callback is already on the stack, the packet is queued
19792    /// instead of being processed re-entrantly.
19793    #[inline]
19794    pub fn rx_from_side(&self, pkt: &Packet, side: RouterSideId) -> TelemetryResult<()> {
19795        if self.side_tx_active() {
19796            return self.rx_queue_from_side(pkt.clone(), side);
19797        }
19798        self.ensure_side_ingress_enabled(side)?;
19799        let data = RouterItem::Packet(pkt.clone());
19800        let item = RouterRxItem {
19801            src: Some(side),
19802            priority: Self::router_item_priority(&data)?,
19803            data,
19804        };
19805        self.rx_item(&item, false)
19806    }
19807
19808    /// Process packed bytes immediately with an explicit ingress side id.
19809    ///
19810    /// If this call occurs while a side TX callback is already on the stack, the bytes are queued
19811    /// instead of being processed re-entrantly.
19812    #[inline]
19813    pub fn rx_packed_from_side(&self, bytes: &[u8], side: RouterSideId) -> TelemetryResult<()> {
19814        if self.side_tx_active() {
19815            return self.rx_packed_queue_from_side(bytes, side);
19816        }
19817        self.ensure_side_ingress_enabled(side)?;
19818        let Some(decoded) = self.decode_side_transport_frame(side, bytes)? else {
19819            return Ok(());
19820        };
19821        let data = RouterItem::Packed(decoded);
19822        let item = RouterRxItem {
19823            src: Some(side),
19824            priority: Self::router_item_priority(&data)?,
19825            data,
19826        };
19827        self.rx_item(&item, false)
19828    }
19829
19830    fn ensure_application_origin_allowed(&self) -> TelemetryResult<()> {
19831        if self.cfg.role.local_services() {
19832            Ok(())
19833        } else {
19834            Err(TelemetryError::PermissionDenied)
19835        }
19836    }
19837
19838    #[cfg(all(test, feature = "discovery"))]
19839    pub(crate) fn debug_end_to_end_acked_destination_count(&self, id: u64) -> Option<usize> {
19840        self.state
19841            .lock()
19842            .forwarded_receipts
19843            .as_ref()?
19844            .destinations
19845            .get(&id)
19846            .map(BTreeSet::len)
19847    }
19848    #[cfg(all(test, feature = "discovery"))]
19849    pub(crate) fn debug_end_to_end_acked_packet_count(&self) -> usize {
19850        self.state
19851            .lock()
19852            .forwarded_receipts
19853            .as_ref()
19854            .map_or(0, |cache| cache.destinations.len())
19855    }
19856
19857    // ---------- PUBLIC API: TX immediate ----------
19858
19859    /// Transmit a decoded packet immediately.
19860    ///
19861    /// The router delivers locally where appropriate and forwards toward eligible sides. If called
19862    /// from inside a side TX callback, the packet is queued instead of being sent re-entrantly.
19863    #[inline]
19864    pub fn tx(&self, pkt: Packet) -> TelemetryResult<()> {
19865        self.ensure_application_origin_allowed()?;
19866        #[cfg(feature = "discovery")]
19867        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19868        if self.side_tx_active() {
19869            return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)));
19870        }
19871        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
19872    }
19873
19874    /// Transmit packed bytes immediately.
19875    ///
19876    /// If called from inside a side TX callback, the bytes are queued instead of being sent
19877    /// re-entrantly.
19878    #[inline]
19879    pub fn tx_packed(&self, pkt: Arc<[u8]>) -> TelemetryResult<()> {
19880        self.ensure_application_origin_allowed()?;
19881        #[cfg(feature = "discovery")]
19882        let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19883        if self.side_tx_active() {
19884            return self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
19885                crate::shared_bytes::convert(pkt),
19886            )));
19887        }
19888        self.tx_item(RouterTxItem::Broadcast(RouterItem::Packed(
19889            crate::shared_bytes::convert(pkt),
19890        )))
19891    }
19892
19893    // ---------- PUBLIC API: TX queue ----------
19894
19895    /// Queue a decoded packet for later transmission.
19896    /// Discovery/time-sync maintenance stays on the worker calling `periodic`.
19897    /// Enqueueing must not serialize topology on a producer's small stack.
19898    #[inline]
19899    pub fn tx_queue(&self, pkt: Packet) -> TelemetryResult<()> {
19900        self.ensure_application_origin_allowed()?;
19901        self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packet(pkt)))
19902    }
19903
19904    /// Queue packed bytes for later transmission.
19905    #[inline]
19906    pub fn tx_packed_queue(&self, data: Arc<[u8]>) -> TelemetryResult<()> {
19907        self.ensure_application_origin_allowed()?;
19908        self.tx_queue_item(RouterTxItem::Broadcast(RouterItem::Packed(
19909            crate::shared_bytes::convert(data),
19910        )))
19911    }
19912
19913    // ---------- PUBLIC API: logging ----------
19914
19915    // Keep one encoder/dispatch closure per representation, rather than four
19916    // copies for direct/queued and implicit/explicit timestamp entry points.
19917    // Queued producers never perform discovery work on their small stacks.
19918    #[inline(never)]
19919    fn log_dispatch<T: LeBytes>(
19920        &self,
19921        ty: DataType,
19922        data: &[T],
19923        timestamp: Option<u64>,
19924        mut queued: bool,
19925    ) -> TelemetryResult<()> {
19926        self.ensure_application_origin_allowed()?;
19927        if !queued {
19928            #[cfg(feature = "discovery")]
19929            let _ = crate::memory_admission::allow_drain(self.poll_discovery())?;
19930            queued = self.side_tx_active();
19931        }
19932        let timestamp = timestamp.unwrap_or_else(|| self.packet_timestamp_ms());
19933        log_raw(self.sender_arc(), ty, data, timestamp, |pkt| {
19934            let item = RouterTxItem::Broadcast(RouterItem::Packet(pkt));
19935            if queued {
19936                self.tx_queue_item(item)
19937            } else {
19938                self.tx_item(item)
19939            }
19940        })
19941    }
19942
19943    /// Build a packet from typed elements and send it immediately.
19944    /// Calls from an active transport callback are deferred to the TX queue.
19945    #[inline]
19946    pub fn log<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
19947        self.log_dispatch(ty, data, None, false)
19948    }
19949
19950    /// Build a packet from typed elements and queue it for later transmission.
19951    #[inline]
19952    pub fn log_queue<T: LeBytes>(&self, ty: DataType, data: &[T]) -> TelemetryResult<()> {
19953        self.log_dispatch(ty, data, None, true)
19954    }
19955
19956    /// Build a packet with an explicit timestamp and send it immediately.
19957    #[inline]
19958    pub fn log_ts<T: LeBytes>(
19959        &self,
19960        ty: DataType,
19961        timestamp: u64,
19962        data: &[T],
19963    ) -> TelemetryResult<()> {
19964        self.log_dispatch(ty, data, Some(timestamp), false)
19965    }
19966
19967    /// Build a packet with an explicit timestamp and queue it for transmission.
19968    #[inline]
19969    pub fn log_queue_ts<T: LeBytes>(
19970        &self,
19971        ty: DataType,
19972        timestamp: u64,
19973        data: &[T],
19974    ) -> TelemetryResult<()> {
19975        self.log_dispatch(ty, data, Some(timestamp), true)
19976    }
19977}
19978
19979#[cfg(all(test, feature = "std", feature = "discovery"))]
19980mod queued_schema_identity_tests {
19981    use super::*;
19982
19983    fn catalog(timestamp: u64, byte: u8) -> Packet {
19984        Packet::new(
19985            crate::DataType::DiscoverySchema,
19986            &[DataEndpoint::Discovery],
19987            "SCHEMA_PEER",
19988            timestamp,
19989            crate::shared_bytes::convert(Arc::<[u8]>::from(vec![byte; 1024])),
19990        )
19991        .unwrap()
19992    }
19993
19994    #[test]
19995    fn inline_and_large_typed_publications_share_headers_and_keep_le_bytes() {
19996        crate::tests::ensure_common_test_schema();
19997        let endpoint = DataEndpoint::named("SD_CARD");
19998        let ty = crate::config::register_data_type_with_description(
19999            "INLINE_TYPED_BOUNDARY",
20000            "",
20001            MessageElement::Dynamic(crate::MessageDataType::UInt32, crate::MessageClass::Data),
20002            &[endpoint],
20003            crate::ReliableMode::None,
20004            16,
20005        )
20006        .unwrap();
20007        let sender: Arc<str> = Arc::from("DAQ_SOURCE");
20008        let endpoints = message_meta(ty);
20009        for count in [
20010            0,
20011            1,
20012            crate::config::STACK_PAYLOAD_SIZE / 4,
20013            crate::config::STACK_PAYLOAD_SIZE / 4 + 1,
20014            64,
20015        ] {
20016            let values = (0..count)
20017                .map(|i| 0x1234_5678u32.wrapping_add(i as u32))
20018                .collect::<Vec<_>>();
20019            log_raw(sender.clone(), ty, &values, 42, |packet| {
20020                let bytes = values
20021                    .iter()
20022                    .flat_map(|v| v.to_le_bytes())
20023                    .collect::<Vec<_>>();
20024                assert_eq!(packet.payload(), bytes);
20025                assert_eq!(packet.sender().as_ptr(), sender.as_ptr());
20026                assert_eq!(
20027                    packet.endpoints().as_ptr(),
20028                    endpoints.endpoints_ref().as_ptr()
20029                );
20030                packet.validate().unwrap();
20031                Ok(())
20032            })
20033            .unwrap();
20034        }
20035        #[derive(Clone, Copy)]
20036        struct BadWidth;
20037        impl LeBytes for BadWidth {
20038            const WIDTH: usize = 0;
20039            fn write_le(self, _: &mut [u8]) {
20040                unreachable!()
20041            }
20042            fn from_le_slice(_: &[u8]) -> Self {
20043                Self
20044            }
20045        }
20046        assert!(matches!(
20047            log_raw(sender.clone(), ty, &[BadWidth], 42, |_| Ok(())),
20048            Err(TelemetryError::BadArg)
20049        ));
20050        #[derive(Clone, Copy)]
20051        struct HugeWidth;
20052        impl LeBytes for HugeWidth {
20053            const WIDTH: usize = usize::MAX;
20054            fn write_le(self, _: &mut [u8]) {
20055                unreachable!()
20056            }
20057            fn from_le_slice(_: &[u8]) -> Self {
20058                Self
20059            }
20060        }
20061        assert!(matches!(
20062            log_raw(sender, ty, &[HugeWidth, HugeWidth], 42, |_| Ok(())),
20063            Err(TelemetryError::PacketTooLarge(_))
20064        ));
20065    }
20066
20067    #[test]
20068    fn queue_accounting_reports_live_cost_without_draining_or_pinning() {
20069        crate::tests::ensure_common_test_schema();
20070        let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 42u64));
20071        let packet = Packet::new(
20072            crate::DataType::named("IMU_DATA"),
20073            &[DataEndpoint::named("SD_CARD")],
20074            "SOURCE",
20075            1,
20076            crate::shared_bytes::convert(Arc::<[u8]>::from([0u8; 24])),
20077        )
20078        .unwrap();
20079        router.tx_queue(packet).unwrap();
20080        assert!(router.has_queued_work());
20081        let expected = router.memory_counters();
20082        let a = router.queue_counters();
20083        assert_eq!(a[0], 1);
20084        assert_eq!(a[1], 0);
20085        assert_eq!(a[2], 1);
20086        assert_eq!(a[3], expected[3]);
20087        assert_eq!(a[4], expected[4]);
20088        assert_eq!(a[5], crate::DataType::named("IMU_DATA").as_u32());
20089        assert_eq!(a[14], 42);
20090        assert_eq!(a[15], 0);
20091        assert_eq!(router.queue_counters(), a);
20092        assert_eq!(router.memory_counters(), expected);
20093        router.drain_queues_with_timeout(0).unwrap();
20094        let empty = router.queue_counters();
20095        assert_eq!(empty[2], 0);
20096        assert_eq!(empty[4], 0);
20097        assert_eq!(router.memory_counters()[4], 0);
20098        assert!(!router.has_queued_work());
20099    }
20100
20101    #[test]
20102    fn repeated_schema_preserves_partial_transfer_but_updates_and_other_sides_do_not_coalesce() {
20103        let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20104        let side = router.add_side_packed("A", |_| Ok(()));
20105        let other_side = router.add_side_packed("B", |_| Ok(()));
20106        let original = catalog(1, 42);
20107        let duplicate = catalog(2, 42);
20108        assert_ne!(original.packet_id(), duplicate.packet_id());
20109        let mut st = router.state.lock();
20110        st.transmit_queue.clear();
20111        let queued = |dst, pkt| TxQueued {
20112            disposable: TX_DISPOSABLE,
20113            item: RouterTxItem::ToSide {
20114                src: None,
20115                dst,
20116                data: RouterItem::Packet(pkt),
20117            },
20118            ignore_local: true,
20119            priority: 255,
20120            delivery_state: QueuedDelivery::Untracked,
20121            accepted_type: 0,
20122            deferred_since_ms: 0,
20123            retry_after_ms: 123,
20124        };
20125        st.push_transmit(queued(side, original.clone())).unwrap();
20126        st.push_transmit(queued(side, duplicate.clone())).unwrap();
20127        assert_eq!(st.transmit_queue.len(), 1);
20128        let pending = st.transmit_queue.iter().next().unwrap();
20129        assert_eq!(pending.retry_after_ms, 123);
20130        assert!(
20131            matches!(&pending.item, RouterTxItem::ToSide { data: RouterItem::Packet(pkt), .. }
20132            if pkt.packet_id() == original.packet_id())
20133        );
20134        st.push_transmit(queued(other_side, duplicate)).unwrap();
20135        assert_eq!(st.transmit_queue.len(), 2);
20136        let changed = catalog(3, 43);
20137        st.push_transmit(queued(side, changed.clone())).unwrap();
20138        assert_eq!(st.transmit_queue.len(), 2);
20139        assert!(st.transmit_queue.iter().any(|queued| matches!(&queued.item,
20140            RouterTxItem::ToSide { dst, data: RouterItem::Packet(pkt), .. }
20141            if *dst == side && pkt.packet_id() == changed.packet_id())));
20142    }
20143
20144    #[test]
20145    fn repeated_announcements_complete_one_fragment_per_dispatch() {
20146        use core::sync::atomic::{AtomicUsize, Ordering};
20147        let allowance = Arc::new(AtomicUsize::new(0));
20148        let accepted = Arc::new(std::sync::Mutex::new(Vec::<Arc<[u8]>>::new()));
20149        let available = allowance.clone();
20150        let frames = accepted.clone();
20151        let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 1u64));
20152        let side = node.add_side_packed_with_options(
20153            "bounded CAN",
20154            move |bytes| {
20155                if available
20156                    .try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
20157                        count.checked_sub(1)
20158                    })
20159                    .is_err()
20160                {
20161                    return Err(TelemetryError::Io("side tx busy"));
20162                }
20163                frames.lock().unwrap().push(Arc::<[u8]>::from(bytes));
20164                Ok(())
20165            },
20166            RouterSideOptions::default().with_small_packet_transport(128),
20167        );
20168        let handler = node.state.lock().sides[side]
20169            .as_ref()
20170            .unwrap()
20171            .tx_handler
20172            .clone();
20173        let mut complete = false;
20174        for timestamp in 1..100 {
20175            // Incompressible bytes ensure this really uses several chunks.
20176            let mut state = 0x517cc1b7u32;
20177            let payload: Vec<u8> = (0..3600)
20178                .map(|_| {
20179                    state ^= state << 13;
20180                    state ^= state >> 17;
20181                    state ^= state << 5;
20182                    state as u8
20183                })
20184                .collect();
20185            let packet = Packet::new(
20186                crate::DataType::DiscoverySchema,
20187                &[crate::DataEndpoint::Discovery],
20188                "SCHEMA_PEER",
20189                timestamp,
20190                crate::shared_bytes::convert(Arc::<[u8]>::from(payload)),
20191            )
20192            .unwrap();
20193            node.state
20194                .lock()
20195                .push_transmit(TxQueued {
20196                    disposable: TX_DISPOSABLE,
20197                    item: RouterTxItem::ToSide {
20198                        src: None,
20199                        dst: side,
20200                        data: RouterItem::Packet(packet),
20201                    },
20202                    ignore_local: true,
20203                    priority: 255,
20204                    delivery_state: QueuedDelivery::Untracked,
20205                    accepted_type: 0,
20206                    deferred_since_ms: 0,
20207                    retry_after_ms: 0,
20208                })
20209                .unwrap();
20210            let data = match &node.state.lock().transmit_queue.iter().next().unwrap().item {
20211                RouterTxItem::ToSide { data, .. } => data.clone(),
20212                _ => panic!("unexpected item"),
20213            };
20214            allowance.store(1, Ordering::Relaxed);
20215            match node.call_side_tx_handler(side, &handler, &data, false) {
20216                Ok(()) => {
20217                    complete = true;
20218                    break;
20219                }
20220                Err(TelemetryError::Io(_)) => {}
20221                other => panic!("unexpected result: {other:?}"),
20222            }
20223        }
20224        assert!(
20225            complete,
20226            "new announcement timestamps must not replay the prefix forever"
20227        );
20228        let received = accepted.lock().unwrap();
20229        // One transfer ID and consecutive fragment indices prove prefix reuse
20230        // did not create incomplete transfers or duplicate accepted fragments.
20231        assert!(
20232            received.len() > 1,
20233            "fixture must exercise fragmentation, not a compressed single frame"
20234        );
20235        let first = &received[0];
20236        assert_eq!(&first[..4], &[83, 68, 84, 3]);
20237        assert_eq!(
20238            u16::from_le_bytes([first[10], first[11]]) as usize,
20239            received.len()
20240        );
20241        let transfer_id = &first[4..8];
20242        for (index, frame) in received.iter().enumerate() {
20243            assert_eq!(&frame[4..8], transfer_id);
20244            assert_eq!(u16::from_le_bytes([frame[8], frame[9]]) as usize, index);
20245        }
20246    }
20247    #[test]
20248    fn schema_receipt_resolves_assigned_numeric_address_without_adding_peers() {
20249        let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20250        let side = router.add_side_packed("A", |_| Ok(()));
20251        let assigned = 0x1234u32;
20252        assert_ne!(sender_address_u32("SCHEMA_PEER"), assigned);
20253        {
20254            let mut st = router.state.lock();
20255            st.discovery_routes
20256                .entry(side)
20257                .or_default()
20258                .announcers
20259                .insert(
20260                    "SCHEMA_PEER".into(),
20261                    Box::new(DiscoverySenderState {
20262                        advertised_address: Some(assigned),
20263                        ..DiscoverySenderState::default()
20264                    }),
20265                );
20266        }
20267        router.note_schema_received(side, "@addr:4660");
20268        router.note_schema_received(side, "@addr:4661");
20269        let st = router.state.lock();
20270        let peers = &st.discovery_routes.get(&side).unwrap().announcers;
20271        assert_eq!(peers.len(), 1);
20272        assert!(peers["SCHEMA_PEER"].has_schema);
20273    }
20274    #[test]
20275    fn busy_handler_returns_after_one_attempt_and_other_errors_keep_retry_budget() {
20276        let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20277        let attempts = core::cell::Cell::new(0);
20278        let result: Result<((), usize), _> = router.retry_with_attempts(8, || {
20279            attempts.set(attempts.get() + 1);
20280            Err(TelemetryError::Io("side tx busy"))
20281        });
20282        assert!(matches!(
20283            result,
20284            Err((TelemetryError::Io("side tx busy"), 1))
20285        ));
20286        assert_eq!(attempts.get(), 1);
20287        attempts.set(0);
20288        let result: Result<((), usize), _> = router.retry_with_attempts(3, || {
20289            attempts.set(attempts.get() + 1);
20290            Err(TelemetryError::Io("permanent handler failure"))
20291        });
20292        assert!(matches!(
20293            result,
20294            Err((TelemetryError::Io("permanent handler failure"), 3))
20295        ));
20296        assert_eq!(attempts.get(), 3);
20297    }
20298    #[test]
20299    fn rejected_first_template_frame_is_retried_self_describing() {
20300        crate::tests::ensure_common_test_schema();
20301        let ty = crate::config::register_data_type_with_description(
20302            "ROUTER_BUSY_FIRST_TEMPLATE",
20303            "",
20304            crate::MessageElement::Static(
20305                1,
20306                crate::MessageDataType::UInt8,
20307                crate::MessageClass::Data,
20308            ),
20309            &[crate::DataEndpoint::named("RADIO")],
20310            crate::ReliableMode::None,
20311            1,
20312        )
20313        .unwrap();
20314        let tx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20315        let rx = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0u64));
20316        let opts = RouterSideOptions::default().with_small_packet_transport(0);
20317        let a = tx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
20318        let b = rx.add_side_packed_with_options("wire", |_| Ok(()), opts.clone());
20319        let packet = Packet::new(
20320            ty,
20321            &[crate::DataEndpoint::named("RADIO")],
20322            "BUSY_PEER",
20323            1,
20324            crate::shared_bytes::convert(Arc::<[u8]>::from([42u8])),
20325        )
20326        .unwrap();
20327        let raw = wire_format::pack_packet(&packet);
20328        // No byte of the dictionary bootstrap reaches the receiver.
20329        let first = tx
20330            .encode_side_transport_frames(
20331                a,
20332                opts.clone(),
20333                crate::shared_bytes::convert(raw.clone()),
20334            )
20335            .unwrap();
20336        assert!(
20337            tx.send_side_frames_resumable(a, first, |_| Err(TelemetryError::Io("side tx busy")))
20338                .is_err()
20339        );
20340        let retry = tx
20341            .encode_side_transport_frames(
20342                a,
20343                opts.clone(),
20344                crate::shared_bytes::convert(raw.clone()),
20345            )
20346            .unwrap();
20347        let mut received = None;
20348        tx.send_side_frames_resumable(a, retry, |frame| {
20349            assert_eq!(
20350                frame[3], SIDE_TRANSPORT_KIND_FULL,
20351                "retry requires the unaccepted dictionary bootstrap"
20352            );
20353            received = rx.decode_side_transport_frame(b, frame)?;
20354            Ok(())
20355        })
20356        .unwrap();
20357        assert_eq!(received, Some(crate::shared_bytes::convert(raw.clone())));
20358        // Compression resumes once a full frame has actually been accepted.
20359        let next = tx
20360            .encode_side_transport_frames(a, opts, crate::shared_bytes::convert(raw.clone()))
20361            .unwrap();
20362        tx.send_side_frames_resumable(a, next, |frame| {
20363            assert_eq!(frame[3], SIDE_TRANSPORT_KIND_COMPACT);
20364            assert_eq!(
20365                rx.decode_side_transport_frame(b, frame)?,
20366                Some(crate::shared_bytes::convert(raw.clone()))
20367            );
20368            Ok(())
20369        })
20370        .unwrap();
20371    }
20372    #[test]
20373    fn already_queued_receipt_precedes_ordered_retry_exhaustion() {
20374        crate::tests::ensure_common_test_schema();
20375        let ep = DataEndpoint::named("RADIO");
20376        let ty = crate::config::register_data_type_with_description(
20377            "QUEUED_RECEIPT_DEADLINE",
20378            "",
20379            MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20380            &[ep],
20381            crate::ReliableMode::Ordered,
20382            1,
20383        )
20384        .unwrap();
20385        for isr in [false, true] {
20386            let now = Arc::new(core::sync::atomic::AtomicU64::new(0));
20387            let clock = now.clone();
20388            let router = Router::new_with_clock(
20389                RouterConfig::new([]).with_hostname("FC"),
20390                Box::new(move || clock.load(Ordering::Relaxed)),
20391            );
20392            let side = router.add_side_packed("CAN", |_| Ok(()));
20393            let packet = Packet::new(ty, &[ep], "FC", 1, Arc::<[u8]>::from([42u8])).unwrap();
20394            let id = packet.packet_id();
20395            let owner = Router::sender_hash("GS");
20396            let mut destinations = BTreeMap::new();
20397            destinations.insert(owner, side);
20398            router.state.lock().end_to_end_reliable_tx.insert(
20399                id,
20400                EndToEndReliableSent {
20401                    #[cfg(feature = "std")]
20402                    completion_epoch: 0,
20403                    #[cfg(feature = "std")]
20404                    pending_type: ty.as_u32(),
20405                    ordered_type: Some(ty.as_u32()),
20406                    waiting_for_predecessor: false,
20407                    data: RouterItem::Packet(packet),
20408                    pending_destinations: destinations,
20409                    tracked_destinations: true,
20410                    last_send_ms: 0,
20411                    retries: runtime_reliable_max_retries(),
20412                    queued: false,
20413                }
20414                .try_box()
20415                .unwrap(),
20416            );
20417            let snapshot = router.export_reliable_pending_stats();
20418            assert_eq!(snapshot.len(), 1);
20419            assert_eq!(snapshot[0].packet_id, id);
20420            assert_eq!(snapshot[0].data_type, ty);
20421            assert_eq!(snapshot[0].pending_destinations, vec![owner]);
20422            assert_eq!(snapshot[0].retries, runtime_reliable_max_retries());
20423            assert!(!snapshot[0].queued);
20424            assert!(!snapshot[0].waiting_for_predecessor);
20425            let ack = Packet::new_with_wire_contract(
20426                DataType::ReliableAck,
20427                message_meta(DataType::ReliableAck).endpoints_ref(),
20428                "E2EACK:GS",
20429                1,
20430                1,
20431                Arc::<[u8]>::from(id.to_le_bytes()),
20432                None,
20433                Arc::from([owner, Router::sender_hash("FC")]),
20434            )
20435            .unwrap();
20436            let packed = wire_format::pack_packet(&ack);
20437            if isr {
20438                router.rx_packed_queue_from_side_isr(&packed, side).unwrap();
20439            } else {
20440                router.rx_packed_queue_from_side(&packed, side).unwrap();
20441            }
20442            now.store(runtime_reliable_retransmit_ms(), Ordering::Relaxed);
20443            router
20444                .drain_queues_with_timeout(1)
20445                .expect("a received ACK must be processed before cancelling its ordered delivery");
20446            assert!(!router.state.lock().end_to_end_reliable_tx.contains_key(&id));
20447            assert!(router.export_reliable_pending_stats().is_empty());
20448        }
20449    }
20450    #[test]
20451    fn embedded_gateway_receipt_with_numeric_sender_clears_host_pending_owner() {
20452        crate::tests::ensure_common_test_schema();
20453        // Captured after the actual gateway UART and Pico mailbox. The owner
20454        // hash is carried in the contract, because the ACK sender is numeric.
20455        let bytes = [
20456            0x0c, 0x01, 0x01, 0x08, 0xd4, 0xe5, 0xea, 0xb6, 0xb7, 0x33, 0xff, 0x90, 0x02, 0xb4,
20457            0xe6, 0xdd, 0x94, 0x07, 0x14, 0x03, 0x44, 0x02, 0x02, 0x06, 0x25, 0xd2, 0x59, 0x1b,
20458            0xbb, 0xfc, 0xa7, 0x5f, 0x43, 0x92, 0x9e, 0x12, 0xbb, 0xfc, 0xa7, 0x26, 0xa2, 0xa0,
20459            0xbe, 0x39, 0x65, 0xe2, 0xd8, 0x61, 0x68, 0x22, 0xfd,
20460        ];
20461        let ack = wire_format::unpack_packet(&bytes).unwrap();
20462        let owner = Router::sender_hash("GB");
20463        assert_eq!(Router::end_to_end_ack_sender_hash(&ack), Some(owner));
20464        let id = Router::decode_end_to_end_reliable_ack(ack.payload()).unwrap();
20465        let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20466        let side = router.add_side_packed("Pico", |_| Ok(()));
20467        let data = ack.clone();
20468        router.state.lock().end_to_end_reliable_tx.insert(
20469            id,
20470            EndToEndReliableSent {
20471                #[cfg(feature = "std")]
20472                completion_epoch: 0,
20473                #[cfg(feature = "std")]
20474                pending_type: data.data_type().as_u32(),
20475                ordered_type: None,
20476                waiting_for_predecessor: false,
20477                data: RouterItem::Packet(data),
20478                pending_destinations: BTreeMap::from([(owner, side)]),
20479                tracked_destinations: true,
20480                last_send_ms: 0,
20481                retries: 0,
20482                queued: false,
20483            }
20484            .try_box()
20485            .unwrap(),
20486        );
20487        router.rx_packed_queue_from_side(&bytes, side).unwrap();
20488        router.drain_queues_with_timeout(1).unwrap();
20489        assert!(router.export_reliable_pending_stats().is_empty());
20490    }
20491
20492    #[test]
20493    fn frozen_delivery_contract_bypasses_dictionary_even_for_best_effort_schema() {
20494        crate::tests::ensure_common_test_schema();
20495        let ep = DataEndpoint::named("RADIO");
20496        let ty = crate::config::register_data_type_with_description(
20497            "MANAGED_CONTROL_WITH_FROZEN_OWNERS",
20498            "",
20499            MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20500            &[ep],
20501            crate::ReliableMode::None,
20502            16,
20503        )
20504        .unwrap();
20505        let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20506        let opts = RouterSideOptions::default().with_template_transport();
20507        let side = router.add_side_packed_with_options("radio", |_| Ok(()), opts.clone());
20508        let packet = Packet::new_with_wire_contract(
20509            ty,
20510            &[ep],
20511            "GS",
20512            1,
20513            1,
20514            Arc::<[u8]>::from([1u8]),
20515            None,
20516            Arc::from([Router::sender_hash("RF")]),
20517        )
20518        .unwrap();
20519        let raw = wire_format::pack_packet(&packet);
20520        assert_eq!(
20521            router
20522                .item_service_priority(&RouterItem::Packet(packet.clone()))
20523                .unwrap(),
20524            254
20525        );
20526        assert_eq!(
20527            router
20528                .item_service_priority(&RouterItem::Packed(crate::shared_bytes::convert(
20529                    raw.clone()
20530                )))
20531                .unwrap(),
20532            254
20533        );
20534        assert!(
20535            !wire_format::peek_routing_frame_info(&raw)
20536                .unwrap()
20537                .envelope
20538                .is_reliable
20539        );
20540        for _ in 0..3 {
20541            let frames = router
20542                .encode_side_transport_frames(
20543                    side,
20544                    opts.clone(),
20545                    crate::shared_bytes::convert(raw.clone()),
20546                )
20547                .unwrap();
20548            let mut count = 0;
20549            router
20550                .send_side_frames_resumable(side, frames, |bytes| {
20551                    assert_eq!(
20552                        bytes,
20553                        raw.as_ref(),
20554                        "delivery contract must remain independently decodable"
20555                    );
20556                    count += 1;
20557                    Ok(())
20558                })
20559                .unwrap();
20560            assert_eq!(count, 1);
20561        }
20562    }
20563
20564    #[test]
20565    fn managed_receive_queues_keep_service_priority_with_low_schema_priority() {
20566        crate::tests::ensure_common_test_schema();
20567        let ep = DataEndpoint::named("RADIO");
20568        let ty = crate::config::register_data_type_with_description(
20569            "MANAGED_RX_SERVICE_PRIORITY",
20570            "",
20571            MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20572            &[ep],
20573            crate::ReliableMode::None,
20574            16,
20575        )
20576        .unwrap();
20577        let router = Router::new(RouterConfig::new([]).with_hostname("RF"));
20578        router
20579            .enable_network_variable(ty, NetworkVariablePermissions::READ_ONLY)
20580            .unwrap();
20581        let side = router.add_side_packed("radio", |_| Ok(()));
20582        router.clear_queues();
20583        let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
20584        let wire = wire_format::pack_packet(&packet);
20585        router.rx_queue(packet.clone()).unwrap();
20586        router.rx_queue_from_side(packet.clone(), side).unwrap();
20587        router.rx_packed_queue(&wire).unwrap();
20588        router.rx_packed_queue_from_side(&wire, side).unwrap();
20589        for _ in 0..4 {
20590            let queued = router
20591                .pop_next_queued_control_rx_item()
20592                .unwrap()
20593                .expect("registered managed controls must precede queued bulk work");
20594            assert_eq!(
20595                queued.priority,
20596                crate::transport_priority(DataType::ManagedVariableValue)
20597            );
20598        }
20599        assert!(router.pop_next_queued_control_rx_item().unwrap().is_none());
20600    }
20601
20602    #[test]
20603    fn managed_value_keeps_service_priority_after_side_backpressure() {
20604        crate::tests::ensure_common_test_schema();
20605        let ep = DataEndpoint::named("RADIO");
20606        let ty = crate::config::register_data_type_with_description(
20607            "MANAGED_RETRY_SERVICE_PRIORITY",
20608            "",
20609            MessageElement::Static(1, crate::MessageDataType::UInt8, crate::MessageClass::Data),
20610            &[ep],
20611            crate::ReliableMode::None,
20612            16,
20613        )
20614        .unwrap();
20615        let router = Router::new(RouterConfig::new([]).with_hostname("GS"));
20616        router
20617            .enable_network_variable(ty, NetworkVariablePermissions::READ_WRITE)
20618            .unwrap();
20619        let side = router.add_side_packed("radio", |_| Err(TelemetryError::Io("side tx busy")));
20620        let packet = Packet::new(ty, &[ep], "GS", 1, Arc::<[u8]>::from([1])).unwrap();
20621        let item = RouterItem::Packet(packet);
20622        for queued in [
20623            RouterTxItem::Broadcast(item.clone()),
20624            RouterTxItem::ToSide {
20625                src: None,
20626                dst: side,
20627                data: item.clone(),
20628            },
20629        ] {
20630            router.tx_queue_item_for_retry(queued, true).unwrap();
20631            let queued = router.state.lock().transmit_queue.pop_front().unwrap();
20632            assert_eq!(
20633                queued.priority,
20634                crate::transport_priority(DataType::ManagedVariableValue),
20635                "a busy side must not demote managed controls to bulk application priority"
20636            );
20637            assert!(queued.retry_after_ms > 0);
20638        }
20639    }
20640    #[test]
20641    fn receipt_budget_does_not_starve_transmit() {
20642        use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
20643        crate::tests::ensure_common_test_schema();
20644        let now = Arc::new(AtomicU64::new(0));
20645        let clock = now.clone();
20646        let count = Arc::new(AtomicUsize::new(0));
20647        let accepted = count.clone();
20648        let node = Router::new_with_clock(
20649            RouterConfig::default(),
20650            Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
20651        );
20652        let side = node.add_side_packed("wire", move |_| {
20653            accepted.fetch_add(1, Ordering::Relaxed);
20654            Ok(())
20655        });
20656        node.clear_queues();
20657        let packet = Packet::new(
20658            crate::DataType::ReliableAck,
20659            &[crate::DataEndpoint::TelemetryError],
20660            "E2EACK:PEER",
20661            1,
20662            Arc::<[u8]>::from([0u8; 8]),
20663        )
20664        .unwrap();
20665        node.state
20666            .lock()
20667            .push_transmit(TxQueued {
20668                disposable: TX_DISPOSABLE,
20669                item: RouterTxItem::ToSide {
20670                    src: None,
20671                    dst: side,
20672                    data: RouterItem::Packet(packet.clone()),
20673                },
20674                ignore_local: true,
20675                priority: 250,
20676                delivery_state: QueuedDelivery::Untracked,
20677                accepted_type: 0,
20678                deferred_since_ms: 0,
20679                retry_after_ms: 0,
20680            })
20681            .unwrap();
20682        node.rx_packed_queue_from_side(
20683            &wire_format::pack_packet(
20684                &Packet::new(
20685                    crate::DataType::ReliableAck,
20686                    &[crate::DataEndpoint::TelemetryError],
20687                    "RX_PEER",
20688                    2,
20689                    Arc::<[u8]>::from([0u8; 8]),
20690                )
20691                .unwrap(),
20692            ),
20693            side,
20694        )
20695        .unwrap();
20696        node.drain_queues_with_timeout(1).unwrap();
20697        assert_eq!(
20698            count.load(Ordering::Relaxed),
20699            1,
20700            "control RX exhausting its cooperative budget must still allow one queued TX"
20701        );
20702    }
20703    #[test]
20704    fn receipt_budget_does_not_starve_bulk_transmit() {
20705        use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
20706        crate::tests::ensure_common_test_schema();
20707        let now = Arc::new(AtomicU64::new(0));
20708        let clock = now.clone();
20709        let count = Arc::new(AtomicUsize::new(0));
20710        let accepted = count.clone();
20711        let node = Router::new_with_clock(
20712            RouterConfig::default(),
20713            Box::new(move || clock.fetch_add(1, Ordering::Relaxed)),
20714        );
20715        let side = node.add_side_packed("wire", move |_| {
20716            accepted.fetch_add(1, Ordering::Relaxed);
20717            Ok(())
20718        });
20719        node.clear_queues();
20720        let packet = Packet::new(
20721            crate::DataType::ReliableAck,
20722            &[crate::DataEndpoint::TelemetryError],
20723            "E2EACK:PEER",
20724            1,
20725            Arc::<[u8]>::from([0u8; 8]),
20726        )
20727        .unwrap();
20728        node.state
20729            .lock()
20730            .push_transmit(TxQueued {
20731                disposable: TX_DISPOSABLE,
20732                item: RouterTxItem::ToSide {
20733                    src: None,
20734                    dst: side,
20735                    data: RouterItem::Packet(packet.clone()),
20736                },
20737                ignore_local: true,
20738                priority: 1,
20739                delivery_state: QueuedDelivery::Untracked,
20740                accepted_type: 0,
20741                deferred_since_ms: 0,
20742                retry_after_ms: 0,
20743            })
20744            .unwrap();
20745        node.rx_packed_queue_from_side(
20746            &wire_format::pack_packet(
20747                &Packet::new(
20748                    crate::DataType::ReliableAck,
20749                    &[crate::DataEndpoint::TelemetryError],
20750                    "RX_PEER",
20751                    2,
20752                    Arc::<[u8]>::from([0u8; 8]),
20753                )
20754                .unwrap(),
20755            ),
20756            side,
20757        )
20758        .unwrap();
20759        node.process_all_queues_with_timeout_impl(1).unwrap();
20760        assert_eq!(
20761            count.load(Ordering::Relaxed),
20762            1,
20763            "control RX exhausting its cooperative budget must still allow one queued bulk TX"
20764        );
20765    }
20766}
20767
20768#[cfg(all(test, feature = "discovery", feature = "std"))]
20769fn untemplated_test_node() -> Router {
20770    Router::new_with_clock(RouterConfig::default(), Box::new(|| 0))
20771}
20772#[cfg(all(test, feature = "discovery", feature = "std"))]
20773fn untemplated_test_options() -> RouterSideOptions {
20774    RouterSideOptions::default().with_small_packet_transport(64)
20775}
20776#[cfg(all(test, feature = "discovery", feature = "std"))]
20777#[allow(clippy::duplicate_mod)]
20778#[path = "tests/untemplated_transport.rs"]
20779mod untemplated_transport_tests;
20780
20781#[cfg(all(test, feature = "std", feature = "discovery"))]
20782mod queued_producer_maintenance_tests {
20783    use super::*;
20784    #[test]
20785    fn queued_publication_defers_discovery_to_worker_maintenance() {
20786        crate::tests::ensure_common_test_schema();
20787        let ty = DataType::named("GPS_DATA");
20788        for variant in 0..4 {
20789            let sent = Arc::new(std::sync::Mutex::new(Vec::new()));
20790            let output = sent.clone();
20791            let router = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20792            router.add_side_packed("wire", move |bytes| {
20793                output
20794                    .lock()
20795                    .unwrap()
20796                    .push(wire_format::unpack_packet(bytes)?.data_type());
20797                Ok(())
20798            });
20799            router.clear_queues();
20800            let packet =
20801                Packet::from_f32_slice(ty, &[1.0, 2.0, 3.0], message_meta(ty).endpoints_ref(), 42)
20802                    .unwrap();
20803            match variant {
20804                0 => router.tx_queue(packet).unwrap(),
20805                1 => router
20806                    .tx_packed_queue(wire_format::pack_packet(&packet))
20807                    .unwrap(),
20808                2 => router.log_queue(ty, &[1.0f32, 2.0, 3.0]).unwrap(),
20809                _ => router.log_queue_ts(ty, 42, &[1.0f32, 2.0, 3.0]).unwrap(),
20810            }
20811            assert!(
20812                sent.lock().unwrap().is_empty(),
20813                "producer invoked a transport"
20814            );
20815            assert_eq!(
20816                router.state.lock().transmit_queue.len(),
20817                1,
20818                "queued API performed discovery on the producer stack (variant {variant})"
20819            );
20820            router.periodic(0).unwrap();
20821            let sent = sent.lock().unwrap();
20822            assert!(sent.contains(&ty));
20823            assert!(
20824                sent.iter().copied().any(discovery::is_discovery_type),
20825                "worker maintenance must still announce the board"
20826            );
20827        }
20828    }
20829}
20830
20831#[cfg(all(test, feature = "std"))]
20832mod bounded_fair_queue_tests {
20833    use super::*;
20834    #[test]
20835    fn small_canonical_reports_survive_fast_senders_fixed_ring_bursts() {
20836        let ep = crate::config::register_endpoint("ROUTER_SMALL_QUEUE_DEST", false).unwrap();
20837        let ty = crate::config::register_data_type_with_description(
20838            "ROUTER_SMALL_QUEUE_REPORT",
20839            "",
20840            crate::MessageElement::Static(
20841                4,
20842                crate::MessageDataType::UInt8,
20843                crate::MessageClass::Data,
20844            ),
20845            &[ep],
20846            crate::ReliableMode::None,
20847            0,
20848        )
20849        .unwrap();
20850        for tx in [false, true] {
20851            for slow_last in [false, true] {
20852                let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20853                let make = |sender| {
20854                    RouterItem::Packed(crate::shared_bytes::convert(wire_format::pack_packet(
20855                        &Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 4])).unwrap(),
20856                    )))
20857                };
20858                let rx = |data| RouterRxItem {
20859                    src: None,
20860                    data,
20861                    priority: 0,
20862                };
20863                let tx_item = |data| TxQueued {
20864                    disposable: TX_DISPOSABLE,
20865                    item: RouterTxItem::Broadcast(data),
20866                    ignore_local: true,
20867                    priority: 0,
20868                    delivery_state: QueuedDelivery::Untracked,
20869                    accepted_type: 0,
20870                    deferred_since_ms: 0,
20871                    retry_after_ms: 0,
20872                };
20873                let mut st = node.state.lock();
20874                st.memory.max_queue_budget = usize::MAX;
20875                if tx {
20876                    st.transmit_queue = BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
20877                } else {
20878                    st.received_queue = BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
20879                }
20880                let order = if slow_last {
20881                    ["FAST", "FAST", "SLOW"]
20882                } else {
20883                    ["SLOW", "FAST", "FAST"]
20884                };
20885                for sender in order.into_iter().chain(core::iter::repeat_n("FAST", 32)) {
20886                    if tx {
20887                        st.push_transmit(tx_item(make(sender))).unwrap();
20888                    } else {
20889                        st.push_received(rx(make(sender))).unwrap();
20890                    }
20891                }
20892                let slow = wire_format::source_address_for_sender("SLOW");
20893                let retained = if tx {
20894                    st.transmit_queue
20895                        .iter()
20896                        .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
20897                } else {
20898                    st.received_queue.iter().any(|item| {
20899                        RouterInner::fair_bulk_owner(&item.data, item.priority) == Some(slow)
20900                    })
20901                };
20902                assert!(
20903                    retained,
20904                    "small report was displaced: tx={tx}, slow_last={slow_last}"
20905                );
20906            }
20907        }
20908    }
20909
20910    #[test]
20911    fn slow_stream_survives_same_board_fast_stream_bursts() {
20912        let ep = crate::config::register_endpoint("ROUTER_STREAM_QUEUE_DEST", false).unwrap();
20913        let ty = crate::config::register_data_type_with_description(
20914            "ROUTER_STREAM_QUEUE_FAST",
20915            "",
20916            crate::MessageElement::Static(
20917                64,
20918                crate::MessageDataType::UInt8,
20919                crate::MessageClass::Data,
20920            ),
20921            &[ep],
20922            crate::ReliableMode::None,
20923            0,
20924        )
20925        .unwrap();
20926        let slow_ty = crate::config::register_data_type_with_description(
20927            "ROUTER_STREAM_QUEUE_SLOW",
20928            "",
20929            crate::MessageElement::Static(
20930                64,
20931                crate::MessageDataType::UInt8,
20932                crate::MessageClass::Data,
20933            ),
20934            &[ep],
20935            crate::ReliableMode::None,
20936            0,
20937        )
20938        .unwrap();
20939        for mode in 0..3 {
20940            for tx in [false, true] {
20941                for slow_last in [false, true] {
20942                    let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
20943                    let make = |sender| {
20944                        let packet = Packet::new(
20945                            if sender == "SLOW" { slow_ty } else { ty },
20946                            &[ep],
20947                            "SAME_BOARD",
20948                            1,
20949                            Arc::<[u8]>::from([1; 64]),
20950                        )
20951                        .unwrap();
20952                        if mode == 1 {
20953                            RouterItem::Packet(packet)
20954                        } else {
20955                            RouterItem::Packed(crate::shared_bytes::convert(
20956                                wire_format::pack_packet(&packet),
20957                            ))
20958                        }
20959                    };
20960                    let rx = |data| RouterRxItem {
20961                        src: None,
20962                        data,
20963                        priority: 0,
20964                    };
20965                    let tx_item = |data| TxQueued {
20966                        disposable: TX_DISPOSABLE,
20967                        item: RouterTxItem::Broadcast(data),
20968                        ignore_local: true,
20969                        priority: 0,
20970                        delivery_state: QueuedDelivery::Untracked,
20971                        accepted_type: 0,
20972                        deferred_since_ms: 0,
20973                        retry_after_ms: 0,
20974                    };
20975                    let mut st = node.state.lock();
20976                    st.memory.max_queue_budget = usize::MAX;
20977                    if mode == 0 {
20978                        if tx {
20979                            st.transmit_queue =
20980                                BoundedDeque::new(8192, 3 * size_of::<TxQueued>(), 1.0);
20981                        } else {
20982                            st.received_queue =
20983                                BoundedDeque::new(8192, 3 * size_of::<RouterRxItem>(), 1.0);
20984                        }
20985                    } else if mode == 1 {
20986                        let cost = if tx {
20987                            tx_item(make("SLOW")).byte_cost()
20988                                + 2 * tx_item(make("FAST")).byte_cost()
20989                        } else {
20990                            rx(make("SLOW")).byte_cost() + 2 * rx(make("FAST")).byte_cost()
20991                        };
20992                        if tx {
20993                            st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
20994                        } else {
20995                            st.received_queue = BoundedDeque::new(cost, cost, 2.0);
20996                        }
20997                    }
20998                    let order = if slow_last {
20999                        ["FAST", "FAST", "SLOW"]
21000                    } else {
21001                        ["SLOW", "FAST", "FAST"]
21002                    };
21003                    for (index, sender) in order
21004                        .into_iter()
21005                        .chain(core::iter::repeat_n("FAST", 32))
21006                        .enumerate()
21007                    {
21008                        if mode == 2 && index == 3 {
21009                            st.memory.max_queue_budget = st.shared_queue_bytes_used();
21010                        }
21011                        if tx {
21012                            st.push_transmit(tx_item(make(sender))).unwrap();
21013                        } else {
21014                            st.push_received(rx(make(sender))).unwrap();
21015                        }
21016                    }
21017                    let is_slow = |data: &RouterItem| match data {
21018                        RouterItem::Packet(p) => p.data_type() == slow_ty,
21019                        RouterItem::Packed(b) => {
21020                            wire_format::scheduling_type(b.as_ref()) == Some(slow_ty)
21021                        }
21022                    };
21023                    let retained = if tx {
21024                        st.transmit_queue.iter().any(|item| {
21025                            let data = match &item.item {
21026                                RouterTxItem::Broadcast(data)
21027                                | RouterTxItem::ToSide { data, .. } => data,
21028                                _ => return false,
21029                            };
21030                            is_slow(data)
21031                        })
21032                    } else {
21033                        st.received_queue.iter().any(|item| is_slow(&item.data))
21034                    };
21035                    assert!(
21036                        retained,
21037                        "slow stream displaced: mode={mode}, tx={tx}, slow_last={slow_last}"
21038                    );
21039                }
21040            }
21041        }
21042    }
21043
21044    #[test]
21045    fn fast_producer_preserves_slow_reports_at_each_queue_boundary() {
21046        let ep = crate::config::register_endpoint("QUEUE_FAIR_DEST", false).unwrap();
21047        let ty = crate::config::register_data_type_with_description(
21048            "QUEUE_FAIR_REPORT",
21049            "",
21050            MessageElement::Static(64, crate::MessageDataType::UInt8, crate::MessageClass::Data),
21051            &[ep],
21052            crate::ReliableMode::None,
21053            0,
21054        )
21055        .unwrap();
21056        for mode in 0..3 {
21057            for tx in [false, true] {
21058                for slow_last in [false, true] {
21059                    for next_sender in ["FAST", "NEW_SLOW"] {
21060                        let router =
21061                            Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
21062                        let make = |sender| {
21063                            RouterItem::Packet(
21064                                Packet::new(ty, &[ep], sender, 1, Arc::<[u8]>::from([1; 64]))
21065                                    .unwrap(),
21066                            )
21067                        };
21068                        let rx = |data| RouterRxItem {
21069                            src: None,
21070                            data,
21071                            priority: 0,
21072                        };
21073                        let tx_item = |data| TxQueued {
21074                            disposable: TX_DISPOSABLE,
21075                            item: RouterTxItem::Broadcast(data),
21076                            ignore_local: true,
21077                            priority: 0,
21078                            delivery_state: QueuedDelivery::Untracked,
21079                            accepted_type: 0,
21080                            deferred_since_ms: 0,
21081                            retry_after_ms: 0,
21082                        };
21083                        let order = if slow_last {
21084                            ["FAST", "FAST", "SLOW"]
21085                        } else {
21086                            ["SLOW", "FAST", "FAST"]
21087                        };
21088                        let mut st = router.state.lock();
21089                        st.memory.max_queue_budget = usize::MAX;
21090                        let cost = if tx {
21091                            order.iter().map(|s| tx_item(make(s)).byte_cost()).sum()
21092                        } else {
21093                            order.iter().map(|s| rx(make(s)).byte_cost()).sum()
21094                        };
21095                        if mode == 0 {
21096                            if tx {
21097                                st.transmit_queue = BoundedDeque::new(cost, cost, 2.0);
21098                            } else {
21099                                st.received_queue = BoundedDeque::new(cost, cost, 2.0);
21100                            }
21101                        } else if mode == 1 {
21102                            if tx {
21103                                st.transmit_queue =
21104                                    BoundedDeque::new(cost * 10, 3 * size_of::<TxQueued>(), 1.0);
21105                            } else {
21106                                st.received_queue = BoundedDeque::new(
21107                                    cost * 10,
21108                                    3 * size_of::<RouterRxItem>(),
21109                                    1.0,
21110                                );
21111                            }
21112                        }
21113                        for sender in order {
21114                            if tx {
21115                                st.push_transmit(tx_item(make(sender))).unwrap();
21116                            } else {
21117                                st.push_received(rx(make(sender))).unwrap();
21118                            }
21119                        }
21120                        if mode == 2 {
21121                            st.memory.max_queue_budget = st.shared_queue_bytes_used();
21122                        }
21123                        if tx {
21124                            st.push_transmit(tx_item(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
21125                        } else {
21126                            st.push_received(rx(make(next_sender))).unwrap_or_else(|error| panic!("admit error {error:?}: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"));
21127                        }
21128                        let slow = wire_format::source_address_for_sender("SLOW");
21129                        let retained = if tx {
21130                            st.transmit_queue
21131                                .iter()
21132                                .any(|item| RouterInner::tx_bulk_owner(item) == Some(slow))
21133                        } else {
21134                            st.received_queue.iter().any(|item| {
21135                                RouterInner::fair_bulk_owner(&item.data, item.priority)
21136                                    == Some(slow)
21137                            })
21138                        };
21139                        assert!(
21140                            retained,
21141                            "slow report evicted: mode={mode} tx={tx} slow_last={slow_last} incoming={next_sender}"
21142                        );
21143                    }
21144                }
21145            }
21146        }
21147    }
21148}
21149
21150#[cfg(all(test, feature = "discovery"))]
21151#[test]
21152fn cow_discovery_rejected_growth_keeps_original_graph_and_routes() {
21153    crate::tests::ensure_common_test_schema();
21154    let node = Router::new_with_clock(RouterConfig::default().with_sender("GB"), Box::new(|| 0));
21155    let side = node.add_side_packed("can", |_| Ok(()));
21156    let baseline = discovery::build_discovery_topology(
21157        "PEER",
21158        0,
21159        &[TopologyBoardNode {
21160            sender_id: "PEER".into(),
21161            reachable_endpoints: Vec::new(),
21162            reachable_timesync_sources: Vec::new(),
21163            connections: Vec::new(),
21164        }],
21165    )
21166    .unwrap();
21167    node.rx_from_side(&baseline, side).unwrap();
21168    node.process_all_queues().unwrap();
21169    node.clear_queues();
21170    let before = node.export_topology();
21171    let original = {
21172        let mut st = node.state.lock();
21173        let graph = st.discovery_routes[&side].announcers["PEER"]
21174            .topology_boards
21175            .clone();
21176        st.memory.max_queue_budget = st.shared_queue_bytes_used();
21177        graph
21178    };
21179    let update = discovery::build_discovery_topology_delta(
21180        "PEER",
21181        1,
21182        &[TopologyBoardNode {
21183            sender_id: "PEER".into(),
21184            reachable_endpoints: Vec::new(),
21185            reachable_timesync_sources: vec!["NEW_CLOCK".into()],
21186            connections: vec!["NEW_NEIGHBOR".into()],
21187        }],
21188        &[],
21189    )
21190    .unwrap();
21191    assert!(
21192        node.rx_from_side(&update, side)
21193            .and_then(|_| node.process_all_queues())
21194            .is_err(),
21195        "growth must fail when no queue budget can be reclaimed"
21196    );
21197    assert_eq!(node.export_topology().routes, before.routes);
21198    let st = node.state.lock();
21199    assert!(Arc::ptr_eq(
21200        &original,
21201        &st.discovery_routes[&side].announcers["PEER"].topology_boards
21202    ));
21203    assert!(original[0].connections.is_empty());
21204    assert!(original[0].reachable_timesync_sources.is_empty());
21205}
21206
21207#[cfg(all(test, feature = "std"))]
21208mod bounded_rx_priority_tests {
21209    use super::*;
21210    #[test]
21211    fn retained_slow_report_cannot_starve_behind_continuous_fast_data() {
21212        let ep =
21213            crate::config::register_endpoint("RouterRxItem_FAIR_EP_RouterInner", false).unwrap();
21214        let ty = crate::config::register_data_type_with_description(
21215            "RouterRxItem_FAIR_DATA_RouterInner",
21216            "",
21217            crate::MessageElement::Static(
21218                1,
21219                crate::MessageDataType::Float32,
21220                crate::MessageClass::Data,
21221            ),
21222            &[ep],
21223            crate::ReliableMode::None,
21224            0,
21225        )
21226        .unwrap();
21227        let node = Router::new_with_clock(RouterConfig::default(), Box::new(|| 0));
21228        let mut st = node.state.lock();
21229        st.memory.max_queue_budget = 65536;
21230        st.received_queue = BoundedDeque::new(32768, 16 * size_of::<RouterRxItem>(), 1.0);
21231        let make = |sender: &str, stamp, priority| {
21232            let packet = Packet::new(ty, &[ep], sender, stamp, Arc::<[u8]>::from([0; 4])).unwrap();
21233            RouterRxItem {
21234                src: None,
21235                data: RouterItem::Packet(packet),
21236                priority,
21237            }
21238        };
21239        st.push_received(make("VALVE", 0, 0)).unwrap();
21240        let mut delivered = false;
21241        for tick in 1..=32u64 {
21242            for index in 0..3 {
21243                st.push_received(make("DAQ", tick * 3 + index, 16)).unwrap();
21244            }
21245            let item = st.pop_received_fair().unwrap();
21246            let sender = match &item.data {
21247                RouterItem::Packet(packet) => packet.sender(),
21248                _ => panic!("expected packet"),
21249            };
21250            if sender == "VALVE" {
21251                delivered = true;
21252                assert!(tick <= u64::from(MAX_PRIORITY_BURST) + 1);
21253                break;
21254            }
21255        }
21256        assert!(
21257            delivered,
21258            "retention alone does not deliver low-rate reports under a continuous higher-priority source"
21259        );
21260        st.received_queue.clear();
21261        st.rx_priority_burst = MAX_PRIORITY_BURST;
21262        st.push_received(make("VALVE", 100, 0)).unwrap();
21263        st.push_received(make("VALVE", 101, 0)).unwrap();
21264        st.push_received(make("DAQ", 102, 16)).unwrap();
21265        st.push_received(make("COMMAND", 103, 254)).unwrap();
21266        assert_eq!(
21267            st.pop_received_fair().unwrap().priority,
21268            254,
21269            "a fairness turn must never delay protected control"
21270        );
21271        let mut first_slow = None;
21272        for tick in 0..=MAX_PRIORITY_BURST {
21273            st.push_received(make("DAQ", 200 + u64::from(tick), 16))
21274                .unwrap();
21275            let item = st.pop_received_fair().unwrap();
21276            if let RouterItem::Packet(packet) = &item.data
21277                && packet.sender() == "VALVE"
21278            {
21279                first_slow = Some(packet.timestamp());
21280                break;
21281            }
21282        }
21283        assert_eq!(
21284            first_slow,
21285            Some(100),
21286            "fairness preserves same-priority FIFO order"
21287        );
21288        st.rx_priority_burst = MAX_PRIORITY_BURST;
21289        drop(st);
21290        node.clear_rx_queue();
21291        assert_eq!(node.state.lock().rx_priority_burst, 0);
21292    }
21293}
21294
21295#[cfg(all(test, feature = "std", feature = "discovery"))]
21296#[path = "tests/accepted_tx_ownership.rs"]
21297mod accepted_tx_ownership;
21298
21299#[cfg(feature = "latest-value-egress")]
21300#[path = "router_latest.rs"]
21301mod latest;
21302
21303#[cfg(all(test, feature = "std", feature = "discovery"))]
21304#[path = "tests/retry_backpressure_native.rs"]
21305mod retry_backpressure_native;
21306
21307#[path = "router_e2e_backpressure.rs"]
21308mod e2e_backpressure;
21309#[cfg(all(test, feature = "std", feature = "discovery", feature = "timesync"))]
21310#[path = "tests/peer_record_footprint.rs"]
21311mod peer_record_footprint;
21312
21313#[cfg(feature = "std")]
21314#[path = "router_completion.rs"]
21315mod completion;
21316#[cfg(feature = "std")]
21317pub use completion::{PackedWriteLease, RetryTracker, RetryTrackerLimits, WriteStart};
21318
21319#[cfg(all(test, feature = "std", feature = "discovery"))]
21320#[path = "tests/actual_bootstrap_arbitration.rs"]
21321mod actual_bootstrap_arbitration;
21322
21323#[cfg(all(test, feature = "discovery", feature = "std"))]
21324#[path = "tests/router_managed_refresh.rs"]
21325mod managed_refresh_tests;
21326
21327#[cfg(all(test, feature = "std", feature = "discovery"))]
21328#[path = "tests/pending_type_metadata.rs"]
21329mod pending_type_metadata;
21330#[cfg(all(test, feature = "std", feature = "discovery"))]
21331#[path = "tests/sender_identity_match.rs"]
21332mod sender_identity_match;
21333
21334#[cfg(all(test, feature = "discovery", feature = "std"))]
21335#[path = "tests/router_cold_cycle_trace.rs"]
21336mod router_cold_cycle_trace;
21337
21338#[cfg(all(test, feature = "std", feature = "discovery"))]
21339#[path = "tests/toside_inspection_metadata.rs"]
21340mod toside_inspection_metadata;
21341
21342#[cfg(all(test, feature = "std", feature = "discovery"))]
21343#[path = "tests/active_queued_types.rs"]
21344mod active_queued_types;