1use std::collections::HashMap;
13use std::hash::{BuildHasher, Hasher};
14use std::io;
15#[cfg(test)]
16use std::io::Write;
17use std::net::{IpAddr, Shutdown, TcpListener, TcpStream, ToSocketAddrs};
18use std::os::unix::io::AsRawFd;
19use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
20use std::sync::{Arc, Mutex, OnceLock};
21use std::thread;
22use std::time::{Duration, Instant};
23
24use socket2::{SockRef, TcpKeepalive};
25
26use rns_core::constants;
27use rns_core::transport::types::{IngressControlConfig, InterfaceId, InterfaceInfo};
28
29use crate::event::{Event, EventSender};
30use crate::event::{LoopContext, LoopSource, SourceStatus};
31use crate::hdlc;
32use crate::interface::loop_writer::LoopEgress;
33use crate::interface::tcp::{read_stream_frames, StreamRead};
34use crate::interface::transmit_buffer::{
35 EgressController, TransmitBuffer, EGRESS_DEAD_TIME, EGRESS_HIGH_WATERMARK,
36};
37use crate::interface::{
38 auto_mtu_for_bitrate, lock_or_recover, wait_while_active, DynamicInterfaceTemplate,
39 EgressControl, InterfaceConfigData, InterfaceFactory, ListenerControl, StartContext,
40 StartResult, Writer,
41};
42use crate::BackbonePeerStateEntry;
43
44pub(crate) const SERVER_BITRATE_GUESS: u64 = 100_000_000;
46pub(crate) const CLIENT_BITRATE_GUESS: u64 = 100_000_000;
48
49pub(crate) const HW_MTU: usize = match auto_mtu_for_bitrate(CLIENT_BITRATE_GUESS) {
51 Some(mtu) => mtu as usize,
52 None => 1_048_576,
53};
54pub(crate) const SERVER_PEER_MTU: u32 = match auto_mtu_for_bitrate(SERVER_BITRATE_GUESS) {
56 Some(mtu) => mtu,
57 None => 1_048_576,
58};
59
60const DP_IC_HIGH_WM_PCT: usize = 90;
61const DP_IC_MID_WM_PCT: usize = 68;
62const DP_IC_LOW_WM_PCT: usize = 10;
63const DP_IC_INTERVAL: Duration = Duration::from_millis(250);
64const DP_IC_RCVBUF: usize = 32_768;
65const DP_IC_IF_HEADROOM: usize = 32;
66const DP_IC_PENALTY: f64 = 1.5;
67
68#[derive(Debug, Clone)]
70pub struct BackboneConfig {
71 pub name: String,
72 pub listen_ip: String,
73 pub listen_port: u16,
74 pub interface_id: InterfaceId,
75 pub mode: u8,
76 pub gravity: i64,
77 pub recursive_prs: bool,
78 pub announces_from_internal: bool,
79 pub announces_to_internal: Option<bool>,
80 pub max_connections: Option<usize>,
81 pub idle_timeout: Option<Duration>,
82 pub write_stall_timeout: Option<Duration>,
83 pub abuse: BackboneAbuseConfig,
84 pub fast_flap: BackboneFastFlapConfig,
85 pub fast_flap_state: Arc<Mutex<BackboneFastFlapMonitor>>,
86 pub ingress_control: IngressControlConfig,
87 pub runtime: Arc<Mutex<BackboneServerRuntime>>,
88 pub peer_state: Arc<Mutex<BackbonePeerMonitor>>,
89}
90
91#[derive(Debug, Clone, Default)]
93pub struct BackboneAbuseConfig {
94 pub max_penalty_duration: Option<Duration>,
95}
96
97#[derive(Debug, Clone, Copy, Eq, PartialEq)]
100pub struct BackboneFastFlapConfig {
101 pub enabled: bool,
102 pub connection_threshold: Duration,
103 pub grace: u64,
104 pub block_duration: Duration,
105}
106
107impl Default for BackboneFastFlapConfig {
108 fn default() -> Self {
109 Self {
110 enabled: true,
111 connection_threshold: Duration::from_secs(20),
112 grace: 5,
113 block_duration: Duration::from_secs(12 * 60 * 60),
114 }
115 }
116}
117
118#[derive(Debug, Clone)]
119struct FastFlapPeerState {
120 first_flap_at: Instant,
121 last_flap_at: Instant,
122 flap_count: u64,
123}
124
125#[derive(Debug, Clone, Copy, Eq, PartialEq)]
127pub struct FastFlapRecord {
128 pub flap_count: u64,
129 pub blocked: bool,
130 pub newly_blocked: bool,
131 pub blocked_until: Option<Instant>,
132 pub elapsed_since_first: Duration,
133}
134
135#[derive(Debug, Default)]
138pub struct BackboneFastFlapMonitor {
139 peers: HashMap<IpAddr, FastFlapPeerState>,
140}
141
142impl BackboneFastFlapMonitor {
143 pub fn new() -> Self {
144 Self::default()
145 }
146
147 pub fn record_disconnect(
148 &mut self,
149 peer_ip: IpAddr,
150 connected_for: Duration,
151 config: &BackboneFastFlapConfig,
152 now: Instant,
153 ) -> Option<FastFlapRecord> {
154 if !config.enabled || connected_for >= config.connection_threshold {
155 return None;
156 }
157
158 self.expire_peer(peer_ip, config, now);
159 let state = self.peers.entry(peer_ip).or_insert(FastFlapPeerState {
160 first_flap_at: now,
161 last_flap_at: now,
162 flap_count: 0,
163 });
164 let was_blocked = state.flap_count > config.grace;
165 state.last_flap_at = now;
166 state.flap_count = state.flap_count.saturating_add(1);
167 let blocked = state.flap_count > config.grace;
168
169 Some(FastFlapRecord {
170 flap_count: state.flap_count,
171 blocked,
172 newly_blocked: blocked && !was_blocked,
173 blocked_until: blocked
174 .then(|| now.checked_add(config.block_duration))
175 .flatten(),
176 elapsed_since_first: now.saturating_duration_since(state.first_flap_at),
177 })
178 }
179
180 pub fn is_blocked(
181 &mut self,
182 peer_ip: IpAddr,
183 config: &BackboneFastFlapConfig,
184 now: Instant,
185 ) -> bool {
186 self.blocked_until(peer_ip, config, now).is_some()
187 }
188
189 pub fn blocked_until(
190 &mut self,
191 peer_ip: IpAddr,
192 config: &BackboneFastFlapConfig,
193 now: Instant,
194 ) -> Option<Instant> {
195 if !config.enabled {
196 return None;
197 }
198 self.expire_peer(peer_ip, config, now);
199 self.peers
200 .get(&peer_ip)
201 .filter(|state| state.flap_count > config.grace)
202 .and_then(|state| state.last_flap_at.checked_add(config.block_duration))
203 }
204
205 pub fn blocked_ip_count(&mut self, config: &BackboneFastFlapConfig, now: Instant) -> usize {
206 if !config.enabled {
207 return 0;
208 }
209 self.expire_all(config, now);
210 self.peers
211 .values()
212 .filter(|state| state.flap_count > config.grace)
213 .count()
214 }
215
216 pub fn blocked_ip_list(
217 &mut self,
218 config: &BackboneFastFlapConfig,
219 now: Instant,
220 ) -> Vec<IpAddr> {
221 if !config.enabled {
222 return Vec::new();
223 }
224 self.expire_all(config, now);
225 let mut blocked: Vec<_> = self
226 .peers
227 .iter()
228 .filter(|(_, state)| state.flap_count > config.grace)
229 .map(|(peer_ip, _)| *peer_ip)
230 .collect();
231 blocked.sort_unstable();
232 blocked
233 }
234
235 pub fn flap_count(&self, peer_ip: IpAddr) -> u64 {
236 self.peers
237 .get(&peer_ip)
238 .map(|state| state.flap_count)
239 .unwrap_or(0)
240 }
241
242 pub fn clear(&mut self, peer_ip: IpAddr) -> bool {
243 self.peers.remove(&peer_ip).is_some()
244 }
245
246 fn expire_peer(&mut self, peer_ip: IpAddr, config: &BackboneFastFlapConfig, now: Instant) {
247 let expired = self.peers.get(&peer_ip).is_some_and(|state| {
248 now.saturating_duration_since(state.last_flap_at) > config.block_duration
249 });
250 if expired {
251 self.peers.remove(&peer_ip);
252 }
253 }
254
255 fn expire_all(&mut self, config: &BackboneFastFlapConfig, now: Instant) {
256 self.peers.retain(|_, state| {
257 now.saturating_duration_since(state.last_flap_at) <= config.block_duration
258 });
259 }
260}
261
262fn shared_fast_flap_monitor() -> Arc<Mutex<BackboneFastFlapMonitor>> {
263 static MONITOR: OnceLock<Arc<Mutex<BackboneFastFlapMonitor>>> = OnceLock::new();
264 Arc::clone(MONITOR.get_or_init(|| Arc::new(Mutex::new(BackboneFastFlapMonitor::new()))))
265}
266
267#[derive(Debug, Clone)]
269pub struct BackboneServerRuntime {
270 pub max_connections: Option<usize>,
271 pub idle_timeout: Option<Duration>,
272 pub write_stall_timeout: Option<Duration>,
273 pub abuse: BackboneAbuseConfig,
274}
275
276impl BackboneServerRuntime {
277 pub fn from_config(config: &BackboneConfig) -> Self {
278 Self {
279 max_connections: config.max_connections,
280 idle_timeout: config.idle_timeout,
281 write_stall_timeout: config.write_stall_timeout,
282 abuse: config.abuse.clone(),
283 }
284 }
285}
286
287#[derive(Debug, Clone)]
288pub struct BackboneRuntimeConfigHandle {
289 pub interface_name: String,
290 pub runtime: Arc<Mutex<BackboneServerRuntime>>,
291 pub startup: BackboneServerRuntime,
292}
293
294#[derive(Debug, Clone)]
295pub struct BackbonePeerStateHandle {
296 pub interface_id: InterfaceId,
297 pub interface_name: String,
298 pub mode: u8,
299 pub gravity: i64,
300 pub announces_to_internal: Option<bool>,
301 pub peer_state: Arc<Mutex<BackbonePeerMonitor>>,
302 pub fast_flap: BackboneFastFlapConfig,
303 pub fast_flap_state: Arc<Mutex<BackboneFastFlapMonitor>>,
304}
305
306impl Default for BackboneConfig {
307 fn default() -> Self {
308 let mut config = BackboneConfig {
309 name: String::new(),
310 listen_ip: "0.0.0.0".into(),
311 listen_port: 0,
312 interface_id: InterfaceId(0),
313 mode: constants::MODE_FULL,
314 gravity: 0,
315 recursive_prs: false,
316 announces_from_internal: true,
317 announces_to_internal: None,
318 max_connections: None,
319 idle_timeout: None,
320 write_stall_timeout: None,
321 abuse: BackboneAbuseConfig::default(),
322 fast_flap: BackboneFastFlapConfig::default(),
323 fast_flap_state: shared_fast_flap_monitor(),
324 ingress_control: IngressControlConfig::enabled(),
325 runtime: Arc::new(Mutex::new(BackboneServerRuntime {
326 max_connections: None,
327 idle_timeout: None,
328 write_stall_timeout: None,
329 abuse: BackboneAbuseConfig::default(),
330 })),
331 peer_state: Arc::new(Mutex::new(BackbonePeerMonitor::new())),
332 };
333 let startup = BackboneServerRuntime::from_config(&config);
334 config.runtime = Arc::new(Mutex::new(startup));
335 config
336 }
337}
338
339struct BackboneWriter {
346 stream: TcpStream,
347 runtime: Arc<Mutex<BackboneServerRuntime>>,
348 interface_name: String,
349 interface_id: InterfaceId,
350 event_tx: EventSender,
351 transmit_buffer: TransmitBuffer,
352 egress_controller: EgressController,
353 egress_control: EgressControl,
354 egress_started: Instant,
355 stall_started: Option<Instant>,
356 disconnect_notified: bool,
357 write_stall_flag: Arc<AtomicBool>,
358}
359
360impl Writer for BackboneWriter {
361 fn send_frame_confirmed(&mut self, data: &[u8]) -> io::Result<()> {
362 if !self
365 .transmit_buffer
366 .append_with_limit(hdlc::frame(data), Some(EGRESS_HIGH_WATERMARK))
367 {
368 return Err(io::Error::new(
369 io::ErrorKind::InvalidInput,
370 "confirmed frame exceeds backbone buffer",
371 ));
372 }
373 self.transmit_buffer.flush();
374 let timeout = lock_or_recover(&self.runtime, "backbone runtime").write_stall_timeout;
375 self.drain_transmit(timeout)
376 }
377 fn send_frame(&mut self, data: &[u8]) -> io::Result<()> {
378 self.send_frames(&[crate::interface::FrameBytes::Owned(data.to_vec())])
379 }
380
381 fn send_frames(&mut self, frames: &[crate::interface::FrameBytes]) -> io::Result<()> {
382 let write_stall_timeout =
383 lock_or_recover(&self.runtime, "backbone runtime").write_stall_timeout;
384 if self.transmit_buffer.buffered_bytes() > 0 {
385 self.drain_transmit(write_stall_timeout)?;
386 if self.transmit_buffer.buffered_bytes() > 0 {
387 return Err(io::Error::new(
388 io::ErrorKind::WouldBlock,
389 "backbone writer still stalled",
390 ));
391 }
392 }
393
394 let elapsed = self.egress_started.elapsed();
395 let _ = self.egress_controller.evaluate(
396 elapsed,
397 self.transmit_buffer.buffered_bytes(),
398 self.transmit_buffer.sendable_bytes(),
399 self.transmit_buffer.total_sent_bytes(),
400 );
401 self.egress_control
402 .set_stalled(self.egress_controller.stalled());
403 for frame in frames {
404 let encoded = hdlc::frame(frame);
405 let encoded_len = encoded.len();
406 if self.egress_control.stalled()
407 || !self
408 .transmit_buffer
409 .append_with_limit(encoded, Some(EGRESS_HIGH_WATERMARK))
410 {
411 self.egress_control.record_drop(encoded_len);
412 log::debug!(
413 "[{}:{}] egress control dropped outbound frame of {} bytes",
414 self.interface_name,
415 self.interface_id.0,
416 encoded_len
417 );
418 }
419 }
420 self.transmit_buffer.flush();
421 self.drain_transmit(write_stall_timeout)
422 }
423
424 fn flush(&mut self) -> io::Result<()> {
425 let timeout = lock_or_recover(&self.runtime, "backbone runtime").write_stall_timeout;
426 self.drain_transmit(timeout)
427 }
428
429 fn egress_control(&self) -> Option<EgressControl> {
430 Some(self.egress_control.clone())
431 }
432
433 fn loop_socket(&self) -> Option<std::os::fd::RawFd> {
434 Some(self.stream.as_raw_fd())
435 }
436
437 fn loop_egress(&mut self) -> Option<LoopEgress> {
438 let runtime = Arc::clone(&self.runtime);
439 Some(
440 LoopEgress::budget(self.egress_control.clone()).with_stall_watch(
441 move || lock_or_recover(&runtime, "backbone runtime").write_stall_timeout,
442 Arc::clone(&self.write_stall_flag),
443 ),
444 )
445 }
446
447 fn shutdown(&mut self) {
448 let _ = self.stream.shutdown(Shutdown::Both);
449 }
450}
451
452impl BackboneWriter {
453 fn drain_transmit(&mut self, write_stall_timeout: Option<Duration>) -> io::Result<()> {
454 if self.transmit_buffer.buffered_bytes() == 0 {
455 return Ok(());
456 }
457
458 let written = self.transmit_buffer.drain_to(&mut self.stream)?;
459 if written > 0 {
460 self.stall_started = None;
461 }
462 let elapsed = self.egress_started.elapsed();
463 let dead = self.egress_controller.evaluate(
464 elapsed,
465 self.transmit_buffer.buffered_bytes(),
466 self.transmit_buffer.sendable_bytes(),
467 self.transmit_buffer.total_sent_bytes(),
468 );
469 self.egress_control
470 .set_stalled(self.egress_controller.stalled());
471 if dead {
472 return Err(self.disconnect_for_write_stall(EGRESS_DEAD_TIME));
473 }
474 if self.transmit_buffer.buffered_bytes() == 0 {
475 return Ok(());
476 }
477
478 let now = Instant::now();
479 let started = self.stall_started.get_or_insert(now);
480 let timeout = write_stall_timeout.unwrap_or(EGRESS_DEAD_TIME);
481 if now.duration_since(*started) >= timeout {
482 return Err(self.disconnect_for_write_stall(timeout));
483 }
484 Err(io::Error::new(
485 io::ErrorKind::WouldBlock,
486 "backbone writer would block",
487 ))
488 }
489
490 fn disconnect_for_write_stall(&mut self, timeout: Duration) -> io::Error {
491 if !self.disconnect_notified {
492 log::warn!(
493 "[{}] backbone client {} disconnected due to write stall timeout ({:?})",
494 self.interface_name,
495 self.interface_id.0,
496 timeout
497 );
498 self.write_stall_flag.store(true, Ordering::Relaxed);
499 let _ = self.stream.shutdown(Shutdown::Both);
500 let _ = self.event_tx.send(Event::InterfaceDown(self.interface_id));
501 self.disconnect_notified = true;
502 }
503 io::Error::new(
504 io::ErrorKind::TimedOut,
505 format!("backbone writer stalled for {:?}", timeout),
506 )
507 }
508}
509
510pub fn start(config: BackboneConfig, tx: EventSender, next_id: Arc<AtomicU64>) -> io::Result<()> {
513 start_with_template(config, tx, next_id, None, None)
514}
515
516fn start_with_template(
517 config: BackboneConfig,
518 tx: EventSender,
519 next_id: Arc<AtomicU64>,
520 dynamic_template: Option<DynamicInterfaceTemplate>,
521 underlay_mark: Option<u32>,
522) -> io::Result<()> {
523 let addr = format!("{}:{}", config.listen_ip, config.listen_port);
524 let listener = TcpListener::bind(&addr)?;
525 super::apply_underlay_mark(listener.as_raw_fd(), underlay_mark)?;
526 listener.set_nonblocking(true)?;
527
528 log::info!(
529 "[{}] backbone server listening on {}",
530 config.name,
531 listener
532 .local_addr()
533 .unwrap_or_else(|_| std::net::SocketAddr::from(([0, 0, 0, 0], config.listen_port)))
534 );
535
536 let ifac_size = dynamic_template
537 .as_ref()
538 .and_then(|template| template.ifac.as_ref())
539 .map(|ifac| ifac.size)
540 .unwrap_or(0);
541 let ingress_thresholds = ingress_thresholds(tx.inbound_queue_snapshot().capacities[0]);
544 let server = Arc::new(Server {
545 name: config.name.clone(),
546 server_interface_id: config.interface_id,
547 tx: tx.clone(),
548 next_id,
549 runtime: Arc::clone(&config.runtime),
550 peer_state: Arc::clone(&config.peer_state),
551 fast_flap: config.fast_flap,
552 fast_flap_state: Arc::clone(&config.fast_flap_state),
553 ingress_control: config.ingress_control,
554 accepted_peer_mode: config.mode,
555 accepted_peer_gravity: config.gravity,
556 accepted_peer_recursive_prs: config.recursive_prs,
557 accepted_peer_announces_from_internal: config.announces_from_internal,
558 accepted_peer_announces_to_internal: config.announces_to_internal,
559 dynamic_template,
560 ifac_size,
561 underlay_mark,
562 ingress_thresholds,
563 state: Mutex::new(ServerState {
564 clients: HashMap::new(),
565 peers: HashMap::new(),
566 ingress_snapshot_at: Instant::now(),
567 }),
568 });
569 tx.register_source(
570 crate::interface::loop_writer::read_key(config.interface_id),
571 Box::new(ListenerSource {
572 listener,
573 server,
574 next_interval: Instant::now() + DP_IC_INTERVAL,
575 }),
576 )
577}
578
579const ACCEPT_BURST: usize = 64;
581
582struct Server {
586 name: String,
587 server_interface_id: InterfaceId,
588 tx: EventSender,
589 next_id: Arc<AtomicU64>,
590 runtime: Arc<Mutex<BackboneServerRuntime>>,
591 peer_state: Arc<Mutex<BackbonePeerMonitor>>,
592 fast_flap: BackboneFastFlapConfig,
593 fast_flap_state: Arc<Mutex<BackboneFastFlapMonitor>>,
594 ingress_control: IngressControlConfig,
595 accepted_peer_mode: u8,
596 accepted_peer_gravity: i64,
597 accepted_peer_recursive_prs: bool,
598 accepted_peer_announces_from_internal: bool,
599 accepted_peer_announces_to_internal: Option<bool>,
600 dynamic_template: Option<DynamicInterfaceTemplate>,
601 ifac_size: usize,
602 underlay_mark: Option<u32>,
603 ingress_thresholds: IngressThresholds,
604 state: Mutex<ServerState>,
605}
606
607struct ServerState {
608 clients: HashMap<usize, ClientState>,
610 peers: HashMap<IpAddr, PeerBehaviorState>,
611 ingress_snapshot_at: Instant,
612}
613
614struct ClientState {
616 id: InterfaceId,
617 peer_ip: IpAddr,
618 peer_port: u16,
619 fd: std::os::fd::RawFd,
622 connected_at: Instant,
623 has_received_data: bool,
624 write_stall_flag: Arc<AtomicBool>,
625 idle_timed_out: bool,
628 dp_ingress_bytes: u64,
629 dp_ingress_packets: u64,
630 dp_ingress_hold: Option<Instant>,
631 dp_ingress_gated: bool,
632 dp_ingress_throttle_count: u64,
633}
634
635impl Server {
636 fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> {
637 lock_or_recover(&self.state, "backbone server state")
638 }
639
640 fn sync_peer_state(&self, state: &mut ServerState) {
643 let mut monitor = lock_or_recover(&self.peer_state, "backbone peer state");
644 monitor.sync_into(&mut state.peers);
645 cleanup_peer_state(&mut state.peers);
646 monitor.upsert_snapshot(&state.peers);
647 }
648
649 fn set_readable(&self, key: usize, fd: std::os::fd::RawFd, readable: bool) {
651 let source = unsafe { std::os::fd::BorrowedFd::borrow_raw(fd) };
653 if let Err(error) = self.tx.set_source_readable(source, key, readable) {
654 log::warn!(
655 "[{}] cannot update peer read interest: {}",
656 self.name,
657 error
658 );
659 }
660 }
661
662 fn throttle_busiest_ingress(&self, state: &mut ServerState) -> Option<usize> {
665 let key = busiest_ingress_key(
666 state
667 .clients
668 .iter()
669 .map(|(key, client)| (*key, client.dp_ingress_packets)),
670 )?;
671 let allocation_divisor = state.clients.len().max(DP_IC_IF_HEADROOM) as f64 * DP_IC_PENALTY;
672 let sample_span = state.ingress_snapshot_at.elapsed();
673 let client = state.clients.get_mut(&key)?;
674 if client.dp_ingress_gated {
675 return None;
676 }
677 let hold = Duration::from_secs_f64(
678 (sample_span.as_secs_f64().max(0.001) * allocation_divisor).max(0.001),
679 );
680 client.dp_ingress_gated = true;
681 client.dp_ingress_hold = Some(Instant::now() + hold);
682 client.dp_ingress_throttle_count = client.dp_ingress_throttle_count.saturating_add(1);
683 log::info!(
684 "ingress throttled on BackboneInterface/{} for {:.3}s",
685 client.id.0,
686 hold.as_secs_f64(),
687 );
688 Some(key)
689 }
690
691 fn release_one_ingress(&self, state: &mut ServerState, now: Instant) {
693 let Some((key, client)) = state.clients.iter_mut().find(|(_, client)| {
694 client.dp_ingress_gated
695 && client
696 .dp_ingress_hold
697 .is_none_or(|hold_until| now >= hold_until)
698 }) else {
699 return;
700 };
701 client.dp_ingress_gated = false;
702 client.dp_ingress_hold = None;
703 log::info!(
704 "released ingress throttle on BackboneInterface/{}",
705 client.id.0
706 );
707 self.set_readable(*key, client.fd, true);
708 }
709
710 fn interval(&self) {
712 let mut state = self.lock();
713 let state = &mut *state;
714 self.sync_peer_state(state);
715 let now = Instant::now();
716 if now.duration_since(state.ingress_snapshot_at) >= DP_IC_INTERVAL {
717 let height = self.tx.inbound_queue_snapshot().heights[0];
718 if height > self.ingress_thresholds.mid {
719 if let Some(key) = self.throttle_busiest_ingress(state) {
720 self.set_readable(key, state.clients[&key].fd, false);
721 }
722 } else if height < self.ingress_thresholds.low {
723 self.release_one_ingress(state, now);
724 }
725 for client in state.clients.values_mut() {
726 client.dp_ingress_bytes = 0;
727 client.dp_ingress_packets = 0;
728 }
729 state.ingress_snapshot_at = now;
730 }
731
732 let idle_timeout = lock_or_recover(&self.runtime, "backbone runtime").idle_timeout;
733 if let Some(timeout) = idle_timeout {
734 for (key, client) in state.clients.iter_mut() {
735 if client.has_received_data
736 || client.idle_timed_out
737 || now.duration_since(client.connected_at) < timeout
738 {
739 continue;
740 }
741 client.idle_timed_out = true;
744 unsafe { libc::shutdown(client.fd, libc::SHUT_RDWR) };
746 self.set_readable(*key, client.fd, true);
747 }
748 }
749 }
750
751 fn admit(
753 self: &Arc<Self>,
754 state: &mut ServerState,
755 stream: TcpStream,
756 peer_addr: std::net::SocketAddr,
757 cx: &mut LoopContext<'_>,
758 ) -> bool {
759 let name = &self.name;
760 if let Err(error) = super::apply_underlay_mark(stream.as_raw_fd(), self.underlay_mark) {
761 log::error!(
762 "[{}] failed to mark accepted underlay socket: {}",
763 name,
764 error
765 );
766 return true;
767 }
768 let peer_ip = peer_addr.ip();
769 let peer_port = peer_addr.port();
770
771 let fast_flap_blocked_until = lock_or_recover(
772 &self.fast_flap_state,
773 "backbone fast-flap state",
774 )
775 .blocked_until(peer_ip, &self.fast_flap, Instant::now());
776 if is_ip_blacklisted(&mut state.peers, peer_ip) || fast_flap_blocked_until.is_some() {
777 let peer = state
778 .peers
779 .entry(peer_ip)
780 .or_insert_with(PeerBehaviorState::new);
781 if let Some(blocked_until) = fast_flap_blocked_until {
782 peer.blacklisted_until = Some(blocked_until);
783 peer.blacklist_reason = Some("fast flapping".into());
784 }
785 peer.reject_count = peer.reject_count.saturating_add(1);
786 lock_or_recover(&self.peer_state, "backbone peer state").upsert_snapshot(&state.peers);
787 log::trace!(target: crate::logging::PATHING_LOG_TARGET,
788 "[{}] rejecting blocked peer {}",
789 name,
790 peer_addr,
791 );
792 return true;
793 }
794
795 let max_connections = lock_or_recover(&self.runtime, "backbone runtime").max_connections;
796 if let Some(max) = max_connections {
797 if state.clients.len() >= max {
798 log::warn!(
799 "[{}] max connections ({}) reached, rejecting {}",
800 name,
801 max,
802 peer_addr
803 );
804 return true;
805 }
806 }
807
808 stream.set_nonblocking(true).ok();
809 stream.set_nodelay(true).ok();
810 set_tcp_keepalive(&stream).ok();
811 SockRef::from(&stream)
812 .set_recv_buffer_size(DP_IC_RCVBUF)
813 .ok();
814
815 #[cfg(target_os = "macos")]
817 {
818 let sock = SockRef::from(&stream);
819 sock.set_nosigpipe(true).ok();
820 }
821
822 let writer_stream = match stream.try_clone() {
824 Ok(s) => s,
825 Err(e) => {
826 log::warn!("[{}] failed to clone client stream: {}", name, e);
827 return true;
828 }
829 };
830 let client_id = InterfaceId(self.next_id.fetch_add(1, Ordering::Relaxed));
831 let key = crate::interface::loop_writer::read_key(client_id);
832
833 log::trace!(target: crate::logging::PATHING_LOG_TARGET,
834 "[{}] backbone client connected: {} → id {}",
835 name,
836 peer_addr,
837 client_id.0
838 );
839
840 let write_stall_flag = Arc::new(AtomicBool::new(false));
841 let writer: Box<dyn Writer> = Box::new(BackboneWriter {
842 stream: writer_stream,
843 runtime: Arc::clone(&self.runtime),
844 interface_name: name.clone(),
845 interface_id: client_id,
846 event_tx: self.tx.clone(),
847 transmit_buffer: TransmitBuffer::new(),
848 egress_controller: EgressController::default(),
849 egress_control: EgressControl::new(EGRESS_HIGH_WATERMARK),
850 egress_started: Instant::now(),
851 stall_started: None,
852 disconnect_notified: false,
853 write_stall_flag: Arc::clone(&write_stall_flag),
854 });
855
856 state.clients.insert(
857 key,
858 ClientState {
859 id: client_id,
860 peer_ip,
861 peer_port,
862 fd: stream.as_raw_fd(),
863 connected_at: Instant::now(),
864 has_received_data: false,
865 write_stall_flag,
866 idle_timed_out: false,
867 dp_ingress_bytes: 0,
868 dp_ingress_packets: 0,
869 dp_ingress_hold: None,
870 dp_ingress_gated: false,
871 dp_ingress_throttle_count: 0,
872 },
873 );
874 state
875 .peers
876 .entry(peer_ip)
877 .or_insert_with(PeerBehaviorState::new)
878 .connected_count += 1;
879 lock_or_recover(&self.peer_state, "backbone peer state").upsert_snapshot(&state.peers);
880
881 let info = InterfaceInfo {
882 id: client_id,
883 name: format!("BackboneInterface/{}", client_id.0),
884 mode: self.accepted_peer_mode,
885 gravity: self.accepted_peer_gravity,
886 recursive_prs: self.accepted_peer_recursive_prs,
887 announces_from_internal: self.accepted_peer_announces_from_internal,
888 announces_to_internal: self.accepted_peer_announces_to_internal,
889 out_capable: true,
890 in_capable: true,
891 bitrate: Some(SERVER_BITRATE_GUESS),
892 airtime_profile: None,
893 announce_rate_target: None,
894 announce_rate_grace: 0,
895 announce_rate_penalty: 0.0,
896 announce_cap: constants::ANNOUNCE_CAP,
897 is_local_client: false,
898 wants_tunnel: false,
899 tunnel_id: None,
900 mtu: SERVER_PEER_MTU,
901 ia_freq: 0.0,
902 ip_freq: 0.0,
903 op_freq: 0.0,
904 op_samples: 0,
905 started: 0.0,
906 ingress_control: self.ingress_control,
907 };
908 let up = if let Some(template) = &self.dynamic_template {
909 Event::DynamicInterfaceUp {
910 id: client_id,
911 writer,
912 registration: Box::new(template.registration(info)),
913 }
914 } else {
915 Event::InterfaceUp(client_id, Some(writer), Some(Box::new(info)))
916 };
917 let connected = Event::BackbonePeerConnected {
918 server_interface_id: self.server_interface_id,
919 peer_interface_id: client_id,
920 peer_ip,
921 peer_port,
922 };
923 if self.tx.push_from_loop([connected, up]).is_err() {
924 state.clients.remove(&key);
925 return false;
926 }
927 cx.add_source(
928 key,
929 Box::new(PeerSource {
930 stream,
931 decoder: hdlc::Decoder::reticulum(HW_MTU, self.ifac_size),
932 key,
933 id: client_id,
934 server: Arc::clone(self),
935 }),
936 );
937 true
938 }
939}
940
941#[derive(Clone, Copy, Debug, Eq, PartialEq)]
942struct IngressThresholds {
943 high: usize,
944 immediate: usize,
945 mid: usize,
946 low: usize,
947}
948
949fn ingress_thresholds(data_capacity: usize) -> IngressThresholds {
950 let high = ((DP_IC_HIGH_WM_PCT * data_capacity) / 100).max(4);
953 IngressThresholds {
954 high,
955 immediate: high.max(128),
956 mid: ((DP_IC_MID_WM_PCT * data_capacity) / 100).max(2),
957 low: (DP_IC_LOW_WM_PCT * data_capacity) / 100,
958 }
959}
960
961fn busiest_ingress_key(producers: impl Iterator<Item = (usize, u64)>) -> Option<usize> {
962 producers
963 .filter(|(_, packets)| *packets > 0)
964 .max_by_key(|(_, packets)| *packets)
965 .map(|(key, _)| key)
966}
967
968#[derive(Debug, Clone)]
969struct PeerBehaviorState {
970 blacklisted_until: Option<Instant>,
971 blacklist_reason: Option<String>,
972 reject_count: u64,
973 connected_count: usize,
974}
975
976impl PeerBehaviorState {
977 fn new() -> Self {
978 Self {
979 blacklisted_until: None,
980 blacklist_reason: None,
981 reject_count: 0,
982 connected_count: 0,
983 }
984 }
985}
986
987#[derive(Debug, Clone, Default)]
988pub struct BackbonePeerMonitor {
989 peers: HashMap<IpAddr, PeerBehaviorState>,
990}
991
992impl BackbonePeerMonitor {
993 pub fn new() -> Self {
994 Self {
995 peers: HashMap::new(),
996 }
997 }
998
999 fn upsert_snapshot(&mut self, peers: &HashMap<IpAddr, PeerBehaviorState>) {
1000 let mut merged = self.peers.clone();
1001
1002 for (peer_ip, state) in peers {
1003 let entry = merged
1004 .entry(*peer_ip)
1005 .or_insert_with(PeerBehaviorState::new);
1006 entry.connected_count = state.connected_count;
1007 entry.reject_count = state.reject_count;
1008 entry.blacklisted_until = state.blacklisted_until;
1009 entry.blacklist_reason = state.blacklist_reason.clone();
1010 }
1011
1012 merged.retain(|peer_ip, state| {
1013 peers.contains_key(peer_ip)
1014 || state.blacklisted_until.is_some()
1015 || state.reject_count > 0
1016 });
1017 self.peers = merged;
1018 }
1019
1020 fn sync_into(&self, peers: &mut HashMap<IpAddr, PeerBehaviorState>) {
1021 for (peer_ip, state) in &self.peers {
1022 let entry = peers.entry(*peer_ip).or_insert_with(PeerBehaviorState::new);
1023 entry.blacklisted_until = state.blacklisted_until;
1024 entry.blacklist_reason = state.blacklist_reason.clone();
1025 entry.reject_count = state.reject_count;
1026 }
1027
1028 peers.retain(|peer_ip, state| {
1029 if state.connected_count > 0 {
1030 return true;
1031 }
1032 self.peers.contains_key(peer_ip)
1033 });
1034 }
1035
1036 pub fn list(&self, interface_name: &str) -> Vec<BackbonePeerStateEntry> {
1037 let now = Instant::now();
1038 let mut entries: Vec<BackbonePeerStateEntry> = self
1039 .peers
1040 .iter()
1041 .map(|(peer_ip, state)| BackbonePeerStateEntry {
1042 interface_name: interface_name.to_string(),
1043 peer_ip: *peer_ip,
1044 connected_count: state.connected_count,
1045 blacklisted_remaining_secs: state
1046 .blacklisted_until
1047 .and_then(|until| (until > now).then(|| (until - now).as_secs_f64())),
1048 blacklist_reason: state.blacklist_reason.clone(),
1049 reject_count: state.reject_count,
1050 })
1051 .collect();
1052 entries.sort_by_key(|a| a.peer_ip);
1053 entries
1054 }
1055
1056 pub fn clear(&mut self, peer_ip: IpAddr) -> bool {
1057 self.peers.remove(&peer_ip).is_some()
1058 }
1059
1060 pub fn blacklist(&mut self, peer_ip: IpAddr, duration: Duration, reason: String) -> bool {
1061 let state = self
1062 .peers
1063 .entry(peer_ip)
1064 .or_insert_with(PeerBehaviorState::new);
1065 state.blacklisted_until = Some(Instant::now() + duration);
1066 state.blacklist_reason = Some(reason);
1067 true
1068 }
1069
1070 #[cfg(test)]
1071 pub fn seed_entry(&mut self, entry: BackbonePeerStateEntry) {
1072 let mut state = PeerBehaviorState::new();
1073 state.connected_count = entry.connected_count;
1074 state.reject_count = entry.reject_count;
1075 state.blacklist_reason = entry.blacklist_reason;
1076 if let Some(remaining) = entry.blacklisted_remaining_secs {
1077 state.blacklisted_until = Some(Instant::now() + Duration::from_secs_f64(remaining));
1078 }
1079 self.peers.insert(entry.peer_ip, state);
1080 }
1081}
1082
1083#[derive(Clone, Copy)]
1084enum DisconnectReason {
1085 RemoteClosed,
1086 IdleTimeout,
1087 WriteStall,
1088}
1089
1090struct ListenerSource {
1093 listener: TcpListener,
1094 server: Arc<Server>,
1095 next_interval: Instant,
1096}
1097
1098impl LoopSource for ListenerSource {
1099 fn fd(&self) -> std::os::fd::RawFd {
1100 self.listener.as_raw_fd()
1101 }
1102
1103 fn on_ready(&mut self, cx: &mut LoopContext<'_>) -> SourceStatus {
1104 let server = &self.server;
1105 let mut state = server.lock();
1106 server.sync_peer_state(&mut state);
1107 for _ in 0..ACCEPT_BURST {
1108 match self.listener.accept() {
1109 Ok((stream, peer_addr)) => {
1110 if !server.admit(&mut state, stream, peer_addr, cx) {
1111 return SourceStatus::Remove;
1112 }
1113 }
1114 Err(error) => match super::tcp_server::accept_error_step(&error) {
1115 super::tcp_server::AcceptStep::Drained => return SourceStatus::Rearm,
1116 super::tcp_server::AcceptStep::Next => {}
1117 super::tcp_server::AcceptStep::Pause => {
1118 log::warn!("[{}] accept error, pausing accepts: {}", server.name, error);
1119 return SourceStatus::Pause;
1120 }
1121 },
1122 }
1123 }
1124 SourceStatus::Rearm
1125 }
1126
1127 fn deadline(&self) -> Option<Instant> {
1128 Some(self.next_interval)
1129 }
1130
1131 fn on_timer(&mut self, _cx: &mut LoopContext<'_>) -> SourceStatus {
1132 self.server.interval();
1133 self.next_interval = Instant::now() + DP_IC_INTERVAL;
1134 SourceStatus::Rearm
1135 }
1136}
1137
1138struct PeerSource {
1140 stream: TcpStream,
1141 decoder: hdlc::Decoder,
1142 key: usize,
1143 id: InterfaceId,
1144 server: Arc<Server>,
1145}
1146
1147impl LoopSource for PeerSource {
1148 fn fd(&self) -> std::os::fd::RawFd {
1149 self.stream.as_raw_fd()
1150 }
1151
1152 fn on_ready(&mut self, cx: &mut LoopContext<'_>) -> SourceStatus {
1153 let server = Arc::clone(&self.server);
1154 if server
1155 .lock()
1156 .clients
1157 .get(&self.key)
1158 .is_some_and(|client| client.dp_ingress_gated)
1159 {
1160 return SourceStatus::Hold;
1162 }
1163 let (mut bytes, mut frames) = (0u64, 0u64);
1164 let read = read_stream_frames(
1165 self.stream.as_raw_fd(),
1166 &mut self.decoder,
1167 cx,
1168 &server.tx,
1169 self.id,
1170 &server.name,
1171 |read_bytes, read_frames| {
1172 bytes += read_bytes as u64;
1173 frames += read_frames as u64;
1174 },
1175 );
1176 let mut state = server.lock();
1177 if bytes > 0 {
1178 if let Some(client) = state.clients.get_mut(&self.key) {
1179 client.has_received_data = true;
1180 client.dp_ingress_bytes = client.dp_ingress_bytes.saturating_add(bytes);
1181 client.dp_ingress_packets = client.dp_ingress_packets.saturating_add(frames);
1182 }
1183 }
1184 let status = match read {
1185 StreamRead::Rearm => SourceStatus::Rearm,
1186 StreamRead::Suspend => SourceStatus::Suspend,
1187 StreamRead::Closed | StreamRead::Failed(_) => {
1188 let reason = match state.clients.get(&self.key) {
1189 Some(client) if client.idle_timed_out => DisconnectReason::IdleTimeout,
1190 Some(client) if client.write_stall_flag.load(Ordering::Relaxed) => {
1191 DisconnectReason::WriteStall
1192 }
1193 _ => DisconnectReason::RemoteClosed,
1194 };
1195 self.disconnect(&mut state, reason);
1196 return SourceStatus::Remove;
1197 }
1198 StreamRead::DriverGone => return SourceStatus::Remove,
1199 };
1200 if bytes > 0
1201 && server.tx.inbound_queue_snapshot().heights[0] >= server.ingress_thresholds.immediate
1202 {
1203 if let Some(key) = server.throttle_busiest_ingress(&mut state) {
1204 if key == self.key {
1205 return SourceStatus::Hold;
1206 }
1207 server.set_readable(key, state.clients[&key].fd, false);
1208 }
1209 }
1210 status
1211 }
1212
1213 fn on_removed(&mut self) {
1214 self.server.lock().clients.remove(&self.key);
1217 }
1218
1219 fn on_abandoned(&mut self) {
1220 let server = Arc::clone(&self.server);
1222 let mut state = server.lock();
1223 self.disconnect(&mut state, DisconnectReason::RemoteClosed);
1224 }
1225}
1226
1227impl PeerSource {
1228 fn disconnect(&self, state: &mut ServerState, reason: DisconnectReason) {
1229 let server = &self.server;
1230 let name = &server.name;
1231 let Some(client) = state.clients.remove(&self.key) else {
1232 return;
1233 };
1234 let client_id = client.id;
1235
1236 match reason {
1237 DisconnectReason::RemoteClosed => {
1238 log::trace!(target: crate::logging::PATHING_LOG_TARGET,
1239 "[{}] backbone client {} disconnected",
1240 name,
1241 client_id.0,
1242 );
1243 }
1244 DisconnectReason::IdleTimeout => {
1245 log::info!(
1246 "[{}] backbone client {} disconnected due to idle timeout",
1247 name,
1248 client_id.0
1249 );
1250 }
1251 DisconnectReason::WriteStall => {
1252 }
1254 }
1255
1256 let connected_for = client.connected_at.elapsed();
1257 if let Some(record) = lock_or_recover(&server.fast_flap_state, "backbone fast-flap state")
1258 .record_disconnect(
1259 client.peer_ip,
1260 connected_for,
1261 &server.fast_flap,
1262 Instant::now(),
1263 )
1264 {
1265 let frequency = if record.elapsed_since_first.is_zero() {
1266 None
1267 } else {
1268 Some(record.flap_count as f64 / record.elapsed_since_first.as_secs_f64())
1269 };
1270 log::debug!(
1271 "[{}] peer {} fast flapped after {:.3}s ({} flaps{})",
1272 name,
1273 client.peer_ip,
1274 connected_for.as_secs_f64(),
1275 record.flap_count,
1276 frequency
1277 .map(|value| format!(", {:.3} Hz", value))
1278 .unwrap_or_default(),
1279 );
1280 if record.blocked {
1281 let peer = state
1282 .peers
1283 .entry(client.peer_ip)
1284 .or_insert_with(PeerBehaviorState::new);
1285 peer.blacklisted_until = record.blocked_until;
1286 peer.blacklist_reason = Some("fast flapping".into());
1287 if record.newly_blocked {
1288 log::warn!(
1289 "[{}] blocking further connections from {} due to fast flapping",
1290 name,
1291 client.peer_ip,
1292 );
1293 }
1294 }
1295 }
1296 let mut events = vec![Event::BackbonePeerDisconnected {
1297 server_interface_id: server.server_interface_id,
1298 peer_interface_id: client.id,
1299 peer_ip: client.peer_ip,
1300 peer_port: client.peer_port,
1301 connected_for,
1302 had_received_data: client.has_received_data,
1303 }];
1304 match reason {
1305 DisconnectReason::IdleTimeout => events.push(Event::BackbonePeerIdleTimeout {
1306 server_interface_id: server.server_interface_id,
1307 peer_interface_id: client.id,
1308 peer_ip: client.peer_ip,
1309 peer_port: client.peer_port,
1310 connected_for,
1311 }),
1312 DisconnectReason::WriteStall => events.push(Event::BackbonePeerWriteStall {
1313 server_interface_id: server.server_interface_id,
1314 peer_interface_id: client.id,
1315 peer_ip: client.peer_ip,
1316 peer_port: client.peer_port,
1317 connected_for,
1318 }),
1319 DisconnectReason::RemoteClosed => {}
1320 }
1321
1322 if let Some(peer) = state.peers.get_mut(&client.peer_ip) {
1323 peer.connected_count = peer.connected_count.saturating_sub(1);
1324 }
1325 lock_or_recover(&server.peer_state, "backbone peer state").upsert_snapshot(&state.peers);
1326 if !matches!(reason, DisconnectReason::WriteStall) {
1328 events.push(Event::InterfaceDown(client_id));
1329 }
1330 let _ = server.tx.push_from_loop(events);
1331 }
1332}
1333
1334fn cleanup_peer_state(peers: &mut HashMap<IpAddr, PeerBehaviorState>) {
1335 let now = Instant::now();
1336 peers.retain(|_, state| {
1337 if matches!(state.blacklisted_until, Some(until) if now >= until) {
1338 state.blacklisted_until = None;
1339 state.blacklist_reason = None;
1340 }
1341 state.blacklisted_until.is_some() || state.connected_count > 0 || state.reject_count > 0
1342 });
1343}
1344
1345fn is_ip_blacklisted(peers: &mut HashMap<IpAddr, PeerBehaviorState>, peer_ip: IpAddr) -> bool {
1346 let now = Instant::now();
1347 if let Some(state) = peers.get_mut(&peer_ip) {
1348 if let Some(until) = state.blacklisted_until {
1349 if now < until {
1350 return true;
1351 }
1352 state.blacklisted_until = None;
1353 }
1354 }
1355 false
1356}
1357
1358fn set_tcp_keepalive(stream: &TcpStream) -> io::Result<()> {
1359 let sock = SockRef::from(stream);
1360 let mut keepalive = TcpKeepalive::new()
1361 .with_time(Duration::from_secs(5))
1362 .with_interval(Duration::from_secs(2));
1363 #[cfg(any(target_os = "linux", target_os = "macos"))]
1364 {
1365 keepalive = keepalive.with_retries(12);
1366 }
1367 sock.set_tcp_keepalive(&keepalive)
1368}
1369
1370#[derive(Debug, Clone)]
1376pub struct BackboneClientConfig {
1377 pub name: String,
1378 pub target_host: String,
1379 pub target_port: u16,
1380 pub interface_id: InterfaceId,
1381 pub reconnect_wait: Duration,
1382 pub max_reconnect_tries: Option<u32>,
1383 pub connect_timeout: Duration,
1384 pub transport_identity: Option<String>,
1385 pub runtime: Arc<Mutex<BackboneClientRuntime>>,
1386}
1387
1388#[derive(Debug, Clone)]
1389pub struct BackboneClientRuntime {
1390 pub reconnect_wait: Duration,
1391 pub max_reconnect_tries: Option<u32>,
1392 pub connect_timeout: Duration,
1393}
1394
1395impl BackboneClientRuntime {
1396 pub fn from_config(config: &BackboneClientConfig) -> Self {
1397 Self {
1398 reconnect_wait: config.reconnect_wait,
1399 max_reconnect_tries: config.max_reconnect_tries,
1400 connect_timeout: config.connect_timeout,
1401 }
1402 }
1403}
1404
1405#[derive(Debug, Clone)]
1406pub struct BackboneClientRuntimeConfigHandle {
1407 pub interface_name: String,
1408 pub runtime: Arc<Mutex<BackboneClientRuntime>>,
1409 pub startup: BackboneClientRuntime,
1410}
1411
1412impl Default for BackboneClientConfig {
1413 fn default() -> Self {
1414 let mut config = BackboneClientConfig {
1415 name: String::new(),
1416 target_host: "127.0.0.1".into(),
1417 target_port: 4242,
1418 interface_id: InterfaceId(0),
1419 reconnect_wait: Duration::from_secs(5),
1420 max_reconnect_tries: None,
1421 connect_timeout: Duration::from_secs(5),
1422 transport_identity: None,
1423 runtime: Arc::new(Mutex::new(BackboneClientRuntime {
1424 reconnect_wait: Duration::from_secs(5),
1425 max_reconnect_tries: None,
1426 connect_timeout: Duration::from_secs(5),
1427 })),
1428 };
1429 let startup = BackboneClientRuntime::from_config(&config);
1430 config.runtime = Arc::new(Mutex::new(startup));
1431 config
1432 }
1433}
1434
1435struct BackboneClientWriter {
1446 stream: TcpStream,
1447 transmit_buffer: TransmitBuffer,
1448 egress_control: EgressControl,
1449}
1450
1451impl Writer for BackboneClientWriter {
1452 fn send_frame_confirmed(&mut self, data: &[u8]) -> io::Result<()> {
1453 if !self
1454 .transmit_buffer
1455 .append_with_limit(hdlc::frame(data), Some(EGRESS_HIGH_WATERMARK))
1456 {
1457 return Err(io::Error::new(
1458 io::ErrorKind::InvalidInput,
1459 "confirmed frame exceeds backbone buffer",
1460 ));
1461 }
1462 self.transmit_buffer.flush();
1463 while self.transmit_buffer.sendable_bytes() > 0 {
1464 if self.transmit_buffer.drain_to(&mut self.stream)? == 0 {
1465 return Err(io::Error::new(
1466 io::ErrorKind::WriteZero,
1467 "backbone writer made no progress",
1468 ));
1469 }
1470 }
1471 Ok(())
1472 }
1473 fn send_frame(&mut self, data: &[u8]) -> io::Result<()> {
1474 self.send_frames(&[crate::interface::FrameBytes::Owned(data.to_vec())])
1475 }
1476
1477 fn send_frames(&mut self, frames: &[crate::interface::FrameBytes]) -> io::Result<()> {
1478 for frame in frames {
1479 let encoded = hdlc::frame(frame);
1480 let encoded_len = encoded.len();
1481 if !self
1482 .transmit_buffer
1483 .append_with_limit(encoded, Some(EGRESS_HIGH_WATERMARK))
1484 {
1485 self.egress_control.record_drop(encoded_len);
1486 }
1487 }
1488 self.transmit_buffer.flush();
1489 while self.transmit_buffer.sendable_bytes() > 0 {
1490 if self.transmit_buffer.drain_to(&mut self.stream)? == 0 {
1491 return Err(io::Error::new(
1492 io::ErrorKind::WriteZero,
1493 "backbone client writer made no progress",
1494 ));
1495 }
1496 }
1497 Ok(())
1498 }
1499
1500 fn egress_control(&self) -> Option<EgressControl> {
1501 Some(self.egress_control.clone())
1502 }
1503
1504 fn loop_socket(&self) -> Option<std::os::fd::RawFd> {
1505 Some(self.stream.as_raw_fd())
1506 }
1507
1508 fn loop_egress(&mut self) -> Option<LoopEgress> {
1509 Some(LoopEgress::budget(self.egress_control.clone()))
1510 }
1511}
1512
1513fn try_connect_client(
1515 config: &BackboneClientConfig,
1516 underlay_mark: Option<u32>,
1517) -> io::Result<TcpStream> {
1518 let runtime = lock_or_recover(&config.runtime, "backbone runtime").clone();
1519 let addr_str = format!("{}:{}", config.target_host, config.target_port);
1520 let addr = addr_str
1521 .to_socket_addrs()?
1522 .next()
1523 .ok_or_else(|| io::Error::new(io::ErrorKind::AddrNotAvailable, "no addresses resolved"))?;
1524
1525 #[cfg(target_os = "linux")]
1526 let stream = if underlay_mark.is_some() {
1527 super::connect_tcp_with_options(&addr, None, underlay_mark, runtime.connect_timeout)?
1528 } else {
1529 TcpStream::connect_timeout(&addr, runtime.connect_timeout)?
1530 };
1531 #[cfg(not(target_os = "linux"))]
1532 let stream = TcpStream::connect_timeout(&addr, runtime.connect_timeout)?;
1533 stream.set_nodelay(true)?;
1534 set_tcp_keepalive(&stream).ok();
1535
1536 #[cfg(target_os = "macos")]
1538 {
1539 let sock = SockRef::from(&stream);
1540 sock.set_nosigpipe(true).ok();
1541 }
1542
1543 Ok(stream)
1544}
1545
1546pub fn start_client(config: BackboneClientConfig, tx: EventSender) -> io::Result<Box<dyn Writer>> {
1548 start_client_with_ifac(config, tx, 0, None, ListenerControl::new())
1549}
1550
1551pub(crate) fn start_client_with_ifac(
1552 config: BackboneClientConfig,
1553 tx: EventSender,
1554 ifac_size: usize,
1555 underlay_mark: Option<u32>,
1556 control: ListenerControl,
1557) -> io::Result<Box<dyn Writer>> {
1558 let stream = try_connect_client(&config, underlay_mark)?;
1559 let writer_stream = stream.try_clone()?;
1560
1561 let id = config.interface_id;
1562 log::info!(
1563 "[{}] backbone client connected to {}:{}",
1564 config.name,
1565 config.target_host,
1566 config.target_port
1567 );
1568
1569 let _ = tx.send(Event::InterfaceUp(id, None, None));
1571
1572 let link = Arc::new(ClientLink {
1573 config,
1574 tx,
1575 ifac_size,
1576 underlay_mark,
1577 control,
1578 });
1579 if let Err(error) = link.serve(stream) {
1580 let _ = link.tx.send(Event::InterfaceDown(id));
1581 return Err(error);
1582 }
1583
1584 Ok(Box::new(BackboneClientWriter {
1585 stream: writer_stream,
1586 transmit_buffer: TransmitBuffer::new(),
1587 egress_control: EgressControl::new(EGRESS_HIGH_WATERMARK),
1588 }))
1589}
1590
1591struct ClientLink {
1593 config: BackboneClientConfig,
1594 tx: EventSender,
1595 ifac_size: usize,
1596 underlay_mark: Option<u32>,
1597 control: ListenerControl,
1598}
1599
1600impl ClientLink {
1601 fn serve(self: &Arc<Self>, stream: TcpStream) -> io::Result<()> {
1603 let source = ClientSource {
1604 stream,
1605 decoder: hdlc::Decoder::reticulum(HW_MTU, self.ifac_size),
1606 link: Arc::clone(self),
1607 };
1608 self.tx.register_source(
1609 crate::interface::loop_writer::read_key(self.config.interface_id),
1610 Box::new(source),
1611 )
1612 }
1613
1614 fn connection_lost(self: &Arc<Self>) {
1618 let id = self.config.interface_id;
1619 let _ = self.tx.push_from_loop([Event::InterfaceDown(id)]);
1620 let max_tries =
1621 lock_or_recover(&self.config.runtime, "backbone runtime").max_reconnect_tries;
1622 if max_tries == Some(0) {
1623 log::debug!(
1628 "[{}] not reconnecting (max_reconnect_tries = 0)",
1629 self.config.name
1630 );
1631 return;
1632 }
1633 let link = Arc::clone(self);
1634 let spawned = thread::Builder::new()
1635 .name(format!("bb-reconn-{}", id.0))
1636 .spawn(move || link.reconnect_loop());
1637 if let Err(error) = spawned {
1638 log::error!(
1639 "[{}] cannot start reconnecting, giving up: {}",
1640 self.config.name,
1641 error
1642 );
1643 }
1644 }
1645
1646 fn reconnect_loop(self: Arc<Self>) {
1647 loop {
1648 let Some(stream) =
1649 client_reconnect(&self.config, &self.tx, self.underlay_mark, &self.control)
1650 else {
1651 if !self.control.should_stop() {
1652 log::error!("[{}] reconnection failed, giving up", self.config.name);
1653 }
1654 return;
1655 };
1656 match self.serve(stream) {
1657 Ok(()) => return,
1658 Err(error) => {
1659 log::warn!(
1660 "[{}] cannot read the new connection: {}",
1661 self.config.name,
1662 error
1663 );
1664 let _ = self.tx.send(Event::InterfaceDown(self.config.interface_id));
1665 }
1666 }
1667 }
1668 }
1669}
1670
1671struct ClientSource {
1673 stream: TcpStream,
1674 decoder: hdlc::Decoder,
1675 link: Arc<ClientLink>,
1676}
1677
1678impl LoopSource for ClientSource {
1679 fn fd(&self) -> std::os::fd::RawFd {
1680 self.stream.as_raw_fd()
1681 }
1682
1683 fn stopped(&self) -> bool {
1684 self.link.control.should_stop()
1685 }
1686
1687 fn on_ready(&mut self, cx: &mut LoopContext<'_>) -> SourceStatus {
1688 let link = &self.link;
1689 if link.control.should_stop() {
1690 return SourceStatus::Remove;
1691 }
1692 let name = &link.config.name;
1693 match read_stream_frames(
1694 self.stream.as_raw_fd(),
1695 &mut self.decoder,
1696 cx,
1697 &link.tx,
1698 link.config.interface_id,
1699 name,
1700 |_, _| {},
1701 ) {
1702 StreamRead::Rearm => SourceStatus::Rearm,
1703 StreamRead::Suspend => SourceStatus::Suspend,
1704 StreamRead::Closed => {
1705 log::warn!("[{}] connection closed", name);
1706 link.connection_lost();
1707 SourceStatus::Remove
1708 }
1709 StreamRead::Failed(error) => {
1710 log::warn!("[{}] read error: {}", name, error);
1711 link.connection_lost();
1712 SourceStatus::Remove
1713 }
1714 StreamRead::DriverGone => SourceStatus::Remove,
1715 }
1716 }
1717}
1718
1719pub(crate) const MAX_BACKOFF_SHIFT: u32 = 6;
1722
1723pub(crate) fn reconnect_backoff(base: Duration, attempt: u32) -> Duration {
1727 let backoff = base * 2u32.pow(attempt.min(MAX_BACKOFF_SHIFT));
1728 let jitter_range = backoff / 4;
1729 if jitter_range.as_nanos() == 0 {
1730 return backoff;
1731 }
1732 let offset = Duration::from_nanos(
1733 (std::hash::RandomState::new().build_hasher().finish() % jitter_range.as_nanos() as u64)
1734 * 2,
1735 );
1736 if offset > jitter_range {
1737 backoff + (offset - jitter_range)
1738 } else {
1739 backoff - (jitter_range - offset)
1740 }
1741}
1742
1743fn client_reconnect(
1747 config: &BackboneClientConfig,
1748 tx: &EventSender,
1749 underlay_mark: Option<u32>,
1750 control: &ListenerControl,
1751) -> Option<TcpStream> {
1752 let mut attempts = 0u32;
1753 loop {
1754 let runtime = lock_or_recover(&config.runtime, "backbone runtime").clone();
1755
1756 let jitter = reconnect_backoff(runtime.reconnect_wait, attempts);
1757 if !wait_while_active(control, jitter) {
1758 return None;
1759 }
1760
1761 attempts += 1;
1762
1763 if let Some(max) = runtime.max_reconnect_tries {
1764 if attempts > max {
1765 let _ = tx.send(Event::InterfaceDown(config.interface_id));
1766 return None;
1767 }
1768 }
1769
1770 log::info!(
1771 "[{}] reconnect attempt {} (backoff {:.1}s) ...",
1772 config.name,
1773 attempts,
1774 jitter.as_secs_f64(),
1775 );
1776
1777 match try_connect_client(config, underlay_mark) {
1778 Ok(new_stream) => {
1779 if control.should_stop() {
1780 return None;
1781 }
1782 let writer_stream = match new_stream.try_clone() {
1783 Ok(s) => s,
1784 Err(e) => {
1785 log::warn!("[{}] failed to clone stream: {}", config.name, e);
1786 continue;
1787 }
1788 };
1789 log::info!(
1790 "[{}] reconnected after {} attempt(s)",
1791 config.name,
1792 attempts
1793 );
1794 let new_writer: Box<dyn Writer> = Box::new(BackboneClientWriter {
1795 stream: writer_stream,
1796 transmit_buffer: TransmitBuffer::new(),
1797 egress_control: EgressControl::new(EGRESS_HIGH_WATERMARK),
1798 });
1799 let _ = tx.send(Event::InterfaceUp(
1800 config.interface_id,
1801 Some(new_writer),
1802 None,
1803 ));
1804 return Some(new_stream);
1805 }
1806 Err(e) => {
1807 log::warn!("[{}] reconnect failed: {}", config.name, e);
1808 }
1809 }
1810 }
1811}
1812
1813#[derive(Clone)]
1820pub(crate) enum BackboneMode {
1821 Server(BackboneConfig),
1822 Client(BackboneClientConfig, u8),
1823}
1824
1825pub struct BackboneInterfaceFactory;
1831
1832pub struct BackboneClientInterfaceFactory;
1835
1836impl InterfaceFactory for BackboneClientInterfaceFactory {
1837 fn type_name(&self) -> &str {
1838 "BackboneClientInterface"
1839 }
1840
1841 fn parse_config(
1842 &self,
1843 name: &str,
1844 id: InterfaceId,
1845 params: &HashMap<String, String>,
1846 ) -> Result<Box<dyn InterfaceConfigData>, String> {
1847 if !params.contains_key("remote") && !params.contains_key("target_host") {
1848 return Err("BackboneClientInterface requires 'remote' or 'target_host'".into());
1849 }
1850 BackboneInterfaceFactory.parse_config(name, id, params)
1851 }
1852
1853 fn start(
1854 &self,
1855 config: Box<dyn InterfaceConfigData>,
1856 ctx: StartContext,
1857 ) -> io::Result<StartResult> {
1858 BackboneInterfaceFactory.start(config, ctx)
1859 }
1860}
1861
1862fn parse_positive_duration_secs(params: &HashMap<String, String>, key: &str) -> Option<Duration> {
1863 params
1864 .get(key)
1865 .and_then(|v| v.parse::<f64>().ok())
1866 .filter(|v| *v > 0.0)
1867 .and_then(|v| Duration::try_from_secs_f64(v).ok())
1868 .filter(|duration| Instant::now().checked_add(*duration).is_some())
1869}
1870
1871fn parse_nonnegative_duration(
1872 params: &HashMap<String, String>,
1873 key: &str,
1874 multiplier: f64,
1875 default: Duration,
1876) -> Result<Duration, String> {
1877 let Some(raw) = params.get(key) else {
1878 return Ok(default);
1879 };
1880 let value = raw
1881 .parse::<f64>()
1882 .map_err(|_| format!("invalid {key} '{raw}' (expected a non-negative number)"))?;
1883 let seconds = value * multiplier;
1884 if !value.is_finite() || value < 0.0 || !seconds.is_finite() || seconds > u64::MAX as f64 {
1885 return Err(format!(
1886 "invalid {key} '{raw}' (expected a finite non-negative number)"
1887 ));
1888 }
1889 let duration = Duration::from_secs_f64(seconds);
1890 if Instant::now().checked_add(duration).is_none() {
1891 return Err(format!("invalid {key} '{raw}' (duration is too large)"));
1892 }
1893 Ok(duration)
1894}
1895
1896fn parse_fast_flap_config(
1897 params: &HashMap<String, String>,
1898) -> Result<BackboneFastFlapConfig, String> {
1899 let defaults = BackboneFastFlapConfig::default();
1900 let enabled = match params.get("block_fast_flapping") {
1901 Some(raw) => crate::config::parse_bool_pub(raw)
1902 .ok_or_else(|| format!("invalid block_fast_flapping '{raw}' (expected yes or no)"))?,
1903 None => defaults.enabled,
1904 };
1905 let connection_threshold = parse_nonnegative_duration(
1906 params,
1907 "fast_flapping_threshold",
1908 1.0,
1909 defaults.connection_threshold,
1910 )?;
1911 let grace = match params.get("fast_flapping_grace") {
1912 Some(raw) => raw.parse::<u64>().map_err(|_| {
1913 format!("invalid fast_flapping_grace '{raw}' (expected a non-negative integer)")
1914 })?,
1915 None => defaults.grace,
1916 };
1917 let block_duration = parse_nonnegative_duration(
1918 params,
1919 "fast_flapping_block_time",
1920 60.0,
1921 defaults.block_duration,
1922 )?;
1923
1924 Ok(BackboneFastFlapConfig {
1925 enabled,
1926 connection_threshold,
1927 grace,
1928 block_duration,
1929 })
1930}
1931
1932impl InterfaceFactory for BackboneInterfaceFactory {
1933 fn type_name(&self) -> &str {
1934 "BackboneInterface"
1935 }
1936
1937 fn parse_config(
1938 &self,
1939 name: &str,
1940 id: InterfaceId,
1941 params: &HashMap<String, String>,
1942 ) -> Result<Box<dyn InterfaceConfigData>, String> {
1943 if let Some(target_host) = params.get("remote").or_else(|| params.get("target_host")) {
1944 let target_host = target_host.clone();
1946 let target_port = params
1947 .get("target_port")
1948 .or_else(|| params.get("port"))
1949 .and_then(|v| v.parse().ok())
1950 .unwrap_or(4242);
1951 let transport_identity = params.get("transport_identity").cloned();
1952 let max_reconnect_tries = match params.get("max_reconnect_tries") {
1954 Some(value) => Some(value.trim().parse::<u32>().map_err(|_| {
1955 format!(
1956 "invalid Backbone max_reconnect_tries '{}' (expected a non-negative integer)",
1957 value
1958 )
1959 })?),
1960 None => None,
1961 };
1962 let priority = match params.get("priority") {
1963 Some(value) => value.parse::<u8>().map_err(|_| {
1964 format!(
1965 "invalid Backbone peer priority '{}' (expected 0..100)",
1966 value
1967 )
1968 })?,
1969 None => crate::driver::BACKBONE_PEER_POOL_CONFIGURED_DEFAULT_PRIORITY,
1970 };
1971 if priority > 100 {
1972 return Err(format!(
1973 "invalid Backbone peer priority '{}' (expected 0..100)",
1974 priority
1975 ));
1976 }
1977 let mut client = BackboneClientConfig {
1978 name: name.to_string(),
1979 target_host,
1980 target_port,
1981 interface_id: id,
1982 max_reconnect_tries,
1983 transport_identity,
1984 ..BackboneClientConfig::default()
1985 };
1986 client.runtime = Arc::new(Mutex::new(BackboneClientRuntime::from_config(&client)));
1987 Ok(Box::new(BackboneMode::Client(client, priority)))
1988 } else {
1989 let listen_ip = params
1991 .get("listen_ip")
1992 .or_else(|| params.get("device"))
1993 .cloned()
1994 .unwrap_or_else(|| "0.0.0.0".into());
1995 let listen_port = params
1996 .get("listen_port")
1997 .or_else(|| params.get("port"))
1998 .and_then(|v| v.parse().ok())
1999 .unwrap_or(4242);
2000 let max_connections = params.get("max_connections").and_then(|v| v.parse().ok());
2001 let idle_timeout = parse_positive_duration_secs(params, "idle_timeout");
2002 let write_stall_timeout = parse_positive_duration_secs(params, "write_stall_timeout");
2003 let abuse = BackboneAbuseConfig {
2004 max_penalty_duration: parse_positive_duration_secs(params, "max_penalty_duration"),
2005 };
2006 let fast_flap = parse_fast_flap_config(params)?;
2007 let mut config = BackboneConfig {
2008 name: name.to_string(),
2009 listen_ip,
2010 listen_port,
2011 interface_id: id,
2012 mode: constants::MODE_FULL,
2013 gravity: 0,
2014 recursive_prs: false,
2015 announces_from_internal: true,
2016 announces_to_internal: None,
2017 max_connections,
2018 idle_timeout,
2019 write_stall_timeout,
2020 abuse,
2021 fast_flap,
2022 fast_flap_state: shared_fast_flap_monitor(),
2023 ingress_control: IngressControlConfig::enabled(),
2024 runtime: Arc::new(Mutex::new(BackboneServerRuntime {
2025 max_connections: None,
2026 idle_timeout: None,
2027 write_stall_timeout: None,
2028 abuse: BackboneAbuseConfig::default(),
2029 })),
2030 peer_state: Arc::new(Mutex::new(BackbonePeerMonitor::new())),
2031 };
2032 let startup = BackboneServerRuntime::from_config(&config);
2033 config.runtime = Arc::new(Mutex::new(startup));
2034 Ok(Box::new(BackboneMode::Server(config)))
2035 }
2036 }
2037
2038 fn start(
2039 &self,
2040 config: Box<dyn InterfaceConfigData>,
2041 ctx: StartContext,
2042 ) -> io::Result<StartResult> {
2043 let mode = *config.into_any().downcast::<BackboneMode>().map_err(|_| {
2044 io::Error::new(
2045 io::ErrorKind::InvalidData,
2046 "wrong config type for BackboneInterface",
2047 )
2048 })?;
2049
2050 match mode {
2051 BackboneMode::Client(cfg, _) => {
2052 let id = cfg.interface_id;
2053 let name = cfg.name.clone();
2054 let info = InterfaceInfo {
2055 id,
2056 name,
2057 mode: ctx.mode,
2058 gravity: ctx.gravity,
2059 recursive_prs: ctx.recursive_prs,
2060 announces_from_internal: ctx.announces_from_internal,
2061 announces_to_internal: ctx.announces_to_internal,
2062 out_capable: true,
2063 in_capable: true,
2064 bitrate: Some(CLIENT_BITRATE_GUESS),
2065 airtime_profile: None,
2066 announce_rate_target: None,
2067 announce_rate_grace: 0,
2068 announce_rate_penalty: 0.0,
2069 announce_cap: constants::ANNOUNCE_CAP,
2070 is_local_client: false,
2071 wants_tunnel: false,
2072 tunnel_id: None,
2073 mtu: HW_MTU as u32,
2074 ingress_control: ctx.ingress_control,
2075 ia_freq: 0.0,
2076 ip_freq: 0.0,
2077 op_freq: 0.0,
2078 op_samples: 0,
2079 started: crate::time::now(),
2080 };
2081 let ifac_size = ctx.ifac.as_ref().map(|ifac| ifac.size).unwrap_or(0);
2082 let control = ListenerControl::new();
2083 let writer = start_client_with_ifac(
2084 cfg,
2085 ctx.tx,
2086 ifac_size,
2087 ctx.underlay_mark,
2088 control.clone(),
2089 )?;
2090 Ok(StartResult::Simple {
2091 control: Some(control),
2092 id,
2093 info,
2094 writer,
2095 interface_type_name: "BackboneInterface".to_string(),
2096 })
2097 }
2098 BackboneMode::Server(mut cfg) => {
2099 cfg.ingress_control = ctx.ingress_control;
2100 cfg.mode = ctx.mode;
2101 cfg.gravity = ctx.gravity;
2102 cfg.recursive_prs = ctx.recursive_prs;
2103 cfg.announces_from_internal = ctx.announces_from_internal;
2104 cfg.announces_to_internal = ctx.announces_to_internal;
2105 let parent_id = cfg.interface_id;
2106 start_with_template(
2107 cfg,
2108 ctx.tx,
2109 ctx.next_dynamic_id,
2110 Some(DynamicInterfaceTemplate {
2111 parent_id,
2112 interface_type: "BackboneInterfacePeer".into(),
2113 ifac: ctx.ifac,
2114 mode: ctx.mode,
2115 gravity: ctx.gravity,
2116 recursive_prs: ctx.recursive_prs,
2117 announces_from_internal: ctx.announces_from_internal,
2118 announces_to_internal: ctx.announces_to_internal,
2119 }),
2120 ctx.underlay_mark,
2121 )?;
2122 Ok(StartResult::Listener { control: None })
2123 }
2124 }
2125 }
2126}
2127
2128pub(crate) fn runtime_handle_from_mode(mode: &BackboneMode) -> Option<BackboneRuntimeConfigHandle> {
2129 match mode {
2130 BackboneMode::Server(config) => Some(BackboneRuntimeConfigHandle {
2131 interface_name: config.name.clone(),
2132 runtime: Arc::clone(&config.runtime),
2133 startup: BackboneServerRuntime::from_config(config),
2134 }),
2135 BackboneMode::Client(_, _) => None,
2136 }
2137}
2138
2139pub(crate) fn peer_state_handle_from_mode(mode: &BackboneMode) -> Option<BackbonePeerStateHandle> {
2140 match mode {
2141 BackboneMode::Server(config) => Some(BackbonePeerStateHandle {
2142 interface_id: config.interface_id,
2143 interface_name: config.name.clone(),
2144 mode: config.mode,
2145 gravity: config.gravity,
2146 announces_to_internal: config.announces_to_internal,
2147 peer_state: Arc::clone(&config.peer_state),
2148 fast_flap: config.fast_flap,
2149 fast_flap_state: Arc::clone(&config.fast_flap_state),
2150 }),
2151 BackboneMode::Client(_, _) => None,
2152 }
2153}
2154
2155pub(crate) fn client_runtime_handle_from_mode(
2156 mode: &BackboneMode,
2157) -> Option<BackboneClientRuntimeConfigHandle> {
2158 match mode {
2159 BackboneMode::Client(config, _) => Some(BackboneClientRuntimeConfigHandle {
2160 interface_name: config.name.clone(),
2161 runtime: Arc::clone(&config.runtime),
2162 startup: BackboneClientRuntime::from_config(config),
2163 }),
2164 BackboneMode::Server(_) => None,
2165 }
2166}
2167
2168pub(crate) fn client_config_from_mode(mode: &BackboneMode) -> Option<BackboneClientConfig> {
2169 match mode {
2170 BackboneMode::Client(config, _) => Some(config.clone()),
2171 BackboneMode::Server(_) => None,
2172 }
2173}
2174
2175pub(crate) fn client_priority_from_mode(mode: &BackboneMode) -> Option<u8> {
2176 match mode {
2177 BackboneMode::Client(_, priority) => Some(*priority),
2178 BackboneMode::Server(_) => None,
2179 }
2180}
2181
2182#[cfg(test)]
2183mod tests {
2184 use super::*;
2185 use std::io::Read;
2186 use std::sync::mpsc;
2187 use std::time::Duration;
2188
2189 fn find_free_port() -> u16 {
2190 crate::test_support::port()
2191 }
2192
2193 fn serve_for(rx: &crate::event::EventReceiver, duration: Duration) {
2196 let deadline = Instant::now() + duration;
2197 let mut ready = polling::Events::new();
2198 while Instant::now() < deadline {
2199 ready.clear();
2200 rx.poll_ready(&mut ready).unwrap();
2201 rx.serve_sources(&ready);
2202 thread::sleep(Duration::from_millis(2));
2203 }
2204 }
2205
2206 fn test_fast_flap_config() -> BackboneFastFlapConfig {
2207 BackboneFastFlapConfig {
2208 enabled: true,
2209 connection_threshold: Duration::from_secs(20),
2210 grace: 5,
2211 block_duration: Duration::from_secs(12 * 60 * 60),
2212 }
2213 }
2214
2215 #[test]
2216 fn fast_flap_defaults_match_upstream() {
2217 assert_eq!(BackboneFastFlapConfig::default(), test_fast_flap_config());
2218 }
2219
2220 #[test]
2221 fn sixth_short_connection_blocks_but_fifth_does_not() {
2222 let config = test_fast_flap_config();
2223 let mut monitor = BackboneFastFlapMonitor::new();
2224 let peer: IpAddr = "192.0.2.10".parse().unwrap();
2225 let start = Instant::now();
2226
2227 for flap in 1..=5 {
2228 let record = monitor
2229 .record_disconnect(
2230 peer,
2231 Duration::from_secs(19),
2232 &config,
2233 start + Duration::from_secs(flap),
2234 )
2235 .unwrap();
2236 assert_eq!(record.flap_count, flap);
2237 assert!(!record.blocked);
2238 assert!(!monitor.is_blocked(peer, &config, start + Duration::from_secs(flap)));
2239 }
2240
2241 let sixth = monitor
2242 .record_disconnect(
2243 peer,
2244 Duration::from_secs(1),
2245 &config,
2246 start + Duration::from_secs(6),
2247 )
2248 .unwrap();
2249 assert_eq!(sixth.flap_count, 6);
2250 assert!(sixth.blocked);
2251 assert!(sixth.newly_blocked);
2252 assert!(monitor.is_blocked(peer, &config, start + Duration::from_secs(6)));
2253 }
2254
2255 #[test]
2256 fn every_short_connection_counts_even_after_receiving_data() {
2257 let config = test_fast_flap_config();
2258 let mut monitor = BackboneFastFlapMonitor::new();
2259 let peer: IpAddr = "192.0.2.11".parse().unwrap();
2260 let now = Instant::now();
2261
2262 let record = monitor
2263 .record_disconnect(peer, Duration::from_secs(2), &config, now)
2264 .unwrap();
2265 assert_eq!(record.flap_count, 1);
2266 assert_eq!(monitor.flap_count(peer), 1);
2267 }
2268
2269 #[test]
2270 fn threshold_boundary_and_long_connections_are_not_counted() {
2271 let config = test_fast_flap_config();
2272 let mut monitor = BackboneFastFlapMonitor::new();
2273 let peer: IpAddr = "192.0.2.12".parse().unwrap();
2274 let now = Instant::now();
2275
2276 assert!(monitor
2277 .record_disconnect(peer, Duration::from_secs(20), &config, now)
2278 .is_none());
2279 assert!(monitor
2280 .record_disconnect(peer, Duration::from_secs(21), &config, now)
2281 .is_none());
2282 assert_eq!(monitor.flap_count(peer), 0);
2283 }
2284
2285 #[test]
2286 fn disabled_fast_flapping_neither_tracks_nor_blocks() {
2287 let mut config = test_fast_flap_config();
2288 config.enabled = false;
2289 let mut monitor = BackboneFastFlapMonitor::new();
2290 let peer: IpAddr = "192.0.2.13".parse().unwrap();
2291 let now = Instant::now();
2292
2293 for offset in 0..10 {
2294 assert!(monitor
2295 .record_disconnect(
2296 peer,
2297 Duration::from_secs(1),
2298 &config,
2299 now + Duration::from_secs(offset),
2300 )
2301 .is_none());
2302 }
2303 assert_eq!(monitor.flap_count(peer), 0);
2304 assert!(!monitor.is_blocked(peer, &config, now + Duration::from_secs(10)));
2305 assert_eq!(monitor.blocked_ip_count(&config, now), 0);
2306 }
2307
2308 #[test]
2309 fn flap_blocks_expire_strictly_after_configured_duration() {
2310 let mut config = test_fast_flap_config();
2311 config.grace = 0;
2312 config.block_duration = Duration::from_secs(60);
2313 let mut monitor = BackboneFastFlapMonitor::new();
2314 let peer: IpAddr = "192.0.2.14".parse().unwrap();
2315 let now = Instant::now();
2316
2317 monitor
2318 .record_disconnect(peer, Duration::from_secs(1), &config, now)
2319 .unwrap();
2320 assert!(monitor.is_blocked(peer, &config, now + Duration::from_secs(60)));
2321 assert!(!monitor.is_blocked(
2322 peer,
2323 &config,
2324 now + Duration::from_secs(60) + Duration::from_nanos(1)
2325 ));
2326 assert_eq!(monitor.flap_count(peer), 0);
2327 }
2328
2329 #[test]
2330 fn fast_flap_history_is_isolated_by_ip() {
2331 let mut config = test_fast_flap_config();
2332 config.grace = 1;
2333 let mut monitor = BackboneFastFlapMonitor::new();
2334 let first: IpAddr = "192.0.2.15".parse().unwrap();
2335 let second: IpAddr = "192.0.2.16".parse().unwrap();
2336 let now = Instant::now();
2337
2338 for offset in 0..2 {
2339 monitor
2340 .record_disconnect(
2341 first,
2342 Duration::from_secs(1),
2343 &config,
2344 now + Duration::from_secs(offset),
2345 )
2346 .unwrap();
2347 }
2348 monitor
2349 .record_disconnect(second, Duration::from_secs(1), &config, now)
2350 .unwrap();
2351
2352 assert!(monitor.is_blocked(first, &config, now + Duration::from_secs(2)));
2353 assert!(!monitor.is_blocked(second, &config, now + Duration::from_secs(2)));
2354 assert_eq!(
2355 monitor.blocked_ip_count(&config, now + Duration::from_secs(2)),
2356 1
2357 );
2358 assert_eq!(
2359 monitor.blocked_ip_list(&config, now + Duration::from_secs(2)),
2360 vec![first],
2361 "tracked peers within the grace allowance must not appear as blocked"
2362 );
2363 }
2364
2365 #[test]
2366 fn blocked_ip_list_is_sorted_current_and_empty_when_disabled() {
2367 let mut config = test_fast_flap_config();
2368 config.grace = 0;
2369 config.block_duration = Duration::from_secs(60);
2370 let mut monitor = BackboneFastFlapMonitor::new();
2371 let first: IpAddr = "203.0.113.20".parse().unwrap();
2372 let second: IpAddr = "192.0.2.10".parse().unwrap();
2373 let now = Instant::now();
2374 monitor
2375 .record_disconnect(first, Duration::from_secs(1), &config, now)
2376 .unwrap();
2377 monitor
2378 .record_disconnect(second, Duration::from_secs(1), &config, now)
2379 .unwrap();
2380
2381 assert_eq!(monitor.blocked_ip_list(&config, now), vec![second, first]);
2382 assert!(monitor
2383 .blocked_ip_list(&config, now + Duration::from_secs(61))
2384 .is_empty());
2385 config.enabled = false;
2386 assert!(monitor.blocked_ip_list(&config, now).is_empty());
2387 }
2388
2389 #[test]
2390 fn peer_monitor_snapshot_clears_expired_blacklist_state() {
2391 let peer: IpAddr = "192.0.2.17".parse().unwrap();
2392 let mut monitor = BackbonePeerMonitor::new();
2393 monitor.blacklist(peer, Duration::from_secs(60), "temporary".into());
2394 let mut peers = HashMap::new();
2395 monitor.sync_into(&mut peers);
2396 let state = peers.get_mut(&peer).unwrap();
2397 state.blacklisted_until = None;
2398 state.blacklist_reason = None;
2399
2400 monitor.upsert_snapshot(&peers);
2401
2402 let entry = monitor.list("test").into_iter().next().unwrap();
2403 assert!(entry.blacklisted_remaining_secs.is_none());
2404 assert!(entry.blacklist_reason.is_none());
2405 }
2406
2407 fn recv_non_peer_event(
2408 rx: &crate::event::EventReceiver,
2409 timeout: Duration,
2410 ) -> Result<Event, mpsc::RecvTimeoutError> {
2411 let deadline = Instant::now() + timeout;
2412 loop {
2413 let remaining = deadline.saturating_duration_since(Instant::now());
2414 if remaining.is_zero() {
2415 return Err(mpsc::RecvTimeoutError::Timeout);
2416 }
2417 let event = rx.recv_timeout(remaining)?;
2418 match event {
2419 Event::BackbonePeerConnected { .. }
2420 | Event::BackbonePeerDisconnected { .. }
2421 | Event::BackbonePeerIdleTimeout { .. }
2422 | Event::BackbonePeerWriteStall { .. }
2423 | Event::BackbonePeerPenalty { .. } => continue,
2424 other => return Ok(other),
2425 }
2426 }
2427 }
2428
2429 fn make_server_config(
2430 port: u16,
2431 interface_id: u64,
2432 max_connections: Option<usize>,
2433 idle_timeout: Option<Duration>,
2434 write_stall_timeout: Option<Duration>,
2435 abuse: BackboneAbuseConfig,
2436 ) -> BackboneConfig {
2437 let mut config = BackboneConfig {
2438 name: "test-backbone".into(),
2439 listen_ip: "127.0.0.1".into(),
2440 listen_port: port,
2441 interface_id: InterfaceId(interface_id),
2442 mode: constants::MODE_FULL,
2443 gravity: 0,
2444 recursive_prs: false,
2445 announces_from_internal: true,
2446 announces_to_internal: None,
2447 max_connections,
2448 idle_timeout,
2449 write_stall_timeout,
2450 abuse,
2451 fast_flap: BackboneFastFlapConfig::default(),
2452 fast_flap_state: Arc::new(Mutex::new(BackboneFastFlapMonitor::new())),
2453 ingress_control: IngressControlConfig::enabled(),
2454 runtime: Arc::new(Mutex::new(BackboneServerRuntime {
2455 max_connections: None,
2456 idle_timeout: None,
2457 write_stall_timeout: None,
2458 abuse: BackboneAbuseConfig::default(),
2459 })),
2460 peer_state: Arc::new(Mutex::new(BackbonePeerMonitor::new())),
2461 };
2462 let startup = BackboneServerRuntime::from_config(&config);
2463 config.runtime = Arc::new(Mutex::new(startup));
2464 config
2465 }
2466
2467 #[test]
2468 fn backbone_accept_connection() {
2469 let port = find_free_port();
2470 let (tx, rx) = crate::event::channel();
2471 let next_id = Arc::new(AtomicU64::new(8000));
2472
2473 let config = make_server_config(port, 80, None, None, None, BackboneAbuseConfig::default());
2474
2475 start(config, tx, next_id).unwrap();
2476 thread::sleep(Duration::from_millis(50));
2477
2478 let _client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2479
2480 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2481 match event {
2482 Event::InterfaceUp(id, writer, info) => {
2483 assert_eq!(id, InterfaceId(8000));
2484 assert!(writer.is_some());
2485 assert!(info.is_some());
2486 let info = info.unwrap();
2487 assert!(info.out_capable);
2488 assert!(info.in_capable);
2489 assert_eq!(info.bitrate, Some(SERVER_BITRATE_GUESS));
2490 assert_eq!(info.mtu, SERVER_PEER_MTU);
2491 }
2492 other => panic!("expected InterfaceUp, got {:?}", other),
2493 }
2494 }
2495
2496 #[test]
2497 fn backbone_class_defaults_preserve_server_and_client_distinction() {
2498 assert_eq!(SERVER_BITRATE_GUESS, 100_000_000);
2499 assert_eq!(SERVER_PEER_MTU, 32_768);
2500 assert_eq!(CLIENT_BITRATE_GUESS, 100_000_000);
2501 assert_eq!(HW_MTU, 32_768);
2502 }
2503
2504 #[test]
2505 fn auto_mtu_thresholds_include_exact_bitrate_boundaries() {
2506 for (bitrate, mtu) in [
2507 (1_000_000_000, Some(524_288)),
2508 (750_000_000, Some(262_144)),
2509 (400_000_000, Some(131_072)),
2510 (200_000_000, Some(65_536)),
2511 (100_000_000, Some(32_768)),
2512 (10_000_000, Some(16_384)),
2513 (5_000_000, Some(8_192)),
2514 (2_000_000, Some(4_096)),
2515 (1_000_000, Some(2_048)),
2516 (62_500, Some(1_024)),
2517 (62_499, None),
2518 ] {
2519 assert_eq!(auto_mtu_for_bitrate(bitrate), mtu, "bitrate {bitrate}");
2520 }
2521 }
2522
2523 #[test]
2524 fn backbone_accepted_connection_inherits_server_mode() {
2525 let port = find_free_port();
2526 let (tx, rx) = crate::event::channel();
2527 let next_id = Arc::new(AtomicU64::new(8050));
2528
2529 let mut config =
2530 make_server_config(port, 85, None, None, None, BackboneAbuseConfig::default());
2531 config.mode = constants::MODE_GATEWAY;
2532
2533 start(config, tx, next_id).unwrap();
2534 thread::sleep(Duration::from_millis(50));
2535
2536 let _client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2537
2538 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2539 match event {
2540 Event::InterfaceUp(_, _, Some(info)) => {
2541 assert_eq!(info.mode, constants::MODE_GATEWAY);
2542 }
2543 other => panic!("expected InterfaceUp with info, got {:?}", other),
2544 }
2545 }
2546
2547 #[test]
2548 fn dataplane_ingress_thresholds_match_upstream_defaults() {
2549 assert_eq!(
2550 ingress_thresholds(1024),
2551 IngressThresholds {
2552 high: 921,
2553 immediate: 921,
2554 mid: 696,
2555 low: 102,
2556 }
2557 );
2558 assert_eq!(
2559 ingress_thresholds(8),
2560 IngressThresholds {
2561 high: 7,
2562 immediate: 128,
2563 mid: 5,
2564 low: 0,
2565 }
2566 );
2567 }
2568
2569 #[test]
2570 fn dataplane_ingress_selects_the_largest_packet_producer() {
2571 assert_eq!(
2572 busiest_ingress_key([(7, 12), (8, 41), (9, 3)].into_iter()),
2573 Some(8)
2574 );
2575 assert_eq!(busiest_ingress_key([(7, 0), (8, 0)].into_iter()), None);
2576 }
2577
2578 #[test]
2579 fn backbone_gates_the_largest_producer_before_data_queue_overflow() {
2580 let port = find_free_port();
2581 let capacities = crate::event::InboundQueueCapacities {
2582 data: 200,
2583 announce: 8,
2584 path_request: 8,
2585 ingress_limited: 8,
2586 };
2587 let (tx, rx) = crate::event::channel_with_queue_capacities(16, capacities);
2588 let observer = tx.clone();
2589 let next_id = Arc::new(AtomicU64::new(8075));
2590 let config = make_server_config(port, 80, None, None, None, BackboneAbuseConfig::default());
2591
2592 start(config, tx, next_id).unwrap();
2593 thread::sleep(Duration::from_millis(50));
2594 let mut client = TcpStream::connect(format!("127.0.0.1:{port}")).unwrap();
2595 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2596
2597 let payload = vec![0u8; 32];
2598 let framed = hdlc::frame(&payload);
2599 let mut burst = Vec::with_capacity(framed.len() * 180);
2600 for _ in 0..180 {
2601 burst.extend_from_slice(&framed);
2602 }
2603 client.write_all(&burst).unwrap();
2604
2605 let deadline = Instant::now() + Duration::from_secs(2);
2606 while observer.inbound_queue_snapshot().heights[0] < 180 && Instant::now() < deadline {
2607 serve_for(&rx, Duration::from_millis(5));
2608 }
2609 serve_for(&rx, Duration::from_millis(100));
2610 let gated = observer.inbound_queue_snapshot();
2611 assert!(
2612 gated.heights[0] >= 180,
2613 "DATA queue never reached the high watermark"
2614 );
2615 assert_eq!(gated.dropped[0], 0);
2616
2617 client.write_all(&hdlc::frame(&[1u8; 32])).unwrap();
2618 serve_for(&rx, Duration::from_millis(300));
2619 let after_marker = observer.inbound_queue_snapshot();
2620 assert_eq!(after_marker.heights[0], gated.heights[0]);
2621 assert_eq!(
2622 after_marker.dropped[0], gated.dropped[0],
2623 "the producer remained readable and overflowed the DATA queue"
2624 );
2625
2626 for _ in 0..gated.heights[0] {
2627 let event = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2628 assert!(matches!(event, Event::Frame { .. }));
2629 }
2630 let released = recv_non_peer_event(&rx, Duration::from_secs(5)).unwrap();
2631 assert!(
2632 matches!(released, Event::Frame { data, .. } if data == vec![1u8; 32]),
2633 "held producer was not released after DATA pressure recovered"
2634 );
2635 }
2636
2637 #[test]
2638 fn backbone_receive_frame() {
2639 let port = find_free_port();
2640 let (tx, rx) = crate::event::channel();
2641 let next_id = Arc::new(AtomicU64::new(8100));
2642
2643 let config = make_server_config(port, 81, None, None, None, BackboneAbuseConfig::default());
2644
2645 start(config, tx, next_id).unwrap();
2646 thread::sleep(Duration::from_millis(50));
2647
2648 let mut client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2649
2650 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2652
2653 client
2656 .write_all(&hdlc::frame(&[0x11; rns_core::constants::HEADER_MINSIZE]))
2657 .unwrap();
2658 let payload: Vec<u8> = (0..32).collect();
2659 client.write_all(&hdlc::frame(&payload)).unwrap();
2660
2661 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2662 match event {
2663 Event::Frame {
2664 interface_id,
2665 data,
2666 rssi: _,
2667 snr: _,
2668 } => {
2669 assert_eq!(interface_id, InterfaceId(8100));
2670 assert_eq!(data, payload);
2671 }
2672 other => panic!("expected Frame, got {:?}", other),
2673 }
2674 }
2675
2676 #[test]
2677 fn backbone_send_to_client() {
2678 let port = find_free_port();
2679 let (tx, rx) = crate::event::channel();
2680 let next_id = Arc::new(AtomicU64::new(8200));
2681
2682 let config = make_server_config(port, 82, None, None, None, BackboneAbuseConfig::default());
2683
2684 start(config, tx, next_id).unwrap();
2685 thread::sleep(Duration::from_millis(50));
2686
2687 let mut client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2688 client
2689 .set_read_timeout(Some(Duration::from_secs(2)))
2690 .unwrap();
2691
2692 let event = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2694 let mut writer = match event {
2695 Event::InterfaceUp(_, Some(w), _) => w,
2696 other => panic!("expected InterfaceUp with writer, got {:?}", other),
2697 };
2698
2699 let payload: Vec<u8> = (0..24).collect();
2701 writer.send_frame(&payload).unwrap();
2702
2703 let mut buf = [0u8; 256];
2705 let n = client.read(&mut buf).unwrap();
2706 let expected = hdlc::frame(&payload);
2707 assert_eq!(&buf[..n], &expected[..]);
2708 }
2709
2710 #[test]
2711 fn backbone_write_backpressure_does_not_starve_reads() {
2712 let port = find_free_port();
2713 let (tx, rx) = crate::event::channel();
2714 let next_id = Arc::new(AtomicU64::new(8250));
2715 let config = make_server_config(port, 82, None, None, None, BackboneAbuseConfig::default());
2716
2717 start(config, tx, next_id).unwrap();
2718 thread::sleep(Duration::from_millis(50));
2719
2720 let mut client = TcpStream::connect(format!("127.0.0.1:{port}")).unwrap();
2721 SockRef::from(&client).set_recv_buffer_size(4096).unwrap();
2722 let event = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2723 let mut writer = match event {
2724 Event::InterfaceUp(_, Some(writer), _) => writer,
2725 other => panic!("expected InterfaceUp with writer, got {other:?}"),
2726 };
2727
2728 let outbound = vec![0x55; 60 * 1024];
2729 let deadline = Instant::now() + Duration::from_secs(3);
2730 let mut backpressured = false;
2731 while Instant::now() < deadline {
2732 match writer.send_frame(&outbound) {
2733 Ok(()) => {}
2734 Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
2735 backpressured = true;
2736 break;
2737 }
2738 Err(error) => panic!("unexpected backbone write error: {error}"),
2739 }
2740 }
2741 assert!(
2742 backpressured,
2743 "failed to create outbound write backpressure"
2744 );
2745
2746 let inbound: Vec<u8> = (0..32).collect();
2747 client.write_all(&hdlc::frame(&inbound)).unwrap();
2748 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2749 match event {
2750 Event::Frame { data, .. } => assert_eq!(data, inbound),
2751 other => panic!("inbound frame was starved by outbound backpressure: {other:?}"),
2752 }
2753 }
2754
2755 #[test]
2756 fn backbone_multiple_clients() {
2757 let port = find_free_port();
2758 let (tx, rx) = crate::event::channel();
2759 let next_id = Arc::new(AtomicU64::new(8300));
2760
2761 let config = make_server_config(port, 83, None, None, None, BackboneAbuseConfig::default());
2762
2763 start(config, tx, next_id).unwrap();
2764 thread::sleep(Duration::from_millis(50));
2765
2766 let _client1 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2767 let _client2 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2768
2769 let mut ids = Vec::new();
2770 for _ in 0..2 {
2771 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2772 match event {
2773 Event::InterfaceUp(id, _, _) => ids.push(id),
2774 other => panic!("expected InterfaceUp, got {:?}", other),
2775 }
2776 }
2777
2778 assert_eq!(ids.len(), 2);
2779 assert_ne!(ids[0], ids[1]);
2780 }
2781
2782 #[test]
2783 fn backbone_client_disconnect() {
2784 let port = find_free_port();
2785 let (tx, rx) = crate::event::channel();
2786 let next_id = Arc::new(AtomicU64::new(8400));
2787
2788 let config = make_server_config(port, 84, None, None, None, BackboneAbuseConfig::default());
2789
2790 start(config, tx, next_id).unwrap();
2791 thread::sleep(Duration::from_millis(50));
2792
2793 let client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2794
2795 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2797
2798 drop(client);
2800
2801 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2803 assert!(
2804 matches!(event, Event::InterfaceDown(InterfaceId(8400))),
2805 "expected InterfaceDown(8400), got {:?}",
2806 event
2807 );
2808 }
2809
2810 #[test]
2811 fn backbone_epoll_multiplexing() {
2812 let port = find_free_port();
2813 let (tx, rx) = crate::event::channel();
2814 let next_id = Arc::new(AtomicU64::new(8500));
2815
2816 let config = make_server_config(port, 85, None, None, None, BackboneAbuseConfig::default());
2817
2818 start(config, tx, next_id).unwrap();
2819 thread::sleep(Duration::from_millis(50));
2820
2821 let mut client1 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2822 let mut client2 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2823
2824 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2826 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2827
2828 let payload1: Vec<u8> = (0..24).collect();
2830 let payload2: Vec<u8> = (100..130).collect();
2831 client1.write_all(&hdlc::frame(&payload1)).unwrap();
2832 client2.write_all(&hdlc::frame(&payload2)).unwrap();
2833
2834 let mut received = Vec::new();
2836 for _ in 0..2 {
2837 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2838 match event {
2839 Event::Frame { data, .. } => received.push(data),
2840 other => panic!("expected Frame, got {:?}", other),
2841 }
2842 }
2843 assert!(received.contains(&payload1));
2844 assert!(received.contains(&payload2));
2845 }
2846
2847 #[test]
2848 fn backbone_bind_port() {
2849 let port = find_free_port();
2850 let (tx, _rx) = crate::event::channel();
2851 let next_id = Arc::new(AtomicU64::new(8600));
2852
2853 let config = make_server_config(port, 86, None, None, None, BackboneAbuseConfig::default());
2854
2855 start(config, tx, next_id).unwrap();
2857 }
2858
2859 #[test]
2860 fn backbone_hdlc_fragmented() {
2861 let port = find_free_port();
2862 let (tx, rx) = crate::event::channel();
2863 let next_id = Arc::new(AtomicU64::new(8700));
2864
2865 let config = make_server_config(port, 87, None, None, None, BackboneAbuseConfig::default());
2866
2867 start(config, tx, next_id).unwrap();
2868 thread::sleep(Duration::from_millis(50));
2869
2870 let mut client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
2871 client.set_nodelay(true).unwrap();
2872
2873 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
2875
2876 let payload: Vec<u8> = (0..32).collect();
2878 let framed = hdlc::frame(&payload);
2879 let mid = framed.len() / 2;
2880
2881 client.write_all(&framed[..mid]).unwrap();
2882 thread::sleep(Duration::from_millis(50));
2883 client.write_all(&framed[mid..]).unwrap();
2884
2885 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
2887 match event {
2888 Event::Frame { data, .. } => {
2889 assert_eq!(data, payload);
2890 }
2891 other => panic!("expected Frame, got {:?}", other),
2892 }
2893 }
2894
2895 fn make_client_config(port: u16, id: u64) -> BackboneClientConfig {
2900 BackboneClientConfig {
2901 name: format!("test-bb-client-{}", port),
2902 target_host: "127.0.0.1".into(),
2903 target_port: port,
2904 interface_id: InterfaceId(id),
2905 reconnect_wait: Duration::from_millis(100),
2906 max_reconnect_tries: Some(2),
2907 connect_timeout: Duration::from_secs(2),
2908 transport_identity: None,
2909 runtime: Arc::new(Mutex::new(BackboneClientRuntime {
2910 reconnect_wait: Duration::from_millis(100),
2911 max_reconnect_tries: Some(2),
2912 connect_timeout: Duration::from_secs(2),
2913 })),
2914 }
2915 }
2916
2917 #[test]
2918 fn backbone_client_connect() {
2919 let port = find_free_port();
2920 let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).unwrap();
2921 let (tx, rx) = crate::event::channel();
2922
2923 let config = make_client_config(port, 9000);
2924 let _writer = start_client(config, tx).unwrap();
2925
2926 let _server_stream = listener.accept().unwrap();
2927
2928 let event = rx.recv_timeout(Duration::from_secs(2)).unwrap();
2929 assert!(matches!(event, Event::InterfaceUp(InterfaceId(9000), _, _)));
2930 }
2931
2932 #[test]
2933 fn backbone_client_receive_frame() {
2934 let port = find_free_port();
2935 let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).unwrap();
2936 let (tx, rx) = crate::event::channel();
2937
2938 let config = make_client_config(port, 9100);
2939 let _writer = start_client(config, tx).unwrap();
2940
2941 let (mut server_stream, _) = listener.accept().unwrap();
2942
2943 let _ = rx.recv_timeout(Duration::from_secs(1)).unwrap();
2945
2946 server_stream
2948 .write_all(&hdlc::frame(&[0x11; rns_core::constants::HEADER_MINSIZE]))
2949 .unwrap();
2950 let payload: Vec<u8> = (0..32).collect();
2951 server_stream.write_all(&hdlc::frame(&payload)).unwrap();
2952
2953 let event = rx.recv_timeout(Duration::from_secs(2)).unwrap();
2954 match event {
2955 Event::Frame {
2956 interface_id,
2957 data,
2958 rssi: _,
2959 snr: _,
2960 } => {
2961 assert_eq!(interface_id, InterfaceId(9100));
2962 assert_eq!(data, payload);
2963 }
2964 other => panic!("expected Frame, got {:?}", other),
2965 }
2966 }
2967
2968 #[test]
2969 fn backbone_client_send_frame() {
2970 let port = find_free_port();
2971 let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).unwrap();
2972 let (tx, _rx) = crate::event::channel();
2973
2974 let config = make_client_config(port, 9200);
2975 let mut writer = start_client(config, tx).unwrap();
2976
2977 let (mut server_stream, _) = listener.accept().unwrap();
2978 server_stream
2979 .set_read_timeout(Some(Duration::from_secs(2)))
2980 .unwrap();
2981
2982 let payload: Vec<u8> = (0..24).collect();
2983 writer.send_frame(&payload).unwrap();
2984
2985 let mut buf = [0u8; 256];
2986 let n = server_stream.read(&mut buf).unwrap();
2987 let expected = hdlc::frame(&payload);
2988 assert_eq!(&buf[..n], &expected[..]);
2989 }
2990
2991 #[test]
2992 fn backbone_max_connections_rejects_excess() {
2993 let port = find_free_port();
2994 let (tx, rx) = crate::event::channel();
2995 let next_id = Arc::new(AtomicU64::new(8800));
2996
2997 let config = make_server_config(
2998 port,
2999 88,
3000 Some(2),
3001 None,
3002 None,
3003 BackboneAbuseConfig::default(),
3004 );
3005
3006 start(config, tx, next_id).unwrap();
3007 thread::sleep(Duration::from_millis(50));
3008
3009 let _client1 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3011 let _client2 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3012
3013 for _ in 0..2 {
3015 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3016 assert!(matches!(event, Event::InterfaceUp(_, _, _)));
3017 }
3018
3019 let client3 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3021 client3
3022 .set_read_timeout(Some(Duration::from_millis(500)))
3023 .unwrap();
3024
3025 thread::sleep(Duration::from_millis(100));
3027
3028 let result = recv_non_peer_event(&rx, Duration::from_millis(500));
3030 assert!(
3031 result.is_err(),
3032 "expected no InterfaceUp for rejected connection, got {:?}",
3033 result
3034 );
3035 }
3036
3037 #[test]
3038 fn backbone_max_connections_allows_after_disconnect() {
3039 let port = find_free_port();
3040 let (tx, rx) = crate::event::channel();
3041 let next_id = Arc::new(AtomicU64::new(8900));
3042
3043 let config = make_server_config(
3044 port,
3045 89,
3046 Some(1),
3047 None,
3048 None,
3049 BackboneAbuseConfig::default(),
3050 );
3051
3052 start(config, tx, next_id).unwrap();
3053 thread::sleep(Duration::from_millis(50));
3054
3055 let client1 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3057 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3058 assert!(matches!(event, Event::InterfaceUp(_, _, _)));
3059
3060 drop(client1);
3062
3063 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3065 assert!(matches!(event, Event::InterfaceDown(_)));
3066
3067 let _client2 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3069 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3070 assert!(
3071 matches!(event, Event::InterfaceUp(_, _, _)),
3072 "expected InterfaceUp after slot freed, got {:?}",
3073 event
3074 );
3075 }
3076
3077 #[test]
3078 fn backbone_client_reconnect() {
3079 let port = find_free_port();
3080 let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).unwrap();
3081 listener.set_nonblocking(false).unwrap();
3082 let (tx, rx) = crate::event::channel();
3083
3084 let config = make_client_config(port, 9300);
3085 let _writer = start_client(config, tx).unwrap();
3086
3087 let (server_stream, _) = listener.accept().unwrap();
3089
3090 let _ = rx.recv_timeout(Duration::from_secs(1)).unwrap();
3092
3093 drop(server_stream);
3094
3095 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3097 assert!(matches!(event, Event::InterfaceDown(InterfaceId(9300))));
3098
3099 let _server_stream2 = listener.accept().unwrap();
3101
3102 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3104 assert!(matches!(event, Event::InterfaceUp(InterfaceId(9300), _, _)));
3105 }
3106
3107 #[test]
3108 fn a_reconnected_backbone_client_is_read_on_the_receiving_thread() {
3109 let port = find_free_port();
3110 let listener = TcpListener::bind(format!("127.0.0.1:{}", port)).unwrap();
3111 let (tx, rx) = crate::event::channel();
3112 let _writer = start_client(make_client_config(port, 9310), tx).unwrap();
3113 let (first, _) = listener.accept().unwrap();
3114 assert!(matches!(
3115 rx.recv_timeout(Duration::from_secs(1)).unwrap(),
3116 Event::InterfaceUp(InterfaceId(9310), None, _)
3117 ));
3118 drop(first);
3119 assert!(matches!(
3120 recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap(),
3121 Event::InterfaceDown(InterfaceId(9310))
3122 ));
3123 let (mut second, _) = listener.accept().unwrap();
3124 assert!(matches!(
3125 recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap(),
3126 Event::InterfaceUp(InterfaceId(9310), Some(_), _)
3127 ));
3128 let payload: Vec<u8> = (0..32).collect();
3129 second.write_all(&hdlc::frame(&payload)).unwrap();
3130 match recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap() {
3131 Event::Frame { data, .. } => assert_eq!(data, payload),
3132 other => panic!("expected Frame, got {:?}", other),
3133 }
3134 let reconnecting = || {
3136 std::fs::read_dir("/proc/self/task")
3137 .unwrap()
3138 .filter_map(|task| std::fs::read_to_string(task.ok()?.path().join("comm")).ok())
3139 .any(|name| name.trim_end() == "bb-reconn-9310")
3140 };
3141 let deadline = Instant::now() + Duration::from_secs(2);
3142 while reconnecting() && Instant::now() < deadline {
3143 thread::sleep(Duration::from_millis(10));
3144 }
3145 assert!(!reconnecting());
3146 }
3147
3148 #[test]
3149 fn backbone_client_keeps_reconnecting_until_server_returns() {
3150 let port = find_free_port();
3154 let listener = TcpListener::bind(("127.0.0.1", port)).unwrap();
3155 let (tx, rx) = crate::event::channel();
3156
3157 let mut config = make_client_config(port, 9360);
3158 config.reconnect_wait = Duration::from_millis(20);
3159 config.max_reconnect_tries = None;
3160 config.connect_timeout = Duration::from_millis(200);
3161 config.runtime = Arc::new(Mutex::new(BackboneClientRuntime::from_config(&config)));
3162 let _writer = start_client(config, tx).unwrap();
3163
3164 let (server_stream, _) = listener.accept().unwrap();
3165 let event = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
3166 assert!(matches!(
3167 event,
3168 Event::InterfaceUp(InterfaceId(9360), None, _)
3169 ));
3170
3171 drop(server_stream);
3173 drop(listener);
3174 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3175 assert!(matches!(event, Event::InterfaceDown(InterfaceId(9360))));
3176
3177 thread::sleep(Duration::from_millis(400));
3179 if let Ok(event) = recv_non_peer_event(&rx, Duration::from_millis(10)) {
3180 panic!("unexpected event while the server is down: {:?}", event);
3181 }
3182
3183 let listener = TcpListener::bind(("127.0.0.1", port)).unwrap();
3184 let (_server_stream, _) = listener.accept().unwrap();
3185 let event = recv_non_peer_event(&rx, Duration::from_secs(5)).unwrap();
3186 assert!(
3187 matches!(event, Event::InterfaceUp(InterfaceId(9360), Some(_), _)),
3188 "expected reconnect InterfaceUp, got {:?}",
3189 event
3190 );
3191 }
3192
3193 #[test]
3194 fn backbone_client_without_reconnect_tries_leaves_reconnecting_to_owner() {
3195 let port = find_free_port();
3196 let listener = TcpListener::bind(("127.0.0.1", port)).unwrap();
3197 let (tx, rx) = crate::event::channel();
3198
3199 let mut config = make_client_config(port, 9370);
3200 config.reconnect_wait = Duration::from_millis(20);
3201 config.max_reconnect_tries = Some(0);
3202 config.runtime = Arc::new(Mutex::new(BackboneClientRuntime::from_config(&config)));
3203 let _writer = start_client(config, tx).unwrap();
3204
3205 let (server_stream, _) = listener.accept().unwrap();
3206 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
3207 drop(server_stream);
3208
3209 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3210 assert!(matches!(event, Event::InterfaceDown(InterfaceId(9370))));
3211 listener.set_nonblocking(true).unwrap();
3214 if let Ok(event) = recv_non_peer_event(&rx, Duration::from_millis(300)) {
3215 panic!("unexpected event after the connection closed: {:?}", event);
3216 }
3217 assert!(listener.accept().is_err());
3218 }
3219
3220 #[test]
3221 fn backbone_client_stop_prevents_reconnect() {
3222 let port = find_free_port();
3223 let listener = TcpListener::bind(("127.0.0.1", port)).unwrap();
3224 let (tx, rx) = crate::event::channel();
3225
3226 let config = make_client_config(port, 9350);
3227 let control = ListenerControl::new();
3228 let _writer = start_client_with_ifac(config, tx, 0, None, control.clone()).unwrap();
3229
3230 let (server_stream, _) = listener.accept().unwrap();
3231 let _ = recv_non_peer_event(&rx, Duration::from_secs(1)).unwrap();
3233
3234 control.request_stop();
3235 drop(server_stream);
3236
3237 if let Ok(event) = recv_non_peer_event(&rx, Duration::from_millis(700)) {
3239 panic!("unexpected event after stop: {:?}", event);
3240 }
3241 }
3242
3243 #[test]
3244 fn backbone_idle_timeout_disconnects_silent_client() {
3245 let port = find_free_port();
3246 let (tx, rx) = crate::event::channel();
3247 let next_id = Arc::new(AtomicU64::new(9400));
3248
3249 let config = make_server_config(
3250 port,
3251 94,
3252 None,
3253 Some(Duration::from_millis(150)),
3254 None,
3255 BackboneAbuseConfig::default(),
3256 );
3257
3258 start(config, tx, next_id).unwrap();
3259 thread::sleep(Duration::from_millis(50));
3260
3261 let _client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3262
3263 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3264 let client_id = match event {
3265 Event::InterfaceUp(id, _, _) => id,
3266 other => panic!("expected InterfaceUp, got {:?}", other),
3267 };
3268
3269 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3270 assert!(matches!(event, Event::InterfaceDown(id) if id == client_id));
3271 }
3272
3273 #[test]
3274 fn backbone_idle_timeout_ignores_client_after_data() {
3275 let port = find_free_port();
3276 let (tx, rx) = crate::event::channel();
3277 let next_id = Arc::new(AtomicU64::new(9500));
3278
3279 let config = make_server_config(
3280 port,
3281 95,
3282 None,
3283 Some(Duration::from_millis(200)),
3284 None,
3285 BackboneAbuseConfig::default(),
3286 );
3287
3288 start(config, tx, next_id).unwrap();
3289 thread::sleep(Duration::from_millis(50));
3290
3291 let mut client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3292
3293 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3294 let client_id = match event {
3295 Event::InterfaceUp(id, _, _) => id,
3296 other => panic!("expected InterfaceUp, got {:?}", other),
3297 };
3298
3299 client.write_all(&hdlc::frame(&[1u8; 24])).unwrap();
3300
3301 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3302 match event {
3303 Event::Frame {
3304 interface_id,
3305 data,
3306 rssi: _,
3307 snr: _,
3308 } => {
3309 assert_eq!(interface_id, client_id);
3310 assert_eq!(data, vec![1u8; 24]);
3311 }
3312 other => panic!("expected Frame, got {:?}", other),
3313 }
3314
3315 let result = recv_non_peer_event(&rx, Duration::from_millis(500));
3316 assert!(
3317 result.is_err(),
3318 "expected no InterfaceDown after client sent data, got {:?}",
3319 result
3320 );
3321 }
3322
3323 #[test]
3324 fn backbone_runtime_idle_timeout_updates_live() {
3325 let port = find_free_port();
3326 let (tx, rx) = crate::event::channel();
3327 let next_id = Arc::new(AtomicU64::new(9650));
3328
3329 let config = make_server_config(port, 97, None, None, None, BackboneAbuseConfig::default());
3330 let runtime = Arc::clone(&config.runtime);
3331
3332 start(config, tx, next_id).unwrap();
3333 thread::sleep(Duration::from_millis(50));
3334
3335 let _client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3336 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3337 let client_id = match event {
3338 Event::InterfaceUp(id, _, _) => id,
3339 other => panic!("expected InterfaceUp, got {:?}", other),
3340 };
3341
3342 {
3343 let mut runtime = runtime.lock().unwrap();
3344 runtime.idle_timeout = Some(Duration::from_millis(150));
3345 }
3346
3347 let event = recv_non_peer_event(&rx, Duration::from_secs(4)).unwrap();
3348 assert!(matches!(event, Event::InterfaceDown(id) if id == client_id));
3349 }
3350
3351 #[test]
3352 fn backbone_write_stall_timeout_disconnects_unwritable_client() {
3353 let port = find_free_port();
3354 let (tx, rx) = crate::event::channel();
3355 let next_id = Arc::new(AtomicU64::new(9660));
3356
3357 let config = make_server_config(
3358 port,
3359 98,
3360 None,
3361 None,
3362 Some(Duration::from_millis(50)),
3363 BackboneAbuseConfig::default(),
3364 );
3365
3366 start(config, tx, next_id).unwrap();
3367 thread::sleep(Duration::from_millis(50));
3368
3369 let client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3370 client
3371 .set_read_timeout(Some(Duration::from_millis(100)))
3372 .unwrap();
3373 let sock = SockRef::from(&client);
3374 sock.set_recv_buffer_size(4096).ok();
3375
3376 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3377 let (client_id, mut writer) = match event {
3378 Event::InterfaceUp(id, Some(writer), _) => (id, writer),
3379 other => panic!("expected InterfaceUp with writer, got {:?}", other),
3380 };
3381
3382 let payload = vec![0x55; 512 * 1024];
3383 let deadline = Instant::now() + Duration::from_secs(3);
3384 let mut stalled = false;
3385 while Instant::now() < deadline {
3386 match writer.send_frame(&payload) {
3387 Ok(()) => {}
3388 Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
3389 thread::sleep(Duration::from_millis(10));
3390 }
3391 Err(ref e) if e.kind() == io::ErrorKind::TimedOut => {
3392 stalled = true;
3393 break;
3394 }
3395 Err(e) => panic!("unexpected send error: {}", e),
3396 }
3397 }
3398
3399 assert!(stalled, "expected writer to time out on persistent stall");
3400 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3401 assert!(matches!(event, Event::InterfaceDown(id) if id == client_id));
3402 }
3403
3404 #[test]
3405 fn shutting_down_backbone_writer_closes_the_client_socket() {
3406 let port = find_free_port();
3407 let (tx, rx) = crate::event::channel();
3408 let next_id = Arc::new(AtomicU64::new(9670));
3409 let config = make_server_config(port, 99, None, None, None, BackboneAbuseConfig::default());
3410
3411 start(config, tx, next_id).unwrap();
3412 thread::sleep(Duration::from_millis(50));
3413
3414 let mut client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3415 client
3416 .set_read_timeout(Some(Duration::from_secs(2)))
3417 .unwrap();
3418 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3419 let mut writer = match event {
3420 Event::InterfaceUp(_, Some(writer), _) => writer,
3421 other => panic!("expected InterfaceUp with writer, got {:?}", other),
3422 };
3423
3424 writer.shutdown();
3425
3426 let mut byte = [0u8; 1];
3427 assert_eq!(client.read(&mut byte).unwrap(), 0);
3428 }
3429
3430 fn wait_for<F>(
3432 rx: &crate::event::EventReceiver,
3433 timeout: Duration,
3434 mut pred: F,
3435 ) -> Option<Event>
3436 where
3437 F: FnMut(&Event) -> bool,
3438 {
3439 let deadline = Instant::now() + timeout;
3440 loop {
3441 let remaining = deadline.saturating_duration_since(Instant::now());
3442 if remaining.is_zero() {
3443 return None;
3444 }
3445 match rx.recv_timeout(remaining) {
3446 Ok(event) if pred(&event) => return Some(event),
3447 Ok(_) => continue,
3448 Err(_) => return None,
3449 }
3450 }
3451 }
3452
3453 #[test]
3454 fn backbone_write_stall_emits_peer_events() {
3455 let port = find_free_port();
3456 let (tx, rx) = crate::event::channel();
3457 let next_id = Arc::new(AtomicU64::new(9700));
3458
3459 let config = make_server_config(
3460 port,
3461 97,
3462 None,
3463 None,
3464 Some(Duration::from_millis(50)), BackboneAbuseConfig::default(),
3466 );
3467
3468 start(config, tx, next_id).unwrap();
3469 thread::sleep(Duration::from_millis(50));
3470
3471 let client = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3473 client
3474 .set_read_timeout(Some(Duration::from_millis(100)))
3475 .unwrap();
3476 let sock = SockRef::from(&client);
3477 sock.set_recv_buffer_size(4096).ok();
3478
3479 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3481 let mut writer = match event {
3482 Event::InterfaceUp(_, Some(w), _) => w,
3483 other => panic!("expected InterfaceUp with writer, got {:?}", other),
3484 };
3485
3486 let payload = vec![0x55; 512 * 1024];
3488 let deadline = Instant::now() + Duration::from_secs(3);
3489 while Instant::now() < deadline {
3490 match writer.send_frame(&payload) {
3491 Ok(()) | Err(_) => {
3492 if Instant::now() + Duration::from_millis(10) > deadline {
3493 break;
3494 }
3495 thread::sleep(Duration::from_millis(10));
3496 }
3497 }
3498 }
3499
3500 let stall_event = wait_for(&rx, Duration::from_secs(3), |e| {
3502 matches!(e, Event::BackbonePeerWriteStall { .. })
3503 });
3504 assert!(
3505 stall_event.is_some(),
3506 "expected BackbonePeerWriteStall event"
3507 );
3508 }
3509
3510 #[test]
3511 fn backbone_blacklisted_peer_rejected_on_connect() {
3512 let port = find_free_port();
3513 let (tx, rx) = crate::event::channel();
3514 let next_id = Arc::new(AtomicU64::new(9800));
3515
3516 let config = make_server_config(port, 98, None, None, None, BackboneAbuseConfig::default());
3517 let peer_state = config.peer_state.clone();
3518
3519 start(config, tx.clone(), next_id).unwrap();
3520 thread::sleep(Duration::from_millis(50));
3521
3522 let client1 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3524 let event = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3525 assert!(
3526 matches!(event, Event::InterfaceUp(_, _, _)),
3527 "first connection should succeed"
3528 );
3529 drop(client1);
3530
3531 thread::sleep(Duration::from_millis(100));
3533 while rx.try_recv().is_ok() {}
3534
3535 peer_state.lock().unwrap().blacklist(
3537 "127.0.0.1".parse().unwrap(),
3538 Duration::from_secs(60),
3539 "test blacklist".into(),
3540 );
3541
3542 let _client2 = TcpStream::connect(format!("127.0.0.1:{}", port)).unwrap();
3544 thread::sleep(Duration::from_millis(200));
3546
3547 let event = rx.try_recv();
3549 if let Ok(Event::InterfaceUp(_, _, _)) = event {
3550 panic!("blacklisted peer should not get InterfaceUp")
3551 }
3552 }
3553
3554 #[test]
3555 fn backbone_six_fast_flaps_reject_before_allocating_seventh_interface() {
3556 let port = find_free_port();
3557 let (tx, rx) = crate::event::channel();
3558 let next_id = Arc::new(AtomicU64::new(9900));
3559 let mut config =
3560 make_server_config(port, 99, None, None, None, BackboneAbuseConfig::default());
3561 config.fast_flap = test_fast_flap_config();
3562 let fast_flap_state = Arc::clone(&config.fast_flap_state);
3563 let peer_state = Arc::clone(&config.peer_state);
3564
3565 start(config, tx, Arc::clone(&next_id)).unwrap();
3566 thread::sleep(Duration::from_millis(50));
3567
3568 for flap in 0..6 {
3569 let mut client = TcpStream::connect(format!("127.0.0.1:{port}")).unwrap();
3570 let up = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3571 assert!(matches!(up, Event::InterfaceUp(_, _, _)));
3572
3573 if flap == 0 {
3575 let payload = vec![0x41; 32];
3576 client.write_all(&hdlc::frame(&payload)).unwrap();
3577 let frame = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3578 assert!(matches!(frame, Event::Frame { data, .. } if data == payload));
3579 }
3580
3581 drop(client);
3582 let down = recv_non_peer_event(&rx, Duration::from_secs(2)).unwrap();
3583 assert!(matches!(down, Event::InterfaceDown(_)));
3584 }
3585
3586 let peer_ip: IpAddr = "127.0.0.1".parse().unwrap();
3587 assert!(
3588 lock_or_recover(&fast_flap_state, "test fast-flap state").is_blocked(
3589 peer_ip,
3590 &test_fast_flap_config(),
3591 Instant::now(),
3592 )
3593 );
3594 assert_eq!(next_id.load(Ordering::Relaxed), 9906);
3595 let entries = lock_or_recover(&peer_state, "test peer state").list("test-backbone");
3596 let entry = entries
3597 .iter()
3598 .find(|entry| entry.peer_ip == peer_ip)
3599 .expect("fast-flapping peer state");
3600 assert_eq!(entry.blacklist_reason.as_deref(), Some("fast flapping"));
3601 assert!(entry.blacklisted_remaining_secs.is_some());
3602
3603 let _rejected = TcpStream::connect(format!("127.0.0.1:{port}")).unwrap();
3604 serve_for(&rx, Duration::from_millis(200));
3605 assert_eq!(
3606 next_id.load(Ordering::Relaxed),
3607 9906,
3608 "rejection must happen before dynamic interface allocation"
3609 );
3610 while let Ok(event) = rx.try_recv() {
3611 assert!(
3612 !matches!(event, Event::InterfaceUp(_, _, _)),
3613 "blocked peer must not register a dynamic interface"
3614 );
3615 }
3616 let entries = lock_or_recover(&peer_state, "test peer state").list("test-backbone");
3617 assert_eq!(
3618 entries
3619 .iter()
3620 .find(|entry| entry.peer_ip == peer_ip)
3621 .unwrap()
3622 .reject_count,
3623 1
3624 );
3625 }
3626
3627 #[test]
3628 fn backbone_parse_config_reads_abuse_settings() {
3629 let factory = BackboneInterfaceFactory;
3630 let mut params = HashMap::new();
3631 params.insert("listen_ip".into(), "127.0.0.1".into());
3632 params.insert("listen_port".into(), "4242".into());
3633 params.insert("idle_timeout".into(), "15".into());
3634 params.insert("write_stall_timeout".into(), "45".into());
3635 params.insert("max_penalty_duration".into(), "3600".into());
3636
3637 let config = factory
3638 .parse_config("test-backbone", InterfaceId(97), ¶ms)
3639 .unwrap();
3640 let mode = *config.into_any().downcast::<BackboneMode>().unwrap();
3641
3642 match mode {
3643 BackboneMode::Server(config) => {
3644 assert_eq!(config.listen_ip, "127.0.0.1");
3645 assert_eq!(config.listen_port, 4242);
3646 assert_eq!(config.idle_timeout, Some(Duration::from_secs(15)));
3647 assert_eq!(config.write_stall_timeout, Some(Duration::from_secs(45)));
3648 assert_eq!(
3649 config.abuse.max_penalty_duration,
3650 Some(Duration::from_secs(3600))
3651 );
3652 }
3653 BackboneMode::Client(_, _) => panic!("expected server config"),
3654 }
3655 }
3656
3657 #[test]
3658 fn backbone_parse_config_reads_fast_flapping_settings() {
3659 let factory = BackboneInterfaceFactory;
3660 let mut params = HashMap::new();
3661 params.insert("listen_ip".into(), "127.0.0.1".into());
3662 params.insert("listen_port".into(), "4242".into());
3663 params.insert("block_fast_flapping".into(), "no".into());
3664 params.insert("fast_flapping_threshold".into(), "2.5".into());
3665 params.insert("fast_flapping_grace".into(), "9".into());
3666 params.insert("fast_flapping_block_time".into(), "1.5".into());
3667
3668 let config = factory
3669 .parse_config("test-backbone", InterfaceId(93), ¶ms)
3670 .unwrap();
3671 let mode = *config.into_any().downcast::<BackboneMode>().unwrap();
3672 let BackboneMode::Server(config) = mode else {
3673 panic!("expected server config");
3674 };
3675
3676 assert!(!config.fast_flap.enabled);
3677 assert_eq!(
3678 config.fast_flap.connection_threshold,
3679 Duration::from_secs_f64(2.5)
3680 );
3681 assert_eq!(config.fast_flap.grace, 9);
3682 assert_eq!(config.fast_flap.block_duration, Duration::from_secs(90));
3683 }
3684
3685 #[test]
3686 fn backbone_parse_config_rejects_invalid_fast_flapping_settings() {
3687 let factory = BackboneInterfaceFactory;
3688 for (key, value) in [
3689 ("block_fast_flapping", "perhaps"),
3690 ("fast_flapping_threshold", "-1"),
3691 ("fast_flapping_threshold", "NaN"),
3692 ("fast_flapping_grace", "-1"),
3693 ("fast_flapping_block_time", "infinite"),
3694 ] {
3695 let mut params = HashMap::new();
3696 params.insert("listen_port".into(), "4242".into());
3697 params.insert(key.into(), value.into());
3698
3699 let err = match factory.parse_config("test-backbone", InterfaceId(92), ¶ms) {
3700 Ok(_) => panic!("{key}={value} should be rejected"),
3701 Err(err) => err,
3702 };
3703 assert!(err.contains(key), "unexpected error: {err}");
3704 }
3705 }
3706
3707 #[test]
3708 fn parsed_backbone_listeners_share_fast_flap_history() {
3709 let factory = BackboneInterfaceFactory;
3710 let mut params = HashMap::new();
3711 params.insert("listen_port".into(), "4242".into());
3712
3713 let first = factory
3714 .parse_config("first", InterfaceId(90), ¶ms)
3715 .unwrap();
3716 let second = factory
3717 .parse_config("second", InterfaceId(91), ¶ms)
3718 .unwrap();
3719 let BackboneMode::Server(first) = *first.into_any().downcast::<BackboneMode>().unwrap()
3720 else {
3721 panic!("expected server config");
3722 };
3723 let BackboneMode::Server(second) = *second.into_any().downcast::<BackboneMode>().unwrap()
3724 else {
3725 panic!("expected server config");
3726 };
3727
3728 assert!(Arc::ptr_eq(&first.fast_flap_state, &second.fast_flap_state));
3729 }
3730
3731 #[test]
3732 fn backbone_parse_config_reads_client_priority() {
3733 let factory = BackboneInterfaceFactory;
3734 let mut params = HashMap::new();
3735 params.insert("remote".into(), "example.com".into());
3736 params.insert("target_port".into(), "4242".into());
3737 params.insert("priority".into(), "87".into());
3738
3739 let config = factory
3740 .parse_config("test-backbone-client", InterfaceId(96), ¶ms)
3741 .unwrap();
3742 let mode = *config.into_any().downcast::<BackboneMode>().unwrap();
3743
3744 match mode {
3745 BackboneMode::Client(_, priority) => assert_eq!(priority, 87),
3746 BackboneMode::Server(_) => panic!("expected client config"),
3747 }
3748 }
3749
3750 #[test]
3751 fn backbone_parse_config_reads_client_max_reconnect_tries() {
3752 let parse = |value: Option<&str>| {
3753 let mut params = HashMap::new();
3754 params.insert("remote".into(), "127.0.0.1".into());
3755 if let Some(value) = value {
3756 params.insert("max_reconnect_tries".into(), value.into());
3757 }
3758 BackboneInterfaceFactory
3759 .parse_config("peer", InterfaceId(1), ¶ms)
3760 .map(|config| {
3761 let mode = config.into_any().downcast::<BackboneMode>().unwrap();
3762 match *mode {
3763 BackboneMode::Client(client, _) => client,
3764 BackboneMode::Server(_) => panic!("expected client mode"),
3765 }
3766 })
3767 };
3768
3769 let client = parse(None).unwrap();
3770 assert_eq!(client.max_reconnect_tries, None);
3771 assert_eq!(client.runtime.lock().unwrap().max_reconnect_tries, None);
3772
3773 let client = parse(Some("7")).unwrap();
3774 assert_eq!(client.max_reconnect_tries, Some(7));
3775 assert_eq!(client.runtime.lock().unwrap().max_reconnect_tries, Some(7));
3776
3777 assert!(parse(Some("-1")).is_err());
3778 assert!(parse(Some("forever")).is_err());
3779 }
3780
3781 #[test]
3782 fn backbone_parse_config_defaults_client_priority() {
3783 let factory = BackboneInterfaceFactory;
3784 let mut params = HashMap::new();
3785 params.insert("remote".into(), "example.com".into());
3786
3787 let config = factory
3788 .parse_config("test-backbone-client", InterfaceId(95), ¶ms)
3789 .unwrap();
3790 let mode = *config.into_any().downcast::<BackboneMode>().unwrap();
3791
3792 match mode {
3793 BackboneMode::Client(_, priority) => assert_eq!(priority, 60),
3794 BackboneMode::Server(_) => panic!("expected client config"),
3795 }
3796 }
3797
3798 #[test]
3799 fn backbone_parse_config_rejects_invalid_client_priority() {
3800 let factory = BackboneInterfaceFactory;
3801 for value in ["-1", "101", "fast"] {
3802 let mut params = HashMap::new();
3803 params.insert("remote".into(), "example.com".into());
3804 params.insert("priority".into(), value.into());
3805
3806 let err = match factory.parse_config("test-backbone-client", InterfaceId(94), ¶ms) {
3807 Ok(_) => panic!("priority {value} should be rejected"),
3808 Err(err) => err,
3809 };
3810 assert!(
3811 err.contains("priority"),
3812 "unexpected error for {value}: {err}"
3813 );
3814 }
3815 }
3816
3817 #[test]
3818 fn positive_duration_param_rejects_values_that_are_not_durations() {
3819 for raw in ["inf", "-inf", "NaN", "1e300", "-1", "0"] {
3820 let params = HashMap::from([("idle_timeout".to_string(), raw.to_string())]);
3821 assert_eq!(
3822 parse_positive_duration_secs(¶ms, "idle_timeout"),
3823 None,
3824 "value {raw} should be ignored"
3825 );
3826 }
3827 let params = HashMap::from([("idle_timeout".to_string(), "2.5".to_string())]);
3828 assert_eq!(
3829 parse_positive_duration_secs(¶ms, "idle_timeout"),
3830 Some(Duration::from_millis(2500))
3831 );
3832 }
3833}