subetha_cxc/
trace_sensor.rs

1//! Item 14: Trace mini-traceroute on the control stream + path asymmetry.
2//!
3//! Two signals about the *shape* of the path, both read without a separate probe
4//! flow:
5//!
6//!  - **Mini-traceroute.** The sender emits a few `Trace` control datagrams at
7//!    ascending IP TTL (1, 2, ...). A datagram whose TTL expires at an
8//!    intermediate router draws an ICMP TimeExceeded back; on Linux that error
9//!    is delivered on the socket's error queue (`IP_RECVERR` +
10//!    `recvmsg(MSG_ERRQUEUE)`), carrying the offending router's address and the
11//!    timestamp machinery for a per-hop RTT - a traceroute riding the transport's
12//!    own socket, no second flow. The TTL that drew each reply is the hop index.
13//!  - **Path asymmetry.** The forward hop count (how many hops the peer says our
14//!    packets crossed, from its `Path` frame) versus the reverse hop count (how
15//!    many hops the peer's feedback crossed, from our own received-TTL cmsg). A
16//!    difference means the two directions are routed differently - which biases
17//!    the per-hop RTT model, since a one-way delay no longer splits evenly.
18//!
19//! The error-queue read is a Linux / BSD capability (the per-platform matrix
20//! lists no Windows path), so the traceroute half is `#[cfg(target_os =
21//! "linux")]`; the asymmetry half is portable (it is pure hop-count arithmetic
22//! over signals the control plane already carries).
23
24use std::net::IpAddr;
25
26/// One discovered hop on the path to the peer.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub struct TraceHop {
29    /// The TTL at which this hop replied (1 = first router, 2 = second, ...).
30    pub ttl: u8,
31    /// The router that sent the ICMP TimeExceeded.
32    pub addr: IpAddr,
33    /// Round-trip time to this hop, microseconds.
34    pub rtt_us: u64,
35}
36
37/// Forward-vs-reverse path asymmetry. The forward hop count is what the peer
38/// reports about our packets; the reverse is what we observe about the peer's.
39#[derive(Debug, Clone, Copy, Default)]
40pub struct PathAsymmetry {
41    forward_hops: u8,
42    reverse_hops: u8,
43    have_forward: bool,
44    have_reverse: bool,
45}
46
47impl PathAsymmetry {
48    pub fn new() -> Self {
49        Self::default()
50    }
51
52    /// Record the forward hop count (from the peer's `Path` frame about us).
53    pub fn observe_forward(&mut self, hops: u8) {
54        self.forward_hops = hops;
55        self.have_forward = true;
56    }
57
58    /// Record the reverse hop count (from our received-TTL cmsg about the peer).
59    pub fn observe_reverse(&mut self, hops: u8) {
60        self.reverse_hops = hops;
61        self.have_reverse = true;
62    }
63
64    pub fn forward(&self) -> Option<u8> {
65        self.have_forward.then_some(self.forward_hops)
66    }
67
68    pub fn reverse(&self) -> Option<u8> {
69        self.have_reverse.then_some(self.reverse_hops)
70    }
71
72    /// `|forward - reverse|`, or `None` until both directions are known. A
73    /// nonzero value means the path is routed asymmetrically.
74    pub fn asymmetry(&self) -> Option<u8> {
75        if self.have_forward && self.have_reverse {
76            Some(self.forward_hops.abs_diff(self.reverse_hops))
77        } else {
78            None
79        }
80    }
81}
82
83/// Enable the ICMP error queue on a socket so an expired-TTL probe's
84/// TimeExceeded is delivered (Linux). A no-op elsewhere.
85#[cfg(target_os = "linux")]
86pub fn enable_icmp_errors(fd: std::os::fd::RawFd) {
87    let on: libc::c_int = 1;
88    // SAFETY: `fd` is a valid socket; `on` is a valid c_int that outlives the
89    // call. IP_RECVERR turns on the per-socket error queue.
90    unsafe {
91        libc::setsockopt(
92            fd,
93            libc::IPPROTO_IP,
94            libc::IP_RECVERR,
95            &on as *const libc::c_int as *const libc::c_void,
96            std::mem::size_of::<libc::c_int>() as libc::socklen_t,
97        );
98    }
99}
100
101#[cfg(not(target_os = "linux"))]
102pub fn enable_icmp_errors(_fd: i32) {}
103
104/// Send `payload` on the **connected** socket `fd` with the IP TTL set to `ttl`
105/// for this one datagram (via an `IP_TTL` cmsg, so the socket's default TTL is
106/// untouched). The socket must already be connected to the peer - `msg_name` is
107/// left null, since a non-null name on a connected socket returns `EISCONN`.
108/// `peer` is used only to skip an IPv6 peer (the hop-limit cmsg is a separate
109/// spelling not needed for the netns / LAN proof). Linux only; a no-op elsewhere.
110#[cfg(target_os = "linux")]
111pub fn send_at_ttl(
112    fd: std::os::fd::RawFd,
113    peer: std::net::SocketAddr,
114    payload: &[u8],
115    ttl: u8,
116) -> std::io::Result<()> {
117    use std::mem::{size_of, zeroed};
118    if !peer.is_ipv4() {
119        return Ok(());
120    }
121    // SAFETY: every pointer below refers to a stack local that outlives the
122    // sendmsg call; the cmsg buffer is sized by CMSG_SPACE and written through
123    // CMSG_FIRSTHDR / CMSG_DATA exactly as the kernel ABI requires.
124    unsafe {
125        let mut iov = libc::iovec {
126            iov_base: payload.as_ptr() as *mut libc::c_void,
127            iov_len: payload.len(),
128        };
129        let mut cbuf = [0u8; 64];
130        let mut msg: libc::msghdr = zeroed();
131        msg.msg_name = std::ptr::null_mut();
132        msg.msg_namelen = 0;
133        msg.msg_iov = &mut iov;
134        msg.msg_iovlen = 1;
135        msg.msg_control = cbuf.as_mut_ptr() as *mut libc::c_void;
136        msg.msg_controllen = libc::CMSG_SPACE(size_of::<libc::c_int>() as u32) as usize;
137
138        let cmsg = libc::CMSG_FIRSTHDR(&msg);
139        if cmsg.is_null() {
140            return Err(std::io::Error::other("CMSG_FIRSTHDR null"));
141        }
142        (*cmsg).cmsg_level = libc::IPPROTO_IP;
143        (*cmsg).cmsg_type = libc::IP_TTL;
144        (*cmsg).cmsg_len = libc::CMSG_LEN(size_of::<libc::c_int>() as u32) as usize;
145        let ttl_i = ttl as libc::c_int;
146        std::ptr::copy_nonoverlapping(
147            &ttl_i as *const libc::c_int as *const u8,
148            libc::CMSG_DATA(cmsg),
149            size_of::<libc::c_int>(),
150        );
151
152        let n = libc::sendmsg(fd, &msg, 0);
153        if n < 0 {
154            return Err(std::io::Error::last_os_error());
155        }
156    }
157    Ok(())
158}
159
160#[cfg(not(target_os = "linux"))]
161pub fn send_at_ttl(
162    _fd: i32,
163    _peer: std::net::SocketAddr,
164    _payload: &[u8],
165    _ttl: u8,
166) -> std::io::Result<()> {
167    Ok(())
168}
169
170/// Drain the socket's error queue, returning, for each ICMP TimeExceeded found,
171/// the offending router address and the bytes of the original probe it expired
172/// (so the caller can read back the TTL it stamped and match the per-hop RTT).
173/// Linux only.
174#[cfg(target_os = "linux")]
175pub fn drain_icmp_errors(fd: std::os::fd::RawFd) -> Vec<(IpAddr, Vec<u8>)> {
176    use std::mem::{size_of, zeroed};
177    let mut hops = Vec::new();
178    // SAFETY: the msghdr and its buffers are stack locals living across each
179    // recvmsg; the cmsg walk uses CMSG_FIRSTHDR / CMSG_NXTHDR / CMSG_DATA on a
180    // buffer the kernel filled, and the offender sockaddr is read from the bytes
181    // immediately after the sock_extended_err the kernel placed.
182    unsafe {
183        loop {
184            let mut from: libc::sockaddr_in = zeroed();
185            let mut buf = [0u8; 512];
186            let mut cbuf = [0u8; 512];
187            let mut iov = libc::iovec {
188                iov_base: buf.as_mut_ptr() as *mut libc::c_void,
189                iov_len: buf.len(),
190            };
191            let mut msg: libc::msghdr = zeroed();
192            msg.msg_name = &mut from as *mut _ as *mut libc::c_void;
193            msg.msg_namelen = size_of::<libc::sockaddr_in>() as libc::socklen_t;
194            msg.msg_iov = &mut iov;
195            msg.msg_iovlen = 1;
196            msg.msg_control = cbuf.as_mut_ptr() as *mut libc::c_void;
197            msg.msg_controllen = cbuf.len();
198
199            let n = libc::recvmsg(fd, &mut msg, libc::MSG_ERRQUEUE | libc::MSG_DONTWAIT);
200            if n < 0 {
201                break;
202            }
203            // The returned iov holds the original UDP payload of the expired
204            // probe, so the caller can read back the TTL it stamped.
205            let payload = buf[..(n as usize).min(buf.len())].to_vec();
206            let mut cmsg = libc::CMSG_FIRSTHDR(&msg);
207            while !cmsg.is_null() {
208                if (*cmsg).cmsg_level == libc::IPPROTO_IP && (*cmsg).cmsg_type == libc::IP_RECVERR {
209                    let ee = libc::CMSG_DATA(cmsg) as *const libc::sock_extended_err;
210                    if (*ee).ee_origin == libc::SO_EE_ORIGIN_ICMP {
211                        // The offender sockaddr_in follows the sock_extended_err
212                        // (the SO_EE_OFFENDER macro is exactly this offset).
213                        let off = (ee as *const u8).add(size_of::<libc::sock_extended_err>())
214                            as *const libc::sockaddr_in;
215                        // s_addr holds the address in network byte order, so its
216                        // in-memory bytes ARE the octets a.b.c.d in order.
217                        let octets = (*off).sin_addr.s_addr.to_ne_bytes();
218                        hops.push((IpAddr::V4(std::net::Ipv4Addr::from(octets)), payload.clone()));
219                    }
220                }
221                cmsg = libc::CMSG_NXTHDR(&msg, cmsg);
222            }
223        }
224    }
225    hops
226}
227
228#[cfg(not(target_os = "linux"))]
229pub fn drain_icmp_errors(_fd: i32) -> Vec<(IpAddr, Vec<u8>)> {
230    Vec::new()
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    #[test]
238    fn asymmetry_is_none_until_both_directions_known() {
239        let mut a = PathAsymmetry::new();
240        assert_eq!(a.asymmetry(), None);
241        a.observe_forward(3);
242        assert_eq!(a.asymmetry(), None, "one direction is not enough");
243        a.observe_reverse(3);
244        assert_eq!(a.asymmetry(), Some(0), "a symmetric path reads 0");
245    }
246
247    #[test]
248    fn asymmetry_counts_the_hop_difference() {
249        let mut a = PathAsymmetry::new();
250        a.observe_forward(5);
251        a.observe_reverse(2);
252        assert_eq!(a.asymmetry(), Some(3), "forward 5 vs reverse 2 -> 3");
253        assert_eq!(a.forward(), Some(5));
254        assert_eq!(a.reverse(), Some(2));
255    }
256}
subetha_cxc/trace_sensor.rs

subetha_cxc/
trace_sensor.rs
