Skip to main content

net_lattice_backend_linux/
lib.rs

1//! Linux backend for Net Lattice: implements `net-lattice-platform`'s provider
2//! traits via Netlink.
3//!
4//! Only ever compiled for `target_os = "linux"` — its dependencies
5//! (`rtnetlink`, Linux-only) are gated the same way in `Cargo.toml`. See
6//! ARCHITECTURE.md for how this crate binds `net-lattice-platform`'s generic
7//! `RouteProvider::Route` associated type to the concrete
8//! `net_lattice_model::route::Route`.
9
10#![cfg(target_os = "linux")]
11
12use std::hash::{Hash, Hasher};
13use std::net::IpAddr;
14
15use futures::{StreamExt, TryStreamExt};
16use net_lattice_core::{Error, Id, PlatformErrorCode, Result};
17use net_lattice_model::dns::{DnsConfig, NewDnsConfig};
18use net_lattice_model::event::{ChangeKind, Event, EventFilter};
19use net_lattice_model::ifaddr::{InterfaceAddress, InterfaceAddressId, NewInterfaceAddress};
20use net_lattice_model::interface::{AdminState, Interface, InterfaceKind, OperationalState};
21use net_lattice_model::mac::MacAddress;
22use net_lattice_model::neighbor::{NeighborEntry, NeighborId, NeighborState};
23use net_lattice_model::route::{Route, RouteId};
24use net_lattice_model::{IpAddress, Network};
25use net_lattice_platform::{
26    AddressMutator, AddressProvider, Capability, CapabilityProvider, DnsMutator, DnsProvider,
27    EventProvider, EventReceiver, InterfaceProvider, NeighborProvider, RouteProvider,
28};
29#[cfg(feature = "async")]
30use net_lattice_platform::{TokioEventProvider, TokioEventReceiver};
31use rtnetlink::packet_route::RouteNetlinkMessage;
32use rtnetlink::packet_route::address::{AddressAttribute, AddressMessage};
33use rtnetlink::packet_route::link::{LinkAttribute, LinkLayerType, LinkMessage, State};
34use rtnetlink::packet_route::neighbour::{
35    NeighbourAddress, NeighbourAttribute, NeighbourMessage, NeighbourState as RtNeighbourState,
36};
37use rtnetlink::packet_route::route::{RouteAddress, RouteAttribute, RouteMessage};
38use rtnetlink::{Handle, MulticastGroup, RouteMessageBuilder};
39
40/// The Linux Netlink-backed implementation of Net Lattice's provider traits.
41pub struct LinuxBackend {
42    runtime: tokio::runtime::Runtime,
43    handle: Handle,
44}
45
46struct LinuxWatch {
47    connection: tokio::task::JoinHandle<()>,
48    events: tokio::task::JoinHandle<()>,
49}
50
51impl Drop for LinuxWatch {
52    fn drop(&mut self) {
53        self.events.abort();
54        self.connection.abort();
55    }
56}
57
58impl LinuxBackend {
59    pub fn new() -> Result<Self> {
60        let runtime =
61            tokio::runtime::Runtime::new().map_err(|err| Error::Platform(io_error_code(&err)))?;
62        // `rtnetlink::new_connection` constructs a `tokio::io::unix::AsyncFd`
63        // socket synchronously, which requires an active Tokio reactor
64        // context at construction time (not just when later polled) -
65        // without `_guard`, this panics with "there is no reactor running"
66        // the moment `add_route`/`remove_route`/`routes` first drives it.
67        let _guard = runtime.enter();
68        let (connection, handle, _) =
69            rtnetlink::new_connection().map_err(|err| Error::Platform(io_error_code(&err)))?;
70        runtime.spawn(connection);
71        Ok(Self { runtime, handle })
72    }
73}
74
75fn io_error_code(err: &std::io::Error) -> PlatformErrorCode {
76    PlatformErrorCode::Linux(err.raw_os_error().unwrap_or(0))
77}
78
79fn rtnetlink_error_code(err: &rtnetlink::Error) -> PlatformErrorCode {
80    match err {
81        rtnetlink::Error::NetlinkError(message) => {
82            PlatformErrorCode::Linux(message.code.map(i32::from).unwrap_or(0))
83        }
84        _ => PlatformErrorCode::Linux(0),
85    }
86}
87
88/// Placeholder identity scheme, same rationale as `synthesize_route_id`: an
89/// interface address has no kernel-assigned numeric ID, so this hashes its
90/// interface and network together (the pair `RTM_GETADDR` itself keys
91/// entries by).
92fn synthesize_interface_address_id(interface_index: u32, network: &Network) -> InterfaceAddressId {
93    let mut hasher = std::collections::hash_map::DefaultHasher::new();
94    interface_index.hash(&mut hasher);
95    network.hash(&mut hasher);
96    InterfaceAddressId::new(hasher.finish())
97}
98
99fn message_to_interface_address(message: &AddressMessage) -> Option<InterfaceAddress> {
100    let interface_index = message.header.index;
101    let prefix_len = message.header.prefix_len;
102
103    let mut address_addr = None;
104    let mut broadcast = None;
105    for attribute in &message.attributes {
106        match attribute {
107            // `IFA_LOCAL`, when present, is the actual assigned address —
108            // `IFA_ADDRESS` alone (no `IFA_LOCAL`) names the *peer* address
109            // on a point-to-point link, so prefer `Local` and only fall
110            // back to `Address` when there is no separate local entry.
111            AddressAttribute::Local(addr) => address_addr = Some(*addr),
112            AddressAttribute::Address(addr) if address_addr.is_none() => {
113                address_addr = Some(*addr);
114            }
115            AddressAttribute::Broadcast(addr) => {
116                broadcast = Some(std_ip_to_ip_address(IpAddr::V4(*addr)));
117            }
118            _ => {}
119        }
120    }
121
122    let network = match address_addr? {
123        IpAddr::V4(addr) => {
124            let prefix = net_lattice_ip::Ipv4PrefixLength::new(prefix_len)?;
125            Network::from(net_lattice_ip::Ipv4Network::new(addr.into(), prefix))
126        }
127        IpAddr::V6(addr) => {
128            let prefix = net_lattice_ip::Ipv6PrefixLength::new(prefix_len)?;
129            Network::from(net_lattice_ip::Ipv6Network::new(addr.into(), prefix))
130        }
131    };
132
133    let mut entry = InterfaceAddress::new(
134        synthesize_interface_address_id(interface_index, &network),
135        interface_index,
136        network,
137    );
138    if let Some(broadcast) = broadcast {
139        entry = entry.with_broadcast(broadcast);
140    }
141    Some(entry)
142}
143
144impl AddressProvider for LinuxBackend {
145    type InterfaceAddress = InterfaceAddress;
146
147    fn addresses(&self) -> Result<Vec<Self::InterfaceAddress>> {
148        self.runtime.block_on(async {
149            let mut messages = self.handle.address().get().execute();
150            let mut addresses = Vec::new();
151            while let Some(message) = messages
152                .try_next()
153                .await
154                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
155            {
156                addresses.extend(message_to_interface_address(&message));
157            }
158            Ok(addresses)
159        })
160    }
161}
162
163impl AddressMutator for LinuxBackend {
164    type NewInterfaceAddress = NewInterfaceAddress;
165    type InterfaceAddress = InterfaceAddress;
166
167    fn add_address(&self, address: Self::NewInterfaceAddress) -> Result<Self::InterfaceAddress> {
168        if matches!(address.address, Network::V6(_)) && address.broadcast.is_some() {
169            return Err(Error::InvalidState);
170        }
171        let interface_index = address.interface_id.value() as u32;
172        let (ip, prefix_len) = network_to_std(address.address);
173        self.runtime.block_on(async {
174            let mut request = self.handle.address().add(interface_index, ip, prefix_len);
175            if let Some(broadcast) = address.broadcast {
176                request
177                    .message_mut()
178                    .attributes
179                    .push(AddressAttribute::Broadcast(broadcast.into()));
180            }
181            request
182                .execute()
183                .await
184                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))
185        })?;
186
187        self.addresses()?
188            .into_iter()
189            .find(|observed| {
190                observed.interface_index == interface_index && observed.address == address.address
191            })
192            .ok_or(Error::InvalidState)
193    }
194
195    fn remove_address(&self, address: Self::InterfaceAddress) -> Result<()> {
196        self.runtime.block_on(async {
197            let mut messages = self.handle.address().get().execute();
198            while let Some(message) = messages
199                .try_next()
200                .await
201                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
202            {
203                if message_to_interface_address(&message)
204                    .is_some_and(|observed| observed.id == address.id)
205                {
206                    return self
207                        .handle
208                        .address()
209                        .del(message)
210                        .execute()
211                        .await
212                        .map_err(|err| Error::Platform(rtnetlink_error_code(&err)));
213                }
214            }
215            Err(Error::NotFound)
216        })
217    }
218}
219
220/// Placeholder identity scheme: a route has no kernel-assigned numeric ID,
221/// so this hashes its defining fields. See ARCHITECTURE.md's open Object
222/// Identity question — this is not a long-term answer, only enough to give
223/// `Stage 0.1` a `RouteId` to work with.
224///
225/// Hashes destination, gateway, and outgoing interface together so that
226/// two routes to the same destination that differ only in gateway or
227/// interface (a common case with multiple default routes, or ECMP-like
228/// setups) don't collide on the same `RouteId`.
229fn synthesize_route_id(message: &RouteMessage) -> RouteId {
230    let mut hasher = std::collections::hash_map::DefaultHasher::new();
231    message.header.destination_prefix_length.hash(&mut hasher);
232    for attribute in &message.attributes {
233        match attribute {
234            RouteAttribute::Destination(addr) => {
235                route_address_to_ip(addr).hash(&mut hasher);
236            }
237            RouteAttribute::Gateway(addr) => {
238                route_address_to_ip(addr).hash(&mut hasher);
239            }
240            RouteAttribute::Oif(index) => {
241                index.hash(&mut hasher);
242            }
243            _ => {}
244        }
245    }
246    RouteId::new(hasher.finish())
247}
248
249fn route_address_to_ip(address: &RouteAddress) -> Option<IpAddr> {
250    match address {
251        RouteAddress::Inet(addr) => Some(IpAddr::V4(*addr)),
252        RouteAddress::Inet6(addr) => Some(IpAddr::V6(*addr)),
253        _ => None,
254    }
255}
256
257fn std_ip_to_ip_address(addr: IpAddr) -> IpAddress {
258    match addr {
259        IpAddr::V4(addr) => IpAddress::from(net_lattice_ip::Ipv4Address::from(addr)),
260        IpAddr::V6(addr) => IpAddress::from(net_lattice_ip::Ipv6Address::from(addr)),
261    }
262}
263
264fn message_to_route(message: &RouteMessage) -> Option<Route> {
265    let mut destination_addr = None;
266    let mut gateway = None;
267    let mut metric = None;
268    let mut interface_index = None;
269
270    for attribute in &message.attributes {
271        match attribute {
272            RouteAttribute::Destination(addr) => {
273                destination_addr = route_address_to_ip(addr);
274            }
275            RouteAttribute::Gateway(addr) => {
276                gateway = route_address_to_ip(addr).map(std_ip_to_ip_address);
277            }
278            RouteAttribute::Priority(priority) => {
279                metric = Some(*priority);
280            }
281            RouteAttribute::Oif(index) => {
282                interface_index = Some(*index);
283            }
284            _ => {}
285        }
286    }
287
288    let destination_addr = destination_addr?;
289    let prefix_len = message.header.destination_prefix_length;
290    let destination = match destination_addr {
291        IpAddr::V4(addr) => {
292            let prefix = net_lattice_ip::Ipv4PrefixLength::new(prefix_len)?;
293            Network::from(net_lattice_ip::Ipv4Network::new(addr.into(), prefix))
294        }
295        IpAddr::V6(addr) => {
296            let prefix = net_lattice_ip::Ipv6PrefixLength::new(prefix_len)?;
297            Network::from(net_lattice_ip::Ipv6Network::new(addr.into(), prefix))
298        }
299    };
300
301    let mut route = Route::new(synthesize_route_id(message), destination);
302    if let Some(gateway) = gateway {
303        route = route.with_gateway(gateway);
304    }
305    if let Some(metric) = metric {
306        route = route.with_metric(metric);
307    }
308    if let Some(interface_index) = interface_index {
309        route = route.with_interface_index(interface_index);
310    }
311    Some(route)
312}
313
314fn ip_address_to_std(address: IpAddress) -> IpAddr {
315    match address {
316        IpAddress::V4(addr) => IpAddr::V4(addr.into()),
317        IpAddress::V6(addr) => IpAddr::V6(addr.into()),
318    }
319}
320
321fn network_to_std(network: Network) -> (IpAddr, u8) {
322    match network {
323        Network::V4(net) => (IpAddr::V4(net.address().into()), net.prefix().value()),
324        Network::V6(net) => (IpAddr::V6(net.address().into()), net.prefix().value()),
325    }
326}
327
328impl RouteProvider for LinuxBackend {
329    type Route = Route;
330
331    fn routes(&self) -> Result<Vec<Self::Route>> {
332        self.runtime.block_on(async {
333            let route_handle = self.handle.route();
334
335            let mut v4 = route_handle
336                .get(RouteMessageBuilder::<std::net::Ipv4Addr>::new().build())
337                .execute();
338            let mut v6 = route_handle
339                .get(RouteMessageBuilder::<std::net::Ipv6Addr>::new().build())
340                .execute();
341
342            let mut routes = Vec::new();
343            while let Some(message) = v4
344                .try_next()
345                .await
346                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
347            {
348                routes.extend(message_to_route(&message));
349            }
350            while let Some(message) = v6
351                .try_next()
352                .await
353                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
354            {
355                routes.extend(message_to_route(&message));
356            }
357            Ok(routes)
358        })
359    }
360
361    fn add_route(&self, route: Self::Route) -> Result<()> {
362        self.runtime.block_on(async {
363            let (destination, prefix_len) = network_to_std(route.destination);
364            let message = match destination {
365                IpAddr::V4(addr) => {
366                    let mut builder = RouteMessageBuilder::<std::net::Ipv4Addr>::new()
367                        .destination_prefix(addr, prefix_len);
368                    if let Some(IpAddr::V4(gateway)) = route.gateway.map(ip_address_to_std) {
369                        builder = builder.gateway(gateway);
370                    }
371                    if let Some(metric) = route.metric {
372                        builder = builder.priority(metric);
373                    }
374                    if let Some(interface_index) = route.interface_index {
375                        builder = builder.output_interface(interface_index);
376                    }
377                    builder.build()
378                }
379                IpAddr::V6(addr) => {
380                    let mut builder = RouteMessageBuilder::<std::net::Ipv6Addr>::new()
381                        .destination_prefix(addr, prefix_len);
382                    if let Some(IpAddr::V6(gateway)) = route.gateway.map(ip_address_to_std) {
383                        builder = builder.gateway(gateway);
384                    }
385                    if let Some(metric) = route.metric {
386                        builder = builder.priority(metric);
387                    }
388                    if let Some(interface_index) = route.interface_index {
389                        builder = builder.output_interface(interface_index);
390                    }
391                    builder.build()
392                }
393            };
394
395            self.handle
396                .route()
397                .add(message)
398                .execute()
399                .await
400                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))
401        })
402    }
403
404    fn remove_route(&self, route: Self::Route) -> Result<()> {
405        self.runtime.block_on(async {
406            let (destination, prefix_len) = network_to_std(route.destination);
407            let message = match destination {
408                IpAddr::V4(addr) => {
409                    let mut builder = RouteMessageBuilder::<std::net::Ipv4Addr>::new()
410                        .destination_prefix(addr, prefix_len);
411                    if let Some(interface_index) = route.interface_index {
412                        builder = builder.output_interface(interface_index);
413                    }
414                    builder.build()
415                }
416                IpAddr::V6(addr) => {
417                    let mut builder = RouteMessageBuilder::<std::net::Ipv6Addr>::new()
418                        .destination_prefix(addr, prefix_len);
419                    if let Some(interface_index) = route.interface_index {
420                        builder = builder.output_interface(interface_index);
421                    }
422                    builder.build()
423                }
424            };
425
426            self.handle
427                .route()
428                .del(message)
429                .execute()
430                .await
431                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))
432        })
433    }
434}
435
436/// Maps Linux `ARPHRD_*` link-layer types (`LinkLayerType`) to the
437/// cross-platform [`InterfaceKind`]. Anything not covered falls back to
438/// `Other`, carrying the raw type code for diagnostics.
439fn link_layer_type_to_kind(link_layer_type: LinkLayerType) -> InterfaceKind {
440    match link_layer_type {
441        LinkLayerType::Ether => InterfaceKind::Ethernet,
442        LinkLayerType::Loopback => InterfaceKind::Loopback,
443        LinkLayerType::Ppp => InterfaceKind::PointToPoint,
444        LinkLayerType::Ieee80211
445        | LinkLayerType::Ieee80211Prism
446        | LinkLayerType::Ieee80211Radiotap => InterfaceKind::Wireless,
447        other => InterfaceKind::Other(u16::from(other) as u32),
448    }
449}
450
451fn message_to_interface(message: &LinkMessage) -> Interface {
452    let index = message.header.index;
453    let mut name = String::new();
454    let mut mac = None;
455    let mut mtu = None;
456    let mut operational_state = OperationalState::Unknown;
457
458    for attribute in &message.attributes {
459        match attribute {
460            LinkAttribute::IfName(value) => name = value.clone(),
461            LinkAttribute::Address(bytes) if bytes.len() == 6 => {
462                let mut octets = [0u8; 6];
463                octets.copy_from_slice(bytes);
464                mac = Some(MacAddress::new(octets));
465            }
466            LinkAttribute::Mtu(value) => mtu = Some(*value),
467            LinkAttribute::OperState(state) => {
468                operational_state = match state {
469                    State::Up => OperationalState::Up,
470                    State::Down | State::LowerLayerDown | State::NotPresent => {
471                        OperationalState::Down
472                    }
473                    State::Dormant => OperationalState::NoCarrier,
474                    _ => OperationalState::Unknown,
475                };
476            }
477            _ => {}
478        }
479    }
480
481    let admin_state = if message
482        .header
483        .flags
484        .contains(rtnetlink::packet_route::link::LinkFlags::Up)
485    {
486        AdminState::Up
487    } else {
488        AdminState::Down
489    };
490
491    let kind = link_layer_type_to_kind(message.header.link_layer_type);
492
493    let mut interface = Interface::new(Id::new(index as u64), index, name, kind)
494        .with_admin_state(admin_state)
495        .with_operational_state(operational_state);
496    if let Some(mac) = mac {
497        interface = interface.with_mac(mac);
498    }
499    if let Some(mtu) = mtu {
500        interface = interface.with_mtu(mtu);
501    }
502    interface
503}
504
505impl InterfaceProvider for LinuxBackend {
506    type Interface = Interface;
507
508    fn interfaces(&self) -> Result<Vec<Self::Interface>> {
509        self.runtime.block_on(async {
510            let mut links = self.handle.link().get().execute();
511            let mut interfaces = Vec::new();
512            while let Some(message) = links
513                .try_next()
514                .await
515                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
516            {
517                interfaces.push(message_to_interface(&message));
518            }
519            Ok(interfaces)
520        })
521    }
522}
523
524/// Placeholder identity scheme, same rationale as `synthesize_route_id`: a
525/// neighbor entry has no kernel-assigned numeric ID, so this hashes its
526/// interface and address together (the pair the kernel itself keys entries
527/// by, via `RTM_GETNEIGH`).
528fn synthesize_neighbor_id(interface_index: u32, address: &IpAddress) -> NeighborId {
529    let mut hasher = std::collections::hash_map::DefaultHasher::new();
530    interface_index.hash(&mut hasher);
531    address.hash(&mut hasher);
532    NeighborId::new(hasher.finish())
533}
534
535/// Maps Linux `NUD_*` neighbor states (`NeighbourState`) to the
536/// cross-platform [`NeighborState`].
537fn neighbour_state_to_state(state: RtNeighbourState) -> NeighborState {
538    match state {
539        RtNeighbourState::Incomplete => NeighborState::Incomplete,
540        RtNeighbourState::Reachable => NeighborState::Reachable,
541        RtNeighbourState::Stale => NeighborState::Stale,
542        RtNeighbourState::Delay => NeighborState::Delay,
543        RtNeighbourState::Probe => NeighborState::Probe,
544        RtNeighbourState::Failed => NeighborState::Failed,
545        RtNeighbourState::Permanent => NeighborState::Permanent,
546        _ => NeighborState::Unknown,
547    }
548}
549
550fn message_to_neighbor(message: &NeighbourMessage) -> Option<NeighborEntry> {
551    let interface_index = message.header.ifindex;
552    let mut address = None;
553    let mut mac = None;
554
555    for attribute in &message.attributes {
556        match attribute {
557            NeighbourAttribute::Destination(NeighbourAddress::Inet(addr)) => {
558                address = Some(std_ip_to_ip_address(IpAddr::V4(*addr)));
559            }
560            NeighbourAttribute::Destination(NeighbourAddress::Inet6(addr)) => {
561                address = Some(std_ip_to_ip_address(IpAddr::V6(*addr)));
562            }
563            NeighbourAttribute::LinkLayerAddress(bytes) if bytes.len() == 6 => {
564                let mut octets = [0u8; 6];
565                octets.copy_from_slice(bytes);
566                mac = Some(MacAddress::new(octets));
567            }
568            _ => {}
569        }
570    }
571
572    let address = address?;
573    let mut entry = NeighborEntry::new(
574        synthesize_neighbor_id(interface_index, &address),
575        interface_index,
576        address,
577    )
578    .with_state(neighbour_state_to_state(message.header.state));
579    if let Some(mac) = mac {
580        entry = entry.with_mac(mac);
581    }
582    Some(entry)
583}
584
585impl NeighborProvider for LinuxBackend {
586    type NeighborEntry = NeighborEntry;
587
588    fn neighbors(&self) -> Result<Vec<Self::NeighborEntry>> {
589        self.runtime.block_on(async {
590            let mut messages = self.handle.neighbours().get().execute();
591            let mut neighbors = Vec::new();
592            while let Some(message) = messages
593                .try_next()
594                .await
595                .map_err(|err| Error::Platform(rtnetlink_error_code(&err)))?
596            {
597                neighbors.extend(message_to_neighbor(&message));
598            }
599            Ok(neighbors)
600        })
601    }
602}
603
604/// Parses the `nameserver`/`search`/`domain` directives out of a
605/// `resolv.conf`-format file (`man 5 resolv.conf`) — the same format on
606/// Linux and BSD/macOS, so this parser is shared verbatim by
607/// `net-lattice-backend-darwin`.
608///
609/// `domain` is folded into `search_domains` too: it is the legacy
610/// single-domain predecessor of `search` and every modern resolver treats a
611/// lone `domain` entry as an implicit one-element search list.
612fn parse_resolv_conf(contents: &str) -> DnsConfig {
613    let mut config = DnsConfig::new();
614    for line in contents.lines() {
615        let line = line.trim();
616        if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
617            continue;
618        }
619        let mut parts = line.split_whitespace();
620        match parts.next() {
621            Some("nameserver") => {
622                if let Some(addr) = parts.next().and_then(|s| s.parse::<IpAddr>().ok()) {
623                    config.nameservers.push(std_ip_to_ip_address(addr));
624                }
625            }
626            Some("search") | Some("domain") => {
627                config
628                    .search_domains
629                    .extend(parts.map(|domain| domain.to_string()));
630            }
631            _ => {}
632        }
633    }
634    config
635}
636
637fn render_resolv_conf(config: &NewDnsConfig) -> String {
638    let mut contents = String::new();
639    for nameserver in &config.nameservers {
640        contents.push_str("nameserver ");
641        contents.push_str(&nameserver.to_string());
642        contents.push('\n');
643    }
644    if !config.search_domains.is_empty() {
645        contents.push_str("search ");
646        contents.push_str(&config.search_domains.join(" "));
647        contents.push('\n');
648    }
649    contents
650}
651
652fn write_resolv_conf(path: &std::path::Path, config: &NewDnsConfig) -> Result<()> {
653    std::fs::write(path, render_resolv_conf(config)).map_err(|err| resolv_conf_error(&err))
654}
655
656fn resolv_conf_error(err: &std::io::Error) -> Error {
657    match err.kind() {
658        std::io::ErrorKind::NotFound => Error::NotFound,
659        std::io::ErrorKind::PermissionDenied => Error::PermissionDenied,
660        _ => Error::Platform(io_error_code(err)),
661    }
662}
663
664impl CapabilityProvider for LinuxBackend {
665    /// `IPV6` unconditionally: every provider this backend implements
666    /// (routes, interfaces, neighbors, addresses) already handles both
667    /// address families. `MONITORING` unconditionally too, now that
668    /// `EventProvider` is implemented below. `VRF`/`NAMESPACES` are left
669    /// unset — Linux genuinely supports both at the kernel level, but Net
670    /// Lattice doesn't implement either yet, and a `Capability` this
671    /// backend can't actually act on would be a lie to the caller.
672    fn capabilities(&self) -> Capability {
673        Capability::IPV6 | Capability::MONITORING | Capability::DNS_MUTATION
674    }
675}
676
677/// Maps one Netlink route/link/neighbour/address notification to an
678/// [`Event`]. Reuses the existing `message_to_*` conversion functions
679/// (built for the one-shot `routes()`/`interfaces()`/... dumps) purely to
680/// derive the same `Id` a consumer would see from those methods — a
681/// notification's `DelFoo` variant tells us an object was removed. Netlink
682/// uses the corresponding `NewFoo` message for both creation and mutation,
683/// so it is reported as `Changed` rather than incorrectly promising an add;
684/// only the `Id` needs deriving here.
685///
686/// Returns `None` for Netlink message kinds this backend doesn't turn into
687/// an `Event` (link property changes, neighbour tables, ...) and for any
688/// notification whose attributes don't parse into an `Id` (extremely rare
689/// for the kernel's own unsolicited notifications, but the parsers are
690/// intentionally lenient rather than panicking on unexpected input).
691fn route_netlink_message_to_event(message: RouteNetlinkMessage) -> Option<Event> {
692    match message {
693        RouteNetlinkMessage::NewRoute(msg) => message_to_route(&msg).map(|route| Event::Route {
694            id: route.id,
695            kind: ChangeKind::Changed,
696        }),
697        RouteNetlinkMessage::DelRoute(msg) => message_to_route(&msg).map(|route| Event::Route {
698            id: route.id,
699            kind: ChangeKind::Removed,
700        }),
701        RouteNetlinkMessage::NewLink(msg) => Some(Event::Interface {
702            id: Id::new(msg.header.index as u64),
703            kind: ChangeKind::Changed,
704        }),
705        RouteNetlinkMessage::DelLink(msg) => Some(Event::Interface {
706            id: Id::new(msg.header.index as u64),
707            kind: ChangeKind::Removed,
708        }),
709        RouteNetlinkMessage::NewNeighbour(msg) => {
710            message_to_neighbor(&msg).map(|entry| Event::Neighbor {
711                id: entry.id,
712                kind: ChangeKind::Changed,
713            })
714        }
715        RouteNetlinkMessage::DelNeighbour(msg) => {
716            message_to_neighbor(&msg).map(|entry| Event::Neighbor {
717                id: entry.id,
718                kind: ChangeKind::Removed,
719            })
720        }
721        RouteNetlinkMessage::NewAddress(msg) => {
722            message_to_interface_address(&msg).map(|addr| Event::Address {
723                id: addr.id,
724                kind: ChangeKind::Changed,
725            })
726        }
727        RouteNetlinkMessage::DelAddress(msg) => {
728            message_to_interface_address(&msg).map(|addr| Event::Address {
729                id: addr.id,
730                kind: ChangeKind::Removed,
731            })
732        }
733        _ => None,
734    }
735}
736
737impl EventProvider for LinuxBackend {
738    type Event = Event;
739    type EventFilter = EventFilter;
740
741    /// Opens a *second*, independent Netlink socket subscribed to the
742    /// `RTNLGRP_LINK`/`RTNLGRP_NEIGH`/`RTNLGRP_IPV4_ROUTE`/
743    /// `RTNLGRP_IPV6_ROUTE`/`RTNLGRP_IPV4_IFADDR`/`RTNLGRP_IPV6_IFADDR`
744    /// multicast groups (`rtnetlink::new_multicast_connection`, the same
745    /// mechanism `ip monitor` uses) — separate from `self.handle`'s socket,
746    /// which only ever sends request/reply traffic. Two background tasks
747    /// are spawned onto `self.runtime`: one drives the new socket's
748    /// connection (required for it to make progress at all, mirroring
749    /// `LinuxBackend::new`'s `connection` task), the other reads
750    /// unsolicited notifications off it and forwards mapped `Event`s into
751    /// the channel `EventReceiver` reads from.
752    ///
753    /// DNS resolver configuration changes (`/etc/resolv.conf`) have no
754    /// Netlink signal — see `Event`'s doc comment — so no `Event::Dns` is
755    /// ever produced by this backend.
756    fn watch(&self) -> Result<EventReceiver<Self::Event>> {
757        self.watch_filtered(EventFilter::ALL)
758    }
759    fn watch_filtered(&self, filter: Self::EventFilter) -> Result<EventReceiver<Self::Event>> {
760        let groups = [
761            MulticastGroup::Link,
762            MulticastGroup::Neigh,
763            MulticastGroup::Ipv4Route,
764            MulticastGroup::Ipv6Route,
765            MulticastGroup::Ipv4Ifaddr,
766            MulticastGroup::Ipv6Ifaddr,
767        ];
768        let _guard = self.runtime.enter();
769        let (connection, _handle, mut messages) = rtnetlink::new_multicast_connection(&groups)
770            .map_err(|err| Error::Platform(io_error_code(&err)))?;
771        let connection = self.runtime.spawn(connection);
772
773        let (sender, receiver) = EventReceiver::bounded();
774        let events = self.runtime.spawn(async move {
775            while let Some((message, _addr)) = messages.next().await {
776                let (_header, payload) = message.into_parts();
777                let rtnetlink::packet_core::NetlinkPayload::InnerMessage(inner) = payload else {
778                    continue;
779                };
780                if let Some(event) = route_netlink_message_to_event(inner)
781                    && filter.matches(event)
782                    && !sender.send(event, Event::resync_all())
783                {
784                    break;
785                }
786            }
787        });
788
789        Ok(receiver.with_subscription(LinuxWatch { connection, events }))
790    }
791}
792
793/// Native async monitoring uses the backend's Tokio reactor and Netlink
794/// multicast socket directly.
795///
796/// Unlike `net_lattice_async::from_receiver`, this implementation does not
797/// place a blocking worker thread between the kernel notification source and
798/// the returned stream.
799#[cfg(feature = "async")]
800impl TokioEventProvider for LinuxBackend {
801    type Event = Event;
802    type EventFilter = EventFilter;
803
804    fn watch_tokio(&self, filter: Self::EventFilter) -> Result<TokioEventReceiver<Self::Event>> {
805        let groups = [
806            MulticastGroup::Link,
807            MulticastGroup::Neigh,
808            MulticastGroup::Ipv4Route,
809            MulticastGroup::Ipv6Route,
810            MulticastGroup::Ipv4Ifaddr,
811            MulticastGroup::Ipv6Ifaddr,
812        ];
813        let _guard = self.runtime.enter();
814        let (connection, _handle, mut messages) = rtnetlink::new_multicast_connection(&groups)
815            .map_err(|err| Error::Platform(io_error_code(&err)))?;
816        let connection = self.runtime.spawn(connection);
817        let (sender, receiver) = TokioEventReceiver::bounded();
818        let events = self.runtime.spawn(async move {
819            while let Some((message, _addr)) = messages.next().await {
820                let (_header, payload) = message.into_parts();
821                let rtnetlink::packet_core::NetlinkPayload::InnerMessage(inner) = payload else {
822                    continue;
823                };
824                if let Some(event) = route_netlink_message_to_event(inner)
825                    && filter.matches(event)
826                    && !sender.send(event, Event::resync_all)
827                {
828                    return;
829                }
830            }
831            let _ = sender.send_error(Error::Disconnected);
832        });
833        Ok(receiver.with_subscription(LinuxWatch { connection, events }))
834    }
835}
836
837impl DnsProvider for LinuxBackend {
838    type DnsConfig = DnsConfig;
839
840    fn dns_config(&self) -> Result<Self::DnsConfig> {
841        let contents =
842            std::fs::read_to_string("/etc/resolv.conf").map_err(|err| resolv_conf_error(&err))?;
843        Ok(parse_resolv_conf(&contents))
844    }
845}
846
847impl DnsMutator for LinuxBackend {
848    type NewDnsConfig = NewDnsConfig;
849
850    fn set_dns_config(&self, config: Self::NewDnsConfig) -> Result<Self::DnsConfig> {
851        write_resolv_conf(std::path::Path::new("/etc/resolv.conf"), &config)?;
852        self.dns_config()
853    }
854}
855
856#[cfg(test)]
857mod tests {
858    use super::*;
859    use net_lattice_ip::{Ipv4Address, Ipv4Network, Ipv4PrefixLength};
860    use std::sync::{Mutex, MutexGuard, OnceLock};
861
862    #[cfg(feature = "async")]
863    fn tokio_route_event(watcher: &mut TokioEventReceiver<Event>, id: RouteId) -> bool {
864        use std::pin::Pin;
865        use std::task::{Context, Poll, Waker};
866        use std::time::Duration;
867
868        let waker = Waker::noop();
869        let mut context = Context::from_waker(waker);
870        for _ in 0..12 {
871            match Pin::new(&mut *watcher).poll_recv(&mut context) {
872                Poll::Ready(Some(Ok(Event::Route { id: event_id, .. }))) if event_id == id => {
873                    return true;
874                }
875                Poll::Ready(Some(_)) | Poll::Pending => thread_sleep(Duration::from_millis(250)),
876                Poll::Ready(None) => return false,
877            }
878        }
879        false
880    }
881
882    #[cfg(feature = "async")]
883    fn thread_sleep(duration: std::time::Duration) {
884        std::thread::sleep(duration);
885    }
886
887    /// The kernel can reject simultaneous Netlink dumps in a shared CI
888    /// network namespace with `EBUSY`. All tests that open a real Netlink
889    /// socket take this guard; pure parser tests intentionally do not.
890    fn kernel_test_guard() -> MutexGuard<'static, ()> {
891        static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
892        GUARD
893            .get_or_init(|| Mutex::new(()))
894            .lock()
895            // A failed kernel assertion must not hide every later test behind
896            // `PoisonError`; they should each report their own OS failure.
897            .unwrap_or_else(std::sync::PoisonError::into_inner)
898    }
899
900    /// Exercises a real round trip through Netlink, no privilege required:
901    /// `RTM_GETROUTE` dumps are readable by any user. This is the one test
902    /// in this module that runs by default and actually proves the backend
903    /// talks to the kernel, rather than only exercising conversion logic.
904    #[test]
905    fn routes_reads_the_real_kernel_routing_table() {
906        let _guard = kernel_test_guard();
907        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
908        let routes = backend
909            .routes()
910            .expect("RTM_GETROUTE dump should not require privilege");
911        // Not asserting on contents: the routing table of the machine
912        // running this test is arbitrary (may even be empty in a minimal
913        // container). Reaching here without an error is the assertion.
914        let _ = routes;
915    }
916
917    /// Exercises a real round trip through Netlink, no privilege required:
918    /// `RTM_GETLINK` dumps are readable by any user, and every Linux system
919    /// has at least `lo`.
920    #[test]
921    fn interfaces_includes_the_loopback_interface() {
922        let _guard = kernel_test_guard();
923        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
924        let interfaces = backend
925            .interfaces()
926            .expect("RTM_GETLINK dump should not require privilege");
927        assert!(
928            interfaces
929                .iter()
930                .any(|iface| iface.name == "lo" && iface.kind == InterfaceKind::Loopback),
931            "expected a `lo` interface classified as Loopback, got: {interfaces:?}"
932        );
933    }
934
935    /// Exercises a real round trip through Netlink, no privilege required:
936    /// `RTM_GETADDR` dumps are readable by any user, and every Linux system
937    /// has at least `lo`'s `127.0.0.1/8`.
938    #[test]
939    fn addresses_includes_loopbacks_address() {
940        let _guard = kernel_test_guard();
941        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
942        let addresses = backend
943            .addresses()
944            .expect("RTM_GETADDR dump should not require privilege");
945        assert!(
946            addresses.iter().any(|addr| matches!(
947                addr.address,
948                Network::V4(net) if net.address() == Ipv4Address::new(127, 0, 0, 1)
949            )),
950            "expected `127.0.0.1` among the assigned addresses, got: {addresses:?}"
951        );
952    }
953
954    /// Exercises a real round trip through Netlink, no privilege required:
955    /// `RTM_GETNEIGH` dumps are readable by any user.
956    #[test]
957    fn neighbors_reads_the_real_kernel_neighbor_table() {
958        let _guard = kernel_test_guard();
959        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
960        let neighbors = backend
961            .neighbors()
962            .expect("RTM_GETNEIGH dump should not require privilege");
963        // Not asserting on contents: the neighbor table of the machine
964        // running this test is arbitrary (may even be empty). Reaching here
965        // without an error is the assertion.
966        let _ = neighbors;
967    }
968
969    /// Opens a real multicast subscription without changing system state.
970    /// The ignored test below verifies delivery through the same subscription
971    /// by adding and removing a temporary route.
972    #[test]
973    fn watch_opens_a_real_netlink_subscription() {
974        use std::time::Duration;
975
976        let _guard = kernel_test_guard();
977        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
978        assert!(backend.capabilities().contains(Capability::MONITORING));
979        let watcher = backend
980            .watch()
981            .expect("failed to subscribe to Netlink multicast groups");
982        assert!(watcher.recv_timeout(Duration::from_millis(1)).is_ok());
983        let filtered = backend
984            .watch_filtered(EventFilter::none())
985            .expect("failed to subscribe to filtered Netlink events");
986        assert_eq!(
987            filtered.recv_timeout(Duration::from_millis(1)).unwrap(),
988            None
989        );
990    }
991
992    /// The feature-gated provider opens the same native Netlink multicast
993    /// subscription, but delivers it through the Tokio-aware transport used
994    /// by `Lattice::watch_async`.
995    #[cfg(feature = "async")]
996    #[test]
997    fn watch_tokio_opens_a_real_netlink_subscription() {
998        let _guard = kernel_test_guard();
999        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
1000        let watcher = backend
1001            .watch_tokio(EventFilter::none())
1002            .expect("failed to subscribe to Netlink multicast groups");
1003        drop(watcher);
1004    }
1005
1006    #[test]
1007    fn parse_resolv_conf_reads_nameservers_and_search_domains() {
1008        let contents = "# comment\n\
1009                         nameserver 1.1.1.1\n\
1010                         nameserver 2606:4700:4700::1111\n\
1011                         search example.com corp.example.com\n";
1012        let config = parse_resolv_conf(contents);
1013        assert_eq!(
1014            config.nameservers,
1015            vec![
1016                IpAddress::from(Ipv4Address::new(1, 1, 1, 1)),
1017                std_ip_to_ip_address("2606:4700:4700::1111".parse().unwrap()),
1018            ]
1019        );
1020        assert_eq!(
1021            config.search_domains,
1022            vec!["example.com".to_string(), "corp.example.com".to_string()]
1023        );
1024    }
1025
1026    #[test]
1027    fn render_resolv_conf_preserves_requested_order() {
1028        let config = NewDnsConfig::with(
1029            vec![
1030                IpAddress::from(Ipv4Address::new(1, 1, 1, 1)),
1031                IpAddress::from(Ipv4Address::new(8, 8, 8, 8)),
1032            ],
1033            vec!["example.test".to_string(), "corp.test".to_string()],
1034        );
1035        assert_eq!(
1036            render_resolv_conf(&config),
1037            "nameserver 1.1.1.1\nnameserver 8.8.8.8\nsearch example.test corp.test\n"
1038        );
1039    }
1040
1041    /// Reads the real `/etc/resolv.conf` present on this test environment.
1042    /// Every Linux system has one (even if empty/symlinked to
1043    /// systemd-resolved's stub), so this exercises the real filesystem read
1044    /// without requiring any specific content.
1045    #[test]
1046    fn dns_config_reads_the_real_resolv_conf() {
1047        let _guard = kernel_test_guard();
1048        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
1049        let config = backend
1050            .dns_config()
1051            .expect("/etc/resolv.conf should be readable");
1052        let _ = config;
1053    }
1054
1055    fn loopback_interface_index(backend: &LinuxBackend) -> u32 {
1056        backend
1057            .runtime
1058            .block_on(async {
1059                let mut links = backend
1060                    .handle
1061                    .link()
1062                    .get()
1063                    .match_name("lo".into())
1064                    .execute();
1065                links
1066                    .try_next()
1067                    .await
1068                    .ok()
1069                    .flatten()
1070                    .map(|link| link.header.index)
1071            })
1072            .expect("this test environment has no `lo` interface")
1073    }
1074
1075    /// Requires `CAP_NET_ADMIN` (root, or `sudo -E cargo test -- --ignored`
1076    /// in this crate). Not run by default because most development and CI
1077    /// environments — including the one this crate was originally written
1078    /// in — don't grant it, and this test would otherwise fail with
1079    /// `PermissionDenied` rather than being skipped.
1080    ///
1081    /// Uses a documentation-only prefix (RFC 5737 `203.0.113.0/24`,
1082    /// TEST-NET-3) on `lo` so it can't collide with or disrupt real
1083    /// routing, and removes what it added regardless of assertion outcome.
1084    #[test]
1085    #[ignore = "requires CAP_NET_ADMIN; run with `sudo -E cargo test -p net-lattice-backend-linux -- --ignored`"]
1086    fn add_then_remove_route_round_trips_through_the_kernel() {
1087        let _guard = kernel_test_guard();
1088        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
1089        let interface_index = loopback_interface_index(&backend);
1090
1091        let destination = Network::from(Ipv4Network::new(
1092            Ipv4Address::new(203, 0, 113, 0),
1093            Ipv4PrefixLength::new(24).unwrap(),
1094        ));
1095        let route = Route::new(RouteId::new(0), destination).with_interface_index(interface_index);
1096
1097        let add_result = backend.add_route(route.clone());
1098        if matches!(
1099            add_result,
1100            Err(Error::PermissionDenied) | Err(Error::Platform(_))
1101        ) {
1102            // Best effort even under #[ignore]: if it's run without the
1103            // capability after all, fail loudly rather than silently
1104            // passing on a no-op.
1105            add_result.expect("add_route failed - are you running with CAP_NET_ADMIN?");
1106        }
1107
1108        let routes = backend
1109            .routes()
1110            .expect("routes() failed after add_route succeeded");
1111        let found = routes
1112            .iter()
1113            .any(|r| r.destination == destination && r.interface_index == Some(interface_index));
1114
1115        // Clean up before asserting, so a failed assertion doesn't leave
1116        // the test route behind on the machine that ran this.
1117        let _ = backend.remove_route(route);
1118
1119        assert!(found, "added route was not present in routes() afterward");
1120
1121        let routes_after_removal = backend
1122            .routes()
1123            .expect("routes() failed after remove_route");
1124        assert!(
1125            !routes_after_removal
1126                .iter()
1127                .any(|r| r.destination == destination && r.interface_index == Some(interface_index)),
1128            "removed route was still present in routes() afterward"
1129        );
1130    }
1131
1132    /// Exercises the complete address-mutation path against the real
1133    /// kernel: create an IPv4 address on `lo`, read the canonical observed
1134    /// record, then remove that exact record. TEST-NET-1 is reserved for
1135    /// documentation and is distinct from the route tests' prefixes.
1136    #[test]
1137    #[ignore = "requires CAP_NET_ADMIN; run with `sudo -E cargo test -p net-lattice-backend-linux add_then_remove_address_round_trips_through_the_kernel -- --ignored`"]
1138    fn add_then_remove_address_round_trips_through_the_kernel() {
1139        let _guard = kernel_test_guard();
1140        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
1141        let interface_index = loopback_interface_index(&backend);
1142        let network = Network::from(Ipv4Network::new(
1143            Ipv4Address::new(192, 0, 2, 9),
1144            Ipv4PrefixLength::new(24).unwrap(),
1145        ));
1146        let requested = NewInterfaceAddress::new(Id::new(interface_index as u64), network);
1147
1148        // A prior interrupted ignored-test run must not turn this run into a
1149        // false duplicate. The address range is test-only and cleanup is
1150        // deliberately best-effort before the actual assertion path.
1151        if let Some(existing) = backend
1152            .addresses()
1153            .expect("addresses() failed before add_address")
1154            .into_iter()
1155            .find(|address| {
1156                address.interface_index == interface_index && address.address == network
1157            })
1158        {
1159            let _ = backend.remove_address(existing);
1160        }
1161
1162        let observed = backend
1163            .add_address(requested)
1164            .expect("add_address failed - are you running with CAP_NET_ADMIN?");
1165        let present = backend
1166            .addresses()
1167            .expect("addresses() failed after add_address")
1168            .into_iter()
1169            .any(|address| address.id == observed.id);
1170
1171        backend
1172            .remove_address(observed.clone())
1173            .expect("remove_address failed after successful add_address");
1174        let absent = !backend
1175            .addresses()
1176            .expect("addresses() failed after remove_address")
1177            .into_iter()
1178            .any(|address| address.id == observed.id);
1179
1180        assert!(
1181            present,
1182            "added address was not present in addresses() afterward"
1183        );
1184        assert!(
1185            absent,
1186            "removed address was still present in addresses() afterward"
1187        );
1188    }
1189
1190    /// End-to-end monitoring verification: a route mutation must travel from
1191    /// the kernel's Netlink multicast group through `watch()` to the caller.
1192    /// It is ignored by default because creating the route requires
1193    /// `CAP_NET_ADMIN`.
1194    #[test]
1195    #[ignore = "requires CAP_NET_ADMIN; run with `sudo -E cargo test -p net-lattice-backend-linux watch_observes_route_changes -- --ignored`"]
1196    fn watch_observes_route_changes() {
1197        use std::time::Duration;
1198
1199        let _guard = kernel_test_guard();
1200        let backend = LinuxBackend::new().expect("failed to open a Netlink connection");
1201        assert!(backend.capabilities().contains(Capability::MONITORING));
1202        let watcher = backend
1203            .watch()
1204            .expect("failed to subscribe to Netlink events");
1205        #[cfg(feature = "async")]
1206        let mut async_watcher = backend
1207            .watch_tokio(EventFilter::none().routes())
1208            .expect("failed to subscribe to async Netlink events");
1209        let interface_index = loopback_interface_index(&backend);
1210        let destination = Network::from(Ipv4Network::new(
1211            Ipv4Address::new(198, 51, 100, 0),
1212            Ipv4PrefixLength::new(24).unwrap(),
1213        ));
1214        let route = Route::new(RouteId::new(0), destination).with_interface_index(interface_index);
1215
1216        backend
1217            .add_route(route.clone())
1218            .expect("failed to add monitoring test route");
1219        let watched_id = backend
1220            .routes()
1221            .expect("failed to read routes after adding test route")
1222            .into_iter()
1223            .find(|candidate| {
1224                candidate.destination == destination
1225                    && candidate.interface_index == Some(interface_index)
1226            })
1227            .expect("test route was not present after it was added")
1228            .id;
1229
1230        let observed = (0..12).any(|_| {
1231            matches!(
1232                watcher.recv_timeout(Duration::from_millis(250)),
1233                Ok(Some(Event::Route { id, .. })) if id == watched_id
1234            )
1235        });
1236        #[cfg(feature = "async")]
1237        let async_observed = tokio_route_event(&mut async_watcher, watched_id);
1238        let selected_watcher = backend
1239            .watch_filtered(EventFilter::none().route(watched_id))
1240            .expect("failed to subscribe to selected Netlink route events");
1241        #[cfg(feature = "async")]
1242        let mut selected_async_watcher = backend
1243            .watch_tokio(EventFilter::none().route(watched_id))
1244            .expect("failed to subscribe to selected async Netlink route events");
1245        let _ = backend.remove_route(route);
1246        let selected_observed = (0..12).any(|_| {
1247            matches!(
1248                selected_watcher.recv_timeout(Duration::from_millis(250)),
1249                Ok(Some(Event::Route { id, kind: ChangeKind::Removed })) if id == watched_id
1250            )
1251        });
1252        #[cfg(feature = "async")]
1253        let selected_async_observed = tokio_route_event(&mut selected_async_watcher, watched_id);
1254        assert!(observed, "watch() did not report the route mutation");
1255        assert!(
1256            selected_observed,
1257            "object route filter did not report removal"
1258        );
1259        #[cfg(feature = "async")]
1260        assert!(
1261            async_observed,
1262            "watch_tokio() did not report the route mutation"
1263        );
1264        #[cfg(feature = "async")]
1265        assert!(
1266            selected_async_observed,
1267            "async object route filter did not report removal"
1268        );
1269    }
1270}