microsandbox_network/engine/netstack/
shared.rs

1//! Shared state between the NetWorker thread, smoltcp poll thread, and tokio
2//! proxy tasks.
3//!
4//! All inter-thread communication flows through [`SharedState`], which holds
5//! lock-free frame queues and cross-platform [`WakePipe`] notifications.
6
7use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
8use std::sync::{
9    Arc, Mutex, OnceLock,
10    atomic::{AtomicU64, Ordering},
11};
12use std::time::{Duration, Instant};
13
14use crossbeam_queue::ArrayQueue;
15use ipnetwork::Ipv6Network;
16use microsandbox_types::HttpConfig;
17use microsandbox_utils::ttl_reverse_index::TtlReverseIndex;
18pub use microsandbox_utils::wake_pipe::WakePipe;
19use parking_lot::RwLock;
20
21use crate::addr::normalize_ip_addr;
22use crate::engine::http_deny::{self, DEFAULT_HTTP_DENY_MESSAGE};
23
24//--------------------------------------------------------------------------------------------------
25// Constants
26//--------------------------------------------------------------------------------------------------
27
28/// Default frame queue capacity. Matches libkrun's virtio queue size.
29pub const DEFAULT_QUEUE_CAPACITY: usize = 1024;
30
31/// Maximum live hostname/address bindings retained by one sandbox.
32pub(crate) const MAX_RESOLVED_HOSTNAME_BINDINGS: usize = 16_384;
33
34/// Maximum live bindings for one normalized hostname, across both families.
35pub(crate) const MAX_BINDINGS_PER_HOSTNAME: usize = 1_024;
36
37//--------------------------------------------------------------------------------------------------
38// Types
39//--------------------------------------------------------------------------------------------------
40
41/// All shared state between the three threads:
42///
43/// - **NetWorker** (libkrun) — pushes guest frames to `tx_ring`, pops
44///   response frames from `rx_ring`.
45/// - **smoltcp poll thread** — pops from `tx_ring`, processes through smoltcp,
46///   pushes responses to `rx_ring`.
47/// - **tokio proxy tasks** — relay data between smoltcp sockets and real
48///   network connections.
49///
50/// Queue naming follows the **guest's perspective** (matching libkrun's
51/// convention): `tx_ring` = "transmit from guest", `rx_ring` = "receive at
52/// guest".
53pub struct SharedState {
54    /// Frames from guest → smoltcp (NetWorker writes, smoltcp reads).
55    pub tx_ring: ArrayQueue<Vec<u8>>,
56
57    /// Frames from smoltcp → guest (smoltcp writes, NetWorker reads).
58    pub rx_ring: ArrayQueue<Vec<u8>>,
59
60    /// Wakes NetWorker: "rx_ring has frames for the guest."
61    /// Written by `SmoltcpDevice::transmit()`. Read end polled by NetWorker's
62    /// epoll loop.
63    pub rx_wake: WakePipe,
64
65    /// Wakes smoltcp poll thread: "tx_ring has frames from the guest."
66    /// Written by `SmoltcpBackend::write_frame()`. Read end polled by the
67    /// poll loop.
68    pub tx_wake: WakePipe,
69
70    /// Wakes smoltcp poll thread: "proxy task has data to write to a smoltcp
71    /// socket." Written by proxy tasks via channels. Read end polled by the
72    /// poll loop.
73    pub proxy_wake: WakePipe,
74
75    /// Optional host-side termination hook used for fatal policy violations.
76    termination_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
77
78    /// Resolved hostname index used to map destination IPs back to queried hostnames.
79    resolved_hostnames: RwLock<TtlReverseIndex<ResolvedHostnameKey, IpAddr>>,
80
81    /// Per-sandbox gateway IPv4. Set once at boot; used by
82    /// `DestinationGroup::Host` rule matching and `host.microsandbox.internal`
83    /// DNS synthesis. `None` in isolated unit tests.
84    gateway_ipv4: OnceLock<Ipv4Addr>,
85
86    /// Per-sandbox gateway IPv6. Set once at boot. See `gateway_ipv4`.
87    gateway_ipv6: OnceLock<Ipv6Addr>,
88
89    /// NAT64 `/96` prefixes used by policy classification.
90    nat64_prefixes: OnceLock<Vec<Ipv6Network>>,
91
92    /// Aggregate network byte counters at the guest/runtime boundary.
93    metrics: NetworkMetrics,
94
95    /// HTTP denial settings installed before the network starts.
96    http: OnceLock<HttpConfig>,
97}
98
99/// Aggregate network byte counters shared with the runtime metrics sampler.
100pub struct NetworkMetrics {
101    tx_bytes: AtomicU64,
102    rx_bytes: AtomicU64,
103}
104
105/// Address family for resolved hostname entries.
106#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
107pub enum ResolvedHostnameFamily {
108    Ipv4,
109    Ipv6,
110}
111
112/// Composite cache key for a single DNS resolution.
113///
114/// `family` partitions entries so that `A` and `AAAA` responses for the
115/// same hostname refresh independently instead of overwriting each other.
116#[derive(Clone, Debug, PartialEq, Eq, Hash)]
117struct ResolvedHostnameKey {
118    hostname: String,
119    family: ResolvedHostnameFamily,
120}
121
122//--------------------------------------------------------------------------------------------------
123// Methods
124//--------------------------------------------------------------------------------------------------
125
126impl SharedState {
127    /// Create shared state with the given queue capacity.
128    pub fn new(queue_capacity: usize) -> Self {
129        Self {
130            tx_ring: ArrayQueue::new(queue_capacity),
131            rx_ring: ArrayQueue::new(queue_capacity),
132            rx_wake: WakePipe::new(),
133            tx_wake: WakePipe::new(),
134            proxy_wake: WakePipe::new(),
135            termination_hook: Mutex::new(None),
136            resolved_hostnames: RwLock::new(TtlReverseIndex::default()),
137            gateway_ipv4: OnceLock::new(),
138            gateway_ipv6: OnceLock::new(),
139            nat64_prefixes: OnceLock::new(),
140            metrics: NetworkMetrics::default(),
141            http: OnceLock::new(),
142        }
143    }
144
145    /// Set the per-sandbox gateway IPs. Called once at boot. Each family is
146    /// only published when active for this sandbox.
147    pub fn set_gateway_ips(&self, ipv4: Option<Ipv4Addr>, ipv6: Option<Ipv6Addr>) {
148        if let Some(ipv4) = ipv4 {
149            let _ = self.gateway_ipv4.set(ipv4);
150        }
151        if let Some(ipv6) = ipv6 {
152            let _ = self.gateway_ipv6.set(ipv6);
153        }
154    }
155
156    /// Gateway IPv4 address, if set.
157    pub fn gateway_ipv4(&self) -> Option<Ipv4Addr> {
158        self.gateway_ipv4.get().copied()
159    }
160
161    /// Gateway IPv6 address, if set.
162    pub fn gateway_ipv6(&self) -> Option<Ipv6Addr> {
163        self.gateway_ipv6.get().copied()
164    }
165
166    /// Install HTTP denial settings. The first call wins.
167    pub fn set_http_config(&self, config: HttpConfig) {
168        let _ = self.http.set(config);
169    }
170
171    /// Whether readable denial responses were explicitly enabled.
172    pub fn http_deny_response_enabled(&self) -> bool {
173        self.http.get().is_some_and(|http| http.deny_response)
174    }
175
176    /// Render the HTTP/HTTPS deny body for `host`.
177    pub fn http_deny_body(&self, host: &str) -> String {
178        let template = self
179            .http
180            .get()
181            .and_then(|http| http.deny_message.as_deref())
182            .unwrap_or(DEFAULT_HTTP_DENY_MESSAGE);
183        http_deny::render_http_deny_message(template, host)
184    }
185
186    /// Set NAT64 prefixes. Called once before policy evaluation starts.
187    pub fn set_nat64_prefixes(&self, prefixes: Vec<Ipv6Network>) {
188        let _ = self.nat64_prefixes.set(prefixes);
189    }
190
191    /// NAT64 prefixes used by policy classification.
192    pub fn nat64_prefixes(&self) -> &[Ipv6Network] {
193        self.nat64_prefixes.get().map(Vec::as_slice).unwrap_or(&[])
194    }
195
196    /// Install a host-side termination hook.
197    pub fn set_termination_hook(&self, hook: Arc<dyn Fn() + Send + Sync>) {
198        *self.termination_hook.lock().unwrap() = Some(hook);
199    }
200
201    /// Trigger host-side termination if a hook is installed.
202    pub fn trigger_termination(&self) {
203        let hook = self.termination_hook.lock().unwrap().clone();
204        if let Some(hook) = hook {
205            hook();
206        }
207    }
208
209    /// Record resolved addresses for a hostname within the given address family.
210    ///
211    /// Addresses from earlier answers stay bound until the TTL of the answer
212    /// that returned them expires.
213    /// Resolvers that rotate their answers hand concurrent lookups of one name
214    /// different addresses, and a connection dialed on any of them must still
215    /// match the name's domain rules.
216    ///
217    /// Returns `false` if the complete answer would exceed either the hostname
218    /// or sandbox binding limit; existing bindings are preserved on rejection.
219    pub fn cache_resolved_hostname(
220        &self,
221        domain: &str,
222        family: ResolvedHostnameFamily,
223        addrs: impl IntoIterator<Item = IpAddr>,
224        ttl: Duration,
225    ) -> bool {
226        let hostname = normalize_hostname(domain);
227        let key = ResolvedHostnameKey { hostname, family };
228        let addrs = addrs.into_iter().map(normalize_ip_addr);
229
230        let now = Instant::now();
231        let mut index = self.resolved_hostnames.write();
232        index.evict_expired(now);
233
234        let bindings_for = |family| {
235            index.member_count(&ResolvedHostnameKey {
236                hostname: key.hostname.clone(),
237                family,
238            })
239        };
240
241        let ipv4_bindings = bindings_for(ResolvedHostnameFamily::Ipv4);
242        let ipv6_bindings = bindings_for(ResolvedHostnameFamily::Ipv6);
243        let hostname_bindings = ipv4_bindings + ipv6_bindings;
244
245        let total_bindings = index.binding_count();
246        let hostname_slots = MAX_BINDINGS_PER_HOSTNAME.saturating_sub(hostname_bindings);
247        let sandbox_slots = MAX_RESOLVED_HOSTNAME_BINDINGS.saturating_sub(total_bindings);
248        let available_slots = hostname_slots.min(sandbox_slots);
249
250        // try_extend takes a total capacity; only new bindings consume slots.
251        let capacity = total_bindings + available_slots;
252        index.try_extend(key, addrs, ttl, now, capacity)
253    }
254
255    /// Clear the resolved addresses for a hostname within the given address family.
256    pub fn clear_resolved_hostname(&self, domain: &str, family: ResolvedHostnameFamily) {
257        let hostname = normalize_hostname(domain);
258        let key = ResolvedHostnameKey { hostname, family };
259        self.resolved_hostnames.write().remove(&key, Instant::now());
260    }
261
262    /// Returns `true` when any resolved hostname for `addr` satisfies `predicate`.
263    pub fn any_resolved_hostname(
264        &self,
265        addr: IpAddr,
266        mut predicate: impl FnMut(&str) -> bool,
267    ) -> bool {
268        let addr = normalize_ip_addr(addr);
269
270        self.resolved_hostnames
271            .read()
272            .member_matches(&addr, Instant::now(), |key| predicate(&key.hostname))
273    }
274
275    /// Best-effort expiry maintenance for resolved hostnames.
276    ///
277    /// This runs outside the hot egress read path. If the index is currently
278    /// busy, cleanup is skipped and retried on the next maintenance pass.
279    pub fn cleanup_resolved_hostnames(&self) {
280        if let Some(mut idx) = self.resolved_hostnames.try_write() {
281            idx.evict_expired(Instant::now());
282        }
283    }
284
285    /// Increment the guest -> runtime byte counter.
286    pub fn add_tx_bytes(&self, bytes: usize) {
287        self.metrics
288            .tx_bytes
289            .fetch_add(bytes as u64, Ordering::Relaxed);
290    }
291
292    /// Increment the runtime -> guest byte counter.
293    pub fn add_rx_bytes(&self, bytes: usize) {
294        self.metrics
295            .rx_bytes
296            .fetch_add(bytes as u64, Ordering::Relaxed);
297    }
298
299    /// Push a runtime -> guest ethernet frame and update RX metrics on success.
300    pub(crate) fn push_rx_frame(&self, frame: Vec<u8>) -> bool {
301        let frame_len = frame.len();
302        if self.rx_ring.push(frame).is_err() {
303            return false;
304        }
305
306        self.add_rx_bytes(frame_len);
307        true
308    }
309
310    /// Push a runtime -> guest ethernet frame, update RX metrics, and wake libkrun.
311    pub(crate) fn push_rx_frame_and_wake(&self, frame: Vec<u8>) -> bool {
312        if !self.push_rx_frame(frame) {
313            return false;
314        }
315
316        self.rx_wake.wake();
317        true
318    }
319
320    /// Total bytes transmitted by the guest into the runtime.
321    pub fn tx_bytes(&self) -> u64 {
322        self.metrics.tx_bytes.load(Ordering::Relaxed)
323    }
324
325    /// Total bytes delivered by the runtime to the guest.
326    pub fn rx_bytes(&self) -> u64 {
327        self.metrics.rx_bytes.load(Ordering::Relaxed)
328    }
329}
330
331impl Default for NetworkMetrics {
332    fn default() -> Self {
333        Self {
334            tx_bytes: AtomicU64::new(0),
335            rx_bytes: AtomicU64::new(0),
336        }
337    }
338}
339
340pub(crate) fn normalize_hostname(domain: &str) -> String {
341    domain.trim_end_matches('.').to_ascii_lowercase()
342}
343
344//--------------------------------------------------------------------------------------------------
345// Tests
346//--------------------------------------------------------------------------------------------------
347
348#[cfg(test)]
349mod tests {
350    use super::*;
351
352    #[test]
353    fn shared_state_queue_push_pop() {
354        let state = SharedState::new(4);
355
356        // Push frames to tx_ring.
357        state.tx_ring.push(vec![1, 2, 3]).unwrap();
358        state.tx_ring.push(vec![4, 5, 6]).unwrap();
359
360        // Pop in FIFO order.
361        assert_eq!(state.tx_ring.pop(), Some(vec![1, 2, 3]));
362        assert_eq!(state.tx_ring.pop(), Some(vec![4, 5, 6]));
363        assert_eq!(state.tx_ring.pop(), None);
364    }
365
366    #[test]
367    fn shared_state_queue_full() {
368        let state = SharedState::new(2);
369
370        state.rx_ring.push(vec![1]).unwrap();
371        state.rx_ring.push(vec![2]).unwrap();
372        // Queue is full — push returns the frame back.
373        assert!(state.rx_ring.push(vec![3]).is_err());
374    }
375
376    #[test]
377    fn push_rx_frame_counts_only_successful_pushes() {
378        let state = SharedState::new(1);
379
380        assert!(state.push_rx_frame(vec![1, 2, 3]));
381        assert_eq!(state.rx_bytes(), 3);
382
383        assert!(!state.push_rx_frame(vec![4, 5]));
384        assert_eq!(state.rx_bytes(), 3);
385    }
386
387    #[test]
388    fn resolved_hostnames_are_isolated_per_family() {
389        let state = SharedState::new(4);
390        let v4: IpAddr = "1.1.1.1".parse().unwrap();
391        let v6: IpAddr = "2606:4700:4700::1111".parse().unwrap();
392
393        state.cache_resolved_hostname(
394            "Example.com.",
395            ResolvedHostnameFamily::Ipv4,
396            [v4],
397            Duration::from_secs(30),
398        );
399        state.cache_resolved_hostname(
400            "example.com",
401            ResolvedHostnameFamily::Ipv6,
402            [v6],
403            Duration::from_secs(30),
404        );
405
406        assert!(state.any_resolved_hostname(v4, |h| h == "example.com"));
407        assert!(state.any_resolved_hostname(v6, |h| h == "example.com"));
408        assert!(!state.any_resolved_hostname(v4, |h| h == "other.example"));
409    }
410
411    #[test]
412    fn resolved_hostnames_keep_earlier_answers() {
413        let state = SharedState::new(4);
414        let first: IpAddr = "142.250.0.1".parse().unwrap();
415        let second: IpAddr = "142.250.0.2".parse().unwrap();
416
417        // Rotating answers must remain usable even beyond the old 64-address
418        // limit, as long as their TTLs have not expired.
419        for n in 1..=65 {
420            let addr = IpAddr::V4(Ipv4Addr::new(142, 250, 0, n));
421            state.cache_resolved_hostname(
422                "fonts.example",
423                ResolvedHostnameFamily::Ipv4,
424                [addr],
425                Duration::from_secs(30),
426            );
427        }
428
429        for n in 1..=65 {
430            let addr = IpAddr::V4(Ipv4Addr::new(142, 250, 0, n));
431            assert!(state.any_resolved_hostname(addr, |h| h == "fonts.example"));
432        }
433
434        state.clear_resolved_hostname("fonts.example", ResolvedHostnameFamily::Ipv4);
435        assert!(!state.any_resolved_hostname(first, |h| h == "fonts.example"));
436        assert!(!state.any_resolved_hostname(second, |h| h == "fonts.example"));
437    }
438
439    #[test]
440    fn resolved_hostnames_normalize_ipv4_mapped_ipv6() {
441        let state = SharedState::new(4);
442        let mapped: IpAddr = "::ffff:169.254.169.254".parse().unwrap();
443        let embedded: IpAddr = "169.254.169.254".parse().unwrap();
444
445        state.cache_resolved_hostname(
446            "metadata.example",
447            ResolvedHostnameFamily::Ipv6,
448            [mapped],
449            Duration::from_secs(30),
450        );
451
452        assert!(state.any_resolved_hostname(embedded, |h| h == "metadata.example"));
453        assert!(state.any_resolved_hostname(mapped, |h| h == "metadata.example"));
454    }
455}
microsandbox_network/engine/netstack/shared.rs

microsandbox_network/engine/netstack/
shared.rs
