microsandbox_network/engine/netstack/
shared.rs

1//! Shared state between the NetWorker thread, smoltcp poll thread, and tokio
2//! proxy tasks.
3//!
4//! All inter-thread communication flows through [`SharedState`], which holds
5//! lock-free frame queues and cross-platform [`WakePipe`] notifications.
6
7use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
8use std::sync::{
9    Arc, Mutex, OnceLock,
10    atomic::{AtomicU64, Ordering},
11};
12use std::time::{Duration, Instant};
13
14use crossbeam_queue::ArrayQueue;
15use ipnetwork::Ipv6Network;
16use microsandbox_types::HttpConfig;
17use microsandbox_utils::ttl_reverse_index::TtlReverseIndex;
18pub use microsandbox_utils::wake_pipe::WakePipe;
19use parking_lot::RwLock;
20
21use crate::addr::normalize_ip_addr;
22use crate::engine::http_deny::{
23    DEFAULT_HTTP_DENY_MESSAGE, http_forbidden_response, json_http_forbidden_response,
24    render_http_deny_message,
25};
26
27//--------------------------------------------------------------------------------------------------
28// Constants
29//--------------------------------------------------------------------------------------------------
30
31/// Default frame queue capacity. Matches libkrun's virtio queue size.
32pub const DEFAULT_QUEUE_CAPACITY: usize = 1024;
33
34/// Maximum live hostname/address bindings retained by one sandbox.
35pub(crate) const MAX_RESOLVED_HOSTNAME_BINDINGS: usize = 16_384;
36
37/// Maximum live bindings for one normalized hostname, across both families.
38pub(crate) const MAX_BINDINGS_PER_HOSTNAME: usize = 1_024;
39
40//--------------------------------------------------------------------------------------------------
41// Types
42//--------------------------------------------------------------------------------------------------
43
44/// All shared state between the three threads:
45///
46/// - **NetWorker** (libkrun) — pushes guest frames to `tx_ring`, pops
47///   response frames from `rx_ring`.
48/// - **smoltcp poll thread** — pops from `tx_ring`, processes through smoltcp,
49///   pushes responses to `rx_ring`.
50/// - **tokio proxy tasks** — relay data between smoltcp sockets and real
51///   network connections.
52///
53/// Queue naming follows the **guest's perspective** (matching libkrun's
54/// convention): `tx_ring` = "transmit from guest", `rx_ring` = "receive at
55/// guest".
56pub struct SharedState {
57    /// Frames from guest → smoltcp (NetWorker writes, smoltcp reads).
58    pub tx_ring: ArrayQueue<Vec<u8>>,
59
60    /// Frames from smoltcp → guest (smoltcp writes, NetWorker reads).
61    pub rx_ring: ArrayQueue<Vec<u8>>,
62
63    /// Wakes NetWorker: "rx_ring has frames for the guest."
64    /// Written by `SmoltcpDevice::transmit()`. Read end polled by NetWorker's
65    /// epoll loop.
66    pub rx_wake: WakePipe,
67
68    /// Wakes smoltcp poll thread: "tx_ring has frames from the guest."
69    /// Written by `SmoltcpBackend::write_frame()`. Read end polled by the
70    /// poll loop.
71    pub tx_wake: WakePipe,
72
73    /// Wakes smoltcp poll thread: "proxy task has data to write to a smoltcp
74    /// socket." Written by proxy tasks via channels. Read end polled by the
75    /// poll loop.
76    pub proxy_wake: WakePipe,
77
78    /// Optional host-side termination hook used for fatal policy violations.
79    termination_hook: Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
80
81    /// Resolved hostname index used to map destination IPs back to queried hostnames.
82    resolved_hostnames: RwLock<TtlReverseIndex<ResolvedHostnameKey, IpAddr>>,
83
84    /// Per-sandbox gateway IPv4. Set once at boot; used by
85    /// `DestinationGroup::Host` rule matching and `host.microsandbox.internal`
86    /// DNS synthesis. `None` in isolated unit tests.
87    gateway_ipv4: OnceLock<Ipv4Addr>,
88
89    /// Per-sandbox gateway IPv6. Set once at boot. See `gateway_ipv4`.
90    gateway_ipv6: OnceLock<Ipv6Addr>,
91
92    /// NAT64 `/96` prefixes used by policy classification.
93    nat64_prefixes: OnceLock<Vec<Ipv6Network>>,
94
95    /// Aggregate network byte counters at the guest/runtime boundary.
96    metrics: NetworkMetrics,
97
98    /// HTTP denial settings installed before the network starts.
99    http: OnceLock<HttpConfig>,
100}
101
102/// Aggregate network byte counters shared with the runtime metrics sampler.
103pub struct NetworkMetrics {
104    tx_bytes: AtomicU64,
105    rx_bytes: AtomicU64,
106}
107
108/// Address family for resolved hostname entries.
109#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
110pub enum ResolvedHostnameFamily {
111    Ipv4,
112    Ipv6,
113}
114
115/// Composite cache key for a single DNS resolution.
116///
117/// `family` partitions entries so that `A` and `AAAA` responses for the
118/// same hostname refresh independently instead of overwriting each other.
119#[derive(Clone, Debug, PartialEq, Eq, Hash)]
120struct ResolvedHostnameKey {
121    hostname: String,
122    family: ResolvedHostnameFamily,
123}
124
125//--------------------------------------------------------------------------------------------------
126// Methods
127//--------------------------------------------------------------------------------------------------
128
129impl SharedState {
130    /// Create shared state with the given queue capacity.
131    pub fn new(queue_capacity: usize) -> Self {
132        Self {
133            tx_ring: ArrayQueue::new(queue_capacity),
134            rx_ring: ArrayQueue::new(queue_capacity),
135            rx_wake: WakePipe::new(),
136            tx_wake: WakePipe::new(),
137            proxy_wake: WakePipe::new(),
138            termination_hook: Mutex::new(None),
139            resolved_hostnames: RwLock::new(TtlReverseIndex::default()),
140            gateway_ipv4: OnceLock::new(),
141            gateway_ipv6: OnceLock::new(),
142            nat64_prefixes: OnceLock::new(),
143            metrics: NetworkMetrics::default(),
144            http: OnceLock::new(),
145        }
146    }
147
148    /// Set the per-sandbox gateway IPs. Called once at boot. Each family is
149    /// only published when active for this sandbox.
150    pub fn set_gateway_ips(&self, ipv4: Option<Ipv4Addr>, ipv6: Option<Ipv6Addr>) {
151        if let Some(ipv4) = ipv4 {
152            let _ = self.gateway_ipv4.set(ipv4);
153        }
154        if let Some(ipv6) = ipv6 {
155            let _ = self.gateway_ipv6.set(ipv6);
156        }
157    }
158
159    /// Gateway IPv4 address, if set.
160    pub fn gateway_ipv4(&self) -> Option<Ipv4Addr> {
161        self.gateway_ipv4.get().copied()
162    }
163
164    /// Gateway IPv6 address, if set.
165    pub fn gateway_ipv6(&self) -> Option<Ipv6Addr> {
166        self.gateway_ipv6.get().copied()
167    }
168
169    /// Install HTTP denial settings. The first call wins.
170    pub fn set_http_config(&self, config: HttpConfig) {
171        let _ = self.http.set(config);
172    }
173
174    /// Whether readable denial responses were explicitly enabled.
175    pub fn http_deny_response_enabled(&self) -> bool {
176        self.http.get().is_some_and(|http| http.deny_response)
177    }
178
179    /// Whether JSON secret denials were explicitly enabled.
180    pub fn secret_deny_response_enabled(&self) -> bool {
181        self.http.get().is_some_and(|http| {
182            http.deny_response
183                && http.deny_response_format == microsandbox_types::HttpDenyResponseFormat::Json
184        })
185    }
186
187    /// Build a network denial using the selected wire contract.
188    pub fn network_denial_response(&self, host: &str) -> Vec<u8> {
189        if self.http.get().is_some_and(|http| {
190            http.deny_response_format == microsandbox_types::HttpDenyResponseFormat::Json
191        }) {
192            json_http_forbidden_response(self.network_deny_message(), host)
193        } else {
194            let template = self
195                .http
196                .get()
197                .and_then(|http| http.deny_message.as_deref())
198                .unwrap_or(DEFAULT_HTTP_DENY_MESSAGE);
199            http_forbidden_response(&render_http_deny_message(template, host))
200        }
201    }
202
203    /// Return the configured network-denial message without interpolation.
204    pub fn network_deny_message(&self) -> &str {
205        self.http
206            .get()
207            .and_then(|http| http.network_deny_message.as_deref())
208            .unwrap_or(crate::engine::http_deny::NETWORK_JSON_DENY_MESSAGE)
209    }
210
211    /// Return the configured secret-denial message without interpolation.
212    pub fn secret_deny_message(&self) -> &str {
213        self.http
214            .get()
215            .and_then(|http| http.secret_deny_message.as_deref())
216            .unwrap_or(crate::engine::http_deny::SECRET_HTTP_DENY_MESSAGE)
217    }
218
219    /// Set NAT64 prefixes. Called once before policy evaluation starts.
220    pub fn set_nat64_prefixes(&self, prefixes: Vec<Ipv6Network>) {
221        let _ = self.nat64_prefixes.set(prefixes);
222    }
223
224    /// NAT64 prefixes used by policy classification.
225    pub fn nat64_prefixes(&self) -> &[Ipv6Network] {
226        self.nat64_prefixes.get().map(Vec::as_slice).unwrap_or(&[])
227    }
228
229    /// Install a host-side termination hook.
230    pub fn set_termination_hook(&self, hook: Arc<dyn Fn() + Send + Sync>) {
231        *self.termination_hook.lock().unwrap() = Some(hook);
232    }
233
234    /// Trigger host-side termination if a hook is installed.
235    pub fn trigger_termination(&self) {
236        let hook = self.termination_hook.lock().unwrap().clone();
237        if let Some(hook) = hook {
238            hook();
239        }
240    }
241
242    /// Record resolved addresses for a hostname within the given address family.
243    ///
244    /// Addresses from earlier answers stay bound until the TTL of the answer
245    /// that returned them expires.
246    /// Resolvers that rotate their answers hand concurrent lookups of one name
247    /// different addresses, and a connection dialed on any of them must still
248    /// match the name's domain rules.
249    ///
250    /// Returns `false` if the complete answer would exceed either the hostname
251    /// or sandbox binding limit; existing bindings are preserved on rejection.
252    pub fn cache_resolved_hostname(
253        &self,
254        domain: &str,
255        family: ResolvedHostnameFamily,
256        addrs: impl IntoIterator<Item = IpAddr>,
257        ttl: Duration,
258    ) -> bool {
259        let hostname = normalize_hostname(domain);
260        let key = ResolvedHostnameKey { hostname, family };
261        let addrs = addrs.into_iter().map(normalize_ip_addr);
262
263        let now = Instant::now();
264        let mut index = self.resolved_hostnames.write();
265        index.evict_expired(now);
266
267        let bindings_for = |family| {
268            index.member_count(&ResolvedHostnameKey {
269                hostname: key.hostname.clone(),
270                family,
271            })
272        };
273
274        let ipv4_bindings = bindings_for(ResolvedHostnameFamily::Ipv4);
275        let ipv6_bindings = bindings_for(ResolvedHostnameFamily::Ipv6);
276        let hostname_bindings = ipv4_bindings + ipv6_bindings;
277
278        let total_bindings = index.binding_count();
279        let hostname_slots = MAX_BINDINGS_PER_HOSTNAME.saturating_sub(hostname_bindings);
280        let sandbox_slots = MAX_RESOLVED_HOSTNAME_BINDINGS.saturating_sub(total_bindings);
281        let available_slots = hostname_slots.min(sandbox_slots);
282
283        // try_extend takes a total capacity; only new bindings consume slots.
284        let capacity = total_bindings + available_slots;
285        index.try_extend(key, addrs, ttl, now, capacity)
286    }
287
288    /// Clear the resolved addresses for a hostname within the given address family.
289    pub fn clear_resolved_hostname(&self, domain: &str, family: ResolvedHostnameFamily) {
290        let hostname = normalize_hostname(domain);
291        let key = ResolvedHostnameKey { hostname, family };
292        self.resolved_hostnames.write().remove(&key, Instant::now());
293    }
294
295    /// Returns `true` when any resolved hostname for `addr` satisfies `predicate`.
296    pub fn any_resolved_hostname(
297        &self,
298        addr: IpAddr,
299        mut predicate: impl FnMut(&str) -> bool,
300    ) -> bool {
301        let addr = normalize_ip_addr(addr);
302
303        self.resolved_hostnames
304            .read()
305            .member_matches(&addr, Instant::now(), |key| predicate(&key.hostname))
306    }
307
308    /// Best-effort expiry maintenance for resolved hostnames.
309    ///
310    /// This runs outside the hot egress read path. If the index is currently
311    /// busy, cleanup is skipped and retried on the next maintenance pass.
312    pub fn cleanup_resolved_hostnames(&self) {
313        if let Some(mut idx) = self.resolved_hostnames.try_write() {
314            idx.evict_expired(Instant::now());
315        }
316    }
317
318    /// Increment the guest -> runtime byte counter.
319    pub fn add_tx_bytes(&self, bytes: usize) {
320        self.metrics
321            .tx_bytes
322            .fetch_add(bytes as u64, Ordering::Relaxed);
323    }
324
325    /// Increment the runtime -> guest byte counter.
326    pub fn add_rx_bytes(&self, bytes: usize) {
327        self.metrics
328            .rx_bytes
329            .fetch_add(bytes as u64, Ordering::Relaxed);
330    }
331
332    /// Push a runtime -> guest ethernet frame and update RX metrics on success.
333    pub(crate) fn push_rx_frame(&self, frame: Vec<u8>) -> bool {
334        let frame_len = frame.len();
335        if self.rx_ring.push(frame).is_err() {
336            return false;
337        }
338
339        self.add_rx_bytes(frame_len);
340        true
341    }
342
343    /// Push a runtime -> guest ethernet frame, update RX metrics, and wake libkrun.
344    pub(crate) fn push_rx_frame_and_wake(&self, frame: Vec<u8>) -> bool {
345        if !self.push_rx_frame(frame) {
346            return false;
347        }
348
349        self.rx_wake.wake();
350        true
351    }
352
353    /// Total bytes transmitted by the guest into the runtime.
354    pub fn tx_bytes(&self) -> u64 {
355        self.metrics.tx_bytes.load(Ordering::Relaxed)
356    }
357
358    /// Total bytes delivered by the runtime to the guest.
359    pub fn rx_bytes(&self) -> u64 {
360        self.metrics.rx_bytes.load(Ordering::Relaxed)
361    }
362}
363
364impl Default for NetworkMetrics {
365    fn default() -> Self {
366        Self {
367            tx_bytes: AtomicU64::new(0),
368            rx_bytes: AtomicU64::new(0),
369        }
370    }
371}
372
373pub(crate) fn normalize_hostname(domain: &str) -> String {
374    domain.trim_end_matches('.').to_ascii_lowercase()
375}
376
377//--------------------------------------------------------------------------------------------------
378// Tests
379//--------------------------------------------------------------------------------------------------
380
381#[cfg(test)]
382mod tests {
383    use super::*;
384
385    #[test]
386    fn explicit_text_format_preserves_legacy_body_and_host_substitution() {
387        for (raw, host, expected) in [
388            (
389                r#"{"deny_response":true,"deny_response_format":"text"}"#,
390                "example.com",
391                "This host is not allowed by the sandbox network policy config.\n\n\
392                Note to agent: `example.com` is not in the allowed-host list. Ask the \
393                user to add it to the sandbox network allow list.\n",
394            ),
395            (
396                r#"{"deny_response":true,"deny_response_format":"text","deny_message":"blocked {host}"}"#,
397                "example.com",
398                "blocked example.com",
399            ),
400            (
401                r#"{"deny_response":true,"deny_response_format":"text","deny_message":""}"#,
402                "example.com",
403                "",
404            ),
405            (
406                r#"{"deny_response_format":"text","deny_message":"blocked {host}"}"#,
407                "  ",
408                "blocked this host",
409            ),
410        ] {
411            let state = SharedState::new(8);
412            state.set_http_config(serde_json::from_str(raw).unwrap());
413            assert!(state.http_deny_response_enabled());
414            assert!(!state.secret_deny_response_enabled());
415            let response = state.network_denial_response(host);
416            let response = std::str::from_utf8(&response).unwrap();
417            let (headers, body) = response.split_once("\r\n\r\n").unwrap();
418            assert!(headers.starts_with("HTTP/1.1 403 Forbidden\r\n"));
419            assert!(headers.contains("Content-Type: text/plain; charset=utf-8\r\n"));
420            assert!(headers.contains(&format!("Content-Length: {}", expected.len())));
421            assert_eq!(body, expected);
422        }
423    }
424
425    #[test]
426    fn shared_state_queue_push_pop() {
427        let state = SharedState::new(4);
428
429        // Push frames to tx_ring.
430        state.tx_ring.push(vec![1, 2, 3]).unwrap();
431        state.tx_ring.push(vec![4, 5, 6]).unwrap();
432
433        // Pop in FIFO order.
434        assert_eq!(state.tx_ring.pop(), Some(vec![1, 2, 3]));
435        assert_eq!(state.tx_ring.pop(), Some(vec![4, 5, 6]));
436        assert_eq!(state.tx_ring.pop(), None);
437    }
438
439    #[test]
440    fn shared_state_queue_full() {
441        let state = SharedState::new(2);
442
443        state.rx_ring.push(vec![1]).unwrap();
444        state.rx_ring.push(vec![2]).unwrap();
445        // Queue is full — push returns the frame back.
446        assert!(state.rx_ring.push(vec![3]).is_err());
447    }
448
449    #[test]
450    fn push_rx_frame_counts_only_successful_pushes() {
451        let state = SharedState::new(1);
452
453        assert!(state.push_rx_frame(vec![1, 2, 3]));
454        assert_eq!(state.rx_bytes(), 3);
455
456        assert!(!state.push_rx_frame(vec![4, 5]));
457        assert_eq!(state.rx_bytes(), 3);
458    }
459
460    #[test]
461    fn resolved_hostnames_are_isolated_per_family() {
462        let state = SharedState::new(4);
463        let v4: IpAddr = "1.1.1.1".parse().unwrap();
464        let v6: IpAddr = "2606:4700:4700::1111".parse().unwrap();
465
466        state.cache_resolved_hostname(
467            "Example.com.",
468            ResolvedHostnameFamily::Ipv4,
469            [v4],
470            Duration::from_secs(30),
471        );
472        state.cache_resolved_hostname(
473            "example.com",
474            ResolvedHostnameFamily::Ipv6,
475            [v6],
476            Duration::from_secs(30),
477        );
478
479        assert!(state.any_resolved_hostname(v4, |h| h == "example.com"));
480        assert!(state.any_resolved_hostname(v6, |h| h == "example.com"));
481        assert!(!state.any_resolved_hostname(v4, |h| h == "other.example"));
482    }
483
484    #[test]
485    fn resolved_hostnames_keep_earlier_answers() {
486        let state = SharedState::new(4);
487        let first: IpAddr = "142.250.0.1".parse().unwrap();
488        let second: IpAddr = "142.250.0.2".parse().unwrap();
489
490        // Rotating answers must remain usable even beyond the old 64-address
491        // limit, as long as their TTLs have not expired.
492        for n in 1..=65 {
493            let addr = IpAddr::V4(Ipv4Addr::new(142, 250, 0, n));
494            state.cache_resolved_hostname(
495                "fonts.example",
496                ResolvedHostnameFamily::Ipv4,
497                [addr],
498                Duration::from_secs(30),
499            );
500        }
501
502        for n in 1..=65 {
503            let addr = IpAddr::V4(Ipv4Addr::new(142, 250, 0, n));
504            assert!(state.any_resolved_hostname(addr, |h| h == "fonts.example"));
505        }
506
507        state.clear_resolved_hostname("fonts.example", ResolvedHostnameFamily::Ipv4);
508        assert!(!state.any_resolved_hostname(first, |h| h == "fonts.example"));
509        assert!(!state.any_resolved_hostname(second, |h| h == "fonts.example"));
510    }
511
512    #[test]
513    fn resolved_hostnames_normalize_ipv4_mapped_ipv6() {
514        let state = SharedState::new(4);
515        let mapped: IpAddr = "::ffff:169.254.169.254".parse().unwrap();
516        let embedded: IpAddr = "169.254.169.254".parse().unwrap();
517
518        state.cache_resolved_hostname(
519            "metadata.example",
520            ResolvedHostnameFamily::Ipv6,
521            [mapped],
522            Duration::from_secs(30),
523        );
524
525        assert!(state.any_resolved_hostname(embedded, |h| h == "metadata.example"));
526        assert!(state.any_resolved_hostname(mapped, |h| h == "metadata.example"));
527    }
528}
microsandbox_network/engine/netstack/shared.rs

microsandbox_network/engine/netstack/
shared.rs
