1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
use std::sync::atomic::Ordering;
use arcbox_virtio_core::QueueConfig;
use crate::header::VirtioNetHeader;
use super::VirtioNet;
impl VirtioNet {
/// Drains the TX virtqueue: walks descriptor chains starting from the
/// current TX cursor up to the guest's latest `avail_idx`, concatenates
/// each chain's read-flagged descriptors into a packet, runs `finalize`
/// to complete any guest-requested checksum offload, strips the
/// virtio-net header, and writes the raw Ethernet frame to the bound
/// host fd. Publishes completions to the used ring itself and returns
/// whether any chain was drained, so the caller can fire the IRQ.
///
/// `finalize` is injected so arcbox-virtio doesn't depend on
/// arcbox-net's checksum helpers.
pub fn drain_tx_queue<F>(&self, qcfg: &QueueConfig, finalize: F) -> bool
where
F: Fn(&mut [u8]),
{
let Some(port) = self.port.get() else {
return false;
};
let Some(ctx) = self.ctx.as_ref() else {
return false;
};
if !qcfg.ready || qcfg.size == 0 {
return false;
}
let host_fd = port.host_fd;
let event_idx = (self.acked_features & arcbox_virtio_core::queue::VIRTIO_F_EVENT_IDX) != 0;
// The queue is the sole accessor of these rings for this call; `ctx.mem`
// wraps the VM-lifetime guest RAM mapping.
let mut queue = arcbox_virtio_core::SplitQueue::new(ctx.mem.clone(), 1, qcfg, event_idx);
queue.set_last_avail_idx(port.last_avail_tx.load(Ordering::Relaxed));
let mut notify = false;
// Drain in a TOCTOU loop: after publishing avail_event the guest may
// have queued more and suppressed its kick on the stale value, so
// re-check the avail ring and drain again.
loop {
let mut completions = Vec::new();
while let Some(chain) = queue.pop_avail() {
// TX descriptors are read-only (guest → host data).
let mut packet_data = Vec::new();
for desc in &chain.descriptors {
if !desc.is_write() {
if let Some(data) = queue.mem().slice(desc.addr as usize, desc.len as usize)
{
packet_data.extend_from_slice(data);
}
}
}
let total_len = packet_data.len() as u32;
finalize(&mut packet_data);
// Strip the virtio-net header (after checksum offload), then
// write the frame to the host fd with a brief EAGAIN/ENOBUFS
// retry: the datapath peer can lag a few frames under bulk
// bursts, and silent loss collapses guest TCP throughput.
if packet_data.len() > VirtioNetHeader::SIZE {
let frame = &packet_data[VirtioNetHeader::SIZE..];
let mut retries = 0;
loop {
// SAFETY: `host_fd` is owned via `NetPort`; `frame` is a
// valid slice of `packet_data` for the write's duration.
let n = unsafe {
libc::write(host_fd, frame.as_ptr().cast::<libc::c_void>(), frame.len())
};
if n >= 0 {
break;
}
let err = std::io::Error::last_os_error();
match err.raw_os_error() {
Some(libc::EAGAIN | libc::ENOBUFS) if retries < 64 => {
std::thread::yield_now();
retries += 1;
}
Some(libc::EAGAIN | libc::ENOBUFS) => break,
_ => {
tracing::warn!("VirtioNet TX write failed: {err}");
break;
}
}
}
}
completions.push((chain.head_idx, total_len));
}
// Publish the completions to the used ring (push_used_batch carries a
// full StoreLoad barrier). Net interrupts on any completed TX buffer.
if !completions.is_empty() {
notify = true;
}
queue.push_used_batch(&completions);
// Refresh avail_event even on an empty drain — the ABX-386 fix so a
// stale value can't make an EVENT_IDX guest suppress its TX kick.
// Only when negotiated: the field sits past the used ring otherwise.
if event_idx {
queue.write_avail_event();
}
// Close the TOCTOU window: SeqCst orders the avail_event store before
// re-reading avail.idx; if the guest queued more, drain again.
std::sync::atomic::fence(Ordering::SeqCst);
if !queue.has_avail() {
break;
}
}
port.last_avail_tx
.store(queue.last_avail_idx(), Ordering::Relaxed);
notify
}
/// Writes a raw Ethernet frame (without virtio-net header) directly to
/// the bound host fd. Intended for out-of-band injection paths; no-op
/// if the port is unbound.
pub fn write_tx_frame(&self, frame: &[u8]) {
let Some(port) = self.port.get() else {
return;
};
// SAFETY: `port.host_fd` is live for as long as `self.port` holds
// the `NetPort`; `frame` is a caller-provided valid slice.
let n = unsafe {
libc::write(
port.host_fd,
frame.as_ptr().cast::<libc::c_void>(),
frame.len(),
)
};
if n < 0 {
let err = std::io::Error::last_os_error();
if err.raw_os_error() != Some(libc::EAGAIN) {
tracing::warn!("VirtioNet::write_tx_frame failed: {err}");
}
}
}
/// Polls the bound host fd for inbound Ethernet frames and injects up
/// to 64 of them into the RX virtqueue described by `rx_qcfg`. Prepends
/// a 12-byte virtio-net header to each frame (`num_buffers` = 1 as
/// VERSION_1 requires without MRG_RXBUF; every other field zero — no
/// offloads on this path). Frames are delivered whole or not at all: a
/// chain that can't hold header + frame completes at zero length and
/// the frame is dropped. Returns `true` if the used ring advanced, so
/// the caller can fire the used-ring IRQ.
///
/// The caller is responsible for building `rx_qcfg` from the device's
/// MMIO state and gating on DRIVER_OK.
#[allow(clippy::too_many_lines)]
pub fn poll_rx(&self, rx_qcfg: &QueueConfig) -> bool {
let Some(port) = self.port.get() else {
return false;
};
let Some(ctx) = self.ctx.as_ref() else {
return false;
};
if !rx_qcfg.ready || rx_qcfg.size == 0 {
return false;
}
let host_fd = port.host_fd;
let event_idx = (self.acked_features & arcbox_virtio_core::queue::VIRTIO_F_EVENT_IDX) != 0;
// RX consumes one avail entry per injected frame, tracked by the guest's
// used.idx (which SplitQueue seeds); the queue is the sole accessor here.
let mut queue = arcbox_virtio_core::SplitQueue::new(ctx.mem.clone(), 0, rx_qcfg, event_idx);
let used0 = queue.mem().read_u16(rx_qcfg.used_addr as usize + 2);
queue.set_last_avail_idx(used0);
// "Published" = we advanced the used ring and the guest must be
// interrupted so it reclaims the descriptors — true both for a real
// frame injection and for a zero-length completion of an unusable
// chain. Returning only on injection would leak the interrupt for a
// chain we consumed but couldn't fill, stalling RX.
let mut published = false;
let hdr_len = VirtioNetHeader::SIZE;
// Injected header: num_buffers = 1 (VERSION_1 headers carry the field
// even without MRG_RXBUF, and the spec pins it to 1 then); every other
// field zero — this path performs no offloads.
let mut hdr = [0u8; VirtioNetHeader::SIZE];
hdr[10..12].copy_from_slice(&1u16.to_le_bytes());
for _ in 0..64 {
// Stop when the guest has no RX buffer posted.
if !queue.has_avail() {
break;
}
// Non-blocking read from the bound fd.
let mut buf = [0u8; 9216]; // MAX_FRAME_SIZE
// SAFETY: `host_fd` is owned by the bound `NetPort`; `buf` is a valid
// mutable stack slice; MSG_DONTWAIT keeps it non-blocking.
let n = unsafe {
libc::recv(
host_fd,
buf.as_mut_ptr().cast::<libc::c_void>(),
buf.len(),
libc::MSG_DONTWAIT,
)
};
if n <= 0 {
break;
}
let frame = &buf[..n as usize];
// The injected buffer is the header followed by the frame,
// scattered across the chain's write-only descriptors.
let total = hdr_len + frame.len();
let Some(chain) = queue.pop_avail() else {
break;
};
let mut written = 0usize;
let mut refused = false;
for desc in &chain.descriptors {
if !desc.is_write() {
continue;
}
if written >= total {
break;
}
let to_write = (total - written).min(desc.len as usize);
if to_write == 0 {
continue;
}
// SAFETY: write-only descriptor buffers are device-owned.
if let Some(out) = unsafe { queue.mem().slice_mut(desc.addr as usize, to_write) } {
for (k, slot) in out.iter_mut().enumerate() {
let pos = written + k;
*slot = if pos < hdr_len {
hdr[pos]
} else {
frame[pos - hdr_len]
};
}
written += to_write;
} else {
// Descriptor outside guest RAM. Skipping it and filling
// later descriptors could still reach `total` while the
// frame's middle is a hole — poison the chain instead.
refused = true;
break;
}
}
// Deliver whole frames only. A partial fit — an undersized chain,
// or a descriptor pointing outside guest RAM (`refused`) —
// completes at zero length: the guest reclaims the popped chain
// (leaving it off the used ring would drain the RX ring dry) and
// the frame is dropped whole; TCP retransmits. Truncated delivery
// would hand the guest a corrupt frame it parses as complete.
// This device never offers MRG_RXBUF (it serves the vmnet bridge
// NIC), so spanning chains is never a legal alternative and a
// conformant guest posts full-frame buffers — the drop fires only
// for undersized or hostile rings. The used ring advances either
// way, so the guest still needs the interrupt below.
let used_len = if !refused && written == total {
written as u32
} else {
0
};
queue.push_used(chain.head_idx, used_len);
published = true;
}
// Publish avail_event for an EVENT_IDX guest (gated: the field sits past
// the used ring when EVENT_IDX is not negotiated). RX publishes the
// current avail.idx (not the consumed cursor): the host polls for frames
// so it only wants a kick for RX buffers posted beyond what it has seen.
if published && event_idx {
let avail_idx_now = queue.mem().read_u16(rx_qcfg.avail_addr as usize + 2);
let avail_event_off = rx_qcfg.used_addr as usize + 4 + rx_qcfg.size as usize * 8;
queue.mem().write_u16(avail_event_off, avail_idx_now);
}
published
}
}