use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicBool, Ordering};
use virtio_bindings::virtio_config::{
VIRTIO_CONFIG_S_ACKNOWLEDGE, VIRTIO_CONFIG_S_DRIVER, VIRTIO_CONFIG_S_DRIVER_OK,
VIRTIO_CONFIG_S_FAILED, VIRTIO_CONFIG_S_FEATURES_OK, VIRTIO_CONFIG_S_NEEDS_RESET,
VIRTIO_F_VERSION_1,
};
use virtio_bindings::virtio_mmio::{VIRTIO_MMIO_INT_CONFIG, VIRTIO_MMIO_INT_VRING};
use virtio_bindings::virtio_net::{
VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
VIRTIO_NET_ERR, VIRTIO_NET_F_CTRL_VQ, VIRTIO_NET_F_MAC, VIRTIO_NET_F_MQ, VIRTIO_NET_OK,
};
use virtio_queue::{DescriptorChain, Error as VirtioQueueError, Queue, QueueOwnedT, QueueT};
use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryMmap};
use vmm_sys_util::eventfd::EventFd;
use super::counters::VirtioNetCounters;
use crate::vmm::PiMutex;
use crate::vmm::net_config::NetConfig;
use crate::vmm::virtio_msix::{IrqSource, MsixState};
pub(crate) const MMIO_MAGIC: u32 = 0x7472_6976; pub(crate) const MMIO_VERSION: u32 = 2; pub(crate) const VENDOR_ID: u32 = 0;
pub const VIRTIO_MMIO_SIZE: u64 = 0x1000;
pub(crate) const NUM_QUEUES: usize = 2;
pub(crate) const QUEUE_MAX_SIZE: u16 = 256;
pub(crate) const MAX_QUEUE_PAIRS: u16 = 256;
#[cfg(test)]
pub(crate) const RXQ: usize = 0;
#[cfg(test)]
pub(crate) const TXQ: usize = 1;
pub const VIRTIO_NET_HDR_LEN: usize = 12;
pub(crate) const MAX_FRAME_SIZE: usize = 65_557;
pub(crate) const S_ACK: u32 = VIRTIO_CONFIG_S_ACKNOWLEDGE;
pub(crate) const S_DRV: u32 = S_ACK | VIRTIO_CONFIG_S_DRIVER;
pub(crate) const S_FEAT: u32 = S_DRV | VIRTIO_CONFIG_S_FEATURES_OK;
#[cfg(test)]
pub(crate) const S_OK: u32 = S_FEAT | VIRTIO_CONFIG_S_DRIVER_OK;
#[repr(C, packed)]
#[derive(Copy, Clone, Default, Debug)]
pub(crate) struct VirtioNetConfig {
pub(crate) mac: [u8; 6],
pub(crate) status: u16,
pub(crate) max_virtqueue_pairs: u16,
pub(crate) mtu: u16,
}
unsafe impl ByteValued for VirtioNetConfig {}
pub(crate) const VIRTIO_NET_CONFIG_SIZE: usize = std::mem::size_of::<VirtioNetConfig>();
const _: () = assert!(std::mem::offset_of!(VirtioNetConfig, mac) == 0x00);
const _: () = assert!(std::mem::offset_of!(VirtioNetConfig, status) == 0x06);
const _: () = assert!(std::mem::offset_of!(VirtioNetConfig, max_virtqueue_pairs) == 0x08);
const _: () = assert!(std::mem::offset_of!(VirtioNetConfig, mtu) == 0x0A);
const _: () = assert!(VIRTIO_NET_CONFIG_SIZE == 12);
pub struct VirtioNet {
queues: Vec<Queue>,
queue_pairs: u16,
curr_queue_pairs: u16,
queue_select: u32,
device_features_sel: u32,
driver_features_sel: u32,
driver_features: u64,
device_status: u32,
interrupt_status: u32,
config_generation: u32,
irq_evt: EventFd,
msix: Option<Arc<PiMutex<MsixState>>>,
mem: Arc<OnceLock<GuestMemoryMmap>>,
mem_unset_warned: Arc<AtomicBool>,
config: VirtioNetConfig,
counters: Arc<VirtioNetCounters>,
tx_frame_scratch: Vec<u8>,
queue_poisoned: Vec<bool>,
}
impl VirtioNet {
pub fn new(config: NetConfig) -> Self {
let irq_evt =
EventFd::new(libc::EFD_NONBLOCK).expect("failed to create virtio-net irq eventfd");
let queue_pairs = config.queue_pairs.clamp(1, MAX_QUEUE_PAIRS);
let num_queues = queue_pairs as usize * 2 + usize::from(queue_pairs > 1);
let queues: Vec<Queue> = (0..num_queues)
.map(|_| Queue::new(QUEUE_MAX_SIZE).expect("valid queue size"))
.collect();
VirtioNet {
queues,
queue_pairs,
curr_queue_pairs: 1,
queue_select: 0,
device_features_sel: 0,
driver_features_sel: 0,
driver_features: 0,
device_status: 0,
interrupt_status: 0,
config_generation: 0,
irq_evt,
msix: None,
mem: Arc::new(OnceLock::new()),
mem_unset_warned: Arc::new(AtomicBool::new(false)),
config: VirtioNetConfig {
mac: config.mac,
status: 0,
max_virtqueue_pairs: if queue_pairs > 1 { queue_pairs } else { 0 },
mtu: 0,
},
counters: Arc::new(VirtioNetCounters::default()),
tx_frame_scratch: Vec::with_capacity(MAX_FRAME_SIZE),
queue_poisoned: vec![false; num_queues],
}
}
pub fn irq_evt(&self) -> &EventFd {
&self.irq_evt
}
pub(crate) fn set_msix_state(&mut self, msix: Arc<PiMutex<MsixState>>) {
self.msix = Some(msix);
}
pub fn set_mem(&mut self, mem: GuestMemoryMmap) {
if self.mem.set(mem).is_err() {
tracing::warn!(
"virtio-net: set_mem called on already-initialised \
device; guest memory binding unchanged (mem is set \
once at boot and preserved across reset())"
);
}
}
pub fn counters(&self) -> Arc<VirtioNetCounters> {
Arc::clone(&self.counters)
}
pub(crate) fn device_status(&self) -> u32 {
self.device_status
}
pub(crate) fn interrupt_status(&self) -> u32 {
self.interrupt_status
}
pub(crate) fn config_generation(&self) -> u32 {
self.config_generation
}
pub(crate) fn num_queues(&self) -> usize {
self.queues.len()
}
#[cfg(test)]
pub(crate) fn curr_queue_pairs(&self) -> u16 {
self.curr_queue_pairs
}
pub(crate) fn device_features_window(&self) -> u32 {
match self.device_features_sel {
0 => self.device_features() as u32,
1 => (self.device_features() >> 32) as u32,
_ => 0,
}
}
pub(crate) fn queue_max_size(&self) -> u32 {
self.selected_queue()
.map(|i| self.queues[i].max_size() as u32)
.unwrap_or(0)
}
pub(crate) fn queue_size(&self) -> u32 {
self.selected_queue()
.map(|i| self.queues[i].size() as u32)
.unwrap_or(0)
}
pub(crate) fn queue_ready(&self) -> u32 {
self.selected_queue()
.map(|i| self.queues[i].ready() as u32)
.unwrap_or(0)
}
pub(crate) fn queue_notify_off(&self) -> u32 {
self.selected_queue().map(|i| i as u32).unwrap_or(0)
}
#[cfg(test)]
pub(crate) fn selected_queue_ring_addrs(&self) -> Option<(u64, u64, u64)> {
self.selected_queue().map(|i| {
(
self.queues[i].desc_table(),
self.queues[i].avail_ring(),
self.queues[i].used_ring(),
)
})
}
pub(crate) fn device_features_sel(&self) -> u32 {
self.device_features_sel
}
pub(crate) fn driver_features_sel(&self) -> u32 {
self.driver_features_sel
}
pub(crate) fn queue_select(&self) -> u32 {
self.queue_select
}
pub(crate) fn config_bytes(&self, offset: usize, data: &mut [u8]) {
let config_bytes = self.config.as_slice();
for (i, byte) in data.iter_mut().enumerate() {
*byte = config_bytes.get(offset + i).copied().unwrap_or(0);
}
}
pub(crate) fn set_device_features_sel(&mut self, sel: u32) {
self.device_features_sel = sel;
}
pub(crate) fn set_driver_features_sel(&mut self, sel: u32) {
self.driver_features_sel = sel;
}
pub(crate) fn set_queue_select(&mut self, sel: u32) {
self.queue_select = sel;
}
pub(crate) fn set_driver_features_window(&mut self, val: u32) {
if !self.features_write_allowed() {
return;
}
match self.driver_features_sel {
0 => self.driver_features = (self.driver_features & 0xFFFF_FFFF_0000_0000) | val as u64,
1 => {
self.driver_features =
(self.driver_features & 0x0000_0000_FFFF_FFFF) | ((val as u64) << 32)
}
_ => {}
}
}
pub(crate) fn set_queue_size(&mut self, size: u16) {
if !self.queue_config_allowed() {
return;
}
if let Some(i) = self.selected_queue() {
self.queues[i].set_size(size);
}
}
pub(crate) fn set_queue_ready(&mut self, ready: bool) {
if !self.queue_config_allowed() {
return;
}
if let Some(i) = self.selected_queue() {
self.queues[i].set_ready(ready);
}
}
pub(crate) fn set_queue_desc_addr(&mut self, lo: Option<u32>, hi: Option<u32>) {
if !self.queue_config_allowed() {
return;
}
if let Some(i) = self.selected_queue() {
self.queues[i].set_desc_table_address(lo, hi);
}
}
pub(crate) fn set_queue_avail_addr(&mut self, lo: Option<u32>, hi: Option<u32>) {
if !self.queue_config_allowed() {
return;
}
if let Some(i) = self.selected_queue() {
self.queues[i].set_avail_ring_address(lo, hi);
}
}
pub(crate) fn set_queue_used_addr(&mut self, lo: Option<u32>, hi: Option<u32>) {
if !self.queue_config_allowed() {
return;
}
if let Some(i) = self.selected_queue() {
self.queues[i].set_used_ring_address(lo, hi);
}
}
pub(crate) fn notify_queue(&mut self, queue_idx: u32) {
let idx = queue_idx as usize;
if self.queue_pairs > 1 && idx == self.queue_pairs as usize * 2 {
self.process_ctrl_queue();
return;
}
if idx % 2 == 1 {
let pair = idx / 2;
if pair < self.curr_queue_pairs as usize {
self.process_tx_loopback(pair);
}
}
}
fn process_ctrl_queue(&mut self) {
if self.device_status & VIRTIO_CONFIG_S_DRIVER_OK == 0 {
return;
}
if self.driver_features & (1u64 << VIRTIO_NET_F_CTRL_VQ) == 0 {
return;
}
let cvq_idx = self.queue_pairs as usize * 2;
let mem_arc = Arc::clone(&self.mem);
let Some(mem) = mem_arc.get() else {
return;
};
if self.queue_poisoned[cvq_idx] {
return;
}
let mut had_used_ring_publish = false;
let mut cvq_just_poisoned = false;
loop {
enum IterStep<C> {
Chain(C),
Empty,
Poisoned(VirtioQueueError),
}
let step: IterStep<_> = {
let q = &mut self.queues[cvq_idx];
match q.iter(mem) {
Ok(mut it) => match it.next() {
Some(c) => IterStep::Chain(c),
None => IterStep::Empty,
},
Err(e) => IterStep::Poisoned(e),
}
};
let chain = match step {
IterStep::Empty => break,
IterStep::Poisoned(err) => {
self.queue_poisoned[cvq_idx] = true;
self.counters.record_invalid_avail_idx();
cvq_just_poisoned = true;
tracing::warn!(
err = %err,
"virtio-net control-vq iter() failed; poisoning + \
NEEDS_RESET until guest reset (any structural queue \
error is non-recoverable)"
);
break;
}
IterStep::Chain(c) => c,
};
let head = chain.head_index();
let Some((status_addr, ack, new_pairs)) = self.eval_ctrl_chain(chain, mem) else {
self.counters.record_ctrl_chain_invalid();
continue;
};
if let Err(e) = mem.write_slice(&[ack], status_addr) {
self.counters.record_ctrl_add_used_failure();
tracing::warn!(
%e,
"virtio-net control-vq status write failed (unmapped GPA); \
dropping chain"
);
continue;
}
if new_pairs.is_none() {
self.counters.record_ctrl_chain_invalid();
}
match self.queues[cvq_idx].add_used(mem, head, 1) {
Ok(()) => {
had_used_ring_publish = true;
if let Some(pairs) = new_pairs {
self.curr_queue_pairs = pairs;
self.counters.record_ctrl_mq_set();
}
}
Err(e) => {
self.counters.record_ctrl_add_used_failure();
tracing::warn!(
head,
%e,
"virtio-net control-vq add_used failed (used-ring \
address unmapped)"
);
}
}
}
if had_used_ring_publish {
self.signal_used(cvq_idx);
}
if cvq_just_poisoned {
self.mark_needs_reset();
}
}
fn eval_ctrl_chain(
&self,
chain: DescriptorChain<&GuestMemoryMmap>,
mem: &GuestMemoryMmap,
) -> Option<(GuestAddress, u8, Option<u16>)> {
const CTRL_CMD_BYTES: usize = 4;
let mut cmd = [0u8; CTRL_CMD_BYTES];
let mut cmd_len = 0usize;
let mut status: Option<GuestAddress> = None;
for desc in chain {
if desc.is_write_only() {
if status.is_none() && desc.len() >= 1 {
status = Some(desc.addr());
}
continue;
}
if cmd_len < CTRL_CMD_BYTES {
let want = (CTRL_CMD_BYTES - cmd_len).min(desc.len() as usize);
if want > 0
&& mem
.read_slice(&mut cmd[cmd_len..cmd_len + want], desc.addr())
.is_ok()
{
cmd_len += want;
}
}
}
let status_addr = status?;
if cmd_len < CTRL_CMD_BYTES
|| u32::from(cmd[0]) != VIRTIO_NET_CTRL_MQ
|| u32::from(cmd[1]) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
{
return Some((status_addr, VIRTIO_NET_ERR as u8, None));
}
let pairs = u16::from_le_bytes([cmd[2], cmd[3]]);
if u32::from(pairs) < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || pairs > self.queue_pairs {
return Some((status_addr, VIRTIO_NET_ERR as u8, None));
}
Some((status_addr, VIRTIO_NET_OK as u8, Some(pairs)))
}
pub(crate) fn ack_interrupt(&mut self, val: u32) {
self.interrupt_status &= !val;
}
pub(crate) fn write_status(&mut self, val: u32) {
if val == 0 {
self.reset();
} else {
self.set_status(val);
}
}
fn device_features(&self) -> u64 {
let mut features = (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_NET_F_MAC);
if self.queue_pairs > 1 && self.msix.is_some() {
features |= (1u64 << VIRTIO_NET_F_MQ) | (1u64 << VIRTIO_NET_F_CTRL_VQ);
}
features
}
fn selected_queue(&self) -> Option<usize> {
let idx = self.queue_select as usize;
if idx < self.queues.len() {
Some(idx)
} else {
None
}
}
fn signal_used(&mut self, queue: usize) {
if let Some(msix) = &self.msix {
let mut m = msix.lock();
if m.enabled() {
m.signal(IrqSource::Vring { queue });
return;
}
}
self.interrupt_status |= VIRTIO_MMIO_INT_VRING;
if let Err(e) = self.irq_evt.write(1) {
tracing::warn!(%e, "virtio-net irq_evt.write failed");
}
}
fn signal_queue_poisoned(&mut self) {
self.mark_needs_reset();
if let Some(msix) = &self.msix {
let mut m = msix.lock();
if m.enabled() {
m.signal(IrqSource::Config);
return;
}
}
self.interrupt_status |= VIRTIO_MMIO_INT_CONFIG;
if let Err(e) = self.irq_evt.write(1) {
tracing::warn!(%e, "virtio-net irq_evt.write failed (poison signal)");
}
}
fn mark_needs_reset(&mut self) {
self.device_status |= VIRTIO_CONFIG_S_NEEDS_RESET;
}
fn queue_config_allowed(&self) -> bool {
self.device_status & S_FEAT == S_FEAT && self.device_status & VIRTIO_CONFIG_S_DRIVER_OK == 0
}
fn features_write_allowed(&self) -> bool {
self.device_status & S_DRV == S_DRV && self.device_status & VIRTIO_CONFIG_S_FEATURES_OK == 0
}
fn process_tx_loopback(&mut self, pair: usize) {
let rx_idx = pair * 2;
let tx_idx = pair * 2 + 1;
if self.device_status & VIRTIO_CONFIG_S_DRIVER_OK == 0 {
return;
}
let mem_arc = Arc::clone(&self.mem);
let Some(mem) = mem_arc.get() else {
if !self.mem_unset_warned.swap(true, Ordering::Relaxed) {
tracing::warn!(
"virtio-net: queue notify before set_mem; \
dropping TX kick until guest memory is wired"
);
}
return;
};
let mut tx_advanced = false;
let mut rx_advanced = false;
let mut tx_just_poisoned = false;
let mut rx_just_poisoned = false;
loop {
let pop_outcome = self.pop_and_capture_tx(mem, tx_idx);
let chain_outcome = match pop_outcome {
TxPopOutcome::Empty => break,
TxPopOutcome::JustPoisoned => {
tx_just_poisoned = true;
break;
}
TxPopOutcome::Chain(c) => c,
};
let TxChainOutcome { head, frame_len } = chain_outcome;
if let Some(len) = frame_len {
let outcome = self.try_loopback_to_rx(mem, len, rx_idx);
self.handle_rx_loopback_outcome(outcome, &mut rx_advanced, &mut rx_just_poisoned);
}
self.tx_add_used(mem, head, frame_len, &mut tx_advanced, tx_idx);
if rx_just_poisoned {
break;
}
}
if self.msix.as_ref().is_some_and(|m| m.lock().enabled()) {
if rx_advanced {
self.signal_used(rx_idx);
}
if tx_advanced {
self.signal_used(tx_idx);
}
} else if rx_advanced || tx_advanced {
self.signal_used(rx_idx);
}
if tx_just_poisoned || rx_just_poisoned {
self.signal_queue_poisoned();
}
}
fn handle_rx_loopback_outcome(
&mut self,
outcome: LoopbackOutcome,
had_used_ring_publish: &mut bool,
rx_just_poisoned: &mut bool,
) {
match outcome {
LoopbackOutcome::Delivered { l2_bytes_written } => {
self.counters.record_rx_delivered(l2_bytes_written);
*had_used_ring_publish = true;
}
LoopbackOutcome::DeliveredButAddUsedFailed => {
}
LoopbackOutcome::RxAlreadyPoisoned => {
self.counters.record_tx_dropped_rx_poisoned();
}
LoopbackOutcome::JustRxPoisoned => {
self.counters.record_tx_dropped_rx_poisoned();
*rx_just_poisoned = true;
}
LoopbackOutcome::NoRxBuffer => {
self.counters.record_tx_dropped_no_rx_buffer();
}
LoopbackOutcome::RxChainInvalid { add_used_ok } => {
if add_used_ok {
*had_used_ring_publish = true;
}
}
}
}
fn tx_add_used(
&mut self,
mem: &GuestMemoryMmap,
head: u16,
frame_len: Option<usize>,
had_used_ring_publish: &mut bool,
tx_idx: usize,
) {
let q = &mut self.queues[tx_idx];
match q.add_used(mem, head, 0) {
Ok(()) => {
if let Some(len) = frame_len {
self.counters.record_tx_completed(len as u64);
}
*had_used_ring_publish = true;
}
Err(e) => {
self.counters.record_tx_add_used_failure();
tracing::warn!(
head,
%e,
"virtio-net TX add_used failed (used-ring address \
likely unmapped); bumped tx_add_used_failures, \
will NOT bump tx_packets"
);
}
}
}
fn pop_and_capture_tx(&mut self, mem: &GuestMemoryMmap, tx_idx: usize) -> TxPopOutcome {
let (chain, head) = match self.pop_tx_chain(mem, tx_idx) {
Ok(v) => v,
Err(outcome) => return outcome,
};
self.capture_tx_frame(chain, head, mem)
}
fn pop_tx_chain<'a>(
&mut self,
mem: &'a GuestMemoryMmap,
tx_idx: usize,
) -> Result<(DescriptorChain<&'a GuestMemoryMmap>, u16), TxPopOutcome> {
if self.queue_poisoned[tx_idx] {
return Err(TxPopOutcome::Empty);
}
enum IterStep<C> {
Chain(C),
Empty,
Poisoned(VirtioQueueError),
}
let step: IterStep<_> = {
let q = &mut self.queues[tx_idx];
match q.iter(mem) {
Ok(mut iter) => match iter.next() {
Some(c) => IterStep::Chain(c),
None => IterStep::Empty,
},
Err(e) => IterStep::Poisoned(e),
}
};
match step {
IterStep::Empty => Err(TxPopOutcome::Empty),
IterStep::Chain(c) => {
let h = c.head_index();
Ok((c, h))
}
IterStep::Poisoned(err) => {
self.queue_poisoned[tx_idx] = true;
self.counters.record_invalid_avail_idx();
tracing::warn!(
err = %err,
"virtio-net TX iter() failed; poisoning TX queue until \
guest reset (any structural queue error is \
non-recoverable; cloud-hypervisor convergence)"
);
Err(TxPopOutcome::JustPoisoned)
}
}
}
fn capture_tx_frame(
&mut self,
chain: DescriptorChain<&GuestMemoryMmap>,
head: u16,
mem: &GuestMemoryMmap,
) -> TxPopOutcome {
self.tx_frame_scratch.clear();
let mut hdr_remaining: usize = VIRTIO_NET_HDR_LEN;
let mut total_data_bytes: usize = 0;
let mut chain_invalid = false;
let mut chain_oversize = false;
for desc in chain {
if desc.is_write_only() {
chain_invalid = true;
break;
}
let mut desc_len = desc.len() as usize;
let mut desc_addr = desc.addr();
if hdr_remaining > 0 {
let skip = hdr_remaining.min(desc_len);
let Some(new_addr) = desc_addr.checked_add(skip as u64) else {
chain_invalid = true;
break;
};
hdr_remaining -= skip;
desc_len -= skip;
desc_addr = new_addr;
}
if desc_len == 0 {
continue;
}
if total_data_bytes + desc_len > MAX_FRAME_SIZE {
chain_oversize = true;
break;
}
let start = self.tx_frame_scratch.len();
self.tx_frame_scratch.resize(start + desc_len, 0);
if mem
.read_slice(
&mut self.tx_frame_scratch[start..start + desc_len],
desc_addr,
)
.is_err()
{
self.tx_frame_scratch.truncate(start);
chain_invalid = true;
break;
}
total_data_bytes += desc_len;
}
if chain_oversize {
self.counters.record_tx_oversize_dropped();
return TxPopOutcome::Chain(TxChainOutcome {
head,
frame_len: None,
});
}
if chain_invalid || hdr_remaining != 0 {
self.counters.record_tx_chain_invalid();
return TxPopOutcome::Chain(TxChainOutcome {
head,
frame_len: None,
});
}
TxPopOutcome::Chain(TxChainOutcome {
head,
frame_len: Some(total_data_bytes),
})
}
fn try_loopback_to_rx(
&mut self,
mem: &GuestMemoryMmap,
frame_len: usize,
rx_idx: usize,
) -> LoopbackOutcome {
let (chain, head) = match self.pop_rx_chain(mem, rx_idx) {
Ok(v) => v,
Err(outcome) => return outcome,
};
let walk = self.write_rx_chain(chain, mem, frame_len);
self.finalize_rx(walk, head, mem, frame_len, rx_idx)
}
fn pop_rx_chain<'a>(
&mut self,
mem: &'a GuestMemoryMmap,
rx_idx: usize,
) -> Result<(DescriptorChain<&'a GuestMemoryMmap>, u16), LoopbackOutcome> {
if self.queue_poisoned[rx_idx] {
return Err(LoopbackOutcome::RxAlreadyPoisoned);
}
enum IterStep<C> {
Chain(C),
NoBuffer,
Poisoned(VirtioQueueError),
}
let step: IterStep<_> = {
let q = &mut self.queues[rx_idx];
if !q.ready() {
return Err(LoopbackOutcome::NoRxBuffer);
}
match q.iter(mem) {
Ok(mut iter) => match iter.next() {
Some(c) => IterStep::Chain(c),
None => IterStep::NoBuffer,
},
Err(e) => IterStep::Poisoned(e),
}
};
match step {
IterStep::NoBuffer => Err(LoopbackOutcome::NoRxBuffer),
IterStep::Chain(c) => {
let h = c.head_index();
Ok((c, h))
}
IterStep::Poisoned(err) => {
self.queue_poisoned[rx_idx] = true;
self.counters.record_invalid_avail_idx();
tracing::warn!(
err = %err,
"virtio-net RX iter() failed; poisoning RX queue until \
guest reset (any structural queue error is \
non-recoverable; cloud-hypervisor convergence)"
);
Err(LoopbackOutcome::JustRxPoisoned)
}
}
}
fn write_rx_chain(
&self,
chain: DescriptorChain<&GuestMemoryMmap>,
mem: &GuestMemoryMmap,
frame_len: usize,
) -> RxWriteResult {
let mut bytes_written: u32 = 0;
let mut hdr_remaining: usize = VIRTIO_NET_HDR_LEN;
let mut frame_pos: usize = 0;
let mut hdr_write_slots: [(GuestAddress, usize); VIRTIO_NET_HDR_LEN] =
[(GuestAddress(0), 0); VIRTIO_NET_HDR_LEN];
let mut hdr_write_count: usize = 0;
let mut chain_invalid: Option<InvalidReason> = None;
for desc in chain {
if !desc.is_write_only() {
chain_invalid = Some(InvalidReason::Shape);
break;
}
let mut desc_addr = desc.addr();
let mut desc_len = desc.len() as usize;
if hdr_remaining > 0 {
let take = hdr_remaining.min(desc_len);
const RX_HDR: [u8; VIRTIO_NET_HDR_LEN] = {
let mut h = [0u8; VIRTIO_NET_HDR_LEN];
h[10] = 1;
h[11] = 0;
h
};
let hdr_start = VIRTIO_NET_HDR_LEN - hdr_remaining;
let hdr_slice = &RX_HDR[hdr_start..hdr_start + take];
if mem.write_slice(hdr_slice, desc_addr).is_err() {
chain_invalid = Some(InvalidReason::WriteFailed);
break;
}
hdr_write_slots[hdr_write_count] = (desc_addr, take);
hdr_write_count += 1;
let Some(new_addr) = desc_addr.checked_add(take as u64) else {
chain_invalid = Some(InvalidReason::Shape);
break;
};
bytes_written = bytes_written
.checked_add(take as u32)
.expect("bytes_written cannot overflow u32 — capped by MAX_FRAME_SIZE+12");
hdr_remaining -= take;
desc_len -= take;
desc_addr = new_addr;
}
if desc_len == 0 || frame_pos == frame_len {
continue;
}
let take = desc_len.min(frame_len - frame_pos);
if mem
.write_slice(
&self.tx_frame_scratch[frame_pos..frame_pos + take],
desc_addr,
)
.is_err()
{
chain_invalid = Some(InvalidReason::WriteFailed);
break;
}
bytes_written = bytes_written
.checked_add(take as u32)
.expect("bytes_written cannot overflow u32 — capped by MAX_FRAME_SIZE+12");
frame_pos += take;
if frame_pos == frame_len && hdr_remaining == 0 {
break;
}
}
RxWriteResult {
bytes_written,
hdr_remaining,
frame_pos,
chain_invalid,
hdr_write_slots,
hdr_write_count,
}
}
fn finalize_rx(
&mut self,
walk: RxWriteResult,
head: u16,
mem: &GuestMemoryMmap,
frame_len: usize,
rx_idx: usize,
) -> LoopbackOutcome {
if let Some(reason) = walk.chain_invalid {
match reason {
InvalidReason::Shape => self.counters.record_rx_chain_invalid(),
InvalidReason::WriteFailed => self.counters.record_rx_write_failed(),
}
const ZEROS: [u8; VIRTIO_NET_HDR_LEN] = [0u8; VIRTIO_NET_HDR_LEN];
for &(addr, len) in &walk.hdr_write_slots[..walk.hdr_write_count] {
let _ = mem.write_slice(&ZEROS[..len], addr);
}
let add_used_ok = match self.queues[rx_idx].add_used(mem, head, 0) {
Ok(()) => true,
Err(e) => {
self.counters.record_rx_add_used_failure();
tracing::warn!(
head,
%e,
"virtio-net RX add_used failed after malformed-chain \
reject (used-ring address likely unmapped); bumped \
rx_add_used_failures"
);
false
}
};
return LoopbackOutcome::RxChainInvalid { add_used_ok };
}
if walk.frame_pos < frame_len || walk.hdr_remaining != 0 {
tracing::debug!(
frame_len,
bytes_written = walk.bytes_written,
hdr_remaining = walk.hdr_remaining,
"virtio-net RX buffer too small for full frame; truncating"
);
}
let hdr_taken = (VIRTIO_NET_HDR_LEN - walk.hdr_remaining) as u32;
let l2_bytes = walk.bytes_written.saturating_sub(hdr_taken) as u64;
match self.queues[rx_idx].add_used(mem, head, walk.bytes_written) {
Ok(()) => LoopbackOutcome::Delivered {
l2_bytes_written: l2_bytes,
},
Err(e) => {
self.counters.record_rx_add_used_failure();
tracing::warn!(
head,
%e,
"virtio-net RX add_used failed after successful frame \
write (used-ring address likely unmapped); bumped \
rx_add_used_failures, will NOT bump rx_packets"
);
LoopbackOutcome::DeliveredButAddUsedFailed
}
}
}
}
enum InvalidReason {
Shape,
WriteFailed,
}
struct RxWriteResult {
bytes_written: u32,
hdr_remaining: usize,
frame_pos: usize,
chain_invalid: Option<InvalidReason>,
hdr_write_slots: [(GuestAddress, usize); VIRTIO_NET_HDR_LEN],
hdr_write_count: usize,
}
enum LoopbackOutcome {
Delivered { l2_bytes_written: u64 },
DeliveredButAddUsedFailed,
NoRxBuffer,
RxChainInvalid { add_used_ok: bool },
JustRxPoisoned,
RxAlreadyPoisoned,
}
enum TxPopOutcome {
Empty,
JustPoisoned,
Chain(TxChainOutcome),
}
struct TxChainOutcome {
head: u16,
frame_len: Option<usize>,
}
impl VirtioNet {
fn set_status(&mut self, val: u32) {
let old = self.device_status;
if val & self.device_status != self.device_status {
tracing::debug!(old, val, "virtio-net set_status: rejected (clears bits)");
return;
}
let new_bits = val & !self.device_status;
let valid = match new_bits {
VIRTIO_CONFIG_S_ACKNOWLEDGE => self.device_status == 0,
VIRTIO_CONFIG_S_DRIVER => self.device_status == S_ACK,
VIRTIO_CONFIG_S_FEATURES_OK => self.device_status == S_DRV,
VIRTIO_CONFIG_S_DRIVER_OK => self.device_status == S_FEAT,
_ => false,
};
if !valid {
tracing::debug!(
old,
val,
"virtio-net set_status: rejected (invalid transition)"
);
return;
}
if new_bits == VIRTIO_CONFIG_S_FEATURES_OK {
let device_features = self.device_features();
let unoffered = self.driver_features & !device_features;
if unoffered != 0 {
self.device_status |= VIRTIO_CONFIG_S_FAILED;
tracing::warn!(
old,
attempted = val,
driver_features = self.driver_features,
device_features,
unoffered,
"virtio-net set_status: driver accepted features not \
offered by device; rejecting FEATURES_OK and setting \
FAILED bit"
);
return;
}
if (self.driver_features & (1u64 << VIRTIO_F_VERSION_1)) == 0 {
self.device_status |= VIRTIO_CONFIG_S_FAILED;
tracing::warn!(
old,
attempted = val,
"virtio-net set_status: VIRTIO_F_VERSION_1 not \
negotiated; rejecting FEATURES_OK and setting FAILED bit"
);
return;
}
}
self.device_status = val;
tracing::debug!(old, new = val, "virtio-net set_status: accepted");
}
fn reset(&mut self) {
self.device_status = 0;
self.interrupt_status = 0;
self.queue_select = 0;
self.device_features_sel = 0;
self.driver_features_sel = 0;
self.driver_features = 0;
self.tx_frame_scratch.clear();
self.queue_poisoned.fill(false);
self.curr_queue_pairs = 1;
if let Some(msix) = &self.msix {
msix.lock().reset_virtio_assignments();
}
for q in &mut self.queues {
q.reset();
}
}
}