use virtio_bindings::virtio_ring;
use crate::error::{Result, VirtioError};
const VRING_AVAIL_F_NO_INTERRUPT: u16 = virtio_ring::VRING_AVAIL_F_NO_INTERRUPT as u16;
pub const VIRTIO_F_EVENT_IDX: u64 = 1 << virtio_ring::VIRTIO_RING_F_EVENT_IDX;
pub mod flags {
use super::virtio_ring;
pub const NEXT: u16 = virtio_ring::VRING_DESC_F_NEXT as u16;
pub const WRITE: u16 = virtio_ring::VRING_DESC_F_WRITE as u16;
pub const INDIRECT: u16 = virtio_ring::VRING_DESC_F_INDIRECT as u16;
}
#[derive(Debug, Clone, Copy, Default)]
#[repr(C)]
pub struct Descriptor {
pub addr: u64,
pub len: u32,
pub flags: u16,
pub next: u16,
}
impl Descriptor {
#[must_use]
pub const fn has_next(&self) -> bool {
self.flags & flags::NEXT != 0
}
#[must_use]
pub const fn is_write_only(&self) -> bool {
self.flags & flags::WRITE != 0
}
#[must_use]
pub const fn is_indirect(&self) -> bool {
self.flags & flags::INDIRECT != 0
}
}
#[derive(Debug)]
pub struct AvailRing {
pub flags: u16,
pub idx: u16,
pub ring: Vec<u16>,
pub used_event: u16,
}
impl AvailRing {
#[must_use]
pub fn new(size: u16) -> Self {
Self {
flags: 0,
idx: 0,
ring: vec![0; size as usize],
used_event: 0,
}
}
}
#[derive(Debug, Clone, Copy, Default)]
#[repr(C)]
pub struct UsedElement {
pub id: u32,
pub len: u32,
}
#[derive(Debug)]
pub struct UsedRing {
pub flags: u16,
pub idx: u16,
pub ring: Vec<UsedElement>,
pub avail_event: u16,
}
impl UsedRing {
#[must_use]
pub fn new(size: u16) -> Self {
Self {
flags: 0,
idx: 0,
ring: vec![UsedElement::default(); size as usize],
avail_event: 0,
}
}
}
#[derive(Debug)]
pub struct VirtQueue {
size: u16,
desc_table: Vec<Descriptor>,
avail: AvailRing,
used: UsedRing,
last_avail_idx: u16,
ready: bool,
event_idx_enabled: bool,
}
impl VirtQueue {
pub fn new(size: u16) -> Result<Self> {
if size == 0 || !size.is_power_of_two() {
return Err(VirtioError::InvalidQueue(
"size must be a power of 2".to_string(),
));
}
if size > 32768 {
return Err(VirtioError::InvalidQueue(
"size must not exceed 32768".to_string(),
));
}
Ok(Self {
size,
desc_table: vec![Descriptor::default(); size as usize],
avail: AvailRing::new(size),
used: UsedRing::new(size),
last_avail_idx: 0,
ready: false,
event_idx_enabled: false,
})
}
#[must_use]
pub const fn size(&self) -> u16 {
self.size
}
#[must_use]
pub const fn is_ready(&self) -> bool {
self.ready
}
pub const fn set_ready(&mut self, ready: bool) {
self.ready = ready;
}
pub fn set_descriptor(&mut self, idx: u16, descriptor: Descriptor) -> Result<()> {
if idx >= self.size {
return Err(VirtioError::InvalidQueue(
"descriptor index out of bounds".to_string(),
));
}
self.desc_table[idx as usize] = descriptor;
Ok(())
}
pub fn add_avail(&mut self, head_idx: u16) -> Result<()> {
if head_idx >= self.size {
return Err(VirtioError::InvalidQueue(
"available index out of bounds".to_string(),
));
}
let ring_idx = (self.avail.idx % self.size) as usize;
self.avail.ring[ring_idx] = head_idx;
self.avail.idx = self.avail.idx.wrapping_add(1);
Ok(())
}
#[must_use]
pub const fn has_available(&self) -> bool {
self.avail.idx != self.last_avail_idx
}
pub fn pop_avail(&mut self) -> Option<(u16, DescriptorChain)> {
if !self.has_available() {
return None;
}
let avail_idx = self.last_avail_idx;
let head_idx = self.avail.ring[(avail_idx % self.size) as usize];
self.last_avail_idx = self.last_avail_idx.wrapping_add(1);
Some((
head_idx,
DescriptorChain {
queue: self,
current: Some(head_idx),
ttl: self.size,
},
))
}
pub fn set_event_idx(&mut self, enabled: bool) {
self.event_idx_enabled = enabled;
}
pub fn push_used(&mut self, head_idx: u16, len: u32) -> bool {
let old_idx = self.used.idx;
self.used.ring[(old_idx % self.size) as usize] = UsedElement {
id: head_idx as u32,
len,
};
self.used.idx = self.used.idx.wrapping_add(1);
self.should_notify(old_idx)
}
pub fn push_used_batch(&mut self, completions: &[(u16, u32)]) -> bool {
if completions.is_empty() {
return false;
}
let old_idx = self.used.idx;
for &(head_idx, len) in completions {
let slot = (self.used.idx % self.size) as usize;
self.used.ring[slot] = UsedElement {
id: head_idx as u32,
len,
};
self.used.idx = self.used.idx.wrapping_add(1);
}
self.should_notify(old_idx)
}
pub fn set_avail_event(&mut self, event: u16) {
self.used.avail_event = event;
}
fn should_notify(&self, old_idx: u16) -> bool {
if self.event_idx_enabled {
Self::needs_notification(old_idx, self.used.idx, self.avail.used_event)
} else {
(self.avail.flags & VRING_AVAIL_F_NO_INTERRUPT) == 0
}
}
fn needs_notification(old_idx: u16, new_idx: u16, used_event: u16) -> bool {
new_idx.wrapping_sub(used_event).wrapping_sub(1) < new_idx.wrapping_sub(old_idx)
}
#[must_use]
pub fn get_descriptor(&self, idx: u16) -> Option<&Descriptor> {
self.desc_table.get(idx as usize)
}
}
pub struct DescriptorChain<'a> {
queue: &'a VirtQueue,
current: Option<u16>,
ttl: u16,
}
impl<'a> Iterator for DescriptorChain<'a> {
type Item = &'a Descriptor;
fn next(&mut self) -> Option<Self::Item> {
if self.ttl == 0 {
return None;
}
let idx = self.current?;
let desc = self.queue.get_descriptor(idx)?;
self.ttl -= 1;
self.current = if desc.has_next() {
Some(desc.next)
} else {
None
};
Some(desc)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_descriptor_default() {
let desc = Descriptor::default();
assert_eq!(desc.addr, 0);
assert_eq!(desc.len, 0);
assert_eq!(desc.flags, 0);
assert_eq!(desc.next, 0);
}
#[test]
fn test_descriptor_has_next() {
let mut desc = Descriptor::default();
assert!(!desc.has_next());
desc.flags = flags::NEXT;
assert!(desc.has_next());
}
#[test]
fn test_descriptor_is_write_only() {
let mut desc = Descriptor::default();
assert!(!desc.is_write_only());
desc.flags = flags::WRITE;
assert!(desc.is_write_only());
}
#[test]
fn test_descriptor_is_indirect() {
let mut desc = Descriptor::default();
assert!(!desc.is_indirect());
desc.flags = flags::INDIRECT;
assert!(desc.is_indirect());
}
#[test]
fn test_descriptor_multiple_flags() {
let desc = Descriptor {
addr: 0x1000,
len: 512,
flags: flags::NEXT | flags::WRITE,
next: 1,
};
assert!(desc.has_next());
assert!(desc.is_write_only());
assert!(!desc.is_indirect());
}
#[test]
fn test_descriptor_clone_copy() {
let desc = Descriptor {
addr: 0xDEADBEEF,
len: 1234,
flags: flags::NEXT,
next: 42,
};
let cloned = desc;
let copied = desc;
assert_eq!(cloned.addr, 0xDEADBEEF);
assert_eq!(copied.addr, 0xDEADBEEF);
}
#[test]
fn test_flag_constants() {
assert_eq!(flags::NEXT, 1);
assert_eq!(flags::WRITE, 2);
assert_eq!(flags::INDIRECT, 4);
}
#[test]
fn test_avail_ring_new() {
let ring = AvailRing::new(256);
assert_eq!(ring.flags, 0);
assert_eq!(ring.idx, 0);
assert_eq!(ring.ring.len(), 256);
assert_eq!(ring.used_event, 0);
}
#[test]
fn test_avail_ring_small() {
let ring = AvailRing::new(1);
assert_eq!(ring.ring.len(), 1);
}
#[test]
fn test_used_element_default() {
let elem = UsedElement::default();
assert_eq!(elem.id, 0);
assert_eq!(elem.len, 0);
}
#[test]
fn test_used_element_clone_copy() {
let elem = UsedElement { id: 42, len: 1024 };
let cloned = elem;
let copied = elem;
assert_eq!(cloned.id, 42);
assert_eq!(copied.len, 1024);
}
#[test]
fn test_used_ring_new() {
let ring = UsedRing::new(128);
assert_eq!(ring.flags, 0);
assert_eq!(ring.idx, 0);
assert_eq!(ring.ring.len(), 128);
assert_eq!(ring.avail_event, 0);
}
#[test]
fn test_virtqueue_new() {
let queue = VirtQueue::new(256).unwrap();
assert_eq!(queue.size(), 256);
assert!(!queue.is_ready());
}
#[test]
fn test_virtqueue_new_power_of_two() {
for size in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] {
assert!(VirtQueue::new(size).is_ok());
}
}
#[test]
fn test_virtqueue_new_invalid_size_zero() {
let result = VirtQueue::new(0);
assert!(result.is_err());
if let Err(VirtioError::InvalidQueue(msg)) = result {
assert!(msg.contains("power of 2"));
}
}
#[test]
fn test_virtqueue_new_invalid_size_not_power_of_two() {
for size in [3, 5, 6, 7, 9, 100, 1000] {
let result = VirtQueue::new(size);
assert!(result.is_err());
}
}
#[test]
fn test_virtqueue_new_too_large() {
let result = VirtQueue::new(32768); assert!(result.is_ok());
}
#[test]
fn test_virtqueue_ready_state() {
let mut queue = VirtQueue::new(16).unwrap();
assert!(!queue.is_ready());
queue.set_ready(true);
assert!(queue.is_ready());
queue.set_ready(false);
assert!(!queue.is_ready());
}
#[test]
fn test_virtqueue_has_available_empty() {
let queue = VirtQueue::new(16).unwrap();
assert!(!queue.has_available());
}
#[test]
fn test_virtqueue_pop_avail_empty() {
let mut queue = VirtQueue::new(16).unwrap();
assert!(queue.pop_avail().is_none());
}
#[test]
fn test_virtqueue_get_descriptor() {
let queue = VirtQueue::new(16).unwrap();
assert!(queue.get_descriptor(0).is_some());
assert!(queue.get_descriptor(15).is_some());
assert!(queue.get_descriptor(16).is_none());
assert!(queue.get_descriptor(100).is_none());
}
#[test]
fn test_virtqueue_push_used() {
let mut queue = VirtQueue::new(16).unwrap();
let _notify = queue.push_used(0, 512);
assert_eq!(queue.used.idx, 1);
assert_eq!(queue.used.ring[0].id, 0);
assert_eq!(queue.used.ring[0].len, 512);
let _notify = queue.push_used(1, 1024);
assert_eq!(queue.used.idx, 2);
assert_eq!(queue.used.ring[1].id, 1);
assert_eq!(queue.used.ring[1].len, 1024);
}
#[test]
fn test_virtqueue_push_used_wrap() {
let mut queue = VirtQueue::new(4).unwrap();
for i in 0..10 {
let _notify = queue.push_used(i, i as u32 * 100);
}
assert_eq!(queue.used.idx, 10);
assert_eq!(queue.used.ring[1].id, 9);
}
#[test]
fn test_virtqueue_simulated_transaction() {
let mut queue = VirtQueue::new(16).unwrap();
queue.set_ready(true);
queue.avail.ring[0] = 0; queue.avail.idx = 1;
assert!(queue.has_available());
let (head_idx, _chain) = queue.pop_avail().unwrap();
assert_eq!(head_idx, 0);
assert!(!queue.has_available());
let _notify = queue.push_used(head_idx, 256);
assert_eq!(queue.used.idx, 1);
}
#[test]
fn test_virtqueue_multiple_descriptors() {
let mut queue = VirtQueue::new(16).unwrap();
queue.set_ready(true);
for i in 0..5 {
queue.avail.ring[i] = i as u16;
}
queue.avail.idx = 5;
for i in 0..5 {
assert!(queue.has_available());
let (head_idx, _) = queue.pop_avail().unwrap();
assert_eq!(head_idx, i as u16);
}
assert!(!queue.has_available());
}
#[test]
fn test_descriptor_chain_single() {
let mut queue = VirtQueue::new(16).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 512,
flags: 0, next: 0,
};
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (head_idx, chain) = queue.pop_avail().unwrap();
assert_eq!(head_idx, 0);
let descs: Vec<_> = chain.collect();
assert_eq!(descs.len(), 1);
assert_eq!(descs[0].addr, 0x1000);
assert_eq!(descs[0].len, 512);
}
#[test]
fn test_descriptor_chain_multiple() {
let mut queue = VirtQueue::new(16).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 256,
flags: flags::NEXT,
next: 1,
};
queue.desc_table[1] = Descriptor {
addr: 0x2000,
len: 512,
flags: flags::NEXT,
next: 2,
};
queue.desc_table[2] = Descriptor {
addr: 0x3000,
len: 1024,
flags: 0, next: 0,
};
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (_, chain) = queue.pop_avail().unwrap();
let descs: Vec<_> = chain.collect();
assert_eq!(descs.len(), 3);
assert_eq!(descs[0].addr, 0x1000);
assert_eq!(descs[1].addr, 0x2000);
assert_eq!(descs[2].addr, 0x3000);
}
#[test]
fn test_descriptor_chain_with_write_flags() {
let mut queue = VirtQueue::new(16).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 256,
flags: flags::NEXT, next: 1,
};
queue.desc_table[1] = Descriptor {
addr: 0x2000,
len: 512,
flags: flags::WRITE, next: 0,
};
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (_, chain) = queue.pop_avail().unwrap();
let descs: Vec<_> = chain.collect();
assert_eq!(descs.len(), 2);
assert!(!descs[0].is_write_only());
assert!(descs[1].is_write_only());
}
#[test]
fn test_descriptor_chain_cycle_terminates() {
let mut queue = VirtQueue::new(16).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 16,
flags: flags::NEXT,
next: 1,
};
queue.desc_table[1] = Descriptor {
addr: 0x2000,
len: 16,
flags: flags::NEXT,
next: 0,
};
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (_, chain) = queue.pop_avail().unwrap();
let descs: Vec<_> = chain.collect();
assert!(
descs.len() <= 16,
"chain iteration exceeded queue size cap: got {}",
descs.len()
);
}
#[test]
fn test_descriptor_chain_self_loop_terminates() {
let mut queue = VirtQueue::new(8).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 16,
flags: flags::NEXT,
next: 0,
};
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (_, chain) = queue.pop_avail().unwrap();
let descs: Vec<_> = chain.collect();
assert!(descs.len() <= 8);
}
#[test]
fn test_avail_idx_wrap() {
let mut queue = VirtQueue::new(4).unwrap();
queue.avail.idx = u16::MAX;
queue.last_avail_idx = u16::MAX - 1;
queue.avail.ring[(queue.last_avail_idx % 4) as usize] = 0;
assert!(queue.has_available());
let (head_idx, _) = queue.pop_avail().unwrap();
assert_eq!(head_idx, 0);
assert_eq!(queue.last_avail_idx, u16::MAX);
queue.avail.ring[(queue.avail.idx % 4) as usize] = 1;
queue.avail.idx = 0;
assert!(queue.has_available());
let (head_idx, _) = queue.pop_avail().unwrap();
assert_eq!(head_idx, 1);
}
#[test]
fn test_used_idx_wrap() {
let mut queue = VirtQueue::new(4).unwrap();
queue.used.idx = u16::MAX;
let _notify = queue.push_used(0, 100);
assert_eq!(queue.used.idx, 0);
assert_eq!(queue.used.ring[3].id, 0);
assert_eq!(queue.used.ring[3].len, 100);
}
#[test]
fn test_queue_min_size() {
let queue = VirtQueue::new(1).unwrap();
assert_eq!(queue.size(), 1);
assert_eq!(queue.desc_table.len(), 1);
assert_eq!(queue.avail.ring.len(), 1);
assert_eq!(queue.used.ring.len(), 1);
}
#[test]
fn test_queue_max_size() {
let queue = VirtQueue::new(32768).unwrap();
assert_eq!(queue.size(), 32768);
assert_eq!(queue.desc_table.len(), 32768);
}
#[test]
fn test_descriptor_chain_out_of_bounds() {
let mut queue = VirtQueue::new(4).unwrap();
queue.desc_table[0] = Descriptor {
addr: 0x1000,
len: 256,
flags: flags::NEXT,
next: 100, };
queue.avail.ring[0] = 0;
queue.avail.idx = 1;
let (_, chain) = queue.pop_avail().unwrap();
let descs: Vec<_> = chain.collect();
assert_eq!(descs.len(), 1);
}
#[test]
fn test_needs_notification_basic() {
assert!(VirtQueue::needs_notification(0, 1, 0));
}
#[test]
fn test_needs_notification_past_event() {
assert!(!VirtQueue::needs_notification(5, 6, 3));
}
#[test]
fn test_needs_notification_wrap() {
assert!(VirtQueue::needs_notification(65534, 0, 65535));
}
#[test]
fn test_needs_notification_wrap_no_notify() {
assert!(!VirtQueue::needs_notification(65534, 65535, 0));
}
#[test]
fn test_push_used_no_interrupt_flag() {
let mut queue = VirtQueue::new(16).unwrap();
queue.avail.flags = VRING_AVAIL_F_NO_INTERRUPT;
let notify = queue.push_used(0, 512);
assert!(!notify); }
#[test]
fn test_push_used_default_notifies() {
let mut queue = VirtQueue::new(16).unwrap();
let notify = queue.push_used(0, 512);
assert!(notify); }
#[test]
fn test_push_used_event_idx_suppresses() {
let mut queue = VirtQueue::new(16).unwrap();
queue.set_event_idx(true);
queue.avail.used_event = 5;
let notify = queue.push_used(0, 512);
assert!(!notify);
}
#[test]
fn test_push_used_event_idx_triggers() {
let mut queue = VirtQueue::new(16).unwrap();
queue.set_event_idx(true);
queue.avail.used_event = 0;
let notify = queue.push_used(0, 512);
assert!(notify);
}
#[test]
fn test_push_used_batch() {
let mut queue = VirtQueue::new(16).unwrap();
let completions = vec![(0, 256), (1, 512), (2, 1024)];
let notify = queue.push_used_batch(&completions);
assert!(notify); assert_eq!(queue.used.idx, 3);
assert_eq!(queue.used.ring[0].id, 0);
assert_eq!(queue.used.ring[1].id, 1);
assert_eq!(queue.used.ring[2].id, 2);
}
#[test]
fn test_push_used_batch_empty() {
let mut queue = VirtQueue::new(16).unwrap();
let notify = queue.push_used_batch(&[]);
assert!(!notify); assert_eq!(queue.used.idx, 0);
}
#[test]
fn test_set_avail_event() {
let mut queue = VirtQueue::new(16).unwrap();
queue.set_avail_event(42);
assert_eq!(queue.used.avail_event, 42);
}
}