#![allow(dead_code)]
use alloc::boxed::Box;
use alloc::collections::VecDeque;
use alloc::rc::Rc;
use alloc::vec::Vec;
use core::cell::RefCell;
use core::ptr;
use core::sync::atomic::{fence, Ordering};
use align_address::Align;
use self::error::VqPackedError;
use super::super::features::Features;
#[cfg(not(feature = "pci"))]
use super::super::transport::mmio::{ComCfg, NotifCfg, NotifCtrl};
#[cfg(feature = "pci")]
use super::super::transport::pci::{ComCfg, NotifCfg, NotifCtrl};
use super::error::VirtqError;
use super::{
AsSliceU8, BuffSpec, Buffer, BufferToken, Bytes, DescrFlags, MemDescr, MemPool, Pinned,
Transfer, TransferState, TransferToken, Virtq, VqIndex, VqSize,
};
use crate::arch::mm::paging::{BasePageSize, PageSize};
use crate::arch::mm::{paging, VirtAddr};
#[derive(Copy, Clone, Debug)]
struct WrapCount(bool);
impl WrapCount {
fn flag_mask() -> u16 {
1 << 7 | 1 << 15
}
fn new() -> Self {
WrapCount(true)
}
fn wrap(&mut self) {
self.0 = !self.0
}
fn as_flags_avail(&self) -> u16 {
if self.0 {
1 << 7
} else {
1 << 15
}
}
fn as_flags_used(&self) -> u16 {
if self.0 {
1 << 7 | 1 << 15
} else {
0
}
}
}
struct DescriptorRing {
ring: &'static mut [Descriptor],
tkn_ref_ring: Box<[*mut TransferToken]>,
write_index: usize,
capacity: usize,
poll_index: usize,
drv_wc: WrapCount,
dev_wc: WrapCount,
}
impl DescriptorRing {
fn new(size: u16) -> Self {
let size = usize::try_from(size).unwrap();
let _mem_len =
(size * core::mem::size_of::<Descriptor>()).align_up(BasePageSize::SIZE as usize);
let ptr = (crate::mm::allocate(_mem_len, true).0 as *const Descriptor) as *mut Descriptor;
let ring: &'static mut [Descriptor] = unsafe { core::slice::from_raw_parts_mut(ptr, size) };
let tkn_ref_ring = vec![ptr::null_mut(); size + 1].into_boxed_slice();
DescriptorRing {
ring,
tkn_ref_ring,
write_index: 0,
capacity: size,
poll_index: 0,
drv_wc: WrapCount::new(),
dev_wc: WrapCount::new(),
}
}
fn poll(&mut self) {
let mut ctrl = self.get_read_ctrler();
if let Some(tkn) = ctrl.poll_next() {
let tkn = unsafe { &mut *(tkn) };
match tkn.await_queue {
Some(_) => {
tkn.state = TransferState::Finished;
let queue = tkn.await_queue.take().unwrap();
queue.borrow_mut().push_back(Transfer {
transfer_tkn: Some(Pinned::from_raw(tkn as *mut TransferToken)),
});
}
None => tkn.state = TransferState::Finished,
}
}
}
fn push_batch(
&mut self,
tkn_lst: Vec<TransferToken>,
) -> (Vec<Pinned<TransferToken>>, usize, u8) {
assert!(!tkn_lst.is_empty());
let mut first_ctrl_settings: (usize, u16, WrapCount) = (0, 0, WrapCount::new());
let mut pind_lst = Vec::with_capacity(tkn_lst.len());
for (i, tkn) in tkn_lst.into_iter().enumerate() {
let mut pinned = Pinned::pin(tkn);
assert!(pinned.buff_tkn.as_ref().unwrap().num_consuming_descr() <= self.capacity);
let mut ctrl = self.get_write_ctrler();
match (
&pinned.buff_tkn.as_ref().unwrap().send_buff,
&pinned.buff_tkn.as_ref().unwrap().recv_buff,
) {
(Some(send_buff), Some(recv_buff)) => {
match (send_buff.get_ctrl_desc(), recv_buff.get_ctrl_desc()) {
(Some(ctrl_desc), Some(_)) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
(None, None) => {
let mut buff_len =
send_buff.as_slice().len() + recv_buff.as_slice().len();
for desc in send_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_NEXT.into());
} else {
ctrl.write_desc(desc, 0);
}
buff_len -= 1;
}
for desc in recv_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(
desc,
DescrFlags::VIRTQ_DESC_F_NEXT
| DescrFlags::VIRTQ_DESC_F_WRITE,
);
} else {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_WRITE.into());
}
buff_len -= 1;
}
}
(None, Some(_)) => {
unreachable!("Indirect buffers mixed with direct buffers!")
} (Some(_), None) => {
unreachable!("Indirect buffers mixed with direct buffers!")
} }
}
(Some(send_buff), None) => {
match send_buff.get_ctrl_desc() {
Some(ctrl_desc) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
None => {
let mut buff_len = send_buff.as_slice().len();
for desc in send_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_NEXT.into());
} else {
ctrl.write_desc(desc, 0);
}
buff_len -= 1;
}
}
}
}
(None, Some(recv_buff)) => {
match recv_buff.get_ctrl_desc() {
Some(ctrl_desc) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
None => {
let mut buff_len = recv_buff.as_slice().len();
for desc in recv_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(
desc,
DescrFlags::VIRTQ_DESC_F_NEXT
| DescrFlags::VIRTQ_DESC_F_WRITE,
);
} else {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_WRITE.into());
}
buff_len -= 1;
}
}
}
}
(None, None) => unreachable!("Empty Transfers are not allowed!"), }
if i == 0 {
first_ctrl_settings = (ctrl.start, ctrl.buff_id, ctrl.wrap_at_init);
} else {
ctrl.make_avail(pinned.raw_addr());
}
pinned.state = TransferState::Processing;
pind_lst.push(pinned);
}
self.tkn_ref_ring[usize::try_from(first_ctrl_settings.1).unwrap()] = pind_lst[0].raw_addr();
fence(Ordering::SeqCst);
self.ring[first_ctrl_settings.0].flags |= first_ctrl_settings.2.as_flags_avail();
(
pind_lst,
first_ctrl_settings.0,
first_ctrl_settings.2 .0 as u8,
)
}
fn push(&mut self, tkn: TransferToken) -> (Pinned<TransferToken>, usize, u8) {
let mut pinned = Pinned::pin(tkn);
assert!(pinned.buff_tkn.as_ref().unwrap().num_consuming_descr() <= self.capacity);
let mut ctrl = self.get_write_ctrler();
match (
&pinned.buff_tkn.as_ref().unwrap().send_buff,
&pinned.buff_tkn.as_ref().unwrap().recv_buff,
) {
(Some(send_buff), Some(recv_buff)) => {
match (send_buff.get_ctrl_desc(), recv_buff.get_ctrl_desc()) {
(Some(ctrl_desc), Some(_)) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
(None, None) => {
let mut buff_len = send_buff.as_slice().len() + recv_buff.as_slice().len();
for desc in send_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_NEXT.into());
} else {
ctrl.write_desc(desc, 0);
}
buff_len -= 1;
}
for desc in recv_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(
desc,
DescrFlags::VIRTQ_DESC_F_NEXT | DescrFlags::VIRTQ_DESC_F_WRITE,
);
} else {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_WRITE.into());
}
buff_len -= 1;
}
}
(None, Some(_)) => unreachable!("Indirect buffers mixed with direct buffers!"), (Some(_), None) => unreachable!("Indirect buffers mixed with direct buffers!"), }
}
(Some(send_buff), None) => {
match send_buff.get_ctrl_desc() {
Some(ctrl_desc) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
None => {
let mut buff_len = send_buff.as_slice().len();
for desc in send_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_NEXT.into());
} else {
ctrl.write_desc(desc, 0);
}
buff_len -= 1;
}
}
}
}
(None, Some(recv_buff)) => {
match recv_buff.get_ctrl_desc() {
Some(ctrl_desc) => {
ctrl.write_desc(ctrl_desc, DescrFlags::VIRTQ_DESC_F_INDIRECT.into());
}
None => {
let mut buff_len = recv_buff.as_slice().len();
for desc in recv_buff.as_slice() {
if buff_len > 1 {
ctrl.write_desc(
desc,
DescrFlags::VIRTQ_DESC_F_NEXT | DescrFlags::VIRTQ_DESC_F_WRITE,
);
} else {
ctrl.write_desc(desc, DescrFlags::VIRTQ_DESC_F_WRITE.into());
}
buff_len -= 1;
}
}
}
}
(None, None) => unreachable!("Empty Transfers are not allowed!"), }
fence(Ordering::SeqCst);
ctrl.make_avail(pinned.raw_addr());
fence(Ordering::SeqCst);
pinned.state = TransferState::Processing;
(pinned, ctrl.start, ctrl.wrap_at_init.0 as u8)
}
fn raw_addr(&self) -> usize {
self.ring.as_ptr() as usize
}
fn get_write_ctrler(&mut self) -> WriteCtrl<'_> {
WriteCtrl {
start: self.write_index,
position: self.write_index,
modulo: self.ring.len(),
wrap_at_init: self.drv_wc,
buff_id: 0,
desc_ring: self,
}
}
fn get_read_ctrler(&mut self) -> ReadCtrl<'_> {
ReadCtrl {
position: self.poll_index,
modulo: self.ring.len(),
desc_ring: self,
}
}
}
struct ReadCtrl<'a> {
position: usize,
modulo: usize,
desc_ring: &'a mut DescriptorRing,
}
impl<'a> ReadCtrl<'a> {
fn poll_next(&mut self) -> Option<*mut TransferToken> {
if self.desc_ring.ring[self.position].flags & WrapCount::flag_mask()
== self.desc_ring.dev_wc.as_flags_used()
{
let tkn = unsafe {
let buff_id = usize::from(self.desc_ring.ring[self.position].buff_id);
let raw_tkn = self.desc_ring.tkn_ref_ring[buff_id];
self.desc_ring.tkn_ref_ring[buff_id] = ptr::null_mut();
assert!(!raw_tkn.is_null());
&mut *raw_tkn
};
let (send_buff, recv_buff) = {
let BufferToken {
send_buff,
recv_buff,
..
} = tkn.buff_tkn.as_mut().unwrap();
(recv_buff.as_mut(), send_buff.as_mut())
};
let write_len = self.desc_ring.ring[self.position].len;
match (send_buff, recv_buff) {
(Some(send_buff), Some(recv_buff)) => {
if send_buff.is_indirect() {
self.update_indirect(Some(send_buff), Some((recv_buff, write_len)));
} else {
self.update_send(send_buff);
self.update_recv((recv_buff, write_len));
}
}
(Some(send_buff), None) => {
if send_buff.is_indirect() {
self.update_indirect(Some(send_buff), None);
} else {
self.update_send(send_buff);
}
}
(None, Some(recv_buff)) => {
if recv_buff.is_indirect() {
self.update_indirect(None, Some((recv_buff, write_len)));
} else {
self.update_recv((recv_buff, write_len));
}
}
(None, None) => unreachable!("Empty Transfers are not allowed..."),
}
Some(tkn as *mut TransferToken)
} else {
None
}
}
fn update_indirect(
&mut self,
send_buff: Option<&mut Buffer>,
recv_buff_spec: Option<(&mut Buffer, u32)>,
) {
match (send_buff, recv_buff_spec) {
(Some(send_buff), Some((recv_buff, mut write_len))) => {
let ctrl_desc = send_buff.get_ctrl_desc_mut().unwrap();
let desc_slice = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
let mut desc_iter = desc_slice.iter_mut();
for _desc in send_buff.as_mut_slice() {
desc_iter.next().unwrap();
}
recv_buff.restr_len(usize::try_from(write_len).unwrap());
for desc in recv_buff.as_mut_slice() {
let ring_desc = desc_iter.next().unwrap();
if write_len >= ring_desc.len {
write_len -= ring_desc.len;
} else {
ring_desc.len = write_len;
desc.len = write_len as usize;
write_len -= ring_desc.len;
assert_eq!(write_len, 0);
}
}
}
(Some(send_buff), None) => {
let ctrl_desc = send_buff.get_ctrl_desc_mut().unwrap();
let desc_slice = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
let mut desc_iter = desc_slice.iter();
for _desc in send_buff.as_mut_slice() {
desc_iter.next().unwrap();
}
}
(None, Some((recv_buff, mut write_len))) => {
let ctrl_desc = recv_buff.get_ctrl_desc_mut().unwrap();
let desc_slice = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
let mut desc_iter = desc_slice.iter_mut();
recv_buff.restr_len(usize::try_from(write_len).unwrap());
for desc in recv_buff.as_mut_slice() {
let ring_desc = desc_iter.next().unwrap();
if write_len >= ring_desc.len {
write_len -= ring_desc.len;
} else {
ring_desc.len = write_len;
desc.len = write_len as usize;
write_len -= ring_desc.len;
assert_eq!(write_len, 0);
}
}
}
(None, None) => unreachable!("Empty transfers are not allowed."),
}
self.reset_ring_pos();
self.incrmt();
}
fn reset_ring_pos(&mut self) {
self.desc_ring.ring[self.position].flags = self.desc_ring.dev_wc.as_flags_used();
}
fn update_recv(&mut self, recv_buff_spec: (&mut Buffer, u32)) {
let (recv_buff, write_len) = recv_buff_spec;
let mut write_len = usize::try_from(write_len).unwrap();
recv_buff.restr_len(write_len);
for desc in recv_buff.as_mut_slice() {
if write_len >= desc.len {
write_len -= desc.len;
} else {
desc.len = write_len;
write_len -= desc.len;
assert_eq!(write_len, 0);
}
self.reset_ring_pos();
self.incrmt();
}
}
fn update_send(&mut self, send_buff: &Buffer) {
for _desc in send_buff.as_slice() {
self.reset_ring_pos();
self.incrmt();
}
}
fn incrmt(&mut self) {
if self.desc_ring.poll_index + 1 == self.modulo {
self.desc_ring.dev_wc.wrap()
}
assert!(self.desc_ring.capacity <= self.desc_ring.ring.len());
self.desc_ring.capacity += 1;
self.desc_ring.poll_index = (self.desc_ring.poll_index + 1) % self.modulo;
self.position = self.desc_ring.poll_index;
}
}
struct WriteCtrl<'a> {
start: usize,
position: usize,
modulo: usize,
wrap_at_init: WrapCount,
buff_id: u16,
desc_ring: &'a mut DescriptorRing,
}
impl<'a> WriteCtrl<'a> {
fn incrmt(&mut self) {
assert!(self.desc_ring.capacity != 0);
self.desc_ring.capacity -= 1;
if self.position + 1 == self.modulo {
self.desc_ring.drv_wc.wrap();
}
self.desc_ring.write_index = (self.desc_ring.write_index + 1) % self.modulo;
self.position = (self.position + 1) % self.modulo;
}
fn write_desc(&mut self, mem_desc: &MemDescr, flags: u16) {
if self.start == self.position {
let desc_ref = &mut self.desc_ring.ring[self.position];
desc_ref.address = paging::virt_to_phys(VirtAddr::from(mem_desc.ptr as u64)).into();
desc_ref.len = mem_desc.len as u32;
desc_ref.buff_id = mem_desc.id.as_ref().unwrap().0;
desc_ref.flags =
flags & !(DescrFlags::VIRTQ_DESC_F_AVAIL) & !(DescrFlags::VIRTQ_DESC_F_USED);
self.buff_id = mem_desc.id.as_ref().unwrap().0;
self.incrmt();
} else {
let desc_ref = &mut self.desc_ring.ring[self.position];
desc_ref.address = paging::virt_to_phys(VirtAddr::from(mem_desc.ptr as u64)).into();
desc_ref.len = mem_desc.len as u32;
desc_ref.buff_id = self.buff_id;
desc_ref.flags =
(flags & !(DescrFlags::VIRTQ_DESC_F_AVAIL) & !(DescrFlags::VIRTQ_DESC_F_USED))
| self.desc_ring.drv_wc.as_flags_avail();
self.incrmt()
}
}
fn make_avail(&mut self, raw_tkn: *mut TransferToken) {
assert!(self.start != self.position);
assert!(self.buff_id != 0);
self.desc_ring.tkn_ref_ring[usize::try_from(self.buff_id).unwrap()] = raw_tkn;
fence(Ordering::SeqCst);
self.desc_ring.ring[self.start].flags |= self.wrap_at_init.as_flags_avail();
}
}
#[derive(Clone, Copy)]
#[repr(C, align(16))]
struct Descriptor {
address: u64,
len: u32,
buff_id: u16,
flags: u16,
}
impl Descriptor {
fn new(add: u64, len: u32, id: u16, flags: u16) -> Self {
Descriptor {
address: add,
len,
buff_id: id,
flags,
}
}
fn to_le_bytes(self) -> [u8; 16] {
let mut desc_bytes: [u8; 16] = [0; 16];
let mem_addr: [u8; 8] = self.address.to_le_bytes();
desc_bytes[0..8].copy_from_slice(&mem_addr);
let mem_len: [u8; 4] = self.len.to_le_bytes();
desc_bytes[8..12].copy_from_slice(&mem_len);
let id: [u8; 2] = self.buff_id.to_le_bytes();
desc_bytes[12..14].copy_from_slice(&id);
let flags: [u8; 2] = self.flags.to_le_bytes();
desc_bytes[14..16].copy_from_slice(&flags);
desc_bytes
}
}
#[repr(C, align(4))]
struct EventSuppr {
event: u16,
flags: u16,
}
struct DrvNotif {
f_notif_idx: bool,
raw: &'static mut EventSuppr,
}
struct DevNotif {
f_notif_idx: bool,
raw: &'static mut EventSuppr,
}
impl EventSuppr {
fn new() -> Self {
EventSuppr { event: 0, flags: 0 }
}
}
impl DrvNotif {
fn enable_notif(&mut self) {
self.raw.flags = 0;
}
fn disable_notif(&mut self) {
self.raw.flags = 0;
}
fn enable_specific(&mut self, at_offset: u16, at_wrap: u8) {
if self.f_notif_idx {
self.raw.flags |= 1 << 1;
self.raw.event = 0;
self.raw.event = at_offset;
self.raw.event |= (at_wrap as u16) << 15;
}
}
}
impl DevNotif {
pub fn enable_notif_specific(&mut self) {
self.f_notif_idx = true;
}
fn is_notif(&self) -> bool {
self.raw.flags & (1 << 0) == 0
}
fn is_notif_specfic(&self, next_off: usize, next_wrap: u8) -> bool {
if self.f_notif_idx {
if self.raw.flags & 1 << 1 == 2 {
let desc_event_off = self.raw.event & !(1 << 15);
let desc_event_wrap = (self.raw.event >> 15) as u8;
desc_event_off == next_off as u16 && desc_event_wrap == next_wrap
} else {
false
}
} else {
false
}
}
}
pub struct PackedVq {
descr_ring: RefCell<DescriptorRing>,
drv_event: RefCell<DrvNotif>,
dev_event: DevNotif,
notif_ctrl: NotifCtrl,
mem_pool: Rc<MemPool>,
size: VqSize,
index: VqIndex,
dropped: RefCell<Vec<Pinned<TransferToken>>>,
}
impl PackedVq {
pub fn enable_notifs(&self) {
self.drv_event.borrow_mut().enable_notif();
}
pub fn disable_notifs(&self) {
self.drv_event.borrow_mut().disable_notif();
}
pub fn clean_up(&self) {
if !self.dropped.borrow().is_empty() {
self.dropped
.borrow_mut()
.retain(|tkn| tkn.state != TransferState::Finished);
}
}
pub fn poll(&self) {
self.descr_ring.borrow_mut().poll();
}
pub fn dispatch_batch(&self, tkns: Vec<TransferToken>, notif: bool) -> Vec<Transfer> {
assert!(!tkns.is_empty());
let (pin_tkn_lst, next_off, next_wrap) = self.descr_ring.borrow_mut().push_batch(tkns);
if notif {
self.drv_event
.borrow_mut()
.enable_specific(next_off as u16, next_wrap);
}
if self.dev_event.is_notif() | self.dev_event.is_notif_specfic(next_off, next_wrap) {
let index = self.index.0.to_le_bytes();
let mut index = index.iter();
let det_notif_data: u16 = (next_off as u16) >> 1;
let flags = (det_notif_data | (u16::from(next_wrap) << 15)).to_le_bytes();
let mut flags = flags.iter();
let mut notif_data: [u8; 4] = [0, 0, 0, 0];
for (i, byte) in notif_data.iter_mut().enumerate() {
if i < 2 {
*byte = *index.next().unwrap();
} else {
*byte = *flags.next().unwrap();
}
}
self.notif_ctrl.notify_dev(¬if_data)
}
let mut transfer_lst = Vec::with_capacity(pin_tkn_lst.len());
for pinned in pin_tkn_lst {
transfer_lst.push(Transfer {
transfer_tkn: Some(pinned),
})
}
transfer_lst
}
pub fn dispatch_batch_await(
&self,
mut tkns: Vec<TransferToken>,
await_queue: Rc<RefCell<VecDeque<Transfer>>>,
notif: bool,
) {
assert!(!tkns.is_empty());
for tkn in tkns.iter_mut() {
tkn.await_queue = Some(Rc::clone(&await_queue));
}
let (pin_tkn_lst, next_off, next_wrap) = self.descr_ring.borrow_mut().push_batch(tkns);
if notif {
self.drv_event
.borrow_mut()
.enable_specific(next_off as u16, next_wrap);
}
if self.dev_event.is_notif() {
let index = self.index.0.to_le_bytes();
let mut index = index.iter();
let det_notif_data: u16 = (next_off as u16) >> 1;
let flags = (det_notif_data | (u16::from(next_wrap) << 15)).to_le_bytes();
let mut flags = flags.iter();
let mut notif_data: [u8; 4] = [0, 0, 0, 0];
for (i, byte) in notif_data.iter_mut().enumerate() {
if i < 2 {
*byte = *index.next().unwrap();
} else {
*byte = *flags.next().unwrap();
}
}
self.notif_ctrl.notify_dev(¬if_data)
}
for pinned in pin_tkn_lst {
pinned.into_raw();
}
}
pub fn dispatch(&self, tkn: TransferToken, notif: bool) -> Transfer {
let (pin_tkn, next_off, next_wrap) = self.descr_ring.borrow_mut().push(tkn);
if notif {
self.drv_event
.borrow_mut()
.enable_specific(next_off as u16, next_wrap);
}
if self.dev_event.is_notif() {
let index = self.index.0.to_le_bytes();
let mut index = index.iter();
let det_notif_data: u16 = (next_off as u16) >> 1;
let flags = (det_notif_data | (u16::from(next_wrap) << 15)).to_le_bytes();
let mut flags = flags.iter();
let mut notif_data: [u8; 4] = [0, 0, 0, 0];
for (i, byte) in notif_data.iter_mut().enumerate() {
if i < 2 {
*byte = *index.next().unwrap();
} else {
*byte = *flags.next().unwrap();
}
}
self.notif_ctrl.notify_dev(¬if_data)
}
Transfer {
transfer_tkn: Some(pin_tkn),
}
}
pub fn early_drop(&self, tkn: Pinned<TransferToken>) {
match tkn.state {
TransferState::Finished => (), TransferState::Ready => {
unreachable!("Early dropped transfers are not allowed to be state == Ready")
}
TransferState::Processing => {
self.dropped.borrow_mut().push(tkn);
}
}
}
pub fn index(&self) -> VqIndex {
self.index
}
pub fn new(
com_cfg: &mut ComCfg,
notif_cfg: &NotifCfg,
size: VqSize,
index: VqIndex,
feats: u64,
) -> Result<Self, VqPackedError> {
if feats & Features::VIRTIO_F_IN_ORDER == Features::VIRTIO_F_IN_ORDER {
info!("PackedVq has no support for VIRTIO_F_IN_ORDER. Aborting...");
return Err(VqPackedError::FeatNotSupported(
feats & Features::VIRTIO_F_IN_ORDER,
));
}
let mut vq_handler = match com_cfg.select_vq(index.into()) {
Some(handler) => handler,
None => return Err(VqPackedError::QueueNotExisting(index.into())),
};
let vq_size = if (size.0 == 0) | (size.0 > 32768) {
return Err(VqPackedError::SizeNotAllowed(size.0));
} else {
vq_handler.set_vq_size(size.0)
};
let descr_ring = RefCell::new(DescriptorRing::new(vq_size));
let _mem_len = core::mem::size_of::<EventSuppr>().align_up(BasePageSize::SIZE as usize);
let drv_event_ptr =
(crate::mm::allocate(_mem_len, true).0 as *const EventSuppr) as *mut EventSuppr;
let dev_event_ptr =
(crate::mm::allocate(_mem_len, true).0 as *const EventSuppr) as *mut EventSuppr;
vq_handler.set_ring_addr(paging::virt_to_phys(VirtAddr::from(
descr_ring.borrow().raw_addr() as u64,
)));
vq_handler.set_drv_ctrl_addr(paging::virt_to_phys(VirtAddr::from(drv_event_ptr as u64)));
vq_handler.set_dev_ctrl_addr(paging::virt_to_phys(VirtAddr::from(dev_event_ptr as u64)));
let drv_event: &'static mut EventSuppr = unsafe { &mut *(drv_event_ptr) };
let dev_event: &'static mut EventSuppr = unsafe { &mut *(dev_event_ptr) };
let drv_event = RefCell::new(DrvNotif {
f_notif_idx: false,
raw: drv_event,
});
let dev_event = DevNotif {
f_notif_idx: false,
raw: dev_event,
};
let mut notif_ctrl = NotifCtrl::new(
(notif_cfg.base()
+ usize::try_from(vq_handler.notif_off()).unwrap()
+ usize::try_from(notif_cfg.multiplier()).unwrap()) as *mut usize,
);
if feats & Features::VIRTIO_F_NOTIFICATION_DATA == Features::VIRTIO_F_NOTIFICATION_DATA {
notif_ctrl.enable_notif_data();
}
if feats & Features::VIRTIO_F_RING_EVENT_IDX == Features::VIRTIO_F_RING_EVENT_IDX {
drv_event.borrow_mut().f_notif_idx = true;
}
let mem_pool = Rc::new(MemPool::new(vq_size));
let dropped: RefCell<Vec<Pinned<TransferToken>>> = RefCell::new(Vec::new());
vq_handler.enable_queue();
info!("Created PackedVq: idx={}, size={}", index.0, vq_size);
Ok(PackedVq {
descr_ring,
drv_event,
dev_event,
notif_ctrl,
mem_pool,
size: VqSize::from(vq_size),
index,
dropped,
})
}
pub fn prep_transfer_from_raw<T: AsSliceU8 + 'static, K: AsSliceU8 + 'static>(
&self,
master: Rc<Virtq>,
send: Option<(*mut T, BuffSpec<'_>)>,
recv: Option<(*mut K, BuffSpec<'_>)>,
) -> Result<TransferToken, VirtqError> {
match (send, recv) {
(None, None) => Err(VirtqError::BufferNotSpecified),
(Some((send_data, send_spec)), None) => {
match send_spec {
BuffSpec::Single(size) => {
let data_slice = unsafe { (*send_data).as_slice_u8() };
if data_slice.len() != size.into() {
return Err(VirtqError::BufferSizeWrong(data_slice.len()));
}
let desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Single {
desc_lst: vec![desc].into_boxed_slice(),
len: data_slice.len(),
next_write: 0,
}),
recv_buff: None,
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
BuffSpec::Multiple(size_lst) => {
let data_slice = unsafe { (*send_data).as_slice_u8() };
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut index = 0usize;
for byte in size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match data_slice.get(index..end_index) {
Some(slice) => slice,
None => return Err(VirtqError::BufferSizeWrong(data_slice.len())),
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Multiple {
desc_lst: desc_lst.into_boxed_slice(),
len: data_slice.len(),
next_write: 0,
}),
recv_buff: None,
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
BuffSpec::Indirect(size_lst) => {
let data_slice = unsafe { (*send_data).as_slice_u8() };
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut index = 0usize;
for byte in size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match data_slice.get(index..end_index) {
Some(slice) => slice,
None => return Err(VirtqError::BufferSizeWrong(data_slice.len())),
};
desc_lst.push(
self.mem_pool
.pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice),
);
index += usize::from(*byte);
}
let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) {
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Indirect {
desc_lst: desc_lst.into_boxed_slice(),
ctrl_desc,
len: data_slice.len(),
next_write: 0,
}),
recv_buff: None,
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
}
}
(None, Some((recv_data, recv_spec))) => {
match recv_spec {
BuffSpec::Single(size) => {
let data_slice = unsafe { (*recv_data).as_slice_u8() };
if data_slice.len() != size.into() {
return Err(VirtqError::BufferSizeWrong(data_slice.len()));
}
let desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: None,
recv_buff: Some(Buffer::Single {
desc_lst: vec![desc].into_boxed_slice(),
len: data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
BuffSpec::Multiple(size_lst) => {
let data_slice = unsafe { (*recv_data).as_slice_u8() };
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut index = 0usize;
for byte in size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match data_slice.get(index..end_index) {
Some(slice) => slice,
None => return Err(VirtqError::BufferSizeWrong(data_slice.len())),
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: None,
recv_buff: Some(Buffer::Multiple {
desc_lst: desc_lst.into_boxed_slice(),
len: data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
BuffSpec::Indirect(size_lst) => {
let data_slice = unsafe { (*recv_data).as_slice_u8() };
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut index = 0usize;
for byte in size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match data_slice.get(index..end_index) {
Some(slice) => slice,
None => return Err(VirtqError::BufferSizeWrong(data_slice.len())),
};
desc_lst.push(
self.mem_pool
.pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice),
);
index += usize::from(*byte);
}
let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) {
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: None,
recv_buff: Some(Buffer::Indirect {
desc_lst: desc_lst.into_boxed_slice(),
ctrl_desc,
len: data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
}
}
(Some((send_data, send_spec)), Some((recv_data, recv_spec))) => {
match (send_spec, recv_spec) {
(BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => {
let send_data_slice = unsafe { (*send_data).as_slice_u8() };
if send_data_slice.len() != send_size.into() {
return Err(VirtqError::BufferSizeWrong(send_data_slice.len()));
}
let send_desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), send_data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let recv_data_slice = unsafe { (*recv_data).as_slice_u8() };
if recv_data_slice.len() != recv_size.into() {
return Err(VirtqError::BufferSizeWrong(recv_data_slice.len()));
}
let recv_desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), recv_data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Single {
desc_lst: vec![send_desc].into_boxed_slice(),
len: send_data_slice.len(),
next_write: 0,
}),
recv_buff: Some(Buffer::Single {
desc_lst: vec![recv_desc].into_boxed_slice(),
len: recv_data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
(BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => {
let send_data_slice = unsafe { (*send_data).as_slice_u8() };
if send_data_slice.len() != send_size.into() {
return Err(VirtqError::BufferSizeWrong(send_data_slice.len()));
}
let send_desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), send_data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let recv_data_slice = unsafe { (*recv_data).as_slice_u8() };
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut index = 0usize;
for byte in recv_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match recv_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(recv_data_slice.len()))
}
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => recv_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Single {
desc_lst: vec![send_desc].into_boxed_slice(),
len: send_data_slice.len(),
next_write: 0,
}),
recv_buff: Some(Buffer::Multiple {
desc_lst: recv_desc_lst.into_boxed_slice(),
len: recv_data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
(BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => {
let send_data_slice = unsafe { (*send_data).as_slice_u8() };
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut index = 0usize;
for byte in send_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match send_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(send_data_slice.len()))
}
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => send_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
let recv_data_slice = unsafe { (*recv_data).as_slice_u8() };
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut index = 0usize;
for byte in recv_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match recv_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(recv_data_slice.len()))
}
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => recv_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Multiple {
desc_lst: send_desc_lst.into_boxed_slice(),
len: send_data_slice.len(),
next_write: 0,
}),
recv_buff: Some(Buffer::Multiple {
desc_lst: recv_desc_lst.into_boxed_slice(),
len: recv_data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
(BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => {
let send_data_slice = unsafe { (*send_data).as_slice_u8() };
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut index = 0usize;
for byte in send_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match send_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(send_data_slice.len()))
}
};
match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), next_slice)
{
Ok(desc) => send_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
};
index += usize::from(*byte);
}
let recv_data_slice = unsafe { (*recv_data).as_slice_u8() };
if recv_data_slice.len() != recv_size.into() {
return Err(VirtqError::BufferSizeWrong(recv_data_slice.len()));
}
let recv_desc = match self
.mem_pool
.pull_from_raw(Rc::clone(&self.mem_pool), recv_data_slice)
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
send_buff: Some(Buffer::Multiple {
desc_lst: send_desc_lst.into_boxed_slice(),
len: send_data_slice.len(),
next_write: 0,
}),
recv_buff: Some(Buffer::Single {
desc_lst: vec![recv_desc].into_boxed_slice(),
len: recv_data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
(BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => {
let send_data_slice = unsafe { (*send_data).as_slice_u8() };
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut index = 0usize;
for byte in send_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match send_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(send_data_slice.len()))
}
};
send_desc_lst.push(
self.mem_pool
.pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice),
);
index += usize::from(*byte);
}
let recv_data_slice = unsafe { (*recv_data).as_slice_u8() };
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut index = 0usize;
for byte in recv_size_lst {
let end_index = index + usize::from(*byte);
let next_slice = match recv_data_slice.get(index..end_index) {
Some(slice) => slice,
None => {
return Err(VirtqError::BufferSizeWrong(recv_data_slice.len()))
}
};
recv_desc_lst.push(
self.mem_pool
.pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice),
);
index += usize::from(*byte);
}
let ctrl_desc = match self
.create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst))
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(BufferToken {
recv_buff: Some(Buffer::Indirect {
desc_lst: recv_desc_lst.into_boxed_slice(),
ctrl_desc: ctrl_desc.no_dealloc_clone(),
len: recv_data_slice.len(),
next_write: 0,
}),
send_buff: Some(Buffer::Indirect {
desc_lst: send_desc_lst.into_boxed_slice(),
ctrl_desc,
len: send_data_slice.len(),
next_write: 0,
}),
vq: master,
ret_send: false,
ret_recv: false,
reusable: false,
}),
await_queue: None,
})
}
(BuffSpec::Indirect(_), BuffSpec::Single(_))
| (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect),
(BuffSpec::Single(_), BuffSpec::Indirect(_))
| (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect),
}
}
}
}
pub fn prep_buffer(
&self,
master: Rc<Virtq>,
send: Option<BuffSpec<'_>>,
recv: Option<BuffSpec<'_>>,
) -> Result<BufferToken, VirtqError> {
match (send, recv) {
(None, None) => Err(VirtqError::BufferNotSpecified),
(Some(spec), None) => {
match spec {
BuffSpec::Single(size) => {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) {
Ok(desc) => {
let buffer = Buffer::Single {
desc_lst: vec![desc].into_boxed_slice(),
len: size.into(),
next_write: 0,
};
Ok(BufferToken {
send_buff: Some(buffer),
recv_buff: None,
vq: master,
ret_send: true,
ret_recv: false,
reusable: true,
})
}
Err(vq_err) => Err(vq_err),
}
}
BuffSpec::Multiple(size_lst) => {
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut len = 0usize;
for size in size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
len += usize::from(*size);
}
let buffer = Buffer::Multiple {
desc_lst: desc_lst.into_boxed_slice(),
len,
next_write: 0,
};
Ok(BufferToken {
send_buff: Some(buffer),
recv_buff: None,
vq: master,
ret_send: true,
ret_recv: false,
reusable: true,
})
}
BuffSpec::Indirect(size_lst) => {
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut len = 0usize;
for size in size_lst {
desc_lst.push(
self.mem_pool
.pull_untracked(Rc::clone(&self.mem_pool), *size),
);
len += usize::from(*size);
}
let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) {
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let buffer = Buffer::Indirect {
desc_lst: desc_lst.into_boxed_slice(),
ctrl_desc,
len,
next_write: 0,
};
Ok(BufferToken {
send_buff: Some(buffer),
recv_buff: None,
vq: master,
ret_send: true,
ret_recv: false,
reusable: true,
})
}
}
}
(None, Some(spec)) => {
match spec {
BuffSpec::Single(size) => {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) {
Ok(desc) => {
let buffer = Buffer::Single {
desc_lst: vec![desc].into_boxed_slice(),
len: size.into(),
next_write: 0,
};
Ok(BufferToken {
send_buff: None,
recv_buff: Some(buffer),
vq: master,
ret_send: false,
ret_recv: true,
reusable: true,
})
}
Err(vq_err) => Err(vq_err),
}
}
BuffSpec::Multiple(size_lst) => {
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut len = 0usize;
for size in size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
len += usize::from(*size);
}
let buffer = Buffer::Multiple {
desc_lst: desc_lst.into_boxed_slice(),
len,
next_write: 0,
};
Ok(BufferToken {
send_buff: None,
recv_buff: Some(buffer),
vq: master,
ret_send: false,
ret_recv: true,
reusable: true,
})
}
BuffSpec::Indirect(size_lst) => {
let mut desc_lst: Vec<MemDescr> = Vec::with_capacity(size_lst.len());
let mut len = 0usize;
for size in size_lst {
desc_lst.push(
self.mem_pool
.pull_untracked(Rc::clone(&self.mem_pool), *size),
);
len += usize::from(*size);
}
let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) {
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let buffer = Buffer::Indirect {
desc_lst: desc_lst.into_boxed_slice(),
ctrl_desc,
len,
next_write: 0,
};
Ok(BufferToken {
send_buff: None,
recv_buff: Some(buffer),
vq: master,
ret_send: false,
ret_recv: true,
reusable: true,
})
}
}
}
(Some(send_spec), Some(recv_spec)) => {
match (send_spec, recv_spec) {
(BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => {
let send_buff =
match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) {
Ok(send_desc) => Some(Buffer::Single {
desc_lst: vec![send_desc].into_boxed_slice(),
len: send_size.into(),
next_write: 0,
}),
Err(vq_err) => return Err(vq_err),
};
let recv_buff =
match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) {
Ok(recv_desc) => Some(Buffer::Single {
desc_lst: vec![recv_desc].into_boxed_slice(),
len: recv_size.into(),
next_write: 0,
}),
Err(vq_err) => return Err(vq_err),
};
Ok(BufferToken {
send_buff,
recv_buff,
vq: master,
ret_send: true,
ret_recv: true,
reusable: true,
})
}
(BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => {
let send_buff =
match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) {
Ok(send_desc) => Some(Buffer::Single {
desc_lst: vec![send_desc].into_boxed_slice(),
len: send_size.into(),
next_write: 0,
}),
Err(vq_err) => return Err(vq_err),
};
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut recv_len = 0usize;
for size in recv_size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => recv_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
recv_len += usize::from(*size);
}
let recv_buff = Some(Buffer::Multiple {
desc_lst: recv_desc_lst.into_boxed_slice(),
len: recv_len,
next_write: 0,
});
Ok(BufferToken {
send_buff,
recv_buff,
vq: master,
ret_send: true,
ret_recv: true,
reusable: true,
})
}
(BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => {
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut send_len = 0usize;
for size in send_size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => send_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
send_len += usize::from(*size);
}
let send_buff = Some(Buffer::Multiple {
desc_lst: send_desc_lst.into_boxed_slice(),
len: send_len,
next_write: 0,
});
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut recv_len = 0usize;
for size in recv_size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => recv_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
recv_len += usize::from(*size);
}
let recv_buff = Some(Buffer::Multiple {
desc_lst: recv_desc_lst.into_boxed_slice(),
len: recv_len,
next_write: 0,
});
Ok(BufferToken {
send_buff,
recv_buff,
vq: master,
ret_send: true,
ret_recv: true,
reusable: true,
})
}
(BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => {
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut send_len = 0usize;
for size in send_size_lst {
match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) {
Ok(desc) => send_desc_lst.push(desc),
Err(vq_err) => return Err(vq_err),
}
send_len += usize::from(*size);
}
let send_buff = Some(Buffer::Multiple {
desc_lst: send_desc_lst.into_boxed_slice(),
len: send_len,
next_write: 0,
});
let recv_buff =
match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) {
Ok(recv_desc) => Some(Buffer::Single {
desc_lst: vec![recv_desc].into_boxed_slice(),
len: recv_size.into(),
next_write: 0,
}),
Err(vq_err) => return Err(vq_err),
};
Ok(BufferToken {
send_buff,
recv_buff,
vq: master,
ret_send: true,
ret_recv: true,
reusable: true,
})
}
(BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => {
let mut send_desc_lst: Vec<MemDescr> =
Vec::with_capacity(send_size_lst.len());
let mut send_len = 0usize;
for size in send_size_lst {
send_desc_lst.push(
self.mem_pool
.pull_untracked(Rc::clone(&self.mem_pool), *size),
);
send_len += usize::from(*size);
}
let mut recv_desc_lst: Vec<MemDescr> =
Vec::with_capacity(recv_size_lst.len());
let mut recv_len = 0usize;
for size in recv_size_lst {
recv_desc_lst.push(
self.mem_pool
.pull_untracked(Rc::clone(&self.mem_pool), *size),
);
recv_len += usize::from(*size);
}
let ctrl_desc = match self
.create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst))
{
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let recv_buff = Some(Buffer::Indirect {
desc_lst: recv_desc_lst.into_boxed_slice(),
ctrl_desc: ctrl_desc.no_dealloc_clone(),
len: recv_len,
next_write: 0,
});
let send_buff = Some(Buffer::Indirect {
desc_lst: send_desc_lst.into_boxed_slice(),
ctrl_desc,
len: send_len,
next_write: 0,
});
Ok(BufferToken {
send_buff,
recv_buff,
vq: master,
ret_send: true,
ret_recv: true,
reusable: true,
})
}
(BuffSpec::Indirect(_), BuffSpec::Single(_))
| (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect),
(BuffSpec::Single(_), BuffSpec::Indirect(_))
| (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect),
}
}
}
}
pub fn size(&self) -> VqSize {
self.size
}
}
impl PackedVq {
fn create_indirect_ctrl(
&self,
send: Option<&Vec<MemDescr>>,
recv: Option<&Vec<MemDescr>>,
) -> Result<MemDescr, VirtqError> {
let len: usize = match (send, recv) {
(None, None) => return Err(VirtqError::BufferNotSpecified),
(None, Some(recv_desc_lst)) => recv_desc_lst.len(),
(Some(send_desc_lst), None) => send_desc_lst.len(),
(Some(send_desc_lst), Some(recv_desc_lst)) => send_desc_lst.len() + recv_desc_lst.len(),
};
let sz_indrct_lst = match Bytes::new(core::mem::size_of::<Descriptor>() * len) {
Some(bytes) => bytes,
None => return Err(VirtqError::BufferToLarge),
};
let ctrl_desc = match self.mem_pool.pull(Rc::clone(&self.mem_pool), sz_indrct_lst) {
Ok(desc) => desc,
Err(vq_err) => return Err(vq_err),
};
let mut crtl_desc_iter = 0usize;
let desc_slice = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(ctrl_desc.ptr as *mut Descriptor, ctrl_desc.len / size)
};
match (send, recv) {
(None, None) => Err(VirtqError::BufferNotSpecified),
(None, Some(recv_desc_lst)) => {
for desc in recv_desc_lst {
desc_slice[crtl_desc_iter] = Descriptor::new(
paging::virt_to_phys(VirtAddr::from(desc.ptr as u64)).into(),
desc.len as u32,
0,
DescrFlags::VIRTQ_DESC_F_WRITE.into(),
);
crtl_desc_iter += 1;
}
Ok(ctrl_desc)
}
(Some(send_desc_lst), None) => {
for desc in send_desc_lst {
desc_slice[crtl_desc_iter] = Descriptor::new(
paging::virt_to_phys(VirtAddr::from(desc.ptr as u64)).into(),
desc.len as u32,
0,
0,
);
crtl_desc_iter += 1;
}
Ok(ctrl_desc)
}
(Some(send_desc_lst), Some(recv_desc_lst)) => {
for desc in send_desc_lst {
desc_slice[crtl_desc_iter] = Descriptor::new(
paging::virt_to_phys(VirtAddr::from(desc.ptr as u64)).into(),
desc.len as u32,
0,
0,
);
crtl_desc_iter += 1;
}
for desc in recv_desc_lst {
desc_slice[crtl_desc_iter] = Descriptor::new(
paging::virt_to_phys(VirtAddr::from(desc.ptr as u64)).into(),
desc.len as u32,
0,
DescrFlags::VIRTQ_DESC_F_WRITE.into(),
);
crtl_desc_iter += 1;
}
Ok(ctrl_desc)
}
}
}
}
pub mod error {
pub enum VqPackedError {
General,
SizeNotAllowed(u16),
QueueNotExisting(u16),
FeatNotSupported(u64),
}
}