#![allow(dead_code)]
#![allow(clippy::type_complexity)]
pub mod packed;
pub mod split;
use alloc::boxed::Box;
use alloc::collections::VecDeque;
use alloc::rc::Rc;
use alloc::vec::Vec;
use core::cell::RefCell;
use core::ops::{BitAnd, Deref, DerefMut};
use align_address::Align;
use zerocopy::AsBytes;
use self::error::{BufferError, VirtqError};
use self::packed::PackedVq;
use self::split::SplitVq;
#[cfg(not(feature = "pci"))]
use super::transport::mmio::{ComCfg, NotifCfg};
#[cfg(feature = "pci")]
use super::transport::pci::{ComCfg, NotifCfg};
use crate::arch::mm::paging::{BasePageSize, PageSize};
use crate::arch::mm::{paging, VirtAddr};
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
pub struct VqIndex(u16);
impl From<u16> for VqIndex {
fn from(val: u16) -> Self {
VqIndex(val)
}
}
impl From<VqIndex> for u16 {
fn from(i: VqIndex) -> Self {
i.0
}
}
impl From<u32> for VqIndex {
fn from(val: u32) -> Self {
if val > u16::MAX as u32 {
VqIndex(u16::MAX)
} else {
VqIndex(val as u16)
}
}
}
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
pub struct VqSize(u16);
impl From<u16> for VqSize {
fn from(val: u16) -> Self {
VqSize(val)
}
}
impl From<u32> for VqSize {
fn from(val: u32) -> Self {
if val > u16::MAX as u32 {
VqSize(u16::MAX)
} else {
VqSize(val as u16)
}
}
}
impl From<VqSize> for u16 {
fn from(val: VqSize) -> Self {
val.0
}
}
pub enum VqType {
Packed,
Split,
}
#[repr(C, align(16))]
struct Descriptor {
address: u64,
len: u32,
buff_id: u16,
flags: u16,
}
pub enum Virtq {
Packed(PackedVq),
Split(SplitVq),
}
impl Virtq {
fn dispatch(&self, tkn: TransferToken, notif: bool) -> Transfer {
match self {
Virtq::Packed(vq) => vq.dispatch(tkn, notif),
Virtq::Split(vq) => vq.dispatch(tkn, notif),
}
}
}
impl Virtq {
pub fn check_bounds<T: AsSliceU8>(data: &T) -> bool {
let slice = data.as_slice_u8();
let start_virt = (&slice[0] as *const u8) as usize;
let end_virt = (&slice[slice.len() - 1] as *const u8) as usize;
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
end_phy == end_phy_calc
}
pub fn check_bounds_slice(slice: &[u8]) -> bool {
let start_virt = (&slice[0] as *const u8) as usize;
let end_virt = (&slice[slice.len() - 1] as *const u8) as usize;
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
end_phy == end_phy_calc
}
pub fn free_raw(ptr: *mut u8, len: usize) {
crate::mm::deallocate(VirtAddr::from(ptr as usize), len);
}
}
impl Virtq {
pub fn enable_notifs(&self) {
match self {
Virtq::Packed(vq) => vq.enable_notifs(),
Virtq::Split(vq) => vq.enable_notifs(),
}
}
pub fn disable_notifs(&self) {
match self {
Virtq::Packed(vq) => vq.disable_notifs(),
Virtq::Split(vq) => vq.disable_notifs(),
}
}
pub fn poll(&self) {
match self {
Virtq::Packed(vq) => vq.poll(),
Virtq::Split(vq) => vq.poll(),
}
}
pub fn clean_up(&self) {
match self {
Virtq::Packed(vq) => vq.clean_up(),
Virtq::Split(vq) => vq.clean_up(),
}
}
pub fn dispatch_batch(tkns: Vec<TransferToken>, notif: bool) -> Vec<Transfer> {
let mut used_vqs: Vec<(Rc<Virtq>, Vec<TransferToken>)> = Vec::new();
for tkn in tkns {
let index = tkn.get_vq().index();
let mut used = false;
let mut index_used = 0usize;
for (pos, (vq, _)) in used_vqs.iter_mut().enumerate() {
if index == vq.index() {
index_used = pos;
used = true;
break;
}
}
if used {
let (_, tkn_lst) = &mut used_vqs[index_used];
tkn_lst.push(tkn);
} else {
let mut new_tkn_lst = Vec::new();
let vq = tkn.get_vq();
new_tkn_lst.push(tkn);
used_vqs.push((vq, new_tkn_lst))
}
}
let mut transfer_lst = Vec::new();
for (vq_ref, tkn_lst) in used_vqs {
match vq_ref.as_ref() {
Virtq::Packed(vq) => {
transfer_lst.append(vq.dispatch_batch(tkn_lst, notif).as_mut())
}
Virtq::Split(vq) => transfer_lst.append(vq.dispatch_batch(tkn_lst, notif).as_mut()),
}
}
transfer_lst
}
pub fn dispatch_batch_await(
tkns: Vec<TransferToken>,
await_queue: Rc<RefCell<VecDeque<Transfer>>>,
notif: bool,
) {
let mut used_vqs: Vec<(Rc<Virtq>, Vec<TransferToken>)> = Vec::new();
for tkn in tkns {
let index = tkn.get_vq().index();
let mut used = false;
let mut index_used = 0usize;
for (pos, (vq, _)) in used_vqs.iter_mut().enumerate() {
if index == vq.index() {
index_used = pos;
used = true;
break;
}
}
if used {
let (_, tkn_lst) = &mut used_vqs[index_used];
tkn_lst.push(tkn);
} else {
let mut new_tkn_lst = Vec::new();
let vq = tkn.get_vq();
new_tkn_lst.push(tkn);
used_vqs.push((vq, new_tkn_lst))
}
}
for (vq, tkn_lst) in used_vqs {
match vq.as_ref() {
Virtq::Packed(vq) => {
vq.dispatch_batch_await(tkn_lst, Rc::clone(&await_queue), notif)
}
Virtq::Split(vq) => {
vq.dispatch_batch_await(tkn_lst, Rc::clone(&await_queue), notif)
}
}
}
}
pub fn new(
com_cfg: &mut ComCfg,
notif_cfg: &NotifCfg,
size: VqSize,
vq_type: VqType,
index: VqIndex,
feats: u64,
) -> Self {
match vq_type {
VqType::Packed => match PackedVq::new(com_cfg, notif_cfg, size, index, feats) {
Ok(packed_vq) => Virtq::Packed(packed_vq),
Err(_vq_error) => panic!("Currently panics if queue fails to be created"),
},
VqType::Split => match SplitVq::new(com_cfg, notif_cfg, size, index, feats) {
Ok(split_vq) => Virtq::Split(split_vq),
Err(_vq_error) => panic!("Currently panics if queue fails to be created"),
},
}
}
pub fn size(&self) -> VqSize {
match self {
Virtq::Packed(vq) => vq.size(),
Virtq::Split(vq) => vq.size(),
}
}
pub fn index(&self) -> VqIndex {
match self {
Virtq::Packed(vq) => vq.index(),
Virtq::Split(vq) => vq.index(),
}
}
pub fn prep_transfer_from_raw<T: AsSliceU8 + 'static, K: AsSliceU8 + 'static>(
&self,
rc_self: Rc<Virtq>,
send: Option<(*mut T, BuffSpec<'_>)>,
recv: Option<(*mut K, BuffSpec<'_>)>,
) -> Result<TransferToken, VirtqError> {
match self {
Virtq::Packed(vq) => vq.prep_transfer_from_raw(rc_self, send, recv),
Virtq::Split(vq) => vq.prep_transfer_from_raw(rc_self, send, recv),
}
}
pub fn prep_buffer(
&self,
rc_self: Rc<Virtq>,
send: Option<BuffSpec<'_>>,
recv: Option<BuffSpec<'_>>,
) -> Result<BufferToken, VirtqError> {
match self {
Virtq::Packed(vq) => vq.prep_buffer(rc_self, send, recv),
Virtq::Split(vq) => vq.prep_buffer(rc_self, send, recv),
}
}
fn early_drop(&self, transfer_tk: Pinned<TransferToken>) {
match self {
Virtq::Packed(vq) => vq.early_drop(transfer_tk),
Virtq::Split(vq) => vq.early_drop(transfer_tk),
}
}
}
pub trait AsSliceU8 {
fn len(&self) -> usize {
core::mem::size_of_val(self)
}
fn as_slice_u8(&self) -> &[u8] {
unsafe { core::slice::from_raw_parts((self as *const _) as *const u8, self.len()) }
}
fn as_slice_u8_mut(&mut self) -> &mut [u8] {
unsafe { core::slice::from_raw_parts_mut((self as *const _) as *mut u8, self.len()) }
}
}
pub struct Transfer {
transfer_tkn: Option<Pinned<TransferToken>>,
}
impl Drop for Transfer {
fn drop(&mut self) {
if let Some(tkn) = self.transfer_tkn.take() {
let vq_ref = Rc::clone(&tkn.buff_tkn.as_ref().unwrap().vq);
vq_ref.early_drop(tkn)
}
}
}
impl Transfer {
pub fn poll(&self) -> bool {
match self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => true,
TransferState::Ready => unreachable!("Transfers owned by other than queue should have Tokens, of Finished or Processing State!"),
TransferState::Processing => false,
}
}
pub fn as_slices(&self) -> Result<(Option<Vec<&[u8]>>, Option<Vec<&[u8]>>), VirtqError> {
match &self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
let send_data = match &self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.send_buff
{
Some(buff) => {
let mut arr = Vec::with_capacity(buff.as_slice().len());
for desc in buff.as_slice() {
arr.push(desc.deref())
}
Some(arr)
}
None => None,
};
let recv_data = match &self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.recv_buff
{
Some(buff) => {
let mut arr = Vec::with_capacity(buff.as_slice().len());
for desc in buff.as_slice() {
arr.push(desc.deref())
}
Some(arr)
}
None => None,
};
Ok((send_data, recv_data))
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(None)),
TransferState::Ready => unreachable!(
"Transfers not owned by a queue Must have state Finished or Processing!"
),
}
}
pub fn as_slices_mut(
&mut self,
) -> Result<(Option<Vec<&mut [u8]>>, Option<Vec<&mut [u8]>>), VirtqError> {
match &self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
let (send_buff, recv_buff) = {
let BufferToken {
send_buff,
recv_buff,
..
} = self
.transfer_tkn
.as_mut()
.unwrap()
.buff_tkn
.as_mut()
.unwrap();
(send_buff.as_mut(), recv_buff.as_mut())
};
let send_data = match send_buff {
Some(buff) => {
let mut arr = Vec::with_capacity(buff.as_slice().len());
for desc in buff.as_mut_slice() {
arr.push(desc.deref_mut())
}
Some(arr)
}
None => None,
};
let recv_data = match recv_buff {
Some(buff) => {
let mut arr = Vec::with_capacity(buff.as_slice().len());
for desc in buff.as_mut_slice() {
arr.push(desc.deref_mut())
}
Some(arr)
}
None => None,
};
Ok((send_data, recv_data))
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(None)),
TransferState::Ready => unreachable!(
"Transfers not owned by a queue Must have state Finished or Processing!"
),
}
}
pub fn ret_scat_cpy(
&self,
) -> Result<(Option<Vec<Box<[u8]>>>, Option<Vec<Box<[u8]>>>), VirtqError> {
match &self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
let send_data = self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.send_buff
.as_ref()
.map(Buffer::scat_cpy);
let recv_data = self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.recv_buff
.as_ref()
.map(Buffer::scat_cpy);
Ok((send_data, recv_data))
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(None)),
TransferState::Ready => unreachable!(
"Transfers not owned by a queue Must have state Finished or Processing!"
),
}
}
pub fn ret_cpy(&self) -> Result<(Option<Box<[u8]>>, Option<Box<[u8]>>), VirtqError> {
match &self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
let send_data = self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.send_buff
.as_ref()
.map(Buffer::cpy);
let recv_data = self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.recv_buff
.as_ref()
.map(Buffer::cpy);
Ok((send_data, recv_data))
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(None)),
TransferState::Ready => unreachable!(
"Transfers not owned by a queue Must have state Finished or Processing!"
),
}
}
pub fn into_raw(
mut self,
) -> Result<(Option<Vec<(*mut u8, usize)>>, Option<Vec<(*mut u8, usize)>>), VirtqError> {
let state = self.transfer_tkn.as_ref().unwrap().state;
match state {
TransferState::Finished => {
let mut transfer_tkn = self.transfer_tkn.take().unwrap().unpin();
let mut buffer_tkn = transfer_tkn.buff_tkn.take().unwrap();
let send_data = if buffer_tkn.ret_send {
match buffer_tkn.send_buff {
Some(buff) => {
buffer_tkn.ret_send = false;
Some(buff.into_raw())
}
None => None,
}
} else {
return Err(VirtqError::NoReuseBuffer);
};
let recv_data = if buffer_tkn.ret_recv {
match buffer_tkn.recv_buff {
Some(buff) => {
buffer_tkn.ret_recv = false;
Some(buff.into_raw())
}
None => None,
}
} else {
return Err(VirtqError::NoReuseBuffer);
};
buffer_tkn.reusable = false;
Ok((send_data, recv_data))
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(Some(self))),
TransferState::Ready => unreachable!(
"Transfers not owned by a queue Must have state Finished or Processing!"
),
}
}
pub fn close(mut self) {
match self.transfer_tkn.as_ref().unwrap().state {
TransferState::Processing => {
let vq = Rc::clone(&self.transfer_tkn.as_ref().unwrap().get_vq());
let transfer_tkn = self.transfer_tkn.take().unwrap();
vq.early_drop(transfer_tkn);
}
TransferState::Ready => {
unreachable!("Transfers MUST have tokens of states Processing or Finished.")
}
TransferState::Finished => (), }
}
pub fn reuse(mut self) -> Result<BufferToken, VirtqError> {
match self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
if self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.reusable
{
let tkn = self
.transfer_tkn
.take()
.unwrap()
.unpin()
.buff_tkn
.take()
.unwrap();
Ok(tkn.reset())
} else {
Err(VirtqError::NoReuseBuffer)
}
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(Some(self))),
TransferState::Ready => unreachable!(
"Transfers coming from outside the queue must be Processing or Finished"
),
}
}
pub fn reuse_reset(mut self) -> Result<BufferToken, VirtqError> {
match self.transfer_tkn.as_ref().unwrap().state {
TransferState::Finished => {
if self
.transfer_tkn
.as_ref()
.unwrap()
.buff_tkn
.as_ref()
.unwrap()
.reusable
{
let tkn = self
.transfer_tkn
.take()
.unwrap()
.unpin()
.buff_tkn
.take()
.unwrap();
Ok(tkn.reset_purge())
} else {
Err(VirtqError::NoReuseBuffer)
}
}
TransferState::Processing => Err(VirtqError::OngoingTransfer(Some(self))),
TransferState::Ready => unreachable!(
"Transfers coming from outside the queue must be Processing or Finished"
),
}
}
}
#[derive(PartialEq, Copy, Clone, Debug)]
enum TransferState {
Finished,
Processing,
Ready,
}
pub struct TransferToken {
state: TransferState,
buff_tkn: Option<BufferToken>,
await_queue: Option<Rc<RefCell<VecDeque<Transfer>>>>,
}
impl TransferToken {
pub fn get_vq(&self) -> Rc<Virtq> {
Rc::clone(&self.buff_tkn.as_ref().unwrap().vq)
}
pub fn dispatch_await(mut self, await_queue: Rc<RefCell<VecDeque<Transfer>>>, notif: bool) {
self.await_queue = Some(Rc::clone(&await_queue));
self.get_vq()
.dispatch(self, notif)
.transfer_tkn
.take()
.unwrap()
.into_raw();
}
pub fn dispatch(self, notif: bool) -> Transfer {
self.get_vq().dispatch(self, notif)
}
pub fn dispatch_blocking(self) -> Result<Transfer, VirtqError> {
let vq = self.get_vq();
let transfer = self.get_vq().dispatch(self, false);
vq.disable_notifs();
while transfer.transfer_tkn.as_ref().unwrap().state != TransferState::Finished {
vq.poll()
}
vq.enable_notifs();
Ok(transfer)
}
}
pub struct BufferToken {
send_buff: Option<Buffer>,
recv_buff: Option<Buffer>,
vq: Rc<Virtq>,
ret_send: bool,
ret_recv: bool,
reusable: bool,
}
impl BufferToken {
fn num_descr(&self) -> usize {
let mut len = 0usize;
if let Some(buffer) = &self.recv_buff {
len += buffer.num_descr();
}
if let Some(buffer) = &self.send_buff {
len += buffer.num_descr();
}
len
}
fn num_consuming_descr(&self) -> usize {
let mut len = 0usize;
if let Some(buffer) = &self.send_buff {
match buffer.get_ctrl_desc() {
Some(_) => len += 1,
None => len += buffer.num_descr(),
}
}
if let Some(buffer) = &self.recv_buff {
match buffer.get_ctrl_desc() {
Some(_) => len += 1,
None => len += buffer.num_descr(),
}
}
len
}
fn reset_purge(mut self) -> Self {
let mut ctrl_desc_cnt = 0usize;
if let Some(buff) = self.send_buff.as_mut() {
buff.reset_write();
let mut init_buff_len = 0usize;
match buff.get_ctrl_desc_mut() {
Some(ctrl_desc) => {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
ind_desc_lst[ctrl_desc_cnt].len = desc._init_len as u32;
ctrl_desc_cnt += 1;
init_buff_len += desc._init_len;
for byte in desc.deref_mut() {
*byte = 0;
}
}
}
None => {
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
init_buff_len += desc._init_len;
for byte in desc.deref_mut() {
*byte = 0;
}
}
}
}
buff.reset_len(init_buff_len);
}
if let Some(buff) = self.recv_buff.as_mut() {
buff.reset_write();
let mut init_buff_len = 0usize;
match buff.get_ctrl_desc_mut() {
Some(ctrl_desc) => {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
ind_desc_lst[ctrl_desc_cnt].len = desc._init_len as u32;
ctrl_desc_cnt += 1;
init_buff_len += desc._init_len;
for byte in desc.deref_mut() {
*byte = 0;
}
}
}
None => {
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
init_buff_len += desc._init_len;
for byte in desc.deref_mut() {
*byte = 0;
}
}
}
}
buff.reset_len(init_buff_len);
}
self
}
fn reset(mut self) -> Self {
let mut ctrl_desc_cnt = 0usize;
if let Some(buff) = self.send_buff.as_mut() {
buff.reset_write();
let mut init_buff_len = 0usize;
match buff.get_ctrl_desc_mut() {
Some(ctrl_desc) => {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
ind_desc_lst[ctrl_desc_cnt].len = desc._init_len as u32;
ctrl_desc_cnt += 1;
init_buff_len += desc._init_len;
}
}
None => {
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
init_buff_len += desc._init_len;
}
}
}
buff.reset_len(init_buff_len);
}
if let Some(buff) = self.recv_buff.as_mut() {
buff.reset_write();
let mut init_buff_len = 0usize;
match buff.get_ctrl_desc_mut() {
Some(ctrl_desc) => {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
ind_desc_lst[ctrl_desc_cnt].len = desc._init_len as u32;
ctrl_desc_cnt += 1;
init_buff_len += desc._init_len;
}
}
None => {
for desc in buff.as_mut_slice() {
desc.len = desc._init_len;
init_buff_len += desc._init_len;
}
}
}
buff.reset_len(init_buff_len);
}
self
}
}
impl BufferToken {
pub fn restr_size(
&mut self,
new_send_len: Option<usize>,
new_recv_len: Option<usize>,
) -> Result<(usize, usize), VirtqError> {
let send_len = match new_send_len {
Some(new_len) => {
match self.send_buff.as_mut() {
Some(send_buff) => {
let mut ctrl_desc_cnt = 0usize;
match send_buff.get_ctrl_desc() {
None => {
if send_buff.len() < new_len {
return Err(VirtqError::General);
} else {
let mut len_now = 0usize;
let mut rest_zero = false;
for desc in send_buff.as_mut_slice() {
len_now += desc.len;
if len_now >= new_len && !rest_zero {
desc.len -= len_now - new_len;
rest_zero = true;
} else if rest_zero {
desc.len = 0;
}
}
send_buff.restr_len(new_len);
new_len
}
}
Some(ctrl_desc) => {
if send_buff.len() < new_len {
return Err(VirtqError::General);
} else {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
let mut len_now = 0usize;
let mut rest_zero = false;
for desc in send_buff.as_mut_slice() {
len_now += desc.len;
if len_now >= new_len && !rest_zero {
desc.len -= len_now - new_len;
ind_desc_lst[ctrl_desc_cnt].len -=
(len_now - new_len) as u32;
rest_zero = true;
} else if rest_zero {
desc.len = 0;
ind_desc_lst[ctrl_desc_cnt].len = 0;
}
ctrl_desc_cnt += 1;
}
send_buff.restr_len(new_len);
new_len
}
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
None => match self.send_buff.as_mut() {
Some(send_buff) => send_buff.len(),
None => 0,
},
};
let recv_len = match new_recv_len {
Some(new_len) => {
match self.recv_buff.as_mut() {
Some(recv_buff) => {
let mut ctrl_desc_cnt = 0usize;
match recv_buff.get_ctrl_desc() {
None => {
if recv_buff.len() < new_len {
return Err(VirtqError::General);
} else {
let mut len_now = 0usize;
let mut rest_zero = false;
for desc in recv_buff.as_mut_slice() {
len_now += desc.len;
if len_now >= new_len && !rest_zero {
desc.len -= len_now - new_len;
rest_zero = true;
} else if rest_zero {
desc.len = 0;
}
}
recv_buff.restr_len(new_len);
new_len
}
}
Some(ctrl_desc) => {
if recv_buff.len() < new_len {
return Err(VirtqError::General);
} else {
let ind_desc_lst = unsafe {
let size = core::mem::size_of::<Descriptor>();
core::slice::from_raw_parts_mut(
ctrl_desc.ptr as *mut Descriptor,
ctrl_desc.len / size,
)
};
let mut len_now = 0usize;
let mut rest_zero = false;
for desc in recv_buff.as_mut_slice() {
len_now += desc.len;
if len_now >= new_len && !rest_zero {
desc.len -= len_now - new_len;
ind_desc_lst[ctrl_desc_cnt].len -=
(len_now - new_len) as u32;
rest_zero = true;
} else if rest_zero {
desc.len = 0;
ind_desc_lst[ctrl_desc_cnt].len = 0;
}
ctrl_desc_cnt += 1;
}
recv_buff.restr_len(new_len);
new_len
}
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
None => match self.recv_buff.as_mut() {
Some(recv_buff) => recv_buff.len(),
None => 0,
},
};
Ok((send_len, recv_len))
}
pub fn len(&self) -> (usize, usize) {
match (self.send_buff.as_ref(), self.recv_buff.as_ref()) {
(Some(send_buff), Some(recv_buff)) => (send_buff.len(), recv_buff.len()),
(Some(send_buff), None) => (send_buff.len(), 0),
(None, Some(recv_buff)) => (0, recv_buff.len()),
(None, None) => unreachable!("Empty BufferToken not allowed!"),
}
}
pub fn raw_ptrs(
&mut self,
) -> (
Option<Box<[(*mut u8, usize)]>>,
Option<Box<[(*mut u8, usize)]>>,
) {
let mut send_ptrs = Vec::new();
let mut recv_ptrs = Vec::new();
if let Some(buff) = self.send_buff.as_mut() {
for desc in buff.as_slice() {
send_ptrs.push((desc.ptr, desc.len()));
}
}
if let Some(buff) = self.recv_buff.as_ref() {
for desc in buff.as_slice() {
recv_ptrs.push((desc.ptr, desc.len()));
}
}
match (send_ptrs.is_empty(), recv_ptrs.is_empty()) {
(true, true) => unreachable!("Empty transfer, Not allowed"),
(false, true) => (Some(send_ptrs.into_boxed_slice()), None),
(true, false) => (None, Some(recv_ptrs.into_boxed_slice())),
(false, false) => (
Some(send_ptrs.into_boxed_slice()),
Some(recv_ptrs.into_boxed_slice()),
),
}
}
pub fn write<K: AsSliceU8 + ?Sized, H: AsSliceU8 + ?Sized>(
mut self,
send: Option<&K>,
recv: Option<&H>,
) -> Result<TransferToken, VirtqError> {
if let Some(data) = send {
match self.send_buff.as_mut() {
Some(buff) => {
if buff.len() < data.as_slice_u8().len() {
return Err(VirtqError::WriteToLarge(self));
} else {
let data_slc = data.as_slice_u8();
let mut from = 0usize;
for i in 0..buff.num_descr() {
let to = if (buff.as_slice()[i].len() + from) > data_slc.len() {
data_slc.len()
} else {
from + buff.as_slice()[i].len()
};
from += buff.next_write(&data_slc[from..to]).unwrap();
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
if let Some(data) = recv {
match self.recv_buff.as_mut() {
Some(buff) => {
let data_slc = data.as_slice_u8();
if buff.len() < data_slc.len() {
return Err(VirtqError::WriteToLarge(self));
} else {
let mut from = 0usize;
for i in 0..buff.num_descr() {
let to = if (buff.as_slice()[i].len() + from) > data_slc.len() {
data_slc.len()
} else {
from + buff.as_slice()[i].len()
};
from += buff.next_write(&data_slc[from..to]).unwrap();
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
Ok(TransferToken {
state: TransferState::Ready,
buff_tkn: Some(self),
await_queue: None,
})
}
pub fn write_seq<K: AsBytes, H: AsBytes + ?Sized>(
mut self,
send_seq: Option<&K>,
recv_seq: Option<&H>,
) -> Result<Self, VirtqError> {
if let Some(data) = send_seq {
match self.send_buff.as_mut() {
Some(buff) => {
match buff.next_write(data.as_bytes()) {
Ok(_) => (), Err(_) => {
return Err(VirtqError::WriteToLarge(self));
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
if let Some(data) = recv_seq {
match self.recv_buff.as_mut() {
Some(buff) => {
match buff.next_write(data.as_bytes()) {
Ok(_) => (), Err(_) => {
return Err(VirtqError::WriteToLarge(self));
}
}
}
None => return Err(VirtqError::NoBufferAvail),
}
}
Ok(self)
}
pub fn provide(self) -> TransferToken {
TransferToken {
state: TransferState::Ready,
buff_tkn: Some(self),
await_queue: None,
}
}
}
enum Buffer {
Single {
desc_lst: Box<[MemDescr]>,
len: usize,
next_write: usize,
},
Multiple {
desc_lst: Box<[MemDescr]>,
len: usize,
next_write: usize,
},
Indirect {
desc_lst: Box<[MemDescr]>,
ctrl_desc: MemDescr,
len: usize,
next_write: usize,
},
}
impl Buffer {
fn reset_len(&mut self, init_len: usize) {
match self {
Buffer::Single { len, .. }
| Buffer::Multiple { len, .. }
| Buffer::Indirect { len, .. } => *len = init_len,
}
}
fn restr_len(&mut self, new_len: usize) {
match self {
Buffer::Single { len, .. }
| Buffer::Multiple { len, .. }
| Buffer::Indirect { len, .. } => *len = new_len,
}
}
fn next_write(&mut self, slice: &[u8]) -> Result<usize, BufferError> {
match self {
Buffer::Single {
desc_lst,
next_write,
..
}
| Buffer::Multiple {
desc_lst,
next_write,
..
}
| Buffer::Indirect {
desc_lst,
next_write,
..
} => {
if (desc_lst.len() - 1) < *next_write {
Err(BufferError::ToManyWrites)
} else if desc_lst.get(*next_write).unwrap().len() < slice.len() {
Err(BufferError::WriteToLarge)
} else {
desc_lst[*next_write].deref_mut()[0..slice.len()].copy_from_slice(slice);
*next_write += 1;
Ok(slice.len())
}
}
}
}
fn reset_write(&mut self) {
match self {
Buffer::Single { next_write, .. }
| Buffer::Multiple { next_write, .. }
| Buffer::Indirect { next_write, .. } => *next_write = 0,
}
}
fn into_raw(self) -> Vec<(*mut u8, usize)> {
match self {
Buffer::Single { mut desc_lst, .. }
| Buffer::Multiple { mut desc_lst, .. }
| Buffer::Indirect { mut desc_lst, .. } => {
let mut arr = Vec::with_capacity(desc_lst.len());
for desc in desc_lst.iter_mut() {
desc.dealloc = Dealloc::Not;
arr.push((desc.ptr, desc._mem_len));
}
arr
}
}
}
fn cpy(&self) -> Box<[u8]> {
match &self {
Buffer::Single { desc_lst, len, .. }
| Buffer::Multiple { desc_lst, len, .. }
| Buffer::Indirect { desc_lst, len, .. } => {
let mut arr = Vec::with_capacity(*len);
for desc in desc_lst.iter() {
arr.append(&mut desc.cpy_into_vec());
}
arr.into_boxed_slice()
}
}
}
fn scat_cpy(&self) -> Vec<Box<[u8]>> {
match &self {
Buffer::Single { desc_lst, .. }
| Buffer::Multiple { desc_lst, .. }
| Buffer::Indirect { desc_lst, .. } => {
let mut arr = Vec::with_capacity(desc_lst.len());
for desc in desc_lst.iter() {
arr.push(desc.cpy_into_box());
}
arr
}
}
}
fn num_descr(&self) -> usize {
match &self {
Buffer::Single { desc_lst, .. }
| Buffer::Multiple { desc_lst, .. }
| Buffer::Indirect { desc_lst, .. } => desc_lst.len(),
}
}
fn len(&self) -> usize {
match &self {
Buffer::Single { len, .. }
| Buffer::Multiple { len, .. }
| Buffer::Indirect { len, .. } => *len,
}
}
fn as_mut_slice(&mut self) -> &mut [MemDescr] {
match self {
Buffer::Single { desc_lst, .. }
| Buffer::Multiple { desc_lst, .. }
| Buffer::Indirect { desc_lst, .. } => desc_lst.as_mut(),
}
}
fn as_slice(&self) -> &[MemDescr] {
match self {
Buffer::Single { desc_lst, .. } => desc_lst.as_ref(),
Buffer::Multiple { desc_lst, .. } => desc_lst.as_ref(),
Buffer::Indirect { desc_lst, .. } => desc_lst.as_ref(),
}
}
fn get_ctrl_desc(&self) -> Option<&MemDescr> {
if let Buffer::Indirect { ctrl_desc, .. } = self {
Some(ctrl_desc)
} else {
None
}
}
fn get_ctrl_desc_mut(&mut self) -> Option<&mut MemDescr> {
if let Buffer::Indirect { ctrl_desc, .. } = self {
Some(ctrl_desc)
} else {
None
}
}
fn is_indirect(&self) -> bool {
matches!(self, Buffer::Indirect { .. })
}
}
struct MemDescr {
ptr: *mut u8,
len: usize,
_init_len: usize,
_mem_len: usize,
id: Option<MemDescrId>,
pool: Rc<MemPool>,
dealloc: Dealloc,
}
impl MemDescr {
fn into_vec(mut self) -> Vec<u8> {
self.dealloc = Dealloc::Not;
unsafe { Vec::from_raw_parts(self.ptr, self._mem_len, 0) }
}
fn cpy_into_vec(&self) -> Vec<u8> {
let mut vec = vec![0u8; self.len];
vec.copy_from_slice(self.deref());
vec
}
fn cpy_into_box(&self) -> Box<[u8]> {
let mut vec = vec![0u8; self.len];
vec.copy_from_slice(self.deref());
vec.into_boxed_slice()
}
fn raw_ptr(&self) -> *mut u8 {
self.ptr
}
fn len(&self) -> usize {
self.len
}
fn no_dealloc_clone(&self) -> Self {
MemDescr {
ptr: self.ptr,
len: self.len,
_init_len: self.len(),
_mem_len: self._mem_len,
id: None,
pool: Rc::clone(&self.pool),
dealloc: Dealloc::Not,
}
}
}
impl Deref for MemDescr {
type Target = [u8];
fn deref(&self) -> &Self::Target {
unsafe { core::slice::from_raw_parts(self.ptr, self.len) }
}
}
impl DerefMut for MemDescr {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) }
}
}
impl Drop for MemDescr {
fn drop(&mut self) {
if let Some(id) = self.id.take() {
self.pool.ret_id(id);
}
match self.dealloc {
Dealloc::Not => (),
Dealloc::AsSlice => unsafe { drop(Vec::from_raw_parts(self.ptr, self._mem_len, 0)) },
Dealloc::AsPage => {
crate::mm::deallocate(VirtAddr::from(self.ptr as usize), self._mem_len)
}
}
}
}
struct MemDescrId(pub u16);
#[derive(Debug, Clone, Copy)]
pub struct Bytes(usize);
impl Bytes {
pub fn new(size: usize) -> Option<Bytes> {
if core::mem::size_of_val(&size) <= core::mem::size_of::<u32>() {
Some(Bytes(size))
} else if core::mem::size_of_val(&size) == core::mem::size_of::<u64>() {
if (size as u64) <= (u32::MAX as u64) {
Some(Bytes(size))
} else {
None
}
} else {
None
}
}
}
impl From<Bytes> for usize {
fn from(byte: Bytes) -> Self {
byte.0
}
}
enum Dealloc {
Not,
AsPage,
AsSlice,
}
struct MemPool {
pool: RefCell<Vec<MemDescrId>>,
limit: u16,
}
impl MemPool {
fn ret_id(&self, id: MemDescrId) {
self.pool.borrow_mut().push(id);
}
fn new(size: u16) -> MemPool {
assert!(core::mem::size_of::<usize>() >= 2);
let mut id_vec = Vec::with_capacity(size as usize);
for i in 1..(size + 1) {
id_vec.push(MemDescrId(i));
}
MemPool {
pool: RefCell::new(id_vec),
limit: size,
}
}
fn pull_from_raw(&self, rc_self: Rc<MemPool>, slice: &[u8]) -> Result<MemDescr, VirtqError> {
assert!(!slice.is_empty());
let start_virt = (&slice[0] as *const u8) as usize;
let end_virt = (&slice[slice.len() - 1] as *const u8) as usize;
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
assert_eq!(end_phy, end_phy_calc);
let desc_id = match self.pool.borrow_mut().pop() {
Some(id) => id,
None => return Err(VirtqError::NoDescrAvail),
};
Ok(MemDescr {
ptr: (&slice[0] as *const u8) as *mut u8,
len: slice.len(),
_init_len: slice.len(),
_mem_len: slice.len(),
id: Some(desc_id),
dealloc: Dealloc::Not,
pool: rc_self,
})
}
fn pull_from_raw_untracked(&self, rc_self: Rc<MemPool>, slice: &[u8]) -> MemDescr {
assert!(!slice.is_empty());
let start_virt = (&slice[0] as *const u8) as usize;
let end_virt = (&slice[slice.len() - 1] as *const u8) as usize;
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
assert_eq!(end_phy, end_phy_calc);
MemDescr {
ptr: (&slice[0] as *const u8) as *mut u8,
len: slice.len(),
_init_len: slice.len(),
_mem_len: slice.len(),
id: None,
dealloc: Dealloc::Not,
pool: rc_self,
}
}
fn pull(&self, rc_self: Rc<MemPool>, bytes: Bytes) -> Result<MemDescr, VirtqError> {
let id = match self.pool.borrow_mut().pop() {
Some(id) => id,
None => return Err(VirtqError::NoDescrAvail),
};
let len = bytes.0;
let _mem_len = len.align_up(BasePageSize::SIZE as usize);
let ptr = (crate::mm::allocate(_mem_len, true).0 as *const u8) as *mut u8;
let start_virt = ptr as usize;
let end_virt = start_virt + (len - 1);
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (len - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
assert_eq!(end_phy, end_phy_calc);
Ok(MemDescr {
ptr,
len,
_init_len: len,
_mem_len,
id: Some(id),
dealloc: Dealloc::AsPage,
pool: rc_self,
})
}
fn pull_untracked(&self, rc_self: Rc<MemPool>, bytes: Bytes) -> MemDescr {
let len = bytes.0;
let _mem_len = len.align_up(BasePageSize::SIZE as usize);
let ptr = (crate::mm::allocate(_mem_len, true).0 as *const u8) as *mut u8;
let start_virt = ptr as usize;
let end_virt = start_virt + (len - 1);
let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (len - 1);
let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt));
assert_eq!(end_phy, end_phy_calc);
MemDescr {
ptr,
len,
_init_len: len,
_mem_len,
id: None,
dealloc: Dealloc::AsPage,
pool: rc_self,
}
}
}
#[derive(Debug, Clone)]
pub enum BuffSpec<'a> {
Single(Bytes),
Multiple(&'a [Bytes]),
Indirect(&'a [Bytes]),
}
pub struct Pinned<T> {
raw_ptr: *mut T,
_drop_inner: bool,
}
impl<T: Sized> Pinned<T> {
fn into_raw(mut self) -> *mut T {
self._drop_inner = false;
self.raw_ptr
}
fn pin(val: T) -> Pinned<T> {
let boxed = Box::new(val);
Pinned {
raw_ptr: Box::into_raw(boxed),
_drop_inner: true,
}
}
fn from_boxed(boxed: Box<T>) -> Pinned<T> {
Pinned {
raw_ptr: Box::into_raw(boxed),
_drop_inner: true,
}
}
fn from_raw(raw_ptr: *mut T) -> Pinned<T> {
Pinned {
raw_ptr,
_drop_inner: true,
}
}
fn unpin(mut self) -> T {
self._drop_inner = false;
unsafe { *Box::from_raw(self.raw_ptr) }
}
fn raw_addr(&self) -> *mut T {
self.raw_ptr
}
}
impl<T> Deref for Pinned<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { &*(self.raw_ptr) }
}
}
impl<T> DerefMut for Pinned<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *(self.raw_ptr) }
}
}
impl<T> Drop for Pinned<T> {
fn drop(&mut self) {
if self._drop_inner {
unsafe {
drop(Box::from_raw(self.raw_ptr));
}
}
}
}
#[allow(dead_code, non_camel_case_types)]
#[derive(Debug, Copy, Clone)]
#[repr(u16)]
pub enum DescrFlags {
VIRTQ_DESC_F_NEXT = 1 << 0,
VIRTQ_DESC_F_WRITE = 1 << 1,
VIRTQ_DESC_F_INDIRECT = 1 << 2,
VIRTQ_DESC_F_AVAIL = 1 << 7,
VIRTQ_DESC_F_USED = 1 << 15,
}
use core::ops::Not;
impl Not for DescrFlags {
type Output = u16;
fn not(self) -> Self::Output {
!(u16::from(self))
}
}
use core::ops::BitOr;
impl BitOr for DescrFlags {
type Output = u16;
fn bitor(self, rhs: DescrFlags) -> Self::Output {
u16::from(self) | u16::from(rhs)
}
}
impl BitOr<DescrFlags> for u16 {
type Output = u16;
fn bitor(self, rhs: DescrFlags) -> Self::Output {
self | u16::from(rhs)
}
}
impl BitAnd for DescrFlags {
type Output = u16;
fn bitand(self, rhs: Self) -> Self::Output {
u16::from(self) & u16::from(rhs)
}
}
impl BitAnd<DescrFlags> for u16 {
type Output = u16;
fn bitand(self, rhs: DescrFlags) -> Self::Output {
self & u16::from(rhs)
}
}
impl PartialEq<DescrFlags> for u16 {
fn eq(&self, other: &DescrFlags) -> bool {
self == other
}
}
impl From<DescrFlags> for u16 {
fn from(flag: DescrFlags) -> Self {
match flag {
DescrFlags::VIRTQ_DESC_F_NEXT => 1 << 0,
DescrFlags::VIRTQ_DESC_F_WRITE => 1 << 1,
DescrFlags::VIRTQ_DESC_F_INDIRECT => 1 << 2,
DescrFlags::VIRTQ_DESC_F_AVAIL => 1 << 7,
DescrFlags::VIRTQ_DESC_F_USED => 1 << 15,
}
}
}
pub mod error {
use super::{BufferToken, Transfer};
#[derive(Debug)]
pub enum BufferError {
WriteToLarge,
ToManyWrites,
}
pub enum VirtqError {
General,
BufferInWithDirect,
BufferNotSpecified,
QueueNotExisting(u16),
NoDescrAvail,
BufferSizeWrong(usize),
NoReuseBuffer,
OngoingTransfer(Option<Transfer>),
NoBufferAvail,
WriteToLarge(BufferToken),
BufferToLarge,
}
impl core::fmt::Debug for VirtqError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
VirtqError::General => write!(f, "Virtq failure due to unknown reasons!"),
VirtqError::NoBufferAvail => write!(f, "Virtq detected write into non existing Buffer!"),
VirtqError::BufferInWithDirect => write!(f, "Virtq detected creation of Token, where Indirect and direct buffers where mixed!"),
VirtqError::BufferNotSpecified => write!(f, "Virtq detected creation of Token, without a BuffSpec"),
VirtqError::QueueNotExisting(_) => write!(f, "Virtq does not exist and can not be used!"),
VirtqError::NoDescrAvail => write!(f, "Virtqs memory pool is exhausted!"),
VirtqError::BufferSizeWrong(_) => write!(f, "Specified Buffer is to small for write!"),
VirtqError::NoReuseBuffer => write!(f, "Buffer can not be reused!"),
VirtqError::OngoingTransfer(_) => write!(f, "Transfer is ongoging and can not be used currently!"),
VirtqError::WriteToLarge(_) => write!(f, "Write is to large for BufferToken!"),
VirtqError::BufferToLarge => write!(f, "Buffer to large for queue! u32::MAX exceeded."),
}
}
}
}