use alloc::vec;
use alloc::vec::Vec;
use core::ptr::{read_volatile, write_volatile};
use core::sync::atomic::{fence, AtomicBool, Ordering};
use spin::Mutex;
use super::super::fs::block_dev::BlockDevice;
use super::super::fs::file::FileError;
const MAGIC_VALUE: usize = 0x00;
const VERSION: usize = 0x04;
const DEVICE_ID: usize = 0x08;
const DEVICE_FEATURES: usize = 0x10;
const DEVICE_FEATURES_SEL: usize = 0x14;
const DRIVER_FEATURES: usize = 0x20;
const DRIVER_FEATURES_SEL: usize = 0x24;
const GUEST_PAGE_SIZE: usize = 0x28;
const QUEUE_SEL: usize = 0x30;
const QUEUE_NUM_MAX: usize = 0x34;
const QUEUE_NUM: usize = 0x38;
const QUEUE_ALIGN: usize = 0x3C;
const QUEUE_PFN: usize = 0x40;
const QUEUE_NOTIFY: usize = 0x50;
const INTERRUPT_STATUS: usize = 0x60;
const INTERRUPT_ACK: usize = 0x64;
const STATUS: usize = 0x70;
const DEVICE_CONFIG: usize = 0x100;
const STATUS_ACKNOWLEDGE: u32 = 1;
const STATUS_DRIVER: u32 = 2;
const STATUS_DRIVER_OK: u32 = 4;
const STATUS_FEATURES_OK: u32 = 8;
const VIRTIO_MMIO_BASE: usize = 0x1000_1000;
const VIRTIO_MMIO_STRIDE: usize = 0x1000;
const VIRTIO_MMIO_SLOTS: usize = 8;
const VIRTIO_MAGIC: u32 = 0x7472_6976;
const VIRTIO_DEV_BLOCK: u32 = 2;
const VIRTQ_SIZE: usize = 8;
const VIRTIO_BLK_T_IN: u32 = 0;
const VIRTIO_BLK_T_OUT: u32 = 1;
const VIRTIO_BLK_S_OK: u8 = 0;
struct MmioRegion {
pub base: usize,
}
impl MmioRegion {
fn new(base: usize) -> Self {
Self { base }
}
#[inline]
fn read32(&self, offset: usize) -> u32 {
unsafe { read_volatile((self.base + offset) as *const u32) }
}
#[inline]
fn write32(&self, offset: usize, val: u32) {
unsafe {
write_volatile((self.base + offset) as *mut u32, val);
}
}
}
#[repr(C, align(16))]
struct VirtqDesc {
addr: u64,
len: u32,
flags: u16,
next: u16,
}
const VIRTQ_DESC_F_NEXT: u16 = 1;
const VIRTQ_DESC_F_WRITE: u16 = 2;
#[repr(C, align(2))]
struct VirtqAvail {
flags: u16,
idx: u16,
ring: [u16; VIRTQ_SIZE],
}
#[repr(C, align(4))]
#[derive(Clone, Copy)]
struct VirtqUsedElem {
id: u32,
len: u32,
}
#[repr(C, align(4))]
struct VirtqUsed {
flags: u16,
idx: u16,
ring: [VirtqUsedElem; VIRTQ_SIZE],
}
struct VirtqMemory {
desc: *mut VirtqDesc,
avail: *mut VirtqAvail,
used: *mut VirtqUsed,
desc_phys: u64,
_backing: Vec<u8>,
}
impl VirtqMemory {
fn new() -> Option<Self> {
let desc_size = core::mem::size_of::<VirtqDesc>() * VIRTQ_SIZE;
let page_size = 4096usize;
let total_size = page_size * 2;
let mut backing = vec![0u8; total_size + page_size];
let base = backing.as_mut_ptr() as usize;
let aligned = (base + page_size - 1) & !(page_size - 1);
let desc = aligned as *mut VirtqDesc;
let avail = (aligned + desc_size) as *mut VirtqAvail;
let used = (aligned + page_size) as *mut VirtqUsed;
unsafe {
for i in 0..VIRTQ_SIZE {
(*desc.add(i)) = VirtqDesc {
addr: 0,
len: 0,
flags: 0,
next: 0,
};
}
(*avail) = VirtqAvail {
flags: 0,
idx: 0,
ring: [0u16; VIRTQ_SIZE],
};
(*used) = VirtqUsed {
flags: 0,
idx: 0,
ring: [VirtqUsedElem { id: 0, len: 0 }; VIRTQ_SIZE],
};
}
Some(Self {
desc,
avail,
used,
desc_phys: aligned as u64,
_backing: backing,
})
}
}
#[repr(C)]
struct BlkReq {
req_type: u32,
reserved: u32,
sector: u64,
}
static VIRTIO_DEV_PTR: Mutex<Option<usize>> = Mutex::new(None);
static VIRTIO_IO_COMPLETE: AtomicBool = AtomicBool::new(false);
pub fn try_handle_interrupt() {
let ptr = *VIRTIO_DEV_PTR.lock();
if let Some(mmio_base) = ptr {
let mmio = MmioRegion::new(mmio_base);
let status = mmio.read32(INTERRUPT_STATUS);
if status & 1 != 0 {
mmio.write32(INTERRUPT_ACK, status);
fence(Ordering::SeqCst);
VIRTIO_IO_COMPLETE.store(true, Ordering::Release);
}
}
}
pub struct VirtioBlkDev {
mmio: MmioRegion,
vq: Mutex<VirtqMemory>,
capacity: usize,
last_used_idx: Mutex<u16>,
}
impl VirtioBlkDev {
pub fn probe() -> Option<Self> {
for slot in 0..VIRTIO_MMIO_SLOTS {
let base = VIRTIO_MMIO_BASE + slot * VIRTIO_MMIO_STRIDE;
let mmio = MmioRegion::new(base);
if mmio.read32(MAGIC_VALUE) != VIRTIO_MAGIC {
continue;
}
if mmio.read32(DEVICE_ID) != VIRTIO_DEV_BLOCK {
continue;
}
crate::serial_println!(
"[VIRTIO] Found virtio-blk at {:#x}, version={}",
base,
mmio.read32(VERSION)
);
return Self::init_device(mmio);
}
crate::serial_println!("[VIRTIO] No virtio-blk device found");
None
}
fn init_device(mmio: MmioRegion) -> Option<Self> {
mmio.write32(STATUS, 0);
fence(Ordering::SeqCst);
mmio.write32(STATUS, STATUS_ACKNOWLEDGE);
fence(Ordering::SeqCst);
mmio.write32(STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
fence(Ordering::SeqCst);
mmio.write32(DEVICE_FEATURES_SEL, 0);
mmio.read32(DEVICE_FEATURES);
mmio.write32(DEVICE_FEATURES_SEL, 1);
mmio.read32(DEVICE_FEATURES);
mmio.write32(DRIVER_FEATURES_SEL, 0);
mmio.write32(DRIVER_FEATURES, 0);
mmio.write32(DRIVER_FEATURES_SEL, 1);
mmio.write32(DRIVER_FEATURES, 0);
mmio.write32(
STATUS,
STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK,
);
fence(Ordering::SeqCst);
let status = mmio.read32(STATUS);
if status & STATUS_FEATURES_OK == 0 {
crate::serial_println!("[VIRTIO] Feature negotiation failed, status={:#x}", status);
return None;
}
let vq = VirtqMemory::new()?;
mmio.write32(QUEUE_SEL, 0);
fence(Ordering::SeqCst);
let max_size = mmio.read32(QUEUE_NUM_MAX);
if max_size == 0 {
return None;
}
let queue_size = core::cmp::min(VIRTQ_SIZE as u32, max_size);
mmio.write32(QUEUE_NUM, queue_size);
fence(Ordering::SeqCst);
let page_size = 4096u32;
mmio.write32(GUEST_PAGE_SIZE, page_size);
mmio.write32(QUEUE_ALIGN, page_size);
mmio.write32(QUEUE_PFN, (vq.desc_phys as u32) / page_size);
fence(Ordering::SeqCst);
mmio.write32(
STATUS,
STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK,
);
fence(Ordering::SeqCst);
let capacity_lo = mmio.read32(DEVICE_CONFIG);
let capacity_hi = mmio.read32(DEVICE_CONFIG + 4);
let capacity = ((capacity_hi as u64) << 32 | capacity_lo as u64) as usize;
mmio.write32(INTERRUPT_ACK, 0x3); fence(Ordering::SeqCst);
*VIRTIO_DEV_PTR.lock() = Some(mmio.base);
crate::serial_println!(
"[VIRTIO] Device ready: {} sectors ({} MB), interrupts enabled",
capacity,
capacity * 512 / (1024 * 1024)
);
Some(Self {
mmio,
vq: Mutex::new(vq),
capacity,
last_used_idx: Mutex::new(0),
})
}
fn submit_io(&self, sector: u64, buf: &mut [u8], is_write: bool) -> Result<(), FileError> {
if buf.len() != 512 {
return Err(FileError::InvalidOperation);
}
if sector >= self.capacity as u64 && self.capacity > 0 {
return Err(FileError::InvalidOperation);
}
let req = BlkReq {
req_type: if is_write {
VIRTIO_BLK_T_OUT
} else {
VIRTIO_BLK_T_IN
},
reserved: 0,
sector,
};
let mut status: u8 = 0xFF;
let req_phys = &req as *const BlkReq as u64;
let buf_phys = buf.as_ptr() as u64;
let status_phys = &mut status as *mut u8 as u64;
let vq = self.vq.lock();
let desc = vq.desc;
unsafe {
(*desc.add(0)).addr = req_phys;
(*desc.add(0)).len = core::mem::size_of::<BlkReq>() as u32;
(*desc.add(0)).flags = VIRTQ_DESC_F_NEXT;
(*desc.add(0)).next = 1;
(*desc.add(1)).addr = buf_phys;
(*desc.add(1)).len = 512;
(*desc.add(1)).flags = VIRTQ_DESC_F_NEXT;
if !is_write {
(*desc.add(1)).flags |= VIRTQ_DESC_F_WRITE;
}
(*desc.add(1)).next = 2;
(*desc.add(2)).addr = status_phys;
(*desc.add(2)).len = 1;
(*desc.add(2)).flags = VIRTQ_DESC_F_WRITE;
(*desc.add(2)).next = 0;
}
fence(Ordering::SeqCst);
let avail = vq.avail;
let used_ptr = vq.used;
unsafe {
let idx = (*avail).idx;
let ring_idx = (idx as usize) % VIRTQ_SIZE;
(*avail).ring[ring_idx] = 0;
fence(Ordering::SeqCst);
(*avail).idx = idx.wrapping_add(1);
}
self.mmio.write32(QUEUE_NOTIFY, 0);
fence(Ordering::SeqCst);
drop(vq);
let mut last = self.last_used_idx.lock();
let used = unsafe { &*used_ptr };
VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
loop {
fence(Ordering::SeqCst);
let used_idx = unsafe { read_volatile(&used.idx) };
if used_idx != *last {
*last = used_idx;
break;
}
drop(last);
unsafe {
riscv::register::sstatus::set_sie();
}
riscv::asm::wfi();
unsafe {
riscv::register::sstatus::clear_sie();
}
last = self.last_used_idx.lock();
}
VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
fence(Ordering::SeqCst);
if status != VIRTIO_BLK_S_OK {
crate::serial_println!("[VIRTIO] I/O error: status={:#x}", status);
return Err(FileError::IoError);
}
Ok(())
}
}
impl BlockDevice for VirtioBlkDev {
fn read_block(&self, block_no: usize, buf: &mut [u8]) -> Result<(), FileError> {
if buf.len() != 512 {
return Err(FileError::InvalidOperation);
}
self.submit_io(block_no as u64, buf, false)
}
fn write_block(&self, block_no: usize, buf: &[u8]) -> Result<(), FileError> {
if buf.len() != 512 {
return Err(FileError::InvalidOperation);
}
let mut tmp = [0u8; 512];
tmp.copy_from_slice(buf);
self.submit_io(block_no as u64, &mut tmp, true)
}
fn block_count(&self) -> usize {
self.capacity
}
}
unsafe impl Send for VirtioBlkDev {}
unsafe impl Sync for VirtioBlkDev {}