1use alloc::vec;
15use alloc::vec::Vec;
16use core::ptr::{read_volatile, write_volatile};
17use core::sync::atomic::{fence, AtomicBool, Ordering};
18use spin::Mutex;
19
20use super::super::fs::block_dev::BlockDevice;
21use super::super::fs::file::FileError;
22
23const MAGIC_VALUE: usize = 0x00;
27const VERSION: usize = 0x04;
28const DEVICE_ID: usize = 0x08;
29const DEVICE_FEATURES: usize = 0x10;
30const DEVICE_FEATURES_SEL: usize = 0x14;
31const DRIVER_FEATURES: usize = 0x20;
32const DRIVER_FEATURES_SEL: usize = 0x24;
33const GUEST_PAGE_SIZE: usize = 0x28;
34const QUEUE_SEL: usize = 0x30;
35const QUEUE_NUM_MAX: usize = 0x34;
36const QUEUE_NUM: usize = 0x38;
37const QUEUE_ALIGN: usize = 0x3C;
38const QUEUE_PFN: usize = 0x40;
39const QUEUE_NOTIFY: usize = 0x50;
40const INTERRUPT_STATUS: usize = 0x60;
41const INTERRUPT_ACK: usize = 0x64;
42const STATUS: usize = 0x70;
43const DEVICE_CONFIG: usize = 0x100;
44
45const STATUS_ACKNOWLEDGE: u32 = 1;
47const STATUS_DRIVER: u32 = 2;
48const STATUS_DRIVER_OK: u32 = 4;
49const STATUS_FEATURES_OK: u32 = 8;
50
51const VIRTIO_MMIO_BASE: usize = 0x1000_1000;
53const VIRTIO_MMIO_STRIDE: usize = 0x1000;
54const VIRTIO_MMIO_SLOTS: usize = 8;
55const VIRTIO_MAGIC: u32 = 0x7472_6976;
56const VIRTIO_DEV_BLOCK: u32 = 2;
57
58const VIRTQ_SIZE: usize = 8;
60
61const VIRTIO_BLK_T_IN: u32 = 0;
63const VIRTIO_BLK_T_OUT: u32 = 1;
64const VIRTIO_BLK_S_OK: u8 = 0;
65
66struct MmioRegion {
71 pub base: usize,
72}
73
74impl MmioRegion {
75 fn new(base: usize) -> Self {
76 Self { base }
77 }
78
79 #[inline]
80 fn read32(&self, offset: usize) -> u32 {
81 unsafe { read_volatile((self.base + offset) as *const u32) }
82 }
83
84 #[inline]
85 fn write32(&self, offset: usize, val: u32) {
86 unsafe {
87 write_volatile((self.base + offset) as *mut u32, val);
88 }
89 }
90}
91
92#[repr(C, align(16))]
97struct VirtqDesc {
98 addr: u64,
99 len: u32,
100 flags: u16,
101 next: u16,
102}
103
104const VIRTQ_DESC_F_NEXT: u16 = 1;
105const VIRTQ_DESC_F_WRITE: u16 = 2;
106
107#[repr(C, align(2))]
108struct VirtqAvail {
109 flags: u16,
110 idx: u16,
111 ring: [u16; VIRTQ_SIZE],
112}
113
114#[repr(C, align(4))]
115#[derive(Clone, Copy)]
116struct VirtqUsedElem {
117 id: u32,
118 len: u32,
119}
120
121#[repr(C, align(4))]
122struct VirtqUsed {
123 flags: u16,
124 idx: u16,
125 ring: [VirtqUsedElem; VIRTQ_SIZE],
126}
127
128struct VirtqMemory {
129 desc: *mut VirtqDesc,
130 avail: *mut VirtqAvail,
131 used: *mut VirtqUsed,
132 desc_phys: u64,
133 _backing: Vec<u8>,
134}
135
136impl VirtqMemory {
137 fn new() -> Option<Self> {
141 let desc_size = core::mem::size_of::<VirtqDesc>() * VIRTQ_SIZE;
142 let page_size = 4096usize;
143 let total_size = page_size * 2;
144 let mut backing = vec![0u8; total_size + page_size];
145 let base = backing.as_mut_ptr() as usize;
146 let aligned = (base + page_size - 1) & !(page_size - 1);
147
148 let desc = aligned as *mut VirtqDesc;
149 let avail = (aligned + desc_size) as *mut VirtqAvail;
150 let used = (aligned + page_size) as *mut VirtqUsed;
151
152 unsafe {
153 for i in 0..VIRTQ_SIZE {
154 (*desc.add(i)) = VirtqDesc {
155 addr: 0,
156 len: 0,
157 flags: 0,
158 next: 0,
159 };
160 }
161 (*avail) = VirtqAvail {
162 flags: 0,
163 idx: 0,
164 ring: [0u16; VIRTQ_SIZE],
165 };
166 (*used) = VirtqUsed {
167 flags: 0,
168 idx: 0,
169 ring: [VirtqUsedElem { id: 0, len: 0 }; VIRTQ_SIZE],
170 };
171 }
172
173 Some(Self {
174 desc,
175 avail,
176 used,
177 desc_phys: aligned as u64,
178 _backing: backing,
179 })
180 }
181}
182
183#[repr(C)]
184struct BlkReq {
185 req_type: u32,
186 reserved: u32,
187 sector: u64,
188}
189
190static VIRTIO_DEV_PTR: Mutex<Option<usize>> = Mutex::new(None);
195static VIRTIO_IO_COMPLETE: AtomicBool = AtomicBool::new(false);
196
197pub fn try_handle_interrupt() {
198 let ptr = *VIRTIO_DEV_PTR.lock();
199 if let Some(mmio_base) = ptr {
200 let mmio = MmioRegion::new(mmio_base);
201 let status = mmio.read32(INTERRUPT_STATUS);
202 if status & 1 != 0 {
203 mmio.write32(INTERRUPT_ACK, status);
204 fence(Ordering::SeqCst);
205 VIRTIO_IO_COMPLETE.store(true, Ordering::Release);
206 }
207 }
208}
209
210pub struct VirtioBlkDev {
215 mmio: MmioRegion,
216 vq: Mutex<VirtqMemory>,
217 capacity: usize,
218 last_used_idx: Mutex<u16>,
219}
220
221impl VirtioBlkDev {
222 pub fn probe() -> Option<Self> {
224 for slot in 0..VIRTIO_MMIO_SLOTS {
225 let base = VIRTIO_MMIO_BASE + slot * VIRTIO_MMIO_STRIDE;
226 let mmio = MmioRegion::new(base);
227
228 if mmio.read32(MAGIC_VALUE) != VIRTIO_MAGIC {
229 continue;
230 }
231 if mmio.read32(DEVICE_ID) != VIRTIO_DEV_BLOCK {
232 continue;
233 }
234
235 crate::serial_println!(
236 "[VIRTIO] Found virtio-blk at {:#x}, version={}",
237 base,
238 mmio.read32(VERSION)
239 );
240
241 return Self::init_device(mmio);
242 }
243 crate::serial_println!("[VIRTIO] No virtio-blk device found");
244 None
245 }
246
247 fn init_device(mmio: MmioRegion) -> Option<Self> {
248 mmio.write32(STATUS, 0);
250 fence(Ordering::SeqCst);
251
252 mmio.write32(STATUS, STATUS_ACKNOWLEDGE);
254 fence(Ordering::SeqCst);
255
256 mmio.write32(STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
258 fence(Ordering::SeqCst);
259
260 mmio.write32(DEVICE_FEATURES_SEL, 0);
262 mmio.read32(DEVICE_FEATURES);
263 mmio.write32(DEVICE_FEATURES_SEL, 1);
264 mmio.read32(DEVICE_FEATURES);
265 mmio.write32(DRIVER_FEATURES_SEL, 0);
266 mmio.write32(DRIVER_FEATURES, 0);
267 mmio.write32(DRIVER_FEATURES_SEL, 1);
268 mmio.write32(DRIVER_FEATURES, 0);
269
270 mmio.write32(
272 STATUS,
273 STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK,
274 );
275 fence(Ordering::SeqCst);
276
277 let status = mmio.read32(STATUS);
278 if status & STATUS_FEATURES_OK == 0 {
279 crate::serial_println!("[VIRTIO] Feature negotiation failed, status={:#x}", status);
280 return None;
281 }
282
283 let vq = VirtqMemory::new()?;
285
286 mmio.write32(QUEUE_SEL, 0);
287 fence(Ordering::SeqCst);
288
289 let max_size = mmio.read32(QUEUE_NUM_MAX);
290 if max_size == 0 {
291 return None;
292 }
293
294 let queue_size = core::cmp::min(VIRTQ_SIZE as u32, max_size);
295 mmio.write32(QUEUE_NUM, queue_size);
296 fence(Ordering::SeqCst);
297
298 let page_size = 4096u32;
299 mmio.write32(GUEST_PAGE_SIZE, page_size);
300 mmio.write32(QUEUE_ALIGN, page_size);
301 mmio.write32(QUEUE_PFN, (vq.desc_phys as u32) / page_size);
302 fence(Ordering::SeqCst);
303
304 mmio.write32(
306 STATUS,
307 STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK,
308 );
309 fence(Ordering::SeqCst);
310
311 let capacity_lo = mmio.read32(DEVICE_CONFIG);
313 let capacity_hi = mmio.read32(DEVICE_CONFIG + 4);
314 let capacity = ((capacity_hi as u64) << 32 | capacity_lo as u64) as usize;
315
316 mmio.write32(INTERRUPT_ACK, 0x3); fence(Ordering::SeqCst);
319
320 *VIRTIO_DEV_PTR.lock() = Some(mmio.base);
322
323 crate::serial_println!(
324 "[VIRTIO] Device ready: {} sectors ({} MB), interrupts enabled",
325 capacity,
326 capacity * 512 / (1024 * 1024)
327 );
328
329 Some(Self {
330 mmio,
331 vq: Mutex::new(vq),
332 capacity,
333 last_used_idx: Mutex::new(0),
334 })
335 }
336
337 fn submit_io(&self, sector: u64, buf: &mut [u8], is_write: bool) -> Result<(), FileError> {
340 if buf.len() != 512 {
341 return Err(FileError::InvalidOperation);
342 }
343 if sector >= self.capacity as u64 && self.capacity > 0 {
344 return Err(FileError::InvalidOperation);
345 }
346
347 let req = BlkReq {
348 req_type: if is_write {
349 VIRTIO_BLK_T_OUT
350 } else {
351 VIRTIO_BLK_T_IN
352 },
353 reserved: 0,
354 sector,
355 };
356
357 let mut status: u8 = 0xFF;
358 let req_phys = &req as *const BlkReq as u64;
359 let buf_phys = buf.as_ptr() as u64;
360 let status_phys = &mut status as *mut u8 as u64;
361
362 let vq = self.vq.lock();
363 let desc = vq.desc;
364
365 unsafe {
367 (*desc.add(0)).addr = req_phys;
368 (*desc.add(0)).len = core::mem::size_of::<BlkReq>() as u32;
369 (*desc.add(0)).flags = VIRTQ_DESC_F_NEXT;
370 (*desc.add(0)).next = 1;
371
372 (*desc.add(1)).addr = buf_phys;
373 (*desc.add(1)).len = 512;
374 (*desc.add(1)).flags = VIRTQ_DESC_F_NEXT;
375 if !is_write {
376 (*desc.add(1)).flags |= VIRTQ_DESC_F_WRITE;
377 }
378 (*desc.add(1)).next = 2;
379
380 (*desc.add(2)).addr = status_phys;
381 (*desc.add(2)).len = 1;
382 (*desc.add(2)).flags = VIRTQ_DESC_F_WRITE;
383 (*desc.add(2)).next = 0;
384 }
385
386 fence(Ordering::SeqCst);
387
388 let avail = vq.avail;
390 let used_ptr = vq.used;
391 unsafe {
392 let idx = (*avail).idx;
393 let ring_idx = (idx as usize) % VIRTQ_SIZE;
394 (*avail).ring[ring_idx] = 0;
395 fence(Ordering::SeqCst);
396 (*avail).idx = idx.wrapping_add(1);
397 }
398
399 self.mmio.write32(QUEUE_NOTIFY, 0);
401 fence(Ordering::SeqCst);
402 drop(vq);
403
404 let mut last = self.last_used_idx.lock();
406 let used = unsafe { &*used_ptr };
407 VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
408 loop {
409 fence(Ordering::SeqCst);
410 let used_idx = unsafe { read_volatile(&used.idx) };
411 if used_idx != *last {
412 *last = used_idx;
413 break;
414 }
415 drop(last);
416 unsafe {
417 riscv::register::sstatus::set_sie();
418 }
419 riscv::asm::wfi();
420 unsafe {
421 riscv::register::sstatus::clear_sie();
422 }
423 last = self.last_used_idx.lock();
424 }
425 VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
426
427 fence(Ordering::SeqCst);
428
429 if status != VIRTIO_BLK_S_OK {
430 crate::serial_println!("[VIRTIO] I/O error: status={:#x}", status);
431 return Err(FileError::IoError);
432 }
433
434 Ok(())
435 }
436}
437
438impl BlockDevice for VirtioBlkDev {
443 fn read_block(&self, block_no: usize, buf: &mut [u8]) -> Result<(), FileError> {
444 if buf.len() != 512 {
445 return Err(FileError::InvalidOperation);
446 }
447 self.submit_io(block_no as u64, buf, false)
448 }
449
450 fn write_block(&self, block_no: usize, buf: &[u8]) -> Result<(), FileError> {
451 if buf.len() != 512 {
452 return Err(FileError::InvalidOperation);
453 }
454 let mut tmp = [0u8; 512];
455 tmp.copy_from_slice(buf);
456 self.submit_io(block_no as u64, &mut tmp, true)
457 }
458
459 fn block_count(&self) -> usize {
460 self.capacity
461 }
462}
463
464unsafe impl Send for VirtioBlkDev {}
465unsafe impl Sync for VirtioBlkDev {}