Skip to main content

os/block/
virtio_blk.rs

1//! VirtIO Block Device Driver (MMIO Transport, Legacy Mode)
2//!
3//! Implements the BlockDevice trait for real QEMU virtio-blk devices
4//! discovered via the MMIO transport on RISC-V virt machines.
5//!
6//! ## Device Discovery
7//! QEMU RISC-V virt places virtio MMIO devices at 0x10001000+ (stride 0x1000).
8//! We scan each slot for MagicValue (0x74726976) and DeviceID 2 (block).
9//!
10//! ## Virtqueue Architecture
11//! Single virtqueue (index 0) for both read and write. Each I/O request uses
12//! a 3-descriptor chain: request header → data buffer → status byte.
13
14use alloc::vec;
15use alloc::vec::Vec;
16use core::ptr::{read_volatile, write_volatile};
17use core::sync::atomic::{fence, AtomicBool, Ordering};
18use spin::Mutex;
19
20use super::super::fs::block_dev::BlockDevice;
21use super::super::fs::file::FileError;
22
23// ============================================================
24// MMIO Register Offsets
25// ============================================================
26const MAGIC_VALUE: usize = 0x00;
27const VERSION: usize = 0x04;
28const DEVICE_ID: usize = 0x08;
29const DEVICE_FEATURES: usize = 0x10;
30const DEVICE_FEATURES_SEL: usize = 0x14;
31const DRIVER_FEATURES: usize = 0x20;
32const DRIVER_FEATURES_SEL: usize = 0x24;
33const GUEST_PAGE_SIZE: usize = 0x28;
34const QUEUE_SEL: usize = 0x30;
35const QUEUE_NUM_MAX: usize = 0x34;
36const QUEUE_NUM: usize = 0x38;
37const QUEUE_ALIGN: usize = 0x3C;
38const QUEUE_PFN: usize = 0x40;
39const QUEUE_NOTIFY: usize = 0x50;
40const INTERRUPT_STATUS: usize = 0x60;
41const INTERRUPT_ACK: usize = 0x64;
42const STATUS: usize = 0x70;
43const DEVICE_CONFIG: usize = 0x100;
44
45// Status register bits
46const STATUS_ACKNOWLEDGE: u32 = 1;
47const STATUS_DRIVER: u32 = 2;
48const STATUS_DRIVER_OK: u32 = 4;
49const STATUS_FEATURES_OK: u32 = 8;
50
51// MMIO discovery
52const VIRTIO_MMIO_BASE: usize = 0x1000_1000;
53const VIRTIO_MMIO_STRIDE: usize = 0x1000;
54const VIRTIO_MMIO_SLOTS: usize = 8;
55const VIRTIO_MAGIC: u32 = 0x7472_6976;
56const VIRTIO_DEV_BLOCK: u32 = 2;
57
58// Virtqueue
59const VIRTQ_SIZE: usize = 8;
60
61// Virtio-blk request codes
62const VIRTIO_BLK_T_IN: u32 = 0;
63const VIRTIO_BLK_T_OUT: u32 = 1;
64const VIRTIO_BLK_S_OK: u8 = 0;
65
66// ============================================================
67// MMIO access helpers
68// ============================================================
69
70struct MmioRegion {
71    pub base: usize,
72}
73
74impl MmioRegion {
75    fn new(base: usize) -> Self {
76        Self { base }
77    }
78
79    #[inline]
80    fn read32(&self, offset: usize) -> u32 {
81        unsafe { read_volatile((self.base + offset) as *const u32) }
82    }
83
84    #[inline]
85    fn write32(&self, offset: usize, val: u32) {
86        unsafe {
87            write_volatile((self.base + offset) as *mut u32, val);
88        }
89    }
90}
91
92// ============================================================
93// Virtqueue structures
94// ============================================================
95
96#[repr(C, align(16))]
97struct VirtqDesc {
98    addr: u64,
99    len: u32,
100    flags: u16,
101    next: u16,
102}
103
104const VIRTQ_DESC_F_NEXT: u16 = 1;
105const VIRTQ_DESC_F_WRITE: u16 = 2;
106
107#[repr(C, align(2))]
108struct VirtqAvail {
109    flags: u16,
110    idx: u16,
111    ring: [u16; VIRTQ_SIZE],
112}
113
114#[repr(C, align(4))]
115#[derive(Clone, Copy)]
116struct VirtqUsedElem {
117    id: u32,
118    len: u32,
119}
120
121#[repr(C, align(4))]
122struct VirtqUsed {
123    flags: u16,
124    idx: u16,
125    ring: [VirtqUsedElem; VIRTQ_SIZE],
126}
127
128struct VirtqMemory {
129    desc: *mut VirtqDesc,
130    avail: *mut VirtqAvail,
131    used: *mut VirtqUsed,
132    desc_phys: u64,
133    _backing: Vec<u8>,
134}
135
136impl VirtqMemory {
137    /// Allocate contiguous page-aligned memory for legacy virtqueue layout:
138    ///   Page 0: Descriptor Table | Available Ring
139    ///   Page 1: Used Ring (must be page-aligned per QUEUE_ALIGN)
140    fn new() -> Option<Self> {
141        let desc_size = core::mem::size_of::<VirtqDesc>() * VIRTQ_SIZE;
142        let page_size = 4096usize;
143        let total_size = page_size * 2;
144        let mut backing = vec![0u8; total_size + page_size];
145        let base = backing.as_mut_ptr() as usize;
146        let aligned = (base + page_size - 1) & !(page_size - 1);
147
148        let desc = aligned as *mut VirtqDesc;
149        let avail = (aligned + desc_size) as *mut VirtqAvail;
150        let used = (aligned + page_size) as *mut VirtqUsed;
151
152        unsafe {
153            for i in 0..VIRTQ_SIZE {
154                (*desc.add(i)) = VirtqDesc {
155                    addr: 0,
156                    len: 0,
157                    flags: 0,
158                    next: 0,
159                };
160            }
161            (*avail) = VirtqAvail {
162                flags: 0,
163                idx: 0,
164                ring: [0u16; VIRTQ_SIZE],
165            };
166            (*used) = VirtqUsed {
167                flags: 0,
168                idx: 0,
169                ring: [VirtqUsedElem { id: 0, len: 0 }; VIRTQ_SIZE],
170            };
171        }
172
173        Some(Self {
174            desc,
175            avail,
176            used,
177            desc_phys: aligned as u64,
178            _backing: backing,
179        })
180    }
181}
182
183#[repr(C)]
184struct BlkReq {
185    req_type: u32,
186    reserved: u32,
187    sector: u64,
188}
189
190// ============================================================
191// Global device reference for interrupt handler
192// ============================================================
193
194static VIRTIO_DEV_PTR: Mutex<Option<usize>> = Mutex::new(None);
195static VIRTIO_IO_COMPLETE: AtomicBool = AtomicBool::new(false);
196
197pub fn try_handle_interrupt() {
198    let ptr = *VIRTIO_DEV_PTR.lock();
199    if let Some(mmio_base) = ptr {
200        let mmio = MmioRegion::new(mmio_base);
201        let status = mmio.read32(INTERRUPT_STATUS);
202        if status & 1 != 0 {
203            mmio.write32(INTERRUPT_ACK, status);
204            fence(Ordering::SeqCst);
205            VIRTIO_IO_COMPLETE.store(true, Ordering::Release);
206        }
207    }
208}
209
210// ============================================================
211// VirtIO Block Device
212// ============================================================
213
214pub struct VirtioBlkDev {
215    mmio: MmioRegion,
216    vq: Mutex<VirtqMemory>,
217    capacity: usize,
218    last_used_idx: Mutex<u16>,
219}
220
221impl VirtioBlkDev {
222    /// Scan MMIO region for a virtio-blk device and initialize it.
223    pub fn probe() -> Option<Self> {
224        for slot in 0..VIRTIO_MMIO_SLOTS {
225            let base = VIRTIO_MMIO_BASE + slot * VIRTIO_MMIO_STRIDE;
226            let mmio = MmioRegion::new(base);
227
228            if mmio.read32(MAGIC_VALUE) != VIRTIO_MAGIC {
229                continue;
230            }
231            if mmio.read32(DEVICE_ID) != VIRTIO_DEV_BLOCK {
232                continue;
233            }
234
235            crate::serial_println!(
236                "[VIRTIO] Found virtio-blk at {:#x}, version={}",
237                base,
238                mmio.read32(VERSION)
239            );
240
241            return Self::init_device(mmio);
242        }
243        crate::serial_println!("[VIRTIO] No virtio-blk device found");
244        None
245    }
246
247    fn init_device(mmio: MmioRegion) -> Option<Self> {
248        // Reset device
249        mmio.write32(STATUS, 0);
250        fence(Ordering::SeqCst);
251
252        // Acknowledge
253        mmio.write32(STATUS, STATUS_ACKNOWLEDGE);
254        fence(Ordering::SeqCst);
255
256        // Driver
257        mmio.write32(STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
258        fence(Ordering::SeqCst);
259
260        // Negotiate features (none)
261        mmio.write32(DEVICE_FEATURES_SEL, 0);
262        mmio.read32(DEVICE_FEATURES);
263        mmio.write32(DEVICE_FEATURES_SEL, 1);
264        mmio.read32(DEVICE_FEATURES);
265        mmio.write32(DRIVER_FEATURES_SEL, 0);
266        mmio.write32(DRIVER_FEATURES, 0);
267        mmio.write32(DRIVER_FEATURES_SEL, 1);
268        mmio.write32(DRIVER_FEATURES, 0);
269
270        // Features OK
271        mmio.write32(
272            STATUS,
273            STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK,
274        );
275        fence(Ordering::SeqCst);
276
277        let status = mmio.read32(STATUS);
278        if status & STATUS_FEATURES_OK == 0 {
279            crate::serial_println!("[VIRTIO] Feature negotiation failed, status={:#x}", status);
280            return None;
281        }
282
283        // Setup virtqueue (legacy mode via QUEUE_PFN)
284        let vq = VirtqMemory::new()?;
285
286        mmio.write32(QUEUE_SEL, 0);
287        fence(Ordering::SeqCst);
288
289        let max_size = mmio.read32(QUEUE_NUM_MAX);
290        if max_size == 0 {
291            return None;
292        }
293
294        let queue_size = core::cmp::min(VIRTQ_SIZE as u32, max_size);
295        mmio.write32(QUEUE_NUM, queue_size);
296        fence(Ordering::SeqCst);
297
298        let page_size = 4096u32;
299        mmio.write32(GUEST_PAGE_SIZE, page_size);
300        mmio.write32(QUEUE_ALIGN, page_size);
301        mmio.write32(QUEUE_PFN, (vq.desc_phys as u32) / page_size);
302        fence(Ordering::SeqCst);
303
304        // Driver OK
305        mmio.write32(
306            STATUS,
307            STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK,
308        );
309        fence(Ordering::SeqCst);
310
311        // Read capacity from config space
312        let capacity_lo = mmio.read32(DEVICE_CONFIG);
313        let capacity_hi = mmio.read32(DEVICE_CONFIG + 4);
314        let capacity = ((capacity_hi as u64) << 32 | capacity_lo as u64) as usize;
315
316        // Enable interrupts on the device
317        mmio.write32(INTERRUPT_ACK, 0x3); // clear any pending interrupts
318        fence(Ordering::SeqCst);
319
320        // Store MMIO base for the external interrupt handler
321        *VIRTIO_DEV_PTR.lock() = Some(mmio.base);
322
323        crate::serial_println!(
324            "[VIRTIO] Device ready: {} sectors ({} MB), interrupts enabled",
325            capacity,
326            capacity * 512 / (1024 * 1024)
327        );
328
329        Some(Self {
330            mmio,
331            vq: Mutex::new(vq),
332            capacity,
333            last_used_idx: Mutex::new(0),
334        })
335    }
336
337    /// Submit a block I/O request using a 3-descriptor chain.
338    /// Polls for completion.
339    fn submit_io(&self, sector: u64, buf: &mut [u8], is_write: bool) -> Result<(), FileError> {
340        if buf.len() != 512 {
341            return Err(FileError::InvalidOperation);
342        }
343        if sector >= self.capacity as u64 && self.capacity > 0 {
344            return Err(FileError::InvalidOperation);
345        }
346
347        let req = BlkReq {
348            req_type: if is_write {
349                VIRTIO_BLK_T_OUT
350            } else {
351                VIRTIO_BLK_T_IN
352            },
353            reserved: 0,
354            sector,
355        };
356
357        let mut status: u8 = 0xFF;
358        let req_phys = &req as *const BlkReq as u64;
359        let buf_phys = buf.as_ptr() as u64;
360        let status_phys = &mut status as *mut u8 as u64;
361
362        let vq = self.vq.lock();
363        let desc = vq.desc;
364
365        // Build 3-descriptor chain
366        unsafe {
367            (*desc.add(0)).addr = req_phys;
368            (*desc.add(0)).len = core::mem::size_of::<BlkReq>() as u32;
369            (*desc.add(0)).flags = VIRTQ_DESC_F_NEXT;
370            (*desc.add(0)).next = 1;
371
372            (*desc.add(1)).addr = buf_phys;
373            (*desc.add(1)).len = 512;
374            (*desc.add(1)).flags = VIRTQ_DESC_F_NEXT;
375            if !is_write {
376                (*desc.add(1)).flags |= VIRTQ_DESC_F_WRITE;
377            }
378            (*desc.add(1)).next = 2;
379
380            (*desc.add(2)).addr = status_phys;
381            (*desc.add(2)).len = 1;
382            (*desc.add(2)).flags = VIRTQ_DESC_F_WRITE;
383            (*desc.add(2)).next = 0;
384        }
385
386        fence(Ordering::SeqCst);
387
388        // Add to available ring
389        let avail = vq.avail;
390        let used_ptr = vq.used;
391        unsafe {
392            let idx = (*avail).idx;
393            let ring_idx = (idx as usize) % VIRTQ_SIZE;
394            (*avail).ring[ring_idx] = 0;
395            fence(Ordering::SeqCst);
396            (*avail).idx = idx.wrapping_add(1);
397        }
398
399        // Notify device
400        self.mmio.write32(QUEUE_NOTIFY, 0);
401        fence(Ordering::SeqCst);
402        drop(vq);
403
404        // Wait for interrupt-driven completion
405        let mut last = self.last_used_idx.lock();
406        let used = unsafe { &*used_ptr };
407        VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
408        loop {
409            fence(Ordering::SeqCst);
410            let used_idx = unsafe { read_volatile(&used.idx) };
411            if used_idx != *last {
412                *last = used_idx;
413                break;
414            }
415            drop(last);
416            unsafe {
417                riscv::register::sstatus::set_sie();
418            }
419            riscv::asm::wfi();
420            unsafe {
421                riscv::register::sstatus::clear_sie();
422            }
423            last = self.last_used_idx.lock();
424        }
425        VIRTIO_IO_COMPLETE.store(false, Ordering::Release);
426
427        fence(Ordering::SeqCst);
428
429        if status != VIRTIO_BLK_S_OK {
430            crate::serial_println!("[VIRTIO] I/O error: status={:#x}", status);
431            return Err(FileError::IoError);
432        }
433
434        Ok(())
435    }
436}
437
438// ============================================================
439// BlockDevice trait implementation
440// ============================================================
441
442impl BlockDevice for VirtioBlkDev {
443    fn read_block(&self, block_no: usize, buf: &mut [u8]) -> Result<(), FileError> {
444        if buf.len() != 512 {
445            return Err(FileError::InvalidOperation);
446        }
447        self.submit_io(block_no as u64, buf, false)
448    }
449
450    fn write_block(&self, block_no: usize, buf: &[u8]) -> Result<(), FileError> {
451        if buf.len() != 512 {
452            return Err(FileError::InvalidOperation);
453        }
454        let mut tmp = [0u8; 512];
455        tmp.copy_from_slice(buf);
456        self.submit_io(block_no as u64, &mut tmp, true)
457    }
458
459    fn block_count(&self) -> usize {
460        self.capacity
461    }
462}
463
464unsafe impl Send for VirtioBlkDev {}
465unsafe impl Sync for VirtioBlkDev {}