1use virtio_bindings::virtio_ring;
7
8use crate::error::{Result, VirtioError};
9
10const VRING_AVAIL_F_NO_INTERRUPT: u16 = virtio_ring::VRING_AVAIL_F_NO_INTERRUPT as u16;
12
13pub const VIRTIO_F_EVENT_IDX: u64 = 1 << virtio_ring::VIRTIO_RING_F_EVENT_IDX;
15
16pub mod flags {
18 use super::virtio_ring;
19
20 pub const NEXT: u16 = virtio_ring::VRING_DESC_F_NEXT as u16;
22 pub const WRITE: u16 = virtio_ring::VRING_DESC_F_WRITE as u16;
24 pub const INDIRECT: u16 = virtio_ring::VRING_DESC_F_INDIRECT as u16;
26}
27
28#[derive(Debug, Clone, Copy, Default)]
30#[repr(C)]
31pub struct Descriptor {
32 pub addr: u64,
34 pub len: u32,
36 pub flags: u16,
38 pub next: u16,
40}
41
42impl Descriptor {
43 #[must_use]
45 pub const fn has_next(&self) -> bool {
46 self.flags & flags::NEXT != 0
47 }
48
49 #[must_use]
51 pub const fn is_write_only(&self) -> bool {
52 self.flags & flags::WRITE != 0
53 }
54
55 #[must_use]
57 pub const fn is_indirect(&self) -> bool {
58 self.flags & flags::INDIRECT != 0
59 }
60}
61
62#[derive(Debug)]
64pub struct AvailRing {
65 pub flags: u16,
67 pub idx: u16,
69 pub ring: Vec<u16>,
71 pub used_event: u16,
73}
74
75impl AvailRing {
76 #[must_use]
78 pub fn new(size: u16) -> Self {
79 Self {
80 flags: 0,
81 idx: 0,
82 ring: vec![0; size as usize],
83 used_event: 0,
84 }
85 }
86}
87
88#[derive(Debug, Clone, Copy, Default)]
90#[repr(C)]
91pub struct UsedElement {
92 pub id: u32,
94 pub len: u32,
96}
97
98#[derive(Debug)]
100pub struct UsedRing {
101 pub flags: u16,
103 pub idx: u16,
105 pub ring: Vec<UsedElement>,
107 pub avail_event: u16,
109}
110
111impl UsedRing {
112 #[must_use]
114 pub fn new(size: u16) -> Self {
115 Self {
116 flags: 0,
117 idx: 0,
118 ring: vec![UsedElement::default(); size as usize],
119 avail_event: 0,
120 }
121 }
122}
123
124#[derive(Debug)]
126pub struct VirtQueue {
127 size: u16,
129 desc_table: Vec<Descriptor>,
131 avail: AvailRing,
133 used: UsedRing,
135 last_avail_idx: u16,
137 ready: bool,
139 event_idx_enabled: bool,
141}
142
143impl VirtQueue {
144 pub fn new(size: u16) -> Result<Self> {
150 if size == 0 || !size.is_power_of_two() {
151 return Err(VirtioError::InvalidQueue(
152 "size must be a power of 2".to_string(),
153 ));
154 }
155
156 if size > 32768 {
157 return Err(VirtioError::InvalidQueue(
158 "size must not exceed 32768".to_string(),
159 ));
160 }
161
162 Ok(Self {
163 size,
164 desc_table: vec![Descriptor::default(); size as usize],
165 avail: AvailRing::new(size),
166 used: UsedRing::new(size),
167 last_avail_idx: 0,
168 ready: false,
169 event_idx_enabled: false,
170 })
171 }
172
173 #[must_use]
175 pub const fn size(&self) -> u16 {
176 self.size
177 }
178
179 #[must_use]
181 pub const fn is_ready(&self) -> bool {
182 self.ready
183 }
184
185 pub const fn set_ready(&mut self, ready: bool) {
187 self.ready = ready;
188 }
189
190 pub fn set_descriptor(&mut self, idx: u16, descriptor: Descriptor) -> Result<()> {
196 if idx >= self.size {
197 return Err(VirtioError::InvalidQueue(
198 "descriptor index out of bounds".to_string(),
199 ));
200 }
201
202 self.desc_table[idx as usize] = descriptor;
203 Ok(())
204 }
205
206 pub fn add_avail(&mut self, head_idx: u16) -> Result<()> {
212 if head_idx >= self.size {
213 return Err(VirtioError::InvalidQueue(
214 "available index out of bounds".to_string(),
215 ));
216 }
217
218 let ring_idx = (self.avail.idx % self.size) as usize;
219 self.avail.ring[ring_idx] = head_idx;
220 self.avail.idx = self.avail.idx.wrapping_add(1);
221 Ok(())
222 }
223
224 #[must_use]
226 pub const fn has_available(&self) -> bool {
227 self.avail.idx != self.last_avail_idx
228 }
229
230 pub fn pop_avail(&mut self) -> Option<(u16, DescriptorChain)> {
234 if !self.has_available() {
235 return None;
236 }
237
238 let avail_idx = self.last_avail_idx;
239 let head_idx = self.avail.ring[(avail_idx % self.size) as usize];
240 self.last_avail_idx = self.last_avail_idx.wrapping_add(1);
241
242 Some((
243 head_idx,
244 DescriptorChain {
245 queue: self,
246 current: Some(head_idx),
247 ttl: self.size,
248 },
249 ))
250 }
251
252 pub fn set_event_idx(&mut self, enabled: bool) {
254 self.event_idx_enabled = enabled;
255 }
256
257 pub fn push_used(&mut self, head_idx: u16, len: u32) -> bool {
262 let old_idx = self.used.idx;
263 self.used.ring[(old_idx % self.size) as usize] = UsedElement {
264 id: head_idx as u32,
265 len,
266 };
267 self.used.idx = self.used.idx.wrapping_add(1);
268 self.should_notify(old_idx)
269 }
270
271 pub fn push_used_batch(&mut self, completions: &[(u16, u32)]) -> bool {
276 if completions.is_empty() {
277 return false;
278 }
279 let old_idx = self.used.idx;
280 for &(head_idx, len) in completions {
281 let slot = (self.used.idx % self.size) as usize;
282 self.used.ring[slot] = UsedElement {
283 id: head_idx as u32,
284 len,
285 };
286 self.used.idx = self.used.idx.wrapping_add(1);
287 }
288 self.should_notify(old_idx)
289 }
290
291 pub fn set_avail_event(&mut self, event: u16) {
293 self.used.avail_event = event;
294 }
295
296 fn should_notify(&self, old_idx: u16) -> bool {
298 if self.event_idx_enabled {
299 Self::needs_notification(old_idx, self.used.idx, self.avail.used_event)
300 } else {
301 (self.avail.flags & VRING_AVAIL_F_NO_INTERRUPT) == 0
302 }
303 }
304
305 fn needs_notification(old_idx: u16, new_idx: u16, used_event: u16) -> bool {
307 new_idx.wrapping_sub(used_event).wrapping_sub(1) < new_idx.wrapping_sub(old_idx)
308 }
309
310 #[must_use]
312 pub fn get_descriptor(&self, idx: u16) -> Option<&Descriptor> {
313 self.desc_table.get(idx as usize)
314 }
315}
316
317pub struct DescriptorChain<'a> {
325 queue: &'a VirtQueue,
326 current: Option<u16>,
327 ttl: u16,
328}
329
330impl<'a> Iterator for DescriptorChain<'a> {
331 type Item = &'a Descriptor;
332
333 fn next(&mut self) -> Option<Self::Item> {
334 if self.ttl == 0 {
335 return None;
336 }
337 let idx = self.current?;
338 let desc = self.queue.get_descriptor(idx)?;
339 self.ttl -= 1;
340
341 self.current = if desc.has_next() {
342 Some(desc.next)
343 } else {
344 None
345 };
346
347 Some(desc)
348 }
349}
350
351#[cfg(test)]
352mod tests {
353 use super::*;
354
355 #[test]
360 fn test_descriptor_default() {
361 let desc = Descriptor::default();
362 assert_eq!(desc.addr, 0);
363 assert_eq!(desc.len, 0);
364 assert_eq!(desc.flags, 0);
365 assert_eq!(desc.next, 0);
366 }
367
368 #[test]
369 fn test_descriptor_has_next() {
370 let mut desc = Descriptor::default();
371 assert!(!desc.has_next());
372
373 desc.flags = flags::NEXT;
374 assert!(desc.has_next());
375 }
376
377 #[test]
378 fn test_descriptor_is_write_only() {
379 let mut desc = Descriptor::default();
380 assert!(!desc.is_write_only());
381
382 desc.flags = flags::WRITE;
383 assert!(desc.is_write_only());
384 }
385
386 #[test]
387 fn test_descriptor_is_indirect() {
388 let mut desc = Descriptor::default();
389 assert!(!desc.is_indirect());
390
391 desc.flags = flags::INDIRECT;
392 assert!(desc.is_indirect());
393 }
394
395 #[test]
396 fn test_descriptor_multiple_flags() {
397 let desc = Descriptor {
398 addr: 0x1000,
399 len: 512,
400 flags: flags::NEXT | flags::WRITE,
401 next: 1,
402 };
403
404 assert!(desc.has_next());
405 assert!(desc.is_write_only());
406 assert!(!desc.is_indirect());
407 }
408
409 #[test]
410 fn test_descriptor_clone_copy() {
411 let desc = Descriptor {
412 addr: 0xDEADBEEF,
413 len: 1234,
414 flags: flags::NEXT,
415 next: 42,
416 };
417
418 let cloned = desc;
419 let copied = desc; assert_eq!(cloned.addr, 0xDEADBEEF);
422 assert_eq!(copied.addr, 0xDEADBEEF);
423 }
424
425 #[test]
430 fn test_flag_constants() {
431 assert_eq!(flags::NEXT, 1);
432 assert_eq!(flags::WRITE, 2);
433 assert_eq!(flags::INDIRECT, 4);
434 }
435
436 #[test]
441 fn test_avail_ring_new() {
442 let ring = AvailRing::new(256);
443 assert_eq!(ring.flags, 0);
444 assert_eq!(ring.idx, 0);
445 assert_eq!(ring.ring.len(), 256);
446 assert_eq!(ring.used_event, 0);
447 }
448
449 #[test]
450 fn test_avail_ring_small() {
451 let ring = AvailRing::new(1);
452 assert_eq!(ring.ring.len(), 1);
453 }
454
455 #[test]
460 fn test_used_element_default() {
461 let elem = UsedElement::default();
462 assert_eq!(elem.id, 0);
463 assert_eq!(elem.len, 0);
464 }
465
466 #[test]
467 fn test_used_element_clone_copy() {
468 let elem = UsedElement { id: 42, len: 1024 };
469 let cloned = elem;
470 let copied = elem; assert_eq!(cloned.id, 42);
473 assert_eq!(copied.len, 1024);
474 }
475
476 #[test]
481 fn test_used_ring_new() {
482 let ring = UsedRing::new(128);
483 assert_eq!(ring.flags, 0);
484 assert_eq!(ring.idx, 0);
485 assert_eq!(ring.ring.len(), 128);
486 assert_eq!(ring.avail_event, 0);
487 }
488
489 #[test]
494 fn test_virtqueue_new() {
495 let queue = VirtQueue::new(256).unwrap();
496 assert_eq!(queue.size(), 256);
497 assert!(!queue.is_ready());
498 }
499
500 #[test]
501 fn test_virtqueue_new_power_of_two() {
502 for size in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] {
504 assert!(VirtQueue::new(size).is_ok());
505 }
506 }
507
508 #[test]
509 fn test_virtqueue_new_invalid_size_zero() {
510 let result = VirtQueue::new(0);
511 assert!(result.is_err());
512 if let Err(VirtioError::InvalidQueue(msg)) = result {
513 assert!(msg.contains("power of 2"));
514 }
515 }
516
517 #[test]
518 fn test_virtqueue_new_invalid_size_not_power_of_two() {
519 for size in [3, 5, 6, 7, 9, 100, 1000] {
520 let result = VirtQueue::new(size);
521 assert!(result.is_err());
522 }
523 }
524
525 #[test]
526 fn test_virtqueue_new_too_large() {
527 let result = VirtQueue::new(32768); assert!(result.is_ok());
529
530 }
533
534 #[test]
535 fn test_virtqueue_ready_state() {
536 let mut queue = VirtQueue::new(16).unwrap();
537
538 assert!(!queue.is_ready());
539 queue.set_ready(true);
540 assert!(queue.is_ready());
541 queue.set_ready(false);
542 assert!(!queue.is_ready());
543 }
544
545 #[test]
546 fn test_virtqueue_has_available_empty() {
547 let queue = VirtQueue::new(16).unwrap();
548 assert!(!queue.has_available());
549 }
550
551 #[test]
552 fn test_virtqueue_pop_avail_empty() {
553 let mut queue = VirtQueue::new(16).unwrap();
554 assert!(queue.pop_avail().is_none());
555 }
556
557 #[test]
558 fn test_virtqueue_get_descriptor() {
559 let queue = VirtQueue::new(16).unwrap();
560
561 assert!(queue.get_descriptor(0).is_some());
563 assert!(queue.get_descriptor(15).is_some());
564
565 assert!(queue.get_descriptor(16).is_none());
567 assert!(queue.get_descriptor(100).is_none());
568 }
569
570 #[test]
571 fn test_virtqueue_push_used() {
572 let mut queue = VirtQueue::new(16).unwrap();
573
574 let _notify = queue.push_used(0, 512);
575 assert_eq!(queue.used.idx, 1);
576 assert_eq!(queue.used.ring[0].id, 0);
577 assert_eq!(queue.used.ring[0].len, 512);
578
579 let _notify = queue.push_used(1, 1024);
580 assert_eq!(queue.used.idx, 2);
581 assert_eq!(queue.used.ring[1].id, 1);
582 assert_eq!(queue.used.ring[1].len, 1024);
583 }
584
585 #[test]
586 fn test_virtqueue_push_used_wrap() {
587 let mut queue = VirtQueue::new(4).unwrap();
588
589 for i in 0..10 {
591 let _notify = queue.push_used(i, i as u32 * 100);
592 }
593
594 assert_eq!(queue.used.idx, 10);
595 assert_eq!(queue.used.ring[1].id, 9);
597 }
598
599 #[test]
600 fn test_virtqueue_simulated_transaction() {
601 let mut queue = VirtQueue::new(16).unwrap();
602 queue.set_ready(true);
603
604 queue.avail.ring[0] = 0; queue.avail.idx = 1;
607
608 assert!(queue.has_available());
610
611 let (head_idx, _chain) = queue.pop_avail().unwrap();
613 assert_eq!(head_idx, 0);
614
615 assert!(!queue.has_available());
617
618 let _notify = queue.push_used(head_idx, 256);
620 assert_eq!(queue.used.idx, 1);
621 }
622
623 #[test]
624 fn test_virtqueue_multiple_descriptors() {
625 let mut queue = VirtQueue::new(16).unwrap();
626 queue.set_ready(true);
627
628 for i in 0..5 {
630 queue.avail.ring[i] = i as u16;
631 }
632 queue.avail.idx = 5;
633
634 for i in 0..5 {
636 assert!(queue.has_available());
637 let (head_idx, _) = queue.pop_avail().unwrap();
638 assert_eq!(head_idx, i as u16);
639 }
640
641 assert!(!queue.has_available());
642 }
643
644 #[test]
649 fn test_descriptor_chain_single() {
650 let mut queue = VirtQueue::new(16).unwrap();
651
652 queue.desc_table[0] = Descriptor {
654 addr: 0x1000,
655 len: 512,
656 flags: 0, next: 0,
658 };
659
660 queue.avail.ring[0] = 0;
661 queue.avail.idx = 1;
662
663 let (head_idx, chain) = queue.pop_avail().unwrap();
664 assert_eq!(head_idx, 0);
665
666 let descs: Vec<_> = chain.collect();
667 assert_eq!(descs.len(), 1);
668 assert_eq!(descs[0].addr, 0x1000);
669 assert_eq!(descs[0].len, 512);
670 }
671
672 #[test]
673 fn test_descriptor_chain_multiple() {
674 let mut queue = VirtQueue::new(16).unwrap();
675
676 queue.desc_table[0] = Descriptor {
678 addr: 0x1000,
679 len: 256,
680 flags: flags::NEXT,
681 next: 1,
682 };
683 queue.desc_table[1] = Descriptor {
684 addr: 0x2000,
685 len: 512,
686 flags: flags::NEXT,
687 next: 2,
688 };
689 queue.desc_table[2] = Descriptor {
690 addr: 0x3000,
691 len: 1024,
692 flags: 0, next: 0,
694 };
695
696 queue.avail.ring[0] = 0;
697 queue.avail.idx = 1;
698
699 let (_, chain) = queue.pop_avail().unwrap();
700 let descs: Vec<_> = chain.collect();
701
702 assert_eq!(descs.len(), 3);
703 assert_eq!(descs[0].addr, 0x1000);
704 assert_eq!(descs[1].addr, 0x2000);
705 assert_eq!(descs[2].addr, 0x3000);
706 }
707
708 #[test]
709 fn test_descriptor_chain_with_write_flags() {
710 let mut queue = VirtQueue::new(16).unwrap();
711
712 queue.desc_table[0] = Descriptor {
714 addr: 0x1000,
715 len: 256,
716 flags: flags::NEXT, next: 1,
718 };
719 queue.desc_table[1] = Descriptor {
720 addr: 0x2000,
721 len: 512,
722 flags: flags::WRITE, next: 0,
724 };
725
726 queue.avail.ring[0] = 0;
727 queue.avail.idx = 1;
728
729 let (_, chain) = queue.pop_avail().unwrap();
730 let descs: Vec<_> = chain.collect();
731
732 assert_eq!(descs.len(), 2);
733 assert!(!descs[0].is_write_only());
734 assert!(descs[1].is_write_only());
735 }
736
737 #[test]
738 fn test_descriptor_chain_cycle_terminates() {
739 let mut queue = VirtQueue::new(16).unwrap();
742
743 queue.desc_table[0] = Descriptor {
745 addr: 0x1000,
746 len: 16,
747 flags: flags::NEXT,
748 next: 1,
749 };
750 queue.desc_table[1] = Descriptor {
751 addr: 0x2000,
752 len: 16,
753 flags: flags::NEXT,
754 next: 0,
755 };
756
757 queue.avail.ring[0] = 0;
758 queue.avail.idx = 1;
759
760 let (_, chain) = queue.pop_avail().unwrap();
761 let descs: Vec<_> = chain.collect();
762
763 assert!(
765 descs.len() <= 16,
766 "chain iteration exceeded queue size cap: got {}",
767 descs.len()
768 );
769 }
770
771 #[test]
772 fn test_descriptor_chain_self_loop_terminates() {
773 let mut queue = VirtQueue::new(8).unwrap();
775
776 queue.desc_table[0] = Descriptor {
777 addr: 0x1000,
778 len: 16,
779 flags: flags::NEXT,
780 next: 0,
781 };
782
783 queue.avail.ring[0] = 0;
784 queue.avail.idx = 1;
785
786 let (_, chain) = queue.pop_avail().unwrap();
787 let descs: Vec<_> = chain.collect();
788 assert!(descs.len() <= 8);
789 }
790
791 #[test]
796 fn test_avail_idx_wrap() {
797 let mut queue = VirtQueue::new(4).unwrap();
798
799 queue.avail.idx = u16::MAX;
801 queue.last_avail_idx = u16::MAX - 1;
802 queue.avail.ring[(queue.last_avail_idx % 4) as usize] = 0;
803
804 assert!(queue.has_available());
805
806 let (head_idx, _) = queue.pop_avail().unwrap();
807 assert_eq!(head_idx, 0);
808 assert_eq!(queue.last_avail_idx, u16::MAX);
809
810 queue.avail.ring[(queue.avail.idx % 4) as usize] = 1;
812 queue.avail.idx = 0; assert!(queue.has_available());
815 let (head_idx, _) = queue.pop_avail().unwrap();
816 assert_eq!(head_idx, 1);
817 }
818
819 #[test]
820 fn test_used_idx_wrap() {
821 let mut queue = VirtQueue::new(4).unwrap();
822 queue.used.idx = u16::MAX;
823
824 let _notify = queue.push_used(0, 100);
825 assert_eq!(queue.used.idx, 0); assert_eq!(queue.used.ring[3].id, 0);
829 assert_eq!(queue.used.ring[3].len, 100);
830 }
831
832 #[test]
833 fn test_queue_min_size() {
834 let queue = VirtQueue::new(1).unwrap();
835 assert_eq!(queue.size(), 1);
836 assert_eq!(queue.desc_table.len(), 1);
837 assert_eq!(queue.avail.ring.len(), 1);
838 assert_eq!(queue.used.ring.len(), 1);
839 }
840
841 #[test]
842 fn test_queue_max_size() {
843 let queue = VirtQueue::new(32768).unwrap();
844 assert_eq!(queue.size(), 32768);
845 assert_eq!(queue.desc_table.len(), 32768);
846 }
847
848 #[test]
849 fn test_descriptor_chain_out_of_bounds() {
850 let mut queue = VirtQueue::new(4).unwrap();
851
852 queue.desc_table[0] = Descriptor {
854 addr: 0x1000,
855 len: 256,
856 flags: flags::NEXT,
857 next: 100, };
859
860 queue.avail.ring[0] = 0;
861 queue.avail.idx = 1;
862
863 let (_, chain) = queue.pop_avail().unwrap();
864 let descs: Vec<_> = chain.collect();
865
866 assert_eq!(descs.len(), 1);
868 }
869
870 #[test]
875 fn test_needs_notification_basic() {
876 assert!(VirtQueue::needs_notification(0, 1, 0));
878 }
879
880 #[test]
881 fn test_needs_notification_past_event() {
882 assert!(!VirtQueue::needs_notification(5, 6, 3));
884 }
885
886 #[test]
887 fn test_needs_notification_wrap() {
888 assert!(VirtQueue::needs_notification(65534, 0, 65535));
890 }
891
892 #[test]
893 fn test_needs_notification_wrap_no_notify() {
894 assert!(!VirtQueue::needs_notification(65534, 65535, 0));
896 }
897
898 #[test]
899 fn test_push_used_no_interrupt_flag() {
900 let mut queue = VirtQueue::new(16).unwrap();
901 queue.avail.flags = VRING_AVAIL_F_NO_INTERRUPT;
903 let notify = queue.push_used(0, 512);
904 assert!(!notify); }
906
907 #[test]
908 fn test_push_used_default_notifies() {
909 let mut queue = VirtQueue::new(16).unwrap();
910 let notify = queue.push_used(0, 512);
911 assert!(notify); }
913
914 #[test]
915 fn test_push_used_event_idx_suppresses() {
916 let mut queue = VirtQueue::new(16).unwrap();
917 queue.set_event_idx(true);
918 queue.avail.used_event = 5;
920 let notify = queue.push_used(0, 512);
922 assert!(!notify);
923 }
924
925 #[test]
926 fn test_push_used_event_idx_triggers() {
927 let mut queue = VirtQueue::new(16).unwrap();
928 queue.set_event_idx(true);
929 queue.avail.used_event = 0;
930 let notify = queue.push_used(0, 512);
932 assert!(notify);
933 }
934
935 #[test]
936 fn test_push_used_batch() {
937 let mut queue = VirtQueue::new(16).unwrap();
938 let completions = vec![(0, 256), (1, 512), (2, 1024)];
939 let notify = queue.push_used_batch(&completions);
940 assert!(notify); assert_eq!(queue.used.idx, 3);
942 assert_eq!(queue.used.ring[0].id, 0);
943 assert_eq!(queue.used.ring[1].id, 1);
944 assert_eq!(queue.used.ring[2].id, 2);
945 }
946
947 #[test]
948 fn test_push_used_batch_empty() {
949 let mut queue = VirtQueue::new(16).unwrap();
950 let notify = queue.push_used_batch(&[]);
951 assert!(!notify); assert_eq!(queue.used.idx, 0);
953 }
954
955 #[test]
956 fn test_set_avail_event() {
957 let mut queue = VirtQueue::new(16).unwrap();
958 queue.set_avail_event(42);
959 assert_eq!(queue.used.avail_event, 42);
960 }
961}