1pub mod canonical;
65
66use core::fmt;
67use core::marker::PhantomData;
68use core::sync::atomic::{Ordering, fence};
69
70use bytemuck::Zeroable;
71use smallvec::SmallVec;
72use thiserror::Error;
73
74use super::desc::{DescFlags, DescTable, Descriptor};
75use super::event::{EventFlags, EventSuppression};
76use super::{Layout, MemOps};
77
78#[derive(Debug, Copy, Clone, Zeroable)]
84pub struct BufferElement {
85 pub addr: u64,
87 pub len: u32,
89 pub writable: bool,
91}
92
93pub(super) type ChainElems = SmallVec<[BufferElement; 4]>;
94
95impl BufferElement {
96 pub fn readable(addr: u64) -> Self {
98 Self {
99 addr,
100 len: 0,
101 writable: false,
102 }
103 }
104
105 pub fn writable(addr: u64, len: u32) -> Self {
107 Self {
108 addr,
109 len,
110 writable: true,
111 }
112 }
113}
114
115#[derive(Debug, Copy, Clone)]
121pub struct UsedBuffer {
122 pub id: u16,
124 pub len: u32,
127}
128
129#[derive(Debug, Copy, Clone)]
134pub struct SubmitResult {
135 pub id: u16,
138 pub notify: bool,
141}
142
143#[derive(Debug, Copy, Clone, PartialEq, Eq)]
145pub enum MemOp {
146 ReadDesc,
148 WriteDesc,
150 ReadEvent,
152 WriteEvent,
154}
155
156impl fmt::Display for MemOp {
157 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
158 match self {
159 Self::ReadDesc => f.write_str("reading descriptor"),
160 Self::WriteDesc => f.write_str("writing descriptor"),
161 Self::ReadEvent => f.write_str("reading event suppression"),
162 Self::WriteEvent => f.write_str("writing event suppression"),
163 }
164 }
165}
166
167#[derive(Error, Debug)]
168pub enum RingError {
169 #[error("Buffer chain is empty")]
170 EmptyChain,
171 #[error("Buffer chain is malformed")]
172 BadChain,
173 #[error("Operation would block")]
174 WouldBlock,
175 #[error("Out of memory")]
176 OutOfMemory,
177 #[error("Failed to allocate descriptor bookkeeping")]
179 Bookkeeping,
180 #[error("Invalid state")]
181 InvalidState,
182 #[error("Invalid memory layout")]
183 InvalidLayout,
184 #[error("Backend memory error while {op} at address 0x{addr:x}, len {len}")]
189 MemError {
190 op: MemOp,
192 addr: u64,
194 len: usize,
196 },
197}
198
199impl RingError {
200 #[inline]
201 fn mem_err(op: MemOp, addr: u64) -> Self {
202 let len = match op {
203 MemOp::ReadDesc | MemOp::WriteDesc => Descriptor::SIZE,
204 MemOp::ReadEvent | MemOp::WriteEvent => EventSuppression::SIZE,
205 };
206
207 Self::MemError { op, addr, len }
208 }
209}
210
211pub struct Readable;
213
214pub struct Writable;
216
217#[derive(Debug, Default)]
225pub struct BufferChainBuilder<T> {
226 elems: ChainElems,
227 split: usize,
228 marker: PhantomData<T>,
229}
230
231impl<T> BufferChainBuilder<T> {
232 pub(super) fn reserve_exact(&mut self, additional: usize) {
233 self.elems.reserve_exact(additional);
234 }
235}
236
237impl BufferChainBuilder<Readable> {
238 pub fn new() -> Self {
240 Self {
241 elems: Default::default(),
242 split: 0,
243 marker: PhantomData,
244 }
245 }
246
247 pub fn readable(mut self, addr: u64, len: u32) -> Self {
249 self.elems.push(BufferElement {
250 addr,
251 len,
252 writable: false,
253 });
254 self.split += 1;
255 self
256 }
257
258 pub fn readables(
260 mut self,
261 elements: impl IntoIterator<Item = impl Into<BufferElement>>,
262 ) -> Self {
263 for elem in elements {
264 let mut elem = elem.into();
265 elem.writable = false;
266 self.elems.push(elem);
267 self.split += 1;
268 }
269
270 self
271 }
272
273 pub fn writable(mut self, addr: u64, len: u32) -> BufferChainBuilder<Writable> {
277 self.elems.push(BufferElement {
278 addr,
279 len,
280 writable: true,
281 });
282
283 BufferChainBuilder {
284 elems: self.elems,
285 split: self.split,
286 marker: PhantomData,
287 }
288 }
289
290 pub fn writables(
294 mut self,
295 elements: impl IntoIterator<Item = impl Into<BufferElement>>,
296 ) -> BufferChainBuilder<Writable> {
297 for elem in elements {
298 let mut elem = elem.into();
299 elem.writable = true;
300 self.elems.push(elem);
301 }
302
303 BufferChainBuilder {
304 elems: self.elems,
305 split: self.split,
306 marker: PhantomData,
307 }
308 }
309
310 pub fn build(self) -> Result<BufferChain, RingError> {
314 if self.elems.is_empty() {
315 return Err(RingError::EmptyChain);
316 }
317
318 Ok(BufferChain {
319 elems: self.elems,
320 split: self.split,
321 })
322 }
323}
324
325impl BufferChainBuilder<Writable> {
326 pub fn writable(mut self, addr: u64, len: u32) -> Self {
328 self.elems.push(BufferElement {
329 addr,
330 len,
331 writable: true,
332 });
333 self
334 }
335
336 pub fn writables(
338 mut self,
339 elements: impl IntoIterator<Item = impl Into<BufferElement>>,
340 ) -> Self {
341 for elem in elements {
342 let mut elem = elem.into();
343 elem.writable = true;
344 self.elems.push(elem);
345 }
346 self
347 }
348
349 pub fn build(self) -> Result<BufferChain, RingError> {
353 if self.elems.is_empty() {
354 return Err(RingError::EmptyChain);
355 }
356
357 Ok(BufferChain {
358 elems: self.elems,
359 split: self.split,
360 })
361 }
362}
363
364#[derive(Debug, Clone)]
369pub struct BufferChain {
370 elems: ChainElems,
372 split: usize,
374}
375
376impl BufferChain {
377 pub(super) fn into_parts(mut self) -> Result<(ChainElems, ChainElems), RingError> {
379 let should_move = self.split <= self.elems.len() - self.split;
380
381 let range = if should_move {
382 0..self.split
383 } else {
384 self.split..self.elems.len()
385 };
386
387 let mut detached = ChainElems::new();
388 detached
389 .try_reserve_exact(range.len())
390 .map_err(|_| RingError::Bookkeeping)?;
391
392 detached.extend(self.elems.drain(range));
393 if self.elems.len() <= self.elems.inline_size() {
394 self.elems.shrink_to_fit();
396 }
397
398 Ok(if should_move {
399 (detached, self.elems)
400 } else {
401 (self.elems, detached)
402 })
403 }
404
405 pub fn elems(&self) -> &[BufferElement] {
407 self.elems.as_slice()
408 }
409
410 pub fn readables(&self) -> &[BufferElement] {
412 &self.elems[..self.split]
413 }
414
415 pub fn writables(&self) -> &[BufferElement] {
417 &self.elems[self.split..]
418 }
419
420 #[allow(clippy::len_without_is_empty)]
423 pub fn len(&self) -> usize {
424 self.elems.len()
425 }
426}
427
428#[derive(Debug, Copy, Clone, PartialEq, Eq)]
433pub struct RingCursor {
434 head: u16,
435 size: u16,
436 wrap: bool,
437}
438
439impl RingCursor {
440 pub(crate) fn new(size: usize) -> Self {
441 Self {
442 head: 0,
443 size: size as u16,
444 wrap: true,
445 }
446 }
447
448 #[inline]
450 fn advance(&mut self) {
451 debug_assert!(self.head.checked_add(1).is_some());
452 self.head += 1;
453 if self.head >= self.size {
454 self.head = 0;
455 self.wrap = !self.wrap;
456 }
457 }
458
459 #[inline]
461 fn advance_by(&mut self, n: u16) {
462 debug_assert!(self.head.checked_add(n).is_some());
463 let new = self.head + n;
464 let wraps = new / self.size;
465 self.head = new % self.size;
466 if !wraps.is_multiple_of(2) {
467 self.wrap = !self.wrap;
468 }
469 }
470
471 #[inline]
473 pub fn head(&self) -> u16 {
474 self.head
475 }
476
477 #[inline]
479 pub fn wrap(&self) -> bool {
480 self.wrap
481 }
482
483 #[inline]
485 pub fn reset(&mut self) {
486 self.head = 0;
487 self.wrap = true;
488 }
489}
490
491pub(super) struct Checkpoint {
495 avail_cursor: RingCursor,
497 used_cursor: RingCursor,
499 num_inflight: usize,
501}
502
503#[derive(Debug)]
516pub struct RingProducer<M> {
517 mem: M,
519 avail_cursor: RingCursor,
521 used_cursor: RingCursor,
523 num_free: usize,
525 desc_table: DescTable,
527 event_flags_shadow: EventFlags,
529 drv_evt_addr: u64,
531 dev_evt_addr: u64,
533 id_free: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
535 id_num: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
537}
538
539impl<M: MemOps> RingProducer<M> {
540 pub fn new(layout: Layout, mem: M) -> Self {
542 let size = layout.desc_table_len() as usize;
543 let raw = layout.desc_table_addr();
544
545 let table = unsafe { DescTable::from_raw_parts(raw, size) };
547 let cursor = RingCursor::new(size);
548
549 const DEFAULT_LEN: usize = DescTable::default_len();
550 let id_free = (0..size as u16).collect::<SmallVec<[_; DEFAULT_LEN]>>();
551 let id_num = SmallVec::<[_; DEFAULT_LEN]>::from_elem(0, size);
552
553 let event_flags_shadow = EventFlags::ENABLE;
555
556 Self {
557 mem,
558 avail_cursor: cursor,
559 used_cursor: cursor,
560 num_free: size,
561 desc_table: table,
562 id_free,
563 id_num,
564 event_flags_shadow,
565 drv_evt_addr: layout.drv_evt_addr(),
566 dev_evt_addr: layout.dev_evt_addr(),
567 }
568 }
569
570 pub fn submit_one(&mut self, addr: u64, len: u32, writable: bool) -> Result<u16, RingError> {
592 if self.num_free < 1 {
593 return Err(RingError::WouldBlock);
594 }
595
596 let id = self.id_free.pop().ok_or(RingError::OutOfMemory)?;
598
599 if self.id_num[id as usize] != 0 {
601 return Err(RingError::InvalidState);
602 }
603
604 self.id_num[id as usize] = 1;
606
607 let head_idx = self.avail_cursor.head();
609 let head_wrap = self.avail_cursor.wrap();
610
611 let mut flags = DescFlags::empty();
612 flags.set(DescFlags::WRITE, writable);
613 let mut desc = Descriptor::new(addr, len, id, flags);
614 desc.mark_avail(head_wrap);
615
616 let addr = self
617 .desc_table
618 .desc_addr(head_idx)
619 .ok_or(RingError::InvalidState)?;
620
621 desc.write_release(&self.mem, addr)
623 .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
624
625 self.avail_cursor.advance();
627 self.num_free -= 1;
628
629 Ok(id)
630 }
631
632 pub fn submit_available_with_notify(
634 &mut self,
635 chain: &BufferChain,
636 ) -> Result<SubmitResult, RingError> {
637 let old = self.avail_cursor;
638 let id = self.submit_available(chain)?;
639 let new = self.avail_cursor;
640 let notify = self.should_notify_device(old, new)?;
641
642 Ok(SubmitResult { id, notify })
643 }
644
645 pub fn submit_one_with_notify(
647 &mut self,
648 addr: u64,
649 len: u32,
650 writable: bool,
651 ) -> Result<SubmitResult, RingError> {
652 let old = self.avail_cursor;
653 let id = self.submit_one(addr, len, writable)?;
654 let new = self.avail_cursor;
655 let notify = self.should_notify_device(old, new)?;
656 Ok(SubmitResult { id, notify })
657 }
658
659 pub fn submit_available(&mut self, chain: &BufferChain) -> Result<u16, RingError> {
682 let total_descs = chain.len();
683 if total_descs == 0 {
684 return Err(RingError::EmptyChain);
685 }
686
687 if self.num_free < total_descs {
688 return Err(RingError::WouldBlock);
689 }
690
691 if total_descs == 1 {
692 let elem = chain.elems()[0];
693 return self.submit_one(elem.addr, elem.len, elem.writable);
694 }
695
696 let head_idx = self.avail_cursor.head();
697 let head_wrap = self.avail_cursor.wrap();
698
699 let id = self.id_free.pop().ok_or(RingError::OutOfMemory)?;
700
701 if self.id_num[id as usize] != 0 {
703 return Err(RingError::InvalidState);
704 }
705
706 self.id_num[id as usize] = total_descs as u16;
708
709 let mut pos = self.avail_cursor;
711 pos.advance();
712
713 for (i, elem) in chain.elems().iter().enumerate().skip(1) {
714 let is_next = i + 1 < total_descs;
715 let mut flags = DescFlags::empty();
716
717 flags.set(DescFlags::NEXT, is_next);
718 flags.set(DescFlags::WRITE, elem.writable);
719
720 let mut desc = Descriptor::new(elem.addr, elem.len, id, flags);
721 desc.mark_avail(pos.wrap());
722
723 let addr = self
724 .desc_table
725 .desc_addr(pos.head())
726 .ok_or(RingError::InvalidState)?;
727
728 self.mem
729 .write_val(addr, desc)
730 .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
731 pos.advance();
732 }
733
734 let head_elem = chain.elems()[0];
736 let mut head_flags = DescFlags::empty();
738 head_flags.set(DescFlags::NEXT, total_descs > 1);
739 head_flags.set(DescFlags::WRITE, head_elem.writable);
740
741 let mut head_desc = Descriptor::new(head_elem.addr, head_elem.len, id, head_flags);
742 head_desc.mark_avail(head_wrap);
743
744 let head_addr = self
745 .desc_table
746 .desc_addr(head_idx)
747 .ok_or(RingError::InvalidState)?;
748
749 head_desc
751 .write_release(&self.mem, head_addr)
752 .map_err(|_| RingError::mem_err(MemOp::WriteDesc, head_addr))?;
753
754 self.num_free -= total_descs;
755 self.avail_cursor = pos;
756
757 Ok(id)
758 }
759
760 pub fn poll_used(&mut self) -> Result<UsedBuffer, RingError> {
770 let idx = self.used_cursor.head();
771 let wrap = self.used_cursor.wrap();
772
773 let addr = self
775 .desc_table
776 .desc_addr(idx)
777 .ok_or(RingError::InvalidState)?;
778
779 let flags = Descriptor::read_flags_acquire(&self.mem, addr)
781 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
782
783 if !flags.is_used(wrap) {
784 return Err(RingError::WouldBlock);
785 }
786
787 let desc = Descriptor::read_body(&self.mem, addr, flags)
788 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
789
790 let id = desc.id;
791 let count = *self
792 .id_num
793 .get(id as usize)
794 .ok_or(RingError::InvalidState)?;
795
796 if count == 0 {
797 return Err(RingError::InvalidState);
798 }
799
800 self.used_cursor.advance_by(count);
802 self.num_free += count as usize;
804 self.id_num[id as usize] = 0;
806 self.id_free.push(id);
808
809 Ok(UsedBuffer { id, len: desc.len })
810 }
811
812 pub fn next_id(&self) -> Result<u16, RingError> {
814 let id = *self.id_free.last().ok_or(RingError::OutOfMemory)?;
815 if self.id_num[id as usize] != 0 {
816 return Err(RingError::InvalidState);
817 }
818 Ok(id)
819 }
820
821 #[inline]
823 pub fn num_free(&self) -> usize {
824 self.num_free
825 }
826
827 #[inline]
829 pub fn num_inflight(&self) -> usize {
830 self.desc_table.len() - self.num_free
831 }
832
833 #[inline]
835 pub fn is_full(&self) -> bool {
836 self.num_free == 0
837 }
838
839 #[inline]
841 #[allow(clippy::len_without_is_empty)]
842 pub fn len(&self) -> usize {
843 self.desc_table.len()
844 }
845
846 #[inline]
848 pub fn mem(&self) -> &M {
849 &self.mem
850 }
851
852 #[inline]
854 pub fn desc_table(&self) -> &DescTable {
855 &self.desc_table
856 }
857
858 #[inline]
863 pub fn avail_cursor(&self) -> RingCursor {
864 self.avail_cursor
865 }
866
867 #[inline]
871 pub fn used_cursor(&self) -> RingCursor {
872 self.used_cursor
873 }
874
875 pub fn should_notify_since(&self, old: RingCursor) -> Result<bool, RingError> {
883 self.should_notify_device(old, self.avail_cursor)
884 }
885
886 pub fn disable_used_notifications(&mut self) -> Result<(), RingError> {
888 if self.event_flags_shadow == EventFlags::DISABLE {
890 return Ok(());
891 }
892
893 let mut evt = self
894 .mem
895 .read_val::<EventSuppression>(self.drv_evt_addr)
896 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
897
898 evt.set_flags(EventFlags::DISABLE);
899
900 evt.write_release(&self.mem, self.drv_evt_addr)
901 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
902 self.event_flags_shadow = EventFlags::DISABLE;
903 Ok(())
904 }
905
906 pub fn enable_used_notifications(&mut self) -> Result<(), RingError> {
908 if self.event_flags_shadow == EventFlags::ENABLE {
909 return Ok(());
910 }
911
912 let mut evt = self
913 .mem
914 .read_val::<EventSuppression>(self.drv_evt_addr)
915 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
916
917 evt.set_flags(EventFlags::ENABLE);
918 evt.write_release(&self.mem, self.drv_evt_addr)
919 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
920
921 self.event_flags_shadow = EventFlags::ENABLE;
922 Ok(())
923 }
924
925 pub fn enable_used_notifications_desc(
932 &mut self,
933 off: u16,
934 wrap: bool,
935 ) -> Result<(), RingError> {
936 let mut evt = self
937 .mem
938 .read_val::<EventSuppression>(self.drv_evt_addr)
939 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
940
941 evt.set_desc_event(off, wrap);
942 evt.set_flags(EventFlags::DESC);
943
944 evt.write_release(&self.mem, self.drv_evt_addr)
946 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
947 self.event_flags_shadow = EventFlags::DESC;
949 Ok(())
950 }
951
952 pub fn enable_used_notifications_for_next(&mut self) -> Result<(), RingError> {
954 let off = self.used_cursor.head();
955 let wrap = self.used_cursor.wrap();
956
957 self.enable_used_notifications_desc(off, wrap)
958 }
959
960 fn should_notify_device(&self, old: RingCursor, new: RingCursor) -> Result<bool, RingError> {
962 fence(Ordering::SeqCst);
972
973 should_notify_evt(&self.mem, self.dev_evt_addr, self.len() as u16, old, new)
974 }
975
976 pub fn reset(&mut self) -> Result<(), RingError> {
989 let table_addr = self.desc_table.base_addr();
990 let size = self.desc_table.len();
991
992 self.desc_table
993 .clear(&self.mem)
994 .map_err(|_| RingError::mem_err(MemOp::WriteDesc, table_addr))?;
995
996 EventSuppression::clear(&self.mem, self.drv_evt_addr)
997 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
998
999 self.avail_cursor.reset();
1000 self.used_cursor.reset();
1001
1002 self.num_free = size;
1003 self.id_free.clear();
1004 self.id_free.extend(0..size as u16);
1005 self.id_num.iter_mut().for_each(|n| *n = 0);
1006 self.event_flags_shadow = EventFlags::ENABLE;
1007 Ok(())
1008 }
1009}
1010
1011#[derive(Debug)]
1024pub struct RingConsumer<M> {
1025 mem: M,
1027 avail_cursor: RingCursor,
1029 used_cursor: RingCursor,
1031 desc_table: DescTable,
1033 id_num: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
1035 num_inflight: usize,
1037 event_flags_shadow: EventFlags,
1039 drv_evt_addr: u64,
1041 dev_evt_addr: u64,
1043}
1044
1045impl<M: MemOps> RingConsumer<M> {
1046 pub fn new(layout: Layout, mem: M) -> Self {
1047 let size = layout.desc_table_len() as usize;
1048 let raw = layout.desc_table_addr();
1049
1050 let table = unsafe { DescTable::from_raw_parts(raw, size) };
1052 let cursor = RingCursor::new(size);
1053 let id_chain_len = SmallVec::<[u16; DescTable::DEFAULT_LEN]>::from_elem(0, size);
1054
1055 let event_flags_shadow = EventFlags::ENABLE;
1057
1058 Self {
1059 mem,
1060 avail_cursor: cursor,
1061 used_cursor: cursor,
1062 desc_table: table,
1063 id_num: id_chain_len,
1064 num_inflight: 0,
1065 event_flags_shadow,
1066 drv_evt_addr: layout.drv_evt_addr(),
1067 dev_evt_addr: layout.dev_evt_addr(),
1068 }
1069 }
1070
1071 pub fn poll_available(&mut self) -> Result<(u16, BufferChain), RingError> {
1084 let idx = self.avail_cursor.head();
1085 let wrap = self.avail_cursor.wrap();
1086
1087 let head_addr = self
1088 .desc_table
1089 .desc_addr(idx)
1090 .ok_or(RingError::InvalidState)?;
1091
1092 let flags = Descriptor::read_flags_acquire(&self.mem, head_addr)
1094 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, head_addr))?;
1095
1096 if !flags.is_avail(wrap) {
1098 return Err(RingError::WouldBlock);
1099 }
1100
1101 let head_desc = Descriptor::read_body(&self.mem, head_addr, flags)
1102 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, head_addr))?;
1103
1104 if flags.contains(DescFlags::INDIRECT) {
1105 return Err(RingError::BadChain);
1106 }
1107
1108 let mut elements = ChainElems::new();
1110 let mut pos = self.avail_cursor;
1111 let mut chain_len: u16 = 1;
1112
1113 let mut steps = 1;
1114 let mut has_next = head_desc.is_next();
1115
1116 let max_steps = self.desc_table.len();
1117
1118 let head_elem = BufferElement::from(&head_desc);
1119 let mut seen_writable = head_elem.writable;
1120 let mut writables: usize = if seen_writable { 1 } else { 0 };
1121 elements.push(head_elem);
1122 pos.advance();
1123
1124 while has_next && steps < max_steps {
1125 let addr = self
1126 .desc_table
1127 .desc_addr(pos.head())
1128 .ok_or(RingError::InvalidState)?;
1129
1130 let desc: Descriptor = self
1132 .mem
1133 .read_val(addr)
1134 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
1135 if desc.flags().contains(DescFlags::INDIRECT) {
1136 return Err(RingError::BadChain);
1137 }
1138
1139 let elem = BufferElement::from(&desc);
1140
1141 if elem.writable {
1142 seen_writable = true;
1143 writables += 1;
1144 } else if seen_writable {
1145 return Err(RingError::BadChain);
1146 }
1147
1148 elements
1149 .try_reserve(1)
1150 .map_err(|_| RingError::Bookkeeping)?;
1151 elements.push(elem);
1152
1153 chain_len += 1;
1154 steps += 1;
1155
1156 has_next = desc.is_next();
1157 pos.advance();
1158 }
1159
1160 if steps >= max_steps && has_next {
1162 return Err(RingError::BadChain);
1163 }
1164
1165 if self.num_inflight + chain_len as usize > self.desc_table.len() {
1168 return Err(RingError::InvalidState);
1169 }
1170
1171 let readables = elements.len() - writables;
1172
1173 let id = head_desc.id;
1175 let id_num = self
1176 .id_num
1177 .get_mut(id as usize)
1178 .ok_or(RingError::InvalidState)?;
1179 if *id_num != 0 {
1180 return Err(RingError::InvalidState);
1181 }
1182
1183 *id_num = chain_len;
1185 self.avail_cursor = pos;
1187 self.num_inflight += chain_len as usize;
1189
1190 Ok((
1191 id,
1192 BufferChain {
1193 elems: elements,
1194 split: readables,
1195 },
1196 ))
1197 }
1198
1199 pub fn submit_used(&mut self, id: u16, written_len: u32) -> Result<(), RingError> {
1211 let chain_len = *self
1213 .id_num
1214 .get(id as usize)
1215 .ok_or(RingError::InvalidState)?;
1216
1217 if chain_len == 0 || chain_len > self.desc_table.len() as u16 {
1218 return Err(RingError::InvalidState);
1219 }
1220
1221 let idx = self.used_cursor.head();
1222 let wrap = self.used_cursor.wrap();
1223
1224 let mut used_desc = Descriptor::new(0, written_len, id, DescFlags::empty());
1226 used_desc.mark_used(wrap);
1227
1228 let addr = self
1229 .desc_table
1230 .desc_addr(idx)
1231 .ok_or(RingError::InvalidState)?;
1232
1233 used_desc
1235 .write_release(&self.mem, addr)
1236 .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
1237
1238 self.used_cursor.advance_by(chain_len);
1240 self.id_num[id as usize] = 0;
1241
1242 self.num_inflight -= chain_len as usize;
1243 Ok(())
1244 }
1245
1246 pub fn peek_available(&self) -> Result<bool, RingError> {
1248 let Some(addr) = self.desc_table.desc_addr(self.avail_cursor.head()) else {
1249 return Err(RingError::InvalidState);
1250 };
1251
1252 let flags = Descriptor::read_flags_acquire(&self.mem, addr)
1253 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
1254
1255 Ok(flags.is_avail(self.avail_cursor.wrap()))
1256 }
1257
1258 pub(super) fn poll_checkpoint(&self) -> Checkpoint {
1260 Checkpoint {
1261 avail_cursor: self.avail_cursor,
1262 used_cursor: self.used_cursor,
1263 num_inflight: self.num_inflight,
1264 }
1265 }
1266
1267 pub(super) fn rollback_polls(&mut self, cp: Checkpoint, ids: impl IntoIterator<Item = u16>) {
1273 let mut desc_count = 0usize;
1274 for id in ids {
1275 let chain_len = core::mem::take(&mut self.id_num[id as usize]);
1276 debug_assert_ne!(chain_len, 0);
1277 desc_count += chain_len as usize;
1278 }
1279
1280 debug_assert!(desc_count <= self.desc_table.len());
1281 debug_assert_eq!(self.used_cursor, cp.used_cursor);
1282 debug_assert_eq!(self.num_inflight, cp.num_inflight + desc_count);
1283 debug_assert_eq!(self.avail_cursor, {
1284 let mut expected = cp.avail_cursor;
1285 expected.advance_by(desc_count as u16);
1286 expected
1287 });
1288
1289 self.avail_cursor = cp.avail_cursor;
1290 self.num_inflight = cp.num_inflight;
1291 }
1292
1293 pub fn submit_used_with_notify(
1295 &mut self,
1296 id: u16,
1297 written_len: u32,
1298 ) -> Result<bool, RingError> {
1299 let old = self.used_cursor;
1300 self.submit_used(id, written_len)?;
1301 let new = self.used_cursor;
1302 self.should_notify_driver(old, new)
1303 }
1304
1305 pub fn num_free(&self) -> usize {
1307 self.desc_table.len() - self.num_inflight
1308 }
1309
1310 pub fn num_inflight(&self) -> usize {
1312 self.num_inflight
1313 }
1314
1315 pub fn is_full(&self) -> bool {
1317 self.num_inflight == self.desc_table.len()
1318 }
1319
1320 #[allow(clippy::len_without_is_empty)]
1322 pub fn len(&self) -> usize {
1323 self.desc_table.len()
1324 }
1325
1326 pub fn mem(&self) -> &M {
1328 &self.mem
1329 }
1330
1331 #[inline]
1333 pub fn avail_cursor(&self) -> RingCursor {
1334 self.avail_cursor
1335 }
1336
1337 #[inline]
1339 pub fn used_cursor(&self) -> RingCursor {
1340 self.used_cursor
1341 }
1342
1343 pub fn disable_avail_notifications(&mut self) -> Result<(), RingError> {
1347 if self.event_flags_shadow == EventFlags::DISABLE {
1348 return Ok(());
1349 }
1350
1351 let mut evt = self
1352 .mem
1353 .read_val::<EventSuppression>(self.dev_evt_addr)
1354 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1355
1356 evt.set_flags(EventFlags::DISABLE);
1357 evt.write_release(&self.mem, self.dev_evt_addr)
1358 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1359
1360 self.event_flags_shadow = EventFlags::DISABLE;
1361 Ok(())
1362 }
1363
1364 pub fn enable_avail_notifications(&mut self) -> Result<(), RingError> {
1366 if self.event_flags_shadow == EventFlags::ENABLE {
1367 return Ok(());
1368 }
1369
1370 let mut evt = self
1371 .mem
1372 .read_val::<EventSuppression>(self.dev_evt_addr)
1373 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1374
1375 evt.set_flags(EventFlags::ENABLE);
1376 evt.write_release(&self.mem, self.dev_evt_addr)
1377 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1378
1379 self.event_flags_shadow = EventFlags::ENABLE;
1380 Ok(())
1381 }
1382
1383 pub fn enable_avail_notifications_desc(
1387 &mut self,
1388 off: u16,
1389 wrap: bool,
1390 ) -> Result<(), RingError> {
1391 let mut evt = self
1393 .mem
1394 .read_val::<EventSuppression>(self.dev_evt_addr)
1395 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1396
1397 evt.set_desc_event(off, wrap);
1398 evt.set_flags(EventFlags::DESC);
1399
1400 evt.write_release(&self.mem, self.dev_evt_addr)
1402 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1403
1404 self.event_flags_shadow = EventFlags::DESC;
1405 Ok(())
1406 }
1407
1408 pub fn enable_avail_notifications_for_next(&mut self) -> Result<(), RingError> {
1411 let off = self.avail_cursor.head();
1412 let wrap = self.avail_cursor.wrap();
1413 self.enable_avail_notifications_desc(off, wrap)
1414 }
1415
1416 fn should_notify_driver(&self, old: RingCursor, new: RingCursor) -> Result<bool, RingError> {
1418 fence(Ordering::SeqCst);
1421
1422 should_notify_evt(&self.mem, self.drv_evt_addr, self.len() as u16, old, new)
1423 }
1424
1425 pub fn reset(&mut self) -> Result<(), RingError> {
1437 EventSuppression::clear(&self.mem, self.dev_evt_addr)
1438 .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1439
1440 self.avail_cursor.reset();
1441 self.used_cursor.reset();
1442 self.id_num.iter_mut().for_each(|n| *n = 0);
1443 self.num_inflight = 0;
1444 self.event_flags_shadow = EventFlags::ENABLE;
1445 Ok(())
1446 }
1447}
1448
1449fn should_notify_evt<M: MemOps>(
1451 mem: &M,
1452 evt_addr: u64,
1453 ring_len: u16,
1454 old: RingCursor,
1455 new: RingCursor,
1456) -> Result<bool, RingError> {
1457 let flags = EventSuppression::read_flags_acquire(mem, evt_addr)
1458 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, evt_addr))?;
1459
1460 let evt = if flags == EventFlags::DESC {
1461 EventSuppression::read_body(mem, evt_addr, flags)
1462 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, evt_addr))?
1463 } else {
1464 EventSuppression::new(0, flags)
1465 };
1466
1467 Ok(should_notify(evt, ring_len, old, new))
1468}
1469
1470#[inline]
1477fn should_notify(evt: EventSuppression, ring_len: u16, old: RingCursor, new: RingCursor) -> bool {
1478 match evt.flags() {
1479 EventFlags::DISABLE => false,
1480 EventFlags::ENABLE => true,
1481 EventFlags::DESC => {
1482 let mut off = evt.desc_event_off();
1483 if off >= ring_len {
1484 return false;
1485 }
1486
1487 let wrap = evt.desc_event_wrap();
1488
1489 if wrap != new.wrap() {
1490 off = off.wrapping_sub(ring_len);
1491 }
1492
1493 ring_need_event(off, new.head(), old.head())
1494 }
1495 _ => false,
1497 }
1498}
1499
1500#[inline(always)]
1501fn ring_need_event(event_idx: u16, new: u16, old: u16) -> bool {
1502 new.wrapping_sub(event_idx).wrapping_sub(1) < new.wrapping_sub(old)
1503}
1504
1505impl From<&Descriptor> for BufferElement {
1506 fn from(desc: &Descriptor) -> Self {
1507 BufferElement {
1508 addr: desc.addr,
1509 len: desc.len,
1510 writable: desc.is_writable(),
1511 }
1512 }
1513}
1514
1515#[cfg(test)]
1516pub(crate) mod tests {
1517 use alloc::sync::Arc;
1518 use alloc::vec::Vec;
1519 use core::cell::UnsafeCell;
1520 use core::num::NonZeroU16;
1521 use core::ptr::{self, NonNull};
1522 use core::sync::atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering};
1523
1524 use bytemuck::{Pod, Zeroable};
1525
1526 use super::super::align_up;
1527 use super::*;
1528 use crate::virtq::event::EventSuppression;
1529 use crate::virtq::{BufferLease, BufferMap};
1530
1531 #[derive(Clone)]
1537 pub struct TestMem {
1538 inner: Arc<TestMemInner>,
1539 }
1540
1541 struct TestMemInner {
1542 storage: UnsafeCell<Vec<u8>>,
1544 base_addr: u64,
1546 }
1547
1548 unsafe impl Send for TestMemInner {}
1551 unsafe impl Sync for TestMemInner {}
1552
1553 impl TestMem {
1554 pub fn new(size: usize) -> Self {
1555 let storage = vec![0u8; size];
1556 let base_addr = storage.as_ptr() as u64;
1557 Self {
1558 inner: Arc::new(TestMemInner {
1559 storage: UnsafeCell::new(storage),
1560 base_addr,
1561 }),
1562 }
1563 }
1564
1565 fn ptr_for_addr(&self, addr: u64) -> *mut u8 {
1567 let storage = unsafe { &mut *self.inner.storage.get() };
1568 let base_ptr = storage.as_mut_ptr();
1569 let offset = (addr - self.inner.base_addr) as usize;
1570 base_ptr.wrapping_add(offset)
1572 }
1573
1574 pub fn base_addr(&self) -> u64 {
1575 self.inner.base_addr
1576 }
1577 }
1578
1579 pub struct TestMapping {
1580 creator: std::thread::ThreadId,
1581 owner: core::mem::ManuallyDrop<TestBufferOwner>,
1582 }
1583
1584 struct TestBufferOwner {
1585 data: NonNull<[u8]>,
1586 _mem: TestMem,
1587 _lease: BufferLease,
1588 }
1589
1590 unsafe impl Send for TestMapping {}
1593
1594 impl AsRef<[u8]> for TestMapping {
1595 fn as_ref(&self) -> &[u8] {
1596 unsafe { self.owner.data.as_ref() }
1598 }
1599 }
1600
1601 impl Drop for TestMapping {
1602 fn drop(&mut self) {
1603 assert_eq!(
1604 self.creator,
1605 std::thread::current().id(),
1606 "mapping dropped on another thread"
1607 );
1608 unsafe { core::mem::ManuallyDrop::drop(&mut self.owner) };
1610 }
1611 }
1612
1613 impl BufferMap for TestMem {
1614 type Mapping = TestMapping;
1615
1616 unsafe fn map_buffer(
1617 &self,
1618 lease: BufferLease,
1619 written: usize,
1620 ) -> Result<Self::Mapping, Self::Error> {
1621 let alloc = lease.allocation();
1622 assert!(written <= alloc.len as usize);
1623
1624 let offset = alloc.addr.checked_sub(self.base_addr()).unwrap() as usize;
1625
1626 let size = unsafe { &*self.inner.storage.get() }.len();
1628 assert!(offset.checked_add(alloc.len as usize).unwrap() <= size);
1629
1630 let data = NonNull::from(unsafe { self.as_slice(alloc.addr, written)? });
1633
1634 Ok(TestMapping {
1635 creator: std::thread::current().id(),
1636 owner: core::mem::ManuallyDrop::new(TestBufferOwner {
1637 data,
1638 _mem: self.clone(),
1639 _lease: lease,
1640 }),
1641 })
1642 }
1643 }
1644
1645 unsafe impl MemOps for TestMem {
1648 type Error = core::convert::Infallible;
1649
1650 fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1651 let src = self.ptr_for_addr(addr);
1652 unsafe {
1653 ptr::copy_nonoverlapping(src, dst.as_mut_ptr(), dst.len());
1654 }
1655 Ok(())
1656 }
1657
1658 fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1659 let dst = self.ptr_for_addr(addr);
1660 unsafe {
1661 ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len());
1662 }
1663 Ok(())
1664 }
1665
1666 fn read_val<T: Pod>(&self, addr: u64) -> Result<T, Self::Error> {
1667 let ptr = self.ptr_for_addr(addr).cast::<T>();
1668 Ok(unsafe { ptr::read_volatile(ptr) })
1669 }
1670
1671 fn write_val<T: Pod>(&self, addr: u64, val: T) -> Result<(), Self::Error> {
1672 let ptr = self.ptr_for_addr(addr).cast::<T>();
1673 unsafe { ptr::write_volatile(ptr, val) };
1674 Ok(())
1675 }
1676
1677 fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1678 let ptr = self.ptr_for_addr(addr).cast::<AtomicU16>();
1679 Ok(unsafe { (*ptr).load(Ordering::Acquire) })
1680 }
1681
1682 fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1683 let ptr = self.ptr_for_addr(addr).cast::<AtomicU16>();
1684 unsafe { (*ptr).store(val, Ordering::Release) };
1685 Ok(())
1686 }
1687
1688 unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1689 let ptr = self.ptr_for_addr(addr);
1690 Ok(unsafe { core::slice::from_raw_parts(ptr, len) })
1691 }
1692
1693 unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1694 let ptr = self.ptr_for_addr(addr);
1695 Ok(unsafe { core::slice::from_raw_parts_mut(ptr, len) })
1696 }
1697 }
1698
1699 #[derive(Clone)]
1701 pub(crate) struct FaultMem(pub Arc<FaultState>);
1702
1703 pub(crate) struct FaultState {
1704 pub mem: TestMem,
1705 pub writes: AtomicUsize,
1706 pub fail_write_at: AtomicUsize,
1707 pub fail_mapping_at: AtomicUsize,
1708 pub map_calls: AtomicUsize,
1709 pub deny_views: AtomicBool,
1710 }
1711
1712 impl FaultMem {
1713 pub(crate) fn new(mem: TestMem) -> Self {
1714 Self(Arc::new(FaultState {
1715 mem,
1716 fail_write_at: AtomicUsize::new(usize::MAX),
1717 writes: AtomicUsize::new(0),
1718 fail_mapping_at: AtomicUsize::new(usize::MAX),
1719 map_calls: AtomicUsize::new(0),
1720 deny_views: AtomicBool::new(false),
1721 }))
1722 }
1723
1724 pub(crate) fn fail_write_at(&self, write: usize) {
1727 self.0.writes.store(0, Ordering::Relaxed);
1728 self.0.fail_write_at.store(write, Ordering::Relaxed);
1729 }
1730
1731 pub(crate) fn allow_writes(&self) {
1732 self.fail_write_at(usize::MAX);
1733 }
1734
1735 pub(crate) fn fail_mapping_at(&self, mapping: usize) {
1737 self.0.map_calls.store(0, Ordering::Relaxed);
1738 self.0.fail_mapping_at.store(mapping, Ordering::Relaxed);
1739 }
1740
1741 pub(crate) fn deny_views(&self) {
1743 self.0.deny_views.store(true, Ordering::Relaxed);
1744 }
1745
1746 fn check_write(&self) -> Result<(), ()> {
1747 let write = self.0.writes.fetch_add(1, Ordering::Relaxed);
1748 if write == self.0.fail_write_at.load(Ordering::Relaxed) {
1749 Err(())
1750 } else {
1751 Ok(())
1752 }
1753 }
1754 }
1755
1756 unsafe impl MemOps for FaultMem {
1759 type Error = ();
1760
1761 fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1762 self.0.mem.read(addr, dst).map_err(|never| match never {})
1763 }
1764
1765 fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1766 self.check_write()?;
1767 self.0.mem.write(addr, src).map_err(|never| match never {})
1768 }
1769
1770 fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1771 self.0
1772 .mem
1773 .load_acquire(addr)
1774 .map_err(|never| match never {})
1775 }
1776
1777 fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1778 self.check_write()?;
1779 self.0
1780 .mem
1781 .store_release(addr, val)
1782 .map_err(|never| match never {})
1783 }
1784
1785 unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1786 if self.0.deny_views.load(Ordering::Relaxed) {
1787 return Err(());
1788 }
1789 unsafe { self.0.mem.as_slice(addr, len) }.map_err(|never| match never {})
1791 }
1792
1793 unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1794 if self.0.deny_views.load(Ordering::Relaxed) {
1795 return Err(());
1796 }
1797 unsafe { self.0.mem.as_mut_slice(addr, len) }.map_err(|never| match never {})
1799 }
1800 }
1801
1802 pub(crate) struct TrackedMapping {
1804 data: TestMapping,
1805 _mem: FaultMem,
1806 }
1807
1808 impl AsRef<[u8]> for TrackedMapping {
1809 fn as_ref(&self) -> &[u8] {
1810 self.data.as_ref()
1811 }
1812 }
1813
1814 impl BufferMap for FaultMem {
1815 type Mapping = TrackedMapping;
1816
1817 unsafe fn map_buffer(
1818 &self,
1819 lease: BufferLease,
1820 written: usize,
1821 ) -> Result<Self::Mapping, Self::Error> {
1822 let call = self.0.map_calls.fetch_add(1, Ordering::Relaxed);
1823 if self.0.deny_views.load(Ordering::Relaxed)
1824 || call == self.0.fail_mapping_at.load(Ordering::Relaxed)
1825 {
1826 return Err(());
1827 }
1828 let data = unsafe { BufferMap::map_buffer(&self.0.mem, lease, written) }
1830 .map_err(|never| match never {})?;
1831 Ok(TrackedMapping {
1832 data,
1833 _mem: self.clone(),
1834 })
1835 }
1836 }
1837
1838 pub struct OwnedRing {
1840 mem: TestMem,
1841 layout: Layout,
1842 }
1843
1844 impl OwnedRing {
1845 pub fn new(size: usize) -> Self {
1846 let num_descs = NonZeroU16::new(size as u16).unwrap();
1847 let needed = Layout::query_size(size);
1848
1849 let padding = Descriptor::ALIGN;
1853 let pool_headroom = 0x100 + 0x8000;
1854 let mem = TestMem::new(needed + padding + pool_headroom);
1855
1856 let aligned_base = align_up(mem.base_addr() as usize, Descriptor::ALIGN) as u64;
1858 let layout = unsafe { Layout::from_base(aligned_base, num_descs).unwrap() };
1859
1860 Self { mem, layout }
1861 }
1862
1863 pub fn layout(&self) -> Layout {
1864 self.layout
1865 }
1866
1867 pub fn mem(&self) -> TestMem {
1868 self.mem.clone()
1869 }
1870
1871 pub fn desc_addr(&self, idx: u16) -> u64 {
1873 self.layout.desc_table_addr() + (idx as u64 * Descriptor::SIZE as u64)
1874 }
1875
1876 pub fn read_desc(&self, idx: u16) -> Descriptor {
1878 self.mem.read_val(self.desc_addr(idx)).unwrap()
1879 }
1880
1881 pub fn write_desc(&self, idx: u16, desc: Descriptor) {
1883 self.mem.write_val(self.desc_addr(idx), desc).unwrap()
1884 }
1885
1886 pub fn read_driver_event(&self) -> EventSuppression {
1888 self.mem.read_val(self.layout.drv_evt_addr()).unwrap()
1889 }
1890
1891 pub fn read_device_event(&self) -> EventSuppression {
1893 self.mem.read_val(self.layout.dev_evt_addr()).unwrap()
1894 }
1895
1896 pub fn len(&self) -> usize {
1897 self.layout.desc_table_len() as usize
1898 }
1899 }
1900
1901 pub(crate) fn make_ring(size: usize) -> OwnedRing {
1903 OwnedRing::new(size)
1904 }
1905
1906 pub(crate) fn make_producer(ring: &OwnedRing) -> RingProducer<TestMem> {
1907 RingProducer::new(ring.layout(), ring.mem())
1908 }
1909
1910 pub(crate) fn make_consumer(ring: &OwnedRing) -> RingConsumer<TestMem> {
1911 RingConsumer::new(ring.layout(), ring.mem())
1912 }
1913
1914 fn assert_invariants(ring: &OwnedRing, prod: &RingProducer<TestMem>) {
1915 let outstanding: u16 = prod.id_num.iter().copied().sum();
1916 assert_eq!(outstanding as usize + prod.num_free, ring.len());
1917
1918 for id in prod.id_free.iter() {
1919 assert_eq!(prod.id_num[*id as usize], 0);
1920 }
1921
1922 for (id, &n) in prod.id_num.iter().enumerate() {
1923 if n > 0 {
1924 assert!(!prod.id_free.contains(&(id as u16)));
1925 }
1926 }
1927 }
1928
1929 #[test]
1930 fn test_initialization() {
1931 let ring = make_ring(8);
1932 let producer = make_producer(&ring);
1933
1934 for i in 0..8u16 {
1936 let desc = ring.read_desc(i);
1937 assert_eq!(desc, Descriptor::zeroed());
1938 assert_eq!(desc.flags, 0);
1939 assert_eq!(desc.addr, 0);
1940 assert_eq!(desc.len, 0);
1941 assert_eq!(desc.id, 0);
1942 }
1943
1944 assert_eq!(producer.avail_cursor.head(), 0);
1946 assert!(producer.avail_cursor.wrap());
1947 assert_eq!(producer.used_cursor.head(), 0);
1948 assert!(producer.used_cursor.wrap());
1949
1950 assert_eq!(producer.id_free.len(), 8);
1952 assert_eq!(producer.num_free, 8);
1953 for i in 0..8 {
1954 assert_eq!(producer.id_num[i], 0);
1955 }
1956 }
1957
1958 #[test]
1959 fn test_buffer_chain_builder_normalizes_element_direction() {
1960 let readable_as_writable = BufferElement {
1961 addr: 0x1000,
1962 len: 16,
1963 writable: true,
1964 };
1965 let writable_as_readable = BufferElement {
1966 addr: 0x2000,
1967 len: 32,
1968 writable: false,
1969 };
1970 let second_writable_as_readable = BufferElement {
1971 addr: 0x3000,
1972 len: 64,
1973 writable: false,
1974 };
1975
1976 let chain = BufferChainBuilder::new()
1977 .readables([readable_as_writable])
1978 .writables([writable_as_readable])
1979 .writables([second_writable_as_readable])
1980 .build()
1981 .unwrap();
1982
1983 assert!(!chain.readables()[0].writable);
1984 assert!(chain.writables().iter().all(|elem| elem.writable));
1985 }
1986
1987 #[test]
1988 fn test_submit_one_descriptor() {
1989 let ring = make_ring(8);
1990 let mut producer = make_producer(&ring);
1991
1992 let addr = 0x1000;
1993 let len = 512;
1994 let writable = false;
1995
1996 let id = producer.submit_one(addr, len, writable).unwrap();
1997
1998 let desc = ring.read_desc(0);
2000
2001 assert_eq!(desc.addr, addr);
2002 assert_eq!(desc.len, len);
2003 assert_eq!(desc.id, id);
2004
2005 let flags = desc.flags();
2007 assert!(flags.contains(DescFlags::AVAIL));
2008 assert!(!flags.contains(DescFlags::USED));
2009 assert!(!flags.contains(DescFlags::WRITE));
2010 assert!(!flags.contains(DescFlags::NEXT));
2011
2012 assert_eq!(producer.num_free, 7);
2014
2015 assert_eq!(producer.avail_cursor.head(), 1);
2017 assert!(producer.avail_cursor.wrap());
2018
2019 assert_eq!(producer.id_num[id as usize], 1);
2021 assert_eq!(producer.id_free.len(), 7);
2022 }
2023
2024 #[test]
2025 fn test_single_descriptor_wrap_toggle() {
2026 let ring = make_ring(4);
2027 let mut producer = make_producer(&ring);
2028
2029 producer.avail_cursor.head = 3;
2031 producer.avail_cursor.wrap = true;
2032 producer.num_free = 1;
2033 producer.id_free.clear();
2034 producer.id_free.push(0);
2035
2036 let _id = producer.submit_one(0x1000, 512, false).unwrap();
2037
2038 assert_eq!(producer.avail_cursor.head(), 0);
2040 assert!(!producer.avail_cursor.wrap());
2041
2042 let desc = ring.read_desc(3);
2044 let flags = desc.flags();
2045 assert!(flags.contains(DescFlags::AVAIL));
2046 assert!(!flags.contains(DescFlags::USED));
2047 }
2048
2049 #[test]
2050 fn test_multi_descriptor_no_wrap() {
2051 let ring = make_ring(8);
2052 let mut producer = make_producer(&ring);
2053
2054 let chain = BufferChainBuilder::new()
2055 .readable(0x1000, 256)
2056 .readable(0x2000, 256)
2057 .writable(0x3000, 512)
2058 .build()
2059 .unwrap();
2060
2061 let id = producer.submit_available(&chain).unwrap();
2062
2063 let head_desc = ring.read_desc(0);
2065 assert_eq!(head_desc.addr, 0x1000);
2066 assert_eq!(head_desc.len, 256);
2067 assert_eq!(head_desc.id, id);
2068
2069 let head_flags = head_desc.flags();
2070 assert!(head_flags.contains(DescFlags::NEXT));
2071 assert!(!head_flags.contains(DescFlags::WRITE));
2072 assert!(head_flags.contains(DescFlags::AVAIL));
2073 assert!(!head_flags.contains(DescFlags::USED));
2074
2075 let mid_desc = ring.read_desc(1);
2077 assert_eq!(mid_desc.addr, 0x2000);
2078 assert_eq!(mid_desc.len, 256);
2079 assert_eq!(mid_desc.id, id);
2080
2081 let mid_flags = mid_desc.flags();
2082 assert!(mid_flags.contains(DescFlags::NEXT));
2083 assert!(!mid_flags.contains(DescFlags::WRITE));
2084
2085 let tail_desc = ring.read_desc(2);
2087 assert_eq!(tail_desc.addr, 0x3000);
2088 assert_eq!(tail_desc.len, 512);
2089 assert_eq!(tail_desc.id, id);
2090
2091 let tail_flags = tail_desc.flags();
2092 assert!(!tail_flags.contains(DescFlags::NEXT));
2093 assert!(tail_flags.contains(DescFlags::WRITE));
2094
2095 assert_eq!(head_desc.id, mid_desc.id);
2097 assert_eq!(mid_desc.id, tail_desc.id);
2098
2099 assert_eq!(producer.num_free, 5);
2101 assert_eq!(producer.avail_cursor.head(), 3);
2102 assert_eq!(producer.id_num[id as usize], 3);
2103 }
2104
2105 #[test]
2106 fn test_multi_descriptor_with_wrap() {
2107 let ring = make_ring(4);
2108 let mut producer = make_producer(&ring);
2109
2110 producer.avail_cursor.head = 2;
2112 producer.avail_cursor.wrap = true;
2113
2114 let chain = BufferChainBuilder::new()
2115 .readable(0x1000, 256)
2116 .readable(0x2000, 256)
2117 .readable(0x3000, 256)
2118 .build()
2119 .unwrap();
2120
2121 let _id = producer.submit_available(&chain).unwrap();
2122
2123 let head_desc = ring.read_desc(2);
2125 let head_flags = head_desc.flags();
2126 assert!(head_flags.contains(DescFlags::AVAIL));
2127 assert!(!head_flags.contains(DescFlags::USED));
2128
2129 let mid_desc = ring.read_desc(3);
2131 let mid_flags = mid_desc.flags();
2132 assert!(mid_flags.contains(DescFlags::AVAIL));
2133 assert!(!mid_flags.contains(DescFlags::USED));
2134
2135 let tail_desc = ring.read_desc(0);
2137 let tail_flags = tail_desc.flags();
2138 assert!(!tail_flags.contains(DescFlags::AVAIL));
2139 assert!(tail_flags.contains(DescFlags::USED));
2140
2141 assert_eq!(producer.avail_cursor.head(), 1);
2143 assert!(!producer.avail_cursor.wrap());
2144 }
2145
2146 #[test]
2147 fn test_ring_full() {
2148 let ring = make_ring(4);
2149 let mut producer = make_producer(&ring);
2150
2151 for _ in 0..4 {
2153 producer.submit_one(0x1000, 256, false).unwrap();
2154 }
2155
2156 assert_eq!(producer.num_free, 0);
2157
2158 let result = producer.submit_one(0x5000, 256, false);
2160 assert!(matches!(result, Err(RingError::WouldBlock)));
2161 }
2162
2163 #[test]
2164 fn test_poll_and_reclaim() {
2165 let ring = make_ring(8);
2166 let mut producer = make_producer(&ring);
2167
2168 let id = producer.submit_one(0x1000, 512, false).unwrap();
2169
2170 let mut desc = ring.read_desc(0);
2172 desc.mark_used(true);
2173 desc.len = 256;
2174 ring.write_desc(0, desc);
2175
2176 let used = producer.poll_used().unwrap();
2178 assert_eq!(used.id, id);
2179 assert_eq!(used.len, 256);
2180
2181 assert_eq!(producer.num_free, 8);
2183 assert_eq!(producer.used_cursor.head(), 1);
2184 assert_eq!(producer.id_num[id as usize], 0);
2185 assert!(producer.id_free.contains(&id));
2186 }
2187
2188 #[test]
2189 fn test_poll_multi_descriptor_chain() {
2190 let ring = make_ring(8);
2191 let mut producer = make_producer(&ring);
2192
2193 let chain = BufferChainBuilder::new()
2194 .readable(0x1000, 256)
2195 .readable(0x2000, 256)
2196 .writable(0x3000, 512)
2197 .build()
2198 .unwrap();
2199
2200 let id = producer.submit_available(&chain).unwrap();
2201
2202 let mut head_desc = ring.read_desc(0);
2204 head_desc.mark_used(true);
2205 head_desc.len = 512;
2206 ring.write_desc(0, head_desc);
2207
2208 let used = producer.poll_used().unwrap();
2210 assert_eq!(used.id, id);
2211 assert_eq!(used.len, 512);
2212
2213 assert_eq!(producer.used_cursor.head(), 3);
2215 assert_eq!(producer.num_free, 8);
2216 }
2217
2218 #[test]
2219 fn test_id_reuse() {
2220 let ring = make_ring(4);
2221 let mut producer = make_producer(&ring);
2222
2223 let id1 = producer.submit_one(0x1000, 256, false).unwrap();
2225
2226 let mut desc = ring.read_desc(0);
2227 desc.mark_used(true);
2228 ring.write_desc(0, desc);
2229
2230 producer.poll_used().unwrap();
2231
2232 let id2 = producer.submit_one(0x2000, 256, false).unwrap();
2234
2235 assert_eq!(id2, id1);
2237 assert_eq!(producer.id_num[id2 as usize], 1);
2238 }
2239
2240 #[test]
2241 fn test_available_descriptor_flags() {
2242 let ring = make_ring(4);
2243 let mut producer = make_producer(&ring);
2244
2245 producer.submit_one(0x1000, 256, false).unwrap();
2246
2247 let desc = ring.read_desc(0);
2248
2249 let flags = desc.flags();
2251 assert_ne!(
2252 flags.contains(DescFlags::AVAIL),
2253 flags.contains(DescFlags::USED)
2254 );
2255
2256 assert!(flags.contains(DescFlags::AVAIL));
2258 assert!(!flags.contains(DescFlags::USED));
2259 }
2260
2261 #[test]
2262 fn test_used_descriptor_flags() {
2263 let ring = make_ring(4);
2264 let mut producer = make_producer(&ring);
2265
2266 producer.submit_one(0x1000, 256, false).unwrap();
2267
2268 let mut desc = ring.read_desc(0);
2269 desc.mark_used(true);
2270 ring.write_desc(0, desc);
2271
2272 let desc = ring.read_desc(0);
2273 let flags = desc.flags();
2274
2275 assert_eq!(
2277 flags.contains(DescFlags::AVAIL),
2278 flags.contains(DescFlags::USED)
2279 );
2280 }
2281
2282 #[test]
2283 fn test_poll_empty_ring() {
2284 let ring = make_ring(4);
2285 let mut producer = make_producer(&ring);
2286
2287 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2289 }
2290
2291 #[test]
2292 fn test_submit_when_full() {
2293 let ring = make_ring(2);
2294 let mut producer = make_producer(&ring);
2295
2296 producer.submit_one(0x1000, 256, false).unwrap();
2297 producer.submit_one(0x2000, 256, false).unwrap();
2298
2299 assert!(matches!(
2301 producer.submit_one(0x3000, 256, false),
2302 Err(RingError::WouldBlock)
2303 ));
2304 }
2305
2306 #[test]
2307 fn test_wrap_stress() {
2308 let ring = make_ring(4);
2309 let mut producer = make_producer(&ring);
2310 let mut consumer = make_consumer(&ring);
2311
2312 for lap in 0..3 {
2314 let expected_wrap = lap % 2 == 0;
2315
2316 for _ in 0..4 {
2317 let id = producer.submit_one(0x1000, 256, false).unwrap();
2318
2319 let (dev_id, _) = consumer.poll_available().unwrap();
2320 assert_eq!(dev_id, id);
2321
2322 consumer.submit_used(dev_id, 256).unwrap();
2323
2324 producer.poll_used().unwrap();
2325 }
2326
2327 assert_eq!(producer.avail_cursor.wrap(), !expected_wrap);
2329 }
2330 assert_invariants(&ring, &producer);
2331 }
2332
2333 #[test]
2334 fn test_next_flag_termination() {
2335 let ring = make_ring(8);
2336 let mut producer = make_producer(&ring);
2337
2338 let chain = BufferChainBuilder::new()
2339 .readable(0x1000, 256)
2340 .readable(0x2000, 256)
2341 .readable(0x3000, 256)
2342 .build()
2343 .unwrap();
2344
2345 producer.submit_available(&chain).unwrap();
2346
2347 for i in 0..2 {
2349 let desc = ring.read_desc(i);
2350 assert!(desc.flags().contains(DescFlags::NEXT));
2351 }
2352
2353 let tail_desc = ring.read_desc(2);
2355 assert!(!tail_desc.flags().contains(DescFlags::NEXT));
2356 }
2357
2358 #[test]
2359 fn test_consumer_initialization() {
2360 let ring = make_ring(8);
2361 let consumer = make_consumer(&ring);
2362
2363 assert_eq!(consumer.avail_cursor.head(), 0);
2364 assert!(consumer.avail_cursor.wrap());
2365 assert_eq!(consumer.used_cursor.head(), 0);
2366 assert!(consumer.used_cursor.wrap());
2367
2368 for i in 0..8 {
2369 assert_eq!(consumer.id_num[i], 0);
2370 }
2371 }
2372
2373 #[test]
2374 fn test_consumer_poll_available_single() {
2375 let ring = make_ring(8);
2376 let mut producer = make_producer(&ring);
2377 let mut consumer = make_consumer(&ring);
2378
2379 let id = producer.submit_one(0x1000, 512, false).unwrap();
2380
2381 let (polled_id, chain) = consumer.poll_available().unwrap();
2382
2383 assert_eq!(polled_id, id);
2384 assert_eq!(chain.len(), 1);
2385 assert_eq!(chain.elems()[0].addr, 0x1000);
2386 assert_eq!(chain.elems()[0].len, 512);
2387 assert!(!chain.elems()[0].writable);
2388
2389 assert_eq!(consumer.id_num[id as usize], 1);
2391 assert_eq!(consumer.avail_cursor.head(), 1);
2392 }
2393
2394 #[test]
2395 fn test_consumer_poll_available_chain() {
2396 let ring = make_ring(8);
2397 let mut producer = make_producer(&ring);
2398 let mut consumer = make_consumer(&ring);
2399
2400 let chain = BufferChainBuilder::new()
2401 .readable(0x1000, 256)
2402 .readable(0x2000, 256)
2403 .writable(0x3000, 512)
2404 .build()
2405 .unwrap();
2406
2407 let id = producer.submit_available(&chain).unwrap();
2408
2409 let (polled_id, polled_chain) = consumer.poll_available().unwrap();
2410
2411 assert_eq!(polled_id, id);
2412 assert_eq!(polled_chain.len(), 3);
2413
2414 assert_eq!(polled_chain.elems()[0].addr, 0x1000);
2415 assert!(!polled_chain.elems()[0].writable);
2416
2417 assert_eq!(polled_chain.elems()[1].addr, 0x2000);
2418 assert!(!polled_chain.elems()[1].writable);
2419
2420 assert_eq!(polled_chain.elems()[2].addr, 0x3000);
2421 assert!(polled_chain.elems()[2].writable);
2422
2423 assert_eq!(consumer.id_num[id as usize], 3);
2424 }
2425
2426 #[test]
2427 fn test_consumer_rejects_duplicate_inflight_id() {
2428 let ring = make_ring(8);
2429 let mut producer = make_producer(&ring);
2430 let mut consumer = make_consumer(&ring);
2431
2432 let id = producer.submit_one(0x1000, 512, false).unwrap();
2433 let (polled_id, _) = consumer.poll_available().unwrap();
2434 assert_eq!(polled_id, id);
2435
2436 let mut desc = Descriptor::new(0x2000, 256, id, DescFlags::empty());
2437 desc.mark_avail(consumer.avail_cursor.wrap());
2438 ring.write_desc(consumer.avail_cursor.head(), desc);
2439
2440 assert!(matches!(
2441 consumer.poll_available(),
2442 Err(RingError::InvalidState)
2443 ));
2444 }
2445
2446 #[test]
2447 fn test_consumer_submit_used() {
2448 let ring = make_ring(8);
2449 let mut producer = make_producer(&ring);
2450 let mut consumer = make_consumer(&ring);
2451
2452 let id = producer.submit_one(0x1000, 512, true).unwrap();
2453
2454 let (polled_id, _) = consumer.poll_available().unwrap();
2455
2456 consumer.submit_used(polled_id, 256).unwrap();
2458
2459 let desc = ring.read_desc(0);
2461
2462 assert_eq!(desc.id, id);
2463 assert_eq!(desc.len, 256);
2464 assert!(desc.is_used(true));
2465
2466 assert_eq!(consumer.used_cursor.head(), 1);
2468 assert_eq!(consumer.id_num[id as usize], 0);
2469 }
2470
2471 #[test]
2472 fn test_consumer_submit_used_multi_descriptor() {
2473 let ring = make_ring(8);
2474 let mut producer = make_producer(&ring);
2475 let mut consumer = make_consumer(&ring);
2476
2477 let chain = BufferChainBuilder::new()
2478 .readable(0x1000, 256)
2479 .writable(0x2000, 512)
2480 .writable(0x3000, 512)
2481 .build()
2482 .unwrap();
2483
2484 producer.submit_available(&chain).unwrap();
2485
2486 let (id, _) = consumer.poll_available().unwrap();
2487
2488 consumer.submit_used(id, 1024).unwrap();
2489
2490 let head_desc = ring.read_desc(0);
2492 assert!(head_desc.is_used(true));
2493 assert_eq!(head_desc.len, 1024);
2494
2495 assert_eq!(consumer.used_cursor.head(), 3);
2497 assert_eq!(consumer.id_num[id as usize], 0);
2498 }
2499
2500 #[test]
2501 fn test_consumer_poll_empty() {
2502 let ring = make_ring(4);
2503 let mut consumer = make_consumer(&ring);
2504
2505 assert!(matches!(
2506 consumer.poll_available(),
2507 Err(RingError::WouldBlock)
2508 ));
2509 }
2510
2511 #[test]
2512 fn test_consumer_peek() {
2513 let ring = make_ring(8);
2514 let mut producer = make_producer(&ring);
2515 let consumer = make_consumer(&ring);
2516
2517 producer.submit_one(0x1000, 512, false).unwrap();
2518 assert!(consumer.peek_available().unwrap());
2519
2520 let empty_ring = make_ring(4);
2521 let empty_consumer = make_consumer(&empty_ring);
2522 assert!(!empty_consumer.peek_available().unwrap());
2523 }
2524
2525 #[test]
2526 fn test_full_roundtrip() {
2527 let ring = make_ring(8);
2528 let mut producer = make_producer(&ring);
2529 let mut consumer = make_consumer(&ring);
2530
2531 let chain = BufferChainBuilder::new()
2532 .readable(0x1000, 256)
2533 .writable(0x2000, 512)
2534 .build()
2535 .unwrap();
2536
2537 let id = producer.submit_available(&chain).unwrap();
2538
2539 let (consumer_id, consumer_chain) = consumer.poll_available().unwrap();
2540
2541 assert_eq!(consumer_id, id);
2542 assert_eq!(consumer_chain.len(), 2);
2543
2544 consumer.submit_used(consumer_id, 512).unwrap();
2545
2546 let used = producer.poll_used().unwrap();
2547 assert_eq!(used.id, id);
2548 assert_eq!(used.len, 512);
2549 }
2550
2551 #[test]
2552 fn ring_initial_poll_used_blocks() {
2553 let ring = make_ring(8);
2554 let mut producer = make_producer(&ring);
2555 for _ in 0..8 {
2557 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2558 }
2559 assert_eq!(producer.num_free, ring.len());
2561 }
2562
2563 #[test]
2564 fn ring_consumer_blocks_until_submit() {
2565 let ring = make_ring(8);
2566 let mut producer = make_producer(&ring);
2567 let mut consumer = make_consumer(&ring);
2568
2569 assert!(matches!(
2570 consumer.poll_available(),
2571 Err(RingError::WouldBlock)
2572 ));
2573
2574 let chain = BufferChainBuilder::new()
2575 .readable(0x1000, 32)
2576 .readable(0x2000, 16)
2577 .build()
2578 .unwrap();
2579
2580 let id = producer.submit_available(&chain).unwrap();
2581
2582 let (cid, polled) = consumer.poll_available().unwrap();
2583 assert_eq!(cid, id);
2584 assert_eq!(polled.len(), chain.len());
2585 }
2586
2587 #[test]
2588 fn test_out_of_order_completion_stream() {
2589 let ring = make_ring(8);
2590 let mut producer = make_producer(&ring);
2591 let mut consumer = make_consumer(&ring);
2592
2593 let id_a = producer.submit_one(0x1000, 256, true).unwrap();
2595 let id_b = producer.submit_one(0x2000, 256, true).unwrap();
2596
2597 let (dev_id_a, chain_a) = consumer.poll_available().unwrap();
2599 assert_eq!(dev_id_a, id_a);
2600 assert_eq!(chain_a.len(), 1);
2601
2602 let (dev_id_b, chain_b) = consumer.poll_available().unwrap();
2603 assert_eq!(dev_id_b, id_b);
2604 assert_eq!(chain_b.len(), 1);
2605
2606 consumer.submit_used(dev_id_b, 128).unwrap();
2608 consumer.submit_used(dev_id_a, 256).unwrap();
2609
2610 let used_b = producer.poll_used().unwrap();
2612 assert_eq!(used_b.id, id_b);
2613 assert_eq!(used_b.len, 128);
2614
2615 let used_a = producer.poll_used().unwrap();
2617 assert_eq!(used_a.id, id_a);
2618 assert_eq!(used_a.len, 256);
2619
2620 assert!(producer.id_free.contains(&id_a));
2622 assert!(producer.id_free.contains(&id_b));
2623 }
2624
2625 #[test]
2626 fn test_mixed_chain_sizes_out_of_order_completion() {
2627 let ring = make_ring(16);
2628 let mut producer = make_producer(&ring);
2629 let mut consumer = make_consumer(&ring);
2630
2631 let chains = vec![
2632 BufferChainBuilder::new()
2633 .readable(0x1000, 10)
2634 .writable(0x2000, 5)
2635 .build()
2636 .unwrap(),
2637 BufferChainBuilder::new()
2638 .readable(0x3000, 8)
2639 .readable(0x3010, 8)
2640 .writable(0x3020, 16)
2641 .build()
2642 .unwrap(),
2643 BufferChainBuilder::new()
2644 .readable(0x4000, 4)
2645 .build()
2646 .unwrap(),
2647 BufferChainBuilder::new()
2648 .readable(0x5000, 4)
2649 .readable(0x5010, 4)
2650 .readable(0x5020, 4)
2651 .writable(0x5030, 4)
2652 .build()
2653 .unwrap(),
2654 ];
2655
2656 for c in &chains {
2657 producer.submit_available(c).unwrap();
2658 }
2659
2660 let mut dev_chain_lens = Vec::new();
2661 for _ in &chains {
2662 let (id, chain) = consumer.poll_available().unwrap();
2663 dev_chain_lens.push((id, chain.len() as u32));
2664 }
2665
2666 let order = [1, 3, 0, 2];
2667 let mut completion = Vec::new();
2668
2669 for &idx in &order {
2670 let (id, len) = dev_chain_lens[idx];
2671 consumer.submit_used(id, len).unwrap();
2672 completion.push((id, len));
2673 }
2674
2675 for (expected_id, expected_len) in &completion {
2676 let used = producer.poll_used().unwrap();
2677 assert_eq!(used.id, *expected_id);
2678 assert_eq!(used.len, *expected_len);
2679 assert_eq!(producer.id_num[*expected_id as usize], 0);
2680 assert!(producer.id_free.contains(expected_id));
2681 }
2682
2683 assert_invariants(&ring, &producer);
2684 }
2685
2686 #[test]
2688 fn test_used_stream_wrap_crossing() {
2689 let ring = make_ring(8);
2690 let mut producer = make_producer(&ring);
2691 let mut consumer = make_consumer(&ring);
2692
2693 let mut ids = Vec::new();
2695 for i in 0..8 {
2696 ids.push(producer.submit_one(0x1000 + i as u64, 1, false).unwrap());
2697 }
2698
2699 for _ in 0..8 {
2701 consumer.poll_available().unwrap();
2702 }
2703
2704 for &id in ids.iter().rev() {
2706 consumer.submit_used(id, 1).unwrap();
2707 }
2708
2709 for _ in 0..8 {
2711 producer.poll_used().unwrap();
2712 }
2713 assert_eq!(producer.used_cursor.head(), 0);
2714 assert!(!producer.used_cursor.wrap()); assert_invariants(&ring, &producer);
2716 }
2717
2718 #[test]
2720 fn test_interleaved_submit_completion() {
2721 let ring = make_ring(8);
2722 let mut producer = make_producer(&ring);
2723 let mut consumer = make_consumer(&ring);
2724
2725 let chain_a = BufferChainBuilder::new()
2727 .readable(0x1000, 8)
2728 .writable(0x2000, 8)
2729 .build()
2730 .unwrap();
2731 let id_a = producer.submit_available(&chain_a).unwrap();
2732
2733 let (dev_id_a, _) = consumer.poll_available().unwrap();
2735 assert_eq!(dev_id_a, id_a);
2736
2737 consumer.submit_used(dev_id_a, 8).unwrap();
2739
2740 let chain_b = BufferChainBuilder::new()
2742 .readable(0x3000, 4)
2743 .readable(0x3010, 4)
2744 .writable(0x3020, 4)
2745 .build()
2746 .unwrap();
2747 let id_b = producer.submit_available(&chain_b).unwrap();
2748
2749 let (dev_id_b, _) = consumer.poll_available().unwrap();
2751 assert_eq!(dev_id_b, id_b);
2752
2753 let used_a = producer.poll_used().unwrap();
2755 assert_eq!(used_a.id, id_a);
2756
2757 consumer.submit_used(dev_id_b, 12).unwrap();
2759
2760 let used_b = producer.poll_used().unwrap();
2762 assert_eq!(used_b.id, id_b);
2763
2764 assert_invariants(&ring, &producer);
2765 }
2766
2767 #[test]
2769 fn test_partial_publish_safety() {
2770 let ring = make_ring(8);
2771 let mut consumer = make_consumer(&ring);
2772 let mut producer = make_producer(&ring);
2773
2774 let chain = BufferChainBuilder::new()
2776 .readable(0x1000, 4)
2777 .readable(0x2000, 4)
2778 .writable(0x3000, 4)
2779 .build()
2780 .unwrap();
2781
2782 let id = producer.id_free.pop().unwrap();
2784 producer.id_num[id as usize] = chain.len() as u16;
2785
2786 let head_idx = producer.avail_cursor.head();
2788 let wrap_start = producer.avail_cursor.wrap();
2789 let mut pos = producer.avail_cursor;
2790 pos.advance();
2791
2792 for (i, elem) in chain.elems().iter().enumerate().skip(1) {
2793 let is_next = i + 1 < chain.len();
2794 let mut flags = DescFlags::empty();
2795 flags.set(DescFlags::NEXT, is_next);
2796 flags.set(DescFlags::WRITE, elem.writable);
2797 let mut d = Descriptor::new(elem.addr, elem.len, id, flags);
2798 d.mark_avail(pos.wrap());
2799 ring.write_desc(pos.head(), d);
2800 pos.advance();
2801 }
2802
2803 assert!(matches!(
2805 consumer.poll_available(),
2806 Err(RingError::WouldBlock)
2807 ));
2808
2809 let head_elem = chain.elems()[0];
2811 let mut head_flags = DescFlags::empty();
2812 head_flags.set(DescFlags::NEXT, true);
2813 head_flags.set(DescFlags::WRITE, head_elem.writable);
2814 let mut head_desc = Descriptor::new(head_elem.addr, head_elem.len, id, head_flags);
2815 head_desc.mark_avail(wrap_start);
2816 ring.write_desc(head_idx, head_desc);
2817 producer.avail_cursor = pos;
2818 producer.num_free -= chain.len();
2819
2820 let (dev_id, dev_chain) = consumer.poll_available().unwrap();
2822 assert_eq!(dev_id, id);
2823 assert_eq!(dev_chain.len(), chain.len());
2824 assert_invariants(&ring, &producer);
2825 }
2826
2827 #[test]
2829 fn test_tail_marked_used_ignored() {
2830 let ring = make_ring(8);
2831 let mut producer = make_producer(&ring);
2832
2833 let chain = BufferChainBuilder::new()
2834 .readable(0x1000, 4)
2835 .readable(0x2000, 4)
2836 .build()
2837 .unwrap();
2838 let id = producer.submit_available(&chain).unwrap();
2839
2840 let mut tail_desc = ring.read_desc(1);
2842 tail_desc.mark_used(producer.used_cursor.wrap());
2843 ring.write_desc(1, tail_desc);
2844
2845 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2847
2848 let mut head_desc = ring.read_desc(0);
2850 head_desc.mark_used(producer.used_cursor.wrap());
2851 ring.write_desc(0, head_desc);
2852
2853 let used = producer.poll_used().unwrap();
2855 assert_eq!(used.id, id);
2856 assert_invariants(&ring, &producer);
2857 }
2858
2859 #[test]
2861 fn test_max_chain_len_rejected() {
2862 let ring = make_ring(8);
2863 let mut producer = make_producer(&ring);
2864
2865 let elems = (0..9).map(|i| BufferElement {
2867 addr: 0x1000 + i as u64,
2868 len: 42,
2869 writable: false,
2870 });
2871
2872 let chain = BufferChainBuilder::new().readables(elems).build().unwrap();
2873
2874 assert!(matches!(
2876 producer.submit_available(&chain),
2877 Err(RingError::WouldBlock)
2878 ));
2879 }
2880
2881 #[test]
2883 fn test_descriptor_state_monotonicity() {
2884 let ring = make_ring(8);
2885 let mut producer = make_producer(&ring);
2886 let mut consumer = make_consumer(&ring);
2887
2888 let mut states = vec![0u8; 8];
2890
2891 for _ in 0..5 {
2892 for state in states.iter_mut() {
2893 let id = producer.submit_one(0x1000, 4, false).unwrap();
2894 *state = (*state).max(1);
2896
2897 let (dev_id, _) = consumer.poll_available().unwrap();
2899 consumer.submit_used(dev_id, 4).unwrap();
2900 *state = (*state).max(2);
2901
2902 let used = producer.poll_used().unwrap();
2903 assert_eq!(used.id, id);
2904 *state = (*state).max(3);
2905 }
2906
2907 assert_invariants(&ring, &producer);
2908 }
2909
2910 for s in states {
2912 assert!(s >= 3);
2913 }
2914 }
2915
2916 #[test]
2918 fn test_random_stress_small() {
2919 use rand::RngExt;
2920 use rand::seq::SliceRandom;
2921
2922 let ring = make_ring(16);
2923 let mut producer = make_producer(&ring);
2924 let mut consumer = make_consumer(&ring);
2925 let mut rng = rand::rng();
2926
2927 let mut active_ids = Vec::new();
2929 for _ in 0..8 {
2930 let len = rng.random_range(1..=4);
2931 let mut b = BufferChainBuilder::new().readable(0x1000, 4);
2932 for i in 1..len {
2933 b = b.readable(0x1000 + i as u64 * 0x10, 4);
2934 }
2935 let chain = b.build().unwrap();
2936 if let Ok(id) = producer.submit_available(&chain) {
2937 active_ids.push(id);
2938 }
2939 }
2940
2941 let mut dev_ids = Vec::new();
2942 while let Ok((id, _)) = consumer.poll_available() {
2943 dev_ids.push(id);
2944 }
2945
2946 dev_ids.shuffle(&mut rng);
2948 for id in &dev_ids {
2949 let chain_len = consumer.id_num[*id as usize];
2950 consumer.submit_used(*id, chain_len as u32 * 4).unwrap();
2951 }
2952 for _ in &dev_ids {
2954 if producer.poll_used().is_ok() {}
2955 }
2956
2957 assert_invariants(&ring, &producer);
2958 }
2959
2960 #[test]
2962 fn test_out_of_order_multi_length() {
2963 let ring = make_ring(16);
2964 let mut producer = make_producer(&ring);
2965 let mut consumer = make_consumer(&ring);
2966
2967 let chain_a = BufferChainBuilder::new()
2968 .readable(0x1000, 4)
2969 .writable(0x2000, 4)
2970 .build()
2971 .unwrap();
2972 let chain_b = BufferChainBuilder::new()
2973 .readable(0x3000, 4)
2974 .readable(0x3010, 4)
2975 .writable(0x3020, 4)
2976 .build()
2977 .unwrap();
2978 let chain_c = BufferChainBuilder::new()
2979 .readable(0x4000, 4)
2980 .build()
2981 .unwrap();
2982
2983 let id_a = producer.submit_available(&chain_a).unwrap();
2984 let id_b = producer.submit_available(&chain_b).unwrap();
2985 let id_c = producer.submit_available(&chain_c).unwrap();
2986
2987 let (d_a, _) = consumer.poll_available().unwrap();
2988 let (d_b, _) = consumer.poll_available().unwrap();
2989 let (d_c, _) = consumer.poll_available().unwrap();
2990 assert_eq!(d_a, id_a);
2991 assert_eq!(d_b, id_b);
2992 assert_eq!(d_c, id_c);
2993
2994 consumer.submit_used(d_b, 12).unwrap();
2996 consumer.submit_used(d_c, 4).unwrap();
2997 consumer.submit_used(d_a, 8).unwrap();
2998
2999 let u_b = producer.poll_used().unwrap();
3000 assert_eq!(u_b.id, id_b);
3001 let u_c = producer.poll_used().unwrap();
3002 assert_eq!(u_c.id, id_c);
3003 let u_a = producer.poll_used().unwrap();
3004 assert_eq!(u_a.id, id_a);
3005
3006 assert_invariants(&ring, &producer);
3007 }
3008
3009 #[test]
3010 fn interleave_submit_and_completion() {
3011 let ring = make_ring(16);
3012 let mut producer = make_producer(&ring);
3013 let mut consumer = make_consumer(&ring);
3014
3015 let chain_a = BufferChainBuilder::new()
3017 .readable(0x1000, 4)
3018 .writable(0x2000, 4)
3019 .build()
3020 .unwrap();
3021 let id_a = producer.submit_available(&chain_a).unwrap();
3022
3023 let (d_a, _) = consumer.poll_available().unwrap();
3025 assert_eq!(d_a, id_a);
3026
3027 consumer.submit_used(d_a, 8).unwrap();
3029
3030 let chain_b = BufferChainBuilder::new()
3032 .readable(0x3000, 4)
3033 .readable(0x3010, 4)
3034 .writable(0x3020, 4)
3035 .build()
3036 .unwrap();
3037 let id_b = producer.submit_available(&chain_b).unwrap();
3038
3039 let u_a = producer.poll_used().unwrap();
3041 assert_eq!(u_a.id, id_a);
3042 assert_eq!(u_a.len, 8);
3043
3044 let (d_b, _) = consumer.poll_available().unwrap();
3046 assert_eq!(d_b, id_b);
3047 consumer.submit_used(d_b, 12).unwrap();
3048
3049 let id_c = producer.submit_one(0x4000, 4, false).unwrap();
3051
3052 let (d_c, _) = consumer.poll_available().unwrap();
3054 assert_eq!(d_c, id_c);
3055 consumer.submit_used(d_c, 4).unwrap();
3056
3057 let u_b = producer.poll_used().unwrap();
3059 assert_eq!(u_b.id, id_b);
3060 assert_eq!(u_b.len, 12);
3061
3062 let u_c = producer.poll_used().unwrap();
3063 assert_eq!(u_c.id, id_c);
3064 assert_eq!(u_c.len, 4);
3065
3066 assert_invariants(&ring, &producer);
3067 }
3068
3069 #[test]
3071 fn producer_disable_used_notifications_writes_driver_disable() {
3072 let ring = make_ring(8);
3073 let mut producer = make_producer(&ring);
3074
3075 assert_eq!(ring.read_driver_event().flags(), EventFlags::ENABLE);
3076 producer.disable_used_notifications().unwrap();
3077 assert_eq!(ring.read_driver_event().flags(), EventFlags::DISABLE);
3078 }
3079
3080 #[test]
3081 fn producer_enable_used_notifications_writes_driver_enable() {
3082 let ring = make_ring(8);
3083 let mut producer = make_producer(&ring);
3084
3085 producer.disable_used_notifications().unwrap();
3086 assert_eq!(ring.read_driver_event().flags(), EventFlags::DISABLE);
3087
3088 producer.enable_used_notifications().unwrap();
3089 assert_eq!(ring.read_driver_event().flags(), EventFlags::ENABLE);
3090 }
3091
3092 #[test]
3093 fn producer_enable_used_notifications_desc_sets_off_wrap_and_flags() {
3094 let ring = make_ring(8);
3095 let mut producer = make_producer(&ring);
3096
3097 producer.enable_used_notifications_desc(5, true).unwrap();
3098
3099 let evt = ring.read_driver_event();
3100 assert_eq!(evt.flags(), EventFlags::DESC);
3101 assert_eq!(evt.desc_event_off(), 5);
3102 assert!(evt.desc_event_wrap());
3103 }
3104
3105 #[test]
3106 fn producer_enable_used_notifications_for_next_programs_used_cursor() {
3107 let ring = make_ring(8);
3108 let mut producer = make_producer(&ring);
3109
3110 producer.enable_used_notifications_for_next().unwrap();
3112
3113 let evt = ring.read_driver_event();
3114 assert_eq!(evt.flags(), EventFlags::DESC);
3115 assert_eq!(evt.desc_event_off(), 0);
3116 assert!(evt.desc_event_wrap());
3117 }
3118
3119 #[test]
3120 fn consumer_disable_avail_notifications_writes_device_disable() {
3121 let ring = make_ring(8);
3122 let mut consumer = make_consumer(&ring);
3123
3124 assert_eq!(ring.read_device_event().flags(), EventFlags::ENABLE);
3125 consumer.disable_avail_notifications().unwrap();
3126 assert_eq!(ring.read_device_event().flags(), EventFlags::DISABLE);
3127 }
3128
3129 #[test]
3130 fn consumer_enable_avail_notifications_writes_device_enable() {
3131 let ring = make_ring(8);
3132 let mut consumer = make_consumer(&ring);
3133
3134 consumer.disable_avail_notifications().unwrap();
3135 assert_eq!(ring.read_device_event().flags(), EventFlags::DISABLE);
3136
3137 consumer.enable_avail_notifications().unwrap();
3138 assert_eq!(ring.read_device_event().flags(), EventFlags::ENABLE);
3139 }
3140
3141 #[test]
3142 fn consumer_enable_avail_notifications_desc_sets_off_wrap_and_flags() {
3143 let ring = make_ring(8);
3144 let mut consumer = make_consumer(&ring);
3145
3146 consumer.enable_avail_notifications_desc(7, false).unwrap();
3147
3148 let evt = ring.read_device_event();
3149 assert_eq!(evt.flags(), EventFlags::DESC);
3150 assert_eq!(evt.desc_event_off(), 7);
3151 assert!(!evt.desc_event_wrap());
3152 }
3153
3154 #[test]
3155 fn consumer_enable_avail_notifications_for_next_programs_avail_cursor() {
3156 let ring = make_ring(8);
3157 let mut consumer = make_consumer(&ring);
3158
3159 consumer.enable_avail_notifications_for_next().unwrap();
3161
3162 let evt = ring.read_device_event();
3163 assert_eq!(evt.flags(), EventFlags::DESC);
3164 assert_eq!(evt.desc_event_off(), 0);
3165 assert!(evt.desc_event_wrap());
3166 }
3167
3168 #[test]
3169 fn producer_does_not_write_device_event_when_toggling_used_notifications() {
3170 let ring = make_ring(8);
3171 let mut producer = make_producer(&ring);
3172
3173 let dev_before = ring.read_device_event();
3174 producer.disable_used_notifications().unwrap();
3175 let dev_after = ring.read_device_event();
3176
3177 assert_eq!(dev_after, dev_before);
3178 }
3179
3180 #[test]
3181 fn consumer_does_not_write_driver_event_when_toggling_avail_notifications() {
3182 let ring = make_ring(8);
3183 let mut consumer = make_consumer(&ring);
3184
3185 let drv_before = ring.read_driver_event();
3186 consumer.disable_avail_notifications().unwrap();
3187 let drv_after = ring.read_driver_event();
3188
3189 assert_eq!(drv_after, drv_before);
3190 }
3191
3192 #[test]
3193 fn should_notify_flags_enable_disable() {
3194 let ring_len = 8;
3195
3196 let old = RingCursor {
3197 head: 0,
3198 size: ring_len,
3199 wrap: true,
3200 };
3201 let new = RingCursor {
3202 head: 1,
3203 size: ring_len,
3204 wrap: true,
3205 };
3206
3207 let evt = EventSuppression::new(0, EventFlags::DISABLE);
3209 assert!(!should_notify(evt, ring_len, old, new));
3210
3211 let evt = EventSuppression::new(0, EventFlags::ENABLE);
3213 assert!(should_notify(evt, ring_len, old, new));
3214 }
3215
3216 #[test]
3217 fn should_notify_desc_no_crossing() {
3218 let ring_len = 8;
3219
3220 let old = RingCursor {
3221 head: 2,
3222 size: ring_len,
3223 wrap: true,
3224 };
3225 let new = RingCursor {
3226 head: 3,
3227 size: ring_len,
3228 wrap: true,
3229 };
3230
3231 let mut evt = EventSuppression::zeroed();
3233 evt.set_desc_event(6, true);
3234 evt.set_flags(EventFlags::DESC);
3235
3236 assert!(!should_notify(evt, ring_len, old, new));
3237 }
3238
3239 #[test]
3240 fn should_notify_desc_wrap_mismatch_adjusts_event_idx() {
3241 let ring_len = 8;
3242
3243 let old = RingCursor {
3244 head: 7,
3245 size: ring_len,
3246 wrap: true,
3247 };
3248 let new = RingCursor {
3249 head: 1,
3250 size: ring_len,
3251 wrap: false,
3252 };
3253
3254 let mut evt = EventSuppression::zeroed();
3255 evt.set_desc_event(7, true);
3256 evt.set_flags(EventFlags::DESC);
3257
3258 assert!(should_notify(evt, ring_len, old, new));
3259 }
3260
3261 #[test]
3262 fn ring_need_event_basic_cases() {
3263 assert!(ring_need_event(4, 5, 2));
3265 assert!(!ring_need_event(4, 5, 5));
3267
3268 let old = 0xFFFE;
3270 let new = 1;
3271 assert!(ring_need_event(0xFFFF, new, old));
3273 }
3274
3275 #[test]
3277 fn bad_device_marks_tail_used() {
3278 let ring = make_ring(8);
3279 let mut producer = make_producer(&ring);
3280
3281 let chain = BufferChainBuilder::new()
3282 .readable(0x1000, 4)
3283 .readable(0x2000, 4)
3284 .build()
3285 .unwrap();
3286 let id = producer.submit_available(&chain).unwrap();
3287
3288 let mut tail = ring.read_desc(1);
3290 tail.mark_used(producer.used_cursor.wrap());
3291 ring.write_desc(1, tail);
3292
3293 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3295
3296 let mut head = ring.read_desc(0);
3298 head.mark_used(producer.used_cursor.wrap());
3299 ring.write_desc(0, head);
3300
3301 let used = producer.poll_used().unwrap();
3302 assert_eq!(used.id, id);
3303 }
3304
3305 #[test]
3306 fn bad_device_wrong_used_bits() {
3307 let ring = make_ring(4);
3308 let mut producer = make_producer(&ring);
3309
3310 let id = producer.submit_one(0x1000, 8, true).unwrap();
3311
3312 let mut d = ring.read_desc(0);
3314 d.mark_avail(producer.used_cursor.wrap());
3316 d.len = 8;
3317 ring.write_desc(0, d);
3318
3319 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3320
3321 let mut d2 = ring.read_desc(0);
3322 d2.mark_used(producer.used_cursor.wrap());
3323 ring.write_desc(0, d2);
3324
3325 let u = producer.poll_used().unwrap();
3326 assert_eq!(u.id, id);
3327 }
3328
3329 #[test]
3330 fn bad_driver_next_never_clears() {
3331 let ring = make_ring(8);
3332 let mut consumer = make_consumer(&ring);
3333 let mut producer = make_producer(&ring);
3334
3335 let id = producer.id_free.pop().unwrap();
3337 producer.id_num[id as usize] = 8;
3338
3339 let mut pos = producer.avail_cursor;
3340 let wrap_start = pos.wrap();
3341
3342 for _ in 0..8 {
3344 let idx = pos.head();
3345 let mut flags = DescFlags::empty();
3346 flags.set(DescFlags::NEXT, true); let mut desc = Descriptor::new(0x1000 + idx as u64 * 0x10, 4, id, flags);
3348 desc.mark_avail(pos.wrap());
3349 ring.write_desc(idx, desc);
3350 pos.advance();
3351 }
3352
3353 let head_idx = producer.avail_cursor.head();
3355 let mut head_flags = DescFlags::empty();
3356 head_flags.set(DescFlags::NEXT, true);
3357 let mut head_desc = Descriptor::new(0x42, 4, id, head_flags);
3358 head_desc.mark_avail(wrap_start);
3359 ring.write_desc(head_idx, head_desc);
3360
3361 assert!(matches!(
3363 consumer.poll_available(),
3364 Err(RingError::BadChain)
3365 ));
3366 }
3367
3368 #[test]
3369 fn bad_driver_interleaved_readables_and_writables() {
3370 let ring = make_ring(8);
3371 let mut consumer = make_consumer(&ring);
3372 let mut producer = make_producer(&ring);
3373
3374 let chain = BufferChainBuilder::new()
3375 .readable(0x1000, 4)
3376 .readable(0x2000, 4)
3377 .writable(0x2000, 4)
3378 .build()
3379 .unwrap();
3380
3381 let _id = producer.submit_available(&chain).unwrap();
3382
3383 let mut first = ring.read_desc(0);
3385 first.flags |= DescFlags::WRITE.bits();
3386 ring.write_desc(0, first);
3387
3388 assert!(matches!(
3389 consumer.poll_available(),
3390 Err(RingError::BadChain)
3391 ));
3392 }
3393
3394 #[test]
3395 fn bad_device_marks_multiple_used_in_chain() {
3396 let ring = make_ring(8);
3397 let mut producer = make_producer(&ring);
3398
3399 let chain = BufferChainBuilder::new()
3400 .readable(0x1000, 4)
3401 .readable(0x2000, 4)
3402 .build()
3403 .unwrap();
3404 let id = producer.submit_available(&chain).unwrap();
3405
3406 let mut head = ring.read_desc(0);
3408 head.mark_used(producer.used_cursor.wrap());
3409 ring.write_desc(0, head);
3410
3411 let mut tail = ring.read_desc(1);
3412 tail.mark_used(producer.used_cursor.wrap());
3413 ring.write_desc(1, tail);
3414
3415 let u = producer.poll_used().unwrap();
3417 assert_eq!(u.id, id);
3418
3419 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3421 }
3422
3423 #[test]
3424 fn bad_device_writes_used_at_wrong_slot() {
3425 let ring = make_ring(8);
3426 let mut producer = make_producer(&ring);
3427
3428 let _id = producer.submit_one(0x1000, 4, true).unwrap();
3429
3430 let mut d = ring.read_desc(3);
3432 d.mark_used(producer.used_cursor.wrap());
3433 ring.write_desc(3, d);
3434
3435 assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3437
3438 let mut d0 = ring.read_desc(0);
3440 d0.mark_used(producer.used_cursor.wrap());
3441 ring.write_desc(0, d0);
3442 let _u = producer.poll_used().unwrap();
3443 }
3444
3445 #[test]
3446 fn bad_driver_reuses_id_while_outstanding() {
3447 let ring = make_ring(8);
3448 let mut producer = make_producer(&ring);
3449
3450 let id = producer.submit_one(0x1000, 4, false).unwrap();
3452 assert_eq!(producer.id_num[id as usize], 1);
3453
3454 producer.id_free.push(id);
3456
3457 let res = producer.submit_one(0x2000, 4, false);
3459 assert!(matches!(res, Err(RingError::InvalidState)));
3460 }
3461
3462 #[test]
3463 fn test_avail_cursor_accessor() {
3464 let ring = make_ring(8);
3465 let mut producer = make_producer(&ring);
3466
3467 let cursor = producer.avail_cursor();
3469 assert_eq!(cursor.head(), 0);
3470 assert!(cursor.wrap());
3471
3472 producer.submit_one(0x1000, 512, false).unwrap();
3474 let cursor = producer.avail_cursor();
3475 assert_eq!(cursor.head(), 1);
3476 assert!(cursor.wrap());
3477 }
3478
3479 #[test]
3480 fn test_should_notify_since() {
3481 let ring = make_ring(8);
3482 let mut producer = make_producer(&ring);
3483
3484 let before = producer.avail_cursor();
3485 producer.submit_one(0x1000, 512, false).unwrap();
3486
3487 let should_notify = producer.should_notify_since(before).unwrap();
3489 assert!(should_notify);
3490 }
3491
3492 #[test]
3493 fn test_batch_notification_single_check() {
3494 let ring = make_ring(8);
3495 let mut producer = make_producer(&ring);
3496 let mut consumer = make_consumer(&ring);
3497
3498 let before = producer.avail_cursor();
3499
3500 producer.submit_one(0x1000, 512, false).unwrap();
3502 producer.submit_one(0x2000, 512, false).unwrap();
3503 producer.submit_one(0x3000, 512, false).unwrap();
3504
3505 let should_notify = producer.should_notify_since(before).unwrap();
3507 assert!(should_notify);
3508
3509 for _ in 0..3 {
3511 let (_, _) = consumer.poll_available().unwrap();
3512 }
3513 }
3514
3515 #[test]
3516 fn test_ring_cursor_reset() {
3517 let mut cursor = RingCursor::new(16);
3518 cursor.advance_by(5);
3519 assert_eq!(cursor.head(), 5);
3520
3521 cursor.reset();
3522 assert_eq!(cursor, RingCursor::new(16));
3523 assert_eq!(cursor.head(), 0);
3524 assert!(cursor.wrap());
3525 }
3526
3527 #[test]
3528 fn test_ring_cursor_reset_after_wrap() {
3529 let mut cursor = RingCursor::new(4);
3530 cursor.advance_by(5);
3532 assert_eq!(cursor.head(), 1);
3533 assert!(!cursor.wrap());
3534
3535 cursor.reset();
3536 assert_eq!(cursor.head(), 0);
3537 assert!(cursor.wrap());
3538 }
3539
3540 #[test]
3541 fn test_ring_producer_reset_matches_new() {
3542 let ring = make_ring(8);
3543 let fresh = make_producer(&ring);
3544
3545 let mut used = make_producer(&ring);
3546 used.submit_one(0x1000, 64, false).unwrap();
3548 used.submit_one(0x2000, 128, true).unwrap();
3549
3550 used.reset().unwrap();
3551
3552 assert_eq!(used.avail_cursor, fresh.avail_cursor);
3553 assert_eq!(used.used_cursor, fresh.used_cursor);
3554 assert_eq!(used.num_free, fresh.num_free);
3555 assert_eq!(used.id_free.len(), fresh.id_free.len());
3556 assert_eq!(used.id_num.as_slice(), fresh.id_num.as_slice());
3557 assert_eq!(used.event_flags_shadow, fresh.event_flags_shadow);
3558 }
3559
3560 #[test]
3561 fn test_ring_producer_reset_id_free_complete() {
3562 let ring = make_ring(8);
3563 let mut producer = make_producer(&ring);
3564
3565 for i in 0..4u64 {
3567 producer.submit_one(0x1000 + i * 0x100, 64, false).unwrap();
3568 }
3569 assert_eq!(producer.num_free, 4);
3570
3571 producer.reset().unwrap();
3572
3573 assert_eq!(producer.num_free, 8);
3574 assert_eq!(producer.id_free.len(), 8);
3575 for id in 0..8u16 {
3577 assert!(producer.id_free.contains(&id));
3578 }
3579 }
3580
3581 #[test]
3582 fn test_ring_producer_failed_reset_preserves_local_state() {
3583 let ring = make_ring(4);
3584 let mem = FaultMem::new(ring.mem());
3585 let mut producer = RingProducer::new(ring.layout(), mem.clone());
3586
3587 producer.submit_one(0x1000, 64, false).unwrap();
3588 producer.submit_one(0x2000, 128, true).unwrap();
3589
3590 let avail_cursor = producer.avail_cursor;
3591 let used_cursor = producer.used_cursor;
3592 let num_free = producer.num_free;
3593 let id_free = producer.id_free.clone();
3594 let id_num = producer.id_num.clone();
3595 let event_flags_shadow = producer.event_flags_shadow;
3596
3597 mem.fail_write_at(1);
3598 assert!(matches!(
3599 producer.reset(),
3600 Err(RingError::MemError {
3601 op: MemOp::WriteDesc,
3602 ..
3603 })
3604 ));
3605
3606 assert_eq!(producer.avail_cursor, avail_cursor);
3607 assert_eq!(producer.used_cursor, used_cursor);
3608 assert_eq!(producer.num_free, num_free);
3609 assert_eq!(producer.id_free, id_free);
3610 assert_eq!(producer.id_num, id_num);
3611 assert_eq!(producer.event_flags_shadow, event_flags_shadow);
3612
3613 mem.allow_writes();
3614 producer.reset().unwrap();
3615
3616 let fresh = RingProducer::new(ring.layout(), mem);
3617 assert_eq!(producer.avail_cursor, fresh.avail_cursor);
3618 assert_eq!(producer.used_cursor, fresh.used_cursor);
3619 assert_eq!(producer.num_free, fresh.num_free);
3620 assert_eq!(producer.id_free, fresh.id_free);
3621 assert_eq!(producer.id_num, fresh.id_num);
3622 assert_eq!(producer.event_flags_shadow, fresh.event_flags_shadow);
3623 }
3624
3625 #[test]
3626 fn test_ring_consumer_reset_matches_new() {
3627 let ring = make_ring(8);
3628 let fresh = make_consumer(&ring);
3629
3630 let mut used = make_consumer(&ring);
3631
3632 let mut producer = make_producer(&ring);
3634 producer.submit_one(0x1000, 64, false).unwrap();
3635
3636 let (id, _chain) = used.poll_available().unwrap();
3638 used.submit_used(id, 64).unwrap();
3639
3640 used.reset().unwrap();
3641
3642 assert_eq!(used.avail_cursor, fresh.avail_cursor);
3643 assert_eq!(used.used_cursor, fresh.used_cursor);
3644 assert_eq!(used.id_num.as_slice(), fresh.id_num.as_slice());
3645 assert_eq!(used.num_inflight, fresh.num_inflight);
3646 assert_eq!(used.event_flags_shadow, fresh.event_flags_shadow);
3647 }
3648
3649 #[test]
3650 fn test_ring_consumer_reset_clears_inflight() {
3651 let ring = make_ring(8);
3652 let mut producer = make_producer(&ring);
3653 let mut consumer = make_consumer(&ring);
3654
3655 producer.submit_one(0x1000, 64, false).unwrap();
3657 producer.submit_one(0x2000, 64, false).unwrap();
3658 let _ = consumer.poll_available().unwrap();
3659 let _ = consumer.poll_available().unwrap();
3660 assert_eq!(consumer.num_inflight, 2);
3661
3662 consumer.reset().unwrap();
3663 assert_eq!(consumer.num_inflight, 0);
3664 }
3665
3666 #[test]
3667 fn test_desc_table_read_after_submit() {
3668 let ring = make_ring(8);
3669 let mut writer = make_producer(&ring);
3670 writer.submit_one(0x1000, 4096, true).unwrap();
3671
3672 let reader = make_producer(&ring);
3673 let addr = reader.desc_table().desc_addr(0).unwrap();
3674 let flags = Descriptor::read_flags_acquire(reader.mem(), addr).unwrap();
3675 let desc = Descriptor::read_body(reader.mem(), addr, flags).unwrap();
3676 assert_eq!(desc.addr, 0x1000);
3677 assert_eq!(desc.len, 4096);
3678 assert!(desc.is_writable());
3679 assert!(desc.is_avail(true));
3680 assert!(!desc.is_used(true));
3681 }
3682
3683 #[test]
3684 fn test_desc_table_out_of_bounds() {
3685 let ring = make_ring(8);
3686 let reader = make_producer(&ring);
3687 assert!(reader.desc_table().desc_addr(8).is_none());
3688 }
3689
3690 #[test]
3691 fn test_desc_table_read_used_descriptor() {
3692 let ring = make_ring(8);
3693 let mut writer = make_producer(&ring);
3694 writer.submit_one(0x1000, 4096, true).unwrap();
3695
3696 let mut consumer = make_consumer(&ring);
3697 let (id, _chain) = consumer.poll_available().unwrap();
3698 consumer.submit_used(id, 128).unwrap();
3699
3700 let reader = make_producer(&ring);
3701 let addr = reader.desc_table().desc_addr(0).unwrap();
3702 let flags = Descriptor::read_flags_acquire(reader.mem(), addr).unwrap();
3703 let desc = Descriptor::read_body(reader.mem(), addr, flags).unwrap();
3704 assert!(desc.is_used(true));
3705 assert!(!desc.is_avail(true));
3706 }
3707}
3708
3709#[cfg(test)]
3711mod virtio_villain {
3712 use super::tests::{OwnedRing, make_consumer, make_producer, make_ring};
3713 use super::*;
3714
3715 fn raw_desc(id: u16, len: u32, flags: DescFlags) -> Descriptor {
3716 Descriptor::new(0x1000 + u64::from(id) * 0x10, len, id, flags)
3717 }
3718
3719 fn avail_desc(id: u16, len: u32, flags: DescFlags) -> Descriptor {
3720 let mut desc = raw_desc(id, len, flags);
3721 desc.mark_avail(true);
3722 desc
3723 }
3724
3725 fn write_avail(ring: &OwnedRing, idx: u16, id: u16, flags: DescFlags) {
3726 write_avail_len(ring, idx, id, 16, flags);
3727 }
3728
3729 fn write_avail_len(ring: &OwnedRing, idx: u16, id: u16, len: u32, flags: DescFlags) {
3730 ring.write_desc(idx, avail_desc(id, len, flags));
3731 }
3732
3733 fn write_driver_event(ring: &OwnedRing, off_wrap: u16, flags: u16) {
3734 write_event(ring, ring.layout().drv_evt_addr(), off_wrap, flags);
3735 }
3736
3737 fn write_device_event(ring: &OwnedRing, off_wrap: u16, flags: u16) {
3738 write_event(ring, ring.layout().dev_evt_addr(), off_wrap, flags);
3739 }
3740
3741 fn write_event(ring: &OwnedRing, addr: u64, off_wrap: u16, flags: u16) {
3742 ring.mem()
3743 .write(
3744 addr,
3745 &[
3746 (off_wrap & 0xff) as u8,
3747 (off_wrap >> 8) as u8,
3748 (flags & 0xff) as u8,
3749 (flags >> 8) as u8,
3750 ],
3751 )
3752 .unwrap();
3753 }
3754
3755 #[test]
3756 fn p0003_chain_with_next_past_ring_is_rejected() {
3757 let ring = make_ring(4);
3758 let mut consumer = make_consumer(&ring);
3759
3760 for idx in 0..ring.len() as u16 {
3761 write_avail(&ring, idx, 0, DescFlags::NEXT);
3762 }
3763
3764 assert!(matches!(
3765 consumer.poll_available(),
3766 Err(RingError::BadChain)
3767 ));
3768 }
3769
3770 #[test]
3771 fn p0005_indirect_without_feature_is_rejected() {
3772 let ring = make_ring(8);
3773 let mut consumer = make_consumer(&ring);
3774
3775 write_avail(&ring, 0, 0, DescFlags::INDIRECT);
3776
3777 assert!(matches!(
3778 consumer.poll_available(),
3779 Err(RingError::BadChain)
3780 ));
3781 }
3782
3783 #[test]
3784 fn p0006_indirect_with_bad_flags_is_rejected() {
3785 let ring = make_ring(8);
3786 let mut consumer = make_consumer(&ring);
3787
3788 write_avail(&ring, 0, 0, DescFlags::INDIRECT | DescFlags::WRITE);
3789
3790 assert!(matches!(
3791 consumer.poll_available(),
3792 Err(RingError::BadChain)
3793 ));
3794 }
3795
3796 #[test]
3797 fn p0007_indirect_tail_in_mixed_chain_is_rejected() {
3798 let ring = make_ring(8);
3799 let mut consumer = make_consumer(&ring);
3800
3801 write_avail(&ring, 0, 0, DescFlags::NEXT);
3802 write_avail(&ring, 1, 0, DescFlags::INDIRECT | DescFlags::WRITE);
3803
3804 assert!(matches!(
3805 consumer.poll_available(),
3806 Err(RingError::BadChain)
3807 ));
3808 }
3809
3810 #[test]
3811 fn p0008_long_chain_is_consumable_at_ring_layer() {
3812 let ring = make_ring(16);
3813 let mut consumer = make_consumer(&ring);
3814
3815 for idx in 0..12u16 {
3816 let flags = if idx == 11 {
3817 DescFlags::WRITE
3818 } else {
3819 DescFlags::NEXT | DescFlags::WRITE
3820 };
3821 write_avail(&ring, idx, 0, flags);
3822 }
3823
3824 let (id, chain) = consumer.poll_available().unwrap();
3825 assert_eq!(id, 0);
3826 assert_eq!(chain.len(), 12);
3827 }
3828
3829 #[test]
3830 fn p0009_unaligned_descriptor_address_is_preserved() {
3831 let ring = make_ring(8);
3832 let mut consumer = make_consumer(&ring);
3833
3834 let mut desc = Descriptor::new(0x1003, 16, 0, DescFlags::empty());
3835 desc.mark_avail(true);
3836 ring.write_desc(0, desc);
3837
3838 let (_, chain) = consumer.poll_available().unwrap();
3839 assert_eq!(chain.elems()[0].addr, 0x1003);
3840 }
3841
3842 #[test]
3843 fn p0010_writable_before_readable_is_rejected() {
3844 let ring = make_ring(8);
3845 let mut consumer = make_consumer(&ring);
3846
3847 write_avail(&ring, 0, 0, DescFlags::NEXT);
3848 write_avail(&ring, 1, 0, DescFlags::NEXT | DescFlags::WRITE);
3849 write_avail(&ring, 2, 0, DescFlags::NEXT);
3850 write_avail(&ring, 3, 0, DescFlags::WRITE);
3851
3852 assert!(matches!(
3853 consumer.poll_available(),
3854 Err(RingError::BadChain)
3855 ));
3856 }
3857
3858 #[test]
3859 fn p0011_stale_wrap_descriptor_is_ignored() {
3860 let ring = make_ring(8);
3861 let mut consumer = make_consumer(&ring);
3862
3863 let mut desc = raw_desc(0, 16, DescFlags::empty());
3864 desc.mark_avail(false);
3865 ring.write_desc(0, desc);
3866
3867 assert!(matches!(
3868 consumer.poll_available(),
3869 Err(RingError::WouldBlock)
3870 ));
3871 }
3872
3873 #[test]
3874 fn p0012_multidescriptor_oob_id_is_rejected() {
3875 let ring = make_ring(8);
3876 let mut consumer = make_consumer(&ring);
3877 let bad_id = ring.len() as u16;
3878
3879 write_avail(&ring, 0, bad_id, DescFlags::NEXT);
3880 write_avail(&ring, 1, bad_id, DescFlags::WRITE);
3881
3882 assert!(matches!(
3883 consumer.poll_available(),
3884 Err(RingError::InvalidState)
3885 ));
3886 }
3887
3888 #[test]
3889 fn p0013_duplicate_inflight_buffer_id_is_rejected() {
3890 let ring = make_ring(8);
3891 let mut consumer = make_consumer(&ring);
3892
3893 write_avail(&ring, 0, 0, DescFlags::NEXT);
3894 write_avail(&ring, 1, 0, DescFlags::WRITE);
3895 write_avail(&ring, 2, 0, DescFlags::NEXT);
3896 write_avail(&ring, 3, 0, DescFlags::WRITE);
3897
3898 let (first_id, first_chain) = consumer.poll_available().unwrap();
3899 assert_eq!(first_id, 0);
3900 assert_eq!(first_chain.len(), 2);
3901
3902 assert!(matches!(
3903 consumer.poll_available(),
3904 Err(RingError::InvalidState)
3905 ));
3906 }
3907
3908 #[test]
3909 fn p0014_reserved_driver_event_flags_do_not_block_processing() {
3910 let ring = make_ring(8);
3911 let mut consumer = make_consumer(&ring);
3912
3913 write_driver_event(&ring, 0, 3);
3914 write_avail(&ring, 0, 0, DescFlags::empty());
3915
3916 let (id, chain) = consumer.poll_available().unwrap();
3917 assert_eq!(id, 0);
3918 assert_eq!(chain.len(), 1);
3919 }
3920
3921 #[test]
3922 fn p0015_wrap_at_zero_is_consumed_correctly() {
3923 let ring = make_ring(2);
3924 let mut producer = make_producer(&ring);
3925 let mut consumer = make_consumer(&ring);
3926
3927 producer.submit_one(0x1000, 16, false).unwrap();
3928 producer.submit_one(0x2000, 16, false).unwrap();
3929
3930 assert_eq!(producer.avail_cursor().head(), 0);
3931 assert!(!producer.avail_cursor().wrap());
3932
3933 assert_eq!(consumer.poll_available().unwrap().1.elems()[0].addr, 0x1000);
3934 assert_eq!(consumer.poll_available().unwrap().1.elems()[0].addr, 0x2000);
3935 assert_eq!(consumer.avail_cursor().head(), 0);
3936 assert!(!consumer.avail_cursor().wrap());
3937 }
3938
3939 #[test]
3940 fn p0016_queue_size_one_next_chain_is_rejected() {
3941 let ring = make_ring(1);
3942 let mut consumer = make_consumer(&ring);
3943
3944 write_avail(&ring, 0, 0, DescFlags::NEXT);
3945
3946 assert!(matches!(
3947 consumer.poll_available(),
3948 Err(RingError::BadChain)
3949 ));
3950 }
3951
3952 #[test]
3953 fn p0017_stale_driver_event_wrap_suppresses_notification() {
3954 let ring = make_ring(8);
3955 let mut consumer = make_consumer(&ring);
3956
3957 write_driver_event(&ring, 0, EventFlags::DESC.bits());
3958 write_avail(&ring, 0, 0, DescFlags::empty());
3959
3960 let (id, _) = consumer.poll_available().unwrap();
3961
3962 assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
3963 }
3964
3965 #[test]
3966 fn p0018_pre_used_descriptor_is_ignored() {
3967 let ring = make_ring(8);
3968 let mut consumer = make_consumer(&ring);
3969
3970 let mut desc = raw_desc(0, 16, DescFlags::empty());
3971 desc.mark_used(true);
3972 ring.write_desc(0, desc);
3973
3974 assert!(matches!(
3975 consumer.poll_available(),
3976 Err(RingError::WouldBlock)
3977 ));
3978 }
3979
3980 #[test]
3981 fn p0019_driver_event_wrap_mismatch_suppresses_notification() {
3982 let ring = make_ring(8);
3983 let mut consumer = make_consumer(&ring);
3984
3985 write_driver_event(&ring, 1, EventFlags::DESC.bits());
3986 write_avail(&ring, 0, 0, DescFlags::empty());
3987
3988 let (id, _) = consumer.poll_available().unwrap();
3989
3990 assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
3991 }
3992
3993 #[test]
3994 fn p0020_ring_full_reports_backpressure() {
3995 let ring = make_ring(4);
3996 let mut producer = make_producer(&ring);
3997
3998 for idx in 0..4 {
3999 producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4000 }
4001
4002 assert!(matches!(
4003 producer.submit_one(0x2000, 16, false),
4004 Err(RingError::WouldBlock)
4005 ));
4006 }
4007
4008 #[test]
4009 fn p0022_indirect_reused_id_is_rejected_before_id_reuse() {
4010 let ring = make_ring(8);
4011 let mut consumer = make_consumer(&ring);
4012
4013 write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4014 write_avail(&ring, 1, 0, DescFlags::empty());
4015
4016 assert!(matches!(
4017 consumer.poll_available(),
4018 Err(RingError::BadChain)
4019 ));
4020 }
4021
4022 #[test]
4023 fn p0023_desc_event_mode_suppresses_before_later_slot() {
4024 let ring = make_ring(8);
4025 let mut consumer = make_consumer(&ring);
4026
4027 write_driver_event(&ring, 0x8001, EventFlags::DESC.bits());
4028 write_avail(&ring, 0, 0, DescFlags::empty());
4029
4030 let (id, _) = consumer.poll_available().unwrap();
4031
4032 assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4033 }
4034
4035 #[test]
4036 fn p0024_single_slot_indirect_is_rejected() {
4037 let ring = make_ring(1);
4038 let mut consumer = make_consumer(&ring);
4039
4040 write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4041
4042 assert!(matches!(
4043 consumer.poll_available(),
4044 Err(RingError::BadChain)
4045 ));
4046 }
4047
4048 #[test]
4049 fn p0025_used_completions_can_arrive_out_of_order() {
4050 let ring = make_ring(8);
4051 let mut producer = make_producer(&ring);
4052 let mut consumer = make_consumer(&ring);
4053
4054 let id0 = producer.submit_one(0x1000, 16, true).unwrap();
4055 let id1 = producer.submit_one(0x2000, 16, true).unwrap();
4056 let id2 = producer.submit_one(0x3000, 16, true).unwrap();
4057
4058 let (dev0, _) = consumer.poll_available().unwrap();
4059 let (dev1, _) = consumer.poll_available().unwrap();
4060 let (dev2, _) = consumer.poll_available().unwrap();
4061
4062 consumer.submit_used(dev2, 32).unwrap();
4063 consumer.submit_used(dev0, 16).unwrap();
4064 consumer.submit_used(dev1, 24).unwrap();
4065
4066 assert_eq!(producer.poll_used().unwrap().id, id2);
4067 assert_eq!(producer.poll_used().unwrap().id, id0);
4068 assert_eq!(producer.poll_used().unwrap().id, id1);
4069 }
4070
4071 #[test]
4072 fn p0026_all_descriptor_flags_set_is_not_available() {
4073 let ring = make_ring(8);
4074 let mut consumer = make_consumer(&ring);
4075
4076 ring.write_desc(0, raw_desc(0, 16, DescFlags::from_bits_truncate(u16::MAX)));
4077
4078 assert!(matches!(
4079 consumer.poll_available(),
4080 Err(RingError::WouldBlock)
4081 ));
4082 }
4083
4084 #[test]
4085 fn p0028_multi_wrap_round_trip_remains_ordered() {
4086 let ring = make_ring(2);
4087 let mut producer = make_producer(&ring);
4088 let mut consumer = make_consumer(&ring);
4089
4090 for idx in 0..8u64 {
4091 let id = producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4092 let (dev_id, _) = consumer.poll_available().unwrap();
4093 assert_eq!(dev_id, id);
4094 consumer.submit_used(dev_id, idx as u32).unwrap();
4095 assert_eq!(producer.poll_used().unwrap().id, id);
4096 }
4097 }
4098
4099 #[test]
4100 fn p0029_desc_event_idx_zero_notifies() {
4101 let ring = make_ring(8);
4102 let mut consumer = make_consumer(&ring);
4103
4104 write_driver_event(&ring, 0x8000, EventFlags::DESC.bits());
4105 write_avail(&ring, 0, 0, DescFlags::empty());
4106
4107 let (id, _) = consumer.poll_available().unwrap();
4108
4109 assert!(consumer.submit_used_with_notify(id, 0).unwrap());
4110 }
4111
4112 #[test]
4113 fn p0031_reserved_driver_event_flags_suppress_notification() {
4114 let ring = make_ring(8);
4115 let mut consumer = make_consumer(&ring);
4116
4117 write_driver_event(&ring, 0, 3);
4118 write_avail(&ring, 0, 0, DescFlags::empty());
4119
4120 let (id, _) = consumer.poll_available().unwrap();
4121
4122 assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4123 }
4124
4125 #[test]
4126 fn p0032_unaligned_indirect_descriptor_is_rejected() {
4127 let ring = make_ring(8);
4128 let mut consumer = make_consumer(&ring);
4129
4130 let mut desc = Descriptor::new(0x1003, 16, 0, DescFlags::INDIRECT);
4131 desc.mark_avail(true);
4132 ring.write_desc(0, desc);
4133
4134 assert!(matches!(
4135 consumer.poll_available(),
4136 Err(RingError::BadChain)
4137 ));
4138 }
4139
4140 #[test]
4141 fn p0033_scatter_gather_chain_is_preserved() {
4142 let ring = make_ring(16);
4143 let mut consumer = make_consumer(&ring);
4144
4145 for idx in 0..8u16 {
4146 let flags = if idx == 7 {
4147 DescFlags::WRITE
4148 } else {
4149 DescFlags::NEXT
4150 };
4151 write_avail(&ring, idx, 0, flags);
4152 }
4153
4154 let (_, chain) = consumer.poll_available().unwrap();
4155 assert_eq!(chain.len(), 8);
4156 assert_eq!(chain.readables().len(), 7);
4157 assert_eq!(chain.writables().len(), 1);
4158 }
4159
4160 #[test]
4161 fn p0034_page_boundary_indirect_descriptor_is_rejected() {
4162 let ring = make_ring(8);
4163 let mut consumer = make_consumer(&ring);
4164
4165 let mut desc = Descriptor::new(0x1ff0, 16, 0, DescFlags::INDIRECT);
4166 desc.mark_avail(true);
4167 ring.write_desc(0, desc);
4168
4169 assert!(matches!(
4170 consumer.poll_available(),
4171 Err(RingError::BadChain)
4172 ));
4173 }
4174
4175 #[test]
4176 fn p0035_both_avail_and_used_clear_is_ignored() {
4177 let ring = make_ring(8);
4178 let mut consumer = make_consumer(&ring);
4179
4180 ring.write_desc(0, raw_desc(0, 16, DescFlags::empty()));
4181
4182 assert!(matches!(
4183 consumer.poll_available(),
4184 Err(RingError::WouldBlock)
4185 ));
4186 }
4187
4188 #[test]
4189 fn p0037_event_flags_can_cycle_between_batches() {
4190 let ring = make_ring(8);
4191 let mut consumer = make_consumer(&ring);
4192
4193 for (idx, (flags, expect_notify)) in [
4194 (EventFlags::ENABLE.bits(), true),
4195 (EventFlags::DISABLE.bits(), false),
4196 (EventFlags::ENABLE.bits(), true),
4197 ]
4198 .into_iter()
4199 .enumerate()
4200 {
4201 write_driver_event(&ring, 0, flags);
4202 write_avail(&ring, idx as u16, idx as u16, DescFlags::empty());
4203 let (id, _) = consumer.poll_available().unwrap();
4204 assert_eq!(
4205 consumer.submit_used_with_notify(id, 0).unwrap(),
4206 expect_notify
4207 );
4208 }
4209 }
4210
4211 #[test]
4212 fn p0038_many_wraps_on_minimum_queue_size() {
4213 let ring = make_ring(1);
4214 let mut producer = make_producer(&ring);
4215 let mut consumer = make_consumer(&ring);
4216
4217 for idx in 0..8u64 {
4218 let id = producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4219 let (dev_id, _) = consumer.poll_available().unwrap();
4220 assert_eq!(dev_id, id);
4221 consumer.submit_used(dev_id, 0).unwrap();
4222 assert_eq!(producer.poll_used().unwrap().id, id);
4223 }
4224 }
4225
4226 #[test]
4227 fn p0039_reserved_device_event_flags_suppress_avail_notification() {
4228 let ring = make_ring(8);
4229 let mut producer = make_producer(&ring);
4230
4231 write_device_event(&ring, 0, 3);
4232
4233 assert!(
4234 !producer
4235 .submit_one_with_notify(0x1000, 16, false)
4236 .unwrap()
4237 .notify
4238 );
4239 }
4240
4241 #[test]
4242 fn p0040_indirect_with_next_is_rejected() {
4243 let ring = make_ring(8);
4244 let mut consumer = make_consumer(&ring);
4245
4246 write_avail(&ring, 0, 0, DescFlags::INDIRECT | DescFlags::NEXT);
4247
4248 assert!(matches!(
4249 consumer.poll_available(),
4250 Err(RingError::BadChain)
4251 ));
4252 }
4253
4254 #[test]
4255 fn p0041_driver_event_offset_at_queue_size_suppresses_notification() {
4256 let ring = make_ring(8);
4257 let mut consumer = make_consumer(&ring);
4258
4259 write_driver_event(&ring, ring.len() as u16, EventFlags::DESC.bits());
4260 write_avail(&ring, 0, 0, DescFlags::empty());
4261
4262 let (id, _) = consumer.poll_available().unwrap();
4263
4264 assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4265 }
4266
4267 #[test]
4268 fn p0042_empty_ring_has_no_available_work() {
4269 let ring = make_ring(8);
4270 let mut consumer = make_consumer(&ring);
4271
4272 assert!(matches!(
4273 consumer.poll_available(),
4274 Err(RingError::WouldBlock)
4275 ));
4276 }
4277
4278 #[test]
4279 fn p0043_id_equal_to_queue_size_is_rejected() {
4280 let ring = make_ring(8);
4281 let mut consumer = make_consumer(&ring);
4282
4283 write_avail(&ring, 0, ring.len() as u16, DescFlags::empty());
4284
4285 assert!(matches!(
4286 consumer.poll_available(),
4287 Err(RingError::InvalidState)
4288 ));
4289 }
4290
4291 #[test]
4292 fn p0044_avail_and_used_set_is_not_available() {
4293 let ring = make_ring(8);
4294 let mut consumer = make_consumer(&ring);
4295
4296 ring.write_desc(0, raw_desc(0, 16, DescFlags::AVAIL | DescFlags::USED));
4297
4298 assert!(!consumer.peek_available().unwrap());
4299 assert!(matches!(
4300 consumer.poll_available(),
4301 Err(RingError::WouldBlock)
4302 ));
4303 }
4304
4305 #[test]
4306 fn p0045_zero_length_descriptor_is_consumable() {
4307 let ring = make_ring(8);
4308 let mut consumer = make_consumer(&ring);
4309
4310 write_avail_len(&ring, 0, 0, 0, DescFlags::WRITE);
4311
4312 let (id, chain) = consumer.poll_available().unwrap();
4313 assert_eq!(id, 0);
4314 assert_eq!(chain.writables()[0].len, 0);
4315 }
4316
4317 #[test]
4318 fn p0046_max_length_descriptor_is_consumable_at_ring_layer() {
4319 let ring = make_ring(8);
4320 let mut consumer = make_consumer(&ring);
4321
4322 write_avail_len(&ring, 0, 0, u32::MAX, DescFlags::WRITE);
4323
4324 let (id, chain) = consumer.poll_available().unwrap();
4325 assert_eq!(id, 0);
4326 assert_eq!(chain.writables()[0].len, u32::MAX);
4327 }
4328
4329 #[test]
4330 fn p0047_zero_descriptor_address_is_preserved_at_ring_layer() {
4331 let ring = make_ring(8);
4332 let mut consumer = make_consumer(&ring);
4333
4334 let mut desc = Descriptor::new(0, 16, 0, DescFlags::WRITE);
4335 desc.mark_avail(true);
4336 ring.write_desc(0, desc);
4337
4338 let (id, chain) = consumer.poll_available().unwrap();
4339 assert_eq!(id, 0);
4340 assert_eq!(chain.writables()[0].addr, 0);
4341 }
4342
4343 #[test]
4344 fn p0049_indirect_entry_with_phase_bits_is_rejected() {
4345 let ring = make_ring(8);
4346 let mut consumer = make_consumer(&ring);
4347
4348 ring.write_desc(
4349 0,
4350 raw_desc(
4351 0,
4352 16,
4353 DescFlags::INDIRECT | DescFlags::AVAIL | DescFlags::USED,
4354 ),
4355 );
4356
4357 assert!(matches!(
4358 consumer.poll_available(),
4359 Err(RingError::WouldBlock)
4360 ));
4361 }
4362
4363 #[test]
4364 fn p0050_nested_indirect_is_rejected_at_head() {
4365 let ring = make_ring(8);
4366 let mut consumer = make_consumer(&ring);
4367
4368 write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4369
4370 assert!(matches!(
4371 consumer.poll_available(),
4372 Err(RingError::BadChain)
4373 ));
4374 }
4375
4376 #[test]
4377 fn p0051_write_flag_controls_readable_writable_split() {
4378 let ring = make_ring(8);
4379 let mut consumer = make_consumer(&ring);
4380
4381 write_avail(&ring, 0, 0, DescFlags::NEXT);
4382 write_avail(&ring, 1, 0, DescFlags::WRITE);
4383
4384 let (_, chain) = consumer.poll_available().unwrap();
4385 assert_eq!(chain.readables().len(), 1);
4386 assert_eq!(chain.writables().len(), 1);
4387 }
4388
4389 #[test]
4390 fn p0052_batch_completion_signals_each_chain_head() {
4391 let ring = make_ring(8);
4392 let mut producer = make_producer(&ring);
4393 let mut consumer = make_consumer(&ring);
4394
4395 let id0 = producer.submit_one(0x1000, 16, true).unwrap();
4396 let id1 = producer.submit_one(0x2000, 16, true).unwrap();
4397
4398 let (dev0, _) = consumer.poll_available().unwrap();
4399 let (dev1, _) = consumer.poll_available().unwrap();
4400 consumer.submit_used(dev0, 16).unwrap();
4401 consumer.submit_used(dev1, 32).unwrap();
4402
4403 assert_eq!(producer.poll_used().unwrap().id, id0);
4404 assert_eq!(producer.poll_used().unwrap().id, id1);
4405 }
4406
4407 #[test]
4408 fn p0053_descriptor_without_avail_bit_is_ignored() {
4409 let ring = make_ring(8);
4410 let mut consumer = make_consumer(&ring);
4411
4412 ring.write_desc(0, raw_desc(0, 16, DescFlags::WRITE));
4413
4414 assert!(!consumer.peek_available().unwrap());
4415 assert!(matches!(
4416 consumer.poll_available(),
4417 Err(RingError::WouldBlock)
4418 ));
4419 }
4420
4421 #[test]
4422 fn p0054_event_suppression_struct_layout_matches_packed_ring() {
4423 assert_eq!(EventSuppression::SIZE, 4);
4424 assert_eq!(EventSuppression::WRAP_OFFSET, 0);
4425 assert_eq!(EventSuppression::FLAGS_OFFSET, 2);
4426 }
4427
4428 #[test]
4429 fn p0055_descriptor_after_chain_tail_is_ignored() {
4430 let ring = make_ring(8);
4431 let mut consumer = make_consumer(&ring);
4432
4433 write_avail(&ring, 0, 0, DescFlags::empty());
4434 write_avail(&ring, 1, 1, DescFlags::WRITE);
4435
4436 let (id, chain) = consumer.poll_available().unwrap();
4437 assert_eq!(id, 0);
4438 assert_eq!(chain.len(), 1);
4439 }
4440
4441 #[test]
4442 fn p0056_next_chain_order_is_preserved() {
4443 let ring = make_ring(8);
4444 let mut consumer = make_consumer(&ring);
4445
4446 for idx in 0..3u16 {
4447 let mut desc = Descriptor::new(
4448 0x1000 + u64::from(idx) * 0x100,
4449 16,
4450 0,
4451 if idx == 2 {
4452 DescFlags::WRITE
4453 } else {
4454 DescFlags::NEXT
4455 },
4456 );
4457 desc.mark_avail(true);
4458 ring.write_desc(idx, desc);
4459 }
4460
4461 let (_, chain) = consumer.poll_available().unwrap();
4462 assert_eq!(chain.elems()[0].addr, 0x1000);
4463 assert_eq!(chain.elems()[1].addr, 0x1100);
4464 assert_eq!(chain.elems()[2].addr, 0x1200);
4465 }
4466
4467 #[test]
4468 fn p0057_device_event_enable_requests_avail_notification() {
4469 let ring = make_ring(8);
4470 let mut producer = make_producer(&ring);
4471
4472 write_device_event(&ring, 0, EventFlags::ENABLE.bits());
4473
4474 assert!(
4475 producer
4476 .submit_one_with_notify(0x1000, 16, false)
4477 .unwrap()
4478 .notify
4479 );
4480 }
4481
4482 #[test]
4483 fn p0058_used_descriptor_carries_id_and_len() {
4484 let ring = make_ring(8);
4485 let mut producer = make_producer(&ring);
4486 let mut consumer = make_consumer(&ring);
4487
4488 let id = producer.submit_one(0x1000, 16, true).unwrap();
4489 let (dev_id, _) = consumer.poll_available().unwrap();
4490 consumer.submit_used(dev_id, 42).unwrap();
4491
4492 let used = producer.poll_used().unwrap();
4493 assert_eq!(used.id, id);
4494 assert_eq!(used.len, 42);
4495 }
4496}
4497
4498#[cfg(test)]
4499mod fuzz;