1use alloc::collections::VecDeque;
5use alloc::vec;
6use alloc::vec::Vec;
7use core::mem::ManuallyDrop;
8
9use bytes::Bytes;
10use smallvec::SmallVec;
11
12use super::*;
13
14#[derive(Debug)]
21pub enum UsedChain {
22 Ack(Token),
24 Data(Token, Segments),
30}
31
32impl UsedChain {
33 pub fn token(&self) -> Token {
35 match self {
36 Self::Ack(token) | Self::Data(token, _) => *token,
37 }
38 }
39
40 pub fn to_bytes(&self) -> Option<Bytes> {
42 match self {
43 Self::Ack(_) => None,
44 Self::Data(_, segments) => Some(segments.to_bytes()),
45 }
46 }
47
48 pub fn segments(&self) -> Option<&Segments> {
50 match self {
51 Self::Ack(_) => None,
52 Self::Data(_, segments) => Some(segments),
53 }
54 }
55
56 pub fn into_bytes(self) -> Option<Bytes> {
58 match self {
59 Self::Ack(_) => None,
60 Self::Data(_, segments) => Some(segments.into_bytes()),
61 }
62 }
63
64 pub fn into_segments(self) -> Option<Segments> {
66 match self {
67 Self::Ack(_) => None,
68 Self::Data(_, segments) => Some(segments),
69 }
70 }
71}
72
73struct Inflight<M> {
74 token: Token,
75 chain: ManuallyDrop<OwnedChain<M>>,
78}
79
80struct InflightTable<M> {
86 by_id: Vec<u16>,
87 live: Vec<Inflight<M>>,
88}
89
90impl<M> InflightTable<M> {
91 const VACANT: u16 = u16::MAX;
92
93 fn new(ring_len: usize) -> Self {
94 Self {
95 by_id: vec![Self::VACANT; ring_len],
96 live: Vec::new(),
97 }
98 }
99
100 fn try_reserve_one(&mut self) -> Result<(), VirtqError> {
101 if self.live.len() > self.by_id.len() {
102 return Err(VirtqError::InvalidState);
103 }
104
105 if self.live.len() == self.by_id.len() {
106 return Err(VirtqError::Backpressure);
107 }
108
109 let result = if self.live.capacity() == 0 {
112 self.live.try_reserve_exact(1)
113 } else {
114 self.live.try_reserve(1)
115 };
116
117 result.map_err(|_| VirtqError::Bookkeeping)
118 }
119
120 fn contains(&self, id: u16) -> bool {
121 self.by_id
122 .get(id as usize)
123 .is_some_and(|slot| *slot != Self::VACANT)
124 }
125
126 fn insert(&mut self, inflight: Inflight<M>) {
127 let id = inflight.token.id;
128 debug_assert!(!self.contains(id));
129 debug_assert!(self.live.len() < Self::VACANT as usize);
130
131 let slot = self.live.len() as u16;
132 self.live.push(inflight);
133 self.by_id[id as usize] = slot;
134 }
135
136 fn remove(&mut self, id: u16) -> Option<Inflight<M>> {
137 let slot = self.by_id.get_mut(id as usize)?;
138 if *slot == Self::VACANT {
139 return None;
140 }
141
142 let index = usize::from(*slot);
143 *slot = Self::VACANT;
144
145 let removed = self.live.swap_remove(index);
146 if let Some(moved) = self.live.get(index) {
147 self.by_id[moved.token.id as usize] = index as u16;
148 }
149
150 Some(removed)
151 }
152
153 fn pop(&mut self) -> Option<Inflight<M>> {
154 let inflight = self.live.pop()?;
155 self.by_id[inflight.token.id as usize] = Self::VACANT;
156 Some(inflight)
157 }
158}
159
160pub struct VirtqProducer<M, N> {
197 inner: RingProducer<M>,
198 notifier: N,
199 pool: SlotPool,
200 next_token: u32,
201 inflight: InflightTable<M>,
202 pending: VecDeque<UsedChain>,
203}
204
205impl<M, N> VirtqProducer<M, N>
206where
207 M: MemOps + Clone,
208 N: Notifier,
209{
210 pub fn new(layout: Layout, mem: M, notifier: N, pool: SlotPool) -> Self {
219 let inner = RingProducer::new(layout, mem);
220 let ring_len = inner.len();
221 let inflight = InflightTable::new(ring_len);
222
223 Self {
224 inner,
225 pool,
226 notifier,
227 inflight,
228 next_token: 0,
229 pending: VecDeque::new(),
230 }
231 }
232
233 pub fn pool(&self) -> &SlotPool {
235 &self.pool
236 }
237
238 pub fn memory(&self) -> &M {
240 self.inner.mem()
241 }
242
243 pub unsafe fn reclaim_slots(&mut self) -> Result<(), VirtqError> {
254 let live = &mut self.inflight.live;
255 if !self.pending.is_empty() {
257 return Err(VirtqError::InvalidState);
258 }
259
260 if live.iter().any(|inf| !self.pool.ptr_eq(&inf.chain.pool)) {
262 return Err(VirtqError::InvalidState);
263 }
264
265 let reserved = live
266 .iter()
267 .flat_map(|inf| inf.chain.buffers.iter().map(|buf| buf.addr));
268
269 if self.pool.num_live() == reserved.clone().count() {
271 return Ok(());
272 }
273
274 let pool = self.pool.swap(reserved)?;
275 for inf in live {
276 inf.chain.pool = pool.clone();
277 }
278
279 self.pool = pool;
280 Ok(())
281 }
282
283 pub fn chain(&self) -> ChainBuilder<M> {
288 ChainBuilder::new(
289 self.inner.mem().clone(),
290 self.pool.clone(),
291 self.inner.num_free(),
292 )
293 }
294
295 pub fn preferred_segment_len(&self) -> usize {
297 self.pool.slot_size()
298 }
299
300 pub fn batch(&mut self) -> SubmitBatch<'_, M, N> {
307 SubmitBatch::new(self)
308 }
309
310 pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
323 let cursor_before = self.inner.avail_cursor();
324 let token = self.publish(chain)?;
325 self.notify_since(cursor_before)?;
326 Ok(token)
327 }
328
329 fn publish(&mut self, send: SendChain<M>) -> Result<Token, VirtqError> {
330 self.inflight.try_reserve_one()?;
331
332 if send.desc_count() > self.inner.num_free() {
333 return Err(VirtqError::Backpressure);
334 }
335 let token_id = self.next_token;
336 let id = self.inner.next_id()?;
337 let token = Token { seq: token_id, id };
338
339 if self.inflight.contains(id) {
341 return Err(VirtqError::InvalidState);
342 }
343
344 let mut descriptors = send.owned.descriptors();
345 let mut builder = BufferChainBuilder::new();
346
347 builder.reserve_exact(send.desc_count());
348 let chain = builder
349 .readables(descriptors.by_ref().take(send.rd_desc_count()))
350 .writables(descriptors)
351 .build()?;
352
353 let published = self.inner.submit_available(&chain);
354
355 let inf = Inflight {
358 token,
359 chain: ManuallyDrop::new(send.owned),
360 };
361 self.inflight.insert(inf);
362 self.next_token = self.next_token.wrapping_add(1);
363
364 let published_id = published?;
365 debug_assert_eq!(published_id, id);
366 Ok(token)
367 }
368
369 fn notify_since(&mut self, cursor: RingCursor) -> Result<bool, VirtqError> {
370 let should_notify = self.inner.should_notify_since(cursor)?;
371 if should_notify {
372 self.notify_now();
373 }
374 Ok(should_notify)
375 }
376
377 fn notify_now(&self) {
378 self.notifier.notify(QueueStats {
379 num_free: self.inner.num_free(),
380 num_inflight: self.inner.num_inflight(),
381 });
382 }
383
384 #[inline]
389 pub fn notify_backpressure(&self) {
390 self.notify_now();
391 }
392
393 #[inline]
397 pub fn used_cursor(&self) -> RingCursor {
398 self.inner.used_cursor()
399 }
400
401 #[inline]
403 pub fn num_free(&self) -> usize {
404 self.inner.num_free()
405 }
406
407 #[inline]
409 pub fn num_inflight(&self) -> usize {
410 self.inner.num_inflight()
411 }
412
413 pub unsafe fn reset(&mut self) -> Result<(), VirtqError> {
432 if !self.pending.is_empty() {
433 return Err(VirtqError::InvalidState);
434 }
435
436 self.inner.reset()?;
437 self.next_token = 0;
438
439 let mut maybe_err = None;
440
441 while let Some(inflight) = self.inflight.pop() {
442 let ret = ManuallyDrop::into_inner(inflight.chain).release();
443 if let Err(err) = ret
444 && maybe_err.is_none()
445 {
446 maybe_err = Some(err);
447 }
448 }
449
450 match maybe_err {
451 Some(error) => Err(error.into()),
452 None => Ok(()),
453 }
454 }
455
456 pub fn set_used_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
476 match kind {
477 SuppressionKind::Enable => self.inner.enable_used_notifications()?,
478 SuppressionKind::Disable => self.inner.disable_used_notifications()?,
479 SuppressionKind::Descriptor(cursor) => self
480 .inner
481 .enable_used_notifications_desc(cursor.head(), cursor.wrap())?,
482 }
483 Ok(())
484 }
485}
486
487impl<M, N> VirtqProducer<M, N>
488where
489 M: BufferMap + Clone,
490 N: Notifier,
491{
492 pub fn poll(&mut self) -> Result<Option<UsedChain>, VirtqError> {
508 if let Some(chain) = self.pending.pop_front() {
509 return Ok(Some(chain));
510 }
511 self.poll_ring()
512 }
513
514 pub fn reclaim(&mut self) -> Result<usize, VirtqError> {
526 let mut count = 0;
527 while let Some(chain) = self.poll_ring()? {
528 if matches!(chain, UsedChain::Data(_, _)) {
529 self.pending
530 .try_reserve(1)
531 .map_err(|_| VirtqError::Bookkeeping)?;
532 self.pending.push_back(chain);
533 }
534 count += 1;
535 }
536 Ok(count)
537 }
538
539 fn poll_ring(&mut self) -> Result<Option<UsedChain>, VirtqError> {
541 let used = match self.inner.poll_used() {
542 Ok(u) => u,
543 Err(RingError::WouldBlock) => return Ok(None),
544 Err(e) => return Err(e.into()),
545 };
546
547 let inf = self
548 .inflight
549 .remove(used.id)
550 .ok_or(VirtqError::InvalidState)?;
551
552 let written = used.len as usize;
553 let Inflight { token, chain } = inf;
554
555 let chain = ManuallyDrop::into_inner(chain);
556 let used = if chain.readable == chain.buffers.len() {
557 chain.release()?;
558 UsedChain::Ack(token)
559 } else {
560 UsedChain::Data(token, chain.into_segments(written)?)
561 };
562
563 Ok(Some(used))
564 }
565
566 pub fn drain(&mut self, mut f: impl FnMut(UsedChain)) -> Result<(), VirtqError> {
583 while let Some(chain) = self.poll()? {
584 f(chain);
585 }
586
587 Ok(())
588 }
589}
590
591#[must_use = "finish the batch explicitly"]
597pub struct SubmitBatch<'a, M, N> {
598 producer: &'a mut VirtqProducer<M, N>,
599 notify_from: Option<RingCursor>,
600}
601
602impl<'a, M, N> SubmitBatch<'a, M, N>
603where
604 M: MemOps + Clone,
605 N: Notifier,
606{
607 fn new(producer: &'a mut VirtqProducer<M, N>) -> Self {
608 Self {
609 producer,
610 notify_from: None,
611 }
612 }
613
614 pub fn chain(&self) -> ChainBuilder<M> {
616 self.producer.chain()
617 }
618
619 pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
621 let cursor_before = self.producer.inner.avail_cursor();
622 let token = self.producer.publish(chain)?;
623
624 if self.notify_from.is_none() {
625 self.notify_from = Some(cursor_before);
626 }
627 Ok(token)
628 }
629
630 pub fn finish(mut self) -> Result<bool, VirtqError> {
635 let Some(notify_from) = self.notify_from.take() else {
636 return Ok(false);
637 };
638
639 self.producer.notify_since(notify_from)
640 }
641
642 pub fn finish_without_notify(self) {}
647}
648
649#[must_use = "call .build() to create a SendChain"]
654pub struct ChainBuilder<M: MemOps> {
655 mem: M,
656 pool: SlotPool,
657 rd_caps: SmallVec<[usize; 4]>,
658 wr_caps: SmallVec<[usize; 4]>,
659 writable_avail: bool,
660 max_descs: usize,
661}
662
663impl<M: MemOps> ChainBuilder<M> {
664 fn new(mem: M, pool: SlotPool, max_descs: usize) -> Self {
665 Self {
666 mem,
667 pool,
668 rd_caps: SmallVec::new(),
669 wr_caps: SmallVec::new(),
670 writable_avail: false,
671 max_descs,
672 }
673 }
674
675 pub fn readable(mut self, cap: usize) -> Self {
681 self.rd_caps.push(cap);
682 self
683 }
684
685 pub fn writable(mut self, cap: usize) -> Self {
694 self.wr_caps.push(cap);
695 self
696 }
697
698 pub fn writable_avail(mut self) -> Self {
703 self.writable_avail = true;
704 self
705 }
706
707 pub fn build(self) -> Result<SendChain<M>, VirtqError> {
716 if self.rd_caps.is_empty() && self.wr_caps.is_empty() && !self.writable_avail {
717 return Err(VirtqError::InvalidState);
718 }
719
720 let slot_size = self.pool.slot_size();
722 let rd_capacity = self.rd_caps.iter().try_fold(0usize, |total, &cap| {
723 total.checked_add(cap).ok_or(AllocError::Overflow)
724 })?;
725
726 let mut caps = self.rd_caps.iter().chain(&self.wr_caps);
727 let desc_count = caps.try_fold(0usize, |total, &cap| {
728 if cap == 0 {
729 return Err(AllocError::InvalidArg);
730 }
731 total
732 .checked_add(cap.div_ceil(slot_size))
733 .ok_or(AllocError::Overflow)
734 })?;
735
736 let remaining_descs = self
737 .max_descs
738 .checked_sub(desc_count)
739 .ok_or(VirtqError::Backpressure)?;
740
741 let mut buffers = SmallVec::new();
743 buffers
744 .try_reserve_exact(desc_count)
745 .map_err(|_| VirtqError::Bookkeeping)?;
746
747 let mut owned = OwnedChain {
748 mem: self.mem,
749 pool: self.pool,
750 buffers,
751 readable: 0,
752 };
753
754 let regions = self
756 .rd_caps
757 .iter()
758 .map(|&len| (len, false))
759 .chain(self.wr_caps.iter().map(|&len| (len, true)));
760
761 for (total_len, wr) in regions {
762 let mut remaining = total_len;
763
764 while remaining > 0 {
765 let len = remaining.min(slot_size);
766 let alloc = owned.pool.alloc(len)?;
767 let capacity = if wr { alloc.len } else { len as u32 };
768
769 let ent = BufferEntry {
770 addr: alloc.addr,
771 capacity,
772 written: 0,
773 };
774
775 owned.buffers.push(ent);
776 owned.readable += usize::from(!wr);
777
778 remaining -= len;
779 }
780 }
781
782 if self.writable_avail {
784 let extra = owned.pool.num_free_upper().min(remaining_descs);
785 owned
786 .buffers
787 .try_reserve_exact(extra)
788 .map_err(|_| VirtqError::Bookkeeping)?;
789
790 for _ in 0..extra {
791 let alloc = owned.pool.alloc(slot_size)?;
792 let ent = BufferEntry {
793 addr: alloc.addr,
794 capacity: alloc.len,
795 written: 0,
796 };
797
798 owned.buffers.push(ent);
799 }
800 }
801
802 if owned.buffers.is_empty() {
804 return Err(VirtqError::Backpressure);
805 }
806
807 Ok(SendChain {
808 owned,
809 rd_capacity,
810 rd_written: 0,
811 write_mode: WriteMode::Unset,
812 })
813 }
814}
815
816#[derive(Debug, Clone, Copy, PartialEq, Eq)]
824enum WriteMode {
825 Unset,
826 Append,
827 Direct,
828}
829
830#[must_use = "dropping without submitting deallocates the buffers"]
852pub struct SendChain<M> {
853 owned: OwnedChain<M>,
854 rd_capacity: usize,
855 rd_written: usize,
856 write_mode: WriteMode,
857}
858
859impl<M: MemOps> SendChain<M> {
860 fn note_write_mode(&mut self, mode: WriteMode) {
863 debug_assert!(
864 self.write_mode == WriteMode::Unset || self.write_mode == mode,
865 "SendChain mixes copy writes (write/write_all) with direct writes (write_seg/with_seg)"
866 );
867 self.write_mode = mode;
868 }
869
870 #[inline]
872 pub fn desc_count(&self) -> usize {
873 self.owned.buffers.len()
874 }
875
876 #[inline]
878 pub fn rd_desc_count(&self) -> usize {
879 self.owned.readable
880 }
881
882 #[inline]
884 pub fn wr_desc_count(&self) -> usize {
885 self.desc_count() - self.rd_desc_count()
886 }
887
888 #[inline]
890 pub fn capacity(&self) -> usize {
891 self.rd_capacity
892 }
893
894 #[inline]
896 pub fn written(&self) -> usize {
897 self.rd_written
898 }
899
900 #[inline]
902 pub fn remaining(&self) -> usize {
903 self.capacity() - self.written()
904 }
905
906 pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
919 if self.rd_desc_count() == 0 {
920 return Err(VirtqError::NoPayloadSegment);
921 }
922
923 self.note_write_mode(WriteMode::Append);
924
925 let mut remaining = &buf[..buf.len().min(self.remaining())];
926 let mut written = 0;
927
928 for buffer in &mut self.owned.buffers[..self.owned.readable] {
929 if remaining.is_empty() {
930 break;
931 }
932
933 let cap = buffer.capacity as usize;
934 let desc_off = buffer.written as usize;
935 let len = (cap - desc_off).min(remaining.len());
936 if len == 0 {
937 continue;
938 }
939
940 let addr = buffer
941 .addr
942 .checked_add(desc_off as u64)
943 .ok_or(VirtqError::MemoryWriteError)?;
944
945 self.owned
946 .mem
947 .write(addr, &remaining[..len])
948 .map_err(|_| VirtqError::MemoryWriteError)?;
949
950 buffer.written += len as u32;
951 self.rd_written += len;
952 written += len;
953 remaining = &remaining[len..];
954 }
955
956 Ok(written)
957 }
958
959 #[inline]
971 pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
972 if self.rd_desc_count() == 0 {
973 return Err(VirtqError::NoPayloadSegment);
974 }
975
976 if buf.len() > self.remaining() {
977 return Err(VirtqError::PayloadTooLarge {
978 recv: buf.len(),
979 limit: self.remaining(),
980 });
981 }
982
983 let written = self.write(buf)?;
984 debug_assert_eq!(written, buf.len());
985 Ok(self)
986 }
987
988 pub fn write_seg(&mut self, index: usize, buf: &[u8]) -> Result<&mut Self, VirtqError> {
999 self.note_write_mode(WriteMode::Direct);
1000
1001 let buffer = self.owned.buffers[..self.owned.readable]
1002 .get_mut(index)
1003 .ok_or(VirtqError::NoPayloadSegment)?;
1004
1005 let cap = buffer.capacity as usize;
1006 if buf.len() > cap {
1007 return Err(VirtqError::PayloadTooLarge {
1008 recv: buf.len(),
1009 limit: cap,
1010 });
1011 }
1012
1013 self.owned
1014 .mem
1015 .write(buffer.addr, buf)
1016 .map_err(|_| VirtqError::MemoryWriteError)?;
1017
1018 let previous = buffer.written as usize;
1019 buffer.written = checked_descriptor_len(buf.len())?;
1020 self.rd_written = self.rd_written - previous + buf.len();
1021 Ok(self)
1022 }
1023
1024 pub fn with_seg<E>(
1035 &mut self,
1036 index: usize,
1037 f: impl FnOnce(&mut [u8]) -> Result<usize, E>,
1038 ) -> Result<&mut Self, E>
1039 where
1040 E: From<VirtqError>,
1041 {
1042 self.note_write_mode(WriteMode::Direct);
1043
1044 let buffer = self.owned.buffers[..self.owned.readable]
1045 .get_mut(index)
1046 .ok_or_else(|| E::from(VirtqError::NoPayloadSegment))?;
1047
1048 let cap = buffer.capacity as usize;
1049
1050 let buf = unsafe {
1052 self.owned
1053 .mem
1054 .as_mut_slice(buffer.addr, cap)
1055 .map_err(|_| E::from(VirtqError::MemoryWriteError))?
1056 };
1057
1058 let written = f(buf)?;
1059 if written > buf.len() {
1060 return Err(E::from(VirtqError::PayloadTooLarge {
1061 recv: written,
1062 limit: buf.len(),
1063 }));
1064 }
1065
1066 let previous = buffer.written as usize;
1067 buffer.written = checked_descriptor_len(written).map_err(E::from)?;
1068 self.rd_written = self.rd_written - previous + written;
1069
1070 Ok(self)
1071 }
1072}
1073
1074#[derive(Debug)]
1078struct BufferEntry {
1079 addr: u64,
1081 capacity: u32,
1083 written: u32,
1085}
1086
1087struct OwnedChain<M> {
1091 mem: M,
1093 pool: SlotPool,
1095 buffers: SmallVec<[BufferEntry; 4]>,
1097 readable: usize,
1099}
1100
1101impl<M> OwnedChain<M> {
1102 fn descriptors(&self) -> impl ExactSizeIterator<Item = BufferElement> + Clone + '_ {
1106 self.buffers.iter().enumerate().map(|(i, buf)| {
1107 let wr = i >= self.readable;
1108 let len = if wr { buf.capacity } else { buf.written };
1109
1110 BufferElement {
1111 addr: buf.addr,
1112 len,
1113 writable: wr,
1114 }
1115 })
1116 }
1117
1118 fn release(mut self) -> Result<(), AllocError> {
1119 self.release_all()
1120 }
1121
1122 fn release_all(&mut self) -> Result<(), AllocError> {
1123 let mut maybe_err = None;
1124 while let Some(buf) = self.buffers.pop() {
1125 if let Err(error) = self.pool.dealloc(buf.addr)
1126 && maybe_err.is_none()
1127 {
1128 maybe_err = Some(error);
1129 }
1130 }
1131
1132 self.readable = 0;
1133 maybe_err.map_or(Ok(()), Err)
1134 }
1135}
1136
1137impl<M: BufferMap> OwnedChain<M> {
1138 fn into_segments(mut self, written: usize) -> Result<Segments, VirtqError> {
1139 let mut remaining = written;
1140 let mut nonempty = 0;
1141
1142 for buf in &mut self.buffers[self.readable..] {
1143 let len = remaining.min(buf.capacity as usize);
1144 buf.written = len as u32;
1145
1146 nonempty += usize::from(len != 0);
1147 remaining -= len;
1148 }
1149
1150 if remaining != 0 {
1151 self.release()?;
1152 return Err(VirtqError::InvalidState);
1153 }
1154
1155 let mut segments = SmallVec::<[Bytes; 4]>::new();
1156 segments
1157 .try_reserve_exact(nonempty)
1158 .map_err(|_| VirtqError::Bookkeeping)?;
1159
1160 while let Some(buf) = self.buffers.last() {
1161 if self.buffers.len() <= self.readable || buf.written == 0 {
1162 let addr = buf.addr;
1163 self.buffers.pop();
1164 self.pool.dealloc(addr)?;
1165
1166 continue;
1167 }
1168
1169 let alloc = Allocation {
1170 addr: buf.addr,
1171 len: buf.capacity,
1172 };
1173
1174 let written = buf.written as usize;
1175 let lease = BufferLease::new(self.pool.clone(), alloc);
1176 self.buffers.pop();
1177
1178 let mapping = unsafe { self.mem.map_buffer(lease, written) }
1181 .map_err(|_| VirtqError::MemoryReadError)?;
1182
1183 segments.push(Bytes::from_owner(mapping));
1184 }
1185
1186 segments.reverse();
1187 Ok(Segments::from_smallvec(segments))
1188 }
1189}
1190
1191impl<M> Drop for OwnedChain<M> {
1192 fn drop(&mut self) {
1193 if let Err(error) = self.release_all() {
1195 log::error!("Failed to release virtqueue buffers: {error}");
1196 debug_assert!(false, "OwnedChain deallocation failed: {error}");
1197 }
1198 }
1199}
1200
1201fn checked_descriptor_len(len: usize) -> Result<u32, VirtqError> {
1202 if len > u32::MAX as usize {
1203 return Err(VirtqError::PayloadTooLarge {
1204 recv: len,
1205 limit: u32::MAX as usize,
1206 });
1207 }
1208 Ok(len as u32)
1209}
1210
1211#[cfg(test)]
1212mod tests {
1213 use alloc::rc::Rc;
1214 use alloc::sync::Arc;
1215 use core::sync::atomic::Ordering;
1216
1217 use super::*;
1218 use crate::virtq::ring::tests::{FaultMem, OwnedRing, TestMem, make_consumer, make_ring};
1219 use crate::virtq::test_utils::*;
1220
1221 fn poll_received<M: MemOps + Clone, N: Notifier>(
1222 consumer: &mut VirtqConsumer<M, N>,
1223 ) -> (RecvChain<M>, ReplyChain<M>) {
1224 consumer.poll(1024).unwrap().unwrap()
1225 }
1226
1227 #[derive(Clone)]
1228 struct CopyingMem<'a>(&'a Rc<TestMem>);
1229
1230 unsafe impl MemOps for CopyingMem<'_> {
1232 type Error = core::convert::Infallible;
1233
1234 fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1235 self.0.read(addr, dst)
1236 }
1237
1238 fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1239 self.0.write(addr, src)
1240 }
1241
1242 fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1243 self.0.load_acquire(addr)
1244 }
1245
1246 fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1247 self.0.store_release(addr, val)
1248 }
1249
1250 unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1251 unsafe { self.0.as_slice(addr, len) }
1253 }
1254
1255 unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1256 unsafe { self.0.as_mut_slice(addr, len) }
1258 }
1259 }
1260
1261 impl BufferMap for CopyingMem<'_> {
1262 type Mapping = Vec<u8>;
1263
1264 unsafe fn map_buffer(
1265 &self,
1266 lease: BufferLease,
1267 written: usize,
1268 ) -> Result<Self::Mapping, Self::Error> {
1269 assert!(written <= lease.allocation().len as usize);
1270 let mut bytes = vec![0; written];
1271 self.read(lease.allocation().addr, &mut bytes)?;
1272 Ok(bytes)
1273 }
1274 }
1275
1276 fn make_virtq_pair(
1277 ring: &OwnedRing,
1278 slot_size: usize,
1279 ) -> (
1280 VirtqProducer<TestMem, TestNotifier>,
1281 VirtqConsumer<TestMem, TestNotifier>,
1282 ) {
1283 let mem = ring.mem();
1284 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1285
1286 let lower = SlotLayout::new(pool_base, slot_size / 2, ring.len()).unwrap();
1287 let upper = SlotLayout::new(lower.end_addr(), slot_size, ring.len()).unwrap();
1288 let pool = SlotPool::new_tiered(lower, upper).unwrap();
1289
1290 let notifier = TestNotifier::new();
1291 let producer = VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool);
1292 let consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1293 (producer, consumer)
1294 }
1295
1296 fn make_chain(
1297 lower_count: usize,
1298 upper_count: usize,
1299 slot_size: usize,
1300 max_descs: usize,
1301 ) -> ChainBuilder<TestMem> {
1302 let lower_len = lower_count * 256;
1303 let mem = TestMem::new(lower_len + upper_count * slot_size);
1304 let upper =
1305 SlotLayout::new(mem.base_addr() + lower_len as u64, slot_size, upper_count).unwrap();
1306
1307 let pool = if lower_count == 0 {
1308 SlotPool::new(upper)
1309 } else {
1310 let lower = SlotLayout::new(mem.base_addr(), 256, lower_count).unwrap();
1311 SlotPool::new_tiered(lower, upper)
1312 }
1313 .unwrap();
1314
1315 ChainBuilder::new(mem, pool, max_descs)
1316 }
1317
1318 fn inflight(seq: u32, id: u16) -> Inflight<TestMem> {
1319 let mem = TestMem::new(8);
1320 let pool_layout = SlotLayout::new(mem.base_addr(), 8, 1).unwrap();
1321 let pool = SlotPool::new(pool_layout).unwrap();
1322 let chain = ChainBuilder::new(mem, pool, 1).readable(8).build().unwrap();
1323 Inflight {
1324 token: Token { seq, id },
1325 chain: ManuallyDrop::new(chain.owned),
1326 }
1327 }
1328
1329 #[test]
1330 fn inflight_table_repairs_moved_entry_after_removal() {
1331 let mut table = InflightTable::new(16);
1332 for (seq, id) in [(0, 3), (1, 7), (2, 5)] {
1333 table.try_reserve_one().unwrap();
1334 table.insert(inflight(seq, id));
1335 }
1336
1337 let removed = table.remove(7).unwrap();
1338 assert_eq!(removed.token.seq, 1);
1339
1340 ManuallyDrop::into_inner(removed.chain).release().unwrap();
1341 assert!(!table.contains(7));
1342
1343 for (id, seq) in [(5, 2), (3, 0)] {
1344 let removed = table.remove(id).unwrap();
1345 assert_eq!(removed.token.seq, seq);
1346 ManuallyDrop::into_inner(removed.chain).release().unwrap();
1347 }
1348 assert!(table.live.is_empty());
1349 assert!(table.remove(7).is_none());
1350 }
1351
1352 #[test]
1353 fn producer_bookkeeping_starts_compact_and_lazy() {
1354 let ring = make_ring(64);
1355 let (producer, _consumer, _notifier) = make_test_producer(&ring);
1356
1357 assert_eq!(producer.inflight.by_id.len(), ring.len());
1358 assert!(producer.inflight.live.is_empty());
1359 assert_eq!(producer.inflight.live.capacity(), 0);
1360 assert_eq!(producer.pending.capacity(), 0);
1361 }
1362
1363 #[test]
1364 fn full_ring_still_reports_backpressure() {
1365 let ring = make_ring(4);
1366 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
1367
1368 for _ in 0..ring.len() {
1369 let chain = producer.chain().readable(1).build().unwrap();
1370 producer.submit(chain).unwrap();
1371 }
1372
1373 assert!(matches!(
1374 producer.chain().readable(1).build(),
1375 Err(VirtqError::Backpressure)
1376 ));
1377 unsafe { producer.reset() }.unwrap();
1379 }
1380
1381 #[test]
1382 fn submission_rechecks_capacity_after_reservation() {
1383 let ring = make_ring(2);
1384 let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1385 let pool = producer.pool.clone();
1386
1387 let chain = producer.chain().writable(64).build().unwrap();
1388 for _ in 0..ring.len() {
1389 let other = producer.chain().writable(32).build().unwrap();
1390 producer.submit(other).unwrap();
1391 }
1392
1393 assert_eq!(pool.num_live(), 3);
1394 assert!(matches!(
1395 producer.submit(chain),
1396 Err(VirtqError::Backpressure)
1397 ));
1398 assert_eq!(pool.num_live(), 2);
1399
1400 unsafe { producer.reset() }.unwrap();
1402 consumer.reset().unwrap();
1403
1404 assert_eq!(pool.num_live(), 0);
1405 }
1406
1407 #[test]
1408 fn publication_failure_keeps_allocations_until_stopped_reset() {
1409 for failed_write in 0..4 {
1410 let ring = make_ring(4);
1411 let orig_mem = FaultMem::new(ring.mem());
1412 let orig_gen = Arc::downgrade(&orig_mem.0);
1413
1414 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1415 let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1416 let pool = SlotPool::new(pool_layout).unwrap();
1417 let notif = TestNotifier::new();
1418
1419 let source = VirtqProducer::new(ring.layout(), orig_mem, notif.clone(), pool.clone());
1420
1421 let chain = source.chain().writable(64).build().unwrap();
1422 drop(source);
1423
1424 let mem = FaultMem::new(ring.mem());
1425 let mut producer =
1426 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1427
1428 mem.fail_write_at(failed_write);
1429
1430 let res = producer.submit(chain);
1431 assert!(matches!(
1432 res,
1433 Err(VirtqError::RingError(RingError::MemError { .. }))
1434 ));
1435 assert_eq!(pool.num_live(), 1);
1436 assert_eq!(producer.inflight.live.len(), 1);
1437 assert!(orig_gen.upgrade().is_some());
1438
1439 mem.allow_writes();
1440 unsafe { producer.reset() }.unwrap();
1442
1443 assert_eq!(pool.num_live(), 0);
1444 assert!(orig_gen.upgrade().is_none());
1445 }
1446 }
1447
1448 #[test]
1449 fn chain_clones_pool_only_for_independent_owners() {
1450 let ring = make_ring(8);
1451 let mem = ring.mem();
1452 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1453 let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1454 let pool = SlotPool::new(pool_layout).unwrap();
1455
1456 let notif = TestNotifier::new();
1457 let mapping = FaultMem::new(mem.clone());
1458
1459 let mut producer =
1460 VirtqProducer::new(ring.layout(), mapping.clone(), notif.clone(), pool.clone());
1461
1462 let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1463
1464 assert_eq!(pool.strong_count(), 2);
1465
1466 let mut chain = producer
1467 .chain()
1468 .readable(192)
1469 .writable(192)
1470 .build()
1471 .unwrap();
1472
1473 chain.write_all(b"request").unwrap();
1474
1475 assert_eq!(pool.num_live(), 6);
1476 assert_eq!(pool.strong_count(), 3);
1477 assert!(chain.owned.buffers.spilled());
1478
1479 let buffers = chain.owned.buffers.as_ptr();
1480 producer.submit(chain).unwrap();
1481
1482 assert_eq!(pool.strong_count(), 3);
1483 assert_eq!(producer.inflight.live[0].chain.buffers.as_ptr(), buffers);
1484
1485 let (recv, reply) = poll_received(&mut consumer);
1486 let ReplyChain::Writable(mut reply) = reply else {
1487 panic!("expected writable reply");
1488 };
1489 reply.write_all(&[0xa5; 70]).unwrap();
1490 consumer.complete(recv, reply).unwrap();
1491
1492 let segments = producer.poll().unwrap().unwrap().into_segments().unwrap();
1493
1494 assert_eq!(segments.segment_count(), 2);
1495 assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1496 assert_eq!(pool.strong_count(), 4);
1497 assert_eq!(pool.num_live(), 2);
1498
1499 let retained = segments.as_slice()[0].slice(1..);
1500 let cloned = retained.clone();
1501
1502 drop(segments);
1503
1504 assert_eq!(pool.num_live(), 1);
1505 assert_eq!(pool.strong_count(), 3);
1506
1507 unsafe { producer.reset() }.unwrap();
1509 consumer.reset().unwrap();
1510
1511 drop(producer);
1512
1513 assert_eq!(pool.strong_count(), 2);
1514 assert_eq!(retained.as_ref(), &[0xa5; 63]);
1515
1516 drop(retained);
1517
1518 assert_eq!(pool.num_live(), 1);
1519
1520 drop(cloned);
1521
1522 assert_eq!(pool.num_live(), 0);
1523 assert_eq!(pool.strong_count(), 1);
1524 assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1525 }
1526
1527 #[test]
1528 fn copied_mapping_crosses_threads_without_its_pool_or_borrowed_backend() {
1529 let ring = make_ring(4);
1530 let mem = Rc::new(ring.mem());
1531 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1532 let pool_layout = SlotLayout::new(base, 64, 1).unwrap();
1533 let pool = SlotPool::new(pool_layout).unwrap();
1534 let notif = TestNotifier::new();
1535
1536 let mut producer =
1537 VirtqProducer::new(ring.layout(), CopyingMem(&mem), notif.clone(), pool.clone());
1538 let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), notif);
1539
1540 let chain = producer.chain().writable(64).build().unwrap();
1541 producer.submit(chain).unwrap();
1542
1543 let (recv, reply) = poll_received(&mut consumer);
1544 let ReplyChain::Writable(mut reply) = reply else {
1545 panic!("expected writable reply");
1546 };
1547 reply.write_all(b"copied").unwrap();
1548 consumer.complete(recv, reply).unwrap();
1549
1550 let bytes = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1551
1552 assert_eq!(pool.num_live(), 0);
1553 assert_eq!(pool.strong_count(), 2);
1554
1555 let mut reused = producer.chain().readable(64).build().unwrap();
1556 reused.write_all(b"reused").unwrap();
1557
1558 std::thread::spawn(move || assert_eq!(bytes.as_ref(), b"copied"))
1559 .join()
1560 .unwrap();
1561
1562 assert_eq!(pool.num_live(), 1);
1563 assert_eq!(pool.strong_count(), 3);
1564
1565 drop(reused);
1566
1567 assert_eq!(pool.num_live(), 0);
1568 }
1569
1570 #[test]
1571 fn completion_uses_its_original_memory_and_allocating_pool() {
1572 let ring = make_ring(4);
1573 let mem = FaultMem::new(ring.mem());
1574 let orig_gen = Arc::downgrade(&mem.0);
1575 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1576
1577 let notifier = TestNotifier::new();
1578 let original_layout = SlotLayout::new(base, 64, 4).unwrap();
1579 let original = SlotPool::new(original_layout).unwrap();
1580 let source = VirtqProducer::new(ring.layout(), mem, TestNotifier::new(), original.clone());
1581
1582 let chain = source.chain().writable(64).build().unwrap();
1583 drop(source);
1584
1585 assert!(orig_gen.upgrade().is_some());
1586
1587 let replacement_layout = SlotLayout::new(original.base_addr() + 0x1000, 64, 4).unwrap();
1588 let replacement = SlotPool::new(replacement_layout).unwrap();
1589
1590 let replacement_mem = FaultMem::new(ring.mem());
1591 replacement_mem.fail_mapping_at(0);
1592
1593 let mut producer = VirtqProducer::new(
1594 ring.layout(),
1595 replacement_mem.clone(),
1596 notifier.clone(),
1597 replacement.clone(),
1598 );
1599 let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), TestNotifier::new());
1600
1601 producer.submit(chain).unwrap();
1602
1603 assert!(matches!(
1605 unsafe { producer.reclaim_slots() },
1606 Err(VirtqError::InvalidState)
1607 ));
1608 assert!(producer.pool().ptr_eq(&replacement));
1609
1610 assert!(orig_gen.upgrade().is_some());
1611
1612 let (recv, reply) = poll_received(&mut consumer);
1613 let ReplyChain::Writable(mut reply) = reply else {
1614 panic!("expected writable reply");
1615 };
1616
1617 reply.write_all(b"retained").unwrap();
1618 consumer.complete(recv, reply).unwrap();
1619
1620 let data = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1621
1622 assert_eq!(data.as_ref(), b"retained");
1623 assert_eq!(original.num_live(), 1);
1624 assert_eq!(replacement.num_live(), 0);
1625 assert_eq!(replacement_mem.0.map_calls.load(Ordering::Relaxed), 0);
1626
1627 let map_calls = orig_gen
1628 .upgrade()
1629 .unwrap()
1630 .map_calls
1631 .load(Ordering::Relaxed);
1632 assert_eq!(map_calls, 1);
1633
1634 drop(producer);
1635
1636 assert_eq!(data.as_ref(), b"retained");
1637
1638 drop(data);
1639
1640 assert_eq!(original.num_live(), 0);
1641 assert!(orig_gen.upgrade().is_none());
1642 }
1643
1644 #[test]
1645 fn mapping_failure_releases_returned_and_unmapped_slots() {
1646 for fail_at in 0..2 {
1647 let ring = make_ring(8);
1648 let mem = FaultMem::new(ring.mem());
1649 mem.fail_mapping_at(fail_at);
1650
1651 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1652 let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1653 let pool = SlotPool::new(pool_layout).unwrap();
1654 let notif = TestNotifier::new();
1655 let mut producer =
1656 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1657
1658 let mut consumer = make_consumer(&ring);
1659
1660 let chain = producer
1661 .chain()
1662 .readable(192)
1663 .writable(256)
1664 .build()
1665 .unwrap();
1666
1667 let mut addresses: Vec<_> = chain
1668 .owned
1669 .buffers
1670 .iter()
1671 .map(|buffer| buffer.addr)
1672 .collect();
1673
1674 producer.submit(chain).unwrap();
1675
1676 let (id, _) = consumer.poll_available().unwrap();
1677 consumer.submit_used(id, 70).unwrap();
1679
1680 assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
1681 assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), fail_at + 1);
1682 assert_eq!(pool.num_live(), 0);
1683 assert_eq!(pool.strong_count(), 2);
1684 assert_eq!(producer.num_inflight(), 0);
1685
1686 if fail_at == 1 {
1687 addresses.swap(3, 4);
1689 }
1690
1691 let repeated = producer
1692 .chain()
1693 .readable(192)
1694 .writable(256)
1695 .build()
1696 .unwrap();
1697
1698 assert_eq!(
1699 repeated
1700 .owned
1701 .buffers
1702 .iter()
1703 .map(|buffer| buffer.addr)
1704 .collect::<Vec<_>>(),
1705 addresses
1706 );
1707 }
1708 }
1709
1710 #[test]
1711 fn empty_and_malformed_completions_skip_mapping_and_release_every_slot() {
1712 let ring = make_ring(4);
1713 let mem = FaultMem::new(ring.mem());
1714 mem.fail_mapping_at(0);
1715
1716 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1717 let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1718 let pool = SlotPool::new(pool_layout).unwrap();
1719 let notif = TestNotifier::new();
1720
1721 let mut producer =
1722 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1723
1724 let mut consumer = make_consumer(&ring);
1725
1726 for (writable, written) in [(false, 0), (true, 0), (true, 129)] {
1727 let builder = producer.chain().readable(64);
1728 let builder = if writable {
1729 builder.writable(128)
1730 } else {
1731 builder
1732 };
1733
1734 let token = producer.submit(builder.build().unwrap()).unwrap();
1735 let (id, _) = consumer.poll_available().unwrap();
1736 consumer.submit_used(id, written).unwrap();
1737
1738 match producer.poll() {
1739 Ok(Some(UsedChain::Ack(returned))) if !writable => assert_eq!(returned, token),
1740 Ok(Some(UsedChain::Data(returned, segments))) if writable && written == 0 => {
1741 assert_eq!(returned, token);
1742 assert_eq!(segments.segment_count(), 0);
1743 }
1744 Err(VirtqError::InvalidState) if written == 129 => {}
1745 other => panic!("unexpected completion: {other:?}"),
1746 }
1747
1748 assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), 0);
1749 assert_eq!(pool.num_live(), 0);
1750 assert_eq!(pool.strong_count(), 2);
1751 assert_eq!(producer.num_inflight(), 0);
1752 }
1753 }
1754
1755 #[test]
1756 fn cancelling_a_chain_preserves_slot_order_after_a_partial_write_error() {
1757 let storage = TestMem::new(16);
1758 let base = storage.base_addr();
1759 let mem = FaultMem::new(storage);
1760 let t1 = SlotLayout::new(base, 2, 2).unwrap();
1761 let t2 = SlotLayout::new(base + 4, 4, 3).unwrap();
1762 let pool = SlotPool::new_tiered(t1, t2).unwrap();
1763
1764 let mut chain = ChainBuilder::new(mem.clone(), pool.clone(), 8)
1765 .readable(9)
1766 .writable_avail()
1767 .build()
1768 .unwrap();
1769
1770 let addresses: Vec<_> = chain
1771 .owned
1772 .buffers
1773 .iter()
1774 .map(|buffer| buffer.addr)
1775 .collect();
1776
1777 mem.fail_write_at(1);
1778 assert!(matches!(
1779 chain.write_all(b"abcdefghi"),
1780 Err(VirtqError::MemoryWriteError)
1781 ));
1782 assert_eq!(chain.written(), 4);
1783
1784 drop(chain);
1785
1786 assert_eq!(pool.num_live(), 0);
1787 assert_eq!(pool.strong_count(), 1);
1788
1789 mem.allow_writes();
1790 let repeated = ChainBuilder::new(mem, pool.clone(), 8)
1791 .readable(9)
1792 .writable_avail()
1793 .build()
1794 .unwrap();
1795
1796 assert_eq!(
1797 repeated
1798 .owned
1799 .buffers
1800 .iter()
1801 .map(|buffer| buffer.addr)
1802 .collect::<Vec<_>>(),
1803 addresses
1804 );
1805 }
1806
1807 #[test]
1808 fn reclaim_buffers_multiple_ring_generations() {
1809 let ring = make_ring(8);
1810 let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1811 let count = ring.len() * 2 + 1;
1812
1813 for _ in 0..count {
1814 let chain = producer.chain().writable(1).build().unwrap();
1815 producer.submit(chain).unwrap();
1816 let (recv, reply) = poll_received(&mut consumer);
1817 consumer.complete(recv, reply).unwrap();
1818 assert_eq!(producer.reclaim().unwrap(), 1);
1819 assert_eq!(producer.num_free(), ring.len());
1820 assert_eq!(producer.pool().num_live(), 0);
1821 }
1822
1823 for seq in 0..count {
1824 let Some(UsedChain::Data(token, segments)) = producer.poll().unwrap() else {
1825 panic!("expected an empty writable completion");
1826 };
1827
1828 assert_eq!(token.seq as usize, seq);
1829 assert!(segments.is_empty());
1830 }
1831
1832 assert!(producer.poll().unwrap().is_none());
1833 }
1834
1835 #[test]
1836 fn pool_swap_skips_pools_without_retained_allocations() {
1837 let ring = make_ring(8);
1838 let (mut producer, _consumer) = make_virtq_pair(&ring, 64);
1839 let pool = producer.pool().clone();
1840
1841 unsafe { producer.reclaim_slots() }.unwrap();
1843 assert!(producer.pool().ptr_eq(&pool));
1844
1845 let chain = producer.chain().writable(64).build().unwrap();
1846 producer.submit(chain).unwrap();
1847 unsafe { producer.reclaim_slots() }.unwrap();
1849 assert!(producer.pool().ptr_eq(&pool));
1850
1851 unsafe { producer.reset() }.unwrap();
1853 assert_eq!(pool.num_live(), 0);
1854 }
1855
1856 #[test]
1857 fn pool_swap_preserves_posted_requests_and_ring_state() {
1858 let ring = make_ring(8);
1859 let mem = ring.mem();
1860 let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1861
1862 let original = producer.pool().clone();
1863 let allocation = original.alloc(64).unwrap();
1864 let mut retained = vec![BufferLease::new(original.clone(), allocation)];
1865
1866 let mut tokens = Vec::new();
1867 for _ in 0..2 {
1868 let chain = producer.chain().writable(64).build().unwrap();
1869 tokens.push(producer.submit(chain).unwrap());
1870 }
1871
1872 let (recv, reply) = poll_received(&mut consumer);
1873 let ReplyChain::Writable(mut reply) = reply else {
1874 panic!("expected writable reply");
1875 };
1876 reply.write_all(b"first").unwrap();
1877 consumer.complete(recv, reply).unwrap();
1878
1879 let mut ring_before = vec![0; Layout::query_size(ring.len())];
1880 mem.read(mem.base_addr(), &mut ring_before).unwrap();
1881
1882 for round in 0..2 {
1883 let previous = producer.pool().clone();
1884 if round != 0 {
1885 let allocation = previous.alloc(64).unwrap();
1886 retained.push(BufferLease::new(previous.clone(), allocation));
1887 }
1888 unsafe { producer.reclaim_slots() }.unwrap();
1891 assert!(!producer.pool().ptr_eq(&previous));
1892
1893 let mut ring_after = vec![0; ring_before.len()];
1894 mem.read(mem.base_addr(), &mut ring_after).unwrap();
1895 assert_eq!(ring_after, ring_before);
1896 }
1897
1898 let pool = producer.pool().clone();
1899 let reused = pool.alloc(64).unwrap();
1900 assert_eq!(reused.addr, allocation.addr);
1901
1902 drop(retained);
1903 assert_eq!(pool.num_live(), 3);
1904
1905 let first = producer.poll().unwrap().unwrap();
1906 assert_eq!(first.token(), tokens[0]);
1907
1908 let first = first.into_bytes().unwrap();
1909 assert_eq!(first.as_ref(), b"first");
1910
1911 drop(first);
1912 assert_eq!(pool.num_live(), 2);
1913
1914 let (recv, reply) = poll_received(&mut consumer);
1915 let ReplyChain::Writable(mut reply) = reply else {
1916 panic!("expected writable reply");
1917 };
1918
1919 reply.write_all(b"second").unwrap();
1920 consumer.complete(recv, reply).unwrap();
1921
1922 let second = producer.poll().unwrap().unwrap();
1923 assert_eq!(second.token(), tokens[1]);
1924
1925 let second = second.into_bytes().unwrap();
1926 assert_eq!(second.as_ref(), b"second");
1927
1928 drop(second);
1929 pool.dealloc(reused.addr).unwrap();
1930 assert_eq!(pool.num_live(), 0);
1931 }
1932
1933 #[test]
1934 fn reset_reclaims_inflight_slots_and_reuses_ring() {
1935 let ring = make_ring(8);
1936 let mem = ring.mem();
1937
1938 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1939 let pool_layout = SlotLayout::new(pool_base, 64, ring.len()).unwrap();
1940 let pool = SlotPool::new(pool_layout).unwrap();
1941 let notifier = TestNotifier::new();
1942
1943 let mut producer = VirtqProducer::new(ring.layout(), mem, notifier, pool.clone());
1944
1945 for _ in 0..ring.len() {
1946 let chain = producer.chain().writable(64).build().unwrap();
1947 producer.submit(chain).unwrap();
1948 }
1949
1950 assert_eq!(pool.num_free(), 0);
1951
1952 unsafe { producer.reset() }.unwrap();
1954
1955 assert_eq!(producer.num_inflight(), 0);
1956 assert_eq!(producer.num_free(), ring.len());
1957 assert_eq!(pool.num_free(), ring.len());
1958 assert!(producer.inflight.live.is_empty());
1959
1960 for _ in 0..ring.len() {
1961 let chain = producer.chain().writable(64).build().unwrap();
1962 producer.submit(chain).unwrap();
1963 }
1964
1965 unsafe { producer.reset() }.unwrap();
1967 }
1968
1969 #[test]
1970 fn stopped_reset_reuses_slot_after_consumer_handles_drop() {
1971 let ring = make_ring(4);
1972 let mem = ring.mem();
1973 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1974 let pool_layout = SlotLayout::new(pool_base, 4, 1).unwrap();
1975 let pool = SlotPool::new(pool_layout).unwrap();
1976 let notifier = TestNotifier::new();
1977 let mut producer =
1978 VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool.clone());
1979 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), notifier.clone());
1980
1981 let sent = producer.chain().writable(4).build().unwrap();
1982 producer.submit(sent).unwrap();
1983 let (recv, reply) = poll_received(&mut consumer);
1984 assert!(matches!(reply, ReplyChain::Writable(_)));
1985
1986 drop((recv, reply, consumer));
1987 assert_eq!(pool.num_live(), 1);
1988
1989 unsafe { producer.reset() }.unwrap();
1991 assert_eq!(pool.num_live(), 0);
1992
1993 let mut replacement = producer.chain().readable(4).build().unwrap();
1994 assert_eq!(replacement.owned.buffers[0].addr, pool_base);
1995 replacement.write_all(b"GOOD").unwrap();
1996 producer.submit(replacement).unwrap();
1997
1998 let mut consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1999 let (recv, reply) = poll_received(&mut consumer);
2000 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"GOOD");
2001 consumer.complete(recv, reply).unwrap();
2002 producer.drain(drop).unwrap();
2003 assert_eq!(pool.num_live(), 0);
2004 }
2005
2006 #[test]
2007 fn reset_and_pool_swap_reject_buffered_writable_completion() {
2008 let ring = make_ring(8);
2009 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2010 let chain = producer.chain().writable(64).build().unwrap();
2011 producer.submit(chain).unwrap();
2012
2013 let (recv, reply) = poll_received(&mut consumer);
2014 let ReplyChain::Writable(mut reply) = reply else {
2015 panic!("expected writable reply");
2016 };
2017
2018 reply.write_all(b"retained").unwrap();
2019 consumer.complete(recv, reply).unwrap();
2020 producer.reclaim().unwrap();
2021
2022 let pool = producer.pool().clone();
2023 assert!(matches!(
2026 unsafe { producer.reclaim_slots() },
2027 Err(VirtqError::InvalidState)
2028 ));
2029 assert!(producer.pool().ptr_eq(&pool));
2030
2031 assert!(matches!(
2033 unsafe { producer.reset() },
2034 Err(VirtqError::InvalidState)
2035 ));
2036
2037 drop(producer.poll().unwrap().unwrap());
2038
2039 unsafe { producer.reset() }.unwrap();
2041 }
2042
2043 #[test]
2044 fn test_chain_readwrite_build() {
2045 let ring = make_ring(16);
2046 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2047
2048 let se = producer.chain().readable(64).writable(128).build().unwrap();
2049 assert_eq!(se.capacity(), 64);
2050 assert_eq!(se.written(), 0);
2051 assert_eq!(se.remaining(), 64);
2052 }
2053
2054 #[test]
2055 fn test_chain_readable_writable_names_build() {
2056 let ring = make_ring(16);
2057 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2058
2059 let se = producer.chain().readable(16).writable(32).build().unwrap();
2060 assert_eq!(se.desc_count(), 2);
2061 assert_eq!(se.rd_desc_count(), 1);
2062 assert_eq!(se.wr_desc_count(), 1);
2063 assert_eq!(se.capacity(), 16);
2064 }
2065
2066 #[test]
2067 fn test_chain_multi_readable_write_all_scatters() {
2068 let ring = make_ring(16);
2069 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2070
2071 let mut se = producer
2072 .chain()
2073 .readable(5)
2074 .readable(6)
2075 .writable(32)
2076 .build()
2077 .unwrap();
2078
2079 se.write_all(b"hello world").unwrap();
2080 assert_eq!(se.written(), 11);
2081
2082 let token = producer.submit(se).unwrap();
2083 let (recv, reply) = poll_received(&mut consumer);
2084 assert_eq!(recv.token(), token);
2085 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2086
2087 let segments = recv.to_segments().unwrap();
2088 assert_eq!(segments.segment_count(), 2);
2089 assert_eq!(segments.as_slice()[0].as_ref(), b"hello");
2090 assert_eq!(segments.as_slice()[1].as_ref(), b" world");
2091
2092 consumer.complete(recv, reply).unwrap();
2093 producer.drain(drop).unwrap();
2094 }
2095
2096 #[test]
2097 fn test_chain_independent_readables_preserve_pool_tiers() {
2098 let ring = make_ring(16);
2099 let layout = ring.layout();
2100 let mem = ring.mem();
2101
2102 let lower = SlotLayout::new(
2103 mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100,
2104 256,
2105 1,
2106 )
2107 .unwrap();
2108
2109 let upper = SlotLayout::new(lower.end_addr(), 4096, 1).unwrap();
2110 let pool = SlotPool::new_tiered(lower, upper).unwrap();
2111 let notifier = TestNotifier::new();
2112 let producer = VirtqProducer::new(layout, mem, notifier, pool.clone());
2113
2114 let send = producer
2115 .chain()
2116 .readable(128)
2117 .readable(4096)
2118 .build()
2119 .unwrap();
2120
2121 let readables = send.owned.descriptors().collect::<Vec<_>>();
2122
2123 assert_eq!(readables.len(), 2);
2124 assert_eq!(readables[0].addr, lower.base_addr());
2125 assert_eq!(readables[1].addr, upper.base_addr());
2126 assert_eq!(pool.num_free_lower(), 0);
2127 assert_eq!(pool.num_free_upper(), 0);
2128
2129 drop(send);
2130
2131 assert_eq!(pool.num_free_lower(), 1);
2132 assert_eq!(pool.num_free_upper(), 1);
2133 }
2134
2135 #[test]
2136 fn chain_avail_uses_pool_availability_at_build() {
2137 for held_count in [0, 2, 3] {
2138 let builder = make_chain(2, 3, 4096, 3);
2139 let pool = builder.pool.clone();
2140 let builder = builder.writable_avail().readable(128);
2141
2142 assert_eq!(pool.num_live(), 0);
2143
2144 let held: Vec<_> = (0..held_count).map(|_| pool.alloc(4096).unwrap()).collect();
2145
2146 let chain = builder.build().unwrap();
2147
2148 let writable = (3 - held_count).min(2);
2149 assert_eq!(chain.rd_desc_count(), 1);
2150 assert_eq!(chain.wr_desc_count(), writable);
2151 assert_eq!(chain.desc_count(), 1 + writable);
2152 assert_eq!(chain.owned.buffers[0].capacity, 128);
2153 assert!(
2154 chain.owned.buffers[1..]
2155 .iter()
2156 .all(|buf| buf.capacity == 4096)
2157 );
2158 assert_eq!(pool.num_free_lower(), 1);
2159 assert_eq!(pool.num_free_upper(), 3 - held_count - writable);
2160 assert_eq!(pool.num_live(), held_count + chain.desc_count());
2161
2162 drop(chain);
2163
2164 assert_eq!(pool.num_live(), held_count);
2165
2166 for allocation in held.into_iter().rev() {
2167 pool.dealloc(allocation.addr).unwrap();
2168 }
2169
2170 assert_eq!(pool.num_live(), 0);
2171 }
2172 }
2173
2174 #[test]
2175 fn chain_avail_respects_mandatory_descriptor_budget() {
2176 for max_descs in 0..3 {
2177 let builder = make_chain(2, 3, 4096, max_descs);
2178 let pool = builder.pool.clone();
2179
2180 let result = builder
2181 .readable(4096)
2182 .readable(128)
2183 .writable_avail()
2184 .build();
2185
2186 if max_descs == 2 {
2187 let chain = result.unwrap();
2188
2189 assert_eq!(chain.rd_desc_count(), 2);
2190 assert_eq!(chain.wr_desc_count(), 0);
2191
2192 drop(chain);
2193 } else {
2194 assert!(matches!(result, Err(VirtqError::Backpressure)));
2195 }
2196
2197 assert_eq!(pool.num_live(), 0);
2198
2199 let lower = pool.alloc(128).unwrap();
2200 let upper = pool.alloc(4096).unwrap();
2201
2202 assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2203 assert_eq!(upper.addr, pool.slot_addr(4).unwrap());
2204
2205 pool.dealloc(lower.addr).unwrap();
2206 pool.dealloc(upper.addr).unwrap();
2207 }
2208 }
2209
2210 #[test]
2211 fn chain_avail_preserves_explicit_writable_capacity_when_full() {
2212 for (upper_count, max_descs) in [(2, 2), (1, 3)] {
2213 let builder = make_chain(1, upper_count, 4096, max_descs);
2214 let pool = builder.pool.clone();
2215
2216 let chain = builder
2217 .writable_avail()
2218 .writable(128)
2219 .readable(128)
2220 .build()
2221 .unwrap();
2222
2223 assert_eq!(chain.rd_desc_count(), 1);
2224 assert_eq!(chain.wr_desc_count(), 1);
2225 assert_eq!(chain.owned.buffers[0].capacity, 128);
2226 assert_eq!(chain.owned.buffers[1].capacity, 4096);
2227 assert!(chain.owned.buffers.iter().all(|buf| buf.written == 0));
2228 assert_eq!(pool.num_free_upper(), upper_count - 1);
2229
2230 drop(chain);
2231
2232 assert_eq!(pool.num_live(), 0);
2233 }
2234 }
2235
2236 #[test]
2237 fn chain_avail_cannot_build_an_empty_chain() {
2238 for max_descs in [0, 2] {
2239 let builder = make_chain(1, 1, 4096, max_descs);
2240 let pool = builder.pool.clone();
2241 let held = (max_descs != 0).then(|| pool.alloc(4096).unwrap());
2242
2243 assert!(matches!(
2244 builder.writable_avail().build(),
2245 Err(VirtqError::Backpressure)
2246 ));
2247 assert_eq!(pool.num_free_lower(), 1);
2248 assert_eq!(pool.num_live(), usize::from(held.is_some()));
2249
2250 if let Some(allocation) = held {
2251 pool.dealloc(allocation.addr).unwrap();
2252 }
2253 }
2254 }
2255
2256 #[test]
2257 fn chain_avail_only_uses_upper_slots() {
2258 let builder = make_chain(4, 2, 4096, 6);
2259 let pool = builder.pool.clone();
2260
2261 let chain = builder.writable_avail().writable_avail().build().unwrap();
2262
2263 assert_eq!(chain.rd_desc_count(), 0);
2264 assert_eq!(chain.wr_desc_count(), 2);
2265 assert_eq!(pool.num_free_lower(), 4);
2266 assert_eq!(pool.num_free_upper(), 0);
2267
2268 drop(chain);
2269
2270 assert_eq!(pool.num_live(), 0);
2271 }
2272
2273 #[test]
2274 fn chain_preserves_region_order_with_odd_slot_sizes() {
2275 let builder = make_chain(1, 4, 3001, 5);
2276 let pool = builder.pool.clone();
2277
2278 let chain = builder
2280 .readable(128)
2281 .readable(6003)
2282 .writable_avail()
2283 .build()
2284 .unwrap();
2285
2286 assert_eq!(chain.rd_desc_count(), 4);
2287
2288 let actual_offsets = chain
2289 .owned
2290 .buffers
2291 .iter()
2292 .map(|buf| {
2293 (
2294 buf.addr,
2295 buf.capacity,
2296 pool.allocation_len(buf.addr).unwrap(),
2297 )
2298 })
2299 .collect::<Vec<_>>();
2300
2301 let expected_offsets = [
2302 (0, 128, 256),
2303 (4, 3001, 3001),
2304 (3, 3001, 3001),
2305 (2, 1, 3001),
2306 (1, 3001, 3001),
2307 ]
2308 .map(|(slot, capacity, allocated)| (pool.slot_addr(slot).unwrap(), capacity, allocated));
2309
2310 assert_eq!(actual_offsets, expected_offsets);
2311
2312 drop(chain);
2313
2314 assert_eq!(pool.num_live(), 0);
2315 }
2316
2317 #[test]
2318 fn chain_failure_releases_only_its_own_slots() {
2319 let builder = make_chain(2, 3, 4096, 4);
2320 let pool = builder.pool.clone();
2321 let held = pool.alloc(4096).unwrap();
2322
2323 assert!(matches!(
2324 builder.readable(128).writable(4096 * 3).build(),
2325 Err(VirtqError::Backpressure)
2326 ));
2327 assert_eq!(pool.live_addrs(), [held.addr]);
2328
2329 let lower = pool.alloc(128).unwrap();
2330 let upper = pool.alloc(4096).unwrap();
2331 let next_upper = pool.alloc(4096).unwrap();
2332
2333 assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2334 assert_eq!(upper.addr, pool.slot_addr(3).unwrap());
2335 assert_eq!(next_upper.addr, pool.slot_addr(2).unwrap());
2336
2337 pool.dealloc(next_upper.addr).unwrap();
2338 pool.dealloc(upper.addr).unwrap();
2339 pool.dealloc(lower.addr).unwrap();
2340 pool.dealloc(held.addr).unwrap();
2341 }
2342
2343 #[test]
2344 fn chain_rejects_invalid_request_sequences() {
2345 for (readable, writable) in [(vec![128, 0], vec![]), (vec![128], vec![4096, 0])] {
2346 let mut builder = make_chain(1, 1, 4096, 4);
2347 let pool = builder.pool.clone();
2348
2349 for cap in readable {
2350 builder = builder.readable(cap);
2351 }
2352 for cap in writable {
2353 builder = builder.writable(cap);
2354 }
2355
2356 assert!(matches!(
2357 builder.build(),
2358 Err(VirtqError::Alloc(AllocError::InvalidArg))
2359 ));
2360 assert_eq!(pool.num_live(), 0);
2361 }
2362
2363 let builder = make_chain(1, 1, 4096, 4);
2364 let pool = builder.pool.clone();
2365
2366 assert!(matches!(
2367 builder.readable(usize::MAX).readable(1).build(),
2368 Err(VirtqError::Alloc(AllocError::Overflow))
2369 ));
2370 assert_eq!(pool.num_live(), 0);
2371 }
2372
2373 #[test]
2374 fn chain_keeps_four_records_inline_and_one_pool_handle() {
2375 for count in [4, 5] {
2376 let builder = make_chain(0, count, 4096, count);
2377 let pool = builder.pool.clone();
2378
2379 let chain = builder
2380 .readable((count - 1) * 4096)
2381 .writable_avail()
2382 .build()
2383 .unwrap();
2384
2385 assert_eq!(chain.desc_count(), count);
2386 assert_eq!(chain.owned.buffers.spilled(), count > 4);
2387 assert_eq!(pool.strong_count(), 2);
2388
2389 drop(chain);
2390
2391 assert_eq!(pool.num_live(), 0);
2392 }
2393 }
2394
2395 #[test]
2396 fn test_chain_multi_readable_appends_across_calls() {
2397 let ring = make_ring(16);
2398 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2399
2400 let mut send = producer.chain().readable(8).build().unwrap();
2401 send.write_all(b"abc").unwrap();
2402 send.write_all(b"def").unwrap();
2403 assert_eq!(send.written(), 6);
2404 assert_eq!(send.remaining(), 2);
2405
2406 producer.submit(send).unwrap();
2407 let (recv, reply) = poll_received(&mut consumer);
2408 let segments = recv.to_segments().unwrap();
2409 assert_eq!(segments.segment_count(), 2);
2410 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2411 assert_eq!(segments.as_slice()[1].as_ref(), b"ef");
2412 consumer.complete(recv, reply).unwrap();
2413 producer.drain(drop).unwrap();
2414 }
2415
2416 #[test]
2417 fn test_chain_readable_splits_logical_capacity() {
2418 let expected_lengths = [4, 4, 4];
2419 let ring = make_ring(16);
2420 let (mut producer, mut consumer, _) = make_test_producer_with_slot_size(&ring, 4);
2421
2422 let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2423 let readables = &se.owned.buffers[..se.rd_desc_count()];
2424
2425 assert_eq!(se.rd_desc_count(), 3);
2426 assert_eq!(se.capacity(), 10);
2427
2428 let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2429 assert_eq!(caps, [4, 4, 2]);
2430
2431 let lengths = readables
2432 .iter()
2433 .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2434 .collect::<Vec<_>>();
2435 assert_eq!(lengths, expected_lengths);
2436
2437 se.write_all(b"abcdefghij").unwrap();
2438 assert_eq!(se.written(), 10);
2439
2440 let token = producer.submit(se).unwrap();
2441 let (recv, reply) = poll_received(&mut consumer);
2442
2443 assert_eq!(recv.token(), token);
2444 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2445
2446 let segments = recv.to_segments().unwrap();
2447
2448 assert_eq!(segments.segment_count(), 3);
2449 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2450 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2451 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2452
2453 consumer.complete(recv, reply).unwrap();
2454 producer.drain(drop).unwrap();
2455
2456 assert_eq!(producer.pool.num_live(), 0);
2457 }
2458
2459 #[test]
2460 fn test_chain_readable_splits_logical_capacity_tiered() {
2461 let expected_lengths = [4, 4, 2];
2462 let ring = make_ring(16);
2463 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2464
2465 let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2466 let readables = &se.owned.buffers[..se.rd_desc_count()];
2467
2468 assert_eq!(se.rd_desc_count(), 3);
2469 assert_eq!(se.capacity(), 10);
2470
2471 let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2472 assert_eq!(caps, [4, 4, 2]);
2473
2474 let lengths = readables
2475 .iter()
2476 .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2477 .collect::<Vec<_>>();
2478 assert_eq!(lengths, expected_lengths);
2479
2480 se.write_all(b"abcdefghij").unwrap();
2481 assert_eq!(se.written(), 10);
2482
2483 let token = producer.submit(se).unwrap();
2484 let (recv, reply) = poll_received(&mut consumer);
2485
2486 assert_eq!(recv.token(), token);
2487 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2488
2489 let segments = recv.to_segments().unwrap();
2490
2491 assert_eq!(segments.segment_count(), 3);
2492 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2493 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2494 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2495
2496 consumer.complete(recv, reply).unwrap();
2497 producer.drain(drop).unwrap();
2498
2499 assert_eq!(producer.pool.num_live(), 0);
2500 }
2501
2502 #[test]
2503 fn test_chain_readable_rejects_zero_capacity_on_build() {
2504 let ring = make_ring(16);
2505 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2506
2507 assert!(matches!(
2508 producer.chain().readable(0).build(),
2509 Err(VirtqError::Alloc(AllocError::InvalidArg))
2510 ));
2511 }
2512
2513 #[test]
2514 fn test_chain_writable_splits_logical_capacity() {
2515 let ring = make_ring(16);
2516 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2517
2518 let se = producer.chain().writable(10).build().unwrap();
2519 let token = producer.submit(se).unwrap();
2520
2521 let (recv, reply) = poll_received(&mut consumer);
2522 let ReplyChain::Writable(mut wc) = reply else {
2523 panic!("expected writable reply");
2524 };
2525 assert_eq!(wc.capacity(), 10);
2526
2527 wc.write_all(b"abcdefghij").unwrap();
2528 consumer.complete(recv, wc).unwrap();
2529
2530 let used = producer.poll().unwrap().unwrap();
2531 assert_eq!(used.token(), token);
2532 let segments = used.segments().unwrap();
2533 assert_eq!(segments.segment_count(), 3);
2534 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2535 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2536 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2537 }
2538
2539 #[test]
2540 fn test_chain_writable_rejects_zero_capacity_on_build() {
2541 let ring = make_ring(16);
2542 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2543
2544 assert!(matches!(
2545 producer.chain().writable(0).build(),
2546 Err(VirtqError::Alloc(AllocError::InvalidArg))
2547 ));
2548 }
2549
2550 #[test]
2551 fn test_chain_multi_readable_write_all_preserves_segments() {
2552 let ring = make_ring(16);
2553 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2554
2555 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2556 se.write_all(b"headbody").unwrap();
2557
2558 producer.submit(se).unwrap();
2559 let (recv, reply) = poll_received(&mut consumer);
2560 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"headbody");
2561 consumer.complete(recv, reply).unwrap();
2562 producer.drain(drop).unwrap();
2563 }
2564
2565 #[test]
2566 fn test_chain_payload_segment_writer_serializes_directly() {
2567 let ring = make_ring(16);
2568 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2569
2570 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2571 se.with_seg(0, |segment| {
2572 segment.copy_from_slice(b"head");
2573 Ok::<usize, VirtqError>(4)
2574 })
2575 .unwrap();
2576 se.with_seg(1, |segment| {
2577 segment.copy_from_slice(b"body");
2578 Ok::<usize, VirtqError>(4)
2579 })
2580 .unwrap();
2581
2582 producer.submit(se).unwrap();
2583 let (recv, reply) = poll_received(&mut consumer);
2584 let segments = recv.to_segments().unwrap();
2585 assert_eq!(segments.segment_count(), 2);
2586 assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2587 consumer.complete(recv, reply).unwrap();
2588 producer.drain(drop).unwrap();
2589 }
2590
2591 #[test]
2592 fn test_chain_payload_segment_write_copies_directly() {
2593 let ring = make_ring(16);
2594 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2595
2596 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2597 se.write_seg(0, b"head").unwrap();
2598 se.write_seg(1, b"body").unwrap();
2599
2600 producer.submit(se).unwrap();
2601 let (recv, reply) = poll_received(&mut consumer);
2602 let segments = recv.to_segments().unwrap();
2603 assert_eq!(segments.segment_count(), 2);
2604 assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2605 consumer.complete(recv, reply).unwrap();
2606 producer.drain(drop).unwrap();
2607 }
2608
2609 #[test]
2610 fn test_chain_multi_writable_used_returns_segments() {
2611 let ring = make_ring(16);
2612 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2613
2614 let se = producer.chain().writable(5).writable(6).build().unwrap();
2615 let token = producer.submit(se).unwrap();
2616
2617 let (recv, reply) = poll_received(&mut consumer);
2618 let ReplyChain::Writable(mut wc) = reply else {
2619 panic!("expected writable reply");
2620 };
2621 assert_eq!(wc.capacity(), 12);
2622
2623 wc.write_all(b"hello world").unwrap();
2624 consumer.complete(recv, wc).unwrap();
2625
2626 let used = producer.poll().unwrap().unwrap();
2627 assert_eq!(used.token(), token);
2628 let segments = used.segments().unwrap();
2629 assert_eq!(segments.segment_count(), 2);
2630 assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2631 assert_eq!(segments.as_slice()[1].as_ref(), b"world");
2632 assert_eq!(segments.to_bytes().as_ref(), b"hello world");
2633 }
2634
2635 #[test]
2636 fn test_chain_multi_writable_short_used_truncates_last_segment() {
2637 let ring = make_ring(16);
2638 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2639
2640 let se = producer.chain().writable(5).writable(6).build().unwrap();
2641 producer.submit(se).unwrap();
2642
2643 let (recv, reply) = poll_received(&mut consumer);
2644 let ReplyChain::Writable(mut wc) = reply else {
2645 panic!("expected writable reply");
2646 };
2647
2648 wc.write_all(b"hello wo").unwrap();
2649 consumer.complete(recv, wc).unwrap();
2650
2651 let used = producer.poll().unwrap().unwrap();
2652 let segments = used.segments().unwrap();
2653 assert_eq!(segments.segment_count(), 2);
2654 assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2655 assert_eq!(segments.as_slice()[1].as_ref(), b"wo");
2656 assert_eq!(segments.to_bytes().as_ref(), b"hello wo");
2657 }
2658
2659 #[test]
2660 fn test_chain_multi_writable_zero_used_returns_empty_segments() {
2661 let ring = make_ring(16);
2662 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2663
2664 let se = producer.chain().writable(5).writable(6).build().unwrap();
2665 producer.submit(se).unwrap();
2666
2667 let (recv, reply) = poll_received(&mut consumer);
2668 consumer.complete(recv, reply).unwrap();
2669
2670 let used = producer.poll().unwrap().unwrap();
2671 let segments = used.segments().unwrap();
2672 assert_eq!(segments.segment_count(), 0);
2673 assert!(segments.is_empty());
2674 assert!(segments.to_bytes().is_empty());
2675 }
2676
2677 #[test]
2678 fn test_chain_readable_only_build() {
2679 let ring = make_ring(16);
2680 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2681
2682 let se = producer.chain().readable(32).build().unwrap();
2683 assert_eq!(se.capacity(), 32);
2684 }
2685
2686 #[test]
2687 fn test_chain_writable_only_build() {
2688 let ring = make_ring(16);
2689 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2690
2691 let se = producer.chain().writable(64).build().unwrap();
2692 assert_eq!(se.capacity(), 0);
2693 }
2694
2695 #[test]
2696 fn test_chain_empty_build_fails() {
2697 let ring = make_ring(16);
2698 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2699
2700 let result = producer.chain().build();
2701 assert!(matches!(result, Err(VirtqError::InvalidState)));
2702 }
2703
2704 #[test]
2705 fn test_send_chain_write_all_and_submit() {
2706 let ring = make_ring(16);
2707 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2708
2709 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2710
2711 se.write_all(b"hello")
2712 .unwrap()
2713 .write_all(b" world")
2714 .unwrap();
2715 assert_eq!(se.written(), 11);
2716 assert_eq!(se.remaining(), 53);
2717 let tok = producer.submit(se).unwrap();
2718
2719 let (recv, reply) = poll_received(&mut consumer);
2720 assert_eq!(recv.token(), tok);
2721 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2722 consumer.complete(recv, reply).unwrap();
2723 producer.drain(drop).unwrap();
2724 }
2725
2726 #[test]
2727 fn test_send_payload_write_all_fluent() {
2728 let ring = make_ring(16);
2729 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2730
2731 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2732 se.write_all(b"hello")
2733 .unwrap()
2734 .write_all(b" world")
2735 .unwrap();
2736 assert_eq!(se.written(), 11);
2737 assert_eq!(se.remaining(), 53);
2738 let tok = producer.submit(se).unwrap();
2739
2740 let (recv, reply) = poll_received(&mut consumer);
2741 assert_eq!(recv.token(), tok);
2742 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2743 consumer.complete(recv, reply).unwrap();
2744 producer.drain(drop).unwrap();
2745 }
2746
2747 #[test]
2748 fn test_send_payload_partial_write() {
2749 let ring = make_ring(16);
2750 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2751
2752 let mut se = producer.chain().readable(8).build().unwrap();
2753 let written = se.write(b"hello world").unwrap();
2754 assert_eq!(written, 8);
2755 assert_eq!(se.remaining(), 0);
2756
2757 producer.submit(se).unwrap();
2758 let (recv, reply) = poll_received(&mut consumer);
2759 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello wo");
2760 consumer.complete(recv, reply).unwrap();
2761 producer.drain(drop).unwrap();
2762 }
2763
2764 #[test]
2765 fn test_send_payload_write_with_serializes_directly() {
2766 let ring = make_ring(16);
2767 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2768
2769 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2770 se.with_seg(0, |buf| {
2771 buf[..5].copy_from_slice(b"hello");
2772 Ok::<usize, VirtqError>(5)
2773 })
2774 .unwrap();
2775
2776 let _tok = producer.submit(se).unwrap();
2777
2778 let (recv, reply) = poll_received(&mut consumer);
2779 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2780 consumer.complete(recv, reply).unwrap();
2781 producer.drain(drop).unwrap();
2782 }
2783
2784 #[test]
2785 fn test_send_chain_single_segment_writer_serializes_directly() {
2786 let ring = make_ring(16);
2787 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2788
2789 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2790 se.with_seg(0, |segment| {
2791 assert_eq!(segment.len(), 64);
2792 segment[..5].copy_from_slice(b"hello");
2793 Ok::<usize, VirtqError>(5)
2794 })
2795 .unwrap();
2796
2797 let _tok = producer.submit(se).unwrap();
2798
2799 let (recv, reply) = poll_received(&mut consumer);
2800 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2801 consumer.complete(recv, reply).unwrap();
2802 producer.drain(drop).unwrap();
2803 }
2804
2805 #[test]
2806 fn test_send_chain_single_segment_writer_rejects_multi_segment() {
2807 let ring = make_ring(16);
2808 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2809
2810 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2811 assert!(matches!(
2812 se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2813 Err(VirtqError::NoPayloadSegment)
2814 ));
2815 }
2816
2817 #[test]
2818 fn test_send_chain_single_segment_writer_rejects_auto_split_chain() {
2819 let ring = make_ring(16);
2820 let (producer, _consumer) = make_virtq_pair(&ring, 4);
2821
2822 let mut se = producer.chain().readable(8).build().unwrap();
2823 assert_eq!(se.rd_desc_count(), 2);
2824 assert!(matches!(
2825 se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2826 Err(VirtqError::NoPayloadSegment)
2827 ));
2828 }
2829
2830 #[test]
2831 fn test_send_payload_segment_set_written_too_large() {
2832 let ring = make_ring(16);
2833 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2834
2835 let mut se = producer.chain().readable(32).writable(64).build().unwrap();
2836 let err = se
2837 .with_seg(0, |_| Ok::<usize, VirtqError>(64))
2838 .err()
2839 .unwrap();
2840 assert!(matches!(
2841 err,
2842 VirtqError::PayloadTooLarge {
2843 recv: 64,
2844 limit: 32
2845 }
2846 ));
2847 }
2848
2849 #[test]
2850 fn test_send_chain_write_too_large() {
2851 let ring = make_ring(16);
2852 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2853
2854 let mut se = producer.chain().readable(4).build().unwrap();
2855 let err = se.write_all(b"too long").err().unwrap();
2856 assert!(matches!(
2857 err,
2858 VirtqError::PayloadTooLarge { recv: 8, limit: 4 }
2859 ));
2860 }
2861
2862 #[test]
2863 fn test_writeonly_has_no_readable_buffer() {
2864 let ring = make_ring(16);
2865 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2866
2867 let mut se = producer.chain().writable(32).build().unwrap();
2868 let err = se.write_all(b"data").err().unwrap();
2869 assert!(matches!(err, VirtqError::NoPayloadSegment));
2870 }
2871
2872 #[test]
2873 fn test_drop_chain_builder_deallocs() {
2874 let ring = make_ring(16);
2875 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2876
2877 {
2878 let _builder = producer.chain().readable(64).writable(128);
2879 }
2881
2882 let se = producer.chain().readable(64).writable(128).build().unwrap();
2884 let tok = producer.submit(se).unwrap();
2885 assert!(tok.id < 16);
2886
2887 unsafe { producer.reset() }.unwrap();
2889 }
2890
2891 #[test]
2892 fn test_drop_send_chain_deallocs() {
2893 let ring = make_ring(16);
2894 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2895
2896 {
2897 let _se = producer.chain().readable(64).writable(128).build().unwrap();
2898 }
2900
2901 let se = producer.chain().readable(64).writable(128).build().unwrap();
2903 let tok = producer.submit(se).unwrap();
2904 assert!(tok.id < 16);
2905
2906 unsafe { producer.reset() }.unwrap();
2908 }
2909
2910 #[test]
2911 fn test_submit_notifies() {
2912 let ring = make_ring(16);
2913 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2914
2915 let initial_count = notifier.notification_count();
2916
2917 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2918 se.write_all(b"hello").unwrap();
2919 producer.submit(se).unwrap();
2920
2921 assert!(notifier.notification_count() > initial_count);
2922
2923 unsafe { producer.reset() }.unwrap();
2925 }
2926
2927 #[test]
2928 fn test_submit_read_only_notifies_by_default() {
2929 let ring = make_ring(16);
2930 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2931
2932 let initial_count = notifier.notification_count();
2933
2934 let mut se = producer.chain().readable(64).build().unwrap();
2935 se.write_all(b"fire-and-forget").unwrap();
2936 producer.submit(se).unwrap();
2937
2938 assert!(notifier.notification_count() > initial_count);
2939
2940 unsafe { producer.reset() }.unwrap();
2942 }
2943
2944 #[test]
2945 fn test_submit_write_only_notifies_by_default() {
2946 let ring = make_ring(16);
2947 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2948
2949 let initial_count = notifier.notification_count();
2950
2951 let se = producer.chain().writable(128).build().unwrap();
2952 producer.submit(se).unwrap();
2953
2954 assert!(notifier.notification_count() > initial_count);
2955
2956 unsafe { producer.reset() }.unwrap();
2958 }
2959
2960 #[test]
2961 fn test_batch_notifies_once_on_finish() {
2962 let ring = make_ring(16);
2963 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2964
2965 let initial_count = notifier.notification_count();
2966
2967 let mut batch = producer.batch();
2968
2969 let mut first = batch.chain().readable(64).build().unwrap();
2970 first.write_all(b"first").unwrap();
2971 batch.submit(first).unwrap();
2972
2973 let mut second = batch.chain().readable(64).build().unwrap();
2974 second.write_all(b"second").unwrap();
2975 batch.submit(second).unwrap();
2976
2977 assert_eq!(notifier.notification_count(), initial_count);
2978 assert!(batch.finish().unwrap());
2979 assert_eq!(notifier.notification_count(), initial_count + 1);
2980
2981 let (recv, reply) = poll_received(&mut consumer);
2982 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"first");
2983 consumer.complete(recv, reply).unwrap();
2984
2985 let (recv, reply) = poll_received(&mut consumer);
2986 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"second");
2987 consumer.complete(recv, reply).unwrap();
2988 producer.drain(drop).unwrap();
2989 }
2990
2991 #[test]
2992 fn test_batch_finish_notifies_from_batch_start_cursor() {
2993 let ring = make_ring(16);
2994 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2995
2996 let cursor = consumer.avail_cursor();
2997 consumer
2998 .set_avail_suppression(SuppressionKind::Descriptor(cursor))
2999 .unwrap();
3000
3001 let mut batch = producer.batch();
3002
3003 let mut first = batch.chain().readable(64).build().unwrap();
3004 first.write_all(b"first").unwrap();
3005 batch.submit(first).unwrap();
3006 assert_eq!(notifier.notification_count(), 0);
3007
3008 let mut second = batch.chain().readable(64).writable(64).build().unwrap();
3009 second.write_all(b"second").unwrap();
3010 batch.submit(second).unwrap();
3011
3012 assert!(batch.finish().unwrap());
3013
3014 assert_eq!(notifier.notification_count(), 1);
3015
3016 unsafe { producer.reset() }.unwrap();
3018 }
3019
3020 #[test]
3021 fn test_empty_batch_finish_does_not_notify() {
3022 let ring = make_ring(16);
3023 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
3024
3025 let batch = producer.batch();
3026 assert!(!batch.finish().unwrap());
3027 assert_eq!(notifier.notification_count(), 0);
3028 }
3029
3030 #[test]
3031 fn test_batch_can_finish_without_notification() {
3032 let ring = make_ring(16);
3033 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
3034
3035 let mut batch = producer.batch();
3036 let mut chain = batch.chain().readable(4).build().unwrap();
3037 chain.write_all(b"data").unwrap();
3038 batch.submit(chain).unwrap();
3039 batch.finish_without_notify();
3040
3041 assert_eq!(notifier.notification_count(), 0);
3042
3043 let (recv, reply) = poll_received(&mut consumer);
3044 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"data");
3045
3046 consumer.complete(recv, reply).unwrap();
3047 producer.drain(drop).unwrap();
3048 }
3049
3050 #[test]
3051 fn test_write_only_round_trip() {
3052 let ring = make_ring(16);
3053 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3054
3055 let se = producer.chain().writable(32).build().unwrap();
3056 let token = producer.submit(se).unwrap();
3057
3058 let (recv, reply) = poll_received(&mut consumer);
3059 assert_eq!(recv.token(), token);
3060 assert!(recv.to_bytes().unwrap().is_empty());
3061
3062 if let ReplyChain::Writable(mut wc) = reply {
3063 wc.write_all(b"filled-by-consumer").unwrap();
3064 consumer.complete(recv, wc).unwrap();
3065 } else {
3066 panic!("expected Writable");
3067 }
3068
3069 let used = producer.poll().unwrap().unwrap();
3070 assert_eq!(used.token(), token);
3071 assert_eq!(used.to_bytes().unwrap().len(), b"filled-by-consumer".len());
3072 assert_eq!(used.to_bytes().unwrap().as_ref(), b"filled-by-consumer");
3073 }
3074
3075 #[test]
3076 fn test_read_only_round_trip() {
3077 let ring = make_ring(16);
3078 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3079
3080 let mut se = producer.chain().readable(32).build().unwrap();
3081 se.write_all(b"fire-and-forget").unwrap();
3082 let token = producer.submit(se).unwrap();
3083
3084 let (recv, reply) = poll_received(&mut consumer);
3085 assert_eq!(recv.token(), token);
3086 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"fire-and-forget");
3087 assert!(matches!(reply, ReplyChain::Ack(_)));
3088 consumer.complete(recv, reply).unwrap();
3089
3090 let used = producer.poll().unwrap().unwrap();
3091 assert!(matches!(used, UsedChain::Ack(t) if t == token));
3092 }
3093
3094 #[test]
3095 fn test_readwrite_round_trip() {
3096 let ring = make_ring(16);
3097 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3098
3099 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3100 se.write_all(b"request data").unwrap();
3101 let token = producer.submit(se).unwrap();
3102
3103 let (recv, reply) = poll_received(&mut consumer);
3104 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"request data");
3105 if let ReplyChain::Writable(mut wc) = reply {
3106 wc.write_all(b"response data").unwrap();
3107 consumer.complete(recv, wc).unwrap();
3108 } else {
3109 panic!("expected Writable");
3110 }
3111
3112 let used = producer.poll().unwrap().unwrap();
3113 assert_eq!(used.token(), token);
3114 assert_eq!(used.to_bytes().unwrap().as_ref(), b"response data");
3115 }
3116
3117 #[test]
3118 fn test_poll_used_requires_owned_mapping() {
3119 let ring = make_ring(16);
3120 let layout = ring.layout();
3121 let test_mem = ring.mem();
3122 let pool_base = test_mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
3123 let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
3124 let pool = SlotPool::new(pool_layout).unwrap();
3125 let notifier = TestNotifier::new();
3126 let mem = FaultMem::new(test_mem);
3127 mem.deny_views();
3128 let mut producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool.clone());
3129 let mut consumer = VirtqConsumer::new(layout, mem, notifier);
3130
3131 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3132 se.write_all(b"request data").unwrap();
3133 producer.submit(se).unwrap();
3134
3135 let (recv, reply) = poll_received(&mut consumer);
3136 if let ReplyChain::Writable(mut wc) = reply {
3137 wc.write_all(b"response data").unwrap();
3138 consumer.complete(recv, wc).unwrap();
3139 } else {
3140 panic!("expected Writable");
3141 }
3142
3143 assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
3144 assert_eq!(pool.num_live(), 0);
3145 }
3146
3147 #[test]
3148 fn test_villain_used_len_exceeding_writable_capacity_is_rejected_and_released() {
3149 let ring = make_ring(16);
3150 let (mut producer, _consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
3151 let mut ring_consumer = make_consumer(&ring);
3152
3153 let se = producer.chain().writable(4).build().unwrap();
3154 producer.submit(se).unwrap();
3155
3156 let (id, _) = ring_consumer.poll_available().unwrap();
3157 ring_consumer.submit_used(id, 8).unwrap();
3158
3159 assert!(matches!(producer.poll(), Err(VirtqError::InvalidState)));
3160 assert_eq!(producer.inner.num_inflight(), 0);
3161
3162 let se = producer.chain().writable(4).build().unwrap();
3163 producer.submit(se).unwrap();
3164 assert_eq!(producer.inner.num_inflight(), 1);
3165
3166 unsafe { producer.reset() }.unwrap();
3168 }
3169
3170 #[test]
3171 fn test_villain_used_descriptor_with_invalid_id_is_rejected() {
3172 let ring = make_ring(16);
3173 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
3174
3175 let se = producer.chain().writable(4).build().unwrap();
3176 producer.submit(se).unwrap();
3177
3178 let mut desc = Descriptor::new(0, 0, ring.len() as u16, DescFlags::empty());
3179 desc.mark_used(true);
3180 ring.write_desc(0, desc);
3181
3182 assert!(matches!(
3183 producer.poll(),
3184 Err(VirtqError::RingError(RingError::InvalidState))
3185 ));
3186 assert_eq!(producer.inner.num_inflight(), 1);
3187
3188 unsafe { producer.reset() }.unwrap();
3190 }
3191}