1use alloc::collections::VecDeque;
5use alloc::vec;
6use alloc::vec::Vec;
7use core::mem::ManuallyDrop;
8
9use bytes::Bytes;
10use smallvec::SmallVec;
11
12use super::*;
13
14#[derive(Debug)]
21pub enum UsedChain {
22 Ack(Token),
24 Data(Token, Segments),
30}
31
32impl UsedChain {
33 pub fn token(&self) -> Token {
35 match self {
36 Self::Ack(token) | Self::Data(token, _) => *token,
37 }
38 }
39
40 pub fn to_bytes(&self) -> Option<Bytes> {
42 match self {
43 Self::Ack(_) => None,
44 Self::Data(_, segments) => Some(segments.to_bytes()),
45 }
46 }
47
48 pub fn segments(&self) -> Option<&Segments> {
50 match self {
51 Self::Ack(_) => None,
52 Self::Data(_, segments) => Some(segments),
53 }
54 }
55
56 pub fn into_bytes(self) -> Option<Bytes> {
58 match self {
59 Self::Ack(_) => None,
60 Self::Data(_, segments) => Some(segments.into_bytes()),
61 }
62 }
63
64 pub fn into_segments(self) -> Option<Segments> {
66 match self {
67 Self::Ack(_) => None,
68 Self::Data(_, segments) => Some(segments),
69 }
70 }
71}
72
73struct Inflight<M> {
74 token: Token,
75 chain: ManuallyDrop<OwnedChain<M>>,
78}
79
80struct InflightTable<M> {
86 by_id: Vec<u16>,
87 live: Vec<Inflight<M>>,
88}
89
90impl<M> InflightTable<M> {
91 const VACANT: u16 = u16::MAX;
92
93 fn new(ring_len: usize) -> Self {
94 Self {
95 by_id: vec![Self::VACANT; ring_len],
96 live: Vec::new(),
97 }
98 }
99
100 fn try_reserve_one(&mut self) -> Result<(), VirtqError> {
101 if self.live.len() > self.by_id.len() {
102 return Err(VirtqError::InvalidState);
103 }
104
105 if self.live.len() == self.by_id.len() {
106 return Err(VirtqError::Backpressure);
107 }
108
109 let result = if self.live.capacity() == 0 {
112 self.live.try_reserve_exact(1)
113 } else {
114 self.live.try_reserve(1)
115 };
116
117 result.map_err(|_| VirtqError::Bookkeeping)
118 }
119
120 fn contains(&self, id: u16) -> bool {
121 self.by_id
122 .get(id as usize)
123 .is_some_and(|slot| *slot != Self::VACANT)
124 }
125
126 fn insert(&mut self, inflight: Inflight<M>) {
127 let id = inflight.token.id;
128 debug_assert!(!self.contains(id));
129 debug_assert!(self.live.len() < Self::VACANT as usize);
130
131 let slot = self.live.len() as u16;
132 self.live.push(inflight);
133 self.by_id[id as usize] = slot;
134 }
135
136 fn remove(&mut self, id: u16) -> Option<Inflight<M>> {
137 let slot = self.by_id.get_mut(id as usize)?;
138 if *slot == Self::VACANT {
139 return None;
140 }
141
142 let index = usize::from(*slot);
143 *slot = Self::VACANT;
144
145 let removed = self.live.swap_remove(index);
146 if let Some(moved) = self.live.get(index) {
147 self.by_id[moved.token.id as usize] = index as u16;
148 }
149
150 Some(removed)
151 }
152
153 fn pop(&mut self) -> Option<Inflight<M>> {
154 let inflight = self.live.pop()?;
155 self.by_id[inflight.token.id as usize] = Self::VACANT;
156 Some(inflight)
157 }
158}
159
160pub struct VirtqProducer<M, N> {
197 inner: RingProducer<M>,
198 notifier: N,
199 pool: SlotPool,
200 next_token: u32,
201 inflight: InflightTable<M>,
202 pending: VecDeque<UsedChain>,
203}
204
205impl<M, N> VirtqProducer<M, N>
206where
207 M: MemOps + Clone,
208 N: Notifier,
209{
210 pub fn new(layout: Layout, mem: M, notifier: N, pool: SlotPool) -> Self {
219 let inner = RingProducer::new(layout, mem);
220 let ring_len = inner.len();
221 let inflight = InflightTable::new(ring_len);
222
223 Self {
224 inner,
225 pool,
226 notifier,
227 inflight,
228 next_token: 0,
229 pending: VecDeque::new(),
230 }
231 }
232
233 pub fn pool(&self) -> &SlotPool {
235 &self.pool
236 }
237
238 pub fn memory(&self) -> &M {
240 self.inner.mem()
241 }
242
243 pub unsafe fn reclaim_slots(&mut self) -> Result<(), VirtqError> {
251 if !self.pending.is_empty() {
252 return Err(VirtqError::InvalidState);
253 }
254
255 let reserved = self
256 .inflight
257 .live
258 .iter()
259 .flat_map(|inflight| inflight.chain.buffers.iter().map(|buffer| buffer.addr));
260
261 unsafe { self.pool.retire_generation(reserved)? };
264 Ok(())
265 }
266
267 pub fn chain(&self) -> ChainBuilder<M> {
272 ChainBuilder::new(
273 self.inner.mem().clone(),
274 self.pool.clone(),
275 self.inner.num_free(),
276 )
277 }
278
279 pub fn preferred_segment_len(&self) -> usize {
281 self.pool.slot_size()
282 }
283
284 pub fn batch(&mut self) -> SubmitBatch<'_, M, N> {
291 SubmitBatch::new(self)
292 }
293
294 pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
307 let cursor_before = self.inner.avail_cursor();
308 let token = self.publish(chain)?;
309 self.notify_since(cursor_before)?;
310 Ok(token)
311 }
312
313 fn publish(&mut self, send: SendChain<M>) -> Result<Token, VirtqError> {
314 self.inflight.try_reserve_one()?;
315
316 if send.desc_count() > self.inner.num_free() {
317 return Err(VirtqError::Backpressure);
318 }
319 let token_id = self.next_token;
320 let id = self.inner.next_id()?;
321 let token = Token { seq: token_id, id };
322
323 if self.inflight.contains(id) {
325 return Err(VirtqError::InvalidState);
326 }
327
328 let mut descriptors = send.owned.descriptors();
329 let mut builder = BufferChainBuilder::new();
330
331 builder.reserve_exact(send.desc_count());
332 let chain = builder
333 .readables(descriptors.by_ref().take(send.rd_desc_count()))
334 .writables(descriptors)
335 .build()?;
336
337 let published = self.inner.submit_available(&chain);
338
339 let inf = Inflight {
342 token,
343 chain: ManuallyDrop::new(send.owned),
344 };
345 self.inflight.insert(inf);
346 self.next_token = self.next_token.wrapping_add(1);
347
348 let published_id = published?;
349 debug_assert_eq!(published_id, id);
350 Ok(token)
351 }
352
353 fn notify_since(&mut self, cursor: RingCursor) -> Result<bool, VirtqError> {
354 let should_notify = self.inner.should_notify_since(cursor)?;
355 if should_notify {
356 self.notify_now();
357 }
358 Ok(should_notify)
359 }
360
361 fn notify_now(&self) {
362 self.notifier.notify(QueueStats {
363 num_free: self.inner.num_free(),
364 num_inflight: self.inner.num_inflight(),
365 });
366 }
367
368 #[inline]
373 pub fn notify_backpressure(&self) {
374 self.notify_now();
375 }
376
377 #[inline]
381 pub fn used_cursor(&self) -> RingCursor {
382 self.inner.used_cursor()
383 }
384
385 #[inline]
387 pub fn num_free(&self) -> usize {
388 self.inner.num_free()
389 }
390
391 #[inline]
393 pub fn num_inflight(&self) -> usize {
394 self.inner.num_inflight()
395 }
396
397 pub unsafe fn reset(&mut self) -> Result<(), VirtqError> {
416 if !self.pending.is_empty() {
417 return Err(VirtqError::InvalidState);
418 }
419
420 self.inner.reset()?;
421 self.next_token = 0;
422
423 let mut maybe_err = None;
424
425 while let Some(inflight) = self.inflight.pop() {
426 let ret = ManuallyDrop::into_inner(inflight.chain).release();
427 if let Err(err) = ret
428 && maybe_err.is_none()
429 {
430 maybe_err = Some(err);
431 }
432 }
433
434 match maybe_err {
435 Some(error) => Err(error.into()),
436 None => Ok(()),
437 }
438 }
439
440 pub fn set_used_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
460 match kind {
461 SuppressionKind::Enable => self.inner.enable_used_notifications()?,
462 SuppressionKind::Disable => self.inner.disable_used_notifications()?,
463 SuppressionKind::Descriptor(cursor) => self
464 .inner
465 .enable_used_notifications_desc(cursor.head(), cursor.wrap())?,
466 }
467 Ok(())
468 }
469}
470
471impl<M, N> VirtqProducer<M, N>
472where
473 M: BufferMap + Clone,
474 N: Notifier,
475{
476 pub fn poll(&mut self) -> Result<Option<UsedChain>, VirtqError> {
492 if let Some(chain) = self.pending.pop_front() {
493 return Ok(Some(chain));
494 }
495 self.poll_ring()
496 }
497
498 pub fn reclaim(&mut self) -> Result<usize, VirtqError> {
510 let mut count = 0;
511 while let Some(chain) = self.poll_ring()? {
512 if matches!(chain, UsedChain::Data(_, _)) {
513 self.pending
514 .try_reserve(1)
515 .map_err(|_| VirtqError::Bookkeeping)?;
516 self.pending.push_back(chain);
517 }
518 count += 1;
519 }
520 Ok(count)
521 }
522
523 fn poll_ring(&mut self) -> Result<Option<UsedChain>, VirtqError> {
525 let used = match self.inner.poll_used() {
526 Ok(u) => u,
527 Err(RingError::WouldBlock) => return Ok(None),
528 Err(e) => return Err(e.into()),
529 };
530
531 let inf = self
532 .inflight
533 .remove(used.id)
534 .ok_or(VirtqError::InvalidState)?;
535
536 let written = used.len as usize;
537 let Inflight { token, chain } = inf;
538
539 let chain = ManuallyDrop::into_inner(chain);
540 let used = if chain.readable == chain.buffers.len() {
541 chain.release()?;
542 UsedChain::Ack(token)
543 } else {
544 UsedChain::Data(token, chain.into_segments(written)?)
545 };
546
547 Ok(Some(used))
548 }
549
550 pub fn drain(&mut self, mut f: impl FnMut(UsedChain)) -> Result<(), VirtqError> {
567 while let Some(chain) = self.poll()? {
568 f(chain);
569 }
570
571 Ok(())
572 }
573}
574
575#[must_use = "finish the batch explicitly"]
581pub struct SubmitBatch<'a, M, N> {
582 producer: &'a mut VirtqProducer<M, N>,
583 notify_from: Option<RingCursor>,
584}
585
586impl<'a, M, N> SubmitBatch<'a, M, N>
587where
588 M: MemOps + Clone,
589 N: Notifier,
590{
591 fn new(producer: &'a mut VirtqProducer<M, N>) -> Self {
592 Self {
593 producer,
594 notify_from: None,
595 }
596 }
597
598 pub fn chain(&self) -> ChainBuilder<M> {
600 self.producer.chain()
601 }
602
603 pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
605 let cursor_before = self.producer.inner.avail_cursor();
606 let token = self.producer.publish(chain)?;
607
608 if self.notify_from.is_none() {
609 self.notify_from = Some(cursor_before);
610 }
611 Ok(token)
612 }
613
614 pub fn finish(mut self) -> Result<bool, VirtqError> {
619 let Some(notify_from) = self.notify_from.take() else {
620 return Ok(false);
621 };
622
623 self.producer.notify_since(notify_from)
624 }
625
626 pub fn finish_without_notify(self) {}
631}
632
633#[must_use = "call .build() to create a SendChain"]
638pub struct ChainBuilder<M: MemOps> {
639 mem: M,
640 pool: SlotPool,
641 rd_caps: SmallVec<[usize; 4]>,
642 wr_caps: SmallVec<[usize; 4]>,
643 writable_avail: bool,
644 max_descs: usize,
645}
646
647impl<M: MemOps> ChainBuilder<M> {
648 fn new(mem: M, pool: SlotPool, max_descs: usize) -> Self {
649 Self {
650 mem,
651 pool,
652 rd_caps: SmallVec::new(),
653 wr_caps: SmallVec::new(),
654 writable_avail: false,
655 max_descs,
656 }
657 }
658
659 pub fn readable(mut self, cap: usize) -> Self {
665 self.rd_caps.push(cap);
666 self
667 }
668
669 pub fn writable(mut self, cap: usize) -> Self {
678 self.wr_caps.push(cap);
679 self
680 }
681
682 pub fn writable_avail(mut self) -> Self {
687 self.writable_avail = true;
688 self
689 }
690
691 pub fn build(self) -> Result<SendChain<M>, VirtqError> {
700 if self.rd_caps.is_empty() && self.wr_caps.is_empty() && !self.writable_avail {
701 return Err(VirtqError::InvalidState);
702 }
703
704 let slot_size = self.pool.slot_size();
706 let rd_capacity = self.rd_caps.iter().try_fold(0usize, |total, &cap| {
707 total.checked_add(cap).ok_or(AllocError::Overflow)
708 })?;
709
710 let mut caps = self.rd_caps.iter().chain(&self.wr_caps);
711 let desc_count = caps.try_fold(0usize, |total, &cap| {
712 if cap == 0 {
713 return Err(AllocError::InvalidArg);
714 }
715 total
716 .checked_add(cap.div_ceil(slot_size))
717 .ok_or(AllocError::Overflow)
718 })?;
719
720 let remaining_descs = self
721 .max_descs
722 .checked_sub(desc_count)
723 .ok_or(VirtqError::Backpressure)?;
724
725 let mut buffers = SmallVec::new();
727 buffers
728 .try_reserve_exact(desc_count)
729 .map_err(|_| VirtqError::Bookkeeping)?;
730
731 let mut owned = OwnedChain {
732 mem: self.mem,
733 pool: self.pool,
734 buffers,
735 readable: 0,
736 };
737
738 let regions = self
740 .rd_caps
741 .iter()
742 .map(|&len| (len, false))
743 .chain(self.wr_caps.iter().map(|&len| (len, true)));
744
745 for (total_len, wr) in regions {
746 let mut remaining = total_len;
747
748 while remaining > 0 {
749 let len = remaining.min(slot_size);
750 let alloc = owned.pool.alloc(len)?;
751 let capacity = if wr { alloc.len } else { len as u32 };
752
753 let ent = BufferEntry {
754 addr: alloc.addr,
755 capacity,
756 written: 0,
757 };
758
759 owned.buffers.push(ent);
760 owned.readable += usize::from(!wr);
761
762 remaining -= len;
763 }
764 }
765
766 if self.writable_avail {
768 let extra = owned.pool.num_free_upper().min(remaining_descs);
769 owned
770 .buffers
771 .try_reserve_exact(extra)
772 .map_err(|_| VirtqError::Bookkeeping)?;
773
774 for _ in 0..extra {
775 let alloc = owned.pool.alloc(slot_size)?;
776 let ent = BufferEntry {
777 addr: alloc.addr,
778 capacity: alloc.len,
779 written: 0,
780 };
781
782 owned.buffers.push(ent);
783 }
784 }
785
786 if owned.buffers.is_empty() {
788 return Err(VirtqError::Backpressure);
789 }
790
791 Ok(SendChain {
792 owned,
793 rd_capacity,
794 rd_written: 0,
795 write_mode: WriteMode::Unset,
796 })
797 }
798}
799
800#[derive(Debug, Clone, Copy, PartialEq, Eq)]
808enum WriteMode {
809 Unset,
810 Append,
811 Direct,
812}
813
814#[must_use = "dropping without submitting deallocates the buffers"]
836pub struct SendChain<M> {
837 owned: OwnedChain<M>,
838 rd_capacity: usize,
839 rd_written: usize,
840 write_mode: WriteMode,
841}
842
843impl<M: MemOps> SendChain<M> {
844 fn note_write_mode(&mut self, mode: WriteMode) {
847 debug_assert!(
848 self.write_mode == WriteMode::Unset || self.write_mode == mode,
849 "SendChain mixes copy writes (write/write_all) with direct writes (write_seg/with_seg)"
850 );
851 self.write_mode = mode;
852 }
853
854 #[inline]
856 pub fn desc_count(&self) -> usize {
857 self.owned.buffers.len()
858 }
859
860 #[inline]
862 pub fn rd_desc_count(&self) -> usize {
863 self.owned.readable
864 }
865
866 #[inline]
868 pub fn wr_desc_count(&self) -> usize {
869 self.desc_count() - self.rd_desc_count()
870 }
871
872 #[inline]
874 pub fn capacity(&self) -> usize {
875 self.rd_capacity
876 }
877
878 #[inline]
880 pub fn written(&self) -> usize {
881 self.rd_written
882 }
883
884 #[inline]
886 pub fn remaining(&self) -> usize {
887 self.capacity() - self.written()
888 }
889
890 pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
903 if self.rd_desc_count() == 0 {
904 return Err(VirtqError::NoPayloadSegment);
905 }
906
907 self.note_write_mode(WriteMode::Append);
908
909 let mut remaining = &buf[..buf.len().min(self.remaining())];
910 let mut written = 0;
911
912 for buffer in &mut self.owned.buffers[..self.owned.readable] {
913 if remaining.is_empty() {
914 break;
915 }
916
917 let cap = buffer.capacity as usize;
918 let desc_off = buffer.written as usize;
919 let len = (cap - desc_off).min(remaining.len());
920 if len == 0 {
921 continue;
922 }
923
924 let addr = buffer
925 .addr
926 .checked_add(desc_off as u64)
927 .ok_or(VirtqError::MemoryWriteError)?;
928
929 self.owned
930 .mem
931 .write(addr, &remaining[..len])
932 .map_err(|_| VirtqError::MemoryWriteError)?;
933
934 buffer.written += len as u32;
935 self.rd_written += len;
936 written += len;
937 remaining = &remaining[len..];
938 }
939
940 Ok(written)
941 }
942
943 #[inline]
955 pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
956 if self.rd_desc_count() == 0 {
957 return Err(VirtqError::NoPayloadSegment);
958 }
959
960 if buf.len() > self.remaining() {
961 return Err(VirtqError::PayloadTooLarge {
962 recv: buf.len(),
963 limit: self.remaining(),
964 });
965 }
966
967 let written = self.write(buf)?;
968 debug_assert_eq!(written, buf.len());
969 Ok(self)
970 }
971
972 pub fn write_seg(&mut self, index: usize, buf: &[u8]) -> Result<&mut Self, VirtqError> {
983 self.note_write_mode(WriteMode::Direct);
984
985 let buffer = self.owned.buffers[..self.owned.readable]
986 .get_mut(index)
987 .ok_or(VirtqError::NoPayloadSegment)?;
988
989 let cap = buffer.capacity as usize;
990 if buf.len() > cap {
991 return Err(VirtqError::PayloadTooLarge {
992 recv: buf.len(),
993 limit: cap,
994 });
995 }
996
997 self.owned
998 .mem
999 .write(buffer.addr, buf)
1000 .map_err(|_| VirtqError::MemoryWriteError)?;
1001
1002 let previous = buffer.written as usize;
1003 buffer.written = checked_descriptor_len(buf.len())?;
1004 self.rd_written = self.rd_written - previous + buf.len();
1005 Ok(self)
1006 }
1007
1008 pub fn with_seg<E>(
1019 &mut self,
1020 index: usize,
1021 f: impl FnOnce(&mut [u8]) -> Result<usize, E>,
1022 ) -> Result<&mut Self, E>
1023 where
1024 E: From<VirtqError>,
1025 {
1026 self.note_write_mode(WriteMode::Direct);
1027
1028 let buffer = self.owned.buffers[..self.owned.readable]
1029 .get_mut(index)
1030 .ok_or_else(|| E::from(VirtqError::NoPayloadSegment))?;
1031
1032 let cap = buffer.capacity as usize;
1033
1034 let buf = unsafe {
1036 self.owned
1037 .mem
1038 .as_mut_slice(buffer.addr, cap)
1039 .map_err(|_| E::from(VirtqError::MemoryWriteError))?
1040 };
1041
1042 let written = f(buf)?;
1043 if written > buf.len() {
1044 return Err(E::from(VirtqError::PayloadTooLarge {
1045 recv: written,
1046 limit: buf.len(),
1047 }));
1048 }
1049
1050 let previous = buffer.written as usize;
1051 buffer.written = checked_descriptor_len(written).map_err(E::from)?;
1052 self.rd_written = self.rd_written - previous + written;
1053
1054 Ok(self)
1055 }
1056}
1057
1058#[derive(Debug)]
1062struct BufferEntry {
1063 addr: u64,
1065 capacity: u32,
1067 written: u32,
1069}
1070
1071struct OwnedChain<M> {
1075 mem: M,
1077 pool: SlotPool,
1079 buffers: SmallVec<[BufferEntry; 4]>,
1081 readable: usize,
1083}
1084
1085impl<M> OwnedChain<M> {
1086 fn descriptors(&self) -> impl ExactSizeIterator<Item = BufferElement> + Clone + '_ {
1090 self.buffers.iter().enumerate().map(|(i, buf)| {
1091 let wr = i >= self.readable;
1092 let len = if wr { buf.capacity } else { buf.written };
1093
1094 BufferElement {
1095 addr: buf.addr,
1096 len,
1097 writable: wr,
1098 }
1099 })
1100 }
1101
1102 fn release(mut self) -> Result<(), AllocError> {
1103 self.release_all()
1104 }
1105
1106 fn release_all(&mut self) -> Result<(), AllocError> {
1107 let mut maybe_err = None;
1108 while let Some(buf) = self.buffers.pop() {
1109 if let Err(error) = self.pool.dealloc(buf.addr)
1110 && maybe_err.is_none()
1111 {
1112 maybe_err = Some(error);
1113 }
1114 }
1115
1116 self.readable = 0;
1117 maybe_err.map_or(Ok(()), Err)
1118 }
1119}
1120
1121impl<M: BufferMap> OwnedChain<M> {
1122 fn into_segments(mut self, written: usize) -> Result<Segments, VirtqError> {
1123 let mut remaining = written;
1124 let mut nonempty = 0;
1125
1126 for buf in &mut self.buffers[self.readable..] {
1127 let len = remaining.min(buf.capacity as usize);
1128 buf.written = len as u32;
1129
1130 nonempty += usize::from(len != 0);
1131 remaining -= len;
1132 }
1133
1134 if remaining != 0 {
1135 self.release()?;
1136 return Err(VirtqError::InvalidState);
1137 }
1138
1139 let mut segments = SmallVec::<[Bytes; 4]>::new();
1140 segments
1141 .try_reserve_exact(nonempty)
1142 .map_err(|_| VirtqError::Bookkeeping)?;
1143
1144 while let Some(buf) = self.buffers.last() {
1145 if self.buffers.len() <= self.readable || buf.written == 0 {
1146 let addr = buf.addr;
1147 self.buffers.pop();
1148 self.pool.dealloc(addr)?;
1149
1150 continue;
1151 }
1152
1153 let alloc = Allocation {
1154 addr: buf.addr,
1155 len: buf.capacity,
1156 };
1157
1158 let written = buf.written as usize;
1159 let lease = BufferLease::new(self.pool.clone(), alloc);
1160 self.buffers.pop();
1161
1162 let mapping = unsafe { self.mem.map_buffer(lease, written) }
1165 .map_err(|_| VirtqError::MemoryReadError)?;
1166
1167 segments.push(Bytes::from_owner(mapping));
1168 }
1169
1170 segments.reverse();
1171 Ok(Segments::from_smallvec(segments))
1172 }
1173}
1174
1175impl<M> Drop for OwnedChain<M> {
1176 fn drop(&mut self) {
1177 if let Err(error) = self.release_all() {
1179 log::error!("Failed to release virtqueue buffers: {error}");
1180 debug_assert!(false, "OwnedChain deallocation failed: {error}");
1181 }
1182 }
1183}
1184
1185fn checked_descriptor_len(len: usize) -> Result<u32, VirtqError> {
1186 if len > u32::MAX as usize {
1187 return Err(VirtqError::PayloadTooLarge {
1188 recv: len,
1189 limit: u32::MAX as usize,
1190 });
1191 }
1192 Ok(len as u32)
1193}
1194
1195#[cfg(test)]
1196mod tests {
1197 use alloc::rc::Rc;
1198 use alloc::sync::Arc;
1199 use core::sync::atomic::Ordering;
1200
1201 use super::*;
1202 use crate::virtq::ring::tests::{FaultMem, OwnedRing, TestMem, make_consumer, make_ring};
1203 use crate::virtq::test_utils::*;
1204
1205 fn poll_received<M: MemOps + Clone, N: Notifier>(
1206 consumer: &mut VirtqConsumer<M, N>,
1207 ) -> (RecvChain<M>, ReplyChain<M>) {
1208 consumer.poll(1024).unwrap().unwrap()
1209 }
1210
1211 #[derive(Clone)]
1212 struct CopyingMem<'a>(&'a Rc<TestMem>);
1213
1214 unsafe impl MemOps for CopyingMem<'_> {
1216 type Error = core::convert::Infallible;
1217
1218 fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1219 self.0.read(addr, dst)
1220 }
1221
1222 fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1223 self.0.write(addr, src)
1224 }
1225
1226 fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1227 self.0.load_acquire(addr)
1228 }
1229
1230 fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1231 self.0.store_release(addr, val)
1232 }
1233
1234 unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1235 unsafe { self.0.as_slice(addr, len) }
1237 }
1238
1239 unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1240 unsafe { self.0.as_mut_slice(addr, len) }
1242 }
1243 }
1244
1245 impl BufferMap for CopyingMem<'_> {
1246 type Mapping = Vec<u8>;
1247
1248 unsafe fn map_buffer(
1249 &self,
1250 lease: BufferLease,
1251 written: usize,
1252 ) -> Result<Self::Mapping, Self::Error> {
1253 assert!(written <= lease.allocation().len as usize);
1254 let mut bytes = vec![0; written];
1255 self.read(lease.allocation().addr, &mut bytes)?;
1256 Ok(bytes)
1257 }
1258 }
1259
1260 fn make_virtq_pair(
1261 ring: &OwnedRing,
1262 slot_size: usize,
1263 ) -> (
1264 VirtqProducer<TestMem, TestNotifier>,
1265 VirtqConsumer<TestMem, TestNotifier>,
1266 ) {
1267 let mem = ring.mem();
1268 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1269
1270 let lower = SlotLayout::new(pool_base, slot_size / 2, ring.len()).unwrap();
1271 let upper = SlotLayout::new(lower.end_addr(), slot_size, ring.len()).unwrap();
1272 let pool = SlotPool::new_tiered(lower, upper).unwrap();
1273
1274 let notifier = TestNotifier::new();
1275 let producer = VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool);
1276 let consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1277 (producer, consumer)
1278 }
1279
1280 fn make_chain(
1281 lower_count: usize,
1282 upper_count: usize,
1283 slot_size: usize,
1284 max_descs: usize,
1285 ) -> ChainBuilder<TestMem> {
1286 let lower_len = lower_count * 256;
1287 let mem = TestMem::new(lower_len + upper_count * slot_size);
1288 let upper =
1289 SlotLayout::new(mem.base_addr() + lower_len as u64, slot_size, upper_count).unwrap();
1290
1291 let pool = if lower_count == 0 {
1292 SlotPool::new(upper)
1293 } else {
1294 let lower = SlotLayout::new(mem.base_addr(), 256, lower_count).unwrap();
1295 SlotPool::new_tiered(lower, upper)
1296 }
1297 .unwrap();
1298
1299 ChainBuilder::new(mem, pool, max_descs)
1300 }
1301
1302 fn inflight(seq: u32, id: u16) -> Inflight<TestMem> {
1303 let mem = TestMem::new(8);
1304 let pool_layout = SlotLayout::new(mem.base_addr(), 8, 1).unwrap();
1305 let pool = SlotPool::new(pool_layout).unwrap();
1306 let chain = ChainBuilder::new(mem, pool, 1).readable(8).build().unwrap();
1307 Inflight {
1308 token: Token { seq, id },
1309 chain: ManuallyDrop::new(chain.owned),
1310 }
1311 }
1312
1313 #[test]
1314 fn inflight_table_repairs_moved_entry_after_removal() {
1315 let mut table = InflightTable::new(16);
1316 for (seq, id) in [(0, 3), (1, 7), (2, 5)] {
1317 table.try_reserve_one().unwrap();
1318 table.insert(inflight(seq, id));
1319 }
1320
1321 let removed = table.remove(7).unwrap();
1322 assert_eq!(removed.token.seq, 1);
1323
1324 ManuallyDrop::into_inner(removed.chain).release().unwrap();
1325 assert!(!table.contains(7));
1326
1327 for (id, seq) in [(5, 2), (3, 0)] {
1328 let removed = table.remove(id).unwrap();
1329 assert_eq!(removed.token.seq, seq);
1330 ManuallyDrop::into_inner(removed.chain).release().unwrap();
1331 }
1332 assert!(table.live.is_empty());
1333 assert!(table.remove(7).is_none());
1334 }
1335
1336 #[test]
1337 fn producer_bookkeeping_starts_compact_and_lazy() {
1338 let ring = make_ring(64);
1339 let (producer, _consumer, _notifier) = make_test_producer(&ring);
1340
1341 assert_eq!(producer.inflight.by_id.len(), ring.len());
1342 assert!(producer.inflight.live.is_empty());
1343 assert_eq!(producer.inflight.live.capacity(), 0);
1344 assert_eq!(producer.pending.capacity(), 0);
1345 }
1346
1347 #[test]
1348 fn full_ring_still_reports_backpressure() {
1349 let ring = make_ring(4);
1350 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
1351
1352 for _ in 0..ring.len() {
1353 let chain = producer.chain().readable(1).build().unwrap();
1354 producer.submit(chain).unwrap();
1355 }
1356
1357 assert!(matches!(
1358 producer.chain().readable(1).build(),
1359 Err(VirtqError::Backpressure)
1360 ));
1361 unsafe { producer.reset() }.unwrap();
1363 }
1364
1365 #[test]
1366 fn submission_rechecks_capacity_after_reservation() {
1367 let ring = make_ring(2);
1368 let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1369 let pool = producer.pool.clone();
1370
1371 let chain = producer.chain().writable(64).build().unwrap();
1372 for _ in 0..ring.len() {
1373 let other = producer.chain().writable(32).build().unwrap();
1374 producer.submit(other).unwrap();
1375 }
1376
1377 assert_eq!(pool.num_live(), 3);
1378 assert!(matches!(
1379 producer.submit(chain),
1380 Err(VirtqError::Backpressure)
1381 ));
1382 assert_eq!(pool.num_live(), 2);
1383
1384 unsafe { producer.reset() }.unwrap();
1386 consumer.reset().unwrap();
1387
1388 assert_eq!(pool.num_live(), 0);
1389 }
1390
1391 #[test]
1392 fn publication_failure_keeps_allocations_until_stopped_reset() {
1393 for failed_write in 0..4 {
1394 let ring = make_ring(4);
1395 let orig_mem = FaultMem::new(ring.mem());
1396 let orig_gen = Arc::downgrade(&orig_mem.0);
1397
1398 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1399 let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1400 let pool = SlotPool::new(pool_layout).unwrap();
1401 let notif = TestNotifier::new();
1402
1403 let source = VirtqProducer::new(ring.layout(), orig_mem, notif.clone(), pool.clone());
1404
1405 let chain = source.chain().writable(64).build().unwrap();
1406 drop(source);
1407
1408 let mem = FaultMem::new(ring.mem());
1409 let mut producer =
1410 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1411
1412 mem.fail_write_at(failed_write);
1413
1414 let res = producer.submit(chain);
1415 assert!(matches!(
1416 res,
1417 Err(VirtqError::RingError(RingError::MemError { .. }))
1418 ));
1419 assert_eq!(pool.num_live(), 1);
1420 assert_eq!(producer.inflight.live.len(), 1);
1421 assert!(orig_gen.upgrade().is_some());
1422
1423 mem.allow_writes();
1424 unsafe { producer.reset() }.unwrap();
1426
1427 assert_eq!(pool.num_live(), 0);
1428 assert!(orig_gen.upgrade().is_none());
1429 }
1430 }
1431
1432 #[test]
1433 fn chain_clones_pool_only_for_independent_owners() {
1434 let ring = make_ring(8);
1435 let mem = ring.mem();
1436 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1437 let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1438 let pool = SlotPool::new(pool_layout).unwrap();
1439
1440 let notif = TestNotifier::new();
1441 let mapping = FaultMem::new(mem.clone());
1442
1443 let mut producer =
1444 VirtqProducer::new(ring.layout(), mapping.clone(), notif.clone(), pool.clone());
1445
1446 let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1447
1448 assert_eq!(pool.strong_count(), 2);
1449
1450 let mut chain = producer
1451 .chain()
1452 .readable(192)
1453 .writable(192)
1454 .build()
1455 .unwrap();
1456
1457 chain.write_all(b"request").unwrap();
1458
1459 assert_eq!(pool.num_live(), 6);
1460 assert_eq!(pool.strong_count(), 3);
1461 assert!(chain.owned.buffers.spilled());
1462
1463 let buffers = chain.owned.buffers.as_ptr();
1464 producer.submit(chain).unwrap();
1465
1466 assert_eq!(pool.strong_count(), 3);
1467 assert_eq!(producer.inflight.live[0].chain.buffers.as_ptr(), buffers);
1468
1469 let (recv, reply) = poll_received(&mut consumer);
1470 let ReplyChain::Writable(mut reply) = reply else {
1471 panic!("expected writable reply");
1472 };
1473 reply.write_all(&[0xa5; 70]).unwrap();
1474 consumer.complete(recv, reply).unwrap();
1475
1476 let segments = producer.poll().unwrap().unwrap().into_segments().unwrap();
1477
1478 assert_eq!(segments.segment_count(), 2);
1479 assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1480 assert_eq!(pool.strong_count(), 4);
1481 assert_eq!(pool.num_live(), 2);
1482
1483 let retained = segments.as_slice()[0].slice(1..);
1484 let cloned = retained.clone();
1485
1486 drop(segments);
1487
1488 assert_eq!(pool.num_live(), 1);
1489 assert_eq!(pool.strong_count(), 3);
1490
1491 unsafe { producer.reset() }.unwrap();
1493 consumer.reset().unwrap();
1494
1495 drop(producer);
1496
1497 assert_eq!(pool.strong_count(), 2);
1498 assert_eq!(retained.as_ref(), &[0xa5; 63]);
1499
1500 drop(retained);
1501
1502 assert_eq!(pool.num_live(), 1);
1503
1504 drop(cloned);
1505
1506 assert_eq!(pool.num_live(), 0);
1507 assert_eq!(pool.strong_count(), 1);
1508 assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1509 }
1510
1511 #[test]
1512 fn copied_mapping_crosses_threads_without_its_pool_or_borrowed_backend() {
1513 let ring = make_ring(4);
1514 let mem = Rc::new(ring.mem());
1515 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1516 let pool_layout = SlotLayout::new(base, 64, 1).unwrap();
1517 let pool = SlotPool::new(pool_layout).unwrap();
1518 let notif = TestNotifier::new();
1519
1520 let mut producer =
1521 VirtqProducer::new(ring.layout(), CopyingMem(&mem), notif.clone(), pool.clone());
1522 let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), notif);
1523
1524 let chain = producer.chain().writable(64).build().unwrap();
1525 producer.submit(chain).unwrap();
1526
1527 let (recv, reply) = poll_received(&mut consumer);
1528 let ReplyChain::Writable(mut reply) = reply else {
1529 panic!("expected writable reply");
1530 };
1531 reply.write_all(b"copied").unwrap();
1532 consumer.complete(recv, reply).unwrap();
1533
1534 let bytes = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1535
1536 assert_eq!(pool.num_live(), 0);
1537 assert_eq!(pool.strong_count(), 2);
1538
1539 let mut reused = producer.chain().readable(64).build().unwrap();
1540 reused.write_all(b"reused").unwrap();
1541
1542 std::thread::spawn(move || assert_eq!(bytes.as_ref(), b"copied"))
1543 .join()
1544 .unwrap();
1545
1546 assert_eq!(pool.num_live(), 1);
1547 assert_eq!(pool.strong_count(), 3);
1548
1549 drop(reused);
1550
1551 assert_eq!(pool.num_live(), 0);
1552 }
1553
1554 #[test]
1555 fn completion_uses_its_original_memory_and_allocating_pool() {
1556 let ring = make_ring(4);
1557 let mem = FaultMem::new(ring.mem());
1558 let orig_gen = Arc::downgrade(&mem.0);
1559 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1560
1561 let notifier = TestNotifier::new();
1562 let original_layout = SlotLayout::new(base, 64, 4).unwrap();
1563 let original = SlotPool::new(original_layout).unwrap();
1564 let source = VirtqProducer::new(ring.layout(), mem, TestNotifier::new(), original.clone());
1565
1566 let chain = source.chain().writable(64).build().unwrap();
1567 drop(source);
1568
1569 assert!(orig_gen.upgrade().is_some());
1570
1571 let replacement_layout = SlotLayout::new(original.base_addr() + 0x1000, 64, 4).unwrap();
1572 let replacement = SlotPool::new(replacement_layout).unwrap();
1573
1574 let replacement_mem = FaultMem::new(ring.mem());
1575 replacement_mem.fail_mapping_at(0);
1576
1577 let mut producer = VirtqProducer::new(
1578 ring.layout(),
1579 replacement_mem.clone(),
1580 notifier.clone(),
1581 replacement.clone(),
1582 );
1583 let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), TestNotifier::new());
1584
1585 producer.submit(chain).unwrap();
1586
1587 assert!(orig_gen.upgrade().is_some());
1588
1589 let (recv, reply) = poll_received(&mut consumer);
1590 let ReplyChain::Writable(mut reply) = reply else {
1591 panic!("expected writable reply");
1592 };
1593
1594 reply.write_all(b"retained").unwrap();
1595 consumer.complete(recv, reply).unwrap();
1596
1597 let data = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1598
1599 assert_eq!(data.as_ref(), b"retained");
1600 assert_eq!(original.num_live(), 1);
1601 assert_eq!(replacement.num_live(), 0);
1602 assert_eq!(replacement_mem.0.map_calls.load(Ordering::Relaxed), 0);
1603
1604 let map_calls = orig_gen
1605 .upgrade()
1606 .unwrap()
1607 .map_calls
1608 .load(Ordering::Relaxed);
1609 assert_eq!(map_calls, 1);
1610
1611 drop(producer);
1612
1613 assert_eq!(data.as_ref(), b"retained");
1614
1615 drop(data);
1616
1617 assert_eq!(original.num_live(), 0);
1618 assert!(orig_gen.upgrade().is_none());
1619 }
1620
1621 #[test]
1622 fn mapping_failure_releases_returned_and_unmapped_slots() {
1623 for fail_at in 0..2 {
1624 let ring = make_ring(8);
1625 let mem = FaultMem::new(ring.mem());
1626 mem.fail_mapping_at(fail_at);
1627
1628 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1629 let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1630 let pool = SlotPool::new(pool_layout).unwrap();
1631 let notif = TestNotifier::new();
1632 let mut producer =
1633 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1634
1635 let mut consumer = make_consumer(&ring);
1636
1637 let chain = producer
1638 .chain()
1639 .readable(192)
1640 .writable(256)
1641 .build()
1642 .unwrap();
1643
1644 let mut addresses: Vec<_> = chain
1645 .owned
1646 .buffers
1647 .iter()
1648 .map(|buffer| buffer.addr)
1649 .collect();
1650
1651 producer.submit(chain).unwrap();
1652
1653 let (id, _) = consumer.poll_available().unwrap();
1654 consumer.submit_used(id, 70).unwrap();
1656
1657 assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
1658 assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), fail_at + 1);
1659 assert_eq!(pool.num_live(), 0);
1660 assert_eq!(pool.strong_count(), 2);
1661 assert_eq!(producer.num_inflight(), 0);
1662
1663 if fail_at == 1 {
1664 addresses.swap(3, 4);
1666 }
1667
1668 let repeated = producer
1669 .chain()
1670 .readable(192)
1671 .writable(256)
1672 .build()
1673 .unwrap();
1674
1675 assert_eq!(
1676 repeated
1677 .owned
1678 .buffers
1679 .iter()
1680 .map(|buffer| buffer.addr)
1681 .collect::<Vec<_>>(),
1682 addresses
1683 );
1684 }
1685 }
1686
1687 #[test]
1688 fn empty_and_malformed_completions_skip_mapping_and_release_every_slot() {
1689 let ring = make_ring(4);
1690 let mem = FaultMem::new(ring.mem());
1691 mem.fail_mapping_at(0);
1692
1693 let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1694 let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1695 let pool = SlotPool::new(pool_layout).unwrap();
1696 let notif = TestNotifier::new();
1697
1698 let mut producer =
1699 VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1700
1701 let mut consumer = make_consumer(&ring);
1702
1703 for (writable, written) in [(false, 0), (true, 0), (true, 129)] {
1704 let builder = producer.chain().readable(64);
1705 let builder = if writable {
1706 builder.writable(128)
1707 } else {
1708 builder
1709 };
1710
1711 let token = producer.submit(builder.build().unwrap()).unwrap();
1712 let (id, _) = consumer.poll_available().unwrap();
1713 consumer.submit_used(id, written).unwrap();
1714
1715 match producer.poll() {
1716 Ok(Some(UsedChain::Ack(returned))) if !writable => assert_eq!(returned, token),
1717 Ok(Some(UsedChain::Data(returned, segments))) if writable && written == 0 => {
1718 assert_eq!(returned, token);
1719 assert_eq!(segments.segment_count(), 0);
1720 }
1721 Err(VirtqError::InvalidState) if written == 129 => {}
1722 other => panic!("unexpected completion: {other:?}"),
1723 }
1724
1725 assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), 0);
1726 assert_eq!(pool.num_live(), 0);
1727 assert_eq!(pool.strong_count(), 2);
1728 assert_eq!(producer.num_inflight(), 0);
1729 }
1730 }
1731
1732 #[test]
1733 fn cancelling_a_chain_preserves_slot_order_after_a_partial_write_error() {
1734 let storage = TestMem::new(16);
1735 let base = storage.base_addr();
1736 let mem = FaultMem::new(storage);
1737 let t1 = SlotLayout::new(base, 2, 2).unwrap();
1738 let t2 = SlotLayout::new(base + 4, 4, 3).unwrap();
1739 let pool = SlotPool::new_tiered(t1, t2).unwrap();
1740
1741 let mut chain = ChainBuilder::new(mem.clone(), pool.clone(), 8)
1742 .readable(9)
1743 .writable_avail()
1744 .build()
1745 .unwrap();
1746
1747 let addresses: Vec<_> = chain
1748 .owned
1749 .buffers
1750 .iter()
1751 .map(|buffer| buffer.addr)
1752 .collect();
1753
1754 mem.fail_write_at(1);
1755 assert!(matches!(
1756 chain.write_all(b"abcdefghi"),
1757 Err(VirtqError::MemoryWriteError)
1758 ));
1759 assert_eq!(chain.written(), 4);
1760
1761 drop(chain);
1762
1763 assert_eq!(pool.num_live(), 0);
1764 assert_eq!(pool.strong_count(), 1);
1765
1766 mem.allow_writes();
1767 let repeated = ChainBuilder::new(mem, pool.clone(), 8)
1768 .readable(9)
1769 .writable_avail()
1770 .build()
1771 .unwrap();
1772
1773 assert_eq!(
1774 repeated
1775 .owned
1776 .buffers
1777 .iter()
1778 .map(|buffer| buffer.addr)
1779 .collect::<Vec<_>>(),
1780 addresses
1781 );
1782 }
1783
1784 #[test]
1785 fn reclaim_buffers_multiple_ring_generations() {
1786 let ring = make_ring(8);
1787 let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1788 let count = ring.len() * 2 + 1;
1789
1790 for _ in 0..count {
1791 let chain = producer.chain().writable(1).build().unwrap();
1792 producer.submit(chain).unwrap();
1793 let (recv, reply) = poll_received(&mut consumer);
1794 consumer.complete(recv, reply).unwrap();
1795 assert_eq!(producer.reclaim().unwrap(), 1);
1796 assert_eq!(producer.num_free(), ring.len());
1797 assert_eq!(producer.pool().num_live(), 0);
1798 }
1799
1800 for seq in 0..count {
1801 let Some(UsedChain::Data(token, segments)) = producer.poll().unwrap() else {
1802 panic!("expected an empty writable completion");
1803 };
1804
1805 assert_eq!(token.seq as usize, seq);
1806 assert!(segments.is_empty());
1807 }
1808
1809 assert!(producer.poll().unwrap().is_none());
1810 }
1811
1812 #[test]
1813 fn reset_reclaims_inflight_slots_and_reuses_ring() {
1814 let ring = make_ring(8);
1815 let mem = ring.mem();
1816 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1817 let pool_layout = SlotLayout::new(pool_base, 64, ring.len()).unwrap();
1818 let pool = SlotPool::new(pool_layout).unwrap();
1819 let notifier = TestNotifier::new();
1820 let mut producer = VirtqProducer::new(ring.layout(), mem, notifier, pool.clone());
1821
1822 for _ in 0..ring.len() {
1823 let chain = producer.chain().writable(64).build().unwrap();
1824 producer.submit(chain).unwrap();
1825 }
1826
1827 assert_eq!(pool.num_free(), 0);
1828
1829 unsafe { producer.reset() }.unwrap();
1831
1832 assert_eq!(producer.num_inflight(), 0);
1833 assert_eq!(producer.num_free(), ring.len());
1834 assert_eq!(pool.num_free(), ring.len());
1835 assert!(producer.inflight.live.is_empty());
1836 assert!(
1837 producer
1838 .inflight
1839 .by_id
1840 .iter()
1841 .all(|slot| *slot == InflightTable::<TestMem>::VACANT)
1842 );
1843
1844 for _ in 0..ring.len() {
1845 let chain = producer.chain().writable(64).build().unwrap();
1846 producer.submit(chain).unwrap();
1847 }
1848
1849 unsafe { producer.reset() }.unwrap();
1851 }
1852
1853 #[test]
1854 fn stopped_reset_reuses_slot_after_consumer_handles_drop() {
1855 let ring = make_ring(4);
1856 let mem = ring.mem();
1857 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1858 let pool_layout = SlotLayout::new(pool_base, 4, 1).unwrap();
1859 let pool = SlotPool::new(pool_layout).unwrap();
1860 let notifier = TestNotifier::new();
1861 let mut producer =
1862 VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool.clone());
1863 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), notifier.clone());
1864
1865 let sent = producer.chain().writable(4).build().unwrap();
1866 producer.submit(sent).unwrap();
1867 let (recv, reply) = poll_received(&mut consumer);
1868 assert!(matches!(reply, ReplyChain::Writable(_)));
1869
1870 drop((recv, reply, consumer));
1871 assert_eq!(pool.num_live(), 1);
1872
1873 unsafe { producer.reset() }.unwrap();
1875 assert_eq!(pool.num_live(), 0);
1876
1877 let mut replacement = producer.chain().readable(4).build().unwrap();
1878 assert_eq!(replacement.owned.buffers[0].addr, pool_base);
1879 replacement.write_all(b"GOOD").unwrap();
1880 producer.submit(replacement).unwrap();
1881
1882 let mut consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1883 let (recv, reply) = poll_received(&mut consumer);
1884 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"GOOD");
1885 consumer.complete(recv, reply).unwrap();
1886 producer.drain(drop).unwrap();
1887 assert_eq!(pool.num_live(), 0);
1888 }
1889
1890 #[test]
1891 fn reset_rejects_buffered_writable_completion() {
1892 let ring = make_ring(8);
1893 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1894 let chain = producer.chain().writable(64).build().unwrap();
1895 producer.submit(chain).unwrap();
1896
1897 let (recv, reply) = poll_received(&mut consumer);
1898 let ReplyChain::Writable(mut reply) = reply else {
1899 panic!("expected writable reply");
1900 };
1901
1902 reply.write_all(b"retained").unwrap();
1903 consumer.complete(recv, reply).unwrap();
1904 producer.reclaim().unwrap();
1905
1906 assert!(matches!(
1908 unsafe { producer.reset() },
1909 Err(VirtqError::InvalidState)
1910 ));
1911
1912 drop(producer.poll().unwrap().unwrap());
1913
1914 unsafe { producer.reset() }.unwrap();
1916 }
1917
1918 #[test]
1919 fn test_chain_readwrite_build() {
1920 let ring = make_ring(16);
1921 let (producer, _consumer, _notifier) = make_test_producer(&ring);
1922
1923 let se = producer.chain().readable(64).writable(128).build().unwrap();
1924 assert_eq!(se.capacity(), 64);
1925 assert_eq!(se.written(), 0);
1926 assert_eq!(se.remaining(), 64);
1927 }
1928
1929 #[test]
1930 fn test_chain_readable_writable_names_build() {
1931 let ring = make_ring(16);
1932 let (producer, _consumer, _notifier) = make_test_producer(&ring);
1933
1934 let se = producer.chain().readable(16).writable(32).build().unwrap();
1935 assert_eq!(se.desc_count(), 2);
1936 assert_eq!(se.rd_desc_count(), 1);
1937 assert_eq!(se.wr_desc_count(), 1);
1938 assert_eq!(se.capacity(), 16);
1939 }
1940
1941 #[test]
1942 fn test_chain_multi_readable_write_all_scatters() {
1943 let ring = make_ring(16);
1944 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1945
1946 let mut se = producer
1947 .chain()
1948 .readable(5)
1949 .readable(6)
1950 .writable(32)
1951 .build()
1952 .unwrap();
1953
1954 se.write_all(b"hello world").unwrap();
1955 assert_eq!(se.written(), 11);
1956
1957 let token = producer.submit(se).unwrap();
1958 let (recv, reply) = poll_received(&mut consumer);
1959 assert_eq!(recv.token(), token);
1960 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
1961
1962 let segments = recv.to_segments().unwrap();
1963 assert_eq!(segments.segment_count(), 2);
1964 assert_eq!(segments.as_slice()[0].as_ref(), b"hello");
1965 assert_eq!(segments.as_slice()[1].as_ref(), b" world");
1966
1967 consumer.complete(recv, reply).unwrap();
1968 producer.drain(drop).unwrap();
1969 }
1970
1971 #[test]
1972 fn test_chain_independent_readables_preserve_pool_tiers() {
1973 let ring = make_ring(16);
1974 let layout = ring.layout();
1975 let mem = ring.mem();
1976
1977 let lower = SlotLayout::new(
1978 mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100,
1979 256,
1980 1,
1981 )
1982 .unwrap();
1983
1984 let upper = SlotLayout::new(lower.end_addr(), 4096, 1).unwrap();
1985 let pool = SlotPool::new_tiered(lower, upper).unwrap();
1986 let notifier = TestNotifier::new();
1987 let producer = VirtqProducer::new(layout, mem, notifier, pool.clone());
1988
1989 let send = producer
1990 .chain()
1991 .readable(128)
1992 .readable(4096)
1993 .build()
1994 .unwrap();
1995
1996 let readables = send.owned.descriptors().collect::<Vec<_>>();
1997
1998 assert_eq!(readables.len(), 2);
1999 assert_eq!(readables[0].addr, lower.base_addr());
2000 assert_eq!(readables[1].addr, upper.base_addr());
2001 assert_eq!(pool.num_free_lower(), 0);
2002 assert_eq!(pool.num_free_upper(), 0);
2003
2004 drop(send);
2005
2006 assert_eq!(pool.num_free_lower(), 1);
2007 assert_eq!(pool.num_free_upper(), 1);
2008 }
2009
2010 #[test]
2011 fn chain_avail_uses_pool_availability_at_build() {
2012 for held_count in [0, 2, 3] {
2013 let builder = make_chain(2, 3, 4096, 3);
2014 let pool = builder.pool.clone();
2015 let builder = builder.writable_avail().readable(128);
2016
2017 assert_eq!(pool.num_live(), 0);
2018
2019 let held: Vec<_> = (0..held_count).map(|_| pool.alloc(4096).unwrap()).collect();
2020
2021 let chain = builder.build().unwrap();
2022
2023 let writable = (3 - held_count).min(2);
2024 assert_eq!(chain.rd_desc_count(), 1);
2025 assert_eq!(chain.wr_desc_count(), writable);
2026 assert_eq!(chain.desc_count(), 1 + writable);
2027 assert_eq!(chain.owned.buffers[0].capacity, 128);
2028 assert!(
2029 chain.owned.buffers[1..]
2030 .iter()
2031 .all(|buf| buf.capacity == 4096)
2032 );
2033 assert_eq!(pool.num_free_lower(), 1);
2034 assert_eq!(pool.num_free_upper(), 3 - held_count - writable);
2035 assert_eq!(pool.num_live(), held_count + chain.desc_count());
2036
2037 drop(chain);
2038
2039 assert_eq!(pool.num_live(), held_count);
2040
2041 for allocation in held.into_iter().rev() {
2042 pool.dealloc(allocation.addr).unwrap();
2043 }
2044
2045 assert_eq!(pool.num_live(), 0);
2046 }
2047 }
2048
2049 #[test]
2050 fn chain_avail_respects_mandatory_descriptor_budget() {
2051 for max_descs in 0..3 {
2052 let builder = make_chain(2, 3, 4096, max_descs);
2053 let pool = builder.pool.clone();
2054
2055 let result = builder
2056 .readable(4096)
2057 .readable(128)
2058 .writable_avail()
2059 .build();
2060
2061 if max_descs == 2 {
2062 let chain = result.unwrap();
2063
2064 assert_eq!(chain.rd_desc_count(), 2);
2065 assert_eq!(chain.wr_desc_count(), 0);
2066
2067 drop(chain);
2068 } else {
2069 assert!(matches!(result, Err(VirtqError::Backpressure)));
2070 }
2071
2072 assert_eq!(pool.num_live(), 0);
2073
2074 let lower = pool.alloc(128).unwrap();
2075 let upper = pool.alloc(4096).unwrap();
2076
2077 assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2078 assert_eq!(upper.addr, pool.slot_addr(4).unwrap());
2079
2080 pool.dealloc(lower.addr).unwrap();
2081 pool.dealloc(upper.addr).unwrap();
2082 }
2083 }
2084
2085 #[test]
2086 fn chain_avail_preserves_explicit_writable_capacity_when_full() {
2087 for (upper_count, max_descs) in [(2, 2), (1, 3)] {
2088 let builder = make_chain(1, upper_count, 4096, max_descs);
2089 let pool = builder.pool.clone();
2090
2091 let chain = builder
2092 .writable_avail()
2093 .writable(128)
2094 .readable(128)
2095 .build()
2096 .unwrap();
2097
2098 assert_eq!(chain.rd_desc_count(), 1);
2099 assert_eq!(chain.wr_desc_count(), 1);
2100 assert_eq!(chain.owned.buffers[0].capacity, 128);
2101 assert_eq!(chain.owned.buffers[1].capacity, 4096);
2102 assert!(chain.owned.buffers.iter().all(|buf| buf.written == 0));
2103 assert_eq!(pool.num_free_upper(), upper_count - 1);
2104
2105 drop(chain);
2106
2107 assert_eq!(pool.num_live(), 0);
2108 }
2109 }
2110
2111 #[test]
2112 fn chain_avail_cannot_build_an_empty_chain() {
2113 for max_descs in [0, 2] {
2114 let builder = make_chain(1, 1, 4096, max_descs);
2115 let pool = builder.pool.clone();
2116 let held = (max_descs != 0).then(|| pool.alloc(4096).unwrap());
2117
2118 assert!(matches!(
2119 builder.writable_avail().build(),
2120 Err(VirtqError::Backpressure)
2121 ));
2122 assert_eq!(pool.num_free_lower(), 1);
2123 assert_eq!(pool.num_live(), usize::from(held.is_some()));
2124
2125 if let Some(allocation) = held {
2126 pool.dealloc(allocation.addr).unwrap();
2127 }
2128 }
2129 }
2130
2131 #[test]
2132 fn chain_avail_only_uses_upper_slots() {
2133 let builder = make_chain(4, 2, 4096, 6);
2134 let pool = builder.pool.clone();
2135
2136 let chain = builder.writable_avail().writable_avail().build().unwrap();
2137
2138 assert_eq!(chain.rd_desc_count(), 0);
2139 assert_eq!(chain.wr_desc_count(), 2);
2140 assert_eq!(pool.num_free_lower(), 4);
2141 assert_eq!(pool.num_free_upper(), 0);
2142
2143 drop(chain);
2144
2145 assert_eq!(pool.num_live(), 0);
2146 }
2147
2148 #[test]
2149 fn chain_preserves_region_order_with_odd_slot_sizes() {
2150 let builder = make_chain(1, 4, 3001, 5);
2151 let pool = builder.pool.clone();
2152
2153 let chain = builder
2155 .readable(128)
2156 .readable(6003)
2157 .writable_avail()
2158 .build()
2159 .unwrap();
2160
2161 assert_eq!(chain.rd_desc_count(), 4);
2162
2163 let actual_offsets = chain
2164 .owned
2165 .buffers
2166 .iter()
2167 .map(|buf| {
2168 (
2169 buf.addr,
2170 buf.capacity,
2171 pool.allocation_len(buf.addr).unwrap(),
2172 )
2173 })
2174 .collect::<Vec<_>>();
2175
2176 let expected_offsets = [
2177 (0, 128, 256),
2178 (4, 3001, 3001),
2179 (3, 3001, 3001),
2180 (2, 1, 3001),
2181 (1, 3001, 3001),
2182 ]
2183 .map(|(slot, capacity, allocated)| (pool.slot_addr(slot).unwrap(), capacity, allocated));
2184
2185 assert_eq!(actual_offsets, expected_offsets);
2186
2187 drop(chain);
2188
2189 assert_eq!(pool.num_live(), 0);
2190 }
2191
2192 #[test]
2193 fn chain_failure_releases_only_its_own_slots() {
2194 let builder = make_chain(2, 3, 4096, 4);
2195 let pool = builder.pool.clone();
2196 let held = pool.alloc(4096).unwrap();
2197
2198 assert!(matches!(
2199 builder.readable(128).writable(4096 * 3).build(),
2200 Err(VirtqError::Backpressure)
2201 ));
2202 assert_eq!(pool.live_addrs(), [held.addr]);
2203
2204 let lower = pool.alloc(128).unwrap();
2205 let upper = pool.alloc(4096).unwrap();
2206 let next_upper = pool.alloc(4096).unwrap();
2207
2208 assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2209 assert_eq!(upper.addr, pool.slot_addr(3).unwrap());
2210 assert_eq!(next_upper.addr, pool.slot_addr(2).unwrap());
2211
2212 pool.dealloc(next_upper.addr).unwrap();
2213 pool.dealloc(upper.addr).unwrap();
2214 pool.dealloc(lower.addr).unwrap();
2215 pool.dealloc(held.addr).unwrap();
2216 }
2217
2218 #[test]
2219 fn chain_rejects_invalid_request_sequences() {
2220 for (readable, writable) in [(vec![128, 0], vec![]), (vec![128], vec![4096, 0])] {
2221 let mut builder = make_chain(1, 1, 4096, 4);
2222 let pool = builder.pool.clone();
2223
2224 for cap in readable {
2225 builder = builder.readable(cap);
2226 }
2227 for cap in writable {
2228 builder = builder.writable(cap);
2229 }
2230
2231 assert!(matches!(
2232 builder.build(),
2233 Err(VirtqError::Alloc(AllocError::InvalidArg))
2234 ));
2235 assert_eq!(pool.num_live(), 0);
2236 }
2237
2238 let builder = make_chain(1, 1, 4096, 4);
2239 let pool = builder.pool.clone();
2240
2241 assert!(matches!(
2242 builder.readable(usize::MAX).readable(1).build(),
2243 Err(VirtqError::Alloc(AllocError::Overflow))
2244 ));
2245 assert_eq!(pool.num_live(), 0);
2246 }
2247
2248 #[test]
2249 fn chain_keeps_four_records_inline_and_one_pool_handle() {
2250 for count in [4, 5] {
2251 let builder = make_chain(0, count, 4096, count);
2252 let pool = builder.pool.clone();
2253
2254 let chain = builder
2255 .readable((count - 1) * 4096)
2256 .writable_avail()
2257 .build()
2258 .unwrap();
2259
2260 assert_eq!(chain.desc_count(), count);
2261 assert_eq!(chain.owned.buffers.spilled(), count > 4);
2262 assert_eq!(pool.strong_count(), 2);
2263
2264 drop(chain);
2265
2266 assert_eq!(pool.num_live(), 0);
2267 }
2268 }
2269
2270 #[test]
2271 fn test_chain_multi_readable_appends_across_calls() {
2272 let ring = make_ring(16);
2273 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2274
2275 let mut send = producer.chain().readable(8).build().unwrap();
2276 send.write_all(b"abc").unwrap();
2277 send.write_all(b"def").unwrap();
2278 assert_eq!(send.written(), 6);
2279 assert_eq!(send.remaining(), 2);
2280
2281 producer.submit(send).unwrap();
2282 let (recv, reply) = poll_received(&mut consumer);
2283 let segments = recv.to_segments().unwrap();
2284 assert_eq!(segments.segment_count(), 2);
2285 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2286 assert_eq!(segments.as_slice()[1].as_ref(), b"ef");
2287 consumer.complete(recv, reply).unwrap();
2288 producer.drain(drop).unwrap();
2289 }
2290
2291 #[test]
2292 fn test_chain_readable_splits_logical_capacity() {
2293 let expected_lengths = [4, 4, 4];
2294 let ring = make_ring(16);
2295 let (mut producer, mut consumer, _) = make_test_producer_with_slot_size(&ring, 4);
2296
2297 let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2298 let readables = &se.owned.buffers[..se.rd_desc_count()];
2299
2300 assert_eq!(se.rd_desc_count(), 3);
2301 assert_eq!(se.capacity(), 10);
2302
2303 let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2304 assert_eq!(caps, [4, 4, 2]);
2305
2306 let lengths = readables
2307 .iter()
2308 .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2309 .collect::<Vec<_>>();
2310 assert_eq!(lengths, expected_lengths);
2311
2312 se.write_all(b"abcdefghij").unwrap();
2313 assert_eq!(se.written(), 10);
2314
2315 let token = producer.submit(se).unwrap();
2316 let (recv, reply) = poll_received(&mut consumer);
2317
2318 assert_eq!(recv.token(), token);
2319 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2320
2321 let segments = recv.to_segments().unwrap();
2322
2323 assert_eq!(segments.segment_count(), 3);
2324 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2325 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2326 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2327
2328 consumer.complete(recv, reply).unwrap();
2329 producer.drain(drop).unwrap();
2330
2331 assert_eq!(producer.pool.num_live(), 0);
2332 }
2333
2334 #[test]
2335 fn test_chain_readable_splits_logical_capacity_tiered() {
2336 let expected_lengths = [4, 4, 2];
2337 let ring = make_ring(16);
2338 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2339
2340 let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2341 let readables = &se.owned.buffers[..se.rd_desc_count()];
2342
2343 assert_eq!(se.rd_desc_count(), 3);
2344 assert_eq!(se.capacity(), 10);
2345
2346 let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2347 assert_eq!(caps, [4, 4, 2]);
2348
2349 let lengths = readables
2350 .iter()
2351 .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2352 .collect::<Vec<_>>();
2353 assert_eq!(lengths, expected_lengths);
2354
2355 se.write_all(b"abcdefghij").unwrap();
2356 assert_eq!(se.written(), 10);
2357
2358 let token = producer.submit(se).unwrap();
2359 let (recv, reply) = poll_received(&mut consumer);
2360
2361 assert_eq!(recv.token(), token);
2362 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2363
2364 let segments = recv.to_segments().unwrap();
2365
2366 assert_eq!(segments.segment_count(), 3);
2367 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2368 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2369 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2370
2371 consumer.complete(recv, reply).unwrap();
2372 producer.drain(drop).unwrap();
2373
2374 assert_eq!(producer.pool.num_live(), 0);
2375 }
2376
2377 #[test]
2378 fn test_chain_readable_rejects_zero_capacity_on_build() {
2379 let ring = make_ring(16);
2380 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2381
2382 assert!(matches!(
2383 producer.chain().readable(0).build(),
2384 Err(VirtqError::Alloc(AllocError::InvalidArg))
2385 ));
2386 }
2387
2388 #[test]
2389 fn test_chain_writable_splits_logical_capacity() {
2390 let ring = make_ring(16);
2391 let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2392
2393 let se = producer.chain().writable(10).build().unwrap();
2394 let token = producer.submit(se).unwrap();
2395
2396 let (recv, reply) = poll_received(&mut consumer);
2397 let ReplyChain::Writable(mut wc) = reply else {
2398 panic!("expected writable reply");
2399 };
2400 assert_eq!(wc.capacity(), 10);
2401
2402 wc.write_all(b"abcdefghij").unwrap();
2403 consumer.complete(recv, wc).unwrap();
2404
2405 let used = producer.poll().unwrap().unwrap();
2406 assert_eq!(used.token(), token);
2407 let segments = used.segments().unwrap();
2408 assert_eq!(segments.segment_count(), 3);
2409 assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2410 assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2411 assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2412 }
2413
2414 #[test]
2415 fn test_chain_writable_rejects_zero_capacity_on_build() {
2416 let ring = make_ring(16);
2417 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2418
2419 assert!(matches!(
2420 producer.chain().writable(0).build(),
2421 Err(VirtqError::Alloc(AllocError::InvalidArg))
2422 ));
2423 }
2424
2425 #[test]
2426 fn test_chain_multi_readable_write_all_preserves_segments() {
2427 let ring = make_ring(16);
2428 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2429
2430 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2431 se.write_all(b"headbody").unwrap();
2432
2433 producer.submit(se).unwrap();
2434 let (recv, reply) = poll_received(&mut consumer);
2435 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"headbody");
2436 consumer.complete(recv, reply).unwrap();
2437 producer.drain(drop).unwrap();
2438 }
2439
2440 #[test]
2441 fn test_chain_payload_segment_writer_serializes_directly() {
2442 let ring = make_ring(16);
2443 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2444
2445 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2446 se.with_seg(0, |segment| {
2447 segment.copy_from_slice(b"head");
2448 Ok::<usize, VirtqError>(4)
2449 })
2450 .unwrap();
2451 se.with_seg(1, |segment| {
2452 segment.copy_from_slice(b"body");
2453 Ok::<usize, VirtqError>(4)
2454 })
2455 .unwrap();
2456
2457 producer.submit(se).unwrap();
2458 let (recv, reply) = poll_received(&mut consumer);
2459 let segments = recv.to_segments().unwrap();
2460 assert_eq!(segments.segment_count(), 2);
2461 assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2462 consumer.complete(recv, reply).unwrap();
2463 producer.drain(drop).unwrap();
2464 }
2465
2466 #[test]
2467 fn test_chain_payload_segment_write_copies_directly() {
2468 let ring = make_ring(16);
2469 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2470
2471 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2472 se.write_seg(0, b"head").unwrap();
2473 se.write_seg(1, b"body").unwrap();
2474
2475 producer.submit(se).unwrap();
2476 let (recv, reply) = poll_received(&mut consumer);
2477 let segments = recv.to_segments().unwrap();
2478 assert_eq!(segments.segment_count(), 2);
2479 assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2480 consumer.complete(recv, reply).unwrap();
2481 producer.drain(drop).unwrap();
2482 }
2483
2484 #[test]
2485 fn test_chain_multi_writable_used_returns_segments() {
2486 let ring = make_ring(16);
2487 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2488
2489 let se = producer.chain().writable(5).writable(6).build().unwrap();
2490 let token = producer.submit(se).unwrap();
2491
2492 let (recv, reply) = poll_received(&mut consumer);
2493 let ReplyChain::Writable(mut wc) = reply else {
2494 panic!("expected writable reply");
2495 };
2496 assert_eq!(wc.capacity(), 12);
2497
2498 wc.write_all(b"hello world").unwrap();
2499 consumer.complete(recv, wc).unwrap();
2500
2501 let used = producer.poll().unwrap().unwrap();
2502 assert_eq!(used.token(), token);
2503 let segments = used.segments().unwrap();
2504 assert_eq!(segments.segment_count(), 2);
2505 assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2506 assert_eq!(segments.as_slice()[1].as_ref(), b"world");
2507 assert_eq!(segments.to_bytes().as_ref(), b"hello world");
2508 }
2509
2510 #[test]
2511 fn test_chain_multi_writable_short_used_truncates_last_segment() {
2512 let ring = make_ring(16);
2513 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2514
2515 let se = producer.chain().writable(5).writable(6).build().unwrap();
2516 producer.submit(se).unwrap();
2517
2518 let (recv, reply) = poll_received(&mut consumer);
2519 let ReplyChain::Writable(mut wc) = reply else {
2520 panic!("expected writable reply");
2521 };
2522
2523 wc.write_all(b"hello wo").unwrap();
2524 consumer.complete(recv, wc).unwrap();
2525
2526 let used = producer.poll().unwrap().unwrap();
2527 let segments = used.segments().unwrap();
2528 assert_eq!(segments.segment_count(), 2);
2529 assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2530 assert_eq!(segments.as_slice()[1].as_ref(), b"wo");
2531 assert_eq!(segments.to_bytes().as_ref(), b"hello wo");
2532 }
2533
2534 #[test]
2535 fn test_chain_multi_writable_zero_used_returns_empty_segments() {
2536 let ring = make_ring(16);
2537 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2538
2539 let se = producer.chain().writable(5).writable(6).build().unwrap();
2540 producer.submit(se).unwrap();
2541
2542 let (recv, reply) = poll_received(&mut consumer);
2543 consumer.complete(recv, reply).unwrap();
2544
2545 let used = producer.poll().unwrap().unwrap();
2546 let segments = used.segments().unwrap();
2547 assert_eq!(segments.segment_count(), 0);
2548 assert!(segments.is_empty());
2549 assert!(segments.to_bytes().is_empty());
2550 }
2551
2552 #[test]
2553 fn test_chain_readable_only_build() {
2554 let ring = make_ring(16);
2555 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2556
2557 let se = producer.chain().readable(32).build().unwrap();
2558 assert_eq!(se.capacity(), 32);
2559 }
2560
2561 #[test]
2562 fn test_chain_writable_only_build() {
2563 let ring = make_ring(16);
2564 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2565
2566 let se = producer.chain().writable(64).build().unwrap();
2567 assert_eq!(se.capacity(), 0);
2568 }
2569
2570 #[test]
2571 fn test_chain_empty_build_fails() {
2572 let ring = make_ring(16);
2573 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2574
2575 let result = producer.chain().build();
2576 assert!(matches!(result, Err(VirtqError::InvalidState)));
2577 }
2578
2579 #[test]
2580 fn test_send_chain_write_all_and_submit() {
2581 let ring = make_ring(16);
2582 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2583
2584 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2585
2586 se.write_all(b"hello")
2587 .unwrap()
2588 .write_all(b" world")
2589 .unwrap();
2590 assert_eq!(se.written(), 11);
2591 assert_eq!(se.remaining(), 53);
2592 let tok = producer.submit(se).unwrap();
2593
2594 let (recv, reply) = poll_received(&mut consumer);
2595 assert_eq!(recv.token(), tok);
2596 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2597 consumer.complete(recv, reply).unwrap();
2598 producer.drain(drop).unwrap();
2599 }
2600
2601 #[test]
2602 fn test_send_payload_write_all_fluent() {
2603 let ring = make_ring(16);
2604 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2605
2606 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2607 se.write_all(b"hello")
2608 .unwrap()
2609 .write_all(b" world")
2610 .unwrap();
2611 assert_eq!(se.written(), 11);
2612 assert_eq!(se.remaining(), 53);
2613 let tok = producer.submit(se).unwrap();
2614
2615 let (recv, reply) = poll_received(&mut consumer);
2616 assert_eq!(recv.token(), tok);
2617 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2618 consumer.complete(recv, reply).unwrap();
2619 producer.drain(drop).unwrap();
2620 }
2621
2622 #[test]
2623 fn test_send_payload_partial_write() {
2624 let ring = make_ring(16);
2625 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2626
2627 let mut se = producer.chain().readable(8).build().unwrap();
2628 let written = se.write(b"hello world").unwrap();
2629 assert_eq!(written, 8);
2630 assert_eq!(se.remaining(), 0);
2631
2632 producer.submit(se).unwrap();
2633 let (recv, reply) = poll_received(&mut consumer);
2634 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello wo");
2635 consumer.complete(recv, reply).unwrap();
2636 producer.drain(drop).unwrap();
2637 }
2638
2639 #[test]
2640 fn test_send_payload_write_with_serializes_directly() {
2641 let ring = make_ring(16);
2642 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2643
2644 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2645 se.with_seg(0, |buf| {
2646 buf[..5].copy_from_slice(b"hello");
2647 Ok::<usize, VirtqError>(5)
2648 })
2649 .unwrap();
2650
2651 let _tok = producer.submit(se).unwrap();
2652
2653 let (recv, reply) = poll_received(&mut consumer);
2654 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2655 consumer.complete(recv, reply).unwrap();
2656 producer.drain(drop).unwrap();
2657 }
2658
2659 #[test]
2660 fn test_send_chain_single_segment_writer_serializes_directly() {
2661 let ring = make_ring(16);
2662 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2663
2664 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2665 se.with_seg(0, |segment| {
2666 assert_eq!(segment.len(), 64);
2667 segment[..5].copy_from_slice(b"hello");
2668 Ok::<usize, VirtqError>(5)
2669 })
2670 .unwrap();
2671
2672 let _tok = producer.submit(se).unwrap();
2673
2674 let (recv, reply) = poll_received(&mut consumer);
2675 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2676 consumer.complete(recv, reply).unwrap();
2677 producer.drain(drop).unwrap();
2678 }
2679
2680 #[test]
2681 fn test_send_chain_single_segment_writer_rejects_multi_segment() {
2682 let ring = make_ring(16);
2683 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2684
2685 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2686 assert!(matches!(
2687 se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2688 Err(VirtqError::NoPayloadSegment)
2689 ));
2690 }
2691
2692 #[test]
2693 fn test_send_chain_single_segment_writer_rejects_auto_split_chain() {
2694 let ring = make_ring(16);
2695 let (producer, _consumer) = make_virtq_pair(&ring, 4);
2696
2697 let mut se = producer.chain().readable(8).build().unwrap();
2698 assert_eq!(se.rd_desc_count(), 2);
2699 assert!(matches!(
2700 se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2701 Err(VirtqError::NoPayloadSegment)
2702 ));
2703 }
2704
2705 #[test]
2706 fn test_send_payload_segment_set_written_too_large() {
2707 let ring = make_ring(16);
2708 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2709
2710 let mut se = producer.chain().readable(32).writable(64).build().unwrap();
2711 let err = se
2712 .with_seg(0, |_| Ok::<usize, VirtqError>(64))
2713 .err()
2714 .unwrap();
2715 assert!(matches!(
2716 err,
2717 VirtqError::PayloadTooLarge {
2718 recv: 64,
2719 limit: 32
2720 }
2721 ));
2722 }
2723
2724 #[test]
2725 fn test_send_chain_write_too_large() {
2726 let ring = make_ring(16);
2727 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2728
2729 let mut se = producer.chain().readable(4).build().unwrap();
2730 let err = se.write_all(b"too long").err().unwrap();
2731 assert!(matches!(
2732 err,
2733 VirtqError::PayloadTooLarge { recv: 8, limit: 4 }
2734 ));
2735 }
2736
2737 #[test]
2738 fn test_writeonly_has_no_readable_buffer() {
2739 let ring = make_ring(16);
2740 let (producer, _consumer, _notifier) = make_test_producer(&ring);
2741
2742 let mut se = producer.chain().writable(32).build().unwrap();
2743 let err = se.write_all(b"data").err().unwrap();
2744 assert!(matches!(err, VirtqError::NoPayloadSegment));
2745 }
2746
2747 #[test]
2748 fn test_drop_chain_builder_deallocs() {
2749 let ring = make_ring(16);
2750 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2751
2752 {
2753 let _builder = producer.chain().readable(64).writable(128);
2754 }
2756
2757 let se = producer.chain().readable(64).writable(128).build().unwrap();
2759 let tok = producer.submit(se).unwrap();
2760 assert!(tok.id < 16);
2761
2762 unsafe { producer.reset() }.unwrap();
2764 }
2765
2766 #[test]
2767 fn test_drop_send_chain_deallocs() {
2768 let ring = make_ring(16);
2769 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2770
2771 {
2772 let _se = producer.chain().readable(64).writable(128).build().unwrap();
2773 }
2775
2776 let se = producer.chain().readable(64).writable(128).build().unwrap();
2778 let tok = producer.submit(se).unwrap();
2779 assert!(tok.id < 16);
2780
2781 unsafe { producer.reset() }.unwrap();
2783 }
2784
2785 #[test]
2786 fn test_submit_notifies() {
2787 let ring = make_ring(16);
2788 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2789
2790 let initial_count = notifier.notification_count();
2791
2792 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2793 se.write_all(b"hello").unwrap();
2794 producer.submit(se).unwrap();
2795
2796 assert!(notifier.notification_count() > initial_count);
2797
2798 unsafe { producer.reset() }.unwrap();
2800 }
2801
2802 #[test]
2803 fn test_submit_read_only_notifies_by_default() {
2804 let ring = make_ring(16);
2805 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2806
2807 let initial_count = notifier.notification_count();
2808
2809 let mut se = producer.chain().readable(64).build().unwrap();
2810 se.write_all(b"fire-and-forget").unwrap();
2811 producer.submit(se).unwrap();
2812
2813 assert!(notifier.notification_count() > initial_count);
2814
2815 unsafe { producer.reset() }.unwrap();
2817 }
2818
2819 #[test]
2820 fn test_submit_write_only_notifies_by_default() {
2821 let ring = make_ring(16);
2822 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2823
2824 let initial_count = notifier.notification_count();
2825
2826 let se = producer.chain().writable(128).build().unwrap();
2827 producer.submit(se).unwrap();
2828
2829 assert!(notifier.notification_count() > initial_count);
2830
2831 unsafe { producer.reset() }.unwrap();
2833 }
2834
2835 #[test]
2836 fn test_batch_notifies_once_on_finish() {
2837 let ring = make_ring(16);
2838 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2839
2840 let initial_count = notifier.notification_count();
2841
2842 let mut batch = producer.batch();
2843
2844 let mut first = batch.chain().readable(64).build().unwrap();
2845 first.write_all(b"first").unwrap();
2846 batch.submit(first).unwrap();
2847
2848 let mut second = batch.chain().readable(64).build().unwrap();
2849 second.write_all(b"second").unwrap();
2850 batch.submit(second).unwrap();
2851
2852 assert_eq!(notifier.notification_count(), initial_count);
2853 assert!(batch.finish().unwrap());
2854 assert_eq!(notifier.notification_count(), initial_count + 1);
2855
2856 let (recv, reply) = poll_received(&mut consumer);
2857 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"first");
2858 consumer.complete(recv, reply).unwrap();
2859
2860 let (recv, reply) = poll_received(&mut consumer);
2861 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"second");
2862 consumer.complete(recv, reply).unwrap();
2863 producer.drain(drop).unwrap();
2864 }
2865
2866 #[test]
2867 fn test_batch_finish_notifies_from_batch_start_cursor() {
2868 let ring = make_ring(16);
2869 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2870
2871 let cursor = consumer.avail_cursor();
2872 consumer
2873 .set_avail_suppression(SuppressionKind::Descriptor(cursor))
2874 .unwrap();
2875
2876 let mut batch = producer.batch();
2877
2878 let mut first = batch.chain().readable(64).build().unwrap();
2879 first.write_all(b"first").unwrap();
2880 batch.submit(first).unwrap();
2881 assert_eq!(notifier.notification_count(), 0);
2882
2883 let mut second = batch.chain().readable(64).writable(64).build().unwrap();
2884 second.write_all(b"second").unwrap();
2885 batch.submit(second).unwrap();
2886
2887 assert!(batch.finish().unwrap());
2888
2889 assert_eq!(notifier.notification_count(), 1);
2890
2891 unsafe { producer.reset() }.unwrap();
2893 }
2894
2895 #[test]
2896 fn test_empty_batch_finish_does_not_notify() {
2897 let ring = make_ring(16);
2898 let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2899
2900 let batch = producer.batch();
2901 assert!(!batch.finish().unwrap());
2902 assert_eq!(notifier.notification_count(), 0);
2903 }
2904
2905 #[test]
2906 fn test_batch_can_finish_without_notification() {
2907 let ring = make_ring(16);
2908 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2909
2910 let mut batch = producer.batch();
2911 let mut chain = batch.chain().readable(4).build().unwrap();
2912 chain.write_all(b"data").unwrap();
2913 batch.submit(chain).unwrap();
2914 batch.finish_without_notify();
2915
2916 assert_eq!(notifier.notification_count(), 0);
2917
2918 let (recv, reply) = poll_received(&mut consumer);
2919 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"data");
2920
2921 consumer.complete(recv, reply).unwrap();
2922 producer.drain(drop).unwrap();
2923 }
2924
2925 #[test]
2926 fn test_write_only_round_trip() {
2927 let ring = make_ring(16);
2928 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2929
2930 let se = producer.chain().writable(32).build().unwrap();
2931 let token = producer.submit(se).unwrap();
2932
2933 let (recv, reply) = poll_received(&mut consumer);
2934 assert_eq!(recv.token(), token);
2935 assert!(recv.to_bytes().unwrap().is_empty());
2936
2937 if let ReplyChain::Writable(mut wc) = reply {
2938 wc.write_all(b"filled-by-consumer").unwrap();
2939 consumer.complete(recv, wc).unwrap();
2940 } else {
2941 panic!("expected Writable");
2942 }
2943
2944 let used = producer.poll().unwrap().unwrap();
2945 assert_eq!(used.token(), token);
2946 assert_eq!(used.to_bytes().unwrap().len(), b"filled-by-consumer".len());
2947 assert_eq!(used.to_bytes().unwrap().as_ref(), b"filled-by-consumer");
2948 }
2949
2950 #[test]
2951 fn test_read_only_round_trip() {
2952 let ring = make_ring(16);
2953 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2954
2955 let mut se = producer.chain().readable(32).build().unwrap();
2956 se.write_all(b"fire-and-forget").unwrap();
2957 let token = producer.submit(se).unwrap();
2958
2959 let (recv, reply) = poll_received(&mut consumer);
2960 assert_eq!(recv.token(), token);
2961 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"fire-and-forget");
2962 assert!(matches!(reply, ReplyChain::Ack(_)));
2963 consumer.complete(recv, reply).unwrap();
2964
2965 let used = producer.poll().unwrap().unwrap();
2966 assert!(matches!(used, UsedChain::Ack(t) if t == token));
2967 }
2968
2969 #[test]
2970 fn test_readwrite_round_trip() {
2971 let ring = make_ring(16);
2972 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2973
2974 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2975 se.write_all(b"request data").unwrap();
2976 let token = producer.submit(se).unwrap();
2977
2978 let (recv, reply) = poll_received(&mut consumer);
2979 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"request data");
2980 if let ReplyChain::Writable(mut wc) = reply {
2981 wc.write_all(b"response data").unwrap();
2982 consumer.complete(recv, wc).unwrap();
2983 } else {
2984 panic!("expected Writable");
2985 }
2986
2987 let used = producer.poll().unwrap().unwrap();
2988 assert_eq!(used.token(), token);
2989 assert_eq!(used.to_bytes().unwrap().as_ref(), b"response data");
2990 }
2991
2992 #[test]
2993 fn test_poll_used_requires_owned_mapping() {
2994 let ring = make_ring(16);
2995 let layout = ring.layout();
2996 let test_mem = ring.mem();
2997 let pool_base = test_mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
2998 let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
2999 let pool = SlotPool::new(pool_layout).unwrap();
3000 let notifier = TestNotifier::new();
3001 let mem = FaultMem::new(test_mem);
3002 mem.deny_views();
3003 let mut producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool.clone());
3004 let mut consumer = VirtqConsumer::new(layout, mem, notifier);
3005
3006 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3007 se.write_all(b"request data").unwrap();
3008 producer.submit(se).unwrap();
3009
3010 let (recv, reply) = poll_received(&mut consumer);
3011 if let ReplyChain::Writable(mut wc) = reply {
3012 wc.write_all(b"response data").unwrap();
3013 consumer.complete(recv, wc).unwrap();
3014 } else {
3015 panic!("expected Writable");
3016 }
3017
3018 assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
3019 assert_eq!(pool.num_live(), 0);
3020 }
3021
3022 #[test]
3023 fn test_villain_used_len_exceeding_writable_capacity_is_rejected_and_released() {
3024 let ring = make_ring(16);
3025 let (mut producer, _consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
3026 let mut ring_consumer = make_consumer(&ring);
3027
3028 let se = producer.chain().writable(4).build().unwrap();
3029 producer.submit(se).unwrap();
3030
3031 let (id, _) = ring_consumer.poll_available().unwrap();
3032 ring_consumer.submit_used(id, 8).unwrap();
3033
3034 assert!(matches!(producer.poll(), Err(VirtqError::InvalidState)));
3035 assert_eq!(producer.inner.num_inflight(), 0);
3036
3037 let se = producer.chain().writable(4).build().unwrap();
3038 producer.submit(se).unwrap();
3039 assert_eq!(producer.inner.num_inflight(), 1);
3040
3041 unsafe { producer.reset() }.unwrap();
3043 }
3044
3045 #[test]
3046 fn test_villain_used_descriptor_with_invalid_id_is_rejected() {
3047 let ring = make_ring(16);
3048 let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
3049
3050 let se = producer.chain().writable(4).build().unwrap();
3051 producer.submit(se).unwrap();
3052
3053 let mut desc = Descriptor::new(0, 0, ring.len() as u16, DescFlags::empty());
3054 desc.mark_used(true);
3055 ring.write_desc(0, desc);
3056
3057 assert!(matches!(
3058 producer.poll(),
3059 Err(VirtqError::RingError(RingError::InvalidState))
3060 ));
3061 assert_eq!(producer.inner.num_inflight(), 1);
3062
3063 unsafe { producer.reset() }.unwrap();
3065 }
3066}