1use alloc::vec::Vec;
5use core::fmt;
6
7use bytes::Bytes;
8use smallvec::SmallVec;
9
10use super::ring::{ChainElems, Checkpoint};
11use super::*;
12
13#[must_use = "dropping without completing leaks the descriptor"]
19pub struct RecvChain<M: MemOps> {
20 state: ChainState<M>,
21}
22
23impl<M: MemOps> fmt::Debug for RecvChain<M> {
24 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
25 f.debug_struct("RecvChain")
26 .field("token", &self.state.token)
27 .field("elems", &self.state.elems)
28 .field("len", &self.state.total)
29 .field("consumed", &self.state.position)
30 .field("desc_index", &self.state.desc_idx)
31 .field("desc_offset", &self.state.desc_off)
32 .finish()
33 }
34}
35
36impl<M: MemOps> RecvChain<M> {
37 fn new(mem: M, token: Token, elems: ChainElems, len: usize) -> Self {
38 Self {
39 state: ChainState::new(mem, token, elems, len),
40 }
41 }
42
43 #[inline]
45 pub fn token(&self) -> Token {
46 self.state.token()
47 }
48
49 #[inline]
51 pub fn len(&self) -> usize {
52 self.state.total()
53 }
54
55 #[inline]
57 pub fn is_empty(&self) -> bool {
58 self.len() == 0
59 }
60
61 #[inline]
63 pub fn consumed(&self) -> usize {
64 self.state.position()
65 }
66
67 #[inline]
69 pub fn remaining(&self) -> usize {
70 self.state.remaining()
71 }
72
73 pub fn read(&mut self, buf: &mut [u8]) -> Result<usize, VirtqError> {
79 let len = buf.len().min(self.remaining());
80 let mut dst = &mut buf[..len];
81 let mut read = 0;
82
83 while !dst.is_empty() {
84 let Some(elem) = self.state.current_elem() else {
85 break;
86 };
87
88 let desc_len = elem.len as usize;
89 let desc_offset = self.state.desc_offset();
90 let len = (desc_len - desc_offset).min(dst.len());
91 let (current, rest) = dst.split_at_mut(len);
92
93 let addr = elem
94 .addr
95 .checked_add(desc_offset as u64)
96 .ok_or(VirtqError::MemoryReadError)?;
97
98 self.state
99 .mem
100 .read(addr, current)
101 .map_err(|_| VirtqError::MemoryReadError)?;
102
103 self.state.advance(len);
104 read += len;
105 dst = rest;
106 }
107
108 Ok(read)
109 }
110
111 #[inline]
113 pub fn read_exact(&mut self, buf: &mut [u8]) -> Result<&mut Self, VirtqError> {
114 if buf.len() > self.remaining() {
115 return Err(VirtqError::ReceiveTooShort {
116 requested: buf.len(),
117 remaining: self.remaining(),
118 });
119 }
120
121 let read = self.read(buf)?;
122 debug_assert_eq!(read, buf.len());
123 Ok(self)
124 }
125
126 pub fn to_segments(&self) -> Result<Segments, VirtqError> {
137 let mut segments = SmallVec::<[Bytes; 4]>::new();
138 segments
139 .try_reserve_exact(self.state.elems.len())
140 .map_err(|_| VirtqError::Bookkeeping)?;
141
142 for elem in &self.state.elems {
143 let mut buf = zeroed_vec(elem.len as usize)?;
144 self.state
145 .mem
146 .read(elem.addr, &mut buf)
147 .map_err(|_| VirtqError::MemoryReadError)?;
148 segments.push(Bytes::from(buf));
149 }
150
151 Ok(Segments::from_smallvec(segments))
152 }
153
154 pub fn to_bytes(&self) -> Result<Bytes, VirtqError> {
165 if self.is_empty() {
166 return Ok(Bytes::new());
167 }
168
169 let mut buf = zeroed_vec(self.len())?;
170 let mut offset = 0;
171
172 for elem in &self.state.elems {
173 let end = offset + elem.len as usize;
174 self.state
175 .mem
176 .read(elem.addr, &mut buf[offset..end])
177 .map_err(|_| VirtqError::MemoryReadError)?;
178 offset = end;
179 }
180
181 Ok(Bytes::from(buf))
182 }
183}
184
185#[must_use = "dropping without completing leaks the descriptor"]
190pub enum ReplyChain<M: MemOps> {
191 Writable(WritableChain<M>),
195 Ack(AckChain),
199}
200
201pub type PolledChain<M> = (RecvChain<M>, ReplyChain<M>);
203
204pub type PolledChains<M> = Vec<PolledChain<M>>;
206
207impl<M: MemOps> ReplyChain<M> {
208 #[inline]
210 pub fn token(&self) -> Token {
211 match self {
212 ReplyChain::Writable(wc) => wc.token(),
213 ReplyChain::Ack(ack) => ack.token(),
214 }
215 }
216
217 #[inline]
219 pub fn written(&self) -> usize {
220 match self {
221 ReplyChain::Writable(wc) => wc.written(),
222 ReplyChain::Ack(_) => 0,
223 }
224 }
225
226 pub fn into_writable(self) -> Result<WritableChain<M>, AckChain> {
231 match self {
232 ReplyChain::Writable(wc) => Ok(wc),
233 ReplyChain::Ack(ack) => Err(ack),
234 }
235 }
236}
237
238#[must_use = "dropping without completing leaks the descriptor"]
249pub struct WritableChain<M: MemOps> {
250 state: ChainState<M>,
251}
252
253impl<M: MemOps> WritableChain<M> {
254 fn new(mem: M, token: Token, elems: ChainElems) -> Self {
255 let capacity = elems.iter().map(|elem| elem.len as usize).sum();
256 Self {
257 state: ChainState::new(mem, token, elems, capacity),
258 }
259 }
260
261 #[inline]
263 pub fn token(&self) -> Token {
264 self.state.token()
265 }
266
267 #[inline]
269 pub fn capacity(&self) -> usize {
270 self.state.total()
271 }
272
273 #[inline]
275 pub fn desc_count(&self) -> usize {
276 self.state.elems.len()
277 }
278
279 #[inline]
281 pub fn written(&self) -> usize {
282 self.state.position()
283 }
284
285 #[inline]
287 pub fn remaining(&self) -> usize {
288 self.state.remaining()
289 }
290
291 pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
303 let mut src = &buf[..buf.len().min(self.remaining())];
304 let mut written = 0;
305
306 while !src.is_empty() {
307 let Some(elem) = self.state.current_elem() else {
308 break;
309 };
310 let desc_capacity = elem.len as usize;
311 let desc_offset = self.state.desc_offset();
312 let len = (desc_capacity - desc_offset).min(src.len());
313
314 let addr = elem
315 .addr
316 .checked_add(desc_offset as u64)
317 .ok_or(VirtqError::MemoryWriteError)?;
318
319 self.state
320 .mem
321 .write(addr, &src[..len])
322 .map_err(|_| VirtqError::MemoryWriteError)?;
323
324 self.state.advance(len);
325 written += len;
326 src = &src[len..];
327 }
328
329 Ok(written)
330 }
331
332 #[inline]
339 pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
340 if buf.len() > self.remaining() {
341 return Err(VirtqError::ReplyTooLarge);
342 }
343
344 let written = self.write(buf)?;
345 debug_assert_eq!(written, buf.len());
346 Ok(self)
347 }
348
349 pub fn rewind(&mut self) {
354 self.state.rewind();
355 }
356}
357
358#[must_use = "dropping without completing leaks the descriptor"]
366pub struct AckChain {
367 token: Token,
368}
369
370impl AckChain {
371 fn new(token: Token) -> Self {
372 Self { token }
373 }
374
375 #[inline]
376 pub fn token(&self) -> Token {
377 self.token
378 }
379}
380
381pub struct VirtqConsumer<M, N> {
419 inner: RingConsumer<M>,
420 mem: M,
421 notifier: N,
422 next_token: u32,
423}
424
425impl<M: MemOps + Clone, N: Notifier> VirtqConsumer<M, N> {
426 pub fn new(layout: Layout, mem: M, notifier: N) -> Self {
434 Self::new_split(layout, mem.clone(), mem, notifier)
435 }
436
437 pub fn new_split(layout: Layout, ring_mem: M, buf_mem: M, notifier: N) -> Self {
439 let inner = RingConsumer::new(layout, ring_mem);
440
441 Self {
442 inner,
443 mem: buf_mem,
444 notifier,
445 next_token: 0,
446 }
447 }
448
449 pub fn poll(&mut self, max_recv_len: usize) -> Result<Option<PolledChain<M>>, VirtqError> {
472 let (id, chain) = match self.inner.poll_available() {
473 Ok(x) => x,
474 Err(RingError::WouldBlock) => return Ok(None),
475 Err(e) => return Err(e.into()),
476 };
477
478 let readables = chain.readables();
479 let writables = chain.writables();
480 if readables.is_empty() && writables.is_empty() {
481 return Err(self.abort_chain(id, VirtqError::BadChain));
482 }
483
484 let recv_len = readables
485 .iter()
486 .fold(0usize, |acc, elem| acc.saturating_add(elem.len as usize));
487
488 let token = Token {
489 seq: self.next_token,
490 id,
491 };
492 self.next_token = self.next_token.wrapping_add(1);
493
494 if recv_len > max_recv_len {
495 return Err(self.abort_chain(
496 id,
497 VirtqError::PayloadTooLarge {
498 recv: recv_len,
499 limit: max_recv_len,
500 },
501 ));
502 }
503
504 let (readables, writables) = match chain.into_parts() {
505 Ok(parts) => parts,
506 Err(error) => return Err(self.abort_chain(id, error.into())),
507 };
508 let chain = RecvChain::new(self.mem.clone(), token, readables, recv_len);
509
510 let reply = if !writables.is_empty() {
511 let mem = self.mem.clone();
512 let writable = WritableChain::new(mem, token, writables);
513 ReplyChain::Writable(writable)
514 } else {
515 let ack = AckChain::new(token);
516 ReplyChain::Ack(ack)
517 };
518
519 Ok(Some((chain, reply)))
520 }
521
522 pub fn poll_batch(
541 &mut self,
542 count: usize,
543 keep_avail: usize,
544 max_recv_len: usize,
545 ) -> Result<Option<PolledChains<M>>, VirtqError> {
546 let Some(total) = count.checked_add(keep_avail) else {
547 return Ok(None);
548 };
549
550 if total > self.inner.num_free() {
552 return Ok(None);
553 }
554
555 let mut polled = PolledChains::new();
556 polled
557 .try_reserve_exact(total)
558 .map_err(|_| VirtqError::Bookkeeping)?;
559
560 let cp = self.inner.poll_checkpoint();
562 let next_token = self.next_token;
563
564 let mut batch_cp = None;
565
566 while polled.len() < total {
567 if keep_avail != 0 && polled.len() == count {
568 batch_cp = Some((self.inner.poll_checkpoint(), self.next_token));
569 }
570
571 if let Some(chain) = self.poll(max_recv_len)? {
572 polled.push(chain);
573 continue;
574 }
575
576 self.rollback_polled(cp, next_token, polled);
577 return Ok(None);
578 }
579
580 if let Some((checkpoint, next_token)) = batch_cp {
581 self.rollback_polled(checkpoint, next_token, polled.drain(count..));
582 }
583
584 Ok(Some(polled))
585 }
586
587 pub fn complete(
599 &mut self,
600 recv: impl Into<RecvChain<M>>,
601 reply: impl Into<ReplyChain<M>>,
602 ) -> Result<(), VirtqError> {
603 let recv = recv.into();
604 let reply = reply.into();
605
606 if recv.token() != reply.token() {
607 return Err(VirtqError::InvalidState);
608 }
609
610 let id = reply.token().id;
611 let written = u32::try_from(reply.written()).map_err(|_| VirtqError::ReplyTooLarge)?;
612
613 if self.inner.submit_used_with_notify(id, written)? {
614 self.notifier.notify(QueueStats {
615 num_free: self.inner.num_free(),
616 num_inflight: self.inner.num_inflight(),
617 });
618 }
619
620 Ok(())
621 }
622
623 fn abort_chain(&mut self, id: u16, err: VirtqError) -> VirtqError {
628 if let Ok(true) = self.inner.submit_used_with_notify(id, 0) {
631 self.notifier.notify(QueueStats {
632 num_free: self.inner.num_free(),
633 num_inflight: self.inner.num_inflight(),
634 });
635 }
636
637 err
638 }
639
640 fn rollback_polled(
645 &mut self,
646 cp: Checkpoint,
647 next: u32,
648 polled: impl IntoIterator<Item = PolledChain<M>>,
649 ) {
650 let ids = polled.into_iter().map(|(recv, reply)| {
652 let id = recv.token().id;
653 drop((recv, reply));
654 id
655 });
656
657 self.inner.rollback_polls(cp, ids);
658 self.next_token = next;
659 }
660
661 #[inline]
666 pub fn avail_cursor(&self) -> RingCursor {
667 self.inner.avail_cursor()
668 }
669
670 #[inline]
675 pub fn used_cursor(&self) -> RingCursor {
676 self.inner.used_cursor()
677 }
678
679 pub fn set_avail_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
698 match kind {
699 SuppressionKind::Enable => self.inner.enable_avail_notifications()?,
700 SuppressionKind::Disable => self.inner.disable_avail_notifications()?,
701 SuppressionKind::Descriptor(cursor) => self
702 .inner
703 .enable_avail_notifications_desc(cursor.head(), cursor.wrap())?,
704 }
705 Ok(())
706 }
707
708 pub fn reset(&mut self) -> Result<(), VirtqError> {
717 if self.inner.num_inflight() != 0 {
718 return Err(VirtqError::InvalidState);
719 }
720
721 self.inner.reset()?;
722 Ok(())
723 }
724}
725
726struct ChainState<M: MemOps> {
727 mem: M,
728 token: Token,
729 elems: ChainElems,
730 total: usize,
731 position: usize,
732 desc_idx: usize,
733 desc_off: usize,
734}
735
736impl<M: MemOps> ChainState<M> {
737 fn new(mem: M, token: Token, elems: ChainElems, total: usize) -> Self {
738 let mut state = Self {
739 mem,
740 token,
741 elems,
742 total,
743 position: 0,
744 desc_idx: 0,
745 desc_off: 0,
746 };
747 state.rewind();
748 state
749 }
750
751 #[inline]
752 fn token(&self) -> Token {
753 self.token
754 }
755
756 #[inline]
757 fn total(&self) -> usize {
758 self.total
759 }
760
761 #[inline]
762 fn position(&self) -> usize {
763 self.position
764 }
765
766 #[inline]
767 fn remaining(&self) -> usize {
768 self.total - self.position
769 }
770
771 #[inline(always)]
772 fn desc_len(&self) -> usize {
773 self.elems
774 .get(self.desc_idx)
775 .map(|elem| elem.len as usize)
776 .unwrap_or(0)
777 }
778
779 #[inline(always)]
780 fn desc_offset(&self) -> usize {
781 self.desc_off
782 }
783
784 #[inline(always)]
785 fn current_elem(&self) -> Option<BufferElement> {
786 self.elems.get(self.desc_idx).copied()
787 }
788
789 #[inline(always)]
790 fn advance(&mut self, len: usize) {
791 debug_assert!(len <= self.desc_len() - self.desc_off);
792 self.desc_off += len;
793 self.position += len;
794
795 while self.current_elem().is_some() && self.desc_off == self.desc_len() {
796 self.desc_idx += 1;
797 self.desc_off = 0;
798 }
799 }
800
801 fn rewind(&mut self) {
802 self.position = 0;
803 self.desc_off = 0;
804 self.desc_idx = self
805 .elems
806 .iter()
807 .position(|elem| elem.len != 0)
808 .unwrap_or(self.elems.len());
809 }
810}
811
812impl<M: MemOps> From<WritableChain<M>> for ReplyChain<M> {
813 fn from(wc: WritableChain<M>) -> Self {
814 ReplyChain::Writable(wc)
815 }
816}
817
818impl<M: MemOps> From<AckChain> for ReplyChain<M> {
819 fn from(ack: AckChain) -> Self {
820 ReplyChain::Ack(ack)
821 }
822}
823
824#[cfg(test)]
825mod tests {
826 use super::*;
827 use crate::flatbuffer_wrappers::ExternalValueSource;
828 use crate::virtq::ring::tests::{TestMem, make_producer, make_ring};
829 use crate::virtq::test_utils::*;
830
831 fn poll_data(
832 consumer: &mut VirtqConsumer<TestMem, TestNotifier>,
833 ) -> (RecvChain<TestMem>, ReplyChain<TestMem>) {
834 consumer.poll(1024).unwrap().unwrap()
835 }
836
837 #[derive(Clone)]
838 struct FailingPayloadReadMem {
839 inner: TestMem,
840 payload_addr: u64,
841 payload_len: usize,
842 }
843
844 unsafe impl MemOps for FailingPayloadReadMem {
847 type Error = ();
848
849 fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
850 let read_end = addr.saturating_add(dst.len() as u64);
851 let payload_end = self.payload_addr.saturating_add(self.payload_len as u64);
852 if addr < payload_end && self.payload_addr < read_end {
853 return Err(());
854 }
855 self.inner.read(addr, dst).map_err(|err| match err {})
856 }
857
858 fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
859 self.inner.write(addr, src).map_err(|err| match err {})
860 }
861
862 fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
863 self.inner.load_acquire(addr).map_err(|err| match err {})
864 }
865
866 fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
867 self.inner
868 .store_release(addr, val)
869 .map_err(|err| match err {})
870 }
871
872 unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
873 unsafe { self.inner.as_slice(addr, len) }.map_err(|err| match err {})
874 }
875
876 unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
877 unsafe { self.inner.as_mut_slice(addr, len) }.map_err(|err| match err {})
878 }
879 }
880
881 #[test]
882 fn test_write_only_recv_is_empty() {
883 let ring = make_ring(16);
884 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
885
886 let se = producer.chain().writable(16).build().unwrap();
887 producer.submit(se).unwrap();
888
889 let (recv, reply) = poll_data(&mut consumer);
890 assert!(recv.to_bytes().unwrap().is_empty());
891 assert!(matches!(reply, ReplyChain::Writable(_)));
892
893 if let ReplyChain::Writable(mut wc) = reply {
894 wc.write_all(b"response").unwrap();
895 consumer.complete(recv, wc).unwrap();
896 }
897 producer.drain(drop).unwrap();
898 }
899
900 #[test]
901 fn test_read_only_ack_reply() {
902 let ring = make_ring(16);
903 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
904
905 let mut se = producer.chain().readable(16).build().unwrap();
906 se.write_all(b"hello").unwrap();
907 producer.submit(se).unwrap();
908
909 let (recv, reply) = poll_data(&mut consumer);
910 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
911 assert!(matches!(reply, ReplyChain::Ack(_)));
912
913 consumer.complete(recv, reply).unwrap();
914 producer.drain(drop).unwrap();
915 }
916
917 #[test]
918 fn test_readwrite_round_trip() {
919 let ring = make_ring(16);
920 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 64);
921 let request: Vec<u8> = (0..5 * 64).map(|i| (i % 251) as u8).collect();
922 let response: Vec<u8> = (0..6 * 64 - 1).map(|i| (i % 239) as u8).collect();
923
924 let mut se = producer
926 .chain()
927 .readable(request.len())
928 .writable(6 * 64)
929 .build()
930 .unwrap();
931 se.write_all(&request).unwrap();
932 producer.submit(se).unwrap();
933
934 let (recv, reply) = poll_data(&mut consumer);
935 assert_eq!(recv.to_bytes().unwrap().as_ref(), request);
936
937 if let ReplyChain::Writable(mut wc) = reply {
938 assert_eq!(wc.capacity(), 6 * 64);
939 assert_eq!(wc.written(), 0);
940 assert_eq!(wc.remaining(), 6 * 64);
941 wc.write_all(&response).unwrap();
942 assert_eq!(wc.written(), response.len());
943 assert_eq!(wc.remaining(), 1);
944 consumer.complete(recv, wc).unwrap();
945 } else {
946 panic!("expected Writable reply for recv+reply chain");
947 }
948
949 let used = producer.poll().unwrap().unwrap();
950 assert_eq!(used.to_bytes().unwrap().as_ref(), response);
951 }
952
953 #[test]
954 fn test_recv_reads_across_descriptor_boundaries() {
955 let ring = make_ring(16);
956 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
957
958 let mut se = producer.chain().readable(4).readable(4).build().unwrap();
959 se.write_all(b"abcdefgh").unwrap();
960 producer.submit(se).unwrap();
961
962 let (mut recv, reply) = poll_data(&mut consumer);
963 assert_eq!(recv.len(), 8);
964 assert_eq!(recv.remaining(), 8);
965 let mut first = [0u8; 2];
966 recv.read_exact(&mut first).unwrap();
967 assert_eq!(&first, b"ab");
968 assert_eq!(recv.consumed(), 2);
969 assert_eq!(recv.remaining(), 6);
970
971 let mut second = [0u8; 3];
972 recv.read_exact(&mut second).unwrap();
973 assert_eq!(&second, b"cde");
974
975 let mut too_long = [0u8; 4];
976 assert!(matches!(
977 recv.read_exact(&mut too_long),
978 Err(VirtqError::ReceiveTooShort {
979 requested: 4,
980 remaining: 3
981 })
982 ));
983
984 let mut final_buf = [0u8; 4];
985 assert_eq!(recv.read(&mut final_buf).unwrap(), 3);
986 assert_eq!(&final_buf[..3], b"fgh");
987 assert_eq!(recv.read(&mut final_buf).unwrap(), 0);
988 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefgh");
989
990 consumer.complete(recv, reply).unwrap();
991 producer.drain(drop).unwrap();
992 }
993
994 #[test]
995 fn test_external_values_copy_shared_memory() {
996 let ring = make_ring(16);
997 let mem = ring.mem();
998 let mut ring_producer = make_producer(&ring);
999 let payload_addr = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1000 mem.write(payload_addr, b"abcdefgh").unwrap();
1001
1002 let chain = BufferChainBuilder::new()
1003 .readable(payload_addr, 4)
1004 .readable(payload_addr + 4, 4)
1005 .build()
1006 .unwrap();
1007 ring_producer.submit_available(&chain).unwrap();
1008
1009 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1010 let (mut recv, reply) = poll_data(&mut consumer);
1011 let contiguous = recv.take_bytes(3).unwrap();
1012 let chunks = recv.take_chunks(5).unwrap();
1013 recv.finish().unwrap();
1014
1015 mem.write(payload_addr, b"xxxxxxxx").unwrap();
1016
1017 assert_eq!(contiguous, b"abc");
1018 assert_eq!(chunks.len(), 1);
1019 assert_eq!(chunks[0].as_ref(), b"defgh");
1020 consumer.complete(recv, reply).unwrap();
1021 }
1022
1023 #[test]
1024 fn test_external_values_validate_remaining_length() {
1025 let ring = make_ring(16);
1026 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1027 let mut chain = producer.chain().readable(4).build().unwrap();
1028 chain.write_all(b"data").unwrap();
1029 producer.submit(chain).unwrap();
1030
1031 let (mut recv, reply) = poll_data(&mut consumer);
1032 for error in [
1033 recv.take_bytes(usize::MAX).unwrap_err(),
1034 recv.take_chunks(usize::MAX).unwrap_err(),
1035 ] {
1036 assert!(matches!(
1037 error.downcast_ref::<VirtqError>(),
1038 Some(VirtqError::ReceiveTooShort {
1039 requested: usize::MAX,
1040 remaining: 4,
1041 })
1042 ));
1043 }
1044
1045 assert!(recv.take_bytes(0).unwrap().is_empty());
1046 assert!(recv.take_chunks(0).unwrap().is_empty());
1047 assert_eq!(recv.remaining(), 4);
1048 assert!(recv.finish().is_err());
1049 assert_eq!(recv.take_bytes(4).unwrap(), b"data");
1050 recv.finish().unwrap();
1051
1052 consumer.complete(recv, reply).unwrap();
1053 producer.drain(drop).unwrap();
1054 }
1055
1056 #[test]
1057 fn test_poll_defers_payload_reads() {
1058 let ring = make_ring(16);
1059 let mem = ring.mem();
1060 let mut ring_producer = make_producer(&ring);
1061 let payload_addr = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1062 mem.write(payload_addr, b"data").unwrap();
1063
1064 let chain = BufferChainBuilder::new()
1065 .readable(payload_addr, 4)
1066 .build()
1067 .unwrap();
1068 ring_producer.submit_available(&chain).unwrap();
1069
1070 let ring_mem = FailingPayloadReadMem {
1071 inner: mem.clone(),
1072 payload_addr,
1073 payload_len: 0,
1074 };
1075 let mem = FailingPayloadReadMem {
1076 inner: mem,
1077 payload_addr,
1078 payload_len: 4,
1079 };
1080
1081 let mut consumer =
1082 VirtqConsumer::new_split(ring.layout(), ring_mem, mem, TestNotifier::new());
1083
1084 let (recv, reply) = consumer.poll(4).unwrap().unwrap();
1085 assert!(matches!(recv.to_bytes(), Err(VirtqError::MemoryReadError)));
1086 consumer.complete(recv, reply).unwrap();
1087 }
1088
1089 #[test]
1090 fn test_writable_partial_write() {
1091 let ring = make_ring(16);
1092 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 8);
1093
1094 let se = producer.chain().writable(8).build().unwrap();
1095 producer.submit(se).unwrap();
1096
1097 let (recv, reply) = poll_data(&mut consumer);
1098
1099 if let ReplyChain::Writable(mut wc) = reply {
1100 let n = wc.write(b"hello world!").unwrap();
1101 assert_eq!(n, 8);
1102 assert_eq!(wc.remaining(), 0);
1103 consumer.complete(recv, wc).unwrap();
1104 } else {
1105 panic!("expected Writable");
1106 }
1107 producer.drain(drop).unwrap();
1108 }
1109
1110 #[test]
1111 fn test_writable_write_all_too_large() {
1112 let ring = make_ring(16);
1113 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
1114
1115 let se = producer.chain().writable(4).build().unwrap();
1116 producer.submit(se).unwrap();
1117 let (recv, reply) = poll_data(&mut consumer);
1118
1119 if let ReplyChain::Writable(mut wc) = reply {
1120 let err = wc.write_all(b"too long").err().unwrap();
1121 assert!(matches!(err, VirtqError::ReplyTooLarge));
1122 consumer.complete(recv, wc).unwrap();
1123 } else {
1124 panic!("expected Writable");
1125 }
1126
1127 producer.drain(drop).unwrap();
1128 }
1129
1130 #[test]
1131 fn test_poll_batch_rolls_back_partial_batch() {
1132 let ring = make_ring(32);
1133 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1134
1135 for _ in 0..20 {
1136 let chain = producer.chain().writable(16).build().unwrap();
1137 producer.submit(chain).unwrap();
1138 }
1139
1140 let cursor = consumer.avail_cursor();
1141 assert!(consumer.poll_batch(21, 0, 0).unwrap().is_none());
1142 assert_eq!(consumer.avail_cursor(), cursor);
1143 assert_eq!(consumer.inner.num_inflight(), 0);
1144 assert_eq!(consumer.next_token, 0);
1145 assert!(producer.poll().unwrap().is_none());
1146
1147 let reserved = consumer.poll_batch(20, 0, 0).unwrap().unwrap();
1148 for (recv, reply) in reserved {
1149 consumer.complete(recv, reply).unwrap();
1150 }
1151 for _ in 0..20 {
1152 assert!(producer.poll().unwrap().is_some());
1153 }
1154 }
1155
1156 #[test]
1157 fn test_poll_batch_keeps_chains_available() {
1158 let ring = make_ring(16);
1159 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1160
1161 for _ in 0..3 {
1162 let chain = producer.chain().writable(16).build().unwrap();
1163 producer.submit(chain).unwrap();
1164 }
1165
1166 let polled = consumer.poll_batch(2, 1, 0).unwrap().unwrap();
1167 assert_eq!(consumer.avail_cursor().head(), 2);
1168 assert_eq!(consumer.inner.num_inflight(), 2);
1169 assert_eq!(consumer.next_token, 2);
1170
1171 for (recv, reply) in polled {
1172 consumer.complete(recv, reply).unwrap();
1173 }
1174
1175 let (recv, reply) = consumer.poll(0).unwrap().unwrap();
1176 assert_eq!(recv.token().seq, 2);
1177 consumer.complete(recv, reply).unwrap();
1178 producer.drain(drop).unwrap();
1179 }
1180
1181 #[test]
1182 fn test_poll_batch_rolls_back_when_keep_avail_is_unmet() {
1183 let ring = make_ring(16);
1184 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1185
1186 for _ in 0..2 {
1187 let chain = producer.chain().writable(16).build().unwrap();
1188 producer.submit(chain).unwrap();
1189 }
1190
1191 let cursor = consumer.avail_cursor();
1192 assert!(consumer.poll_batch(2, 1, 0).unwrap().is_none());
1193 assert_eq!(consumer.avail_cursor(), cursor);
1194 assert_eq!(consumer.inner.num_inflight(), 0);
1195 assert_eq!(consumer.next_token, 0);
1196
1197 let polled = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1198 for (recv, reply) in polled {
1199 consumer.complete(recv, reply).unwrap();
1200 }
1201 producer.drain(drop).unwrap();
1202 }
1203
1204 #[test]
1205 fn test_poll_batch_preserves_existing_inflight_chain() {
1206 let ring = make_ring(16);
1207 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1208
1209 for _ in 0..3 {
1210 let chain = producer.chain().writable(16).build().unwrap();
1211 producer.submit(chain).unwrap();
1212 }
1213
1214 let existing = consumer.poll(0).unwrap().unwrap();
1215 let cursor = consumer.avail_cursor();
1216 assert!(consumer.poll_batch(3, 0, 0).unwrap().is_none());
1217 assert_eq!(consumer.avail_cursor(), cursor);
1218 assert_eq!(consumer.inner.num_inflight(), 1);
1219
1220 let reserved = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1221 consumer.complete(existing.0, existing.1).unwrap();
1222 for (recv, reply) in reserved {
1223 consumer.complete(recv, reply).unwrap();
1224 }
1225 producer.drain(drop).unwrap();
1226 }
1227
1228 #[test]
1229 fn test_poll_batch_rolls_back_multi_descriptor_chain() {
1230 let ring = make_ring(16);
1231 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1232
1233 let chain = producer.chain().writable(8).writable(8).build().unwrap();
1234 producer.submit(chain).unwrap();
1235
1236 let cursor = consumer.avail_cursor();
1237 assert!(consumer.poll_batch(2, 0, 0).unwrap().is_none());
1238 assert_eq!(consumer.avail_cursor(), cursor);
1239 assert_eq!(consumer.inner.num_inflight(), 0);
1240
1241 let mut reserved = consumer.poll_batch(1, 0, 0).unwrap().unwrap();
1242 let (recv, reply) = reserved.pop().unwrap();
1243 let ReplyChain::Writable(writable) = &reply else {
1244 panic!("expected writable chain");
1245 };
1246 assert_eq!(writable.desc_count(), 2);
1247 consumer.complete(recv, reply).unwrap();
1248 producer.drain(drop).unwrap();
1249 }
1250
1251 #[test]
1252 fn test_poll_batch_rolls_back_across_wrap() {
1253 let ring = make_ring(4);
1254 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1255
1256 for _ in 0..3 {
1257 let chain = producer.chain().writable(16).build().unwrap();
1258 producer.submit(chain).unwrap();
1259 }
1260 let reserved = consumer.poll_batch(3, 0, 0).unwrap().unwrap();
1261 for (recv, reply) in reserved {
1262 consumer.complete(recv, reply).unwrap();
1263 }
1264 for _ in 0..3 {
1265 assert!(producer.poll().unwrap().is_some());
1266 }
1267
1268 let chain = producer.chain().writable(16).build().unwrap();
1269 producer.submit(chain).unwrap();
1270
1271 let cursor = consumer.avail_cursor();
1272 assert_eq!(cursor.head(), 3);
1273 assert!(consumer.poll_batch(2, 0, 0).unwrap().is_none());
1274 assert_eq!(consumer.avail_cursor(), cursor);
1275
1276 let mut reserved = consumer.poll_batch(1, 0, 0).unwrap().unwrap();
1277 let (recv, reply) = reserved.pop().unwrap();
1278 consumer.complete(recv, reply).unwrap();
1279 producer.drain(drop).unwrap();
1280 }
1281
1282 #[test]
1283 fn test_poll_batch_rolls_back_token_wrap() {
1284 let ring = make_ring(16);
1285 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
1286 consumer.next_token = u32::MAX;
1287 for _ in 0..2 {
1288 let chain = producer.chain().writable(16).build().unwrap();
1289 producer.submit(chain).unwrap();
1290 }
1291 let avail = consumer.avail_cursor();
1292 let used = consumer.used_cursor();
1293 let notifications = notifier.notification_count();
1294
1295 assert!(consumer.poll_batch(3, 0, 0).unwrap().is_none());
1297 assert_eq!(consumer.avail_cursor(), avail);
1298 assert_eq!(consumer.used_cursor(), used);
1299 assert_eq!(consumer.inner.num_inflight(), 0);
1300 assert_eq!(consumer.next_token, u32::MAX);
1301 assert_eq!(notifier.notification_count(), notifications);
1302 assert!(producer.poll().unwrap().is_none());
1303
1304 let polled = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1306 assert_eq!(polled[0].0.token().seq, u32::MAX);
1307 assert_eq!(polled[1].0.token().seq, 0);
1308 assert_eq!(consumer.next_token, 1);
1309 for (recv, reply) in polled {
1310 consumer.complete(recv, reply).unwrap();
1311 }
1312 producer.drain(drop).unwrap();
1313 }
1314
1315 #[test]
1316 fn test_poll_batch_keeps_multi_descriptor_chains_available() {
1317 let ring = make_ring(16);
1318 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1319 let chain = producer.chain().writable(8).build().unwrap();
1320 let first_token = producer.submit(chain).unwrap();
1321 let chain = producer.chain().writable(8).writable(8).build().unwrap();
1322 let second_token = producer.submit(chain).unwrap();
1323 let chain = producer
1324 .chain()
1325 .writable(8)
1326 .writable(8)
1327 .writable(8)
1328 .build()
1329 .unwrap();
1330 let third_token = producer.submit(chain).unwrap();
1331
1332 let mut polled = consumer.poll_batch(1, 2, 0).unwrap().unwrap();
1334 assert_eq!(polled.len(), 1);
1335 assert_eq!(consumer.avail_cursor().head(), 1);
1336 assert_eq!(consumer.used_cursor().head(), 0);
1337 assert_eq!(consumer.inner.num_inflight(), 1);
1338 assert_eq!(consumer.next_token, 1);
1339 let first = polled.pop().unwrap();
1340 assert_eq!(first.0.token().id, first_token.id);
1341
1342 let second = consumer.poll(0).unwrap().unwrap();
1344 assert_eq!(second.0.token().id, second_token.id);
1345 assert_eq!(second.0.token().seq, 1);
1346 assert_eq!(consumer.inner.num_inflight(), 3);
1347 let third = consumer.poll(0).unwrap().unwrap();
1348 assert_eq!(third.0.token().id, third_token.id);
1349 assert_eq!(third.0.token().seq, 2);
1350 assert_eq!(consumer.inner.num_inflight(), 6);
1351 consumer.complete(first.0, first.1).unwrap();
1352 consumer.complete(second.0, second.1).unwrap();
1353 consumer.complete(third.0, third.1).unwrap();
1354 producer.drain(drop).unwrap();
1355 }
1356
1357 #[test]
1358 fn test_poll_batch_zero_count_keeps_full_ring_available() {
1359 let ring = make_ring(32);
1360 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1361 for _ in 0..32 {
1362 let chain = producer.chain().writable(16).build().unwrap();
1363 producer.submit(chain).unwrap();
1364 }
1365 let avail = consumer.avail_cursor();
1366 let used = consumer.used_cursor();
1367
1368 let polled = consumer.poll_batch(0, 32, 0).unwrap().unwrap();
1370 assert!(polled.is_empty());
1371 assert_eq!(consumer.avail_cursor(), avail);
1372 assert_eq!(consumer.used_cursor(), used);
1373 assert_eq!(consumer.inner.num_inflight(), 0);
1374 assert_eq!(consumer.next_token, 0);
1375 assert!(producer.poll().unwrap().is_none());
1376
1377 let polled = consumer.poll_batch(32, 0, 0).unwrap().unwrap();
1379 assert_eq!(polled.len(), 32);
1380 for (recv, reply) in polled {
1381 consumer.complete(recv, reply).unwrap();
1382 }
1383 producer.drain(drop).unwrap();
1384 }
1385
1386 #[test]
1387 fn test_poll_batch_error_after_partial_batch_is_not_rolled_back() {
1388 let ring = make_ring(16);
1389 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1390 let chain = producer.chain().writable(16).build().unwrap();
1391 producer.submit(chain).unwrap();
1392 let mut chain = producer.chain().readable(8).build().unwrap();
1393 chain.write_all(b"too much").unwrap();
1394 let rejected = producer.submit(chain).unwrap();
1395
1396 assert!(matches!(
1398 consumer.poll_batch(2, 0, 4),
1399 Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1400 ));
1401 assert_eq!(consumer.avail_cursor().head(), 2);
1402 assert_eq!(consumer.used_cursor().head(), 1);
1403 assert_eq!(consumer.inner.num_inflight(), 1);
1404 assert_eq!(consumer.next_token, 2);
1405
1406 assert_eq!(producer.poll().unwrap().unwrap().token(), rejected);
1408 assert!(producer.poll().unwrap().is_none());
1409
1410 drop(consumer);
1411 unsafe { producer.reset().unwrap() };
1413 assert_eq!(producer.pool().num_live(), 0);
1414 }
1415
1416 #[test]
1417 fn test_poll_batch_rejects_out_of_range_id() {
1418 let ring = make_ring(16);
1419 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1420 let chain = producer.chain().writable(16).build().unwrap();
1421 producer.submit(chain).unwrap();
1422 let mut desc = ring.read_desc(0);
1423 desc.id = 16;
1424 ring.write_desc(0, desc);
1425
1426 assert!(matches!(
1428 consumer.poll_batch(1, 0, 0),
1429 Err(VirtqError::RingError(RingError::InvalidState))
1430 ));
1431 assert_eq!(consumer.avail_cursor().head(), 0);
1432 assert_eq!(consumer.used_cursor().head(), 0);
1433 assert_eq!(consumer.inner.num_inflight(), 0);
1434 assert_eq!(consumer.next_token, 0);
1435 assert!(producer.poll().unwrap().is_none());
1436
1437 drop(consumer);
1438 unsafe { producer.reset().unwrap() };
1440 assert_eq!(producer.pool().num_live(), 0);
1441 }
1442
1443 #[test]
1444 fn test_poll_batch_rejects_duplicate_id_after_partial_batch() {
1445 let ring = make_ring(16);
1446 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1447 let chain = producer.chain().writable(16).build().unwrap();
1448 let first = producer.submit(chain).unwrap();
1449 let chain = producer.chain().writable(16).build().unwrap();
1450 producer.submit(chain).unwrap();
1451 let mut desc = ring.read_desc(1);
1452 desc.id = first.id;
1453 ring.write_desc(1, desc);
1454
1455 assert!(matches!(
1457 consumer.poll_batch(2, 0, 0),
1458 Err(VirtqError::RingError(RingError::InvalidState))
1459 ));
1460 assert_eq!(consumer.avail_cursor().head(), 1);
1461 assert_eq!(consumer.used_cursor().head(), 0);
1462 assert_eq!(consumer.inner.num_inflight(), 1);
1463 assert_eq!(consumer.next_token, 1);
1464 assert!(producer.poll().unwrap().is_none());
1465
1466 drop(consumer);
1467 unsafe { producer.reset().unwrap() };
1469 assert_eq!(producer.pool().num_live(), 0);
1470 }
1471
1472 #[test]
1473 fn test_poll_too_large_returns_payload_error() {
1474 let ring = make_ring(16);
1475 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1476
1477 let mut se = producer.chain().readable(8).writable(16).build().unwrap();
1478 se.write_all(b"too much").unwrap();
1479 producer.submit(se).unwrap();
1480
1481 assert!(matches!(
1482 consumer.poll(4),
1483 Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1484 ));
1485 producer.drain(drop).unwrap();
1486 }
1487
1488 #[test]
1489 fn test_poll_too_large_returns_descriptor() {
1490 let ring = make_ring(16);
1491 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1492
1493 let mut se = producer.chain().readable(8).writable(16).build().unwrap();
1494 se.write_all(b"too much").unwrap();
1495 let token = producer.submit(se).unwrap();
1496
1497 assert!(matches!(
1500 consumer.poll(4),
1501 Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1502 ));
1503
1504 let used = producer.poll().unwrap().unwrap();
1506 assert_eq!(used.token(), token);
1507
1508 let se2 = producer.chain().writable(16).build().unwrap();
1510 producer.submit(se2).unwrap();
1511 let (recv, reply) = poll_data(&mut consumer);
1512 consumer.complete(recv, reply).unwrap();
1513 assert!(producer.poll().unwrap().is_some());
1514 }
1515
1516 #[test]
1517 fn test_villain_indirect_descriptor_does_not_mark_high_level_inflight() {
1518 let ring = make_ring(16);
1519 let mem = ring.mem();
1520 let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1521
1522 let mut desc = Descriptor::new(0x1000, 16, 0, DescFlags::INDIRECT);
1523 desc.mark_avail(true);
1524 ring.write_desc(0, desc);
1525
1526 assert!(matches!(
1527 consumer.poll(1024),
1528 Err(VirtqError::RingError(RingError::BadChain))
1529 ));
1530 assert_eq!(consumer.inner.num_inflight(), 0);
1531 }
1532
1533 #[test]
1534 fn test_villain_bad_chain_does_not_mark_high_level_inflight() {
1535 let ring = make_ring(16);
1536 let mem = ring.mem();
1537 let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1538
1539 let mut first = Descriptor::new(0x1000, 16, 0, DescFlags::NEXT | DescFlags::WRITE);
1540 first.mark_avail(true);
1541 ring.write_desc(0, first);
1542
1543 let mut second = Descriptor::new(0x2000, 16, 0, DescFlags::empty());
1544 second.mark_avail(true);
1545 ring.write_desc(1, second);
1546
1547 assert!(matches!(
1548 consumer.poll(1024),
1549 Err(VirtqError::RingError(RingError::BadChain))
1550 ));
1551 assert_eq!(consumer.inner.num_inflight(), 0);
1552 }
1553
1554 #[test]
1555 fn test_writable_chain_writes_single_segment() {
1556 let ring = make_ring(16);
1557 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1558
1559 let se = producer.chain().writable(16).build().unwrap();
1560 producer.submit(se).unwrap();
1561 let (recv, reply) = poll_data(&mut consumer);
1562
1563 let ReplyChain::Writable(mut wc) = reply else {
1564 panic!("expected Writable");
1565 };
1566 wc.write_all(b"hello").unwrap();
1567 consumer.complete(recv, wc).unwrap();
1568
1569 let used = producer.poll().unwrap().unwrap();
1570 assert_eq!(used.to_bytes().unwrap().as_ref(), b"hello");
1571 }
1572
1573 #[test]
1574 fn test_writable_rewind() {
1575 let ring = make_ring(16);
1576 let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 16);
1577
1578 let se = producer.chain().writable(16).build().unwrap();
1579 producer.submit(se).unwrap();
1580
1581 let (recv, reply) = poll_data(&mut consumer);
1582
1583 if let ReplyChain::Writable(mut wc) = reply {
1584 wc.write_all(b"first").unwrap();
1585 assert_eq!(wc.written(), 5);
1586 wc.rewind();
1587 assert_eq!(wc.written(), 0);
1588 assert_eq!(wc.remaining(), 16);
1589 wc.write_all(b"second").unwrap();
1590 assert_eq!(wc.written(), 6);
1591 consumer.complete(recv, wc).unwrap();
1592 } else {
1593 panic!("expected Writable");
1594 }
1595 producer.drain(drop).unwrap();
1596 }
1597
1598 #[test]
1599 fn test_writable_reply_scatters_across_segments() {
1600 let ring = make_ring(16);
1601 let mem = ring.mem();
1602 let mut ring_producer = make_producer(&ring);
1603 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1604
1605 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1606 let chain = BufferChainBuilder::new()
1607 .writable(base, 4)
1608 .writable(base + 4, 4)
1609 .build()
1610 .unwrap();
1611 let id = ring_producer.submit_available(&chain).unwrap();
1612
1613 let (recv, reply) = poll_data(&mut consumer);
1614 assert!(recv.to_bytes().unwrap().is_empty());
1615
1616 let ReplyChain::Writable(mut wc) = reply else {
1617 panic!("expected Writable");
1618 };
1619 assert_eq!(wc.capacity(), 8);
1620 wc.write_all(b"abcdefgh").unwrap();
1621 assert_eq!(wc.written(), 8);
1622 consumer.complete(recv, wc).unwrap();
1623
1624 let mut first = [0u8; 4];
1625 let mut second = [0u8; 4];
1626 mem.read(base, &mut first).unwrap();
1627 mem.read(base + 4, &mut second).unwrap();
1628 assert_eq!(&first, b"abcd");
1629 assert_eq!(&second, b"efgh");
1630
1631 let used = ring_producer.poll_used().unwrap();
1632 assert_eq!(used.id, id);
1633 assert_eq!(used.len, 8);
1634 }
1635
1636 #[test]
1637 fn test_writable_short_write_reports_contiguous_used_length() {
1638 let ring = make_ring(16);
1639 let mem = ring.mem();
1640 let mut ring_producer = make_producer(&ring);
1641 let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1642 mem.write(base, &[0xff; 8]).unwrap();
1643
1644 let chain = BufferChainBuilder::new()
1645 .writable(base, 4)
1646 .writable(base + 4, 4)
1647 .build()
1648 .unwrap();
1649 let id = ring_producer.submit_available(&chain).unwrap();
1650
1651 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1652 let (recv, reply) = poll_data(&mut consumer);
1653 let ReplyChain::Writable(mut writable) = reply else {
1654 panic!("expected writable reply");
1655 };
1656
1657 writable.write_all(b"abc").unwrap();
1658 writable.write_all(b"def").unwrap();
1659 assert_eq!(writable.written(), 6);
1660 assert_eq!(writable.remaining(), 2);
1661
1662 consumer.complete(recv, writable).unwrap();
1663
1664 let mut contents = [0u8; 8];
1665 mem.read(base, &mut contents).unwrap();
1666 assert_eq!(&contents, b"abcdef\xff\xff");
1667
1668 let used = ring_producer.poll_used().unwrap();
1669 assert_eq!(used.id, id);
1670 assert_eq!(used.len, 6);
1671 }
1672
1673 #[test]
1674 fn test_multiple_pending_replies() {
1675 let ring = make_ring(16);
1676 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1677
1678 let se1 = producer.chain().writable(16).build().unwrap();
1679 producer.submit(se1).unwrap();
1680 let se2 = producer.chain().writable(16).build().unwrap();
1681 producer.submit(se2).unwrap();
1682
1683 let (e1, c1) = poll_data(&mut consumer);
1684 let (e2, c2) = poll_data(&mut consumer);
1685
1686 consumer.complete(e2, c2).unwrap();
1688 consumer.complete(e1, c1).unwrap();
1689 producer.drain(drop).unwrap();
1690 }
1691
1692 #[test]
1693 fn test_mismatched_completion_fails_closed() {
1694 let ring = make_ring(16);
1695 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1696
1697 let mut first = producer.chain().readable(1).build().unwrap();
1698 first.write_all(b"a").unwrap();
1699 producer.submit(first).unwrap();
1700
1701 let mut second = producer.chain().readable(1).build().unwrap();
1702 second.write_all(b"b").unwrap();
1703 producer.submit(second).unwrap();
1704
1705 let (recv1, reply1) = poll_data(&mut consumer);
1706 let (recv2, reply2) = poll_data(&mut consumer);
1707
1708 assert!(matches!(
1709 consumer.complete(recv1, reply2),
1710 Err(VirtqError::InvalidState)
1711 ));
1712 assert_eq!(consumer.inner.num_inflight(), 2);
1713 assert!(producer.poll().unwrap().is_none());
1714 assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1715
1716 drop((recv2, reply1));
1717
1718 unsafe { producer.reset() }.unwrap();
1720 }
1721
1722 #[test]
1723 fn test_recv_to_bytes_preserves_reader_position() {
1724 let ring = make_ring(16);
1725 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1726
1727 let mut se = producer.chain().readable(16).build().unwrap();
1728 se.write_all(b"abc").unwrap();
1729 producer.submit(se).unwrap();
1730
1731 let (mut recv, reply) = poll_data(&mut consumer);
1732 let mut first = [0u8; 1];
1733 recv.read_exact(&mut first).unwrap();
1734 let data = recv.to_bytes().unwrap();
1735 assert_eq!(&first, b"a");
1736 assert_eq!(data.as_ref(), b"abc");
1737 assert_eq!(recv.consumed(), 1);
1738 consumer.complete(recv, reply).unwrap();
1739 producer.drain(drop).unwrap();
1740 }
1741
1742 #[test]
1743 fn test_virtq_consumer_reset() {
1744 let ring = make_ring(16);
1745 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1746
1747 let se = producer.chain().writable(16).build().unwrap();
1749 producer.submit(se).unwrap();
1750
1751 let (recv, reply) = poll_data(&mut consumer);
1752 assert!(consumer.inner.num_inflight() > 0);
1753 assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1754
1755 consumer.complete(recv, reply).unwrap();
1757
1758 consumer.reset().unwrap();
1759
1760 assert_eq!(consumer.inner.num_inflight(), 0);
1761
1762 unsafe { producer.reset() }.unwrap();
1764 }
1765
1766 #[test]
1767 fn test_virtq_consumer_reset_clears_inflight() {
1768 let ring = make_ring(16);
1769 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1770
1771 let se1 = producer.chain().writable(16).build().unwrap();
1773 producer.submit(se1).unwrap();
1774 let se2 = producer.chain().writable(16).build().unwrap();
1775 producer.submit(se2).unwrap();
1776
1777 let (e1, c1) = poll_data(&mut consumer);
1778 let (e2, c2) = poll_data(&mut consumer);
1779 consumer.complete(e1, c1).unwrap();
1781 consumer.complete(e2, c2).unwrap();
1782
1783 consumer.reset().unwrap();
1784
1785 assert_eq!(consumer.inner.num_inflight(), 0);
1786
1787 unsafe { producer.reset() }.unwrap();
1789 }
1790
1791 #[test]
1792 fn failed_completion_keeps_reset_blocked() {
1793 use crate::virtq::ring::tests::FaultMem;
1794
1795 let ring = make_ring(4);
1796 let (mut producer, _, _) = make_test_producer(&ring);
1797 let mem = FaultMem::new(ring.mem());
1798 let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1799 let chain = producer.chain().writable(8).build().unwrap();
1800 producer.submit(chain).unwrap();
1801 let (recv, reply) = consumer.poll(0).unwrap().unwrap();
1802
1803 mem.fail_write_at(0);
1804 assert!(matches!(
1805 consumer.complete(recv, reply),
1806 Err(VirtqError::RingError(RingError::MemError { .. }))
1807 ));
1808 mem.allow_writes();
1809 assert_eq!(consumer.inner.num_inflight(), 1);
1810 assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1811
1812 unsafe { producer.reset() }.unwrap();
1814 }
1815}