1mod access;
151mod buffer;
152mod consumer;
153mod desc;
154mod event;
155mod pool;
156mod producer;
157mod ring;
158
159#[cfg(all(test, loom))]
160mod concurrency;
161
162use alloc::vec::Vec;
163use core::num::NonZeroU16;
164
165pub use access::*;
166pub use buffer::*;
167pub use consumer::*;
168pub use desc::*;
169pub use event::*;
170pub use pool::*;
171pub use producer::*;
172pub use ring::*;
173use thiserror::Error;
174
175pub const G2H_LOWER_SLOT_SIZE: usize = 256;
177pub const G2H_LOWER_SLOT_COUNT: usize = crate::vmem::PAGE_SIZE / G2H_LOWER_SLOT_SIZE;
179
180const _: () = assert!(G2H_LOWER_SLOT_COUNT * G2H_LOWER_SLOT_SIZE == crate::vmem::PAGE_SIZE);
181
182pub trait Notifier {
184 fn notify(&self, stats: QueueStats);
185}
186
187#[derive(Error, Debug)]
189pub enum VirtqError {
190 #[error("Ring error: {0}")]
191 RingError(RingError),
192 #[error("Allocation error: {0}")]
193 Alloc(AllocError),
194 #[error("Ring or pool temporarily full")]
195 Backpressure,
196 #[error("Allocation exceeds pool capacity")]
197 OutOfMemory,
198 #[error("Failed to allocate virtqueue storage")]
200 Bookkeeping,
201 #[error("Invalid chain received")]
202 BadChain,
203 #[error("Payload data too large: received {recv} bytes, limit {limit} bytes")]
204 PayloadTooLarge { recv: usize, limit: usize },
205 #[error("Receive data too short: requested {requested} bytes, only {remaining} bytes remain")]
206 ReceiveTooShort { requested: usize, remaining: usize },
207 #[error("Reply data too large for allocated buffer")]
208 ReplyTooLarge,
209 #[error("Internal state error")]
210 InvalidState,
211 #[error("Memory write error")]
212 MemoryWriteError,
213 #[error("Memory read error")]
214 MemoryReadError,
215 #[error("No payload segment in this chain")]
216 NoPayloadSegment,
217}
218
219impl VirtqError {
220 #[inline(always)]
222 pub fn is_transient(&self) -> bool {
223 matches!(self, Self::Backpressure)
224 }
225}
226
227impl From<RingError> for VirtqError {
228 fn from(e: RingError) -> Self {
229 match e {
230 RingError::WouldBlock => Self::Backpressure,
231 RingError::Bookkeeping => Self::Bookkeeping,
232 other => Self::RingError(other),
233 }
234 }
235}
236
237impl From<AllocError> for VirtqError {
238 fn from(e: AllocError) -> Self {
239 match e {
240 AllocError::NoSpace => Self::Backpressure,
241 AllocError::OutOfMemory => Self::OutOfMemory,
242 AllocError::Bookkeeping => Self::Bookkeeping,
243 other => Self::Alloc(other),
244 }
245 }
246}
247
248#[derive(Clone, Copy, Debug)]
261pub struct Layout {
262 desc_table_addr: u64,
264 desc_table_len: u16,
266 drv_evt_addr: u64,
268 dev_evt_addr: u64,
270}
271
272#[inline]
273const fn align_up(val: usize, align: usize) -> usize {
274 val.next_multiple_of(align)
275}
276
277impl Layout {
278 pub const unsafe fn from_base(base: u64, num_descs: NonZeroU16) -> Result<Self, RingError> {
289 let num_descs = num_descs.get() as usize;
290 if !num_descs.is_power_of_two() {
291 return Err(RingError::InvalidLayout);
292 }
293
294 if !base.is_multiple_of(Descriptor::ALIGN as u64) {
295 return Err(RingError::InvalidLayout);
296 }
297
298 if base
299 .checked_add(Layout::query_size(num_descs) as u64)
300 .is_none()
301 {
302 return Err(RingError::InvalidLayout);
303 }
304
305 let desc_size = num_descs * Descriptor::SIZE;
306 let event_size = EventSuppression::SIZE;
307 let event_align = EventSuppression::ALIGN;
308
309 let drv_evt_offset = align_up(desc_size, event_align);
310 let dev_evt_offset = align_up(drv_evt_offset + event_size, event_align);
311
312 Ok(Self {
313 desc_table_addr: base,
314 desc_table_len: num_descs as u16,
315 drv_evt_addr: base + drv_evt_offset as u64,
316 dev_evt_addr: base + dev_evt_offset as u64,
317 })
318 }
319
320 pub const fn desc_table_addr(&self) -> u64 {
322 self.desc_table_addr
323 }
324
325 pub const fn desc_table_len(&self) -> u16 {
327 self.desc_table_len
328 }
329
330 pub const fn drv_evt_addr(&self) -> u64 {
332 self.drv_evt_addr
333 }
334
335 pub const fn dev_evt_addr(&self) -> u64 {
337 self.dev_evt_addr
338 }
339
340 pub const fn query_size(num_descs: usize) -> usize {
343 let desc_size = num_descs * Descriptor::SIZE;
344 let event_size = EventSuppression::SIZE;
345 let event_align = EventSuppression::ALIGN;
346
347 let drv_evt_offset = align_up(desc_size, event_align);
349 let dev_evt_offset = align_up(drv_evt_offset + event_size, event_align);
350
351 dev_evt_offset + event_size
352 }
353}
354
355#[derive(Debug, Clone, Copy)]
360pub struct QueueStats {
361 pub num_free: usize,
363 pub num_inflight: usize,
365}
366
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
372pub enum SuppressionKind {
373 Enable,
375 Disable,
377 Descriptor(RingCursor),
379}
380
381#[derive(Copy, Clone, Debug, PartialEq, Eq)]
386pub struct Token {
387 pub seq: u32,
389 pub id: u16,
391}
392
393impl From<BufferElement> for Allocation {
394 fn from(value: BufferElement) -> Self {
395 Allocation {
396 addr: value.addr,
397 len: value.len,
398 }
399 }
400}
401
402const _: () = {
403 #[allow(clippy::panic)]
404 #[allow(clippy::unwrap_used)]
405 const fn verify_layout(num_descs: usize) {
406 let base = 0x1000u64;
407
408 let layout =
410 match unsafe { Layout::from_base(base, NonZeroU16::new(num_descs as u16).unwrap()) } {
411 Ok(l) => l,
412 Err(_) => panic!("from_base failed"),
413 };
414
415 let expected_size = Layout::query_size(num_descs);
416
417 assert!(layout.desc_table_addr() == base);
418 assert!(layout.desc_table_len() as usize == num_descs);
419 assert!(
420 layout
421 .drv_evt_addr()
422 .is_multiple_of(EventSuppression::ALIGN as u64)
423 );
424 assert!(
425 layout
426 .dev_evt_addr()
427 .is_multiple_of(EventSuppression::ALIGN as u64)
428 );
429
430 let desc_end = base + (num_descs * Descriptor::SIZE) as u64;
432 assert!(layout.drv_evt_addr() >= desc_end);
433 assert!(layout.dev_evt_addr() >= layout.drv_evt_addr() + EventSuppression::SIZE as u64);
434
435 let layout_end = layout.dev_evt_addr() + EventSuppression::SIZE as u64;
437 assert!(base + expected_size as u64 == layout_end);
438 }
439
440 unsafe {
441 assert!(Layout::from_base(u64::MAX, NonZeroU16::new(1).unwrap()).is_err());
442 }
443
444 verify_layout(1);
445 verify_layout(2);
446 verify_layout(4);
447 verify_layout(8);
448 verify_layout(16);
449 verify_layout(32);
450 verify_layout(64);
451 verify_layout(128);
452 verify_layout(256);
453 verify_layout(512);
454 verify_layout(1024);
455};
456
457pub(crate) fn zeroed_vec(length: usize) -> Result<Vec<u8>, VirtqError> {
458 let mut bytes = Vec::new();
459 bytes
460 .try_reserve_exact(length)
461 .map_err(|_| VirtqError::Bookkeeping)?;
462
463 bytes.resize(length, 0);
464 Ok(bytes)
465}
466
467#[cfg(test)]
469pub(crate) mod test_utils {
470 use alloc::sync::Arc;
471 use core::sync::atomic::{AtomicUsize, Ordering};
472
473 use super::*;
474 use crate::virtq::ring::tests::{OwnedRing, TestMem};
475
476 #[derive(Debug, Clone)]
478 pub(crate) struct TestNotifier {
479 pub(crate) count: Arc<AtomicUsize>,
480 }
481
482 impl TestNotifier {
483 pub(crate) fn new() -> Self {
484 Self {
485 count: Arc::new(AtomicUsize::new(0)),
486 }
487 }
488
489 pub(crate) fn notification_count(&self) -> usize {
490 self.count.load(Ordering::Relaxed)
491 }
492 }
493
494 impl Notifier for TestNotifier {
495 fn notify(&self, _stats: QueueStats) {
496 self.count.fetch_add(1, Ordering::Relaxed);
497 }
498 }
499
500 type TestProducer = VirtqProducer<TestMem, TestNotifier>;
501 type TestConsumer = VirtqConsumer<TestMem, TestNotifier>;
502
503 pub(crate) fn make_test_producer(
506 ring: &OwnedRing,
507 ) -> (TestProducer, TestConsumer, TestNotifier) {
508 make_test_producer_with_slot_size(ring, 128)
509 }
510
511 pub(crate) fn make_test_producer_with_slot_size(
512 ring: &OwnedRing,
513 slot_size: usize,
514 ) -> (TestProducer, TestConsumer, TestNotifier) {
515 let layout = ring.layout();
516 let mem = ring.mem();
517
518 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
520 let pool_layout = SlotLayout::new(pool_base, slot_size, 0x8000 / slot_size).unwrap();
521 let pool = SlotPool::new(pool_layout).unwrap();
522 let notifier = TestNotifier::new();
523
524 let producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool);
525 let consumer = VirtqConsumer::new(layout, mem, notifier.clone());
526
527 (producer, consumer, notifier)
528 }
529}
530
531#[cfg(test)]
532mod tests {
533 use alloc::sync::Arc;
534 use core::sync::atomic::{AtomicUsize, Ordering};
535
536 use super::*;
537 use crate::virtq::ring::tests::{TestMem, make_ring};
538 use crate::virtq::test_utils::*;
539
540 fn send_readwrite(
542 producer: &mut VirtqProducer<TestMem, TestNotifier>,
543 entry_data: &[u8],
544 used_cap: usize,
545 ) -> Token {
546 let mut se = producer
547 .chain()
548 .readable(entry_data.len())
549 .writable(used_cap)
550 .build()
551 .unwrap();
552 se.write_all(entry_data).unwrap();
553 producer.submit(se).unwrap()
554 }
555
556 fn poll_received(
557 consumer: &mut VirtqConsumer<TestMem, TestNotifier>,
558 ) -> (RecvChain<TestMem>, ReplyChain<TestMem>) {
559 consumer.poll(1024).unwrap().unwrap()
560 }
561
562 #[test]
563 fn test_payload_capacity_overflow_is_an_error() {
564 assert!(matches!(
565 zeroed_vec(usize::MAX),
566 Err(VirtqError::Bookkeeping)
567 ));
568 }
569
570 #[test]
571 fn test_submit_notifies() {
572 let ring = make_ring(16);
573 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
574
575 let initial_count = notifier.notification_count();
576
577 let token = send_readwrite(&mut producer, b"hello", 64);
578 assert!(notifier.notification_count() > initial_count);
579
580 let (recv, reply) = poll_received(&mut consumer);
581 assert_eq!(recv.token(), token);
582 consumer.complete(recv, reply).unwrap();
583 producer.drain(drop).unwrap();
584 }
585
586 #[test]
587 fn test_multiple_submits() {
588 let ring = make_ring(16);
589 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
590
591 let tok1 = send_readwrite(&mut producer, b"request1", 64);
592 let tok2 = send_readwrite(&mut producer, b"request2", 64);
593 let tok3 = send_readwrite(&mut producer, b"request3", 64);
594
595 for _ in 0..3 {
597 let (recv, reply) = poll_received(&mut consumer);
598 consumer.complete(recv, reply).unwrap();
599 }
600
601 let used1 = producer.poll().unwrap().unwrap();
603 let used2 = producer.poll().unwrap().unwrap();
604 let used3 = producer.poll().unwrap().unwrap();
605 assert!(
606 [used1.token(), used2.token(), used3.token()].contains(&tok1)
607 && [used1.token(), used2.token(), used3.token()].contains(&tok2)
608 && [used1.token(), used2.token(), used3.token()].contains(&tok3)
609 );
610 }
611
612 #[test]
613 fn test_completion_batching_with_suppression() {
614 let ring = make_ring(16);
615 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
616
617 let tok1 = send_readwrite(&mut producer, b"req1", 64);
619 let tok2 = send_readwrite(&mut producer, b"req2", 64);
620 let tok3 = send_readwrite(&mut producer, b"req3", 64);
621
622 let cursor = producer.used_cursor();
624 producer
625 .set_used_suppression(SuppressionKind::Descriptor(cursor))
626 .unwrap();
627
628 for _ in 0..3 {
630 let (recv, reply) = poll_received(&mut consumer);
631 let ReplyChain::Writable(mut wc) = reply else {
632 panic!("expected writable reply");
633 };
634 wc.write_all(b"used-data").unwrap();
635 consumer.complete(recv, wc).unwrap();
636 }
637
638 let mut responses = Vec::new();
640 producer
641 .drain(|reply| {
642 responses.push(reply.token());
643 })
644 .unwrap();
645
646 assert_eq!(responses.len(), 3);
647 assert!(responses.contains(&tok1));
648 assert!(responses.contains(&tok2));
649 assert!(responses.contains(&tok3));
650 }
651
652 #[test]
653 fn test_notifier_receives_context() {
654 #[derive(Debug, Clone)]
655 struct CtxNotifier {
656 last_num_free: Arc<AtomicUsize>,
657 last_num_inflight: Arc<AtomicUsize>,
658 count: Arc<AtomicUsize>,
659 }
660
661 impl Notifier for CtxNotifier {
662 fn notify(&self, stats: QueueStats) {
663 self.last_num_free.store(stats.num_free, Ordering::Relaxed);
664 self.last_num_inflight
665 .store(stats.num_inflight, Ordering::Relaxed);
666 self.count.fetch_add(1, Ordering::Relaxed);
667 }
668 }
669
670 let ring = make_ring(16);
671 let layout = ring.layout();
672 let mem = ring.mem();
673 let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
674 let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
675 let pool = SlotPool::new(pool_layout).unwrap();
676 let notifier = CtxNotifier {
677 last_num_free: Arc::new(AtomicUsize::new(0)),
678 last_num_inflight: Arc::new(AtomicUsize::new(0)),
679 count: Arc::new(AtomicUsize::new(0)),
680 };
681
682 let mut producer = VirtqProducer::new(layout, mem, notifier.clone(), pool);
683
684 let mut se = producer.chain().readable(4).writable(32).build().unwrap();
685 se.write_all(b"test").unwrap();
686 producer.submit(se).unwrap();
687 assert_eq!(notifier.count.load(Ordering::Relaxed), 1);
688 assert!(notifier.last_num_inflight.load(Ordering::Relaxed) > 0);
689
690 unsafe { producer.reset() }.unwrap();
692 }
693
694 #[test]
695 fn test_chain_batch() {
696 let ring = make_ring(16);
697 let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
698
699 let initial_count = notifier.notification_count();
700
701 let mut se1 = producer.chain().readable(64).writable(128).build().unwrap();
703 se1.write_all(b"first-ent").unwrap();
704 let _tok1 = producer.submit(se1).unwrap();
705
706 let mut se2 = producer.chain().readable(64).writable(64).build().unwrap();
708 se2.write_all(b"copy-ent").unwrap();
709 let _tok2 = producer.submit(se2).unwrap();
710
711 let se3 = producer.chain().writable(32).build().unwrap();
713 let tok3 = producer.submit(se3).unwrap();
714
715 assert!(notifier.notification_count() > initial_count);
717
718 let (recv1, reply1) = poll_received(&mut consumer);
720 assert_eq!(recv1.to_bytes().unwrap().as_ref(), b"first-ent");
721 consumer.complete(recv1, reply1).unwrap();
722
723 let (recv2, reply2) = poll_received(&mut consumer);
724 assert_eq!(recv2.to_bytes().unwrap().as_ref(), b"copy-ent");
725 consumer.complete(recv2, reply2).unwrap();
726
727 let (recv3, reply3) = poll_received(&mut consumer);
728 let ReplyChain::Writable(mut wc) = reply3 else {
729 panic!("expected writable reply");
730 };
731 wc.write_all(b"resp").unwrap();
732 consumer.complete(recv3, wc).unwrap();
733
734 let _ = producer.poll().unwrap().unwrap();
736 let _ = producer.poll().unwrap().unwrap();
737
738 let used = producer.poll().unwrap().unwrap();
739 assert_eq!(used.token(), tok3);
740 assert_eq!(used.to_bytes().unwrap().as_ref(), b"resp");
741 }
742
743 #[test]
744 fn test_chain_write_send() {
745 let ring = make_ring(16);
746 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
747
748 let mut se = producer.chain().readable(64).writable(128).build().unwrap();
749 se.write_all(b"hello").unwrap();
750 let token = producer.submit(se).unwrap();
751
752 let (recv, reply) = poll_received(&mut consumer);
754 assert_eq!(recv.token(), token);
755 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
756
757 let ReplyChain::Writable(mut wc) = reply else {
759 panic!("expected writable reply");
760 };
761 wc.write_all(b"world").unwrap();
762 consumer.complete(recv, wc).unwrap();
763 let used = producer.poll().unwrap().unwrap();
764 assert_eq!(used.to_bytes().unwrap().as_ref(), b"world");
765 }
766
767 #[test]
768 fn test_full_round_trip() {
769 let ring = make_ring(16);
770 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
771
772 let token = send_readwrite(&mut producer, b"round-trip-recv", 128);
774
775 let (recv, reply) = poll_received(&mut consumer);
777 assert_eq!(recv.token(), token);
778 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"round-trip-recv");
779
780 let ReplyChain::Writable(mut wc) = reply else {
781 panic!("expected writable reply");
782 };
783 assert!(wc.capacity() >= 128);
784 wc.write_all(b"round-trip-rsp").unwrap();
785 consumer.complete(recv, wc).unwrap();
786
787 let used = producer.poll().unwrap().unwrap();
789 assert_eq!(used.token(), token);
790 assert_eq!(used.to_bytes().unwrap().as_ref(), b"round-trip-rsp");
791 }
792
793 #[test]
794 fn test_cancel_submits_zero_length() {
795 let ring = make_ring(16);
796 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
797
798 let token = send_readwrite(&mut producer, b"recv-data", 64);
799
800 let (recv, reply) = poll_received(&mut consumer);
801 consumer.complete(recv, reply).unwrap();
802
803 let used = producer.poll().unwrap().unwrap();
804 assert_eq!(used.token(), token);
805 assert_eq!(used.to_bytes().unwrap().len(), 0);
806 assert!(used.to_bytes().unwrap().is_empty());
807 }
808
809 #[test]
810 fn test_hold_reply_and_complete() {
811 let ring = make_ring(16);
812 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
813
814 let token = send_readwrite(&mut producer, b"deferred", 64);
815
816 let (recv, reply) = poll_received(&mut consumer);
818 assert_eq!(recv.token(), token);
819 assert_eq!(recv.to_bytes().unwrap().as_ref(), b"deferred");
820
821 let ReplyChain::Writable(mut wc) = reply else {
822 panic!("expected writable reply");
823 };
824 wc.write_all(b"deferred-used").unwrap();
825 consumer.complete(recv, wc).unwrap();
826
827 let used = producer.poll().unwrap().unwrap();
828 assert_eq!(used.token(), token);
829 assert_eq!(used.to_bytes().unwrap().as_ref(), b"deferred-used");
830 }
831
832 #[test]
833 fn test_concurrent_pending_replies() {
834 let ring = make_ring(16);
835 let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
836
837 let tok1 = send_readwrite(&mut producer, b"first", 64);
838 let tok2 = send_readwrite(&mut producer, b"second", 64);
839
840 let (recv1, reply1) = poll_received(&mut consumer);
842 assert_eq!(recv1.token(), tok1);
843 assert_eq!(recv1.to_bytes().unwrap().as_ref(), b"first");
844
845 let (recv2, reply2) = poll_received(&mut consumer);
846 assert_eq!(recv2.token(), tok2);
847 assert_eq!(recv2.to_bytes().unwrap().as_ref(), b"second");
848
849 let ReplyChain::Writable(mut wc2) = reply2 else {
851 panic!("expected writable");
852 };
853 wc2.write_all(b"resp2").unwrap();
854 consumer.complete(recv2, wc2).unwrap();
855
856 let ReplyChain::Writable(mut wc1) = reply1 else {
857 panic!("expected writable");
858 };
859 wc1.write_all(b"resp1").unwrap();
860 consumer.complete(recv1, wc1).unwrap();
861
862 let used1 = producer.poll().unwrap().unwrap();
863 let used2 = producer.poll().unwrap().unwrap();
864 let mut responses: Vec<_> = vec![
865 (used1.token(), used1.to_bytes().unwrap().to_vec()),
866 (used2.token(), used2.to_bytes().unwrap().to_vec()),
867 ];
868 responses.sort_by_key(|(t, _)| t.seq);
869
870 let expected_first = responses.iter().find(|(t, _)| *t == tok1).unwrap();
871 let expected_second = responses.iter().find(|(t, _)| *t == tok2).unwrap();
872 assert_eq!(&expected_first.1[..], b"resp1");
873 assert_eq!(&expected_second.1[..], b"resp2");
874 }
875
876 fn send_readonly(
878 producer: &mut VirtqProducer<TestMem, TestNotifier>,
879 entry_data: &[u8],
880 ) -> Token {
881 let mut se = producer.chain().readable(entry_data.len()).build().unwrap();
882 se.write_all(entry_data).unwrap();
883 producer.submit(se).unwrap()
884 }
885
886 #[test]
887 fn test_reclaim_frees_ring_slots() {
888 let ring = make_ring(4);
889 let (mut producer, mut consumer, _) = make_test_producer(&ring);
890
891 send_readonly(&mut producer, b"a");
893 send_readonly(&mut producer, b"b");
894 send_readonly(&mut producer, b"c");
895 send_readonly(&mut producer, b"d");
896 assert_eq!(producer.num_inflight(), 4);
897
898 let res = producer.chain().readable(1).build();
899 assert!(
900 matches!(res, Err(VirtqError::Backpressure)),
901 "expected Backpressure from full ring"
902 );
903
904 while let Some(result) = consumer.poll(1024).unwrap() {
906 let (recv, reply) = result;
907 consumer.complete(recv, reply).unwrap();
908 }
909
910 let count = producer.reclaim().unwrap();
912 assert_eq!(count, 4, "expected 4 reclaimed entries");
913 assert_eq!(producer.num_inflight(), 0);
914
915 send_readonly(&mut producer, b"e");
917
918 unsafe { producer.reset() }.unwrap();
920 }
921
922 #[test]
923 fn test_reclaim_buffers_rw_completions() {
924 let ring = make_ring(4);
925 let (mut producer, mut consumer, _) = make_test_producer(&ring);
926
927 let tok = send_readwrite(&mut producer, b"request", 64);
929
930 let (recv, reply) = poll_received(&mut consumer);
932 let ReplyChain::Writable(mut wc) = reply else {
933 panic!("expected writable");
934 };
935 wc.write_all(b"response-data").unwrap();
936 consumer.complete(recv, wc).unwrap();
937
938 let count = producer.reclaim().unwrap();
940 assert_eq!(count, 1);
941
942 let used = producer.poll().unwrap().unwrap();
944 assert_eq!(used.token(), tok);
945 assert_eq!(used.to_bytes().unwrap().as_ref(), b"response-data");
946 }
947
948 #[test]
949 fn test_reclaim_discards_readonly_completions() {
950 let ring = make_ring(8);
951 let (mut producer, mut consumer, _) = make_test_producer(&ring);
952
953 let _tok_ro1 = send_readonly(&mut producer, b"log1");
955 let tok_rw = send_readwrite(&mut producer, b"call", 64);
956 let _tok_ro2 = send_readonly(&mut producer, b"log2");
957
958 let (recv1, reply1) = poll_received(&mut consumer);
960 consumer.complete(recv1, reply1).unwrap(); let (recv2, reply2) = poll_received(&mut consumer);
963 let ReplyChain::Writable(mut wc) = reply2 else {
964 panic!("expected writable");
965 };
966 wc.write_all(b"result").unwrap();
967 consumer.complete(recv2, wc).unwrap(); let (recv3, reply3) = poll_received(&mut consumer);
970 consumer.complete(recv3, reply3).unwrap(); let count = producer.reclaim().unwrap();
974 assert_eq!(count, 3);
975
976 let used = producer.poll().unwrap().unwrap();
978 assert_eq!(used.token(), tok_rw);
979 assert_eq!(used.to_bytes().unwrap().as_ref(), b"result");
980
981 assert!(producer.poll().unwrap().is_none());
983 }
984
985 #[test]
986 fn test_reclaim_mixed_with_poll() {
987 let ring = make_ring(8);
988 let (mut producer, mut consumer, _) = make_test_producer(&ring);
989
990 send_readonly(&mut producer, b"x");
992 let tok_rw = send_readwrite(&mut producer, b"y", 64);
993
994 let (recv1, reply1) = poll_received(&mut consumer);
995 consumer.complete(recv1, reply1).unwrap();
996
997 let (recv2, reply2) = poll_received(&mut consumer);
998 let ReplyChain::Writable(mut wc) = reply2 else {
999 panic!("expected writable");
1000 };
1001 wc.write_all(b"reply").unwrap();
1002 consumer.complete(recv2, wc).unwrap();
1003
1004 let used1 = producer.poll().unwrap().unwrap();
1006 assert!(matches!(used1, UsedChain::Ack(_)));
1007
1008 let count = producer.reclaim().unwrap();
1010 assert_eq!(count, 1);
1011
1012 let used2 = producer.poll().unwrap().unwrap();
1014 assert_eq!(used2.token(), tok_rw);
1015 assert_eq!(used2.to_bytes().unwrap().as_ref(), b"reply");
1016 }
1017
1018 #[test]
1020 fn test_reclaim_submit_no_token_collision() {
1021 let ring = make_ring(8);
1022 let (mut producer, mut consumer, _) = make_test_producer(&ring);
1023
1024 let tok_old = send_readonly(&mut producer, b"log");
1026
1027 let (recv, reply) = poll_received(&mut consumer);
1028 consumer.complete(recv, reply).unwrap();
1029
1030 let count = producer.reclaim().unwrap();
1031 assert_eq!(count, 1);
1032
1033 let tok_new = send_readwrite(&mut producer, b"call", 64);
1035
1036 assert_ne!(
1038 tok_old, tok_new,
1039 "tokens must be unique across reclaim/submit cycles"
1040 );
1041
1042 let (recv, reply) = poll_received(&mut consumer);
1044 let ReplyChain::Writable(mut wc) = reply else {
1045 panic!("expected writable");
1046 };
1047 wc.write_all(b"result").unwrap();
1048 consumer.complete(recv, wc).unwrap();
1049
1050 let used = producer.poll().unwrap().unwrap();
1052 assert_eq!(used.token(), tok_new);
1053 assert_eq!(used.to_bytes().unwrap().as_ref(), b"result");
1054
1055 assert!(producer.poll().unwrap().is_none());
1057 }
1058
1059 #[test]
1061 fn test_reclaim_readonly_does_not_leak_pending() {
1062 let ring = make_ring(4);
1063 let (mut producer, mut consumer, _) = make_test_producer(&ring);
1064
1065 for _ in 0..10 {
1066 for _ in 0..4 {
1068 send_readonly(&mut producer, b"msg");
1069 }
1070
1071 while let Some(result) = consumer.poll(1024).unwrap() {
1073 let (recv, reply) = result;
1074 consumer.complete(recv, reply).unwrap();
1075 }
1076
1077 let count = producer.reclaim().unwrap();
1079 assert_eq!(count, 4);
1080
1081 assert!(
1083 producer.poll().unwrap().is_none(),
1084 "pending should be empty after reclaiming RO entries"
1085 );
1086 }
1087 }
1088}