Skip to main content

hyperlight_common/virtq/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4//! Packed Virtqueue Implementation
5//!
6//! This module provides a high-level API for virtio packed virtqueues, built on top of
7//! the lower-level ring primitives. It implements the VIRTIO 1.1+ packed ring format
8//! with proper memory ordering and event suppression support.
9//!
10//! # Architecture
11//!
12//! The implementation is split into layers:
13//!
14//! - **High-level API** ([`VirtqProducer`], [`VirtqConsumer`]): Manages buffer allocation,
15//!   chain lifecycle, and notification decisions. This is the recommended API
16//!   for most use cases.
17//!
18//! - **Ring primitives** ([`RingProducer`], [`RingConsumer`]): Low-level descriptor ring
19//!   operations with explicit buffer chain management. Use this when you need full control
20//!   over buffer layouts or custom allocation strategies.
21//!
22//! - **Descriptor and event types** ([`Descriptor`], [`EventSuppression`]): Raw virtio
23//!   data structures for direct memory manipulation.
24//!
25//! # Quick Start
26//!
27//! ## Single Readable/Writable Chain
28//!
29//! ```ignore
30//! // Producer (driver) side - build and submit a send chain
31//! let mut chain = producer.chain()
32//!     .readable(64)
33//!     .writable(128)
34//!     .build()?;
35//! chain.write_all(b"request data")?;
36//! let token = producer.submit(chain)?;
37//! // ... wait for notification ...
38//! if let Some(used) = producer.poll()? {
39//!     match used {
40//!         UsedChain::Data(_, segments) => process(segments),
41//!         UsedChain::Ack(_) => {}
42//!     }
43//! }
44//!
45//! // Consumer (device) side - receive a chain and reply/ack it
46//! if let Some((recv, reply)) = consumer.poll(max_recv_len)? {
47//!     let request = recv.to_bytes()?;
48//!     match reply {
49//!         ReplyChain::Writable(mut wc) => {
50//!             let response = handle(request);
51//!             wc.write_all(&response)?;
52//!             consumer.complete(recv, wc)?;
53//!         }
54//!         ReplyChain::Ack(ack) => {
55//!             consumer.complete(recv, ack)?;
56//!         }
57//!     }
58//! }
59//!
60//! // Multiple pending completions (no borrow on consumer)
61//! let mut pending = Vec::new();
62//! while let Some((recv, reply)) = consumer.poll(max_recv_len)? {
63//!     let result = process(&recv);
64//!     pending.push((result, recv, reply));
65//! }
66//! for (result, recv, reply) in pending {
67//!     consumer.complete(recv, reply)?;
68//! }
69//! ```
70//!
71//! ## Multiple Chains
72//!
73//! Each submit checks event suppression and notifies independently. Use
74//! [`VirtqProducer::batch`] when a higher-level protocol wants to publish
75//! multiple chains and kick the queue once.
76//!
77//! ```ignore
78//! let mut batch = producer.batch();
79//! for data in entries {
80//!     let mut chain = batch.chain()
81//!         .readable(data.len())
82//!         .writable(64)
83//!         .build()?;
84//!     chain.write_all(data)?;
85//!     batch.submit(chain)?;
86//! }
87//! batch.finish()?;
88//! ```
89//!
90//! ## Completion Batching with Event Suppression
91//!
92//! To receive a single notification when multiple requests complete:
93//!
94//! ```ignore
95//! // Submit chains
96//! for data in entries {
97//!     let mut chain = producer.chain()
98//!         .readable(data.len())
99//!         .writable(64)
100//!         .build()?;
101//!     chain.write_all(data)?;
102//!     producer.submit(chain)?;
103//! }
104//!
105//! // Tell device: "notify me only after completing past this cursor"
106//! let cursor = producer.used_cursor();
107//! producer.set_used_suppression(SuppressionKind::Descriptor(cursor))?;
108//!
109//! // Wait for single notification, then drain all responses
110//! producer.drain(|used| {
111//!     if let UsedChain::Data(token, data) = used {
112//!         handle_response(token, data);
113//!     }
114//! })?;
115//! ```
116//!
117//! # Event Suppression
118//!
119//! Both sides can control when they want to be notified using [`SuppressionKind`]:
120//!
121//! - [`SuppressionKind::Enable`]: Always notify (default, lowest latency)
122//! - [`SuppressionKind::Disable`]: Never notify (polling mode, lowest overhead)
123//! - [`SuppressionKind::Descriptor`]: Notify at specific ring position (batching)
124//!
125//! See [`VirtqProducer::set_used_suppression`] and [`VirtqConsumer::set_avail_suppression`].
126//!
127//! # Low-Level API
128//!
129//! For advanced use cases, the ring module exposes lower-level primitives:
130//!
131//! - [`RingProducer`] / [`RingConsumer`]: Direct ring access with [`BufferChain`] submission
132//! - [`BufferChainBuilder`]: Construct scatter-gather buffer lists
133//! - [`RingCursor`]: Track ring positions for event suppression
134//!
135//! Example using low-level API:
136//!
137//! ```ignore
138//! let chain = BufferChainBuilder::new()
139//!     .readable(header_addr, header_len)
140//!     .readable(data_addr, data_len)
141//!     .writable(response_addr, response_len)
142//!     .build()?;
143//!
144//! let result = ring_producer.submit_available_with_notify(&chain)?;
145//! if result.notify {
146//!     kick_device();
147//! }
148//! ```
149
150mod access;
151mod buffer;
152mod consumer;
153mod desc;
154mod event;
155mod pool;
156mod producer;
157mod ring;
158
159#[cfg(all(test, loom))]
160mod concurrency;
161
162use alloc::vec::Vec;
163use core::num::NonZeroU16;
164
165pub use access::*;
166pub use buffer::*;
167pub use consumer::*;
168pub use desc::*;
169pub use event::*;
170pub use pool::*;
171pub use producer::*;
172pub use ring::*;
173use thiserror::Error;
174
175/// Capacity of each fixed G2H lower-tier slot.
176pub const G2H_LOWER_SLOT_SIZE: usize = 256;
177/// Number of G2H lower-tier slots occupying the first pool page.
178pub const G2H_LOWER_SLOT_COUNT: usize = crate::vmem::PAGE_SIZE / G2H_LOWER_SLOT_SIZE;
179
180const _: () = assert!(G2H_LOWER_SLOT_COUNT * G2H_LOWER_SLOT_SIZE == crate::vmem::PAGE_SIZE);
181
182/// A trait for notifying the consumer about virtqueue events.
183pub trait Notifier {
184    fn notify(&self, stats: QueueStats);
185}
186
187/// Errors that can occur in the virtqueue operations.
188#[derive(Error, Debug)]
189pub enum VirtqError {
190    #[error("Ring error: {0}")]
191    RingError(RingError),
192    #[error("Allocation error: {0}")]
193    Alloc(AllocError),
194    #[error("Ring or pool temporarily full")]
195    Backpressure,
196    #[error("Allocation exceeds pool capacity")]
197    OutOfMemory,
198    /// Virtqueue-owned storage could not be allocated.
199    #[error("Failed to allocate virtqueue storage")]
200    Bookkeeping,
201    #[error("Invalid chain received")]
202    BadChain,
203    #[error("Payload data too large: received {recv} bytes, limit {limit} bytes")]
204    PayloadTooLarge { recv: usize, limit: usize },
205    #[error("Receive data too short: requested {requested} bytes, only {remaining} bytes remain")]
206    ReceiveTooShort { requested: usize, remaining: usize },
207    #[error("Reply data too large for allocated buffer")]
208    ReplyTooLarge,
209    #[error("Internal state error")]
210    InvalidState,
211    #[error("Memory write error")]
212    MemoryWriteError,
213    #[error("Memory read error")]
214    MemoryReadError,
215    #[error("No payload segment in this chain")]
216    NoPayloadSegment,
217}
218
219impl VirtqError {
220    /// Check if this error is transient or unrecoverable.
221    #[inline(always)]
222    pub fn is_transient(&self) -> bool {
223        matches!(self, Self::Backpressure)
224    }
225}
226
227impl From<RingError> for VirtqError {
228    fn from(e: RingError) -> Self {
229        match e {
230            RingError::WouldBlock => Self::Backpressure,
231            RingError::Bookkeeping => Self::Bookkeeping,
232            other => Self::RingError(other),
233        }
234    }
235}
236
237impl From<AllocError> for VirtqError {
238    fn from(e: AllocError) -> Self {
239        match e {
240            AllocError::NoSpace => Self::Backpressure,
241            AllocError::OutOfMemory => Self::OutOfMemory,
242            AllocError::Bookkeeping => Self::Bookkeeping,
243            other => Self::Alloc(other),
244        }
245    }
246}
247
248/// Layout of a packed virtqueue ring in shared memory.
249///
250/// Describes the memory addresses for the descriptor table and event suppression
251/// structures. Use [`from_base`](Self::from_base) to compute the layout from a
252/// base address, or [`query_size`](Self::query_size) to determine memory requirements.
253///
254/// # Memory Layout
255///
256/// The packed ring consists of:
257/// 1. Descriptor table: `num_descs` × 16 bytes, aligned to 16 bytes
258/// 2. Driver event suppression: 4 bytes, aligned to 4 bytes
259/// 3. Device event suppression: 4 bytes, aligned to 4 bytes
260#[derive(Clone, Copy, Debug)]
261pub struct Layout {
262    /// Packed ring descriptor table base in shared memory.
263    desc_table_addr: u64,
264    /// Number of descriptors (ring size, must be power of 2).
265    desc_table_len: u16,
266    /// Driver-written event suppression area in shared memory.
267    drv_evt_addr: u64,
268    /// Device-written event suppression area in shared memory.
269    dev_evt_addr: u64,
270}
271
272#[inline]
273const fn align_up(val: usize, align: usize) -> usize {
274    val.next_multiple_of(align)
275}
276
277impl Layout {
278    /// Create a Layout from a base address and number of descriptors.
279    ///
280    /// The base address must be aligned to `Descriptor::ALIGN`.
281    /// The number of descriptors must be a power of 2.
282    /// The memory region starting at `base` must be at least `Layout::query_size(num_descs)` bytes.
283    ///
284    /// # Safety
285    /// - `base` must be valid for `Layout::query_size(num_descs)` bytes.
286    /// - `base` must be aligned to `Descriptor::ALIGN`.
287    /// - Memory must remain valid for the lifetime of the ring.
288    pub const unsafe fn from_base(base: u64, num_descs: NonZeroU16) -> Result<Self, RingError> {
289        let num_descs = num_descs.get() as usize;
290        if !num_descs.is_power_of_two() {
291            return Err(RingError::InvalidLayout);
292        }
293
294        if !base.is_multiple_of(Descriptor::ALIGN as u64) {
295            return Err(RingError::InvalidLayout);
296        }
297
298        if base
299            .checked_add(Layout::query_size(num_descs) as u64)
300            .is_none()
301        {
302            return Err(RingError::InvalidLayout);
303        }
304
305        let desc_size = num_descs * Descriptor::SIZE;
306        let event_size = EventSuppression::SIZE;
307        let event_align = EventSuppression::ALIGN;
308
309        let drv_evt_offset = align_up(desc_size, event_align);
310        let dev_evt_offset = align_up(drv_evt_offset + event_size, event_align);
311
312        Ok(Self {
313            desc_table_addr: base,
314            desc_table_len: num_descs as u16,
315            drv_evt_addr: base + drv_evt_offset as u64,
316            dev_evt_addr: base + dev_evt_offset as u64,
317        })
318    }
319
320    /// Packed ring descriptor table base in shared memory.
321    pub const fn desc_table_addr(&self) -> u64 {
322        self.desc_table_addr
323    }
324
325    /// Number of descriptors in the ring.
326    pub const fn desc_table_len(&self) -> u16 {
327        self.desc_table_len
328    }
329
330    /// Driver-written event suppression area in shared memory.
331    pub const fn drv_evt_addr(&self) -> u64 {
332        self.drv_evt_addr
333    }
334
335    /// Device-written event suppression area in shared memory.
336    pub const fn dev_evt_addr(&self) -> u64 {
337        self.dev_evt_addr
338    }
339
340    /// Calculate the memory size needed for a ring with `num_descs` descriptors,
341    /// accounting for alignment requirements.
342    pub const fn query_size(num_descs: usize) -> usize {
343        let desc_size = num_descs * Descriptor::SIZE;
344        let event_size = EventSuppression::SIZE;
345        let event_align = EventSuppression::ALIGN;
346
347        // desc table at offset 0, then aligned events
348        let drv_evt_offset = align_up(desc_size, event_align);
349        let dev_evt_offset = align_up(drv_evt_offset + event_size, event_align);
350
351        dev_evt_offset + event_size
352    }
353}
354
355/// Statistics about the current virtqueue state.
356///
357/// Provided to the [`Notifier`] when sending notifications, allowing
358/// the notifier to make decisions based on queue pressure.
359#[derive(Debug, Clone, Copy)]
360pub struct QueueStats {
361    /// Number of free descriptor slots available.
362    pub num_free: usize,
363    /// Number of descriptors currently in-flight (submitted but not completed).
364    pub num_inflight: usize,
365}
366
367/// Event suppression mode for controlling when notifications are sent.
368///
369/// This configures when the other side should signal (interrupt/kick) us
370/// about new data. Used to optimize batching and reduce interrupt overhead.
371#[derive(Debug, Clone, Copy, PartialEq, Eq)]
372pub enum SuppressionKind {
373    /// Always signal after each operation (default behavior).
374    Enable,
375    /// Never signal.
376    Disable,
377    /// Signal only when reaching a specific descriptor position.
378    Descriptor(RingCursor),
379}
380
381/// A token representing a sent chain in the virtqueue.
382///
383/// Tokens uniquely identify in-flight requests and are used to correlate
384/// requests with their responses.
385#[derive(Copy, Clone, Debug, PartialEq, Eq)]
386pub struct Token {
387    /// Monotonically increasing generation counter.
388    pub seq: u32,
389    /// Descriptor ID this token maps to.
390    pub id: u16,
391}
392
393impl From<BufferElement> for Allocation {
394    fn from(value: BufferElement) -> Self {
395        Allocation {
396            addr: value.addr,
397            len: value.len,
398        }
399    }
400}
401
402const _: () = {
403    #[allow(clippy::panic)]
404    #[allow(clippy::unwrap_used)]
405    const fn verify_layout(num_descs: usize) {
406        let base = 0x1000u64;
407
408        // Safety: base is aligned and we're only checking layout math
409        let layout =
410            match unsafe { Layout::from_base(base, NonZeroU16::new(num_descs as u16).unwrap()) } {
411                Ok(l) => l,
412                Err(_) => panic!("from_base failed"),
413            };
414
415        let expected_size = Layout::query_size(num_descs);
416
417        assert!(layout.desc_table_addr() == base);
418        assert!(layout.desc_table_len() as usize == num_descs);
419        assert!(
420            layout
421                .drv_evt_addr()
422                .is_multiple_of(EventSuppression::ALIGN as u64)
423        );
424        assert!(
425            layout
426                .dev_evt_addr()
427                .is_multiple_of(EventSuppression::ALIGN as u64)
428        );
429
430        // Events don't overlap with descriptor table
431        let desc_end = base + (num_descs * Descriptor::SIZE) as u64;
432        assert!(layout.drv_evt_addr() >= desc_end);
433        assert!(layout.dev_evt_addr() >= layout.drv_evt_addr() + EventSuppression::SIZE as u64);
434
435        // Total size from query_size covers entire layout
436        let layout_end = layout.dev_evt_addr() + EventSuppression::SIZE as u64;
437        assert!(base + expected_size as u64 == layout_end);
438    }
439
440    unsafe {
441        assert!(Layout::from_base(u64::MAX, NonZeroU16::new(1).unwrap()).is_err());
442    }
443
444    verify_layout(1);
445    verify_layout(2);
446    verify_layout(4);
447    verify_layout(8);
448    verify_layout(16);
449    verify_layout(32);
450    verify_layout(64);
451    verify_layout(128);
452    verify_layout(256);
453    verify_layout(512);
454    verify_layout(1024);
455};
456
457pub(crate) fn zeroed_vec(length: usize) -> Result<Vec<u8>, VirtqError> {
458    let mut bytes = Vec::new();
459    bytes
460        .try_reserve_exact(length)
461        .map_err(|_| VirtqError::Bookkeeping)?;
462
463    bytes.resize(length, 0);
464    Ok(bytes)
465}
466
467/// Shared test utilities for virtqueue tests.
468#[cfg(test)]
469pub(crate) mod test_utils {
470    use alloc::sync::Arc;
471    use core::sync::atomic::{AtomicUsize, Ordering};
472
473    use super::*;
474    use crate::virtq::ring::tests::{OwnedRing, TestMem};
475
476    /// Simple notifier that tracks notification count.
477    #[derive(Debug, Clone)]
478    pub(crate) struct TestNotifier {
479        pub(crate) count: Arc<AtomicUsize>,
480    }
481
482    impl TestNotifier {
483        pub(crate) fn new() -> Self {
484            Self {
485                count: Arc::new(AtomicUsize::new(0)),
486            }
487        }
488
489        pub(crate) fn notification_count(&self) -> usize {
490            self.count.load(Ordering::Relaxed)
491        }
492    }
493
494    impl Notifier for TestNotifier {
495        fn notify(&self, _stats: QueueStats) {
496            self.count.fetch_add(1, Ordering::Relaxed);
497        }
498    }
499
500    type TestProducer = VirtqProducer<TestMem, TestNotifier>;
501    type TestConsumer = VirtqConsumer<TestMem, TestNotifier>;
502
503    /// Create test infrastructure: a producer, consumer, and notifier backed
504    /// by the supplied [`OwnedRing`].
505    pub(crate) fn make_test_producer(
506        ring: &OwnedRing,
507    ) -> (TestProducer, TestConsumer, TestNotifier) {
508        make_test_producer_with_slot_size(ring, 128)
509    }
510
511    pub(crate) fn make_test_producer_with_slot_size(
512        ring: &OwnedRing,
513        slot_size: usize,
514    ) -> (TestProducer, TestConsumer, TestNotifier) {
515        let layout = ring.layout();
516        let mem = ring.mem();
517
518        // Pool needs to be in memory accessible via mem - use memory after ring layout
519        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
520        let pool_layout = SlotLayout::new(pool_base, slot_size, 0x8000 / slot_size).unwrap();
521        let pool = SlotPool::new(pool_layout).unwrap();
522        let notifier = TestNotifier::new();
523
524        let producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool);
525        let consumer = VirtqConsumer::new(layout, mem, notifier.clone());
526
527        (producer, consumer, notifier)
528    }
529}
530
531#[cfg(test)]
532mod tests {
533    use alloc::sync::Arc;
534    use core::sync::atomic::{AtomicUsize, Ordering};
535
536    use super::*;
537    use crate::virtq::ring::tests::{TestMem, make_ring};
538    use crate::virtq::test_utils::*;
539
540    /// Helper: build and submit a readable+writable chain using the chain() builder.
541    fn send_readwrite(
542        producer: &mut VirtqProducer<TestMem, TestNotifier>,
543        entry_data: &[u8],
544        used_cap: usize,
545    ) -> Token {
546        let mut se = producer
547            .chain()
548            .readable(entry_data.len())
549            .writable(used_cap)
550            .build()
551            .unwrap();
552        se.write_all(entry_data).unwrap();
553        producer.submit(se).unwrap()
554    }
555
556    fn poll_received(
557        consumer: &mut VirtqConsumer<TestMem, TestNotifier>,
558    ) -> (RecvChain<TestMem>, ReplyChain<TestMem>) {
559        consumer.poll(1024).unwrap().unwrap()
560    }
561
562    #[test]
563    fn test_payload_capacity_overflow_is_an_error() {
564        assert!(matches!(
565            zeroed_vec(usize::MAX),
566            Err(VirtqError::Bookkeeping)
567        ));
568    }
569
570    #[test]
571    fn test_submit_notifies() {
572        let ring = make_ring(16);
573        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
574
575        let initial_count = notifier.notification_count();
576
577        let token = send_readwrite(&mut producer, b"hello", 64);
578        assert!(notifier.notification_count() > initial_count);
579
580        let (recv, reply) = poll_received(&mut consumer);
581        assert_eq!(recv.token(), token);
582        consumer.complete(recv, reply).unwrap();
583        producer.drain(drop).unwrap();
584    }
585
586    #[test]
587    fn test_multiple_submits() {
588        let ring = make_ring(16);
589        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
590
591        let tok1 = send_readwrite(&mut producer, b"request1", 64);
592        let tok2 = send_readwrite(&mut producer, b"request2", 64);
593        let tok3 = send_readwrite(&mut producer, b"request3", 64);
594
595        // Consumer sees all requests
596        for _ in 0..3 {
597            let (recv, reply) = poll_received(&mut consumer);
598            consumer.complete(recv, reply).unwrap();
599        }
600
601        // All completions available
602        let used1 = producer.poll().unwrap().unwrap();
603        let used2 = producer.poll().unwrap().unwrap();
604        let used3 = producer.poll().unwrap().unwrap();
605        assert!(
606            [used1.token(), used2.token(), used3.token()].contains(&tok1)
607                && [used1.token(), used2.token(), used3.token()].contains(&tok2)
608                && [used1.token(), used2.token(), used3.token()].contains(&tok3)
609        );
610    }
611
612    #[test]
613    fn test_completion_batching_with_suppression() {
614        let ring = make_ring(16);
615        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
616
617        // Submit entries
618        let tok1 = send_readwrite(&mut producer, b"req1", 64);
619        let tok2 = send_readwrite(&mut producer, b"req2", 64);
620        let tok3 = send_readwrite(&mut producer, b"req3", 64);
621
622        // Set up reply batching via used suppression
623        let cursor = producer.used_cursor();
624        producer
625            .set_used_suppression(SuppressionKind::Descriptor(cursor))
626            .unwrap();
627
628        // Consumer processes requests
629        for _ in 0..3 {
630            let (recv, reply) = poll_received(&mut consumer);
631            let ReplyChain::Writable(mut wc) = reply else {
632                panic!("expected writable reply");
633            };
634            wc.write_all(b"used-data").unwrap();
635            consumer.complete(recv, wc).unwrap();
636        }
637
638        // Producer can drain all responses
639        let mut responses = Vec::new();
640        producer
641            .drain(|reply| {
642                responses.push(reply.token());
643            })
644            .unwrap();
645
646        assert_eq!(responses.len(), 3);
647        assert!(responses.contains(&tok1));
648        assert!(responses.contains(&tok2));
649        assert!(responses.contains(&tok3));
650    }
651
652    #[test]
653    fn test_notifier_receives_context() {
654        #[derive(Debug, Clone)]
655        struct CtxNotifier {
656            last_num_free: Arc<AtomicUsize>,
657            last_num_inflight: Arc<AtomicUsize>,
658            count: Arc<AtomicUsize>,
659        }
660
661        impl Notifier for CtxNotifier {
662            fn notify(&self, stats: QueueStats) {
663                self.last_num_free.store(stats.num_free, Ordering::Relaxed);
664                self.last_num_inflight
665                    .store(stats.num_inflight, Ordering::Relaxed);
666                self.count.fetch_add(1, Ordering::Relaxed);
667            }
668        }
669
670        let ring = make_ring(16);
671        let layout = ring.layout();
672        let mem = ring.mem();
673        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
674        let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
675        let pool = SlotPool::new(pool_layout).unwrap();
676        let notifier = CtxNotifier {
677            last_num_free: Arc::new(AtomicUsize::new(0)),
678            last_num_inflight: Arc::new(AtomicUsize::new(0)),
679            count: Arc::new(AtomicUsize::new(0)),
680        };
681
682        let mut producer = VirtqProducer::new(layout, mem, notifier.clone(), pool);
683
684        let mut se = producer.chain().readable(4).writable(32).build().unwrap();
685        se.write_all(b"test").unwrap();
686        producer.submit(se).unwrap();
687        assert_eq!(notifier.count.load(Ordering::Relaxed), 1);
688        assert!(notifier.last_num_inflight.load(Ordering::Relaxed) > 0);
689
690        // SAFETY: No consumer is attached to this ring.
691        unsafe { producer.reset() }.unwrap();
692    }
693
694    #[test]
695    fn test_chain_batch() {
696        let ring = make_ring(16);
697        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
698
699        let initial_count = notifier.notification_count();
700
701        // First readable chain
702        let mut se1 = producer.chain().readable(64).writable(128).build().unwrap();
703        se1.write_all(b"first-ent").unwrap();
704        let _tok1 = producer.submit(se1).unwrap();
705
706        // Write-based recv
707        let mut se2 = producer.chain().readable(64).writable(64).build().unwrap();
708        se2.write_all(b"copy-ent").unwrap();
709        let _tok2 = producer.submit(se2).unwrap();
710
711        // Completion-only chain
712        let se3 = producer.chain().writable(32).build().unwrap();
713        let tok3 = producer.submit(se3).unwrap();
714
715        // Each submit may notify independently
716        assert!(notifier.notification_count() > initial_count);
717
718        // Consumer sees all three entries
719        let (recv1, reply1) = poll_received(&mut consumer);
720        assert_eq!(recv1.to_bytes().unwrap().as_ref(), b"first-ent");
721        consumer.complete(recv1, reply1).unwrap();
722
723        let (recv2, reply2) = poll_received(&mut consumer);
724        assert_eq!(recv2.to_bytes().unwrap().as_ref(), b"copy-ent");
725        consumer.complete(recv2, reply2).unwrap();
726
727        let (recv3, reply3) = poll_received(&mut consumer);
728        let ReplyChain::Writable(mut wc) = reply3 else {
729            panic!("expected writable reply");
730        };
731        wc.write_all(b"resp").unwrap();
732        consumer.complete(recv3, wc).unwrap();
733
734        // Drain completions
735        let _ = producer.poll().unwrap().unwrap();
736        let _ = producer.poll().unwrap().unwrap();
737
738        let used = producer.poll().unwrap().unwrap();
739        assert_eq!(used.token(), tok3);
740        assert_eq!(used.to_bytes().unwrap().as_ref(), b"resp");
741    }
742
743    #[test]
744    fn test_chain_write_send() {
745        let ring = make_ring(16);
746        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
747
748        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
749        se.write_all(b"hello").unwrap();
750        let token = producer.submit(se).unwrap();
751
752        // Consumer sees the data
753        let (recv, reply) = poll_received(&mut consumer);
754        assert_eq!(recv.token(), token);
755        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
756
757        // Write response
758        let ReplyChain::Writable(mut wc) = reply else {
759            panic!("expected writable reply");
760        };
761        wc.write_all(b"world").unwrap();
762        consumer.complete(recv, wc).unwrap();
763        let used = producer.poll().unwrap().unwrap();
764        assert_eq!(used.to_bytes().unwrap().as_ref(), b"world");
765    }
766
767    #[test]
768    fn test_full_round_trip() {
769        let ring = make_ring(16);
770        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
771
772        // Send an recv
773        let token = send_readwrite(&mut producer, b"round-trip-recv", 128);
774
775        // Consumer receives and responds
776        let (recv, reply) = poll_received(&mut consumer);
777        assert_eq!(recv.token(), token);
778        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"round-trip-recv");
779
780        let ReplyChain::Writable(mut wc) = reply else {
781            panic!("expected writable reply");
782        };
783        assert!(wc.capacity() >= 128);
784        wc.write_all(b"round-trip-rsp").unwrap();
785        consumer.complete(recv, wc).unwrap();
786
787        // Producer gets the reply
788        let used = producer.poll().unwrap().unwrap();
789        assert_eq!(used.token(), token);
790        assert_eq!(used.to_bytes().unwrap().as_ref(), b"round-trip-rsp");
791    }
792
793    #[test]
794    fn test_cancel_submits_zero_length() {
795        let ring = make_ring(16);
796        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
797
798        let token = send_readwrite(&mut producer, b"recv-data", 64);
799
800        let (recv, reply) = poll_received(&mut consumer);
801        consumer.complete(recv, reply).unwrap();
802
803        let used = producer.poll().unwrap().unwrap();
804        assert_eq!(used.token(), token);
805        assert_eq!(used.to_bytes().unwrap().len(), 0);
806        assert!(used.to_bytes().unwrap().is_empty());
807    }
808
809    #[test]
810    fn test_hold_reply_and_complete() {
811        let ring = make_ring(16);
812        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
813
814        let token = send_readwrite(&mut producer, b"deferred", 64);
815
816        // Poll and hold the reply
817        let (recv, reply) = poll_received(&mut consumer);
818        assert_eq!(recv.token(), token);
819        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"deferred");
820
821        let ReplyChain::Writable(mut wc) = reply else {
822            panic!("expected writable reply");
823        };
824        wc.write_all(b"deferred-used").unwrap();
825        consumer.complete(recv, wc).unwrap();
826
827        let used = producer.poll().unwrap().unwrap();
828        assert_eq!(used.token(), token);
829        assert_eq!(used.to_bytes().unwrap().as_ref(), b"deferred-used");
830    }
831
832    #[test]
833    fn test_concurrent_pending_replies() {
834        let ring = make_ring(16);
835        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
836
837        let tok1 = send_readwrite(&mut producer, b"first", 64);
838        let tok2 = send_readwrite(&mut producer, b"second", 64);
839
840        // Poll both
841        let (recv1, reply1) = poll_received(&mut consumer);
842        assert_eq!(recv1.token(), tok1);
843        assert_eq!(recv1.to_bytes().unwrap().as_ref(), b"first");
844
845        let (recv2, reply2) = poll_received(&mut consumer);
846        assert_eq!(recv2.token(), tok2);
847        assert_eq!(recv2.to_bytes().unwrap().as_ref(), b"second");
848
849        // Complete second first (out of order)
850        let ReplyChain::Writable(mut wc2) = reply2 else {
851            panic!("expected writable");
852        };
853        wc2.write_all(b"resp2").unwrap();
854        consumer.complete(recv2, wc2).unwrap();
855
856        let ReplyChain::Writable(mut wc1) = reply1 else {
857            panic!("expected writable");
858        };
859        wc1.write_all(b"resp1").unwrap();
860        consumer.complete(recv1, wc1).unwrap();
861
862        let used1 = producer.poll().unwrap().unwrap();
863        let used2 = producer.poll().unwrap().unwrap();
864        let mut responses: Vec<_> = vec![
865            (used1.token(), used1.to_bytes().unwrap().to_vec()),
866            (used2.token(), used2.to_bytes().unwrap().to_vec()),
867        ];
868        responses.sort_by_key(|(t, _)| t.seq);
869
870        let expected_first = responses.iter().find(|(t, _)| *t == tok1).unwrap();
871        let expected_second = responses.iter().find(|(t, _)| *t == tok2).unwrap();
872        assert_eq!(&expected_first.1[..], b"resp1");
873        assert_eq!(&expected_second.1[..], b"resp2");
874    }
875
876    /// Helper: submit a read-only chain (readable data, no writable reply).
877    fn send_readonly(
878        producer: &mut VirtqProducer<TestMem, TestNotifier>,
879        entry_data: &[u8],
880    ) -> Token {
881        let mut se = producer.chain().readable(entry_data.len()).build().unwrap();
882        se.write_all(entry_data).unwrap();
883        producer.submit(se).unwrap()
884    }
885
886    #[test]
887    fn test_reclaim_frees_ring_slots() {
888        let ring = make_ring(4);
889        let (mut producer, mut consumer, _) = make_test_producer(&ring);
890
891        // Fill the ring with ReadOnly entries
892        send_readonly(&mut producer, b"a");
893        send_readonly(&mut producer, b"b");
894        send_readonly(&mut producer, b"c");
895        send_readonly(&mut producer, b"d");
896        assert_eq!(producer.num_inflight(), 4);
897
898        let res = producer.chain().readable(1).build();
899        assert!(
900            matches!(res, Err(VirtqError::Backpressure)),
901            "expected Backpressure from full ring"
902        );
903
904        // Consumer acks all entries
905        while let Some(result) = consumer.poll(1024).unwrap() {
906            let (recv, reply) = result;
907            consumer.complete(recv, reply).unwrap();
908        }
909
910        // Reclaim should free ring slots without losing data
911        let count = producer.reclaim().unwrap();
912        assert_eq!(count, 4, "expected 4 reclaimed entries");
913        assert_eq!(producer.num_inflight(), 0);
914
915        // Ring should have space now
916        send_readonly(&mut producer, b"e");
917
918        // SAFETY: The consumer completed all handles and stays inactive.
919        unsafe { producer.reset() }.unwrap();
920    }
921
922    #[test]
923    fn test_reclaim_buffers_rw_completions() {
924        let ring = make_ring(4);
925        let (mut producer, mut consumer, _) = make_test_producer(&ring);
926
927        // Submit a ReadWrite recv
928        let tok = send_readwrite(&mut producer, b"request", 64);
929
930        // Consumer processes and writes response
931        let (recv, reply) = poll_received(&mut consumer);
932        let ReplyChain::Writable(mut wc) = reply else {
933            panic!("expected writable");
934        };
935        wc.write_all(b"response-data").unwrap();
936        consumer.complete(recv, wc).unwrap();
937
938        // Reclaim buffers the reply (doesn't discard it)
939        let count = producer.reclaim().unwrap();
940        assert_eq!(count, 1);
941
942        // poll() should return the buffered reply
943        let used = producer.poll().unwrap().unwrap();
944        assert_eq!(used.token(), tok);
945        assert_eq!(used.to_bytes().unwrap().as_ref(), b"response-data");
946    }
947
948    #[test]
949    fn test_reclaim_discards_readonly_completions() {
950        let ring = make_ring(8);
951        let (mut producer, mut consumer, _) = make_test_producer(&ring);
952
953        // Submit 3 entries: RO, RW, RO
954        let _tok_ro1 = send_readonly(&mut producer, b"log1");
955        let tok_rw = send_readwrite(&mut producer, b"call", 64);
956        let _tok_ro2 = send_readonly(&mut producer, b"log2");
957
958        // Consumer processes all 3
959        let (recv1, reply1) = poll_received(&mut consumer);
960        consumer.complete(recv1, reply1).unwrap(); // ack RO
961
962        let (recv2, reply2) = poll_received(&mut consumer);
963        let ReplyChain::Writable(mut wc) = reply2 else {
964            panic!("expected writable");
965        };
966        wc.write_all(b"result").unwrap();
967        consumer.complete(recv2, wc).unwrap(); // complete RW
968
969        let (recv3, reply3) = poll_received(&mut consumer);
970        consumer.complete(recv3, reply3).unwrap(); // ack RO
971
972        // Reclaim all 3 - RO completions are discarded, only RW is buffered
973        let count = producer.reclaim().unwrap();
974        assert_eq!(count, 3);
975
976        // poll() returns only the RW reply
977        let used = producer.poll().unwrap().unwrap();
978        assert_eq!(used.token(), tok_rw);
979        assert_eq!(used.to_bytes().unwrap().as_ref(), b"result");
980
981        // No more - RO completions were discarded
982        assert!(producer.poll().unwrap().is_none());
983    }
984
985    #[test]
986    fn test_reclaim_mixed_with_poll() {
987        let ring = make_ring(8);
988        let (mut producer, mut consumer, _) = make_test_producer(&ring);
989
990        // Submit and complete 2 entries
991        send_readonly(&mut producer, b"x");
992        let tok_rw = send_readwrite(&mut producer, b"y", 64);
993
994        let (recv1, reply1) = poll_received(&mut consumer);
995        consumer.complete(recv1, reply1).unwrap();
996
997        let (recv2, reply2) = poll_received(&mut consumer);
998        let ReplyChain::Writable(mut wc) = reply2 else {
999            panic!("expected writable");
1000        };
1001        wc.write_all(b"reply").unwrap();
1002        consumer.complete(recv2, wc).unwrap();
1003
1004        // poll() consumes first recv directly from ring
1005        let used1 = producer.poll().unwrap().unwrap();
1006        assert!(matches!(used1, UsedChain::Ack(_)));
1007
1008        // reclaim() buffers second recv
1009        let count = producer.reclaim().unwrap();
1010        assert_eq!(count, 1);
1011
1012        // poll() returns the buffered one
1013        let used2 = producer.poll().unwrap().unwrap();
1014        assert_eq!(used2.token(), tok_rw);
1015        assert_eq!(used2.to_bytes().unwrap().as_ref(), b"reply");
1016    }
1017
1018    /// reclaim + submit must not cause token collisions.
1019    #[test]
1020    fn test_reclaim_submit_no_token_collision() {
1021        let ring = make_ring(8);
1022        let (mut producer, mut consumer, _) = make_test_producer(&ring);
1023
1024        // Submit and complete a ReadOnly recv
1025        let tok_old = send_readonly(&mut producer, b"log");
1026
1027        let (recv, reply) = poll_received(&mut consumer);
1028        consumer.complete(recv, reply).unwrap();
1029
1030        let count = producer.reclaim().unwrap();
1031        assert_eq!(count, 1);
1032
1033        // Submit a new ReadWrite recv - may reuse the same descriptor ID
1034        let tok_new = send_readwrite(&mut producer, b"call", 64);
1035
1036        // Tokens must differ even if the descriptor ID was recycled
1037        assert_ne!(
1038            tok_old, tok_new,
1039            "tokens must be unique across reclaim/submit cycles"
1040        );
1041
1042        // Complete the ReadWrite recv
1043        let (recv, reply) = poll_received(&mut consumer);
1044        let ReplyChain::Writable(mut wc) = reply else {
1045            panic!("expected writable");
1046        };
1047        wc.write_all(b"result").unwrap();
1048        consumer.complete(recv, wc).unwrap();
1049
1050        // Poll returns only the RW reply (RO was discarded by reclaim)
1051        let used = producer.poll().unwrap().unwrap();
1052        assert_eq!(used.token(), tok_new);
1053        assert_eq!(used.to_bytes().unwrap().as_ref(), b"result");
1054
1055        // No stale RO reply in the queue
1056        assert!(producer.poll().unwrap().is_none());
1057    }
1058
1059    /// Verify that repeated oneshot submit/reclaim cycles do not accumulate pending completions.
1060    #[test]
1061    fn test_reclaim_readonly_does_not_leak_pending() {
1062        let ring = make_ring(4);
1063        let (mut producer, mut consumer, _) = make_test_producer(&ring);
1064
1065        for _ in 0..10 {
1066            // Fill the ring
1067            for _ in 0..4 {
1068                send_readonly(&mut producer, b"msg");
1069            }
1070
1071            // Consumer acks all
1072            while let Some(result) = consumer.poll(1024).unwrap() {
1073                let (recv, reply) = result;
1074                consumer.complete(recv, reply).unwrap();
1075            }
1076
1077            // Reclaim frees ring slots; empty completions are discarded
1078            let count = producer.reclaim().unwrap();
1079            assert_eq!(count, 4);
1080
1081            // No completions should be buffered in pending
1082            assert!(
1083                producer.poll().unwrap().is_none(),
1084                "pending should be empty after reclaiming RO entries"
1085            );
1086        }
1087    }
1088}