Skip to main content

hyperlight_common/virtq/
ring.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4//! Packed Virtqueue Ring Implementation
5//!
6//! This module implements the packed virtqueue format from the VIRTIO specification.
7//! Packed virtqueues use a single descriptor ring where descriptors cycle through
8//! available and used states, providing better cache locality and simpler memory
9//! layout compared to split virtqueues.
10//!
11//! # Descriptor State Machine
12//!
13//! Each descriptor transitions through states using AVAIL and USED flags:
14//!
15//! ```text
16//!                    Driver publishes
17//!     ┌─────────┐    (AVAIL=wrap)     ┌───────────┐
18//!     │  Free   │ ──────────────────> │ Available │
19//!     └─────────┘                     └───────────┘
20//!          ^                                │
21//!          │                                │ Device consumes
22//!          │ Driver reclaims                │ and marks used
23//!          │ (polls USED=wrap)              │ (USED=wrap)
24//!          │                                v
25//!     ┌─────────┐                     ┌───────────┐
26//!     │Reclaimed│ <────────────────── │   Used    │
27//!     └─────────┘                     └───────────┘
28//! ```
29//!
30//! # Wrap Counter
31//!
32//! The wrap counter solves ring wraparound ambiguity. When cursors wrap around
33//! the ring, the wrap counter toggles, changing how AVAIL/USED flags are interpreted:
34//!
35//! - **wrap=true**: AVAIL=1, USED=0 means "available"; AVAIL=1, USED=1 means "used"
36//! - **wrap=false**: AVAIL=0, USED=1 means "available"; AVAIL=0, USED=0 means "used"
37//!
38//! # Buffer Chains
39//!
40//! Multiple buffers can be chained using the NEXT flag. All descriptors in a chain
41//! share the same ID, and only the head descriptor's AVAIL/USED flags matter for
42//! state transitions:
43//!
44//! ```text
45//! Chain with 3 buffers (ID=5):
46//! ┌──────────────┐    ┌──────────────┐    ┌──────────────┐
47//! │ Desc[0]      │    │ Desc[1]      │    │ Desc[2]      │
48//! │ id=42        │───>│ id=42        │───>│ id=42        │
49//! │ flags=NEXT   │    │ flags=NEXT   │    │ flags=0      │
50//! │ AVAIL/USED   │    │ (ignored)    │    │ (ignored)    │
51//! └──────────────┘    └──────────────┘    └──────────────┘
52//!       HEAD              MIDDLE               TAIL
53//! ```
54//!
55//! # Event Suppression
56//!
57//! Both sides can control when they want to be notified:
58//!
59//! - **ENABLE**: Always notify (default)
60//! - **DISABLE**: Never notify (for polling mode)
61//! - **DESC**: Notify only when a specific descriptor index is reached
62//! ```
63
64pub mod canonical;
65
66use core::fmt;
67use core::marker::PhantomData;
68use core::sync::atomic::{Ordering, fence};
69
70use bytemuck::Zeroable;
71use smallvec::SmallVec;
72use thiserror::Error;
73
74use super::desc::{DescFlags, DescTable, Descriptor};
75use super::event::{EventFlags, EventSuppression};
76use super::{Layout, MemOps};
77
78/// A single buffer element in a scatter-gather list.
79///
80/// Represents one contiguous memory region that the device will read from
81/// or write to. Multiple elements can be chained together to form a
82/// [`BufferChain`].
83#[derive(Debug, Copy, Clone, Zeroable)]
84pub struct BufferElement {
85    /// Physical address of buffer
86    pub addr: u64,
87    /// Length of the buffer in bytes
88    pub len: u32,
89    /// Whether this buffer is writable by the device
90    pub writable: bool,
91}
92
93pub(super) type ChainElems = SmallVec<[BufferElement; 4]>;
94
95impl BufferElement {
96    /// Create a readable buffer element
97    pub fn readable(addr: u64) -> Self {
98        Self {
99            addr,
100            len: 0,
101            writable: false,
102        }
103    }
104
105    /// Create a writable buffer element
106    pub fn writable(addr: u64, len: u32) -> Self {
107        Self {
108            addr,
109            len,
110            writable: true,
111        }
112    }
113}
114
115/// A buffer returned from the ring after being used by the device.
116///
117/// When the device completes processing a buffer chain, it returns this
118/// structure containing the original descriptor ID and the number of bytes
119/// written (for chains with writable buffers).
120#[derive(Debug, Copy, Clone)]
121pub struct UsedBuffer {
122    /// Descriptor ID that was assigned when the buffer was submitted
123    pub id: u16,
124    /// Number of bytes written by the device to writable buffers.
125    /// For read-only chains, this may be 0 or the total readable length.
126    pub len: u32,
127}
128
129/// Result of submitting a buffer to the ring.
130///
131/// Contains the assigned descriptor ID and whether the other side
132/// needs to be notified about the new buffer.
133#[derive(Debug, Copy, Clone)]
134pub struct SubmitResult {
135    /// Descriptor ID assigned to the submitted buffer chain
136    /// Use this ID to correlate completions with submissions.
137    pub id: u16,
138    /// Whether the device should be notified immediately based on the other
139    /// side's event suppression settings.
140    pub notify: bool,
141}
142
143/// Memory operation that failed in the backend.
144#[derive(Debug, Copy, Clone, PartialEq, Eq)]
145pub enum MemOp {
146    /// Reading a descriptor from the descriptor table.
147    ReadDesc,
148    /// Writing a descriptor to the descriptor table.
149    WriteDesc,
150    /// Reading an event suppression structure.
151    ReadEvent,
152    /// Writing an event suppression structure.
153    WriteEvent,
154}
155
156impl fmt::Display for MemOp {
157    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
158        match self {
159            Self::ReadDesc => f.write_str("reading descriptor"),
160            Self::WriteDesc => f.write_str("writing descriptor"),
161            Self::ReadEvent => f.write_str("reading event suppression"),
162            Self::WriteEvent => f.write_str("writing event suppression"),
163        }
164    }
165}
166
167#[derive(Error, Debug)]
168pub enum RingError {
169    #[error("Buffer chain is empty")]
170    EmptyChain,
171    #[error("Buffer chain is malformed")]
172    BadChain,
173    #[error("Operation would block")]
174    WouldBlock,
175    #[error("Out of memory")]
176    OutOfMemory,
177    /// Cached descriptor storage could not be allocated.
178    #[error("Failed to allocate descriptor bookkeeping")]
179    Bookkeeping,
180    #[error("Invalid state")]
181    InvalidState,
182    #[error("Invalid memory layout")]
183    InvalidLayout,
184    /// A backend memory operation failed.
185    ///
186    /// A failed write may have partially modified shared memory. After a write
187    /// error, retry reset or discard the endpoint before reuse.
188    #[error("Backend memory error while {op} at address 0x{addr:x}, len {len}")]
189    MemError {
190        /// Memory operation that failed.
191        op: MemOp,
192        /// Address passed to the memory backend.
193        addr: u64,
194        /// Number of bytes requested for the operation.
195        len: usize,
196    },
197}
198
199impl RingError {
200    #[inline]
201    fn mem_err(op: MemOp, addr: u64) -> Self {
202        let len = match op {
203            MemOp::ReadDesc | MemOp::WriteDesc => Descriptor::SIZE,
204            MemOp::ReadEvent | MemOp::WriteEvent => EventSuppression::SIZE,
205        };
206
207        Self::MemError { op, addr, len }
208    }
209}
210
211/// Type-state: Can add readable buffers
212pub struct Readable;
213
214/// Type-state: Can add writable buffers (no more readables allowed)
215pub struct Writable;
216
217/// A builder for buffer chains using type-state to enforce readable/writable order.
218///
219/// Upholds invariants: at least one buffer must be present in the chain,
220/// and readable buffers must be added before writable buffers.
221///
222/// The builder stores up to four buffer elements inline to avoid allocation for
223/// common small chains. Larger chains are still supported and spill to the heap.
224#[derive(Debug, Default)]
225pub struct BufferChainBuilder<T> {
226    elems: ChainElems,
227    split: usize,
228    marker: PhantomData<T>,
229}
230
231impl<T> BufferChainBuilder<T> {
232    pub(super) fn reserve_exact(&mut self, additional: usize) {
233        self.elems.reserve_exact(additional);
234    }
235}
236
237impl BufferChainBuilder<Readable> {
238    /// Create a new builder in the [`Readable`] state.
239    pub fn new() -> Self {
240        Self {
241            elems: Default::default(),
242            split: 0,
243            marker: PhantomData,
244        }
245    }
246
247    /// Add a readable buffer (device reads from this).
248    pub fn readable(mut self, addr: u64, len: u32) -> Self {
249        self.elems.push(BufferElement {
250            addr,
251            len,
252            writable: false,
253        });
254        self.split += 1;
255        self
256    }
257
258    /// Add multiple readable buffers from an iterator.
259    pub fn readables(
260        mut self,
261        elements: impl IntoIterator<Item = impl Into<BufferElement>>,
262    ) -> Self {
263        for elem in elements {
264            let mut elem = elem.into();
265            elem.writable = false;
266            self.elems.push(elem);
267            self.split += 1;
268        }
269
270        self
271    }
272
273    /// Add a writable buffer (device writes to this).
274    ///
275    /// This transitions to Writable state so no more readable buffers can be added.
276    pub fn writable(mut self, addr: u64, len: u32) -> BufferChainBuilder<Writable> {
277        self.elems.push(BufferElement {
278            addr,
279            len,
280            writable: true,
281        });
282
283        BufferChainBuilder {
284            elems: self.elems,
285            split: self.split,
286            marker: PhantomData,
287        }
288    }
289
290    /// Add multiple writable buffers from an iterator.
291    ///
292    /// This transitions to Writable state so no more readable buffers can be added.
293    pub fn writables(
294        mut self,
295        elements: impl IntoIterator<Item = impl Into<BufferElement>>,
296    ) -> BufferChainBuilder<Writable> {
297        for elem in elements {
298            let mut elem = elem.into();
299            elem.writable = true;
300            self.elems.push(elem);
301        }
302
303        BufferChainBuilder {
304            elems: self.elems,
305            split: self.split,
306            marker: PhantomData,
307        }
308    }
309
310    /// Build a buffer chain with only readable buffers.
311    ///
312    /// Chain must have at least one buffer otherwise an error is returned.
313    pub fn build(self) -> Result<BufferChain, RingError> {
314        if self.elems.is_empty() {
315            return Err(RingError::EmptyChain);
316        }
317
318        Ok(BufferChain {
319            elems: self.elems,
320            split: self.split,
321        })
322    }
323}
324
325impl BufferChainBuilder<Writable> {
326    /// Add writable buffer
327    pub fn writable(mut self, addr: u64, len: u32) -> Self {
328        self.elems.push(BufferElement {
329            addr,
330            len,
331            writable: true,
332        });
333        self
334    }
335
336    /// Add multiple writable buffers from an iterator.
337    pub fn writables(
338        mut self,
339        elements: impl IntoIterator<Item = impl Into<BufferElement>>,
340    ) -> Self {
341        for elem in elements {
342            let mut elem = elem.into();
343            elem.writable = true;
344            self.elems.push(elem);
345        }
346        self
347    }
348
349    /// Build the buffer chain.
350    ///
351    /// Chain must have at least one buffer otherwise an error is returned.
352    pub fn build(self) -> Result<BufferChain, RingError> {
353        if self.elems.is_empty() {
354            return Err(RingError::EmptyChain);
355        }
356
357        Ok(BufferChain {
358            elems: self.elems,
359            split: self.split,
360        })
361    }
362}
363
364/// A chain of buffers ready for submission to the virtqueue.
365///
366/// Contains a scatter-gather list of [`BufferElement`]s, divided into
367/// readable (driver->device) and writable (device->driver) sections.
368#[derive(Debug, Clone)]
369pub struct BufferChain {
370    /// All buffer elements (readable followed by writable)
371    elems: ChainElems,
372    /// Split index between readable and writable buffers
373    split: usize,
374}
375
376impl BufferChain {
377    /// Split descriptor ownership, reusing heap storage when a half needs it.
378    pub(super) fn into_parts(mut self) -> Result<(ChainElems, ChainElems), RingError> {
379        let should_move = self.split <= self.elems.len() - self.split;
380
381        let range = if should_move {
382            0..self.split
383        } else {
384            self.split..self.elems.len()
385        };
386
387        let mut detached = ChainElems::new();
388        detached
389            .try_reserve_exact(range.len())
390            .map_err(|_| RingError::Bookkeeping)?;
391
392        detached.extend(self.elems.drain(range));
393        if self.elems.len() <= self.elems.inline_size() {
394            // Shrinking into inline storage only copies and frees.
395            self.elems.shrink_to_fit();
396        }
397
398        Ok(if should_move {
399            (detached, self.elems)
400        } else {
401            (self.elems, detached)
402        })
403    }
404
405    /// Get all buffer elements in the chain.
406    pub fn elems(&self) -> &[BufferElement] {
407        self.elems.as_slice()
408    }
409
410    /// Get readable buffers in chain
411    pub fn readables(&self) -> &[BufferElement] {
412        &self.elems[..self.split]
413    }
414
415    /// Get writable buffers in chain
416    pub fn writables(&self) -> &[BufferElement] {
417        &self.elems[self.split..]
418    }
419
420    /// Get total number of buffers in chain
421    // Note: buffer chain cannot be empty by construction
422    #[allow(clippy::len_without_is_empty)]
423    pub fn len(&self) -> usize {
424        self.elems.len()
425    }
426}
427
428/// Tracks position in a ring buffer with wrap-around handling.
429///
430/// The cursor maintains both an index into the ring and a wrap counter
431/// that toggles each time the index wraps around.
432#[derive(Debug, Copy, Clone, PartialEq, Eq)]
433pub struct RingCursor {
434    head: u16,
435    size: u16,
436    wrap: bool,
437}
438
439impl RingCursor {
440    pub(crate) fn new(size: usize) -> Self {
441        Self {
442            head: 0,
443            size: size as u16,
444            wrap: true,
445        }
446    }
447
448    /// Advance to next position, wrapping around and toggling wrap counter if needed
449    #[inline]
450    fn advance(&mut self) {
451        debug_assert!(self.head.checked_add(1).is_some());
452        self.head += 1;
453        if self.head >= self.size {
454            self.head = 0;
455            self.wrap = !self.wrap;
456        }
457    }
458
459    /// Advance by n positions using modular arithmetic.
460    #[inline]
461    fn advance_by(&mut self, n: u16) {
462        debug_assert!(self.head.checked_add(n).is_some());
463        let new = self.head + n;
464        let wraps = new / self.size;
465        self.head = new % self.size;
466        if !wraps.is_multiple_of(2) {
467            self.wrap = !self.wrap;
468        }
469    }
470
471    /// Get current head index
472    #[inline]
473    pub fn head(&self) -> u16 {
474        self.head
475    }
476
477    /// Get current wrap counter
478    #[inline]
479    pub fn wrap(&self) -> bool {
480        self.wrap
481    }
482
483    /// Reset cursor to initial state.
484    #[inline]
485    pub fn reset(&mut self) {
486        self.head = 0;
487        self.wrap = true;
488    }
489}
490
491/// Local [`RingConsumer`] state needed to undo a sequence of polls.
492///
493/// No chains may be completed between capture and rollback.
494pub(super) struct Checkpoint {
495    /// Position of the next available chain.
496    avail_cursor: RingCursor,
497    /// Position of the next completion.
498    used_cursor: RingCursor,
499    /// Number of descriptors awaiting completion.
500    num_inflight: usize,
501}
502
503/// Producer (driver) side of a packed virtqueue.
504///
505/// The producer submits buffer chains for the device to process and polls
506/// for completions. This is typically used by the driver/guest side.
507///
508/// # Lifecycle
509///
510/// 1.Submit: Call [`submit_available`](Self::submit_available) or
511///    [`submit_one`](Self::submit_one) to make buffers available to device
512/// 2. Notify: If `SubmitResult::notify` is true, signal the device
513/// 3. Poll: Call [`poll_used`](Self::poll_used) to check for completions
514/// 4. Process: Handle completed buffers and reuse descriptor IDs
515#[derive(Debug)]
516pub struct RingProducer<M> {
517    /// Memory accessor
518    mem: M,
519    /// Next available descriptor position
520    avail_cursor: RingCursor,
521    /// Next used descriptor position
522    used_cursor: RingCursor,
523    /// Free slots in the ring
524    num_free: usize,
525    /// Descriptor table in shared memory
526    desc_table: DescTable,
527    /// Shadow of driver event flags (last written value)
528    event_flags_shadow: EventFlags,
529    // controls when device notifies about used buffers
530    drv_evt_addr: u64,
531    // reads device event to check if device wants notification
532    dev_evt_addr: u64,
533    /// stack of free IDs, allows out-of-order completion
534    id_free: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
535    // chain length per ID, index = ID,
536    id_num: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
537}
538
539impl<M: MemOps> RingProducer<M> {
540    /// Create a new producer from a memory layout and accessor.
541    pub fn new(layout: Layout, mem: M) -> Self {
542        let size = layout.desc_table_len() as usize;
543        let raw = layout.desc_table_addr();
544
545        // SAFETY: Layout fields are private and from_base validates ring geometry.
546        let table = unsafe { DescTable::from_raw_parts(raw, size) };
547        let cursor = RingCursor::new(size);
548
549        const DEFAULT_LEN: usize = DescTable::default_len();
550        let id_free = (0..size as u16).collect::<SmallVec<[_; DEFAULT_LEN]>>();
551        let id_num = SmallVec::<[_; DEFAULT_LEN]>::from_elem(0, size);
552
553        // Notification enabled by default
554        let event_flags_shadow = EventFlags::ENABLE;
555
556        Self {
557            mem,
558            avail_cursor: cursor,
559            used_cursor: cursor,
560            num_free: size,
561            desc_table: table,
562            id_free,
563            id_num,
564            event_flags_shadow,
565            drv_evt_addr: layout.drv_evt_addr(),
566            dev_evt_addr: layout.dev_evt_addr(),
567        }
568    }
569
570    /// Fast path: submit exactly one descriptor
571    ///
572    /// This is more efficient than [`submit_available`](Self::submit_available)
573    /// for single-buffer submissions as it avoids chain iteration overhead.
574    ///
575    /// # Arguments
576    ///
577    /// * `addr` - physical address of the buffer
578    /// * `len` - Length of the buffer in bytes
579    /// * `writable` - If true, device writes to buffer; if false, device reads
580    ///
581    /// # Returns
582    ///
583    /// The descriptor ID assigned to this buffer, for matching with completions.
584    ///
585    /// # Errors
586    ///
587    /// - [`RingError::WouldBlock`] - No free descriptor slots
588    /// - [`RingError::OutOfMemory`] - No free descriptor IDs (internal error)
589    /// - [`RingError::InvalidState`] - ID tracking corrupted (internal error)
590    /// - [`RingError::MemError`] - Backend memory error while publishing the descriptor
591    pub fn submit_one(&mut self, addr: u64, len: u32, writable: bool) -> Result<u16, RingError> {
592        if self.num_free < 1 {
593            return Err(RingError::WouldBlock);
594        }
595
596        // Allocate ID and record chain length
597        let id = self.id_free.pop().ok_or(RingError::OutOfMemory)?;
598
599        // We should never reuse an ID that is still outstanding
600        if self.id_num[id as usize] != 0 {
601            return Err(RingError::InvalidState);
602        }
603
604        // Record chain length for single descriptor
605        self.id_num[id as usize] = 1;
606
607        // Build and publish the head descriptor
608        let head_idx = self.avail_cursor.head();
609        let head_wrap = self.avail_cursor.wrap();
610
611        let mut flags = DescFlags::empty();
612        flags.set(DescFlags::WRITE, writable);
613        let mut desc = Descriptor::new(addr, len, id, flags);
614        desc.mark_avail(head_wrap);
615
616        let addr = self
617            .desc_table
618            .desc_addr(head_idx)
619            .ok_or(RingError::InvalidState)?;
620
621        // Release publish
622        desc.write_release(&self.mem, addr)
623            .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
624
625        // Advance state
626        self.avail_cursor.advance();
627        self.num_free -= 1;
628
629        Ok(id)
630    }
631
632    /// Submit a buffer chain to the ring, returning whether to notify the device.
633    pub fn submit_available_with_notify(
634        &mut self,
635        chain: &BufferChain,
636    ) -> Result<SubmitResult, RingError> {
637        let old = self.avail_cursor;
638        let id = self.submit_available(chain)?;
639        let new = self.avail_cursor;
640        let notify = self.should_notify_device(old, new)?;
641
642        Ok(SubmitResult { id, notify })
643    }
644
645    /// Submit a single-buffer descriptor with notification check.
646    pub fn submit_one_with_notify(
647        &mut self,
648        addr: u64,
649        len: u32,
650        writable: bool,
651    ) -> Result<SubmitResult, RingError> {
652        let old = self.avail_cursor;
653        let id = self.submit_one(addr, len, writable)?;
654        let new = self.avail_cursor;
655        let notify = self.should_notify_device(old, new)?;
656        Ok(SubmitResult { id, notify })
657    }
658
659    /// Submit a buffer chain to the ring.
660    ///
661    /// Writes all descriptors in the chain to the ring, linking them with
662    /// NEXT flags. The head descriptor is written last with release semantics
663    /// to ensure atomicity of the chain.
664    ///
665    /// # Arguments
666    ///
667    /// * `chain` - The buffer chain to submit
668    ///
669    /// # Returns
670    ///
671    /// The descriptor ID assigned to this chain. All descriptors in the chain
672    /// share this ID for correlation during completion.
673    ///
674    /// # Errors
675    ///
676    /// - [`RingError::EmptyChain`] - Chain has no buffers
677    /// - [`RingError::WouldBlock`] - Not enough free descriptor slots
678    /// - [`RingError::OutOfMemory`] - No free descriptor IDs (internal error)
679    /// - [`RingError::InvalidState`] - ID tracking or descriptor-table state is corrupted
680    /// - [`RingError::MemError`] - Backend memory error while publishing descriptors
681    pub fn submit_available(&mut self, chain: &BufferChain) -> Result<u16, RingError> {
682        let total_descs = chain.len();
683        if total_descs == 0 {
684            return Err(RingError::EmptyChain);
685        }
686
687        if self.num_free < total_descs {
688            return Err(RingError::WouldBlock);
689        }
690
691        if total_descs == 1 {
692            let elem = chain.elems()[0];
693            return self.submit_one(elem.addr, elem.len, elem.writable);
694        }
695
696        let head_idx = self.avail_cursor.head();
697        let head_wrap = self.avail_cursor.wrap();
698
699        let id = self.id_free.pop().ok_or(RingError::OutOfMemory)?;
700
701        // We should never reuse an ID that is still outstanding
702        if self.id_num[id as usize] != 0 {
703            return Err(RingError::InvalidState);
704        }
705
706        // Record chain length
707        self.id_num[id as usize] = total_descs as u16;
708
709        // Write tail elements first; head last.
710        let mut pos = self.avail_cursor;
711        pos.advance();
712
713        for (i, elem) in chain.elems().iter().enumerate().skip(1) {
714            let is_next = i + 1 < total_descs;
715            let mut flags = DescFlags::empty();
716
717            flags.set(DescFlags::NEXT, is_next);
718            flags.set(DescFlags::WRITE, elem.writable);
719
720            let mut desc = Descriptor::new(elem.addr, elem.len, id, flags);
721            desc.mark_avail(pos.wrap());
722
723            let addr = self
724                .desc_table
725                .desc_addr(pos.head())
726                .ok_or(RingError::InvalidState)?;
727
728            self.mem
729                .write_val(addr, desc)
730                .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
731            pos.advance();
732        }
733
734        // Head descriptor
735        let head_elem = chain.elems()[0];
736        // Record chain length
737        let mut head_flags = DescFlags::empty();
738        head_flags.set(DescFlags::NEXT, total_descs > 1);
739        head_flags.set(DescFlags::WRITE, head_elem.writable);
740
741        let mut head_desc = Descriptor::new(head_elem.addr, head_elem.len, id, head_flags);
742        head_desc.mark_avail(head_wrap);
743
744        let head_addr = self
745            .desc_table
746            .desc_addr(head_idx)
747            .ok_or(RingError::InvalidState)?;
748
749        // Release publish
750        head_desc
751            .write_release(&self.mem, head_addr)
752            .map_err(|_| RingError::mem_err(MemOp::WriteDesc, head_addr))?;
753
754        self.num_free -= total_descs;
755        self.avail_cursor = pos;
756
757        Ok(id)
758    }
759
760    /// Poll the ring for a used buffer.
761    ///
762    /// Checks if the device has marked any buffers as used. If so, returns
763    /// the completion information and reclaims the descriptor(s).
764    ///
765    /// # Returns
766    ///
767    /// - `Ok(UsedBuffer)` - A buffer chain was completed
768    /// - `Err(RingError::WouldBlock)` - No completions available
769    pub fn poll_used(&mut self) -> Result<UsedBuffer, RingError> {
770        let idx = self.used_cursor.head();
771        let wrap = self.used_cursor.wrap();
772
773        // Read the descriptor at next_used position with ordering
774        let addr = self
775            .desc_table
776            .desc_addr(idx)
777            .ok_or(RingError::InvalidState)?;
778
779        // Acquire flags then fields (publish point)
780        let flags = Descriptor::read_flags_acquire(&self.mem, addr)
781            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
782
783        if !flags.is_used(wrap) {
784            return Err(RingError::WouldBlock);
785        }
786
787        let desc = Descriptor::read_body(&self.mem, addr, flags)
788            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
789
790        let id = desc.id;
791        let count = *self
792            .id_num
793            .get(id as usize)
794            .ok_or(RingError::InvalidState)?;
795
796        if count == 0 {
797            return Err(RingError::InvalidState);
798        }
799
800        // Advance used cursor by number of reclaimed descriptors
801        self.used_cursor.advance_by(count);
802        // Update number of free descriptors
803        self.num_free += count as usize;
804        // SAFETY: id is valid because we checked above
805        self.id_num[id as usize] = 0;
806        // Return ID to free stack
807        self.id_free.push(id);
808
809        Ok(UsedBuffer { id, len: desc.len })
810    }
811
812    /// Get the next available descriptor ID without consuming it.
813    pub fn next_id(&self) -> Result<u16, RingError> {
814        let id = *self.id_free.last().ok_or(RingError::OutOfMemory)?;
815        if self.id_num[id as usize] != 0 {
816            return Err(RingError::InvalidState);
817        }
818        Ok(id)
819    }
820
821    /// Get number of free descriptors in the ring.
822    #[inline]
823    pub fn num_free(&self) -> usize {
824        self.num_free
825    }
826
827    /// Get number of inflight (submitted but not yet used) descriptors.
828    #[inline]
829    pub fn num_inflight(&self) -> usize {
830        self.desc_table.len() - self.num_free
831    }
832
833    /// Check if the ring is full (no free descriptors).
834    #[inline]
835    pub fn is_full(&self) -> bool {
836        self.num_free == 0
837    }
838
839    /// Get descriptor table length
840    #[inline]
841    #[allow(clippy::len_without_is_empty)]
842    pub fn len(&self) -> usize {
843        self.desc_table.len()
844    }
845
846    /// Get memory accessor reference
847    #[inline]
848    pub fn mem(&self) -> &M {
849        &self.mem
850    }
851
852    /// Get descriptor table reference
853    #[inline]
854    pub fn desc_table(&self) -> &DescTable {
855        &self.desc_table
856    }
857
858    /// Get a snapshot of the current available cursor position.
859    ///
860    /// Used for batch operations to track the cursor before submitting
861    /// multiple chains, enabling proper event suppression checks.
862    #[inline]
863    pub fn avail_cursor(&self) -> RingCursor {
864        self.avail_cursor
865    }
866
867    /// Get a snapshot of the current used cursor position.
868    ///
869    /// Used for setting up DESC mode event suppression at specific positions.
870    #[inline]
871    pub fn used_cursor(&self) -> RingCursor {
872        self.used_cursor
873    }
874
875    /// Check if device should be notified given a cursor snapshot from before batch start.
876    ///
877    /// This is used for batching: record cursor before first submit, then after all
878    /// submits call this to determine if notification is needed based on event suppression.
879    ///
880    /// # Arguments
881    /// * `old` - Cursor position snapshot taken before batch started
882    pub fn should_notify_since(&self, old: RingCursor) -> Result<bool, RingError> {
883        self.should_notify_device(old, self.avail_cursor)
884    }
885
886    /// Driver disables used-buffer notifications from device to driver.
887    pub fn disable_used_notifications(&mut self) -> Result<(), RingError> {
888        // Avoid redundant MMIO writes if already disabled
889        if self.event_flags_shadow == EventFlags::DISABLE {
890            return Ok(());
891        }
892
893        let mut evt = self
894            .mem
895            .read_val::<EventSuppression>(self.drv_evt_addr)
896            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
897
898        evt.set_flags(EventFlags::DISABLE);
899
900        evt.write_release(&self.mem, self.drv_evt_addr)
901            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
902        self.event_flags_shadow = EventFlags::DISABLE;
903        Ok(())
904    }
905
906    /// Driver enables used-buffer notifications from device to driver.
907    pub fn enable_used_notifications(&mut self) -> Result<(), RingError> {
908        if self.event_flags_shadow == EventFlags::ENABLE {
909            return Ok(());
910        }
911
912        let mut evt = self
913            .mem
914            .read_val::<EventSuppression>(self.drv_evt_addr)
915            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
916
917        evt.set_flags(EventFlags::ENABLE);
918        evt.write_release(&self.mem, self.drv_evt_addr)
919            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
920
921        self.event_flags_shadow = EventFlags::ENABLE;
922        Ok(())
923    }
924
925    /// Driver enables descriptor-specific used notifications (EVENT_IDX / DESC mode).
926    ///
927    /// This tells the device: "Interrupt me when you reach used index (off, wrap)".
928    ///
929    /// This enables batching on the device side - it can complete multiple requests
930    /// before triggering an interrupt.
931    pub fn enable_used_notifications_desc(
932        &mut self,
933        off: u16,
934        wrap: bool,
935    ) -> Result<(), RingError> {
936        let mut evt = self
937            .mem
938            .read_val::<EventSuppression>(self.drv_evt_addr)
939            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.drv_evt_addr))?;
940
941        evt.set_desc_event(off, wrap);
942        evt.set_flags(EventFlags::DESC);
943
944        // Now publish flags = DESC with Release semantics.
945        evt.write_release(&self.mem, self.drv_evt_addr)
946            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
947        // cache shadow
948        self.event_flags_shadow = EventFlags::DESC;
949        Ok(())
950    }
951
952    /// Convenience: enable DESC mode for "next used cursor" like Linux enable_cb_prepare.
953    pub fn enable_used_notifications_for_next(&mut self) -> Result<(), RingError> {
954        let off = self.used_cursor.head();
955        let wrap = self.used_cursor.wrap();
956
957        self.enable_used_notifications_desc(off, wrap)
958    }
959
960    /// Check whether the device should be notified about new available descriptors.
961    fn should_notify_device(&self, old: RingCursor, new: RingCursor) -> Result<bool, RingError> {
962        // VIRTIO 1.1 "The driver MUST perform a suitable memory barrier before
963        // reading the Device Event Suppression structure".
964        //
965        // After publishing descriptors with store-release on the AVAIL/USED flags,
966        // we need a full barrier before reading event suppression, because
967        // release+acquire across different memory locations does NOT provide
968        // Store/Load ordering on weakly-ordered architectures e.g. aarch64.
969        //
970        // Linux kernel uses virtio_mb() full barrier in virtqueue_kick_prepare_packed.
971        fence(Ordering::SeqCst);
972
973        should_notify_evt(&self.mem, self.dev_evt_addr, self.len() as u16, old, new)
974    }
975
976    /// Reset producer state and its shared ring image to the canonical empty state.
977    ///
978    /// The peer must not access the ring during this operation. This clears
979    /// every descriptor and sets the driver event to `ENABLE`. The consumer
980    /// separately owns the device event. This low-level operation does not
981    /// reclaim payload allocations or reconcile higher-level in-flight tracking.
982    ///
983    /// # Errors
984    ///
985    /// Returns [`RingError::MemError`] if descriptor or event normalization
986    /// cannot be written to shared memory. Local bookkeeping remains unchanged
987    /// on error.
988    pub fn reset(&mut self) -> Result<(), RingError> {
989        let table_addr = self.desc_table.base_addr();
990        let size = self.desc_table.len();
991
992        self.desc_table
993            .clear(&self.mem)
994            .map_err(|_| RingError::mem_err(MemOp::WriteDesc, table_addr))?;
995
996        EventSuppression::clear(&self.mem, self.drv_evt_addr)
997            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.drv_evt_addr))?;
998
999        self.avail_cursor.reset();
1000        self.used_cursor.reset();
1001
1002        self.num_free = size;
1003        self.id_free.clear();
1004        self.id_free.extend(0..size as u16);
1005        self.id_num.iter_mut().for_each(|n| *n = 0);
1006        self.event_flags_shadow = EventFlags::ENABLE;
1007        Ok(())
1008    }
1009}
1010
1011/// Consumer (device) side of a packed virtqueue.
1012///
1013/// The consumer polls for available buffer chains submitted by the driver,
1014/// processes them, and marks them as used. This is typically used by the
1015/// device/host side.
1016///
1017/// # Lifecycle
1018///
1019/// 1. **Poll**: Call [`poll_available`](Self::poll_available) to get buffers
1020/// 2. **Process**: Read from readable buffers, write to writable buffers
1021/// 3. **Complete**: Call [`submit_used`](Self::submit_used) to return buffers
1022/// 4. **Notify**: If `submit_used_with_notify` returns true, signal the driver
1023#[derive(Debug)]
1024pub struct RingConsumer<M> {
1025    /// Memory accessor
1026    mem: M,
1027    /// Cursor for reading available (driver-published) descriptors
1028    avail_cursor: RingCursor,
1029    /// Cursor for writing used descriptors
1030    used_cursor: RingCursor,
1031    /// Shared descriptor table
1032    desc_table: DescTable,
1033    /// Per-ID chain length learned when polling (index = ID)
1034    id_num: SmallVec<[u16; DescTable::DEFAULT_LEN]>,
1035    /// Number of descriptors consumed from avail stream but not yet posted as used.
1036    num_inflight: usize,
1037    /// Shadow of device event flags (last written value)
1038    event_flags_shadow: EventFlags,
1039    // reads driver event to control when device should notify
1040    drv_evt_addr: u64,
1041    // write device_event (checks if device wants notification about available buffers)
1042    dev_evt_addr: u64,
1043}
1044
1045impl<M: MemOps> RingConsumer<M> {
1046    pub fn new(layout: Layout, mem: M) -> Self {
1047        let size = layout.desc_table_len() as usize;
1048        let raw = layout.desc_table_addr();
1049
1050        // SAFETY: Layout fields are private and from_base validates ring geometry.
1051        let table = unsafe { DescTable::from_raw_parts(raw, size) };
1052        let cursor = RingCursor::new(size);
1053        let id_chain_len = SmallVec::<[u16; DescTable::DEFAULT_LEN]>::from_elem(0, size);
1054
1055        // Notification enabled by default
1056        let event_flags_shadow = EventFlags::ENABLE;
1057
1058        Self {
1059            mem,
1060            avail_cursor: cursor,
1061            used_cursor: cursor,
1062            desc_table: table,
1063            id_num: id_chain_len,
1064            num_inflight: 0,
1065            event_flags_shadow,
1066            drv_evt_addr: layout.drv_evt_addr(),
1067            dev_evt_addr: layout.dev_evt_addr(),
1068        }
1069    }
1070
1071    /// Poll for an available buffer chain.
1072    ///
1073    /// Returns the chain ID and a [`BufferChain`] containing all buffers.
1074    /// The chain ID must be passed to [`submit_used`](Self::submit_used)
1075    /// when processing is complete.
1076    ///
1077    /// # Returns
1078    ///
1079    /// - `Ok((id, chain))` - A buffer chain is available
1080    /// - `Err(RingError::WouldBlock)` - No buffers available
1081    /// - `Err(RingError::BadChain)` - Malformed chain (driver bug)
1082    /// - `Err(RingError::Bookkeeping)` - Descriptor storage allocation failed
1083    pub fn poll_available(&mut self) -> Result<(u16, BufferChain), RingError> {
1084        let idx = self.avail_cursor.head();
1085        let wrap = self.avail_cursor.wrap();
1086
1087        let head_addr = self
1088            .desc_table
1089            .desc_addr(idx)
1090            .ok_or(RingError::InvalidState)?;
1091
1092        // Acquire: flags then fields (publish point)
1093        let flags = Descriptor::read_flags_acquire(&self.mem, head_addr)
1094            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, head_addr))?;
1095
1096        // Check if head descriptor is available to consume
1097        if !flags.is_avail(wrap) {
1098            return Err(RingError::WouldBlock);
1099        }
1100
1101        let head_desc = Descriptor::read_body(&self.mem, head_addr, flags)
1102            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, head_addr))?;
1103
1104        if flags.contains(DescFlags::INDIRECT) {
1105            return Err(RingError::BadChain);
1106        }
1107
1108        // Build chain (head + tails), tracking readable/writable split inline.
1109        let mut elements = ChainElems::new();
1110        let mut pos = self.avail_cursor;
1111        let mut chain_len: u16 = 1;
1112
1113        let mut steps = 1;
1114        let mut has_next = head_desc.is_next();
1115
1116        let max_steps = self.desc_table.len();
1117
1118        let head_elem = BufferElement::from(&head_desc);
1119        let mut seen_writable = head_elem.writable;
1120        let mut writables: usize = if seen_writable { 1 } else { 0 };
1121        elements.push(head_elem);
1122        pos.advance();
1123
1124        while has_next && steps < max_steps {
1125            let addr = self
1126                .desc_table
1127                .desc_addr(pos.head())
1128                .ok_or(RingError::InvalidState)?;
1129
1130            // tail reads does not need ordering because head has been already validated
1131            let desc: Descriptor = self
1132                .mem
1133                .read_val(addr)
1134                .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
1135            if desc.flags().contains(DescFlags::INDIRECT) {
1136                return Err(RingError::BadChain);
1137            }
1138
1139            let elem = BufferElement::from(&desc);
1140
1141            if elem.writable {
1142                seen_writable = true;
1143                writables += 1;
1144            } else if seen_writable {
1145                return Err(RingError::BadChain);
1146            }
1147
1148            elements
1149                .try_reserve(1)
1150                .map_err(|_| RingError::Bookkeeping)?;
1151            elements.push(elem);
1152
1153            chain_len += 1;
1154            steps += 1;
1155
1156            has_next = desc.is_next();
1157            pos.advance();
1158        }
1159
1160        // Detect malformed chains, this means we reached max_steps but still have NEXT set.
1161        if steps >= max_steps && has_next {
1162            return Err(RingError::BadChain);
1163        }
1164
1165        // Check if next inflight will exceed ring capacity - this should never happen if driver is
1166        // well-behaved and we correctly track inflight count.
1167        if self.num_inflight + chain_len as usize > self.desc_table.len() {
1168            return Err(RingError::InvalidState);
1169        }
1170
1171        let readables = elements.len() - writables;
1172
1173        // Since driver wrote the same id everywhere, head_desc.id is valid.
1174        let id = head_desc.id;
1175        let id_num = self
1176            .id_num
1177            .get_mut(id as usize)
1178            .ok_or(RingError::InvalidState)?;
1179        if *id_num != 0 {
1180            return Err(RingError::InvalidState);
1181        }
1182
1183        // Record chain length for later used submission
1184        *id_num = chain_len;
1185        // Advance avail cursor to first slot after chain
1186        self.avail_cursor = pos;
1187        // Update inflight count
1188        self.num_inflight += chain_len as usize;
1189
1190        Ok((
1191            id,
1192            BufferChain {
1193                elems: elements,
1194                split: readables,
1195            },
1196        ))
1197    }
1198
1199    /// Publish a single used descriptor for the chain identified by id.
1200    /// written_len is the total bytes produced by the device (for writable part).
1201    ///
1202    /// # Arguments
1203    ///
1204    /// * `id` - The chain ID from `poll_available`
1205    /// * `written_len` - Total bytes written to writable buffers
1206    ///
1207    /// # Errors
1208    ///
1209    /// - [`RingError::InvalidState`] - Unknown ID or already completed
1210    pub fn submit_used(&mut self, id: u16, written_len: u32) -> Result<(), RingError> {
1211        // Lookup chain length
1212        let chain_len = *self
1213            .id_num
1214            .get(id as usize)
1215            .ok_or(RingError::InvalidState)?;
1216
1217        if chain_len == 0 || chain_len > self.desc_table.len() as u16 {
1218            return Err(RingError::InvalidState);
1219        }
1220
1221        let idx = self.used_cursor.head();
1222        let wrap = self.used_cursor.wrap();
1223
1224        // addr is unused for used descriptor according to packed-virtqueue spec
1225        let mut used_desc = Descriptor::new(0, written_len, id, DescFlags::empty());
1226        used_desc.mark_used(wrap);
1227
1228        let addr = self
1229            .desc_table
1230            .desc_addr(idx)
1231            .ok_or(RingError::InvalidState)?;
1232
1233        // Release publish (flags written last inside write_release)
1234        used_desc
1235            .write_release(&self.mem, addr)
1236            .map_err(|_| RingError::mem_err(MemOp::WriteDesc, addr))?;
1237
1238        // Advance used cursor by whole chain length
1239        self.used_cursor.advance_by(chain_len);
1240        self.id_num[id as usize] = 0;
1241
1242        self.num_inflight -= chain_len as usize;
1243        Ok(())
1244    }
1245
1246    /// Try to peek whether the next chain is available without consuming it.
1247    pub fn peek_available(&self) -> Result<bool, RingError> {
1248        let Some(addr) = self.desc_table.desc_addr(self.avail_cursor.head()) else {
1249            return Err(RingError::InvalidState);
1250        };
1251
1252        let flags = Descriptor::read_flags_acquire(&self.mem, addr)
1253            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
1254
1255        Ok(flags.is_avail(self.avail_cursor.wrap()))
1256    }
1257
1258    /// Capture the local state needed to undo subsequent polls.
1259    pub(super) fn poll_checkpoint(&self) -> Checkpoint {
1260        Checkpoint {
1261            avail_cursor: self.avail_cursor,
1262            used_cursor: self.used_cursor,
1263            num_inflight: self.num_inflight,
1264        }
1265    }
1266
1267    /// Undo a batch's successful polls without publishing completions.
1268    ///
1269    /// `ids` must cover every chain polled since `cp` on this consumer.
1270    /// Each pair of handles must be dropped before its ID is yielded,
1271    /// with no intervening completions.
1272    pub(super) fn rollback_polls(&mut self, cp: Checkpoint, ids: impl IntoIterator<Item = u16>) {
1273        let mut desc_count = 0usize;
1274        for id in ids {
1275            let chain_len = core::mem::take(&mut self.id_num[id as usize]);
1276            debug_assert_ne!(chain_len, 0);
1277            desc_count += chain_len as usize;
1278        }
1279
1280        debug_assert!(desc_count <= self.desc_table.len());
1281        debug_assert_eq!(self.used_cursor, cp.used_cursor);
1282        debug_assert_eq!(self.num_inflight, cp.num_inflight + desc_count);
1283        debug_assert_eq!(self.avail_cursor, {
1284            let mut expected = cp.avail_cursor;
1285            expected.advance_by(desc_count as u16);
1286            expected
1287        });
1288
1289        self.avail_cursor = cp.avail_cursor;
1290        self.num_inflight = cp.num_inflight;
1291    }
1292
1293    /// Submit a used descriptor and return whether to notify the driver.
1294    pub fn submit_used_with_notify(
1295        &mut self,
1296        id: u16,
1297        written_len: u32,
1298    ) -> Result<bool, RingError> {
1299        let old = self.used_cursor;
1300        self.submit_used(id, written_len)?;
1301        let new = self.used_cursor;
1302        self.should_notify_driver(old, new)
1303    }
1304
1305    /// Get number of free descriptors in the ring.
1306    pub fn num_free(&self) -> usize {
1307        self.desc_table.len() - self.num_inflight
1308    }
1309
1310    /// Get number of inflight (submitted but not yet used) descriptors.
1311    pub fn num_inflight(&self) -> usize {
1312        self.num_inflight
1313    }
1314
1315    /// Check if the ring is full (no free descriptors).
1316    pub fn is_full(&self) -> bool {
1317        self.num_inflight == self.desc_table.len()
1318    }
1319
1320    /// Get descriptor table length
1321    #[allow(clippy::len_without_is_empty)]
1322    pub fn len(&self) -> usize {
1323        self.desc_table.len()
1324    }
1325
1326    /// Get memory accessor reference
1327    pub fn mem(&self) -> &M {
1328        &self.mem
1329    }
1330
1331    /// Get a snapshot of the current avail cursor position.
1332    #[inline]
1333    pub fn avail_cursor(&self) -> RingCursor {
1334        self.avail_cursor
1335    }
1336
1337    /// Get a snapshot of the current used cursor position.
1338    #[inline]
1339    pub fn used_cursor(&self) -> RingCursor {
1340        self.used_cursor
1341    }
1342
1343    /// Device disables available-buffer notifications from driver to device.
1344    ///
1345    /// This is the device-side mirror of "disable callbacks" but for avail kicks.
1346    pub fn disable_avail_notifications(&mut self) -> Result<(), RingError> {
1347        if self.event_flags_shadow == EventFlags::DISABLE {
1348            return Ok(());
1349        }
1350
1351        let mut evt = self
1352            .mem
1353            .read_val::<EventSuppression>(self.dev_evt_addr)
1354            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1355
1356        evt.set_flags(EventFlags::DISABLE);
1357        evt.write_release(&self.mem, self.dev_evt_addr)
1358            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1359
1360        self.event_flags_shadow = EventFlags::DISABLE;
1361        Ok(())
1362    }
1363
1364    /// Device enables available-buffer notifications from driver to device.
1365    pub fn enable_avail_notifications(&mut self) -> Result<(), RingError> {
1366        if self.event_flags_shadow == EventFlags::ENABLE {
1367            return Ok(());
1368        }
1369
1370        let mut evt = self
1371            .mem
1372            .read_val::<EventSuppression>(self.dev_evt_addr)
1373            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1374
1375        evt.set_flags(EventFlags::ENABLE);
1376        evt.write_release(&self.mem, self.dev_evt_addr)
1377            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1378
1379        self.event_flags_shadow = EventFlags::ENABLE;
1380        Ok(())
1381    }
1382
1383    /// Device enables descriptor-specific available notifications (EVENT_IDX / DESC mode).
1384    ///
1385    /// This tells the driver: "Kick me when you reach avail index (off, wrap)".
1386    pub fn enable_avail_notifications_desc(
1387        &mut self,
1388        off: u16,
1389        wrap: bool,
1390    ) -> Result<(), RingError> {
1391        // Update off_wrap first
1392        let mut evt = self
1393            .mem
1394            .read_val::<EventSuppression>(self.dev_evt_addr)
1395            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, self.dev_evt_addr))?;
1396
1397        evt.set_desc_event(off, wrap);
1398        evt.set_flags(EventFlags::DESC);
1399
1400        // Now publish flags = DESC with Release semantics.
1401        evt.write_release(&self.mem, self.dev_evt_addr)
1402            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1403
1404        self.event_flags_shadow = EventFlags::DESC;
1405        Ok(())
1406    }
1407
1408    /// Convenience: enable DESC mode for "next avail cursor" (device wants a kick when new
1409    /// buffers arrive at the next index it will poll).
1410    pub fn enable_avail_notifications_for_next(&mut self) -> Result<(), RingError> {
1411        let off = self.avail_cursor.head();
1412        let wrap = self.avail_cursor.wrap();
1413        self.enable_avail_notifications_desc(off, wrap)
1414    }
1415
1416    /// Decide whether the device should notify the driver about newly used descriptors.
1417    fn should_notify_driver(&self, old: RingCursor, new: RingCursor) -> Result<bool, RingError> {
1418        // VIRTIO 1.1: Full memory barrier required before reading the
1419        // Driver Event Suppression structure. See also should_notify_device()
1420        fence(Ordering::SeqCst);
1421
1422        should_notify_evt(&self.mem, self.drv_evt_addr, self.len() as u16, old, new)
1423    }
1424
1425    /// Reset consumer state and normalize its event-suppression structure.
1426    ///
1427    /// The peer must not access the ring during this operation. Descriptor
1428    /// contents remain producer-owned. This lets a fresh consumer adopt a
1429    /// canonical prefill. A higher-level caller must first rule out outstanding
1430    /// descriptor views.
1431    ///
1432    /// # Errors
1433    ///
1434    /// Returns [`RingError::MemError`] if the device event cannot be normalized
1435    /// in shared memory. Local bookkeeping remains unchanged on error.
1436    pub fn reset(&mut self) -> Result<(), RingError> {
1437        EventSuppression::clear(&self.mem, self.dev_evt_addr)
1438            .map_err(|_| RingError::mem_err(MemOp::WriteEvent, self.dev_evt_addr))?;
1439
1440        self.avail_cursor.reset();
1441        self.used_cursor.reset();
1442        self.id_num.iter_mut().for_each(|n| *n = 0);
1443        self.num_inflight = 0;
1444        self.event_flags_shadow = EventFlags::ENABLE;
1445        Ok(())
1446    }
1447}
1448
1449/// Read an event-suppression structure and decide whether to notify the peer.
1450fn should_notify_evt<M: MemOps>(
1451    mem: &M,
1452    evt_addr: u64,
1453    ring_len: u16,
1454    old: RingCursor,
1455    new: RingCursor,
1456) -> Result<bool, RingError> {
1457    let flags = EventSuppression::read_flags_acquire(mem, evt_addr)
1458        .map_err(|_| RingError::mem_err(MemOp::ReadEvent, evt_addr))?;
1459
1460    let evt = if flags == EventFlags::DESC {
1461        EventSuppression::read_body(mem, evt_addr, flags)
1462            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, evt_addr))?
1463    } else {
1464        EventSuppression::new(0, flags)
1465    };
1466
1467    Ok(should_notify(evt, ring_len, old, new))
1468}
1469
1470/// Common packed-ring notification decision:
1471/// - `old` and `new` are the ring indices (head) before/after publishing a batch
1472/// - `new.wrap()` is the wrap counter corresponding to `new.head()`
1473/// - `evt.desc_event_wrap()` is compared against `new.wrap()`
1474///
1475/// This is compatible with Linux `virtqueue_kick_prepare_packed` logic
1476#[inline]
1477fn should_notify(evt: EventSuppression, ring_len: u16, old: RingCursor, new: RingCursor) -> bool {
1478    match evt.flags() {
1479        EventFlags::DISABLE => false,
1480        EventFlags::ENABLE => true,
1481        EventFlags::DESC => {
1482            let mut off = evt.desc_event_off();
1483            if off >= ring_len {
1484                return false;
1485            }
1486
1487            let wrap = evt.desc_event_wrap();
1488
1489            if wrap != new.wrap() {
1490                off = off.wrapping_sub(ring_len);
1491            }
1492
1493            ring_need_event(off, new.head(), old.head())
1494        }
1495        // treat as disabled if invalid
1496        _ => false,
1497    }
1498}
1499
1500#[inline(always)]
1501fn ring_need_event(event_idx: u16, new: u16, old: u16) -> bool {
1502    new.wrapping_sub(event_idx).wrapping_sub(1) < new.wrapping_sub(old)
1503}
1504
1505impl From<&Descriptor> for BufferElement {
1506    fn from(desc: &Descriptor) -> Self {
1507        BufferElement {
1508            addr: desc.addr,
1509            len: desc.len,
1510            writable: desc.is_writable(),
1511        }
1512    }
1513}
1514
1515#[cfg(test)]
1516pub(crate) mod tests {
1517    use alloc::sync::Arc;
1518    use alloc::vec::Vec;
1519    use core::cell::UnsafeCell;
1520    use core::num::NonZeroU16;
1521    use core::ptr::{self, NonNull};
1522    use core::sync::atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering};
1523
1524    use bytemuck::{Pod, Zeroable};
1525
1526    use super::super::align_up;
1527    use super::*;
1528    use crate::virtq::event::EventSuppression;
1529    use crate::virtq::{BufferLease, BufferMap};
1530
1531    /// Test MemOps implementation that maintains pointer provenance.
1532    ///
1533    /// Wraps shared storage behind Arc for cheap cloning. This allows
1534    /// producer and consumer to share the same backing memory without
1535    /// Arc appearing in the type signatures.
1536    #[derive(Clone)]
1537    pub struct TestMem {
1538        inner: Arc<TestMemInner>,
1539    }
1540
1541    struct TestMemInner {
1542        /// The backing storage - UnsafeCell for interior mutability
1543        storage: UnsafeCell<Vec<u8>>,
1544        /// Base address (the address we tell the ring about)
1545        base_addr: u64,
1546    }
1547
1548    // Safety: TestMemInner's UnsafeCell is only accessed from test code
1549    // with no real concurrency in unit tests (loom tests use LoomMem).
1550    unsafe impl Send for TestMemInner {}
1551    unsafe impl Sync for TestMemInner {}
1552
1553    impl TestMem {
1554        pub fn new(size: usize) -> Self {
1555            let storage = vec![0u8; size];
1556            let base_addr = storage.as_ptr() as u64;
1557            Self {
1558                inner: Arc::new(TestMemInner {
1559                    storage: UnsafeCell::new(storage),
1560                    base_addr,
1561                }),
1562            }
1563        }
1564
1565        /// Get a pointer with proper provenance for the given address
1566        fn ptr_for_addr(&self, addr: u64) -> *mut u8 {
1567            let storage = unsafe { &mut *self.inner.storage.get() };
1568            let base_ptr = storage.as_mut_ptr();
1569            let offset = (addr - self.inner.base_addr) as usize;
1570            // Use wrapping_add to maintain provenance from base_ptr
1571            base_ptr.wrapping_add(offset)
1572        }
1573
1574        pub fn base_addr(&self) -> u64 {
1575            self.inner.base_addr
1576        }
1577    }
1578
1579    pub struct TestMapping {
1580        creator: std::thread::ThreadId,
1581        owner: core::mem::ManuallyDrop<TestBufferOwner>,
1582    }
1583
1584    struct TestBufferOwner {
1585        data: NonNull<[u8]>,
1586        _mem: TestMem,
1587        _lease: BufferLease,
1588    }
1589
1590    // SAFETY: The view is immutable. Drop checks the creator thread before
1591    // destroying the owner containing the Rc-backed lease.
1592    unsafe impl Send for TestMapping {}
1593
1594    impl AsRef<[u8]> for TestMapping {
1595        fn as_ref(&self) -> &[u8] {
1596            // SAFETY: The mapping owns the backing and its initialized immutable prefix.
1597            unsafe { self.owner.data.as_ref() }
1598        }
1599    }
1600
1601    impl Drop for TestMapping {
1602        fn drop(&mut self) {
1603            assert_eq!(
1604                self.creator,
1605                std::thread::current().id(),
1606                "mapping dropped on another thread"
1607            );
1608            // SAFETY: The creator thread releases the mapping before its lease.
1609            unsafe { core::mem::ManuallyDrop::drop(&mut self.owner) };
1610        }
1611    }
1612
1613    impl BufferMap for TestMem {
1614        type Mapping = TestMapping;
1615
1616        unsafe fn map_buffer(
1617            &self,
1618            lease: BufferLease,
1619            written: usize,
1620        ) -> Result<Self::Mapping, Self::Error> {
1621            let alloc = lease.allocation();
1622            assert!(written <= alloc.len as usize);
1623
1624            let offset = alloc.addr.checked_sub(self.base_addr()).unwrap() as usize;
1625
1626            // SAFETY: Test storage is never resized.
1627            let size = unsafe { &*self.inner.storage.get() }.len();
1628            assert!(offset.checked_add(alloc.len as usize).unwrap() <= size);
1629
1630            // SAFETY: The caller owns the allocation and excludes writes. The
1631            // cloned TestMem retains its stable backing after this borrow.
1632            let data = NonNull::from(unsafe { self.as_slice(alloc.addr, written)? });
1633
1634            Ok(TestMapping {
1635                creator: std::thread::current().id(),
1636                owner: core::mem::ManuallyDrop::new(TestBufferOwner {
1637                    data,
1638                    _mem: self.clone(),
1639                    _lease: lease,
1640                }),
1641            })
1642        }
1643    }
1644
1645    // SAFETY: TestMem translates addresses into its owned backing storage. Unit
1646    // tests construct layouts within that storage and avoid concurrent access.
1647    unsafe impl MemOps for TestMem {
1648        type Error = core::convert::Infallible;
1649
1650        fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1651            let src = self.ptr_for_addr(addr);
1652            unsafe {
1653                ptr::copy_nonoverlapping(src, dst.as_mut_ptr(), dst.len());
1654            }
1655            Ok(())
1656        }
1657
1658        fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1659            let dst = self.ptr_for_addr(addr);
1660            unsafe {
1661                ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len());
1662            }
1663            Ok(())
1664        }
1665
1666        fn read_val<T: Pod>(&self, addr: u64) -> Result<T, Self::Error> {
1667            let ptr = self.ptr_for_addr(addr).cast::<T>();
1668            Ok(unsafe { ptr::read_volatile(ptr) })
1669        }
1670
1671        fn write_val<T: Pod>(&self, addr: u64, val: T) -> Result<(), Self::Error> {
1672            let ptr = self.ptr_for_addr(addr).cast::<T>();
1673            unsafe { ptr::write_volatile(ptr, val) };
1674            Ok(())
1675        }
1676
1677        fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1678            let ptr = self.ptr_for_addr(addr).cast::<AtomicU16>();
1679            Ok(unsafe { (*ptr).load(Ordering::Acquire) })
1680        }
1681
1682        fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1683            let ptr = self.ptr_for_addr(addr).cast::<AtomicU16>();
1684            unsafe { (*ptr).store(val, Ordering::Release) };
1685            Ok(())
1686        }
1687
1688        unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1689            let ptr = self.ptr_for_addr(addr);
1690            Ok(unsafe { core::slice::from_raw_parts(ptr, len) })
1691        }
1692
1693        unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1694            let ptr = self.ptr_for_addr(addr);
1695            Ok(unsafe { core::slice::from_raw_parts_mut(ptr, len) })
1696        }
1697    }
1698
1699    /// Shared fault injection over real test memory and buffer owners.
1700    #[derive(Clone)]
1701    pub(crate) struct FaultMem(pub Arc<FaultState>);
1702
1703    pub(crate) struct FaultState {
1704        pub mem: TestMem,
1705        pub writes: AtomicUsize,
1706        pub fail_write_at: AtomicUsize,
1707        pub fail_mapping_at: AtomicUsize,
1708        pub map_calls: AtomicUsize,
1709        pub deny_views: AtomicBool,
1710    }
1711
1712    impl FaultMem {
1713        pub(crate) fn new(mem: TestMem) -> Self {
1714            Self(Arc::new(FaultState {
1715                mem,
1716                fail_write_at: AtomicUsize::new(usize::MAX),
1717                writes: AtomicUsize::new(0),
1718                fail_mapping_at: AtomicUsize::new(usize::MAX),
1719                map_calls: AtomicUsize::new(0),
1720                deny_views: AtomicBool::new(false),
1721            }))
1722        }
1723
1724        /// Reset the write count and fail at this zero-based index.
1725        /// Byte writes and release stores share the count.
1726        pub(crate) fn fail_write_at(&self, write: usize) {
1727            self.0.writes.store(0, Ordering::Relaxed);
1728            self.0.fail_write_at.store(write, Ordering::Relaxed);
1729        }
1730
1731        pub(crate) fn allow_writes(&self) {
1732            self.fail_write_at(usize::MAX);
1733        }
1734
1735        /// Fail the zero-based mapping attempt after resetting its counter.
1736        pub(crate) fn fail_mapping_at(&self, mapping: usize) {
1737            self.0.map_calls.store(0, Ordering::Relaxed);
1738            self.0.fail_mapping_at.store(mapping, Ordering::Relaxed);
1739        }
1740
1741        /// Reject borrowed slices and owned mappings.
1742        pub(crate) fn deny_views(&self) {
1743            self.0.deny_views.store(true, Ordering::Relaxed);
1744        }
1745
1746        fn check_write(&self) -> Result<(), ()> {
1747            let write = self.0.writes.fetch_add(1, Ordering::Relaxed);
1748            if write == self.0.fail_write_at.load(Ordering::Relaxed) {
1749                Err(())
1750            } else {
1751                Ok(())
1752            }
1753        }
1754    }
1755
1756    // SAFETY: Successful operations delegate to TestMem with the same address
1757    // and ownership preconditions. Injected failures perform no memory access.
1758    unsafe impl MemOps for FaultMem {
1759        type Error = ();
1760
1761        fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1762            self.0.mem.read(addr, dst).map_err(|never| match never {})
1763        }
1764
1765        fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1766            self.check_write()?;
1767            self.0.mem.write(addr, src).map_err(|never| match never {})
1768        }
1769
1770        fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1771            self.0
1772                .mem
1773                .load_acquire(addr)
1774                .map_err(|never| match never {})
1775        }
1776
1777        fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1778            self.check_write()?;
1779            self.0
1780                .mem
1781                .store_release(addr, val)
1782                .map_err(|never| match never {})
1783        }
1784
1785        unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1786            if self.0.deny_views.load(Ordering::Relaxed) {
1787                return Err(());
1788            }
1789            // SAFETY: The caller supplies TestMem's immutable slice preconditions.
1790            unsafe { self.0.mem.as_slice(addr, len) }.map_err(|never| match never {})
1791        }
1792
1793        unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1794            if self.0.deny_views.load(Ordering::Relaxed) {
1795                return Err(());
1796            }
1797            // SAFETY: The caller supplies exclusive access to this range.
1798            unsafe { self.0.mem.as_mut_slice(addr, len) }.map_err(|never| match never {})
1799        }
1800    }
1801
1802    /// Retains the backend generation for lifetime assertions.
1803    pub(crate) struct TrackedMapping {
1804        data: TestMapping,
1805        _mem: FaultMem,
1806    }
1807
1808    impl AsRef<[u8]> for TrackedMapping {
1809        fn as_ref(&self) -> &[u8] {
1810            self.data.as_ref()
1811        }
1812    }
1813
1814    impl BufferMap for FaultMem {
1815        type Mapping = TrackedMapping;
1816
1817        unsafe fn map_buffer(
1818            &self,
1819            lease: BufferLease,
1820            written: usize,
1821        ) -> Result<Self::Mapping, Self::Error> {
1822            let call = self.0.map_calls.fetch_add(1, Ordering::Relaxed);
1823            if self.0.deny_views.load(Ordering::Relaxed)
1824                || call == self.0.fail_mapping_at.load(Ordering::Relaxed)
1825            {
1826                return Err(());
1827            }
1828            // SAFETY: The caller supplies TestMem's mapping preconditions.
1829            let data = unsafe { BufferMap::map_buffer(&self.0.mem, lease, written) }
1830                .map_err(|never| match never {})?;
1831            Ok(TrackedMapping {
1832                data,
1833                _mem: self.clone(),
1834            })
1835        }
1836    }
1837
1838    /// Owns the descriptor table and event suppression structures
1839    pub struct OwnedRing {
1840        mem: TestMem,
1841        layout: Layout,
1842    }
1843
1844    impl OwnedRing {
1845        pub fn new(size: usize) -> Self {
1846            let num_descs = NonZeroU16::new(size as u16).unwrap();
1847            let needed = Layout::query_size(size);
1848
1849            // Add padding for alignment, plus extra space for pool buffers
1850            // used by high-level API tests (pool offset = ring_end + 0x100,
1851            // pool size = 0x8000).
1852            let padding = Descriptor::ALIGN;
1853            let pool_headroom = 0x100 + 0x8000;
1854            let mem = TestMem::new(needed + padding + pool_headroom);
1855
1856            // Align the base address
1857            let aligned_base = align_up(mem.base_addr() as usize, Descriptor::ALIGN) as u64;
1858            let layout = unsafe { Layout::from_base(aligned_base, num_descs).unwrap() };
1859
1860            Self { mem, layout }
1861        }
1862
1863        pub fn layout(&self) -> Layout {
1864            self.layout
1865        }
1866
1867        pub fn mem(&self) -> TestMem {
1868            self.mem.clone()
1869        }
1870
1871        /// Get address of descriptor at index
1872        pub fn desc_addr(&self, idx: u16) -> u64 {
1873            self.layout.desc_table_addr() + (idx as u64 * Descriptor::SIZE as u64)
1874        }
1875
1876        /// Read descriptor directly (for test verification)
1877        pub fn read_desc(&self, idx: u16) -> Descriptor {
1878            self.mem.read_val(self.desc_addr(idx)).unwrap()
1879        }
1880
1881        /// Write descriptor directly (for test manipulation)
1882        pub fn write_desc(&self, idx: u16, desc: Descriptor) {
1883            self.mem.write_val(self.desc_addr(idx), desc).unwrap()
1884        }
1885
1886        /// Read driver event directly
1887        pub fn read_driver_event(&self) -> EventSuppression {
1888            self.mem.read_val(self.layout.drv_evt_addr()).unwrap()
1889        }
1890
1891        /// Read device event directly
1892        pub fn read_device_event(&self) -> EventSuppression {
1893            self.mem.read_val(self.layout.dev_evt_addr()).unwrap()
1894        }
1895
1896        pub fn len(&self) -> usize {
1897            self.layout.desc_table_len() as usize
1898        }
1899    }
1900
1901    // Share the TestMem between producer and consumer via reference
1902    pub(crate) fn make_ring(size: usize) -> OwnedRing {
1903        OwnedRing::new(size)
1904    }
1905
1906    pub(crate) fn make_producer(ring: &OwnedRing) -> RingProducer<TestMem> {
1907        RingProducer::new(ring.layout(), ring.mem())
1908    }
1909
1910    pub(crate) fn make_consumer(ring: &OwnedRing) -> RingConsumer<TestMem> {
1911        RingConsumer::new(ring.layout(), ring.mem())
1912    }
1913
1914    fn assert_invariants(ring: &OwnedRing, prod: &RingProducer<TestMem>) {
1915        let outstanding: u16 = prod.id_num.iter().copied().sum();
1916        assert_eq!(outstanding as usize + prod.num_free, ring.len());
1917
1918        for id in prod.id_free.iter() {
1919            assert_eq!(prod.id_num[*id as usize], 0);
1920        }
1921
1922        for (id, &n) in prod.id_num.iter().enumerate() {
1923            if n > 0 {
1924                assert!(!prod.id_free.contains(&(id as u16)));
1925            }
1926        }
1927    }
1928
1929    #[test]
1930    fn test_initialization() {
1931        let ring = make_ring(8);
1932        let producer = make_producer(&ring);
1933
1934        // All descriptors should be zeroed
1935        for i in 0..8u16 {
1936            let desc = ring.read_desc(i);
1937            assert_eq!(desc, Descriptor::zeroed());
1938            assert_eq!(desc.flags, 0);
1939            assert_eq!(desc.addr, 0);
1940            assert_eq!(desc.len, 0);
1941            assert_eq!(desc.id, 0);
1942        }
1943
1944        // Cursors start at head=0, wrap=true
1945        assert_eq!(producer.avail_cursor.head(), 0);
1946        assert!(producer.avail_cursor.wrap());
1947        assert_eq!(producer.used_cursor.head(), 0);
1948        assert!(producer.used_cursor.wrap());
1949
1950        // All IDs free, id_num zeroed, num_free == size
1951        assert_eq!(producer.id_free.len(), 8);
1952        assert_eq!(producer.num_free, 8);
1953        for i in 0..8 {
1954            assert_eq!(producer.id_num[i], 0);
1955        }
1956    }
1957
1958    #[test]
1959    fn test_buffer_chain_builder_normalizes_element_direction() {
1960        let readable_as_writable = BufferElement {
1961            addr: 0x1000,
1962            len: 16,
1963            writable: true,
1964        };
1965        let writable_as_readable = BufferElement {
1966            addr: 0x2000,
1967            len: 32,
1968            writable: false,
1969        };
1970        let second_writable_as_readable = BufferElement {
1971            addr: 0x3000,
1972            len: 64,
1973            writable: false,
1974        };
1975
1976        let chain = BufferChainBuilder::new()
1977            .readables([readable_as_writable])
1978            .writables([writable_as_readable])
1979            .writables([second_writable_as_readable])
1980            .build()
1981            .unwrap();
1982
1983        assert!(!chain.readables()[0].writable);
1984        assert!(chain.writables().iter().all(|elem| elem.writable));
1985    }
1986
1987    #[test]
1988    fn test_submit_one_descriptor() {
1989        let ring = make_ring(8);
1990        let mut producer = make_producer(&ring);
1991
1992        let addr = 0x1000;
1993        let len = 512;
1994        let writable = false;
1995
1996        let id = producer.submit_one(addr, len, writable).unwrap();
1997
1998        // Check descriptor was written correctly
1999        let desc = ring.read_desc(0);
2000
2001        assert_eq!(desc.addr, addr);
2002        assert_eq!(desc.len, len);
2003        assert_eq!(desc.id, id);
2004
2005        // AVAIL should match wrap (true), USED should be inverse (false)
2006        let flags = desc.flags();
2007        assert!(flags.contains(DescFlags::AVAIL));
2008        assert!(!flags.contains(DescFlags::USED));
2009        assert!(!flags.contains(DescFlags::WRITE));
2010        assert!(!flags.contains(DescFlags::NEXT));
2011
2012        // num_free should be decremented
2013        assert_eq!(producer.num_free, 7);
2014
2015        // Cursor advanced
2016        assert_eq!(producer.avail_cursor.head(), 1);
2017        assert!(producer.avail_cursor.wrap());
2018
2019        // ID allocated and chain length recorded
2020        assert_eq!(producer.id_num[id as usize], 1);
2021        assert_eq!(producer.id_free.len(), 7);
2022    }
2023
2024    #[test]
2025    fn test_single_descriptor_wrap_toggle() {
2026        let ring = make_ring(4);
2027        let mut producer = make_producer(&ring);
2028
2029        // Advance to last slot
2030        producer.avail_cursor.head = 3;
2031        producer.avail_cursor.wrap = true;
2032        producer.num_free = 1;
2033        producer.id_free.clear();
2034        producer.id_free.push(0);
2035
2036        let _id = producer.submit_one(0x1000, 512, false).unwrap();
2037
2038        // After submission, cursor should wrap
2039        assert_eq!(producer.avail_cursor.head(), 0);
2040        assert!(!producer.avail_cursor.wrap());
2041
2042        // Descriptor should have old wrap bits
2043        let desc = ring.read_desc(3);
2044        let flags = desc.flags();
2045        assert!(flags.contains(DescFlags::AVAIL));
2046        assert!(!flags.contains(DescFlags::USED));
2047    }
2048
2049    #[test]
2050    fn test_multi_descriptor_no_wrap() {
2051        let ring = make_ring(8);
2052        let mut producer = make_producer(&ring);
2053
2054        let chain = BufferChainBuilder::new()
2055            .readable(0x1000, 256)
2056            .readable(0x2000, 256)
2057            .writable(0x3000, 512)
2058            .build()
2059            .unwrap();
2060
2061        let id = producer.submit_available(&chain).unwrap();
2062
2063        // Check head descriptor
2064        let head_desc = ring.read_desc(0);
2065        assert_eq!(head_desc.addr, 0x1000);
2066        assert_eq!(head_desc.len, 256);
2067        assert_eq!(head_desc.id, id);
2068
2069        let head_flags = head_desc.flags();
2070        assert!(head_flags.contains(DescFlags::NEXT));
2071        assert!(!head_flags.contains(DescFlags::WRITE));
2072        assert!(head_flags.contains(DescFlags::AVAIL));
2073        assert!(!head_flags.contains(DescFlags::USED));
2074
2075        // Check middle descriptor
2076        let mid_desc = ring.read_desc(1);
2077        assert_eq!(mid_desc.addr, 0x2000);
2078        assert_eq!(mid_desc.len, 256);
2079        assert_eq!(mid_desc.id, id);
2080
2081        let mid_flags = mid_desc.flags();
2082        assert!(mid_flags.contains(DescFlags::NEXT));
2083        assert!(!mid_flags.contains(DescFlags::WRITE));
2084
2085        // Check tail descriptor
2086        let tail_desc = ring.read_desc(2);
2087        assert_eq!(tail_desc.addr, 0x3000);
2088        assert_eq!(tail_desc.len, 512);
2089        assert_eq!(tail_desc.id, id);
2090
2091        let tail_flags = tail_desc.flags();
2092        assert!(!tail_flags.contains(DescFlags::NEXT));
2093        assert!(tail_flags.contains(DescFlags::WRITE));
2094
2095        // All descriptors have same ID
2096        assert_eq!(head_desc.id, mid_desc.id);
2097        assert_eq!(mid_desc.id, tail_desc.id);
2098
2099        // Check state updates
2100        assert_eq!(producer.num_free, 5);
2101        assert_eq!(producer.avail_cursor.head(), 3);
2102        assert_eq!(producer.id_num[id as usize], 3);
2103    }
2104
2105    #[test]
2106    fn test_multi_descriptor_with_wrap() {
2107        let ring = make_ring(4);
2108        let mut producer = make_producer(&ring);
2109
2110        // Position head near end
2111        producer.avail_cursor.head = 2;
2112        producer.avail_cursor.wrap = true;
2113
2114        let chain = BufferChainBuilder::new()
2115            .readable(0x1000, 256)
2116            .readable(0x2000, 256)
2117            .readable(0x3000, 256)
2118            .build()
2119            .unwrap();
2120
2121        let _id = producer.submit_available(&chain).unwrap();
2122
2123        // Head at index 2 with wrap=true
2124        let head_desc = ring.read_desc(2);
2125        let head_flags = head_desc.flags();
2126        assert!(head_flags.contains(DescFlags::AVAIL));
2127        assert!(!head_flags.contains(DescFlags::USED));
2128
2129        // Middle at index 3 with wrap=true (before boundary)
2130        let mid_desc = ring.read_desc(3);
2131        let mid_flags = mid_desc.flags();
2132        assert!(mid_flags.contains(DescFlags::AVAIL));
2133        assert!(!mid_flags.contains(DescFlags::USED));
2134
2135        // Tail at index 0 with wrap=false (after boundary)
2136        let tail_desc = ring.read_desc(0);
2137        let tail_flags = tail_desc.flags();
2138        assert!(!tail_flags.contains(DescFlags::AVAIL));
2139        assert!(tail_flags.contains(DescFlags::USED));
2140
2141        // Cursor should have wrapped
2142        assert_eq!(producer.avail_cursor.head(), 1);
2143        assert!(!producer.avail_cursor.wrap());
2144    }
2145
2146    #[test]
2147    fn test_ring_full() {
2148        let ring = make_ring(4);
2149        let mut producer = make_producer(&ring);
2150
2151        // Fill ring completely
2152        for _ in 0..4 {
2153            producer.submit_one(0x1000, 256, false).unwrap();
2154        }
2155
2156        assert_eq!(producer.num_free, 0);
2157
2158        // Next submit should fail
2159        let result = producer.submit_one(0x5000, 256, false);
2160        assert!(matches!(result, Err(RingError::WouldBlock)));
2161    }
2162
2163    #[test]
2164    fn test_poll_and_reclaim() {
2165        let ring = make_ring(8);
2166        let mut producer = make_producer(&ring);
2167
2168        let id = producer.submit_one(0x1000, 512, false).unwrap();
2169
2170        // Manually mark as used (simulate device)
2171        let mut desc = ring.read_desc(0);
2172        desc.mark_used(true);
2173        desc.len = 256;
2174        ring.write_desc(0, desc);
2175
2176        // Poll should return the used buffer
2177        let used = producer.poll_used().unwrap();
2178        assert_eq!(used.id, id);
2179        assert_eq!(used.len, 256);
2180
2181        // State should be updated
2182        assert_eq!(producer.num_free, 8);
2183        assert_eq!(producer.used_cursor.head(), 1);
2184        assert_eq!(producer.id_num[id as usize], 0);
2185        assert!(producer.id_free.contains(&id));
2186    }
2187
2188    #[test]
2189    fn test_poll_multi_descriptor_chain() {
2190        let ring = make_ring(8);
2191        let mut producer = make_producer(&ring);
2192
2193        let chain = BufferChainBuilder::new()
2194            .readable(0x1000, 256)
2195            .readable(0x2000, 256)
2196            .writable(0x3000, 512)
2197            .build()
2198            .unwrap();
2199
2200        let id = producer.submit_available(&chain).unwrap();
2201
2202        // Mark only head as used
2203        let mut head_desc = ring.read_desc(0);
2204        head_desc.mark_used(true);
2205        head_desc.len = 512;
2206        ring.write_desc(0, head_desc);
2207
2208        // Poll should reclaim all 3 descriptors
2209        let used = producer.poll_used().unwrap();
2210        assert_eq!(used.id, id);
2211        assert_eq!(used.len, 512);
2212
2213        // Should have skipped 3 descriptors
2214        assert_eq!(producer.used_cursor.head(), 3);
2215        assert_eq!(producer.num_free, 8);
2216    }
2217
2218    #[test]
2219    fn test_id_reuse() {
2220        let ring = make_ring(4);
2221        let mut producer = make_producer(&ring);
2222
2223        // Submit and complete first buffer
2224        let id1 = producer.submit_one(0x1000, 256, false).unwrap();
2225
2226        let mut desc = ring.read_desc(0);
2227        desc.mark_used(true);
2228        ring.write_desc(0, desc);
2229
2230        producer.poll_used().unwrap();
2231
2232        // Submit another buffer - should reuse ID
2233        let id2 = producer.submit_one(0x2000, 256, false).unwrap();
2234
2235        // ID should be reused (LIFO from stack)
2236        assert_eq!(id2, id1);
2237        assert_eq!(producer.id_num[id2 as usize], 1);
2238    }
2239
2240    #[test]
2241    fn test_available_descriptor_flags() {
2242        let ring = make_ring(4);
2243        let mut producer = make_producer(&ring);
2244
2245        producer.submit_one(0x1000, 256, false).unwrap();
2246
2247        let desc = ring.read_desc(0);
2248
2249        // Available descriptor: AVAIL != USED
2250        let flags = desc.flags();
2251        assert_ne!(
2252            flags.contains(DescFlags::AVAIL),
2253            flags.contains(DescFlags::USED)
2254        );
2255
2256        // ... and AVAIL=true, USED=false for wrap=true
2257        assert!(flags.contains(DescFlags::AVAIL));
2258        assert!(!flags.contains(DescFlags::USED));
2259    }
2260
2261    #[test]
2262    fn test_used_descriptor_flags() {
2263        let ring = make_ring(4);
2264        let mut producer = make_producer(&ring);
2265
2266        producer.submit_one(0x1000, 256, false).unwrap();
2267
2268        let mut desc = ring.read_desc(0);
2269        desc.mark_used(true);
2270        ring.write_desc(0, desc);
2271
2272        let desc = ring.read_desc(0);
2273        let flags = desc.flags();
2274
2275        // Used descriptor: AVAIL == USED
2276        assert_eq!(
2277            flags.contains(DescFlags::AVAIL),
2278            flags.contains(DescFlags::USED)
2279        );
2280    }
2281
2282    #[test]
2283    fn test_poll_empty_ring() {
2284        let ring = make_ring(4);
2285        let mut producer = make_producer(&ring);
2286
2287        // Poll without any submitted buffers
2288        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2289    }
2290
2291    #[test]
2292    fn test_submit_when_full() {
2293        let ring = make_ring(2);
2294        let mut producer = make_producer(&ring);
2295
2296        producer.submit_one(0x1000, 256, false).unwrap();
2297        producer.submit_one(0x2000, 256, false).unwrap();
2298
2299        // Ring is full
2300        assert!(matches!(
2301            producer.submit_one(0x3000, 256, false),
2302            Err(RingError::WouldBlock)
2303        ));
2304    }
2305
2306    #[test]
2307    fn test_wrap_stress() {
2308        let ring = make_ring(4);
2309        let mut producer = make_producer(&ring);
2310        let mut consumer = make_consumer(&ring);
2311
2312        // Do multiple full laps
2313        for lap in 0..3 {
2314            let expected_wrap = lap % 2 == 0;
2315
2316            for _ in 0..4 {
2317                let id = producer.submit_one(0x1000, 256, false).unwrap();
2318
2319                let (dev_id, _) = consumer.poll_available().unwrap();
2320                assert_eq!(dev_id, id);
2321
2322                consumer.submit_used(dev_id, 256).unwrap();
2323
2324                producer.poll_used().unwrap();
2325            }
2326
2327            // After full lap, wrap should toggle
2328            assert_eq!(producer.avail_cursor.wrap(), !expected_wrap);
2329        }
2330        assert_invariants(&ring, &producer);
2331    }
2332
2333    #[test]
2334    fn test_next_flag_termination() {
2335        let ring = make_ring(8);
2336        let mut producer = make_producer(&ring);
2337
2338        let chain = BufferChainBuilder::new()
2339            .readable(0x1000, 256)
2340            .readable(0x2000, 256)
2341            .readable(0x3000, 256)
2342            .build()
2343            .unwrap();
2344
2345        producer.submit_available(&chain).unwrap();
2346
2347        // First two should have NEXT
2348        for i in 0..2 {
2349            let desc = ring.read_desc(i);
2350            assert!(desc.flags().contains(DescFlags::NEXT));
2351        }
2352
2353        // Last should not have NEXT
2354        let tail_desc = ring.read_desc(2);
2355        assert!(!tail_desc.flags().contains(DescFlags::NEXT));
2356    }
2357
2358    #[test]
2359    fn test_consumer_initialization() {
2360        let ring = make_ring(8);
2361        let consumer = make_consumer(&ring);
2362
2363        assert_eq!(consumer.avail_cursor.head(), 0);
2364        assert!(consumer.avail_cursor.wrap());
2365        assert_eq!(consumer.used_cursor.head(), 0);
2366        assert!(consumer.used_cursor.wrap());
2367
2368        for i in 0..8 {
2369            assert_eq!(consumer.id_num[i], 0);
2370        }
2371    }
2372
2373    #[test]
2374    fn test_consumer_poll_available_single() {
2375        let ring = make_ring(8);
2376        let mut producer = make_producer(&ring);
2377        let mut consumer = make_consumer(&ring);
2378
2379        let id = producer.submit_one(0x1000, 512, false).unwrap();
2380
2381        let (polled_id, chain) = consumer.poll_available().unwrap();
2382
2383        assert_eq!(polled_id, id);
2384        assert_eq!(chain.len(), 1);
2385        assert_eq!(chain.elems()[0].addr, 0x1000);
2386        assert_eq!(chain.elems()[0].len, 512);
2387        assert!(!chain.elems()[0].writable);
2388
2389        // Chain length recorded
2390        assert_eq!(consumer.id_num[id as usize], 1);
2391        assert_eq!(consumer.avail_cursor.head(), 1);
2392    }
2393
2394    #[test]
2395    fn test_consumer_poll_available_chain() {
2396        let ring = make_ring(8);
2397        let mut producer = make_producer(&ring);
2398        let mut consumer = make_consumer(&ring);
2399
2400        let chain = BufferChainBuilder::new()
2401            .readable(0x1000, 256)
2402            .readable(0x2000, 256)
2403            .writable(0x3000, 512)
2404            .build()
2405            .unwrap();
2406
2407        let id = producer.submit_available(&chain).unwrap();
2408
2409        let (polled_id, polled_chain) = consumer.poll_available().unwrap();
2410
2411        assert_eq!(polled_id, id);
2412        assert_eq!(polled_chain.len(), 3);
2413
2414        assert_eq!(polled_chain.elems()[0].addr, 0x1000);
2415        assert!(!polled_chain.elems()[0].writable);
2416
2417        assert_eq!(polled_chain.elems()[1].addr, 0x2000);
2418        assert!(!polled_chain.elems()[1].writable);
2419
2420        assert_eq!(polled_chain.elems()[2].addr, 0x3000);
2421        assert!(polled_chain.elems()[2].writable);
2422
2423        assert_eq!(consumer.id_num[id as usize], 3);
2424    }
2425
2426    #[test]
2427    fn test_consumer_rejects_duplicate_inflight_id() {
2428        let ring = make_ring(8);
2429        let mut producer = make_producer(&ring);
2430        let mut consumer = make_consumer(&ring);
2431
2432        let id = producer.submit_one(0x1000, 512, false).unwrap();
2433        let (polled_id, _) = consumer.poll_available().unwrap();
2434        assert_eq!(polled_id, id);
2435
2436        let mut desc = Descriptor::new(0x2000, 256, id, DescFlags::empty());
2437        desc.mark_avail(consumer.avail_cursor.wrap());
2438        ring.write_desc(consumer.avail_cursor.head(), desc);
2439
2440        assert!(matches!(
2441            consumer.poll_available(),
2442            Err(RingError::InvalidState)
2443        ));
2444    }
2445
2446    #[test]
2447    fn test_consumer_submit_used() {
2448        let ring = make_ring(8);
2449        let mut producer = make_producer(&ring);
2450        let mut consumer = make_consumer(&ring);
2451
2452        let id = producer.submit_one(0x1000, 512, true).unwrap();
2453
2454        let (polled_id, _) = consumer.poll_available().unwrap();
2455
2456        // Submit as used
2457        consumer.submit_used(polled_id, 256).unwrap();
2458
2459        // Check descriptor marked used
2460        let desc = ring.read_desc(0);
2461
2462        assert_eq!(desc.id, id);
2463        assert_eq!(desc.len, 256);
2464        assert!(desc.is_used(true));
2465
2466        // Cursor advanced, chain length cleared
2467        assert_eq!(consumer.used_cursor.head(), 1);
2468        assert_eq!(consumer.id_num[id as usize], 0);
2469    }
2470
2471    #[test]
2472    fn test_consumer_submit_used_multi_descriptor() {
2473        let ring = make_ring(8);
2474        let mut producer = make_producer(&ring);
2475        let mut consumer = make_consumer(&ring);
2476
2477        let chain = BufferChainBuilder::new()
2478            .readable(0x1000, 256)
2479            .writable(0x2000, 512)
2480            .writable(0x3000, 512)
2481            .build()
2482            .unwrap();
2483
2484        producer.submit_available(&chain).unwrap();
2485
2486        let (id, _) = consumer.poll_available().unwrap();
2487
2488        consumer.submit_used(id, 1024).unwrap();
2489
2490        // Only head marked used
2491        let head_desc = ring.read_desc(0);
2492        assert!(head_desc.is_used(true));
2493        assert_eq!(head_desc.len, 1024);
2494
2495        // Cursor skipped entire chain
2496        assert_eq!(consumer.used_cursor.head(), 3);
2497        assert_eq!(consumer.id_num[id as usize], 0);
2498    }
2499
2500    #[test]
2501    fn test_consumer_poll_empty() {
2502        let ring = make_ring(4);
2503        let mut consumer = make_consumer(&ring);
2504
2505        assert!(matches!(
2506            consumer.poll_available(),
2507            Err(RingError::WouldBlock)
2508        ));
2509    }
2510
2511    #[test]
2512    fn test_consumer_peek() {
2513        let ring = make_ring(8);
2514        let mut producer = make_producer(&ring);
2515        let consumer = make_consumer(&ring);
2516
2517        producer.submit_one(0x1000, 512, false).unwrap();
2518        assert!(consumer.peek_available().unwrap());
2519
2520        let empty_ring = make_ring(4);
2521        let empty_consumer = make_consumer(&empty_ring);
2522        assert!(!empty_consumer.peek_available().unwrap());
2523    }
2524
2525    #[test]
2526    fn test_full_roundtrip() {
2527        let ring = make_ring(8);
2528        let mut producer = make_producer(&ring);
2529        let mut consumer = make_consumer(&ring);
2530
2531        let chain = BufferChainBuilder::new()
2532            .readable(0x1000, 256)
2533            .writable(0x2000, 512)
2534            .build()
2535            .unwrap();
2536
2537        let id = producer.submit_available(&chain).unwrap();
2538
2539        let (consumer_id, consumer_chain) = consumer.poll_available().unwrap();
2540
2541        assert_eq!(consumer_id, id);
2542        assert_eq!(consumer_chain.len(), 2);
2543
2544        consumer.submit_used(consumer_id, 512).unwrap();
2545
2546        let used = producer.poll_used().unwrap();
2547        assert_eq!(used.id, id);
2548        assert_eq!(used.len, 512);
2549    }
2550
2551    #[test]
2552    fn ring_initial_poll_used_blocks() {
2553        let ring = make_ring(8);
2554        let mut producer = make_producer(&ring);
2555        // No submissions yet: all descriptors zero.
2556        for _ in 0..8 {
2557            assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2558        }
2559        // Invariants: num_free == ring size
2560        assert_eq!(producer.num_free, ring.len());
2561    }
2562
2563    #[test]
2564    fn ring_consumer_blocks_until_submit() {
2565        let ring = make_ring(8);
2566        let mut producer = make_producer(&ring);
2567        let mut consumer = make_consumer(&ring);
2568
2569        assert!(matches!(
2570            consumer.poll_available(),
2571            Err(RingError::WouldBlock)
2572        ));
2573
2574        let chain = BufferChainBuilder::new()
2575            .readable(0x1000, 32)
2576            .readable(0x2000, 16)
2577            .build()
2578            .unwrap();
2579
2580        let id = producer.submit_available(&chain).unwrap();
2581
2582        let (cid, polled) = consumer.poll_available().unwrap();
2583        assert_eq!(cid, id);
2584        assert_eq!(polled.len(), chain.len());
2585    }
2586
2587    #[test]
2588    fn test_out_of_order_completion_stream() {
2589        let ring = make_ring(8);
2590        let mut producer = make_producer(&ring);
2591        let mut consumer = make_consumer(&ring);
2592
2593        // Driver submits two single-descriptor chains A then B
2594        let id_a = producer.submit_one(0x1000, 256, true).unwrap();
2595        let id_b = producer.submit_one(0x2000, 256, true).unwrap();
2596
2597        // Device polls them in ring order (A then B)
2598        let (dev_id_a, chain_a) = consumer.poll_available().unwrap();
2599        assert_eq!(dev_id_a, id_a);
2600        assert_eq!(chain_a.len(), 1);
2601
2602        let (dev_id_b, chain_b) = consumer.poll_available().unwrap();
2603        assert_eq!(dev_id_b, id_b);
2604        assert_eq!(chain_b.len(), 1);
2605
2606        // Device completes B first, then A
2607        consumer.submit_used(dev_id_b, 128).unwrap();
2608        consumer.submit_used(dev_id_a, 256).unwrap();
2609
2610        // Driver polls used stream: should see B (first completion)
2611        let used_b = producer.poll_used().unwrap();
2612        assert_eq!(used_b.id, id_b);
2613        assert_eq!(used_b.len, 128);
2614
2615        // Then sees A
2616        let used_a = producer.poll_used().unwrap();
2617        assert_eq!(used_a.id, id_a);
2618        assert_eq!(used_a.len, 256);
2619
2620        // IDs recycled
2621        assert!(producer.id_free.contains(&id_a));
2622        assert!(producer.id_free.contains(&id_b));
2623    }
2624
2625    #[test]
2626    fn test_mixed_chain_sizes_out_of_order_completion() {
2627        let ring = make_ring(16);
2628        let mut producer = make_producer(&ring);
2629        let mut consumer = make_consumer(&ring);
2630
2631        let chains = vec![
2632            BufferChainBuilder::new()
2633                .readable(0x1000, 10)
2634                .writable(0x2000, 5)
2635                .build()
2636                .unwrap(),
2637            BufferChainBuilder::new()
2638                .readable(0x3000, 8)
2639                .readable(0x3010, 8)
2640                .writable(0x3020, 16)
2641                .build()
2642                .unwrap(),
2643            BufferChainBuilder::new()
2644                .readable(0x4000, 4)
2645                .build()
2646                .unwrap(),
2647            BufferChainBuilder::new()
2648                .readable(0x5000, 4)
2649                .readable(0x5010, 4)
2650                .readable(0x5020, 4)
2651                .writable(0x5030, 4)
2652                .build()
2653                .unwrap(),
2654        ];
2655
2656        for c in &chains {
2657            producer.submit_available(c).unwrap();
2658        }
2659
2660        let mut dev_chain_lens = Vec::new();
2661        for _ in &chains {
2662            let (id, chain) = consumer.poll_available().unwrap();
2663            dev_chain_lens.push((id, chain.len() as u32));
2664        }
2665
2666        let order = [1, 3, 0, 2];
2667        let mut completion = Vec::new();
2668
2669        for &idx in &order {
2670            let (id, len) = dev_chain_lens[idx];
2671            consumer.submit_used(id, len).unwrap();
2672            completion.push((id, len));
2673        }
2674
2675        for (expected_id, expected_len) in &completion {
2676            let used = producer.poll_used().unwrap();
2677            assert_eq!(used.id, *expected_id);
2678            assert_eq!(used.len, *expected_len);
2679            assert_eq!(producer.id_num[*expected_id as usize], 0);
2680            assert!(producer.id_free.contains(expected_id));
2681        }
2682
2683        assert_invariants(&ring, &producer);
2684    }
2685
2686    // Used stream wrap crossing
2687    #[test]
2688    fn test_used_stream_wrap_crossing() {
2689        let ring = make_ring(8);
2690        let mut producer = make_producer(&ring);
2691        let mut consumer = make_consumer(&ring);
2692
2693        // Submit enough single descriptors to make used writes wrap
2694        let mut ids = Vec::new();
2695        for i in 0..8 {
2696            ids.push(producer.submit_one(0x1000 + i as u64, 1, false).unwrap());
2697        }
2698
2699        // Device polls all
2700        for _ in 0..8 {
2701            consumer.poll_available().unwrap();
2702        }
2703
2704        // Complete all in order except we simulate out-of-order by reversing
2705        for &id in ids.iter().rev() {
2706            consumer.submit_used(id, 1).unwrap();
2707        }
2708
2709        // Producer polls used; after consuming size descriptors used_cursor should wrap
2710        for _ in 0..8 {
2711            producer.poll_used().unwrap();
2712        }
2713        assert_eq!(producer.used_cursor.head(), 0);
2714        assert!(!producer.used_cursor.wrap()); // flipped once
2715        assert_invariants(&ring, &producer);
2716    }
2717
2718    // Interleaved availability and completion
2719    #[test]
2720    fn test_interleaved_submit_completion() {
2721        let ring = make_ring(8);
2722        let mut producer = make_producer(&ring);
2723        let mut consumer = make_consumer(&ring);
2724
2725        // Submit chain A (len 2)
2726        let chain_a = BufferChainBuilder::new()
2727            .readable(0x1000, 8)
2728            .writable(0x2000, 8)
2729            .build()
2730            .unwrap();
2731        let id_a = producer.submit_available(&chain_a).unwrap();
2732
2733        // Device polls A
2734        let (dev_id_a, _) = consumer.poll_available().unwrap();
2735        assert_eq!(dev_id_a, id_a);
2736
2737        // Device completes A
2738        consumer.submit_used(dev_id_a, 8).unwrap();
2739
2740        // Submit chain B (len 3) before driver reclaims A
2741        let chain_b = BufferChainBuilder::new()
2742            .readable(0x3000, 4)
2743            .readable(0x3010, 4)
2744            .writable(0x3020, 4)
2745            .build()
2746            .unwrap();
2747        let id_b = producer.submit_available(&chain_b).unwrap();
2748
2749        // Device polls B
2750        let (dev_id_b, _) = consumer.poll_available().unwrap();
2751        assert_eq!(dev_id_b, id_b);
2752
2753        // Driver reclaims A
2754        let used_a = producer.poll_used().unwrap();
2755        assert_eq!(used_a.id, id_a);
2756
2757        // Device completes B
2758        consumer.submit_used(dev_id_b, 12).unwrap();
2759
2760        // Driver reclaims B
2761        let used_b = producer.poll_used().unwrap();
2762        assert_eq!(used_b.id, id_b);
2763
2764        assert_invariants(&ring, &producer);
2765    }
2766
2767    // Partial publish safety (head not published yet)
2768    #[test]
2769    fn test_partial_publish_safety() {
2770        let ring = make_ring(8);
2771        let mut consumer = make_consumer(&ring);
2772        let mut producer = make_producer(&ring);
2773
2774        // Build chain manually: write tails only
2775        let chain = BufferChainBuilder::new()
2776            .readable(0x1000, 4)
2777            .readable(0x2000, 4)
2778            .writable(0x3000, 4)
2779            .build()
2780            .unwrap();
2781
2782        // Simulate manual tail writes without head publish
2783        let id = producer.id_free.pop().unwrap();
2784        producer.id_num[id as usize] = chain.len() as u16;
2785
2786        // Emulate internal position logic
2787        let head_idx = producer.avail_cursor.head();
2788        let wrap_start = producer.avail_cursor.wrap();
2789        let mut pos = producer.avail_cursor;
2790        pos.advance();
2791
2792        for (i, elem) in chain.elems().iter().enumerate().skip(1) {
2793            let is_next = i + 1 < chain.len();
2794            let mut flags = DescFlags::empty();
2795            flags.set(DescFlags::NEXT, is_next);
2796            flags.set(DescFlags::WRITE, elem.writable);
2797            let mut d = Descriptor::new(elem.addr, elem.len, id, flags);
2798            d.mark_avail(pos.wrap());
2799            ring.write_desc(pos.head(), d);
2800            pos.advance();
2801        }
2802
2803        // Head not published yet: consumer must not see chain
2804        assert!(matches!(
2805            consumer.poll_available(),
2806            Err(RingError::WouldBlock)
2807        ));
2808
2809        // Now publish head
2810        let head_elem = chain.elems()[0];
2811        let mut head_flags = DescFlags::empty();
2812        head_flags.set(DescFlags::NEXT, true);
2813        head_flags.set(DescFlags::WRITE, head_elem.writable);
2814        let mut head_desc = Descriptor::new(head_elem.addr, head_elem.len, id, head_flags);
2815        head_desc.mark_avail(wrap_start);
2816        ring.write_desc(head_idx, head_desc);
2817        producer.avail_cursor = pos;
2818        producer.num_free -= chain.len();
2819
2820        // Consumer can now see the chain
2821        let (dev_id, dev_chain) = consumer.poll_available().unwrap();
2822        assert_eq!(dev_id, id);
2823        assert_eq!(dev_chain.len(), chain.len());
2824        assert_invariants(&ring, &producer);
2825    }
2826
2827    // Tail misuse negative test
2828    #[test]
2829    fn test_tail_marked_used_ignored() {
2830        let ring = make_ring(8);
2831        let mut producer = make_producer(&ring);
2832
2833        let chain = BufferChainBuilder::new()
2834            .readable(0x1000, 4)
2835            .readable(0x2000, 4)
2836            .build()
2837            .unwrap();
2838        let id = producer.submit_available(&chain).unwrap();
2839
2840        // Incorrectly mark tail (index 1) used
2841        let mut tail_desc = ring.read_desc(1);
2842        tail_desc.mark_used(producer.used_cursor.wrap());
2843        ring.write_desc(1, tail_desc);
2844
2845        // Poll should return WouldBlock (head not used yet)
2846        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
2847
2848        // Mark head used properly
2849        let mut head_desc = ring.read_desc(0);
2850        head_desc.mark_used(producer.used_cursor.wrap());
2851        ring.write_desc(0, head_desc);
2852
2853        // Now poll succeeds
2854        let used = producer.poll_used().unwrap();
2855        assert_eq!(used.id, id);
2856        assert_invariants(&ring, &producer);
2857    }
2858
2859    // Max chain length boundary
2860    #[test]
2861    fn test_max_chain_len_rejected() {
2862        let ring = make_ring(8);
2863        let mut producer = make_producer(&ring);
2864
2865        // Try chain longer than ring size
2866        let elems = (0..9).map(|i| BufferElement {
2867            addr: 0x1000 + i as u64,
2868            len: 42,
2869            writable: false,
2870        });
2871
2872        let chain = BufferChainBuilder::new().readables(elems).build().unwrap();
2873
2874        // Submit_available should reject when num_free < total_descs
2875        assert!(matches!(
2876            producer.submit_available(&chain),
2877            Err(RingError::WouldBlock)
2878        ));
2879    }
2880
2881    // Descriptor state monotonicity after many cycles
2882    #[test]
2883    fn test_descriptor_state_monotonicity() {
2884        let ring = make_ring(8);
2885        let mut producer = make_producer(&ring);
2886        let mut consumer = make_consumer(&ring);
2887
2888        // Track states: 0=zero/init, 1=available, 2=used, 3=reclaimed
2889        let mut states = vec![0u8; 8];
2890
2891        for _ in 0..5 {
2892            for state in states.iter_mut() {
2893                let id = producer.submit_one(0x1000, 4, false).unwrap();
2894                // mark available
2895                *state = (*state).max(1);
2896
2897                // device polls and completes
2898                let (dev_id, _) = consumer.poll_available().unwrap();
2899                consumer.submit_used(dev_id, 4).unwrap();
2900                *state = (*state).max(2);
2901
2902                let used = producer.poll_used().unwrap();
2903                assert_eq!(used.id, id);
2904                *state = (*state).max(3);
2905            }
2906
2907            assert_invariants(&ring, &producer);
2908        }
2909
2910        // Ensure monotonic progression (never decrease)
2911        for s in states {
2912            assert!(s >= 3);
2913        }
2914    }
2915
2916    // Large multi-lap random submission/completion
2917    #[test]
2918    fn test_random_stress_small() {
2919        use rand::RngExt;
2920        use rand::seq::SliceRandom;
2921
2922        let ring = make_ring(16);
2923        let mut producer = make_producer(&ring);
2924        let mut consumer = make_consumer(&ring);
2925        let mut rng = rand::rng();
2926
2927        // Submit initial set
2928        let mut active_ids = Vec::new();
2929        for _ in 0..8 {
2930            let len = rng.random_range(1..=4);
2931            let mut b = BufferChainBuilder::new().readable(0x1000, 4);
2932            for i in 1..len {
2933                b = b.readable(0x1000 + i as u64 * 0x10, 4);
2934            }
2935            let chain = b.build().unwrap();
2936            if let Ok(id) = producer.submit_available(&chain) {
2937                active_ids.push(id);
2938            }
2939        }
2940
2941        let mut dev_ids = Vec::new();
2942        while let Ok((id, _)) = consumer.poll_available() {
2943            dev_ids.push(id);
2944        }
2945
2946        // Randomly complete
2947        dev_ids.shuffle(&mut rng);
2948        for id in &dev_ids {
2949            let chain_len = consumer.id_num[*id as usize];
2950            consumer.submit_used(*id, chain_len as u32 * 4).unwrap();
2951        }
2952        // Driver reclaim
2953        for _ in &dev_ids {
2954            if producer.poll_used().is_ok() {}
2955        }
2956
2957        assert_invariants(&ring, &producer);
2958    }
2959
2960    // Out-of-order multi-length explicit
2961    #[test]
2962    fn test_out_of_order_multi_length() {
2963        let ring = make_ring(16);
2964        let mut producer = make_producer(&ring);
2965        let mut consumer = make_consumer(&ring);
2966
2967        let chain_a = BufferChainBuilder::new()
2968            .readable(0x1000, 4)
2969            .writable(0x2000, 4)
2970            .build()
2971            .unwrap();
2972        let chain_b = BufferChainBuilder::new()
2973            .readable(0x3000, 4)
2974            .readable(0x3010, 4)
2975            .writable(0x3020, 4)
2976            .build()
2977            .unwrap();
2978        let chain_c = BufferChainBuilder::new()
2979            .readable(0x4000, 4)
2980            .build()
2981            .unwrap();
2982
2983        let id_a = producer.submit_available(&chain_a).unwrap();
2984        let id_b = producer.submit_available(&chain_b).unwrap();
2985        let id_c = producer.submit_available(&chain_c).unwrap();
2986
2987        let (d_a, _) = consumer.poll_available().unwrap();
2988        let (d_b, _) = consumer.poll_available().unwrap();
2989        let (d_c, _) = consumer.poll_available().unwrap();
2990        assert_eq!(d_a, id_a);
2991        assert_eq!(d_b, id_b);
2992        assert_eq!(d_c, id_c);
2993
2994        // Complete B, then C, then A
2995        consumer.submit_used(d_b, 12).unwrap();
2996        consumer.submit_used(d_c, 4).unwrap();
2997        consumer.submit_used(d_a, 8).unwrap();
2998
2999        let u_b = producer.poll_used().unwrap();
3000        assert_eq!(u_b.id, id_b);
3001        let u_c = producer.poll_used().unwrap();
3002        assert_eq!(u_c.id, id_c);
3003        let u_a = producer.poll_used().unwrap();
3004        assert_eq!(u_a.id, id_a);
3005
3006        assert_invariants(&ring, &producer);
3007    }
3008
3009    #[test]
3010    fn interleave_submit_and_completion() {
3011        let ring = make_ring(16);
3012        let mut producer = make_producer(&ring);
3013        let mut consumer = make_consumer(&ring);
3014
3015        // Submit A (len 2)
3016        let chain_a = BufferChainBuilder::new()
3017            .readable(0x1000, 4)
3018            .writable(0x2000, 4)
3019            .build()
3020            .unwrap();
3021        let id_a = producer.submit_available(&chain_a).unwrap();
3022
3023        // Device polls A
3024        let (d_a, _) = consumer.poll_available().unwrap();
3025        assert_eq!(d_a, id_a);
3026
3027        // Immediately complete A
3028        consumer.submit_used(d_a, 8).unwrap();
3029
3030        // Submit B (len 3)
3031        let chain_b = BufferChainBuilder::new()
3032            .readable(0x3000, 4)
3033            .readable(0x3010, 4)
3034            .writable(0x3020, 4)
3035            .build()
3036            .unwrap();
3037        let id_b = producer.submit_available(&chain_b).unwrap();
3038
3039        // Driver polls used: gets A
3040        let u_a = producer.poll_used().unwrap();
3041        assert_eq!(u_a.id, id_a);
3042        assert_eq!(u_a.len, 8);
3043
3044        // Device polls B and submits used for it
3045        let (d_b, _) = consumer.poll_available().unwrap();
3046        assert_eq!(d_b, id_b);
3047        consumer.submit_used(d_b, 12).unwrap();
3048
3049        // Submit C (len 1)
3050        let id_c = producer.submit_one(0x4000, 4, false).unwrap();
3051
3052        // Device polls C and completes it
3053        let (d_c, _) = consumer.poll_available().unwrap();
3054        assert_eq!(d_c, id_c);
3055        consumer.submit_used(d_c, 4).unwrap();
3056
3057        // Driver polls used: gets B then C
3058        let u_b = producer.poll_used().unwrap();
3059        assert_eq!(u_b.id, id_b);
3060        assert_eq!(u_b.len, 12);
3061
3062        let u_c = producer.poll_used().unwrap();
3063        assert_eq!(u_c.id, id_c);
3064        assert_eq!(u_c.len, 4);
3065
3066        assert_invariants(&ring, &producer);
3067    }
3068
3069    // Event suppression tests
3070    #[test]
3071    fn producer_disable_used_notifications_writes_driver_disable() {
3072        let ring = make_ring(8);
3073        let mut producer = make_producer(&ring);
3074
3075        assert_eq!(ring.read_driver_event().flags(), EventFlags::ENABLE);
3076        producer.disable_used_notifications().unwrap();
3077        assert_eq!(ring.read_driver_event().flags(), EventFlags::DISABLE);
3078    }
3079
3080    #[test]
3081    fn producer_enable_used_notifications_writes_driver_enable() {
3082        let ring = make_ring(8);
3083        let mut producer = make_producer(&ring);
3084
3085        producer.disable_used_notifications().unwrap();
3086        assert_eq!(ring.read_driver_event().flags(), EventFlags::DISABLE);
3087
3088        producer.enable_used_notifications().unwrap();
3089        assert_eq!(ring.read_driver_event().flags(), EventFlags::ENABLE);
3090    }
3091
3092    #[test]
3093    fn producer_enable_used_notifications_desc_sets_off_wrap_and_flags() {
3094        let ring = make_ring(8);
3095        let mut producer = make_producer(&ring);
3096
3097        producer.enable_used_notifications_desc(5, true).unwrap();
3098
3099        let evt = ring.read_driver_event();
3100        assert_eq!(evt.flags(), EventFlags::DESC);
3101        assert_eq!(evt.desc_event_off(), 5);
3102        assert!(evt.desc_event_wrap());
3103    }
3104
3105    #[test]
3106    fn producer_enable_used_notifications_for_next_programs_used_cursor() {
3107        let ring = make_ring(8);
3108        let mut producer = make_producer(&ring);
3109
3110        // initial used cursor: head=0, wrap=true
3111        producer.enable_used_notifications_for_next().unwrap();
3112
3113        let evt = ring.read_driver_event();
3114        assert_eq!(evt.flags(), EventFlags::DESC);
3115        assert_eq!(evt.desc_event_off(), 0);
3116        assert!(evt.desc_event_wrap());
3117    }
3118
3119    #[test]
3120    fn consumer_disable_avail_notifications_writes_device_disable() {
3121        let ring = make_ring(8);
3122        let mut consumer = make_consumer(&ring);
3123
3124        assert_eq!(ring.read_device_event().flags(), EventFlags::ENABLE);
3125        consumer.disable_avail_notifications().unwrap();
3126        assert_eq!(ring.read_device_event().flags(), EventFlags::DISABLE);
3127    }
3128
3129    #[test]
3130    fn consumer_enable_avail_notifications_writes_device_enable() {
3131        let ring = make_ring(8);
3132        let mut consumer = make_consumer(&ring);
3133
3134        consumer.disable_avail_notifications().unwrap();
3135        assert_eq!(ring.read_device_event().flags(), EventFlags::DISABLE);
3136
3137        consumer.enable_avail_notifications().unwrap();
3138        assert_eq!(ring.read_device_event().flags(), EventFlags::ENABLE);
3139    }
3140
3141    #[test]
3142    fn consumer_enable_avail_notifications_desc_sets_off_wrap_and_flags() {
3143        let ring = make_ring(8);
3144        let mut consumer = make_consumer(&ring);
3145
3146        consumer.enable_avail_notifications_desc(7, false).unwrap();
3147
3148        let evt = ring.read_device_event();
3149        assert_eq!(evt.flags(), EventFlags::DESC);
3150        assert_eq!(evt.desc_event_off(), 7);
3151        assert!(!evt.desc_event_wrap());
3152    }
3153
3154    #[test]
3155    fn consumer_enable_avail_notifications_for_next_programs_avail_cursor() {
3156        let ring = make_ring(8);
3157        let mut consumer = make_consumer(&ring);
3158
3159        // initial avail cursor: head=0, wrap=true
3160        consumer.enable_avail_notifications_for_next().unwrap();
3161
3162        let evt = ring.read_device_event();
3163        assert_eq!(evt.flags(), EventFlags::DESC);
3164        assert_eq!(evt.desc_event_off(), 0);
3165        assert!(evt.desc_event_wrap());
3166    }
3167
3168    #[test]
3169    fn producer_does_not_write_device_event_when_toggling_used_notifications() {
3170        let ring = make_ring(8);
3171        let mut producer = make_producer(&ring);
3172
3173        let dev_before = ring.read_device_event();
3174        producer.disable_used_notifications().unwrap();
3175        let dev_after = ring.read_device_event();
3176
3177        assert_eq!(dev_after, dev_before);
3178    }
3179
3180    #[test]
3181    fn consumer_does_not_write_driver_event_when_toggling_avail_notifications() {
3182        let ring = make_ring(8);
3183        let mut consumer = make_consumer(&ring);
3184
3185        let drv_before = ring.read_driver_event();
3186        consumer.disable_avail_notifications().unwrap();
3187        let drv_after = ring.read_driver_event();
3188
3189        assert_eq!(drv_after, drv_before);
3190    }
3191
3192    #[test]
3193    fn should_notify_flags_enable_disable() {
3194        let ring_len = 8;
3195
3196        let old = RingCursor {
3197            head: 0,
3198            size: ring_len,
3199            wrap: true,
3200        };
3201        let new = RingCursor {
3202            head: 1,
3203            size: ring_len,
3204            wrap: true,
3205        };
3206
3207        // DISABLE -> never notify
3208        let evt = EventSuppression::new(0, EventFlags::DISABLE);
3209        assert!(!should_notify(evt, ring_len, old, new));
3210
3211        // ENABLE -> always notify
3212        let evt = EventSuppression::new(0, EventFlags::ENABLE);
3213        assert!(should_notify(evt, ring_len, old, new));
3214    }
3215
3216    #[test]
3217    fn should_notify_desc_no_crossing() {
3218        let ring_len = 8;
3219
3220        let old = RingCursor {
3221            head: 2,
3222            size: ring_len,
3223            wrap: true,
3224        };
3225        let new = RingCursor {
3226            head: 3,
3227            size: ring_len,
3228            wrap: true,
3229        };
3230
3231        // event at 6, we did not cross it
3232        let mut evt = EventSuppression::zeroed();
3233        evt.set_desc_event(6, true);
3234        evt.set_flags(EventFlags::DESC);
3235
3236        assert!(!should_notify(evt, ring_len, old, new));
3237    }
3238
3239    #[test]
3240    fn should_notify_desc_wrap_mismatch_adjusts_event_idx() {
3241        let ring_len = 8;
3242
3243        let old = RingCursor {
3244            head: 7,
3245            size: ring_len,
3246            wrap: true,
3247        };
3248        let new = RingCursor {
3249            head: 1,
3250            size: ring_len,
3251            wrap: false,
3252        };
3253
3254        let mut evt = EventSuppression::zeroed();
3255        evt.set_desc_event(7, true);
3256        evt.set_flags(EventFlags::DESC);
3257
3258        assert!(should_notify(evt, ring_len, old, new));
3259    }
3260
3261    #[test]
3262    fn ring_need_event_basic_cases() {
3263        // If event_idx == new-1, should be true
3264        assert!(ring_need_event(4, 5, 2));
3265        // If no progress, should be false
3266        assert!(!ring_need_event(4, 5, 5));
3267
3268        // Wrapping arithmetic sanity: old near u16::MAX
3269        let old = 0xFFFE;
3270        let new = 1;
3271        // event at 0xFFFF is considered "just before wrap"
3272        assert!(ring_need_event(0xFFFF, new, old));
3273    }
3274
3275    // Bad device/driver tests
3276    #[test]
3277    fn bad_device_marks_tail_used() {
3278        let ring = make_ring(8);
3279        let mut producer = make_producer(&ring);
3280
3281        let chain = BufferChainBuilder::new()
3282            .readable(0x1000, 4)
3283            .readable(0x2000, 4)
3284            .build()
3285            .unwrap();
3286        let id = producer.submit_available(&chain).unwrap();
3287
3288        // Bad device: mark index 1 (tail) used
3289        let mut tail = ring.read_desc(1);
3290        tail.mark_used(producer.used_cursor.wrap());
3291        ring.write_desc(1, tail);
3292
3293        // Driver must not consume it
3294        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3295
3296        // Now mark head properly, driver must consume
3297        let mut head = ring.read_desc(0);
3298        head.mark_used(producer.used_cursor.wrap());
3299        ring.write_desc(0, head);
3300
3301        let used = producer.poll_used().unwrap();
3302        assert_eq!(used.id, id);
3303    }
3304
3305    #[test]
3306    fn bad_device_wrong_used_bits() {
3307        let ring = make_ring(4);
3308        let mut producer = make_producer(&ring);
3309
3310        let id = producer.submit_one(0x1000, 8, true).unwrap();
3311
3312        // Malformed: set AVAIL but clear USED (should be equal for used)
3313        let mut d = ring.read_desc(0);
3314        // Force flags to look like "available" despite intent
3315        d.mark_avail(producer.used_cursor.wrap());
3316        d.len = 8;
3317        ring.write_desc(0, d);
3318
3319        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3320
3321        let mut d2 = ring.read_desc(0);
3322        d2.mark_used(producer.used_cursor.wrap());
3323        ring.write_desc(0, d2);
3324
3325        let u = producer.poll_used().unwrap();
3326        assert_eq!(u.id, id);
3327    }
3328
3329    #[test]
3330    fn bad_driver_next_never_clears() {
3331        let ring = make_ring(8);
3332        let mut consumer = make_consumer(&ring);
3333        let mut producer = make_producer(&ring);
3334
3335        // Allocate an ID and pretend one huge chain
3336        let id = producer.id_free.pop().unwrap();
3337        producer.id_num[id as usize] = 8;
3338
3339        let mut pos = producer.avail_cursor;
3340        let wrap_start = pos.wrap();
3341
3342        // Write every descriptor with NEXT set and same id
3343        for _ in 0..8 {
3344            let idx = pos.head();
3345            let mut flags = DescFlags::empty();
3346            flags.set(DescFlags::NEXT, true); // incorrect: last should NOT have NEXT
3347            let mut desc = Descriptor::new(0x1000 + idx as u64 * 0x10, 4, id, flags);
3348            desc.mark_avail(pos.wrap());
3349            ring.write_desc(idx, desc);
3350            pos.advance();
3351        }
3352
3353        // Publish head last (simulate driver behavior)
3354        let head_idx = producer.avail_cursor.head();
3355        let mut head_flags = DescFlags::empty();
3356        head_flags.set(DescFlags::NEXT, true);
3357        let mut head_desc = Descriptor::new(0x42, 4, id, head_flags);
3358        head_desc.mark_avail(wrap_start);
3359        ring.write_desc(head_idx, head_desc);
3360
3361        // Consumer should detect invalid chain via step guard
3362        assert!(matches!(
3363            consumer.poll_available(),
3364            Err(RingError::BadChain)
3365        ));
3366    }
3367
3368    #[test]
3369    fn bad_driver_interleaved_readables_and_writables() {
3370        let ring = make_ring(8);
3371        let mut consumer = make_consumer(&ring);
3372        let mut producer = make_producer(&ring);
3373
3374        let chain = BufferChainBuilder::new()
3375            .readable(0x1000, 4)
3376            .readable(0x2000, 4)
3377            .writable(0x2000, 4)
3378            .build()
3379            .unwrap();
3380
3381        let _id = producer.submit_available(&chain).unwrap();
3382
3383        // now change first descriptor to writable (bad driver)
3384        let mut first = ring.read_desc(0);
3385        first.flags |= DescFlags::WRITE.bits();
3386        ring.write_desc(0, first);
3387
3388        assert!(matches!(
3389            consumer.poll_available(),
3390            Err(RingError::BadChain)
3391        ));
3392    }
3393
3394    #[test]
3395    fn bad_device_marks_multiple_used_in_chain() {
3396        let ring = make_ring(8);
3397        let mut producer = make_producer(&ring);
3398
3399        let chain = BufferChainBuilder::new()
3400            .readable(0x1000, 4)
3401            .readable(0x2000, 4)
3402            .build()
3403            .unwrap();
3404        let id = producer.submit_available(&chain).unwrap();
3405
3406        // Bad device: mark head and tail used
3407        let mut head = ring.read_desc(0);
3408        head.mark_used(producer.used_cursor.wrap());
3409        ring.write_desc(0, head);
3410
3411        let mut tail = ring.read_desc(1);
3412        tail.mark_used(producer.used_cursor.wrap());
3413        ring.write_desc(1, tail);
3414
3415        // Driver consumes once
3416        let u = producer.poll_used().unwrap();
3417        assert_eq!(u.id, id);
3418
3419        // Next poll should block; no duplicate consumption
3420        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3421    }
3422
3423    #[test]
3424    fn bad_device_writes_used_at_wrong_slot() {
3425        let ring = make_ring(8);
3426        let mut producer = make_producer(&ring);
3427
3428        let _id = producer.submit_one(0x1000, 4, true).unwrap();
3429
3430        // Wrong slot: mark index 3 used while next_used is 0
3431        let mut d = ring.read_desc(3);
3432        d.mark_used(producer.used_cursor.wrap());
3433        ring.write_desc(3, d);
3434
3435        // Driver should still block (polls only slot 0)
3436        assert!(matches!(producer.poll_used(), Err(RingError::WouldBlock)));
3437
3438        // Now mark slot 0 correctly, driver can consume
3439        let mut d0 = ring.read_desc(0);
3440        d0.mark_used(producer.used_cursor.wrap());
3441        ring.write_desc(0, d0);
3442        let _u = producer.poll_used().unwrap();
3443    }
3444
3445    #[test]
3446    fn bad_driver_reuses_id_while_outstanding() {
3447        let ring = make_ring(8);
3448        let mut producer = make_producer(&ring);
3449
3450        // Submit first buffer: allocate ID
3451        let id = producer.submit_one(0x1000, 4, false).unwrap();
3452        assert_eq!(producer.id_num[id as usize], 1);
3453
3454        // push the same ID back into free list while it's still outstanding.
3455        producer.id_free.push(id);
3456
3457        // Next submit should fail because ID is still outstanding.
3458        let res = producer.submit_one(0x2000, 4, false);
3459        assert!(matches!(res, Err(RingError::InvalidState)));
3460    }
3461
3462    #[test]
3463    fn test_avail_cursor_accessor() {
3464        let ring = make_ring(8);
3465        let mut producer = make_producer(&ring);
3466
3467        // Initial cursor
3468        let cursor = producer.avail_cursor();
3469        assert_eq!(cursor.head(), 0);
3470        assert!(cursor.wrap());
3471
3472        // After submit
3473        producer.submit_one(0x1000, 512, false).unwrap();
3474        let cursor = producer.avail_cursor();
3475        assert_eq!(cursor.head(), 1);
3476        assert!(cursor.wrap());
3477    }
3478
3479    #[test]
3480    fn test_should_notify_since() {
3481        let ring = make_ring(8);
3482        let mut producer = make_producer(&ring);
3483
3484        let before = producer.avail_cursor();
3485        producer.submit_one(0x1000, 512, false).unwrap();
3486
3487        // Default is ENABLE mode, so should notify
3488        let should_notify = producer.should_notify_since(before).unwrap();
3489        assert!(should_notify);
3490    }
3491
3492    #[test]
3493    fn test_batch_notification_single_check() {
3494        let ring = make_ring(8);
3495        let mut producer = make_producer(&ring);
3496        let mut consumer = make_consumer(&ring);
3497
3498        let before = producer.avail_cursor();
3499
3500        // Submit multiple descriptors
3501        producer.submit_one(0x1000, 512, false).unwrap();
3502        producer.submit_one(0x2000, 512, false).unwrap();
3503        producer.submit_one(0x3000, 512, false).unwrap();
3504
3505        // Single notification check for the entire batch
3506        let should_notify = producer.should_notify_since(before).unwrap();
3507        assert!(should_notify);
3508
3509        // Consumer sees all 3 descriptors
3510        for _ in 0..3 {
3511            let (_, _) = consumer.poll_available().unwrap();
3512        }
3513    }
3514
3515    #[test]
3516    fn test_ring_cursor_reset() {
3517        let mut cursor = RingCursor::new(16);
3518        cursor.advance_by(5);
3519        assert_eq!(cursor.head(), 5);
3520
3521        cursor.reset();
3522        assert_eq!(cursor, RingCursor::new(16));
3523        assert_eq!(cursor.head(), 0);
3524        assert!(cursor.wrap());
3525    }
3526
3527    #[test]
3528    fn test_ring_cursor_reset_after_wrap() {
3529        let mut cursor = RingCursor::new(4);
3530        // Advance past the wrap point
3531        cursor.advance_by(5);
3532        assert_eq!(cursor.head(), 1);
3533        assert!(!cursor.wrap());
3534
3535        cursor.reset();
3536        assert_eq!(cursor.head(), 0);
3537        assert!(cursor.wrap());
3538    }
3539
3540    #[test]
3541    fn test_ring_producer_reset_matches_new() {
3542        let ring = make_ring(8);
3543        let fresh = make_producer(&ring);
3544
3545        let mut used = make_producer(&ring);
3546        // Mutate state
3547        used.submit_one(0x1000, 64, false).unwrap();
3548        used.submit_one(0x2000, 128, true).unwrap();
3549
3550        used.reset().unwrap();
3551
3552        assert_eq!(used.avail_cursor, fresh.avail_cursor);
3553        assert_eq!(used.used_cursor, fresh.used_cursor);
3554        assert_eq!(used.num_free, fresh.num_free);
3555        assert_eq!(used.id_free.len(), fresh.id_free.len());
3556        assert_eq!(used.id_num.as_slice(), fresh.id_num.as_slice());
3557        assert_eq!(used.event_flags_shadow, fresh.event_flags_shadow);
3558    }
3559
3560    #[test]
3561    fn test_ring_producer_reset_id_free_complete() {
3562        let ring = make_ring(8);
3563        let mut producer = make_producer(&ring);
3564
3565        // Submit and consume several descriptors
3566        for i in 0..4u64 {
3567            producer.submit_one(0x1000 + i * 0x100, 64, false).unwrap();
3568        }
3569        assert_eq!(producer.num_free, 4);
3570
3571        producer.reset().unwrap();
3572
3573        assert_eq!(producer.num_free, 8);
3574        assert_eq!(producer.id_free.len(), 8);
3575        // All IDs 0..8 should be present
3576        for id in 0..8u16 {
3577            assert!(producer.id_free.contains(&id));
3578        }
3579    }
3580
3581    #[test]
3582    fn test_ring_producer_failed_reset_preserves_local_state() {
3583        let ring = make_ring(4);
3584        let mem = FaultMem::new(ring.mem());
3585        let mut producer = RingProducer::new(ring.layout(), mem.clone());
3586
3587        producer.submit_one(0x1000, 64, false).unwrap();
3588        producer.submit_one(0x2000, 128, true).unwrap();
3589
3590        let avail_cursor = producer.avail_cursor;
3591        let used_cursor = producer.used_cursor;
3592        let num_free = producer.num_free;
3593        let id_free = producer.id_free.clone();
3594        let id_num = producer.id_num.clone();
3595        let event_flags_shadow = producer.event_flags_shadow;
3596
3597        mem.fail_write_at(1);
3598        assert!(matches!(
3599            producer.reset(),
3600            Err(RingError::MemError {
3601                op: MemOp::WriteDesc,
3602                ..
3603            })
3604        ));
3605
3606        assert_eq!(producer.avail_cursor, avail_cursor);
3607        assert_eq!(producer.used_cursor, used_cursor);
3608        assert_eq!(producer.num_free, num_free);
3609        assert_eq!(producer.id_free, id_free);
3610        assert_eq!(producer.id_num, id_num);
3611        assert_eq!(producer.event_flags_shadow, event_flags_shadow);
3612
3613        mem.allow_writes();
3614        producer.reset().unwrap();
3615
3616        let fresh = RingProducer::new(ring.layout(), mem);
3617        assert_eq!(producer.avail_cursor, fresh.avail_cursor);
3618        assert_eq!(producer.used_cursor, fresh.used_cursor);
3619        assert_eq!(producer.num_free, fresh.num_free);
3620        assert_eq!(producer.id_free, fresh.id_free);
3621        assert_eq!(producer.id_num, fresh.id_num);
3622        assert_eq!(producer.event_flags_shadow, fresh.event_flags_shadow);
3623    }
3624
3625    #[test]
3626    fn test_ring_consumer_reset_matches_new() {
3627        let ring = make_ring(8);
3628        let fresh = make_consumer(&ring);
3629
3630        let mut used = make_consumer(&ring);
3631
3632        // Submit from producer side so consumer has something to poll
3633        let mut producer = make_producer(&ring);
3634        producer.submit_one(0x1000, 64, false).unwrap();
3635
3636        // Consumer polls the available descriptor
3637        let (id, _chain) = used.poll_available().unwrap();
3638        used.submit_used(id, 64).unwrap();
3639
3640        used.reset().unwrap();
3641
3642        assert_eq!(used.avail_cursor, fresh.avail_cursor);
3643        assert_eq!(used.used_cursor, fresh.used_cursor);
3644        assert_eq!(used.id_num.as_slice(), fresh.id_num.as_slice());
3645        assert_eq!(used.num_inflight, fresh.num_inflight);
3646        assert_eq!(used.event_flags_shadow, fresh.event_flags_shadow);
3647    }
3648
3649    #[test]
3650    fn test_ring_consumer_reset_clears_inflight() {
3651        let ring = make_ring(8);
3652        let mut producer = make_producer(&ring);
3653        let mut consumer = make_consumer(&ring);
3654
3655        // Submit and poll two items (consume but do not complete)
3656        producer.submit_one(0x1000, 64, false).unwrap();
3657        producer.submit_one(0x2000, 64, false).unwrap();
3658        let _ = consumer.poll_available().unwrap();
3659        let _ = consumer.poll_available().unwrap();
3660        assert_eq!(consumer.num_inflight, 2);
3661
3662        consumer.reset().unwrap();
3663        assert_eq!(consumer.num_inflight, 0);
3664    }
3665
3666    #[test]
3667    fn test_desc_table_read_after_submit() {
3668        let ring = make_ring(8);
3669        let mut writer = make_producer(&ring);
3670        writer.submit_one(0x1000, 4096, true).unwrap();
3671
3672        let reader = make_producer(&ring);
3673        let addr = reader.desc_table().desc_addr(0).unwrap();
3674        let flags = Descriptor::read_flags_acquire(reader.mem(), addr).unwrap();
3675        let desc = Descriptor::read_body(reader.mem(), addr, flags).unwrap();
3676        assert_eq!(desc.addr, 0x1000);
3677        assert_eq!(desc.len, 4096);
3678        assert!(desc.is_writable());
3679        assert!(desc.is_avail(true));
3680        assert!(!desc.is_used(true));
3681    }
3682
3683    #[test]
3684    fn test_desc_table_out_of_bounds() {
3685        let ring = make_ring(8);
3686        let reader = make_producer(&ring);
3687        assert!(reader.desc_table().desc_addr(8).is_none());
3688    }
3689
3690    #[test]
3691    fn test_desc_table_read_used_descriptor() {
3692        let ring = make_ring(8);
3693        let mut writer = make_producer(&ring);
3694        writer.submit_one(0x1000, 4096, true).unwrap();
3695
3696        let mut consumer = make_consumer(&ring);
3697        let (id, _chain) = consumer.poll_available().unwrap();
3698        consumer.submit_used(id, 128).unwrap();
3699
3700        let reader = make_producer(&ring);
3701        let addr = reader.desc_table().desc_addr(0).unwrap();
3702        let flags = Descriptor::read_flags_acquire(reader.mem(), addr).unwrap();
3703        let desc = Descriptor::read_body(reader.mem(), addr, flags).unwrap();
3704        assert!(desc.is_used(true));
3705        assert!(!desc.is_avail(true));
3706    }
3707}
3708
3709// Adopted from https://github.com/weltling/virtio-villain/tree/main/tests/packed
3710#[cfg(test)]
3711mod virtio_villain {
3712    use super::tests::{OwnedRing, make_consumer, make_producer, make_ring};
3713    use super::*;
3714
3715    fn raw_desc(id: u16, len: u32, flags: DescFlags) -> Descriptor {
3716        Descriptor::new(0x1000 + u64::from(id) * 0x10, len, id, flags)
3717    }
3718
3719    fn avail_desc(id: u16, len: u32, flags: DescFlags) -> Descriptor {
3720        let mut desc = raw_desc(id, len, flags);
3721        desc.mark_avail(true);
3722        desc
3723    }
3724
3725    fn write_avail(ring: &OwnedRing, idx: u16, id: u16, flags: DescFlags) {
3726        write_avail_len(ring, idx, id, 16, flags);
3727    }
3728
3729    fn write_avail_len(ring: &OwnedRing, idx: u16, id: u16, len: u32, flags: DescFlags) {
3730        ring.write_desc(idx, avail_desc(id, len, flags));
3731    }
3732
3733    fn write_driver_event(ring: &OwnedRing, off_wrap: u16, flags: u16) {
3734        write_event(ring, ring.layout().drv_evt_addr(), off_wrap, flags);
3735    }
3736
3737    fn write_device_event(ring: &OwnedRing, off_wrap: u16, flags: u16) {
3738        write_event(ring, ring.layout().dev_evt_addr(), off_wrap, flags);
3739    }
3740
3741    fn write_event(ring: &OwnedRing, addr: u64, off_wrap: u16, flags: u16) {
3742        ring.mem()
3743            .write(
3744                addr,
3745                &[
3746                    (off_wrap & 0xff) as u8,
3747                    (off_wrap >> 8) as u8,
3748                    (flags & 0xff) as u8,
3749                    (flags >> 8) as u8,
3750                ],
3751            )
3752            .unwrap();
3753    }
3754
3755    #[test]
3756    fn p0003_chain_with_next_past_ring_is_rejected() {
3757        let ring = make_ring(4);
3758        let mut consumer = make_consumer(&ring);
3759
3760        for idx in 0..ring.len() as u16 {
3761            write_avail(&ring, idx, 0, DescFlags::NEXT);
3762        }
3763
3764        assert!(matches!(
3765            consumer.poll_available(),
3766            Err(RingError::BadChain)
3767        ));
3768    }
3769
3770    #[test]
3771    fn p0005_indirect_without_feature_is_rejected() {
3772        let ring = make_ring(8);
3773        let mut consumer = make_consumer(&ring);
3774
3775        write_avail(&ring, 0, 0, DescFlags::INDIRECT);
3776
3777        assert!(matches!(
3778            consumer.poll_available(),
3779            Err(RingError::BadChain)
3780        ));
3781    }
3782
3783    #[test]
3784    fn p0006_indirect_with_bad_flags_is_rejected() {
3785        let ring = make_ring(8);
3786        let mut consumer = make_consumer(&ring);
3787
3788        write_avail(&ring, 0, 0, DescFlags::INDIRECT | DescFlags::WRITE);
3789
3790        assert!(matches!(
3791            consumer.poll_available(),
3792            Err(RingError::BadChain)
3793        ));
3794    }
3795
3796    #[test]
3797    fn p0007_indirect_tail_in_mixed_chain_is_rejected() {
3798        let ring = make_ring(8);
3799        let mut consumer = make_consumer(&ring);
3800
3801        write_avail(&ring, 0, 0, DescFlags::NEXT);
3802        write_avail(&ring, 1, 0, DescFlags::INDIRECT | DescFlags::WRITE);
3803
3804        assert!(matches!(
3805            consumer.poll_available(),
3806            Err(RingError::BadChain)
3807        ));
3808    }
3809
3810    #[test]
3811    fn p0008_long_chain_is_consumable_at_ring_layer() {
3812        let ring = make_ring(16);
3813        let mut consumer = make_consumer(&ring);
3814
3815        for idx in 0..12u16 {
3816            let flags = if idx == 11 {
3817                DescFlags::WRITE
3818            } else {
3819                DescFlags::NEXT | DescFlags::WRITE
3820            };
3821            write_avail(&ring, idx, 0, flags);
3822        }
3823
3824        let (id, chain) = consumer.poll_available().unwrap();
3825        assert_eq!(id, 0);
3826        assert_eq!(chain.len(), 12);
3827    }
3828
3829    #[test]
3830    fn p0009_unaligned_descriptor_address_is_preserved() {
3831        let ring = make_ring(8);
3832        let mut consumer = make_consumer(&ring);
3833
3834        let mut desc = Descriptor::new(0x1003, 16, 0, DescFlags::empty());
3835        desc.mark_avail(true);
3836        ring.write_desc(0, desc);
3837
3838        let (_, chain) = consumer.poll_available().unwrap();
3839        assert_eq!(chain.elems()[0].addr, 0x1003);
3840    }
3841
3842    #[test]
3843    fn p0010_writable_before_readable_is_rejected() {
3844        let ring = make_ring(8);
3845        let mut consumer = make_consumer(&ring);
3846
3847        write_avail(&ring, 0, 0, DescFlags::NEXT);
3848        write_avail(&ring, 1, 0, DescFlags::NEXT | DescFlags::WRITE);
3849        write_avail(&ring, 2, 0, DescFlags::NEXT);
3850        write_avail(&ring, 3, 0, DescFlags::WRITE);
3851
3852        assert!(matches!(
3853            consumer.poll_available(),
3854            Err(RingError::BadChain)
3855        ));
3856    }
3857
3858    #[test]
3859    fn p0011_stale_wrap_descriptor_is_ignored() {
3860        let ring = make_ring(8);
3861        let mut consumer = make_consumer(&ring);
3862
3863        let mut desc = raw_desc(0, 16, DescFlags::empty());
3864        desc.mark_avail(false);
3865        ring.write_desc(0, desc);
3866
3867        assert!(matches!(
3868            consumer.poll_available(),
3869            Err(RingError::WouldBlock)
3870        ));
3871    }
3872
3873    #[test]
3874    fn p0012_multidescriptor_oob_id_is_rejected() {
3875        let ring = make_ring(8);
3876        let mut consumer = make_consumer(&ring);
3877        let bad_id = ring.len() as u16;
3878
3879        write_avail(&ring, 0, bad_id, DescFlags::NEXT);
3880        write_avail(&ring, 1, bad_id, DescFlags::WRITE);
3881
3882        assert!(matches!(
3883            consumer.poll_available(),
3884            Err(RingError::InvalidState)
3885        ));
3886    }
3887
3888    #[test]
3889    fn p0013_duplicate_inflight_buffer_id_is_rejected() {
3890        let ring = make_ring(8);
3891        let mut consumer = make_consumer(&ring);
3892
3893        write_avail(&ring, 0, 0, DescFlags::NEXT);
3894        write_avail(&ring, 1, 0, DescFlags::WRITE);
3895        write_avail(&ring, 2, 0, DescFlags::NEXT);
3896        write_avail(&ring, 3, 0, DescFlags::WRITE);
3897
3898        let (first_id, first_chain) = consumer.poll_available().unwrap();
3899        assert_eq!(first_id, 0);
3900        assert_eq!(first_chain.len(), 2);
3901
3902        assert!(matches!(
3903            consumer.poll_available(),
3904            Err(RingError::InvalidState)
3905        ));
3906    }
3907
3908    #[test]
3909    fn p0014_reserved_driver_event_flags_do_not_block_processing() {
3910        let ring = make_ring(8);
3911        let mut consumer = make_consumer(&ring);
3912
3913        write_driver_event(&ring, 0, 3);
3914        write_avail(&ring, 0, 0, DescFlags::empty());
3915
3916        let (id, chain) = consumer.poll_available().unwrap();
3917        assert_eq!(id, 0);
3918        assert_eq!(chain.len(), 1);
3919    }
3920
3921    #[test]
3922    fn p0015_wrap_at_zero_is_consumed_correctly() {
3923        let ring = make_ring(2);
3924        let mut producer = make_producer(&ring);
3925        let mut consumer = make_consumer(&ring);
3926
3927        producer.submit_one(0x1000, 16, false).unwrap();
3928        producer.submit_one(0x2000, 16, false).unwrap();
3929
3930        assert_eq!(producer.avail_cursor().head(), 0);
3931        assert!(!producer.avail_cursor().wrap());
3932
3933        assert_eq!(consumer.poll_available().unwrap().1.elems()[0].addr, 0x1000);
3934        assert_eq!(consumer.poll_available().unwrap().1.elems()[0].addr, 0x2000);
3935        assert_eq!(consumer.avail_cursor().head(), 0);
3936        assert!(!consumer.avail_cursor().wrap());
3937    }
3938
3939    #[test]
3940    fn p0016_queue_size_one_next_chain_is_rejected() {
3941        let ring = make_ring(1);
3942        let mut consumer = make_consumer(&ring);
3943
3944        write_avail(&ring, 0, 0, DescFlags::NEXT);
3945
3946        assert!(matches!(
3947            consumer.poll_available(),
3948            Err(RingError::BadChain)
3949        ));
3950    }
3951
3952    #[test]
3953    fn p0017_stale_driver_event_wrap_suppresses_notification() {
3954        let ring = make_ring(8);
3955        let mut consumer = make_consumer(&ring);
3956
3957        write_driver_event(&ring, 0, EventFlags::DESC.bits());
3958        write_avail(&ring, 0, 0, DescFlags::empty());
3959
3960        let (id, _) = consumer.poll_available().unwrap();
3961
3962        assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
3963    }
3964
3965    #[test]
3966    fn p0018_pre_used_descriptor_is_ignored() {
3967        let ring = make_ring(8);
3968        let mut consumer = make_consumer(&ring);
3969
3970        let mut desc = raw_desc(0, 16, DescFlags::empty());
3971        desc.mark_used(true);
3972        ring.write_desc(0, desc);
3973
3974        assert!(matches!(
3975            consumer.poll_available(),
3976            Err(RingError::WouldBlock)
3977        ));
3978    }
3979
3980    #[test]
3981    fn p0019_driver_event_wrap_mismatch_suppresses_notification() {
3982        let ring = make_ring(8);
3983        let mut consumer = make_consumer(&ring);
3984
3985        write_driver_event(&ring, 1, EventFlags::DESC.bits());
3986        write_avail(&ring, 0, 0, DescFlags::empty());
3987
3988        let (id, _) = consumer.poll_available().unwrap();
3989
3990        assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
3991    }
3992
3993    #[test]
3994    fn p0020_ring_full_reports_backpressure() {
3995        let ring = make_ring(4);
3996        let mut producer = make_producer(&ring);
3997
3998        for idx in 0..4 {
3999            producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4000        }
4001
4002        assert!(matches!(
4003            producer.submit_one(0x2000, 16, false),
4004            Err(RingError::WouldBlock)
4005        ));
4006    }
4007
4008    #[test]
4009    fn p0022_indirect_reused_id_is_rejected_before_id_reuse() {
4010        let ring = make_ring(8);
4011        let mut consumer = make_consumer(&ring);
4012
4013        write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4014        write_avail(&ring, 1, 0, DescFlags::empty());
4015
4016        assert!(matches!(
4017            consumer.poll_available(),
4018            Err(RingError::BadChain)
4019        ));
4020    }
4021
4022    #[test]
4023    fn p0023_desc_event_mode_suppresses_before_later_slot() {
4024        let ring = make_ring(8);
4025        let mut consumer = make_consumer(&ring);
4026
4027        write_driver_event(&ring, 0x8001, EventFlags::DESC.bits());
4028        write_avail(&ring, 0, 0, DescFlags::empty());
4029
4030        let (id, _) = consumer.poll_available().unwrap();
4031
4032        assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4033    }
4034
4035    #[test]
4036    fn p0024_single_slot_indirect_is_rejected() {
4037        let ring = make_ring(1);
4038        let mut consumer = make_consumer(&ring);
4039
4040        write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4041
4042        assert!(matches!(
4043            consumer.poll_available(),
4044            Err(RingError::BadChain)
4045        ));
4046    }
4047
4048    #[test]
4049    fn p0025_used_completions_can_arrive_out_of_order() {
4050        let ring = make_ring(8);
4051        let mut producer = make_producer(&ring);
4052        let mut consumer = make_consumer(&ring);
4053
4054        let id0 = producer.submit_one(0x1000, 16, true).unwrap();
4055        let id1 = producer.submit_one(0x2000, 16, true).unwrap();
4056        let id2 = producer.submit_one(0x3000, 16, true).unwrap();
4057
4058        let (dev0, _) = consumer.poll_available().unwrap();
4059        let (dev1, _) = consumer.poll_available().unwrap();
4060        let (dev2, _) = consumer.poll_available().unwrap();
4061
4062        consumer.submit_used(dev2, 32).unwrap();
4063        consumer.submit_used(dev0, 16).unwrap();
4064        consumer.submit_used(dev1, 24).unwrap();
4065
4066        assert_eq!(producer.poll_used().unwrap().id, id2);
4067        assert_eq!(producer.poll_used().unwrap().id, id0);
4068        assert_eq!(producer.poll_used().unwrap().id, id1);
4069    }
4070
4071    #[test]
4072    fn p0026_all_descriptor_flags_set_is_not_available() {
4073        let ring = make_ring(8);
4074        let mut consumer = make_consumer(&ring);
4075
4076        ring.write_desc(0, raw_desc(0, 16, DescFlags::from_bits_truncate(u16::MAX)));
4077
4078        assert!(matches!(
4079            consumer.poll_available(),
4080            Err(RingError::WouldBlock)
4081        ));
4082    }
4083
4084    #[test]
4085    fn p0028_multi_wrap_round_trip_remains_ordered() {
4086        let ring = make_ring(2);
4087        let mut producer = make_producer(&ring);
4088        let mut consumer = make_consumer(&ring);
4089
4090        for idx in 0..8u64 {
4091            let id = producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4092            let (dev_id, _) = consumer.poll_available().unwrap();
4093            assert_eq!(dev_id, id);
4094            consumer.submit_used(dev_id, idx as u32).unwrap();
4095            assert_eq!(producer.poll_used().unwrap().id, id);
4096        }
4097    }
4098
4099    #[test]
4100    fn p0029_desc_event_idx_zero_notifies() {
4101        let ring = make_ring(8);
4102        let mut consumer = make_consumer(&ring);
4103
4104        write_driver_event(&ring, 0x8000, EventFlags::DESC.bits());
4105        write_avail(&ring, 0, 0, DescFlags::empty());
4106
4107        let (id, _) = consumer.poll_available().unwrap();
4108
4109        assert!(consumer.submit_used_with_notify(id, 0).unwrap());
4110    }
4111
4112    #[test]
4113    fn p0031_reserved_driver_event_flags_suppress_notification() {
4114        let ring = make_ring(8);
4115        let mut consumer = make_consumer(&ring);
4116
4117        write_driver_event(&ring, 0, 3);
4118        write_avail(&ring, 0, 0, DescFlags::empty());
4119
4120        let (id, _) = consumer.poll_available().unwrap();
4121
4122        assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4123    }
4124
4125    #[test]
4126    fn p0032_unaligned_indirect_descriptor_is_rejected() {
4127        let ring = make_ring(8);
4128        let mut consumer = make_consumer(&ring);
4129
4130        let mut desc = Descriptor::new(0x1003, 16, 0, DescFlags::INDIRECT);
4131        desc.mark_avail(true);
4132        ring.write_desc(0, desc);
4133
4134        assert!(matches!(
4135            consumer.poll_available(),
4136            Err(RingError::BadChain)
4137        ));
4138    }
4139
4140    #[test]
4141    fn p0033_scatter_gather_chain_is_preserved() {
4142        let ring = make_ring(16);
4143        let mut consumer = make_consumer(&ring);
4144
4145        for idx in 0..8u16 {
4146            let flags = if idx == 7 {
4147                DescFlags::WRITE
4148            } else {
4149                DescFlags::NEXT
4150            };
4151            write_avail(&ring, idx, 0, flags);
4152        }
4153
4154        let (_, chain) = consumer.poll_available().unwrap();
4155        assert_eq!(chain.len(), 8);
4156        assert_eq!(chain.readables().len(), 7);
4157        assert_eq!(chain.writables().len(), 1);
4158    }
4159
4160    #[test]
4161    fn p0034_page_boundary_indirect_descriptor_is_rejected() {
4162        let ring = make_ring(8);
4163        let mut consumer = make_consumer(&ring);
4164
4165        let mut desc = Descriptor::new(0x1ff0, 16, 0, DescFlags::INDIRECT);
4166        desc.mark_avail(true);
4167        ring.write_desc(0, desc);
4168
4169        assert!(matches!(
4170            consumer.poll_available(),
4171            Err(RingError::BadChain)
4172        ));
4173    }
4174
4175    #[test]
4176    fn p0035_both_avail_and_used_clear_is_ignored() {
4177        let ring = make_ring(8);
4178        let mut consumer = make_consumer(&ring);
4179
4180        ring.write_desc(0, raw_desc(0, 16, DescFlags::empty()));
4181
4182        assert!(matches!(
4183            consumer.poll_available(),
4184            Err(RingError::WouldBlock)
4185        ));
4186    }
4187
4188    #[test]
4189    fn p0037_event_flags_can_cycle_between_batches() {
4190        let ring = make_ring(8);
4191        let mut consumer = make_consumer(&ring);
4192
4193        for (idx, (flags, expect_notify)) in [
4194            (EventFlags::ENABLE.bits(), true),
4195            (EventFlags::DISABLE.bits(), false),
4196            (EventFlags::ENABLE.bits(), true),
4197        ]
4198        .into_iter()
4199        .enumerate()
4200        {
4201            write_driver_event(&ring, 0, flags);
4202            write_avail(&ring, idx as u16, idx as u16, DescFlags::empty());
4203            let (id, _) = consumer.poll_available().unwrap();
4204            assert_eq!(
4205                consumer.submit_used_with_notify(id, 0).unwrap(),
4206                expect_notify
4207            );
4208        }
4209    }
4210
4211    #[test]
4212    fn p0038_many_wraps_on_minimum_queue_size() {
4213        let ring = make_ring(1);
4214        let mut producer = make_producer(&ring);
4215        let mut consumer = make_consumer(&ring);
4216
4217        for idx in 0..8u64 {
4218            let id = producer.submit_one(0x1000 + idx * 0x10, 16, false).unwrap();
4219            let (dev_id, _) = consumer.poll_available().unwrap();
4220            assert_eq!(dev_id, id);
4221            consumer.submit_used(dev_id, 0).unwrap();
4222            assert_eq!(producer.poll_used().unwrap().id, id);
4223        }
4224    }
4225
4226    #[test]
4227    fn p0039_reserved_device_event_flags_suppress_avail_notification() {
4228        let ring = make_ring(8);
4229        let mut producer = make_producer(&ring);
4230
4231        write_device_event(&ring, 0, 3);
4232
4233        assert!(
4234            !producer
4235                .submit_one_with_notify(0x1000, 16, false)
4236                .unwrap()
4237                .notify
4238        );
4239    }
4240
4241    #[test]
4242    fn p0040_indirect_with_next_is_rejected() {
4243        let ring = make_ring(8);
4244        let mut consumer = make_consumer(&ring);
4245
4246        write_avail(&ring, 0, 0, DescFlags::INDIRECT | DescFlags::NEXT);
4247
4248        assert!(matches!(
4249            consumer.poll_available(),
4250            Err(RingError::BadChain)
4251        ));
4252    }
4253
4254    #[test]
4255    fn p0041_driver_event_offset_at_queue_size_suppresses_notification() {
4256        let ring = make_ring(8);
4257        let mut consumer = make_consumer(&ring);
4258
4259        write_driver_event(&ring, ring.len() as u16, EventFlags::DESC.bits());
4260        write_avail(&ring, 0, 0, DescFlags::empty());
4261
4262        let (id, _) = consumer.poll_available().unwrap();
4263
4264        assert!(!consumer.submit_used_with_notify(id, 0).unwrap());
4265    }
4266
4267    #[test]
4268    fn p0042_empty_ring_has_no_available_work() {
4269        let ring = make_ring(8);
4270        let mut consumer = make_consumer(&ring);
4271
4272        assert!(matches!(
4273            consumer.poll_available(),
4274            Err(RingError::WouldBlock)
4275        ));
4276    }
4277
4278    #[test]
4279    fn p0043_id_equal_to_queue_size_is_rejected() {
4280        let ring = make_ring(8);
4281        let mut consumer = make_consumer(&ring);
4282
4283        write_avail(&ring, 0, ring.len() as u16, DescFlags::empty());
4284
4285        assert!(matches!(
4286            consumer.poll_available(),
4287            Err(RingError::InvalidState)
4288        ));
4289    }
4290
4291    #[test]
4292    fn p0044_avail_and_used_set_is_not_available() {
4293        let ring = make_ring(8);
4294        let mut consumer = make_consumer(&ring);
4295
4296        ring.write_desc(0, raw_desc(0, 16, DescFlags::AVAIL | DescFlags::USED));
4297
4298        assert!(!consumer.peek_available().unwrap());
4299        assert!(matches!(
4300            consumer.poll_available(),
4301            Err(RingError::WouldBlock)
4302        ));
4303    }
4304
4305    #[test]
4306    fn p0045_zero_length_descriptor_is_consumable() {
4307        let ring = make_ring(8);
4308        let mut consumer = make_consumer(&ring);
4309
4310        write_avail_len(&ring, 0, 0, 0, DescFlags::WRITE);
4311
4312        let (id, chain) = consumer.poll_available().unwrap();
4313        assert_eq!(id, 0);
4314        assert_eq!(chain.writables()[0].len, 0);
4315    }
4316
4317    #[test]
4318    fn p0046_max_length_descriptor_is_consumable_at_ring_layer() {
4319        let ring = make_ring(8);
4320        let mut consumer = make_consumer(&ring);
4321
4322        write_avail_len(&ring, 0, 0, u32::MAX, DescFlags::WRITE);
4323
4324        let (id, chain) = consumer.poll_available().unwrap();
4325        assert_eq!(id, 0);
4326        assert_eq!(chain.writables()[0].len, u32::MAX);
4327    }
4328
4329    #[test]
4330    fn p0047_zero_descriptor_address_is_preserved_at_ring_layer() {
4331        let ring = make_ring(8);
4332        let mut consumer = make_consumer(&ring);
4333
4334        let mut desc = Descriptor::new(0, 16, 0, DescFlags::WRITE);
4335        desc.mark_avail(true);
4336        ring.write_desc(0, desc);
4337
4338        let (id, chain) = consumer.poll_available().unwrap();
4339        assert_eq!(id, 0);
4340        assert_eq!(chain.writables()[0].addr, 0);
4341    }
4342
4343    #[test]
4344    fn p0049_indirect_entry_with_phase_bits_is_rejected() {
4345        let ring = make_ring(8);
4346        let mut consumer = make_consumer(&ring);
4347
4348        ring.write_desc(
4349            0,
4350            raw_desc(
4351                0,
4352                16,
4353                DescFlags::INDIRECT | DescFlags::AVAIL | DescFlags::USED,
4354            ),
4355        );
4356
4357        assert!(matches!(
4358            consumer.poll_available(),
4359            Err(RingError::WouldBlock)
4360        ));
4361    }
4362
4363    #[test]
4364    fn p0050_nested_indirect_is_rejected_at_head() {
4365        let ring = make_ring(8);
4366        let mut consumer = make_consumer(&ring);
4367
4368        write_avail(&ring, 0, 0, DescFlags::INDIRECT);
4369
4370        assert!(matches!(
4371            consumer.poll_available(),
4372            Err(RingError::BadChain)
4373        ));
4374    }
4375
4376    #[test]
4377    fn p0051_write_flag_controls_readable_writable_split() {
4378        let ring = make_ring(8);
4379        let mut consumer = make_consumer(&ring);
4380
4381        write_avail(&ring, 0, 0, DescFlags::NEXT);
4382        write_avail(&ring, 1, 0, DescFlags::WRITE);
4383
4384        let (_, chain) = consumer.poll_available().unwrap();
4385        assert_eq!(chain.readables().len(), 1);
4386        assert_eq!(chain.writables().len(), 1);
4387    }
4388
4389    #[test]
4390    fn p0052_batch_completion_signals_each_chain_head() {
4391        let ring = make_ring(8);
4392        let mut producer = make_producer(&ring);
4393        let mut consumer = make_consumer(&ring);
4394
4395        let id0 = producer.submit_one(0x1000, 16, true).unwrap();
4396        let id1 = producer.submit_one(0x2000, 16, true).unwrap();
4397
4398        let (dev0, _) = consumer.poll_available().unwrap();
4399        let (dev1, _) = consumer.poll_available().unwrap();
4400        consumer.submit_used(dev0, 16).unwrap();
4401        consumer.submit_used(dev1, 32).unwrap();
4402
4403        assert_eq!(producer.poll_used().unwrap().id, id0);
4404        assert_eq!(producer.poll_used().unwrap().id, id1);
4405    }
4406
4407    #[test]
4408    fn p0053_descriptor_without_avail_bit_is_ignored() {
4409        let ring = make_ring(8);
4410        let mut consumer = make_consumer(&ring);
4411
4412        ring.write_desc(0, raw_desc(0, 16, DescFlags::WRITE));
4413
4414        assert!(!consumer.peek_available().unwrap());
4415        assert!(matches!(
4416            consumer.poll_available(),
4417            Err(RingError::WouldBlock)
4418        ));
4419    }
4420
4421    #[test]
4422    fn p0054_event_suppression_struct_layout_matches_packed_ring() {
4423        assert_eq!(EventSuppression::SIZE, 4);
4424        assert_eq!(EventSuppression::WRAP_OFFSET, 0);
4425        assert_eq!(EventSuppression::FLAGS_OFFSET, 2);
4426    }
4427
4428    #[test]
4429    fn p0055_descriptor_after_chain_tail_is_ignored() {
4430        let ring = make_ring(8);
4431        let mut consumer = make_consumer(&ring);
4432
4433        write_avail(&ring, 0, 0, DescFlags::empty());
4434        write_avail(&ring, 1, 1, DescFlags::WRITE);
4435
4436        let (id, chain) = consumer.poll_available().unwrap();
4437        assert_eq!(id, 0);
4438        assert_eq!(chain.len(), 1);
4439    }
4440
4441    #[test]
4442    fn p0056_next_chain_order_is_preserved() {
4443        let ring = make_ring(8);
4444        let mut consumer = make_consumer(&ring);
4445
4446        for idx in 0..3u16 {
4447            let mut desc = Descriptor::new(
4448                0x1000 + u64::from(idx) * 0x100,
4449                16,
4450                0,
4451                if idx == 2 {
4452                    DescFlags::WRITE
4453                } else {
4454                    DescFlags::NEXT
4455                },
4456            );
4457            desc.mark_avail(true);
4458            ring.write_desc(idx, desc);
4459        }
4460
4461        let (_, chain) = consumer.poll_available().unwrap();
4462        assert_eq!(chain.elems()[0].addr, 0x1000);
4463        assert_eq!(chain.elems()[1].addr, 0x1100);
4464        assert_eq!(chain.elems()[2].addr, 0x1200);
4465    }
4466
4467    #[test]
4468    fn p0057_device_event_enable_requests_avail_notification() {
4469        let ring = make_ring(8);
4470        let mut producer = make_producer(&ring);
4471
4472        write_device_event(&ring, 0, EventFlags::ENABLE.bits());
4473
4474        assert!(
4475            producer
4476                .submit_one_with_notify(0x1000, 16, false)
4477                .unwrap()
4478                .notify
4479        );
4480    }
4481
4482    #[test]
4483    fn p0058_used_descriptor_carries_id_and_len() {
4484        let ring = make_ring(8);
4485        let mut producer = make_producer(&ring);
4486        let mut consumer = make_consumer(&ring);
4487
4488        let id = producer.submit_one(0x1000, 16, true).unwrap();
4489        let (dev_id, _) = consumer.poll_available().unwrap();
4490        consumer.submit_used(dev_id, 42).unwrap();
4491
4492        let used = producer.poll_used().unwrap();
4493        assert_eq!(used.id, id);
4494        assert_eq!(used.len, 42);
4495    }
4496}
4497
4498#[cfg(test)]
4499mod fuzz;