Skip to main content

hyperlight_common/virtq/
consumer.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4use alloc::vec::Vec;
5use core::fmt;
6
7use bytes::Bytes;
8use smallvec::SmallVec;
9
10use super::ring::{ChainElems, Checkpoint};
11use super::*;
12
13/// Stateful reader over device-readable descriptors received from the producer.
14///
15/// Reads copy directly from shared memory into caller-provided final storage.
16/// The chain must be returned together with its paired [`ReplyChain`] through
17/// [`VirtqConsumer::complete`] before the descriptors can be reused.
18#[must_use = "dropping without completing leaks the descriptor"]
19pub struct RecvChain<M: MemOps> {
20    state: ChainState<M>,
21}
22
23impl<M: MemOps> fmt::Debug for RecvChain<M> {
24    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
25        f.debug_struct("RecvChain")
26            .field("token", &self.state.token)
27            .field("elems", &self.state.elems)
28            .field("len", &self.state.total)
29            .field("consumed", &self.state.position)
30            .field("desc_index", &self.state.desc_idx)
31            .field("desc_offset", &self.state.desc_off)
32            .finish()
33    }
34}
35
36impl<M: MemOps> RecvChain<M> {
37    fn new(mem: M, token: Token, elems: ChainElems, len: usize) -> Self {
38        Self {
39            state: ChainState::new(mem, token, elems, len),
40        }
41    }
42
43    /// The token identifying this chain.
44    #[inline]
45    pub fn token(&self) -> Token {
46        self.state.token()
47    }
48
49    /// Total readable payload length.
50    #[inline]
51    pub fn len(&self) -> usize {
52        self.state.total()
53    }
54
55    /// Whether this chain has no readable payload.
56    #[inline]
57    pub fn is_empty(&self) -> bool {
58        self.len() == 0
59    }
60
61    /// Number of bytes consumed by the stateful reader.
62    #[inline]
63    pub fn consumed(&self) -> usize {
64        self.state.position()
65    }
66
67    /// Number of bytes still available to the stateful reader.
68    #[inline]
69    pub fn remaining(&self) -> usize {
70        self.state.remaining()
71    }
72
73    /// Read bytes sequentially across descriptor boundaries.
74    ///
75    /// Returns the number of bytes copied, which may be smaller than `buf.len()`
76    /// at the end of the chain. If a later memory read fails, the cursor remains
77    /// advanced past any earlier chunks copied by the same call.
78    pub fn read(&mut self, buf: &mut [u8]) -> Result<usize, VirtqError> {
79        let len = buf.len().min(self.remaining());
80        let mut dst = &mut buf[..len];
81        let mut read = 0;
82
83        while !dst.is_empty() {
84            let Some(elem) = self.state.current_elem() else {
85                break;
86            };
87
88            let desc_len = elem.len as usize;
89            let desc_offset = self.state.desc_offset();
90            let len = (desc_len - desc_offset).min(dst.len());
91            let (current, rest) = dst.split_at_mut(len);
92
93            let addr = elem
94                .addr
95                .checked_add(desc_offset as u64)
96                .ok_or(VirtqError::MemoryReadError)?;
97
98            self.state
99                .mem
100                .read(addr, current)
101                .map_err(|_| VirtqError::MemoryReadError)?;
102
103            self.state.advance(len);
104            read += len;
105            dst = rest;
106        }
107
108        Ok(read)
109    }
110
111    /// Read exactly `buf.len()` bytes or return an error.
112    #[inline]
113    pub fn read_exact(&mut self, buf: &mut [u8]) -> Result<&mut Self, VirtqError> {
114        if buf.len() > self.remaining() {
115            return Err(VirtqError::ReceiveTooShort {
116                requested: buf.len(),
117                remaining: self.remaining(),
118            });
119        }
120
121        let read = self.read(buf)?;
122        debug_assert_eq!(read, buf.len());
123        Ok(self)
124    }
125
126    /// Copy the complete payload into descriptor-preserving owned segments.
127    ///
128    /// This does not change the stateful read position. Each call takes a new
129    /// snapshot of shared memory; callers should validate and use the returned
130    /// owned value rather than reading the same untrusted payload again.
131    ///
132    /// # Errors
133    ///
134    /// Returns [`VirtqError::Bookkeeping`] for allocation failures or
135    /// [`VirtqError::MemoryReadError`] for unreadable payloads.
136    pub fn to_segments(&self) -> Result<Segments, VirtqError> {
137        let mut segments = SmallVec::<[Bytes; 4]>::new();
138        segments
139            .try_reserve_exact(self.state.elems.len())
140            .map_err(|_| VirtqError::Bookkeeping)?;
141
142        for elem in &self.state.elems {
143            let mut buf = zeroed_vec(elem.len as usize)?;
144            self.state
145                .mem
146                .read(elem.addr, &mut buf)
147                .map_err(|_| VirtqError::MemoryReadError)?;
148            segments.push(Bytes::from(buf));
149        }
150
151        Ok(Segments::from_smallvec(segments))
152    }
153
154    /// Copy the complete payload directly into one contiguous allocation.
155    ///
156    /// This does not change the stateful read position. Each call takes a new
157    /// snapshot of shared memory; callers should validate and use the returned
158    /// owned value rather than reading the same untrusted payload again.
159    ///
160    /// # Errors
161    ///
162    /// Returns [`VirtqError::Bookkeeping`] for allocation failures or
163    /// [`VirtqError::MemoryReadError`] for unreadable payloads.
164    pub fn to_bytes(&self) -> Result<Bytes, VirtqError> {
165        if self.is_empty() {
166            return Ok(Bytes::new());
167        }
168
169        let mut buf = zeroed_vec(self.len())?;
170        let mut offset = 0;
171
172        for elem in &self.state.elems {
173            let end = offset + elem.len as usize;
174            self.state
175                .mem
176                .read(elem.addr, &mut buf[offset..end])
177                .map_err(|_| VirtqError::MemoryReadError)?;
178            offset = end;
179        }
180
181        Ok(Bytes::from(buf))
182    }
183}
184
185/// Consumer-side chain reply, either writable or ack-only.
186///
187/// Created by [`VirtqConsumer::poll`]. Must be submitted back via
188/// [`VirtqConsumer::complete`] to release the descriptor.
189#[must_use = "dropping without completing leaks the descriptor"]
190pub enum ReplyChain<M: MemOps> {
191    /// Reply with writable buffer capacity.
192    /// Use the `write*` methods on [`WritableChain`] to fill the
193    /// response buffer.
194    Writable(WritableChain<M>),
195    /// Ack-only reply (for chains with only readable buffers). No response
196    /// buffer. Pass it back with the paired [`RecvChain`] through
197    /// [`VirtqConsumer::complete`] to acknowledge.
198    Ack(AckChain),
199}
200
201/// One polled chain and its matching completion capability.
202pub type PolledChain<M> = (RecvChain<M>, ReplyChain<M>);
203
204/// An exact batch returned by [`VirtqConsumer::poll_batch`].
205pub type PolledChains<M> = Vec<PolledChain<M>>;
206
207impl<M: MemOps> ReplyChain<M> {
208    /// The token identifying this reply.
209    #[inline]
210    pub fn token(&self) -> Token {
211        match self {
212            ReplyChain::Writable(wc) => wc.token(),
213            ReplyChain::Ack(ack) => ack.token(),
214        }
215    }
216
217    /// Number of bytes written (0 for Ack).
218    #[inline]
219    pub fn written(&self) -> usize {
220        match self {
221            ReplyChain::Writable(wc) => wc.written(),
222            ReplyChain::Ack(_) => 0,
223        }
224    }
225
226    /// Convert into the writable form.
227    ///
228    /// Returns the [`AckChain`] unchanged as `Err` for ack-only replies, so the
229    /// completion capability is never silently dropped.
230    pub fn into_writable(self) -> Result<WritableChain<M>, AckChain> {
231        match self {
232            ReplyChain::Writable(wc) => Ok(wc),
233            ReplyChain::Ack(ack) => Err(ack),
234        }
235    }
236}
237
238/// A reply chain with writable buffer capacity.
239///
240/// # Example
241///
242/// ```ignore
243/// if let ReplyChain::Writable(mut wc) = reply {
244///     wc.write_all(b"response data")?;
245///     consumer.complete(recv, wc)?;
246/// }
247/// ```
248#[must_use = "dropping without completing leaks the descriptor"]
249pub struct WritableChain<M: MemOps> {
250    state: ChainState<M>,
251}
252
253impl<M: MemOps> WritableChain<M> {
254    fn new(mem: M, token: Token, elems: ChainElems) -> Self {
255        let capacity = elems.iter().map(|elem| elem.len as usize).sum();
256        Self {
257            state: ChainState::new(mem, token, elems, capacity),
258        }
259    }
260
261    /// The token identifying this writable reply.
262    #[inline]
263    pub fn token(&self) -> Token {
264        self.state.token()
265    }
266
267    /// Total reply capacity in bytes.
268    #[inline]
269    pub fn capacity(&self) -> usize {
270        self.state.total()
271    }
272
273    /// Number of writable descriptors in this chain.
274    #[inline]
275    pub fn desc_count(&self) -> usize {
276        self.state.elems.len()
277    }
278
279    /// Number of bytes written so far.
280    #[inline]
281    pub fn written(&self) -> usize {
282        self.state.position()
283    }
284
285    /// Remaining reply capacity.
286    #[inline]
287    pub fn remaining(&self) -> usize {
288        self.state.remaining()
289    }
290
291    /// Write bytes into writable buffers, returning how many were written.
292    ///
293    /// Appends at the current write position. If `buf` is larger than the
294    /// remaining capacity, writes as many bytes as will fit (partial write).
295    /// Segmentation is intentionally hidden; host-side writes must go through
296    /// [`MemOps::write`]. If a later memory write fails, the cursor and written
297    /// length retain any earlier chunks written by the same call.
298    ///
299    /// # Errors
300    ///
301    /// - [`VirtqError::MemoryWriteError`] - underlying MemOps write failed
302    pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
303        let mut src = &buf[..buf.len().min(self.remaining())];
304        let mut written = 0;
305
306        while !src.is_empty() {
307            let Some(elem) = self.state.current_elem() else {
308                break;
309            };
310            let desc_capacity = elem.len as usize;
311            let desc_offset = self.state.desc_offset();
312            let len = (desc_capacity - desc_offset).min(src.len());
313
314            let addr = elem
315                .addr
316                .checked_add(desc_offset as u64)
317                .ok_or(VirtqError::MemoryWriteError)?;
318
319            self.state
320                .mem
321                .write(addr, &src[..len])
322                .map_err(|_| VirtqError::MemoryWriteError)?;
323
324            self.state.advance(len);
325            written += len;
326            src = &src[len..];
327        }
328
329        Ok(written)
330    }
331
332    /// Write the entire buffer or return an error.
333    ///
334    /// # Errors
335    ///
336    /// - [`VirtqError::ReplyTooLarge`] - buf exceeds remaining capacity
337    /// - [`VirtqError::MemoryWriteError`] - underlying MemOps write failed
338    #[inline]
339    pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
340        if buf.len() > self.remaining() {
341            return Err(VirtqError::ReplyTooLarge);
342        }
343
344        let written = self.write(buf)?;
345        debug_assert_eq!(written, buf.len());
346        Ok(self)
347    }
348
349    /// Rewind the write cursor to the beginning.
350    ///
351    /// Previously written bytes in shared memory are not zeroed; the
352    /// `written` count is simply reset to 0.
353    pub fn rewind(&mut self) {
354        self.state.rewind();
355    }
356}
357
358/// An ack-only reply for chains with no writable buffers.
359///
360/// No response buffer - pass it back with the paired [`RecvChain`] through
361/// [`VirtqConsumer::complete`] to acknowledge processing and release the descriptor.
362///
363/// This wrapper keeps ack replies as a must-use completion capability instead
364/// of exposing a bare token that could be accidentally ignored.
365#[must_use = "dropping without completing leaks the descriptor"]
366pub struct AckChain {
367    token: Token,
368}
369
370impl AckChain {
371    fn new(token: Token) -> Self {
372        Self { token }
373    }
374
375    #[inline]
376    pub fn token(&self) -> Token {
377        self.token
378    }
379}
380
381/// A high-level virtqueue consumer (device side).
382///
383/// The consumer receives chains from the producer (driver), processes them,
384/// and sends back replies. This is typically used on the device/host side.
385///
386/// # Example
387///
388/// ```ignore
389/// let mut consumer = VirtqConsumer::new(layout, mem, notifier);
390///
391/// // Poll and process
392/// while let Some((recv, reply)) = consumer.poll(MAX_RECV_LEN)? {
393///     let data = recv.to_bytes()?;
394///     match reply {
395///         ReplyChain::Writable(mut wc) => {
396///             let response = handle_request(data);
397///             wc.write_all(&response)?;
398///             consumer.complete(recv, wc)?;
399///         }
400///         ReplyChain::Ack(ack) => {
401///             consumer.complete(recv, ack)?;
402///         }
403///     }
404/// }
405///
406/// // Or defer completions
407/// let mut pending = Vec::new();
408/// while let Some((recv, reply)) = consumer.poll(MAX_RECV_LEN)? {
409///     let result = process(&recv);
410///     pending.push((result, recv, reply));
411/// }
412///
413/// for (result, recv, reply) in pending {
414///     // ... complete later ...
415///     consumer.complete(recv, reply)?;
416/// }
417/// ```
418pub struct VirtqConsumer<M, N> {
419    inner: RingConsumer<M>,
420    mem: M,
421    notifier: N,
422    next_token: u32,
423}
424
425impl<M: MemOps + Clone, N: Notifier> VirtqConsumer<M, N> {
426    /// Create a new virtqueue consumer.
427    ///
428    /// # Arguments
429    ///
430    /// * `layout` - Ring memory layout
431    /// * `mem` - Memory ops implementation for reading/writing to shared memory
432    /// * `notifier` - Callback for notifying the driver about replies
433    pub fn new(layout: Layout, mem: M, notifier: N) -> Self {
434        Self::new_split(layout, mem.clone(), mem, notifier)
435    }
436
437    /// Create a consumer with separate ring and buffer memory accessors.
438    pub fn new_split(layout: Layout, ring_mem: M, buf_mem: M, notifier: N) -> Self {
439        let inner = RingConsumer::new(layout, ring_mem);
440
441        Self {
442            inner,
443            mem: buf_mem,
444            notifier,
445            next_token: 0,
446        }
447    }
448
449    /// Poll for a single incoming chain from the driver.
450    ///
451    /// Returns a stateful [`RecvChain`] reader and a [`ReplyChain`] writable
452    /// reply or ack capability. Both are independent owned values with no
453    /// borrow on the consumer, but they must be returned together through
454    /// [`complete`](Self::complete).
455    ///
456    /// On [`VirtqError::BadChain`] and [`VirtqError::PayloadTooLarge`] the
457    /// descriptor is returned to the driver (completed with zero length) before
458    /// the error is propagated, so a rejected chain does not leak.
459    /// Bookkeeping failures after acquiring the chain also return its descriptor.
460    ///
461    /// # Arguments
462    ///
463    /// * `max_recv_len` - Maximum readable payload size. Payloads larger
464    ///   than this return [`VirtqError::PayloadTooLarge`].
465    ///
466    /// # Errors
467    ///
468    /// - [`VirtqError::BadChain`] - Descriptor chain format not recognized
469    /// - [`VirtqError::Bookkeeping`] - Descriptor storage allocation failed
470    /// - [`VirtqError::RingError`] - Invalid ring state or memory access failure
471    pub fn poll(&mut self, max_recv_len: usize) -> Result<Option<PolledChain<M>>, VirtqError> {
472        let (id, chain) = match self.inner.poll_available() {
473            Ok(x) => x,
474            Err(RingError::WouldBlock) => return Ok(None),
475            Err(e) => return Err(e.into()),
476        };
477
478        let readables = chain.readables();
479        let writables = chain.writables();
480        if readables.is_empty() && writables.is_empty() {
481            return Err(self.abort_chain(id, VirtqError::BadChain));
482        }
483
484        let recv_len = readables
485            .iter()
486            .fold(0usize, |acc, elem| acc.saturating_add(elem.len as usize));
487
488        let token = Token {
489            seq: self.next_token,
490            id,
491        };
492        self.next_token = self.next_token.wrapping_add(1);
493
494        if recv_len > max_recv_len {
495            return Err(self.abort_chain(
496                id,
497                VirtqError::PayloadTooLarge {
498                    recv: recv_len,
499                    limit: max_recv_len,
500                },
501            ));
502        }
503
504        let (readables, writables) = match chain.into_parts() {
505            Ok(parts) => parts,
506            Err(error) => return Err(self.abort_chain(id, error.into())),
507        };
508        let chain = RecvChain::new(self.mem.clone(), token, readables, recv_len);
509
510        let reply = if !writables.is_empty() {
511            let mem = self.mem.clone();
512            let writable = WritableChain::new(mem, token, writables);
513            ReplyChain::Writable(writable)
514        } else {
515            let ack = AckChain::new(token);
516            ReplyChain::Ack(ack)
517        };
518
519        Ok(Some((chain, reply)))
520    }
521
522    /// Poll `count` chains while keeping at least `keep_avail` available.
523    ///
524    /// Returns `None` and restores the consumer's local state when fewer than
525    /// `count + keep_avail` chains are available. Complete each returned chain
526    /// through [`complete`](Self::complete).
527    ///
528    /// # Arguments
529    ///
530    /// * `count` - Exact number of chains to return.
531    /// * `keep_avail` - Minimum number of chains to leave available.
532    /// * `max_recv_len` - Maximum readable payload size accepted for each chain
533    ///   independently.
534    ///
535    /// # Errors
536    ///
537    /// Polling errors propagate without rollback. Earlier chains in the batch
538    /// can remain in flight without their handles being returned.
539    /// Batch storage allocation failure leaves the queue unchanged.
540    pub fn poll_batch(
541        &mut self,
542        count: usize,
543        keep_avail: usize,
544        max_recv_len: usize,
545    ) -> Result<Option<PolledChains<M>>, VirtqError> {
546        let Some(total) = count.checked_add(keep_avail) else {
547            return Ok(None);
548        };
549
550        // Every chain consumes at least one descriptor.
551        if total > self.inner.num_free() {
552            return Ok(None);
553        }
554
555        let mut polled = PolledChains::new();
556        polled
557            .try_reserve_exact(total)
558            .map_err(|_| VirtqError::Bookkeeping)?;
559
560        // Polling changes only local bookkeeping until a chain is completed.
561        let cp = self.inner.poll_checkpoint();
562        let next_token = self.next_token;
563
564        let mut batch_cp = None;
565
566        while polled.len() < total {
567            if keep_avail != 0 && polled.len() == count {
568                batch_cp = Some((self.inner.poll_checkpoint(), self.next_token));
569            }
570
571            if let Some(chain) = self.poll(max_recv_len)? {
572                polled.push(chain);
573                continue;
574            }
575
576            self.rollback_polled(cp, next_token, polled);
577            return Ok(None);
578        }
579
580        if let Some((checkpoint, next_token)) = batch_cp {
581            self.rollback_polled(checkpoint, next_token, polled.drain(count..));
582        }
583
584        Ok(Some(polled))
585    }
586
587    /// Submit both halves of a received chain back to the ring.
588    ///
589    /// Consuming the [`RecvChain`] prevents further reads once its descriptors
590    /// can be reused by the producer. `reply` accepts both [`WritableChain`]
591    /// (with written byte count) and [`AckChain`] (zero-length) through
592    /// [`ReplyChain`]. The two halves must have matching tokens.
593    ///
594    /// A mismatched pair returns [`VirtqError::InvalidState`] without returning
595    /// either descriptor. This fails closed: the descriptors remain in flight
596    /// because completing either could invalidate another still-live
597    /// [`RecvChain`].
598    pub fn complete(
599        &mut self,
600        recv: impl Into<RecvChain<M>>,
601        reply: impl Into<ReplyChain<M>>,
602    ) -> Result<(), VirtqError> {
603        let recv = recv.into();
604        let reply = reply.into();
605
606        if recv.token() != reply.token() {
607            return Err(VirtqError::InvalidState);
608        }
609
610        let id = reply.token().id;
611        let written = u32::try_from(reply.written()).map_err(|_| VirtqError::ReplyTooLarge)?;
612
613        if self.inner.submit_used_with_notify(id, written)? {
614            self.notifier.notify(QueueStats {
615                num_free: self.inner.num_free(),
616                num_inflight: self.inner.num_inflight(),
617            });
618        }
619
620        Ok(())
621    }
622
623    /// Return a consumed descriptor to the driver with zero written length.
624    ///
625    /// The ring's `poll_available` removes the descriptor from the available
626    /// ring before [`poll`](Self::poll) validates the chain.
627    fn abort_chain(&mut self, id: u16, err: VirtqError) -> VirtqError {
628        // Best effort: failing to return the descriptor means the ring is
629        // already in an unrecoverable state, so surface the original error.
630        if let Ok(true) = self.inner.submit_used_with_notify(id, 0) {
631            self.notifier.notify(QueueStats {
632                num_free: self.inner.num_free(),
633                num_inflight: self.inner.num_inflight(),
634            });
635        }
636
637        err
638    }
639
640    /// Restore the saved ring state and token counter for an uncompleted batch.
641    ///
642    /// `polled` must contain every chain acquired since `checkpoint` on this consumer,
643    /// with no intervening completions.
644    fn rollback_polled(
645        &mut self,
646        cp: Checkpoint,
647        next: u32,
648        polled: impl IntoIterator<Item = PolledChain<M>>,
649    ) {
650        // No chain handle may survive when its descriptor becomes pollable again.
651        let ids = polled.into_iter().map(|(recv, reply)| {
652            let id = recv.token().id;
653            drop((recv, reply));
654            id
655        });
656
657        self.inner.rollback_polls(cp, ids);
658        self.next_token = next;
659    }
660
661    /// Get the current available cursor position.
662    ///
663    /// Returns the position where the next available descriptor will be
664    /// consumed. Useful for setting up descriptor-based event suppression.
665    #[inline]
666    pub fn avail_cursor(&self) -> RingCursor {
667        self.inner.avail_cursor()
668    }
669
670    /// Get the current used cursor position.
671    ///
672    /// Returns the position where the next used descriptor will be written.
673    /// Useful for setting up descriptor-based event suppression.
674    #[inline]
675    pub fn used_cursor(&self) -> RingCursor {
676        self.inner.used_cursor()
677    }
678
679    /// Configure event suppression for available buffer notifications.
680    ///
681    /// This controls when the driver (producer) signals us about new buffers:
682    ///
683    /// - [`SuppressionKind::Enable`] - Always signal (default) - good for latency
684    /// - [`SuppressionKind::Disable`] - Never signal - caller must poll
685    /// - [`SuppressionKind::Descriptor`] - Signal only at specific cursor position
686    ///
687    /// # Example: Polling Mode
688    /// ```ignore
689    /// consumer.set_avail_suppression(SuppressionKind::Disable)?;
690    /// loop {
691    ///     while let Some((chain, reply)) = consumer.poll(1024)? {
692    ///         process(chain, reply);
693    ///     }
694    ///     // ... do other work ...
695    /// }
696    /// ```
697    pub fn set_avail_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
698        match kind {
699            SuppressionKind::Enable => self.inner.enable_avail_notifications()?,
700            SuppressionKind::Disable => self.inner.disable_avail_notifications()?,
701            SuppressionKind::Descriptor(cursor) => self
702                .inner
703                .enable_avail_notifications_desc(cursor.head(), cursor.wrap())?,
704        }
705        Ok(())
706    }
707
708    /// Reset ring and inflight state to initial values.
709    ///
710    /// Fails while a chain polled by this consumer remains uncompleted.
711    ///
712    /// # Errors
713    ///
714    /// - [`VirtqError::InvalidState`] - one or more chains are still in flight
715    /// - [`VirtqError::RingError`] - device-event normalization failed
716    pub fn reset(&mut self) -> Result<(), VirtqError> {
717        if self.inner.num_inflight() != 0 {
718            return Err(VirtqError::InvalidState);
719        }
720
721        self.inner.reset()?;
722        Ok(())
723    }
724}
725
726struct ChainState<M: MemOps> {
727    mem: M,
728    token: Token,
729    elems: ChainElems,
730    total: usize,
731    position: usize,
732    desc_idx: usize,
733    desc_off: usize,
734}
735
736impl<M: MemOps> ChainState<M> {
737    fn new(mem: M, token: Token, elems: ChainElems, total: usize) -> Self {
738        let mut state = Self {
739            mem,
740            token,
741            elems,
742            total,
743            position: 0,
744            desc_idx: 0,
745            desc_off: 0,
746        };
747        state.rewind();
748        state
749    }
750
751    #[inline]
752    fn token(&self) -> Token {
753        self.token
754    }
755
756    #[inline]
757    fn total(&self) -> usize {
758        self.total
759    }
760
761    #[inline]
762    fn position(&self) -> usize {
763        self.position
764    }
765
766    #[inline]
767    fn remaining(&self) -> usize {
768        self.total - self.position
769    }
770
771    #[inline(always)]
772    fn desc_len(&self) -> usize {
773        self.elems
774            .get(self.desc_idx)
775            .map(|elem| elem.len as usize)
776            .unwrap_or(0)
777    }
778
779    #[inline(always)]
780    fn desc_offset(&self) -> usize {
781        self.desc_off
782    }
783
784    #[inline(always)]
785    fn current_elem(&self) -> Option<BufferElement> {
786        self.elems.get(self.desc_idx).copied()
787    }
788
789    #[inline(always)]
790    fn advance(&mut self, len: usize) {
791        debug_assert!(len <= self.desc_len() - self.desc_off);
792        self.desc_off += len;
793        self.position += len;
794
795        while self.current_elem().is_some() && self.desc_off == self.desc_len() {
796            self.desc_idx += 1;
797            self.desc_off = 0;
798        }
799    }
800
801    fn rewind(&mut self) {
802        self.position = 0;
803        self.desc_off = 0;
804        self.desc_idx = self
805            .elems
806            .iter()
807            .position(|elem| elem.len != 0)
808            .unwrap_or(self.elems.len());
809    }
810}
811
812impl<M: MemOps> From<WritableChain<M>> for ReplyChain<M> {
813    fn from(wc: WritableChain<M>) -> Self {
814        ReplyChain::Writable(wc)
815    }
816}
817
818impl<M: MemOps> From<AckChain> for ReplyChain<M> {
819    fn from(ack: AckChain) -> Self {
820        ReplyChain::Ack(ack)
821    }
822}
823
824#[cfg(test)]
825mod tests {
826    use super::*;
827    use crate::flatbuffer_wrappers::ExternalValueSource;
828    use crate::virtq::ring::tests::{TestMem, make_producer, make_ring};
829    use crate::virtq::test_utils::*;
830
831    fn poll_data(
832        consumer: &mut VirtqConsumer<TestMem, TestNotifier>,
833    ) -> (RecvChain<TestMem>, ReplyChain<TestMem>) {
834        consumer.poll(1024).unwrap().unwrap()
835    }
836
837    #[derive(Clone)]
838    struct FailingPayloadReadMem {
839        inner: TestMem,
840        payload_addr: u64,
841        payload_len: usize,
842    }
843
844    // SAFETY: All operations delegate to TestMem. Reads overlapping the
845    // configured payload range return an error before accessing memory.
846    unsafe impl MemOps for FailingPayloadReadMem {
847        type Error = ();
848
849        fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
850            let read_end = addr.saturating_add(dst.len() as u64);
851            let payload_end = self.payload_addr.saturating_add(self.payload_len as u64);
852            if addr < payload_end && self.payload_addr < read_end {
853                return Err(());
854            }
855            self.inner.read(addr, dst).map_err(|err| match err {})
856        }
857
858        fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
859            self.inner.write(addr, src).map_err(|err| match err {})
860        }
861
862        fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
863            self.inner.load_acquire(addr).map_err(|err| match err {})
864        }
865
866        fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
867            self.inner
868                .store_release(addr, val)
869                .map_err(|err| match err {})
870        }
871
872        unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
873            unsafe { self.inner.as_slice(addr, len) }.map_err(|err| match err {})
874        }
875
876        unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
877            unsafe { self.inner.as_mut_slice(addr, len) }.map_err(|err| match err {})
878        }
879    }
880
881    #[test]
882    fn test_write_only_recv_is_empty() {
883        let ring = make_ring(16);
884        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
885
886        let se = producer.chain().writable(16).build().unwrap();
887        producer.submit(se).unwrap();
888
889        let (recv, reply) = poll_data(&mut consumer);
890        assert!(recv.to_bytes().unwrap().is_empty());
891        assert!(matches!(reply, ReplyChain::Writable(_)));
892
893        if let ReplyChain::Writable(mut wc) = reply {
894            wc.write_all(b"response").unwrap();
895            consumer.complete(recv, wc).unwrap();
896        }
897        producer.drain(drop).unwrap();
898    }
899
900    #[test]
901    fn test_read_only_ack_reply() {
902        let ring = make_ring(16);
903        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
904
905        let mut se = producer.chain().readable(16).build().unwrap();
906        se.write_all(b"hello").unwrap();
907        producer.submit(se).unwrap();
908
909        let (recv, reply) = poll_data(&mut consumer);
910        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
911        assert!(matches!(reply, ReplyChain::Ack(_)));
912
913        consumer.complete(recv, reply).unwrap();
914        producer.drain(drop).unwrap();
915    }
916
917    #[test]
918    fn test_readwrite_round_trip() {
919        let ring = make_ring(16);
920        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 64);
921        let request: Vec<u8> = (0..5 * 64).map(|i| (i % 251) as u8).collect();
922        let response: Vec<u8> = (0..6 * 64 - 1).map(|i| (i % 239) as u8).collect();
923
924        // Both descriptor halves exceed inline capacity.
925        let mut se = producer
926            .chain()
927            .readable(request.len())
928            .writable(6 * 64)
929            .build()
930            .unwrap();
931        se.write_all(&request).unwrap();
932        producer.submit(se).unwrap();
933
934        let (recv, reply) = poll_data(&mut consumer);
935        assert_eq!(recv.to_bytes().unwrap().as_ref(), request);
936
937        if let ReplyChain::Writable(mut wc) = reply {
938            assert_eq!(wc.capacity(), 6 * 64);
939            assert_eq!(wc.written(), 0);
940            assert_eq!(wc.remaining(), 6 * 64);
941            wc.write_all(&response).unwrap();
942            assert_eq!(wc.written(), response.len());
943            assert_eq!(wc.remaining(), 1);
944            consumer.complete(recv, wc).unwrap();
945        } else {
946            panic!("expected Writable reply for recv+reply chain");
947        }
948
949        let used = producer.poll().unwrap().unwrap();
950        assert_eq!(used.to_bytes().unwrap().as_ref(), response);
951    }
952
953    #[test]
954    fn test_recv_reads_across_descriptor_boundaries() {
955        let ring = make_ring(16);
956        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
957
958        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
959        se.write_all(b"abcdefgh").unwrap();
960        producer.submit(se).unwrap();
961
962        let (mut recv, reply) = poll_data(&mut consumer);
963        assert_eq!(recv.len(), 8);
964        assert_eq!(recv.remaining(), 8);
965        let mut first = [0u8; 2];
966        recv.read_exact(&mut first).unwrap();
967        assert_eq!(&first, b"ab");
968        assert_eq!(recv.consumed(), 2);
969        assert_eq!(recv.remaining(), 6);
970
971        let mut second = [0u8; 3];
972        recv.read_exact(&mut second).unwrap();
973        assert_eq!(&second, b"cde");
974
975        let mut too_long = [0u8; 4];
976        assert!(matches!(
977            recv.read_exact(&mut too_long),
978            Err(VirtqError::ReceiveTooShort {
979                requested: 4,
980                remaining: 3
981            })
982        ));
983
984        let mut final_buf = [0u8; 4];
985        assert_eq!(recv.read(&mut final_buf).unwrap(), 3);
986        assert_eq!(&final_buf[..3], b"fgh");
987        assert_eq!(recv.read(&mut final_buf).unwrap(), 0);
988        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefgh");
989
990        consumer.complete(recv, reply).unwrap();
991        producer.drain(drop).unwrap();
992    }
993
994    #[test]
995    fn test_external_values_copy_shared_memory() {
996        let ring = make_ring(16);
997        let mem = ring.mem();
998        let mut ring_producer = make_producer(&ring);
999        let payload_addr = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1000        mem.write(payload_addr, b"abcdefgh").unwrap();
1001
1002        let chain = BufferChainBuilder::new()
1003            .readable(payload_addr, 4)
1004            .readable(payload_addr + 4, 4)
1005            .build()
1006            .unwrap();
1007        ring_producer.submit_available(&chain).unwrap();
1008
1009        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1010        let (mut recv, reply) = poll_data(&mut consumer);
1011        let contiguous = recv.take_bytes(3).unwrap();
1012        let chunks = recv.take_chunks(5).unwrap();
1013        recv.finish().unwrap();
1014
1015        mem.write(payload_addr, b"xxxxxxxx").unwrap();
1016
1017        assert_eq!(contiguous, b"abc");
1018        assert_eq!(chunks.len(), 1);
1019        assert_eq!(chunks[0].as_ref(), b"defgh");
1020        consumer.complete(recv, reply).unwrap();
1021    }
1022
1023    #[test]
1024    fn test_external_values_validate_remaining_length() {
1025        let ring = make_ring(16);
1026        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1027        let mut chain = producer.chain().readable(4).build().unwrap();
1028        chain.write_all(b"data").unwrap();
1029        producer.submit(chain).unwrap();
1030
1031        let (mut recv, reply) = poll_data(&mut consumer);
1032        for error in [
1033            recv.take_bytes(usize::MAX).unwrap_err(),
1034            recv.take_chunks(usize::MAX).unwrap_err(),
1035        ] {
1036            assert!(matches!(
1037                error.downcast_ref::<VirtqError>(),
1038                Some(VirtqError::ReceiveTooShort {
1039                    requested: usize::MAX,
1040                    remaining: 4,
1041                })
1042            ));
1043        }
1044
1045        assert!(recv.take_bytes(0).unwrap().is_empty());
1046        assert!(recv.take_chunks(0).unwrap().is_empty());
1047        assert_eq!(recv.remaining(), 4);
1048        assert!(recv.finish().is_err());
1049        assert_eq!(recv.take_bytes(4).unwrap(), b"data");
1050        recv.finish().unwrap();
1051
1052        consumer.complete(recv, reply).unwrap();
1053        producer.drain(drop).unwrap();
1054    }
1055
1056    #[test]
1057    fn test_poll_defers_payload_reads() {
1058        let ring = make_ring(16);
1059        let mem = ring.mem();
1060        let mut ring_producer = make_producer(&ring);
1061        let payload_addr = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1062        mem.write(payload_addr, b"data").unwrap();
1063
1064        let chain = BufferChainBuilder::new()
1065            .readable(payload_addr, 4)
1066            .build()
1067            .unwrap();
1068        ring_producer.submit_available(&chain).unwrap();
1069
1070        let ring_mem = FailingPayloadReadMem {
1071            inner: mem.clone(),
1072            payload_addr,
1073            payload_len: 0,
1074        };
1075        let mem = FailingPayloadReadMem {
1076            inner: mem,
1077            payload_addr,
1078            payload_len: 4,
1079        };
1080
1081        let mut consumer =
1082            VirtqConsumer::new_split(ring.layout(), ring_mem, mem, TestNotifier::new());
1083
1084        let (recv, reply) = consumer.poll(4).unwrap().unwrap();
1085        assert!(matches!(recv.to_bytes(), Err(VirtqError::MemoryReadError)));
1086        consumer.complete(recv, reply).unwrap();
1087    }
1088
1089    #[test]
1090    fn test_writable_partial_write() {
1091        let ring = make_ring(16);
1092        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 8);
1093
1094        let se = producer.chain().writable(8).build().unwrap();
1095        producer.submit(se).unwrap();
1096
1097        let (recv, reply) = poll_data(&mut consumer);
1098
1099        if let ReplyChain::Writable(mut wc) = reply {
1100            let n = wc.write(b"hello world!").unwrap();
1101            assert_eq!(n, 8);
1102            assert_eq!(wc.remaining(), 0);
1103            consumer.complete(recv, wc).unwrap();
1104        } else {
1105            panic!("expected Writable");
1106        }
1107        producer.drain(drop).unwrap();
1108    }
1109
1110    #[test]
1111    fn test_writable_write_all_too_large() {
1112        let ring = make_ring(16);
1113        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
1114
1115        let se = producer.chain().writable(4).build().unwrap();
1116        producer.submit(se).unwrap();
1117        let (recv, reply) = poll_data(&mut consumer);
1118
1119        if let ReplyChain::Writable(mut wc) = reply {
1120            let err = wc.write_all(b"too long").err().unwrap();
1121            assert!(matches!(err, VirtqError::ReplyTooLarge));
1122            consumer.complete(recv, wc).unwrap();
1123        } else {
1124            panic!("expected Writable");
1125        }
1126
1127        producer.drain(drop).unwrap();
1128    }
1129
1130    #[test]
1131    fn test_poll_batch_rolls_back_partial_batch() {
1132        let ring = make_ring(32);
1133        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1134
1135        for _ in 0..20 {
1136            let chain = producer.chain().writable(16).build().unwrap();
1137            producer.submit(chain).unwrap();
1138        }
1139
1140        let cursor = consumer.avail_cursor();
1141        assert!(consumer.poll_batch(21, 0, 0).unwrap().is_none());
1142        assert_eq!(consumer.avail_cursor(), cursor);
1143        assert_eq!(consumer.inner.num_inflight(), 0);
1144        assert_eq!(consumer.next_token, 0);
1145        assert!(producer.poll().unwrap().is_none());
1146
1147        let reserved = consumer.poll_batch(20, 0, 0).unwrap().unwrap();
1148        for (recv, reply) in reserved {
1149            consumer.complete(recv, reply).unwrap();
1150        }
1151        for _ in 0..20 {
1152            assert!(producer.poll().unwrap().is_some());
1153        }
1154    }
1155
1156    #[test]
1157    fn test_poll_batch_keeps_chains_available() {
1158        let ring = make_ring(16);
1159        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1160
1161        for _ in 0..3 {
1162            let chain = producer.chain().writable(16).build().unwrap();
1163            producer.submit(chain).unwrap();
1164        }
1165
1166        let polled = consumer.poll_batch(2, 1, 0).unwrap().unwrap();
1167        assert_eq!(consumer.avail_cursor().head(), 2);
1168        assert_eq!(consumer.inner.num_inflight(), 2);
1169        assert_eq!(consumer.next_token, 2);
1170
1171        for (recv, reply) in polled {
1172            consumer.complete(recv, reply).unwrap();
1173        }
1174
1175        let (recv, reply) = consumer.poll(0).unwrap().unwrap();
1176        assert_eq!(recv.token().seq, 2);
1177        consumer.complete(recv, reply).unwrap();
1178        producer.drain(drop).unwrap();
1179    }
1180
1181    #[test]
1182    fn test_poll_batch_rolls_back_when_keep_avail_is_unmet() {
1183        let ring = make_ring(16);
1184        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1185
1186        for _ in 0..2 {
1187            let chain = producer.chain().writable(16).build().unwrap();
1188            producer.submit(chain).unwrap();
1189        }
1190
1191        let cursor = consumer.avail_cursor();
1192        assert!(consumer.poll_batch(2, 1, 0).unwrap().is_none());
1193        assert_eq!(consumer.avail_cursor(), cursor);
1194        assert_eq!(consumer.inner.num_inflight(), 0);
1195        assert_eq!(consumer.next_token, 0);
1196
1197        let polled = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1198        for (recv, reply) in polled {
1199            consumer.complete(recv, reply).unwrap();
1200        }
1201        producer.drain(drop).unwrap();
1202    }
1203
1204    #[test]
1205    fn test_poll_batch_preserves_existing_inflight_chain() {
1206        let ring = make_ring(16);
1207        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1208
1209        for _ in 0..3 {
1210            let chain = producer.chain().writable(16).build().unwrap();
1211            producer.submit(chain).unwrap();
1212        }
1213
1214        let existing = consumer.poll(0).unwrap().unwrap();
1215        let cursor = consumer.avail_cursor();
1216        assert!(consumer.poll_batch(3, 0, 0).unwrap().is_none());
1217        assert_eq!(consumer.avail_cursor(), cursor);
1218        assert_eq!(consumer.inner.num_inflight(), 1);
1219
1220        let reserved = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1221        consumer.complete(existing.0, existing.1).unwrap();
1222        for (recv, reply) in reserved {
1223            consumer.complete(recv, reply).unwrap();
1224        }
1225        producer.drain(drop).unwrap();
1226    }
1227
1228    #[test]
1229    fn test_poll_batch_rolls_back_multi_descriptor_chain() {
1230        let ring = make_ring(16);
1231        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1232
1233        let chain = producer.chain().writable(8).writable(8).build().unwrap();
1234        producer.submit(chain).unwrap();
1235
1236        let cursor = consumer.avail_cursor();
1237        assert!(consumer.poll_batch(2, 0, 0).unwrap().is_none());
1238        assert_eq!(consumer.avail_cursor(), cursor);
1239        assert_eq!(consumer.inner.num_inflight(), 0);
1240
1241        let mut reserved = consumer.poll_batch(1, 0, 0).unwrap().unwrap();
1242        let (recv, reply) = reserved.pop().unwrap();
1243        let ReplyChain::Writable(writable) = &reply else {
1244            panic!("expected writable chain");
1245        };
1246        assert_eq!(writable.desc_count(), 2);
1247        consumer.complete(recv, reply).unwrap();
1248        producer.drain(drop).unwrap();
1249    }
1250
1251    #[test]
1252    fn test_poll_batch_rolls_back_across_wrap() {
1253        let ring = make_ring(4);
1254        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1255
1256        for _ in 0..3 {
1257            let chain = producer.chain().writable(16).build().unwrap();
1258            producer.submit(chain).unwrap();
1259        }
1260        let reserved = consumer.poll_batch(3, 0, 0).unwrap().unwrap();
1261        for (recv, reply) in reserved {
1262            consumer.complete(recv, reply).unwrap();
1263        }
1264        for _ in 0..3 {
1265            assert!(producer.poll().unwrap().is_some());
1266        }
1267
1268        let chain = producer.chain().writable(16).build().unwrap();
1269        producer.submit(chain).unwrap();
1270
1271        let cursor = consumer.avail_cursor();
1272        assert_eq!(cursor.head(), 3);
1273        assert!(consumer.poll_batch(2, 0, 0).unwrap().is_none());
1274        assert_eq!(consumer.avail_cursor(), cursor);
1275
1276        let mut reserved = consumer.poll_batch(1, 0, 0).unwrap().unwrap();
1277        let (recv, reply) = reserved.pop().unwrap();
1278        consumer.complete(recv, reply).unwrap();
1279        producer.drain(drop).unwrap();
1280    }
1281
1282    #[test]
1283    fn test_poll_batch_rolls_back_token_wrap() {
1284        let ring = make_ring(16);
1285        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
1286        consumer.next_token = u32::MAX;
1287        for _ in 0..2 {
1288            let chain = producer.chain().writable(16).build().unwrap();
1289            producer.submit(chain).unwrap();
1290        }
1291        let avail = consumer.avail_cursor();
1292        let used = consumer.used_cursor();
1293        let notifications = notifier.notification_count();
1294
1295        // The unsuccessful batch crosses the token counter's wrap boundary.
1296        assert!(consumer.poll_batch(3, 0, 0).unwrap().is_none());
1297        assert_eq!(consumer.avail_cursor(), avail);
1298        assert_eq!(consumer.used_cursor(), used);
1299        assert_eq!(consumer.inner.num_inflight(), 0);
1300        assert_eq!(consumer.next_token, u32::MAX);
1301        assert_eq!(notifier.notification_count(), notifications);
1302        assert!(producer.poll().unwrap().is_none());
1303
1304        // A retry receives the same token sequence.
1305        let polled = consumer.poll_batch(2, 0, 0).unwrap().unwrap();
1306        assert_eq!(polled[0].0.token().seq, u32::MAX);
1307        assert_eq!(polled[1].0.token().seq, 0);
1308        assert_eq!(consumer.next_token, 1);
1309        for (recv, reply) in polled {
1310            consumer.complete(recv, reply).unwrap();
1311        }
1312        producer.drain(drop).unwrap();
1313    }
1314
1315    #[test]
1316    fn test_poll_batch_keeps_multi_descriptor_chains_available() {
1317        let ring = make_ring(16);
1318        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1319        let chain = producer.chain().writable(8).build().unwrap();
1320        let first_token = producer.submit(chain).unwrap();
1321        let chain = producer.chain().writable(8).writable(8).build().unwrap();
1322        let second_token = producer.submit(chain).unwrap();
1323        let chain = producer
1324            .chain()
1325            .writable(8)
1326            .writable(8)
1327            .writable(8)
1328            .build()
1329            .unwrap();
1330        let third_token = producer.submit(chain).unwrap();
1331
1332        // The two spare chains occupy five descriptors, not two.
1333        let mut polled = consumer.poll_batch(1, 2, 0).unwrap().unwrap();
1334        assert_eq!(polled.len(), 1);
1335        assert_eq!(consumer.avail_cursor().head(), 1);
1336        assert_eq!(consumer.used_cursor().head(), 0);
1337        assert_eq!(consumer.inner.num_inflight(), 1);
1338        assert_eq!(consumer.next_token, 1);
1339        let first = polled.pop().unwrap();
1340        assert_eq!(first.0.token().id, first_token.id);
1341
1342        // Both spare chains remain available in their original order.
1343        let second = consumer.poll(0).unwrap().unwrap();
1344        assert_eq!(second.0.token().id, second_token.id);
1345        assert_eq!(second.0.token().seq, 1);
1346        assert_eq!(consumer.inner.num_inflight(), 3);
1347        let third = consumer.poll(0).unwrap().unwrap();
1348        assert_eq!(third.0.token().id, third_token.id);
1349        assert_eq!(third.0.token().seq, 2);
1350        assert_eq!(consumer.inner.num_inflight(), 6);
1351        consumer.complete(first.0, first.1).unwrap();
1352        consumer.complete(second.0, second.1).unwrap();
1353        consumer.complete(third.0, third.1).unwrap();
1354        producer.drain(drop).unwrap();
1355    }
1356
1357    #[test]
1358    fn test_poll_batch_zero_count_keeps_full_ring_available() {
1359        let ring = make_ring(32);
1360        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1361        for _ in 0..32 {
1362            let chain = producer.chain().writable(16).build().unwrap();
1363            producer.submit(chain).unwrap();
1364        }
1365        let avail = consumer.avail_cursor();
1366        let used = consumer.used_cursor();
1367
1368        // Polling the spare capacity wraps the cursor through the entire ring.
1369        let polled = consumer.poll_batch(0, 32, 0).unwrap().unwrap();
1370        assert!(polled.is_empty());
1371        assert_eq!(consumer.avail_cursor(), avail);
1372        assert_eq!(consumer.used_cursor(), used);
1373        assert_eq!(consumer.inner.num_inflight(), 0);
1374        assert_eq!(consumer.next_token, 0);
1375        assert!(producer.poll().unwrap().is_none());
1376
1377        // Every descriptor can still be polled and completed.
1378        let polled = consumer.poll_batch(32, 0, 0).unwrap().unwrap();
1379        assert_eq!(polled.len(), 32);
1380        for (recv, reply) in polled {
1381            consumer.complete(recv, reply).unwrap();
1382        }
1383        producer.drain(drop).unwrap();
1384    }
1385
1386    #[test]
1387    fn test_poll_batch_error_after_partial_batch_is_not_rolled_back() {
1388        let ring = make_ring(16);
1389        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1390        let chain = producer.chain().writable(16).build().unwrap();
1391        producer.submit(chain).unwrap();
1392        let mut chain = producer.chain().readable(8).build().unwrap();
1393        chain.write_all(b"too much").unwrap();
1394        let rejected = producer.submit(chain).unwrap();
1395
1396        // Rejecting the second chain publishes a completion, so rewind is invalid.
1397        assert!(matches!(
1398            consumer.poll_batch(2, 0, 4),
1399            Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1400        ));
1401        assert_eq!(consumer.avail_cursor().head(), 2);
1402        assert_eq!(consumer.used_cursor().head(), 1);
1403        assert_eq!(consumer.inner.num_inflight(), 1);
1404        assert_eq!(consumer.next_token, 2);
1405
1406        // The rejected chain is returned. The earlier chain remains in flight.
1407        assert_eq!(producer.poll().unwrap().unwrap().token(), rejected);
1408        assert!(producer.poll().unwrap().is_none());
1409
1410        drop(consumer);
1411        // SAFETY: The peer is stopped and all consumer chain handles are dropped.
1412        unsafe { producer.reset().unwrap() };
1413        assert_eq!(producer.pool().num_live(), 0);
1414    }
1415
1416    #[test]
1417    fn test_poll_batch_rejects_out_of_range_id() {
1418        let ring = make_ring(16);
1419        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1420        let chain = producer.chain().writable(16).build().unwrap();
1421        producer.submit(chain).unwrap();
1422        let mut desc = ring.read_desc(0);
1423        desc.id = 16;
1424        ring.write_desc(0, desc);
1425
1426        // An untrusted ID must be rejected before it enters local bookkeeping.
1427        assert!(matches!(
1428            consumer.poll_batch(1, 0, 0),
1429            Err(VirtqError::RingError(RingError::InvalidState))
1430        ));
1431        assert_eq!(consumer.avail_cursor().head(), 0);
1432        assert_eq!(consumer.used_cursor().head(), 0);
1433        assert_eq!(consumer.inner.num_inflight(), 0);
1434        assert_eq!(consumer.next_token, 0);
1435        assert!(producer.poll().unwrap().is_none());
1436
1437        drop(consumer);
1438        // SAFETY: The peer is stopped and all consumer chain handles are dropped.
1439        unsafe { producer.reset().unwrap() };
1440        assert_eq!(producer.pool().num_live(), 0);
1441    }
1442
1443    #[test]
1444    fn test_poll_batch_rejects_duplicate_id_after_partial_batch() {
1445        let ring = make_ring(16);
1446        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1447        let chain = producer.chain().writable(16).build().unwrap();
1448        let first = producer.submit(chain).unwrap();
1449        let chain = producer.chain().writable(16).build().unwrap();
1450        producer.submit(chain).unwrap();
1451        let mut desc = ring.read_desc(1);
1452        desc.id = first.id;
1453        ring.write_desc(1, desc);
1454
1455        // A duplicate ID is a ring error, not an availability shortage.
1456        assert!(matches!(
1457            consumer.poll_batch(2, 0, 0),
1458            Err(VirtqError::RingError(RingError::InvalidState))
1459        ));
1460        assert_eq!(consumer.avail_cursor().head(), 1);
1461        assert_eq!(consumer.used_cursor().head(), 0);
1462        assert_eq!(consumer.inner.num_inflight(), 1);
1463        assert_eq!(consumer.next_token, 1);
1464        assert!(producer.poll().unwrap().is_none());
1465
1466        drop(consumer);
1467        // SAFETY: The peer is stopped and all consumer chain handles are dropped.
1468        unsafe { producer.reset().unwrap() };
1469        assert_eq!(producer.pool().num_live(), 0);
1470    }
1471
1472    #[test]
1473    fn test_poll_too_large_returns_payload_error() {
1474        let ring = make_ring(16);
1475        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1476
1477        let mut se = producer.chain().readable(8).writable(16).build().unwrap();
1478        se.write_all(b"too much").unwrap();
1479        producer.submit(se).unwrap();
1480
1481        assert!(matches!(
1482            consumer.poll(4),
1483            Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1484        ));
1485        producer.drain(drop).unwrap();
1486    }
1487
1488    #[test]
1489    fn test_poll_too_large_returns_descriptor() {
1490        let ring = make_ring(16);
1491        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1492
1493        let mut se = producer.chain().readable(8).writable(16).build().unwrap();
1494        se.write_all(b"too much").unwrap();
1495        let token = producer.submit(se).unwrap();
1496
1497        // Oversized payload is rejected, but the descriptor must be returned to
1498        // the driver so the ring slot is not leaked.
1499        assert!(matches!(
1500            consumer.poll(4),
1501            Err(VirtqError::PayloadTooLarge { recv: 8, limit: 4 })
1502        ));
1503
1504        // The producer can reclaim the rejected chain; the queue is not wedged.
1505        let used = producer.poll().unwrap().unwrap();
1506        assert_eq!(used.token(), token);
1507
1508        // A subsequent normal exchange still round-trips end to end.
1509        let se2 = producer.chain().writable(16).build().unwrap();
1510        producer.submit(se2).unwrap();
1511        let (recv, reply) = poll_data(&mut consumer);
1512        consumer.complete(recv, reply).unwrap();
1513        assert!(producer.poll().unwrap().is_some());
1514    }
1515
1516    #[test]
1517    fn test_villain_indirect_descriptor_does_not_mark_high_level_inflight() {
1518        let ring = make_ring(16);
1519        let mem = ring.mem();
1520        let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1521
1522        let mut desc = Descriptor::new(0x1000, 16, 0, DescFlags::INDIRECT);
1523        desc.mark_avail(true);
1524        ring.write_desc(0, desc);
1525
1526        assert!(matches!(
1527            consumer.poll(1024),
1528            Err(VirtqError::RingError(RingError::BadChain))
1529        ));
1530        assert_eq!(consumer.inner.num_inflight(), 0);
1531    }
1532
1533    #[test]
1534    fn test_villain_bad_chain_does_not_mark_high_level_inflight() {
1535        let ring = make_ring(16);
1536        let mem = ring.mem();
1537        let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1538
1539        let mut first = Descriptor::new(0x1000, 16, 0, DescFlags::NEXT | DescFlags::WRITE);
1540        first.mark_avail(true);
1541        ring.write_desc(0, first);
1542
1543        let mut second = Descriptor::new(0x2000, 16, 0, DescFlags::empty());
1544        second.mark_avail(true);
1545        ring.write_desc(1, second);
1546
1547        assert!(matches!(
1548            consumer.poll(1024),
1549            Err(VirtqError::RingError(RingError::BadChain))
1550        ));
1551        assert_eq!(consumer.inner.num_inflight(), 0);
1552    }
1553
1554    #[test]
1555    fn test_writable_chain_writes_single_segment() {
1556        let ring = make_ring(16);
1557        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1558
1559        let se = producer.chain().writable(16).build().unwrap();
1560        producer.submit(se).unwrap();
1561        let (recv, reply) = poll_data(&mut consumer);
1562
1563        let ReplyChain::Writable(mut wc) = reply else {
1564            panic!("expected Writable");
1565        };
1566        wc.write_all(b"hello").unwrap();
1567        consumer.complete(recv, wc).unwrap();
1568
1569        let used = producer.poll().unwrap().unwrap();
1570        assert_eq!(used.to_bytes().unwrap().as_ref(), b"hello");
1571    }
1572
1573    #[test]
1574    fn test_writable_rewind() {
1575        let ring = make_ring(16);
1576        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 16);
1577
1578        let se = producer.chain().writable(16).build().unwrap();
1579        producer.submit(se).unwrap();
1580
1581        let (recv, reply) = poll_data(&mut consumer);
1582
1583        if let ReplyChain::Writable(mut wc) = reply {
1584            wc.write_all(b"first").unwrap();
1585            assert_eq!(wc.written(), 5);
1586            wc.rewind();
1587            assert_eq!(wc.written(), 0);
1588            assert_eq!(wc.remaining(), 16);
1589            wc.write_all(b"second").unwrap();
1590            assert_eq!(wc.written(), 6);
1591            consumer.complete(recv, wc).unwrap();
1592        } else {
1593            panic!("expected Writable");
1594        }
1595        producer.drain(drop).unwrap();
1596    }
1597
1598    #[test]
1599    fn test_writable_reply_scatters_across_segments() {
1600        let ring = make_ring(16);
1601        let mem = ring.mem();
1602        let mut ring_producer = make_producer(&ring);
1603        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1604
1605        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1606        let chain = BufferChainBuilder::new()
1607            .writable(base, 4)
1608            .writable(base + 4, 4)
1609            .build()
1610            .unwrap();
1611        let id = ring_producer.submit_available(&chain).unwrap();
1612
1613        let (recv, reply) = poll_data(&mut consumer);
1614        assert!(recv.to_bytes().unwrap().is_empty());
1615
1616        let ReplyChain::Writable(mut wc) = reply else {
1617            panic!("expected Writable");
1618        };
1619        assert_eq!(wc.capacity(), 8);
1620        wc.write_all(b"abcdefgh").unwrap();
1621        assert_eq!(wc.written(), 8);
1622        consumer.complete(recv, wc).unwrap();
1623
1624        let mut first = [0u8; 4];
1625        let mut second = [0u8; 4];
1626        mem.read(base, &mut first).unwrap();
1627        mem.read(base + 4, &mut second).unwrap();
1628        assert_eq!(&first, b"abcd");
1629        assert_eq!(&second, b"efgh");
1630
1631        let used = ring_producer.poll_used().unwrap();
1632        assert_eq!(used.id, id);
1633        assert_eq!(used.len, 8);
1634    }
1635
1636    #[test]
1637    fn test_writable_short_write_reports_contiguous_used_length() {
1638        let ring = make_ring(16);
1639        let mem = ring.mem();
1640        let mut ring_producer = make_producer(&ring);
1641        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1642        mem.write(base, &[0xff; 8]).unwrap();
1643
1644        let chain = BufferChainBuilder::new()
1645            .writable(base, 4)
1646            .writable(base + 4, 4)
1647            .build()
1648            .unwrap();
1649        let id = ring_producer.submit_available(&chain).unwrap();
1650
1651        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1652        let (recv, reply) = poll_data(&mut consumer);
1653        let ReplyChain::Writable(mut writable) = reply else {
1654            panic!("expected writable reply");
1655        };
1656
1657        writable.write_all(b"abc").unwrap();
1658        writable.write_all(b"def").unwrap();
1659        assert_eq!(writable.written(), 6);
1660        assert_eq!(writable.remaining(), 2);
1661
1662        consumer.complete(recv, writable).unwrap();
1663
1664        let mut contents = [0u8; 8];
1665        mem.read(base, &mut contents).unwrap();
1666        assert_eq!(&contents, b"abcdef\xff\xff");
1667
1668        let used = ring_producer.poll_used().unwrap();
1669        assert_eq!(used.id, id);
1670        assert_eq!(used.len, 6);
1671    }
1672
1673    #[test]
1674    fn test_multiple_pending_replies() {
1675        let ring = make_ring(16);
1676        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1677
1678        let se1 = producer.chain().writable(16).build().unwrap();
1679        producer.submit(se1).unwrap();
1680        let se2 = producer.chain().writable(16).build().unwrap();
1681        producer.submit(se2).unwrap();
1682
1683        let (e1, c1) = poll_data(&mut consumer);
1684        let (e2, c2) = poll_data(&mut consumer);
1685
1686        // Complete in reverse order
1687        consumer.complete(e2, c2).unwrap();
1688        consumer.complete(e1, c1).unwrap();
1689        producer.drain(drop).unwrap();
1690    }
1691
1692    #[test]
1693    fn test_mismatched_completion_fails_closed() {
1694        let ring = make_ring(16);
1695        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1696
1697        let mut first = producer.chain().readable(1).build().unwrap();
1698        first.write_all(b"a").unwrap();
1699        producer.submit(first).unwrap();
1700
1701        let mut second = producer.chain().readable(1).build().unwrap();
1702        second.write_all(b"b").unwrap();
1703        producer.submit(second).unwrap();
1704
1705        let (recv1, reply1) = poll_data(&mut consumer);
1706        let (recv2, reply2) = poll_data(&mut consumer);
1707
1708        assert!(matches!(
1709            consumer.complete(recv1, reply2),
1710            Err(VirtqError::InvalidState)
1711        ));
1712        assert_eq!(consumer.inner.num_inflight(), 2);
1713        assert!(producer.poll().unwrap().is_none());
1714        assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1715
1716        drop((recv2, reply1));
1717
1718        // SAFETY: All consumer handles were dropped. The consumer stays inactive.
1719        unsafe { producer.reset() }.unwrap();
1720    }
1721
1722    #[test]
1723    fn test_recv_to_bytes_preserves_reader_position() {
1724        let ring = make_ring(16);
1725        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1726
1727        let mut se = producer.chain().readable(16).build().unwrap();
1728        se.write_all(b"abc").unwrap();
1729        producer.submit(se).unwrap();
1730
1731        let (mut recv, reply) = poll_data(&mut consumer);
1732        let mut first = [0u8; 1];
1733        recv.read_exact(&mut first).unwrap();
1734        let data = recv.to_bytes().unwrap();
1735        assert_eq!(&first, b"a");
1736        assert_eq!(data.as_ref(), b"abc");
1737        assert_eq!(recv.consumed(), 1);
1738        consumer.complete(recv, reply).unwrap();
1739        producer.drain(drop).unwrap();
1740    }
1741
1742    #[test]
1743    fn test_virtq_consumer_reset() {
1744        let ring = make_ring(16);
1745        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1746
1747        // Submit and poll (but do not complete)
1748        let se = producer.chain().writable(16).build().unwrap();
1749        producer.submit(se).unwrap();
1750
1751        let (recv, reply) = poll_data(&mut consumer);
1752        assert!(consumer.inner.num_inflight() > 0);
1753        assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1754
1755        // Complete first so we do not leak
1756        consumer.complete(recv, reply).unwrap();
1757
1758        consumer.reset().unwrap();
1759
1760        assert_eq!(consumer.inner.num_inflight(), 0);
1761
1762        // SAFETY: The consumer completed all handles, reset, and stays inactive.
1763        unsafe { producer.reset() }.unwrap();
1764    }
1765
1766    #[test]
1767    fn test_virtq_consumer_reset_clears_inflight() {
1768        let ring = make_ring(16);
1769        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1770
1771        // Submit two entries and poll both
1772        let se1 = producer.chain().writable(16).build().unwrap();
1773        producer.submit(se1).unwrap();
1774        let se2 = producer.chain().writable(16).build().unwrap();
1775        producer.submit(se2).unwrap();
1776
1777        let (e1, c1) = poll_data(&mut consumer);
1778        let (e2, c2) = poll_data(&mut consumer);
1779        // Complete both before reset
1780        consumer.complete(e1, c1).unwrap();
1781        consumer.complete(e2, c2).unwrap();
1782
1783        consumer.reset().unwrap();
1784
1785        assert_eq!(consumer.inner.num_inflight(), 0);
1786
1787        // SAFETY: The consumer completed all handles, reset, and stays inactive.
1788        unsafe { producer.reset() }.unwrap();
1789    }
1790
1791    #[test]
1792    fn failed_completion_keeps_reset_blocked() {
1793        use crate::virtq::ring::tests::FaultMem;
1794
1795        let ring = make_ring(4);
1796        let (mut producer, _, _) = make_test_producer(&ring);
1797        let mem = FaultMem::new(ring.mem());
1798        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), TestNotifier::new());
1799        let chain = producer.chain().writable(8).build().unwrap();
1800        producer.submit(chain).unwrap();
1801        let (recv, reply) = consumer.poll(0).unwrap().unwrap();
1802
1803        mem.fail_write_at(0);
1804        assert!(matches!(
1805            consumer.complete(recv, reply),
1806            Err(VirtqError::RingError(RingError::MemError { .. }))
1807        ));
1808        mem.allow_writes();
1809        assert_eq!(consumer.inner.num_inflight(), 1);
1810        assert!(matches!(consumer.reset(), Err(VirtqError::InvalidState)));
1811
1812        // SAFETY: Failed completion dropped both handles. The consumer stays inactive.
1813        unsafe { producer.reset() }.unwrap();
1814    }
1815}