Skip to main content

hyperlight_common/virtq/
producer.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4use alloc::collections::VecDeque;
5use alloc::vec;
6use alloc::vec::Vec;
7use core::mem::ManuallyDrop;
8
9use bytes::Bytes;
10use smallvec::SmallVec;
11
12use super::*;
13
14/// A used chain observed by the driver (producer) side.
15///
16/// Read-only chains are returned as [`Ack`](Self::Ack). Chains with a writable
17/// buffer complete as [`Data`](Self::Data), even when the device wrote zero
18/// bytes. Non-empty segments use backend-owned mappings. Borrowed mappings
19/// keep their pool slots allocated until the last [`Bytes`] owner drops.
20#[derive(Debug)]
21pub enum UsedChain {
22    /// Acknowledgement for a read-only/fire-and-forget chain.
23    Ack(Token),
24    /// Data written by the consumer into the chain's writable buffers.
25    ///
26    /// The payload may contain zero bytes when the consumer writes zero bytes;
27    /// that is still a data used chain because the submitted chain had
28    /// writable capacity.
29    Data(Token, Segments),
30}
31
32impl UsedChain {
33    /// Token identifying which submitted chain this used chain corresponds to.
34    pub fn token(&self) -> Token {
35        match self {
36            Self::Ack(token) | Self::Data(token, _) => *token,
37        }
38    }
39
40    /// Data written by the consumer as contiguous bytes, if this chain has data.
41    pub fn to_bytes(&self) -> Option<Bytes> {
42        match self {
43            Self::Ack(_) => None,
44            Self::Data(_, segments) => Some(segments.to_bytes()),
45        }
46    }
47
48    /// Segments written by the consumer, if this chain has data.
49    pub fn segments(&self) -> Option<&Segments> {
50        match self {
51            Self::Ack(_) => None,
52            Self::Data(_, segments) => Some(segments),
53        }
54    }
55
56    /// Consume the used chain and return written data as contiguous bytes, if present.
57    pub fn into_bytes(self) -> Option<Bytes> {
58        match self {
59            Self::Ack(_) => None,
60            Self::Data(_, segments) => Some(segments.into_bytes()),
61        }
62    }
63
64    /// Consume the used chain and return written segments, if present.
65    pub fn into_segments(self) -> Option<Segments> {
66        match self {
67            Self::Ack(_) => None,
68            Self::Data(_, segments) => Some(segments),
69        }
70    }
71}
72
73struct Inflight<M> {
74    token: Token,
75    // Automatic cleanup could recycle buffers still accessible to the peer.
76    // Only completion or a stopped reset permits releasing this owner.
77    chain: ManuallyDrop<OwnedChain<M>>,
78}
79
80/// Compact in-flight chains with constant-time descriptor-ID lookup.
81///
82/// Descriptor IDs span the full ring, but live chains are normally bounded by
83/// the much smaller buffer pool. `by_id` maps each descriptor ID to a packed
84/// `live` index. Removal uses `swap_remove` and repairs the moved entry's map.
85struct InflightTable<M> {
86    by_id: Vec<u16>,
87    live: Vec<Inflight<M>>,
88}
89
90impl<M> InflightTable<M> {
91    const VACANT: u16 = u16::MAX;
92
93    fn new(ring_len: usize) -> Self {
94        Self {
95            by_id: vec![Self::VACANT; ring_len],
96            live: Vec::new(),
97        }
98    }
99
100    fn try_reserve_one(&mut self) -> Result<(), VirtqError> {
101        if self.live.len() > self.by_id.len() {
102            return Err(VirtqError::InvalidState);
103        }
104
105        if self.live.len() == self.by_id.len() {
106            return Err(VirtqError::Backpressure);
107        }
108
109        // Producers with one live chain should not pay for four large inline
110        // chain records.
111        let result = if self.live.capacity() == 0 {
112            self.live.try_reserve_exact(1)
113        } else {
114            self.live.try_reserve(1)
115        };
116
117        result.map_err(|_| VirtqError::Bookkeeping)
118    }
119
120    fn contains(&self, id: u16) -> bool {
121        self.by_id
122            .get(id as usize)
123            .is_some_and(|slot| *slot != Self::VACANT)
124    }
125
126    fn insert(&mut self, inflight: Inflight<M>) {
127        let id = inflight.token.id;
128        debug_assert!(!self.contains(id));
129        debug_assert!(self.live.len() < Self::VACANT as usize);
130
131        let slot = self.live.len() as u16;
132        self.live.push(inflight);
133        self.by_id[id as usize] = slot;
134    }
135
136    fn remove(&mut self, id: u16) -> Option<Inflight<M>> {
137        let slot = self.by_id.get_mut(id as usize)?;
138        if *slot == Self::VACANT {
139            return None;
140        }
141
142        let index = usize::from(*slot);
143        *slot = Self::VACANT;
144
145        let removed = self.live.swap_remove(index);
146        if let Some(moved) = self.live.get(index) {
147            self.by_id[moved.token.id as usize] = index as u16;
148        }
149
150        Some(removed)
151    }
152
153    fn pop(&mut self) -> Option<Inflight<M>> {
154        let inflight = self.live.pop()?;
155        self.by_id[inflight.token.id as usize] = Self::VACANT;
156        Some(inflight)
157    }
158}
159
160/// A high-level virtqueue producer (driver side).
161///
162/// The producer sends chains to the consumer (device), and receives used chains.
163/// This is used on the driver/guest side.
164///
165/// # Threading
166///
167/// The producer and its pool use single-threaded allocation state.
168/// [`BufferMap`] supplies the complete `Send` owner for returned [`Bytes`].
169///
170/// # Cleanup
171///
172/// Dropping a producer does not stop its peer, so inflight owners are
173/// deliberately leaked to avoid freeing buffers the peer may still use.
174/// Drain completions for all submitted chains, or stop the peer and call
175/// [`reset`](Self::reset), before dropping the producer.
176///
177/// # Example
178///
179/// ```ignore
180/// let mut producer = VirtqProducer::new(layout, mem, notifier, pool);
181///
182/// // Build and submit a chain
183/// let mut chain = producer.chain().readable(64).writable(64).build()?;
184/// chain.write_all(b"hello")?;
185/// let token = producer.submit(chain)?;
186///
187/// // Later, poll for the used chain
188/// if let Some(used) = producer.poll()? {
189///     assert_eq!(used.token(), token);
190///     match used {
191///         UsedChain::Data(_, segments) => println!("Got used chain: {:?}", segments),
192///         UsedChain::Ack(_) => println!("Got ack"),
193///     }
194/// }
195/// ```
196pub struct VirtqProducer<M, N> {
197    inner: RingProducer<M>,
198    notifier: N,
199    pool: SlotPool,
200    next_token: u32,
201    inflight: InflightTable<M>,
202    pending: VecDeque<UsedChain>,
203}
204
205impl<M, N> VirtqProducer<M, N>
206where
207    M: MemOps + Clone,
208    N: Notifier,
209{
210    /// Create a new virtqueue producer.
211    ///
212    /// # Arguments
213    ///
214    /// * `layout` - Ring memory layout (descriptor table and event suppression addresses)
215    /// * `mem` - Memory operations implementation for reading/writing to shared memory
216    /// * `notifier` - Callback for notifying the device (consumer) about new chains
217    /// * `pool` - Buffer allocator for chain payload and reply data
218    pub fn new(layout: Layout, mem: M, notifier: N, pool: SlotPool) -> Self {
219        let inner = RingProducer::new(layout, mem);
220        let ring_len = inner.len();
221        let inflight = InflightTable::new(ring_len);
222
223        Self {
224            inner,
225            pool,
226            notifier,
227            inflight,
228            next_token: 0,
229            pending: VecDeque::new(),
230        }
231    }
232
233    /// Borrow the pool used for new chains.
234    pub fn pool(&self) -> &SlotPool {
235        &self.pool
236    }
237
238    /// Borrow the backend used for ring access and new chains.
239    pub fn memory(&self) -> &M {
240        self.inner.mem()
241    }
242
243    /// Reclaim retained scratch slots while preserving in-flight chains.
244    ///
245    /// # Safety
246    ///
247    /// Every live allocation outside this producer's in-flight chains must
248    /// belong to a returned view with backing independent of reused scratch.
249    /// No unsent chains or builders may remain. Peer accesses must be stopped.
250    pub unsafe fn reclaim_slots(&mut self) -> Result<(), VirtqError> {
251        if !self.pending.is_empty() {
252            return Err(VirtqError::InvalidState);
253        }
254
255        let reserved = self
256            .inflight
257            .live
258            .iter()
259            .flat_map(|inflight| inflight.chain.buffers.iter().map(|buffer| buffer.addr));
260
261        // SAFETY: In-flight records preserve all non-retired allocations.
262        // The caller guarantees independent backing and excludes untracked chains.
263        unsafe { self.pool.retire_generation(reserved)? };
264        Ok(())
265    }
266
267    /// Begin building a descriptor chain for submission.
268    ///
269    /// The builder captures the current free-descriptor budget.
270    /// Submission checks capacity again.
271    pub fn chain(&self) -> ChainBuilder<M> {
272        ChainBuilder::new(
273            self.inner.mem().clone(),
274            self.pool.clone(),
275            self.inner.num_free(),
276        )
277    }
278
279    /// Preferred size of one bulk payload segment.
280    pub fn preferred_segment_len(&self) -> usize {
281        self.pool.slot_size()
282    }
283
284    /// Begin a batch of submissions.
285    ///
286    /// Chains submitted through the returned [`SubmitBatch`] are published to
287    /// the ring immediately, but the consumer is notified at most once when
288    /// [`SubmitBatch::finish`] is called. This mirrors the virtio pattern of
289    /// adding multiple buffers and then kicking the queue once.
290    pub fn batch(&mut self) -> SubmitBatch<'_, M, N> {
291        SubmitBatch::new(self)
292    }
293
294    /// Submit a [`SendChain`] to the ring.
295    ///
296    /// Publishes the descriptor chain, stores the in-flight tracking state,
297    /// and notifies the consumer if event suppression allows. Notifications
298    /// are layout-neutral; use [`batch`](Self::batch) when a higher-level
299    /// protocol wants to publish multiple chains and kick once.
300    ///
301    /// # Errors
302    ///
303    /// * [`VirtqError::Backpressure`] - ring or in-flight tracking is full
304    /// * [`VirtqError::RingError`] - publication or notification memory access failed
305    /// * [`VirtqError::InvalidState`] - descriptor ID collision
306    pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
307        let cursor_before = self.inner.avail_cursor();
308        let token = self.publish(chain)?;
309        self.notify_since(cursor_before)?;
310        Ok(token)
311    }
312
313    fn publish(&mut self, send: SendChain<M>) -> Result<Token, VirtqError> {
314        self.inflight.try_reserve_one()?;
315
316        if send.desc_count() > self.inner.num_free() {
317            return Err(VirtqError::Backpressure);
318        }
319        let token_id = self.next_token;
320        let id = self.inner.next_id()?;
321        let token = Token { seq: token_id, id };
322
323        // A free descriptor id must never already be tracked as inflight.
324        if self.inflight.contains(id) {
325            return Err(VirtqError::InvalidState);
326        }
327
328        let mut descriptors = send.owned.descriptors();
329        let mut builder = BufferChainBuilder::new();
330
331        builder.reserve_exact(send.desc_count());
332        let chain = builder
333            .readables(descriptors.by_ref().take(send.rd_desc_count()))
334            .writables(descriptors)
335            .build()?;
336
337        let published = self.inner.submit_available(&chain);
338
339        // A failed write can still publish descriptors. Keep their allocations
340        // until a stopped reset, including when submission reports an error.
341        let inf = Inflight {
342            token,
343            chain: ManuallyDrop::new(send.owned),
344        };
345        self.inflight.insert(inf);
346        self.next_token = self.next_token.wrapping_add(1);
347
348        let published_id = published?;
349        debug_assert_eq!(published_id, id);
350        Ok(token)
351    }
352
353    fn notify_since(&mut self, cursor: RingCursor) -> Result<bool, VirtqError> {
354        let should_notify = self.inner.should_notify_since(cursor)?;
355        if should_notify {
356            self.notify_now();
357        }
358        Ok(should_notify)
359    }
360
361    fn notify_now(&self) {
362        self.notifier.notify(QueueStats {
363            num_free: self.inner.num_free(),
364            num_inflight: self.inner.num_inflight(),
365        });
366    }
367
368    /// Signal backpressure to the consumer.
369    ///
370    /// Bypasses event suppression. Call this when submit fails with a
371    /// backpressure error and the consumer needs to drain.
372    #[inline]
373    pub fn notify_backpressure(&self) {
374        self.notify_now();
375    }
376
377    /// Get the current used cursor position.
378    ///
379    /// Useful for setting up descriptor-based event suppression.
380    #[inline]
381    pub fn used_cursor(&self) -> RingCursor {
382        self.inner.used_cursor()
383    }
384
385    /// Number of free (unsubmitted) descriptors in the ring.
386    #[inline]
387    pub fn num_free(&self) -> usize {
388        self.inner.num_free()
389    }
390
391    /// Number of submitted descriptors not yet polled as used.
392    #[inline]
393    pub fn num_inflight(&self) -> usize {
394        self.inner.num_inflight()
395    }
396
397    /// Reset a stopped producer and release transport-owned allocations.
398    ///
399    /// Buffered writable completions are guest-owned and make this operation fail.
400    /// Owner-backed payloads already returned to callers are not tracked as
401    /// in-flight and remain allocated.
402    ///
403    /// # Safety
404    ///
405    /// All consumer-side chain handles must be dropped. The peer must stay
406    /// stopped until its consumer is reset or replaced. Surviving handles
407    /// could access recycled buffers, violating ownership of borrowed views.
408    ///
409    /// ```compile_fail,E0133
410    /// # use hyperlight_common::virtq::{MemOps, Notifier, VirtqProducer};
411    /// fn reset<M: MemOps + Clone, N: Notifier>(producer: &mut VirtqProducer<M, N>) {
412    ///     producer.reset().unwrap();
413    /// }
414    /// ```
415    pub unsafe fn reset(&mut self) -> Result<(), VirtqError> {
416        if !self.pending.is_empty() {
417            return Err(VirtqError::InvalidState);
418        }
419
420        self.inner.reset()?;
421        self.next_token = 0;
422
423        let mut maybe_err = None;
424
425        while let Some(inflight) = self.inflight.pop() {
426            let ret = ManuallyDrop::into_inner(inflight.chain).release();
427            if let Err(err) = ret
428                && maybe_err.is_none()
429            {
430                maybe_err = Some(err);
431            }
432        }
433
434        match maybe_err {
435            Some(error) => Err(error.into()),
436            None => Ok(()),
437        }
438    }
439
440    /// Configure event suppression for used buffer notifications.
441    ///
442    /// This controls when the device (consumer) signals us about completed buffers:
443    ///
444    /// - [`SuppressionKind::Enable`]: Always signal (default) - good for latency
445    /// - [`SuppressionKind::Disable`]: Never signal - caller must poll
446    /// - [`SuppressionKind::Descriptor`]: Signal only at specific cursor position
447    ///
448    /// # Example: Used-chain batching
449    ///
450    /// ```ignore
451    /// // Submit chains, then suppress notifications until all are used
452    /// let mut se = producer.chain().readable(64).writable(128).build()?;
453    /// se.write_all(b"entry1")?;
454    /// producer.submit(se)?;
455    /// let cursor = producer.used_cursor();
456    /// producer.set_used_suppression(SuppressionKind::Descriptor(cursor))?;
457    /// // Device will notify only after reaching that cursor position
458    /// ```
459    pub fn set_used_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
460        match kind {
461            SuppressionKind::Enable => self.inner.enable_used_notifications()?,
462            SuppressionKind::Disable => self.inner.disable_used_notifications()?,
463            SuppressionKind::Descriptor(cursor) => self
464                .inner
465                .enable_used_notifications_desc(cursor.head(), cursor.wrap())?,
466        }
467        Ok(())
468    }
469}
470
471impl<M, N> VirtqProducer<M, N>
472where
473    M: BufferMap + Clone,
474    N: Notifier,
475{
476    /// Poll for a single used chain from the device.
477    ///
478    /// Returns buffered used chains from prior [`reclaim`](Self::reclaim)
479    /// calls first, then checks the ring for newly used chains.
480    ///
481    /// Returns `Ok(Some(used))` if a used chain is available, `Ok(None)` if no
482    /// used chains are ready (would block), or an error if the device misbehaved.
483    ///
484    /// Data used chains contain [`Bytes`] owned by the [`BufferMap`] backend.
485    /// Borrowed views retain their pool allocations until the last clone drops.
486    ///
487    /// # Errors
488    ///
489    /// - [`VirtqError::InvalidState`] - Device returned invalid descriptor ID or
490    ///   wrote more data than the writable buffer capacity
491    pub fn poll(&mut self) -> Result<Option<UsedChain>, VirtqError> {
492        if let Some(chain) = self.pending.pop_front() {
493            return Ok(Some(chain));
494        }
495        self.poll_ring()
496    }
497
498    /// Reclaim ring slots and pool allocations from used descriptors.
499    ///
500    /// Processes all available used chains from the ring: frees readable
501    /// buffer allocations immediately, and buffers writable data for
502    /// later retrieval via [`poll`](Self::poll).
503    ///
504    /// Read-only ack used chains are discarded immediately.
505    ///
506    /// Use this to free resources under backpressure without losing
507    /// writable data. Returns the number of chains reclaimed.
508    /// Buffered completions can span multiple ring reuse cycles.
509    pub fn reclaim(&mut self) -> Result<usize, VirtqError> {
510        let mut count = 0;
511        while let Some(chain) = self.poll_ring()? {
512            if matches!(chain, UsedChain::Data(_, _)) {
513                self.pending
514                    .try_reserve(1)
515                    .map_err(|_| VirtqError::Bookkeeping)?;
516                self.pending.push_back(chain);
517            }
518            count += 1;
519        }
520        Ok(count)
521    }
522
523    /// Poll one used chain directly from the ring (bypassing pending buffer).
524    fn poll_ring(&mut self) -> Result<Option<UsedChain>, VirtqError> {
525        let used = match self.inner.poll_used() {
526            Ok(u) => u,
527            Err(RingError::WouldBlock) => return Ok(None),
528            Err(e) => return Err(e.into()),
529        };
530
531        let inf = self
532            .inflight
533            .remove(used.id)
534            .ok_or(VirtqError::InvalidState)?;
535
536        let written = used.len as usize;
537        let Inflight { token, chain } = inf;
538
539        let chain = ManuallyDrop::into_inner(chain);
540        let used = if chain.readable == chain.buffers.len() {
541            chain.release()?;
542            UsedChain::Ack(token)
543        } else {
544            UsedChain::Data(token, chain.into_segments(written)?)
545        };
546
547        Ok(Some(used))
548    }
549
550    /// Drain all available used chains, calling the provided closure for each.
551    ///
552    /// This is a convenience method that repeatedly calls [`poll`](Self::poll)
553    /// until no more used chains are available.
554    ///
555    /// # Arguments
556    ///
557    /// * `f` - Closure called for each used chain
558    ///
559    /// # Example
560    ///
561    /// ```ignore
562    /// producer.drain(|used| {
563    ///     println!("Got used chain for {:?}", used.token());
564    /// })?;
565    /// ```
566    pub fn drain(&mut self, mut f: impl FnMut(UsedChain)) -> Result<(), VirtqError> {
567        while let Some(chain) = self.poll()? {
568            f(chain);
569        }
570
571        Ok(())
572    }
573}
574
575/// A scoped batch of producer submissions.
576///
577/// Submissions are published immediately, while notification is delayed until
578/// [`finish`](Self::finish). [`finish_without_notify`](Self::finish_without_notify)
579/// supports protocols whose peer is already scheduled to inspect the queue.
580#[must_use = "finish the batch explicitly"]
581pub struct SubmitBatch<'a, M, N> {
582    producer: &'a mut VirtqProducer<M, N>,
583    notify_from: Option<RingCursor>,
584}
585
586impl<'a, M, N> SubmitBatch<'a, M, N>
587where
588    M: MemOps + Clone,
589    N: Notifier,
590{
591    fn new(producer: &'a mut VirtqProducer<M, N>) -> Self {
592        Self {
593            producer,
594            notify_from: None,
595        }
596    }
597
598    /// Begin building a descriptor chain for this batch.
599    pub fn chain(&self) -> ChainBuilder<M> {
600        self.producer.chain()
601    }
602
603    /// Publish a chain as part of this batch without notifying yet.
604    pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
605        let cursor_before = self.producer.inner.avail_cursor();
606        let token = self.producer.publish(chain)?;
607
608        if self.notify_from.is_none() {
609            self.notify_from = Some(cursor_before);
610        }
611        Ok(token)
612    }
613
614    /// Finish the batch and notify the consumer once if event suppression
615    /// requires it for the whole published range.
616    ///
617    /// Returns `true` if a notification was sent.
618    pub fn finish(mut self) -> Result<bool, VirtqError> {
619        let Some(notify_from) = self.notify_from.take() else {
620            return Ok(false);
621        };
622
623        self.producer.notify_since(notify_from)
624    }
625
626    /// Finish the batch without notifying the consumer.
627    ///
628    /// Use this only when another protocol event guarantees that the consumer
629    /// will inspect the published descriptors.
630    pub fn finish_without_notify(self) {}
631}
632
633/// Builder for configuring a descriptor chain's buffer layout.
634///
635/// If dropped without building, no resources are leaked (allocations are
636/// deferred to [`build`](Self::build)).
637#[must_use = "call .build() to create a SendChain"]
638pub struct ChainBuilder<M: MemOps> {
639    mem: M,
640    pool: SlotPool,
641    rd_caps: SmallVec<[usize; 4]>,
642    wr_caps: SmallVec<[usize; 4]>,
643    writable_avail: bool,
644    max_descs: usize,
645}
646
647impl<M: MemOps> ChainBuilder<M> {
648    fn new(mem: M, pool: SlotPool, max_descs: usize) -> Self {
649        Self {
650            mem,
651            pool,
652            rd_caps: SmallVec::new(),
653            wr_caps: SmallVec::new(),
654            writable_avail: false,
655            max_descs,
656        }
657    }
658
659    /// Request a device-readable buffer of `cap` bytes.
660    ///
661    /// The producer writes data into readable buffers before submission; the
662    /// consumer reads that data after polling the chain.
663    /// The actual allocation is deferred to [`build`](Self::build).
664    pub fn readable(mut self, cap: usize) -> Self {
665        self.rd_caps.push(cap);
666        self
667    }
668
669    /// Request a device-writable buffer of `cap` bytes.
670    ///
671    /// The writable buffer is filled by the consumer and returned via
672    /// [`VirtqProducer::poll`] as [`UsedChain`].
673    ///
674    /// Multiple writable buffers are completed as ordered [`Segments`]. The
675    /// consumer writes them sequentially, because the virtio used ring reports
676    /// one aggregate written length rather than per-descriptor lengths.
677    pub fn writable(mut self, cap: usize) -> Self {
678        self.wr_caps.push(cap);
679        self
680    }
681
682    /// Request available upper-tier buffers within the ring budget.
683    ///
684    /// [`build`](Self::build) allocates these after all explicit requests.
685    /// This may add zero buffers. The complete chain must be nonempty.
686    pub fn writable_avail(mut self) -> Self {
687        self.writable_avail = true;
688        self
689    }
690
691    /// Allocate buffers and return a [`SendChain`] for writing.
692    ///
693    /// # Errors
694    ///
695    /// * [`VirtqError::InvalidState`] - no buffers requested
696    /// * [`VirtqError::Backpressure`] - insufficient pool slots or ring descriptors
697    /// * [`VirtqError::Bookkeeping`] - buffer record storage allocation failed
698    /// * [`VirtqError::Alloc`] - zero-length request or buffer allocation failed
699    pub fn build(self) -> Result<SendChain<M>, VirtqError> {
700        if self.rd_caps.is_empty() && self.wr_caps.is_empty() && !self.writable_avail {
701            return Err(VirtqError::InvalidState);
702        }
703
704        // Count explicit descriptors against the captured ring budget.
705        let slot_size = self.pool.slot_size();
706        let rd_capacity = self.rd_caps.iter().try_fold(0usize, |total, &cap| {
707            total.checked_add(cap).ok_or(AllocError::Overflow)
708        })?;
709
710        let mut caps = self.rd_caps.iter().chain(&self.wr_caps);
711        let desc_count = caps.try_fold(0usize, |total, &cap| {
712            if cap == 0 {
713                return Err(AllocError::InvalidArg);
714            }
715            total
716                .checked_add(cap.div_ceil(slot_size))
717                .ok_or(AllocError::Overflow)
718        })?;
719
720        let remaining_descs = self
721            .max_descs
722            .checked_sub(desc_count)
723            .ok_or(VirtqError::Backpressure)?;
724
725        // Reserve explicit records before taking slots. OwnedChain handles rollback.
726        let mut buffers = SmallVec::new();
727        buffers
728            .try_reserve_exact(desc_count)
729            .map_err(|_| VirtqError::Bookkeeping)?;
730
731        let mut owned = OwnedChain {
732            mem: self.mem,
733            pool: self.pool,
734            buffers,
735            readable: 0,
736        };
737
738        // Allocate readable regions before writable ones, splitting each at the upper slot size.
739        let regions = self
740            .rd_caps
741            .iter()
742            .map(|&len| (len, false))
743            .chain(self.wr_caps.iter().map(|&len| (len, true)));
744
745        for (total_len, wr) in regions {
746            let mut remaining = total_len;
747
748            while remaining > 0 {
749                let len = remaining.min(slot_size);
750                let alloc = owned.pool.alloc(len)?;
751                let capacity = if wr { alloc.len } else { len as u32 };
752
753                let ent = BufferEntry {
754                    addr: alloc.addr,
755                    capacity,
756                    written: 0,
757                };
758
759                owned.buffers.push(ent);
760                owned.readable += usize::from(!wr);
761
762                remaining -= len;
763            }
764        }
765
766        // Use spare descriptors for upper-tier slots left after the explicit allocations.
767        if self.writable_avail {
768            let extra = owned.pool.num_free_upper().min(remaining_descs);
769            owned
770                .buffers
771                .try_reserve_exact(extra)
772                .map_err(|_| VirtqError::Bookkeeping)?;
773
774            for _ in 0..extra {
775                let alloc = owned.pool.alloc(slot_size)?;
776                let ent = BufferEntry {
777                    addr: alloc.addr,
778                    capacity: alloc.len,
779                    written: 0,
780                };
781
782                owned.buffers.push(ent);
783            }
784        }
785
786        // An availability-only request can leave no buffers to publish.
787        if owned.buffers.is_empty() {
788            return Err(VirtqError::Backpressure);
789        }
790
791        Ok(SendChain {
792            owned,
793            rd_capacity,
794            rd_written: 0,
795            write_mode: WriteMode::Unset,
796        })
797    }
798}
799
800/// Tracks which write API a [`SendChain`] payload uses, so the two paths are
801/// not mixed.
802///
803/// Copy writes ([`SendChain::write`]/[`SendChain::write_all`]) append at an
804/// aggregate cursor, while direct writes
805/// ([`SendChain::write_seg`]/[`SendChain::with_seg`]) set per-segment lengths
806/// absolutely. Mixing them would corrupt the written-length accounting.
807#[derive(Debug, Clone, Copy, PartialEq, Eq)]
808enum WriteMode {
809    Unset,
810    Append,
811    Direct,
812}
813
814/// A configured send chain ready for writing and submission.
815///
816/// Created by [`ChainBuilder::build`]. Write readable payload bytes directly on
817/// the chain, then submit via [`VirtqProducer::submit`].
818///
819/// # Examples
820///
821/// ```ignore
822/// let mut sc = producer.chain().readable(64).writable(128).build()?;
823/// sc.write_all(b"header")?.write_all(b" body")?;
824/// let tok = producer.submit(sc)?;
825///
826/// let mut sc = producer.chain().readable(128).build()?;
827/// sc.with_seg(0, |buf| serialize_into(buf))?;
828/// let tok = producer.submit(sc)?;
829/// ```
830///
831/// Copy writes (`write`/`write_all`) and direct writes (`write_seg`/`with_seg`)
832/// must not be mixed on the same chain; doing so panics in debug builds.
833///
834/// If dropped without submitting, allocated buffers are returned to the pool.
835#[must_use = "dropping without submitting deallocates the buffers"]
836pub struct SendChain<M> {
837    owned: OwnedChain<M>,
838    rd_capacity: usize,
839    rd_written: usize,
840    write_mode: WriteMode,
841}
842
843impl<M: MemOps> SendChain<M> {
844    /// Record that this chain uses `mode`, asserting it is not mixed with the
845    /// other write path.
846    fn note_write_mode(&mut self, mode: WriteMode) {
847        debug_assert!(
848            self.write_mode == WriteMode::Unset || self.write_mode == mode,
849            "SendChain mixes copy writes (write/write_all) with direct writes (write_seg/with_seg)"
850        );
851        self.write_mode = mode;
852    }
853
854    /// Total number of descriptors in this chain.
855    #[inline]
856    pub fn desc_count(&self) -> usize {
857        self.owned.buffers.len()
858    }
859
860    /// Number of readable descriptors in this chain.
861    #[inline]
862    pub fn rd_desc_count(&self) -> usize {
863        self.owned.readable
864    }
865
866    /// Number of writable descriptors in this chain.
867    #[inline]
868    pub fn wr_desc_count(&self) -> usize {
869        self.desc_count() - self.rd_desc_count()
870    }
871
872    /// Total producer-written readable capacity in bytes.
873    #[inline]
874    pub fn capacity(&self) -> usize {
875        self.rd_capacity
876    }
877
878    /// Number of producer-written readable bytes written so far.
879    #[inline]
880    pub fn written(&self) -> usize {
881        self.rd_written
882    }
883
884    /// Remaining producer-written readable capacity.
885    #[inline]
886    pub fn remaining(&self) -> usize {
887        self.capacity() - self.written()
888    }
889
890    /// Write bytes into payload segments, returning how many bytes were written.
891    ///
892    /// Appends at the current aggregate write position and scatters across
893    /// readable segments in chain order. Uses [`MemOps::write`] (volatile on
894    /// host side). If `buf` is larger than the remaining capacity, writes as
895    /// many bytes as will fit. If a later memory write fails, the cursor and
896    /// written length retain any earlier chunks written by the same call.
897    ///
898    /// # Errors
899    ///
900    /// - [`VirtqError::NoPayloadSegment`] - no readable buffer allocated
901    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
902    pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
903        if self.rd_desc_count() == 0 {
904            return Err(VirtqError::NoPayloadSegment);
905        }
906
907        self.note_write_mode(WriteMode::Append);
908
909        let mut remaining = &buf[..buf.len().min(self.remaining())];
910        let mut written = 0;
911
912        for buffer in &mut self.owned.buffers[..self.owned.readable] {
913            if remaining.is_empty() {
914                break;
915            }
916
917            let cap = buffer.capacity as usize;
918            let desc_off = buffer.written as usize;
919            let len = (cap - desc_off).min(remaining.len());
920            if len == 0 {
921                continue;
922            }
923
924            let addr = buffer
925                .addr
926                .checked_add(desc_off as u64)
927                .ok_or(VirtqError::MemoryWriteError)?;
928
929            self.owned
930                .mem
931                .write(addr, &remaining[..len])
932                .map_err(|_| VirtqError::MemoryWriteError)?;
933
934            buffer.written += len as u32;
935            self.rd_written += len;
936            written += len;
937            remaining = &remaining[len..];
938        }
939
940        Ok(written)
941    }
942
943    /// Write the entire buffer into payload segments.
944    ///
945    /// Appends at the current aggregate write position and scatters across
946    /// readable segments in chain order. Uses [`MemOps::write`] (volatile on
947    /// host side).
948    ///
949    /// # Errors
950    ///
951    /// - [`VirtqError::PayloadTooLarge`] - buf exceeds remaining capacity
952    /// - [`VirtqError::NoPayloadSegment`] - no readable buffer allocated
953    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
954    #[inline]
955    pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
956        if self.rd_desc_count() == 0 {
957            return Err(VirtqError::NoPayloadSegment);
958        }
959
960        if buf.len() > self.remaining() {
961            return Err(VirtqError::PayloadTooLarge {
962                recv: buf.len(),
963                limit: self.remaining(),
964            });
965        }
966
967        let written = self.write(buf)?;
968        debug_assert_eq!(written, buf.len());
969        Ok(self)
970    }
971
972    /// Write bytes into one readable segment by index.
973    ///
974    /// Writes from the start of the selected segment and records `buf.len()` as
975    /// that segment's descriptor length.
976    ///
977    /// # Errors
978    ///
979    /// - [`VirtqError::NoPayloadSegment`] - `index` does not name a payload segment
980    /// - [`VirtqError::PayloadTooLarge`] - `buf` exceeds the segment capacity
981    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
982    pub fn write_seg(&mut self, index: usize, buf: &[u8]) -> Result<&mut Self, VirtqError> {
983        self.note_write_mode(WriteMode::Direct);
984
985        let buffer = self.owned.buffers[..self.owned.readable]
986            .get_mut(index)
987            .ok_or(VirtqError::NoPayloadSegment)?;
988
989        let cap = buffer.capacity as usize;
990        if buf.len() > cap {
991            return Err(VirtqError::PayloadTooLarge {
992                recv: buf.len(),
993                limit: cap,
994            });
995        }
996
997        self.owned
998            .mem
999            .write(buffer.addr, buf)
1000            .map_err(|_| VirtqError::MemoryWriteError)?;
1001
1002        let previous = buffer.written as usize;
1003        buffer.written = checked_descriptor_len(buf.len())?;
1004        self.rd_written = self.rd_written - previous + buf.len();
1005        Ok(self)
1006    }
1007
1008    /// Serialize directly into one readable segment by index.
1009    ///
1010    /// The closure returns the number of valid bytes it wrote. The written
1011    /// length for that segment is recorded on success.
1012    ///
1013    /// # Errors
1014    ///
1015    /// - [`VirtqError::NoPayloadSegment`] - `index` does not name a payload segment
1016    /// - [`VirtqError::PayloadTooLarge`] - closure reports more bytes than segment capacity
1017    /// - [`VirtqError::MemoryWriteError`] - the memory backend cannot expose a mutable slice
1018    pub fn with_seg<E>(
1019        &mut self,
1020        index: usize,
1021        f: impl FnOnce(&mut [u8]) -> Result<usize, E>,
1022    ) -> Result<&mut Self, E>
1023    where
1024        E: From<VirtqError>,
1025    {
1026        self.note_write_mode(WriteMode::Direct);
1027
1028        let buffer = self.owned.buffers[..self.owned.readable]
1029            .get_mut(index)
1030            .ok_or_else(|| E::from(VirtqError::NoPayloadSegment))?;
1031
1032        let cap = buffer.capacity as usize;
1033
1034        // SAFETY: This unpublished chain owns the allocation exclusively.
1035        let buf = unsafe {
1036            self.owned
1037                .mem
1038                .as_mut_slice(buffer.addr, cap)
1039                .map_err(|_| E::from(VirtqError::MemoryWriteError))?
1040        };
1041
1042        let written = f(buf)?;
1043        if written > buf.len() {
1044            return Err(E::from(VirtqError::PayloadTooLarge {
1045                recv: written,
1046                limit: buf.len(),
1047            }));
1048        }
1049
1050        let previous = buffer.written as usize;
1051        buffer.written = checked_descriptor_len(written).map_err(E::from)?;
1052        self.rd_written = self.rd_written - previous + written;
1053
1054        Ok(self)
1055    }
1056}
1057
1058/// Tracks a pool slot's capacity and initialized length for writes and reply mappings.
1059///
1060/// [`BufferElement::len`] carries only the length published to the peer.
1061#[derive(Debug)]
1062struct BufferEntry {
1063    /// Buffer base address used for memory access and release through the pool.
1064    addr: u64,
1065    /// Usable bytes,
1066    capacity: u32,
1067    /// Initialized prefix length
1068    written: u32,
1069}
1070
1071/// Buffer ownership passed from [`SendChain`] to [`Inflight`] at publication.
1072///
1073/// Keeps the original memory backend and pool for reply mappings and slot release.
1074struct OwnedChain<M> {
1075    /// Original backend for writing payloads and mapping completed replies.
1076    mem: M,
1077    /// Shared allocator.
1078    pool: SlotPool,
1079    /// Allocation records in descriptor order.
1080    buffers: SmallVec<[BufferEntry; 4]>,
1081    /// Number of leading device readable buffers.
1082    readable: usize,
1083}
1084
1085impl<M> OwnedChain<M> {
1086    /// One [`BufferElement`] per direct descriptor, in chain order.
1087    ///
1088    /// The ring adds the descriptor IDs and flags that link the buffers.
1089    fn descriptors(&self) -> impl ExactSizeIterator<Item = BufferElement> + Clone + '_ {
1090        self.buffers.iter().enumerate().map(|(i, buf)| {
1091            let wr = i >= self.readable;
1092            let len = if wr { buf.capacity } else { buf.written };
1093
1094            BufferElement {
1095                addr: buf.addr,
1096                len,
1097                writable: wr,
1098            }
1099        })
1100    }
1101
1102    fn release(mut self) -> Result<(), AllocError> {
1103        self.release_all()
1104    }
1105
1106    fn release_all(&mut self) -> Result<(), AllocError> {
1107        let mut maybe_err = None;
1108        while let Some(buf) = self.buffers.pop() {
1109            if let Err(error) = self.pool.dealloc(buf.addr)
1110                && maybe_err.is_none()
1111            {
1112                maybe_err = Some(error);
1113            }
1114        }
1115
1116        self.readable = 0;
1117        maybe_err.map_or(Ok(()), Err)
1118    }
1119}
1120
1121impl<M: BufferMap> OwnedChain<M> {
1122    fn into_segments(mut self, written: usize) -> Result<Segments, VirtqError> {
1123        let mut remaining = written;
1124        let mut nonempty = 0;
1125
1126        for buf in &mut self.buffers[self.readable..] {
1127            let len = remaining.min(buf.capacity as usize);
1128            buf.written = len as u32;
1129
1130            nonempty += usize::from(len != 0);
1131            remaining -= len;
1132        }
1133
1134        if remaining != 0 {
1135            self.release()?;
1136            return Err(VirtqError::InvalidState);
1137        }
1138
1139        let mut segments = SmallVec::<[Bytes; 4]>::new();
1140        segments
1141            .try_reserve_exact(nonempty)
1142            .map_err(|_| VirtqError::Bookkeeping)?;
1143
1144        while let Some(buf) = self.buffers.last() {
1145            if self.buffers.len() <= self.readable || buf.written == 0 {
1146                let addr = buf.addr;
1147                self.buffers.pop();
1148                self.pool.dealloc(addr)?;
1149
1150                continue;
1151            }
1152
1153            let alloc = Allocation {
1154                addr: buf.addr,
1155                len: buf.capacity,
1156            };
1157
1158            let written = buf.written as usize;
1159            let lease = BufferLease::new(self.pool.clone(), alloc);
1160            self.buffers.pop();
1161
1162            // SAFETY: Completion returns exclusive ownership of the initialized
1163            // prefix. The mapper owns the slot until its borrowed view drops.
1164            let mapping = unsafe { self.mem.map_buffer(lease, written) }
1165                .map_err(|_| VirtqError::MemoryReadError)?;
1166
1167            segments.push(Bytes::from_owner(mapping));
1168        }
1169
1170        segments.reverse();
1171        Ok(Segments::from_smallvec(segments))
1172    }
1173}
1174
1175impl<M> Drop for OwnedChain<M> {
1176    fn drop(&mut self) {
1177        // best effort: if the pool deallocation fails, we can't do much about it here
1178        if let Err(error) = self.release_all() {
1179            log::error!("Failed to release virtqueue buffers: {error}");
1180            debug_assert!(false, "OwnedChain deallocation failed: {error}");
1181        }
1182    }
1183}
1184
1185fn checked_descriptor_len(len: usize) -> Result<u32, VirtqError> {
1186    if len > u32::MAX as usize {
1187        return Err(VirtqError::PayloadTooLarge {
1188            recv: len,
1189            limit: u32::MAX as usize,
1190        });
1191    }
1192    Ok(len as u32)
1193}
1194
1195#[cfg(test)]
1196mod tests {
1197    use alloc::rc::Rc;
1198    use alloc::sync::Arc;
1199    use core::sync::atomic::Ordering;
1200
1201    use super::*;
1202    use crate::virtq::ring::tests::{FaultMem, OwnedRing, TestMem, make_consumer, make_ring};
1203    use crate::virtq::test_utils::*;
1204
1205    fn poll_received<M: MemOps + Clone, N: Notifier>(
1206        consumer: &mut VirtqConsumer<M, N>,
1207    ) -> (RecvChain<M>, ReplyChain<M>) {
1208        consumer.poll(1024).unwrap().unwrap()
1209    }
1210
1211    #[derive(Clone)]
1212    struct CopyingMem<'a>(&'a Rc<TestMem>);
1213
1214    // SAFETY: All bounded operations delegate to TestMem.
1215    unsafe impl MemOps for CopyingMem<'_> {
1216        type Error = core::convert::Infallible;
1217
1218        fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1219            self.0.read(addr, dst)
1220        }
1221
1222        fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1223            self.0.write(addr, src)
1224        }
1225
1226        fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1227            self.0.load_acquire(addr)
1228        }
1229
1230        fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1231            self.0.store_release(addr, val)
1232        }
1233
1234        unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1235            // SAFETY: The caller supplies TestMem's slice preconditions.
1236            unsafe { self.0.as_slice(addr, len) }
1237        }
1238
1239        unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1240            // SAFETY: The caller supplies exclusive access to this range.
1241            unsafe { self.0.as_mut_slice(addr, len) }
1242        }
1243    }
1244
1245    impl BufferMap for CopyingMem<'_> {
1246        type Mapping = Vec<u8>;
1247
1248        unsafe fn map_buffer(
1249            &self,
1250            lease: BufferLease,
1251            written: usize,
1252        ) -> Result<Self::Mapping, Self::Error> {
1253            assert!(written <= lease.allocation().len as usize);
1254            let mut bytes = vec![0; written];
1255            self.read(lease.allocation().addr, &mut bytes)?;
1256            Ok(bytes)
1257        }
1258    }
1259
1260    fn make_virtq_pair(
1261        ring: &OwnedRing,
1262        slot_size: usize,
1263    ) -> (
1264        VirtqProducer<TestMem, TestNotifier>,
1265        VirtqConsumer<TestMem, TestNotifier>,
1266    ) {
1267        let mem = ring.mem();
1268        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1269
1270        let lower = SlotLayout::new(pool_base, slot_size / 2, ring.len()).unwrap();
1271        let upper = SlotLayout::new(lower.end_addr(), slot_size, ring.len()).unwrap();
1272        let pool = SlotPool::new_tiered(lower, upper).unwrap();
1273
1274        let notifier = TestNotifier::new();
1275        let producer = VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool);
1276        let consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1277        (producer, consumer)
1278    }
1279
1280    fn make_chain(
1281        lower_count: usize,
1282        upper_count: usize,
1283        slot_size: usize,
1284        max_descs: usize,
1285    ) -> ChainBuilder<TestMem> {
1286        let lower_len = lower_count * 256;
1287        let mem = TestMem::new(lower_len + upper_count * slot_size);
1288        let upper =
1289            SlotLayout::new(mem.base_addr() + lower_len as u64, slot_size, upper_count).unwrap();
1290
1291        let pool = if lower_count == 0 {
1292            SlotPool::new(upper)
1293        } else {
1294            let lower = SlotLayout::new(mem.base_addr(), 256, lower_count).unwrap();
1295            SlotPool::new_tiered(lower, upper)
1296        }
1297        .unwrap();
1298
1299        ChainBuilder::new(mem, pool, max_descs)
1300    }
1301
1302    fn inflight(seq: u32, id: u16) -> Inflight<TestMem> {
1303        let mem = TestMem::new(8);
1304        let pool_layout = SlotLayout::new(mem.base_addr(), 8, 1).unwrap();
1305        let pool = SlotPool::new(pool_layout).unwrap();
1306        let chain = ChainBuilder::new(mem, pool, 1).readable(8).build().unwrap();
1307        Inflight {
1308            token: Token { seq, id },
1309            chain: ManuallyDrop::new(chain.owned),
1310        }
1311    }
1312
1313    #[test]
1314    fn inflight_table_repairs_moved_entry_after_removal() {
1315        let mut table = InflightTable::new(16);
1316        for (seq, id) in [(0, 3), (1, 7), (2, 5)] {
1317            table.try_reserve_one().unwrap();
1318            table.insert(inflight(seq, id));
1319        }
1320
1321        let removed = table.remove(7).unwrap();
1322        assert_eq!(removed.token.seq, 1);
1323
1324        ManuallyDrop::into_inner(removed.chain).release().unwrap();
1325        assert!(!table.contains(7));
1326
1327        for (id, seq) in [(5, 2), (3, 0)] {
1328            let removed = table.remove(id).unwrap();
1329            assert_eq!(removed.token.seq, seq);
1330            ManuallyDrop::into_inner(removed.chain).release().unwrap();
1331        }
1332        assert!(table.live.is_empty());
1333        assert!(table.remove(7).is_none());
1334    }
1335
1336    #[test]
1337    fn producer_bookkeeping_starts_compact_and_lazy() {
1338        let ring = make_ring(64);
1339        let (producer, _consumer, _notifier) = make_test_producer(&ring);
1340
1341        assert_eq!(producer.inflight.by_id.len(), ring.len());
1342        assert!(producer.inflight.live.is_empty());
1343        assert_eq!(producer.inflight.live.capacity(), 0);
1344        assert_eq!(producer.pending.capacity(), 0);
1345    }
1346
1347    #[test]
1348    fn full_ring_still_reports_backpressure() {
1349        let ring = make_ring(4);
1350        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
1351
1352        for _ in 0..ring.len() {
1353            let chain = producer.chain().readable(1).build().unwrap();
1354            producer.submit(chain).unwrap();
1355        }
1356
1357        assert!(matches!(
1358            producer.chain().readable(1).build(),
1359            Err(VirtqError::Backpressure)
1360        ));
1361        // SAFETY: The consumer has never polled the ring and stays inactive.
1362        unsafe { producer.reset() }.unwrap();
1363    }
1364
1365    #[test]
1366    fn submission_rechecks_capacity_after_reservation() {
1367        let ring = make_ring(2);
1368        let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1369        let pool = producer.pool.clone();
1370
1371        let chain = producer.chain().writable(64).build().unwrap();
1372        for _ in 0..ring.len() {
1373            let other = producer.chain().writable(32).build().unwrap();
1374            producer.submit(other).unwrap();
1375        }
1376
1377        assert_eq!(pool.num_live(), 3);
1378        assert!(matches!(
1379            producer.submit(chain),
1380            Err(VirtqError::Backpressure)
1381        ));
1382        assert_eq!(pool.num_live(), 2);
1383
1384        // SAFETY: The consumer has never polled the ring and is reset next.
1385        unsafe { producer.reset() }.unwrap();
1386        consumer.reset().unwrap();
1387
1388        assert_eq!(pool.num_live(), 0);
1389    }
1390
1391    #[test]
1392    fn publication_failure_keeps_allocations_until_stopped_reset() {
1393        for failed_write in 0..4 {
1394            let ring = make_ring(4);
1395            let orig_mem = FaultMem::new(ring.mem());
1396            let orig_gen = Arc::downgrade(&orig_mem.0);
1397
1398            let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1399            let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1400            let pool = SlotPool::new(pool_layout).unwrap();
1401            let notif = TestNotifier::new();
1402
1403            let source = VirtqProducer::new(ring.layout(), orig_mem, notif.clone(), pool.clone());
1404
1405            let chain = source.chain().writable(64).build().unwrap();
1406            drop(source);
1407
1408            let mem = FaultMem::new(ring.mem());
1409            let mut producer =
1410                VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1411
1412            mem.fail_write_at(failed_write);
1413
1414            let res = producer.submit(chain);
1415            assert!(matches!(
1416                res,
1417                Err(VirtqError::RingError(RingError::MemError { .. }))
1418            ));
1419            assert_eq!(pool.num_live(), 1);
1420            assert_eq!(producer.inflight.live.len(), 1);
1421            assert!(orig_gen.upgrade().is_some());
1422
1423            mem.allow_writes();
1424            // SAFETY: No consumer is attached to this ring.
1425            unsafe { producer.reset() }.unwrap();
1426
1427            assert_eq!(pool.num_live(), 0);
1428            assert!(orig_gen.upgrade().is_none());
1429        }
1430    }
1431
1432    #[test]
1433    fn chain_clones_pool_only_for_independent_owners() {
1434        let ring = make_ring(8);
1435        let mem = ring.mem();
1436        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1437        let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1438        let pool = SlotPool::new(pool_layout).unwrap();
1439
1440        let notif = TestNotifier::new();
1441        let mapping = FaultMem::new(mem.clone());
1442
1443        let mut producer =
1444            VirtqProducer::new(ring.layout(), mapping.clone(), notif.clone(), pool.clone());
1445
1446        let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1447
1448        assert_eq!(pool.strong_count(), 2);
1449
1450        let mut chain = producer
1451            .chain()
1452            .readable(192)
1453            .writable(192)
1454            .build()
1455            .unwrap();
1456
1457        chain.write_all(b"request").unwrap();
1458
1459        assert_eq!(pool.num_live(), 6);
1460        assert_eq!(pool.strong_count(), 3);
1461        assert!(chain.owned.buffers.spilled());
1462
1463        let buffers = chain.owned.buffers.as_ptr();
1464        producer.submit(chain).unwrap();
1465
1466        assert_eq!(pool.strong_count(), 3);
1467        assert_eq!(producer.inflight.live[0].chain.buffers.as_ptr(), buffers);
1468
1469        let (recv, reply) = poll_received(&mut consumer);
1470        let ReplyChain::Writable(mut reply) = reply else {
1471            panic!("expected writable reply");
1472        };
1473        reply.write_all(&[0xa5; 70]).unwrap();
1474        consumer.complete(recv, reply).unwrap();
1475
1476        let segments = producer.poll().unwrap().unwrap().into_segments().unwrap();
1477
1478        assert_eq!(segments.segment_count(), 2);
1479        assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1480        assert_eq!(pool.strong_count(), 4);
1481        assert_eq!(pool.num_live(), 2);
1482
1483        let retained = segments.as_slice()[0].slice(1..);
1484        let cloned = retained.clone();
1485
1486        drop(segments);
1487
1488        assert_eq!(pool.num_live(), 1);
1489        assert_eq!(pool.strong_count(), 3);
1490
1491        // SAFETY: All consumer handles were completed. The consumer is reset next.
1492        unsafe { producer.reset() }.unwrap();
1493        consumer.reset().unwrap();
1494
1495        drop(producer);
1496
1497        assert_eq!(pool.strong_count(), 2);
1498        assert_eq!(retained.as_ref(), &[0xa5; 63]);
1499
1500        drop(retained);
1501
1502        assert_eq!(pool.num_live(), 1);
1503
1504        drop(cloned);
1505
1506        assert_eq!(pool.num_live(), 0);
1507        assert_eq!(pool.strong_count(), 1);
1508        assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1509    }
1510
1511    #[test]
1512    fn copied_mapping_crosses_threads_without_its_pool_or_borrowed_backend() {
1513        let ring = make_ring(4);
1514        let mem = Rc::new(ring.mem());
1515        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1516        let pool_layout = SlotLayout::new(base, 64, 1).unwrap();
1517        let pool = SlotPool::new(pool_layout).unwrap();
1518        let notif = TestNotifier::new();
1519
1520        let mut producer =
1521            VirtqProducer::new(ring.layout(), CopyingMem(&mem), notif.clone(), pool.clone());
1522        let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), notif);
1523
1524        let chain = producer.chain().writable(64).build().unwrap();
1525        producer.submit(chain).unwrap();
1526
1527        let (recv, reply) = poll_received(&mut consumer);
1528        let ReplyChain::Writable(mut reply) = reply else {
1529            panic!("expected writable reply");
1530        };
1531        reply.write_all(b"copied").unwrap();
1532        consumer.complete(recv, reply).unwrap();
1533
1534        let bytes = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1535
1536        assert_eq!(pool.num_live(), 0);
1537        assert_eq!(pool.strong_count(), 2);
1538
1539        let mut reused = producer.chain().readable(64).build().unwrap();
1540        reused.write_all(b"reused").unwrap();
1541
1542        std::thread::spawn(move || assert_eq!(bytes.as_ref(), b"copied"))
1543            .join()
1544            .unwrap();
1545
1546        assert_eq!(pool.num_live(), 1);
1547        assert_eq!(pool.strong_count(), 3);
1548
1549        drop(reused);
1550
1551        assert_eq!(pool.num_live(), 0);
1552    }
1553
1554    #[test]
1555    fn completion_uses_its_original_memory_and_allocating_pool() {
1556        let ring = make_ring(4);
1557        let mem = FaultMem::new(ring.mem());
1558        let orig_gen = Arc::downgrade(&mem.0);
1559        let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1560
1561        let notifier = TestNotifier::new();
1562        let original_layout = SlotLayout::new(base, 64, 4).unwrap();
1563        let original = SlotPool::new(original_layout).unwrap();
1564        let source = VirtqProducer::new(ring.layout(), mem, TestNotifier::new(), original.clone());
1565
1566        let chain = source.chain().writable(64).build().unwrap();
1567        drop(source);
1568
1569        assert!(orig_gen.upgrade().is_some());
1570
1571        let replacement_layout = SlotLayout::new(original.base_addr() + 0x1000, 64, 4).unwrap();
1572        let replacement = SlotPool::new(replacement_layout).unwrap();
1573
1574        let replacement_mem = FaultMem::new(ring.mem());
1575        replacement_mem.fail_mapping_at(0);
1576
1577        let mut producer = VirtqProducer::new(
1578            ring.layout(),
1579            replacement_mem.clone(),
1580            notifier.clone(),
1581            replacement.clone(),
1582        );
1583        let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), TestNotifier::new());
1584
1585        producer.submit(chain).unwrap();
1586
1587        assert!(orig_gen.upgrade().is_some());
1588
1589        let (recv, reply) = poll_received(&mut consumer);
1590        let ReplyChain::Writable(mut reply) = reply else {
1591            panic!("expected writable reply");
1592        };
1593
1594        reply.write_all(b"retained").unwrap();
1595        consumer.complete(recv, reply).unwrap();
1596
1597        let data = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1598
1599        assert_eq!(data.as_ref(), b"retained");
1600        assert_eq!(original.num_live(), 1);
1601        assert_eq!(replacement.num_live(), 0);
1602        assert_eq!(replacement_mem.0.map_calls.load(Ordering::Relaxed), 0);
1603
1604        let map_calls = orig_gen
1605            .upgrade()
1606            .unwrap()
1607            .map_calls
1608            .load(Ordering::Relaxed);
1609        assert_eq!(map_calls, 1);
1610
1611        drop(producer);
1612
1613        assert_eq!(data.as_ref(), b"retained");
1614
1615        drop(data);
1616
1617        assert_eq!(original.num_live(), 0);
1618        assert!(orig_gen.upgrade().is_none());
1619    }
1620
1621    #[test]
1622    fn mapping_failure_releases_returned_and_unmapped_slots() {
1623        for fail_at in 0..2 {
1624            let ring = make_ring(8);
1625            let mem = FaultMem::new(ring.mem());
1626            mem.fail_mapping_at(fail_at);
1627
1628            let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1629            let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1630            let pool = SlotPool::new(pool_layout).unwrap();
1631            let notif = TestNotifier::new();
1632            let mut producer =
1633                VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1634
1635            let mut consumer = make_consumer(&ring);
1636
1637            let chain = producer
1638                .chain()
1639                .readable(192)
1640                .writable(256)
1641                .build()
1642                .unwrap();
1643
1644            let mut addresses: Vec<_> = chain
1645                .owned
1646                .buffers
1647                .iter()
1648                .map(|buffer| buffer.addr)
1649                .collect();
1650
1651            producer.submit(chain).unwrap();
1652
1653            let (id, _) = consumer.poll_available().unwrap();
1654            // TestMem's zeroed backing supplies initialized bytes for the mapped prefix.
1655            consumer.submit_used(id, 70).unwrap();
1656
1657            assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
1658            assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), fail_at + 1);
1659            assert_eq!(pool.num_live(), 0);
1660            assert_eq!(pool.strong_count(), 2);
1661            assert_eq!(producer.num_inflight(), 0);
1662
1663            if fail_at == 1 {
1664                // The mapper releases the failed lease before completed owners unwind.
1665                addresses.swap(3, 4);
1666            }
1667
1668            let repeated = producer
1669                .chain()
1670                .readable(192)
1671                .writable(256)
1672                .build()
1673                .unwrap();
1674
1675            assert_eq!(
1676                repeated
1677                    .owned
1678                    .buffers
1679                    .iter()
1680                    .map(|buffer| buffer.addr)
1681                    .collect::<Vec<_>>(),
1682                addresses
1683            );
1684        }
1685    }
1686
1687    #[test]
1688    fn empty_and_malformed_completions_skip_mapping_and_release_every_slot() {
1689        let ring = make_ring(4);
1690        let mem = FaultMem::new(ring.mem());
1691        mem.fail_mapping_at(0);
1692
1693        let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1694        let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1695        let pool = SlotPool::new(pool_layout).unwrap();
1696        let notif = TestNotifier::new();
1697
1698        let mut producer =
1699            VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1700
1701        let mut consumer = make_consumer(&ring);
1702
1703        for (writable, written) in [(false, 0), (true, 0), (true, 129)] {
1704            let builder = producer.chain().readable(64);
1705            let builder = if writable {
1706                builder.writable(128)
1707            } else {
1708                builder
1709            };
1710
1711            let token = producer.submit(builder.build().unwrap()).unwrap();
1712            let (id, _) = consumer.poll_available().unwrap();
1713            consumer.submit_used(id, written).unwrap();
1714
1715            match producer.poll() {
1716                Ok(Some(UsedChain::Ack(returned))) if !writable => assert_eq!(returned, token),
1717                Ok(Some(UsedChain::Data(returned, segments))) if writable && written == 0 => {
1718                    assert_eq!(returned, token);
1719                    assert_eq!(segments.segment_count(), 0);
1720                }
1721                Err(VirtqError::InvalidState) if written == 129 => {}
1722                other => panic!("unexpected completion: {other:?}"),
1723            }
1724
1725            assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), 0);
1726            assert_eq!(pool.num_live(), 0);
1727            assert_eq!(pool.strong_count(), 2);
1728            assert_eq!(producer.num_inflight(), 0);
1729        }
1730    }
1731
1732    #[test]
1733    fn cancelling_a_chain_preserves_slot_order_after_a_partial_write_error() {
1734        let storage = TestMem::new(16);
1735        let base = storage.base_addr();
1736        let mem = FaultMem::new(storage);
1737        let t1 = SlotLayout::new(base, 2, 2).unwrap();
1738        let t2 = SlotLayout::new(base + 4, 4, 3).unwrap();
1739        let pool = SlotPool::new_tiered(t1, t2).unwrap();
1740
1741        let mut chain = ChainBuilder::new(mem.clone(), pool.clone(), 8)
1742            .readable(9)
1743            .writable_avail()
1744            .build()
1745            .unwrap();
1746
1747        let addresses: Vec<_> = chain
1748            .owned
1749            .buffers
1750            .iter()
1751            .map(|buffer| buffer.addr)
1752            .collect();
1753
1754        mem.fail_write_at(1);
1755        assert!(matches!(
1756            chain.write_all(b"abcdefghi"),
1757            Err(VirtqError::MemoryWriteError)
1758        ));
1759        assert_eq!(chain.written(), 4);
1760
1761        drop(chain);
1762
1763        assert_eq!(pool.num_live(), 0);
1764        assert_eq!(pool.strong_count(), 1);
1765
1766        mem.allow_writes();
1767        let repeated = ChainBuilder::new(mem, pool.clone(), 8)
1768            .readable(9)
1769            .writable_avail()
1770            .build()
1771            .unwrap();
1772
1773        assert_eq!(
1774            repeated
1775                .owned
1776                .buffers
1777                .iter()
1778                .map(|buffer| buffer.addr)
1779                .collect::<Vec<_>>(),
1780            addresses
1781        );
1782    }
1783
1784    #[test]
1785    fn reclaim_buffers_multiple_ring_generations() {
1786        let ring = make_ring(8);
1787        let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1788        let count = ring.len() * 2 + 1;
1789
1790        for _ in 0..count {
1791            let chain = producer.chain().writable(1).build().unwrap();
1792            producer.submit(chain).unwrap();
1793            let (recv, reply) = poll_received(&mut consumer);
1794            consumer.complete(recv, reply).unwrap();
1795            assert_eq!(producer.reclaim().unwrap(), 1);
1796            assert_eq!(producer.num_free(), ring.len());
1797            assert_eq!(producer.pool().num_live(), 0);
1798        }
1799
1800        for seq in 0..count {
1801            let Some(UsedChain::Data(token, segments)) = producer.poll().unwrap() else {
1802                panic!("expected an empty writable completion");
1803            };
1804
1805            assert_eq!(token.seq as usize, seq);
1806            assert!(segments.is_empty());
1807        }
1808
1809        assert!(producer.poll().unwrap().is_none());
1810    }
1811
1812    #[test]
1813    fn reset_reclaims_inflight_slots_and_reuses_ring() {
1814        let ring = make_ring(8);
1815        let mem = ring.mem();
1816        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1817        let pool_layout = SlotLayout::new(pool_base, 64, ring.len()).unwrap();
1818        let pool = SlotPool::new(pool_layout).unwrap();
1819        let notifier = TestNotifier::new();
1820        let mut producer = VirtqProducer::new(ring.layout(), mem, notifier, pool.clone());
1821
1822        for _ in 0..ring.len() {
1823            let chain = producer.chain().writable(64).build().unwrap();
1824            producer.submit(chain).unwrap();
1825        }
1826
1827        assert_eq!(pool.num_free(), 0);
1828
1829        // SAFETY: No consumer is attached to this ring.
1830        unsafe { producer.reset() }.unwrap();
1831
1832        assert_eq!(producer.num_inflight(), 0);
1833        assert_eq!(producer.num_free(), ring.len());
1834        assert_eq!(pool.num_free(), ring.len());
1835        assert!(producer.inflight.live.is_empty());
1836        assert!(
1837            producer
1838                .inflight
1839                .by_id
1840                .iter()
1841                .all(|slot| *slot == InflightTable::<TestMem>::VACANT)
1842        );
1843
1844        for _ in 0..ring.len() {
1845            let chain = producer.chain().writable(64).build().unwrap();
1846            producer.submit(chain).unwrap();
1847        }
1848
1849        // SAFETY: No consumer is attached to this ring.
1850        unsafe { producer.reset() }.unwrap();
1851    }
1852
1853    #[test]
1854    fn stopped_reset_reuses_slot_after_consumer_handles_drop() {
1855        let ring = make_ring(4);
1856        let mem = ring.mem();
1857        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1858        let pool_layout = SlotLayout::new(pool_base, 4, 1).unwrap();
1859        let pool = SlotPool::new(pool_layout).unwrap();
1860        let notifier = TestNotifier::new();
1861        let mut producer =
1862            VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool.clone());
1863        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), notifier.clone());
1864
1865        let sent = producer.chain().writable(4).build().unwrap();
1866        producer.submit(sent).unwrap();
1867        let (recv, reply) = poll_received(&mut consumer);
1868        assert!(matches!(reply, ReplyChain::Writable(_)));
1869
1870        drop((recv, reply, consumer));
1871        assert_eq!(pool.num_live(), 1);
1872
1873        // SAFETY: The consumer and all its chain handles have been dropped.
1874        unsafe { producer.reset() }.unwrap();
1875        assert_eq!(pool.num_live(), 0);
1876
1877        let mut replacement = producer.chain().readable(4).build().unwrap();
1878        assert_eq!(replacement.owned.buffers[0].addr, pool_base);
1879        replacement.write_all(b"GOOD").unwrap();
1880        producer.submit(replacement).unwrap();
1881
1882        let mut consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1883        let (recv, reply) = poll_received(&mut consumer);
1884        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"GOOD");
1885        consumer.complete(recv, reply).unwrap();
1886        producer.drain(drop).unwrap();
1887        assert_eq!(pool.num_live(), 0);
1888    }
1889
1890    #[test]
1891    fn reset_rejects_buffered_writable_completion() {
1892        let ring = make_ring(8);
1893        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1894        let chain = producer.chain().writable(64).build().unwrap();
1895        producer.submit(chain).unwrap();
1896
1897        let (recv, reply) = poll_received(&mut consumer);
1898        let ReplyChain::Writable(mut reply) = reply else {
1899            panic!("expected writable reply");
1900        };
1901
1902        reply.write_all(b"retained").unwrap();
1903        consumer.complete(recv, reply).unwrap();
1904        producer.reclaim().unwrap();
1905
1906        // SAFETY: The consumer completed its handles and stays inactive.
1907        assert!(matches!(
1908            unsafe { producer.reset() },
1909            Err(VirtqError::InvalidState)
1910        ));
1911
1912        drop(producer.poll().unwrap().unwrap());
1913
1914        // SAFETY: The consumer completed its handles and stays inactive.
1915        unsafe { producer.reset() }.unwrap();
1916    }
1917
1918    #[test]
1919    fn test_chain_readwrite_build() {
1920        let ring = make_ring(16);
1921        let (producer, _consumer, _notifier) = make_test_producer(&ring);
1922
1923        let se = producer.chain().readable(64).writable(128).build().unwrap();
1924        assert_eq!(se.capacity(), 64);
1925        assert_eq!(se.written(), 0);
1926        assert_eq!(se.remaining(), 64);
1927    }
1928
1929    #[test]
1930    fn test_chain_readable_writable_names_build() {
1931        let ring = make_ring(16);
1932        let (producer, _consumer, _notifier) = make_test_producer(&ring);
1933
1934        let se = producer.chain().readable(16).writable(32).build().unwrap();
1935        assert_eq!(se.desc_count(), 2);
1936        assert_eq!(se.rd_desc_count(), 1);
1937        assert_eq!(se.wr_desc_count(), 1);
1938        assert_eq!(se.capacity(), 16);
1939    }
1940
1941    #[test]
1942    fn test_chain_multi_readable_write_all_scatters() {
1943        let ring = make_ring(16);
1944        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
1945
1946        let mut se = producer
1947            .chain()
1948            .readable(5)
1949            .readable(6)
1950            .writable(32)
1951            .build()
1952            .unwrap();
1953
1954        se.write_all(b"hello world").unwrap();
1955        assert_eq!(se.written(), 11);
1956
1957        let token = producer.submit(se).unwrap();
1958        let (recv, reply) = poll_received(&mut consumer);
1959        assert_eq!(recv.token(), token);
1960        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
1961
1962        let segments = recv.to_segments().unwrap();
1963        assert_eq!(segments.segment_count(), 2);
1964        assert_eq!(segments.as_slice()[0].as_ref(), b"hello");
1965        assert_eq!(segments.as_slice()[1].as_ref(), b" world");
1966
1967        consumer.complete(recv, reply).unwrap();
1968        producer.drain(drop).unwrap();
1969    }
1970
1971    #[test]
1972    fn test_chain_independent_readables_preserve_pool_tiers() {
1973        let ring = make_ring(16);
1974        let layout = ring.layout();
1975        let mem = ring.mem();
1976
1977        let lower = SlotLayout::new(
1978            mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100,
1979            256,
1980            1,
1981        )
1982        .unwrap();
1983
1984        let upper = SlotLayout::new(lower.end_addr(), 4096, 1).unwrap();
1985        let pool = SlotPool::new_tiered(lower, upper).unwrap();
1986        let notifier = TestNotifier::new();
1987        let producer = VirtqProducer::new(layout, mem, notifier, pool.clone());
1988
1989        let send = producer
1990            .chain()
1991            .readable(128)
1992            .readable(4096)
1993            .build()
1994            .unwrap();
1995
1996        let readables = send.owned.descriptors().collect::<Vec<_>>();
1997
1998        assert_eq!(readables.len(), 2);
1999        assert_eq!(readables[0].addr, lower.base_addr());
2000        assert_eq!(readables[1].addr, upper.base_addr());
2001        assert_eq!(pool.num_free_lower(), 0);
2002        assert_eq!(pool.num_free_upper(), 0);
2003
2004        drop(send);
2005
2006        assert_eq!(pool.num_free_lower(), 1);
2007        assert_eq!(pool.num_free_upper(), 1);
2008    }
2009
2010    #[test]
2011    fn chain_avail_uses_pool_availability_at_build() {
2012        for held_count in [0, 2, 3] {
2013            let builder = make_chain(2, 3, 4096, 3);
2014            let pool = builder.pool.clone();
2015            let builder = builder.writable_avail().readable(128);
2016
2017            assert_eq!(pool.num_live(), 0);
2018
2019            let held: Vec<_> = (0..held_count).map(|_| pool.alloc(4096).unwrap()).collect();
2020
2021            let chain = builder.build().unwrap();
2022
2023            let writable = (3 - held_count).min(2);
2024            assert_eq!(chain.rd_desc_count(), 1);
2025            assert_eq!(chain.wr_desc_count(), writable);
2026            assert_eq!(chain.desc_count(), 1 + writable);
2027            assert_eq!(chain.owned.buffers[0].capacity, 128);
2028            assert!(
2029                chain.owned.buffers[1..]
2030                    .iter()
2031                    .all(|buf| buf.capacity == 4096)
2032            );
2033            assert_eq!(pool.num_free_lower(), 1);
2034            assert_eq!(pool.num_free_upper(), 3 - held_count - writable);
2035            assert_eq!(pool.num_live(), held_count + chain.desc_count());
2036
2037            drop(chain);
2038
2039            assert_eq!(pool.num_live(), held_count);
2040
2041            for allocation in held.into_iter().rev() {
2042                pool.dealloc(allocation.addr).unwrap();
2043            }
2044
2045            assert_eq!(pool.num_live(), 0);
2046        }
2047    }
2048
2049    #[test]
2050    fn chain_avail_respects_mandatory_descriptor_budget() {
2051        for max_descs in 0..3 {
2052            let builder = make_chain(2, 3, 4096, max_descs);
2053            let pool = builder.pool.clone();
2054
2055            let result = builder
2056                .readable(4096)
2057                .readable(128)
2058                .writable_avail()
2059                .build();
2060
2061            if max_descs == 2 {
2062                let chain = result.unwrap();
2063
2064                assert_eq!(chain.rd_desc_count(), 2);
2065                assert_eq!(chain.wr_desc_count(), 0);
2066
2067                drop(chain);
2068            } else {
2069                assert!(matches!(result, Err(VirtqError::Backpressure)));
2070            }
2071
2072            assert_eq!(pool.num_live(), 0);
2073
2074            let lower = pool.alloc(128).unwrap();
2075            let upper = pool.alloc(4096).unwrap();
2076
2077            assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2078            assert_eq!(upper.addr, pool.slot_addr(4).unwrap());
2079
2080            pool.dealloc(lower.addr).unwrap();
2081            pool.dealloc(upper.addr).unwrap();
2082        }
2083    }
2084
2085    #[test]
2086    fn chain_avail_preserves_explicit_writable_capacity_when_full() {
2087        for (upper_count, max_descs) in [(2, 2), (1, 3)] {
2088            let builder = make_chain(1, upper_count, 4096, max_descs);
2089            let pool = builder.pool.clone();
2090
2091            let chain = builder
2092                .writable_avail()
2093                .writable(128)
2094                .readable(128)
2095                .build()
2096                .unwrap();
2097
2098            assert_eq!(chain.rd_desc_count(), 1);
2099            assert_eq!(chain.wr_desc_count(), 1);
2100            assert_eq!(chain.owned.buffers[0].capacity, 128);
2101            assert_eq!(chain.owned.buffers[1].capacity, 4096);
2102            assert!(chain.owned.buffers.iter().all(|buf| buf.written == 0));
2103            assert_eq!(pool.num_free_upper(), upper_count - 1);
2104
2105            drop(chain);
2106
2107            assert_eq!(pool.num_live(), 0);
2108        }
2109    }
2110
2111    #[test]
2112    fn chain_avail_cannot_build_an_empty_chain() {
2113        for max_descs in [0, 2] {
2114            let builder = make_chain(1, 1, 4096, max_descs);
2115            let pool = builder.pool.clone();
2116            let held = (max_descs != 0).then(|| pool.alloc(4096).unwrap());
2117
2118            assert!(matches!(
2119                builder.writable_avail().build(),
2120                Err(VirtqError::Backpressure)
2121            ));
2122            assert_eq!(pool.num_free_lower(), 1);
2123            assert_eq!(pool.num_live(), usize::from(held.is_some()));
2124
2125            if let Some(allocation) = held {
2126                pool.dealloc(allocation.addr).unwrap();
2127            }
2128        }
2129    }
2130
2131    #[test]
2132    fn chain_avail_only_uses_upper_slots() {
2133        let builder = make_chain(4, 2, 4096, 6);
2134        let pool = builder.pool.clone();
2135
2136        let chain = builder.writable_avail().writable_avail().build().unwrap();
2137
2138        assert_eq!(chain.rd_desc_count(), 0);
2139        assert_eq!(chain.wr_desc_count(), 2);
2140        assert_eq!(pool.num_free_lower(), 4);
2141        assert_eq!(pool.num_free_upper(), 0);
2142
2143        drop(chain);
2144
2145        assert_eq!(pool.num_live(), 0);
2146    }
2147
2148    #[test]
2149    fn chain_preserves_region_order_with_odd_slot_sizes() {
2150        let builder = make_chain(1, 4, 3001, 5);
2151        let pool = builder.pool.clone();
2152
2153        // The final short readable must fall back to an upper slot.
2154        let chain = builder
2155            .readable(128)
2156            .readable(6003)
2157            .writable_avail()
2158            .build()
2159            .unwrap();
2160
2161        assert_eq!(chain.rd_desc_count(), 4);
2162
2163        let actual_offsets = chain
2164            .owned
2165            .buffers
2166            .iter()
2167            .map(|buf| {
2168                (
2169                    buf.addr,
2170                    buf.capacity,
2171                    pool.allocation_len(buf.addr).unwrap(),
2172                )
2173            })
2174            .collect::<Vec<_>>();
2175
2176        let expected_offsets = [
2177            (0, 128, 256),
2178            (4, 3001, 3001),
2179            (3, 3001, 3001),
2180            (2, 1, 3001),
2181            (1, 3001, 3001),
2182        ]
2183        .map(|(slot, capacity, allocated)| (pool.slot_addr(slot).unwrap(), capacity, allocated));
2184
2185        assert_eq!(actual_offsets, expected_offsets);
2186
2187        drop(chain);
2188
2189        assert_eq!(pool.num_live(), 0);
2190    }
2191
2192    #[test]
2193    fn chain_failure_releases_only_its_own_slots() {
2194        let builder = make_chain(2, 3, 4096, 4);
2195        let pool = builder.pool.clone();
2196        let held = pool.alloc(4096).unwrap();
2197
2198        assert!(matches!(
2199            builder.readable(128).writable(4096 * 3).build(),
2200            Err(VirtqError::Backpressure)
2201        ));
2202        assert_eq!(pool.live_addrs(), [held.addr]);
2203
2204        let lower = pool.alloc(128).unwrap();
2205        let upper = pool.alloc(4096).unwrap();
2206        let next_upper = pool.alloc(4096).unwrap();
2207
2208        assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2209        assert_eq!(upper.addr, pool.slot_addr(3).unwrap());
2210        assert_eq!(next_upper.addr, pool.slot_addr(2).unwrap());
2211
2212        pool.dealloc(next_upper.addr).unwrap();
2213        pool.dealloc(upper.addr).unwrap();
2214        pool.dealloc(lower.addr).unwrap();
2215        pool.dealloc(held.addr).unwrap();
2216    }
2217
2218    #[test]
2219    fn chain_rejects_invalid_request_sequences() {
2220        for (readable, writable) in [(vec![128, 0], vec![]), (vec![128], vec![4096, 0])] {
2221            let mut builder = make_chain(1, 1, 4096, 4);
2222            let pool = builder.pool.clone();
2223
2224            for cap in readable {
2225                builder = builder.readable(cap);
2226            }
2227            for cap in writable {
2228                builder = builder.writable(cap);
2229            }
2230
2231            assert!(matches!(
2232                builder.build(),
2233                Err(VirtqError::Alloc(AllocError::InvalidArg))
2234            ));
2235            assert_eq!(pool.num_live(), 0);
2236        }
2237
2238        let builder = make_chain(1, 1, 4096, 4);
2239        let pool = builder.pool.clone();
2240
2241        assert!(matches!(
2242            builder.readable(usize::MAX).readable(1).build(),
2243            Err(VirtqError::Alloc(AllocError::Overflow))
2244        ));
2245        assert_eq!(pool.num_live(), 0);
2246    }
2247
2248    #[test]
2249    fn chain_keeps_four_records_inline_and_one_pool_handle() {
2250        for count in [4, 5] {
2251            let builder = make_chain(0, count, 4096, count);
2252            let pool = builder.pool.clone();
2253
2254            let chain = builder
2255                .readable((count - 1) * 4096)
2256                .writable_avail()
2257                .build()
2258                .unwrap();
2259
2260            assert_eq!(chain.desc_count(), count);
2261            assert_eq!(chain.owned.buffers.spilled(), count > 4);
2262            assert_eq!(pool.strong_count(), 2);
2263
2264            drop(chain);
2265
2266            assert_eq!(pool.num_live(), 0);
2267        }
2268    }
2269
2270    #[test]
2271    fn test_chain_multi_readable_appends_across_calls() {
2272        let ring = make_ring(16);
2273        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2274
2275        let mut send = producer.chain().readable(8).build().unwrap();
2276        send.write_all(b"abc").unwrap();
2277        send.write_all(b"def").unwrap();
2278        assert_eq!(send.written(), 6);
2279        assert_eq!(send.remaining(), 2);
2280
2281        producer.submit(send).unwrap();
2282        let (recv, reply) = poll_received(&mut consumer);
2283        let segments = recv.to_segments().unwrap();
2284        assert_eq!(segments.segment_count(), 2);
2285        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2286        assert_eq!(segments.as_slice()[1].as_ref(), b"ef");
2287        consumer.complete(recv, reply).unwrap();
2288        producer.drain(drop).unwrap();
2289    }
2290
2291    #[test]
2292    fn test_chain_readable_splits_logical_capacity() {
2293        let expected_lengths = [4, 4, 4];
2294        let ring = make_ring(16);
2295        let (mut producer, mut consumer, _) = make_test_producer_with_slot_size(&ring, 4);
2296
2297        let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2298        let readables = &se.owned.buffers[..se.rd_desc_count()];
2299
2300        assert_eq!(se.rd_desc_count(), 3);
2301        assert_eq!(se.capacity(), 10);
2302
2303        let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2304        assert_eq!(caps, [4, 4, 2]);
2305
2306        let lengths = readables
2307            .iter()
2308            .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2309            .collect::<Vec<_>>();
2310        assert_eq!(lengths, expected_lengths);
2311
2312        se.write_all(b"abcdefghij").unwrap();
2313        assert_eq!(se.written(), 10);
2314
2315        let token = producer.submit(se).unwrap();
2316        let (recv, reply) = poll_received(&mut consumer);
2317
2318        assert_eq!(recv.token(), token);
2319        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2320
2321        let segments = recv.to_segments().unwrap();
2322
2323        assert_eq!(segments.segment_count(), 3);
2324        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2325        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2326        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2327
2328        consumer.complete(recv, reply).unwrap();
2329        producer.drain(drop).unwrap();
2330
2331        assert_eq!(producer.pool.num_live(), 0);
2332    }
2333
2334    #[test]
2335    fn test_chain_readable_splits_logical_capacity_tiered() {
2336        let expected_lengths = [4, 4, 2];
2337        let ring = make_ring(16);
2338        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2339
2340        let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2341        let readables = &se.owned.buffers[..se.rd_desc_count()];
2342
2343        assert_eq!(se.rd_desc_count(), 3);
2344        assert_eq!(se.capacity(), 10);
2345
2346        let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2347        assert_eq!(caps, [4, 4, 2]);
2348
2349        let lengths = readables
2350            .iter()
2351            .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2352            .collect::<Vec<_>>();
2353        assert_eq!(lengths, expected_lengths);
2354
2355        se.write_all(b"abcdefghij").unwrap();
2356        assert_eq!(se.written(), 10);
2357
2358        let token = producer.submit(se).unwrap();
2359        let (recv, reply) = poll_received(&mut consumer);
2360
2361        assert_eq!(recv.token(), token);
2362        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2363
2364        let segments = recv.to_segments().unwrap();
2365
2366        assert_eq!(segments.segment_count(), 3);
2367        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2368        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2369        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2370
2371        consumer.complete(recv, reply).unwrap();
2372        producer.drain(drop).unwrap();
2373
2374        assert_eq!(producer.pool.num_live(), 0);
2375    }
2376
2377    #[test]
2378    fn test_chain_readable_rejects_zero_capacity_on_build() {
2379        let ring = make_ring(16);
2380        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2381
2382        assert!(matches!(
2383            producer.chain().readable(0).build(),
2384            Err(VirtqError::Alloc(AllocError::InvalidArg))
2385        ));
2386    }
2387
2388    #[test]
2389    fn test_chain_writable_splits_logical_capacity() {
2390        let ring = make_ring(16);
2391        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2392
2393        let se = producer.chain().writable(10).build().unwrap();
2394        let token = producer.submit(se).unwrap();
2395
2396        let (recv, reply) = poll_received(&mut consumer);
2397        let ReplyChain::Writable(mut wc) = reply else {
2398            panic!("expected writable reply");
2399        };
2400        assert_eq!(wc.capacity(), 10);
2401
2402        wc.write_all(b"abcdefghij").unwrap();
2403        consumer.complete(recv, wc).unwrap();
2404
2405        let used = producer.poll().unwrap().unwrap();
2406        assert_eq!(used.token(), token);
2407        let segments = used.segments().unwrap();
2408        assert_eq!(segments.segment_count(), 3);
2409        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2410        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2411        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2412    }
2413
2414    #[test]
2415    fn test_chain_writable_rejects_zero_capacity_on_build() {
2416        let ring = make_ring(16);
2417        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2418
2419        assert!(matches!(
2420            producer.chain().writable(0).build(),
2421            Err(VirtqError::Alloc(AllocError::InvalidArg))
2422        ));
2423    }
2424
2425    #[test]
2426    fn test_chain_multi_readable_write_all_preserves_segments() {
2427        let ring = make_ring(16);
2428        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2429
2430        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2431        se.write_all(b"headbody").unwrap();
2432
2433        producer.submit(se).unwrap();
2434        let (recv, reply) = poll_received(&mut consumer);
2435        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"headbody");
2436        consumer.complete(recv, reply).unwrap();
2437        producer.drain(drop).unwrap();
2438    }
2439
2440    #[test]
2441    fn test_chain_payload_segment_writer_serializes_directly() {
2442        let ring = make_ring(16);
2443        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2444
2445        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2446        se.with_seg(0, |segment| {
2447            segment.copy_from_slice(b"head");
2448            Ok::<usize, VirtqError>(4)
2449        })
2450        .unwrap();
2451        se.with_seg(1, |segment| {
2452            segment.copy_from_slice(b"body");
2453            Ok::<usize, VirtqError>(4)
2454        })
2455        .unwrap();
2456
2457        producer.submit(se).unwrap();
2458        let (recv, reply) = poll_received(&mut consumer);
2459        let segments = recv.to_segments().unwrap();
2460        assert_eq!(segments.segment_count(), 2);
2461        assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2462        consumer.complete(recv, reply).unwrap();
2463        producer.drain(drop).unwrap();
2464    }
2465
2466    #[test]
2467    fn test_chain_payload_segment_write_copies_directly() {
2468        let ring = make_ring(16);
2469        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2470
2471        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2472        se.write_seg(0, b"head").unwrap();
2473        se.write_seg(1, b"body").unwrap();
2474
2475        producer.submit(se).unwrap();
2476        let (recv, reply) = poll_received(&mut consumer);
2477        let segments = recv.to_segments().unwrap();
2478        assert_eq!(segments.segment_count(), 2);
2479        assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2480        consumer.complete(recv, reply).unwrap();
2481        producer.drain(drop).unwrap();
2482    }
2483
2484    #[test]
2485    fn test_chain_multi_writable_used_returns_segments() {
2486        let ring = make_ring(16);
2487        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2488
2489        let se = producer.chain().writable(5).writable(6).build().unwrap();
2490        let token = producer.submit(se).unwrap();
2491
2492        let (recv, reply) = poll_received(&mut consumer);
2493        let ReplyChain::Writable(mut wc) = reply else {
2494            panic!("expected writable reply");
2495        };
2496        assert_eq!(wc.capacity(), 12);
2497
2498        wc.write_all(b"hello world").unwrap();
2499        consumer.complete(recv, wc).unwrap();
2500
2501        let used = producer.poll().unwrap().unwrap();
2502        assert_eq!(used.token(), token);
2503        let segments = used.segments().unwrap();
2504        assert_eq!(segments.segment_count(), 2);
2505        assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2506        assert_eq!(segments.as_slice()[1].as_ref(), b"world");
2507        assert_eq!(segments.to_bytes().as_ref(), b"hello world");
2508    }
2509
2510    #[test]
2511    fn test_chain_multi_writable_short_used_truncates_last_segment() {
2512        let ring = make_ring(16);
2513        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2514
2515        let se = producer.chain().writable(5).writable(6).build().unwrap();
2516        producer.submit(se).unwrap();
2517
2518        let (recv, reply) = poll_received(&mut consumer);
2519        let ReplyChain::Writable(mut wc) = reply else {
2520            panic!("expected writable reply");
2521        };
2522
2523        wc.write_all(b"hello wo").unwrap();
2524        consumer.complete(recv, wc).unwrap();
2525
2526        let used = producer.poll().unwrap().unwrap();
2527        let segments = used.segments().unwrap();
2528        assert_eq!(segments.segment_count(), 2);
2529        assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2530        assert_eq!(segments.as_slice()[1].as_ref(), b"wo");
2531        assert_eq!(segments.to_bytes().as_ref(), b"hello wo");
2532    }
2533
2534    #[test]
2535    fn test_chain_multi_writable_zero_used_returns_empty_segments() {
2536        let ring = make_ring(16);
2537        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2538
2539        let se = producer.chain().writable(5).writable(6).build().unwrap();
2540        producer.submit(se).unwrap();
2541
2542        let (recv, reply) = poll_received(&mut consumer);
2543        consumer.complete(recv, reply).unwrap();
2544
2545        let used = producer.poll().unwrap().unwrap();
2546        let segments = used.segments().unwrap();
2547        assert_eq!(segments.segment_count(), 0);
2548        assert!(segments.is_empty());
2549        assert!(segments.to_bytes().is_empty());
2550    }
2551
2552    #[test]
2553    fn test_chain_readable_only_build() {
2554        let ring = make_ring(16);
2555        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2556
2557        let se = producer.chain().readable(32).build().unwrap();
2558        assert_eq!(se.capacity(), 32);
2559    }
2560
2561    #[test]
2562    fn test_chain_writable_only_build() {
2563        let ring = make_ring(16);
2564        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2565
2566        let se = producer.chain().writable(64).build().unwrap();
2567        assert_eq!(se.capacity(), 0);
2568    }
2569
2570    #[test]
2571    fn test_chain_empty_build_fails() {
2572        let ring = make_ring(16);
2573        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2574
2575        let result = producer.chain().build();
2576        assert!(matches!(result, Err(VirtqError::InvalidState)));
2577    }
2578
2579    #[test]
2580    fn test_send_chain_write_all_and_submit() {
2581        let ring = make_ring(16);
2582        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2583
2584        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2585
2586        se.write_all(b"hello")
2587            .unwrap()
2588            .write_all(b" world")
2589            .unwrap();
2590        assert_eq!(se.written(), 11);
2591        assert_eq!(se.remaining(), 53);
2592        let tok = producer.submit(se).unwrap();
2593
2594        let (recv, reply) = poll_received(&mut consumer);
2595        assert_eq!(recv.token(), tok);
2596        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2597        consumer.complete(recv, reply).unwrap();
2598        producer.drain(drop).unwrap();
2599    }
2600
2601    #[test]
2602    fn test_send_payload_write_all_fluent() {
2603        let ring = make_ring(16);
2604        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2605
2606        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2607        se.write_all(b"hello")
2608            .unwrap()
2609            .write_all(b" world")
2610            .unwrap();
2611        assert_eq!(se.written(), 11);
2612        assert_eq!(se.remaining(), 53);
2613        let tok = producer.submit(se).unwrap();
2614
2615        let (recv, reply) = poll_received(&mut consumer);
2616        assert_eq!(recv.token(), tok);
2617        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2618        consumer.complete(recv, reply).unwrap();
2619        producer.drain(drop).unwrap();
2620    }
2621
2622    #[test]
2623    fn test_send_payload_partial_write() {
2624        let ring = make_ring(16);
2625        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2626
2627        let mut se = producer.chain().readable(8).build().unwrap();
2628        let written = se.write(b"hello world").unwrap();
2629        assert_eq!(written, 8);
2630        assert_eq!(se.remaining(), 0);
2631
2632        producer.submit(se).unwrap();
2633        let (recv, reply) = poll_received(&mut consumer);
2634        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello wo");
2635        consumer.complete(recv, reply).unwrap();
2636        producer.drain(drop).unwrap();
2637    }
2638
2639    #[test]
2640    fn test_send_payload_write_with_serializes_directly() {
2641        let ring = make_ring(16);
2642        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2643
2644        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2645        se.with_seg(0, |buf| {
2646            buf[..5].copy_from_slice(b"hello");
2647            Ok::<usize, VirtqError>(5)
2648        })
2649        .unwrap();
2650
2651        let _tok = producer.submit(se).unwrap();
2652
2653        let (recv, reply) = poll_received(&mut consumer);
2654        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2655        consumer.complete(recv, reply).unwrap();
2656        producer.drain(drop).unwrap();
2657    }
2658
2659    #[test]
2660    fn test_send_chain_single_segment_writer_serializes_directly() {
2661        let ring = make_ring(16);
2662        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2663
2664        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2665        se.with_seg(0, |segment| {
2666            assert_eq!(segment.len(), 64);
2667            segment[..5].copy_from_slice(b"hello");
2668            Ok::<usize, VirtqError>(5)
2669        })
2670        .unwrap();
2671
2672        let _tok = producer.submit(se).unwrap();
2673
2674        let (recv, reply) = poll_received(&mut consumer);
2675        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2676        consumer.complete(recv, reply).unwrap();
2677        producer.drain(drop).unwrap();
2678    }
2679
2680    #[test]
2681    fn test_send_chain_single_segment_writer_rejects_multi_segment() {
2682        let ring = make_ring(16);
2683        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2684
2685        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2686        assert!(matches!(
2687            se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2688            Err(VirtqError::NoPayloadSegment)
2689        ));
2690    }
2691
2692    #[test]
2693    fn test_send_chain_single_segment_writer_rejects_auto_split_chain() {
2694        let ring = make_ring(16);
2695        let (producer, _consumer) = make_virtq_pair(&ring, 4);
2696
2697        let mut se = producer.chain().readable(8).build().unwrap();
2698        assert_eq!(se.rd_desc_count(), 2);
2699        assert!(matches!(
2700            se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2701            Err(VirtqError::NoPayloadSegment)
2702        ));
2703    }
2704
2705    #[test]
2706    fn test_send_payload_segment_set_written_too_large() {
2707        let ring = make_ring(16);
2708        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2709
2710        let mut se = producer.chain().readable(32).writable(64).build().unwrap();
2711        let err = se
2712            .with_seg(0, |_| Ok::<usize, VirtqError>(64))
2713            .err()
2714            .unwrap();
2715        assert!(matches!(
2716            err,
2717            VirtqError::PayloadTooLarge {
2718                recv: 64,
2719                limit: 32
2720            }
2721        ));
2722    }
2723
2724    #[test]
2725    fn test_send_chain_write_too_large() {
2726        let ring = make_ring(16);
2727        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2728
2729        let mut se = producer.chain().readable(4).build().unwrap();
2730        let err = se.write_all(b"too long").err().unwrap();
2731        assert!(matches!(
2732            err,
2733            VirtqError::PayloadTooLarge { recv: 8, limit: 4 }
2734        ));
2735    }
2736
2737    #[test]
2738    fn test_writeonly_has_no_readable_buffer() {
2739        let ring = make_ring(16);
2740        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2741
2742        let mut se = producer.chain().writable(32).build().unwrap();
2743        let err = se.write_all(b"data").err().unwrap();
2744        assert!(matches!(err, VirtqError::NoPayloadSegment));
2745    }
2746
2747    #[test]
2748    fn test_drop_chain_builder_deallocs() {
2749        let ring = make_ring(16);
2750        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2751
2752        {
2753            let _builder = producer.chain().readable(64).writable(128);
2754            // dropped without build
2755        }
2756
2757        // Ring should still be fully usable
2758        let se = producer.chain().readable(64).writable(128).build().unwrap();
2759        let tok = producer.submit(se).unwrap();
2760        assert!(tok.id < 16);
2761
2762        // SAFETY: The consumer has never polled the ring and stays inactive.
2763        unsafe { producer.reset() }.unwrap();
2764    }
2765
2766    #[test]
2767    fn test_drop_send_chain_deallocs() {
2768        let ring = make_ring(16);
2769        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2770
2771        {
2772            let _se = producer.chain().readable(64).writable(128).build().unwrap();
2773            // dropped without submit
2774        }
2775
2776        // Ring should still be fully usable
2777        let se = producer.chain().readable(64).writable(128).build().unwrap();
2778        let tok = producer.submit(se).unwrap();
2779        assert!(tok.id < 16);
2780
2781        // SAFETY: The consumer has never polled the ring and stays inactive.
2782        unsafe { producer.reset() }.unwrap();
2783    }
2784
2785    #[test]
2786    fn test_submit_notifies() {
2787        let ring = make_ring(16);
2788        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2789
2790        let initial_count = notifier.notification_count();
2791
2792        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2793        se.write_all(b"hello").unwrap();
2794        producer.submit(se).unwrap();
2795
2796        assert!(notifier.notification_count() > initial_count);
2797
2798        // SAFETY: The consumer has never polled the ring and stays inactive.
2799        unsafe { producer.reset() }.unwrap();
2800    }
2801
2802    #[test]
2803    fn test_submit_read_only_notifies_by_default() {
2804        let ring = make_ring(16);
2805        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2806
2807        let initial_count = notifier.notification_count();
2808
2809        let mut se = producer.chain().readable(64).build().unwrap();
2810        se.write_all(b"fire-and-forget").unwrap();
2811        producer.submit(se).unwrap();
2812
2813        assert!(notifier.notification_count() > initial_count);
2814
2815        // SAFETY: The consumer has never polled the ring and stays inactive.
2816        unsafe { producer.reset() }.unwrap();
2817    }
2818
2819    #[test]
2820    fn test_submit_write_only_notifies_by_default() {
2821        let ring = make_ring(16);
2822        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2823
2824        let initial_count = notifier.notification_count();
2825
2826        let se = producer.chain().writable(128).build().unwrap();
2827        producer.submit(se).unwrap();
2828
2829        assert!(notifier.notification_count() > initial_count);
2830
2831        // SAFETY: The consumer has never polled the ring and stays inactive.
2832        unsafe { producer.reset() }.unwrap();
2833    }
2834
2835    #[test]
2836    fn test_batch_notifies_once_on_finish() {
2837        let ring = make_ring(16);
2838        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2839
2840        let initial_count = notifier.notification_count();
2841
2842        let mut batch = producer.batch();
2843
2844        let mut first = batch.chain().readable(64).build().unwrap();
2845        first.write_all(b"first").unwrap();
2846        batch.submit(first).unwrap();
2847
2848        let mut second = batch.chain().readable(64).build().unwrap();
2849        second.write_all(b"second").unwrap();
2850        batch.submit(second).unwrap();
2851
2852        assert_eq!(notifier.notification_count(), initial_count);
2853        assert!(batch.finish().unwrap());
2854        assert_eq!(notifier.notification_count(), initial_count + 1);
2855
2856        let (recv, reply) = poll_received(&mut consumer);
2857        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"first");
2858        consumer.complete(recv, reply).unwrap();
2859
2860        let (recv, reply) = poll_received(&mut consumer);
2861        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"second");
2862        consumer.complete(recv, reply).unwrap();
2863        producer.drain(drop).unwrap();
2864    }
2865
2866    #[test]
2867    fn test_batch_finish_notifies_from_batch_start_cursor() {
2868        let ring = make_ring(16);
2869        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2870
2871        let cursor = consumer.avail_cursor();
2872        consumer
2873            .set_avail_suppression(SuppressionKind::Descriptor(cursor))
2874            .unwrap();
2875
2876        let mut batch = producer.batch();
2877
2878        let mut first = batch.chain().readable(64).build().unwrap();
2879        first.write_all(b"first").unwrap();
2880        batch.submit(first).unwrap();
2881        assert_eq!(notifier.notification_count(), 0);
2882
2883        let mut second = batch.chain().readable(64).writable(64).build().unwrap();
2884        second.write_all(b"second").unwrap();
2885        batch.submit(second).unwrap();
2886
2887        assert!(batch.finish().unwrap());
2888
2889        assert_eq!(notifier.notification_count(), 1);
2890
2891        // SAFETY: The consumer has never polled the ring and stays inactive.
2892        unsafe { producer.reset() }.unwrap();
2893    }
2894
2895    #[test]
2896    fn test_empty_batch_finish_does_not_notify() {
2897        let ring = make_ring(16);
2898        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2899
2900        let batch = producer.batch();
2901        assert!(!batch.finish().unwrap());
2902        assert_eq!(notifier.notification_count(), 0);
2903    }
2904
2905    #[test]
2906    fn test_batch_can_finish_without_notification() {
2907        let ring = make_ring(16);
2908        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2909
2910        let mut batch = producer.batch();
2911        let mut chain = batch.chain().readable(4).build().unwrap();
2912        chain.write_all(b"data").unwrap();
2913        batch.submit(chain).unwrap();
2914        batch.finish_without_notify();
2915
2916        assert_eq!(notifier.notification_count(), 0);
2917
2918        let (recv, reply) = poll_received(&mut consumer);
2919        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"data");
2920
2921        consumer.complete(recv, reply).unwrap();
2922        producer.drain(drop).unwrap();
2923    }
2924
2925    #[test]
2926    fn test_write_only_round_trip() {
2927        let ring = make_ring(16);
2928        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2929
2930        let se = producer.chain().writable(32).build().unwrap();
2931        let token = producer.submit(se).unwrap();
2932
2933        let (recv, reply) = poll_received(&mut consumer);
2934        assert_eq!(recv.token(), token);
2935        assert!(recv.to_bytes().unwrap().is_empty());
2936
2937        if let ReplyChain::Writable(mut wc) = reply {
2938            wc.write_all(b"filled-by-consumer").unwrap();
2939            consumer.complete(recv, wc).unwrap();
2940        } else {
2941            panic!("expected Writable");
2942        }
2943
2944        let used = producer.poll().unwrap().unwrap();
2945        assert_eq!(used.token(), token);
2946        assert_eq!(used.to_bytes().unwrap().len(), b"filled-by-consumer".len());
2947        assert_eq!(used.to_bytes().unwrap().as_ref(), b"filled-by-consumer");
2948    }
2949
2950    #[test]
2951    fn test_read_only_round_trip() {
2952        let ring = make_ring(16);
2953        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2954
2955        let mut se = producer.chain().readable(32).build().unwrap();
2956        se.write_all(b"fire-and-forget").unwrap();
2957        let token = producer.submit(se).unwrap();
2958
2959        let (recv, reply) = poll_received(&mut consumer);
2960        assert_eq!(recv.token(), token);
2961        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"fire-and-forget");
2962        assert!(matches!(reply, ReplyChain::Ack(_)));
2963        consumer.complete(recv, reply).unwrap();
2964
2965        let used = producer.poll().unwrap().unwrap();
2966        assert!(matches!(used, UsedChain::Ack(t) if t == token));
2967    }
2968
2969    #[test]
2970    fn test_readwrite_round_trip() {
2971        let ring = make_ring(16);
2972        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2973
2974        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2975        se.write_all(b"request data").unwrap();
2976        let token = producer.submit(se).unwrap();
2977
2978        let (recv, reply) = poll_received(&mut consumer);
2979        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"request data");
2980        if let ReplyChain::Writable(mut wc) = reply {
2981            wc.write_all(b"response data").unwrap();
2982            consumer.complete(recv, wc).unwrap();
2983        } else {
2984            panic!("expected Writable");
2985        }
2986
2987        let used = producer.poll().unwrap().unwrap();
2988        assert_eq!(used.token(), token);
2989        assert_eq!(used.to_bytes().unwrap().as_ref(), b"response data");
2990    }
2991
2992    #[test]
2993    fn test_poll_used_requires_owned_mapping() {
2994        let ring = make_ring(16);
2995        let layout = ring.layout();
2996        let test_mem = ring.mem();
2997        let pool_base = test_mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
2998        let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
2999        let pool = SlotPool::new(pool_layout).unwrap();
3000        let notifier = TestNotifier::new();
3001        let mem = FaultMem::new(test_mem);
3002        mem.deny_views();
3003        let mut producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool.clone());
3004        let mut consumer = VirtqConsumer::new(layout, mem, notifier);
3005
3006        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3007        se.write_all(b"request data").unwrap();
3008        producer.submit(se).unwrap();
3009
3010        let (recv, reply) = poll_received(&mut consumer);
3011        if let ReplyChain::Writable(mut wc) = reply {
3012            wc.write_all(b"response data").unwrap();
3013            consumer.complete(recv, wc).unwrap();
3014        } else {
3015            panic!("expected Writable");
3016        }
3017
3018        assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
3019        assert_eq!(pool.num_live(), 0);
3020    }
3021
3022    #[test]
3023    fn test_villain_used_len_exceeding_writable_capacity_is_rejected_and_released() {
3024        let ring = make_ring(16);
3025        let (mut producer, _consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
3026        let mut ring_consumer = make_consumer(&ring);
3027
3028        let se = producer.chain().writable(4).build().unwrap();
3029        producer.submit(se).unwrap();
3030
3031        let (id, _) = ring_consumer.poll_available().unwrap();
3032        ring_consumer.submit_used(id, 8).unwrap();
3033
3034        assert!(matches!(producer.poll(), Err(VirtqError::InvalidState)));
3035        assert_eq!(producer.inner.num_inflight(), 0);
3036
3037        let se = producer.chain().writable(4).build().unwrap();
3038        producer.submit(se).unwrap();
3039        assert_eq!(producer.inner.num_inflight(), 1);
3040
3041        // SAFETY: Both consumers stay inactive and no chain handles survive.
3042        unsafe { producer.reset() }.unwrap();
3043    }
3044
3045    #[test]
3046    fn test_villain_used_descriptor_with_invalid_id_is_rejected() {
3047        let ring = make_ring(16);
3048        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
3049
3050        let se = producer.chain().writable(4).build().unwrap();
3051        producer.submit(se).unwrap();
3052
3053        let mut desc = Descriptor::new(0, 0, ring.len() as u16, DescFlags::empty());
3054        desc.mark_used(true);
3055        ring.write_desc(0, desc);
3056
3057        assert!(matches!(
3058            producer.poll(),
3059            Err(VirtqError::RingError(RingError::InvalidState))
3060        ));
3061        assert_eq!(producer.inner.num_inflight(), 1);
3062
3063        // SAFETY: The consumer has never polled the ring and stays inactive.
3064        unsafe { producer.reset() }.unwrap();
3065    }
3066}