Skip to main content

hyperlight_common/virtq/
producer.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4use alloc::collections::VecDeque;
5use alloc::vec;
6use alloc::vec::Vec;
7use core::mem::ManuallyDrop;
8
9use bytes::Bytes;
10use smallvec::SmallVec;
11
12use super::*;
13
14/// A used chain observed by the driver (producer) side.
15///
16/// Read-only chains are returned as [`Ack`](Self::Ack). Chains with a writable
17/// buffer complete as [`Data`](Self::Data), even when the device wrote zero
18/// bytes. Non-empty segments use backend-owned mappings. Borrowed mappings
19/// keep their pool slots allocated until the last [`Bytes`] owner drops.
20#[derive(Debug)]
21pub enum UsedChain {
22    /// Acknowledgement for a read-only/fire-and-forget chain.
23    Ack(Token),
24    /// Data written by the consumer into the chain's writable buffers.
25    ///
26    /// The payload may contain zero bytes when the consumer writes zero bytes;
27    /// that is still a data used chain because the submitted chain had
28    /// writable capacity.
29    Data(Token, Segments),
30}
31
32impl UsedChain {
33    /// Token identifying which submitted chain this used chain corresponds to.
34    pub fn token(&self) -> Token {
35        match self {
36            Self::Ack(token) | Self::Data(token, _) => *token,
37        }
38    }
39
40    /// Data written by the consumer as contiguous bytes, if this chain has data.
41    pub fn to_bytes(&self) -> Option<Bytes> {
42        match self {
43            Self::Ack(_) => None,
44            Self::Data(_, segments) => Some(segments.to_bytes()),
45        }
46    }
47
48    /// Segments written by the consumer, if this chain has data.
49    pub fn segments(&self) -> Option<&Segments> {
50        match self {
51            Self::Ack(_) => None,
52            Self::Data(_, segments) => Some(segments),
53        }
54    }
55
56    /// Consume the used chain and return written data as contiguous bytes, if present.
57    pub fn into_bytes(self) -> Option<Bytes> {
58        match self {
59            Self::Ack(_) => None,
60            Self::Data(_, segments) => Some(segments.into_bytes()),
61        }
62    }
63
64    /// Consume the used chain and return written segments, if present.
65    pub fn into_segments(self) -> Option<Segments> {
66        match self {
67            Self::Ack(_) => None,
68            Self::Data(_, segments) => Some(segments),
69        }
70    }
71}
72
73struct Inflight<M> {
74    token: Token,
75    // Automatic cleanup could recycle buffers still accessible to the peer.
76    // Only completion or a stopped reset permits releasing this owner.
77    chain: ManuallyDrop<OwnedChain<M>>,
78}
79
80/// Compact in-flight chains with constant-time descriptor-ID lookup.
81///
82/// Descriptor IDs span the full ring, but live chains are normally bounded by
83/// the much smaller buffer pool. `by_id` maps each descriptor ID to a packed
84/// `live` index. Removal uses `swap_remove` and repairs the moved entry's map.
85struct InflightTable<M> {
86    by_id: Vec<u16>,
87    live: Vec<Inflight<M>>,
88}
89
90impl<M> InflightTable<M> {
91    const VACANT: u16 = u16::MAX;
92
93    fn new(ring_len: usize) -> Self {
94        Self {
95            by_id: vec![Self::VACANT; ring_len],
96            live: Vec::new(),
97        }
98    }
99
100    fn try_reserve_one(&mut self) -> Result<(), VirtqError> {
101        if self.live.len() > self.by_id.len() {
102            return Err(VirtqError::InvalidState);
103        }
104
105        if self.live.len() == self.by_id.len() {
106            return Err(VirtqError::Backpressure);
107        }
108
109        // Producers with one live chain should not pay for four large inline
110        // chain records.
111        let result = if self.live.capacity() == 0 {
112            self.live.try_reserve_exact(1)
113        } else {
114            self.live.try_reserve(1)
115        };
116
117        result.map_err(|_| VirtqError::Bookkeeping)
118    }
119
120    fn contains(&self, id: u16) -> bool {
121        self.by_id
122            .get(id as usize)
123            .is_some_and(|slot| *slot != Self::VACANT)
124    }
125
126    fn insert(&mut self, inflight: Inflight<M>) {
127        let id = inflight.token.id;
128        debug_assert!(!self.contains(id));
129        debug_assert!(self.live.len() < Self::VACANT as usize);
130
131        let slot = self.live.len() as u16;
132        self.live.push(inflight);
133        self.by_id[id as usize] = slot;
134    }
135
136    fn remove(&mut self, id: u16) -> Option<Inflight<M>> {
137        let slot = self.by_id.get_mut(id as usize)?;
138        if *slot == Self::VACANT {
139            return None;
140        }
141
142        let index = usize::from(*slot);
143        *slot = Self::VACANT;
144
145        let removed = self.live.swap_remove(index);
146        if let Some(moved) = self.live.get(index) {
147            self.by_id[moved.token.id as usize] = index as u16;
148        }
149
150        Some(removed)
151    }
152
153    fn pop(&mut self) -> Option<Inflight<M>> {
154        let inflight = self.live.pop()?;
155        self.by_id[inflight.token.id as usize] = Self::VACANT;
156        Some(inflight)
157    }
158}
159
160/// A high-level virtqueue producer (driver side).
161///
162/// The producer sends chains to the consumer (device), and receives used chains.
163/// This is used on the driver/guest side.
164///
165/// # Threading
166///
167/// The producer and its pool use single-threaded allocation state.
168/// [`BufferMap`] supplies the complete `Send` owner for returned [`Bytes`].
169///
170/// # Cleanup
171///
172/// Dropping a producer does not stop its peer, so inflight owners are
173/// deliberately leaked to avoid freeing buffers the peer may still use.
174/// Drain completions for all submitted chains, or stop the peer and call
175/// [`reset`](Self::reset), before dropping the producer.
176///
177/// # Example
178///
179/// ```ignore
180/// let mut producer = VirtqProducer::new(layout, mem, notifier, pool);
181///
182/// // Build and submit a chain
183/// let mut chain = producer.chain().readable(64).writable(64).build()?;
184/// chain.write_all(b"hello")?;
185/// let token = producer.submit(chain)?;
186///
187/// // Later, poll for the used chain
188/// if let Some(used) = producer.poll()? {
189///     assert_eq!(used.token(), token);
190///     match used {
191///         UsedChain::Data(_, segments) => println!("Got used chain: {:?}", segments),
192///         UsedChain::Ack(_) => println!("Got ack"),
193///     }
194/// }
195/// ```
196pub struct VirtqProducer<M, N> {
197    inner: RingProducer<M>,
198    notifier: N,
199    pool: SlotPool,
200    next_token: u32,
201    inflight: InflightTable<M>,
202    pending: VecDeque<UsedChain>,
203}
204
205impl<M, N> VirtqProducer<M, N>
206where
207    M: MemOps + Clone,
208    N: Notifier,
209{
210    /// Create a new virtqueue producer.
211    ///
212    /// # Arguments
213    ///
214    /// * `layout` - Ring memory layout (descriptor table and event suppression addresses)
215    /// * `mem` - Memory operations implementation for reading/writing to shared memory
216    /// * `notifier` - Callback for notifying the device (consumer) about new chains
217    /// * `pool` - Buffer allocator for chain payload and reply data
218    pub fn new(layout: Layout, mem: M, notifier: N, pool: SlotPool) -> Self {
219        let inner = RingProducer::new(layout, mem);
220        let ring_len = inner.len();
221        let inflight = InflightTable::new(ring_len);
222
223        Self {
224            inner,
225            pool,
226            notifier,
227            inflight,
228            next_token: 0,
229            pending: VecDeque::new(),
230        }
231    }
232
233    /// Borrow the pool used for new chains.
234    pub fn pool(&self) -> &SlotPool {
235        &self.pool
236    }
237
238    /// Borrow the backend used for ring access and new chains.
239    pub fn memory(&self) -> &M {
240        self.inner.mem()
241    }
242
243    /// Reclaim scratch slots held by retained views without changing in-flight chains.
244    ///
245    /// Retained views keep their original pools. Buffered completions or chains
246    /// from another allocator return [`VirtqError::InvalidState`].
247    ///
248    /// # Safety
249    ///
250    /// Allocations held outside this producer must have backing independent of
251    /// reused scratch. After a swap, old pool handles must not allocate again.
252    /// No unsent chains or builders may remain. Peer accesses must be stopped.
253    pub unsafe fn reclaim_slots(&mut self) -> Result<(), VirtqError> {
254        let live = &mut self.inflight.live;
255        // All pending used chains must be drained before we can swap pools
256        if !self.pending.is_empty() {
257            return Err(VirtqError::InvalidState);
258        }
259
260        // inflight chains must have the same pool as the producer before we swap
261        if live.iter().any(|inf| !self.pool.ptr_eq(&inf.chain.pool)) {
262            return Err(VirtqError::InvalidState);
263        }
264
265        let reserved = live
266            .iter()
267            .flat_map(|inf| inf.chain.buffers.iter().map(|buf| buf.addr));
268
269        // No retained slots need reclaiming when all live slots are in-flight.
270        if self.pool.num_live() == reserved.clone().count() {
271            return Ok(());
272        }
273
274        let pool = self.pool.swap(reserved)?;
275        for inf in live {
276            inf.chain.pool = pool.clone();
277        }
278
279        self.pool = pool;
280        Ok(())
281    }
282
283    /// Begin building a descriptor chain for submission.
284    ///
285    /// The builder captures the current free-descriptor budget.
286    /// Submission checks capacity again.
287    pub fn chain(&self) -> ChainBuilder<M> {
288        ChainBuilder::new(
289            self.inner.mem().clone(),
290            self.pool.clone(),
291            self.inner.num_free(),
292        )
293    }
294
295    /// Preferred size of one bulk payload segment.
296    pub fn preferred_segment_len(&self) -> usize {
297        self.pool.slot_size()
298    }
299
300    /// Begin a batch of submissions.
301    ///
302    /// Chains submitted through the returned [`SubmitBatch`] are published to
303    /// the ring immediately, but the consumer is notified at most once when
304    /// [`SubmitBatch::finish`] is called. This mirrors the virtio pattern of
305    /// adding multiple buffers and then kicking the queue once.
306    pub fn batch(&mut self) -> SubmitBatch<'_, M, N> {
307        SubmitBatch::new(self)
308    }
309
310    /// Submit a [`SendChain`] to the ring.
311    ///
312    /// Publishes the descriptor chain, stores the in-flight tracking state,
313    /// and notifies the consumer if event suppression allows. Notifications
314    /// are layout-neutral; use [`batch`](Self::batch) when a higher-level
315    /// protocol wants to publish multiple chains and kick once.
316    ///
317    /// # Errors
318    ///
319    /// * [`VirtqError::Backpressure`] - ring or in-flight tracking is full
320    /// * [`VirtqError::RingError`] - publication or notification memory access failed
321    /// * [`VirtqError::InvalidState`] - descriptor ID collision
322    pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
323        let cursor_before = self.inner.avail_cursor();
324        let token = self.publish(chain)?;
325        self.notify_since(cursor_before)?;
326        Ok(token)
327    }
328
329    fn publish(&mut self, send: SendChain<M>) -> Result<Token, VirtqError> {
330        self.inflight.try_reserve_one()?;
331
332        if send.desc_count() > self.inner.num_free() {
333            return Err(VirtqError::Backpressure);
334        }
335        let token_id = self.next_token;
336        let id = self.inner.next_id()?;
337        let token = Token { seq: token_id, id };
338
339        // A free descriptor id must never already be tracked as inflight.
340        if self.inflight.contains(id) {
341            return Err(VirtqError::InvalidState);
342        }
343
344        let mut descriptors = send.owned.descriptors();
345        let mut builder = BufferChainBuilder::new();
346
347        builder.reserve_exact(send.desc_count());
348        let chain = builder
349            .readables(descriptors.by_ref().take(send.rd_desc_count()))
350            .writables(descriptors)
351            .build()?;
352
353        let published = self.inner.submit_available(&chain);
354
355        // A failed write can still publish descriptors. Keep their allocations
356        // until a stopped reset, including when submission reports an error.
357        let inf = Inflight {
358            token,
359            chain: ManuallyDrop::new(send.owned),
360        };
361        self.inflight.insert(inf);
362        self.next_token = self.next_token.wrapping_add(1);
363
364        let published_id = published?;
365        debug_assert_eq!(published_id, id);
366        Ok(token)
367    }
368
369    fn notify_since(&mut self, cursor: RingCursor) -> Result<bool, VirtqError> {
370        let should_notify = self.inner.should_notify_since(cursor)?;
371        if should_notify {
372            self.notify_now();
373        }
374        Ok(should_notify)
375    }
376
377    fn notify_now(&self) {
378        self.notifier.notify(QueueStats {
379            num_free: self.inner.num_free(),
380            num_inflight: self.inner.num_inflight(),
381        });
382    }
383
384    /// Signal backpressure to the consumer.
385    ///
386    /// Bypasses event suppression. Call this when submit fails with a
387    /// backpressure error and the consumer needs to drain.
388    #[inline]
389    pub fn notify_backpressure(&self) {
390        self.notify_now();
391    }
392
393    /// Get the current used cursor position.
394    ///
395    /// Useful for setting up descriptor-based event suppression.
396    #[inline]
397    pub fn used_cursor(&self) -> RingCursor {
398        self.inner.used_cursor()
399    }
400
401    /// Number of free (unsubmitted) descriptors in the ring.
402    #[inline]
403    pub fn num_free(&self) -> usize {
404        self.inner.num_free()
405    }
406
407    /// Number of submitted descriptors not yet polled as used.
408    #[inline]
409    pub fn num_inflight(&self) -> usize {
410        self.inner.num_inflight()
411    }
412
413    /// Reset a stopped producer and release transport-owned allocations.
414    ///
415    /// Buffered writable completions are guest-owned and make this operation fail.
416    /// Owner-backed payloads already returned to callers are not tracked as
417    /// in-flight and remain allocated.
418    ///
419    /// # Safety
420    ///
421    /// All consumer-side chain handles must be dropped. The peer must stay
422    /// stopped until its consumer is reset or replaced. Surviving handles
423    /// could access recycled buffers, violating ownership of borrowed views.
424    ///
425    /// ```compile_fail,E0133
426    /// # use hyperlight_common::virtq::{MemOps, Notifier, VirtqProducer};
427    /// fn reset<M: MemOps + Clone, N: Notifier>(producer: &mut VirtqProducer<M, N>) {
428    ///     producer.reset().unwrap();
429    /// }
430    /// ```
431    pub unsafe fn reset(&mut self) -> Result<(), VirtqError> {
432        if !self.pending.is_empty() {
433            return Err(VirtqError::InvalidState);
434        }
435
436        self.inner.reset()?;
437        self.next_token = 0;
438
439        let mut maybe_err = None;
440
441        while let Some(inflight) = self.inflight.pop() {
442            let ret = ManuallyDrop::into_inner(inflight.chain).release();
443            if let Err(err) = ret
444                && maybe_err.is_none()
445            {
446                maybe_err = Some(err);
447            }
448        }
449
450        match maybe_err {
451            Some(error) => Err(error.into()),
452            None => Ok(()),
453        }
454    }
455
456    /// Configure event suppression for used buffer notifications.
457    ///
458    /// This controls when the device (consumer) signals us about completed buffers:
459    ///
460    /// - [`SuppressionKind::Enable`]: Always signal (default) - good for latency
461    /// - [`SuppressionKind::Disable`]: Never signal - caller must poll
462    /// - [`SuppressionKind::Descriptor`]: Signal only at specific cursor position
463    ///
464    /// # Example: Used-chain batching
465    ///
466    /// ```ignore
467    /// // Submit chains, then suppress notifications until all are used
468    /// let mut se = producer.chain().readable(64).writable(128).build()?;
469    /// se.write_all(b"entry1")?;
470    /// producer.submit(se)?;
471    /// let cursor = producer.used_cursor();
472    /// producer.set_used_suppression(SuppressionKind::Descriptor(cursor))?;
473    /// // Device will notify only after reaching that cursor position
474    /// ```
475    pub fn set_used_suppression(&mut self, kind: SuppressionKind) -> Result<(), VirtqError> {
476        match kind {
477            SuppressionKind::Enable => self.inner.enable_used_notifications()?,
478            SuppressionKind::Disable => self.inner.disable_used_notifications()?,
479            SuppressionKind::Descriptor(cursor) => self
480                .inner
481                .enable_used_notifications_desc(cursor.head(), cursor.wrap())?,
482        }
483        Ok(())
484    }
485}
486
487impl<M, N> VirtqProducer<M, N>
488where
489    M: BufferMap + Clone,
490    N: Notifier,
491{
492    /// Poll for a single used chain from the device.
493    ///
494    /// Returns buffered used chains from prior [`reclaim`](Self::reclaim)
495    /// calls first, then checks the ring for newly used chains.
496    ///
497    /// Returns `Ok(Some(used))` if a used chain is available, `Ok(None)` if no
498    /// used chains are ready (would block), or an error if the device misbehaved.
499    ///
500    /// Data used chains contain [`Bytes`] owned by the [`BufferMap`] backend.
501    /// Borrowed views retain their pool allocations until the last clone drops.
502    ///
503    /// # Errors
504    ///
505    /// - [`VirtqError::InvalidState`] - Device returned invalid descriptor ID or
506    ///   wrote more data than the writable buffer capacity
507    pub fn poll(&mut self) -> Result<Option<UsedChain>, VirtqError> {
508        if let Some(chain) = self.pending.pop_front() {
509            return Ok(Some(chain));
510        }
511        self.poll_ring()
512    }
513
514    /// Reclaim ring slots and pool allocations from used descriptors.
515    ///
516    /// Processes all available used chains from the ring: frees readable
517    /// buffer allocations immediately, and buffers writable data for
518    /// later retrieval via [`poll`](Self::poll).
519    ///
520    /// Read-only ack used chains are discarded immediately.
521    ///
522    /// Use this to free resources under backpressure without losing
523    /// writable data. Returns the number of chains reclaimed.
524    /// Buffered completions can span multiple ring reuse cycles.
525    pub fn reclaim(&mut self) -> Result<usize, VirtqError> {
526        let mut count = 0;
527        while let Some(chain) = self.poll_ring()? {
528            if matches!(chain, UsedChain::Data(_, _)) {
529                self.pending
530                    .try_reserve(1)
531                    .map_err(|_| VirtqError::Bookkeeping)?;
532                self.pending.push_back(chain);
533            }
534            count += 1;
535        }
536        Ok(count)
537    }
538
539    /// Poll one used chain directly from the ring (bypassing pending buffer).
540    fn poll_ring(&mut self) -> Result<Option<UsedChain>, VirtqError> {
541        let used = match self.inner.poll_used() {
542            Ok(u) => u,
543            Err(RingError::WouldBlock) => return Ok(None),
544            Err(e) => return Err(e.into()),
545        };
546
547        let inf = self
548            .inflight
549            .remove(used.id)
550            .ok_or(VirtqError::InvalidState)?;
551
552        let written = used.len as usize;
553        let Inflight { token, chain } = inf;
554
555        let chain = ManuallyDrop::into_inner(chain);
556        let used = if chain.readable == chain.buffers.len() {
557            chain.release()?;
558            UsedChain::Ack(token)
559        } else {
560            UsedChain::Data(token, chain.into_segments(written)?)
561        };
562
563        Ok(Some(used))
564    }
565
566    /// Drain all available used chains, calling the provided closure for each.
567    ///
568    /// This is a convenience method that repeatedly calls [`poll`](Self::poll)
569    /// until no more used chains are available.
570    ///
571    /// # Arguments
572    ///
573    /// * `f` - Closure called for each used chain
574    ///
575    /// # Example
576    ///
577    /// ```ignore
578    /// producer.drain(|used| {
579    ///     println!("Got used chain for {:?}", used.token());
580    /// })?;
581    /// ```
582    pub fn drain(&mut self, mut f: impl FnMut(UsedChain)) -> Result<(), VirtqError> {
583        while let Some(chain) = self.poll()? {
584            f(chain);
585        }
586
587        Ok(())
588    }
589}
590
591/// A scoped batch of producer submissions.
592///
593/// Submissions are published immediately, while notification is delayed until
594/// [`finish`](Self::finish). [`finish_without_notify`](Self::finish_without_notify)
595/// supports protocols whose peer is already scheduled to inspect the queue.
596#[must_use = "finish the batch explicitly"]
597pub struct SubmitBatch<'a, M, N> {
598    producer: &'a mut VirtqProducer<M, N>,
599    notify_from: Option<RingCursor>,
600}
601
602impl<'a, M, N> SubmitBatch<'a, M, N>
603where
604    M: MemOps + Clone,
605    N: Notifier,
606{
607    fn new(producer: &'a mut VirtqProducer<M, N>) -> Self {
608        Self {
609            producer,
610            notify_from: None,
611        }
612    }
613
614    /// Begin building a descriptor chain for this batch.
615    pub fn chain(&self) -> ChainBuilder<M> {
616        self.producer.chain()
617    }
618
619    /// Publish a chain as part of this batch without notifying yet.
620    pub fn submit(&mut self, chain: SendChain<M>) -> Result<Token, VirtqError> {
621        let cursor_before = self.producer.inner.avail_cursor();
622        let token = self.producer.publish(chain)?;
623
624        if self.notify_from.is_none() {
625            self.notify_from = Some(cursor_before);
626        }
627        Ok(token)
628    }
629
630    /// Finish the batch and notify the consumer once if event suppression
631    /// requires it for the whole published range.
632    ///
633    /// Returns `true` if a notification was sent.
634    pub fn finish(mut self) -> Result<bool, VirtqError> {
635        let Some(notify_from) = self.notify_from.take() else {
636            return Ok(false);
637        };
638
639        self.producer.notify_since(notify_from)
640    }
641
642    /// Finish the batch without notifying the consumer.
643    ///
644    /// Use this only when another protocol event guarantees that the consumer
645    /// will inspect the published descriptors.
646    pub fn finish_without_notify(self) {}
647}
648
649/// Builder for configuring a descriptor chain's buffer layout.
650///
651/// If dropped without building, no resources are leaked (allocations are
652/// deferred to [`build`](Self::build)).
653#[must_use = "call .build() to create a SendChain"]
654pub struct ChainBuilder<M: MemOps> {
655    mem: M,
656    pool: SlotPool,
657    rd_caps: SmallVec<[usize; 4]>,
658    wr_caps: SmallVec<[usize; 4]>,
659    writable_avail: bool,
660    max_descs: usize,
661}
662
663impl<M: MemOps> ChainBuilder<M> {
664    fn new(mem: M, pool: SlotPool, max_descs: usize) -> Self {
665        Self {
666            mem,
667            pool,
668            rd_caps: SmallVec::new(),
669            wr_caps: SmallVec::new(),
670            writable_avail: false,
671            max_descs,
672        }
673    }
674
675    /// Request a device-readable buffer of `cap` bytes.
676    ///
677    /// The producer writes data into readable buffers before submission; the
678    /// consumer reads that data after polling the chain.
679    /// The actual allocation is deferred to [`build`](Self::build).
680    pub fn readable(mut self, cap: usize) -> Self {
681        self.rd_caps.push(cap);
682        self
683    }
684
685    /// Request a device-writable buffer of `cap` bytes.
686    ///
687    /// The writable buffer is filled by the consumer and returned via
688    /// [`VirtqProducer::poll`] as [`UsedChain`].
689    ///
690    /// Multiple writable buffers are completed as ordered [`Segments`]. The
691    /// consumer writes them sequentially, because the virtio used ring reports
692    /// one aggregate written length rather than per-descriptor lengths.
693    pub fn writable(mut self, cap: usize) -> Self {
694        self.wr_caps.push(cap);
695        self
696    }
697
698    /// Request available upper-tier buffers within the ring budget.
699    ///
700    /// [`build`](Self::build) allocates these after all explicit requests.
701    /// This may add zero buffers. The complete chain must be nonempty.
702    pub fn writable_avail(mut self) -> Self {
703        self.writable_avail = true;
704        self
705    }
706
707    /// Allocate buffers and return a [`SendChain`] for writing.
708    ///
709    /// # Errors
710    ///
711    /// * [`VirtqError::InvalidState`] - no buffers requested
712    /// * [`VirtqError::Backpressure`] - insufficient pool slots or ring descriptors
713    /// * [`VirtqError::Bookkeeping`] - buffer record storage allocation failed
714    /// * [`VirtqError::Alloc`] - zero-length request or buffer allocation failed
715    pub fn build(self) -> Result<SendChain<M>, VirtqError> {
716        if self.rd_caps.is_empty() && self.wr_caps.is_empty() && !self.writable_avail {
717            return Err(VirtqError::InvalidState);
718        }
719
720        // Count explicit descriptors against the captured ring budget.
721        let slot_size = self.pool.slot_size();
722        let rd_capacity = self.rd_caps.iter().try_fold(0usize, |total, &cap| {
723            total.checked_add(cap).ok_or(AllocError::Overflow)
724        })?;
725
726        let mut caps = self.rd_caps.iter().chain(&self.wr_caps);
727        let desc_count = caps.try_fold(0usize, |total, &cap| {
728            if cap == 0 {
729                return Err(AllocError::InvalidArg);
730            }
731            total
732                .checked_add(cap.div_ceil(slot_size))
733                .ok_or(AllocError::Overflow)
734        })?;
735
736        let remaining_descs = self
737            .max_descs
738            .checked_sub(desc_count)
739            .ok_or(VirtqError::Backpressure)?;
740
741        // Reserve explicit records before taking slots. OwnedChain handles rollback.
742        let mut buffers = SmallVec::new();
743        buffers
744            .try_reserve_exact(desc_count)
745            .map_err(|_| VirtqError::Bookkeeping)?;
746
747        let mut owned = OwnedChain {
748            mem: self.mem,
749            pool: self.pool,
750            buffers,
751            readable: 0,
752        };
753
754        // Allocate readable regions before writable ones, splitting each at the upper slot size.
755        let regions = self
756            .rd_caps
757            .iter()
758            .map(|&len| (len, false))
759            .chain(self.wr_caps.iter().map(|&len| (len, true)));
760
761        for (total_len, wr) in regions {
762            let mut remaining = total_len;
763
764            while remaining > 0 {
765                let len = remaining.min(slot_size);
766                let alloc = owned.pool.alloc(len)?;
767                let capacity = if wr { alloc.len } else { len as u32 };
768
769                let ent = BufferEntry {
770                    addr: alloc.addr,
771                    capacity,
772                    written: 0,
773                };
774
775                owned.buffers.push(ent);
776                owned.readable += usize::from(!wr);
777
778                remaining -= len;
779            }
780        }
781
782        // Use spare descriptors for upper-tier slots left after the explicit allocations.
783        if self.writable_avail {
784            let extra = owned.pool.num_free_upper().min(remaining_descs);
785            owned
786                .buffers
787                .try_reserve_exact(extra)
788                .map_err(|_| VirtqError::Bookkeeping)?;
789
790            for _ in 0..extra {
791                let alloc = owned.pool.alloc(slot_size)?;
792                let ent = BufferEntry {
793                    addr: alloc.addr,
794                    capacity: alloc.len,
795                    written: 0,
796                };
797
798                owned.buffers.push(ent);
799            }
800        }
801
802        // An availability-only request can leave no buffers to publish.
803        if owned.buffers.is_empty() {
804            return Err(VirtqError::Backpressure);
805        }
806
807        Ok(SendChain {
808            owned,
809            rd_capacity,
810            rd_written: 0,
811            write_mode: WriteMode::Unset,
812        })
813    }
814}
815
816/// Tracks which write API a [`SendChain`] payload uses, so the two paths are
817/// not mixed.
818///
819/// Copy writes ([`SendChain::write`]/[`SendChain::write_all`]) append at an
820/// aggregate cursor, while direct writes
821/// ([`SendChain::write_seg`]/[`SendChain::with_seg`]) set per-segment lengths
822/// absolutely. Mixing them would corrupt the written-length accounting.
823#[derive(Debug, Clone, Copy, PartialEq, Eq)]
824enum WriteMode {
825    Unset,
826    Append,
827    Direct,
828}
829
830/// A configured send chain ready for writing and submission.
831///
832/// Created by [`ChainBuilder::build`]. Write readable payload bytes directly on
833/// the chain, then submit via [`VirtqProducer::submit`].
834///
835/// # Examples
836///
837/// ```ignore
838/// let mut sc = producer.chain().readable(64).writable(128).build()?;
839/// sc.write_all(b"header")?.write_all(b" body")?;
840/// let tok = producer.submit(sc)?;
841///
842/// let mut sc = producer.chain().readable(128).build()?;
843/// sc.with_seg(0, |buf| serialize_into(buf))?;
844/// let tok = producer.submit(sc)?;
845/// ```
846///
847/// Copy writes (`write`/`write_all`) and direct writes (`write_seg`/`with_seg`)
848/// must not be mixed on the same chain; doing so panics in debug builds.
849///
850/// If dropped without submitting, allocated buffers are returned to the pool.
851#[must_use = "dropping without submitting deallocates the buffers"]
852pub struct SendChain<M> {
853    owned: OwnedChain<M>,
854    rd_capacity: usize,
855    rd_written: usize,
856    write_mode: WriteMode,
857}
858
859impl<M: MemOps> SendChain<M> {
860    /// Record that this chain uses `mode`, asserting it is not mixed with the
861    /// other write path.
862    fn note_write_mode(&mut self, mode: WriteMode) {
863        debug_assert!(
864            self.write_mode == WriteMode::Unset || self.write_mode == mode,
865            "SendChain mixes copy writes (write/write_all) with direct writes (write_seg/with_seg)"
866        );
867        self.write_mode = mode;
868    }
869
870    /// Total number of descriptors in this chain.
871    #[inline]
872    pub fn desc_count(&self) -> usize {
873        self.owned.buffers.len()
874    }
875
876    /// Number of readable descriptors in this chain.
877    #[inline]
878    pub fn rd_desc_count(&self) -> usize {
879        self.owned.readable
880    }
881
882    /// Number of writable descriptors in this chain.
883    #[inline]
884    pub fn wr_desc_count(&self) -> usize {
885        self.desc_count() - self.rd_desc_count()
886    }
887
888    /// Total producer-written readable capacity in bytes.
889    #[inline]
890    pub fn capacity(&self) -> usize {
891        self.rd_capacity
892    }
893
894    /// Number of producer-written readable bytes written so far.
895    #[inline]
896    pub fn written(&self) -> usize {
897        self.rd_written
898    }
899
900    /// Remaining producer-written readable capacity.
901    #[inline]
902    pub fn remaining(&self) -> usize {
903        self.capacity() - self.written()
904    }
905
906    /// Write bytes into payload segments, returning how many bytes were written.
907    ///
908    /// Appends at the current aggregate write position and scatters across
909    /// readable segments in chain order. Uses [`MemOps::write`] (volatile on
910    /// host side). If `buf` is larger than the remaining capacity, writes as
911    /// many bytes as will fit. If a later memory write fails, the cursor and
912    /// written length retain any earlier chunks written by the same call.
913    ///
914    /// # Errors
915    ///
916    /// - [`VirtqError::NoPayloadSegment`] - no readable buffer allocated
917    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
918    pub fn write(&mut self, buf: &[u8]) -> Result<usize, VirtqError> {
919        if self.rd_desc_count() == 0 {
920            return Err(VirtqError::NoPayloadSegment);
921        }
922
923        self.note_write_mode(WriteMode::Append);
924
925        let mut remaining = &buf[..buf.len().min(self.remaining())];
926        let mut written = 0;
927
928        for buffer in &mut self.owned.buffers[..self.owned.readable] {
929            if remaining.is_empty() {
930                break;
931            }
932
933            let cap = buffer.capacity as usize;
934            let desc_off = buffer.written as usize;
935            let len = (cap - desc_off).min(remaining.len());
936            if len == 0 {
937                continue;
938            }
939
940            let addr = buffer
941                .addr
942                .checked_add(desc_off as u64)
943                .ok_or(VirtqError::MemoryWriteError)?;
944
945            self.owned
946                .mem
947                .write(addr, &remaining[..len])
948                .map_err(|_| VirtqError::MemoryWriteError)?;
949
950            buffer.written += len as u32;
951            self.rd_written += len;
952            written += len;
953            remaining = &remaining[len..];
954        }
955
956        Ok(written)
957    }
958
959    /// Write the entire buffer into payload segments.
960    ///
961    /// Appends at the current aggregate write position and scatters across
962    /// readable segments in chain order. Uses [`MemOps::write`] (volatile on
963    /// host side).
964    ///
965    /// # Errors
966    ///
967    /// - [`VirtqError::PayloadTooLarge`] - buf exceeds remaining capacity
968    /// - [`VirtqError::NoPayloadSegment`] - no readable buffer allocated
969    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
970    #[inline]
971    pub fn write_all(&mut self, buf: &[u8]) -> Result<&mut Self, VirtqError> {
972        if self.rd_desc_count() == 0 {
973            return Err(VirtqError::NoPayloadSegment);
974        }
975
976        if buf.len() > self.remaining() {
977            return Err(VirtqError::PayloadTooLarge {
978                recv: buf.len(),
979                limit: self.remaining(),
980            });
981        }
982
983        let written = self.write(buf)?;
984        debug_assert_eq!(written, buf.len());
985        Ok(self)
986    }
987
988    /// Write bytes into one readable segment by index.
989    ///
990    /// Writes from the start of the selected segment and records `buf.len()` as
991    /// that segment's descriptor length.
992    ///
993    /// # Errors
994    ///
995    /// - [`VirtqError::NoPayloadSegment`] - `index` does not name a payload segment
996    /// - [`VirtqError::PayloadTooLarge`] - `buf` exceeds the segment capacity
997    /// - [`VirtqError::MemoryWriteError`] - underlying write failed
998    pub fn write_seg(&mut self, index: usize, buf: &[u8]) -> Result<&mut Self, VirtqError> {
999        self.note_write_mode(WriteMode::Direct);
1000
1001        let buffer = self.owned.buffers[..self.owned.readable]
1002            .get_mut(index)
1003            .ok_or(VirtqError::NoPayloadSegment)?;
1004
1005        let cap = buffer.capacity as usize;
1006        if buf.len() > cap {
1007            return Err(VirtqError::PayloadTooLarge {
1008                recv: buf.len(),
1009                limit: cap,
1010            });
1011        }
1012
1013        self.owned
1014            .mem
1015            .write(buffer.addr, buf)
1016            .map_err(|_| VirtqError::MemoryWriteError)?;
1017
1018        let previous = buffer.written as usize;
1019        buffer.written = checked_descriptor_len(buf.len())?;
1020        self.rd_written = self.rd_written - previous + buf.len();
1021        Ok(self)
1022    }
1023
1024    /// Serialize directly into one readable segment by index.
1025    ///
1026    /// The closure returns the number of valid bytes it wrote. The written
1027    /// length for that segment is recorded on success.
1028    ///
1029    /// # Errors
1030    ///
1031    /// - [`VirtqError::NoPayloadSegment`] - `index` does not name a payload segment
1032    /// - [`VirtqError::PayloadTooLarge`] - closure reports more bytes than segment capacity
1033    /// - [`VirtqError::MemoryWriteError`] - the memory backend cannot expose a mutable slice
1034    pub fn with_seg<E>(
1035        &mut self,
1036        index: usize,
1037        f: impl FnOnce(&mut [u8]) -> Result<usize, E>,
1038    ) -> Result<&mut Self, E>
1039    where
1040        E: From<VirtqError>,
1041    {
1042        self.note_write_mode(WriteMode::Direct);
1043
1044        let buffer = self.owned.buffers[..self.owned.readable]
1045            .get_mut(index)
1046            .ok_or_else(|| E::from(VirtqError::NoPayloadSegment))?;
1047
1048        let cap = buffer.capacity as usize;
1049
1050        // SAFETY: This unpublished chain owns the allocation exclusively.
1051        let buf = unsafe {
1052            self.owned
1053                .mem
1054                .as_mut_slice(buffer.addr, cap)
1055                .map_err(|_| E::from(VirtqError::MemoryWriteError))?
1056        };
1057
1058        let written = f(buf)?;
1059        if written > buf.len() {
1060            return Err(E::from(VirtqError::PayloadTooLarge {
1061                recv: written,
1062                limit: buf.len(),
1063            }));
1064        }
1065
1066        let previous = buffer.written as usize;
1067        buffer.written = checked_descriptor_len(written).map_err(E::from)?;
1068        self.rd_written = self.rd_written - previous + written;
1069
1070        Ok(self)
1071    }
1072}
1073
1074/// Tracks a pool slot's capacity and initialized length for writes and reply mappings.
1075///
1076/// [`BufferElement::len`] carries only the length published to the peer.
1077#[derive(Debug)]
1078struct BufferEntry {
1079    /// Buffer base address used for memory access and release through the pool.
1080    addr: u64,
1081    /// Usable bytes,
1082    capacity: u32,
1083    /// Initialized prefix length
1084    written: u32,
1085}
1086
1087/// Buffer ownership passed from [`SendChain`] to [`Inflight`] at publication.
1088///
1089/// Keeps the original memory backend and the pool owning its slot reservations.
1090struct OwnedChain<M> {
1091    /// Original backend for writing payloads and mapping completed replies.
1092    mem: M,
1093    /// Shared allocator.
1094    pool: SlotPool,
1095    /// Allocation records in descriptor order.
1096    buffers: SmallVec<[BufferEntry; 4]>,
1097    /// Number of leading device readable buffers.
1098    readable: usize,
1099}
1100
1101impl<M> OwnedChain<M> {
1102    /// One [`BufferElement`] per direct descriptor, in chain order.
1103    ///
1104    /// The ring adds the descriptor IDs and flags that link the buffers.
1105    fn descriptors(&self) -> impl ExactSizeIterator<Item = BufferElement> + Clone + '_ {
1106        self.buffers.iter().enumerate().map(|(i, buf)| {
1107            let wr = i >= self.readable;
1108            let len = if wr { buf.capacity } else { buf.written };
1109
1110            BufferElement {
1111                addr: buf.addr,
1112                len,
1113                writable: wr,
1114            }
1115        })
1116    }
1117
1118    fn release(mut self) -> Result<(), AllocError> {
1119        self.release_all()
1120    }
1121
1122    fn release_all(&mut self) -> Result<(), AllocError> {
1123        let mut maybe_err = None;
1124        while let Some(buf) = self.buffers.pop() {
1125            if let Err(error) = self.pool.dealloc(buf.addr)
1126                && maybe_err.is_none()
1127            {
1128                maybe_err = Some(error);
1129            }
1130        }
1131
1132        self.readable = 0;
1133        maybe_err.map_or(Ok(()), Err)
1134    }
1135}
1136
1137impl<M: BufferMap> OwnedChain<M> {
1138    fn into_segments(mut self, written: usize) -> Result<Segments, VirtqError> {
1139        let mut remaining = written;
1140        let mut nonempty = 0;
1141
1142        for buf in &mut self.buffers[self.readable..] {
1143            let len = remaining.min(buf.capacity as usize);
1144            buf.written = len as u32;
1145
1146            nonempty += usize::from(len != 0);
1147            remaining -= len;
1148        }
1149
1150        if remaining != 0 {
1151            self.release()?;
1152            return Err(VirtqError::InvalidState);
1153        }
1154
1155        let mut segments = SmallVec::<[Bytes; 4]>::new();
1156        segments
1157            .try_reserve_exact(nonempty)
1158            .map_err(|_| VirtqError::Bookkeeping)?;
1159
1160        while let Some(buf) = self.buffers.last() {
1161            if self.buffers.len() <= self.readable || buf.written == 0 {
1162                let addr = buf.addr;
1163                self.buffers.pop();
1164                self.pool.dealloc(addr)?;
1165
1166                continue;
1167            }
1168
1169            let alloc = Allocation {
1170                addr: buf.addr,
1171                len: buf.capacity,
1172            };
1173
1174            let written = buf.written as usize;
1175            let lease = BufferLease::new(self.pool.clone(), alloc);
1176            self.buffers.pop();
1177
1178            // SAFETY: Completion returns exclusive ownership of the initialized
1179            // prefix. The mapper owns the slot until its borrowed view drops.
1180            let mapping = unsafe { self.mem.map_buffer(lease, written) }
1181                .map_err(|_| VirtqError::MemoryReadError)?;
1182
1183            segments.push(Bytes::from_owner(mapping));
1184        }
1185
1186        segments.reverse();
1187        Ok(Segments::from_smallvec(segments))
1188    }
1189}
1190
1191impl<M> Drop for OwnedChain<M> {
1192    fn drop(&mut self) {
1193        // best effort: if the pool deallocation fails, we can't do much about it here
1194        if let Err(error) = self.release_all() {
1195            log::error!("Failed to release virtqueue buffers: {error}");
1196            debug_assert!(false, "OwnedChain deallocation failed: {error}");
1197        }
1198    }
1199}
1200
1201fn checked_descriptor_len(len: usize) -> Result<u32, VirtqError> {
1202    if len > u32::MAX as usize {
1203        return Err(VirtqError::PayloadTooLarge {
1204            recv: len,
1205            limit: u32::MAX as usize,
1206        });
1207    }
1208    Ok(len as u32)
1209}
1210
1211#[cfg(test)]
1212mod tests {
1213    use alloc::rc::Rc;
1214    use alloc::sync::Arc;
1215    use core::sync::atomic::Ordering;
1216
1217    use super::*;
1218    use crate::virtq::ring::tests::{FaultMem, OwnedRing, TestMem, make_consumer, make_ring};
1219    use crate::virtq::test_utils::*;
1220
1221    fn poll_received<M: MemOps + Clone, N: Notifier>(
1222        consumer: &mut VirtqConsumer<M, N>,
1223    ) -> (RecvChain<M>, ReplyChain<M>) {
1224        consumer.poll(1024).unwrap().unwrap()
1225    }
1226
1227    #[derive(Clone)]
1228    struct CopyingMem<'a>(&'a Rc<TestMem>);
1229
1230    // SAFETY: All bounded operations delegate to TestMem.
1231    unsafe impl MemOps for CopyingMem<'_> {
1232        type Error = core::convert::Infallible;
1233
1234        fn read(&self, addr: u64, dst: &mut [u8]) -> Result<(), Self::Error> {
1235            self.0.read(addr, dst)
1236        }
1237
1238        fn write(&self, addr: u64, src: &[u8]) -> Result<(), Self::Error> {
1239            self.0.write(addr, src)
1240        }
1241
1242        fn load_acquire(&self, addr: u64) -> Result<u16, Self::Error> {
1243            self.0.load_acquire(addr)
1244        }
1245
1246        fn store_release(&self, addr: u64, val: u16) -> Result<(), Self::Error> {
1247            self.0.store_release(addr, val)
1248        }
1249
1250        unsafe fn as_slice(&self, addr: u64, len: usize) -> Result<&[u8], Self::Error> {
1251            // SAFETY: The caller supplies TestMem's slice preconditions.
1252            unsafe { self.0.as_slice(addr, len) }
1253        }
1254
1255        unsafe fn as_mut_slice(&self, addr: u64, len: usize) -> Result<&mut [u8], Self::Error> {
1256            // SAFETY: The caller supplies exclusive access to this range.
1257            unsafe { self.0.as_mut_slice(addr, len) }
1258        }
1259    }
1260
1261    impl BufferMap for CopyingMem<'_> {
1262        type Mapping = Vec<u8>;
1263
1264        unsafe fn map_buffer(
1265            &self,
1266            lease: BufferLease,
1267            written: usize,
1268        ) -> Result<Self::Mapping, Self::Error> {
1269            assert!(written <= lease.allocation().len as usize);
1270            let mut bytes = vec![0; written];
1271            self.read(lease.allocation().addr, &mut bytes)?;
1272            Ok(bytes)
1273        }
1274    }
1275
1276    fn make_virtq_pair(
1277        ring: &OwnedRing,
1278        slot_size: usize,
1279    ) -> (
1280        VirtqProducer<TestMem, TestNotifier>,
1281        VirtqConsumer<TestMem, TestNotifier>,
1282    ) {
1283        let mem = ring.mem();
1284        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1285
1286        let lower = SlotLayout::new(pool_base, slot_size / 2, ring.len()).unwrap();
1287        let upper = SlotLayout::new(lower.end_addr(), slot_size, ring.len()).unwrap();
1288        let pool = SlotPool::new_tiered(lower, upper).unwrap();
1289
1290        let notifier = TestNotifier::new();
1291        let producer = VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool);
1292        let consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1293        (producer, consumer)
1294    }
1295
1296    fn make_chain(
1297        lower_count: usize,
1298        upper_count: usize,
1299        slot_size: usize,
1300        max_descs: usize,
1301    ) -> ChainBuilder<TestMem> {
1302        let lower_len = lower_count * 256;
1303        let mem = TestMem::new(lower_len + upper_count * slot_size);
1304        let upper =
1305            SlotLayout::new(mem.base_addr() + lower_len as u64, slot_size, upper_count).unwrap();
1306
1307        let pool = if lower_count == 0 {
1308            SlotPool::new(upper)
1309        } else {
1310            let lower = SlotLayout::new(mem.base_addr(), 256, lower_count).unwrap();
1311            SlotPool::new_tiered(lower, upper)
1312        }
1313        .unwrap();
1314
1315        ChainBuilder::new(mem, pool, max_descs)
1316    }
1317
1318    fn inflight(seq: u32, id: u16) -> Inflight<TestMem> {
1319        let mem = TestMem::new(8);
1320        let pool_layout = SlotLayout::new(mem.base_addr(), 8, 1).unwrap();
1321        let pool = SlotPool::new(pool_layout).unwrap();
1322        let chain = ChainBuilder::new(mem, pool, 1).readable(8).build().unwrap();
1323        Inflight {
1324            token: Token { seq, id },
1325            chain: ManuallyDrop::new(chain.owned),
1326        }
1327    }
1328
1329    #[test]
1330    fn inflight_table_repairs_moved_entry_after_removal() {
1331        let mut table = InflightTable::new(16);
1332        for (seq, id) in [(0, 3), (1, 7), (2, 5)] {
1333            table.try_reserve_one().unwrap();
1334            table.insert(inflight(seq, id));
1335        }
1336
1337        let removed = table.remove(7).unwrap();
1338        assert_eq!(removed.token.seq, 1);
1339
1340        ManuallyDrop::into_inner(removed.chain).release().unwrap();
1341        assert!(!table.contains(7));
1342
1343        for (id, seq) in [(5, 2), (3, 0)] {
1344            let removed = table.remove(id).unwrap();
1345            assert_eq!(removed.token.seq, seq);
1346            ManuallyDrop::into_inner(removed.chain).release().unwrap();
1347        }
1348        assert!(table.live.is_empty());
1349        assert!(table.remove(7).is_none());
1350    }
1351
1352    #[test]
1353    fn producer_bookkeeping_starts_compact_and_lazy() {
1354        let ring = make_ring(64);
1355        let (producer, _consumer, _notifier) = make_test_producer(&ring);
1356
1357        assert_eq!(producer.inflight.by_id.len(), ring.len());
1358        assert!(producer.inflight.live.is_empty());
1359        assert_eq!(producer.inflight.live.capacity(), 0);
1360        assert_eq!(producer.pending.capacity(), 0);
1361    }
1362
1363    #[test]
1364    fn full_ring_still_reports_backpressure() {
1365        let ring = make_ring(4);
1366        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
1367
1368        for _ in 0..ring.len() {
1369            let chain = producer.chain().readable(1).build().unwrap();
1370            producer.submit(chain).unwrap();
1371        }
1372
1373        assert!(matches!(
1374            producer.chain().readable(1).build(),
1375            Err(VirtqError::Backpressure)
1376        ));
1377        // SAFETY: The consumer has never polled the ring and stays inactive.
1378        unsafe { producer.reset() }.unwrap();
1379    }
1380
1381    #[test]
1382    fn submission_rechecks_capacity_after_reservation() {
1383        let ring = make_ring(2);
1384        let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1385        let pool = producer.pool.clone();
1386
1387        let chain = producer.chain().writable(64).build().unwrap();
1388        for _ in 0..ring.len() {
1389            let other = producer.chain().writable(32).build().unwrap();
1390            producer.submit(other).unwrap();
1391        }
1392
1393        assert_eq!(pool.num_live(), 3);
1394        assert!(matches!(
1395            producer.submit(chain),
1396            Err(VirtqError::Backpressure)
1397        ));
1398        assert_eq!(pool.num_live(), 2);
1399
1400        // SAFETY: The consumer has never polled the ring and is reset next.
1401        unsafe { producer.reset() }.unwrap();
1402        consumer.reset().unwrap();
1403
1404        assert_eq!(pool.num_live(), 0);
1405    }
1406
1407    #[test]
1408    fn publication_failure_keeps_allocations_until_stopped_reset() {
1409        for failed_write in 0..4 {
1410            let ring = make_ring(4);
1411            let orig_mem = FaultMem::new(ring.mem());
1412            let orig_gen = Arc::downgrade(&orig_mem.0);
1413
1414            let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1415            let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1416            let pool = SlotPool::new(pool_layout).unwrap();
1417            let notif = TestNotifier::new();
1418
1419            let source = VirtqProducer::new(ring.layout(), orig_mem, notif.clone(), pool.clone());
1420
1421            let chain = source.chain().writable(64).build().unwrap();
1422            drop(source);
1423
1424            let mem = FaultMem::new(ring.mem());
1425            let mut producer =
1426                VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1427
1428            mem.fail_write_at(failed_write);
1429
1430            let res = producer.submit(chain);
1431            assert!(matches!(
1432                res,
1433                Err(VirtqError::RingError(RingError::MemError { .. }))
1434            ));
1435            assert_eq!(pool.num_live(), 1);
1436            assert_eq!(producer.inflight.live.len(), 1);
1437            assert!(orig_gen.upgrade().is_some());
1438
1439            mem.allow_writes();
1440            // SAFETY: No consumer is attached to this ring.
1441            unsafe { producer.reset() }.unwrap();
1442
1443            assert_eq!(pool.num_live(), 0);
1444            assert!(orig_gen.upgrade().is_none());
1445        }
1446    }
1447
1448    #[test]
1449    fn chain_clones_pool_only_for_independent_owners() {
1450        let ring = make_ring(8);
1451        let mem = ring.mem();
1452        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1453        let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1454        let pool = SlotPool::new(pool_layout).unwrap();
1455
1456        let notif = TestNotifier::new();
1457        let mapping = FaultMem::new(mem.clone());
1458
1459        let mut producer =
1460            VirtqProducer::new(ring.layout(), mapping.clone(), notif.clone(), pool.clone());
1461
1462        let mut consumer = VirtqConsumer::new(ring.layout(), mem, TestNotifier::new());
1463
1464        assert_eq!(pool.strong_count(), 2);
1465
1466        let mut chain = producer
1467            .chain()
1468            .readable(192)
1469            .writable(192)
1470            .build()
1471            .unwrap();
1472
1473        chain.write_all(b"request").unwrap();
1474
1475        assert_eq!(pool.num_live(), 6);
1476        assert_eq!(pool.strong_count(), 3);
1477        assert!(chain.owned.buffers.spilled());
1478
1479        let buffers = chain.owned.buffers.as_ptr();
1480        producer.submit(chain).unwrap();
1481
1482        assert_eq!(pool.strong_count(), 3);
1483        assert_eq!(producer.inflight.live[0].chain.buffers.as_ptr(), buffers);
1484
1485        let (recv, reply) = poll_received(&mut consumer);
1486        let ReplyChain::Writable(mut reply) = reply else {
1487            panic!("expected writable reply");
1488        };
1489        reply.write_all(&[0xa5; 70]).unwrap();
1490        consumer.complete(recv, reply).unwrap();
1491
1492        let segments = producer.poll().unwrap().unwrap().into_segments().unwrap();
1493
1494        assert_eq!(segments.segment_count(), 2);
1495        assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1496        assert_eq!(pool.strong_count(), 4);
1497        assert_eq!(pool.num_live(), 2);
1498
1499        let retained = segments.as_slice()[0].slice(1..);
1500        let cloned = retained.clone();
1501
1502        drop(segments);
1503
1504        assert_eq!(pool.num_live(), 1);
1505        assert_eq!(pool.strong_count(), 3);
1506
1507        // SAFETY: All consumer handles were completed. The consumer is reset next.
1508        unsafe { producer.reset() }.unwrap();
1509        consumer.reset().unwrap();
1510
1511        drop(producer);
1512
1513        assert_eq!(pool.strong_count(), 2);
1514        assert_eq!(retained.as_ref(), &[0xa5; 63]);
1515
1516        drop(retained);
1517
1518        assert_eq!(pool.num_live(), 1);
1519
1520        drop(cloned);
1521
1522        assert_eq!(pool.num_live(), 0);
1523        assert_eq!(pool.strong_count(), 1);
1524        assert_eq!(mapping.0.map_calls.load(Ordering::Relaxed), 2);
1525    }
1526
1527    #[test]
1528    fn copied_mapping_crosses_threads_without_its_pool_or_borrowed_backend() {
1529        let ring = make_ring(4);
1530        let mem = Rc::new(ring.mem());
1531        let base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1532        let pool_layout = SlotLayout::new(base, 64, 1).unwrap();
1533        let pool = SlotPool::new(pool_layout).unwrap();
1534        let notif = TestNotifier::new();
1535
1536        let mut producer =
1537            VirtqProducer::new(ring.layout(), CopyingMem(&mem), notif.clone(), pool.clone());
1538        let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), notif);
1539
1540        let chain = producer.chain().writable(64).build().unwrap();
1541        producer.submit(chain).unwrap();
1542
1543        let (recv, reply) = poll_received(&mut consumer);
1544        let ReplyChain::Writable(mut reply) = reply else {
1545            panic!("expected writable reply");
1546        };
1547        reply.write_all(b"copied").unwrap();
1548        consumer.complete(recv, reply).unwrap();
1549
1550        let bytes = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1551
1552        assert_eq!(pool.num_live(), 0);
1553        assert_eq!(pool.strong_count(), 2);
1554
1555        let mut reused = producer.chain().readable(64).build().unwrap();
1556        reused.write_all(b"reused").unwrap();
1557
1558        std::thread::spawn(move || assert_eq!(bytes.as_ref(), b"copied"))
1559            .join()
1560            .unwrap();
1561
1562        assert_eq!(pool.num_live(), 1);
1563        assert_eq!(pool.strong_count(), 3);
1564
1565        drop(reused);
1566
1567        assert_eq!(pool.num_live(), 0);
1568    }
1569
1570    #[test]
1571    fn completion_uses_its_original_memory_and_allocating_pool() {
1572        let ring = make_ring(4);
1573        let mem = FaultMem::new(ring.mem());
1574        let orig_gen = Arc::downgrade(&mem.0);
1575        let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1576
1577        let notifier = TestNotifier::new();
1578        let original_layout = SlotLayout::new(base, 64, 4).unwrap();
1579        let original = SlotPool::new(original_layout).unwrap();
1580        let source = VirtqProducer::new(ring.layout(), mem, TestNotifier::new(), original.clone());
1581
1582        let chain = source.chain().writable(64).build().unwrap();
1583        drop(source);
1584
1585        assert!(orig_gen.upgrade().is_some());
1586
1587        let replacement_layout = SlotLayout::new(original.base_addr() + 0x1000, 64, 4).unwrap();
1588        let replacement = SlotPool::new(replacement_layout).unwrap();
1589
1590        let replacement_mem = FaultMem::new(ring.mem());
1591        replacement_mem.fail_mapping_at(0);
1592
1593        let mut producer = VirtqProducer::new(
1594            ring.layout(),
1595            replacement_mem.clone(),
1596            notifier.clone(),
1597            replacement.clone(),
1598        );
1599        let mut consumer = VirtqConsumer::new(ring.layout(), ring.mem(), TestNotifier::new());
1600
1601        producer.submit(chain).unwrap();
1602
1603        // SAFETY: No views exist and the consumer has not accessed the ring.
1604        assert!(matches!(
1605            unsafe { producer.reclaim_slots() },
1606            Err(VirtqError::InvalidState)
1607        ));
1608        assert!(producer.pool().ptr_eq(&replacement));
1609
1610        assert!(orig_gen.upgrade().is_some());
1611
1612        let (recv, reply) = poll_received(&mut consumer);
1613        let ReplyChain::Writable(mut reply) = reply else {
1614            panic!("expected writable reply");
1615        };
1616
1617        reply.write_all(b"retained").unwrap();
1618        consumer.complete(recv, reply).unwrap();
1619
1620        let data = producer.poll().unwrap().unwrap().into_bytes().unwrap();
1621
1622        assert_eq!(data.as_ref(), b"retained");
1623        assert_eq!(original.num_live(), 1);
1624        assert_eq!(replacement.num_live(), 0);
1625        assert_eq!(replacement_mem.0.map_calls.load(Ordering::Relaxed), 0);
1626
1627        let map_calls = orig_gen
1628            .upgrade()
1629            .unwrap()
1630            .map_calls
1631            .load(Ordering::Relaxed);
1632        assert_eq!(map_calls, 1);
1633
1634        drop(producer);
1635
1636        assert_eq!(data.as_ref(), b"retained");
1637
1638        drop(data);
1639
1640        assert_eq!(original.num_live(), 0);
1641        assert!(orig_gen.upgrade().is_none());
1642    }
1643
1644    #[test]
1645    fn mapping_failure_releases_returned_and_unmapped_slots() {
1646        for fail_at in 0..2 {
1647            let ring = make_ring(8);
1648            let mem = FaultMem::new(ring.mem());
1649            mem.fail_mapping_at(fail_at);
1650
1651            let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1652            let pool_layout = SlotLayout::new(base, 64, 8).unwrap();
1653            let pool = SlotPool::new(pool_layout).unwrap();
1654            let notif = TestNotifier::new();
1655            let mut producer =
1656                VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1657
1658            let mut consumer = make_consumer(&ring);
1659
1660            let chain = producer
1661                .chain()
1662                .readable(192)
1663                .writable(256)
1664                .build()
1665                .unwrap();
1666
1667            let mut addresses: Vec<_> = chain
1668                .owned
1669                .buffers
1670                .iter()
1671                .map(|buffer| buffer.addr)
1672                .collect();
1673
1674            producer.submit(chain).unwrap();
1675
1676            let (id, _) = consumer.poll_available().unwrap();
1677            // TestMem's zeroed backing supplies initialized bytes for the mapped prefix.
1678            consumer.submit_used(id, 70).unwrap();
1679
1680            assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
1681            assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), fail_at + 1);
1682            assert_eq!(pool.num_live(), 0);
1683            assert_eq!(pool.strong_count(), 2);
1684            assert_eq!(producer.num_inflight(), 0);
1685
1686            if fail_at == 1 {
1687                // The mapper releases the failed lease before completed owners unwind.
1688                addresses.swap(3, 4);
1689            }
1690
1691            let repeated = producer
1692                .chain()
1693                .readable(192)
1694                .writable(256)
1695                .build()
1696                .unwrap();
1697
1698            assert_eq!(
1699                repeated
1700                    .owned
1701                    .buffers
1702                    .iter()
1703                    .map(|buffer| buffer.addr)
1704                    .collect::<Vec<_>>(),
1705                addresses
1706            );
1707        }
1708    }
1709
1710    #[test]
1711    fn empty_and_malformed_completions_skip_mapping_and_release_every_slot() {
1712        let ring = make_ring(4);
1713        let mem = FaultMem::new(ring.mem());
1714        mem.fail_mapping_at(0);
1715
1716        let base = ring.mem().base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1717        let pool_layout = SlotLayout::new(base, 64, 4).unwrap();
1718        let pool = SlotPool::new(pool_layout).unwrap();
1719        let notif = TestNotifier::new();
1720
1721        let mut producer =
1722            VirtqProducer::new(ring.layout(), mem.clone(), notif.clone(), pool.clone());
1723
1724        let mut consumer = make_consumer(&ring);
1725
1726        for (writable, written) in [(false, 0), (true, 0), (true, 129)] {
1727            let builder = producer.chain().readable(64);
1728            let builder = if writable {
1729                builder.writable(128)
1730            } else {
1731                builder
1732            };
1733
1734            let token = producer.submit(builder.build().unwrap()).unwrap();
1735            let (id, _) = consumer.poll_available().unwrap();
1736            consumer.submit_used(id, written).unwrap();
1737
1738            match producer.poll() {
1739                Ok(Some(UsedChain::Ack(returned))) if !writable => assert_eq!(returned, token),
1740                Ok(Some(UsedChain::Data(returned, segments))) if writable && written == 0 => {
1741                    assert_eq!(returned, token);
1742                    assert_eq!(segments.segment_count(), 0);
1743                }
1744                Err(VirtqError::InvalidState) if written == 129 => {}
1745                other => panic!("unexpected completion: {other:?}"),
1746            }
1747
1748            assert_eq!(mem.0.map_calls.load(Ordering::Relaxed), 0);
1749            assert_eq!(pool.num_live(), 0);
1750            assert_eq!(pool.strong_count(), 2);
1751            assert_eq!(producer.num_inflight(), 0);
1752        }
1753    }
1754
1755    #[test]
1756    fn cancelling_a_chain_preserves_slot_order_after_a_partial_write_error() {
1757        let storage = TestMem::new(16);
1758        let base = storage.base_addr();
1759        let mem = FaultMem::new(storage);
1760        let t1 = SlotLayout::new(base, 2, 2).unwrap();
1761        let t2 = SlotLayout::new(base + 4, 4, 3).unwrap();
1762        let pool = SlotPool::new_tiered(t1, t2).unwrap();
1763
1764        let mut chain = ChainBuilder::new(mem.clone(), pool.clone(), 8)
1765            .readable(9)
1766            .writable_avail()
1767            .build()
1768            .unwrap();
1769
1770        let addresses: Vec<_> = chain
1771            .owned
1772            .buffers
1773            .iter()
1774            .map(|buffer| buffer.addr)
1775            .collect();
1776
1777        mem.fail_write_at(1);
1778        assert!(matches!(
1779            chain.write_all(b"abcdefghi"),
1780            Err(VirtqError::MemoryWriteError)
1781        ));
1782        assert_eq!(chain.written(), 4);
1783
1784        drop(chain);
1785
1786        assert_eq!(pool.num_live(), 0);
1787        assert_eq!(pool.strong_count(), 1);
1788
1789        mem.allow_writes();
1790        let repeated = ChainBuilder::new(mem, pool.clone(), 8)
1791            .readable(9)
1792            .writable_avail()
1793            .build()
1794            .unwrap();
1795
1796        assert_eq!(
1797            repeated
1798                .owned
1799                .buffers
1800                .iter()
1801                .map(|buffer| buffer.addr)
1802                .collect::<Vec<_>>(),
1803            addresses
1804        );
1805    }
1806
1807    #[test]
1808    fn reclaim_buffers_multiple_ring_generations() {
1809        let ring = make_ring(8);
1810        let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1811        let count = ring.len() * 2 + 1;
1812
1813        for _ in 0..count {
1814            let chain = producer.chain().writable(1).build().unwrap();
1815            producer.submit(chain).unwrap();
1816            let (recv, reply) = poll_received(&mut consumer);
1817            consumer.complete(recv, reply).unwrap();
1818            assert_eq!(producer.reclaim().unwrap(), 1);
1819            assert_eq!(producer.num_free(), ring.len());
1820            assert_eq!(producer.pool().num_live(), 0);
1821        }
1822
1823        for seq in 0..count {
1824            let Some(UsedChain::Data(token, segments)) = producer.poll().unwrap() else {
1825                panic!("expected an empty writable completion");
1826            };
1827
1828            assert_eq!(token.seq as usize, seq);
1829            assert!(segments.is_empty());
1830        }
1831
1832        assert!(producer.poll().unwrap().is_none());
1833    }
1834
1835    #[test]
1836    fn pool_swap_skips_pools_without_retained_allocations() {
1837        let ring = make_ring(8);
1838        let (mut producer, _consumer) = make_virtq_pair(&ring, 64);
1839        let pool = producer.pool().clone();
1840
1841        // SAFETY: No allocations or peer accesses exist.
1842        unsafe { producer.reclaim_slots() }.unwrap();
1843        assert!(producer.pool().ptr_eq(&pool));
1844
1845        let chain = producer.chain().writable(64).build().unwrap();
1846        producer.submit(chain).unwrap();
1847        // SAFETY: Only the posted chain owns an allocation. The peer stays stopped.
1848        unsafe { producer.reclaim_slots() }.unwrap();
1849        assert!(producer.pool().ptr_eq(&pool));
1850
1851        // SAFETY: The consumer has not accessed the ring and stays inactive.
1852        unsafe { producer.reset() }.unwrap();
1853        assert_eq!(pool.num_live(), 0);
1854    }
1855
1856    #[test]
1857    fn pool_swap_preserves_posted_requests_and_ring_state() {
1858        let ring = make_ring(8);
1859        let mem = ring.mem();
1860        let (mut producer, mut consumer) = make_virtq_pair(&ring, 64);
1861
1862        let original = producer.pool().clone();
1863        let allocation = original.alloc(64).unwrap();
1864        let mut retained = vec![BufferLease::new(original.clone(), allocation)];
1865
1866        let mut tokens = Vec::new();
1867        for _ in 0..2 {
1868            let chain = producer.chain().writable(64).build().unwrap();
1869            tokens.push(producer.submit(chain).unwrap());
1870        }
1871
1872        let (recv, reply) = poll_received(&mut consumer);
1873        let ReplyChain::Writable(mut reply) = reply else {
1874            panic!("expected writable reply");
1875        };
1876        reply.write_all(b"first").unwrap();
1877        consumer.complete(recv, reply).unwrap();
1878
1879        let mut ring_before = vec![0; Layout::query_size(ring.len())];
1880        mem.read(mem.base_addr(), &mut ring_before).unwrap();
1881
1882        for round in 0..2 {
1883            let previous = producer.pool().clone();
1884            if round != 0 {
1885                let allocation = previous.alloc(64).unwrap();
1886                retained.push(BufferLease::new(previous.clone(), allocation));
1887            }
1888            // SAFETY: The retained lease has no views. No peer handles remain,
1889            // and the consumer stays stopped during the swap.
1890            unsafe { producer.reclaim_slots() }.unwrap();
1891            assert!(!producer.pool().ptr_eq(&previous));
1892
1893            let mut ring_after = vec![0; ring_before.len()];
1894            mem.read(mem.base_addr(), &mut ring_after).unwrap();
1895            assert_eq!(ring_after, ring_before);
1896        }
1897
1898        let pool = producer.pool().clone();
1899        let reused = pool.alloc(64).unwrap();
1900        assert_eq!(reused.addr, allocation.addr);
1901
1902        drop(retained);
1903        assert_eq!(pool.num_live(), 3);
1904
1905        let first = producer.poll().unwrap().unwrap();
1906        assert_eq!(first.token(), tokens[0]);
1907
1908        let first = first.into_bytes().unwrap();
1909        assert_eq!(first.as_ref(), b"first");
1910
1911        drop(first);
1912        assert_eq!(pool.num_live(), 2);
1913
1914        let (recv, reply) = poll_received(&mut consumer);
1915        let ReplyChain::Writable(mut reply) = reply else {
1916            panic!("expected writable reply");
1917        };
1918
1919        reply.write_all(b"second").unwrap();
1920        consumer.complete(recv, reply).unwrap();
1921
1922        let second = producer.poll().unwrap().unwrap();
1923        assert_eq!(second.token(), tokens[1]);
1924
1925        let second = second.into_bytes().unwrap();
1926        assert_eq!(second.as_ref(), b"second");
1927
1928        drop(second);
1929        pool.dealloc(reused.addr).unwrap();
1930        assert_eq!(pool.num_live(), 0);
1931    }
1932
1933    #[test]
1934    fn reset_reclaims_inflight_slots_and_reuses_ring() {
1935        let ring = make_ring(8);
1936        let mem = ring.mem();
1937
1938        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1939        let pool_layout = SlotLayout::new(pool_base, 64, ring.len()).unwrap();
1940        let pool = SlotPool::new(pool_layout).unwrap();
1941        let notifier = TestNotifier::new();
1942
1943        let mut producer = VirtqProducer::new(ring.layout(), mem, notifier, pool.clone());
1944
1945        for _ in 0..ring.len() {
1946            let chain = producer.chain().writable(64).build().unwrap();
1947            producer.submit(chain).unwrap();
1948        }
1949
1950        assert_eq!(pool.num_free(), 0);
1951
1952        // SAFETY: No consumer is attached to this ring.
1953        unsafe { producer.reset() }.unwrap();
1954
1955        assert_eq!(producer.num_inflight(), 0);
1956        assert_eq!(producer.num_free(), ring.len());
1957        assert_eq!(pool.num_free(), ring.len());
1958        assert!(producer.inflight.live.is_empty());
1959
1960        for _ in 0..ring.len() {
1961            let chain = producer.chain().writable(64).build().unwrap();
1962            producer.submit(chain).unwrap();
1963        }
1964
1965        // SAFETY: No consumer is attached to this ring.
1966        unsafe { producer.reset() }.unwrap();
1967    }
1968
1969    #[test]
1970    fn stopped_reset_reuses_slot_after_consumer_handles_drop() {
1971        let ring = make_ring(4);
1972        let mem = ring.mem();
1973        let pool_base = mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
1974        let pool_layout = SlotLayout::new(pool_base, 4, 1).unwrap();
1975        let pool = SlotPool::new(pool_layout).unwrap();
1976        let notifier = TestNotifier::new();
1977        let mut producer =
1978            VirtqProducer::new(ring.layout(), mem.clone(), notifier.clone(), pool.clone());
1979        let mut consumer = VirtqConsumer::new(ring.layout(), mem.clone(), notifier.clone());
1980
1981        let sent = producer.chain().writable(4).build().unwrap();
1982        producer.submit(sent).unwrap();
1983        let (recv, reply) = poll_received(&mut consumer);
1984        assert!(matches!(reply, ReplyChain::Writable(_)));
1985
1986        drop((recv, reply, consumer));
1987        assert_eq!(pool.num_live(), 1);
1988
1989        // SAFETY: The consumer and all its chain handles have been dropped.
1990        unsafe { producer.reset() }.unwrap();
1991        assert_eq!(pool.num_live(), 0);
1992
1993        let mut replacement = producer.chain().readable(4).build().unwrap();
1994        assert_eq!(replacement.owned.buffers[0].addr, pool_base);
1995        replacement.write_all(b"GOOD").unwrap();
1996        producer.submit(replacement).unwrap();
1997
1998        let mut consumer = VirtqConsumer::new(ring.layout(), mem, notifier);
1999        let (recv, reply) = poll_received(&mut consumer);
2000        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"GOOD");
2001        consumer.complete(recv, reply).unwrap();
2002        producer.drain(drop).unwrap();
2003        assert_eq!(pool.num_live(), 0);
2004    }
2005
2006    #[test]
2007    fn reset_and_pool_swap_reject_buffered_writable_completion() {
2008        let ring = make_ring(8);
2009        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2010        let chain = producer.chain().writable(64).build().unwrap();
2011        producer.submit(chain).unwrap();
2012
2013        let (recv, reply) = poll_received(&mut consumer);
2014        let ReplyChain::Writable(mut reply) = reply else {
2015            panic!("expected writable reply");
2016        };
2017
2018        reply.write_all(b"retained").unwrap();
2019        consumer.complete(recv, reply).unwrap();
2020        producer.reclaim().unwrap();
2021
2022        let pool = producer.pool().clone();
2023        // SAFETY: No allocations are held outside the producer. The consumer
2024        // completed its handles and stays inactive.
2025        assert!(matches!(
2026            unsafe { producer.reclaim_slots() },
2027            Err(VirtqError::InvalidState)
2028        ));
2029        assert!(producer.pool().ptr_eq(&pool));
2030
2031        // SAFETY: The consumer completed its handles and stays inactive.
2032        assert!(matches!(
2033            unsafe { producer.reset() },
2034            Err(VirtqError::InvalidState)
2035        ));
2036
2037        drop(producer.poll().unwrap().unwrap());
2038
2039        // SAFETY: The consumer completed its handles and stays inactive.
2040        unsafe { producer.reset() }.unwrap();
2041    }
2042
2043    #[test]
2044    fn test_chain_readwrite_build() {
2045        let ring = make_ring(16);
2046        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2047
2048        let se = producer.chain().readable(64).writable(128).build().unwrap();
2049        assert_eq!(se.capacity(), 64);
2050        assert_eq!(se.written(), 0);
2051        assert_eq!(se.remaining(), 64);
2052    }
2053
2054    #[test]
2055    fn test_chain_readable_writable_names_build() {
2056        let ring = make_ring(16);
2057        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2058
2059        let se = producer.chain().readable(16).writable(32).build().unwrap();
2060        assert_eq!(se.desc_count(), 2);
2061        assert_eq!(se.rd_desc_count(), 1);
2062        assert_eq!(se.wr_desc_count(), 1);
2063        assert_eq!(se.capacity(), 16);
2064    }
2065
2066    #[test]
2067    fn test_chain_multi_readable_write_all_scatters() {
2068        let ring = make_ring(16);
2069        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2070
2071        let mut se = producer
2072            .chain()
2073            .readable(5)
2074            .readable(6)
2075            .writable(32)
2076            .build()
2077            .unwrap();
2078
2079        se.write_all(b"hello world").unwrap();
2080        assert_eq!(se.written(), 11);
2081
2082        let token = producer.submit(se).unwrap();
2083        let (recv, reply) = poll_received(&mut consumer);
2084        assert_eq!(recv.token(), token);
2085        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2086
2087        let segments = recv.to_segments().unwrap();
2088        assert_eq!(segments.segment_count(), 2);
2089        assert_eq!(segments.as_slice()[0].as_ref(), b"hello");
2090        assert_eq!(segments.as_slice()[1].as_ref(), b" world");
2091
2092        consumer.complete(recv, reply).unwrap();
2093        producer.drain(drop).unwrap();
2094    }
2095
2096    #[test]
2097    fn test_chain_independent_readables_preserve_pool_tiers() {
2098        let ring = make_ring(16);
2099        let layout = ring.layout();
2100        let mem = ring.mem();
2101
2102        let lower = SlotLayout::new(
2103            mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100,
2104            256,
2105            1,
2106        )
2107        .unwrap();
2108
2109        let upper = SlotLayout::new(lower.end_addr(), 4096, 1).unwrap();
2110        let pool = SlotPool::new_tiered(lower, upper).unwrap();
2111        let notifier = TestNotifier::new();
2112        let producer = VirtqProducer::new(layout, mem, notifier, pool.clone());
2113
2114        let send = producer
2115            .chain()
2116            .readable(128)
2117            .readable(4096)
2118            .build()
2119            .unwrap();
2120
2121        let readables = send.owned.descriptors().collect::<Vec<_>>();
2122
2123        assert_eq!(readables.len(), 2);
2124        assert_eq!(readables[0].addr, lower.base_addr());
2125        assert_eq!(readables[1].addr, upper.base_addr());
2126        assert_eq!(pool.num_free_lower(), 0);
2127        assert_eq!(pool.num_free_upper(), 0);
2128
2129        drop(send);
2130
2131        assert_eq!(pool.num_free_lower(), 1);
2132        assert_eq!(pool.num_free_upper(), 1);
2133    }
2134
2135    #[test]
2136    fn chain_avail_uses_pool_availability_at_build() {
2137        for held_count in [0, 2, 3] {
2138            let builder = make_chain(2, 3, 4096, 3);
2139            let pool = builder.pool.clone();
2140            let builder = builder.writable_avail().readable(128);
2141
2142            assert_eq!(pool.num_live(), 0);
2143
2144            let held: Vec<_> = (0..held_count).map(|_| pool.alloc(4096).unwrap()).collect();
2145
2146            let chain = builder.build().unwrap();
2147
2148            let writable = (3 - held_count).min(2);
2149            assert_eq!(chain.rd_desc_count(), 1);
2150            assert_eq!(chain.wr_desc_count(), writable);
2151            assert_eq!(chain.desc_count(), 1 + writable);
2152            assert_eq!(chain.owned.buffers[0].capacity, 128);
2153            assert!(
2154                chain.owned.buffers[1..]
2155                    .iter()
2156                    .all(|buf| buf.capacity == 4096)
2157            );
2158            assert_eq!(pool.num_free_lower(), 1);
2159            assert_eq!(pool.num_free_upper(), 3 - held_count - writable);
2160            assert_eq!(pool.num_live(), held_count + chain.desc_count());
2161
2162            drop(chain);
2163
2164            assert_eq!(pool.num_live(), held_count);
2165
2166            for allocation in held.into_iter().rev() {
2167                pool.dealloc(allocation.addr).unwrap();
2168            }
2169
2170            assert_eq!(pool.num_live(), 0);
2171        }
2172    }
2173
2174    #[test]
2175    fn chain_avail_respects_mandatory_descriptor_budget() {
2176        for max_descs in 0..3 {
2177            let builder = make_chain(2, 3, 4096, max_descs);
2178            let pool = builder.pool.clone();
2179
2180            let result = builder
2181                .readable(4096)
2182                .readable(128)
2183                .writable_avail()
2184                .build();
2185
2186            if max_descs == 2 {
2187                let chain = result.unwrap();
2188
2189                assert_eq!(chain.rd_desc_count(), 2);
2190                assert_eq!(chain.wr_desc_count(), 0);
2191
2192                drop(chain);
2193            } else {
2194                assert!(matches!(result, Err(VirtqError::Backpressure)));
2195            }
2196
2197            assert_eq!(pool.num_live(), 0);
2198
2199            let lower = pool.alloc(128).unwrap();
2200            let upper = pool.alloc(4096).unwrap();
2201
2202            assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2203            assert_eq!(upper.addr, pool.slot_addr(4).unwrap());
2204
2205            pool.dealloc(lower.addr).unwrap();
2206            pool.dealloc(upper.addr).unwrap();
2207        }
2208    }
2209
2210    #[test]
2211    fn chain_avail_preserves_explicit_writable_capacity_when_full() {
2212        for (upper_count, max_descs) in [(2, 2), (1, 3)] {
2213            let builder = make_chain(1, upper_count, 4096, max_descs);
2214            let pool = builder.pool.clone();
2215
2216            let chain = builder
2217                .writable_avail()
2218                .writable(128)
2219                .readable(128)
2220                .build()
2221                .unwrap();
2222
2223            assert_eq!(chain.rd_desc_count(), 1);
2224            assert_eq!(chain.wr_desc_count(), 1);
2225            assert_eq!(chain.owned.buffers[0].capacity, 128);
2226            assert_eq!(chain.owned.buffers[1].capacity, 4096);
2227            assert!(chain.owned.buffers.iter().all(|buf| buf.written == 0));
2228            assert_eq!(pool.num_free_upper(), upper_count - 1);
2229
2230            drop(chain);
2231
2232            assert_eq!(pool.num_live(), 0);
2233        }
2234    }
2235
2236    #[test]
2237    fn chain_avail_cannot_build_an_empty_chain() {
2238        for max_descs in [0, 2] {
2239            let builder = make_chain(1, 1, 4096, max_descs);
2240            let pool = builder.pool.clone();
2241            let held = (max_descs != 0).then(|| pool.alloc(4096).unwrap());
2242
2243            assert!(matches!(
2244                builder.writable_avail().build(),
2245                Err(VirtqError::Backpressure)
2246            ));
2247            assert_eq!(pool.num_free_lower(), 1);
2248            assert_eq!(pool.num_live(), usize::from(held.is_some()));
2249
2250            if let Some(allocation) = held {
2251                pool.dealloc(allocation.addr).unwrap();
2252            }
2253        }
2254    }
2255
2256    #[test]
2257    fn chain_avail_only_uses_upper_slots() {
2258        let builder = make_chain(4, 2, 4096, 6);
2259        let pool = builder.pool.clone();
2260
2261        let chain = builder.writable_avail().writable_avail().build().unwrap();
2262
2263        assert_eq!(chain.rd_desc_count(), 0);
2264        assert_eq!(chain.wr_desc_count(), 2);
2265        assert_eq!(pool.num_free_lower(), 4);
2266        assert_eq!(pool.num_free_upper(), 0);
2267
2268        drop(chain);
2269
2270        assert_eq!(pool.num_live(), 0);
2271    }
2272
2273    #[test]
2274    fn chain_preserves_region_order_with_odd_slot_sizes() {
2275        let builder = make_chain(1, 4, 3001, 5);
2276        let pool = builder.pool.clone();
2277
2278        // The final short readable must fall back to an upper slot.
2279        let chain = builder
2280            .readable(128)
2281            .readable(6003)
2282            .writable_avail()
2283            .build()
2284            .unwrap();
2285
2286        assert_eq!(chain.rd_desc_count(), 4);
2287
2288        let actual_offsets = chain
2289            .owned
2290            .buffers
2291            .iter()
2292            .map(|buf| {
2293                (
2294                    buf.addr,
2295                    buf.capacity,
2296                    pool.allocation_len(buf.addr).unwrap(),
2297                )
2298            })
2299            .collect::<Vec<_>>();
2300
2301        let expected_offsets = [
2302            (0, 128, 256),
2303            (4, 3001, 3001),
2304            (3, 3001, 3001),
2305            (2, 1, 3001),
2306            (1, 3001, 3001),
2307        ]
2308        .map(|(slot, capacity, allocated)| (pool.slot_addr(slot).unwrap(), capacity, allocated));
2309
2310        assert_eq!(actual_offsets, expected_offsets);
2311
2312        drop(chain);
2313
2314        assert_eq!(pool.num_live(), 0);
2315    }
2316
2317    #[test]
2318    fn chain_failure_releases_only_its_own_slots() {
2319        let builder = make_chain(2, 3, 4096, 4);
2320        let pool = builder.pool.clone();
2321        let held = pool.alloc(4096).unwrap();
2322
2323        assert!(matches!(
2324            builder.readable(128).writable(4096 * 3).build(),
2325            Err(VirtqError::Backpressure)
2326        ));
2327        assert_eq!(pool.live_addrs(), [held.addr]);
2328
2329        let lower = pool.alloc(128).unwrap();
2330        let upper = pool.alloc(4096).unwrap();
2331        let next_upper = pool.alloc(4096).unwrap();
2332
2333        assert_eq!(lower.addr, pool.slot_addr(1).unwrap());
2334        assert_eq!(upper.addr, pool.slot_addr(3).unwrap());
2335        assert_eq!(next_upper.addr, pool.slot_addr(2).unwrap());
2336
2337        pool.dealloc(next_upper.addr).unwrap();
2338        pool.dealloc(upper.addr).unwrap();
2339        pool.dealloc(lower.addr).unwrap();
2340        pool.dealloc(held.addr).unwrap();
2341    }
2342
2343    #[test]
2344    fn chain_rejects_invalid_request_sequences() {
2345        for (readable, writable) in [(vec![128, 0], vec![]), (vec![128], vec![4096, 0])] {
2346            let mut builder = make_chain(1, 1, 4096, 4);
2347            let pool = builder.pool.clone();
2348
2349            for cap in readable {
2350                builder = builder.readable(cap);
2351            }
2352            for cap in writable {
2353                builder = builder.writable(cap);
2354            }
2355
2356            assert!(matches!(
2357                builder.build(),
2358                Err(VirtqError::Alloc(AllocError::InvalidArg))
2359            ));
2360            assert_eq!(pool.num_live(), 0);
2361        }
2362
2363        let builder = make_chain(1, 1, 4096, 4);
2364        let pool = builder.pool.clone();
2365
2366        assert!(matches!(
2367            builder.readable(usize::MAX).readable(1).build(),
2368            Err(VirtqError::Alloc(AllocError::Overflow))
2369        ));
2370        assert_eq!(pool.num_live(), 0);
2371    }
2372
2373    #[test]
2374    fn chain_keeps_four_records_inline_and_one_pool_handle() {
2375        for count in [4, 5] {
2376            let builder = make_chain(0, count, 4096, count);
2377            let pool = builder.pool.clone();
2378
2379            let chain = builder
2380                .readable((count - 1) * 4096)
2381                .writable_avail()
2382                .build()
2383                .unwrap();
2384
2385            assert_eq!(chain.desc_count(), count);
2386            assert_eq!(chain.owned.buffers.spilled(), count > 4);
2387            assert_eq!(pool.strong_count(), 2);
2388
2389            drop(chain);
2390
2391            assert_eq!(pool.num_live(), 0);
2392        }
2393    }
2394
2395    #[test]
2396    fn test_chain_multi_readable_appends_across_calls() {
2397        let ring = make_ring(16);
2398        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2399
2400        let mut send = producer.chain().readable(8).build().unwrap();
2401        send.write_all(b"abc").unwrap();
2402        send.write_all(b"def").unwrap();
2403        assert_eq!(send.written(), 6);
2404        assert_eq!(send.remaining(), 2);
2405
2406        producer.submit(send).unwrap();
2407        let (recv, reply) = poll_received(&mut consumer);
2408        let segments = recv.to_segments().unwrap();
2409        assert_eq!(segments.segment_count(), 2);
2410        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2411        assert_eq!(segments.as_slice()[1].as_ref(), b"ef");
2412        consumer.complete(recv, reply).unwrap();
2413        producer.drain(drop).unwrap();
2414    }
2415
2416    #[test]
2417    fn test_chain_readable_splits_logical_capacity() {
2418        let expected_lengths = [4, 4, 4];
2419        let ring = make_ring(16);
2420        let (mut producer, mut consumer, _) = make_test_producer_with_slot_size(&ring, 4);
2421
2422        let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2423        let readables = &se.owned.buffers[..se.rd_desc_count()];
2424
2425        assert_eq!(se.rd_desc_count(), 3);
2426        assert_eq!(se.capacity(), 10);
2427
2428        let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2429        assert_eq!(caps, [4, 4, 2]);
2430
2431        let lengths = readables
2432            .iter()
2433            .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2434            .collect::<Vec<_>>();
2435        assert_eq!(lengths, expected_lengths);
2436
2437        se.write_all(b"abcdefghij").unwrap();
2438        assert_eq!(se.written(), 10);
2439
2440        let token = producer.submit(se).unwrap();
2441        let (recv, reply) = poll_received(&mut consumer);
2442
2443        assert_eq!(recv.token(), token);
2444        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2445
2446        let segments = recv.to_segments().unwrap();
2447
2448        assert_eq!(segments.segment_count(), 3);
2449        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2450        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2451        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2452
2453        consumer.complete(recv, reply).unwrap();
2454        producer.drain(drop).unwrap();
2455
2456        assert_eq!(producer.pool.num_live(), 0);
2457    }
2458
2459    #[test]
2460    fn test_chain_readable_splits_logical_capacity_tiered() {
2461        let expected_lengths = [4, 4, 2];
2462        let ring = make_ring(16);
2463        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2464
2465        let mut se = producer.chain().readable(10).writable(32).build().unwrap();
2466        let readables = &se.owned.buffers[..se.rd_desc_count()];
2467
2468        assert_eq!(se.rd_desc_count(), 3);
2469        assert_eq!(se.capacity(), 10);
2470
2471        let caps = readables.iter().map(|buf| buf.capacity).collect::<Vec<_>>();
2472        assert_eq!(caps, [4, 4, 2]);
2473
2474        let lengths = readables
2475            .iter()
2476            .map(|buf| producer.pool.allocation_len(buf.addr).unwrap())
2477            .collect::<Vec<_>>();
2478        assert_eq!(lengths, expected_lengths);
2479
2480        se.write_all(b"abcdefghij").unwrap();
2481        assert_eq!(se.written(), 10);
2482
2483        let token = producer.submit(se).unwrap();
2484        let (recv, reply) = poll_received(&mut consumer);
2485
2486        assert_eq!(recv.token(), token);
2487        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"abcdefghij");
2488
2489        let segments = recv.to_segments().unwrap();
2490
2491        assert_eq!(segments.segment_count(), 3);
2492        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2493        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2494        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2495
2496        consumer.complete(recv, reply).unwrap();
2497        producer.drain(drop).unwrap();
2498
2499        assert_eq!(producer.pool.num_live(), 0);
2500    }
2501
2502    #[test]
2503    fn test_chain_readable_rejects_zero_capacity_on_build() {
2504        let ring = make_ring(16);
2505        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2506
2507        assert!(matches!(
2508            producer.chain().readable(0).build(),
2509            Err(VirtqError::Alloc(AllocError::InvalidArg))
2510        ));
2511    }
2512
2513    #[test]
2514    fn test_chain_writable_splits_logical_capacity() {
2515        let ring = make_ring(16);
2516        let (mut producer, mut consumer) = make_virtq_pair(&ring, 4);
2517
2518        let se = producer.chain().writable(10).build().unwrap();
2519        let token = producer.submit(se).unwrap();
2520
2521        let (recv, reply) = poll_received(&mut consumer);
2522        let ReplyChain::Writable(mut wc) = reply else {
2523            panic!("expected writable reply");
2524        };
2525        assert_eq!(wc.capacity(), 10);
2526
2527        wc.write_all(b"abcdefghij").unwrap();
2528        consumer.complete(recv, wc).unwrap();
2529
2530        let used = producer.poll().unwrap().unwrap();
2531        assert_eq!(used.token(), token);
2532        let segments = used.segments().unwrap();
2533        assert_eq!(segments.segment_count(), 3);
2534        assert_eq!(segments.as_slice()[0].as_ref(), b"abcd");
2535        assert_eq!(segments.as_slice()[1].as_ref(), b"efgh");
2536        assert_eq!(segments.as_slice()[2].as_ref(), b"ij");
2537    }
2538
2539    #[test]
2540    fn test_chain_writable_rejects_zero_capacity_on_build() {
2541        let ring = make_ring(16);
2542        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2543
2544        assert!(matches!(
2545            producer.chain().writable(0).build(),
2546            Err(VirtqError::Alloc(AllocError::InvalidArg))
2547        ));
2548    }
2549
2550    #[test]
2551    fn test_chain_multi_readable_write_all_preserves_segments() {
2552        let ring = make_ring(16);
2553        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2554
2555        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2556        se.write_all(b"headbody").unwrap();
2557
2558        producer.submit(se).unwrap();
2559        let (recv, reply) = poll_received(&mut consumer);
2560        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"headbody");
2561        consumer.complete(recv, reply).unwrap();
2562        producer.drain(drop).unwrap();
2563    }
2564
2565    #[test]
2566    fn test_chain_payload_segment_writer_serializes_directly() {
2567        let ring = make_ring(16);
2568        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2569
2570        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2571        se.with_seg(0, |segment| {
2572            segment.copy_from_slice(b"head");
2573            Ok::<usize, VirtqError>(4)
2574        })
2575        .unwrap();
2576        se.with_seg(1, |segment| {
2577            segment.copy_from_slice(b"body");
2578            Ok::<usize, VirtqError>(4)
2579        })
2580        .unwrap();
2581
2582        producer.submit(se).unwrap();
2583        let (recv, reply) = poll_received(&mut consumer);
2584        let segments = recv.to_segments().unwrap();
2585        assert_eq!(segments.segment_count(), 2);
2586        assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2587        consumer.complete(recv, reply).unwrap();
2588        producer.drain(drop).unwrap();
2589    }
2590
2591    #[test]
2592    fn test_chain_payload_segment_write_copies_directly() {
2593        let ring = make_ring(16);
2594        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2595
2596        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2597        se.write_seg(0, b"head").unwrap();
2598        se.write_seg(1, b"body").unwrap();
2599
2600        producer.submit(se).unwrap();
2601        let (recv, reply) = poll_received(&mut consumer);
2602        let segments = recv.to_segments().unwrap();
2603        assert_eq!(segments.segment_count(), 2);
2604        assert_eq!(segments.to_bytes().as_ref(), b"headbody");
2605        consumer.complete(recv, reply).unwrap();
2606        producer.drain(drop).unwrap();
2607    }
2608
2609    #[test]
2610    fn test_chain_multi_writable_used_returns_segments() {
2611        let ring = make_ring(16);
2612        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2613
2614        let se = producer.chain().writable(5).writable(6).build().unwrap();
2615        let token = producer.submit(se).unwrap();
2616
2617        let (recv, reply) = poll_received(&mut consumer);
2618        let ReplyChain::Writable(mut wc) = reply else {
2619            panic!("expected writable reply");
2620        };
2621        assert_eq!(wc.capacity(), 12);
2622
2623        wc.write_all(b"hello world").unwrap();
2624        consumer.complete(recv, wc).unwrap();
2625
2626        let used = producer.poll().unwrap().unwrap();
2627        assert_eq!(used.token(), token);
2628        let segments = used.segments().unwrap();
2629        assert_eq!(segments.segment_count(), 2);
2630        assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2631        assert_eq!(segments.as_slice()[1].as_ref(), b"world");
2632        assert_eq!(segments.to_bytes().as_ref(), b"hello world");
2633    }
2634
2635    #[test]
2636    fn test_chain_multi_writable_short_used_truncates_last_segment() {
2637        let ring = make_ring(16);
2638        let (mut producer, mut consumer, _notifier) = make_test_producer_with_slot_size(&ring, 6);
2639
2640        let se = producer.chain().writable(5).writable(6).build().unwrap();
2641        producer.submit(se).unwrap();
2642
2643        let (recv, reply) = poll_received(&mut consumer);
2644        let ReplyChain::Writable(mut wc) = reply else {
2645            panic!("expected writable reply");
2646        };
2647
2648        wc.write_all(b"hello wo").unwrap();
2649        consumer.complete(recv, wc).unwrap();
2650
2651        let used = producer.poll().unwrap().unwrap();
2652        let segments = used.segments().unwrap();
2653        assert_eq!(segments.segment_count(), 2);
2654        assert_eq!(segments.as_slice()[0].as_ref(), b"hello ");
2655        assert_eq!(segments.as_slice()[1].as_ref(), b"wo");
2656        assert_eq!(segments.to_bytes().as_ref(), b"hello wo");
2657    }
2658
2659    #[test]
2660    fn test_chain_multi_writable_zero_used_returns_empty_segments() {
2661        let ring = make_ring(16);
2662        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2663
2664        let se = producer.chain().writable(5).writable(6).build().unwrap();
2665        producer.submit(se).unwrap();
2666
2667        let (recv, reply) = poll_received(&mut consumer);
2668        consumer.complete(recv, reply).unwrap();
2669
2670        let used = producer.poll().unwrap().unwrap();
2671        let segments = used.segments().unwrap();
2672        assert_eq!(segments.segment_count(), 0);
2673        assert!(segments.is_empty());
2674        assert!(segments.to_bytes().is_empty());
2675    }
2676
2677    #[test]
2678    fn test_chain_readable_only_build() {
2679        let ring = make_ring(16);
2680        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2681
2682        let se = producer.chain().readable(32).build().unwrap();
2683        assert_eq!(se.capacity(), 32);
2684    }
2685
2686    #[test]
2687    fn test_chain_writable_only_build() {
2688        let ring = make_ring(16);
2689        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2690
2691        let se = producer.chain().writable(64).build().unwrap();
2692        assert_eq!(se.capacity(), 0);
2693    }
2694
2695    #[test]
2696    fn test_chain_empty_build_fails() {
2697        let ring = make_ring(16);
2698        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2699
2700        let result = producer.chain().build();
2701        assert!(matches!(result, Err(VirtqError::InvalidState)));
2702    }
2703
2704    #[test]
2705    fn test_send_chain_write_all_and_submit() {
2706        let ring = make_ring(16);
2707        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2708
2709        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2710
2711        se.write_all(b"hello")
2712            .unwrap()
2713            .write_all(b" world")
2714            .unwrap();
2715        assert_eq!(se.written(), 11);
2716        assert_eq!(se.remaining(), 53);
2717        let tok = producer.submit(se).unwrap();
2718
2719        let (recv, reply) = poll_received(&mut consumer);
2720        assert_eq!(recv.token(), tok);
2721        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2722        consumer.complete(recv, reply).unwrap();
2723        producer.drain(drop).unwrap();
2724    }
2725
2726    #[test]
2727    fn test_send_payload_write_all_fluent() {
2728        let ring = make_ring(16);
2729        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2730
2731        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2732        se.write_all(b"hello")
2733            .unwrap()
2734            .write_all(b" world")
2735            .unwrap();
2736        assert_eq!(se.written(), 11);
2737        assert_eq!(se.remaining(), 53);
2738        let tok = producer.submit(se).unwrap();
2739
2740        let (recv, reply) = poll_received(&mut consumer);
2741        assert_eq!(recv.token(), tok);
2742        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello world");
2743        consumer.complete(recv, reply).unwrap();
2744        producer.drain(drop).unwrap();
2745    }
2746
2747    #[test]
2748    fn test_send_payload_partial_write() {
2749        let ring = make_ring(16);
2750        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2751
2752        let mut se = producer.chain().readable(8).build().unwrap();
2753        let written = se.write(b"hello world").unwrap();
2754        assert_eq!(written, 8);
2755        assert_eq!(se.remaining(), 0);
2756
2757        producer.submit(se).unwrap();
2758        let (recv, reply) = poll_received(&mut consumer);
2759        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello wo");
2760        consumer.complete(recv, reply).unwrap();
2761        producer.drain(drop).unwrap();
2762    }
2763
2764    #[test]
2765    fn test_send_payload_write_with_serializes_directly() {
2766        let ring = make_ring(16);
2767        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2768
2769        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2770        se.with_seg(0, |buf| {
2771            buf[..5].copy_from_slice(b"hello");
2772            Ok::<usize, VirtqError>(5)
2773        })
2774        .unwrap();
2775
2776        let _tok = producer.submit(se).unwrap();
2777
2778        let (recv, reply) = poll_received(&mut consumer);
2779        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2780        consumer.complete(recv, reply).unwrap();
2781        producer.drain(drop).unwrap();
2782    }
2783
2784    #[test]
2785    fn test_send_chain_single_segment_writer_serializes_directly() {
2786        let ring = make_ring(16);
2787        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
2788
2789        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2790        se.with_seg(0, |segment| {
2791            assert_eq!(segment.len(), 64);
2792            segment[..5].copy_from_slice(b"hello");
2793            Ok::<usize, VirtqError>(5)
2794        })
2795        .unwrap();
2796
2797        let _tok = producer.submit(se).unwrap();
2798
2799        let (recv, reply) = poll_received(&mut consumer);
2800        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"hello");
2801        consumer.complete(recv, reply).unwrap();
2802        producer.drain(drop).unwrap();
2803    }
2804
2805    #[test]
2806    fn test_send_chain_single_segment_writer_rejects_multi_segment() {
2807        let ring = make_ring(16);
2808        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2809
2810        let mut se = producer.chain().readable(4).readable(4).build().unwrap();
2811        assert!(matches!(
2812            se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2813            Err(VirtqError::NoPayloadSegment)
2814        ));
2815    }
2816
2817    #[test]
2818    fn test_send_chain_single_segment_writer_rejects_auto_split_chain() {
2819        let ring = make_ring(16);
2820        let (producer, _consumer) = make_virtq_pair(&ring, 4);
2821
2822        let mut se = producer.chain().readable(8).build().unwrap();
2823        assert_eq!(se.rd_desc_count(), 2);
2824        assert!(matches!(
2825            se.with_seg(2, |_| Ok::<usize, VirtqError>(0)),
2826            Err(VirtqError::NoPayloadSegment)
2827        ));
2828    }
2829
2830    #[test]
2831    fn test_send_payload_segment_set_written_too_large() {
2832        let ring = make_ring(16);
2833        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2834
2835        let mut se = producer.chain().readable(32).writable(64).build().unwrap();
2836        let err = se
2837            .with_seg(0, |_| Ok::<usize, VirtqError>(64))
2838            .err()
2839            .unwrap();
2840        assert!(matches!(
2841            err,
2842            VirtqError::PayloadTooLarge {
2843                recv: 64,
2844                limit: 32
2845            }
2846        ));
2847    }
2848
2849    #[test]
2850    fn test_send_chain_write_too_large() {
2851        let ring = make_ring(16);
2852        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2853
2854        let mut se = producer.chain().readable(4).build().unwrap();
2855        let err = se.write_all(b"too long").err().unwrap();
2856        assert!(matches!(
2857            err,
2858            VirtqError::PayloadTooLarge { recv: 8, limit: 4 }
2859        ));
2860    }
2861
2862    #[test]
2863    fn test_writeonly_has_no_readable_buffer() {
2864        let ring = make_ring(16);
2865        let (producer, _consumer, _notifier) = make_test_producer(&ring);
2866
2867        let mut se = producer.chain().writable(32).build().unwrap();
2868        let err = se.write_all(b"data").err().unwrap();
2869        assert!(matches!(err, VirtqError::NoPayloadSegment));
2870    }
2871
2872    #[test]
2873    fn test_drop_chain_builder_deallocs() {
2874        let ring = make_ring(16);
2875        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2876
2877        {
2878            let _builder = producer.chain().readable(64).writable(128);
2879            // dropped without build
2880        }
2881
2882        // Ring should still be fully usable
2883        let se = producer.chain().readable(64).writable(128).build().unwrap();
2884        let tok = producer.submit(se).unwrap();
2885        assert!(tok.id < 16);
2886
2887        // SAFETY: The consumer has never polled the ring and stays inactive.
2888        unsafe { producer.reset() }.unwrap();
2889    }
2890
2891    #[test]
2892    fn test_drop_send_chain_deallocs() {
2893        let ring = make_ring(16);
2894        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
2895
2896        {
2897            let _se = producer.chain().readable(64).writable(128).build().unwrap();
2898            // dropped without submit
2899        }
2900
2901        // Ring should still be fully usable
2902        let se = producer.chain().readable(64).writable(128).build().unwrap();
2903        let tok = producer.submit(se).unwrap();
2904        assert!(tok.id < 16);
2905
2906        // SAFETY: The consumer has never polled the ring and stays inactive.
2907        unsafe { producer.reset() }.unwrap();
2908    }
2909
2910    #[test]
2911    fn test_submit_notifies() {
2912        let ring = make_ring(16);
2913        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2914
2915        let initial_count = notifier.notification_count();
2916
2917        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
2918        se.write_all(b"hello").unwrap();
2919        producer.submit(se).unwrap();
2920
2921        assert!(notifier.notification_count() > initial_count);
2922
2923        // SAFETY: The consumer has never polled the ring and stays inactive.
2924        unsafe { producer.reset() }.unwrap();
2925    }
2926
2927    #[test]
2928    fn test_submit_read_only_notifies_by_default() {
2929        let ring = make_ring(16);
2930        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2931
2932        let initial_count = notifier.notification_count();
2933
2934        let mut se = producer.chain().readable(64).build().unwrap();
2935        se.write_all(b"fire-and-forget").unwrap();
2936        producer.submit(se).unwrap();
2937
2938        assert!(notifier.notification_count() > initial_count);
2939
2940        // SAFETY: The consumer has never polled the ring and stays inactive.
2941        unsafe { producer.reset() }.unwrap();
2942    }
2943
2944    #[test]
2945    fn test_submit_write_only_notifies_by_default() {
2946        let ring = make_ring(16);
2947        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
2948
2949        let initial_count = notifier.notification_count();
2950
2951        let se = producer.chain().writable(128).build().unwrap();
2952        producer.submit(se).unwrap();
2953
2954        assert!(notifier.notification_count() > initial_count);
2955
2956        // SAFETY: The consumer has never polled the ring and stays inactive.
2957        unsafe { producer.reset() }.unwrap();
2958    }
2959
2960    #[test]
2961    fn test_batch_notifies_once_on_finish() {
2962        let ring = make_ring(16);
2963        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2964
2965        let initial_count = notifier.notification_count();
2966
2967        let mut batch = producer.batch();
2968
2969        let mut first = batch.chain().readable(64).build().unwrap();
2970        first.write_all(b"first").unwrap();
2971        batch.submit(first).unwrap();
2972
2973        let mut second = batch.chain().readable(64).build().unwrap();
2974        second.write_all(b"second").unwrap();
2975        batch.submit(second).unwrap();
2976
2977        assert_eq!(notifier.notification_count(), initial_count);
2978        assert!(batch.finish().unwrap());
2979        assert_eq!(notifier.notification_count(), initial_count + 1);
2980
2981        let (recv, reply) = poll_received(&mut consumer);
2982        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"first");
2983        consumer.complete(recv, reply).unwrap();
2984
2985        let (recv, reply) = poll_received(&mut consumer);
2986        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"second");
2987        consumer.complete(recv, reply).unwrap();
2988        producer.drain(drop).unwrap();
2989    }
2990
2991    #[test]
2992    fn test_batch_finish_notifies_from_batch_start_cursor() {
2993        let ring = make_ring(16);
2994        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
2995
2996        let cursor = consumer.avail_cursor();
2997        consumer
2998            .set_avail_suppression(SuppressionKind::Descriptor(cursor))
2999            .unwrap();
3000
3001        let mut batch = producer.batch();
3002
3003        let mut first = batch.chain().readable(64).build().unwrap();
3004        first.write_all(b"first").unwrap();
3005        batch.submit(first).unwrap();
3006        assert_eq!(notifier.notification_count(), 0);
3007
3008        let mut second = batch.chain().readable(64).writable(64).build().unwrap();
3009        second.write_all(b"second").unwrap();
3010        batch.submit(second).unwrap();
3011
3012        assert!(batch.finish().unwrap());
3013
3014        assert_eq!(notifier.notification_count(), 1);
3015
3016        // SAFETY: The consumer has never polled the ring and stays inactive.
3017        unsafe { producer.reset() }.unwrap();
3018    }
3019
3020    #[test]
3021    fn test_empty_batch_finish_does_not_notify() {
3022        let ring = make_ring(16);
3023        let (mut producer, _consumer, notifier) = make_test_producer(&ring);
3024
3025        let batch = producer.batch();
3026        assert!(!batch.finish().unwrap());
3027        assert_eq!(notifier.notification_count(), 0);
3028    }
3029
3030    #[test]
3031    fn test_batch_can_finish_without_notification() {
3032        let ring = make_ring(16);
3033        let (mut producer, mut consumer, notifier) = make_test_producer(&ring);
3034
3035        let mut batch = producer.batch();
3036        let mut chain = batch.chain().readable(4).build().unwrap();
3037        chain.write_all(b"data").unwrap();
3038        batch.submit(chain).unwrap();
3039        batch.finish_without_notify();
3040
3041        assert_eq!(notifier.notification_count(), 0);
3042
3043        let (recv, reply) = poll_received(&mut consumer);
3044        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"data");
3045
3046        consumer.complete(recv, reply).unwrap();
3047        producer.drain(drop).unwrap();
3048    }
3049
3050    #[test]
3051    fn test_write_only_round_trip() {
3052        let ring = make_ring(16);
3053        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3054
3055        let se = producer.chain().writable(32).build().unwrap();
3056        let token = producer.submit(se).unwrap();
3057
3058        let (recv, reply) = poll_received(&mut consumer);
3059        assert_eq!(recv.token(), token);
3060        assert!(recv.to_bytes().unwrap().is_empty());
3061
3062        if let ReplyChain::Writable(mut wc) = reply {
3063            wc.write_all(b"filled-by-consumer").unwrap();
3064            consumer.complete(recv, wc).unwrap();
3065        } else {
3066            panic!("expected Writable");
3067        }
3068
3069        let used = producer.poll().unwrap().unwrap();
3070        assert_eq!(used.token(), token);
3071        assert_eq!(used.to_bytes().unwrap().len(), b"filled-by-consumer".len());
3072        assert_eq!(used.to_bytes().unwrap().as_ref(), b"filled-by-consumer");
3073    }
3074
3075    #[test]
3076    fn test_read_only_round_trip() {
3077        let ring = make_ring(16);
3078        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3079
3080        let mut se = producer.chain().readable(32).build().unwrap();
3081        se.write_all(b"fire-and-forget").unwrap();
3082        let token = producer.submit(se).unwrap();
3083
3084        let (recv, reply) = poll_received(&mut consumer);
3085        assert_eq!(recv.token(), token);
3086        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"fire-and-forget");
3087        assert!(matches!(reply, ReplyChain::Ack(_)));
3088        consumer.complete(recv, reply).unwrap();
3089
3090        let used = producer.poll().unwrap().unwrap();
3091        assert!(matches!(used, UsedChain::Ack(t) if t == token));
3092    }
3093
3094    #[test]
3095    fn test_readwrite_round_trip() {
3096        let ring = make_ring(16);
3097        let (mut producer, mut consumer, _notifier) = make_test_producer(&ring);
3098
3099        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3100        se.write_all(b"request data").unwrap();
3101        let token = producer.submit(se).unwrap();
3102
3103        let (recv, reply) = poll_received(&mut consumer);
3104        assert_eq!(recv.to_bytes().unwrap().as_ref(), b"request data");
3105        if let ReplyChain::Writable(mut wc) = reply {
3106            wc.write_all(b"response data").unwrap();
3107            consumer.complete(recv, wc).unwrap();
3108        } else {
3109            panic!("expected Writable");
3110        }
3111
3112        let used = producer.poll().unwrap().unwrap();
3113        assert_eq!(used.token(), token);
3114        assert_eq!(used.to_bytes().unwrap().as_ref(), b"response data");
3115    }
3116
3117    #[test]
3118    fn test_poll_used_requires_owned_mapping() {
3119        let ring = make_ring(16);
3120        let layout = ring.layout();
3121        let test_mem = ring.mem();
3122        let pool_base = test_mem.base_addr() + Layout::query_size(ring.len()) as u64 + 0x100;
3123        let pool_layout = SlotLayout::new(pool_base, 128, 0x8000 / 128).unwrap();
3124        let pool = SlotPool::new(pool_layout).unwrap();
3125        let notifier = TestNotifier::new();
3126        let mem = FaultMem::new(test_mem);
3127        mem.deny_views();
3128        let mut producer = VirtqProducer::new(layout, mem.clone(), notifier.clone(), pool.clone());
3129        let mut consumer = VirtqConsumer::new(layout, mem, notifier);
3130
3131        let mut se = producer.chain().readable(64).writable(128).build().unwrap();
3132        se.write_all(b"request data").unwrap();
3133        producer.submit(se).unwrap();
3134
3135        let (recv, reply) = poll_received(&mut consumer);
3136        if let ReplyChain::Writable(mut wc) = reply {
3137            wc.write_all(b"response data").unwrap();
3138            consumer.complete(recv, wc).unwrap();
3139        } else {
3140            panic!("expected Writable");
3141        }
3142
3143        assert!(matches!(producer.poll(), Err(VirtqError::MemoryReadError)));
3144        assert_eq!(pool.num_live(), 0);
3145    }
3146
3147    #[test]
3148    fn test_villain_used_len_exceeding_writable_capacity_is_rejected_and_released() {
3149        let ring = make_ring(16);
3150        let (mut producer, _consumer, _notifier) = make_test_producer_with_slot_size(&ring, 4);
3151        let mut ring_consumer = make_consumer(&ring);
3152
3153        let se = producer.chain().writable(4).build().unwrap();
3154        producer.submit(se).unwrap();
3155
3156        let (id, _) = ring_consumer.poll_available().unwrap();
3157        ring_consumer.submit_used(id, 8).unwrap();
3158
3159        assert!(matches!(producer.poll(), Err(VirtqError::InvalidState)));
3160        assert_eq!(producer.inner.num_inflight(), 0);
3161
3162        let se = producer.chain().writable(4).build().unwrap();
3163        producer.submit(se).unwrap();
3164        assert_eq!(producer.inner.num_inflight(), 1);
3165
3166        // SAFETY: Both consumers stay inactive and no chain handles survive.
3167        unsafe { producer.reset() }.unwrap();
3168    }
3169
3170    #[test]
3171    fn test_villain_used_descriptor_with_invalid_id_is_rejected() {
3172        let ring = make_ring(16);
3173        let (mut producer, _consumer, _notifier) = make_test_producer(&ring);
3174
3175        let se = producer.chain().writable(4).build().unwrap();
3176        producer.submit(se).unwrap();
3177
3178        let mut desc = Descriptor::new(0, 0, ring.len() as u16, DescFlags::empty());
3179        desc.mark_used(true);
3180        ring.write_desc(0, desc);
3181
3182        assert!(matches!(
3183            producer.poll(),
3184            Err(VirtqError::RingError(RingError::InvalidState))
3185        ));
3186        assert_eq!(producer.inner.num_inflight(), 1);
3187
3188        // SAFETY: The consumer has never polled the ring and stays inactive.
3189        unsafe { producer.reset() }.unwrap();
3190    }
3191}