Skip to main content

hyperlight_common/virtq/ring/
canonical.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2026 The Hyperlight Authors.
3
4//! Canonical packed virtqueue images.
5//!
6//! A canonical image starts at the initial wrap round. Available descriptors
7//! occupy a complete prefix, unused descriptors are zero, and both event
8//! structures are enabled at offset zero. This form can be restored as bytes
9//! and validated before either peer resumes.
10//!
11//! Ring resets normalize the structures owned by each peer. Buffer range
12//! policy remains with the integration through the validator callback.
13
14use alloc::vec::Vec;
15
16use bytemuck::Zeroable;
17use fixedbitset::FixedBitSet;
18use smallvec::SmallVec;
19use thiserror::Error;
20
21use super::super::desc::{DescFlags, DescTable, Descriptor};
22use super::super::event::{EventFlags, EventSuppression};
23use super::super::{Layout, MemOps};
24use super::{BufferChain, BufferElement, MemOp, RingError};
25
26/// Why a descriptor does not belong to a canonical packed-ring image.
27#[derive(Error, Debug, Copy, Clone, PartialEq, Eq)]
28pub enum DescError {
29    /// An unused descriptor was not completely zeroed.
30    #[error("unused descriptor is not zeroed")]
31    ExpectedZero,
32    /// The raw descriptor contains unsupported or reserved flag bits.
33    #[error("descriptor contains unknown flags")]
34    UnknownFlags,
35    /// The descriptor is not available in the initial packed-ring wrap round.
36    #[error("descriptor is not initially available")]
37    NotAvailable,
38    /// Indirect descriptor tables are unsupported.
39    #[error("indirect descriptor is unsupported")]
40    Indirect,
41    /// A chain's NEXT flag extends beyond the available descriptor prefix.
42    #[error("chain continues beyond the available descriptor prefix")]
43    ChainContinues,
44    /// A chain ID is outside the descriptor-table bounds.
45    #[error("descriptor ID is out of range")]
46    IdOutOfRange,
47    /// A tail descriptor does not carry its head descriptor's ID.
48    #[error("descriptor ID differs within a chain")]
49    IdMismatch,
50    /// Two available chains use the same descriptor ID.
51    #[error("descriptor ID is already used by another chain")]
52    DuplicateId,
53    /// A readable descriptor follows a writable descriptor.
54    #[error("readable descriptor follows a writable descriptor")]
55    ReadableAfterWritable,
56}
57
58/// Validation failure for a canonical packed-ring image.
59#[derive(Error, Debug)]
60pub enum ImageError {
61    /// Reading the shared ring image failed.
62    #[error(transparent)]
63    Ring(#[from] RingError),
64    /// The available-descriptor prefix exceeds its allowed limit.
65    #[error("available descriptor count {available} exceeds limit {capacity}")]
66    DescCount {
67        /// Number of descriptors supplied or discovered as available.
68        available: usize,
69        /// Maximum allowed number of available descriptors.
70        capacity: usize,
71    },
72    /// An event-suppression structure is not the canonical enabled value.
73    #[error("event suppression at address 0x{addr:x} is not canonical")]
74    Event {
75        /// Address of the invalid event-suppression structure.
76        addr: u64,
77    },
78    /// A descriptor violates the canonical packed-ring structure.
79    #[error("descriptor {index} is not canonical: {reason}")]
80    Desc {
81        /// Descriptor-table index.
82        index: u16,
83        /// Structural validation failure.
84        reason: DescError,
85    },
86    /// The caller rejected a descriptor's payload range or attributes.
87    #[error("descriptor {index} buffer at 0x{addr:x} with length {len} was rejected")]
88    Buffer {
89        /// Descriptor-table index.
90        index: u16,
91        /// Buffer address from the descriptor.
92        addr: u64,
93        /// Buffer length from the descriptor.
94        len: u32,
95    },
96}
97
98impl ImageError {
99    fn desc_count(available: usize, capacity: usize) -> Self {
100        Self::DescCount {
101            available,
102            capacity,
103        }
104    }
105
106    fn event(addr: u64) -> Self {
107        Self::Event { addr }
108    }
109
110    fn desc(index: u16, reason: DescError) -> Self {
111        Self::Desc { index, reason }
112    }
113
114    fn buffer(index: u16, addr: u64, len: u32) -> Self {
115        Self::Buffer { index, addr, len }
116    }
117}
118
119/// One available descriptor chain from a validated canonical ring image.
120#[derive(Debug, Clone)]
121pub struct CanonChain {
122    id: u16,
123    inner: BufferChain,
124}
125
126impl CanonChain {
127    fn new(id: u16, chain: BufferChain) -> Self {
128        Self { id, inner: chain }
129    }
130
131    /// Descriptor ID shared by every buffer in the chain.
132    pub fn id(&self) -> u16 {
133        self.id
134    }
135
136    /// Validated buffers in descriptor order.
137    pub fn buffers(&self) -> &BufferChain {
138        &self.inner
139    }
140
141    /// Consume the image metadata and return its buffer chain.
142    pub fn into_buffers(self) -> BufferChain {
143        self.inner
144    }
145}
146
147/// Validate a packed ring while neither peer can modify it.
148///
149/// The first `avail_descs` descriptors must form complete available chains
150/// beginning at descriptor zero. Every remaining descriptor must be zeroed,
151/// and both event-suppression structures must be the canonical enabled value.
152/// `validate_buf` supplies integration-specific address, length, and
153/// direction bounds without embedding them in the ring implementation.
154///
155/// The returned chains preserve descriptor IDs and chain boundaries for
156/// cross-checking against producer and pool ownership.
157///
158/// # Errors
159///
160/// Returns [`ImageError`] if event state is not normalized, descriptor
161/// structure is malformed, an unused descriptor is not zero, a buffer is
162/// rejected by `validate_buf`, or shared memory cannot be read.
163pub fn validate_canon_image<M, F>(
164    mem: &M,
165    layout: Layout,
166    avail_descs: usize,
167    mut validate_buf: F,
168) -> Result<Vec<CanonChain>, ImageError>
169where
170    M: MemOps,
171    F: FnMut(u16, BufferElement) -> bool,
172{
173    let cap = layout.desc_table_len() as usize;
174    if avail_descs > cap {
175        return Err(ImageError::desc_count(avail_descs, cap));
176    }
177
178    let canon_evt = EventSuppression::new(0, EventFlags::ENABLE);
179    for addr in [layout.drv_evt_addr(), layout.dev_evt_addr()] {
180        let evt = mem
181            .read_val::<EventSuppression>(addr)
182            .map_err(|_| RingError::mem_err(MemOp::ReadEvent, addr))?;
183
184        if evt != canon_evt {
185            return Err(ImageError::event(addr));
186        }
187    }
188
189    // SAFETY: `Layout` validates the table base, alignment, and descriptor count.
190    let table = unsafe { DescTable::from_raw_parts(layout.desc_table_addr(), cap) };
191
192    let mut seen_ids = FixedBitSet::with_capacity(cap);
193    let mut chains = Vec::new();
194    let mut pos = 0usize;
195
196    while pos < avail_descs {
197        let head_idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
198        let (head, _) = read_canon_avail_desc(mem, &table, head_idx)?;
199        let id_idx = head.id as usize;
200        if id_idx >= cap {
201            return Err(ImageError::desc(head_idx, DescError::IdOutOfRange));
202        }
203
204        if seen_ids.contains(id_idx) {
205            return Err(ImageError::desc(head_idx, DescError::DuplicateId));
206        }
207
208        seen_ids.insert(id_idx);
209
210        let mut elems = SmallVec::<[BufferElement; 4]>::new();
211        let mut split = 0usize;
212
213        loop {
214            let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
215            let (desc, flags) = read_canon_avail_desc(mem, &table, idx)?;
216            if desc.id != head.id {
217                return Err(ImageError::desc(idx, DescError::IdMismatch));
218            }
219
220            let elem = BufferElement::from(&desc);
221            if !elem.writable && split != elems.len() {
222                return Err(ImageError::desc(idx, DescError::ReadableAfterWritable));
223            }
224
225            split += usize::from(!elem.writable);
226
227            if !validate_buf(idx, elem) {
228                return Err(ImageError::buffer(idx, elem.addr, elem.len));
229            }
230
231            elems.push(elem);
232            pos += 1;
233
234            if !flags.contains(DescFlags::NEXT) {
235                break;
236            }
237            if pos >= avail_descs {
238                return Err(ImageError::desc(idx, DescError::ChainContinues));
239            }
240        }
241
242        let canon = CanonChain::new(head.id, BufferChain { elems, split });
243        chains.push(canon);
244    }
245
246    let empty = Descriptor::zeroed();
247    for pos in avail_descs..cap {
248        let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
249        let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
250
251        let desc = mem
252            .read_val::<Descriptor>(addr)
253            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
254
255        if desc != empty {
256            return Err(ImageError::desc(idx, DescError::ExpectedZero));
257        }
258    }
259
260    Ok(chains)
261}
262
263/// Validate a canonical image with at most `max_avail_descs` available descriptors.
264///
265/// The first zero descriptor ends the prefix. [`validate_canon_image`] then
266/// checks the prefix and zero tail. Keep the image unchanged during both passes.
267///
268/// # Errors
269///
270/// Returns [`ImageError`] if the prefix exceeds the limit, a descriptor cannot
271/// be read, or [`validate_canon_image`] rejects the image or its buffers.
272pub fn validate_canon_prefix<M, F>(
273    mem: &M,
274    layout: Layout,
275    max_avail_descs: usize,
276    validate_buf: F,
277) -> Result<Vec<CanonChain>, ImageError>
278where
279    M: MemOps,
280    F: FnMut(u16, BufferElement) -> bool,
281{
282    let cap = layout.desc_table_len() as usize;
283    // SAFETY: `Layout` validates the table base, alignment, and descriptor count.
284    let table = unsafe { DescTable::from_raw_parts(layout.desc_table_addr(), cap) };
285    let empty = Descriptor::zeroed();
286    let mut avail_descs = cap;
287
288    for pos in 0..cap {
289        let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
290        let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
291        let desc = mem
292            .read_val::<Descriptor>(addr)
293            .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
294
295        if desc == empty {
296            avail_descs = pos;
297            break;
298        }
299    }
300
301    if avail_descs > max_avail_descs {
302        return Err(ImageError::desc_count(avail_descs, max_avail_descs));
303    }
304
305    validate_canon_image(mem, layout, avail_descs, validate_buf)
306}
307
308fn read_canon_avail_desc<M: MemOps>(
309    mem: &M,
310    table: &DescTable,
311    idx: u16,
312) -> Result<(Descriptor, DescFlags), ImageError> {
313    let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
314    let desc = mem
315        .read_val::<Descriptor>(addr)
316        .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
317
318    let flags = DescFlags::from_bits(desc.flags)
319        .ok_or_else(|| ImageError::desc(idx, DescError::UnknownFlags))?;
320
321    if flags.contains(DescFlags::INDIRECT) {
322        return Err(ImageError::desc(idx, DescError::Indirect));
323    }
324    if !flags.is_avail(true) {
325        return Err(ImageError::desc(idx, DescError::NotAvailable));
326    }
327
328    Ok((desc, flags))
329}
330
331#[cfg(test)]
332mod tests {
333    use super::super::BufferChainBuilder;
334    use super::super::tests::{OwnedRing, make_consumer, make_producer, make_ring};
335    use super::*;
336
337    fn writable_chain(base: u64, lengths: &[u32]) -> BufferChain {
338        BufferChainBuilder::new()
339            .writables(lengths.iter().scan(base, |addr, &len| {
340                let element = BufferElement {
341                    addr: *addr,
342                    len,
343                    writable: true,
344                };
345                *addr += len as u64;
346                Some(element)
347            }))
348            .build()
349            .unwrap()
350    }
351
352    fn validate_all(ring: &OwnedRing, avail_descs: usize) -> Result<Vec<CanonChain>, ImageError> {
353        validate_canon_image(&ring.mem(), ring.layout(), avail_descs, |_, _| true)
354    }
355
356    #[test]
357    fn canon_reset_normalizes_empty_image() {
358        let ring = make_ring(8);
359        let mut producer = make_producer(&ring);
360        let mut consumer = make_consumer(&ring);
361
362        producer.submit_one(0x1000, 64, true).unwrap();
363        producer.enable_used_notifications_desc(3, false).unwrap();
364        consumer.enable_avail_notifications_desc(5, false).unwrap();
365
366        producer.reset().unwrap();
367        consumer.reset().unwrap();
368
369        assert!(validate_all(&ring, 0).unwrap().is_empty());
370        assert_eq!(
371            ring.read_driver_event(),
372            EventSuppression::new(0, EventFlags::ENABLE)
373        );
374        assert_eq!(
375            ring.read_device_event(),
376            EventSuppression::new(0, EventFlags::ENABLE)
377        );
378        for index in 0..ring.len() as u16 {
379            assert_eq!(ring.read_desc(index), Descriptor::zeroed());
380        }
381    }
382
383    #[test]
384    fn canon_multi_desc_refill_is_visible_to_fresh_consumer() {
385        let ring = make_ring(8);
386        let mut producer = make_producer(&ring);
387        let mut consumer = make_consumer(&ring);
388
389        producer.reset().unwrap();
390        consumer.reset().unwrap();
391
392        let chains = [
393            writable_chain(0x1000, &[64, 128, 256]),
394            writable_chain(0x2000, &[512, 1024]),
395            writable_chain(0x4000, &[64, 64, 64]),
396        ];
397        let mut expected = Vec::new();
398        for chain in &chains {
399            let id = producer.submit_available(chain).unwrap();
400            expected.push((id, chain.len()));
401        }
402
403        assert_eq!(producer.num_free(), 0);
404        assert_eq!(producer.avail_cursor().head(), 0);
405        assert!(!producer.avail_cursor().wrap());
406
407        let image = validate_canon_image(&ring.mem(), ring.layout(), ring.len(), |_, elem| {
408            elem.writable
409                && elem.len > 0
410                && elem
411                    .addr
412                    .checked_add(elem.len as u64)
413                    .is_some_and(|end| end <= 0x5000)
414        })
415        .unwrap();
416
417        assert_eq!(image.len(), expected.len());
418        for (validated, (id, len)) in image.iter().zip(&expected) {
419            assert_eq!(validated.id(), *id);
420            assert_eq!(validated.buffers().len(), *len);
421            assert!(validated.buffers().elems().iter().all(|elem| elem.writable));
422            assert_eq!(producer.id_num[*id as usize] as usize, *len);
423        }
424
425        let mut fresh = make_consumer(&ring);
426        for (expected_id, expected_len) in expected {
427            let (id, chain) = fresh.poll_available().unwrap();
428            assert_eq!(id, expected_id);
429            assert_eq!(chain.len(), expected_len);
430        }
431        assert!(matches!(fresh.poll_available(), Err(RingError::WouldBlock)));
432    }
433
434    #[test]
435    fn canon_image_rejects_invalid_ids() {
436        let duplicate_ring = make_ring(4);
437        let mut producer = make_producer(&duplicate_ring);
438        producer.submit_one(0x1000, 64, true).unwrap();
439        producer.submit_one(0x2000, 64, true).unwrap();
440
441        let head_id = duplicate_ring.read_desc(0).id;
442        let mut duplicate = duplicate_ring.read_desc(1);
443        duplicate.id = head_id;
444        duplicate_ring.write_desc(1, duplicate);
445
446        assert!(matches!(
447            validate_all(&duplicate_ring, 2),
448            Err(ImageError::Desc {
449                index: 1,
450                reason: DescError::DuplicateId,
451            })
452        ));
453
454        let mismatch_ring = make_ring(4);
455        let mut producer = make_producer(&mismatch_ring);
456        producer
457            .submit_available(&writable_chain(0x3000, &[64, 64]))
458            .unwrap();
459
460        let mut tail = mismatch_ring.read_desc(1);
461        tail.id = tail.id.wrapping_sub(1);
462        mismatch_ring.write_desc(1, tail);
463
464        assert!(matches!(
465            validate_all(&mismatch_ring, 2),
466            Err(ImageError::Desc {
467                index: 1,
468                reason: DescError::IdMismatch,
469            })
470        ));
471
472        let range_ring = make_ring(4);
473        let mut producer = make_producer(&range_ring);
474        producer.submit_one(0x4000, 64, true).unwrap();
475
476        let mut out_of_range = range_ring.read_desc(0);
477        out_of_range.id = range_ring.len() as u16;
478        range_ring.write_desc(0, out_of_range);
479
480        assert!(matches!(
481            validate_all(&range_ring, 1),
482            Err(ImageError::Desc {
483                index: 0,
484                reason: DescError::IdOutOfRange,
485            })
486        ));
487    }
488
489    #[test]
490    fn canon_image_rejects_invalid_flags_and_wrap_state() {
491        let unknown_ring = make_ring(4);
492        let mut producer = make_producer(&unknown_ring);
493        producer.submit_one(0x1000, 64, true).unwrap();
494
495        let mut unknown = unknown_ring.read_desc(0);
496        unknown.flags |= 1 << 3;
497        unknown_ring.write_desc(0, unknown);
498
499        assert!(matches!(
500            validate_all(&unknown_ring, 1),
501            Err(ImageError::Desc {
502                index: 0,
503                reason: DescError::UnknownFlags,
504            })
505        ));
506
507        let used_ring = make_ring(4);
508        let mut producer = make_producer(&used_ring);
509        producer.submit_one(0x2000, 64, true).unwrap();
510
511        let mut used = used_ring.read_desc(0);
512        used.mark_used(true);
513        used_ring.write_desc(0, used);
514
515        assert!(matches!(
516            validate_all(&used_ring, 1),
517            Err(ImageError::Desc {
518                index: 0,
519                reason: DescError::NotAvailable,
520            })
521        ));
522
523        let indirect_ring = make_ring(4);
524        let mut producer = make_producer(&indirect_ring);
525        producer.submit_one(0x3000, 64, true).unwrap();
526
527        let mut indirect = indirect_ring.read_desc(0);
528        indirect.flags |= DescFlags::INDIRECT.bits();
529        indirect_ring.write_desc(0, indirect);
530
531        assert!(matches!(
532            validate_all(&indirect_ring, 1),
533            Err(ImageError::Desc {
534                index: 0,
535                reason: DescError::Indirect,
536            })
537        ));
538    }
539
540    #[test]
541    fn canon_image_rejects_malformed_chain_and_unused_desc() {
542        let chain_ring = make_ring(4);
543        let mut producer = make_producer(&chain_ring);
544        producer
545            .submit_available(&writable_chain(0x1000, &[64, 64]))
546            .unwrap();
547
548        let mut tail = chain_ring.read_desc(1);
549        tail.flags |= DescFlags::NEXT.bits();
550        chain_ring.write_desc(1, tail);
551
552        assert!(matches!(
553            validate_all(&chain_ring, 2),
554            Err(ImageError::Desc {
555                index: 1,
556                reason: DescError::ChainContinues,
557            })
558        ));
559
560        let unused_ring = make_ring(4);
561        let mut producer = make_producer(&unused_ring);
562        producer.submit_one(0x2000, 64, true).unwrap();
563        unused_ring.write_desc(3, Descriptor::new(0x3000, 64, 0, DescFlags::empty()));
564
565        assert!(matches!(
566            validate_all(&unused_ring, 1),
567            Err(ImageError::Desc {
568                index: 3,
569                reason: DescError::ExpectedZero,
570            })
571        ));
572
573        let direction_ring = make_ring(4);
574        let mut producer = make_producer(&direction_ring);
575        producer
576            .submit_available(&writable_chain(0x4000, &[64, 64]))
577            .unwrap();
578
579        let mut readable_tail = direction_ring.read_desc(1);
580        readable_tail.flags &= !DescFlags::WRITE.bits();
581        direction_ring.write_desc(1, readable_tail);
582
583        assert!(matches!(
584            validate_all(&direction_ring, 2),
585            Err(ImageError::Desc {
586                index: 1,
587                reason: DescError::ReadableAfterWritable,
588            })
589        ));
590    }
591
592    #[test]
593    fn canon_image_rejects_noncanon_evt_and_buffer_bounds() {
594        let evt_ring = make_ring(4);
595        evt_ring
596            .mem()
597            .write_val(
598                evt_ring.layout().drv_evt_addr(),
599                EventSuppression::new(1, EventFlags::ENABLE),
600            )
601            .unwrap();
602
603        match validate_all(&evt_ring, 0) {
604            Err(ImageError::Event { addr }) => {
605                let expected = evt_ring.layout().drv_evt_addr();
606                assert_eq!(addr, expected);
607            }
608            other => unreachable!("unexpected result: {other:?}"),
609        }
610
611        let bounds = make_ring(4);
612        let mut producer = make_producer(&bounds);
613        producer.submit_one(u64::MAX - 15, 32, true).unwrap();
614
615        let res = validate_canon_image(&bounds.mem(), bounds.layout(), 1, |_, element| {
616            element
617                .addr
618                .checked_add(element.len as u64)
619                .is_some_and(|end| end <= 0x8000)
620        });
621
622        match res {
623            Err(ImageError::Buffer { index, addr, len }) => {
624                assert_eq!(index, 0);
625                assert_eq!(addr, u64::MAX - 15);
626                assert_eq!(len, 32);
627            }
628            other => unreachable!("unexpected result: {other:?}"),
629        }
630    }
631
632    #[test]
633    fn canon_image_rejects_avail_count_over_capacity() {
634        let ring = make_ring(4);
635        assert!(matches!(
636            validate_all(&ring, 5),
637            Err(ImageError::DescCount {
638                available: 5,
639                capacity: 4,
640            })
641        ));
642    }
643
644    #[test]
645    fn canon_prefix_rejects_incomplete_chains_and_gaps() {
646        let ring = make_ring(4);
647        let mut producer = make_producer(&ring);
648        producer
649            .submit_available(&writable_chain(0x1000, &[64, 64]))
650            .unwrap();
651
652        ring.write_desc(1, Descriptor::zeroed());
653        let err = validate_canon_prefix(&ring.mem(), ring.layout(), ring.len(), |_, _| true);
654
655        assert!(matches!(
656            err,
657            Err(ImageError::Desc {
658                index: 0,
659                reason: DescError::ChainContinues,
660            })
661        ));
662
663        for avail in 0..=1 {
664            let ring = make_ring(4);
665            let mut producer = make_producer(&ring);
666
667            for _ in 0..avail {
668                producer.submit_one(0x1000, 64, true).unwrap();
669            }
670
671            ring.write_desc(3, Descriptor::new(0x2000, 64, 0, DescFlags::empty()));
672            let err = validate_canon_prefix(&ring.mem(), ring.layout(), ring.len(), |_, _| true);
673
674            assert!(matches!(
675                err,
676                Err(ImageError::Desc {
677                    index: 3,
678                    reason: DescError::ExpectedZero,
679                })
680            ));
681        }
682    }
683}