1use alloc::vec::Vec;
15
16use bytemuck::Zeroable;
17use fixedbitset::FixedBitSet;
18use smallvec::SmallVec;
19use thiserror::Error;
20
21use super::super::desc::{DescFlags, DescTable, Descriptor};
22use super::super::event::{EventFlags, EventSuppression};
23use super::super::{Layout, MemOps};
24use super::{BufferChain, BufferElement, MemOp, RingError};
25
26#[derive(Error, Debug, Copy, Clone, PartialEq, Eq)]
28pub enum DescError {
29 #[error("unused descriptor is not zeroed")]
31 ExpectedZero,
32 #[error("descriptor contains unknown flags")]
34 UnknownFlags,
35 #[error("descriptor is not initially available")]
37 NotAvailable,
38 #[error("indirect descriptor is unsupported")]
40 Indirect,
41 #[error("chain continues beyond the available descriptor prefix")]
43 ChainContinues,
44 #[error("descriptor ID is out of range")]
46 IdOutOfRange,
47 #[error("descriptor ID differs within a chain")]
49 IdMismatch,
50 #[error("descriptor ID is already used by another chain")]
52 DuplicateId,
53 #[error("readable descriptor follows a writable descriptor")]
55 ReadableAfterWritable,
56}
57
58#[derive(Error, Debug)]
60pub enum ImageError {
61 #[error(transparent)]
63 Ring(#[from] RingError),
64 #[error("available descriptor count {available} exceeds limit {capacity}")]
66 DescCount {
67 available: usize,
69 capacity: usize,
71 },
72 #[error("event suppression at address 0x{addr:x} is not canonical")]
74 Event {
75 addr: u64,
77 },
78 #[error("descriptor {index} is not canonical: {reason}")]
80 Desc {
81 index: u16,
83 reason: DescError,
85 },
86 #[error("descriptor {index} buffer at 0x{addr:x} with length {len} was rejected")]
88 Buffer {
89 index: u16,
91 addr: u64,
93 len: u32,
95 },
96}
97
98impl ImageError {
99 fn desc_count(available: usize, capacity: usize) -> Self {
100 Self::DescCount {
101 available,
102 capacity,
103 }
104 }
105
106 fn event(addr: u64) -> Self {
107 Self::Event { addr }
108 }
109
110 fn desc(index: u16, reason: DescError) -> Self {
111 Self::Desc { index, reason }
112 }
113
114 fn buffer(index: u16, addr: u64, len: u32) -> Self {
115 Self::Buffer { index, addr, len }
116 }
117}
118
119#[derive(Debug, Clone)]
121pub struct CanonChain {
122 id: u16,
123 inner: BufferChain,
124}
125
126impl CanonChain {
127 fn new(id: u16, chain: BufferChain) -> Self {
128 Self { id, inner: chain }
129 }
130
131 pub fn id(&self) -> u16 {
133 self.id
134 }
135
136 pub fn buffers(&self) -> &BufferChain {
138 &self.inner
139 }
140
141 pub fn into_buffers(self) -> BufferChain {
143 self.inner
144 }
145}
146
147pub fn validate_canon_image<M, F>(
164 mem: &M,
165 layout: Layout,
166 avail_descs: usize,
167 mut validate_buf: F,
168) -> Result<Vec<CanonChain>, ImageError>
169where
170 M: MemOps,
171 F: FnMut(u16, BufferElement) -> bool,
172{
173 let cap = layout.desc_table_len() as usize;
174 if avail_descs > cap {
175 return Err(ImageError::desc_count(avail_descs, cap));
176 }
177
178 let canon_evt = EventSuppression::new(0, EventFlags::ENABLE);
179 for addr in [layout.drv_evt_addr(), layout.dev_evt_addr()] {
180 let evt = mem
181 .read_val::<EventSuppression>(addr)
182 .map_err(|_| RingError::mem_err(MemOp::ReadEvent, addr))?;
183
184 if evt != canon_evt {
185 return Err(ImageError::event(addr));
186 }
187 }
188
189 let table = unsafe { DescTable::from_raw_parts(layout.desc_table_addr(), cap) };
191
192 let mut seen_ids = FixedBitSet::with_capacity(cap);
193 let mut chains = Vec::new();
194 let mut pos = 0usize;
195
196 while pos < avail_descs {
197 let head_idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
198 let (head, _) = read_canon_avail_desc(mem, &table, head_idx)?;
199 let id_idx = head.id as usize;
200 if id_idx >= cap {
201 return Err(ImageError::desc(head_idx, DescError::IdOutOfRange));
202 }
203
204 if seen_ids.contains(id_idx) {
205 return Err(ImageError::desc(head_idx, DescError::DuplicateId));
206 }
207
208 seen_ids.insert(id_idx);
209
210 let mut elems = SmallVec::<[BufferElement; 4]>::new();
211 let mut split = 0usize;
212
213 loop {
214 let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
215 let (desc, flags) = read_canon_avail_desc(mem, &table, idx)?;
216 if desc.id != head.id {
217 return Err(ImageError::desc(idx, DescError::IdMismatch));
218 }
219
220 let elem = BufferElement::from(&desc);
221 if !elem.writable && split != elems.len() {
222 return Err(ImageError::desc(idx, DescError::ReadableAfterWritable));
223 }
224
225 split += usize::from(!elem.writable);
226
227 if !validate_buf(idx, elem) {
228 return Err(ImageError::buffer(idx, elem.addr, elem.len));
229 }
230
231 elems.push(elem);
232 pos += 1;
233
234 if !flags.contains(DescFlags::NEXT) {
235 break;
236 }
237 if pos >= avail_descs {
238 return Err(ImageError::desc(idx, DescError::ChainContinues));
239 }
240 }
241
242 let canon = CanonChain::new(head.id, BufferChain { elems, split });
243 chains.push(canon);
244 }
245
246 let empty = Descriptor::zeroed();
247 for pos in avail_descs..cap {
248 let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
249 let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
250
251 let desc = mem
252 .read_val::<Descriptor>(addr)
253 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
254
255 if desc != empty {
256 return Err(ImageError::desc(idx, DescError::ExpectedZero));
257 }
258 }
259
260 Ok(chains)
261}
262
263pub fn validate_canon_prefix<M, F>(
273 mem: &M,
274 layout: Layout,
275 max_avail_descs: usize,
276 validate_buf: F,
277) -> Result<Vec<CanonChain>, ImageError>
278where
279 M: MemOps,
280 F: FnMut(u16, BufferElement) -> bool,
281{
282 let cap = layout.desc_table_len() as usize;
283 let table = unsafe { DescTable::from_raw_parts(layout.desc_table_addr(), cap) };
285 let empty = Descriptor::zeroed();
286 let mut avail_descs = cap;
287
288 for pos in 0..cap {
289 let idx = u16::try_from(pos).map_err(|_| RingError::InvalidState)?;
290 let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
291 let desc = mem
292 .read_val::<Descriptor>(addr)
293 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
294
295 if desc == empty {
296 avail_descs = pos;
297 break;
298 }
299 }
300
301 if avail_descs > max_avail_descs {
302 return Err(ImageError::desc_count(avail_descs, max_avail_descs));
303 }
304
305 validate_canon_image(mem, layout, avail_descs, validate_buf)
306}
307
308fn read_canon_avail_desc<M: MemOps>(
309 mem: &M,
310 table: &DescTable,
311 idx: u16,
312) -> Result<(Descriptor, DescFlags), ImageError> {
313 let addr = table.desc_addr(idx).ok_or(RingError::InvalidState)?;
314 let desc = mem
315 .read_val::<Descriptor>(addr)
316 .map_err(|_| RingError::mem_err(MemOp::ReadDesc, addr))?;
317
318 let flags = DescFlags::from_bits(desc.flags)
319 .ok_or_else(|| ImageError::desc(idx, DescError::UnknownFlags))?;
320
321 if flags.contains(DescFlags::INDIRECT) {
322 return Err(ImageError::desc(idx, DescError::Indirect));
323 }
324 if !flags.is_avail(true) {
325 return Err(ImageError::desc(idx, DescError::NotAvailable));
326 }
327
328 Ok((desc, flags))
329}
330
331#[cfg(test)]
332mod tests {
333 use super::super::BufferChainBuilder;
334 use super::super::tests::{OwnedRing, make_consumer, make_producer, make_ring};
335 use super::*;
336
337 fn writable_chain(base: u64, lengths: &[u32]) -> BufferChain {
338 BufferChainBuilder::new()
339 .writables(lengths.iter().scan(base, |addr, &len| {
340 let element = BufferElement {
341 addr: *addr,
342 len,
343 writable: true,
344 };
345 *addr += len as u64;
346 Some(element)
347 }))
348 .build()
349 .unwrap()
350 }
351
352 fn validate_all(ring: &OwnedRing, avail_descs: usize) -> Result<Vec<CanonChain>, ImageError> {
353 validate_canon_image(&ring.mem(), ring.layout(), avail_descs, |_, _| true)
354 }
355
356 #[test]
357 fn canon_reset_normalizes_empty_image() {
358 let ring = make_ring(8);
359 let mut producer = make_producer(&ring);
360 let mut consumer = make_consumer(&ring);
361
362 producer.submit_one(0x1000, 64, true).unwrap();
363 producer.enable_used_notifications_desc(3, false).unwrap();
364 consumer.enable_avail_notifications_desc(5, false).unwrap();
365
366 producer.reset().unwrap();
367 consumer.reset().unwrap();
368
369 assert!(validate_all(&ring, 0).unwrap().is_empty());
370 assert_eq!(
371 ring.read_driver_event(),
372 EventSuppression::new(0, EventFlags::ENABLE)
373 );
374 assert_eq!(
375 ring.read_device_event(),
376 EventSuppression::new(0, EventFlags::ENABLE)
377 );
378 for index in 0..ring.len() as u16 {
379 assert_eq!(ring.read_desc(index), Descriptor::zeroed());
380 }
381 }
382
383 #[test]
384 fn canon_multi_desc_refill_is_visible_to_fresh_consumer() {
385 let ring = make_ring(8);
386 let mut producer = make_producer(&ring);
387 let mut consumer = make_consumer(&ring);
388
389 producer.reset().unwrap();
390 consumer.reset().unwrap();
391
392 let chains = [
393 writable_chain(0x1000, &[64, 128, 256]),
394 writable_chain(0x2000, &[512, 1024]),
395 writable_chain(0x4000, &[64, 64, 64]),
396 ];
397 let mut expected = Vec::new();
398 for chain in &chains {
399 let id = producer.submit_available(chain).unwrap();
400 expected.push((id, chain.len()));
401 }
402
403 assert_eq!(producer.num_free(), 0);
404 assert_eq!(producer.avail_cursor().head(), 0);
405 assert!(!producer.avail_cursor().wrap());
406
407 let image = validate_canon_image(&ring.mem(), ring.layout(), ring.len(), |_, elem| {
408 elem.writable
409 && elem.len > 0
410 && elem
411 .addr
412 .checked_add(elem.len as u64)
413 .is_some_and(|end| end <= 0x5000)
414 })
415 .unwrap();
416
417 assert_eq!(image.len(), expected.len());
418 for (validated, (id, len)) in image.iter().zip(&expected) {
419 assert_eq!(validated.id(), *id);
420 assert_eq!(validated.buffers().len(), *len);
421 assert!(validated.buffers().elems().iter().all(|elem| elem.writable));
422 assert_eq!(producer.id_num[*id as usize] as usize, *len);
423 }
424
425 let mut fresh = make_consumer(&ring);
426 for (expected_id, expected_len) in expected {
427 let (id, chain) = fresh.poll_available().unwrap();
428 assert_eq!(id, expected_id);
429 assert_eq!(chain.len(), expected_len);
430 }
431 assert!(matches!(fresh.poll_available(), Err(RingError::WouldBlock)));
432 }
433
434 #[test]
435 fn canon_image_rejects_invalid_ids() {
436 let duplicate_ring = make_ring(4);
437 let mut producer = make_producer(&duplicate_ring);
438 producer.submit_one(0x1000, 64, true).unwrap();
439 producer.submit_one(0x2000, 64, true).unwrap();
440
441 let head_id = duplicate_ring.read_desc(0).id;
442 let mut duplicate = duplicate_ring.read_desc(1);
443 duplicate.id = head_id;
444 duplicate_ring.write_desc(1, duplicate);
445
446 assert!(matches!(
447 validate_all(&duplicate_ring, 2),
448 Err(ImageError::Desc {
449 index: 1,
450 reason: DescError::DuplicateId,
451 })
452 ));
453
454 let mismatch_ring = make_ring(4);
455 let mut producer = make_producer(&mismatch_ring);
456 producer
457 .submit_available(&writable_chain(0x3000, &[64, 64]))
458 .unwrap();
459
460 let mut tail = mismatch_ring.read_desc(1);
461 tail.id = tail.id.wrapping_sub(1);
462 mismatch_ring.write_desc(1, tail);
463
464 assert!(matches!(
465 validate_all(&mismatch_ring, 2),
466 Err(ImageError::Desc {
467 index: 1,
468 reason: DescError::IdMismatch,
469 })
470 ));
471
472 let range_ring = make_ring(4);
473 let mut producer = make_producer(&range_ring);
474 producer.submit_one(0x4000, 64, true).unwrap();
475
476 let mut out_of_range = range_ring.read_desc(0);
477 out_of_range.id = range_ring.len() as u16;
478 range_ring.write_desc(0, out_of_range);
479
480 assert!(matches!(
481 validate_all(&range_ring, 1),
482 Err(ImageError::Desc {
483 index: 0,
484 reason: DescError::IdOutOfRange,
485 })
486 ));
487 }
488
489 #[test]
490 fn canon_image_rejects_invalid_flags_and_wrap_state() {
491 let unknown_ring = make_ring(4);
492 let mut producer = make_producer(&unknown_ring);
493 producer.submit_one(0x1000, 64, true).unwrap();
494
495 let mut unknown = unknown_ring.read_desc(0);
496 unknown.flags |= 1 << 3;
497 unknown_ring.write_desc(0, unknown);
498
499 assert!(matches!(
500 validate_all(&unknown_ring, 1),
501 Err(ImageError::Desc {
502 index: 0,
503 reason: DescError::UnknownFlags,
504 })
505 ));
506
507 let used_ring = make_ring(4);
508 let mut producer = make_producer(&used_ring);
509 producer.submit_one(0x2000, 64, true).unwrap();
510
511 let mut used = used_ring.read_desc(0);
512 used.mark_used(true);
513 used_ring.write_desc(0, used);
514
515 assert!(matches!(
516 validate_all(&used_ring, 1),
517 Err(ImageError::Desc {
518 index: 0,
519 reason: DescError::NotAvailable,
520 })
521 ));
522
523 let indirect_ring = make_ring(4);
524 let mut producer = make_producer(&indirect_ring);
525 producer.submit_one(0x3000, 64, true).unwrap();
526
527 let mut indirect = indirect_ring.read_desc(0);
528 indirect.flags |= DescFlags::INDIRECT.bits();
529 indirect_ring.write_desc(0, indirect);
530
531 assert!(matches!(
532 validate_all(&indirect_ring, 1),
533 Err(ImageError::Desc {
534 index: 0,
535 reason: DescError::Indirect,
536 })
537 ));
538 }
539
540 #[test]
541 fn canon_image_rejects_malformed_chain_and_unused_desc() {
542 let chain_ring = make_ring(4);
543 let mut producer = make_producer(&chain_ring);
544 producer
545 .submit_available(&writable_chain(0x1000, &[64, 64]))
546 .unwrap();
547
548 let mut tail = chain_ring.read_desc(1);
549 tail.flags |= DescFlags::NEXT.bits();
550 chain_ring.write_desc(1, tail);
551
552 assert!(matches!(
553 validate_all(&chain_ring, 2),
554 Err(ImageError::Desc {
555 index: 1,
556 reason: DescError::ChainContinues,
557 })
558 ));
559
560 let unused_ring = make_ring(4);
561 let mut producer = make_producer(&unused_ring);
562 producer.submit_one(0x2000, 64, true).unwrap();
563 unused_ring.write_desc(3, Descriptor::new(0x3000, 64, 0, DescFlags::empty()));
564
565 assert!(matches!(
566 validate_all(&unused_ring, 1),
567 Err(ImageError::Desc {
568 index: 3,
569 reason: DescError::ExpectedZero,
570 })
571 ));
572
573 let direction_ring = make_ring(4);
574 let mut producer = make_producer(&direction_ring);
575 producer
576 .submit_available(&writable_chain(0x4000, &[64, 64]))
577 .unwrap();
578
579 let mut readable_tail = direction_ring.read_desc(1);
580 readable_tail.flags &= !DescFlags::WRITE.bits();
581 direction_ring.write_desc(1, readable_tail);
582
583 assert!(matches!(
584 validate_all(&direction_ring, 2),
585 Err(ImageError::Desc {
586 index: 1,
587 reason: DescError::ReadableAfterWritable,
588 })
589 ));
590 }
591
592 #[test]
593 fn canon_image_rejects_noncanon_evt_and_buffer_bounds() {
594 let evt_ring = make_ring(4);
595 evt_ring
596 .mem()
597 .write_val(
598 evt_ring.layout().drv_evt_addr(),
599 EventSuppression::new(1, EventFlags::ENABLE),
600 )
601 .unwrap();
602
603 match validate_all(&evt_ring, 0) {
604 Err(ImageError::Event { addr }) => {
605 let expected = evt_ring.layout().drv_evt_addr();
606 assert_eq!(addr, expected);
607 }
608 other => unreachable!("unexpected result: {other:?}"),
609 }
610
611 let bounds = make_ring(4);
612 let mut producer = make_producer(&bounds);
613 producer.submit_one(u64::MAX - 15, 32, true).unwrap();
614
615 let res = validate_canon_image(&bounds.mem(), bounds.layout(), 1, |_, element| {
616 element
617 .addr
618 .checked_add(element.len as u64)
619 .is_some_and(|end| end <= 0x8000)
620 });
621
622 match res {
623 Err(ImageError::Buffer { index, addr, len }) => {
624 assert_eq!(index, 0);
625 assert_eq!(addr, u64::MAX - 15);
626 assert_eq!(len, 32);
627 }
628 other => unreachable!("unexpected result: {other:?}"),
629 }
630 }
631
632 #[test]
633 fn canon_image_rejects_avail_count_over_capacity() {
634 let ring = make_ring(4);
635 assert!(matches!(
636 validate_all(&ring, 5),
637 Err(ImageError::DescCount {
638 available: 5,
639 capacity: 4,
640 })
641 ));
642 }
643
644 #[test]
645 fn canon_prefix_rejects_incomplete_chains_and_gaps() {
646 let ring = make_ring(4);
647 let mut producer = make_producer(&ring);
648 producer
649 .submit_available(&writable_chain(0x1000, &[64, 64]))
650 .unwrap();
651
652 ring.write_desc(1, Descriptor::zeroed());
653 let err = validate_canon_prefix(&ring.mem(), ring.layout(), ring.len(), |_, _| true);
654
655 assert!(matches!(
656 err,
657 Err(ImageError::Desc {
658 index: 0,
659 reason: DescError::ChainContinues,
660 })
661 ));
662
663 for avail in 0..=1 {
664 let ring = make_ring(4);
665 let mut producer = make_producer(&ring);
666
667 for _ in 0..avail {
668 producer.submit_one(0x1000, 64, true).unwrap();
669 }
670
671 ring.write_desc(3, Descriptor::new(0x2000, 64, 0, DescFlags::empty()));
672 let err = validate_canon_prefix(&ring.mem(), ring.layout(), ring.len(), |_, _| true);
673
674 assert!(matches!(
675 err,
676 Err(ImageError::Desc {
677 index: 3,
678 reason: DescError::ExpectedZero,
679 })
680 ));
681 }
682 }
683}