1use alloc::vec::Vec;
10
11use anyhow::Result;
12pub use bytes::Buf;
13use bytes::Bytes;
14
15use crate::flatbuffer_wrappers::{ExternalValueSink, ExternalValueSource};
16use crate::virtq::{MemOps, RecvChain, Segments, VirtqError, zeroed_vec};
17
18pub const SIZE_PREFIX_LEN: usize = core::mem::size_of::<u32>();
20
21#[repr(u8)]
23#[derive(Debug, Clone, Copy, PartialEq, Eq, bytemuck::CheckedBitPattern, bytemuck::NoUninit)]
24pub enum MsgKind {
25 Request = 0x01,
27 Response = 0x02,
29 StreamChunk = 0x03,
31 StreamEnd = 0x04,
33 Cancel = 0x05,
35 Log = 0x06,
37 SnapshotCheckpoint = 0x07,
39}
40
41impl TryFrom<u8> for MsgKind {
42 type Error = u8;
43
44 fn try_from(value: u8) -> Result<Self, Self::Error> {
45 match value {
46 0x01 => Ok(Self::Request),
47 0x02 => Ok(Self::Response),
48 0x03 => Ok(Self::StreamChunk),
49 0x04 => Ok(Self::StreamEnd),
50 0x05 => Ok(Self::Cancel),
51 0x06 => Ok(Self::Log),
52 0x07 => Ok(Self::SnapshotCheckpoint),
53 other => Err(other),
54 }
55 }
56}
57
58#[repr(u64)]
60#[derive(Debug, Clone, Copy, PartialEq, Eq, bytemuck::CheckedBitPattern, bytemuck::NoUninit)]
61pub enum MailboxValue {
62 CheckpointPending = 0,
64 CheckpointComplete = 1,
66}
67
68impl MailboxValue {
69 pub fn as_bytes(&self) -> &[u8] {
71 bytemuck::bytes_of(self)
72 }
73
74 pub fn raw(self) -> u64 {
76 self as u64
77 }
78}
79
80impl TryFrom<u64> for MailboxValue {
81 type Error = u64;
82
83 fn try_from(value: u64) -> Result<Self, Self::Error> {
84 match value {
85 0 => Ok(Self::CheckpointPending),
86 1 => Ok(Self::CheckpointComplete),
87 other => Err(other),
88 }
89 }
90}
91
92#[derive(Debug, Clone, Copy, PartialEq, Eq, bytemuck::CheckedBitPattern, bytemuck::NoUninit)]
94#[repr(C)]
95pub struct MsgHeader {
96 pub kind: MsgKind,
98 reserved: [u8; 3],
100 pub cid: u32,
102 pub payload_len: u32,
104}
105
106impl MsgHeader {
107 pub const SIZE: usize = core::mem::size_of::<Self>();
108
109 pub const fn new(kind: MsgKind, cid: u32, payload_len: u32) -> Self {
111 Self {
112 kind,
113 reserved: [0; 3],
114 cid,
115 payload_len,
116 }
117 }
118
119 pub fn as_bytes(&self) -> &[u8] {
121 bytemuck::bytes_of(self)
122 }
123
124 pub fn from_bytes(bytes: &[u8]) -> Option<Self> {
126 let header: Self = bytemuck::checked::try_pod_read_unaligned(bytes).ok()?;
127 (header.reserved == [0; 3]).then_some(header)
128 }
129}
130
131#[derive(Debug)]
133pub struct EncodedMessage<'a> {
134 header: MsgHeader,
135 control: &'a [u8],
136 externals: ExternalValues<'a>,
137 total_len: usize,
138}
139
140impl<'a> EncodedMessage<'a> {
141 pub fn new(
143 kind: MsgKind,
144 cid: u32,
145 control: &'a [u8],
146 externals: ExternalValues<'a>,
147 ) -> Option<Self> {
148 let payload_len = control.len().checked_add(externals.total_len())?;
149 let payload_len = u32::try_from(payload_len).ok()?;
150 let total_len = MsgHeader::SIZE.checked_add(payload_len as usize)?;
151
152 Some(Self {
153 header: MsgHeader::new(kind, cid, payload_len),
154 control,
155 externals,
156 total_len,
157 })
158 }
159
160 pub fn new_snapshot_cp() -> Self {
162 let total_len = MsgHeader::SIZE;
163 let externals = ExternalValues::new();
164
165 Self {
166 header: MsgHeader::new(MsgKind::SnapshotCheckpoint, 0, 0),
167 control: &[],
168 externals,
169 total_len,
170 }
171 }
172
173 pub fn as_buf(&self) -> impl Buf + '_ {
175 EncodedMessageBuf::new(
176 self.header.as_bytes(),
177 self.control,
178 &self.externals.chunks,
179 self.total_len,
180 )
181 }
182
183 pub fn chunks(&self) -> impl Iterator<Item = &[u8]> + '_ {
185 core::iter::once(self.header.as_bytes())
186 .chain(core::iter::once(self.control))
187 .chain(self.externals.chunks())
188 }
189
190 pub fn external_chunks(&self) -> impl Iterator<Item = &[u8]> + '_ {
192 self.externals.chunks()
193 }
194
195 pub const fn header(&self) -> &MsgHeader {
197 &self.header
198 }
199
200 pub const fn control(&self) -> &[u8] {
202 self.control
203 }
204
205 pub const fn external_len(&self) -> usize {
207 self.payload_len() - self.control.len()
208 }
209
210 pub const fn prefix_len(&self) -> usize {
212 MsgHeader::SIZE + self.control.len()
213 }
214
215 pub const fn payload_len(&self) -> usize {
217 self.header.payload_len as usize
218 }
219
220 pub const fn total_len(&self) -> usize {
222 self.total_len
223 }
224}
225
226struct EncodedMessageBuf<'a> {
230 header: &'a [u8],
231 control: &'a [u8],
232 externals: &'a [&'a [u8]],
233 index: usize,
234 offset: usize,
235 remaining: usize,
236}
237
238impl<'a> EncodedMessageBuf<'a> {
239 fn new(
240 header: &'a [u8],
241 control: &'a [u8],
242 externals: &'a [&'a [u8]],
243 remaining: usize,
244 ) -> Self {
245 let mut this = Self {
246 header,
247 control,
248 externals,
249 index: 0,
250 offset: 0,
251 remaining,
252 };
253
254 this.skip_empty_chunks();
255 this
256 }
257
258 fn current(&self) -> Option<&[u8]> {
259 match self.index {
260 0 => Some(self.header),
261 1 => Some(self.control),
262 index => self.externals.get(index - 2).copied(),
263 }
264 }
265
266 fn skip_empty_chunks(&mut self) {
267 while self
268 .current()
269 .is_some_and(|chunk| self.offset >= chunk.len())
270 {
271 self.index += 1;
272 self.offset = 0;
273 }
274 }
275}
276
277impl Buf for EncodedMessageBuf<'_> {
278 fn remaining(&self) -> usize {
279 self.remaining
280 }
281
282 fn chunk(&self) -> &[u8] {
283 if self.remaining == 0 {
284 return &[];
285 }
286
287 #[allow(clippy::expect_used)] let chunk = self.current().expect("message length mismatch");
289 &chunk[self.offset..]
290 }
291
292 fn advance(&mut self, cnt: usize) {
293 assert!(cnt <= self.remaining, "cannot advance past remaining bytes");
294
295 self.remaining -= cnt;
296 let mut cnt = cnt;
297
298 while cnt != 0 {
299 #[allow(clippy::expect_used)] let chunk = self.current().expect("message length mismatch");
301 let advanced = cnt.min(chunk.len() - self.offset);
302
303 self.offset += advanced;
304 cnt -= advanced;
305 self.skip_empty_chunks();
306 }
307 }
308}
309
310#[derive(Debug, Default)]
312pub struct ExternalValues<'a> {
313 chunks: Vec<&'a [u8]>,
314 total_len: usize,
315}
316
317impl<'a> ExternalValues<'a> {
318 pub fn new() -> Self {
320 Self::default()
321 }
322
323 fn chunks(&self) -> impl Iterator<Item = &[u8]> + '_ {
325 self.chunks.iter().copied()
326 }
327
328 pub const fn total_len(&self) -> usize {
330 self.total_len
331 }
332}
333
334impl<'a> ExternalValueSink<'a> for ExternalValues<'a> {
335 fn push_bytes(&mut self, value: &'a [u8]) -> Result<()> {
336 if value.is_empty() {
337 return Ok(());
338 }
339
340 self.total_len = self
341 .total_len
342 .checked_add(value.len())
343 .ok_or_else(|| anyhow::anyhow!("external value length overflow"))?;
344
345 self.chunks.push(value);
346 Ok(())
347 }
348
349 fn push_chunks(&mut self, value: &'a [Bytes]) -> Result<()> {
350 let total_len = value
351 .iter()
352 .try_fold(self.total_len, |len, chunk| len.checked_add(chunk.len()))
353 .ok_or_else(|| anyhow::anyhow!("external value length overflow"))?;
354
355 let chunks = value
356 .iter()
357 .map(Bytes::as_ref)
358 .filter(|chunk| !chunk.is_empty());
359
360 self.chunks.extend(chunks);
361 self.total_len = total_len;
362 Ok(())
363 }
364}
365
366impl<M: MemOps> ExternalValueSource for RecvChain<M> {
368 fn take_bytes(&mut self, length: usize) -> Result<Vec<u8>> {
369 let mut value = external_buffer(length, self.remaining())?;
370 self.read_exact(&mut value)?;
371 Ok(value)
372 }
373
374 fn take_chunks(&mut self, length: usize) -> Result<Vec<Bytes>> {
375 if length == 0 {
376 return Ok(Vec::new());
377 }
378
379 let mut value = external_buffer(length, self.remaining())?;
380 let mut chunks = Vec::new();
381
382 chunks
383 .try_reserve_exact(1)
384 .map_err(|_| VirtqError::Bookkeeping)?;
385
386 self.read_exact(&mut value)?;
387
388 chunks.push(Bytes::from(value));
389 Ok(chunks)
390 }
391
392 fn finish(&mut self) -> Result<()> {
393 finish_external_values(self.remaining())
394 }
395}
396
397impl ExternalValueSource for Segments {
398 fn take_bytes(&mut self, length: usize) -> Result<Vec<u8>> {
399 let segments = take_external_segments(self, length)?;
400 let mut value = Vec::new();
401
402 value
403 .try_reserve_exact(length)
404 .map_err(|_| VirtqError::Bookkeeping)?;
405
406 for seg in segments.iter() {
407 value.extend_from_slice(seg);
408 }
409
410 Ok(value)
411 }
412
413 fn take_chunks(&mut self, length: usize) -> Result<Vec<Bytes>> {
414 Ok(take_external_segments(self, length)?.into_chunks())
415 }
416
417 fn finish(&mut self) -> Result<()> {
418 finish_external_values(self.len())
419 }
420}
421
422fn external_buffer(length: usize, remaining: usize) -> Result<Vec<u8>> {
423 if length > remaining {
424 return Err(VirtqError::ReceiveTooShort {
425 requested: length,
426 remaining,
427 }
428 .into());
429 }
430 Ok(zeroed_vec(length)?)
431}
432
433fn take_external_segments(segments: &mut Segments, length: usize) -> Result<Segments> {
434 segments.split_to(length).ok_or_else(|| {
435 VirtqError::ReceiveTooShort {
436 requested: length,
437 remaining: segments.len(),
438 }
439 .into()
440 })
441}
442
443fn finish_external_values(remaining: usize) -> Result<()> {
444 if remaining != 0 {
445 anyhow::bail!("Virtqueue message has {remaining} trailing external bytes");
446 }
447 Ok(())
448}
449
450pub fn size_prefix_payload_len(prefix: &[u8]) -> Option<usize> {
452 let prefix = <[u8; SIZE_PREFIX_LEN]>::try_from(prefix).ok()?;
454 usize::try_from(u32::from_le_bytes(prefix)).ok()
455}
456
457pub const fn size_prefixed_len(payload_len: usize) -> Option<usize> {
459 SIZE_PREFIX_LEN.checked_add(payload_len)
461}
462
463#[cfg(test)]
464mod tests {
465 use bytemuck::checked::try_pod_read_unaligned;
466
467 use super::*;
468 use crate::flatbuffer_wrappers::ExternalValueSink;
469
470 #[test]
472 fn mailbox_value_wire_contract() {
473 for (value, raw) in [
474 (MailboxValue::CheckpointPending, 0u64),
475 (MailboxValue::CheckpointComplete, 1),
476 ] {
477 assert_eq!(value.raw(), raw);
478 assert_eq!(value.as_bytes(), raw.to_ne_bytes());
479 assert_eq!(MailboxValue::try_from(raw), Ok(value));
480
481 let bits = try_pod_read_unaligned::<MailboxValue>(value.as_bytes()).unwrap();
482 assert_eq!(bits, value);
483 }
484
485 for raw in [2u64, 3, 0xa000, u64::MAX] {
486 assert_eq!(MailboxValue::try_from(raw), Err(raw));
487 assert!(try_pod_read_unaligned::<MailboxValue>(&raw.to_ne_bytes()).is_err());
488 }
489 }
490
491 #[test]
492 fn header_contains_framing_fields() {
493 let header = MsgHeader::new(MsgKind::Response, 0x1234_5678, 4096);
494
495 assert_eq!(MsgHeader::SIZE, 12);
496 assert_eq!(header.kind, MsgKind::Response);
497 assert_eq!(header.cid, 0x1234_5678);
498 assert_eq!(header.payload_len, 4096);
499 assert_eq!(header.reserved, [0; 3]);
500 assert_eq!(
501 header.as_bytes(),
502 &[0x02, 0, 0, 0, 0x78, 0x56, 0x34, 0x12, 0, 0x10, 0, 0]
503 );
504 assert_eq!(MsgHeader::from_bytes(header.as_bytes()), Some(header));
505 }
506
507 #[test]
508 fn header_decodes_unaligned_bytes() {
509 let header = MsgHeader::new(MsgKind::Response, 7, 4096);
510 let mut storage = [0u32; 4];
511 let bytes = &mut bytemuck::cast_slice_mut(&mut storage)[1..1 + MsgHeader::SIZE];
512 bytes.copy_from_slice(header.as_bytes());
513
514 assert_eq!(MsgHeader::from_bytes(bytes), Some(header));
515 }
516
517 #[test]
518 fn rejects_invalid_wire_headers() {
519 let header = MsgHeader::new(MsgKind::Request, 1, 4);
520 let mut bytes = [0; MsgHeader::SIZE];
521 bytes.copy_from_slice(header.as_bytes());
522
523 for index in 1..4 {
524 bytes[index] = 1;
525 assert_eq!(MsgHeader::from_bytes(&bytes), None);
526 bytes[index] = 0;
527 }
528
529 bytes[0] = u8::MAX;
530 assert_eq!(MsgHeader::from_bytes(&bytes), None);
531
532 bytes[0] = MsgKind::Request as u8;
533 for len in 0..MsgHeader::SIZE {
534 assert_eq!(MsgHeader::from_bytes(&bytes[..len]), None);
535 }
536
537 let mut oversized = [0; MsgHeader::SIZE + 1];
538 oversized[..MsgHeader::SIZE].copy_from_slice(&bytes);
539 assert_eq!(MsgHeader::from_bytes(&oversized), None);
540 }
541
542 #[test]
543 fn encoded_message_yields_wire_chunks_in_order() {
544 let chunks = [
545 bytes::Bytes::from_static(b"ef"),
546 bytes::Bytes::from_static(b"gh"),
547 ];
548 let mut external_values = ExternalValues::new();
549 external_values.push_bytes(b"cd").unwrap();
550 external_values.push_chunks(&chunks).unwrap();
551
552 let message = EncodedMessage::new(MsgKind::Request, 7, b"ab", external_values).unwrap();
553 let visited: Vec<_> = message.chunks().map(<[u8]>::to_vec).collect();
554
555 assert_eq!(message.total_len(), MsgHeader::SIZE + 8);
556 assert_eq!(message.prefix_len(), MsgHeader::SIZE + 2);
557 assert_eq!(message.payload_len(), 8);
558 assert_eq!(visited[1..], [b"ab", b"cd", b"ef", b"gh"]);
559 }
560
561 #[test]
562 fn encoded_message_buf_skips_empty_chunks() {
563 let mut external_values = ExternalValues::new();
564 external_values.chunks.push(&[]);
565 external_values.push_bytes(b"ab").unwrap();
566
567 let message = EncodedMessage::new(MsgKind::Request, 7, &[], external_values).unwrap();
568 let expected = message.chunks().flatten().copied().collect::<Vec<_>>();
569 let mut cursor = message.as_buf();
570 let mut actual = vec![0; cursor.remaining()];
571
572 cursor.copy_to_slice(&mut actual);
573
574 assert_eq!(actual, expected);
575 assert!(!cursor.has_remaining());
576 }
577
578 #[test]
579 fn encoded_message_rejects_length_overflow() {
580 let external_values = ExternalValues {
581 chunks: Vec::new(),
582 total_len: usize::MAX,
583 };
584
585 assert!(EncodedMessage::new(MsgKind::Request, 7, b"x", external_values).is_none());
586
587 let mut external_values = ExternalValues {
588 chunks: Vec::new(),
589 total_len: usize::MAX,
590 };
591 assert!(external_values.push_bytes(b"x").is_err());
592 assert!(external_values.chunks.is_empty());
593
594 let chunks = [Bytes::from_static(b"x")];
595 assert!(external_values.push_chunks(&chunks).is_err());
596 assert!(external_values.chunks.is_empty());
597 }
598
599 #[test]
600 fn segments_source_flattens_only_contiguous_values() {
601 let first = Bytes::from_static(b"ab");
602 let second = Bytes::from_static(b"cd");
603 let second_ptr = second.as_ptr();
604 let mut source = Segments::new([first, second]);
605
606 assert!(source.take_bytes(usize::MAX).is_err());
607 assert!(source.take_chunks(usize::MAX).is_err());
608 assert!(source.take_bytes(0).unwrap().is_empty());
609 assert!(source.take_chunks(0).unwrap().is_empty());
610 assert_eq!(source.len(), 4);
611 assert!(source.finish().is_err());
612
613 let contiguous = source.take_bytes(3).unwrap();
614 let chunks = source.take_chunks(1).unwrap();
615 source.finish().unwrap();
616
617 assert_eq!(contiguous, b"abc");
618 assert_eq!(chunks.len(), 1);
619 assert_eq!(chunks[0].as_ref(), b"d");
620 assert_eq!(chunks[0].as_ptr(), second_ptr.wrapping_add(1));
621 }
622
623 #[test]
624 fn segments_source_reads_fragmented_values_in_order() {
625 #[cfg(miri)]
626 let count = 64;
627 #[cfg(not(miri))]
628 let count = 1024;
629
630 let mut source = Segments::new(
631 (0..count).flat_map(|_| [Bytes::from_static(b"ab"), Bytes::from_static(b"cd")]),
632 );
633
634 for _ in 0..count {
635 assert_eq!(source.take_bytes(1).unwrap(), b"a");
636 assert_eq!(
637 source.take_chunks(2).unwrap(),
638 [Bytes::from_static(b"b"), Bytes::from_static(b"c")]
639 );
640 assert_eq!(source.take_bytes(1).unwrap(), b"d");
641 }
642
643 source.finish().unwrap();
644 }
645
646 #[test]
647 fn size_prefix_helpers_validate_length() {
648 assert_eq!(size_prefix_payload_len(&4u32.to_le_bytes()), Some(4));
649 assert_eq!(size_prefix_payload_len(&[0; 3]), None);
650 assert_eq!(size_prefixed_len(4), Some(SIZE_PREFIX_LEN + 4));
651 assert_eq!(size_prefixed_len(usize::MAX), None);
652 }
653}