fracpack/
fracpack.rs

1// TODO: fix reading structs and tuples which have unknown fields
2// TODO: option to allow/disallow unknown fields during verify and unpack
3// TODO: replace 'a with 'de; change macro to look for 'de specifically instead of assuming
4
5//! Rust support for the fracpack format.
6//!
7//! [Psibase](https://psibase.io) uses a new binary format, `fracpack`, which has the following goals:
8//!
9//! - Quickly pack and unpack data, making it suitable for service-to-service communication, node-to-node communication, blockchain-to-outside communication, and database storage.
10//! - Forwards and backwards compatibility; it supports adding new optional fields to the end of structs and tuples, even when they are embedded in variable-length vectors, fixed-length arrays, optional, and other structs and tuples.
11//! - Option to read without unpacking (almost zero-copy); helps to efficiently handle large data. TODO: this library doesn't implement this feature yet.
12//! - Doesn't require a code generator to support either C++ or Rust; macros and metaprogramming handle it.
13//! - Efficient compression when combined with the compression algorithm from Cap 'n' Proto.
14//!
15//! # Example use
16//!
17//! ```
18//! use fracpack::{Pack, Unpack, Result};
19//!
20//! #[derive(Pack, Unpack, PartialEq, Debug)]
21//! #[fracpack(fracpack_mod = "fracpack")]
22//! struct Example {
23//!     a_string: String,
24//!     a_tuple: (u32, String),
25//! }
26//!
27//! let orig = Example {
28//!     a_string: "content".into(),
29//!     a_tuple: (1234, "5678".into()),
30//! };
31//!
32//! // Convert to fracpack format
33//! let packed: Vec<u8> = orig.packed();
34//!
35//! // Convert from fracpack format
36//! let unpacked = Example::unpacked(&packed)?;
37//!
38//! assert_eq!(orig, unpacked);
39//! # Ok::<(), fracpack::Error>(())
40//! ```
41//!
42//! Note: `#[fracpack(fracpack_mod = "fracpack")]` is only needed when using the `fracpack`
43//! library directly instead of through the [psibase crate](https://docs.rs/psibase).
44//!
45//! # Caution
46//!
47//! It's easy to accidentally convert from a fixed-size
48//! array reference (`&[T;7]`) to a slice (`&[T]`). This matters
49//! to fracpack, which has different, and incompatible, encodings
50//! for the two types.
51
52use custom_error::custom_error;
53pub use psibase_macros::{Pack, ToSchema, Unpack};
54use std::{cell::RefCell, hash::Hash, mem, rc::Rc, sync::Arc};
55
56mod schema;
57pub use schema::*;
58
59mod nested;
60pub use nested::*;
61
62custom_error! {pub Error
63    ReadPastEnd         = "Read past end",
64    BadOffset           = "Bad offset",
65    BadSize             = "Bad size",
66    BadUTF8             = "Bad UTF-8 encoding",
67    BadEnumIndex        = "Bad enum index",
68    ExtraData           = "Extra data in buffer",
69    BadScalar           = "Bad scalar value",
70    ExtraEmptyOptional  = "Trailing empty optionals must be omitted",
71    PtrEmptyList        = "A pointer to an empty list must use zero offset",
72    HasUnknown          = "Unknown fields not allowed",
73    UnknownType         = "Unknown type",
74    ExpectedStringKey   = "Map keys must be strings",
75}
76pub type Result<T> = std::result::Result<T, Error>;
77
78#[derive(Debug)]
79pub struct FracInputStream<'a> {
80    pub data: &'a [u8],
81    pub pos: u32,
82    pub has_unknown: bool,
83    pub known_end: bool,
84}
85
86impl<'a> FracInputStream<'a> {
87    fn new(data: &'a [u8]) -> Self {
88        FracInputStream {
89            data,
90            pos: 0,
91            has_unknown: false,
92            known_end: true,
93        }
94    }
95
96    pub fn consume_trailing_optional(
97        &mut self,
98        fixed_pos: u32,
99        heap_pos: u32,
100        last_empty: bool,
101    ) -> Result<()> {
102        let mut fixed_stream = FracInputStream {
103            data: &self.data[0..heap_pos as usize],
104            pos: fixed_pos,
105            has_unknown: false,
106            known_end: true,
107        };
108        consume_trailing_optional(&mut fixed_stream, self, last_empty)
109    }
110
111    fn unpack_at<T: Unpack<'a>>(&self, pos: &mut u32) -> Result<T> {
112        assert!(!T::VARIABLE_SIZE);
113        let mut tmp = FracInputStream {
114            data: self.data,
115            pos: *pos,
116            has_unknown: false,
117            known_end: true,
118        };
119        let result = T::unpack(&mut tmp)?;
120        *pos = tmp.pos;
121        Ok(result)
122    }
123    fn verify_at<'b, T: Unpack<'b>>(&self, pos: &mut u32) -> Result<()> {
124        assert!(!T::VARIABLE_SIZE);
125        let mut tmp = FracInputStream {
126            data: self.data,
127            pos: *pos,
128            has_unknown: false,
129            known_end: true,
130        };
131        T::verify(&mut tmp)?;
132        *pos = tmp.pos;
133        Ok(())
134    }
135    pub fn advance(&mut self, len: u32) -> Result<u32> {
136        assert!(self.known_end);
137        let old_pos = self.pos;
138        let Some(pos) = self.pos.checked_add(len) else {
139            return Err(Error::ReadPastEnd);
140        };
141        if pos as usize > self.data.len() {
142            return Err(Error::ReadPastEnd);
143        }
144        self.pos = pos;
145        Ok(old_pos)
146    }
147    pub fn read_fixed(&mut self, len: u32) -> Result<Self> {
148        let old_pos = self.advance(len)?;
149        let result = FracInputStream {
150            data: &self.data[0..self.pos as usize],
151            pos: old_pos,
152            has_unknown: false,
153            known_end: true,
154        };
155        Ok(result)
156    }
157    pub(crate) fn set_pos(&mut self, pos: u32) -> Result<()> {
158        if self.known_end {
159            if self.pos != pos {
160                return Err(Error::BadOffset);
161            }
162        } else {
163            if self.pos > pos {
164                return Err(Error::BadOffset);
165            }
166            if pos > self.data.len() as u32 {
167                return Err(Error::BadOffset);
168            }
169            self.pos = pos;
170            self.known_end = true;
171        }
172        Ok(())
173    }
174    pub(crate) fn remaining(&self) -> u32 {
175        self.data.len() as u32 - self.pos
176    }
177    pub fn finish(&self) -> Result<()> {
178        if self.known_end {
179            if self.pos != self.data.len() as u32 {
180                return Err(Error::ExtraData);
181            }
182        }
183        Ok(())
184    }
185}
186
187pub(crate) fn consume_trailing_optional(
188    fixed_stream: &mut FracInputStream,
189    stream: &mut FracInputStream,
190    mut last_empty: bool,
191) -> Result<()> {
192    while fixed_stream.remaining() > 0 {
193        let orig_pos = fixed_stream.pos;
194        let offset = u32::unpack(fixed_stream)?;
195        last_empty = offset == 1;
196        if offset > 1 {
197            let Some(new_pos) = orig_pos.checked_add(offset) else {
198                return Err(Error::ReadPastEnd);
199            };
200            stream.set_pos(new_pos)?;
201            stream.known_end = false;
202        }
203        stream.has_unknown = true;
204    }
205    if last_empty {
206        return Err(Error::ExtraEmptyOptional);
207    }
208    Ok(())
209}
210
211/// Use this trait on generic functions instead of [Unpack] when
212/// the deserialized data may only be owned instead of borrowed from
213/// the source.
214///
215/// ```
216/// use fracpack::{UnpackOwned, Error};
217///
218/// pub fn get_unpacked<T: UnpackOwned>(packed: &[u8]) -> Result<T, Error> {
219///     T::unpacked(packed)
220/// }
221/// ```
222pub trait UnpackOwned: for<'a> Unpack<'a> {}
223impl<T> UnpackOwned for T where T: for<'a> Unpack<'a> {}
224
225/// Convert to fracpack format
226///
227/// Use [`#[derive(Pack)]`](psibase_macros::Pack) to implement
228/// this trait; manually implementing it is unsupported.
229pub trait Pack {
230    #[doc(hidden)]
231    const FIXED_SIZE: u32;
232
233    #[doc(hidden)]
234    const VARIABLE_SIZE: bool;
235
236    #[doc(hidden)]
237    const IS_OPTIONAL: bool = false;
238
239    /// Convert to fracpack format
240    ///
241    /// This packs `self` into the end of `dest`.
242    ///
243    /// Example:
244    ///
245    /// ```rust
246    /// use fracpack::Pack;
247    ///
248    /// fn convert(src: &(u32, String)) -> Vec<u8> {
249    ///     let mut bytes = Vec::new();
250    ///     src.pack(&mut bytes);
251    ///     bytes
252    /// }
253    /// ```
254    ///
255    /// See [Pack::packed], which is often more convenient.
256    fn pack(&self, dest: &mut Vec<u8>);
257
258    /// Convert to fracpack format
259    ///
260    /// This packs `self` and returns the result.
261    ///
262    /// Example:
263    ///
264    /// ```rust
265    /// use fracpack::Pack;
266    ///
267    /// fn packSomething(a: u32, b: &str) -> Vec<u8> {
268    ///     (a, b).packed()
269    /// }
270    /// ```
271    fn packed(&self) -> Vec<u8> {
272        let mut bytes = Vec::new();
273        self.pack(&mut bytes);
274        bytes
275    }
276
277    #[doc(hidden)]
278    fn is_empty_container(&self) -> bool {
279        false
280    }
281
282    #[doc(hidden)]
283    fn is_empty_optional(&self) -> bool {
284        false
285    }
286
287    #[doc(hidden)]
288    fn embedded_fixed_pack(&self, dest: &mut Vec<u8>) {
289        if Self::VARIABLE_SIZE {
290            dest.extend_from_slice(&0_u32.to_le_bytes());
291        } else {
292            Self::pack(self, dest);
293        }
294    }
295
296    #[doc(hidden)]
297    fn embedded_fixed_repack(&self, fixed_pos: u32, heap_pos: u32, dest: &mut Vec<u8>) {
298        if Self::VARIABLE_SIZE && !self.is_empty_container() {
299            dest[fixed_pos as usize..fixed_pos as usize + 4]
300                .copy_from_slice(&(heap_pos - fixed_pos).to_le_bytes());
301        }
302    }
303
304    #[doc(hidden)]
305    fn embedded_variable_pack(&self, dest: &mut Vec<u8>) {
306        if Self::VARIABLE_SIZE && !self.is_empty_container() {
307            self.pack(dest);
308        }
309    }
310}
311
312/// Unpack fracpack data
313///
314/// Use [`#[derive(Unpack)]`](psibase_macros::Unpack) to implement
315/// this trait; manually implementing it is unsupported.
316pub trait Unpack<'a>: Sized {
317    #[doc(hidden)]
318    const FIXED_SIZE: u32;
319
320    #[doc(hidden)]
321    const VARIABLE_SIZE: bool;
322
323    #[doc(hidden)]
324    const IS_OPTIONAL: bool = false;
325
326    /// Convert from fracpack format. Also verifies the integrity of the data.
327    ///
328    /// This unpacks `Self` from `src` starting at position `pos`.
329    ///
330    /// Example:
331    ///
332    /// ```rust
333    /// use fracpack::Unpack;
334    ///
335    /// fn unpackSomething(src: &[u8]) -> String {
336    ///     let mut pos = 0;
337    ///     String::unpack(src, &mut pos).unwrap()
338    /// }
339    /// ```
340    ///
341    /// See [Pack::unpacked], which is often more convenient.
342    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self>;
343
344    /// Convert from fracpack format. Also verifies the integrity of the data.
345    ///
346    /// This unpacks `Self` from `src`.
347    ///
348    /// Example:
349    ///
350    /// ```rust
351    /// use fracpack::Unpack;
352    ///
353    /// fn unpackSomething(src: &[u8]) -> (String, String) {
354    ///     <(String, String)>::unpacked(src).unwrap()
355    /// }
356    /// ```
357    fn unpacked(src: &'a [u8]) -> Result<Self> {
358        let mut stream = FracInputStream::new(src);
359        let result = Self::unpack(&mut stream)?;
360        stream.finish()?;
361        Ok(result)
362    }
363
364    /// Verify the integrity of fracpack data. You don't need to call this if
365    /// using [Pack::unpack] since it verifies integrity during unpack.
366    fn verify(src: &mut FracInputStream) -> Result<()>;
367
368    /// Verify the integrity of fracpack data, plus make sure there is no
369    /// leftover data after it.
370    fn verify_no_extra(src: &[u8]) -> Result<()> {
371        let mut stream = FracInputStream::new(src);
372        Self::verify(&mut stream)?;
373        stream.finish()?;
374        Ok(())
375    }
376
377    #[doc(hidden)]
378    fn new_empty_container() -> Result<Self> {
379        Err(Error::BadOffset)
380    }
381
382    #[doc(hidden)]
383    fn new_empty_optional() -> Result<Self> {
384        panic!("new_empty_optional must be implemented when IS_OPTIONAL == true");
385    }
386
387    #[doc(hidden)]
388    fn embedded_variable_unpack(
389        src: &mut FracInputStream<'a>,
390        fixed_pos: &mut u32,
391    ) -> Result<Self> {
392        let orig_pos = *fixed_pos;
393        let offset = src.unpack_at::<u32>(fixed_pos)?;
394        if offset == 0 {
395            return Self::new_empty_container();
396        }
397        let Some(new_pos) = orig_pos.checked_add(offset) else {
398            return Err(Error::ReadPastEnd);
399        };
400        src.set_pos(new_pos)?;
401        if Self::new_empty_container().is_ok() {
402            if src.unpack_at::<u32>(&mut src.pos.clone())? == 0 {
403                return Err(Error::PtrEmptyList);
404            }
405        }
406        Self::unpack(src)
407    }
408
409    #[doc(hidden)]
410    fn is_empty_optional(src: &FracInputStream, fixed_pos: &mut u32) -> Result<bool> {
411        Ok(Self::IS_OPTIONAL && src.unpack_at::<u32>(fixed_pos)? == 1)
412    }
413
414    #[doc(hidden)]
415    fn embedded_unpack(src: &mut FracInputStream<'a>, fixed_pos: &mut u32) -> Result<Self> {
416        if Self::VARIABLE_SIZE {
417            Self::embedded_variable_unpack(src, fixed_pos)
418        } else {
419            src.unpack_at::<Self>(fixed_pos)
420        }
421    }
422
423    #[doc(hidden)]
424    fn embedded_variable_verify(src: &mut FracInputStream, fixed_pos: &mut u32) -> Result<()> {
425        let orig_pos = *fixed_pos;
426        let offset = src.unpack_at::<u32>(fixed_pos)?;
427        if offset == 0 {
428            Self::new_empty_container()?;
429            return Ok(());
430        }
431        let Some(new_pos) = orig_pos.checked_add(offset) else {
432            return Err(Error::ReadPastEnd);
433        };
434        src.set_pos(new_pos)?;
435        if Self::new_empty_container().is_ok() {
436            if src.unpack_at::<u32>(&mut src.pos.clone())? == 0 {
437                return Err(Error::PtrEmptyList);
438            }
439        }
440        Self::verify(src)
441    }
442
443    #[doc(hidden)]
444    fn embedded_verify(src: &mut FracInputStream, fixed_pos: &mut u32) -> Result<()> {
445        if Self::VARIABLE_SIZE {
446            Self::embedded_variable_verify(src, fixed_pos)
447        } else {
448            src.verify_at::<Self>(fixed_pos)
449        }
450    }
451}
452
453fn read_u8_arr<const SIZE: usize>(src: &[u8], pos: &mut u32) -> Result<[u8; SIZE]> {
454    let mut bytes: [u8; SIZE] = [0; SIZE];
455    bytes.copy_from_slice(
456        src.get(*pos as usize..*pos as usize + SIZE)
457            .ok_or(Error::ReadPastEnd)?,
458    );
459    *pos += SIZE as u32;
460    Ok(bytes)
461}
462
463impl<'a, T: Pack> Pack for &'a T {
464    const FIXED_SIZE: u32 = T::FIXED_SIZE;
465    const VARIABLE_SIZE: bool = T::VARIABLE_SIZE;
466    const IS_OPTIONAL: bool = T::IS_OPTIONAL;
467
468    fn pack(&self, dest: &mut Vec<u8>) {
469        (*self).pack(dest)
470    }
471
472    fn is_empty_container(&self) -> bool {
473        (*self).is_empty_container()
474    }
475
476    fn embedded_fixed_pack(&self, dest: &mut Vec<u8>) {
477        (*self).embedded_fixed_pack(dest)
478    }
479
480    fn embedded_fixed_repack(&self, fixed_pos: u32, heap_pos: u32, dest: &mut Vec<u8>) {
481        (*self).embedded_fixed_repack(fixed_pos, heap_pos, dest)
482    }
483
484    fn embedded_variable_pack(&self, dest: &mut Vec<u8>) {
485        (*self).embedded_variable_pack(dest)
486    }
487}
488
489impl Pack for bool {
490    const FIXED_SIZE: u32 = mem::size_of::<Self>() as u32;
491    const VARIABLE_SIZE: bool = false;
492    fn pack(&self, dest: &mut Vec<u8>) {
493        dest.push(if *self { 1 } else { 0 });
494    }
495}
496
497impl<'a> Unpack<'a> for bool {
498    const FIXED_SIZE: u32 = mem::size_of::<Self>() as u32;
499    const VARIABLE_SIZE: bool = false;
500    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
501        match u8::unpack(src)? {
502            0 => Ok(false),
503            1 => Ok(true),
504            _ => Err(Error::BadScalar),
505        }
506    }
507    fn verify(src: &mut FracInputStream) -> Result<()> {
508        Self::unpack(src)?;
509        Ok(())
510    }
511}
512
513macro_rules! scalar_impl {
514    ($t:ty) => {
515        impl Pack for $t {
516            const FIXED_SIZE: u32 = mem::size_of::<Self>() as u32;
517            const VARIABLE_SIZE: bool = false;
518            fn pack(&self, dest: &mut Vec<u8>) {
519                dest.extend_from_slice(&self.to_le_bytes());
520            }
521        }
522        impl<'a> Unpack<'a> for $t {
523            const FIXED_SIZE: u32 = mem::size_of::<Self>() as u32;
524            const VARIABLE_SIZE: bool = false;
525            fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
526                Ok(Self::from_le_bytes(
527                    read_u8_arr(src.data, &mut src.pos)?.into(),
528                ))
529            }
530            fn verify(src: &mut FracInputStream) -> Result<()> {
531                if (src.pos as u64 + <Self as Unpack>::FIXED_SIZE as u64 > src.data.len() as u64) {
532                    Err(Error::ReadPastEnd)
533                } else {
534                    src.pos += <Self as Unpack>::FIXED_SIZE;
535                    Ok(())
536                }
537            }
538        }
539    };
540} // scalar_impl
541
542scalar_impl! {i8}
543scalar_impl! {i16}
544scalar_impl! {i32}
545scalar_impl! {i64}
546scalar_impl! {u8}
547scalar_impl! {u16}
548scalar_impl! {u32}
549scalar_impl! {u64}
550scalar_impl! {f32}
551scalar_impl! {f64}
552
553macro_rules! pack_ptr {
554    ($ptr:ident, $to_ref:ident) => {
555        impl<T: Pack> Pack for $ptr<T> {
556            const FIXED_SIZE: u32 = T::FIXED_SIZE;
557            const VARIABLE_SIZE: bool = T::VARIABLE_SIZE;
558            const IS_OPTIONAL: bool = T::IS_OPTIONAL;
559
560            fn pack(&self, dest: &mut Vec<u8>) {
561                self.$to_ref().pack(dest)
562            }
563
564            fn is_empty_container(&self) -> bool {
565                self.$to_ref().is_empty_container()
566            }
567
568            fn embedded_fixed_pack(&self, dest: &mut Vec<u8>) {
569                self.$to_ref().embedded_fixed_pack(dest)
570            }
571
572            fn embedded_fixed_repack(&self, fixed_pos: u32, heap_pos: u32, dest: &mut Vec<u8>) {
573                self.$to_ref()
574                    .embedded_fixed_repack(fixed_pos, heap_pos, dest)
575            }
576
577            fn embedded_variable_pack(&self, dest: &mut Vec<u8>) {
578                self.$to_ref().embedded_variable_pack(dest)
579            }
580        }
581    };
582}
583
584macro_rules! unpack_ptr {
585    ($ptr:ident) => {
586        impl<'a, T: Unpack<'a>> Unpack<'a> for $ptr<T> {
587            const FIXED_SIZE: u32 = T::FIXED_SIZE;
588            const VARIABLE_SIZE: bool = T::VARIABLE_SIZE;
589            const IS_OPTIONAL: bool = T::IS_OPTIONAL;
590
591            fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
592                Ok(Self::new(<T>::unpack(src)?))
593            }
594
595            fn verify(src: &mut FracInputStream) -> Result<()> {
596                <T>::verify(src)
597            }
598
599            fn new_empty_container() -> Result<Self> {
600                Ok(Self::new(<T>::new_empty_container()?))
601            }
602
603            fn new_empty_optional() -> Result<Self> {
604                Ok(Self::new(<T>::new_empty_optional()?))
605            }
606
607            fn embedded_variable_unpack(
608                src: &mut FracInputStream<'a>,
609                fixed_pos: &mut u32,
610            ) -> Result<Self> {
611                Ok(Self::new(<T>::embedded_variable_unpack(src, fixed_pos)?))
612            }
613
614            fn embedded_unpack(src: &mut FracInputStream<'a>, fixed_pos: &mut u32) -> Result<Self> {
615                Ok(Self::new(<T>::embedded_unpack(src, fixed_pos)?))
616            }
617
618            fn embedded_variable_verify(
619                src: &mut FracInputStream,
620                fixed_pos: &mut u32,
621            ) -> Result<()> {
622                <T>::embedded_variable_verify(src, fixed_pos)
623            }
624
625            fn embedded_verify(src: &mut FracInputStream, fixed_pos: &mut u32) -> Result<()> {
626                <T>::embedded_verify(src, fixed_pos)
627            }
628        }
629    };
630}
631
632pack_ptr!(Box, as_ref);
633pack_ptr!(Rc, as_ref);
634pack_ptr!(Arc, as_ref);
635pack_ptr!(RefCell, borrow);
636
637unpack_ptr!(Box);
638unpack_ptr!(Rc);
639unpack_ptr!(Arc);
640unpack_ptr!(RefCell);
641
642impl<T: Pack> Pack for Option<T> {
643    const FIXED_SIZE: u32 = 4;
644    const VARIABLE_SIZE: bool = true;
645    const IS_OPTIONAL: bool = true;
646
647    fn pack(&self, dest: &mut Vec<u8>) {
648        let fixed_pos = dest.len() as u32;
649        Self::embedded_fixed_pack(self, dest);
650        let heap_pos = dest.len() as u32;
651        Self::embedded_fixed_repack(self, fixed_pos, heap_pos, dest);
652        Self::embedded_variable_pack(self, dest);
653    }
654
655    fn is_empty_optional(&self) -> bool {
656        self.is_none()
657    }
658
659    fn embedded_fixed_pack(&self, dest: &mut Vec<u8>) {
660        if T::IS_OPTIONAL || !T::VARIABLE_SIZE {
661            dest.extend_from_slice(&1u32.to_le_bytes())
662        } else {
663            match self {
664                Some(x) => x.embedded_fixed_pack(dest),
665                None => dest.extend_from_slice(&1u32.to_le_bytes()),
666            }
667        }
668    }
669
670    fn embedded_fixed_repack(&self, fixed_pos: u32, heap_pos: u32, dest: &mut Vec<u8>) {
671        if let Some(x) = self {
672            if T::IS_OPTIONAL || !T::VARIABLE_SIZE {
673                dest[fixed_pos as usize..fixed_pos as usize + 4]
674                    .copy_from_slice(&(heap_pos - fixed_pos).to_le_bytes())
675            } else {
676                x.embedded_fixed_repack(fixed_pos, heap_pos, dest)
677            }
678        }
679    }
680
681    fn embedded_variable_pack(&self, dest: &mut Vec<u8>) {
682        if let Some(x) = self {
683            if !x.is_empty_container() {
684                x.pack(dest)
685            }
686        }
687    }
688}
689
690impl<'a, T: Unpack<'a>> Unpack<'a> for Option<T> {
691    const FIXED_SIZE: u32 = 4;
692    const VARIABLE_SIZE: bool = true;
693    const IS_OPTIONAL: bool = true;
694
695    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
696        let mut fixed_pos = src.advance(Self::FIXED_SIZE)?;
697        Self::embedded_unpack(src, &mut fixed_pos)
698    }
699
700    fn verify(src: &mut FracInputStream) -> Result<()> {
701        let mut fixed_pos = src.advance(Self::FIXED_SIZE)?;
702        Self::embedded_verify(src, &mut fixed_pos)
703    }
704
705    fn new_empty_optional() -> Result<Self> {
706        Ok(None)
707    }
708
709    fn embedded_unpack(src: &mut FracInputStream<'a>, fixed_pos: &mut u32) -> Result<Self> {
710        let mut tmp_pos = *fixed_pos;
711        if Self::is_empty_optional(src, &mut tmp_pos)? {
712            *fixed_pos = tmp_pos;
713            return Ok(None);
714        }
715        Ok(Some(<T>::embedded_variable_unpack(src, fixed_pos)?))
716    }
717
718    fn embedded_verify(src: &mut FracInputStream, fixed_pos: &mut u32) -> Result<()> {
719        let mut tmp_pos = *fixed_pos;
720        if Self::is_empty_optional(src, &mut tmp_pos)? {
721            *fixed_pos = tmp_pos;
722            return Ok(());
723        }
724        T::embedded_variable_verify(src, fixed_pos)
725    }
726}
727
728trait BytesConversion<'a>: Sized {
729    fn fracpack_verify_if_str(bytes: &[u8]) -> Result<()>;
730    fn fracpack_from_bytes(bytes: &'a [u8]) -> Result<Self>;
731    fn fracpack_as_bytes(&'a self) -> &'a [u8];
732}
733
734impl<'a> BytesConversion<'a> for String {
735    fn fracpack_verify_if_str(bytes: &[u8]) -> Result<()> {
736        std::str::from_utf8(bytes).or(Err(Error::BadUTF8))?;
737        Ok(())
738    }
739    fn fracpack_from_bytes(bytes: &'a [u8]) -> Result<Self> {
740        Self::from_utf8(bytes.to_vec()).or(Err(Error::BadUTF8))
741    }
742    fn fracpack_as_bytes(&'a self) -> &'a [u8] {
743        self.as_bytes()
744    }
745}
746
747impl<'a> BytesConversion<'a> for &'a str {
748    fn fracpack_verify_if_str(bytes: &[u8]) -> Result<()> {
749        std::str::from_utf8(bytes).or(Err(Error::BadUTF8))?;
750        Ok(())
751    }
752    fn fracpack_from_bytes(bytes: &'a [u8]) -> Result<Self> {
753        std::str::from_utf8(bytes).or(Err(Error::BadUTF8))
754    }
755    fn fracpack_as_bytes(&self) -> &'a [u8] {
756        self.as_bytes()
757    }
758}
759
760impl<'a> BytesConversion<'a> for &'a [u8] {
761    fn fracpack_verify_if_str(_bytes: &[u8]) -> Result<()> {
762        Ok(())
763    }
764    fn fracpack_from_bytes(bytes: &'a [u8]) -> Result<Self> {
765        Ok(bytes)
766    }
767    fn fracpack_as_bytes(&self) -> &'a [u8] {
768        self
769    }
770}
771
772macro_rules! bytes_impl {
773    ($t:ty) => {
774        impl<'a> Pack for $t {
775            const FIXED_SIZE: u32 = 4;
776            const VARIABLE_SIZE: bool = true;
777
778            fn pack(&self, dest: &mut Vec<u8>) {
779                dest.extend_from_slice(&(self.len() as u32).to_le_bytes());
780                dest.extend_from_slice(self.fracpack_as_bytes());
781            }
782
783            fn is_empty_container(&self) -> bool {
784                self.is_empty()
785            }
786        }
787
788        impl<'a> Unpack<'a> for $t {
789            const FIXED_SIZE: u32 = 4;
790            const VARIABLE_SIZE: bool = true;
791
792            fn unpack(src: &mut FracInputStream<'a>) -> Result<$t> {
793                let len = u32::unpack(src)?;
794                let bytes = src
795                    .data
796                    .get(src.pos as usize..(src.pos + len) as usize)
797                    .ok_or(Error::ReadPastEnd)?;
798                src.pos += len;
799                <$t>::fracpack_from_bytes(bytes)
800            }
801
802            fn verify(src: &mut FracInputStream) -> Result<()> {
803                let len = u32::unpack(src)?;
804                let bytes = src
805                    .data
806                    .get(src.pos as usize..(src.pos + len) as usize)
807                    .ok_or(Error::ReadPastEnd)?;
808                src.pos += len;
809                <$t>::fracpack_verify_if_str(bytes)?;
810                Ok(())
811            }
812
813            fn new_empty_container() -> Result<Self> {
814                Ok(Default::default())
815            }
816        }
817    };
818} // bytes_impl
819
820bytes_impl! {String}
821bytes_impl! {&'a str}
822bytes_impl! {&'a [u8]}
823
824impl<T: Pack> Pack for Vec<T> {
825    const FIXED_SIZE: u32 = 4;
826    const VARIABLE_SIZE: bool = true;
827
828    // TODO: optimize scalar
829    fn pack(&self, dest: &mut Vec<u8>) {
830        let num_bytes = self.len() as u32 * T::FIXED_SIZE;
831        dest.extend_from_slice(&num_bytes.to_le_bytes());
832        dest.reserve(num_bytes as usize);
833        let start = dest.len();
834        for x in self {
835            T::embedded_fixed_pack(x, dest);
836        }
837        for (i, x) in self.iter().enumerate() {
838            let heap_pos = dest.len() as u32;
839            T::embedded_fixed_repack(x, start as u32 + (i as u32) * T::FIXED_SIZE, heap_pos, dest);
840            T::embedded_variable_pack(x, dest);
841        }
842    }
843
844    fn is_empty_container(&self) -> bool {
845        self.is_empty()
846    }
847}
848
849impl<'a, T: Unpack<'a>> Unpack<'a> for Vec<T> {
850    const FIXED_SIZE: u32 = 4;
851    const VARIABLE_SIZE: bool = true;
852
853    // TODO: optimize scalar
854    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
855        let num_bytes = u32::unpack(src)?;
856        if num_bytes % T::FIXED_SIZE != 0 {
857            return Err(Error::BadSize);
858        }
859        let len = (num_bytes / T::FIXED_SIZE) as usize;
860        let mut fixed_pos = src.advance(num_bytes)?;
861        let mut result = Self::with_capacity(len);
862        for _ in 0..len {
863            result.push(T::embedded_unpack(src, &mut fixed_pos)?);
864        }
865        Ok(result)
866    }
867
868    // TODO: optimize scalar
869    fn verify(src: &mut FracInputStream) -> Result<()> {
870        let num_bytes = u32::unpack(src)?;
871        if num_bytes % T::FIXED_SIZE != 0 {
872            return Err(Error::BadSize);
873        }
874        let mut fixed_pos = src.advance(num_bytes)?;
875        for _ in 0..num_bytes / T::FIXED_SIZE {
876            T::embedded_verify(src, &mut fixed_pos)?;
877        }
878        Ok(())
879    }
880
881    fn new_empty_container() -> Result<Self> {
882        Ok(Default::default())
883    }
884}
885
886impl<T: Pack, const N: usize> Pack for [T; N] {
887    const VARIABLE_SIZE: bool = T::VARIABLE_SIZE;
888    const FIXED_SIZE: u32 = if T::VARIABLE_SIZE {
889        4
890    } else {
891        T::FIXED_SIZE * N as u32
892    };
893
894    fn pack(&self, dest: &mut Vec<u8>) {
895        let start = dest.len();
896        for item in self {
897            item.embedded_fixed_pack(dest);
898        }
899        for (i, item) in self.iter().enumerate() {
900            let heap_pos = dest.len() as u32;
901            item.embedded_fixed_repack(start as u32 + (i as u32) * T::FIXED_SIZE, heap_pos, dest);
902            item.embedded_variable_pack(dest);
903        }
904    }
905}
906
907impl<'a, T: Unpack<'a>, const N: usize> Unpack<'a> for [T; N] {
908    const VARIABLE_SIZE: bool = T::VARIABLE_SIZE;
909    const FIXED_SIZE: u32 = if T::VARIABLE_SIZE {
910        4
911    } else {
912        T::FIXED_SIZE * N as u32
913    };
914
915    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
916        let total_size: u32 = T::FIXED_SIZE * N as u32;
917        let mut fixed_pos = src.advance(total_size)?;
918
919        let mut items: Vec<T> = Vec::with_capacity(N);
920        for _ in 0..N {
921            items.push(T::embedded_unpack(src, &mut fixed_pos)?);
922        }
923
924        let result: [T; N] = items.try_into().unwrap_or_else(|v: Vec<T>| {
925            panic!(
926                "Expected a fixed array of length {} but it was {}",
927                N,
928                v.len()
929            )
930        });
931        Ok(result)
932    }
933
934    fn verify(src: &mut FracInputStream) -> Result<()> {
935        let total_size: u32 = T::FIXED_SIZE * N as u32;
936        let mut fixed_pos = src.advance(total_size)?;
937        for _ in 0..N {
938            T::embedded_verify(src, &mut fixed_pos)?;
939        }
940        Ok(())
941    }
942}
943
944macro_rules! container_impl {
945    (impl<$($n:ident),+> Pack+Unpack for $t:ty $(where $($w:tt)*)?) =>
946    {
947        impl <$($n:Pack),+> Pack for $t $(where $($w)*)?
948{
949    const FIXED_SIZE: u32 = 4;
950    const VARIABLE_SIZE: bool = true;
951
952    fn pack(&self, dest: &mut Vec<u8>) {
953        let num_bytes = self.len() as u32 * <&$t as IntoIterator>::Item::FIXED_SIZE;
954        dest.extend_from_slice(&num_bytes.to_le_bytes());
955        dest.reserve(num_bytes as usize);
956        let start = dest.len();
957        for x in self {
958            <&$t as IntoIterator>::Item::embedded_fixed_pack(&x, dest);
959        }
960        for (i, x) in self.into_iter().enumerate() {
961            let heap_pos = dest.len() as u32;
962            <&$t as IntoIterator>::Item::embedded_fixed_repack(&x, start as u32 + (i as u32) * <&$t as IntoIterator>::Item::FIXED_SIZE, heap_pos, dest);
963            <&$t as IntoIterator>::Item::embedded_variable_pack(&x, dest);
964        }
965    }
966
967    fn is_empty_container(&self) -> bool {
968        self.into_iter().next().is_none()
969    }
970}
971
972impl<'a, $($n: Unpack<'a>),+> Unpack<'a> for $t $(where $($w)*)? {
973    const FIXED_SIZE: u32 = 4;
974    const VARIABLE_SIZE: bool = true;
975
976    fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
977        let num_bytes = u32::unpack(src)?;
978        if num_bytes % <$t as IntoIterator>::Item::FIXED_SIZE != 0 {
979            return Err(Error::BadSize);
980        }
981        let len = (num_bytes / <$t as IntoIterator>::Item::FIXED_SIZE) as usize;
982        let mut fixed_pos = src.advance(num_bytes)?;
983        (0..len).map(|_| {
984            <$t as IntoIterator>::Item::embedded_unpack(src, &mut fixed_pos)
985        }).collect()
986    }
987
988    fn verify(src: &mut FracInputStream) -> Result<()> {
989        let num_bytes = u32::unpack(src)?;
990        if num_bytes % <$t as IntoIterator>::Item::FIXED_SIZE != 0 {
991            return Err(Error::BadSize);
992        }
993        let mut fixed_pos = src.advance(num_bytes)?;
994        for _ in 0..num_bytes / <$t as IntoIterator>::Item::FIXED_SIZE {
995            <$t as IntoIterator>::Item::embedded_verify(src, &mut fixed_pos)?;
996        }
997        Ok(())
998    }
999
1000    fn new_empty_container() -> Result<Self> {
1001        Ok(Default::default())
1002    }
1003}
1004    }
1005}
1006
1007container_impl!(impl<K, V> Pack+Unpack for indexmap::IndexMap<K, V> where K: Hash, K: Eq);
1008
1009macro_rules! tuple_impls {
1010    ($($len:expr => ($($n:tt $name:ident)*))+) => {
1011        $(
1012            impl<$($name: Pack),*> Pack for ($($name,)*)
1013            {
1014                const VARIABLE_SIZE: bool = true;
1015                const FIXED_SIZE: u32 = 4;
1016
1017                #[allow(non_snake_case)]
1018                fn pack(&self, dest: &mut Vec<u8>) {
1019                    let _trailing_empty_index = [
1020                        $({!<$name as Pack>::is_empty_optional(&self.$n)}),*
1021                    ].iter().rposition(|is_non_empty: &bool| *is_non_empty).map_or(0, |idx| idx + 1);
1022
1023                    let mut _heap: u32 = 0;
1024                    $({
1025                        if $n < _trailing_empty_index {
1026                            _heap += <$name as Pack>::FIXED_SIZE;
1027                        }
1028                    })*
1029                    assert!(_heap as u16 as u32 == _heap); // TODO: return error
1030                    (_heap as u16).pack(dest);
1031
1032
1033
1034                    $(
1035                        let $name = dest.len() as u32;
1036                        if $n < _trailing_empty_index {
1037                            self.$n.embedded_fixed_pack(dest);
1038                        }
1039                    )*
1040                    $(
1041                        if $n < _trailing_empty_index {
1042                            let heap_pos = dest.len() as u32;
1043                            self.$n.embedded_fixed_repack($name, heap_pos, dest);
1044                            self.$n.embedded_variable_pack(dest);
1045                        }
1046                    )*
1047                }
1048            }
1049
1050            impl<'a, $($name: Unpack<'a>),*> Unpack<'a> for ($($name,)*)
1051            {
1052                const VARIABLE_SIZE: bool = true;
1053                const FIXED_SIZE: u32 = 4;
1054
1055                #[allow(non_snake_case,unused_mut)]
1056                fn unpack(src: &mut FracInputStream<'a>) -> Result<Self> {
1057                    let fixed_size = u16::unpack(src)?;
1058                    let mut fixed_pos = src.advance(fixed_size as u32)?;
1059                    let heap_pos = src.pos;
1060                    let mut last_empty = false;
1061                    let _trailing_optional_index = [$(<$name as Unpack>::IS_OPTIONAL,)*].iter().rposition(|is_optional: &bool| !is_optional).map_or(0, |idx| idx + 1);
1062                    $(
1063                        let $name = if $n < _trailing_optional_index || fixed_pos < heap_pos {
1064                            last_empty = <$name as Unpack>::is_empty_optional(src, &mut fixed_pos.clone())?;
1065                            $name::embedded_unpack(src, &mut fixed_pos)?
1066                        } else {
1067                            $name::new_empty_optional()?
1068                        };
1069                    )*
1070                    src.consume_trailing_optional(fixed_pos, heap_pos, last_empty)?;
1071                    Ok(($($name,)*))
1072                }
1073
1074                #[allow(unused_mut)]
1075                fn verify(src: &mut FracInputStream) -> Result<()> {
1076                    let fixed_size = u16::unpack(src)?;
1077                    let mut fixed_pos = src.advance(fixed_size as u32)?;
1078                    let heap_pos = src.pos;
1079                    let mut last_empty = false;
1080                    let _trailing_optional_index = [$(<$name as Unpack>::IS_OPTIONAL,)*].iter().rposition(|is_optional: &bool| !is_optional).map_or(0, |idx| idx + 1);
1081                    $(
1082                        if $n < _trailing_optional_index || fixed_pos < heap_pos {
1083                            last_empty = <$name as Unpack>::is_empty_optional(src, &mut fixed_pos.clone())?;
1084                            $name::embedded_verify(src, &mut fixed_pos)?;
1085                        }
1086                    )*
1087                    src.consume_trailing_optional(fixed_pos, heap_pos, last_empty)?;
1088                    Ok(())
1089                }
1090            }
1091        )+
1092    }
1093}
1094
1095tuple_impls! {
1096    0 => ()
1097    1 => (0 T0)
1098    2 => (0 T0 1 T1)
1099    3 => (0 T0 1 T1 2 T2)
1100    4 => (0 T0 1 T1 2 T2 3 T3)
1101    5 => (0 T0 1 T1 2 T2 3 T3 4 T4)
1102    6 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5)
1103    7 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6)
1104    8 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7)
1105    9 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8)
1106    10 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9)
1107    11 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10)
1108    12 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11)
1109    13 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12)
1110    14 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13)
1111    15 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14)
1112    16 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14 15 T15)
1113}
fracpack/fracpack.rs

fracpack/
fracpack.rs
