Skip to main content

tycho_common/
hex_bytes.rs

1use std::{
2    borrow::Borrow,
3    clone::Clone,
4    fmt::{Debug, Display, Formatter, LowerHex, Result as FmtResult},
5    ops::Deref,
6    str::FromStr,
7};
8
9use deepsize::{Context, DeepSizeOf};
10#[cfg(feature = "diesel")]
11use diesel::{
12    deserialize::{self, FromSql, FromSqlRow},
13    expression::AsExpression,
14    pg::Pg,
15    serialize::{self, ToSql},
16    sql_types::Binary,
17};
18use rand::Rng;
19use serde::{Deserialize, Serialize};
20use thiserror::Error;
21
22use crate::serde_primitives::hex_bytes;
23
24/// Wrapper type around Bytes to deserialize/serialize from/to hex
25#[derive(Clone, Default, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
26#[cfg_attr(feature = "diesel", derive(AsExpression, FromSqlRow,))]
27#[cfg_attr(feature = "diesel", diesel(sql_type = Binary))]
28pub struct Bytes(#[serde(with = "hex_bytes")] pub bytes::Bytes);
29
30impl DeepSizeOf for Bytes {
31    fn deep_size_of_children(&self, _ctx: &mut Context) -> usize {
32        // Note: This may overcount memory if the underlying bytes are shared (e.g. via Arc).
33        // We cannot detect shared ownership here because Context’s internal tracking is private
34        // At the same time, this might also underreport memory if:
35        // - the bytes::Bytes are instantiated as Shared internally, which adds 24 bytes of overhead
36        // - the bytes::Bytes has capacity greater than its length, as we only count the length here
37        self.0.len()
38    }
39}
40
41fn bytes_to_hex(b: &Bytes) -> String {
42    hex::encode(b.0.as_ref())
43}
44
45impl Debug for Bytes {
46    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
47        write!(f, "Bytes(0x{})", bytes_to_hex(self))
48    }
49}
50
51impl Display for Bytes {
52    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
53        write!(f, "0x{}", bytes_to_hex(self))
54    }
55}
56
57impl LowerHex for Bytes {
58    fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
59        write!(f, "0x{}", bytes_to_hex(self))
60    }
61}
62
63impl Bytes {
64    pub fn new() -> Self {
65        Self(bytes::Bytes::new())
66    }
67    /// This function converts the internal byte array into a `Vec<u8>`
68    ///
69    /// # Returns
70    ///
71    /// A `Vec<u8>` containing the bytes from the `Bytes` struct.
72    ///
73    /// # Example
74    ///
75    /// ```
76    /// let bytes = Bytes::from(vec![0x01, 0x02, 0x03]);
77    /// let vec = bytes.to_vec();
78    /// assert_eq!(vec, vec![0x01, 0x02, 0x03]);
79    /// ```
80    pub fn to_vec(&self) -> Vec<u8> {
81        self.as_ref().to_vec()
82    }
83
84    /// Left-pads the byte array to the specified length with the given padding byte.
85    ///
86    /// This function creates a new `Bytes` instance by prepending the specified padding byte
87    /// to the current byte array until its total length matches the desired length.
88    ///
89    /// If the current length of the byte array is greater than or equal to the specified length,
90    /// the original byte array is returned unchanged.
91    ///
92    /// # Arguments
93    ///
94    /// * `length` - The desired total length of the resulting byte array.
95    /// * `pad_byte` - The byte value to use for padding. Commonly `0x00`.
96    ///
97    /// # Returns
98    ///
99    /// A new `Bytes` instance with the byte array left-padded to the desired length.
100    ///
101    /// # Example
102    ///
103    /// ```
104    /// let bytes = Bytes::from(vec![0x01, 0x02, 0x03]);
105    /// let padded = bytes.lpad(6, 0x00);
106    /// assert_eq!(padded.to_vec(), vec![0x00, 0x00, 0x00, 0x01, 0x02, 0x03]);
107    /// ```
108    pub fn lpad(&self, length: usize, pad_byte: u8) -> Bytes {
109        let mut padded_vec = vec![pad_byte; length.saturating_sub(self.len())];
110        padded_vec.extend_from_slice(self.as_ref());
111
112        Bytes(bytes::Bytes::from(padded_vec))
113    }
114
115    /// Right-pads the byte array to the specified length with the given padding byte.
116    ///
117    /// This function creates a new `Bytes` instance by appending the specified padding byte
118    /// to the current byte array until its total length matches the desired length.
119    ///
120    /// If the current length of the byte array is greater than or equal to the specified length,
121    /// the original byte array is returned unchanged.
122    ///
123    /// # Arguments
124    ///
125    /// * `length` - The desired total length of the resulting byte array.
126    /// * `pad_byte` - The byte value to use for padding. Commonly `0x00`.
127    ///
128    /// # Returns
129    ///
130    /// A new `Bytes` instance with the byte array right-padded to the desired length.
131    ///
132    /// # Example
133    ///
134    /// ```
135    /// let bytes = Bytes::from(vec![0x01, 0x02, 0x03]);
136    /// let padded = bytes.rpad(6, 0x00);
137    /// assert_eq!(padded.to_vec(), vec![0x01, 0x02, 0x03, 0x00, 0x00, 0x00]);
138    /// ```
139    pub fn rpad(&self, length: usize, pad_byte: u8) -> Bytes {
140        let mut padded_vec = self.to_vec();
141        padded_vec.resize(length, pad_byte);
142
143        Bytes(bytes::Bytes::from(padded_vec))
144    }
145
146    /// Creates a `Bytes` object of the specified length, filled with zeros.
147    ///
148    /// # Arguments
149    ///
150    /// * `length` - The length of the `Bytes` object to be created.
151    ///
152    /// # Returns
153    ///
154    /// A `Bytes` object of the specified length, where each byte is set to zero.
155    ///
156    /// # Example
157    ///
158    /// ```
159    /// let b = Bytes::zero(5);
160    /// assert_eq!(b, Bytes::from(vec![0, 0, 0, 0, 0]));
161    /// ```
162    pub fn zero(length: usize) -> Bytes {
163        Bytes::from(vec![0u8; length])
164    }
165
166    /// Creates a `Bytes` object of the specified length, filled with random bytes.
167    ///
168    /// # Arguments
169    ///
170    /// * `length` - The length of the `Bytes` object to be created.
171    ///
172    /// # Returns
173    ///
174    /// A `Bytes` object of the specified length, filled with random bytes.
175    ///
176    /// # Example
177    ///
178    /// ```
179    /// let random_bytes = Bytes::random(5);
180    /// assert_eq!(random_bytes.len(), 5);
181    /// ```
182    pub fn random(length: usize) -> Bytes {
183        let mut data = vec![0u8; length];
184        rand::thread_rng().fill(&mut data[..]);
185        Bytes::from(data)
186    }
187
188    /// Checks if the byte array is full of zeros.
189    ///
190    /// # Returns
191    ///
192    /// A boolean value indicating whether all bytes in the byte array are zero.
193    ///
194    /// # Example
195    ///
196    /// ```
197    /// let b = Bytes::zero(5);
198    /// assert!(b.is_zero());
199    /// ```
200    pub fn is_zero(&self) -> bool {
201        self.as_ref().iter().all(|b| *b == 0)
202    }
203}
204
205impl Deref for Bytes {
206    type Target = [u8];
207
208    #[inline]
209    fn deref(&self) -> &[u8] {
210        self.as_ref()
211    }
212}
213
214impl AsRef<[u8]> for Bytes {
215    fn as_ref(&self) -> &[u8] {
216        self.0.as_ref()
217    }
218}
219
220impl Borrow<[u8]> for Bytes {
221    fn borrow(&self) -> &[u8] {
222        self.as_ref()
223    }
224}
225
226impl IntoIterator for Bytes {
227    type Item = u8;
228    type IntoIter = bytes::buf::IntoIter<bytes::Bytes>;
229
230    fn into_iter(self) -> Self::IntoIter {
231        self.0.into_iter()
232    }
233}
234
235impl<'a> IntoIterator for &'a Bytes {
236    type Item = &'a u8;
237    type IntoIter = core::slice::Iter<'a, u8>;
238
239    fn into_iter(self) -> Self::IntoIter {
240        self.as_ref().iter()
241    }
242}
243
244impl From<&[u8]> for Bytes {
245    fn from(src: &[u8]) -> Self {
246        Self(bytes::Bytes::copy_from_slice(src))
247    }
248}
249
250impl From<bytes::Bytes> for Bytes {
251    fn from(src: bytes::Bytes) -> Self {
252        Self(src)
253    }
254}
255
256impl From<Bytes> for bytes::Bytes {
257    fn from(src: Bytes) -> Self {
258        src.0
259    }
260}
261
262impl From<Vec<u8>> for Bytes {
263    fn from(src: Vec<u8>) -> Self {
264        Self(src.into())
265    }
266}
267
268impl From<Bytes> for Vec<u8> {
269    fn from(value: Bytes) -> Self {
270        value.to_vec()
271    }
272}
273
274impl<const N: usize> From<[u8; N]> for Bytes {
275    fn from(src: [u8; N]) -> Self {
276        src.to_vec().into()
277    }
278}
279
280impl<'a, const N: usize> From<&'a [u8; N]> for Bytes {
281    fn from(src: &'a [u8; N]) -> Self {
282        src.to_vec().into()
283    }
284}
285
286impl PartialEq<[u8]> for Bytes {
287    fn eq(&self, other: &[u8]) -> bool {
288        self.as_ref() == other
289    }
290}
291
292impl PartialEq<Bytes> for [u8] {
293    fn eq(&self, other: &Bytes) -> bool {
294        *other == *self
295    }
296}
297
298impl PartialEq<Vec<u8>> for Bytes {
299    fn eq(&self, other: &Vec<u8>) -> bool {
300        self.as_ref() == &other[..]
301    }
302}
303
304impl PartialEq<Bytes> for Vec<u8> {
305    fn eq(&self, other: &Bytes) -> bool {
306        *other == *self
307    }
308}
309
310impl PartialEq<bytes::Bytes> for Bytes {
311    fn eq(&self, other: &bytes::Bytes) -> bool {
312        other == self.as_ref()
313    }
314}
315
316#[derive(Debug, Clone, Error)]
317#[error("Failed to parse bytes: {0}")]
318pub struct ParseBytesError(String);
319
320impl FromStr for Bytes {
321    type Err = ParseBytesError;
322
323    fn from_str(value: &str) -> Result<Self, Self::Err> {
324        if let Some(value) = value.strip_prefix("0x") {
325            hex::decode(value)
326        } else {
327            hex::decode(value)
328        }
329        .map(Into::into)
330        .map_err(|e| ParseBytesError(format!("Invalid hex: {e}")))
331    }
332}
333
334#[expect(
335    clippy::fallible_impl_from,
336    reason = "public API; replacing it with TryFrom is a breaking change"
337)]
338impl From<&str> for Bytes {
339    fn from(value: &str) -> Self {
340        value.parse().unwrap()
341    }
342}
343
344#[cfg(feature = "diesel")]
345impl ToSql<Binary, Pg> for Bytes {
346    fn to_sql<'b>(&'b self, out: &mut serialize::Output<'b, '_, Pg>) -> serialize::Result {
347        let bytes_slice: &[u8] = &self.0;
348        <&[u8] as ToSql<Binary, Pg>>::to_sql(&bytes_slice, &mut out.reborrow())
349    }
350}
351
352#[cfg(feature = "diesel")]
353impl FromSql<Binary, Pg> for Bytes {
354    fn from_sql(
355        bytes: <diesel::pg::Pg as diesel::backend::Backend>::RawValue<'_>,
356    ) -> deserialize::Result<Self> {
357        let byte_vec: Vec<u8> = <Vec<u8> as FromSql<Binary, Pg>>::from_sql(bytes)?;
358        Ok(Bytes(bytes::Bytes::from(byte_vec)))
359    }
360}
361
362macro_rules! impl_from_uint_for_bytes {
363    ($($t:ty),*) => {
364        $(
365            impl From<$t> for Bytes {
366                fn from(src: $t) -> Self {
367                    let size = std::mem::size_of::<$t>();
368                    let mut buf = vec![0u8; size];
369                    buf.copy_from_slice(&src.to_be_bytes());
370
371                    Self(bytes::Bytes::from(buf))
372                }
373            }
374        )*
375    };
376}
377
378impl_from_uint_for_bytes!(u8, u16, u32, u64, u128);
379
380/// A big-endian byte string holds a value outside the integer type it was converted into.
381#[derive(Debug, Clone, PartialEq, Eq, Error)]
382#[error("{value} ({len} bytes) does not fit into {target}: at most {max} significant bytes")]
383pub struct IntegerFromBytesError {
384    value: Bytes,
385    len: usize,
386    target: &'static str,
387    max: usize,
388}
389
390impl IntegerFromBytesError {
391    fn new(value: Bytes, target: &'static str, max: usize) -> Self {
392        let len = value.len();
393        Self { value, len, target, max }
394    }
395}
396
397/// Drops the leading bytes that carry no information: zero bytes for an unsigned value, and for a
398/// signed one the sign-extension bytes (`0x00` before a byte with a clear sign bit, `0xFF` before a
399/// byte with a set sign bit).
400fn significant_bytes(bytes: &[u8], signed: bool) -> &[u8] {
401    let Some(&first) = bytes.first() else { return bytes };
402    let fill = if signed && first & 0x80 != 0 { 0xFF } else { 0x00 };
403    let mut start = 0;
404    while start + 1 < bytes.len() && bytes[start] == fill {
405        let next_sign_matches = !signed || (bytes[start + 1] & 0x80) == (fill & 0x80);
406        if !next_sign_matches {
407            break;
408        }
409        start += 1;
410    }
411    &bytes[start..]
412}
413
414macro_rules! impl_try_from_bytes_for_int {
415    ($signed:literal, $($t:ty),*) => {
416        $(
417            impl TryFrom<Bytes> for $t {
418                type Error = IntegerFromBytesError;
419
420                /// Interprets the bytes as a big-endian integer, sign-extending for signed
421                /// types. Fails when the value does not fit the target type.
422                fn try_from(src: Bytes) -> Result<Self, Self::Error> {
423                    const SIZE: usize = std::mem::size_of::<$t>();
424                    let digits = significant_bytes(src.as_ref(), $signed);
425                    let Some(pad) = SIZE.checked_sub(digits.len()) else {
426                        return Err(IntegerFromBytesError::new(src, stringify!($t), SIZE));
427                    };
428                    let negative = $signed && digits.first().is_some_and(|&b| b & 0x80 != 0);
429                    let mut buf = if negative { [0xFFu8; SIZE] } else { [0x00u8; SIZE] };
430                    buf[pad..].copy_from_slice(digits);
431                    Ok(<$t>::from_be_bytes(buf))
432                }
433            }
434        )*
435    };
436}
437
438impl_try_from_bytes_for_int!(false, u8, u16, u32, u64, u128);
439impl_try_from_bytes_for_int!(true, i8, i16, i32, i64, i128);
440
441#[cfg(test)]
442mod tests {
443    use super::*;
444
445    #[test]
446    fn test_from_bytes() {
447        let b = bytes::Bytes::from("0123456789abcdef");
448        let wrapped_b = Bytes::from(b.clone());
449        let expected = Bytes(b);
450
451        assert_eq!(wrapped_b, expected);
452    }
453
454    #[test]
455    fn test_from_slice() {
456        let arr = [1, 35, 69, 103, 137, 171, 205, 239];
457        let b = Bytes::from(&arr);
458        let expected = Bytes(bytes::Bytes::from(arr.to_vec()));
459
460        assert_eq!(b, expected);
461    }
462
463    #[test]
464    fn hex_formatting() {
465        let b = Bytes::from(vec![1, 35, 69, 103, 137, 171, 205, 239]);
466        let expected = String::from("0x0123456789abcdef");
467        assert_eq!(format!("{b:x}"), expected);
468        assert_eq!(format!("{b}"), expected);
469    }
470
471    #[test]
472    fn test_from_str() {
473        let b = Bytes::from_str("0x1213");
474        assert!(b.is_ok());
475        let b = b.unwrap();
476        assert_eq!(b.as_ref(), hex::decode("1213").unwrap());
477
478        let b = Bytes::from_str("1213");
479        let b = b.unwrap();
480        assert_eq!(b.as_ref(), hex::decode("1213").unwrap());
481    }
482
483    #[test]
484    fn test_debug_formatting() {
485        let b = Bytes::from(vec![1, 35, 69, 103, 137, 171, 205, 239]);
486        assert_eq!(format!("{b:?}"), "Bytes(0x0123456789abcdef)");
487        assert_eq!(format!("{b:#?}"), "Bytes(0x0123456789abcdef)");
488    }
489
490    #[test]
491    fn test_to_vec() {
492        let vec = vec![1, 35, 69, 103, 137, 171, 205, 239];
493        let b = Bytes::from(vec.clone());
494
495        assert_eq!(b.to_vec(), vec);
496    }
497
498    #[test]
499    fn test_vec_partialeq() {
500        let vec = vec![1, 35, 69, 103, 137, 171, 205, 239];
501        let b = Bytes::from(vec.clone());
502        assert_eq!(b, vec);
503        assert_eq!(vec, b);
504
505        let wrong_vec = vec![1, 3, 52, 137];
506        assert_ne!(b, wrong_vec);
507        assert_ne!(wrong_vec, b);
508    }
509
510    #[test]
511    fn test_bytes_partialeq() {
512        let b = bytes::Bytes::from("0123456789abcdef");
513        let wrapped_b = Bytes::from(b.clone());
514        assert_eq!(wrapped_b, b);
515
516        let wrong_b = bytes::Bytes::from("0123absd");
517        assert_ne!(wrong_b, b);
518    }
519
520    #[test]
521    fn test_u128_from_bytes() {
522        let data = Bytes::from(vec![4, 3, 2, 1]);
523        let result = u128::try_from(data).unwrap();
524        assert_eq!(result, u128::from_str("67305985").unwrap());
525    }
526
527    #[test]
528    fn test_i128_from_bytes() {
529        let data = Bytes::from(vec![4, 3, 2, 1]);
530        let result = i128::try_from(data).unwrap();
531        assert_eq!(result, i128::from_str("67305985").unwrap());
532    }
533
534    #[test]
535    fn test_i32_from_bytes() {
536        let data = Bytes::from(vec![4, 3, 2, 1]);
537        let result = i32::try_from(data).unwrap();
538        assert_eq!(result, i32::from_str("67305985").unwrap());
539    }
540
541    #[test]
542    fn int_from_bytes_sign_extends_negative_values() {
543        assert_eq!(i32::try_from(Bytes::from(vec![0xff])).unwrap(), -1);
544        assert_eq!(i32::try_from(Bytes::from(vec![0xff, 0xff, 0xfe])).unwrap(), -2);
545        assert_eq!(i128::try_from(Bytes::from(vec![0x80])).unwrap(), -128);
546    }
547
548    #[test]
549    fn uint_from_bytes_accepts_leading_zero_bytes() {
550        let mut data = vec![0u8; 16];
551        data.insert(0, 0);
552        data[1] = 0x80;
553        assert_eq!(u128::try_from(Bytes::from(data)).unwrap(), 1u128 << 127);
554        assert_eq!(u8::try_from(Bytes::from(vec![0, 0, 0, 7])).unwrap(), 7);
555        assert_eq!(u8::try_from(Bytes::from(vec![0, 0])).unwrap(), 0);
556        assert_eq!(u8::try_from(Bytes::from(vec![])).unwrap(), 0);
557    }
558
559    #[test]
560    fn int_from_bytes_accepts_redundant_sign_extension_bytes() {
561        assert_eq!(i8::try_from(Bytes::from(vec![0xff, 0xff, 0x80])).unwrap(), -128);
562        assert_eq!(i8::try_from(Bytes::from(vec![0x00, 0x00, 0x7f])).unwrap(), 127);
563        assert_eq!(i16::try_from(Bytes::from(vec![0x00, 0x80])).unwrap(), 128);
564    }
565
566    #[test]
567    fn int_from_bytes_rejects_values_that_do_not_fit() {
568        let address = Bytes::from(vec![0x11u8; 20]);
569        let err = u128::try_from(address.clone()).unwrap_err();
570        assert_eq!(
571            err.to_string(),
572            format!("{address} (20 bytes) does not fit into u128: at most 16 significant bytes")
573        );
574        assert!(i128::try_from(address).is_err());
575        assert!(u8::try_from(Bytes::from(vec![0x01, 0x00])).is_err());
576        assert!(i8::try_from(Bytes::from(vec![0x00, 0x80])).is_err());
577        assert!(i8::try_from(Bytes::from(vec![0xff, 0x7f])).is_err());
578    }
579}
580
581#[cfg(feature = "diesel")]
582#[cfg(test)]
583mod diesel_tests {
584    use diesel::{insert_into, table, Insertable, Queryable};
585    use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl, SimpleAsyncConnection};
586
587    use super::*;
588
589    async fn setup_db() -> AsyncPgConnection {
590        let db_url = std::env::var("DATABASE_URL").unwrap();
591        let mut conn = AsyncPgConnection::establish(&db_url)
592            .await
593            .unwrap();
594        conn.begin_test_transaction()
595            .await
596            .unwrap();
597        conn
598    }
599
600    #[tokio::test]
601    async fn test_bytes_db_round_trip() {
602        table! {
603            bytes_table (id) {
604                id -> Int4,
605                data -> Binary,
606            }
607        }
608
609        #[derive(Insertable)]
610        #[diesel(table_name = bytes_table)]
611        struct NewByteEntry {
612            data: Bytes,
613        }
614
615        #[derive(Queryable, PartialEq)]
616        struct ByteEntry {
617            id: i32,
618            data: Bytes,
619        }
620
621        let mut conn = setup_db().await;
622        let example_bytes = Bytes::from_str("0x0123456789abcdef").unwrap();
623
624        conn.batch_execute(
625            r"
626            CREATE TEMPORARY TABLE bytes_table (
627                id SERIAL PRIMARY KEY,
628                data BYTEA NOT NULL
629            );
630        ",
631        )
632        .await
633        .unwrap();
634
635        let new_entry = NewByteEntry { data: example_bytes.clone() };
636
637        let inserted: Vec<ByteEntry> = insert_into(bytes_table::table)
638            .values(&new_entry)
639            .get_results(&mut conn)
640            .await
641            .unwrap();
642
643        assert_eq!(inserted[0].data, example_bytes);
644    }
645}