forest/utils/encoding/
mod.rs

1// Copyright 2019-2026 ChainSafe Systems
2// SPDX-License-Identifier: Apache-2.0, MIT
3
4use crate::shim::address::Address;
5use blake2b_simd::Params;
6use fil_actors_shared::filecoin_proofs_api::ProverId;
7use fvm_ipld_encoding::strict_bytes::{Deserialize, Serialize};
8use serde::{Deserializer, Serializer, de, ser};
9
10mod fallback_de_ipld_dagcbor;
11pub mod hex;
12
13/// Limit the number of bytes that are used for pre-allocating `Vec<Cid>`s. This follows what `serde` is
14/// doing internally with `serde::private::size_hint::cautious()`.
15/// The limit is set to 1 MiB, which is a reasonable upper bound for most use cases.
16fn size_hint_cautious_cid(size_hint: usize) -> usize {
17    const MAX_PREALLOC_BYTES: usize = 1024 * 1024;
18    size_hint.min(MAX_PREALLOC_BYTES / std::mem::size_of::<cid::Cid>())
19}
20
21/// This method will attempt to de-serialize given bytes using the regular
22/// `serde_ipld_dagcbor::from_slice`. Due to a historical issue in Lotus (see more in
23/// [FIP-0027](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0027.md), we must still
24/// support strings with invalid UTF-8 bytes. On a failure, it
25/// will retry the operation using the fallback that will de-serialize
26/// strings with invalid UTF-8 bytes as bytes.
27pub fn from_slice_with_fallback<'a, T: serde::de::Deserialize<'a>>(
28    bytes: &'a [u8],
29) -> anyhow::Result<T> {
30    match serde_ipld_dagcbor::from_slice(bytes) {
31        Ok(v) => Ok(v),
32        Err(err) => fallback_de_ipld_dagcbor::from_slice(bytes).map_err(|fallback_err| {
33            anyhow::anyhow!(
34                "Fallback deserialization failed: {fallback_err}. Original error: {err}"
35            )
36        }),
37    }
38}
39
40mod cid_de_cbor;
41pub use cid_de_cbor::extract_cids;
42
43/// `serde_bytes` with max length check
44pub mod serde_byte_array {
45    use super::*;
46    /// lotus use cbor-gen for generating codec for types, it has a length limit
47    /// for byte array as `2 << 20`
48    ///
49    /// <https://github.com/whyrusleeping/cbor-gen/blob/f57984553008dd4285df16d4ec2760f97977d713/gen.go#L16>
50    pub const BYTE_ARRAY_MAX_LEN: usize = 2 << 20;
51
52    /// checked if `input > crate::utils::BYTE_ARRAY_MAX_LEN`
53    pub fn serialize<T, S>(bytes: &T, serializer: S) -> Result<S::Ok, S::Error>
54    where
55        T: ?Sized + Serialize + AsRef<[u8]>,
56        S: Serializer,
57    {
58        let len = bytes.as_ref().len();
59        if len > BYTE_ARRAY_MAX_LEN {
60            return Err(ser::Error::custom::<String>(
61                "Array exceed max length".into(),
62            ));
63        }
64
65        Serialize::serialize(bytes, serializer)
66    }
67
68    /// checked if `output > crate::utils::ByteArrayMaxLen`
69    pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
70    where
71        T: Deserialize<'de> + AsRef<[u8]>,
72        D: Deserializer<'de>,
73    {
74        Deserialize::deserialize(deserializer).and_then(|bytes: T| {
75            if bytes.as_ref().len() > BYTE_ARRAY_MAX_LEN {
76                Err(de::Error::custom::<String>(
77                    "Array exceed max length".into(),
78                ))
79            } else {
80                Ok(bytes)
81            }
82        })
83    }
84}
85
86/// Generates BLAKE2b hash of fixed 32 bytes size.
87///
88/// # Example
89/// ```
90/// # use forest::doctest_private::blake2b_256;
91///
92/// let ingest: Vec<u8> = vec![];
93/// let hash = blake2b_256(&ingest);
94/// assert_eq!(hash.len(), 32);
95/// ```
96pub fn blake2b_256(ingest: &[u8]) -> [u8; 32] {
97    let digest = Params::new()
98        .hash_length(32)
99        .to_state()
100        .update(ingest)
101        .finalize();
102
103    let mut ret = [0u8; 32];
104    ret.clone_from_slice(digest.as_bytes());
105    ret
106}
107
108/// Generates Keccak-256 hash of fixed 32 bytes size.
109///
110/// # Example
111/// ```
112/// # use forest::doctest_private::keccak_256;
113/// let ingest: Vec<u8> = vec![];
114/// let hash = keccak_256(&ingest);
115/// assert_eq!(hash.len(), 32);
116/// ```
117pub fn keccak_256(ingest: &[u8]) -> [u8; 32] {
118    let mut ret: [u8; 32] = Default::default();
119    keccak_hash::keccak_256(ingest, &mut ret);
120    ret
121}
122
123pub fn prover_id_from_u64(id: u64) -> ProverId {
124    let mut prover_id = ProverId::default();
125    let prover_bytes = Address::new_id(id).payload().to_raw_bytes();
126    assert!(prover_bytes.len() <= prover_id.len());
127    #[allow(clippy::indexing_slicing)]
128    prover_id[..prover_bytes.len()].copy_from_slice(&prover_bytes);
129    prover_id
130}
131
132#[cfg(test)]
133mod tests {
134    use ipld_core::ipld::Ipld;
135    use itertools::Itertools as _;
136    use rand::Rng;
137    use serde::{Deserialize, Serialize};
138    use serde_ipld_dagcbor::to_vec;
139
140    use super::*;
141    use crate::utils::encoding::serde_byte_array::BYTE_ARRAY_MAX_LEN;
142
143    #[test]
144    fn vector_hashing() {
145        let ing_vec = vec![1, 2, 3];
146
147        assert_eq!(blake2b_256(&ing_vec), blake2b_256(&[1, 2, 3]));
148        assert_ne!(blake2b_256(&ing_vec), blake2b_256(&[1, 2, 3, 4]));
149    }
150
151    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
152    struct ByteArray {
153        #[serde(with = "serde_byte_array")]
154        pub inner: Vec<u8>,
155    }
156
157    #[test]
158    fn can_serialize_byte_array() {
159        for len in [0, 1, BYTE_ARRAY_MAX_LEN] {
160            let bytes = ByteArray {
161                inner: vec![0; len],
162            };
163
164            assert!(serde_ipld_dagcbor::to_vec(&bytes).is_ok());
165        }
166    }
167
168    #[test]
169    fn cannot_serialize_byte_array_overflow() {
170        let bytes = ByteArray {
171            inner: vec![0; BYTE_ARRAY_MAX_LEN + 1],
172        };
173
174        let err = serde_ipld_dagcbor::to_vec(&bytes).unwrap_err();
175        assert!(
176            format!("{err}").contains("Struct value cannot be serialized."),
177            "{err}"
178        );
179    }
180
181    #[test]
182    fn can_deserialize_byte_array() {
183        for len in [0, 1, BYTE_ARRAY_MAX_LEN] {
184            let bytes = ByteArray {
185                inner: vec![0; len],
186            };
187
188            let encoding = serde_ipld_dagcbor::to_vec(&bytes).unwrap();
189            assert_eq!(
190                from_slice_with_fallback::<ByteArray>(&encoding).unwrap(),
191                bytes
192            );
193        }
194    }
195
196    #[test]
197    fn cannot_deserialize_byte_array_overflow() {
198        let max_length_bytes = ByteArray {
199            inner: vec![0; BYTE_ARRAY_MAX_LEN],
200        };
201
202        // prefix: 2 ^ 21 -> 2 ^ 21 + 1
203        let mut overflow_encoding = serde_ipld_dagcbor::to_vec(&max_length_bytes).unwrap();
204        let encoding_len = overflow_encoding.len();
205        overflow_encoding[encoding_len - BYTE_ARRAY_MAX_LEN - 1] = 1;
206        overflow_encoding.push(0);
207
208        assert!(
209            format!(
210                "{}",
211                from_slice_with_fallback::<ByteArray>(&overflow_encoding)
212                    .err()
213                    .unwrap()
214            )
215            .contains("Array exceed max length")
216        );
217    }
218
219    #[test]
220    fn parity_tests() {
221        use cs_serde_bytes;
222
223        #[derive(Deserialize, Serialize)]
224        struct A(#[serde(with = "fvm_ipld_encoding::strict_bytes")] Vec<u8>);
225
226        #[derive(Deserialize, Serialize)]
227        struct B(#[serde(with = "cs_serde_bytes")] Vec<u8>);
228
229        let mut array = [0; 1024];
230        crate::utils::rand::forest_rng().fill(&mut array);
231
232        let a = A(array.to_vec());
233        let b = B(array.to_vec());
234
235        assert_eq!(
236            serde_json::to_string_pretty(&a).unwrap(),
237            serde_json::to_string_pretty(&b).unwrap()
238        );
239    }
240
241    #[test]
242    fn test_fallback_deserialization() {
243        // where the regular deserialization fails with invalid UTF-8 strings, the fallback should
244        // succeed.
245
246        // Valid UTF-8, should return the same results.
247        let ipld_string = Ipld::String("cthulhu".to_string());
248        let serialized = to_vec(&ipld_string).unwrap();
249        assert_eq!(
250            ipld_string,
251            serde_ipld_dagcbor::from_slice::<Ipld>(&serialized).unwrap()
252        );
253        assert_eq!(
254            ipld_string,
255            from_slice_with_fallback::<Ipld>(&serialized).unwrap()
256        );
257
258        // Invalid UTF-8, regular deserialization fails, fallback succeeds. We can
259        // extract the bytes.
260        let corrupted = serialized
261            .iter()
262            .take(serialized.len() - 2)
263            .chain(&[0xa0, 0xa1])
264            .copied()
265            .collect_vec();
266        assert!(
267            matches!(from_slice_with_fallback::<Ipld>(&corrupted).unwrap(), Ipld::Bytes(bytes) if bytes == [0x63, 0x74, 0x68, 0x75, 0x6c, 0xa0, 0xa1])
268        )
269    }
270
271    #[test]
272    fn size_hint_cautious_test() {
273        assert_eq!(size_hint_cautious_cid(0), 0);
274        assert_eq!(
275            size_hint_cautious_cid(1024 * 1024),
276            1024 * 1024 / std::mem::size_of::<cid::Cid>()
277        );
278    }
279}
forest/utils/encoding/mod.rs

forest/utils/encoding/
mod.rs
