noosphere_core/data/
body_chunk.rs

1use anyhow::{anyhow, Result};
2use cid::Cid;
3use fastcdc::v2020::FastCDC;
4use libipld_cbor::DagCborCodec;
5use serde::{Deserialize, Serialize};
6
7use noosphere_storage::BlockStore;
8
9/// The maximum size of a body chunk as produced by [BodyChunkIpld]
10pub const BODY_CHUNK_MAX_SIZE: u32 = 1024 * 1024; // ~1mb/chunk worst case, ~.5mb/chunk average case
11
12/// A body chunk is a simplified flexible byte layout used for linking
13/// chunks of bytes. This is necessary to support cases when body contents
14/// byte size exceeds the IPFS block size (~1MB). This may be replaced with
15/// a more sophisticated layout structure in the future.
16#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
17pub struct BodyChunkIpld {
18    /// A chunk of bytes
19    pub bytes: Vec<u8>,
20    /// An optional pointer to the next chunk of bytes, if there are any remaining
21    pub next: Option<Cid>,
22}
23
24impl BodyChunkIpld {
25    /// Chunk and encode a slice of bytes as linked [BodyChunkIpld], storing the chunks in storage
26    /// and returning the [Cid] of the head of the list.
27    // TODO(#498): Re-write to address potentially unbounded memory overhead
28    pub async fn store_bytes<S: BlockStore>(bytes: &[u8], store: &mut S) -> Result<Cid> {
29        let chunks = FastCDC::new(
30            bytes,
31            fastcdc::v2020::MINIMUM_MIN,
32            BODY_CHUNK_MAX_SIZE / 2,
33            BODY_CHUNK_MAX_SIZE,
34        );
35        let mut byte_chunks = Vec::new();
36
37        for chunk in chunks {
38            let length = chunk.length;
39            let offset = chunk.offset;
40            let end = offset + length;
41            let bytes = &bytes[offset..end];
42
43            byte_chunks.push(bytes);
44        }
45
46        let mut next_chunk_cid = None;
47
48        for byte_chunk in byte_chunks.into_iter().rev() {
49            next_chunk_cid = Some(
50                store
51                    .save::<DagCborCodec, _>(&BodyChunkIpld {
52                        bytes: Vec::from(byte_chunk),
53                        next: next_chunk_cid,
54                    })
55                    .await?,
56            );
57        }
58
59        next_chunk_cid.ok_or_else(|| anyhow!("No CID; did you try to store zero bytes?"))
60    }
61
62    /// Fold all bytes in the [BodyChunkIpld] chain into a single buffer and return it
63    // TODO(#498): Re-write to address potentially unbounded memory overhead
64    pub async fn load_all_bytes<S: BlockStore>(&self, store: &S) -> Result<Vec<u8>> {
65        let mut all_bytes = self.bytes.clone();
66        let mut next_cid = self.next;
67
68        while let Some(cid) = next_cid {
69            let BodyChunkIpld { mut bytes, next } = store.load::<DagCborCodec, _>(&cid).await?;
70
71            all_bytes.append(&mut bytes);
72            next_cid = next;
73        }
74
75        Ok(all_bytes)
76    }
77}
noosphere_core/data/body_chunk.rs

noosphere_core/data/
body_chunk.rs
