netget/server/libp2p/
ipfs.rs

1//! The IPFS data model, minimally: multihashes (sha2-256 only), CIDv0 and CIDv1 in binary and
2//! text (base58btc, and base32 as `b…`), the `raw` and `dag-pb` codecs, and UnixFS nodes.
3//!
4//! A block's CID is always computed here from its bytes. Nothing a peer or the model says
5//! about a block's identity is taken on trust: a block is sent, or reported, only when its
6//! bytes hash to the CID it is sent or reported under.
7//!
8//! Protobuf nesting is fixed by the schema (PBNode → PBLink, PBNode.Data → UnixFS Data); no
9//! decoder here recurses, so a peer cannot buy stack depth.
10use super::wire::{base58, get_uvarint, pb_bytes, pb_fields_bounded, pb_varint, put_uvarint};
11use super::wire::{unbase58, PbValue};
12use anyhow::{bail, ensure, Context, Result};
13use sha2::Digest;
14
15/// Multicodec `raw`: the block is the bytes.
16pub const RAW: u64 = 0x55;
17/// Multicodec `dag-pb`: a protobuf PBNode, the codec UnixFS is written in.
18pub const DAG_PB: u64 = 0x70;
19/// Multihash `sha2-256`, 32-byte digests.
20pub const SHA2_256: u64 = 0x12;
21/// A block is at most this long (Bitswap's own limit, which Kubo enforces too).
22pub const MAX_BLOCK: usize = 2 * 1024 * 1024;
23/// A CID in text is at most this long; a sha2-256 CIDv1 in base32 is 59 characters.
24pub const MAX_CID_TEXT: usize = 128;
25/// Links one dag-pb node may carry before it is refused (Kubo writes at most 174 per file
26/// node and 1024-ish per directory shard).
27pub const MAX_LINKS: usize = 4096;
28
29/// A content identifier: version, codec and a sha2-256 digest. Other hash functions are not
30/// representable, so a CID naming one is refused at parse time.
31#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
32pub struct Cid {
33    pub version: u8,
34    pub codec: u64,
35    pub digest: [u8; 32],
36}
37
38/// Why a CID is not one NetGet can serve or verify.
39pub fn codec_name(codec: u64) -> String {
40    match codec {
41        RAW => "raw".into(),
42        DAG_PB => "dag-pb".into(),
43        0x71 => "dag-cbor".into(),
44        0x0129 => "dag-json".into(),
45        0x72 => "libp2p-key".into(),
46        c => format!("{c:#x}"),
47    }
48}
49
50fn hash_name(code: u64) -> String {
51    match code {
52        0x00 => "identity".into(),
53        0x11 => "sha1".into(),
54        0x12 => "sha2-256".into(),
55        0x13 => "sha2-512".into(),
56        0x1e => "blake3".into(),
57        0xb220 => "blake2b-256".into(),
58        c => format!("{c:#x}"),
59    }
60}
61
62/// `sha2-256` multihash of `data`: the 34 bytes `0x12 0x20 ‖ digest`.
63pub fn sha256(data: &[u8]) -> [u8; 32] {
64    sha2::Sha256::digest(data).into()
65}
66
67/// A multihash's digest, refusing every function but sha2-256 with 32-byte output.
68pub fn parse_multihash(b: &[u8]) -> Result<([u8; 32], usize)> {
69    let (code, n) = get_uvarint(b)?;
70    let (len, m) = get_uvarint(&b[n..])?;
71    ensure!(
72        code == SHA2_256,
73        "multihash function {} is not supported (only sha2-256)",
74        hash_name(code)
75    );
76    ensure!(
77        len == 32,
78        "a sha2-256 multihash must be 32 bytes, not {len}"
79    );
80    let start = n + m;
81    ensure!(b.len() >= start + 32, "truncated multihash");
82    let mut d = [0u8; 32];
83    d.copy_from_slice(&b[start..start + 32]);
84    Ok((d, start + 32))
85}
86
87pub fn multihash_bytes(digest: &[u8; 32]) -> Vec<u8> {
88    let mut out = vec![SHA2_256 as u8, 32];
89    out.extend_from_slice(digest);
90    out
91}
92
93impl Cid {
94    pub fn v1(codec: u64, data: &[u8]) -> Cid {
95        Cid {
96            version: 1,
97            codec,
98            digest: sha256(data),
99        }
100    }
101    /// Whether `data` hashes to this CID. The codec is the sender's claim about the bytes; the
102    /// digest is what is checked.
103    pub fn matches(&self, data: &[u8]) -> bool {
104        sha256(data) == self.digest
105    }
106    /// The same block under any CID version: two CIDs name the same block when codec and
107    /// digest agree.
108    pub fn same_block(&self, other: &Cid) -> bool {
109        self.codec == other.codec && self.digest == other.digest
110    }
111    pub fn multihash(&self) -> Vec<u8> {
112        multihash_bytes(&self.digest)
113    }
114    pub fn to_bytes(&self) -> Vec<u8> {
115        if self.version == 0 {
116            return self.multihash();
117        }
118        let mut out = Vec::with_capacity(38);
119        put_uvarint(&mut out, 1);
120        put_uvarint(&mut out, self.codec);
121        out.extend(self.multihash());
122        out
123    }
124    /// Binary CID: v0 is a bare sha2-256 multihash, v1 is `0x01 ‖ codec ‖ multihash`.
125    pub fn from_bytes(b: &[u8]) -> Result<Cid> {
126        ensure!(!b.is_empty(), "empty CID");
127        if b.len() == 34 && b[0] == SHA2_256 as u8 && b[1] == 32 {
128            let (digest, _) = parse_multihash(b)?;
129            return Ok(Cid {
130                version: 0,
131                codec: DAG_PB,
132                digest,
133            });
134        }
135        let (version, n) = get_uvarint(b)?;
136        ensure!(version == 1, "CID version {version} is not supported");
137        let (codec, m) = get_uvarint(&b[n..])?;
138        let (digest, used) = parse_multihash(&b[n + m..])?;
139        ensure!(n + m + used == b.len(), "trailing bytes after the CID");
140        Ok(Cid {
141            version: 1,
142            codec,
143            digest,
144        })
145    }
146    /// Refuse codecs NetGet cannot interpret (everything but raw and dag-pb).
147    pub fn supported(&self) -> Result<()> {
148        ensure!(
149            self.codec == RAW || self.codec == DAG_PB,
150            "codec {} is not supported (only raw and dag-pb)",
151            codec_name(self.codec)
152        );
153        Ok(())
154    }
155    /// Bitswap's block prefix: version, codec, hash function, digest length.
156    pub fn prefix(&self) -> Vec<u8> {
157        let mut out = Vec::new();
158        put_uvarint(&mut out, u64::from(self.version));
159        put_uvarint(&mut out, self.codec);
160        put_uvarint(&mut out, SHA2_256);
161        put_uvarint(&mut out, 32);
162        out
163    }
164    /// The CID a Bitswap 1.1+ block names: its prefix applied to its bytes.
165    pub fn from_prefix(prefix: &[u8], data: &[u8]) -> Result<Cid> {
166        let mut b = prefix;
167        let mut next = || -> Result<u64> {
168            let (v, n) = get_uvarint(b)?;
169            b = &b[n..];
170            Ok(v)
171        };
172        let (version, codec, hash, len) = (next()?, next()?, next()?, next()?);
173        ensure!(version <= 1, "CID version {version} is not supported");
174        ensure!(
175            hash == SHA2_256,
176            "multihash function {} is not supported (only sha2-256)",
177            hash_name(hash)
178        );
179        ensure!(len == 32, "a sha2-256 digest is 32 bytes, not {len}");
180        ensure!(
181            version == 1 || codec == DAG_PB,
182            "a CIDv0 is always dag-pb, not {}",
183            codec_name(codec)
184        );
185        Ok(Cid {
186            version: version as u8,
187            codec,
188            digest: sha256(data),
189        })
190    }
191    /// Text: `Qm…` (base58btc) for v0, `b…` (base32, lower case) for v1.
192    pub fn parse(text: &str) -> Result<Cid> {
193        let s = text.trim();
194        ensure!(!s.is_empty(), "empty CID");
195        ensure!(
196            s.len() <= MAX_CID_TEXT,
197            "CID text longer than {MAX_CID_TEXT}"
198        );
199        if s.len() == 46 && s.starts_with("Qm") {
200            return Cid::from_bytes(&unbase58(s)?);
201        }
202        let (base, rest) = s.split_at(1);
203        let bytes = match base {
204            "b" => base32_decode(rest)?,
205            "B" => base32_decode(&rest.to_ascii_lowercase())?,
206            "z" => unbase58(rest)?,
207            "f" | "F" => hex::decode(rest).context("not a base16 CID")?,
208            _ => bail!("{s:?} is not a CID (expected Qm… or a base32 b… / base58 z… CIDv1)"),
209        };
210        let cid = Cid::from_bytes(&bytes).with_context(|| format!("{s:?} is not a CID"))?;
211        ensure!(cid.version == 1, "a multibase CID must be version 1");
212        Ok(cid)
213    }
214}
215
216impl std::fmt::Display for Cid {
217    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
218        if self.version == 0 {
219            write!(f, "{}", base58(&self.to_bytes()))
220        } else {
221            write!(f, "b{}", base32_encode(&self.to_bytes()))
222        }
223    }
224}
225
226/// Every CID text a sha2-256 digest can stand for, in the two codecs NetGet knows: what a
227/// DHT key (a bare multihash) may mean.
228pub fn cids_for_digest(digest: &[u8; 32]) -> Vec<String> {
229    let v0 = Cid {
230        version: 0,
231        codec: DAG_PB,
232        digest: *digest,
233    };
234    let v1pb = Cid { version: 1, ..v0 };
235    let v1raw = Cid { codec: RAW, ..v1pb };
236    vec![v0.to_string(), v1pb.to_string(), v1raw.to_string()]
237}
238
239const B32: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
240
241/// RFC 4648 base32, lower case, no padding (multibase `b`).
242pub fn base32_encode(b: &[u8]) -> String {
243    let mut out = String::with_capacity(b.len() * 8 / 5 + 1);
244    let (mut acc, mut bits) = (0u32, 0u32);
245    for &byte in b {
246        acc = (acc << 8) | u32::from(byte);
247        bits += 8;
248        while bits >= 5 {
249            bits -= 5;
250            out.push(B32[((acc >> bits) & 31) as usize] as char);
251        }
252    }
253    if bits > 0 {
254        out.push(B32[((acc << (5 - bits)) & 31) as usize] as char);
255    }
256    out
257}
258
259pub fn base32_decode(s: &str) -> Result<Vec<u8>> {
260    let mut out = Vec::with_capacity(s.len() * 5 / 8);
261    let (mut acc, mut bits) = (0u32, 0u32);
262    for c in s.bytes() {
263        let v = B32
264            .iter()
265            .position(|&x| x == c)
266            .with_context(|| format!("{:?} is not base32", c as char))? as u32;
267        acc = (acc << 5) | v;
268        bits += 5;
269        if bits >= 8 {
270            bits -= 8;
271            out.push((acc >> bits) as u8);
272        }
273        acc &= (1 << bits) - 1;
274    }
275    Ok(out)
276}
277
278// ---------------------------------------------------------------------------------------
279// dag-pb and UnixFS
280
281/// A dag-pb link: the child's CID, its name in a directory, and the size of its subtree.
282#[derive(Clone, Debug, PartialEq)]
283pub struct Link {
284    pub cid: Cid,
285    pub name: String,
286    pub size: u64,
287}
288
289/// What a UnixFS node says about itself.
290#[derive(Clone, Debug, PartialEq, Default)]
291pub struct UnixFs {
292    /// raw, directory, file, metadata, symlink, hamt-shard.
293    pub kind: String,
294    pub data: Vec<u8>,
295    pub file_size: Option<u64>,
296    pub block_sizes: Vec<u64>,
297}
298
299/// A decoded dag-pb node.
300#[derive(Clone, Debug, PartialEq, Default)]
301pub struct PbNode {
302    pub links: Vec<Link>,
303    pub data: Option<Vec<u8>>,
304}
305
306fn unixfs_kind(t: u64) -> String {
307    match t {
308        0 => "raw",
309        1 => "directory",
310        2 => "file",
311        3 => "metadata",
312        4 => "symlink",
313        5 => "hamt-shard",
314        _ => "unknown",
315    }
316    .into()
317}
318
319/// The canonical dag-pb encoding: every link (field 2) before the data (field 1).
320pub fn encode_pb_node(node: &PbNode) -> Vec<u8> {
321    let mut out = Vec::new();
322    for l in &node.links {
323        let mut link = Vec::new();
324        pb_bytes(&mut link, 1, &l.cid.to_bytes());
325        pb_bytes(&mut link, 2, l.name.as_bytes());
326        pb_varint(&mut link, 3, l.size);
327        pb_bytes(&mut out, 2, &link);
328    }
329    if let Some(d) = &node.data {
330        pb_bytes(&mut out, 1, d);
331    }
332    out
333}
334
335pub fn decode_pb_node(b: &[u8]) -> Result<PbNode> {
336    let mut node = PbNode::default();
337    for (f, v) in pb_fields_bounded(b, MAX_LINKS + 1)? {
338        match (f, v) {
339            (1, PbValue::Bytes(d)) => node.data = Some(d.to_vec()),
340            (2, PbValue::Bytes(l)) => {
341                let (mut hash, mut name, mut size) = (None, String::new(), 0);
342                for (lf, lv) in pb_fields_bounded(l, 16)? {
343                    match (lf, lv) {
344                        (1, PbValue::Bytes(h)) => hash = Some(Cid::from_bytes(h)?),
345                        (2, PbValue::Bytes(n)) => {
346                            name =
347                                String::from_utf8(n.to_vec()).context("link name is not UTF-8")?
348                        }
349                        (3, PbValue::Varint(s)) => size = s,
350                        _ => bail!("malformed dag-pb link"),
351                    }
352                }
353                node.links.push(Link {
354                    cid: hash.context("a dag-pb link has no hash")?,
355                    name,
356                    size,
357                });
358            }
359            _ => bail!("malformed dag-pb node (field {f})"),
360        }
361    }
362    Ok(node)
363}
364
365/// UnixFS `Data`: Type, Data, filesize, blocksizes, in field order as go-unixfs writes them.
366pub fn encode_unixfs(u: &UnixFs) -> Vec<u8> {
367    let kind = match u.kind.as_str() {
368        "raw" => 0,
369        "directory" => 1,
370        "metadata" => 3,
371        "symlink" => 4,
372        "hamt-shard" => 5,
373        _ => 2,
374    };
375    let mut out = Vec::new();
376    pb_varint(&mut out, 1, kind);
377    if !u.data.is_empty() {
378        pb_bytes(&mut out, 2, &u.data);
379    }
380    if let Some(s) = u.file_size {
381        pb_varint(&mut out, 3, s);
382    }
383    for s in &u.block_sizes {
384        pb_varint(&mut out, 4, *s);
385    }
386    out
387}
388
389pub fn decode_unixfs(b: &[u8]) -> Result<UnixFs> {
390    let mut u = UnixFs::default();
391    let mut kind = None;
392    for (f, v) in pb_fields_bounded(b, MAX_LINKS + 16)? {
393        match (f, v) {
394            (1, PbValue::Varint(t)) => kind = Some(t),
395            (2, PbValue::Bytes(d)) => u.data = d.to_vec(),
396            (3, PbValue::Varint(s)) => u.file_size = Some(s),
397            (4, PbValue::Varint(s)) => u.block_sizes.push(s),
398            // hashType, fanout, mode, mtime: read past.
399            (5..=8, _) => {}
400            _ => bail!("malformed UnixFS data (field {f})"),
401        }
402    }
403    u.kind = unixfs_kind(kind.context("UnixFS data has no type")?);
404    Ok(u)
405}
406
407/// A single-block UnixFS file holding `content`, as `ipfs add` writes a file no larger than
408/// its chunk size when it does not use raw leaves.
409pub fn unixfs_file_block(content: &[u8]) -> Vec<u8> {
410    encode_pb_node(&PbNode {
411        links: Vec::new(),
412        data: Some(encode_unixfs(&UnixFs {
413            kind: "file".into(),
414            data: content.to_vec(),
415            file_size: Some(content.len() as u64),
416            block_sizes: Vec::new(),
417        })),
418    })
419}
420
421/// What a block looks like to whoever receives it: its codec, and for dag-pb its UnixFS
422/// meaning and links.
423pub struct Described {
424    pub codec: &'static str,
425    /// The bytes worth showing: the block for raw, the UnixFS data for a dag-pb file.
426    pub content: Vec<u8>,
427    pub unixfs_type: Option<String>,
428    pub file_size: Option<u64>,
429    pub links: Vec<Link>,
430}
431
432pub fn describe(cid: &Cid, data: &[u8]) -> Result<Described> {
433    match cid.codec {
434        RAW => Ok(Described {
435            codec: "raw",
436            content: data.to_vec(),
437            unixfs_type: None,
438            file_size: None,
439            links: Vec::new(),
440        }),
441        DAG_PB => {
442            let node = decode_pb_node(data)?;
443            let u = node.data.as_deref().map(decode_unixfs).transpose()?;
444            Ok(Described {
445                codec: "dag-pb",
446                content: u.as_ref().map(|u| u.data.clone()).unwrap_or_default(),
447                unixfs_type: u.as_ref().map(|u| u.kind.clone()),
448                file_size: u.as_ref().and_then(|u| u.file_size),
449                links: node.links,
450            })
451        }
452        c => bail!("codec {} is not supported", codec_name(c)),
453    }
454}
netget/server/libp2p/ipfs.rs

netget/server/libp2p/
ipfs.rs
