filecoin_proofs/
commitment_reader.rs

1use std::cmp::min;
2use std::io::{self, Read};
3
4use anyhow::{ensure, Result};
5use filecoin_hashers::{HashFunction, Hasher};
6use rayon::prelude::{ParallelIterator, ParallelSlice};
7
8use crate::{constants::DefaultPieceHasher, pieces::piece_hash};
9
10const BUFFER_SIZE: usize = 4096;
11
12/// Calculates comm-d of the data piped through to it.
13/// Data must be bit padded and power of 2 bytes.
14pub struct CommitmentReader<R> {
15    source: R,
16    buffer: Vec<u8>,
17    buffer_pos: usize,
18    current_tree: Vec<<DefaultPieceHasher as Hasher>::Domain>,
19}
20
21impl<R: Read> CommitmentReader<R> {
22    pub fn new(source: R) -> Self {
23        CommitmentReader {
24            source,
25            buffer: vec![0u8; BUFFER_SIZE],
26            buffer_pos: 0,
27            current_tree: Vec::new(),
28        }
29    }
30
31    /// Attempt to generate the next hash, but only if the buffers are full.
32    fn try_hash(&mut self) {
33        // Get more bytes in case we cannot iterate cleanly in 64 byte chunks.
34        if self.buffer_pos % 64 != 0 {
35            return;
36        }
37
38        for chunk in self.buffer[..self.buffer_pos].chunks_exact(64) {
39            // WARNING: keep in sync with DefaultPieceHasher and its .node impl
40            let hash = <DefaultPieceHasher as Hasher>::Function::hash(chunk);
41            self.current_tree.push(hash);
42        }
43        self.buffer_pos = 0;
44
45        // TODO: reduce hashes when possible, instead of keeping them around.
46    }
47
48    pub fn finish(self) -> Result<<DefaultPieceHasher as Hasher>::Domain> {
49        ensure!(self.buffer_pos == 0, "not enough inputs provided");
50
51        let CommitmentReader { current_tree, .. } = self;
52
53        let mut current_row = current_tree;
54
55        while current_row.len() > 1 {
56            let next_row = current_row
57                .par_chunks(2)
58                .map(|chunk| piece_hash(chunk[0].as_ref(), chunk[1].as_ref()))
59                .collect::<Vec<_>>();
60
61            current_row = next_row;
62        }
63        debug_assert_eq!(current_row.len(), 1);
64
65        Ok(current_row
66            .into_iter()
67            .next()
68            .expect("should have been caught by debug build: len==1"))
69    }
70}
71
72impl<R: Read> Read for CommitmentReader<R> {
73    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
74        let start = self.buffer_pos;
75        let left = BUFFER_SIZE - self.buffer_pos;
76        let end = start + min(left, buf.len());
77
78        // fill the buffer as much as possible
79        let r = self.source.read(&mut self.buffer[start..end])?;
80
81        // write the data, we read
82        buf[..r].copy_from_slice(&self.buffer[start..start + r]);
83
84        self.buffer_pos += r;
85
86        // try to hash
87        self.try_hash();
88
89        Ok(r)
90    }
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96
97    use std::io::Cursor;
98
99    use fr32::Fr32Reader;
100    use storage_proofs_core::pieces::generate_piece_commitment_bytes_from_source;
101
102    use crate::types::{PaddedBytesAmount, UnpaddedBytesAmount};
103
104    #[test]
105    fn test_commitment_reader() {
106        let piece_size = 127 * 8;
107        let source = vec![255u8; piece_size];
108        let mut fr32_reader = Fr32Reader::new(Cursor::new(&source));
109
110        let commitment1 = generate_piece_commitment_bytes_from_source::<DefaultPieceHasher>(
111            &mut fr32_reader,
112            PaddedBytesAmount::from(UnpaddedBytesAmount(piece_size as u64)).into(),
113        )
114        .expect("failed to generate piece commitment bytes from source");
115
116        let fr32_reader = Fr32Reader::new(Cursor::new(&source));
117        let mut commitment_reader = CommitmentReader::new(fr32_reader);
118        io::copy(&mut commitment_reader, &mut io::sink()).expect("io copy failed");
119
120        let commitment2 = commitment_reader.finish().expect("failed to finish");
121
122        assert_eq!(&commitment1[..], AsRef::<[u8]>::as_ref(&commitment2));
123    }
124}
filecoin_proofs/commitment_reader.rs

filecoin_proofs/
commitment_reader.rs
