Skip to main content

speed/
speed.rs

1//! How fast the parts of building and checking a tree are, on one file held
2//! in memory, so that no disk is in the way.
3//!
4//! ```sh
5//! cargo run --release --example speed -- <MiB> [folder]
6//! ```
7//!
8//! It prints, for a file of that many MiB of bytes that look random: chunking
9//! alone, hashing alone, building the file into a store in memory, checking
10//! every chunk of it again, fetching it into a second store as a receiver
11//! would, and writing it as a bundle and reading that into a third.
12//!
13//! With a folder, it also builds that folder from disk into a store in
14//! memory, and prints the best of five runs: by then the system holds the
15//! files in memory, so it is the library that is measured and not the disk.
16
17use scionic_merkletree::{
18    build, export, fetch, import, record_of, verify, BuildOptions, Builder, Chunking, ExtraData,
19    Hash, MemoryStore, Responder, Stored,
20};
21use std::time::Instant;
22
23/// Bytes that look random and cost little to make: a xorshift generator.
24fn filler(length: usize) -> Vec<u8> {
25    let mut state = 0x9E37_79B9_7F4A_7C15_u64;
26    let mut bytes = Vec::with_capacity(length + 8);
27    while bytes.len() < length {
28        state ^= state << 13;
29        state ^= state >> 7;
30        state ^= state << 17;
31        bytes.extend_from_slice(&state.to_le_bytes());
32    }
33    bytes.truncate(length);
34    bytes
35}
36
37fn report(step: &str, started: Instant, bytes: usize) {
38    let seconds = started.elapsed().as_secs_f64();
39    let rate = bytes as f64 / (1 << 20) as f64 / seconds.max(1e-9);
40    println!("{step:<22} {seconds:>7.3} s  {rate:>8.0} MiB/s");
41}
42
43fn main() -> Result<(), Box<dyn std::error::Error>> {
44    let megabytes: usize = match std::env::args().nth(1) {
45        Some(argument) => argument.parse()?,
46        None => 1024,
47    };
48    let data = filler(megabytes << 20);
49
50    let started = Instant::now();
51    let chunks = Chunking::FastCdcV1.cut(&data);
52    report("chunked", started, data.len());
53    println!("{:<22} {} chunks", "", chunks.len());
54
55    let started = Instant::now();
56    let mut mixed = 0_u8;
57    for chunk in &chunks {
58        mixed ^= Hash::of(chunk).as_bytes()[0];
59    }
60    report("hashed", started, data.len());
61
62    let store = MemoryStore::new();
63    let started = Instant::now();
64    let mut builder = Builder::new(&store, Chunking::FastCdcV1);
65    let file = builder.file_from_bytes(&data)?;
66    report("built", started, data.len());
67
68    let root = builder.finish(file, ExtraData::new())?;
69    let started = Instant::now();
70    verify(&store, &root)?;
71    report("checked", started, data.len());
72    println!("root {root} ({mixed:02x})");
73
74    // The tree is fetched from the first store into a second one, as a
75    // receiver would take it from a peer, and then written as a bundle and
76    // read back into a third. Each checks every byte as it comes.
77    let fetched = MemoryStore::new();
78    let started = Instant::now();
79    fetch(&fetched, &root, &mut Responder::new(&store))?;
80    report("fetched", started, data.len());
81
82    let imported = MemoryStore::new();
83    let mut bundle = Vec::with_capacity(data.len() + (1 << 20));
84    let started = Instant::now();
85    export(&store, &root, &mut bundle)?;
86    import(&imported, &root, &mut bundle.as_slice())?;
87    report("bundled", started, data.len());
88
89    if let Some(folder) = std::env::args().nth(2) {
90        let mut best = f64::MAX;
91        let (mut bytes, mut files) = (0, 0);
92        for _ in 0..5 {
93            let store = MemoryStore::new();
94            let started = Instant::now();
95            let root = build(&store, &folder, &BuildOptions::new())?;
96            best = best.min(started.elapsed().as_secs_f64());
97            bytes = record_of(&store, &root)?.top().size;
98            files = store.count(Stored::FileList);
99        }
100        let rate = bytes as f64 / (1 << 20) as f64 / best.max(1e-9);
101        println!("{:<22} {best:>7.3} s  {rate:>8.0} MiB/s", "built from disk");
102        println!("{:<22} {files} different files, {bytes} bytes", "");
103    }
104    Ok(())
105}