lance-encoding 4.0.1

Encoders and decoders for the Lance file format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main};
use lance_encoding::buffer::LanceBuffer;

const NUM_VALUES: &[usize] = &[1024 * 1024, 32 * 1024, 8 * 1024];

fn bench_zip(c: &mut Criterion) {
    for num_values in NUM_VALUES {
        let num_values = *num_values;
        let mut group = c.benchmark_group(format!("zip_{}Ki", num_values / 1024));

        group.throughput(Throughput::Bytes((num_values * 6) as u64));

        group.bench_function("2_4_zip_into_6", move |b| {
            // Zip together a 2-byte-per-buffer and an 8-byte-per-buffer array, each with 1Mi items
            let random_shorts: Vec<u8> =
                (0..num_values * 2).map(|_| rand::random::<u8>()).collect();
            let random_ints: Vec<u8> = (0..num_values * 4).map(|_| rand::random::<u8>()).collect();
            let mut buffers = vec![
                (LanceBuffer::from(random_shorts), 16),
                (LanceBuffer::from(random_ints), 32),
            ];
            let buffers = &mut buffers;

            b.iter(move || {
                let buffers = buffers
                    .iter_mut()
                    .map(|(buf, bits_per_value)| (buf.clone(), *bits_per_value))
                    .collect::<Vec<_>>();
                black_box(LanceBuffer::zip_into_one(buffers, num_values as u64).unwrap());
            })
        });

        group.bench_function("2_2_2_zip_into_6", move |b| {
            // Zip together a 2-byte-per-buffer and an 8-byte-per-buffer array, each with 1Mi items
            let random_shorts1: Vec<u8> =
                (0..num_values * 2).map(|_| rand::random::<u8>()).collect();
            let random_shorts2: Vec<u8> =
                (0..num_values * 2).map(|_| rand::random::<u8>()).collect();
            let random_shorts3: Vec<u8> =
                (0..num_values * 2).map(|_| rand::random::<u8>()).collect();
            let mut buffers = vec![
                (LanceBuffer::from(random_shorts1), 16),
                (LanceBuffer::from(random_shorts2), 16),
                (LanceBuffer::from(random_shorts3), 16),
            ];
            let buffers = &mut buffers;

            b.iter(move || {
                let buffers = buffers
                    .iter_mut()
                    .map(|(buf, bits_per_value)| (buf.clone(), *bits_per_value))
                    .collect::<Vec<_>>();
                black_box(LanceBuffer::zip_into_one(buffers, num_values as u64).unwrap());
            })
        });
    }
}

#[cfg(target_os = "linux")]
criterion_group!(
    name=benches;
    config = Criterion::default().significance_level(0.1).sample_size(10)
        .with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None)));
    targets = bench_zip);

// Non-linux version does not support pprof.
#[cfg(not(target_os = "linux"))]
criterion_group!(
    name=benches;
    config = Criterion::default().significance_level(0.1).sample_size(10);
    targets = bench_zip);

criterion_main!(benches);