array-format 0.9.0

A block-backed, footer-indexed container for storing multiple n-dimensional arrays in a single file, with a delta/overlay architecture and pluggable compression and storage backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Layered writes (flush) and compaction.
//!
//! Each flush creates a new overlay sidecar that shadows earlier versions of
//! changed chunks. Compact merges all layers into one.
//!
//! ```sh
//! cargo run --example 06_layers_and_compact
//! ```

use array_format::{ArrayFile, FileConfig, InMemoryStorage, NoCompression};
use ndarray::Array;

#[tokio::main]
async fn main() {
    let mut file = ArrayFile::create_memory(FileConfig::new(NoCompression))
        .await
        .unwrap();
    file.define_array::<i32>("values", vec!["x".into()], vec![4], None, None)
        .unwrap();

    // Layer 1: write initial data
    let v1 = Array::from_vec(vec![1i32, 2, 3, 4]).into_dyn();
    file.write_array("values", vec![0], v1.view())
        .await
        .unwrap();
    let ov1 = InMemoryStorage::new();
    file.flush_memory(&ov1).await.unwrap();
    println!("after flush 1: {} layer(s)", file.num_layers()); // 2

    // Layer 2: patch a single element
    let patch = Array::from_vec(vec![99i32]).into_dyn();
    file.write_array("values", vec![2], patch.view())
        .await
        .unwrap();
    let ov2 = InMemoryStorage::new();
    file.flush_memory(&ov2).await.unwrap();
    println!("after flush 2: {} layer(s)", file.num_layers()); // 3

    let before = file
        .read_array::<i32>("values", vec![], vec![])
        .await
        .unwrap();
    println!("before compact = {:?}", before.as_slice().unwrap()); // [1, 2, 99, 4]

    // Compact: merge everything into the base layer
    file.compact().await.unwrap();
    println!("after compact:  {} layer(s)", file.num_layers()); // 1

    let after = file
        .read_array::<i32>("values", vec![], vec![])
        .await
        .unwrap();
    println!("after compact  = {:?}", after.as_slice().unwrap()); // [1, 2, 99, 4]
    assert_eq!(after[[2]], 99);
}