array-format 0.11.0

A block-backed, footer-indexed container for storing multiple n-dimensional arrays in a single file, with a delta/overlay architecture and pluggable compression and storage backends.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! File-level metadata via a placeholder array.
//!
//! Attributes attach to arrays, so to store metadata about the *file* as a
//! whole (title, provenance, schema version, …) without any real data, define a
//! scalar placeholder array with an empty shape and hang the attributes on it.
//! No data is ever written to it, so `flush` skips stats and it costs almost
//! nothing.
//!
//! ```sh
//! cargo run --example 10_file_metadata
//! ```

use array_format::{ArrayFile, AttributeValue, FileConfig, NoCompression};

/// Reserved array name used to hold file-level metadata.
const FILE_META: &str = "__file__";

#[tokio::main]
async fn main() {
    let mut file = ArrayFile::create_memory(FileConfig::new(NoCompression))
        .await
        .unwrap();

    // A scalar placeholder: empty shape means no dimensions and no chunks, so
    // nothing is ever stored for it beyond its attributes.
    file.define_array::<u8>(FILE_META, vec![], vec![], None, None)
        .unwrap();

    file.set_attribute(
        FILE_META,
        "title",
        AttributeValue::String("My Dataset".into()),
    )
    .unwrap();
    file.set_attribute(FILE_META, "schema_version", AttributeValue::Int32(3))
        .unwrap();
    file.set_attribute(
        FILE_META,
        "authors",
        AttributeValue::StringList(vec!["alice".into(), "bob".into()]),
    )
    .unwrap();

    // Persist — a metadata-only file is completely valid.
    file.flush().await.unwrap();

    // Read the file-level metadata back.
    let title = file.get_attribute(FILE_META, "title").unwrap();
    let version = file.get_attribute(FILE_META, "schema_version").unwrap();
    let authors = file.get_attribute(FILE_META, "authors").unwrap();
    println!("title          = {title:?}");
    println!("schema_version = {version:?}");
    println!("authors        = {authors:?}");

    // The placeholder shows up in list_arrays like any array; filter it out to
    // present only real data arrays to users.
    let data_arrays: Vec<String> = file
        .list_arrays()
        .into_iter()
        .map(|m| m.name)
        .filter(|name| name != FILE_META)
        .collect();
    println!("data arrays    = {data_arrays:?}"); // [] — metadata only

    assert_eq!(title, Some(&AttributeValue::String("My Dataset".into())));
    assert!(data_arrays.is_empty());
}