array_format/
lib.rs

1//! # array-format
2//!
3//! A block-backed, footer-indexed container for storing many n-dimensional
4//! arrays in a single file.
5//!
6//! The format uses a **delta/overlay architecture**: each flush produces a
7//! self-describing sidecar file that stacks on top of the base, recording only
8//! the chunks that changed. Reads fall through to older layers for unchanged
9//! chunks, and layers can be merged back into a single file with
10//! [`compact`](ArrayFile::compact).
11//!
12//! ## Features
13//!
14//! - Store many arrays in one object (or a small set of related sidecar files).
15//! - Append arrays and update individual chunks without rewriting the whole file.
16//! - Per-block compression (LZ4, Zstd, or none) recorded in the block table, so
17//!   readers need no configuration to decode a file.
18//! - Chunked or single-chunk layouts with coordinate-addressed reads.
19//! - Logical deletes with periodic compaction to reclaim space.
20//! - Works with any [`object_store`]-compatible backend (local filesystem,
21//!   S3, GCS, Azure) via [`ObjectStoreBackend`](storage::ObjectStoreBackend),
22//!   plus an [`InMemoryStorage`] backend for tests.
23//!
24//! ## Quick start
25//!
26//! ```
27//! use array_format::{ArrayFile, FileConfig, Lz4Codec};
28//! use ndarray::Array;
29//!
30//! # async fn example() -> array_format::Result<()> {
31//! // An in-memory file; use `ArrayFile::create(store, path, config)` for on-disk.
32//! let mut file = ArrayFile::create_memory(FileConfig::new(Lz4Codec)).await?;
33//!
34//! // Define and write a 1-D f32 array.
35//! file.define_array::<f32>("signal", vec!["t".into()], vec![4], None, None)?;
36//! let data = Array::from_vec(vec![1.0f32, 2.0, 3.0, 4.0]).into_dyn();
37//! file.write_array("signal", vec![0], data.view()).await?;
38//!
39//! // Read it back — `vec![], vec![]` means "the whole array".
40//! let out = file.read_array::<f32>("signal", vec![], vec![]).await?;
41//! assert_eq!(out.len(), 4);
42//! # Ok(())
43//! # }
44//! ```
45//!
46//! ## Architecture
47//!
48//! The crate is organized in four layers:
49//!
50//! | Layer | Purpose | Key types |
51//! |-------|---------|-----------|
52//! | 0 — Core | Primitives | [`DType`], [`ChunkAddress`], [`BlockId`], [`Error`] |
53//! | 1 — Metadata | Footer model | [`BlockMeta`], [`Footer`] |
54//! | 2 — Traits | Extension points | [`CompressionCodec`], [`Storage`] |
55//! | 3 — Runtime | Read / write / compact | [`ArrayFile`] |
56//!
57//! The [`CompressionCodec`] and [`Storage`] traits are the extension points:
58//! implement them to plug in custom compression algorithms or storage backends.
59//!
60//! [`ChunkAddress`]: address::ChunkAddress
61//! [`BlockId`]: address::BlockId
62//! [`BlockMeta`]: block::BlockMeta
63//! [`Footer`]: footer::Footer
64//! [`Storage`]: storage::Storage
65//! [`object_store`]: https://docs.rs/object_store
66
67// ── Layer 0: Core types ─────────────────────────────────────────────
68pub mod address;
69pub mod delta;
70pub mod dtype;
71pub mod error;
72
73// ── Layer 1: Metadata ───────────────────────────────────────────────
74pub mod block;
75pub mod footer;
76pub mod layout;
77
78// ── Layer 2: Extension traits ───────────────────────────────────────
79pub mod codec;
80pub mod storage;
81
82// ── Layer 3: Runtime ────────────────────────────────────────────────
83pub mod array;
84pub mod file;
85pub mod stats;
86
87pub mod ndarray_ext;
88pub mod timestamp;
89
90// ── Public re-exports ───────────────────────────────────────────────
91pub use array::ArrayElement;
92pub use codec::{CompressionCodec, Lz4Codec, NoCompression, ZstdCodec};
93pub use delta::DeltaCache;
94pub use dtype::DType;
95pub use error::{Error, Result};
96pub use file::{
97    ArrayFile, DEFAULT_BLOCK_TARGET_SIZE, DEFAULT_CACHE_CAPACITY, DEFAULT_IO_CACHE_CAPACITY,
98    FileConfig, MergedArrayMeta,
99};
100pub use layout::{AttributeValue, FillValue};
101pub use stats::{ArrayStats, StatValue, StatsFile};
102pub use storage::InMemoryStorage;
103pub use timestamp::TimestampNs;
array_format/lib.rs

array_format/
lib.rs
