oxigeo_zarr/lib.rs
1//! OxiGeo Zarr Driver - Pure Rust Zarr v2/v3 Support
2//!
3//! This crate provides a pure Rust implementation of the Zarr storage specification
4//! for chunked, compressed, N-dimensional arrays. It supports both Zarr v2 and v3
5//! specifications with various storage backends and compression codecs.
6//!
7//! # Features
8//!
9//! - **Zarr Versions**: Zarr v2 and v3 arrays (chunk read/write, codec and
10//! filter pipelines, v3 sharding)
11//! - **Storage Backends**: Filesystem, S3, HTTP, and in-memory storage
12//! (with byte-range reads used for cloud-efficient shard access)
13//! - **Compression**: Zstd, Gzip, LZ4 codecs (all Pure Rust via OxiARC),
14//! plus the v3 `crc32c` checksum codec
15//! - **Filters** (opt-in cargo features): `shuffle`, `delta`, `scale-offset`
16//! - **Async I/O**: Async support for cloud storage backends
17//! - **Parallel**: Parallel chunk reading and writing
18//! - **Caching**: LRU caching for chunks
19//!
20//! # Zarr Format
21//!
22//! Zarr is a format for the storage of chunked, compressed, N-dimensional arrays.
23//! It was designed for use in parallel computing and is widely used in scientific
24//! computing, particularly for earth observation and climate data.
25//!
26//! ## Key Concepts
27//!
28//! - **Array**: N-dimensional data structure
29//! - **Chunk**: Fixed-size sub-array for storage
30//! - **Codec**: Compression/encoding method
31//! - **Filter**: Data transformation before/after codec
32//! - **Group**: Hierarchical organization of arrays
33//! - **Attributes**: Metadata attached to arrays/groups
34//!
35//! # Example - Reading a Zarr v2 Array
36//!
37//! ```no_run
38//! # #[cfg(all(feature = "v2", feature = "filesystem"))]
39//! # fn demo() -> oxigeo_zarr::Result<()> {
40//! use oxigeo_zarr::{ZarrReaderV2, FilesystemStore};
41//!
42//! // Open a Zarr v2 array stored under the "temperature" path.
43//! let store = FilesystemStore::open("data.zarr")?;
44//! let reader = ZarrReaderV2::open(store, "temperature")?;
45//!
46//! println!("Shape: {:?}", reader.shape());
47//! println!("Chunks: {:?}", reader.chunks());
48//! println!("Data type: {}", reader.dtype());
49//!
50//! // Read one chunk (decoded element bytes in the array's stored order).
51//! let chunk_data = reader.read_chunk(&[0, 0])?;
52//! # let _ = chunk_data;
53//! # Ok(())
54//! # }
55//! ```
56//!
57//! # Example - Writing a Zarr v2 Array
58//!
59//! ```no_run
60//! # #[cfg(all(feature = "v2", feature = "filesystem", feature = "zstd"))]
61//! # fn demo() -> oxigeo_zarr::Result<()> {
62//! use oxigeo_zarr::{ZarrWriterV2, FilesystemStore};
63//! use oxigeo_zarr::codecs::CompressorConfig;
64//! use oxigeo_zarr::metadata::v2::ArrayMetadataV2;
65//!
66//! let store = FilesystemStore::create("output.zarr")?;
67//! let metadata = ArrayMetadataV2::new(vec![100, 200], vec![10, 20], "<f4")
68//! .with_compressor(CompressorConfig::Zstd { level: 3 });
69//!
70//! let mut writer = ZarrWriterV2::create(store, "temperature", metadata)?;
71//!
72//! // Write one 10x20 float32 chunk.
73//! let chunk_data = vec![0u8; 10 * 20 * 4];
74//! writer.write_chunk(&[0, 0], &chunk_data)?;
75//! writer.finalize()?;
76//! # Ok(())
77//! # }
78//! ```
79//!
80//! # Storage Backends
81//!
82//! ## Filesystem
83//!
84//! ```ignore
85//! use oxigeo_zarr::FilesystemStore;
86//!
87//! let store = FilesystemStore::open("data.zarr")?;
88//! ```
89//!
90//! ## S3
91//!
92//! ```ignore
93//! use oxigeo_zarr::S3Store;
94//!
95//! let store = S3Store::new("bucket-name", "prefix/data.zarr").await?;
96//! ```
97//!
98//! ## HTTP
99//!
100//! ```ignore
101//! use oxigeo_zarr::HttpStore;
102//!
103//! let store = HttpStore::new("https://example.com/data.zarr")?;
104//! ```
105
106#![cfg_attr(not(feature = "std"), no_std)]
107#![warn(clippy::all)]
108// Pedantic disabled to reduce noise - default clippy::all is sufficient
109// #![warn(clippy::pedantic)]
110#![deny(clippy::unwrap_used)]
111#![allow(clippy::module_name_repetitions)]
112#![allow(clippy::similar_names)]
113#![allow(clippy::too_many_arguments)]
114// Allow dead code for internal structures
115#![allow(dead_code)]
116// Allow partial documentation during development
117#![allow(missing_docs)]
118// Allow manual div_ceil for chunk calculations
119#![allow(clippy::manual_div_ceil)]
120// Allow expect() for internal state invariants
121#![allow(clippy::expect_used)]
122// Allow complex types for zarr data structures
123#![allow(clippy::type_complexity)]
124// Allow collapsible match for zarr format handling
125#![allow(clippy::collapsible_match)]
126// Allow manual strip for path parsing
127#![allow(clippy::manual_strip)]
128// Allow vec push after creation for chunk building
129#![allow(clippy::vec_init_then_push)]
130// Allow should_implement_trait for builder patterns
131#![allow(clippy::should_implement_trait)]
132// Allow doc list item overindentation in complex nested lists
133#![allow(clippy::doc_overindented_list_items)]
134
135#[cfg(feature = "alloc")]
136extern crate alloc;
137
138#[cfg(feature = "std")]
139extern crate std;
140
141pub mod chunk;
142pub mod codecs;
143pub mod consolidation;
144pub mod dimension;
145pub mod error;
146pub mod filters;
147pub mod metadata;
148pub mod reader;
149pub mod sharding;
150pub mod storage;
151pub mod transformers;
152pub mod writer;
153
154// Re-export commonly used types
155pub use chunk::{ChunkCoord, ChunkGrid, ChunkIndex};
156pub use consolidation::{ConsolidatedMetadata, ConsolidatedStore, consolidate_metadata};
157pub use dimension::{Dimension, DimensionSeparator, Shape};
158pub use error::{Result, ZarrError};
159pub use reader::ZarrReader;
160#[cfg(feature = "v2")]
161pub use reader::ZarrReaderV2;
162#[cfg(feature = "v3")]
163pub use reader::v3::ZarrV3Reader;
164pub use storage::{Store, StoreKey};
165pub use writer::ZarrWriter;
166#[cfg(feature = "v2")]
167pub use writer::ZarrWriterV2;
168#[cfg(feature = "v3")]
169pub use writer::v3::ZarrV3Writer;
170
171#[cfg(feature = "filesystem")]
172pub use storage::filesystem::FilesystemStore;
173
174#[cfg(feature = "s3")]
175pub use storage::s3::S3Storage;
176
177#[cfg(feature = "http")]
178pub use storage::http::HttpStorage;
179
180#[cfg(feature = "memory")]
181pub use storage::memory::MemoryStore;
182
183#[cfg(feature = "cache")]
184pub use storage::cache::CachingStorage;
185
186/// Crate version
187pub const VERSION: &str = env!("CARGO_PKG_VERSION");
188
189/// Crate name
190pub const NAME: &str = env!("CARGO_PKG_NAME");
191
192/// Zarr specification version 2
193pub const ZARR_VERSION_2: u8 = 2;
194
195/// Zarr specification version 3
196pub const ZARR_VERSION_3: u8 = 3;
197
198#[cfg(test)]
199mod tests {
200 use super::*;
201
202 #[test]
203 fn test_version() {
204 assert!(!VERSION.is_empty());
205 assert_eq!(NAME, "oxigeo-zarr");
206 }
207
208 #[test]
209 fn test_zarr_versions() {
210 assert_eq!(ZARR_VERSION_2, 2);
211 assert_eq!(ZARR_VERSION_3, 3);
212 }
213}