zarrs 0.23.9

A library for the Zarr storage format for multidimensional arrays and metadata
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! The `optional` array to bytes codec (Experimental).
//!
//! Encodes optional (nullable) data by separating the mask and data encoding.
//! This codec is designed for the `Optional` data type, which represents nullable values.
//!
//! <div class="warning">
//! This codec is experimental and may be incompatible with other Zarr V3 implementations.
//! </div>
//!
//! ### Compatible Implementations
//! None
//!
//! ### Specification
//! - <https://codec.zarrs.dev/array_to_bytes/optional>
//!
//! The `optional` codec separates encoding of the mask (bool array)
//! and the data (flattened bytes, excluding missing elements).
//! This allows for efficient storage when many elements are missing, and enables
//! independent compression strategies for mask and data.
//!
//! Missing elements are not encoded at all in the flattened data, saving space.
//! The mask and data are encoded through independent codec chains specified
//! by `mask_codecs` and `data_codecs` configuration parameters.
//!
//! The encoded format is:
//! - 8 bytes: mask length (u64 little-endian)
//! - 8 bytes: data length (u64 little-endian)
//! - N bytes: encoded mask
//! - M bytes: encoded data (only valid elements)
//!
//! #### Fill Value Encoding
//!
//! The `"fill_value"` metadata for optional types is encoded as follows:
//! - A `null` fill value (`None`) is represented as `null`.
//! - A non-null fill value (`Some(value)`) is represented as a single-element array containing the inner fill value.
//!
//! For nested optional types, this representation is applied recursively.
//!
//! | Rust Type | Fill Value | `"fill_value"` JSON | Fill Value Bytes |
//! |------|-------|------|-------|
//! | `Option<UInt8>` | `None` | `null` | `[0]` |
//! | `Option<UInt8>` | `Some(42)` | `[42]` | `[42, 1]` |
//! | `Option<Option<UInt8>>` | `None` | `null` | `[0]` |
//! | `Option<Option<UInt8>>` | `Some(None)` | `[null]` | `[0, 1]` |
//! | `Option<Option<UInt8>>` | `Some(Some(42))` | `[[42]]` | `[42, 1, 1]` |
//!
//! The fill value bytes are the in-memory representation in `zarrs`.
//! This is an implementation detail.
//!
//! ### Codec `name` Aliases (Zarr V3)
//! - `optional`
//! - `https://codec.zarrs.dev/array_to_bytes/optional`
//!
//! ### Codec `id` Aliases (Zarr V2)
//! None
//!
//! ### Codec `configuration` Example - [`OptionalCodecConfiguration`]:
//! ```rust
//! # let JSON = r#"
//! {
//!     "mask_codecs": [
//!         {
//!             "name": "packbits",
//!             "configuration": {}
//!         },
//!         {
//!             "name": "gzip",
//!             "configuration": {"level": 5}
//!         }
//!     ],
//!     "data_codecs": [
//!         {
//!             "name": "bytes",
//!             "configuration": {"endian": "little"}
//!         },
//!         {
//!             "name": "gzip",
//!             "configuration": {"level": 5}
//!         }
//!     ]
//! }
//! # "#;
//! # use zarrs::metadata_ext::codec::optional::OptionalCodecConfiguration;
//! # let configuration: OptionalCodecConfiguration = serde_json::from_str(JSON).unwrap();
//! ```

mod optional_codec;

use std::sync::Arc;

pub use optional_codec::OptionalCodec;
use zarrs_metadata::v3::MetadataV3;

use zarrs_codec::{Codec, CodecPluginV3, CodecTraitsV3};
pub use zarrs_metadata_ext::codec::optional::{
    OptionalCodecConfiguration, OptionalCodecConfigurationV1,
};
use zarrs_plugin::PluginCreateError;

zarrs_plugin::impl_extension_aliases!(OptionalCodec,
  v3: "zarrs.optional", []
);

// Register the V3 codec.
inventory::submit! {
    CodecPluginV3::new::<OptionalCodec>()
}

impl CodecTraitsV3 for OptionalCodec {
    fn create(metadata: &MetadataV3) -> Result<Codec, PluginCreateError> {
        crate::warn_experimental_extension(metadata.name(), "codec");
        let configuration: OptionalCodecConfiguration = metadata.to_typed_configuration()?;
        let codec = Arc::new(OptionalCodec::new_with_configuration(&configuration)?);
        Ok(Codec::ArrayToBytes(codec))
    }
}