zarrs/array/codec/bytes_to_bytes/
fletcher32.rs

1//! The `fletcher32` bytes to bytes codec (Experimental).
2//!
3//! <div class="warning">
4//! This codec is experimental and may be incompatible with other Zarr V3 implementations.
5//! </div>
6//!
7//! Appends a fletcher32 checksum of the input bytestream.
8//!
9//! This codec requires the `fletcher32` feature, which is disabled by default.
10//!
11//! ### Compatible Implementations
12//! This codec is fully compatible with the `numcodecs.fletcher32` codec in `zarr-python`.
13//!
14//! ### Specification
15//! - <https://github.com/zarr-developers/zarr-extensions/tree/numcodecs/codecs/numcodecs.fletcher32>
16//! - <https://codec.zarrs.dev/bytes_to_bytes/fletcher32>
17//!
18//! ### Codec `name` Aliases (Zarr V3)
19//! - `numcodecs.fletcher32`
20//! - `https://codec.zarrs.dev/bytes_to_bytes/fletcher32`
21//!
22//! ### Codec `id` Aliases (Zarr V2)
23//! - `fletcher32`
24//!
25//! ### Codec `configuration` Example - [`Fletcher32CodecConfiguration`]:
26//! ```rust
27//! # let JSON = r#"
28//! {}
29//! # "#;
30//! # use zarrs::metadata_ext::codec::fletcher32::Fletcher32CodecConfiguration;
31//! # serde_json::from_str::<Fletcher32CodecConfiguration>(JSON).unwrap();
32//! ```
33
34mod fletcher32_codec;
35
36use std::sync::Arc;
37
38pub use fletcher32_codec::Fletcher32Codec;
39use zarrs_metadata::v2::MetadataV2;
40use zarrs_metadata::v3::MetadataV3;
41
42use zarrs_codec::{Codec, CodecPluginV2, CodecPluginV3, CodecTraitsV2, CodecTraitsV3};
43pub use zarrs_metadata_ext::codec::fletcher32::{
44    Fletcher32CodecConfiguration, Fletcher32CodecConfigurationV1,
45};
46use zarrs_plugin::PluginCreateError;
47
48zarrs_plugin::impl_extension_aliases!(Fletcher32Codec,
49    v3: "numcodecs.fletcher32", ["numcodecs.fletcher32", "https://codec.zarrs.dev/bytes_to_bytes/fletcher32"],
50    v2: "fletcher32"
51);
52
53// Register the V3 codec.
54inventory::submit! {
55    CodecPluginV3::new::<Fletcher32Codec>()
56}
57// Register the V2 codec.
58inventory::submit! {
59    CodecPluginV2::new::<Fletcher32Codec>()
60}
61
62impl CodecTraitsV3 for Fletcher32Codec {
63    fn create(metadata: &MetadataV3) -> Result<Codec, PluginCreateError> {
64        let configuration = metadata.to_typed_configuration()?;
65        let codec = Arc::new(Fletcher32Codec::new_with_configuration(&configuration));
66        Ok(Codec::BytesToBytes(codec))
67    }
68}
69
70impl CodecTraitsV2 for Fletcher32Codec {
71    fn create(metadata: &MetadataV2) -> Result<Codec, PluginCreateError> {
72        let configuration: Fletcher32CodecConfiguration = metadata.to_typed_configuration()?;
73        let codec = Arc::new(Fletcher32Codec::new_with_configuration(&configuration));
74        Ok(Codec::BytesToBytes(codec))
75    }
76}
77
78const CHECKSUM_SIZE: usize = size_of::<u32>();
79
80#[cfg(test)]
81mod tests {
82    use std::borrow::Cow;
83    use std::sync::Arc;
84
85    use super::*;
86    use crate::array::BytesRepresentation;
87    use zarrs_codec::{
88        BytesPartialDecoderTraits, BytesToBytesCodecTraits, CodecMetadataOptions, CodecOptions,
89        CodecTraits,
90    };
91    use zarrs_storage::byte_range::ByteRange;
92
93    const JSON1: &str = r"{}";
94
95    #[test]
96    fn codec_fletcher32_configuration_none() {
97        let codec_configuration: Fletcher32CodecConfiguration =
98            serde_json::from_str(r"{}").unwrap();
99        let codec = Fletcher32Codec::new_with_configuration(&codec_configuration);
100        let configuration = codec
101            .configuration_v3(&CodecMetadataOptions::default())
102            .unwrap();
103        assert_eq!(serde_json::to_string(&configuration).unwrap(), r"{}");
104    }
105
106    #[test]
107    fn codec_fletcher32() {
108        let elements: Vec<u8> = (0..6).collect();
109        let bytes = elements;
110        let bytes_representation = BytesRepresentation::FixedSize(bytes.len() as u64);
111
112        let codec_configuration: Fletcher32CodecConfiguration =
113            serde_json::from_str(JSON1).unwrap();
114        let codec = Fletcher32Codec::new_with_configuration(&codec_configuration);
115
116        let encoded = codec
117            .encode(Cow::Borrowed(&bytes), &CodecOptions::default())
118            .unwrap();
119        let decoded = codec
120            .decode(
121                encoded.clone(),
122                &bytes_representation,
123                &CodecOptions::default(),
124            )
125            .unwrap();
126        assert_eq!(bytes, decoded.to_vec());
127
128        // Check that the checksum is correct
129        let checksum: &[u8; 4] = &encoded[encoded.len() - size_of::<u32>()..encoded.len()]
130            .try_into()
131            .unwrap();
132        println!("checksum {checksum:?}");
133        assert_eq!(checksum, &[9, 6, 14, 8]);
134    }
135
136    #[test]
137    fn codec_fletcher32_partial_decode() {
138        let elements: Vec<u8> = (0..32).collect();
139        let bytes = elements;
140        let bytes_representation = BytesRepresentation::FixedSize(bytes.len() as u64);
141
142        let codec_configuration: Fletcher32CodecConfiguration =
143            serde_json::from_str(JSON1).unwrap();
144        let codec = Arc::new(Fletcher32Codec::new_with_configuration(
145            &codec_configuration,
146        ));
147
148        let encoded = codec
149            .encode(Cow::Owned(bytes), &CodecOptions::default())
150            .unwrap();
151        let decoded_regions = [ByteRange::FromStart(3, Some(2))];
152        let input_handle = Arc::new(encoded);
153        let partial_decoder = codec
154            .partial_decoder(
155                input_handle.clone(),
156                &bytes_representation,
157                &CodecOptions::default(),
158            )
159            .unwrap();
160        assert_eq!(partial_decoder.size_held(), input_handle.size_held()); // fletcher32 partial decoder does not hold bytes
161        let decoded_partial_chunk = partial_decoder
162            .partial_decode_many(
163                Box::new(decoded_regions.into_iter()),
164                &CodecOptions::default(),
165            )
166            .unwrap()
167            .unwrap();
168        let answer: &[Vec<u8>] = &[vec![3, 4]];
169        assert_eq!(
170            answer,
171            decoded_partial_chunk
172                .into_iter()
173                .map(|v| v.to_vec())
174                .collect::<Vec<_>>()
175        );
176    }
177
178    #[cfg(feature = "async")]
179    #[tokio::test]
180    async fn codec_fletcher32_async_partial_decode() {
181        let elements: Vec<u8> = (0..32).collect();
182        let bytes = elements;
183        let bytes_representation = BytesRepresentation::FixedSize(bytes.len() as u64);
184
185        let codec_configuration: Fletcher32CodecConfiguration =
186            serde_json::from_str(JSON1).unwrap();
187        let codec = Arc::new(Fletcher32Codec::new_with_configuration(
188            &codec_configuration,
189        ));
190
191        let encoded = codec
192            .encode(Cow::Owned(bytes), &CodecOptions::default())
193            .unwrap();
194        let decoded_regions = [ByteRange::FromStart(3, Some(2))];
195        let input_handle = Arc::new(encoded);
196        let partial_decoder = codec
197            .async_partial_decoder(
198                input_handle,
199                &bytes_representation,
200                &CodecOptions::default(),
201            )
202            .await
203            .unwrap();
204        let decoded_partial_chunk = partial_decoder
205            .partial_decode_many(
206                Box::new(decoded_regions.into_iter()),
207                &CodecOptions::default(),
208            )
209            .await
210            .unwrap()
211            .unwrap();
212        let answer: &[Vec<u8>] = &[vec![3, 4]];
213        assert_eq!(
214            answer,
215            decoded_partial_chunk
216                .into_iter()
217                .map(|v| v.to_vec())
218                .collect::<Vec<_>>()
219        );
220    }
221}
zarrs/array/codec/bytes_to_bytes/fletcher32.rs

zarrs/array/codec/bytes_to_bytes/
fletcher32.rs
