pub struct FillValue(/* private fields */);Expand description
A fill value.
Provides an element value to use for uninitialised portions of the Zarr array.
For optional data types, the fill value includes an extra trailing byte:
0x00suffix indicates null/missing0x01suffix indicates non-null (inner bytes precede the suffix)
For nested optional types, each nesting level adds its own suffix byte.
Implementations§
Source§impl FillValue
impl FillValue
Sourcepub fn new(bytes: Vec<u8>) -> FillValue
pub fn new(bytes: Vec<u8>) -> FillValue
Create a new fill value composed of bytes.
Examples found in repository?
examples/custom_data_type_fixed_size.rs (line 189)
181 fn fill_value(
182 &self,
183 fill_value_metadata: &FillValueMetadata,
184 _version: ZarrVersion,
185 ) -> Result<FillValue, DataTypeFillValueMetadataError> {
186 let element_metadata: CustomDataTypeFixedSizeMetadata = fill_value_metadata
187 .as_custom()
188 .ok_or(DataTypeFillValueMetadataError)?;
189 Ok(FillValue::new(element_metadata.to_ne_bytes().to_vec()))
190 }
191
192 fn metadata_fill_value(
193 &self,
194 fill_value: &FillValue,
195 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
196 let element = CustomDataTypeFixedSizeMetadata::from_ne_bytes(
197 fill_value
198 .as_ne_bytes()
199 .try_into()
200 .map_err(|_| DataTypeFillValueError)?,
201 );
202 Ok(FillValueMetadata::from(element))
203 }
204
205 fn size(&self) -> zarrs::array::DataTypeSize {
206 DataTypeSize::Fixed(size_of::<CustomDataTypeFixedSizeBytes>())
207 }
208
209 fn as_any(&self) -> &dyn std::any::Any {
210 self
211 }
212}
213
214/// Add support for the `bytes` codec. This must be implemented for fixed-size data types, even if they just pass-through the data type.
215impl BytesDataTypeTraits for CustomDataTypeFixedSize {
216 fn encode<'a>(
217 &self,
218 bytes: std::borrow::Cow<'a, [u8]>,
219 endianness: Option<zarrs_metadata::Endianness>,
220 ) -> Result<std::borrow::Cow<'a, [u8]>, BytesCodecEndiannessMissingError> {
221 if let Some(endianness) = endianness {
222 if endianness != Endianness::native() {
223 let mut bytes = bytes.into_owned();
224 for bytes in bytes
225 .as_chunks_mut::<{ size_of::<CustomDataTypeFixedSizeBytes>() }>()
226 .0
227 {
228 let value = CustomDataTypeFixedSizeElement::from_ne_bytes(bytes);
229 if endianness == Endianness::Little {
230 *bytes = value.to_le_bytes();
231 } else {
232 *bytes = value.to_be_bytes();
233 }
234 }
235 Ok(Cow::Owned(bytes))
236 } else {
237 Ok(bytes)
238 }
239 } else {
240 Err(BytesCodecEndiannessMissingError)
241 }
242 }
243
244 fn decode<'a>(
245 &self,
246 bytes: std::borrow::Cow<'a, [u8]>,
247 endianness: Option<zarrs_metadata::Endianness>,
248 ) -> Result<std::borrow::Cow<'a, [u8]>, BytesCodecEndiannessMissingError> {
249 if let Some(endianness) = endianness {
250 if endianness != Endianness::native() {
251 let mut bytes = bytes.into_owned();
252 for bytes in bytes
253 .as_chunks_mut::<{ size_of::<u64>() + size_of::<f32>() }>()
254 .0
255 {
256 let value = if endianness == Endianness::Little {
257 CustomDataTypeFixedSizeElement::from_le_bytes(bytes)
258 } else {
259 CustomDataTypeFixedSizeElement::from_be_bytes(bytes)
260 };
261 *bytes = value.to_ne_bytes();
262 }
263 Ok(Cow::Owned(bytes))
264 } else {
265 Ok(bytes)
266 }
267 } else {
268 Err(BytesCodecEndiannessMissingError)
269 }
270 }
271}
272
273// Register codec support
274zarrs_data_type::register_data_type_extension_codec!(
275 CustomDataTypeFixedSize,
276 zarrs_data_type::codec_traits::bytes::BytesDataTypePlugin,
277 zarrs_data_type::codec_traits::bytes::BytesDataTypeTraits
278);
279
280fn main() {
281 let store = std::sync::Arc::new(MemoryStore::default());
282 let array_path = "/array";
283 let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
284 let array = ArrayBuilder::new(
285 vec![4, 1], // array shape
286 vec![2, 1], // regular chunk shape
287 Arc::new(CustomDataTypeFixedSize),
288 FillValue::new(fill_value.to_ne_bytes().to_vec()),
289 )
290 .array_to_array_codecs(vec![
291 #[cfg(feature = "transpose")]
292 Arc::new(zarrs::array::codec::TransposeCodec::new(
293 zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
294 )),
295 ])
296 .bytes_to_bytes_codecs(vec![
297 #[cfg(feature = "gzip")]
298 Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
299 #[cfg(feature = "crc32c")]
300 Arc::new(zarrs::array::codec::Crc32cCodec::new()),
301 ])
302 // .storage_transformers(vec![].into())
303 .build(store, array_path)
304 .unwrap();
305 println!("{}", array.metadata().to_string_pretty());
306
307 let data = [
308 CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
309 CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
310 ];
311 array.store_chunk(&[0, 0], &data).unwrap();
312
313 let data: Vec<CustomDataTypeFixedSizeElement> =
314 array.retrieve_array_subset(&array.subset_all()).unwrap();
315
316 assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
317 assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
318 assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
319 assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
320
321 println!("{data:#?}");
322}More examples
examples/custom_data_type_float8_e3m4.rs (line 63)
54 fn fill_value(
55 &self,
56 fill_value_metadata: &FillValueMetadata,
57 _version: ZarrVersion,
58 ) -> Result<FillValue, DataTypeFillValueMetadataError> {
59 let element_metadata: f32 = fill_value_metadata
60 .as_f32()
61 .ok_or(DataTypeFillValueMetadataError)?;
62 let element = CustomDataTypeFloat8e3m4Element::from(element_metadata);
63 Ok(FillValue::new(element.into_ne_bytes().to_vec()))
64 }
65
66 fn metadata_fill_value(
67 &self,
68 fill_value: &FillValue,
69 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
70 let element = CustomDataTypeFloat8e3m4Element::from_ne_bytes(
71 fill_value
72 .as_ne_bytes()
73 .try_into()
74 .map_err(|_| DataTypeFillValueError)?,
75 );
76 Ok(FillValueMetadata::from(element.into_f32()))
77 }
78
79 fn size(&self) -> zarrs::array::DataTypeSize {
80 DataTypeSize::Fixed(1)
81 }
82
83 fn as_any(&self) -> &dyn std::any::Any {
84 self
85 }
86}
87
88// Add support for the `bytes` codec using the helper macro (component size 1 = passthrough).
89zarrs_data_type::codec_traits::impl_bytes_data_type_traits!(CustomDataTypeFloat8e3m4, 1);
90
91// FIXME: Not tested for correctness. Prefer a supporting crate.
92fn float32_to_float8_e3m4(val: f32) -> u8 {
93 let bits = val.to_bits();
94 let sign = ((bits >> 24) & 0x80) as u8;
95 let unbiased_exponent = ((bits >> 23) & 0xFF) as i16 - 127;
96 let mantissa = ((bits >> 19) & 0x0F) as u8;
97
98 let biased_to_exponent = unbiased_exponent + 3;
99
100 if biased_to_exponent < 0 {
101 // Flush denormals and underflowing values to zero
102 sign
103 } else if biased_to_exponent > 7 {
104 // Overflow: return ±Infinity
105 sign | 0b01110000
106 } else {
107 sign | ((biased_to_exponent as u8) << 4) | mantissa
108 }
109}
110
111// FIXME: Not tested for correctness. Prefer a supporting crate.
112fn float8_e3m4_to_float32(val: u8) -> f32 {
113 let sign = (val & 0b10000000) as u32;
114 let biased_exponent = ((val >> 4) & 0b111) as i16;
115 let mantissa = (val & 0b1111) as u32;
116
117 let f32_bits = if biased_exponent == 0 {
118 // Subnormal
119 return f32::from_bits(sign << 24 | mantissa << 19);
120 } else if biased_exponent == 7 {
121 // Infinity or NaN
122 if mantissa == 0 {
123 (sign << 24) | 0x7F800000 // ±Infinity
124 } else {
125 (sign << 24) | 0x7F800000 | (mantissa << 19) // NaN
126 }
127 } else {
128 let unbiased_exponent = biased_exponent - 3;
129 let biased_to_exponent = (unbiased_exponent + 127) as u32;
130 let new_mantissa = mantissa << 19;
131 (sign << 24) | (biased_to_exponent << 23) | new_mantissa
132 };
133 f32::from_bits(f32_bits)
134}
135
136impl From<f32> for CustomDataTypeFloat8e3m4Element {
137 fn from(value: f32) -> Self {
138 Self(float32_to_float8_e3m4(value))
139 }
140}
141
142impl CustomDataTypeFloat8e3m4Element {
143 fn into_ne_bytes(self) -> [u8; 1] {
144 [self.0]
145 }
146
147 fn from_ne_bytes(bytes: [u8; 1]) -> Self {
148 Self(bytes[0])
149 }
150
151 fn into_f32(self) -> f32 {
152 float8_e3m4_to_float32(self.0)
153 }
154}
155
156/// This defines how an in-memory CustomDataTypeFloat8e3m4Element is converted into ArrayBytes before encoding via the codec pipeline.
157impl Element for CustomDataTypeFloat8e3m4Element {
158 fn validate_data_type(data_type: &DataType) -> Result<(), ElementError> {
159 data_type
160 .is::<CustomDataTypeFloat8e3m4>()
161 .then_some(())
162 .ok_or(ElementError::IncompatibleElementType)
163 }
164
165 fn to_array_bytes<'a>(
166 data_type: &DataType,
167 elements: &'a [Self],
168 ) -> Result<zarrs::array::ArrayBytes<'a>, ElementError> {
169 Self::validate_data_type(data_type)?;
170 let mut bytes: Vec<u8> = Vec::with_capacity(elements.len());
171 for element in elements {
172 bytes.push(element.0);
173 }
174 Ok(ArrayBytes::Fixed(Cow::Owned(bytes)))
175 }
176
177 fn into_array_bytes(
178 data_type: &DataType,
179 elements: Vec<Self>,
180 ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
181 Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
182 }
183}
184
185/// This defines how ArrayBytes are converted into a CustomDataTypeFloat8e3m4Element after decoding via the codec pipeline.
186impl ElementOwned for CustomDataTypeFloat8e3m4Element {
187 fn from_array_bytes(
188 data_type: &DataType,
189 bytes: ArrayBytes<'_>,
190 ) -> Result<Vec<Self>, ElementError> {
191 Self::validate_data_type(data_type)?;
192 let bytes = bytes.into_fixed()?;
193 let bytes_len = bytes.len();
194 let mut elements = Vec::with_capacity(bytes_len);
195 // NOTE: Could memcpy here
196 for byte in bytes.iter() {
197 elements.push(CustomDataTypeFloat8e3m4Element(*byte))
198 }
199 Ok(elements)
200 }
201}
202
203fn main() {
204 let store = std::sync::Arc::new(MemoryStore::default());
205 let array_path = "/array";
206 let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
207 let array = ArrayBuilder::new(
208 vec![6, 1], // array shape
209 vec![5, 1], // regular chunk shape
210 Arc::new(CustomDataTypeFloat8e3m4),
211 FillValue::new(fill_value.into_ne_bytes().to_vec()),
212 )
213 .array_to_array_codecs(vec![
214 #[cfg(feature = "transpose")]
215 Arc::new(zarrs::array::codec::TransposeCodec::new(
216 zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
217 )),
218 ])
219 .bytes_to_bytes_codecs(vec![
220 #[cfg(feature = "gzip")]
221 Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
222 #[cfg(feature = "crc32c")]
223 Arc::new(zarrs::array::codec::Crc32cCodec::new()),
224 ])
225 // .storage_transformers(vec![].into())
226 .build(store, array_path)
227 .unwrap();
228 println!("{}", array.metadata().to_string_pretty());
229
230 let data = [
231 CustomDataTypeFloat8e3m4Element::from(2.34),
232 CustomDataTypeFloat8e3m4Element::from(3.45),
233 CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
234 CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
235 CustomDataTypeFloat8e3m4Element::from(f32::NAN),
236 ];
237 array.store_chunk(&[0, 0], &data).unwrap();
238
239 let data: Vec<CustomDataTypeFloat8e3m4Element> =
240 array.retrieve_array_subset(&array.subset_all()).unwrap();
241
242 for f in &data {
243 println!(
244 "float8_e3m4: {:08b} f32: {}",
245 f.into_ne_bytes()[0],
246 f.into_f32()
247 );
248 }
249
250 assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
251 assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
252 assert_eq!(
253 data[2],
254 CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
255 );
256 assert_eq!(
257 data[3],
258 CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
259 );
260 assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
261 assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
262}examples/custom_data_type_uint4.rs (line 65)
55 fn fill_value(
56 &self,
57 fill_value_metadata: &FillValueMetadata,
58 _version: ZarrVersion,
59 ) -> Result<FillValue, DataTypeFillValueMetadataError> {
60 let element_metadata: u64 = fill_value_metadata
61 .as_u64()
62 .ok_or(DataTypeFillValueMetadataError)?;
63 let element = CustomDataTypeUInt4Element::try_from(element_metadata)
64 .map_err(|_| DataTypeFillValueMetadataError)?;
65 Ok(FillValue::new(element.into_ne_bytes().to_vec()))
66 }
67
68 fn metadata_fill_value(
69 &self,
70 fill_value: &FillValue,
71 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
72 let element = CustomDataTypeUInt4Element::from_ne_bytes(
73 fill_value
74 .as_ne_bytes()
75 .try_into()
76 .map_err(|_| DataTypeFillValueError)?,
77 );
78 Ok(FillValueMetadata::from(element.into_u8()))
79 }
80
81 fn size(&self) -> zarrs::array::DataTypeSize {
82 DataTypeSize::Fixed(1)
83 }
84
85 fn as_any(&self) -> &dyn Any {
86 self
87 }
88
89 /// Allow u8 as compatible element type.
90 fn compatible_element_types(&self) -> &'static [std::any::TypeId] {
91 const TYPES: [std::any::TypeId; 1] = [std::any::TypeId::of::<u8>()];
92 &TYPES
93 }
94}
95
96// Add support for the `bytes` codec using the helper macro (component size 1 = passthrough).
97zarrs_data_type::codec_traits::impl_bytes_data_type_traits!(CustomDataTypeUInt4, 1);
98
99/// Add support for the `packbits` codec.
100impl PackBitsDataTypeTraits for CustomDataTypeUInt4 {
101 fn component_size_bits(&self) -> u64 {
102 4
103 }
104
105 fn num_components(&self) -> u64 {
106 1
107 }
108
109 fn sign_extension(&self) -> bool {
110 false
111 }
112}
113
114// Register packbits codec support
115zarrs_data_type::register_data_type_extension_codec!(
116 CustomDataTypeUInt4,
117 zarrs_data_type::codec_traits::packbits::PackBitsDataTypePlugin,
118 zarrs_data_type::codec_traits::packbits::PackBitsDataTypeTraits
119);
120
121impl TryFrom<u64> for CustomDataTypeUInt4Element {
122 type Error = u64;
123
124 fn try_from(value: u64) -> Result<Self, Self::Error> {
125 if value < 16 {
126 Ok(Self(value as u8))
127 } else {
128 Err(value)
129 }
130 }
131}
132
133impl CustomDataTypeUInt4Element {
134 fn into_ne_bytes(self) -> [u8; 1] {
135 [self.0]
136 }
137
138 fn from_ne_bytes(bytes: [u8; 1]) -> Self {
139 Self(bytes[0])
140 }
141
142 fn into_u8(self) -> u8 {
143 self.0
144 }
145}
146
147/// This defines how an in-memory CustomDataTypeUInt4Element is converted into ArrayBytes before encoding via the codec pipeline.
148impl Element for CustomDataTypeUInt4Element {
149 fn validate_data_type(data_type: &DataType) -> Result<(), ElementError> {
150 // Check if the data type matches our custom data type
151 data_type
152 .is::<CustomDataTypeUInt4>()
153 .then_some(())
154 .ok_or(ElementError::IncompatibleElementType)
155 }
156
157 fn to_array_bytes<'a>(
158 data_type: &DataType,
159 elements: &'a [Self],
160 ) -> Result<zarrs::array::ArrayBytes<'a>, ElementError> {
161 Self::validate_data_type(data_type)?;
162 let mut bytes: Vec<u8> = Vec::with_capacity(std::mem::size_of_val(elements));
163 for element in elements {
164 bytes.push(element.0);
165 }
166 Ok(ArrayBytes::Fixed(Cow::Owned(bytes)))
167 }
168
169 fn into_array_bytes(
170 data_type: &DataType,
171 elements: Vec<Self>,
172 ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
173 Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
174 }
175}
176
177/// This defines how ArrayBytes are converted into a CustomDataTypeUInt4Element after decoding via the codec pipeline.
178impl ElementOwned for CustomDataTypeUInt4Element {
179 fn from_array_bytes(
180 data_type: &DataType,
181 bytes: ArrayBytes<'_>,
182 ) -> Result<Vec<Self>, ElementError> {
183 Self::validate_data_type(data_type)?;
184 let bytes = bytes.into_fixed()?;
185 let bytes_len = bytes.len();
186 let mut elements = Vec::with_capacity(bytes_len / size_of::<CustomDataTypeUInt4Element>());
187 for byte in bytes.iter() {
188 elements.push(CustomDataTypeUInt4Element(*byte))
189 }
190 Ok(elements)
191 }
192}
193
194fn main() {
195 let store = std::sync::Arc::new(MemoryStore::default());
196 let array_path = "/array";
197 let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
198 let array = ArrayBuilder::new(
199 vec![6, 1], // array shape
200 vec![5, 1], // regular chunk shape
201 Arc::new(CustomDataTypeUInt4),
202 FillValue::new(fill_value.into_ne_bytes().to_vec()),
203 )
204 .array_to_array_codecs(vec![
205 #[cfg(feature = "transpose")]
206 Arc::new(zarrs::array::codec::TransposeCodec::new(
207 zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
208 )),
209 ])
210 .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
211 .bytes_to_bytes_codecs(vec![
212 #[cfg(feature = "gzip")]
213 Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
214 #[cfg(feature = "crc32c")]
215 Arc::new(zarrs::array::codec::Crc32cCodec::new()),
216 ])
217 // .storage_transformers(vec![].into())
218 .build(store, array_path)
219 .unwrap();
220 println!("{}", array.metadata().to_string_pretty());
221
222 let data = [
223 CustomDataTypeUInt4Element::try_from(1).unwrap(),
224 CustomDataTypeUInt4Element::try_from(2).unwrap(),
225 CustomDataTypeUInt4Element::try_from(3).unwrap(),
226 CustomDataTypeUInt4Element::try_from(4).unwrap(),
227 CustomDataTypeUInt4Element::try_from(5).unwrap(),
228 ];
229 array.store_chunk(&[0, 0], &data).unwrap();
230
231 let data: Vec<CustomDataTypeUInt4Element> =
232 array.retrieve_array_subset(&array.subset_all()).unwrap();
233
234 for f in &data {
235 println!("uint4: {:08b} u8: {}", f.into_u8(), f.into_u8());
236 }
237
238 assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
239 assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
240 assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
241 assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
242 assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
243 assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());
244
245 let data: Vec<CustomDataTypeUInt4Element> = array.retrieve_array_subset(&[1..3, 0..1]).unwrap();
246 assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
247 assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
248}examples/custom_data_type_uint12.rs (line 63)
53 fn fill_value(
54 &self,
55 fill_value_metadata: &FillValueMetadata,
56 _version: ZarrVersion,
57 ) -> Result<FillValue, DataTypeFillValueMetadataError> {
58 let element_metadata: u64 = fill_value_metadata
59 .as_u64()
60 .ok_or(DataTypeFillValueMetadataError)?;
61 let element = CustomDataTypeUInt12Element::try_from(element_metadata)
62 .map_err(|_| DataTypeFillValueMetadataError)?;
63 Ok(FillValue::new(element.into_le_bytes().to_vec()))
64 }
65
66 fn metadata_fill_value(
67 &self,
68 fill_value: &FillValue,
69 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
70 let element = CustomDataTypeUInt12Element::from_le_bytes(
71 fill_value
72 .as_ne_bytes()
73 .try_into()
74 .map_err(|_| DataTypeFillValueError)?,
75 );
76 Ok(FillValueMetadata::from(element.into_u16()))
77 }
78
79 fn size(&self) -> zarrs::array::DataTypeSize {
80 DataTypeSize::Fixed(2)
81 }
82
83 fn as_any(&self) -> &dyn std::any::Any {
84 self
85 }
86
87 /// Allow u16 as compatible element type.
88 fn compatible_element_types(&self) -> &'static [std::any::TypeId] {
89 const TYPES: [std::any::TypeId; 1] = [std::any::TypeId::of::<u16>()];
90 &TYPES
91 }
92}
93
94// Add support for the `bytes` codec using the helper macro (component size 1 = passthrough).
95zarrs_data_type::codec_traits::impl_bytes_data_type_traits!(CustomDataTypeUInt12, 1);
96
97/// Add support for the `packbits` codec.
98impl PackBitsDataTypeTraits for CustomDataTypeUInt12 {
99 fn component_size_bits(&self) -> u64 {
100 12
101 }
102
103 fn num_components(&self) -> u64 {
104 1
105 }
106
107 fn sign_extension(&self) -> bool {
108 false
109 }
110}
111
112// Register packbits codec support
113zarrs_data_type::register_data_type_extension_codec!(
114 CustomDataTypeUInt12,
115 zarrs_data_type::codec_traits::packbits::PackBitsDataTypePlugin,
116 zarrs_data_type::codec_traits::packbits::PackBitsDataTypeTraits
117);
118
119impl TryFrom<u64> for CustomDataTypeUInt12Element {
120 type Error = u64;
121
122 fn try_from(value: u64) -> Result<Self, Self::Error> {
123 if value < 4096 {
124 Ok(Self(value as u16))
125 } else {
126 Err(value)
127 }
128 }
129}
130
131impl CustomDataTypeUInt12Element {
132 fn into_le_bytes(self) -> [u8; 2] {
133 self.0.to_le_bytes()
134 }
135
136 fn from_le_bytes(bytes: [u8; 2]) -> Self {
137 Self(u16::from_le_bytes(bytes))
138 }
139
140 fn into_u16(self) -> u16 {
141 self.0
142 }
143}
144
145/// This defines how an in-memory CustomDataTypeUInt12Element is converted into ArrayBytes before encoding via the codec pipeline.
146impl Element for CustomDataTypeUInt12Element {
147 fn validate_data_type(data_type: &DataType) -> Result<(), ElementError> {
148 // Check if the data type matches our custom data type
149 data_type
150 .is::<CustomDataTypeUInt12>()
151 .then_some(())
152 .ok_or(ElementError::IncompatibleElementType)
153 }
154
155 fn to_array_bytes<'a>(
156 data_type: &DataType,
157 elements: &'a [Self],
158 ) -> Result<zarrs::array::ArrayBytes<'a>, ElementError> {
159 Self::validate_data_type(data_type)?;
160 let mut bytes: Vec<u8> = Vec::with_capacity(std::mem::size_of_val(elements));
161 for element in elements {
162 bytes.extend_from_slice(&element.into_le_bytes());
163 }
164 Ok(ArrayBytes::Fixed(Cow::Owned(bytes)))
165 }
166
167 fn into_array_bytes(
168 data_type: &DataType,
169 elements: Vec<Self>,
170 ) -> Result<zarrs::array::ArrayBytes<'static>, ElementError> {
171 Ok(Self::to_array_bytes(data_type, &elements)?.into_owned())
172 }
173}
174
175/// This defines how ArrayBytes are converted into a CustomDataTypeUInt12Element after decoding via the codec pipeline.
176impl ElementOwned for CustomDataTypeUInt12Element {
177 fn from_array_bytes(
178 data_type: &DataType,
179 bytes: ArrayBytes<'_>,
180 ) -> Result<Vec<Self>, ElementError> {
181 Self::validate_data_type(data_type)?;
182 let bytes = bytes.into_fixed()?;
183 let bytes_len = bytes.len();
184 let mut elements = Vec::with_capacity(bytes_len / size_of::<CustomDataTypeUInt12Element>());
185 for chunk in bytes.as_chunks::<2>().0 {
186 elements.push(CustomDataTypeUInt12Element::from_le_bytes(*chunk))
187 }
188 Ok(elements)
189 }
190}
191
192fn main() {
193 let store = std::sync::Arc::new(MemoryStore::default());
194 let array_path = "/array";
195 let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
196 let array = ArrayBuilder::new(
197 vec![4096, 1], // array shape
198 vec![5, 1], // regular chunk shape
199 Arc::new(CustomDataTypeUInt12),
200 FillValue::new(fill_value.into_le_bytes().to_vec()),
201 )
202 .array_to_array_codecs(vec![
203 #[cfg(feature = "transpose")]
204 Arc::new(zarrs::array::codec::TransposeCodec::new(
205 zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
206 )),
207 ])
208 .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
209 .bytes_to_bytes_codecs(vec![
210 #[cfg(feature = "gzip")]
211 Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
212 #[cfg(feature = "crc32c")]
213 Arc::new(zarrs::array::codec::Crc32cCodec::new()),
214 ])
215 // .storage_transformers(vec![].into())
216 .build(store, array_path)
217 .unwrap();
218 println!("{}", array.metadata().to_string_pretty());
219
220 let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
221 .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
222 .collect();
223
224 array
225 .store_array_subset(&array.subset_all(), &data)
226 .unwrap();
227
228 let mut data: Vec<CustomDataTypeUInt12Element> =
229 array.retrieve_array_subset(&array.subset_all()).unwrap();
230
231 for (i, d) in data.drain(0..4096).enumerate() {
232 let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
233 assert_eq!(d, element);
234 let element_pd: Vec<CustomDataTypeUInt12Element> = array
235 .retrieve_array_subset(&[(i as u64)..i as u64 + 1, 0..1])
236 .unwrap();
237 assert_eq!(element_pd[0], element);
238 }
239}examples/custom_data_type_variable_size.rs (line 123)
117 fn fill_value(
118 &self,
119 fill_value_metadata: &FillValueMetadata,
120 _version: ZarrVersion,
121 ) -> Result<FillValue, DataTypeFillValueMetadataError> {
122 if let Some(f) = fill_value_metadata.as_f32() {
123 Ok(FillValue::new(f.to_ne_bytes().to_vec()))
124 } else if fill_value_metadata.is_null() {
125 Ok(FillValue::new(vec![]))
126 } else if let Some(bytes) = fill_value_metadata.as_bytes() {
127 Ok(FillValue::new(bytes))
128 } else {
129 Err(DataTypeFillValueMetadataError)
130 }
131 }Sourcepub fn new_optional_null() -> FillValue
pub fn new_optional_null() -> FillValue
Create a null optional fill value.
Returns a fill value with a single 0x00 byte indicating null.
Can be chained with into_optional to create nested optional fill values.
§Examples
// Null fill value for Option<T>
let null_fill = FillValue::new_optional_null();
assert_eq!(null_fill.as_ne_bytes(), &[0]);
// Some(None) fill value for Option<Option<T>>
let some_null = FillValue::new_optional_null().into_optional();
assert_eq!(some_null.as_ne_bytes(), &[0, 1]);Examples found in repository?
examples/data_type_optional_nested.rs (line 23)
11fn main() -> Result<(), Box<dyn std::error::Error>> {
12 // Create an in-memory store
13 // let store = Arc::new(zarrs::filesystem::FilesystemStore::new(
14 // "zarrs/tests/data/v3/array_optional_nested.zarr",
15 // )?);
16 let store = Arc::new(zarrs::storage::store::MemoryStore::new());
17
18 // Build the codec chains for the optional codec
19 let array = ArrayBuilder::new(
20 vec![4, 4], // 4x4 array
21 vec![2, 2], // 2x2 chunks
22 data_type::uint8().to_optional().to_optional(), // Optional optional uint8 => Option<Option<u8>>
23 FillValue::new_optional_null().into_optional(), // Fill value => Some(None)
24 )
25 .dimension_names(["y", "x"].into())
26 .attributes(
27 serde_json::json!({
28 "description": r#"A 4x4 array of optional optional uint8 values with some missing data.
29The fill value is null on the inner optional layer, i.e. Some(None).
30N marks missing (`None`=`null`) values. SN marks `Some(None)`=`[null]` values:
31 N SN 2 3
32 N 5 N 7
33 SN SN N N
34 SN SN N N"#,
35 })
36 .as_object()
37 .unwrap()
38 .clone(),
39 )
40 .build(store.clone(), "/array")?;
41 array.store_metadata_opt(
42 &zarrs::array::ArrayMetadataOptions::default().with_include_zarrs_metadata(false),
43 )?;
44
45 println!("Array metadata:\n{}", array.metadata().to_string_pretty());
46
47 // Create some data with missing values
48 let data = ndarray::array![
49 [None, Some(None), Some(Some(2u8)), Some(Some(3u8))],
50 [None, Some(Some(5u8)), None, Some(Some(7u8))],
51 [Some(None), Some(None), None, None],
52 [Some(None), Some(None), None, None],
53 ]
54 .into_dyn();
55
56 // Write the data
57 array.store_array_subset(&array.subset_all(), data.clone())?;
58 println!("Data written to array.");
59
60 // Read back the data
61 let data_read: ArrayD<Option<Option<u8>>> = array.retrieve_array_subset(&array.subset_all())?;
62
63 // Verify data integrity
64 assert_eq!(data, data_read);
65
66 // Display the data in a grid format
67 println!(
68 "Data grid. N marks missing (`None`=`null`) values. SN marks `Some(None)`=`[null]` values"
69 );
70 println!(" 0 1 2 3");
71 for y in 0..4 {
72 print!("{} ", y);
73 for x in 0..4 {
74 match data_read[[y, x]] {
75 Some(Some(value)) => print!("{:3} ", value),
76 Some(None) => print!(" SN "),
77 None => print!(" N "),
78 }
79 }
80 println!();
81 }
82 Ok(())
83}More examples
examples/data_type_optional.rs (line 30)
18fn main() -> Result<(), Box<dyn std::error::Error>> {
19 // Create an in-memory store
20 // let store = Arc::new(zarrs::filesystem::FilesystemStore::new(
21 // "zarrs/tests/data/v3/array_optional.zarr",
22 // )?);
23 let store = Arc::new(zarrs::storage::store::MemoryStore::new());
24
25 // Build the codec chains for the optional codec
26 let array = ArrayBuilder::new(
27 vec![4, 4], // 4x4 array
28 vec![2, 2], // 2x2 chunks
29 data_type::uint8().to_optional(), // Optional uint8
30 FillValue::new_optional_null(), // Null fill value: [0]
31 )
32 .dimension_names(["y", "x"].into())
33 .attributes(
34 serde_json::json!({
35 "description": r#"A 4x4 array of optional uint8 values with some missing data.
36N marks missing (`None`=`null`) values:
37 0 N 2 3
38 N 5 N 7
39 8 9 N N
4012 N N N"#,
41 })
42 .as_object()
43 .unwrap()
44 .clone(),
45 )
46 .build(store.clone(), "/array")?;
47 array.store_metadata_opt(
48 &zarrs::array::ArrayMetadataOptions::default().with_include_zarrs_metadata(false),
49 )?;
50
51 println!("Array metadata:\n{}", array.metadata().to_string_pretty());
52
53 // Create some data with missing values
54 let data = ndarray::array![
55 [Some(0u8), None, Some(2u8), Some(3u8)],
56 [None, Some(5u8), None, Some(7u8)],
57 [Some(8u8), Some(9u8), None, None],
58 [Some(12u8), None, None, None],
59 ]
60 .into_dyn();
61
62 // Write the data
63 array.store_array_subset(&array.subset_all(), data.clone())?;
64
65 // Read back the data
66 let data_read: ArrayD<Option<u8>> = array.retrieve_array_subset(&array.subset_all())?;
67
68 // Verify data integrity
69 assert_eq!(data, data_read);
70
71 // Display the data in a grid format
72 println!("Data grid, N marks missing (`None`=`null`) values");
73 println!(" 0 1 2 3");
74 for y in 0..4 {
75 print!("{} ", y);
76 for x in 0..4 {
77 match data_read[[y, x]] {
78 Some(value) => print!("{:2} ", value),
79 None => print!(" N "),
80 }
81 }
82 println!();
83 }
84
85 // Print the raw bytes in all chunks
86 println!("Raw bytes in all chunks:");
87 let chunk_grid_shape = array.chunk_grid_shape();
88 for chunk_y in 0..chunk_grid_shape[0] {
89 for chunk_x in 0..chunk_grid_shape[1] {
90 let chunk_indices = vec![chunk_y, chunk_x];
91 let chunk_key = array.chunk_key(&chunk_indices);
92 println!(" Chunk [{}, {}] (key: {}):", chunk_y, chunk_x, chunk_key);
93
94 if let Some(chunk_bytes) = store.get(&chunk_key)? {
95 println!(" Size: {} bytes", chunk_bytes.len());
96
97 if chunk_bytes.len() >= 16 {
98 // Parse first 8 bytes as mask size (little-endian u64)
99 let mask_size = u64::from_le_bytes([
100 chunk_bytes[0],
101 chunk_bytes[1],
102 chunk_bytes[2],
103 chunk_bytes[3],
104 chunk_bytes[4],
105 chunk_bytes[5],
106 chunk_bytes[6],
107 chunk_bytes[7],
108 ]) as usize;
109
110 // Parse second 8 bytes as data size (little-endian u64)
111 let data_size = u64::from_le_bytes([
112 chunk_bytes[8],
113 chunk_bytes[9],
114 chunk_bytes[10],
115 chunk_bytes[11],
116 chunk_bytes[12],
117 chunk_bytes[13],
118 chunk_bytes[14],
119 chunk_bytes[15],
120 ]) as usize;
121
122 // Display mask size header with raw bytes
123 print!(" Mask size: 0b");
124 for byte in &chunk_bytes[0..8] {
125 print!("{:08b}", byte);
126 }
127 println!(" -> {} bytes", mask_size);
128
129 // Display data size header with raw bytes
130 print!(" Data size: 0b");
131 for byte in &chunk_bytes[8..16] {
132 print!("{:08b}", byte);
133 }
134 println!(" -> {} bytes", data_size);
135
136 // Show mask and data sections separately
137 if chunk_bytes.len() >= 16 + mask_size + data_size {
138 let mask_start = 16;
139 let data_start = 16 + mask_size;
140
141 // Show mask as binary
142 if mask_size > 0 {
143 println!(" Mask (binary):");
144 print!(" ");
145 for byte in &chunk_bytes[mask_start..mask_start + mask_size] {
146 print!("0b{:08b} ", byte);
147 }
148 println!();
149 }
150
151 // Show data as binary
152 if data_size > 0 {
153 println!(" Data (binary):");
154 print!(" ");
155 for byte in &chunk_bytes[data_start..data_start + data_size] {
156 print!("0b{:08b} ", byte);
157 }
158 println!();
159 }
160 }
161 } else {
162 panic!(" Chunk too small to parse headers");
163 }
164 } else {
165 println!(" Chunk missing (fill value chunk)");
166 }
167 }
168 }
169 Ok(())
170}Sourcepub fn as_ne_bytes(&self) -> &[u8] ⓘ
pub fn as_ne_bytes(&self) -> &[u8] ⓘ
Return the byte representation of the fill value.
Examples found in repository?
examples/custom_data_type_fixed_size.rs (line 198)
192 fn metadata_fill_value(
193 &self,
194 fill_value: &FillValue,
195 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
196 let element = CustomDataTypeFixedSizeMetadata::from_ne_bytes(
197 fill_value
198 .as_ne_bytes()
199 .try_into()
200 .map_err(|_| DataTypeFillValueError)?,
201 );
202 Ok(FillValueMetadata::from(element))
203 }More examples
examples/custom_data_type_uint4.rs (line 74)
68 fn metadata_fill_value(
69 &self,
70 fill_value: &FillValue,
71 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
72 let element = CustomDataTypeUInt4Element::from_ne_bytes(
73 fill_value
74 .as_ne_bytes()
75 .try_into()
76 .map_err(|_| DataTypeFillValueError)?,
77 );
78 Ok(FillValueMetadata::from(element.into_u8()))
79 }examples/custom_data_type_uint12.rs (line 72)
66 fn metadata_fill_value(
67 &self,
68 fill_value: &FillValue,
69 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
70 let element = CustomDataTypeUInt12Element::from_le_bytes(
71 fill_value
72 .as_ne_bytes()
73 .try_into()
74 .map_err(|_| DataTypeFillValueError)?,
75 );
76 Ok(FillValueMetadata::from(element.into_u16()))
77 }examples/custom_data_type_float8_e3m4.rs (line 72)
66 fn metadata_fill_value(
67 &self,
68 fill_value: &FillValue,
69 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
70 let element = CustomDataTypeFloat8e3m4Element::from_ne_bytes(
71 fill_value
72 .as_ne_bytes()
73 .try_into()
74 .map_err(|_| DataTypeFillValueError)?,
75 );
76 Ok(FillValueMetadata::from(element.into_f32()))
77 }examples/custom_data_type_variable_size.rs (line 137)
133 fn metadata_fill_value(
134 &self,
135 fill_value: &FillValue,
136 ) -> Result<FillValueMetadata, DataTypeFillValueError> {
137 let fill_value = fill_value.as_ne_bytes();
138 if fill_value.is_empty() {
139 Ok(FillValueMetadata::Null)
140 } else if fill_value.len() == 4 {
141 let value = f32::from_ne_bytes(fill_value.try_into().unwrap());
142 Ok(FillValueMetadata::from(value))
143 } else {
144 Err(DataTypeFillValueError)
145 }
146 }Sourcepub fn equals_all(&self, bytes: &[u8]) -> bool
pub fn equals_all(&self, bytes: &[u8]) -> bool
Check if the bytes are equal to a sequence of the fill value.
Sourcepub fn into_optional(self) -> FillValue
pub fn into_optional(self) -> FillValue
Wrap this fill value as a non-null optional fill value.
Appends a 0x01 suffix byte indicating the value is non-null.
Can be chained to create nested optional fill values.
§Examples
// Single level optional (Some(u8))
let opt_fill = FillValue::from(42u8).into_optional();
assert_eq!(opt_fill.as_ne_bytes(), &[42, 1]);
// Double level optional (Some(Some(u8)))
let nested = FillValue::from(42u8).into_optional().into_optional();
assert_eq!(nested.as_ne_bytes(), &[42, 1, 1]);Examples found in repository?
examples/data_type_optional_nested.rs (line 23)
11fn main() -> Result<(), Box<dyn std::error::Error>> {
12 // Create an in-memory store
13 // let store = Arc::new(zarrs::filesystem::FilesystemStore::new(
14 // "zarrs/tests/data/v3/array_optional_nested.zarr",
15 // )?);
16 let store = Arc::new(zarrs::storage::store::MemoryStore::new());
17
18 // Build the codec chains for the optional codec
19 let array = ArrayBuilder::new(
20 vec![4, 4], // 4x4 array
21 vec![2, 2], // 2x2 chunks
22 data_type::uint8().to_optional().to_optional(), // Optional optional uint8 => Option<Option<u8>>
23 FillValue::new_optional_null().into_optional(), // Fill value => Some(None)
24 )
25 .dimension_names(["y", "x"].into())
26 .attributes(
27 serde_json::json!({
28 "description": r#"A 4x4 array of optional optional uint8 values with some missing data.
29The fill value is null on the inner optional layer, i.e. Some(None).
30N marks missing (`None`=`null`) values. SN marks `Some(None)`=`[null]` values:
31 N SN 2 3
32 N 5 N 7
33 SN SN N N
34 SN SN N N"#,
35 })
36 .as_object()
37 .unwrap()
38 .clone(),
39 )
40 .build(store.clone(), "/array")?;
41 array.store_metadata_opt(
42 &zarrs::array::ArrayMetadataOptions::default().with_include_zarrs_metadata(false),
43 )?;
44
45 println!("Array metadata:\n{}", array.metadata().to_string_pretty());
46
47 // Create some data with missing values
48 let data = ndarray::array![
49 [None, Some(None), Some(Some(2u8)), Some(Some(3u8))],
50 [None, Some(Some(5u8)), None, Some(Some(7u8))],
51 [Some(None), Some(None), None, None],
52 [Some(None), Some(None), None, None],
53 ]
54 .into_dyn();
55
56 // Write the data
57 array.store_array_subset(&array.subset_all(), data.clone())?;
58 println!("Data written to array.");
59
60 // Read back the data
61 let data_read: ArrayD<Option<Option<u8>>> = array.retrieve_array_subset(&array.subset_all())?;
62
63 // Verify data integrity
64 assert_eq!(data, data_read);
65
66 // Display the data in a grid format
67 println!(
68 "Data grid. N marks missing (`None`=`null`) values. SN marks `Some(None)`=`[null]` values"
69 );
70 println!(" 0 1 2 3");
71 for y in 0..4 {
72 print!("{} ", y);
73 for x in 0..4 {
74 match data_read[[y, x]] {
75 Some(Some(value)) => print!("{:3} ", value),
76 Some(None) => print!(" SN "),
77 None => print!(" N "),
78 }
79 }
80 println!();
81 }
82 Ok(())
83}Trait Implementations§
impl Eq for FillValue
Source§impl From<FillValue> for ArrayBuilderFillValue
impl From<FillValue> for ArrayBuilderFillValue
impl StructuralPartialEq for FillValue
Auto Trait Implementations§
impl Freeze for FillValue
impl RefUnwindSafe for FillValue
impl Send for FillValue
impl Sync for FillValue
impl Unpin for FillValue
impl UnsafeUnpin for FillValue
impl UnwindSafe for FillValue
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<Q, K> Equivalent<K> for Q
impl<Q, K> Equivalent<K> for Q
Source§impl<Q, K> Equivalent<K> for Q
impl<Q, K> Equivalent<K> for Q
Source§fn equivalent(&self, key: &K) -> bool
fn equivalent(&self, key: &K) -> bool
Compare self to
key and return true if they are equal.Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more