mzdata_bindata/
array.rs

1use std::borrow::Cow;
2use std::fmt::{self, Formatter};
3use std::io::prelude::*;
4use std::mem;
5use std::num;
6
7use base64_simd;
8use bytemuck::Pod;
9use flate2::write::{ZlibDecoder, ZlibEncoder};
10use flate2::Compression;
11use num_traits::ToBytes;
12
13use mzdata_param::{ParamList, Unit};
14
15#[allow(unused)]
16use crate::encodings::{
17    to_bytes, vec_as_bytes, ArrayRetrievalError, ArrayType, BinaryCompressionType, BinaryDataArrayType,
18    Bytes,
19};
20use crate::traits::{ByteArrayView, ByteArrayViewMut};
21
22#[allow(unused)]
23use super::encodings::{
24    reverse_transpose_f32, reverse_transpose_f64, reverse_transpose_i32, reverse_transpose_i64,
25    transpose_f32, transpose_f64, transpose_i32, transpose_i64,
26};
27
28/// Represents a data array that holds a byte buffer that may be compressed, base64 encoded,
29/// or raw little endian bytes, and provides views of those bytes as a small range of supported
30/// types.
31///
32/// This type is modeled after the `<binaryDataArray>` element in mzML.
33///
34/// # Note
35/// This type tries to walk a fine line between convenience and performance and as such is easy
36/// to misuse. All operations that view the byte buffer as arbitrary data need that data to be
37/// decoded and decompressed in order to borrow it. If the byte buffer is not already stored
38/// decoded, the operation will copy and decode the buffer in its entirety before performing any
39/// other operation, so repeated method calls may incur excessive overhead. If this is happening,
40/// please use [`DataArray::decode_and_store`] to store the decoded representation explicitly.
41///
42/// Normally, [`SpectrumSource`](crate::io::SpectrumSource)-implementing file readers will eagerly decode all arrays
43/// as soon as they are ready. If they are operating in lazy mode, the buffers will need to be decoded
44/// explicitly, again using [`DataArray::decode_and_store`] or operations should make as much use of the
45/// copied arrays as possible instead.
46#[derive(Default, Clone)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub struct DataArray {
49    /// The raw data of the array, stored in a variety of states controlled by [`Self::Compression`], as native bytes
50    pub data: Bytes,
51    /// The *kind* of data stored in the array, as in bit-width primitive types
52    pub dtype: BinaryDataArrayType,
53    /// How [`Self::data`] is encoded and compressed.
54    pub compression: BinaryCompressionType,
55    /// The *what* the data stored in the array was measuring, e.g. an m/z array
56    pub name: ArrayType,
57    /// Additional metadata parameters
58    pub params: Option<Box<ParamList>>,
59    /// The [`Unit`] for the data measurements
60    pub unit: Unit,
61    /// A cache for the number of elements stored in the array
62    item_count: Option<num::NonZero<usize>>,
63    /// An identifier reference to a data processing method other than the default method
64    data_processing_reference: Option<Box<str>>,
65}
66
67impl core::fmt::Debug for DataArray {
68    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
69        f.debug_struct("DataArray")
70            .field("name", &self.name)
71            .field("data size", &self.data.len())
72            .field("dtype", &self.dtype)
73            .field("compression", &self.compression)
74            .field("params", &self.params)
75            .field("unit", &self.unit)
76            .field("data_processing_ref", &self.data_processing_reference)
77            .finish()
78    }
79}
80
81const EMPTY_BUFFER: [u8; 0] = [];
82
83
84impl<'transient, 'lifespan: 'transient> DataArray {
85    pub fn new() -> DataArray {
86        DataArray {
87            ..Default::default()
88        }
89    }
90
91    pub fn from_name(name: &ArrayType) -> DataArray {
92        DataArray {
93            dtype: name.preferred_dtype(),
94            name: name.clone(),
95            compression: BinaryCompressionType::Decoded,
96            ..Default::default()
97        }
98    }
99
100    pub fn from_name_and_type(name: &ArrayType, dtype: BinaryDataArrayType) -> DataArray {
101        DataArray {
102            dtype,
103            name: name.clone(),
104            compression: BinaryCompressionType::Decoded,
105            ..Default::default()
106        }
107    }
108
109    pub fn from_name_type_size(
110        name: &ArrayType,
111        dtype: BinaryDataArrayType,
112        size: usize,
113    ) -> DataArray {
114        DataArray {
115            dtype,
116            name: name.clone(),
117            data: Bytes::with_capacity(size),
118            compression: BinaryCompressionType::Decoded,
119            ..Default::default()
120        }
121    }
122
123    pub fn slice(&self, start: usize, end: usize) -> Result<DataArray, ArrayRetrievalError> {
124        if end < start || (end - start) % self.dtype.size_of() != 0 {
125            Err(ArrayRetrievalError::DataTypeSizeMismatch)
126        } else {
127            let data = self.decode()?;
128            let slice = data[start..end].to_vec();
129            let subset = Self::wrap(&self.name, self.dtype, slice);
130            Ok(subset)
131        }
132    }
133
134    pub fn slice_buffer(
135        &self,
136        start: usize,
137        end: usize,
138    ) -> Result<Cow<'_, [u8]>, ArrayRetrievalError> {
139        if end < start || (end - start) % self.dtype.size_of() != 0 {
140            Err(ArrayRetrievalError::DataTypeSizeMismatch)
141        } else {
142            let data = self.decode()?;
143            match data {
144                Cow::Borrowed(view) => Ok(Cow::Borrowed(&view[start..end])),
145                Cow::Owned(view) => {
146                    let data = &view[start..end];
147                    Ok(Cow::Owned(data.to_owned()))
148                }
149            }
150        }
151    }
152
153    /// This method assumes the data are already in native byte order
154    pub fn wrap(name: &ArrayType, dtype: BinaryDataArrayType, data: Bytes) -> DataArray {
155        DataArray {
156            dtype,
157            name: name.clone(),
158            data,
159            compression: BinaryCompressionType::Decoded,
160            ..Default::default()
161        }
162    }
163
164    /// This method assumes the data are already in native byte order
165    fn set_buffer_of_type(&mut self, data_buffer: Vec<u8>) -> Result<usize, ArrayRetrievalError> {
166        if data_buffer.is_empty() {
167            self.item_count = None;
168        } else {
169            self.item_count = num::NonZero::try_from(data_buffer.len() / self.dtype().size_of()).ok();
170        }
171        self.data = data_buffer;
172        Ok(self.data.len())
173    }
174
175    /// Directly set the data buffer from a slice of a [`Pod`] type, copying it.
176    ///
177    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
178    /// the size of [`DataArray::dtype`].
179    pub fn update_buffer<T: Pod + ToBytes>(
180        &mut self,
181        data_buffer: &[T],
182    ) -> Result<usize, ArrayRetrievalError> {
183        if self.dtype.size_of() != mem::size_of::<T>() {
184            Err(ArrayRetrievalError::DataTypeSizeMismatch)
185        } else {
186            let n = data_buffer.len();
187            self.item_count = num::NonZero::try_from(n).ok();
188            self.data = to_bytes(data_buffer);
189            Ok(self.data.len())
190        }
191    }
192
193    /// Add the value of a [`Pod`] implementing type `T` to the array, converting it
194    /// to native byte representation.
195    ///
196    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
197    /// the size of [`DataArray::dtype`].
198    pub fn push<T: Pod>(&mut self, value: T) -> Result<(), ArrayRetrievalError> {
199        if !matches!(self.compression, BinaryCompressionType::Decoded) {
200            self.decode_and_store()?;
201        };
202        if self.dtype.size_of() != mem::size_of::<T>() {
203            Err(ArrayRetrievalError::DataTypeSizeMismatch)
204        } else {
205            let data = bytemuck::bytes_of(&value);
206            self.data.extend_from_slice(data);
207            self.item_count = self.item_count.map(|i| i.saturating_add(1));
208            Ok(())
209        }
210    }
211
212    /// Add the values from a slice of a [`Pod`] implementing type `T` to the array, converting the slice
213    /// a slice of `T` in its native byte representation.
214    ///
215    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
216    /// the size of [`DataArray::dtype`].
217    pub fn extend<T: Pod>(&mut self, values: &[T]) -> Result<(), ArrayRetrievalError> {
218        if !matches!(self.compression, BinaryCompressionType::Decoded) {
219            self.decode_and_store()?;
220        };
221        if self.dtype.size_of() != mem::size_of::<T>() {
222            Err(ArrayRetrievalError::DataTypeSizeMismatch)
223        } else {
224            self.item_count = self.item_count.map(|i| i.saturating_add(values.len()));
225            let data = bytemuck::cast_slice(values);
226            self.data.extend_from_slice(data);
227            Ok(())
228        }
229    }
230
231    /// Add the values from a buffer of bytes to the array directly.
232    ///
233    /// This will return an error if `values.len()` is not a multiple of
234    /// the size of [`DataArray::dtype`].
235    pub fn extend_raw(&mut self, values: &[u8]) -> Result<(), ArrayRetrievalError> {
236        if values.len() % self.dtype.size_of() != 0 {
237            Err(ArrayRetrievalError::DataTypeSizeMismatch)
238        } else {
239            self.data.extend_from_slice(values);
240            Ok(())
241        }
242    }
243
244    /// Add the values from an iterator over [`Pod`] type `T`.
245    ///
246    /// This is more efficient than repeated calls to [`DataArray::push`] or collecting
247    /// to a `Vec` and then using [`DataArray::extend`], but equivalent to either.
248    ///
249    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
250    /// the size of [`DataArray::dtype`].
251    pub fn extend_iter<T: Pod>(&mut self, iter: impl IntoIterator<Item=T>) -> Result<(), ArrayRetrievalError> {
252        if mem::size_of::<T>() != self.dtype.size_of() {
253            return Err(ArrayRetrievalError::DataTypeSizeMismatch)
254        }
255        for val in iter.into_iter() {
256            let vb = bytemuck::bytes_of(&val);
257            self.data.extend_from_slice(vb);
258        }
259        Ok(())
260    }
261
262    /// Encode the data buffer to a byte array using the requested [`BinaryCompressionType`] to
263    /// compress it and then base64 encode it.
264    pub fn encode_bytestring(&self, compression: BinaryCompressionType) -> Bytes {
265        if self.compression == compression {
266            log::trace!("Fast-path encoding {}:{}", self.name, self.dtype);
267            return self.data.clone();
268        }
269        let bytestring = match self.compression {
270            BinaryCompressionType::Decoded => Cow::Borrowed(self.data.as_slice()),
271            _ => self.decode().expect("Failed to decode binary data"),
272        };
273        match compression {
274            BinaryCompressionType::Decoded => panic!("Should never happen"),
275            BinaryCompressionType::Zlib => {
276                let compressed = Self::compress_zlib(&bytestring);
277                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
278            }
279            BinaryCompressionType::NoCompression => {
280                base64_simd::STANDARD.encode_type::<Bytes>(bytestring.as_ref())
281            }
282            #[cfg(feature = "numpress")]
283            BinaryCompressionType::NumpressLinear => {
284                if self.dtype != BinaryDataArrayType::Float64 {
285                    panic!("Cannot Numpress non-float64 data!");
286                }
287                if bytestring.is_empty() {
288                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
289                }
290                let compressed =
291                    Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring)).unwrap();
292                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
293            }
294            #[cfg(feature = "numpress")]
295            BinaryCompressionType::NumpressSLOF => {
296                let compressed = match self.dtype {
297                    BinaryDataArrayType::Float32 => {
298                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
299                    },
300                    BinaryDataArrayType::Float64 => {
301                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
302                    },
303                    _ => {
304                        panic!("Cannot Numpress non-float data!");
305                    }
306                };
307                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
308            }
309            #[cfg(feature = "numpress")]
310            BinaryCompressionType::NumpressLinearZlib => {
311                if self.dtype != BinaryDataArrayType::Float64 {
312                    panic!("Cannot Numpress non-float64 data!");
313                }
314                if bytestring.is_empty() {
315                    let compressed = Self::compress_zlib(&bytestring);
316                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
317                }
318                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
319                    .inspect_err(|e| {
320                        log::error!("Failed to compress buffer with numpress: {e}");
321                    })
322                    .unwrap();
323                let compressed = Self::compress_zlib(&compressed);
324                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
325            }
326            #[cfg(all(feature = "numpress", feature = "zstd"))]
327            BinaryCompressionType::NumpressLinearZstd => {
328                if self.dtype != BinaryDataArrayType::Float64 {
329                    panic!("Cannot Numpress non-float64 data!");
330                }
331                if bytestring.is_empty() {
332                    let compressed = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
333                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
334                }
335                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
336                    .inspect_err(|e| {
337                        log::error!("Failed to compress buffer with numpress: {e}");
338                    })
339                    .unwrap();
340                let compressed = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
341                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
342            }
343            #[cfg(feature = "numpress")]
344            BinaryCompressionType::NumpressSLOFZlib => {
345                if bytestring.is_empty() {
346                    let bytestring = Self::compress_zlib(&bytestring);
347                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
348                }
349                let compressed = match self.dtype {
350                    BinaryDataArrayType::Float32 => {
351                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
352                    },
353                    BinaryDataArrayType::Float64 => {
354                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
355                    },
356                    _ => {
357                        panic!("Cannot Numpress non-float data!");
358                    }
359                };
360                let bytestring = Self::compress_zlib(&compressed);
361                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
362            }
363            #[cfg(all(feature = "numpress", feature = "zstd"))]
364            BinaryCompressionType::NumpressSLOFZstd => {
365                if bytestring.is_empty() {
366                    let bytestring = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
367                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
368                }
369                let compressed = match self.dtype {
370                    BinaryDataArrayType::Float32 => {
371                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
372                    },
373                    BinaryDataArrayType::Float64 => {
374                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
375                    },
376                    _ => {
377                        panic!("Cannot Numpress non-float data!");
378                    }
379                };
380                let bytestring = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
381                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
382            }
383            #[cfg(feature = "zstd")]
384            BinaryCompressionType::Zstd => {
385                let compressed = Self::compress_zstd(&bytestring, self.dtype, false);
386                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
387            }
388            #[cfg(feature = "zstd")]
389            BinaryCompressionType::ShuffleZstd => {
390                let compressed = Self::compress_zstd(&bytestring, self.dtype, true);
391                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
392            }
393            #[cfg(feature = "zstd")]
394            BinaryCompressionType::DeltaShuffleZstd => {
395                let compressed = Self::compress_delta_zstd(&bytestring, self.dtype, true);
396                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
397            }
398            #[cfg(feature = "zstd")]
399            BinaryCompressionType::ZstdDict => {
400                let compressed = Self::compress_dict_zstd(&bytestring, self.dtype);
401                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
402            }
403            _ => {
404                panic!("Compresion type {:?} is unsupported", compression)
405            }
406        }
407    }
408
409    /// Decode the compressed data, if needed, and store that buffer in `self.data`. After
410    /// decoding `self.compression` will always be [`BinaryCompressionType::Decoded`].
411    ///
412    /// The return value is the content of `self.compression` after decoding.
413    ///
414    /// This may fail if the decoding fails for any reason.
415    pub fn decode_and_store(&mut self) -> Result<BinaryCompressionType, ArrayRetrievalError> {
416        match self.decode() {
417            Ok(data) => {
418                match data {
419                    // The only time this is a borrow is when the data are already
420                    // decoded.
421                    Cow::Borrowed(_view) => Ok(self.compression),
422                    Cow::Owned(buffer) => {
423                        let n = buffer.len() / self.dtype.size_of();
424                        self.item_count = n.try_into().ok();
425                        self.data = buffer;
426                        self.compression = BinaryCompressionType::Decoded;
427                        Ok(self.compression)
428                    }
429                }
430            }
431            Err(err) => Err(err),
432        }
433    }
434
435    /// Decompress and base64-decode encoded bytes, and return the data.
436    ///
437    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
438    /// more buffers may be allocated to hold the decompressed and decoded bytes.
439    pub fn decode(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
440        if self.data.is_empty() {
441            return Ok(Cow::Borrowed(&EMPTY_BUFFER));
442        }
443
444        macro_rules! base64_decode {
445            () => {
446                base64_simd::STANDARD
447                    .decode_type::<Bytes>(&self.data)
448                    .unwrap_or_else(|e| panic!("Failed to decode base64 array: {}", e))
449            };
450        }
451
452        match self.compression {
453            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(self.data.as_slice())),
454            BinaryCompressionType::NoCompression => {
455                let bytestring = base64_decode!();
456                Ok(Cow::Owned(bytestring))
457            }
458            BinaryCompressionType::Zlib => {
459                let mut bytestring = base64_decode!();
460                bytestring = Self::decompress_zlib(&bytestring);
461                Ok(Cow::Owned(bytestring))
462            }
463            #[cfg(feature = "zstd")]
464            BinaryCompressionType::Zstd => {
465                let bytestring = base64_decode!();
466                Ok(Cow::Owned(Self::decompress_zstd(
467                    &bytestring,
468                    self.dtype,
469                    false,
470                )))
471            }
472            #[cfg(feature = "zstd")]
473            BinaryCompressionType::ShuffleZstd => {
474                let bytestring = base64_decode!();
475                Ok(Cow::Owned(Self::decompress_zstd(
476                    &bytestring,
477                    self.dtype,
478                    true,
479                )))
480            }
481            #[cfg(feature = "zstd")]
482            BinaryCompressionType::DeltaShuffleZstd => {
483                let bytestring = base64_decode!();
484                Ok(Cow::Owned(Self::decompress_delta_zstd(
485                    &bytestring,
486                    self.dtype,
487                    true,
488                )))
489            }
490
491            #[cfg(feature = "zstd")]
492            BinaryCompressionType::ZstdDict => {
493                let bytestring = base64_decode!();
494                Ok(Cow::Owned(Self::decompress_dict_zstd(
495                    &bytestring,
496                    self.dtype,
497                )))
498            }
499            #[cfg(feature = "numpress")]
500            BinaryCompressionType::NumpressLinear => match self.dtype {
501                BinaryDataArrayType::Float64 => {
502                    let bytestring = base64_decode!();
503                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
504                    let view = vec_as_bytes(decoded);
505                    Ok(Cow::Owned(view))
506                }
507                _ => Err(ArrayRetrievalError::DecompressionError(
508                    self.compression.unsupported_msg(Some(
509                        format!("Not compatible with {:?}", self.dtype).as_str(),
510                    )),
511                )),
512            },
513
514            #[cfg(feature = "numpress")]
515            BinaryCompressionType::NumpressSLOF => {
516                let bytestring = base64_decode!();
517                Self::decompress_numpress_slof(&bytestring, self.dtype)
518            }
519
520            #[cfg(feature = "numpress")]
521            BinaryCompressionType::NumpressLinearZlib => match self.dtype {
522                BinaryDataArrayType::Float64 => {
523                    let bytestring = base64_decode!();
524                    let bytestring = Self::decompress_zlib(&bytestring);
525                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
526                    let view = vec_as_bytes(decoded);
527                    Ok(Cow::Owned(view))
528                }
529                _ => Err(ArrayRetrievalError::DecompressionError(
530                    self.compression.unsupported_msg(Some(
531                        format!("Not compatible with {:?}", self.dtype).as_str(),
532                    )),
533                )),
534            },
535
536            #[cfg(feature = "numpress")]
537            BinaryCompressionType::NumpressSLOFZlib => {
538                let bytestring = base64_decode!();
539                let bytestring = Self::decompress_zlib(&bytestring);
540                Self::decompress_numpress_slof(&bytestring, self.dtype)
541            }
542
543            #[cfg(all(feature = "numpress", feature = "zstd"))]
544            BinaryCompressionType::NumpressLinearZstd => match self.dtype {
545                BinaryDataArrayType::Float64 => {
546                    let bytestring = base64_decode!();
547                    let bytestring = Self::decompress_zstd(
548                        &bytestring,
549                        BinaryDataArrayType::Unknown,
550                        false,
551                    );
552                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
553                    let view = vec_as_bytes(decoded);
554                    Ok(Cow::Owned(view))
555                }
556                _ => Err(ArrayRetrievalError::DecompressionError(
557                    self.compression.unsupported_msg(Some(
558                        format!("Not compatible with {:?}", self.dtype).as_str(),
559                    )),
560                )),
561            },
562
563            #[cfg(all(feature = "numpress", feature = "zstd"))]
564            BinaryCompressionType::NumpressSLOFZstd => {
565                let bytestring = base64_decode!();
566                let bytestring = Self::decompress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
567                Self::decompress_numpress_slof(&bytestring, self.dtype)
568            }
569
570            mode => Err(ArrayRetrievalError::DecompressionError(format!(
571                "Cannot decode array encoded with {:?}",
572                mode
573            ))),
574        }
575    }
576
577    pub(crate) fn decoded_slice(
578        &'lifespan self,
579        start: usize,
580        end: usize,
581    ) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
582        if start > end || (end - start) % self.dtype.size_of() != 0 {
583            return Err(ArrayRetrievalError::DataTypeSizeMismatch);
584        }
585        match self.compression {
586            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(&self.data.as_slice()[start..end])),
587            _ => {
588                Ok(Cow::Owned(self.decode()?[start..end].to_vec()))
589            }
590        }
591    }
592
593    pub fn decode_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
594        if self.data.is_empty() || matches!(self.compression, BinaryCompressionType::Decoded) {
595            Ok(&mut self.data)
596        } else {
597            match self.decode()? {
598                Cow::Borrowed(_) => {
599                    Ok(&mut self.data)
600                },
601                Cow::Owned(owned) => {
602                    self.data = owned;
603                    self.compression = BinaryCompressionType::Decoded;
604                    Ok(&mut self.data)
605                },
606            }
607        }
608
609    }
610
611    pub fn clear(&mut self) {
612        self.data.clear();
613        self.params = None;
614        self.item_count = None;
615    }
616
617    /// The reverse of [`DataArray::decode_and_store`], this method compresses `self.data` to the desired
618    /// compression method and stores that buffer as `self.data`.
619    pub fn store_compressed(
620        &mut self,
621        compression: BinaryCompressionType,
622    ) -> Result<(), ArrayRetrievalError> {
623        if self.compression == compression {
624            Ok(())
625        } else {
626            self.item_count = self.data_len().ok().and_then(|n| n.try_into().ok());
627            let bytes = self.encode_bytestring(compression);
628            self.data = bytes;
629            self.compression = compression;
630            Ok(())
631        }
632    }
633
634    /// Recode the stored data as the requested binary data type.
635    pub fn store_as(&mut self, dtype: BinaryDataArrayType) -> Result<usize, ArrayRetrievalError> {
636        if self.dtype == dtype {
637            return Ok(self.data.len());
638        }
639        match dtype {
640            BinaryDataArrayType::Float32 => {
641                let view = self.to_f32()?;
642                #[cfg(target_endian = "big")]
643                {
644                    let mut recast = to_bytes(&view);
645                    dtype.swap_bytes(&mut recast)?;
646                    self.dtype = dtype;
647                    self.set_buffer_of_type(recast)
648                }
649                #[cfg(not(target_endian = "big"))]
650                {
651                    let recast = to_bytes(&view);
652                    self.dtype = dtype;
653                    self.set_buffer_of_type(recast)
654                }
655
656            }
657            BinaryDataArrayType::Float64 => {
658                let view = self.to_f64()?;
659                #[cfg(target_endian = "big")]
660                {
661                    let mut recast = to_bytes(&view);
662                    dtype.swap_bytes(&mut recast)?;
663                    self.dtype = dtype;
664                    self.set_buffer_of_type(recast)
665                }
666                #[cfg(not(target_endian = "big"))]
667                {
668                    let recast = to_bytes(&view);
669                    self.dtype = dtype;
670                    self.set_buffer_of_type(recast)
671                }
672            }
673            BinaryDataArrayType::Int32 => {
674                let view = self.to_i32()?;
675                #[cfg(target_endian = "big")]
676                {
677                    let mut recast = to_bytes(&view);
678                    dtype.swap_bytes(&mut recast)?;
679                    self.dtype = dtype;
680                    self.set_buffer_of_type(recast)
681                }
682                #[cfg(not(target_endian = "big"))]
683                {
684                    let recast = to_bytes(&view);
685                    self.dtype = dtype;
686                    self.set_buffer_of_type(recast)
687                }
688            }
689            BinaryDataArrayType::Int64 => {
690                let view = self.to_i64()?;
691                #[cfg(target_endian = "big")]
692                {
693                    let mut recast = to_bytes(&view);
694                    dtype.swap_bytes(&mut recast)?;
695                    self.dtype = dtype;
696                    self.set_buffer_of_type(recast)
697                }
698                #[cfg(not(target_endian = "big"))]
699                {
700                    let recast = to_bytes(&view);
701                    self.dtype = dtype;
702                    self.set_buffer_of_type(recast)
703                }
704            }
705            _ => Ok(0),
706        }
707    }
708
709    /// Test if the the array describes an ion mobility quantity.
710    ///
711    /// # See also
712    /// [`ArrayType::is_ion_mobility`]
713    pub const fn is_ion_mobility(&self) -> bool {
714        self.name.is_ion_mobility()
715    }
716
717    /// The size of the raw byte buffer
718    pub fn raw_len(&self) -> usize {
719        self.data.len()
720    }
721
722    /// Get the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
723    pub fn data_processing_reference(&self) -> Option<&str> {
724        self.data_processing_reference.as_deref()
725    }
726
727    /// Set the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
728    pub fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
729        self.data_processing_reference = data_processing_reference;
730    }
731}
732
733
734/// [`DataArray`] implements several compression codecs, some of which require additional dependencies.
735impl DataArray {
736    pub fn compress_zlib(bytestring: &[u8]) -> Bytes {
737        let result = Bytes::new();
738        let mut compressor = ZlibEncoder::new(result, Compression::best());
739        compressor.write_all(bytestring).expect("Error compressing");
740        compressor.finish().expect("Error compressing")
741    }
742
743    pub fn decompress_zlib(bytestring: &[u8]) -> Bytes {
744        let result = Bytes::new();
745        let mut decompressor = ZlibDecoder::new(result);
746        decompressor
747            .write_all(bytestring)
748            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
749        let buf = decompressor
750            .finish()
751            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
752        buf
753    }
754
755    #[cfg(feature = "numpress")]
756    pub fn compress_numpress_linear(data: &[f64]) -> Result<Bytes, ArrayRetrievalError> {
757        if data.is_empty() {
758            return Ok(Bytes::new());
759        }
760        let scaling = numpress::optimal_scaling(data);
761        match numpress::numpress_compress(data, scaling) {
762            Ok(data) => Ok(data),
763            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
764        }
765    }
766
767    #[cfg(feature = "numpress")]
768    pub fn compress_numpress_slof<T: numpress::AsFloat64>(data: &[T]) -> Result<Bytes, ArrayRetrievalError> {
769        let scaling = numpress::optimal_slof_fixed_point(data);
770        let mut buf = Bytes::new();
771        match numpress::encode_slof(data, &mut buf, scaling) {
772            Ok(_) => Ok(buf),
773            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
774        }
775    }
776
777    #[cfg(feature = "numpress")]
778    pub fn decompress_numpress_linear(data: &[u8]) -> Result<Vec<f64>, ArrayRetrievalError> {
779        if data.is_empty() {
780            return Ok(Vec::new())
781        }
782        match numpress::numpress_decompress(data) {
783            Ok(data) => Ok(data),
784            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
785        }
786    }
787
788    #[cfg(feature = "numpress")]
789    pub fn decompress_numpress_slof(data: &[u8], dtype: BinaryDataArrayType) -> Result<Cow<'static, [u8]>, ArrayRetrievalError> {
790        use log::trace;
791
792        let mut buf = Vec::new();
793        let decoded = match numpress::decode_slof(data, &mut buf) {
794            Ok(_) => buf,
795            Err(e) => return Err(ArrayRetrievalError::DecompressionError(e.to_string())),
796        };
797        trace!("Numpress SLOF decoded to {} points", decoded.len());
798        match dtype {
799            BinaryDataArrayType::Float64 => {
800                let view = vec_as_bytes(decoded);
801                Ok(Cow::Owned(view))
802            },
803            BinaryDataArrayType::Float32 => {
804                let n = decoded.len() * BinaryDataArrayType::Float32.size_of();
805                trace!("Mapping to {n} bytes for f32 storage");
806                let mut view: Vec<u8> = Vec::with_capacity(n);
807                for val in decoded {
808                    let val = val as f32;
809                    view.extend(bytemuck::bytes_of(&val))
810                }
811                Ok(Cow::Owned(view))
812            },
813            _ => {
814                Err(ArrayRetrievalError::DecompressionError(
815                    BinaryCompressionType::NumpressSLOF.unsupported_msg(Some(
816                        format!("Not compatible with {:?}", dtype).as_str(),
817                    )),
818                ))
819            }
820        }
821    }
822
823    #[cfg(feature = "zstd")]
824    /// Compress the byte buffer using Zstandard compression.
825    ///
826    /// The default compression level is controlled by [`zstd::DEFAULT_COMPRESSION_LEVEL`], but the `MZDATA_ZSTD_LEVEL`
827    /// environment variable can be used to raise or lower it as desired.
828    pub(crate) fn compress_zstd(
829        bytestring: &[u8],
830        dtype: BinaryDataArrayType,
831        shuffle: bool,
832    ) -> Bytes {
833        let level: i32 = std::env::var("MZDATA_ZSTD_LEVEL")
834            .map(|v| v.parse())
835            .unwrap_or(Ok(zstd::DEFAULT_COMPRESSION_LEVEL))
836            .unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL);
837        if !shuffle {
838            return zstd::bulk::compress(bytestring, level).unwrap();
839        }
840        match dtype {
841            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
842                zstd::bulk::compress(bytestring, level).unwrap()
843            }
844            BinaryDataArrayType::Float64 => {
845                zstd::bulk::compress(&transpose_f64(bytemuck::cast_slice(bytestring)), level)
846                    .unwrap()
847            }
848            BinaryDataArrayType::Float32 => {
849                zstd::bulk::compress(&transpose_f32(bytemuck::cast_slice(bytestring)), level)
850                    .unwrap()
851            }
852            BinaryDataArrayType::Int64 => {
853                zstd::bulk::compress(&transpose_i64(bytemuck::cast_slice(bytestring)), level)
854                    .unwrap()
855            }
856            BinaryDataArrayType::Int32 => {
857                zstd::bulk::compress(&transpose_i32(bytemuck::cast_slice(bytestring)), level)
858                    .unwrap()
859            }
860        }
861    }
862
863    #[cfg(feature = "zstd")]
864    pub(crate) fn compress_delta_zstd(
865        bytestring: &[u8],
866        dtype: BinaryDataArrayType,
867        shuffle: bool,
868    ) -> Bytes {
869        use bytemuck::cast_slice;
870
871        use super::delta_encoding;
872
873        match dtype {
874            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
875                Self::compress_zstd(bytestring, dtype, shuffle)
876            }
877            BinaryDataArrayType::Float64 => {
878                let mut buf = cast_slice::<_, f64>(bytestring).to_vec();
879                delta_encoding(&mut buf);
880                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
881            }
882            BinaryDataArrayType::Float32 => {
883                let mut buf = cast_slice::<_, f32>(bytestring).to_vec();
884                delta_encoding(&mut buf);
885                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
886            }
887            BinaryDataArrayType::Int64 => {
888                let mut buf = cast_slice::<_, i64>(bytestring).to_vec();
889                delta_encoding(&mut buf);
890                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
891            }
892            BinaryDataArrayType::Int32 => {
893                let mut buf = cast_slice::<_, i32>(bytestring).to_vec();
894                delta_encoding(&mut buf);
895                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
896            }
897        }
898    }
899
900    #[cfg(feature = "zstd")]
901    pub fn compress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
902        use super::encodings::dictionary_encoding;
903        log::trace!("Dictionary encoding {} bytes as {dtype}", bytestring.len());
904        if bytestring.is_empty() {
905            return Self::compress_zstd(&bytestring, dtype, false);
906        }
907        match dtype {
908            BinaryDataArrayType::Float64 => {
909                let compressed =
910                    dictionary_encoding(bytemuck::cast_slice::<u8, f64>(&bytestring))
911                        .unwrap();
912                let compressed = Self::compress_zstd(&compressed, dtype, false);
913                compressed
914            }
915            BinaryDataArrayType::Float32 => {
916                let compressed =
917                    dictionary_encoding(bytemuck::cast_slice::<u8, f32>(&bytestring))
918                        .unwrap();
919                let compressed = Self::compress_zstd(&compressed, dtype, false);
920                compressed
921            }
922            BinaryDataArrayType::Int64 => {
923                let compressed =
924                    dictionary_encoding(bytemuck::cast_slice::<u8, i64>(&bytestring))
925                        .unwrap();
926                let compressed = Self::compress_zstd(&compressed, dtype, false);
927                compressed
928            }
929            BinaryDataArrayType::Int32 => {
930                let compressed =
931                    dictionary_encoding(bytemuck::cast_slice::<u8, i32>(&bytestring))
932                        .unwrap();
933                let compressed = Self::compress_zstd(&compressed, dtype, false);
934                compressed
935            }
936            _ => {
937                let compressed =
938                    dictionary_encoding(bytemuck::cast_slice::<u8, u8>(&bytestring))
939                        .unwrap();
940                let compressed = Self::compress_zstd(&compressed, dtype, false);
941                compressed
942            }
943        }
944    }
945
946    #[cfg(feature = "zstd")]
947    pub(crate) fn decompress_zstd(data: &[u8], dtype: BinaryDataArrayType, shuffle: bool) -> Bytes {
948        let mut decoder = zstd::Decoder::new(std::io::Cursor::new(data)).unwrap();
949        let mut buf = Vec::new();
950        decoder.read_to_end(&mut buf).unwrap();
951        if !shuffle {
952            return buf;
953        }
954        match dtype {
955            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => buf,
956            BinaryDataArrayType::Float64 => reverse_transpose_f64(&buf),
957            BinaryDataArrayType::Float32 => reverse_transpose_f32(&buf),
958            BinaryDataArrayType::Int64 => reverse_transpose_i64(&buf),
959            BinaryDataArrayType::Int32 => reverse_transpose_i32(&buf),
960        }
961    }
962
963    #[cfg(feature = "zstd")]
964    pub(crate) fn decompress_delta_zstd(
965        data: &[u8],
966        dtype: BinaryDataArrayType,
967        shuffle: bool,
968    ) -> Bytes {
969        use super::delta_decoding;
970
971        let mut delta = Self::decompress_zstd(data, dtype, shuffle);
972        match dtype {
973            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => delta,
974            BinaryDataArrayType::Float64 => {
975                let buf = bytemuck::cast_slice_mut::<_, f64>(&mut delta);
976                delta_decoding(buf);
977                delta
978            }
979            BinaryDataArrayType::Float32 => {
980                let buf = bytemuck::cast_slice_mut::<_, f32>(&mut delta);
981                delta_decoding(buf);
982                delta
983            }
984            BinaryDataArrayType::Int64 => {
985                let buf = bytemuck::cast_slice_mut::<_, i64>(&mut delta);
986                delta_decoding(buf);
987                delta
988            }
989            BinaryDataArrayType::Int32 => {
990                let buf = bytemuck::cast_slice_mut::<_, i32>(&mut delta);
991                delta_decoding(buf);
992                delta
993            }
994        }
995    }
996
997    #[cfg(feature = "zstd")]
998    pub(crate) fn decompress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
999        use super::encodings::dictionary_decoding;
1000
1001        let data = Self::decompress_zstd(bytestring, dtype, false);
1002        match dtype {
1003            BinaryDataArrayType::ASCII | BinaryDataArrayType::Unknown => dictionary_decoding(&data).unwrap(),
1004            BinaryDataArrayType::Float64 => {
1005                to_bytes(&dictionary_decoding::<f64>(&data).unwrap())
1006            },
1007            BinaryDataArrayType::Float32 => {
1008                to_bytes(&dictionary_decoding::<f32>(&data).unwrap())
1009            },
1010            BinaryDataArrayType::Int64 => {
1011                to_bytes(&dictionary_decoding::<i64>(&data).unwrap())
1012            },
1013            BinaryDataArrayType::Int32 => {
1014                to_bytes(&dictionary_decoding::<i32>(&data).unwrap())
1015            },
1016        }
1017    }
1018}
1019
1020impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan> for DataArray {
1021    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1022        self.decode()
1023    }
1024
1025    fn dtype(&self) -> BinaryDataArrayType {
1026        self.dtype
1027    }
1028
1029    fn data_len(&'lifespan self) -> Result<usize, ArrayRetrievalError> {
1030        if let Some(z) = self.item_count {
1031            Ok(z.get())
1032        } else {
1033            let view = self.view()?;
1034            let n = view.len();
1035            Ok(n / self.dtype().size_of())
1036        }
1037    }
1038
1039    fn unit(&self) -> Unit {
1040        self.unit
1041    }
1042
1043    fn data_processing_reference(&self) -> Option<&str> {
1044        self.data_processing_reference()
1045    }
1046
1047    fn name(&self) -> &ArrayType {
1048        &self.name
1049    }
1050}
1051
1052impl<'transient, 'lifespan: 'transient> ByteArrayViewMut<'transient, 'lifespan> for DataArray {
1053    fn view_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
1054        self.decode_mut()
1055    }
1056
1057    fn unit_mut(&mut self) -> &mut Unit {
1058        &mut self.unit
1059    }
1060
1061    fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
1062        self.set_data_processing_reference(data_processing_reference);
1063    }
1064}
1065
1066mzdata_param::impl_param_described_deferred!(DataArray);
1067
1068/// Represent a slice of a [`DataArray`] that manages offsets and decoding automatically.
1069#[derive(Clone, Debug)]
1070pub struct DataArraySlice<'a> {
1071    source: &'a DataArray,
1072    pub start: usize,
1073    pub end: usize,
1074}
1075
1076impl<'a> DataArraySlice<'a> {
1077    pub fn new(source: &'a DataArray, mut start: usize, mut end: usize) -> Self {
1078        if start > end {
1079            mem::swap(&mut start, &mut end);
1080        }
1081        Self { source, start, end }
1082    }
1083
1084    pub fn decode(&'a self) -> Result<Cow<'a, [u8]>, ArrayRetrievalError> {
1085        self.source.decoded_slice(self.start, self.end)
1086    }
1087
1088    pub const fn is_ion_mobility(&self) -> bool {
1089        self.source.is_ion_mobility()
1090    }
1091}
1092
1093impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan>
1094    for DataArraySlice<'lifespan>
1095{
1096    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1097        self.decode()
1098    }
1099
1100    fn dtype(&self) -> BinaryDataArrayType {
1101        self.source.dtype()
1102    }
1103
1104    fn unit(&self) -> Unit {
1105        self.source.unit
1106    }
1107
1108    fn name(&self) -> &ArrayType {
1109        self.source.name()
1110    }
1111
1112    fn data_processing_reference(&self) -> Option<&str> {
1113        self.source.data_processing_reference()
1114    }
1115}
1116
1117#[cfg(test)]
1118mod test {
1119    use super::*;
1120    use std::fs;
1121    use std::io;
1122
1123    use super::DataArray;
1124
1125    fn make_array_from_file() -> io::Result<DataArray> {
1126        let mut fh = fs::File::open("../../test/data/mz_f64_zlib_bas64.txt")?;
1127        let mut buf = String::new();
1128        fh.read_to_string(&mut buf)?;
1129        let bytes: Vec<u8> = buf.into();
1130        let mut da = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
1131        da.compression = BinaryCompressionType::Zlib;
1132        *da.unit_mut() = Unit::MZ;
1133        assert_eq!(da.unit(), Unit::MZ);
1134        assert!(!da.is_ion_mobility());
1135        assert_eq!(da.name(), &ArrayType::MZArray);
1136        Ok(da)
1137    }
1138
1139    #[cfg(feature = "zstd")]
1140    fn make_array_from_file_im_zstd() -> io::Result<DataArray> {
1141        let mut fh = fs::File::open("../../test/data/im_f64_zstd_base64.txt")?;
1142        let mut buf = String::new();
1143        fh.read_to_string(&mut buf)?;
1144        let bytes: Vec<u8> = buf.into();
1145        let mut da = DataArray::wrap(&ArrayType::MeanInverseReducedIonMobilityArray, BinaryDataArrayType::Float64, bytes);
1146        da.compression = BinaryCompressionType::Zstd;
1147        *da.unit_mut() = Unit::VoltSecondPerSquareCentimeter;
1148        assert_eq!(da.unit(), Unit::VoltSecondPerSquareCentimeter);
1149        assert!(da.is_ion_mobility());
1150        assert_eq!(da.name(), &ArrayType::MeanInverseReducedIonMobilityArray);
1151        Ok(da)
1152    }
1153
1154    #[test]
1155    fn test_decode() -> io::Result<()> {
1156        let mut da = make_array_from_file()?;
1157        da.decode_and_store()?;
1158        let view = da.to_f64()?;
1159        assert_eq!(view.len(), 19800);
1160        Ok(())
1161    }
1162
1163    #[test]
1164    fn test_decode_store() -> io::Result<()> {
1165        let mut da = make_array_from_file()?;
1166        da.decode_and_store()?;
1167        let back = da.clone();
1168        da.store_as(BinaryDataArrayType::Float32)?;
1169        let view = da.to_f64()?;
1170        assert_eq!(view.len(), 19800);
1171        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1172            let err = (a - b).abs();
1173            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1174        }
1175        for (a, b) in back.iter_f64()?.zip(da.iter_f32()?.map(|x| x as f64)) {
1176            let err = (a - b).abs();
1177            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1178        }
1179        da.store_as(BinaryDataArrayType::Float64)?;
1180        let view = da.to_f64()?;
1181        assert_eq!(view.len(), 19800);
1182        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1183            let err = (a - b).abs();
1184            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1185        }
1186        Ok(())
1187    }
1188
1189    #[cfg(feature = "zstd")]
1190    #[test]
1191    fn test_decode_delta_zstd() {
1192        let points: Vec<f64> = (0..200_000usize).map(|i| i as f64 * 0.01 + 100.0).collect();
1193        let mut da = DataArray::from_name(&ArrayType::MZArray);
1194        da.extend(&points).unwrap();
1195
1196        let decoded_len = da.data.len();
1197        da.store_compressed(BinaryCompressionType::ShuffleZstd)
1198            .unwrap();
1199        let zstd_len = da.data.len();
1200
1201        da.store_compressed(BinaryCompressionType::Zlib).unwrap();
1202        let zlib_len = da.data.len();
1203
1204        da.decode_and_store().unwrap();
1205
1206        da.store_compressed(BinaryCompressionType::DeltaShuffleZstd)
1207            .unwrap();
1208        let delta_zstd_len = da.data.len();
1209        eprintln!("decoded: {decoded_len};\nzlib: {zlib_len};\nzstd: {zstd_len};\ndelta-zstd: {delta_zstd_len}");
1210        da.decode_and_store().unwrap();
1211        let view = da.to_f64().unwrap();
1212        let err: f64 = points
1213            .iter()
1214            .zip(view.iter())
1215            .map(|(a, b)| {
1216                assert!((a - b).abs() < 1e-3, "{a} - {b} = {}", a - b);
1217                (a - b).abs()
1218            })
1219            .sum();
1220        let mean_err = err / (points.len() as f64);
1221        eprintln!("mean abs error: {mean_err:0.8}")
1222    }
1223
1224    #[cfg(feature = "zstd")]
1225    #[test]
1226    fn test_decode_zstd() -> io::Result<()> {
1227        let mut da = make_array_from_file()?;
1228        let zlib_len = da.data.len();
1229        da.decode_and_store()?;
1230
1231        let decoded_len = da.data.len();
1232
1233        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1234
1235        let zstd_len = da.data.len();
1236
1237        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nzstd: {zstd_len}");
1238        da.decode_and_store()?;
1239
1240        let mut da_ref = make_array_from_file()?;
1241        da_ref.decode_and_store()?;
1242        assert_eq!(da.data, da_ref.data);
1243        Ok(())
1244    }
1245
1246    #[cfg(feature = "numpress")]
1247    #[test]
1248    fn test_numpress_linear() -> io::Result<()> {
1249        let mut da = make_array_from_file()?;
1250        let zlib_len = da.data.len();
1251        da.decode_and_store()?;
1252
1253        let decoded_len = da.data.len();
1254
1255        da.store_compressed(BinaryCompressionType::NumpressLinear)?;
1256        let numpress_len = da.data.len();
1257
1258        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nnumpress: {numpress_len}");
1259        da.decode_and_store()?;
1260
1261        let mut da_ref = make_array_from_file()?;
1262        da_ref.decode_and_store()?;
1263        for (a, b) in da.iter_f64()?.zip(da_ref.iter_f64()?) {
1264            assert!((a - b).abs() < 1e-3, "{a} - {b} = {} which is too large a deviation", (a - b).abs())
1265        }
1266
1267        Ok(())
1268    }
1269
1270    #[test]
1271    fn test_decode_roundtrip() -> io::Result<()> {
1272        let mut da = make_array_from_file()?;
1273        let compressed_size = da.data_len()?;
1274        da.decode_and_store()?;
1275        let view = da.to_f64()?;
1276        assert_eq!(view.len(), 19800);
1277        drop(view);
1278        da.store_compressed(BinaryCompressionType::Zlib)?;
1279        assert_eq!(da.compression, BinaryCompressionType::Zlib);
1280        assert_eq!(da.data_len()?, compressed_size);
1281        let view = da.to_f64()?;
1282        assert_eq!(view.len(), 19800);
1283        Ok(())
1284    }
1285
1286    #[test]
1287    fn test_decode_empty() {
1288        let mut da = DataArray::wrap(
1289            &ArrayType::MZArray,
1290            BinaryDataArrayType::Float64,
1291            Vec::new(),
1292        );
1293        da.compression = BinaryCompressionType::Zlib;
1294
1295        assert_eq!(da.data.len(), 0);
1296        assert_eq!(da.data_len().unwrap(), 0);
1297        assert_eq!(da.decode().unwrap().len(), 0);
1298        assert_eq!(da.to_f64().unwrap().len(), 0);
1299    }
1300
1301
1302    #[cfg(feature = "zstd")]
1303    #[test]
1304    fn test_dict_from_base64() -> io::Result<()> {
1305        let mut da = make_array_from_file_im_zstd()?;
1306
1307        da.decode_and_store()?;
1308        assert_eq!(da.data_len()?, 221);
1309
1310        da.store_compressed(BinaryCompressionType::ZstdDict)?;
1311        assert_eq!(da.data_len()?, 221);
1312
1313        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1314        assert_eq!(da.data_len()?, 221);
1315
1316        Ok(())
1317    }
1318}
mzdata_bindata/array.rs

mzdata_bindata/
array.rs
