mzdata_bindata/
array.rs

1use std::borrow::Cow;
2use std::fmt::{self, Formatter};
3use std::io::prelude::*;
4use std::mem;
5use std::num;
6
7use base64_simd;
8use bytemuck::Pod;
9use flate2::write::{ZlibDecoder, ZlibEncoder};
10use flate2::Compression;
11use num_traits::ToBytes;
12
13use mzdata_param::{ParamList, Unit};
14
15use crate::encodings::{
16    to_bytes, vec_as_bytes, ArrayRetrievalError, ArrayType, BinaryCompressionType, BinaryDataArrayType,
17    Bytes,
18};
19use crate::traits::{ByteArrayView, ByteArrayViewMut};
20
21#[allow(unused)]
22use super::encodings::{
23    reverse_transpose_f32, reverse_transpose_f64, reverse_transpose_i32, reverse_transpose_i64,
24    transpose_f32, transpose_f64, transpose_i32, transpose_i64,
25};
26
27/// Represents a data array that holds a byte buffer that may be compressed, base64 encoded,
28/// or raw little endian bytes, and provides views of those bytes as a small range of supported
29/// types.
30///
31/// This type is modeled after the `<binaryDataArray>` element in mzML.
32///
33/// # Note
34/// This type tries to walk a fine line between convenience and performance and as such is easy
35/// to misuse. All operations that view the byte buffer as arbitrary data need that data to be
36/// decoded and decompressed in order to borrow it. If the byte buffer is not already stored
37/// decoded, the operation will copy and decode the buffer in its entirety before performing any
38/// other operation, so repeated method calls may incur excessive overhead. If this is happening,
39/// please use [`DataArray::decode_and_store`] to store the decoded representation explicitly.
40///
41/// Normally, [`SpectrumSource`](crate::io::SpectrumSource)-implementing file readers will eagerly decode all arrays
42/// as soon as they are ready. If they are operating in lazy mode, the buffers will need to be decoded
43/// explicitly, again using [`DataArray::decode_and_store`] or operations should make as much use of the
44/// copied arrays as possible instead.
45#[derive(Default, Clone)]
46#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
47pub struct DataArray {
48    /// The raw data of the array, stored in a variety of states controlled by [`Self::Compression`], as native bytes
49    pub data: Bytes,
50    /// The *kind* of data stored in the array, as in bit-width primitive types
51    pub dtype: BinaryDataArrayType,
52    /// How [`Self::data`] is encoded and compressed.
53    pub compression: BinaryCompressionType,
54    /// The *what* the data stored in the array was measuring, e.g. an m/z array
55    pub name: ArrayType,
56    /// Additional metadata parameters
57    pub params: Option<Box<ParamList>>,
58    /// The [`Unit`] for the data measurements
59    pub unit: Unit,
60    /// A cache for the number of elements stored in the array
61    item_count: Option<num::NonZero<usize>>,
62    /// An identifier reference to a data processing method other than the default method
63    data_processing_reference: Option<Box<str>>,
64}
65
66impl core::fmt::Debug for DataArray {
67    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
68        f.debug_struct("DataArray")
69            .field("name", &self.name)
70            .field("data size", &self.data.len())
71            .field("dtype", &self.dtype)
72            .field("compression", &self.compression)
73            .field("params", &self.params)
74            .field("unit", &self.unit)
75            .field("data_processing_ref", &self.data_processing_reference)
76            .finish()
77    }
78}
79
80const EMPTY_BUFFER: [u8; 0] = [];
81
82
83impl<'transient, 'lifespan: 'transient> DataArray {
84    pub fn new() -> DataArray {
85        DataArray {
86            ..Default::default()
87        }
88    }
89
90    pub fn from_name(name: &ArrayType) -> DataArray {
91        DataArray {
92            dtype: name.preferred_dtype(),
93            name: name.clone(),
94            compression: BinaryCompressionType::Decoded,
95            ..Default::default()
96        }
97    }
98
99    pub fn from_name_and_type(name: &ArrayType, dtype: BinaryDataArrayType) -> DataArray {
100        DataArray {
101            dtype,
102            name: name.clone(),
103            compression: BinaryCompressionType::Decoded,
104            ..Default::default()
105        }
106    }
107
108    pub fn from_name_type_size(
109        name: &ArrayType,
110        dtype: BinaryDataArrayType,
111        size: usize,
112    ) -> DataArray {
113        DataArray {
114            dtype,
115            name: name.clone(),
116            data: Bytes::with_capacity(size),
117            compression: BinaryCompressionType::Decoded,
118            ..Default::default()
119        }
120    }
121
122    pub fn slice(&self, start: usize, end: usize) -> Result<DataArray, ArrayRetrievalError> {
123        if end < start || (end - start) % self.dtype.size_of() != 0 {
124            Err(ArrayRetrievalError::DataTypeSizeMismatch)
125        } else {
126            let data = self.decode()?;
127            let slice = data[start..end].to_vec();
128            let subset = Self::wrap(&self.name, self.dtype, slice);
129            Ok(subset)
130        }
131    }
132
133    pub fn slice_buffer(
134        &self,
135        start: usize,
136        end: usize,
137    ) -> Result<Cow<'_, [u8]>, ArrayRetrievalError> {
138        if end < start || (end - start) % self.dtype.size_of() != 0 {
139            Err(ArrayRetrievalError::DataTypeSizeMismatch)
140        } else {
141            let data = self.decode()?;
142            match data {
143                Cow::Borrowed(view) => Ok(Cow::Borrowed(&view[start..end])),
144                Cow::Owned(view) => {
145                    let data = &view[start..end];
146                    Ok(Cow::Owned(data.to_owned()))
147                }
148            }
149        }
150    }
151
152    /// This method assumes the data are already in native byte order
153    pub fn wrap(name: &ArrayType, dtype: BinaryDataArrayType, data: Bytes) -> DataArray {
154        DataArray {
155            dtype,
156            name: name.clone(),
157            data,
158            compression: BinaryCompressionType::Decoded,
159            ..Default::default()
160        }
161    }
162
163    /// This method assumes the data are already in native byte order
164    fn set_buffer_of_type(&mut self, data_buffer: Vec<u8>) -> Result<usize, ArrayRetrievalError> {
165        if data_buffer.is_empty() {
166            self.item_count = None;
167        } else {
168            self.item_count = num::NonZero::try_from(data_buffer.len() / self.dtype().size_of()).ok();
169        }
170        self.data = data_buffer;
171        Ok(self.data.len())
172    }
173
174    /// Directly set the data buffer from a slice of a [`Pod`] type, copying it.
175    ///
176    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
177    /// the size of [`DataArray::dtype`].
178    pub fn update_buffer<T: Pod + ToBytes>(
179        &mut self,
180        data_buffer: &[T],
181    ) -> Result<usize, ArrayRetrievalError> {
182        if self.dtype.size_of() != mem::size_of::<T>() {
183            Err(ArrayRetrievalError::DataTypeSizeMismatch)
184        } else {
185            let n = data_buffer.len();
186            self.item_count = num::NonZero::try_from(n).ok();
187            self.data = to_bytes(data_buffer);
188            Ok(self.data.len())
189        }
190    }
191
192    /// Add the value of a [`Pod`] implementing type `T` to the array, converting it
193    /// to native byte representation.
194    ///
195    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
196    /// the size of [`DataArray::dtype`].
197    pub fn push<T: Pod>(&mut self, value: T) -> Result<(), ArrayRetrievalError> {
198        if !matches!(self.compression, BinaryCompressionType::Decoded) {
199            self.decode_and_store()?;
200        };
201        if self.dtype.size_of() != mem::size_of::<T>() {
202            Err(ArrayRetrievalError::DataTypeSizeMismatch)
203        } else {
204            let data = bytemuck::bytes_of(&value);
205            self.data.extend_from_slice(data);
206            self.item_count = self.item_count.map(|i| i.saturating_add(1));
207            Ok(())
208        }
209    }
210
211    /// Add the values from a slice of a [`Pod`] implementing type `T` to the array, converting the slice
212    /// a slice of `T` in its native byte representation.
213    ///
214    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
215    /// the size of [`DataArray::dtype`].
216    pub fn extend<T: Pod>(&mut self, values: &[T]) -> Result<(), ArrayRetrievalError> {
217        if !matches!(self.compression, BinaryCompressionType::Decoded) {
218            self.decode_and_store()?;
219        };
220        if self.dtype.size_of() != mem::size_of::<T>() {
221            Err(ArrayRetrievalError::DataTypeSizeMismatch)
222        } else {
223            self.item_count = self.item_count.map(|i| i.saturating_add(values.len()));
224            let data = bytemuck::cast_slice(values);
225            self.data.extend_from_slice(data);
226            Ok(())
227        }
228    }
229
230    /// Add the values from a buffer of bytes to the array directly.
231    ///
232    /// This will return an error if `values.len()` is not a multiple of
233    /// the size of [`DataArray::dtype`].
234    pub fn extend_raw(&mut self, values: &[u8]) -> Result<(), ArrayRetrievalError> {
235        if values.len() % self.dtype.size_of() != 0 {
236            Err(ArrayRetrievalError::DataTypeSizeMismatch)
237        } else {
238            self.data.extend_from_slice(values);
239            Ok(())
240        }
241    }
242
243    /// Add the values from an iterator over [`Pod`] type `T`.
244    ///
245    /// This is more efficient than repeated calls to [`DataArray::push`] or collecting
246    /// to a `Vec` and then using [`DataArray::extend`], but equivalent to either.
247    ///
248    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
249    /// the size of [`DataArray::dtype`].
250    pub fn extend_iter<T: Pod>(&mut self, iter: impl IntoIterator<Item=T>) -> Result<(), ArrayRetrievalError> {
251        if mem::size_of::<T>() != self.dtype.size_of() {
252            return Err(ArrayRetrievalError::DataTypeSizeMismatch)
253        }
254        for val in iter.into_iter() {
255            let vb = bytemuck::bytes_of(&val);
256            self.data.extend_from_slice(vb);
257        }
258        Ok(())
259    }
260
261    /// Encode the data buffer to a byte array using the requested [`BinaryCompressionType`] to
262    /// compress it and then base64 encode it.
263    pub fn encode_bytestring(&self, compression: BinaryCompressionType) -> Bytes {
264        if self.compression == compression {
265            log::trace!("Fast-path encoding {}:{}", self.name, self.dtype);
266            return self.data.clone();
267        }
268        let bytestring = match self.compression {
269            BinaryCompressionType::Decoded => Cow::Borrowed(self.data.as_slice()),
270            _ => self.decode().expect("Failed to decode binary data"),
271        };
272        match compression {
273            BinaryCompressionType::Decoded => panic!("Should never happen"),
274            BinaryCompressionType::Zlib => {
275                let compressed = Self::compress_zlib(&bytestring);
276                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
277            }
278            BinaryCompressionType::NoCompression => {
279                base64_simd::STANDARD.encode_type::<Bytes>(bytestring.as_ref())
280            }
281            #[cfg(feature = "numpress")]
282            BinaryCompressionType::NumpressLinear => {
283                if self.dtype != BinaryDataArrayType::Float64 {
284                    panic!("Cannot Numpress non-float64 data!");
285                }
286                if bytestring.is_empty() {
287                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
288                }
289                let compressed =
290                    Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring)).unwrap();
291                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
292            }
293            #[cfg(feature = "numpress")]
294            BinaryCompressionType::NumpressSLOF => {
295                let compressed = match self.dtype {
296                    BinaryDataArrayType::Float32 => {
297                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
298                    },
299                    BinaryDataArrayType::Float64 => {
300                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
301                    },
302                    _ => {
303                        panic!("Cannot Numpress non-float data!");
304                    }
305                };
306                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
307            }
308            #[cfg(feature = "numpress")]
309            BinaryCompressionType::NumpressLinearZlib => {
310                if self.dtype != BinaryDataArrayType::Float64 {
311                    panic!("Cannot Numpress non-float64 data!");
312                }
313                if bytestring.is_empty() {
314                    let compressed = Self::compress_zlib(&bytestring);
315                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
316                }
317                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
318                    .inspect_err(|e| {
319                        log::error!("Failed to compress buffer with numpress: {e}");
320                    })
321                    .unwrap();
322                let compressed = Self::compress_zlib(&compressed);
323                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
324            }
325            #[cfg(all(feature = "numpress", feature = "zstd"))]
326            BinaryCompressionType::NumpressLinearZstd => {
327                if self.dtype != BinaryDataArrayType::Float64 {
328                    panic!("Cannot Numpress non-float64 data!");
329                }
330                if bytestring.is_empty() {
331                    let compressed = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
332                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
333                }
334                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
335                    .inspect_err(|e| {
336                        log::error!("Failed to compress buffer with numpress: {e}");
337                    })
338                    .unwrap();
339                let compressed = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
340                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
341            }
342            #[cfg(feature = "numpress")]
343            BinaryCompressionType::NumpressSLOFZlib => {
344                if bytestring.is_empty() {
345                    let bytestring = Self::compress_zlib(&bytestring);
346                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
347                }
348                let compressed = match self.dtype {
349                    BinaryDataArrayType::Float32 => {
350                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
351                    },
352                    BinaryDataArrayType::Float64 => {
353                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
354                    },
355                    _ => {
356                        panic!("Cannot Numpress non-float data!");
357                    }
358                };
359                let bytestring = Self::compress_zlib(&compressed);
360                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
361            }
362            #[cfg(all(feature = "numpress", feature = "zstd"))]
363            BinaryCompressionType::NumpressSLOFZstd => {
364                if bytestring.is_empty() {
365                    let bytestring = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
366                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
367                }
368                let compressed = match self.dtype {
369                    BinaryDataArrayType::Float32 => {
370                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
371                    },
372                    BinaryDataArrayType::Float64 => {
373                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
374                    },
375                    _ => {
376                        panic!("Cannot Numpress non-float data!");
377                    }
378                };
379                let bytestring = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
380                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
381            }
382            #[cfg(feature = "zstd")]
383            BinaryCompressionType::Zstd => {
384                let compressed = Self::compress_zstd(&bytestring, self.dtype, false);
385                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
386            }
387            #[cfg(feature = "zstd")]
388            BinaryCompressionType::ShuffleZstd => {
389                let compressed = Self::compress_zstd(&bytestring, self.dtype, true);
390                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
391            }
392            #[cfg(feature = "zstd")]
393            BinaryCompressionType::DeltaShuffleZstd => {
394                let compressed = Self::compress_delta_zstd(&bytestring, self.dtype, true);
395                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
396            }
397            #[cfg(feature = "zstd")]
398            BinaryCompressionType::ZstdDict => {
399                let compressed = Self::compress_dict_zstd(&bytestring, self.dtype);
400                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
401            }
402            _ => {
403                panic!("Compresion type {:?} is unsupported", compression)
404            }
405        }
406    }
407
408    /// Decode the compressed data, if needed, and store that buffer in `self.data`. After
409    /// decoding `self.compression` will always be [`BinaryCompressionType::Decoded`].
410    ///
411    /// The return value is the content of `self.compression` after decoding.
412    ///
413    /// This may fail if the decoding fails for any reason.
414    pub fn decode_and_store(&mut self) -> Result<BinaryCompressionType, ArrayRetrievalError> {
415        match self.decode() {
416            Ok(data) => {
417                match data {
418                    // The only time this is a borrow is when the data are already
419                    // decoded.
420                    Cow::Borrowed(_view) => Ok(self.compression),
421                    Cow::Owned(buffer) => {
422                        let n = buffer.len() / self.dtype.size_of();
423                        self.item_count = n.try_into().ok();
424                        self.data = buffer;
425                        self.compression = BinaryCompressionType::Decoded;
426                        Ok(self.compression)
427                    }
428                }
429            }
430            Err(err) => Err(err),
431        }
432    }
433
434    /// Decompress and base64-decode encoded bytes, and return the data.
435    ///
436    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
437    /// more buffers may be allocated to hold the decompressed and decoded bytes.
438    pub fn decode(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
439        if self.data.is_empty() {
440            return Ok(Cow::Borrowed(&EMPTY_BUFFER));
441        }
442
443        macro_rules! base64_decode {
444            () => {
445                base64_simd::STANDARD
446                    .decode_type::<Bytes>(&self.data)
447                    .unwrap_or_else(|e| panic!("Failed to decode base64 array: {}", e))
448            };
449        }
450
451        match self.compression {
452            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(self.data.as_slice())),
453            BinaryCompressionType::NoCompression => {
454                let bytestring = base64_decode!();
455                Ok(Cow::Owned(bytestring))
456            }
457            BinaryCompressionType::Zlib => {
458                let mut bytestring = base64_decode!();
459                bytestring = Self::decompress_zlib(&bytestring);
460                Ok(Cow::Owned(bytestring))
461            }
462            #[cfg(feature = "zstd")]
463            BinaryCompressionType::Zstd => {
464                let bytestring = base64_decode!();
465                Ok(Cow::Owned(Self::decompress_zstd(
466                    &bytestring,
467                    self.dtype,
468                    false,
469                )))
470            }
471            #[cfg(feature = "zstd")]
472            BinaryCompressionType::ShuffleZstd => {
473                let bytestring = base64_decode!();
474                Ok(Cow::Owned(Self::decompress_zstd(
475                    &bytestring,
476                    self.dtype,
477                    true,
478                )))
479            }
480            #[cfg(feature = "zstd")]
481            BinaryCompressionType::DeltaShuffleZstd => {
482                let bytestring = base64_decode!();
483                Ok(Cow::Owned(Self::decompress_delta_zstd(
484                    &bytestring,
485                    self.dtype,
486                    true,
487                )))
488            }
489
490            #[cfg(feature = "zstd")]
491            BinaryCompressionType::ZstdDict => {
492                let bytestring = base64_decode!();
493                Ok(Cow::Owned(Self::decompress_dict_zstd(
494                    &bytestring,
495                    self.dtype,
496                )))
497            }
498            #[cfg(feature = "numpress")]
499            BinaryCompressionType::NumpressLinear => match self.dtype {
500                BinaryDataArrayType::Float64 => {
501                    let bytestring = base64_decode!();
502                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
503                    let view = vec_as_bytes(decoded);
504                    Ok(Cow::Owned(view))
505                }
506                _ => Err(ArrayRetrievalError::DecompressionError(
507                    self.compression.unsupported_msg(Some(
508                        format!("Not compatible with {:?}", self.dtype).as_str(),
509                    )),
510                )),
511            },
512
513            #[cfg(feature = "numpress")]
514            BinaryCompressionType::NumpressSLOF => {
515                let bytestring = base64_decode!();
516                Self::decompress_numpress_slof(&bytestring, self.dtype)
517            }
518
519            #[cfg(feature = "numpress")]
520            BinaryCompressionType::NumpressLinearZlib => match self.dtype {
521                BinaryDataArrayType::Float64 => {
522                    let bytestring = base64_decode!();
523                    let bytestring = Self::decompress_zlib(&bytestring);
524                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
525                    let view = vec_as_bytes(decoded);
526                    Ok(Cow::Owned(view))
527                }
528                _ => Err(ArrayRetrievalError::DecompressionError(
529                    self.compression.unsupported_msg(Some(
530                        format!("Not compatible with {:?}", self.dtype).as_str(),
531                    )),
532                )),
533            },
534
535            #[cfg(feature = "numpress")]
536            BinaryCompressionType::NumpressSLOFZlib => {
537                let bytestring = base64_decode!();
538                let bytestring = Self::decompress_zlib(&bytestring);
539                Self::decompress_numpress_slof(&bytestring, self.dtype)
540            }
541
542            #[cfg(all(feature = "numpress", feature = "zstd"))]
543            BinaryCompressionType::NumpressLinearZstd => match self.dtype {
544                BinaryDataArrayType::Float64 => {
545                    let bytestring = base64_decode!();
546                    let bytestring = Self::decompress_zstd(
547                        &bytestring,
548                        BinaryDataArrayType::Unknown,
549                        false,
550                    );
551                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
552                    let view = vec_as_bytes(decoded);
553                    Ok(Cow::Owned(view))
554                }
555                _ => Err(ArrayRetrievalError::DecompressionError(
556                    self.compression.unsupported_msg(Some(
557                        format!("Not compatible with {:?}", self.dtype).as_str(),
558                    )),
559                )),
560            },
561
562            #[cfg(all(feature = "numpress", feature = "zstd"))]
563            BinaryCompressionType::NumpressSLOFZstd => {
564                let bytestring = base64_decode!();
565                let bytestring = Self::decompress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
566                Self::decompress_numpress_slof(&bytestring, self.dtype)
567            }
568
569            mode => Err(ArrayRetrievalError::DecompressionError(format!(
570                "Cannot decode array encoded with {:?}",
571                mode
572            ))),
573        }
574    }
575
576    pub(crate) fn decoded_slice(
577        &'lifespan self,
578        start: usize,
579        end: usize,
580    ) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
581        if start > end || (end - start) % self.dtype.size_of() != 0 {
582            return Err(ArrayRetrievalError::DataTypeSizeMismatch);
583        }
584        match self.compression {
585            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(&self.data.as_slice()[start..end])),
586            _ => {
587                Ok(Cow::Owned(self.decode()?[start..end].to_vec()))
588            }
589        }
590    }
591
592    pub fn decode_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
593        if self.data.is_empty() || matches!(self.compression, BinaryCompressionType::Decoded) {
594            Ok(&mut self.data)
595        } else {
596            match self.decode()? {
597                Cow::Borrowed(_) => {
598                    Ok(&mut self.data)
599                },
600                Cow::Owned(owned) => {
601                    self.data = owned;
602                    self.compression = BinaryCompressionType::Decoded;
603                    Ok(&mut self.data)
604                },
605            }
606        }
607
608    }
609
610    pub fn clear(&mut self) {
611        self.data.clear();
612        self.params = None;
613        self.item_count = None;
614    }
615
616    /// The reverse of [`DataArray::decode_and_store`], this method compresses `self.data` to the desired
617    /// compression method and stores that buffer as `self.data`.
618    pub fn store_compressed(
619        &mut self,
620        compression: BinaryCompressionType,
621    ) -> Result<(), ArrayRetrievalError> {
622        if self.compression == compression {
623            Ok(())
624        } else {
625            self.item_count = self.data_len().ok().and_then(|n| n.try_into().ok());
626            let bytes = self.encode_bytestring(compression);
627            self.data = bytes;
628            self.compression = compression;
629            Ok(())
630        }
631    }
632
633    /// Recode the stored data as the requested binary data type.
634    pub fn store_as(&mut self, dtype: BinaryDataArrayType) -> Result<usize, ArrayRetrievalError> {
635        if self.dtype == dtype {
636            return Ok(self.data.len());
637        }
638        match dtype {
639            BinaryDataArrayType::Float32 => {
640                let view = self.to_f32()?;
641                #[cfg(target_endian = "big")]
642                {
643                    let mut recast = to_bytes(&view);
644                    dtype.swap_bytes(&mut recast)?;
645                    self.dtype = dtype;
646                    self.set_buffer_of_type(recast)
647                }
648                #[cfg(not(target_endian = "big"))]
649                {
650                    let recast = to_bytes(&view);
651                    self.dtype = dtype;
652                    self.set_buffer_of_type(recast)
653                }
654
655            }
656            BinaryDataArrayType::Float64 => {
657                let view = self.to_f64()?;
658                #[cfg(target_endian = "big")]
659                {
660                    let mut recast = to_bytes(&view);
661                    dtype.swap_bytes(&mut recast)?;
662                    self.dtype = dtype;
663                    self.set_buffer_of_type(recast)
664                }
665                #[cfg(not(target_endian = "big"))]
666                {
667                    let recast = to_bytes(&view);
668                    self.dtype = dtype;
669                    self.set_buffer_of_type(recast)
670                }
671            }
672            BinaryDataArrayType::Int32 => {
673                let view = self.to_i32()?;
674                #[cfg(target_endian = "big")]
675                {
676                    let mut recast = to_bytes(&view);
677                    dtype.swap_bytes(&mut recast)?;
678                    self.dtype = dtype;
679                    self.set_buffer_of_type(recast)
680                }
681                #[cfg(not(target_endian = "big"))]
682                {
683                    let recast = to_bytes(&view);
684                    self.dtype = dtype;
685                    self.set_buffer_of_type(recast)
686                }
687            }
688            BinaryDataArrayType::Int64 => {
689                let view = self.to_i64()?;
690                #[cfg(target_endian = "big")]
691                {
692                    let mut recast = to_bytes(&view);
693                    dtype.swap_bytes(&mut recast)?;
694                    self.dtype = dtype;
695                    self.set_buffer_of_type(recast)
696                }
697                #[cfg(not(target_endian = "big"))]
698                {
699                    let recast = to_bytes(&view);
700                    self.dtype = dtype;
701                    self.set_buffer_of_type(recast)
702                }
703            }
704            _ => Ok(0),
705        }
706    }
707
708    /// Test if the the array describes an ion mobility quantity.
709    ///
710    /// # See also
711    /// [`ArrayType::is_ion_mobility`]
712    pub const fn is_ion_mobility(&self) -> bool {
713        self.name.is_ion_mobility()
714    }
715
716    /// The size of the raw byte buffer
717    pub fn raw_len(&self) -> usize {
718        self.data.len()
719    }
720
721    /// Get the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
722    pub fn data_processing_reference(&self) -> Option<&str> {
723        self.data_processing_reference.as_deref()
724    }
725
726    /// Set the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
727    pub fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
728        self.data_processing_reference = data_processing_reference;
729    }
730}
731
732
733/// [`DataArray`] implements several compression codecs, some of which require additional dependencies.
734impl DataArray {
735    pub fn compress_zlib(bytestring: &[u8]) -> Bytes {
736        let result = Bytes::new();
737        let mut compressor = ZlibEncoder::new(result, Compression::best());
738        compressor.write_all(bytestring).expect("Error compressing");
739        compressor.finish().expect("Error compressing")
740    }
741
742    pub fn decompress_zlib(bytestring: &[u8]) -> Bytes {
743        let result = Bytes::new();
744        let mut decompressor = ZlibDecoder::new(result);
745        decompressor
746            .write_all(bytestring)
747            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
748        let buf = decompressor
749            .finish()
750            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
751        buf
752    }
753
754    #[cfg(feature = "numpress")]
755    pub fn compress_numpress_linear(data: &[f64]) -> Result<Bytes, ArrayRetrievalError> {
756        if data.is_empty() {
757            return Ok(Bytes::new());
758        }
759        let scaling = numpress::optimal_scaling(data);
760        match numpress::numpress_compress(data, scaling) {
761            Ok(data) => Ok(data),
762            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
763        }
764    }
765
766    #[cfg(feature = "numpress")]
767    pub fn compress_numpress_slof<T: numpress::AsFloat64>(data: &[T]) -> Result<Bytes, ArrayRetrievalError> {
768        let scaling = numpress::optimal_slof_fixed_point(data);
769        let mut buf = Bytes::new();
770        match numpress::encode_slof(data, &mut buf, scaling) {
771            Ok(_) => Ok(buf),
772            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
773        }
774    }
775
776    #[cfg(feature = "numpress")]
777    pub fn decompress_numpress_linear(data: &[u8]) -> Result<Vec<f64>, ArrayRetrievalError> {
778        if data.is_empty() {
779            return Ok(Vec::new())
780        }
781        match numpress::numpress_decompress(data) {
782            Ok(data) => Ok(data),
783            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
784        }
785    }
786
787    #[cfg(feature = "numpress")]
788    pub fn decompress_numpress_slof(data: &[u8], dtype: BinaryDataArrayType) -> Result<Cow<'static, [u8]>, ArrayRetrievalError> {
789        use log::trace;
790
791        let mut buf = Vec::new();
792        let decoded = match numpress::decode_slof(data, &mut buf) {
793            Ok(_) => buf,
794            Err(e) => return Err(ArrayRetrievalError::DecompressionError(e.to_string())),
795        };
796        trace!("Numpress SLOF decoded to {} points", decoded.len());
797        match dtype {
798            BinaryDataArrayType::Float64 => {
799                let view = vec_as_bytes(decoded);
800                Ok(Cow::Owned(view))
801            },
802            BinaryDataArrayType::Float32 => {
803                let n = decoded.len() * BinaryDataArrayType::Float32.size_of();
804                trace!("Mapping to {n} bytes for f32 storage");
805                let mut view: Vec<u8> = Vec::with_capacity(n);
806                for val in decoded {
807                    let val = val as f32;
808                    view.extend(bytemuck::bytes_of(&val))
809                }
810                Ok(Cow::Owned(view))
811            },
812            _ => {
813                Err(ArrayRetrievalError::DecompressionError(
814                    BinaryCompressionType::NumpressSLOF.unsupported_msg(Some(
815                        format!("Not compatible with {:?}", dtype).as_str(),
816                    )),
817                ))
818            }
819        }
820    }
821
822    #[cfg(feature = "zstd")]
823    /// Compress the byte buffer using Zstandard compression.
824    ///
825    /// The default compression level is controlled by [`zstd::DEFAULT_COMPRESSION_LEVEL`], but the `MZDATA_ZSTD_LEVEL`
826    /// environment variable can be used to raise or lower it as desired.
827    pub(crate) fn compress_zstd(
828        bytestring: &[u8],
829        dtype: BinaryDataArrayType,
830        shuffle: bool,
831    ) -> Bytes {
832        let level: i32 = std::env::var("MZDATA_ZSTD_LEVEL")
833            .map(|v| v.parse())
834            .unwrap_or(Ok(zstd::DEFAULT_COMPRESSION_LEVEL))
835            .unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL);
836        if !shuffle {
837            return zstd::bulk::compress(bytestring, level).unwrap();
838        }
839        match dtype {
840            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
841                zstd::bulk::compress(bytestring, level).unwrap()
842            }
843            BinaryDataArrayType::Float64 => {
844                zstd::bulk::compress(&transpose_f64(bytemuck::cast_slice(bytestring)), level)
845                    .unwrap()
846            }
847            BinaryDataArrayType::Float32 => {
848                zstd::bulk::compress(&transpose_f32(bytemuck::cast_slice(bytestring)), level)
849                    .unwrap()
850            }
851            BinaryDataArrayType::Int64 => {
852                zstd::bulk::compress(&transpose_i64(bytemuck::cast_slice(bytestring)), level)
853                    .unwrap()
854            }
855            BinaryDataArrayType::Int32 => {
856                zstd::bulk::compress(&transpose_i32(bytemuck::cast_slice(bytestring)), level)
857                    .unwrap()
858            }
859        }
860    }
861
862    #[cfg(feature = "zstd")]
863    pub(crate) fn compress_delta_zstd(
864        bytestring: &[u8],
865        dtype: BinaryDataArrayType,
866        shuffle: bool,
867    ) -> Bytes {
868        use bytemuck::cast_slice;
869
870        use super::delta_encoding;
871
872        match dtype {
873            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
874                Self::compress_zstd(bytestring, dtype, shuffle)
875            }
876            BinaryDataArrayType::Float64 => {
877                let mut buf = cast_slice::<_, f64>(bytestring).to_vec();
878                delta_encoding(&mut buf);
879                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
880            }
881            BinaryDataArrayType::Float32 => {
882                let mut buf = cast_slice::<_, f32>(bytestring).to_vec();
883                delta_encoding(&mut buf);
884                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
885            }
886            BinaryDataArrayType::Int64 => {
887                let mut buf = cast_slice::<_, i64>(bytestring).to_vec();
888                delta_encoding(&mut buf);
889                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
890            }
891            BinaryDataArrayType::Int32 => {
892                let mut buf = cast_slice::<_, i32>(bytestring).to_vec();
893                delta_encoding(&mut buf);
894                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
895            }
896        }
897    }
898
899    #[cfg(feature = "zstd")]
900    pub fn compress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
901        use super::encodings::dictionary_encoding;
902        log::trace!("Dictionary encoding {} bytes as {dtype}", bytestring.len());
903        if bytestring.is_empty() {
904            return Self::compress_zstd(&bytestring, dtype, false);
905        }
906        match dtype {
907            BinaryDataArrayType::Float64 => {
908                let compressed =
909                    dictionary_encoding(bytemuck::cast_slice::<u8, f64>(&bytestring))
910                        .unwrap();
911                let compressed = Self::compress_zstd(&compressed, dtype, false);
912                compressed
913            }
914            BinaryDataArrayType::Float32 => {
915                let compressed =
916                    dictionary_encoding(bytemuck::cast_slice::<u8, f32>(&bytestring))
917                        .unwrap();
918                let compressed = Self::compress_zstd(&compressed, dtype, false);
919                compressed
920            }
921            BinaryDataArrayType::Int64 => {
922                let compressed =
923                    dictionary_encoding(bytemuck::cast_slice::<u8, i64>(&bytestring))
924                        .unwrap();
925                let compressed = Self::compress_zstd(&compressed, dtype, false);
926                compressed
927            }
928            BinaryDataArrayType::Int32 => {
929                let compressed =
930                    dictionary_encoding(bytemuck::cast_slice::<u8, i32>(&bytestring))
931                        .unwrap();
932                let compressed = Self::compress_zstd(&compressed, dtype, false);
933                compressed
934            }
935            _ => {
936                let compressed =
937                    dictionary_encoding(bytemuck::cast_slice::<u8, u8>(&bytestring))
938                        .unwrap();
939                let compressed = Self::compress_zstd(&compressed, dtype, false);
940                compressed
941            }
942        }
943    }
944
945    #[cfg(feature = "zstd")]
946    pub(crate) fn decompress_zstd(data: &[u8], dtype: BinaryDataArrayType, shuffle: bool) -> Bytes {
947        let mut decoder = zstd::Decoder::new(std::io::Cursor::new(data)).unwrap();
948        let mut buf = Vec::new();
949        decoder.read_to_end(&mut buf).unwrap();
950        if !shuffle {
951            return buf;
952        }
953        match dtype {
954            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => buf,
955            BinaryDataArrayType::Float64 => reverse_transpose_f64(&buf),
956            BinaryDataArrayType::Float32 => reverse_transpose_f32(&buf),
957            BinaryDataArrayType::Int64 => reverse_transpose_i64(&buf),
958            BinaryDataArrayType::Int32 => reverse_transpose_i32(&buf),
959        }
960    }
961
962    #[cfg(feature = "zstd")]
963    pub(crate) fn decompress_delta_zstd(
964        data: &[u8],
965        dtype: BinaryDataArrayType,
966        shuffle: bool,
967    ) -> Bytes {
968        use super::delta_decoding;
969
970        let mut delta = Self::decompress_zstd(data, dtype, shuffle);
971        match dtype {
972            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => delta,
973            BinaryDataArrayType::Float64 => {
974                let buf = bytemuck::cast_slice_mut::<_, f64>(&mut delta);
975                delta_decoding(buf);
976                delta
977            }
978            BinaryDataArrayType::Float32 => {
979                let buf = bytemuck::cast_slice_mut::<_, f32>(&mut delta);
980                delta_decoding(buf);
981                delta
982            }
983            BinaryDataArrayType::Int64 => {
984                let buf = bytemuck::cast_slice_mut::<_, i64>(&mut delta);
985                delta_decoding(buf);
986                delta
987            }
988            BinaryDataArrayType::Int32 => {
989                let buf = bytemuck::cast_slice_mut::<_, i32>(&mut delta);
990                delta_decoding(buf);
991                delta
992            }
993        }
994    }
995
996    #[cfg(feature = "zstd")]
997    pub(crate) fn decompress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
998        use super::encodings::dictionary_decoding;
999
1000        let data = Self::decompress_zstd(bytestring, dtype, false);
1001        match dtype {
1002            BinaryDataArrayType::ASCII | BinaryDataArrayType::Unknown => dictionary_decoding(&data).unwrap(),
1003            BinaryDataArrayType::Float64 => {
1004                to_bytes(&dictionary_decoding::<f64>(&data).unwrap())
1005            },
1006            BinaryDataArrayType::Float32 => {
1007                to_bytes(&dictionary_decoding::<f32>(&data).unwrap())
1008            },
1009            BinaryDataArrayType::Int64 => {
1010                to_bytes(&dictionary_decoding::<i64>(&data).unwrap())
1011            },
1012            BinaryDataArrayType::Int32 => {
1013                to_bytes(&dictionary_decoding::<i32>(&data).unwrap())
1014            },
1015        }
1016    }
1017}
1018
1019impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan> for DataArray {
1020    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1021        self.decode()
1022    }
1023
1024    fn dtype(&self) -> BinaryDataArrayType {
1025        self.dtype
1026    }
1027
1028    fn data_len(&'lifespan self) -> Result<usize, ArrayRetrievalError> {
1029        if let Some(z) = self.item_count {
1030            Ok(z.get())
1031        } else {
1032            let view = self.view()?;
1033            let n = view.len();
1034            Ok(n / self.dtype().size_of())
1035        }
1036    }
1037
1038    fn unit(&self) -> Unit {
1039        self.unit
1040    }
1041
1042    fn data_processing_reference(&self) -> Option<&str> {
1043        self.data_processing_reference()
1044    }
1045
1046    fn name(&self) -> &ArrayType {
1047        &self.name
1048    }
1049}
1050
1051impl<'transient, 'lifespan: 'transient> ByteArrayViewMut<'transient, 'lifespan> for DataArray {
1052    fn view_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
1053        self.decode_mut()
1054    }
1055
1056    fn unit_mut(&mut self) -> &mut Unit {
1057        &mut self.unit
1058    }
1059
1060    fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
1061        self.set_data_processing_reference(data_processing_reference);
1062    }
1063}
1064
1065mzdata_param::impl_param_described_deferred!(DataArray);
1066
1067/// Represent a slice of a [`DataArray`] that manages offsets and decoding automatically.
1068#[derive(Clone, Debug)]
1069pub struct DataArraySlice<'a> {
1070    source: &'a DataArray,
1071    pub start: usize,
1072    pub end: usize,
1073}
1074
1075impl<'a> DataArraySlice<'a> {
1076    pub fn new(source: &'a DataArray, mut start: usize, mut end: usize) -> Self {
1077        if start > end {
1078            mem::swap(&mut start, &mut end);
1079        }
1080        Self { source, start, end }
1081    }
1082
1083    pub fn decode(&'a self) -> Result<Cow<'a, [u8]>, ArrayRetrievalError> {
1084        self.source.decoded_slice(self.start, self.end)
1085    }
1086
1087    pub const fn is_ion_mobility(&self) -> bool {
1088        self.source.is_ion_mobility()
1089    }
1090}
1091
1092impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan>
1093    for DataArraySlice<'lifespan>
1094{
1095    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1096        self.decode()
1097    }
1098
1099    fn dtype(&self) -> BinaryDataArrayType {
1100        self.source.dtype()
1101    }
1102
1103    fn unit(&self) -> Unit {
1104        self.source.unit
1105    }
1106
1107    fn name(&self) -> &ArrayType {
1108        self.source.name()
1109    }
1110
1111    fn data_processing_reference(&self) -> Option<&str> {
1112        self.source.data_processing_reference()
1113    }
1114}
1115
1116#[cfg(test)]
1117mod test {
1118    use super::*;
1119    use std::fs;
1120    use std::io;
1121
1122    use super::DataArray;
1123
1124    fn make_array_from_file() -> io::Result<DataArray> {
1125        let mut fh = fs::File::open("../../test/data/mz_f64_zlib_bas64.txt")?;
1126        let mut buf = String::new();
1127        fh.read_to_string(&mut buf)?;
1128        let bytes: Vec<u8> = buf.into();
1129        let mut da = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
1130        da.compression = BinaryCompressionType::Zlib;
1131        *da.unit_mut() = Unit::MZ;
1132        assert_eq!(da.unit(), Unit::MZ);
1133        assert!(!da.is_ion_mobility());
1134        assert_eq!(da.name(), &ArrayType::MZArray);
1135        Ok(da)
1136    }
1137
1138    #[cfg(feature = "zstd")]
1139    fn make_array_from_file_im_zstd() -> io::Result<DataArray> {
1140        let mut fh = fs::File::open("../../test/data/im_f64_zstd_base64.txt")?;
1141        let mut buf = String::new();
1142        fh.read_to_string(&mut buf)?;
1143        let bytes: Vec<u8> = buf.into();
1144        let mut da = DataArray::wrap(&ArrayType::MeanInverseReducedIonMobilityArray, BinaryDataArrayType::Float64, bytes);
1145        da.compression = BinaryCompressionType::Zstd;
1146        *da.unit_mut() = Unit::VoltSecondPerSquareCentimeter;
1147        assert_eq!(da.unit(), Unit::VoltSecondPerSquareCentimeter);
1148        assert!(da.is_ion_mobility());
1149        assert_eq!(da.name(), &ArrayType::MeanInverseReducedIonMobilityArray);
1150        Ok(da)
1151    }
1152
1153    #[test]
1154    fn test_decode() -> io::Result<()> {
1155        let mut da = make_array_from_file()?;
1156        da.decode_and_store()?;
1157        let view = da.to_f64()?;
1158        assert_eq!(view.len(), 19800);
1159        Ok(())
1160    }
1161
1162    #[test]
1163    fn test_decode_store() -> io::Result<()> {
1164        let mut da = make_array_from_file()?;
1165        da.decode_and_store()?;
1166        let back = da.clone();
1167        da.store_as(BinaryDataArrayType::Float32)?;
1168        let view = da.to_f64()?;
1169        assert_eq!(view.len(), 19800);
1170        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1171            let err = (a - b).abs();
1172            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1173        }
1174        for (a, b) in back.iter_f64()?.zip(da.iter_f32()?.map(|x| x as f64)) {
1175            let err = (a - b).abs();
1176            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1177        }
1178        da.store_as(BinaryDataArrayType::Float64)?;
1179        let view = da.to_f64()?;
1180        assert_eq!(view.len(), 19800);
1181        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1182            let err = (a - b).abs();
1183            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1184        }
1185        Ok(())
1186    }
1187
1188    #[cfg(feature = "zstd")]
1189    #[test]
1190    fn test_decode_delta_zstd() {
1191        let points: Vec<f64> = (0..200_000usize).map(|i| i as f64 * 0.01 + 100.0).collect();
1192        let mut da = DataArray::from_name(&ArrayType::MZArray);
1193        da.extend(&points).unwrap();
1194
1195        let decoded_len = da.data.len();
1196        da.store_compressed(BinaryCompressionType::ShuffleZstd)
1197            .unwrap();
1198        let zstd_len = da.data.len();
1199
1200        da.store_compressed(BinaryCompressionType::Zlib).unwrap();
1201        let zlib_len = da.data.len();
1202
1203        da.decode_and_store().unwrap();
1204
1205        da.store_compressed(BinaryCompressionType::DeltaShuffleZstd)
1206            .unwrap();
1207        let delta_zstd_len = da.data.len();
1208        eprintln!("decoded: {decoded_len};\nzlib: {zlib_len};\nzstd: {zstd_len};\ndelta-zstd: {delta_zstd_len}");
1209        da.decode_and_store().unwrap();
1210        let view = da.to_f64().unwrap();
1211        let err: f64 = points
1212            .iter()
1213            .zip(view.iter())
1214            .map(|(a, b)| {
1215                assert!((a - b).abs() < 1e-3, "{a} - {b} = {}", a - b);
1216                (a - b).abs()
1217            })
1218            .sum();
1219        let mean_err = err / (points.len() as f64);
1220        eprintln!("mean abs error: {mean_err:0.8}")
1221    }
1222
1223    #[cfg(feature = "zstd")]
1224    #[test]
1225    fn test_decode_zstd() -> io::Result<()> {
1226        let mut da = make_array_from_file()?;
1227        let zlib_len = da.data.len();
1228        da.decode_and_store()?;
1229
1230        let decoded_len = da.data.len();
1231
1232        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1233
1234        let zstd_len = da.data.len();
1235
1236        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nzstd: {zstd_len}");
1237        da.decode_and_store()?;
1238
1239        let mut da_ref = make_array_from_file()?;
1240        da_ref.decode_and_store()?;
1241        assert_eq!(da.data, da_ref.data);
1242        Ok(())
1243    }
1244
1245    #[cfg(feature = "numpress")]
1246    #[test]
1247    fn test_numpress_linear() -> io::Result<()> {
1248        let mut da = make_array_from_file()?;
1249        let zlib_len = da.data.len();
1250        da.decode_and_store()?;
1251
1252        let decoded_len = da.data.len();
1253
1254        da.store_compressed(BinaryCompressionType::NumpressLinear)?;
1255        let numpress_len = da.data.len();
1256
1257        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nnumpress: {numpress_len}");
1258        da.decode_and_store()?;
1259
1260        let mut da_ref = make_array_from_file()?;
1261        da_ref.decode_and_store()?;
1262        for (a, b) in da.iter_f64()?.zip(da_ref.iter_f64()?) {
1263            assert!((a - b).abs() < 1e-3, "{a} - {b} = {} which is too large a deviation", (a - b).abs())
1264        }
1265
1266        Ok(())
1267    }
1268
1269    #[test]
1270    fn test_decode_roundtrip() -> io::Result<()> {
1271        let mut da = make_array_from_file()?;
1272        let compressed_size = da.data_len()?;
1273        da.decode_and_store()?;
1274        let view = da.to_f64()?;
1275        assert_eq!(view.len(), 19800);
1276        drop(view);
1277        da.store_compressed(BinaryCompressionType::Zlib)?;
1278        assert_eq!(da.compression, BinaryCompressionType::Zlib);
1279        assert_eq!(da.data_len()?, compressed_size);
1280        let view = da.to_f64()?;
1281        assert_eq!(view.len(), 19800);
1282        Ok(())
1283    }
1284
1285    #[test]
1286    fn test_decode_empty() {
1287        let mut da = DataArray::wrap(
1288            &ArrayType::MZArray,
1289            BinaryDataArrayType::Float64,
1290            Vec::new(),
1291        );
1292        da.compression = BinaryCompressionType::Zlib;
1293
1294        assert_eq!(da.data.len(), 0);
1295        assert_eq!(da.data_len().unwrap(), 0);
1296        assert_eq!(da.decode().unwrap().len(), 0);
1297        assert_eq!(da.to_f64().unwrap().len(), 0);
1298    }
1299
1300
1301    #[cfg(feature = "zstd")]
1302    #[test]
1303    fn test_dict_from_base64() -> io::Result<()> {
1304        let mut da = make_array_from_file_im_zstd()?;
1305
1306        da.decode_and_store()?;
1307        assert_eq!(da.data_len()?, 221);
1308
1309        da.store_compressed(BinaryCompressionType::ZstdDict)?;
1310        assert_eq!(da.data_len()?, 221);
1311
1312        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1313        assert_eq!(da.data_len()?, 221);
1314
1315        Ok(())
1316    }
1317}
mzdata_bindata/array.rs

mzdata_bindata/
array.rs
