mzdata_bindata/
array.rs

1use std::borrow::Cow;
2use std::fmt::{self, Formatter};
3use std::io::prelude::*;
4use std::mem;
5use std::num;
6
7use base64_simd;
8use bytemuck::Pod;
9use flate2::write::{ZlibDecoder, ZlibEncoder};
10use flate2::Compression;
11use num_traits::ToBytes;
12
13use mzdata_param::{ParamList, Unit};
14
15#[allow(unused)]
16use crate::encodings::{
17    to_bytes, vec_as_bytes, ArrayRetrievalError, ArrayType, BinaryCompressionType, BinaryDataArrayType,
18    Bytes,
19};
20use crate::traits::{ByteArrayView, ByteArrayViewMut};
21
22#[allow(unused)]
23use super::encodings::{
24    reverse_transpose_f32, reverse_transpose_f64, reverse_transpose_i32, reverse_transpose_i64,
25    transpose_f32, transpose_f64, transpose_i32, transpose_i64,
26};
27
28/// Represents a data array that holds a byte buffer that may be compressed, base64 encoded,
29/// or raw little endian bytes, and provides views of those bytes as a small range of supported
30/// types.
31///
32/// This type is modeled after the `<binaryDataArray>` element in mzML.
33///
34/// # Note
35/// This type tries to walk a fine line between convenience and performance and as such is easy
36/// to misuse. All operations that view the byte buffer as arbitrary data need that data to be
37/// decoded and decompressed in order to borrow it. If the byte buffer is not already stored
38/// decoded, the operation will copy and decode the buffer in its entirety before performing any
39/// other operation, so repeated method calls may incur excessive overhead. If this is happening,
40/// please use [`DataArray::decode_and_store`] to store the decoded representation explicitly.
41///
42/// Normally, [`SpectrumSource`](crate::io::SpectrumSource)-implementing file readers will eagerly decode all arrays
43/// as soon as they are ready. If they are operating in lazy mode, the buffers will need to be decoded
44/// explicitly, again using [`DataArray::decode_and_store`] or operations should make as much use of the
45/// copied arrays as possible instead.
46#[derive(Default, Clone)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub struct DataArray {
49    /// The raw data of the array, stored in a variety of states controlled by [`Self::Compression`], as native bytes
50    pub data: Bytes,
51    /// The *kind* of data stored in the array, as in bit-width primitive types
52    pub dtype: BinaryDataArrayType,
53    /// How [`Self::data`] is encoded and compressed.
54    pub compression: BinaryCompressionType,
55    /// The *what* the data stored in the array was measuring, e.g. an m/z array
56    pub name: ArrayType,
57    /// Additional metadata parameters
58    pub params: Option<Box<ParamList>>,
59    /// The [`Unit`] for the data measurements
60    pub unit: Unit,
61    /// A cache for the number of elements stored in the array
62    item_count: Option<num::NonZero<usize>>,
63    /// An identifier reference to a data processing method other than the default method
64    data_processing_reference: Option<Box<str>>,
65}
66
67impl core::fmt::Debug for DataArray {
68    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
69        f.debug_struct("DataArray")
70            .field("name", &self.name)
71            .field("data size", &self.data.len())
72            .field("dtype", &self.dtype)
73            .field("compression", &self.compression)
74            .field("params", &self.params)
75            .field("unit", &self.unit)
76            .field("data_processing_ref", &self.data_processing_reference)
77            .finish()
78    }
79}
80
81const EMPTY_BUFFER: [u8; 0] = [];
82
83
84impl<'transient, 'lifespan: 'transient> DataArray {
85    /// An alias of [`Default::default`], no configuration is
86    /// performed whatsoever.
87    pub fn new() -> DataArray {
88        DataArray {
89            ..Default::default()
90        }
91    }
92
93    /// Create a new [`DataArray`] for a given `name` in its preferred data type
94    /// returned by [`ArrayType::preferred_dtype`] and unit returned by [`ArrayType::preferred_unit`]
95    pub fn from_name(name: &ArrayType) -> DataArray {
96        DataArray {
97            dtype: name.preferred_dtype(),
98            name: name.clone(),
99            compression: BinaryCompressionType::Decoded,
100            unit: name.preferred_unit().unwrap_or_default(),
101            ..Default::default()
102        }
103    }
104
105    /// Create a new [`DataArray`] with a given `name` and `dtype`, using
106    /// `name`'s preferred units. Does not allocate any additional storage.
107    pub fn from_name_and_type(name: &ArrayType, dtype: BinaryDataArrayType) -> DataArray {
108        DataArray {
109            dtype,
110            name: name.clone(),
111            compression: BinaryCompressionType::Decoded,
112            unit: name.preferred_unit().unwrap_or_default(),
113            ..Default::default()
114        }
115    }
116
117    /// Create a new [`DataArray`] with a given `name` and `dtype`, using
118    /// `name`'s preferred units. Pre-allocates but does not fill
119    /// `size` storage capacity in bytes (not elements of `dtype`).
120    pub fn from_name_type_size(
121        name: &ArrayType,
122        dtype: BinaryDataArrayType,
123        size: usize,
124    ) -> DataArray {
125        DataArray {
126            dtype,
127            name: name.clone(),
128            data: Bytes::with_capacity(size),
129            compression: BinaryCompressionType::Decoded,
130            unit: name.preferred_unit().unwrap_or_default(),
131            ..Default::default()
132        }
133    }
134
135    /// Extract a subrange of the encoded byte buffer as a new [`DataArray`].
136    ///
137    /// `start` and `end` are **byte offsets** into the decoded buffer, not element
138    /// indices, so they must be aligned to a multiple of [`DataArray::dtype`]'s size.
139    ///
140    /// # Examples
141    /// ```
142    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
143    ///
144    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
145    /// array.extend(&[1.0f64, 2.0, 3.0, 4.0]).unwrap();
146    ///
147    /// // Each `f64` is 8 bytes wide, so the 2nd and 3rd elements occupy bytes [8, 24).
148    /// let middle = array.slice(8, 24).unwrap();
149    /// assert_eq!(&*middle.to_f64().unwrap(), &[2.0f64, 3.0f64][..]);
150    /// ```
151    pub fn slice(&self, start: usize, end: usize) -> Result<DataArray, ArrayRetrievalError> {
152        if end < start || (end - start) % self.dtype.size_of() != 0 {
153            Err(ArrayRetrievalError::DataTypeSizeMismatch)
154        } else {
155            let data = self.decode()?;
156            let slice = data[start..end].to_vec();
157            let subset = Self::wrap(&self.name, self.dtype, slice);
158            Ok(subset)
159        }
160    }
161
162    /// Retrieve a slice of raw bytes with no concern for value type alignment.
163    ///
164    /// `start` and `end` are **byte offsets** into the decoded buffer
165    pub fn slice_buffer(
166        &self,
167        start: usize,
168        end: usize,
169    ) -> Result<Cow<'_, [u8]>, ArrayRetrievalError> {
170        if end < start || (end - start) % self.dtype.size_of() != 0 {
171            Err(ArrayRetrievalError::DataTypeSizeMismatch)
172        } else {
173            let data = self.decode()?;
174            match data {
175                Cow::Borrowed(view) => Ok(Cow::Borrowed(&view[start..end])),
176                Cow::Owned(view) => {
177                    let data = &view[start..end];
178                    Ok(Cow::Owned(data.to_owned()))
179                }
180            }
181        }
182    }
183
184    /// This method assumes the data are already in native byte order.
185    ///
186    /// This is not endian-safe *yet*, but is valid for in-memory operations.
187    ///
188    /// # Examples
189    /// ```
190    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
191    ///
192    /// let values: [f64; 3] = [100.5, 200.25, 300.0];
193    /// let bytes: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
194    /// let array = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
195    ///
196    /// let decoded = array.to_f64().unwrap();
197    /// assert_eq!(&*decoded, &values[..]);
198    /// ```
199    pub fn wrap(name: &ArrayType, dtype: BinaryDataArrayType, data: Bytes) -> DataArray {
200        DataArray {
201            dtype,
202            name: name.clone(),
203            data,
204            compression: BinaryCompressionType::Decoded,
205            ..Default::default()
206        }
207    }
208
209    /// This method assumes the data are already in native byte order
210    fn set_buffer_of_type(&mut self, data_buffer: Vec<u8>) -> Result<usize, ArrayRetrievalError> {
211        if data_buffer.is_empty() {
212            self.item_count = None;
213        } else {
214            self.item_count = num::NonZero::try_from(data_buffer.len() / self.dtype().size_of()).ok();
215        }
216        self.data = data_buffer;
217        Ok(self.data.len())
218    }
219
220    /// Directly set the data buffer from a slice of a [`Pod`] type, copying it.
221    ///
222    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
223    /// the size of [`DataArray::dtype`].
224    pub fn update_buffer<T: Pod + ToBytes>(
225        &mut self,
226        data_buffer: &[T],
227    ) -> Result<usize, ArrayRetrievalError> {
228        if self.dtype.size_of() != mem::size_of::<T>() {
229            Err(ArrayRetrievalError::DataTypeSizeMismatch)
230        } else {
231            let n = data_buffer.len();
232            self.item_count = num::NonZero::try_from(n).ok();
233            self.data = to_bytes(data_buffer);
234            Ok(self.data.len())
235        }
236    }
237
238    /// Add the value of a [`Pod`] implementing type `T` to the array, converting it
239    /// to native byte representation.
240    ///
241    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
242    /// the size of [`DataArray::dtype`].
243    ///
244    /// # Examples
245    /// ```
246    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
247    ///
248    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
249    /// array.push(500.5f64).unwrap();
250    /// array.push(501.0f64).unwrap();
251    ///
252    /// assert_eq!(&*array.to_f64().unwrap(), &[500.5f64, 501.0f64][..]);
253    /// ```
254    pub fn push<T: Pod>(&mut self, value: T) -> Result<(), ArrayRetrievalError> {
255        if !matches!(self.compression, BinaryCompressionType::Decoded) {
256            self.decode_and_store()?;
257        };
258        if self.dtype.size_of() != mem::size_of::<T>() {
259            Err(ArrayRetrievalError::DataTypeSizeMismatch)
260        } else {
261            let data = bytemuck::bytes_of(&value);
262            self.data.extend_from_slice(data);
263            self.item_count = self.item_count.map(|i| i.saturating_add(1));
264            Ok(())
265        }
266    }
267
268    /// Add the values from a slice of a [`Pod`] implementing type `T` to the array, converting the slice
269    /// a slice of `T` in its native byte representation.
270    ///
271    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
272    /// the size of [`DataArray::dtype`].
273    ///
274    /// # Examples
275    /// ```
276    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
277    ///
278    /// let mut array = DataArray::from_name_and_type(&ArrayType::IntensityArray, BinaryDataArrayType::Float32);
279    /// array.extend(&[10.0f32, 20.0, 30.0]).unwrap();
280    ///
281    /// assert_eq!(&*array.to_f32().unwrap(), &[10.0f32, 20.0f32, 30.0f32][..]);
282    /// ```
283    pub fn extend<T: Pod>(&mut self, values: &[T]) -> Result<(), ArrayRetrievalError> {
284        if !matches!(self.compression, BinaryCompressionType::Decoded) {
285            self.decode_and_store()?;
286        };
287        if self.dtype.size_of() != mem::size_of::<T>() {
288            Err(ArrayRetrievalError::DataTypeSizeMismatch)
289        } else {
290            self.item_count = self.item_count.map(|i| i.saturating_add(values.len()));
291            let data = bytemuck::cast_slice(values);
292            self.data.extend_from_slice(data);
293            Ok(())
294        }
295    }
296
297    /// Add the values from a buffer of bytes to the array directly.
298    ///
299    /// This will return an error if `values.len()` is not a multiple of
300    /// the size of [`DataArray::dtype`].
301    pub fn extend_raw(&mut self, values: &[u8]) -> Result<(), ArrayRetrievalError> {
302        if values.len() % self.dtype.size_of() != 0 {
303            Err(ArrayRetrievalError::DataTypeSizeMismatch)
304        } else {
305            self.data.extend_from_slice(values);
306            Ok(())
307        }
308    }
309
310    /// Add the values from an iterator over [`Pod`] type `T`.
311    ///
312    /// This is more efficient than repeated calls to [`DataArray::push`] or collecting
313    /// to a `Vec` and then using [`DataArray::extend`], but equivalent to either.
314    ///
315    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
316    /// the size of [`DataArray::dtype`].
317    pub fn extend_iter<T: Pod>(&mut self, iter: impl IntoIterator<Item=T>) -> Result<(), ArrayRetrievalError> {
318        if mem::size_of::<T>() != self.dtype.size_of() {
319            return Err(ArrayRetrievalError::DataTypeSizeMismatch)
320        }
321        for val in iter.into_iter() {
322            let vb = bytemuck::bytes_of(&val);
323            self.data.extend_from_slice(vb);
324        }
325        Ok(())
326    }
327
328    /// Encode the data buffer to a byte array using the requested [`BinaryCompressionType`] to
329    /// compress it and then base64 encode it.
330    ///
331    /// This method takes the current byte buffer in [`Self::data`] and does the following:
332    ///     1. If [`Self::compression`] is `compression`, just return a copy of the data as-is,
333    ///         it is pre-coded properly.
334    ///     2. If [`Self::compression`] is not [`BinaryCompressionType::Decoded`], copy and decode
335    ///         [`Self::data`], otherwise borrow the raw byte data.
336    ///     3. Apply `compression` to the decoded raw bytes, performing one or more transformations
337    ///         and then base64 encode the resulting compressed byte array.
338    ///     4. Return the base64 encoded, compressed byte array
339    ///
340    /// # Panics
341    /// This method will panic if compression fails for any reason, or if an unsupported
342    /// compression method is requested. Some compression methods are only available if
343    /// appropriate features are enabled.
344    pub fn encode_bytestring(&self, compression: BinaryCompressionType) -> Bytes {
345        if self.compression == compression {
346            log::trace!("Fast-path encoding {}:{}", self.name, self.dtype);
347            return self.data.clone();
348        }
349        let bytestring = match self.compression {
350            BinaryCompressionType::Decoded => Cow::Borrowed(self.data.as_slice()),
351            _ => self.decode().expect("Failed to decode binary data"),
352        };
353        match compression {
354            BinaryCompressionType::Decoded => panic!("Should never happen"),
355            BinaryCompressionType::Zlib => {
356                let compressed = Self::compress_zlib(&bytestring);
357                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
358            }
359            BinaryCompressionType::NoCompression => {
360                base64_simd::STANDARD.encode_type::<Bytes>(bytestring.as_ref())
361            }
362            #[cfg(feature = "numpress")]
363            BinaryCompressionType::NumpressLinear => {
364                if self.dtype != BinaryDataArrayType::Float64 {
365                    panic!("Cannot Numpress non-float64 data!");
366                }
367                if bytestring.is_empty() {
368                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
369                }
370                let compressed =
371                    Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring)).unwrap();
372                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
373            }
374            #[cfg(feature = "numpress")]
375            BinaryCompressionType::NumpressSLOF => {
376                let compressed = match self.dtype {
377                    BinaryDataArrayType::Float32 => {
378                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
379                    },
380                    BinaryDataArrayType::Float64 => {
381                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
382                    },
383                    _ => {
384                        panic!("Cannot Numpress non-float data!");
385                    }
386                };
387                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
388            }
389            #[cfg(feature = "numpress")]
390            BinaryCompressionType::NumpressLinearZlib => {
391                if self.dtype != BinaryDataArrayType::Float64 {
392                    panic!("Cannot Numpress non-float64 data!");
393                }
394                if bytestring.is_empty() {
395                    let compressed = Self::compress_zlib(&bytestring);
396                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
397                }
398                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
399                    .inspect_err(|e| {
400                        log::error!("Failed to compress buffer with numpress: {e}");
401                    })
402                    .unwrap();
403                let compressed = Self::compress_zlib(&compressed);
404                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
405            }
406            #[cfg(all(feature = "numpress", feature = "zstd"))]
407            BinaryCompressionType::NumpressLinearZstd => {
408                if self.dtype != BinaryDataArrayType::Float64 {
409                    panic!("Cannot Numpress non-float64 data!");
410                }
411                if bytestring.is_empty() {
412                    let compressed = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
413                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
414                }
415                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
416                    .inspect_err(|e| {
417                        log::error!("Failed to compress buffer with numpress: {e}");
418                    })
419                    .unwrap();
420                let compressed = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
421                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
422            }
423            #[cfg(feature = "numpress")]
424            BinaryCompressionType::NumpressSLOFZlib => {
425                if bytestring.is_empty() {
426                    let bytestring = Self::compress_zlib(&bytestring);
427                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
428                }
429                let compressed = match self.dtype {
430                    BinaryDataArrayType::Float32 => {
431                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
432                    },
433                    BinaryDataArrayType::Float64 => {
434                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
435                    },
436                    _ => {
437                        panic!("Cannot Numpress non-float data!");
438                    }
439                };
440                let bytestring = Self::compress_zlib(&compressed);
441                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
442            }
443            #[cfg(all(feature = "numpress", feature = "zstd"))]
444            BinaryCompressionType::NumpressSLOFZstd => {
445                if bytestring.is_empty() {
446                    let bytestring = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
447                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
448                }
449                let compressed = match self.dtype {
450                    BinaryDataArrayType::Float32 => {
451                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
452                    },
453                    BinaryDataArrayType::Float64 => {
454                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
455                    },
456                    _ => {
457                        panic!("Cannot Numpress non-float data!");
458                    }
459                };
460                let bytestring = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
461                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
462            }
463            #[cfg(feature = "zstd")]
464            BinaryCompressionType::Zstd => {
465                let compressed = Self::compress_zstd(&bytestring, self.dtype, false);
466                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
467            }
468            #[cfg(feature = "zstd")]
469            BinaryCompressionType::ShuffleZstd => {
470                let compressed = Self::compress_zstd(&bytestring, self.dtype, true);
471                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
472            }
473            #[cfg(feature = "zstd")]
474            BinaryCompressionType::DeltaShuffleZstd => {
475                let compressed = Self::compress_delta_zstd(&bytestring, self.dtype, true);
476                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
477            }
478            #[cfg(feature = "zstd")]
479            BinaryCompressionType::ZstdDict => {
480                let compressed = Self::compress_dict_zstd(&bytestring, self.dtype);
481                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
482            }
483            _ => {
484                panic!("Compresion type {:?} is unsupported", compression)
485            }
486        }
487    }
488
489    /// Decode the compressed data, if needed, and store that buffer in `self.data`. After
490    /// decoding `self.compression` will always be [`BinaryCompressionType::Decoded`].
491    ///
492    /// The return value is the content of `self.compression` after decoding.
493    ///
494    /// This may fail if the decoding fails for any reason.
495    ///
496    /// # Examples
497    /// ```
498    /// use mzdata_bindata::{ArrayType, BinaryCompressionType, BinaryDataArrayType, ByteArrayView, DataArray};
499    ///
500    /// let values: [f64; 4] = [1.0, 2.0, 3.0, 4.0];
501    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
502    /// array.extend(&values).unwrap();
503    ///
504    /// array.store_compressed(BinaryCompressionType::Zlib).unwrap();
505    /// assert_eq!(array.compression, BinaryCompressionType::Zlib);
506    ///
507    /// array.decode_and_store().unwrap();
508    /// assert_eq!(array.compression, BinaryCompressionType::Decoded);
509    /// assert_eq!(&*array.to_f64().unwrap(), &values[..]);
510    /// ```
511    pub fn decode_and_store(&mut self) -> Result<BinaryCompressionType, ArrayRetrievalError> {
512        match self.decode() {
513            Ok(data) => {
514                match data {
515                    // The only time this is a borrow is when the data are already
516                    // decoded.
517                    Cow::Borrowed(_view) => Ok(self.compression),
518                    Cow::Owned(buffer) => {
519                        let n = buffer.len() / self.dtype.size_of();
520                        self.item_count = n.try_into().ok();
521                        self.data = buffer;
522                        self.compression = BinaryCompressionType::Decoded;
523                        Ok(self.compression)
524                    }
525                }
526            }
527            Err(err) => Err(err),
528        }
529    }
530
531    /// Decompress and base64-decode encoded bytes, and return the data.
532    ///
533    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
534    /// more buffers may be allocated to hold the decompressed and decoded bytes.
535    ///
536    /// # Panics
537    /// This method panics if [`Self::compression`] fails for any reason or is not supported.
538    /// Several compression methods require certain features be enabled.
539    pub fn decode(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
540        if self.data.is_empty() {
541            return Ok(Cow::Borrowed(&EMPTY_BUFFER));
542        }
543
544        macro_rules! base64_decode {
545            () => {
546                base64_simd::STANDARD
547                    .decode_type::<Bytes>(&self.data)
548                    .map_err(|_| ArrayRetrievalError::MalformedBase64Encoding)?
549            };
550        }
551
552        match self.compression {
553            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(self.data.as_slice())),
554            BinaryCompressionType::NoCompression => {
555                let bytestring = base64_decode!();
556                Ok(Cow::Owned(bytestring))
557            }
558            BinaryCompressionType::Zlib => {
559                let mut bytestring = base64_decode!();
560                bytestring = Self::decompress_zlib(&bytestring);
561                Ok(Cow::Owned(bytestring))
562            }
563            #[cfg(feature = "zstd")]
564            BinaryCompressionType::Zstd => {
565                let bytestring = base64_decode!();
566                Ok(Cow::Owned(Self::decompress_zstd(
567                    &bytestring,
568                    self.dtype,
569                    false,
570                )))
571            }
572            #[cfg(feature = "zstd")]
573            BinaryCompressionType::ShuffleZstd => {
574                let bytestring = base64_decode!();
575                Ok(Cow::Owned(Self::decompress_zstd(
576                    &bytestring,
577                    self.dtype,
578                    true,
579                )))
580            }
581            #[cfg(feature = "zstd")]
582            BinaryCompressionType::DeltaShuffleZstd => {
583                let bytestring = base64_decode!();
584                Ok(Cow::Owned(Self::decompress_delta_zstd(
585                    &bytestring,
586                    self.dtype,
587                    true,
588                )))
589            }
590
591            #[cfg(feature = "zstd")]
592            BinaryCompressionType::ZstdDict => {
593                let bytestring = base64_decode!();
594                Ok(Cow::Owned(Self::decompress_dict_zstd(
595                    &bytestring,
596                    self.dtype,
597                )))
598            }
599            #[cfg(feature = "numpress")]
600            BinaryCompressionType::NumpressLinear => match self.dtype {
601                BinaryDataArrayType::Float64 => {
602                    let bytestring = base64_decode!();
603                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
604                    let view = vec_as_bytes(decoded);
605                    Ok(Cow::Owned(view))
606                }
607                _ => Err(ArrayRetrievalError::DecompressionError(
608                    self.compression.unsupported_msg(Some(
609                        format!("Not compatible with {:?}", self.dtype).as_str(),
610                    )),
611                )),
612            },
613
614            #[cfg(feature = "numpress")]
615            BinaryCompressionType::NumpressSLOF => {
616                let bytestring = base64_decode!();
617                Self::decompress_numpress_slof(&bytestring, self.dtype)
618            }
619
620            #[cfg(feature = "numpress")]
621            BinaryCompressionType::NumpressLinearZlib => match self.dtype {
622                BinaryDataArrayType::Float64 => {
623                    let bytestring = base64_decode!();
624                    let bytestring = Self::decompress_zlib(&bytestring);
625                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
626                    let view = vec_as_bytes(decoded);
627                    Ok(Cow::Owned(view))
628                }
629                _ => Err(ArrayRetrievalError::DecompressionError(
630                    self.compression.unsupported_msg(Some(
631                        format!("Not compatible with {:?}", self.dtype).as_str(),
632                    )),
633                )),
634            },
635
636            #[cfg(feature = "numpress")]
637            BinaryCompressionType::NumpressSLOFZlib => {
638                let bytestring = base64_decode!();
639                let bytestring = Self::decompress_zlib(&bytestring);
640                Self::decompress_numpress_slof(&bytestring, self.dtype)
641            }
642
643            #[cfg(all(feature = "numpress", feature = "zstd"))]
644            BinaryCompressionType::NumpressLinearZstd => match self.dtype {
645                BinaryDataArrayType::Float64 => {
646                    let bytestring = base64_decode!();
647                    let bytestring = Self::decompress_zstd(
648                        &bytestring,
649                        BinaryDataArrayType::Unknown,
650                        false,
651                    );
652                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
653                    let view = vec_as_bytes(decoded);
654                    Ok(Cow::Owned(view))
655                }
656                _ => Err(ArrayRetrievalError::DecompressionError(
657                    self.compression.unsupported_msg(Some(
658                        format!("Not compatible with {:?}", self.dtype).as_str(),
659                    )),
660                )),
661            },
662
663            #[cfg(all(feature = "numpress", feature = "zstd"))]
664            BinaryCompressionType::NumpressSLOFZstd => {
665                let bytestring = base64_decode!();
666                let bytestring = Self::decompress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
667                Self::decompress_numpress_slof(&bytestring, self.dtype)
668            }
669
670            mode => Err(ArrayRetrievalError::DecompressionError(format!(
671                "Cannot decode array encoded with {:?}",
672                mode
673            ))),
674        }
675    }
676
677    /// Get a slice of the raw bytes aligned to [`Self::dtype`].
678    ///
679    /// If the data are already decoded, this does not create a copy of the data.
680    ///
681    /// This is an aligned version of [`Self::slice_buffer`].
682    pub(crate) fn decoded_slice(
683        &'lifespan self,
684        start: usize,
685        end: usize,
686    ) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
687        if start > end || (end - start) % self.dtype.size_of() != 0 {
688            return Err(ArrayRetrievalError::DataTypeSizeMismatch);
689        }
690        match self.compression {
691            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(&self.data.as_slice()[start..end])),
692            _ => {
693                Ok(Cow::Owned(self.decode()?[start..end].to_vec()))
694            }
695        }
696    }
697
698    /// Get a mutable view of the raw bytes.
699    ///
700    /// If the data are already decoded, this just borrows [`Self::data`], otherwise
701    /// the [`Self::decode`] is called [`Self::data`] is updated to the decoded copy first,
702    /// then borrowed.
703    ///
704    /// Note that care **MUST** be taken here to avoid writing invalid values when working on the
705    /// raw bytes directly. Prefer to update the native values indirectly via
706    /// [`ByteArrayViewMut::coerce_mut`].
707    pub fn decode_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
708        if self.data.is_empty() || matches!(self.compression, BinaryCompressionType::Decoded) {
709            Ok(&mut self.data)
710        } else {
711            match self.decode()? {
712                Cow::Borrowed(_) => {
713                    Ok(&mut self.data)
714                },
715                Cow::Owned(owned) => {
716                    self.data = owned;
717                    self.compression = BinaryCompressionType::Decoded;
718                    Ok(&mut self.data)
719                },
720            }
721        }
722
723    }
724
725    /// Erase all the data in the array, calling [`Vec::clear`] on [`Self::data`],
726    /// and erasing all [`Param`] values. All other properties remain unchanged.
727    pub fn clear(&mut self) {
728        self.data.clear();
729        self.params = None;
730        self.item_count = None;
731    }
732
733    /// The reverse of [`DataArray::decode_and_store`], this method compresses `self.data` to the desired
734    /// compression method and stores that buffer as `self.data`.
735    pub fn store_compressed(
736        &mut self,
737        compression: BinaryCompressionType,
738    ) -> Result<(), ArrayRetrievalError> {
739        if self.compression == compression {
740            Ok(())
741        } else {
742            self.item_count = self.data_len().ok().and_then(|n| n.try_into().ok());
743            let bytes = self.encode_bytestring(compression);
744            self.data = bytes;
745            self.compression = compression;
746            Ok(())
747        }
748    }
749
750    /// Recode the stored data as the requested binary data type.
751    ///
752    /// This is a no-op if the [`Self::dtype`] matches the `dtype` argument.
753    /// The data will be decoded and decompressed after this call ends. Any previous
754    /// compression will need to be re-applied.
755    ///
756    /// ## Note
757    /// When down-casting to a lower precision type, e.g. `f64` to `f32`, data may be lost.
758    /// Take care to do this only when necessary or when the loss of precision is acceptable.
759    pub fn store_as(&mut self, dtype: BinaryDataArrayType) -> Result<usize, ArrayRetrievalError> {
760        if self.dtype == dtype {
761            return Ok(self.data.len());
762        }
763        let z = match dtype {
764            BinaryDataArrayType::Float32 => {
765                let view = self.to_f32()?;
766                #[cfg(target_endian = "big")]
767                {
768                    let mut recast = to_bytes(&view);
769                    dtype.swap_bytes(&mut recast)?;
770                    self.dtype = dtype;
771                    self.set_buffer_of_type(recast)
772                }
773                #[cfg(not(target_endian = "big"))]
774                {
775                    let recast = to_bytes(&view);
776                    self.dtype = dtype;
777                    self.set_buffer_of_type(recast)
778                }
779
780            }
781            BinaryDataArrayType::Float64 => {
782                let view = self.to_f64()?;
783                #[cfg(target_endian = "big")]
784                {
785                    let mut recast = to_bytes(&view);
786                    dtype.swap_bytes(&mut recast)?;
787                    self.dtype = dtype;
788                    self.set_buffer_of_type(recast)
789                }
790                #[cfg(not(target_endian = "big"))]
791                {
792                    let recast = to_bytes(&view);
793                    self.dtype = dtype;
794                    self.set_buffer_of_type(recast)
795                }
796            }
797            BinaryDataArrayType::Int32 => {
798                let view = self.to_i32()?;
799                #[cfg(target_endian = "big")]
800                {
801                    let mut recast = to_bytes(&view);
802                    dtype.swap_bytes(&mut recast)?;
803                    self.dtype = dtype;
804                    self.set_buffer_of_type(recast)
805                }
806                #[cfg(not(target_endian = "big"))]
807                {
808                    let recast = to_bytes(&view);
809                    self.dtype = dtype;
810                    self.set_buffer_of_type(recast)
811                }
812            }
813            BinaryDataArrayType::Int64 => {
814                let view = self.to_i64()?;
815                #[cfg(target_endian = "big")]
816                {
817                    let mut recast = to_bytes(&view);
818                    dtype.swap_bytes(&mut recast)?;
819                    self.dtype = dtype;
820                    self.set_buffer_of_type(recast)
821                }
822                #[cfg(not(target_endian = "big"))]
823                {
824                    let recast = to_bytes(&view);
825                    self.dtype = dtype;
826                    self.set_buffer_of_type(recast)
827                }
828            }
829            _ => Ok(0),
830        }?;
831        self.compression = BinaryCompressionType::Decoded;
832        Ok(z)
833    }
834
835    /// Test if the the array describes an ion mobility quantity.
836    ///
837    /// # See also
838    /// [`ArrayType::is_ion_mobility`]
839    pub const fn is_ion_mobility(&self) -> bool {
840        self.name.is_ion_mobility()
841    }
842
843    /// The size of the raw byte buffer
844    pub fn raw_len(&self) -> usize {
845        self.data.len()
846    }
847
848    /// Get the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
849    pub fn data_processing_reference(&self) -> Option<&str> {
850        self.data_processing_reference.as_deref()
851    }
852
853    /// Set the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
854    pub fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
855        self.data_processing_reference = data_processing_reference;
856    }
857}
858
859
860/// [`DataArray`] implements several compression codecs, some of which require additional dependencies.
861///
862/// These methods are often cumbersome and not easy to use as they require a lot of extra context. They
863/// live here as a historical artefact. They shouldn't be used directly in the vast majority of cases.
864impl DataArray {
865
866    /// Compress the provided bytes using `zlib` at the highest compression setting
867    pub(crate) fn compress_zlib(bytestring: &[u8]) -> Bytes {
868        let result = Bytes::new();
869        let mut compressor = ZlibEncoder::new(result, Compression::best());
870        compressor.write_all(bytestring).expect("Error compressing");
871        compressor.finish().expect("Error compressing")
872    }
873
874    /// Decompress the provided bytes using `zlib`.
875    ///
876    /// # Panics
877    /// If the data are not `zlib` compressed.
878    pub(crate) fn decompress_zlib(bytestring: &[u8]) -> Bytes {
879        let result = Bytes::new();
880        let mut decompressor = ZlibDecoder::new(result);
881        decompressor
882            .write_all(bytestring)
883            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
884        let buf = decompressor
885            .finish()
886            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
887        buf
888    }
889
890    #[cfg(feature = "numpress")]
891    /// Encode the provided `f64`` data using MS-Numpress Linear encoding.
892    ///
893    /// The data **MUST** be pre-sorted or else the regression model will provide
894    /// worse results.
895    pub(crate) fn compress_numpress_linear(data: &[f64]) -> Result<Bytes, ArrayRetrievalError> {
896        if data.is_empty() {
897            return Ok(Bytes::new());
898        }
899        let scaling = numpress::optimal_scaling(data);
900        match numpress::numpress_compress(data, scaling) {
901            Ok(data) => Ok(data),
902            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
903        }
904    }
905
906    #[cfg(feature = "numpress")]
907    pub(crate) fn compress_numpress_slof<T: numpress::AsFloat64>(data: &[T]) -> Result<Bytes, ArrayRetrievalError> {
908        let scaling = numpress::optimal_slof_fixed_point(data);
909        let mut buf = Bytes::new();
910        match numpress::encode_slof(data, &mut buf, scaling) {
911            Ok(_) => Ok(buf),
912            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
913        }
914    }
915
916    #[cfg(feature = "numpress")]
917    /// Decode a MS-Numpress Linear encoded byte array.
918    pub(crate) fn decompress_numpress_linear(data: &[u8]) -> Result<Vec<f64>, ArrayRetrievalError> {
919        if data.is_empty() {
920            return Ok(Vec::new())
921        }
922        match numpress::numpress_decompress(data) {
923            Ok(data) => Ok(data),
924            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
925        }
926    }
927
928    #[cfg(feature = "numpress")]
929    pub(crate) fn decompress_numpress_slof(data: &[u8], dtype: BinaryDataArrayType) -> Result<Cow<'static, [u8]>, ArrayRetrievalError> {
930        use log::trace;
931
932        let mut buf = Vec::new();
933        let decoded = match numpress::decode_slof(data, &mut buf) {
934            Ok(_) => buf,
935            Err(e) => return Err(ArrayRetrievalError::DecompressionError(e.to_string())),
936        };
937        trace!("Numpress SLOF decoded to {} points", decoded.len());
938        match dtype {
939            BinaryDataArrayType::Float64 => {
940                let view = vec_as_bytes(decoded);
941                Ok(Cow::Owned(view))
942            },
943            BinaryDataArrayType::Float32 => {
944                let n = decoded.len() * BinaryDataArrayType::Float32.size_of();
945                trace!("Mapping to {n} bytes for f32 storage");
946                let mut view: Vec<u8> = Vec::with_capacity(n);
947                for val in decoded {
948                    let val = val as f32;
949                    view.extend(bytemuck::bytes_of(&val))
950                }
951                Ok(Cow::Owned(view))
952            },
953            _ => {
954                Err(ArrayRetrievalError::DecompressionError(
955                    BinaryCompressionType::NumpressSLOF.unsupported_msg(Some(
956                        format!("Not compatible with {:?}", dtype).as_str(),
957                    )),
958                ))
959            }
960        }
961    }
962
963    #[cfg(feature = "zstd")]
964    /// Compress the byte buffer using Zstandard compression.
965    ///
966    /// The default compression level is controlled by [`zstd::DEFAULT_COMPRESSION_LEVEL`], but the `MZDATA_ZSTD_LEVEL`
967    /// environment variable can be used to raise or lower it as desired.
968    pub(crate) fn compress_zstd(
969        bytestring: &[u8],
970        dtype: BinaryDataArrayType,
971        shuffle: bool,
972    ) -> Bytes {
973        let level: i32 = std::env::var("MZDATA_ZSTD_LEVEL")
974            .map(|v| v.parse())
975            .unwrap_or(Ok(zstd::DEFAULT_COMPRESSION_LEVEL))
976            .unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL);
977        if !shuffle {
978            return zstd::bulk::compress(bytestring, level).unwrap();
979        }
980        match dtype {
981            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
982                zstd::bulk::compress(bytestring, level).unwrap()
983            }
984            BinaryDataArrayType::Float64 => {
985                zstd::bulk::compress(&transpose_f64(bytemuck::cast_slice(bytestring)), level)
986                    .unwrap()
987            }
988            BinaryDataArrayType::Float32 => {
989                zstd::bulk::compress(&transpose_f32(bytemuck::cast_slice(bytestring)), level)
990                    .unwrap()
991            }
992            BinaryDataArrayType::Int64 => {
993                zstd::bulk::compress(&transpose_i64(bytemuck::cast_slice(bytestring)), level)
994                    .unwrap()
995            }
996            BinaryDataArrayType::Int32 => {
997                zstd::bulk::compress(&transpose_i32(bytemuck::cast_slice(bytestring)), level)
998                    .unwrap()
999            }
1000        }
1001    }
1002
1003    #[cfg(feature = "zstd")]
1004    pub(crate) fn compress_delta_zstd(
1005        bytestring: &[u8],
1006        dtype: BinaryDataArrayType,
1007        shuffle: bool,
1008    ) -> Bytes {
1009        use bytemuck::cast_slice;
1010
1011        use super::delta_encoding;
1012
1013        match dtype {
1014            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
1015                Self::compress_zstd(bytestring, dtype, shuffle)
1016            }
1017            BinaryDataArrayType::Float64 => {
1018                let mut buf = cast_slice::<_, f64>(bytestring).to_vec();
1019                delta_encoding(&mut buf);
1020                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
1021            }
1022            BinaryDataArrayType::Float32 => {
1023                let mut buf = cast_slice::<_, f32>(bytestring).to_vec();
1024                delta_encoding(&mut buf);
1025                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
1026            }
1027            BinaryDataArrayType::Int64 => {
1028                let mut buf = cast_slice::<_, i64>(bytestring).to_vec();
1029                delta_encoding(&mut buf);
1030                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
1031            }
1032            BinaryDataArrayType::Int32 => {
1033                let mut buf = cast_slice::<_, i32>(bytestring).to_vec();
1034                delta_encoding(&mut buf);
1035                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
1036            }
1037        }
1038    }
1039
1040    #[cfg(feature = "zstd")]
1041    pub(crate) fn compress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
1042        use super::encodings::dictionary_encoding;
1043        log::trace!("Dictionary encoding {} bytes as {dtype}", bytestring.len());
1044        if bytestring.is_empty() {
1045            return Self::compress_zstd(&bytestring, dtype, false);
1046        }
1047        match dtype {
1048            BinaryDataArrayType::Float64 => {
1049                let compressed =
1050                    dictionary_encoding(bytemuck::cast_slice::<u8, f64>(&bytestring))
1051                        .unwrap();
1052                let compressed = Self::compress_zstd(&compressed, dtype, false);
1053                compressed
1054            }
1055            BinaryDataArrayType::Float32 => {
1056                let compressed =
1057                    dictionary_encoding(bytemuck::cast_slice::<u8, f32>(&bytestring))
1058                        .unwrap();
1059                let compressed = Self::compress_zstd(&compressed, dtype, false);
1060                compressed
1061            }
1062            BinaryDataArrayType::Int64 => {
1063                let compressed =
1064                    dictionary_encoding(bytemuck::cast_slice::<u8, i64>(&bytestring))
1065                        .unwrap();
1066                let compressed = Self::compress_zstd(&compressed, dtype, false);
1067                compressed
1068            }
1069            BinaryDataArrayType::Int32 => {
1070                let compressed =
1071                    dictionary_encoding(bytemuck::cast_slice::<u8, i32>(&bytestring))
1072                        .unwrap();
1073                let compressed = Self::compress_zstd(&compressed, dtype, false);
1074                compressed
1075            }
1076            _ => {
1077                let compressed =
1078                    dictionary_encoding(bytemuck::cast_slice::<u8, u8>(&bytestring))
1079                        .unwrap();
1080                let compressed = Self::compress_zstd(&compressed, dtype, false);
1081                compressed
1082            }
1083        }
1084    }
1085
1086    #[cfg(feature = "zstd")]
1087    pub(crate) fn decompress_zstd(data: &[u8], dtype: BinaryDataArrayType, shuffle: bool) -> Bytes {
1088        let mut decoder = zstd::Decoder::new(std::io::Cursor::new(data)).unwrap();
1089        let mut buf = Vec::new();
1090        decoder.read_to_end(&mut buf).unwrap();
1091        if !shuffle {
1092            return buf;
1093        }
1094        match dtype {
1095            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => buf,
1096            BinaryDataArrayType::Float64 => reverse_transpose_f64(&buf),
1097            BinaryDataArrayType::Float32 => reverse_transpose_f32(&buf),
1098            BinaryDataArrayType::Int64 => reverse_transpose_i64(&buf),
1099            BinaryDataArrayType::Int32 => reverse_transpose_i32(&buf),
1100        }
1101    }
1102
1103    #[cfg(feature = "zstd")]
1104    pub(crate) fn decompress_delta_zstd(
1105        data: &[u8],
1106        dtype: BinaryDataArrayType,
1107        shuffle: bool,
1108    ) -> Bytes {
1109        use super::delta_decoding;
1110
1111        let mut delta = Self::decompress_zstd(data, dtype, shuffle);
1112        match dtype {
1113            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => delta,
1114            BinaryDataArrayType::Float64 => {
1115                let buf = bytemuck::cast_slice_mut::<_, f64>(&mut delta);
1116                delta_decoding(buf);
1117                delta
1118            }
1119            BinaryDataArrayType::Float32 => {
1120                let buf = bytemuck::cast_slice_mut::<_, f32>(&mut delta);
1121                delta_decoding(buf);
1122                delta
1123            }
1124            BinaryDataArrayType::Int64 => {
1125                let buf = bytemuck::cast_slice_mut::<_, i64>(&mut delta);
1126                delta_decoding(buf);
1127                delta
1128            }
1129            BinaryDataArrayType::Int32 => {
1130                let buf = bytemuck::cast_slice_mut::<_, i32>(&mut delta);
1131                delta_decoding(buf);
1132                delta
1133            }
1134        }
1135    }
1136
1137    #[cfg(feature = "zstd")]
1138    pub(crate) fn decompress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
1139        use super::encodings::dictionary_decoding;
1140
1141        let data = Self::decompress_zstd(bytestring, dtype, false);
1142        match dtype {
1143            BinaryDataArrayType::ASCII | BinaryDataArrayType::Unknown => dictionary_decoding(&data).unwrap(),
1144            BinaryDataArrayType::Float64 => {
1145                to_bytes(&dictionary_decoding::<f64>(&data).unwrap())
1146            },
1147            BinaryDataArrayType::Float32 => {
1148                to_bytes(&dictionary_decoding::<f32>(&data).unwrap())
1149            },
1150            BinaryDataArrayType::Int64 => {
1151                to_bytes(&dictionary_decoding::<i64>(&data).unwrap())
1152            },
1153            BinaryDataArrayType::Int32 => {
1154                to_bytes(&dictionary_decoding::<i32>(&data).unwrap())
1155            },
1156        }
1157    }
1158}
1159
1160impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan> for DataArray {
1161    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1162        self.decode()
1163    }
1164
1165    fn dtype(&self) -> BinaryDataArrayType {
1166        self.dtype
1167    }
1168
1169    fn data_len(&'lifespan self) -> Result<usize, ArrayRetrievalError> {
1170        if let Some(z) = self.item_count {
1171            Ok(z.get())
1172        } else {
1173            let view = self.view()?;
1174            let n = view.len();
1175            Ok(n / self.dtype().size_of())
1176        }
1177    }
1178
1179    fn unit(&self) -> Unit {
1180        self.unit
1181    }
1182
1183    fn data_processing_reference(&self) -> Option<&str> {
1184        self.data_processing_reference()
1185    }
1186
1187    fn name(&self) -> &ArrayType {
1188        &self.name
1189    }
1190}
1191
1192impl<'transient, 'lifespan: 'transient> ByteArrayViewMut<'transient, 'lifespan> for DataArray {
1193    fn view_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
1194        self.decode_mut()
1195    }
1196
1197    fn unit_mut(&mut self) -> &mut Unit {
1198        &mut self.unit
1199    }
1200
1201    fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
1202        self.set_data_processing_reference(data_processing_reference);
1203    }
1204}
1205
1206mzdata_param::impl_param_described_deferred!(DataArray);
1207
1208/// Represent a slice of a [`DataArray`] that manages offsets and decoding automatically.
1209///
1210/// This **does not** copy the data.
1211#[derive(Clone, Debug)]
1212pub struct DataArraySlice<'a> {
1213    source: &'a DataArray,
1214    /// The start of the slice, in bytes
1215    pub start: usize,
1216    /// The end of the slice, in bytes
1217    pub end: usize,
1218}
1219
1220impl<'a> DataArraySlice<'a> {
1221    /// Construct a new [`DataArraySlice`]
1222    pub fn new(source: &'a DataArray, mut start: usize, mut end: usize) -> Self {
1223        if start > end {
1224            mem::swap(&mut start, &mut end);
1225        }
1226        Self { source, start, end }
1227    }
1228
1229    /// Decompress and base64-decode encoded bytes, and return the data.
1230    ///
1231    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
1232    /// more buffers may be allocated to hold the decompressed and decoded bytes.
1233    ///
1234    /// This is a thin wrapper around [`DataArray::decoded_slice`].
1235    pub fn decode(&'a self) -> Result<Cow<'a, [u8]>, ArrayRetrievalError> {
1236        self.source.decoded_slice(self.start, self.end)
1237    }
1238
1239    /// Test if the the array describes an ion mobility quantity.
1240    ///
1241    /// # See also
1242    /// [`ArrayType::is_ion_mobility`]
1243    pub const fn is_ion_mobility(&self) -> bool {
1244        self.source.is_ion_mobility()
1245    }
1246
1247    /// Get an reference to the source [`DataArray`]
1248    pub fn source(&self) -> &'a DataArray {
1249        self.source
1250    }
1251}
1252
1253impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan>
1254    for DataArraySlice<'lifespan>
1255{
1256    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
1257        self.decode()
1258    }
1259
1260    fn dtype(&self) -> BinaryDataArrayType {
1261        self.source.dtype()
1262    }
1263
1264    fn unit(&self) -> Unit {
1265        self.source.unit
1266    }
1267
1268    fn name(&self) -> &ArrayType {
1269        self.source.name()
1270    }
1271
1272    fn data_processing_reference(&self) -> Option<&str> {
1273        self.source.data_processing_reference()
1274    }
1275}
1276
1277#[cfg(test)]
1278mod test {
1279    use super::*;
1280    use std::fs;
1281    use std::io;
1282
1283    use super::DataArray;
1284
1285    fn make_array_from_file() -> io::Result<DataArray> {
1286        let mut fh = fs::File::open("../../test/data/mz_f64_zlib_bas64.txt")?;
1287        let mut buf = String::new();
1288        fh.read_to_string(&mut buf)?;
1289        let bytes: Vec<u8> = buf.into();
1290        let mut da = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
1291        da.compression = BinaryCompressionType::Zlib;
1292        *da.unit_mut() = Unit::MZ;
1293        assert_eq!(da.unit(), Unit::MZ);
1294        assert!(!da.is_ion_mobility());
1295        assert_eq!(da.name(), &ArrayType::MZArray);
1296        Ok(da)
1297    }
1298
1299    #[cfg(feature = "zstd")]
1300    fn make_array_from_file_im_zstd() -> io::Result<DataArray> {
1301        let mut fh = fs::File::open("../../test/data/im_f64_zstd_base64.txt")?;
1302        let mut buf = String::new();
1303        fh.read_to_string(&mut buf)?;
1304        let bytes: Vec<u8> = buf.into();
1305        let mut da = DataArray::wrap(&ArrayType::MeanInverseReducedIonMobilityArray, BinaryDataArrayType::Float64, bytes);
1306        da.compression = BinaryCompressionType::Zstd;
1307        *da.unit_mut() = Unit::VoltSecondPerSquareCentimeter;
1308        assert_eq!(da.unit(), Unit::VoltSecondPerSquareCentimeter);
1309        assert!(da.is_ion_mobility());
1310        assert_eq!(da.name(), &ArrayType::MeanInverseReducedIonMobilityArray);
1311        Ok(da)
1312    }
1313
1314    #[test]
1315    fn test_decode() -> io::Result<()> {
1316        let mut da = make_array_from_file()?;
1317        da.decode_and_store()?;
1318        let view = da.to_f64()?;
1319        assert_eq!(view.len(), 19800);
1320        Ok(())
1321    }
1322
1323    #[test]
1324    fn test_decode_store() -> io::Result<()> {
1325        let mut da = make_array_from_file()?;
1326        da.decode_and_store()?;
1327        let back = da.clone();
1328        da.store_as(BinaryDataArrayType::Float32)?;
1329        let view = da.to_f64()?;
1330        assert_eq!(view.len(), 19800);
1331        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1332            let err = (a - b).abs();
1333            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1334        }
1335        for (a, b) in back.iter_f64()?.zip(da.iter_f32()?.map(|x| x as f64)) {
1336            let err = (a - b).abs();
1337            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1338        }
1339        da.store_as(BinaryDataArrayType::Float64)?;
1340        let view = da.to_f64()?;
1341        assert_eq!(view.len(), 19800);
1342        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
1343            let err = (a - b).abs();
1344            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
1345        }
1346        Ok(())
1347    }
1348
1349    #[cfg(feature = "zstd")]
1350    #[test]
1351    fn test_decode_delta_zstd() {
1352        let points: Vec<f64> = (0..200_000usize).map(|i| i as f64 * 0.01 + 100.0).collect();
1353        let mut da = DataArray::from_name(&ArrayType::MZArray);
1354        da.extend(&points).unwrap();
1355
1356        let decoded_len = da.data.len();
1357        da.store_compressed(BinaryCompressionType::ShuffleZstd)
1358            .unwrap();
1359        let zstd_len = da.data.len();
1360
1361        da.store_compressed(BinaryCompressionType::Zlib).unwrap();
1362        let zlib_len = da.data.len();
1363
1364        da.decode_and_store().unwrap();
1365
1366        da.store_compressed(BinaryCompressionType::DeltaShuffleZstd)
1367            .unwrap();
1368        let delta_zstd_len = da.data.len();
1369        eprintln!("decoded: {decoded_len};\nzlib: {zlib_len};\nzstd: {zstd_len};\ndelta-zstd: {delta_zstd_len}");
1370        da.decode_and_store().unwrap();
1371        let view = da.to_f64().unwrap();
1372        let err: f64 = points
1373            .iter()
1374            .zip(view.iter())
1375            .map(|(a, b)| {
1376                assert!((a - b).abs() < 1e-3, "{a} - {b} = {}", a - b);
1377                (a - b).abs()
1378            })
1379            .sum();
1380        let mean_err = err / (points.len() as f64);
1381        eprintln!("mean abs error: {mean_err:0.8}")
1382    }
1383
1384    #[cfg(feature = "zstd")]
1385    #[test]
1386    fn test_decode_zstd() -> io::Result<()> {
1387        let mut da = make_array_from_file()?;
1388        let zlib_len = da.data.len();
1389        da.decode_and_store()?;
1390
1391        let decoded_len = da.data.len();
1392
1393        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1394
1395        let zstd_len = da.data.len();
1396
1397        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nzstd: {zstd_len}");
1398        da.decode_and_store()?;
1399
1400        let mut da_ref = make_array_from_file()?;
1401        da_ref.decode_and_store()?;
1402        assert_eq!(da.data, da_ref.data);
1403        Ok(())
1404    }
1405
1406    #[cfg(feature = "numpress")]
1407    #[test]
1408    fn test_numpress_linear() -> io::Result<()> {
1409        let mut da = make_array_from_file()?;
1410        let zlib_len = da.data.len();
1411        da.decode_and_store()?;
1412
1413        let decoded_len = da.data.len();
1414
1415        da.store_compressed(BinaryCompressionType::NumpressLinear)?;
1416        let numpress_len = da.data.len();
1417
1418        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nnumpress: {numpress_len}");
1419        da.decode_and_store()?;
1420
1421        let mut da_ref = make_array_from_file()?;
1422        da_ref.decode_and_store()?;
1423        for (a, b) in da.iter_f64()?.zip(da_ref.iter_f64()?) {
1424            assert!((a - b).abs() < 1e-3, "{a} - {b} = {} which is too large a deviation", (a - b).abs())
1425        }
1426
1427        Ok(())
1428    }
1429
1430    #[test]
1431    fn test_decode_roundtrip() -> io::Result<()> {
1432        let mut da = make_array_from_file()?;
1433        let compressed_size = da.data_len()?;
1434        da.decode_and_store()?;
1435        let view = da.to_f64()?;
1436        assert_eq!(view.len(), 19800);
1437        drop(view);
1438        da.store_compressed(BinaryCompressionType::Zlib)?;
1439        assert_eq!(da.compression, BinaryCompressionType::Zlib);
1440        assert_eq!(da.data_len()?, compressed_size);
1441        let view = da.to_f64()?;
1442        assert_eq!(view.len(), 19800);
1443        Ok(())
1444    }
1445
1446    #[test]
1447    fn test_decode_empty() {
1448        let mut da = DataArray::wrap(
1449            &ArrayType::MZArray,
1450            BinaryDataArrayType::Float64,
1451            Vec::new(),
1452        );
1453        da.compression = BinaryCompressionType::Zlib;
1454
1455        assert_eq!(da.data.len(), 0);
1456        assert_eq!(da.data_len().unwrap(), 0);
1457        assert_eq!(da.decode().unwrap().len(), 0);
1458        assert_eq!(da.to_f64().unwrap().len(), 0);
1459    }
1460
1461
1462    #[cfg(feature = "zstd")]
1463    #[test]
1464    fn test_dict_from_base64() -> io::Result<()> {
1465        let mut da = make_array_from_file_im_zstd()?;
1466
1467        da.decode_and_store()?;
1468        assert_eq!(da.data_len()?, 221);
1469
1470        da.store_compressed(BinaryCompressionType::ZstdDict)?;
1471        assert_eq!(da.data_len()?, 221);
1472
1473        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
1474        assert_eq!(da.data_len()?, 221);
1475
1476        Ok(())
1477    }
1478}
mzdata_bindata/array.rs

mzdata_bindata/
array.rs
