mzdata_bindata/

array.rs

use std::borrow::Cow;
use std::fmt::{self, Formatter};
use std::io::prelude::*;
use std::mem;
use std::num;

use base64_simd;
use bytemuck::Pod;
use flate2::write::{ZlibDecoder, ZlibEncoder};
use flate2::Compression;
use num_traits::ToBytes;

use mzdata_param::{ParamList, Unit};

#[allow(unused)]
use crate::encodings::{
    to_bytes, vec_as_bytes, ArrayRetrievalError, ArrayType, BinaryCompressionType, BinaryDataArrayType,
    Bytes,
};
use crate::traits::{ByteArrayView, ByteArrayViewMut};

#[allow(unused)]
use super::encodings::{
    reverse_transpose_f32, reverse_transpose_f64, reverse_transpose_i32, reverse_transpose_i64,
    transpose_f32, transpose_f64, transpose_i32, transpose_i64,
};

/// Represents a data array that holds a byte buffer that may be compressed, base64 encoded,
/// or raw little endian bytes, and provides views of those bytes as a small range of supported
/// types.
///
/// This type is modeled after the `<binaryDataArray>` element in mzML.
///
/// # Note
/// This type tries to walk a fine line between convenience and performance and as such is easy
/// to misuse. All operations that view the byte buffer as arbitrary data need that data to be
/// decoded and decompressed in order to borrow it. If the byte buffer is not already stored
/// decoded, the operation will copy and decode the buffer in its entirety before performing any
/// other operation, so repeated method calls may incur excessive overhead. If this is happening,
/// please use [`DataArray::decode_and_store`] to store the decoded representation explicitly.
///
/// Normally, [`SpectrumSource`](crate::io::SpectrumSource)-implementing file readers will eagerly decode all arrays
/// as soon as they are ready. If they are operating in lazy mode, the buffers will need to be decoded
/// explicitly, again using [`DataArray::decode_and_store`] or operations should make as much use of the
/// copied arrays as possible instead.
#[derive(Default, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct DataArray {
    /// The raw data of the array, stored in a variety of states controlled by [`Self::Compression`], as native bytes
    pub data: Bytes,
    /// The *kind* of data stored in the array, as in bit-width primitive types
    pub dtype: BinaryDataArrayType,
    /// How [`Self::data`] is encoded and compressed.
    pub compression: BinaryCompressionType,
    /// The *what* the data stored in the array was measuring, e.g. an m/z array
    pub name: ArrayType,
    /// Additional metadata parameters
    pub params: Option<Box<ParamList>>,
    /// The [`Unit`] for the data measurements
    pub unit: Unit,
    /// A cache for the number of elements stored in the array
    item_count: Option<num::NonZero<usize>>,
    /// An identifier reference to a data processing method other than the default method
    data_processing_reference: Option<Box<str>>,
}

impl core::fmt::Debug for DataArray {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        f.debug_struct("DataArray")
            .field("name", &self.name)
            .field("data size", &self.data.len())
            .field("dtype", &self.dtype)
            .field("compression", &self.compression)
            .field("params", &self.params)
            .field("unit", &self.unit)
            .field("data_processing_ref", &self.data_processing_reference)
            .finish()
    }
}

const EMPTY_BUFFER: [u8; 0] = [];


impl<'transient, 'lifespan: 'transient> DataArray {
    /// An alias of [`Default::default`], no configuration is
    /// performed whatsoever.
    pub fn new() -> DataArray {
        DataArray {
            ..Default::default()
        }
    }

    /// Create a new [`DataArray`] for a given `name` in its preferred data type
    /// returned by [`ArrayType::preferred_dtype`] and unit returned by [`ArrayType::preferred_unit`]
    pub fn from_name(name: &ArrayType) -> DataArray {
        DataArray {
            dtype: name.preferred_dtype(),
            name: name.clone(),
            compression: BinaryCompressionType::Decoded,
            unit: name.preferred_unit().unwrap_or_default(),
            ..Default::default()
        }
    }

    /// Create a new [`DataArray`] with a given `name` and `dtype`, using
    /// `name`'s preferred units. Does not allocate any additional storage.
    pub fn from_name_and_type(name: &ArrayType, dtype: BinaryDataArrayType) -> DataArray {
        DataArray {
            dtype,
            name: name.clone(),
            compression: BinaryCompressionType::Decoded,
            unit: name.preferred_unit().unwrap_or_default(),
            ..Default::default()
        }
    }

    /// Create a new [`DataArray`] with a given `name` and `dtype`, using
    /// `name`'s preferred units. Pre-allocates but does not fill
    /// `size` storage capacity in bytes (not elements of `dtype`).
    pub fn from_name_type_size(
        name: &ArrayType,
        dtype: BinaryDataArrayType,
        size: usize,
    ) -> DataArray {
        DataArray {
            dtype,
            name: name.clone(),
            data: Bytes::with_capacity(size),
            compression: BinaryCompressionType::Decoded,
            unit: name.preferred_unit().unwrap_or_default(),
            ..Default::default()
        }
    }

    /// Extract a subrange of the encoded byte buffer as a new [`DataArray`].
    ///
    /// `start` and `end` are **byte offsets** into the decoded buffer, not element
    /// indices, so they must be aligned to a multiple of [`DataArray::dtype`]'s size.
    ///
    /// # Examples
    /// ```
    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
    ///
    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
    /// array.extend(&[1.0f64, 2.0, 3.0, 4.0]).unwrap();
    ///
    /// // Each `f64` is 8 bytes wide, so the 2nd and 3rd elements occupy bytes [8, 24).
    /// let middle = array.slice(8, 24).unwrap();
    /// assert_eq!(&*middle.to_f64().unwrap(), &[2.0f64, 3.0f64][..]);
    /// ```
    pub fn slice(&self, start: usize, end: usize) -> Result<DataArray, ArrayRetrievalError> {
        if end < start || (end - start) % self.dtype.size_of() != 0 {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            let data = self.decode()?;
            let slice = data[start..end].to_vec();
            let subset = Self::wrap(&self.name, self.dtype, slice);
            Ok(subset)
        }
    }

    /// Retrieve a slice of raw bytes with no concern for value type alignment.
    ///
    /// `start` and `end` are **byte offsets** into the decoded buffer
    pub fn slice_buffer(
        &self,
        start: usize,
        end: usize,
    ) -> Result<Cow<'_, [u8]>, ArrayRetrievalError> {
        if end < start || (end - start) % self.dtype.size_of() != 0 {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            let data = self.decode()?;
            match data {
                Cow::Borrowed(view) => Ok(Cow::Borrowed(&view[start..end])),
                Cow::Owned(view) => {
                    let data = &view[start..end];
                    Ok(Cow::Owned(data.to_owned()))
                }
            }
        }
    }

    /// This method assumes the data are already in native byte order.
    ///
    /// This is not endian-safe *yet*, but is valid for in-memory operations.
    ///
    /// # Examples
    /// ```
    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
    ///
    /// let values: [f64; 3] = [100.5, 200.25, 300.0];
    /// let bytes: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
    /// let array = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
    ///
    /// let decoded = array.to_f64().unwrap();
    /// assert_eq!(&*decoded, &values[..]);
    /// ```
    pub fn wrap(name: &ArrayType, dtype: BinaryDataArrayType, data: Bytes) -> DataArray {
        DataArray {
            dtype,
            name: name.clone(),
            data,
            compression: BinaryCompressionType::Decoded,
            ..Default::default()
        }
    }

    /// This method assumes the data are already in native byte order
    fn set_buffer_of_type(&mut self, data_buffer: Vec<u8>) -> Result<usize, ArrayRetrievalError> {
        if data_buffer.is_empty() {
            self.item_count = None;
        } else {
            self.item_count = num::NonZero::try_from(data_buffer.len() / self.dtype().size_of()).ok();
        }
        self.data = data_buffer;
        Ok(self.data.len())
    }

    /// Directly set the data buffer from a slice of a [`Pod`] type, copying it.
    ///
    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
    /// the size of [`DataArray::dtype`].
    pub fn update_buffer<T: Pod + ToBytes>(
        &mut self,
        data_buffer: &[T],
    ) -> Result<usize, ArrayRetrievalError> {
        if self.dtype.size_of() != mem::size_of::<T>() {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            let n = data_buffer.len();
            self.item_count = num::NonZero::try_from(n).ok();
            self.data = to_bytes(data_buffer);
            Ok(self.data.len())
        }
    }

    /// Add the value of a [`Pod`] implementing type `T` to the array, converting it
    /// to native byte representation.
    ///
    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
    /// the size of [`DataArray::dtype`].
    ///
    /// # Examples
    /// ```
    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
    ///
    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
    /// array.push(500.5f64).unwrap();
    /// array.push(501.0f64).unwrap();
    ///
    /// assert_eq!(&*array.to_f64().unwrap(), &[500.5f64, 501.0f64][..]);
    /// ```
    pub fn push<T: Pod>(&mut self, value: T) -> Result<(), ArrayRetrievalError> {
        if !matches!(self.compression, BinaryCompressionType::Decoded) {
            self.decode_and_store()?;
        };
        if self.dtype.size_of() != mem::size_of::<T>() {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            let data = bytemuck::bytes_of(&value);
            self.data.extend_from_slice(data);
            self.item_count = self.item_count.map(|i| i.saturating_add(1));
            Ok(())
        }
    }

    /// Add the values from a slice of a [`Pod`] implementing type `T` to the array, converting the slice
    /// a slice of `T` in its native byte representation.
    ///
    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
    /// the size of [`DataArray::dtype`].
    ///
    /// # Examples
    /// ```
    /// use mzdata_bindata::{ArrayType, BinaryDataArrayType, ByteArrayView, DataArray};
    ///
    /// let mut array = DataArray::from_name_and_type(&ArrayType::IntensityArray, BinaryDataArrayType::Float32);
    /// array.extend(&[10.0f32, 20.0, 30.0]).unwrap();
    ///
    /// assert_eq!(&*array.to_f32().unwrap(), &[10.0f32, 20.0f32, 30.0f32][..]);
    /// ```
    pub fn extend<T: Pod>(&mut self, values: &[T]) -> Result<(), ArrayRetrievalError> {
        if !matches!(self.compression, BinaryCompressionType::Decoded) {
            self.decode_and_store()?;
        };
        if self.dtype.size_of() != mem::size_of::<T>() {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            self.item_count = self.item_count.map(|i| i.saturating_add(values.len()));
            let data = bytemuck::cast_slice(values);
            self.data.extend_from_slice(data);
            Ok(())
        }
    }

    /// Add the values from a buffer of bytes to the array directly.
    ///
    /// This will return an error if `values.len()` is not a multiple of
    /// the size of [`DataArray::dtype`].
    pub fn extend_raw(&mut self, values: &[u8]) -> Result<(), ArrayRetrievalError> {
        if values.len() % self.dtype.size_of() != 0 {
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        } else {
            self.data.extend_from_slice(values);
            Ok(())
        }
    }

    /// Add the values from an iterator over [`Pod`] type `T`.
    ///
    /// This is more efficient than repeated calls to [`DataArray::push`] or collecting
    /// to a `Vec` and then using [`DataArray::extend`], but equivalent to either.
    ///
    /// This will return an error if `std::mem::size_of::<T>()` is not equal to
    /// the size of [`DataArray::dtype`].
    pub fn extend_iter<T: Pod>(&mut self, iter: impl IntoIterator<Item=T>) -> Result<(), ArrayRetrievalError> {
        if mem::size_of::<T>() != self.dtype.size_of() {
            return Err(ArrayRetrievalError::DataTypeSizeMismatch)
        }
        for val in iter.into_iter() {
            let vb = bytemuck::bytes_of(&val);
            self.data.extend_from_slice(vb);
        }
        Ok(())
    }

    /// Encode the data buffer to a byte array using the requested [`BinaryCompressionType`] to
    /// compress it and then base64 encode it.
    ///
    /// This method takes the current byte buffer in [`Self::data`] and does the following:
    ///     1. If [`Self::compression`] is `compression`, just return a copy of the data as-is,
    ///         it is pre-coded properly.
    ///     2. If [`Self::compression`] is not [`BinaryCompressionType::Decoded`], copy and decode
    ///         [`Self::data`], otherwise borrow the raw byte data.
    ///     3. Apply `compression` to the decoded raw bytes, performing one or more transformations
    ///         and then base64 encode the resulting compressed byte array.
    ///     4. Return the base64 encoded, compressed byte array
    ///
    /// # Panics
    /// This method will panic if compression fails for any reason, or if an unsupported
    /// compression method is requested. Some compression methods are only available if
    /// appropriate features are enabled.
    pub fn encode_bytestring(&self, compression: BinaryCompressionType) -> Bytes {
        if self.compression == compression {
            log::trace!("Fast-path encoding {}:{}", self.name, self.dtype);
            return self.data.clone();
        }
        let bytestring = match self.compression {
            BinaryCompressionType::Decoded => Cow::Borrowed(self.data.as_slice()),
            _ => self.decode().expect("Failed to decode binary data"),
        };
        match compression {
            BinaryCompressionType::Decoded => panic!("Should never happen"),
            BinaryCompressionType::Zlib => {
                let compressed = Self::compress_zlib(&bytestring);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            BinaryCompressionType::NoCompression => {
                base64_simd::STANDARD.encode_type::<Bytes>(bytestring.as_ref())
            }
            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressLinear => {
                if self.dtype != BinaryDataArrayType::Float64 {
                    panic!("Cannot Numpress non-float64 data!");
                }
                if bytestring.is_empty() {
                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
                }
                let compressed =
                    Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring)).unwrap();
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressSLOF => {
                let compressed = match self.dtype {
                    BinaryDataArrayType::Float32 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
                    },
                    BinaryDataArrayType::Float64 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
                    },
                    _ => {
                        panic!("Cannot Numpress non-float data!");
                    }
                };
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressLinearZlib => {
                if self.dtype != BinaryDataArrayType::Float64 {
                    panic!("Cannot Numpress non-float64 data!");
                }
                if bytestring.is_empty() {
                    let compressed = Self::compress_zlib(&bytestring);
                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
                }
                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
                    .inspect_err(|e| {
                        log::error!("Failed to compress buffer with numpress: {e}");
                    })
                    .unwrap();
                let compressed = Self::compress_zlib(&compressed);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(all(feature = "numpress", feature = "zstd"))]
            BinaryCompressionType::NumpressLinearZstd => {
                if self.dtype != BinaryDataArrayType::Float64 {
                    panic!("Cannot Numpress non-float64 data!");
                }
                if bytestring.is_empty() {
                    let compressed = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
                    return base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
                }
                let compressed = Self::compress_numpress_linear(bytemuck::cast_slice(&bytestring))
                    .inspect_err(|e| {
                        log::error!("Failed to compress buffer with numpress: {e}");
                    })
                    .unwrap();
                let compressed = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressSLOFZlib => {
                if bytestring.is_empty() {
                    let bytestring = Self::compress_zlib(&bytestring);
                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
                }
                let compressed = match self.dtype {
                    BinaryDataArrayType::Float32 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
                    },
                    BinaryDataArrayType::Float64 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
                    },
                    _ => {
                        panic!("Cannot Numpress non-float data!");
                    }
                };
                let bytestring = Self::compress_zlib(&compressed);
                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
            }
            #[cfg(all(feature = "numpress", feature = "zstd"))]
            BinaryCompressionType::NumpressSLOFZstd => {
                if bytestring.is_empty() {
                    let bytestring = Self::compress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
                    return base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
                }
                let compressed = match self.dtype {
                    BinaryDataArrayType::Float32 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f32>(&bytestring)).unwrap()
                    },
                    BinaryDataArrayType::Float64 => {
                        Self::compress_numpress_slof(bytemuck::cast_slice::<u8, f64>(&bytestring)).unwrap()
                    },
                    _ => {
                        panic!("Cannot Numpress non-float data!");
                    }
                };
                let bytestring = Self::compress_zstd(&compressed, BinaryDataArrayType::Unknown, false);
                base64_simd::STANDARD.encode_type::<Bytes>(&bytestring)
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::Zstd => {
                let compressed = Self::compress_zstd(&bytestring, self.dtype, false);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::ShuffleZstd => {
                let compressed = Self::compress_zstd(&bytestring, self.dtype, true);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::DeltaShuffleZstd => {
                let compressed = Self::compress_delta_zstd(&bytestring, self.dtype, true);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::ZstdDict => {
                let compressed = Self::compress_dict_zstd(&bytestring, self.dtype);
                base64_simd::STANDARD.encode_type::<Bytes>(&compressed)
            }
            _ => {
                panic!("Compresion type {:?} is unsupported", compression)
            }
        }
    }

    /// Decode the compressed data, if needed, and store that buffer in `self.data`. After
    /// decoding `self.compression` will always be [`BinaryCompressionType::Decoded`].
    ///
    /// The return value is the content of `self.compression` after decoding.
    ///
    /// This may fail if the decoding fails for any reason.
    ///
    /// # Examples
    /// ```
    /// use mzdata_bindata::{ArrayType, BinaryCompressionType, BinaryDataArrayType, ByteArrayView, DataArray};
    ///
    /// let values: [f64; 4] = [1.0, 2.0, 3.0, 4.0];
    /// let mut array = DataArray::from_name_and_type(&ArrayType::MZArray, BinaryDataArrayType::Float64);
    /// array.extend(&values).unwrap();
    ///
    /// array.store_compressed(BinaryCompressionType::Zlib).unwrap();
    /// assert_eq!(array.compression, BinaryCompressionType::Zlib);
    ///
    /// array.decode_and_store().unwrap();
    /// assert_eq!(array.compression, BinaryCompressionType::Decoded);
    /// assert_eq!(&*array.to_f64().unwrap(), &values[..]);
    /// ```
    pub fn decode_and_store(&mut self) -> Result<BinaryCompressionType, ArrayRetrievalError> {
        match self.decode() {
            Ok(data) => {
                match data {
                    // The only time this is a borrow is when the data are already
                    // decoded.
                    Cow::Borrowed(_view) => Ok(self.compression),
                    Cow::Owned(buffer) => {
                        let n = buffer.len() / self.dtype.size_of();
                        self.item_count = n.try_into().ok();
                        self.data = buffer;
                        self.compression = BinaryCompressionType::Decoded;
                        Ok(self.compression)
                    }
                }
            }
            Err(err) => Err(err),
        }
    }

    /// Decompress and base64-decode encoded bytes, and return the data.
    ///
    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
    /// more buffers may be allocated to hold the decompressed and decoded bytes.
    ///
    /// # Panics
    /// This method panics if [`Self::compression`] fails for any reason or is not supported.
    /// Several compression methods require certain features be enabled.
    pub fn decode(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
        if self.data.is_empty() {
            return Ok(Cow::Borrowed(&EMPTY_BUFFER));
        }

        macro_rules! base64_decode {
            () => {
                base64_simd::STANDARD
                    .decode_type::<Bytes>(&self.data)
                    .map_err(|_| ArrayRetrievalError::MalformedBase64Encoding)?
            };
        }

        match self.compression {
            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(self.data.as_slice())),
            BinaryCompressionType::NoCompression => {
                let bytestring = base64_decode!();
                Ok(Cow::Owned(bytestring))
            }
            BinaryCompressionType::Zlib => {
                let mut bytestring = base64_decode!();
                bytestring = Self::decompress_zlib(&bytestring);
                Ok(Cow::Owned(bytestring))
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::Zstd => {
                let bytestring = base64_decode!();
                Ok(Cow::Owned(Self::decompress_zstd(
                    &bytestring,
                    self.dtype,
                    false,
                )))
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::ShuffleZstd => {
                let bytestring = base64_decode!();
                Ok(Cow::Owned(Self::decompress_zstd(
                    &bytestring,
                    self.dtype,
                    true,
                )))
            }
            #[cfg(feature = "zstd")]
            BinaryCompressionType::DeltaShuffleZstd => {
                let bytestring = base64_decode!();
                Ok(Cow::Owned(Self::decompress_delta_zstd(
                    &bytestring,
                    self.dtype,
                    true,
                )))
            }

            #[cfg(feature = "zstd")]
            BinaryCompressionType::ZstdDict => {
                let bytestring = base64_decode!();
                Ok(Cow::Owned(Self::decompress_dict_zstd(
                    &bytestring,
                    self.dtype,
                )))
            }
            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressLinear => match self.dtype {
                BinaryDataArrayType::Float64 => {
                    let bytestring = base64_decode!();
                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
                    let view = vec_as_bytes(decoded);
                    Ok(Cow::Owned(view))
                }
                _ => Err(ArrayRetrievalError::DecompressionError(
                    self.compression.unsupported_msg(Some(
                        format!("Not compatible with {:?}", self.dtype).as_str(),
                    )),
                )),
            },

            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressSLOF => {
                let bytestring = base64_decode!();
                Self::decompress_numpress_slof(&bytestring, self.dtype)
            }

            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressLinearZlib => match self.dtype {
                BinaryDataArrayType::Float64 => {
                    let bytestring = base64_decode!();
                    let bytestring = Self::decompress_zlib(&bytestring);
                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
                    let view = vec_as_bytes(decoded);
                    Ok(Cow::Owned(view))
                }
                _ => Err(ArrayRetrievalError::DecompressionError(
                    self.compression.unsupported_msg(Some(
                        format!("Not compatible with {:?}", self.dtype).as_str(),
                    )),
                )),
            },

            #[cfg(feature = "numpress")]
            BinaryCompressionType::NumpressSLOFZlib => {
                let bytestring = base64_decode!();
                let bytestring = Self::decompress_zlib(&bytestring);
                Self::decompress_numpress_slof(&bytestring, self.dtype)
            }

            #[cfg(all(feature = "numpress", feature = "zstd"))]
            BinaryCompressionType::NumpressLinearZstd => match self.dtype {
                BinaryDataArrayType::Float64 => {
                    let bytestring = base64_decode!();
                    let bytestring = Self::decompress_zstd(
                        &bytestring,
                        BinaryDataArrayType::Unknown,
                        false,
                    );
                    let decoded = Self::decompress_numpress_linear(&bytestring)?;
                    let view = vec_as_bytes(decoded);
                    Ok(Cow::Owned(view))
                }
                _ => Err(ArrayRetrievalError::DecompressionError(
                    self.compression.unsupported_msg(Some(
                        format!("Not compatible with {:?}", self.dtype).as_str(),
                    )),
                )),
            },

            #[cfg(all(feature = "numpress", feature = "zstd"))]
            BinaryCompressionType::NumpressSLOFZstd => {
                let bytestring = base64_decode!();
                let bytestring = Self::decompress_zstd(&bytestring, BinaryDataArrayType::Unknown, false);
                Self::decompress_numpress_slof(&bytestring, self.dtype)
            }

            mode => Err(ArrayRetrievalError::DecompressionError(format!(
                "Cannot decode array encoded with {:?}",
                mode
            ))),
        }
    }

    /// Get a slice of the raw bytes aligned to [`Self::dtype`].
    ///
    /// If the data are already decoded, this does not create a copy of the data.
    ///
    /// This is an aligned version of [`Self::slice_buffer`].
    pub(crate) fn decoded_slice(
        &'lifespan self,
        start: usize,
        end: usize,
    ) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
        if start > end || (end - start) % self.dtype.size_of() != 0 {
            return Err(ArrayRetrievalError::DataTypeSizeMismatch);
        }
        match self.compression {
            BinaryCompressionType::Decoded => Ok(Cow::Borrowed(&self.data.as_slice()[start..end])),
            _ => {
                Ok(Cow::Owned(self.decode()?[start..end].to_vec()))
            }
        }
    }

    /// Get a mutable view of the raw bytes.
    ///
    /// If the data are already decoded, this just borrows [`Self::data`], otherwise
    /// the [`Self::decode`] is called [`Self::data`] is updated to the decoded copy first,
    /// then borrowed.
    ///
    /// Note that care **MUST** be taken here to avoid writing invalid values when working on the
    /// raw bytes directly. Prefer to update the native values indirectly via
    /// [`ByteArrayViewMut::coerce_mut`].
    pub fn decode_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
        if self.data.is_empty() || matches!(self.compression, BinaryCompressionType::Decoded) {
            Ok(&mut self.data)
        } else {
            match self.decode()? {
                Cow::Borrowed(_) => {
                    Ok(&mut self.data)
                },
                Cow::Owned(owned) => {
                    self.data = owned;
                    self.compression = BinaryCompressionType::Decoded;
                    Ok(&mut self.data)
                },
            }
        }

    }

    /// Erase all the data in the array, calling [`Vec::clear`] on [`Self::data`],
    /// and erasing all [`Param`] values. All other properties remain unchanged.
    pub fn clear(&mut self) {
        self.data.clear();
        self.params = None;
        self.item_count = None;
    }

    /// The reverse of [`DataArray::decode_and_store`], this method compresses `self.data` to the desired
    /// compression method and stores that buffer as `self.data`.
    pub fn store_compressed(
        &mut self,
        compression: BinaryCompressionType,
    ) -> Result<(), ArrayRetrievalError> {
        if self.compression == compression {
            Ok(())
        } else {
            self.item_count = self.data_len().ok().and_then(|n| n.try_into().ok());
            let bytes = self.encode_bytestring(compression);
            self.data = bytes;
            self.compression = compression;
            Ok(())
        }
    }

    /// Recode the stored data as the requested binary data type.
    ///
    /// This is a no-op if the [`Self::dtype`] matches the `dtype` argument.
    /// The data will be decoded and decompressed after this call ends. Any previous
    /// compression will need to be re-applied.
    ///
    /// ## Note
    /// When down-casting to a lower precision type, e.g. `f64` to `f32`, data may be lost.
    /// Take care to do this only when necessary or when the loss of precision is acceptable.
    pub fn store_as(&mut self, dtype: BinaryDataArrayType) -> Result<usize, ArrayRetrievalError> {
        if self.dtype == dtype {
            return Ok(self.data.len());
        }
        let z = match dtype {
            BinaryDataArrayType::Float32 => {
                let view = self.to_f32()?;
                #[cfg(target_endian = "big")]
                {
                    let mut recast = to_bytes(&view);
                    dtype.swap_bytes(&mut recast)?;
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
                #[cfg(not(target_endian = "big"))]
                {
                    let recast = to_bytes(&view);
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }

            }
            BinaryDataArrayType::Float64 => {
                let view = self.to_f64()?;
                #[cfg(target_endian = "big")]
                {
                    let mut recast = to_bytes(&view);
                    dtype.swap_bytes(&mut recast)?;
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
                #[cfg(not(target_endian = "big"))]
                {
                    let recast = to_bytes(&view);
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
            }
            BinaryDataArrayType::Int32 => {
                let view = self.to_i32()?;
                #[cfg(target_endian = "big")]
                {
                    let mut recast = to_bytes(&view);
                    dtype.swap_bytes(&mut recast)?;
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
                #[cfg(not(target_endian = "big"))]
                {
                    let recast = to_bytes(&view);
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
            }
            BinaryDataArrayType::Int64 => {
                let view = self.to_i64()?;
                #[cfg(target_endian = "big")]
                {
                    let mut recast = to_bytes(&view);
                    dtype.swap_bytes(&mut recast)?;
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
                #[cfg(not(target_endian = "big"))]
                {
                    let recast = to_bytes(&view);
                    self.dtype = dtype;
                    self.set_buffer_of_type(recast)
                }
            }
            _ => Ok(0),
        }?;
        self.compression = BinaryCompressionType::Decoded;
        Ok(z)
    }

    /// Test if the the array describes an ion mobility quantity.
    ///
    /// # See also
    /// [`ArrayType::is_ion_mobility`]
    pub const fn is_ion_mobility(&self) -> bool {
        self.name.is_ion_mobility()
    }

    /// The size of the raw byte buffer
    pub fn raw_len(&self) -> usize {
        self.data.len()
    }

    /// Get the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
    pub fn data_processing_reference(&self) -> Option<&str> {
        self.data_processing_reference.as_deref()
    }

    /// Set the identifier referencing a [`DataProcessing`](crate::meta::DataProcessing)
    pub fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
        self.data_processing_reference = data_processing_reference;
    }
}


/// [`DataArray`] implements several compression codecs, some of which require additional dependencies.
///
/// These methods are often cumbersome and not easy to use as they require a lot of extra context. They
/// live here as a historical artefact. They shouldn't be used directly in the vast majority of cases.
impl DataArray {

    /// Compress the provided bytes using `zlib` at the highest compression setting
    pub(crate) fn compress_zlib(bytestring: &[u8]) -> Bytes {
        let result = Bytes::new();
        let mut compressor = ZlibEncoder::new(result, Compression::best());
        compressor.write_all(bytestring).expect("Error compressing");
        compressor.finish().expect("Error compressing")
    }

    /// Decompress the provided bytes using `zlib`.
    ///
    /// # Panics
    /// If the data are not `zlib` compressed.
    pub(crate) fn decompress_zlib(bytestring: &[u8]) -> Bytes {
        let result = Bytes::new();
        let mut decompressor = ZlibDecoder::new(result);
        decompressor
            .write_all(bytestring)
            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
        let buf = decompressor
            .finish()
            .unwrap_or_else(|e| panic!("Decompression error: {}", e));
        buf
    }

    #[cfg(feature = "numpress")]
    /// Encode the provided `f64`` data using MS-Numpress Linear encoding.
    ///
    /// The data **MUST** be pre-sorted or else the regression model will provide
    /// worse results.
    pub(crate) fn compress_numpress_linear(data: &[f64]) -> Result<Bytes, ArrayRetrievalError> {
        if data.is_empty() {
            return Ok(Bytes::new());
        }
        let scaling = numpress::optimal_scaling(data);
        match numpress::numpress_compress(data, scaling) {
            Ok(data) => Ok(data),
            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
        }
    }

    #[cfg(feature = "numpress")]
    pub(crate) fn compress_numpress_slof<T: numpress::AsFloat64>(data: &[T]) -> Result<Bytes, ArrayRetrievalError> {
        let scaling = numpress::optimal_slof_fixed_point(data);
        let mut buf = Bytes::new();
        match numpress::encode_slof(data, &mut buf, scaling) {
            Ok(_) => Ok(buf),
            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
        }
    }

    #[cfg(feature = "numpress")]
    /// Decode a MS-Numpress Linear encoded byte array.
    pub(crate) fn decompress_numpress_linear(data: &[u8]) -> Result<Vec<f64>, ArrayRetrievalError> {
        if data.is_empty() {
            return Ok(Vec::new())
        }
        match numpress::numpress_decompress(data) {
            Ok(data) => Ok(data),
            Err(e) => Err(ArrayRetrievalError::DecompressionError(e.to_string())),
        }
    }

    #[cfg(feature = "numpress")]
    pub(crate) fn decompress_numpress_slof(data: &[u8], dtype: BinaryDataArrayType) -> Result<Cow<'static, [u8]>, ArrayRetrievalError> {
        use log::trace;

        let mut buf = Vec::new();
        let decoded = match numpress::decode_slof(data, &mut buf) {
            Ok(_) => buf,
            Err(e) => return Err(ArrayRetrievalError::DecompressionError(e.to_string())),
        };
        trace!("Numpress SLOF decoded to {} points", decoded.len());
        match dtype {
            BinaryDataArrayType::Float64 => {
                let view = vec_as_bytes(decoded);
                Ok(Cow::Owned(view))
            },
            BinaryDataArrayType::Float32 => {
                let n = decoded.len() * BinaryDataArrayType::Float32.size_of();
                trace!("Mapping to {n} bytes for f32 storage");
                let mut view: Vec<u8> = Vec::with_capacity(n);
                for val in decoded {
                    let val = val as f32;
                    view.extend(bytemuck::bytes_of(&val))
                }
                Ok(Cow::Owned(view))
            },
            _ => {
                Err(ArrayRetrievalError::DecompressionError(
                    BinaryCompressionType::NumpressSLOF.unsupported_msg(Some(
                        format!("Not compatible with {:?}", dtype).as_str(),
                    )),
                ))
            }
        }
    }

    #[cfg(feature = "zstd")]
    /// Compress the byte buffer using Zstandard compression.
    ///
    /// The default compression level is controlled by [`zstd::DEFAULT_COMPRESSION_LEVEL`], but the `MZDATA_ZSTD_LEVEL`
    /// environment variable can be used to raise or lower it as desired.
    pub(crate) fn compress_zstd(
        bytestring: &[u8],
        dtype: BinaryDataArrayType,
        shuffle: bool,
    ) -> Bytes {
        let level: i32 = std::env::var("MZDATA_ZSTD_LEVEL")
            .map(|v| v.parse())
            .unwrap_or(Ok(zstd::DEFAULT_COMPRESSION_LEVEL))
            .unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL);
        if !shuffle {
            return zstd::bulk::compress(bytestring, level).unwrap();
        }
        match dtype {
            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
                zstd::bulk::compress(bytestring, level).unwrap()
            }
            BinaryDataArrayType::Float64 => {
                zstd::bulk::compress(&transpose_f64(bytemuck::cast_slice(bytestring)), level)
                    .unwrap()
            }
            BinaryDataArrayType::Float32 => {
                zstd::bulk::compress(&transpose_f32(bytemuck::cast_slice(bytestring)), level)
                    .unwrap()
            }
            BinaryDataArrayType::Int64 => {
                zstd::bulk::compress(&transpose_i64(bytemuck::cast_slice(bytestring)), level)
                    .unwrap()
            }
            BinaryDataArrayType::Int32 => {
                zstd::bulk::compress(&transpose_i32(bytemuck::cast_slice(bytestring)), level)
                    .unwrap()
            }
        }
    }

    #[cfg(feature = "zstd")]
    pub(crate) fn compress_delta_zstd(
        bytestring: &[u8],
        dtype: BinaryDataArrayType,
        shuffle: bool,
    ) -> Bytes {
        use bytemuck::cast_slice;

        use super::delta_encoding;

        match dtype {
            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => {
                Self::compress_zstd(bytestring, dtype, shuffle)
            }
            BinaryDataArrayType::Float64 => {
                let mut buf = cast_slice::<_, f64>(bytestring).to_vec();
                delta_encoding(&mut buf);
                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
            }
            BinaryDataArrayType::Float32 => {
                let mut buf = cast_slice::<_, f32>(bytestring).to_vec();
                delta_encoding(&mut buf);
                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
            }
            BinaryDataArrayType::Int64 => {
                let mut buf = cast_slice::<_, i64>(bytestring).to_vec();
                delta_encoding(&mut buf);
                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
            }
            BinaryDataArrayType::Int32 => {
                let mut buf = cast_slice::<_, i32>(bytestring).to_vec();
                delta_encoding(&mut buf);
                Self::compress_zstd(cast_slice(&buf), dtype, shuffle)
            }
        }
    }

    #[cfg(feature = "zstd")]
    pub(crate) fn compress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
        use super::encodings::dictionary_encoding;
        log::trace!("Dictionary encoding {} bytes as {dtype}", bytestring.len());
        if bytestring.is_empty() {
            return Self::compress_zstd(&bytestring, dtype, false);
        }
        match dtype {
            BinaryDataArrayType::Float64 => {
                let compressed =
                    dictionary_encoding(bytemuck::cast_slice::<u8, f64>(&bytestring))
                        .unwrap();
                let compressed = Self::compress_zstd(&compressed, dtype, false);
                compressed
            }
            BinaryDataArrayType::Float32 => {
                let compressed =
                    dictionary_encoding(bytemuck::cast_slice::<u8, f32>(&bytestring))
                        .unwrap();
                let compressed = Self::compress_zstd(&compressed, dtype, false);
                compressed
            }
            BinaryDataArrayType::Int64 => {
                let compressed =
                    dictionary_encoding(bytemuck::cast_slice::<u8, i64>(&bytestring))
                        .unwrap();
                let compressed = Self::compress_zstd(&compressed, dtype, false);
                compressed
            }
            BinaryDataArrayType::Int32 => {
                let compressed =
                    dictionary_encoding(bytemuck::cast_slice::<u8, i32>(&bytestring))
                        .unwrap();
                let compressed = Self::compress_zstd(&compressed, dtype, false);
                compressed
            }
            _ => {
                let compressed =
                    dictionary_encoding(bytemuck::cast_slice::<u8, u8>(&bytestring))
                        .unwrap();
                let compressed = Self::compress_zstd(&compressed, dtype, false);
                compressed
            }
        }
    }

    #[cfg(feature = "zstd")]
    pub(crate) fn decompress_zstd(data: &[u8], dtype: BinaryDataArrayType, shuffle: bool) -> Bytes {
        let mut decoder = zstd::Decoder::new(std::io::Cursor::new(data)).unwrap();
        let mut buf = Vec::new();
        decoder.read_to_end(&mut buf).unwrap();
        if !shuffle {
            return buf;
        }
        match dtype {
            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => buf,
            BinaryDataArrayType::Float64 => reverse_transpose_f64(&buf),
            BinaryDataArrayType::Float32 => reverse_transpose_f32(&buf),
            BinaryDataArrayType::Int64 => reverse_transpose_i64(&buf),
            BinaryDataArrayType::Int32 => reverse_transpose_i32(&buf),
        }
    }

    #[cfg(feature = "zstd")]
    pub(crate) fn decompress_delta_zstd(
        data: &[u8],
        dtype: BinaryDataArrayType,
        shuffle: bool,
    ) -> Bytes {
        use super::delta_decoding;

        let mut delta = Self::decompress_zstd(data, dtype, shuffle);
        match dtype {
            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => delta,
            BinaryDataArrayType::Float64 => {
                let buf = bytemuck::cast_slice_mut::<_, f64>(&mut delta);
                delta_decoding(buf);
                delta
            }
            BinaryDataArrayType::Float32 => {
                let buf = bytemuck::cast_slice_mut::<_, f32>(&mut delta);
                delta_decoding(buf);
                delta
            }
            BinaryDataArrayType::Int64 => {
                let buf = bytemuck::cast_slice_mut::<_, i64>(&mut delta);
                delta_decoding(buf);
                delta
            }
            BinaryDataArrayType::Int32 => {
                let buf = bytemuck::cast_slice_mut::<_, i32>(&mut delta);
                delta_decoding(buf);
                delta
            }
        }
    }

    #[cfg(feature = "zstd")]
    pub(crate) fn decompress_dict_zstd(bytestring: &[u8], dtype: BinaryDataArrayType) -> Bytes {
        use super::encodings::dictionary_decoding;

        let data = Self::decompress_zstd(bytestring, dtype, false);
        match dtype {
            BinaryDataArrayType::ASCII | BinaryDataArrayType::Unknown => dictionary_decoding(&data).unwrap(),
            BinaryDataArrayType::Float64 => {
                to_bytes(&dictionary_decoding::<f64>(&data).unwrap())
            },
            BinaryDataArrayType::Float32 => {
                to_bytes(&dictionary_decoding::<f32>(&data).unwrap())
            },
            BinaryDataArrayType::Int64 => {
                to_bytes(&dictionary_decoding::<i64>(&data).unwrap())
            },
            BinaryDataArrayType::Int32 => {
                to_bytes(&dictionary_decoding::<i32>(&data).unwrap())
            },
        }
    }
}

impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan> for DataArray {
    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
        self.decode()
    }

    fn dtype(&self) -> BinaryDataArrayType {
        self.dtype
    }

    fn data_len(&'lifespan self) -> Result<usize, ArrayRetrievalError> {
        if let Some(z) = self.item_count {
            Ok(z.get())
        } else {
            let view = self.view()?;
            let n = view.len();
            Ok(n / self.dtype().size_of())
        }
    }

    fn unit(&self) -> Unit {
        self.unit
    }

    fn data_processing_reference(&self) -> Option<&str> {
        self.data_processing_reference()
    }

    fn name(&self) -> &ArrayType {
        &self.name
    }
}

impl<'transient, 'lifespan: 'transient> ByteArrayViewMut<'transient, 'lifespan> for DataArray {
    fn view_mut(&'transient mut self) -> Result<&'transient mut Bytes, ArrayRetrievalError> {
        self.decode_mut()
    }

    fn unit_mut(&mut self) -> &mut Unit {
        &mut self.unit
    }

    fn set_data_processing_reference(&mut self, data_processing_reference: Option<Box<str>>) {
        self.set_data_processing_reference(data_processing_reference);
    }
}

mzdata_param::impl_param_described_deferred!(DataArray);

/// Represent a slice of a [`DataArray`] that manages offsets and decoding automatically.
///
/// This **does not** copy the data.
#[derive(Clone, Debug)]
pub struct DataArraySlice<'a> {
    source: &'a DataArray,
    /// The start of the slice, in bytes
    pub start: usize,
    /// The end of the slice, in bytes
    pub end: usize,
}

impl<'a> DataArraySlice<'a> {
    /// Construct a new [`DataArraySlice`]
    pub fn new(source: &'a DataArray, mut start: usize, mut end: usize) -> Self {
        if start > end {
            mem::swap(&mut start, &mut end);
        }
        Self { source, start, end }
    }

    /// Decompress and base64-decode encoded bytes, and return the data.
    ///
    /// If the data were already decoded, the existing bytes are returned. Otherwise one or
    /// more buffers may be allocated to hold the decompressed and decoded bytes.
    ///
    /// This is a thin wrapper around [`DataArray::decoded_slice`].
    pub fn decode(&'a self) -> Result<Cow<'a, [u8]>, ArrayRetrievalError> {
        self.source.decoded_slice(self.start, self.end)
    }

    /// Test if the the array describes an ion mobility quantity.
    ///
    /// # See also
    /// [`ArrayType::is_ion_mobility`]
    pub const fn is_ion_mobility(&self) -> bool {
        self.source.is_ion_mobility()
    }

    /// Get an reference to the source [`DataArray`]
    pub fn source(&self) -> &'a DataArray {
        self.source
    }
}

impl<'transient, 'lifespan: 'transient> ByteArrayView<'transient, 'lifespan>
    for DataArraySlice<'lifespan>
{
    fn view(&'lifespan self) -> Result<Cow<'lifespan, [u8]>, ArrayRetrievalError> {
        self.decode()
    }

    fn dtype(&self) -> BinaryDataArrayType {
        self.source.dtype()
    }

    fn unit(&self) -> Unit {
        self.source.unit
    }

    fn name(&self) -> &ArrayType {
        self.source.name()
    }

    fn data_processing_reference(&self) -> Option<&str> {
        self.source.data_processing_reference()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::fs;
    use std::io;

    use super::DataArray;

    fn make_array_from_file() -> io::Result<DataArray> {
        let mut fh = fs::File::open("../../test/data/mz_f64_zlib_bas64.txt")?;
        let mut buf = String::new();
        fh.read_to_string(&mut buf)?;
        let bytes: Vec<u8> = buf.into();
        let mut da = DataArray::wrap(&ArrayType::MZArray, BinaryDataArrayType::Float64, bytes);
        da.compression = BinaryCompressionType::Zlib;
        *da.unit_mut() = Unit::MZ;
        assert_eq!(da.unit(), Unit::MZ);
        assert!(!da.is_ion_mobility());
        assert_eq!(da.name(), &ArrayType::MZArray);
        Ok(da)
    }

    #[cfg(feature = "zstd")]
    fn make_array_from_file_im_zstd() -> io::Result<DataArray> {
        let mut fh = fs::File::open("../../test/data/im_f64_zstd_base64.txt")?;
        let mut buf = String::new();
        fh.read_to_string(&mut buf)?;
        let bytes: Vec<u8> = buf.into();
        let mut da = DataArray::wrap(&ArrayType::MeanInverseReducedIonMobilityArray, BinaryDataArrayType::Float64, bytes);
        da.compression = BinaryCompressionType::Zstd;
        *da.unit_mut() = Unit::VoltSecondPerSquareCentimeter;
        assert_eq!(da.unit(), Unit::VoltSecondPerSquareCentimeter);
        assert!(da.is_ion_mobility());
        assert_eq!(da.name(), &ArrayType::MeanInverseReducedIonMobilityArray);
        Ok(da)
    }

    #[test]
    fn test_decode() -> io::Result<()> {
        let mut da = make_array_from_file()?;
        da.decode_and_store()?;
        let view = da.to_f64()?;
        assert_eq!(view.len(), 19800);
        Ok(())
    }

    #[test]
    fn test_decode_store() -> io::Result<()> {
        let mut da = make_array_from_file()?;
        da.decode_and_store()?;
        let back = da.clone();
        da.store_as(BinaryDataArrayType::Float32)?;
        let view = da.to_f64()?;
        assert_eq!(view.len(), 19800);
        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
            let err = (a - b).abs();
            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
        }
        for (a, b) in back.iter_f64()?.zip(da.iter_f32()?.map(|x| x as f64)) {
            let err = (a - b).abs();
            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
        }
        da.store_as(BinaryDataArrayType::Float64)?;
        let view = da.to_f64()?;
        assert_eq!(view.len(), 19800);
        for (a, b) in back.iter_f64()?.zip(view.iter().copied()) {
            let err = (a - b).abs();
            assert!((a - b).abs() < 1e-3, "{} - {} = {}", a, b, err);
        }
        Ok(())
    }

    #[cfg(feature = "zstd")]
    #[test]
    fn test_decode_delta_zstd() {
        let points: Vec<f64> = (0..200_000usize).map(|i| i as f64 * 0.01 + 100.0).collect();
        let mut da = DataArray::from_name(&ArrayType::MZArray);
        da.extend(&points).unwrap();

        let decoded_len = da.data.len();
        da.store_compressed(BinaryCompressionType::ShuffleZstd)
            .unwrap();
        let zstd_len = da.data.len();

        da.store_compressed(BinaryCompressionType::Zlib).unwrap();
        let zlib_len = da.data.len();

        da.decode_and_store().unwrap();

        da.store_compressed(BinaryCompressionType::DeltaShuffleZstd)
            .unwrap();
        let delta_zstd_len = da.data.len();
        eprintln!("decoded: {decoded_len};\nzlib: {zlib_len};\nzstd: {zstd_len};\ndelta-zstd: {delta_zstd_len}");
        da.decode_and_store().unwrap();
        let view = da.to_f64().unwrap();
        let err: f64 = points
            .iter()
            .zip(view.iter())
            .map(|(a, b)| {
                assert!((a - b).abs() < 1e-3, "{a} - {b} = {}", a - b);
                (a - b).abs()
            })
            .sum();
        let mean_err = err / (points.len() as f64);
        eprintln!("mean abs error: {mean_err:0.8}")
    }

    #[cfg(feature = "zstd")]
    #[test]
    fn test_decode_zstd() -> io::Result<()> {
        let mut da = make_array_from_file()?;
        let zlib_len = da.data.len();
        da.decode_and_store()?;

        let decoded_len = da.data.len();

        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;

        let zstd_len = da.data.len();

        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nzstd: {zstd_len}");
        da.decode_and_store()?;

        let mut da_ref = make_array_from_file()?;
        da_ref.decode_and_store()?;
        assert_eq!(da.data, da_ref.data);
        Ok(())
    }

    #[cfg(feature = "numpress")]
    #[test]
    fn test_numpress_linear() -> io::Result<()> {
        let mut da = make_array_from_file()?;
        let zlib_len = da.data.len();
        da.decode_and_store()?;

        let decoded_len = da.data.len();

        da.store_compressed(BinaryCompressionType::NumpressLinear)?;
        let numpress_len = da.data.len();

        eprintln!("zlib: {zlib_len};\ndecoded: {decoded_len};\nnumpress: {numpress_len}");
        da.decode_and_store()?;

        let mut da_ref = make_array_from_file()?;
        da_ref.decode_and_store()?;
        for (a, b) in da.iter_f64()?.zip(da_ref.iter_f64()?) {
            assert!((a - b).abs() < 1e-3, "{a} - {b} = {} which is too large a deviation", (a - b).abs())
        }

        Ok(())
    }

    #[test]
    fn test_decode_roundtrip() -> io::Result<()> {
        let mut da = make_array_from_file()?;
        let compressed_size = da.data_len()?;
        da.decode_and_store()?;
        let view = da.to_f64()?;
        assert_eq!(view.len(), 19800);
        drop(view);
        da.store_compressed(BinaryCompressionType::Zlib)?;
        assert_eq!(da.compression, BinaryCompressionType::Zlib);
        assert_eq!(da.data_len()?, compressed_size);
        let view = da.to_f64()?;
        assert_eq!(view.len(), 19800);
        Ok(())
    }

    #[test]
    fn test_decode_empty() {
        let mut da = DataArray::wrap(
            &ArrayType::MZArray,
            BinaryDataArrayType::Float64,
            Vec::new(),
        );
        da.compression = BinaryCompressionType::Zlib;

        assert_eq!(da.data.len(), 0);
        assert_eq!(da.data_len().unwrap(), 0);
        assert_eq!(da.decode().unwrap().len(), 0);
        assert_eq!(da.to_f64().unwrap().len(), 0);
    }


    #[test]
    fn test_coerce_from_mut() {
        let mut values = [1.25_f64, 2.5];
        let bytes = bytemuck::cast_slice_mut(&mut values);
        let view = <DataArray as ByteArrayViewMut>::coerce_from_mut::<f64>(bytes).unwrap();
        assert_eq!(view, &[1.25, 2.5]);
        view[1] = -3.5;
        assert_eq!(values, [1.25, -3.5]);

        let mut empty = [];
        assert!(
            <DataArray as ByteArrayViewMut>::coerce_from_mut::<f64>(&mut empty)
                .unwrap()
                .is_empty()
        );
        assert!(
            <DataArray as ByteArrayViewMut>::coerce_from_mut::<[u8; 0]>(&mut empty)
                .unwrap()
                .is_empty()
        );
    }

    #[test]
    fn test_coerce_from_mut_invalid_buffer() {
        let mut values = [0_u64; 2];
        let bytes: &mut [u8] = bytemuck::cast_slice_mut(&mut values);
        assert_eq!(
            <DataArray as ByteArrayViewMut>::coerce_from_mut::<u64>(&mut bytes[1..9]),
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        );
        assert_eq!(
            <DataArray as ByteArrayViewMut>::coerce_from_mut::<u64>(&mut bytes[..7]),
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        );
        assert_eq!(
            <DataArray as ByteArrayViewMut>::coerce_from_mut::<[u8; 0]>(&mut bytes[..1]),
            Err(ArrayRetrievalError::DataTypeSizeMismatch)
        );
    }

    #[test]
    fn test_iter_unaligned_numeric_data() {
        fn check<T: bytemuck::Pod + std::fmt::Debug + PartialEq>(
            dtype: BinaryDataArrayType,
            encoded: &[u8],
            expected: &[T],
        ) {
            let alignment = std::mem::align_of::<T>();
            let mut bytes = vec![0u8; encoded.len() + alignment + 1];
            let start = (1..=alignment)
                .find(|offset| !bytes[*offset..].as_ptr().cast::<T>().is_aligned())
                .unwrap();
            bytes[start..start + encoded.len()].copy_from_slice(encoded);
            let array = DataArray::wrap(&ArrayType::MZArray, dtype, bytes);
            let slice = DataArraySlice::new(&array, start, start + encoded.len());
            let mut iter = slice.iter_type::<T>().unwrap();
            for value in expected {
                assert_eq!(iter.next(), Some(*value));
            }
            assert_eq!(iter.next(), None);
            assert_eq!(iter.next(), None);

            let mut iter = crate::traits::DataSliceIter::<T>::new(Cow::Borrowed(
                &array.data[start..start + encoded.len() + 1],
            ));
            assert_eq!(iter.by_ref().collect::<Vec<_>>(), expected);
            assert_eq!(iter.next(), None);
        }

        check(
            BinaryDataArrayType::Float64,
            &[1.25_f64.to_le_bytes(), (-2.5_f64).to_le_bytes()].concat(),
            &[1.25_f64, -2.5],
        );
        check(
            BinaryDataArrayType::Float32,
            &[1.25_f32.to_le_bytes(), (-2.5_f32).to_le_bytes()].concat(),
            &[1.25_f32, -2.5],
        );
        check(
            BinaryDataArrayType::Int64,
            &[42_i64.to_le_bytes(), (-17_i64).to_le_bytes()].concat(),
            &[42_i64, -17],
        );
        check(
            BinaryDataArrayType::Int32,
            &[42_i32.to_le_bytes(), (-17_i32).to_le_bytes()].concat(),
            &[42_i32, -17],
        );
    }

    #[cfg(feature = "zstd")]
    #[test]
    fn test_dict_from_base64() -> io::Result<()> {
        let mut da = make_array_from_file_im_zstd()?;

        da.decode_and_store()?;
        assert_eq!(da.data_len()?, 221);

        da.store_compressed(BinaryCompressionType::ZstdDict)?;
        assert_eq!(da.data_len()?, 221);

        da.store_compressed(BinaryCompressionType::ShuffleZstd)?;
        assert_eq!(da.data_len()?, 221);

        Ok(())
    }
}