mzdata_bindata/
encodings.rs

1use bytemuck::{self, Pod};
2
3use std::{
4    fmt::Display,
5    io,
6    ops::{AddAssign, Mul},
7};
8
9use thiserror::{self, Error};
10use num_traits::{Num, ToBytes};
11
12#[cfg(feature = "numpress")]
13use numpress;
14
15use mzdata_param::{curie, Param, ControlledVocabulary, ParamCow, Unit, CURIE, ValueRef};
16
17pub type Bytes = Vec<u8>;
18
19/// Convert the values in `data` into an owned buffer of little-endian bytes
20pub fn to_bytes<T: Pod + ToBytes>(data: &[T]) -> Bytes {
21    let n = data.len();
22    let mut buf = Vec::with_capacity(n * size_of::<T>());
23    for v in data {
24        buf.extend_from_slice(v.to_le_bytes().as_ref());
25    }
26    buf
27}
28
29pub fn as_bytes<T: Pod>(data: &[T]) -> &[u8] {
30    bytemuck::cast_slice(data)
31}
32
33pub fn vec_as_bytes<T: Pod>(data: Vec<T>) -> Bytes {
34    let mut buf = Bytes::with_capacity(data.len() * std::mem::size_of::<T>());
35    for val in data {
36        buf.extend_from_slice(bytemuck::bytes_of(&val));
37    }
38    buf
39}
40
41mod byte_rotation {
42    use super::*;
43
44    pub fn transpose_bytes_into<T: Pod, const N: usize>(data: &[T], buffer: &mut Vec<u8>) {
45        assert_eq!(core::mem::size_of::<T>(), N);
46        let bytes = bytemuck::cast_slice::<T, [u8; N]>(data);
47        buffer.clear();
48        let delta = (data.len() * N).saturating_sub(buffer.capacity());
49        if delta > 0 {
50            buffer.reserve(delta);
51        }
52
53        #[cfg(target_endian = "little")]
54        {
55            for i in 0..N {
56                buffer.extend(bytes.iter().map(|b| b[i]))
57            }
58        }
59        #[cfg(target_endian = "big")]
60        {
61            for i in (0..N).rev() {
62                buffer.extend(bytes.iter().map(|b| b[i]))
63            }
64        }
65    }
66
67    pub fn transpose_bytes<T: Pod, const N: usize>(data: &[T]) -> Bytes {
68        let mut result = Bytes::with_capacity(data.len() * N);
69        transpose_bytes_into::<T, N>(data, &mut result);
70        result
71    }
72
73    pub fn transpose_4bytes<T: Pod>(data: &[T]) -> Bytes {
74        assert_eq!(std::mem::size_of::<T>(), 4);
75        transpose_bytes::<_, 4>(data)
76    }
77
78    pub fn transpose_8bytes<T: Pod>(data: &[T]) -> Bytes {
79        assert_eq!(std::mem::size_of::<T>(), 8);
80        transpose_bytes::<_, 8>(data)
81    }
82
83    pub fn transpose_i32(data: &[i32]) -> Bytes {
84        transpose_4bytes(data)
85    }
86
87    pub fn transpose_f32(data: &[f32]) -> Bytes {
88        transpose_4bytes(data)
89    }
90
91    pub fn transpose_i64(data: &[i64]) -> Bytes {
92        transpose_8bytes(data)
93    }
94
95    pub fn transpose_f64(data: &[f64]) -> Bytes {
96        transpose_8bytes(data)
97    }
98
99    pub fn reverse_transpose_bytes_into<const N: usize>(data: &[u8], buffer: &mut Vec<u8>) {
100        let rem = data.len() % N;
101        assert_eq!(rem, 0);
102        let n_entries = data.len() / N;
103        buffer.clear();
104        buffer.resize(data.len(), 0);
105
106        #[cfg(target_endian = "little")]
107        {
108            for (i, band) in data.chunks_exact(n_entries).enumerate() {
109                for (j, byte) in band.iter().copied().enumerate() {
110                    bytemuck::cast_slice_mut::<_, [u8; N]>(buffer)[j][i] = byte;
111                }
112            }
113        }
114        #[cfg(target_endian = "big")]
115        {
116            for (i, band) in data.chunks_exact(n_entries).enumerate() {
117                for (j, byte) in band.iter().copied().enumerate() {
118                    bytemuck::cast_slice_mut::<_, [u8; N]>(buffer)[j][(N - 1) - i] = byte;
119                }
120            }
121        }
122    }
123
124    pub fn reverse_transpose_bytes<const N: usize>(data: &[u8]) -> Bytes {
125        let mut result: Bytes = vec![0; data.len()];
126        reverse_transpose_bytes_into::<N>(data, &mut result);
127        result
128    }
129
130    pub fn reverse_transpose_4bytes<T: Pod>(data: &[u8]) -> Bytes {
131        assert_eq!(std::mem::size_of::<T>(), 4);
132        reverse_transpose_bytes::<4>(data)
133    }
134
135    pub fn reverse_transpose_8bytes<T: Pod>(data: &[u8]) -> Bytes {
136        assert_eq!(std::mem::size_of::<T>(), 8);
137        reverse_transpose_bytes::<8>(data)
138    }
139
140    pub fn reverse_transpose_i32(data: &[u8]) -> Vec<u8> {
141        reverse_transpose_4bytes::<i32>(data)
142    }
143
144    pub fn reverse_transpose_f32(data: &[u8]) -> Vec<u8> {
145        reverse_transpose_4bytes::<f32>(data)
146    }
147
148    pub fn reverse_transpose_i64(data: &[u8]) -> Vec<u8> {
149        reverse_transpose_8bytes::<i64>(data)
150    }
151
152    pub fn reverse_transpose_f64(data: &[u8]) -> Vec<u8> {
153        reverse_transpose_8bytes::<f64>(data)
154    }
155}
156
157mod dictionary_encoding {
158    use super::*;
159    use io::prelude::*;
160    use num_traits::ops::bytes::{FromBytes, ToBytes};
161    use std::{
162        borrow::Cow,
163        collections::{HashMap, HashSet},
164        hash::Hash,
165        io::BufWriter,
166    };
167
168    trait DictValue<const W: usize>:
169        Pod + ToBytes<Bytes = [u8; W]> + Hash + Eq + Ord + FromBytes<Bytes = [u8; W]>
170    {
171    }
172
173    macro_rules! impl_dict_value {
174        ($val:ty, $size:literal) => {
175            impl DictValue<$size> for $val {}
176        };
177    }
178
179    impl_dict_value!(u8, 1);
180    impl_dict_value!(u16, 2);
181    impl_dict_value!(u32, 4);
182    impl_dict_value!(u64, 8);
183
184    trait DictIndex<const W: usize>:
185        Pod + ToBytes<Bytes = [u8; W]> + FromBytes<Bytes = [u8; W]>
186    {
187        fn from_usize(index: usize) -> Self;
188        fn to_usize(&self) -> usize;
189    }
190
191    macro_rules! impl_dict_index {
192        ($idx:ty, $size:literal) => {
193            impl DictIndex<$size> for $idx {
194                fn from_usize(index: usize) -> Self {
195                    index as Self
196                }
197
198                fn to_usize(&self) -> usize {
199                    *self as usize
200                }
201            }
202        };
203    }
204
205    impl_dict_index!(u8, 1);
206    impl_dict_index!(u16, 2);
207    impl_dict_index!(u32, 4);
208    impl_dict_index!(u64, 8);
209
210    #[derive(Default, Debug)]
211    pub struct DictionaryEncoder {
212        shuffle: bool,
213        buffer: Vec<u8>,
214    }
215
216    impl DictionaryEncoder {
217        pub fn new(shuffle: bool) -> Self {
218            Self {
219                shuffle,
220                buffer: Vec::new(),
221            }
222        }
223
224        fn build_value_map<T: Pod, const W1: usize, V: DictValue<W1>>(
225            &self,
226            data: &[T],
227        ) -> (Vec<V>, HashMap<V, usize>) {
228            debug_assert_eq!(core::mem::size_of::<T>(), core::mem::size_of::<V>());
229            debug_assert_eq!(core::mem::size_of::<T>(), W1);
230            let mut value_codes = HashSet::new();
231            for v in data {
232                let k: V = *bytemuck::from_bytes(bytemuck::bytes_of(v));
233                value_codes.insert(k);
234            }
235
236            let mut value_codes: Vec<_> = value_codes.into_iter().collect();
237            value_codes.sort();
238
239            let byte_map: HashMap<V, usize> = value_codes
240                .iter()
241                .enumerate()
242                .map(|(i, k)| (*k, i))
243                .collect();
244            (value_codes, byte_map)
245        }
246
247        fn create_writer<
248            T: Pod,
249            const W1: usize,
250            V: Pod + Ord + Hash + ToBytes<Bytes = [u8; W1]> + Eq,
251            const W2: usize,
252            K: DictIndex<W2>,
253        >(
254            &self,
255            data: &[T],
256            value_codes: &[V],
257        ) -> BufWriter<Vec<u8>> {
258            let data_offset = 16 + std::mem::size_of_val(value_codes);
259            let dict_buffer: Vec<u8> =
260                Vec::with_capacity(data_offset + data.len() * core::mem::size_of::<K>());
261            BufWriter::new(dict_buffer)
262        }
263
264        fn encode_dict_indices<
265            T: Pod,
266            const W1: usize,
267            V: Pod + Ord + Hash + ToBytes<Bytes = [u8; W1]> + Eq,
268            const W2: usize,
269            K: DictIndex<W2>,
270        >(
271            &mut self,
272            data: &[T],
273            value_codes: &[V],
274            byte_map: HashMap<V, usize>,
275        ) -> io::Result<Vec<u8>> {
276            let data_offset = 16 + std::mem::size_of_val(value_codes);
277            let mut writer = self.create_writer::<T, W1, V, W2, K>(data, value_codes);
278            writer.write_all(&(data_offset as u64).to_le_bytes())?;
279            writer.write_all(&(value_codes.len() as u64).to_le_bytes())?;
280
281            if self.shuffle {
282                // This isn't endian-correct yet, see note in `transpose_bytes_into`
283                byte_rotation::transpose_bytes_into::<V, W1>(value_codes, &mut self.buffer);
284                writer.write_all(&self.buffer)?;
285                // for v in self.buffer.chunks(W1) {
286                //     writer.write_all(v)?;
287                // }
288            } else {
289                for v in value_codes.iter() {
290                    let bts = v.to_le_bytes();
291                    writer.write_all(&bts)?;
292                }
293            }
294
295            if self.shuffle {
296                let mut buf = Vec::with_capacity(data.len());
297                for v in data {
298                    let i = *byte_map
299                        .get(bytemuck::from_bytes(bytemuck::bytes_of(v)))
300                        .unwrap();
301                    let ik: K = K::from_usize(i);
302                    buf.push(ik);
303                }
304                // This isn't endian-correct
305                byte_rotation::transpose_bytes_into::<K, W2>(&buf, &mut self.buffer);
306                writer.write_all(&self.buffer)?;
307            } else {
308                for v in data {
309                    let i = *byte_map
310                        .get(bytemuck::from_bytes(bytemuck::bytes_of(v)))
311                        .unwrap();
312                    let ik: K = K::from_usize(i);
313                    writer.write_all(&ik.to_le_bytes())?;
314                }
315            }
316
317            writer.flush()?;
318            let val = writer.into_inner().unwrap();
319            Ok(val)
320        }
321
322        fn encode_values<T: Pod, const W1: usize, V: DictValue<W1>>(
323            &mut self,
324            data: &[T],
325        ) -> Result<Vec<u8>, io::Error> {
326            let (value_codes, byte_map) = self.build_value_map(data);
327            let n_value_codes = value_codes.len();
328
329            if n_value_codes <= 2usize.pow(8) {
330                self.encode_dict_indices::<T, W1, V, 1, u8>(data, &value_codes, byte_map)
331            } else if n_value_codes <= 2usize.pow(16) {
332                self.encode_dict_indices::<T, W1, V, 2, u16>(data, &value_codes, byte_map)
333            } else if n_value_codes <= 2usize.pow(32) {
334                self.encode_dict_indices::<T, W1, V, 4, u32>(data, &value_codes, byte_map)
335            } else if n_value_codes <= 2usize.pow(64) {
336                self.encode_dict_indices::<T, W1, V, 8, u64>(data, &value_codes, byte_map)
337            } else {
338                Err(io::Error::new(
339                    io::ErrorKind::Unsupported,
340                    "Cannot encode a dictionary with more than 2 ** 64 values",
341                ))
342            }
343        }
344
345        pub fn encode<T: Pod>(&mut self, data: &[T]) -> io::Result<Bytes> {
346            if data.is_empty() {
347                return Ok(Vec::new());
348            }
349            let z_val = core::mem::size_of::<T>();
350            if z_val <= 1 {
351                self.encode_values::<T, 1, u8>(data)
352            } else if z_val <= 2 {
353                self.encode_values::<T, 2, u16>(data)
354            } else if z_val <= 4 {
355                self.encode_values::<T, 4, u32>(data)
356            } else if z_val <= 8 {
357                self.encode_values::<T, 8, u64>(data)
358            } else {
359                Err(io::Error::new(
360                    io::ErrorKind::Unsupported,
361                    "Cannot encode a dictionary with more than 2 ** 64 keys",
362                ))
363            }
364        }
365    }
366
367    #[derive(Default, Debug)]
368    pub struct DictionaryDecoder {
369        shuffle: bool,
370        buffer: Vec<u8>,
371    }
372
373    impl DictionaryDecoder {
374        pub fn new(shuffle: bool) -> Self {
375            Self {
376                shuffle,
377                buffer: Default::default(),
378            }
379        }
380
381        fn make_reader<'a>(&self, buffer: &'a [u8]) -> io::BufReader<&'a [u8]> {
382            io::BufReader::new(buffer)
383        }
384
385        fn decode_value_buffer<T: Pod, const W1: usize, V: DictValue<W1>>(
386            &mut self,
387            buffer: &[u8],
388            n_values: usize,
389        ) -> Vec<T> {
390            macro_rules! decode_chunk {
391                ($chunk:ident) => {{
392                    let chunk_a: [u8; W1] = $chunk.try_into().unwrap();
393                    let val = V::from_le_bytes(&chunk_a);
394                    let val: T = *bytemuck::from_bytes(bytemuck::bytes_of(&val));
395                    val
396                }};
397            }
398            let mut value_buffer = Vec::with_capacity(n_values);
399            if self.shuffle {
400                let blocks = match core::mem::size_of::<T>() {
401                    1 => Cow::Borrowed(buffer),
402                    2 => {
403                        byte_rotation::reverse_transpose_bytes_into::<2>(buffer, &mut self.buffer);
404                        Cow::Borrowed(self.buffer.as_slice())
405                    }
406                    4 => {
407                        byte_rotation::reverse_transpose_bytes_into::<4>(buffer, &mut self.buffer);
408                        Cow::Borrowed(self.buffer.as_slice())
409                    }
410                    8 => {
411                        byte_rotation::reverse_transpose_bytes_into::<8>(buffer, &mut self.buffer);
412                        Cow::Borrowed(self.buffer.as_slice())
413                    }
414                    x => {
415                        panic!("Unsupported size {x}");
416                    }
417                };
418                for chunk in blocks.chunks_exact(W1) {
419                    let val = decode_chunk!(chunk);
420                    value_buffer.push(val);
421                }
422            } else {
423                for chunk in buffer.chunks_exact(W1) {
424                    let val = decode_chunk!(chunk);
425                    value_buffer.push(val);
426                }
427            };
428            value_buffer
429        }
430
431        fn decode_index_buffer<T: Pod, const W2: usize, K: DictIndex<W2>>(
432            &mut self,
433            value_codes: &[T],
434            index_buffer: &[u8],
435        ) -> Vec<T> {
436            let mut result = Vec::with_capacity(index_buffer.len() / W2);
437            if self.shuffle {
438                byte_rotation::reverse_transpose_bytes_into::<W2>(index_buffer, &mut self.buffer);
439                for chunk in self.buffer.chunks_exact(W2) {
440                    let b: [u8; W2] = chunk.try_into().unwrap();
441                    let k: usize = K::from_le_bytes(&b).to_usize();
442                    result.push(value_codes[k])
443                }
444            } else {
445                for chunk in index_buffer.chunks_exact(W2) {
446                    let b: [u8; W2] = chunk.try_into().unwrap();
447                    let k: usize = K::from_le_bytes(&b).to_usize();
448                    result.push(value_codes[k])
449                }
450            }
451            result
452        }
453
454        pub fn decode<T: Pod>(&mut self, buffer: &[u8]) -> io::Result<Vec<T>> {
455            if buffer.is_empty() {
456                return Ok(Vec::new());
457            }
458            let mut reader = self.make_reader(buffer);
459            let mut z_buf = [0u8; 8];
460            reader.read_exact(&mut z_buf)?;
461            let data_offset = u64::from_le_bytes(z_buf);
462            if data_offset == 0 {
463                return Ok(Vec::new());
464            }
465            let mut z_buf = [0u8; 8];
466            reader.read_exact(&mut z_buf)?;
467            let n_value_codes = u64::from_le_bytes(z_buf);
468            if n_value_codes == 0 {
469                return Ok(Vec::new());
470            }
471            let value_buffer = &buffer[16..(data_offset as usize)];
472            let value_width = (data_offset - 16) / n_value_codes;
473            let index_buffer = &buffer[data_offset as usize..];
474
475            let n_value_codes = n_value_codes as usize;
476
477            macro_rules! decode_indices {
478                ($values:ident) => {
479                    if n_value_codes <= 2usize.pow(8) {
480                        self.decode_index_buffer::<T, 1, u8>(&$values, index_buffer)
481                    } else if n_value_codes <= 2usize.pow(16) {
482                        self.decode_index_buffer::<T, 2, u16>(&$values, index_buffer)
483                    } else if n_value_codes <= 2usize.pow(32) {
484                        self.decode_index_buffer::<T, 4, u32>(&$values, index_buffer)
485                    } else if n_value_codes <= 2usize.pow(64) {
486                        self.decode_index_buffer::<T, 8, u64>(&$values, index_buffer)
487                    } else {
488                        return Err(io::Error::new(
489                            io::ErrorKind::Unsupported,
490                            "Cannot decode a dictionary with more than 2 ** 64 indices",
491                        ));
492                    }
493                };
494            }
495
496            let values = if value_width <= 1 {
497                let values = self.decode_value_buffer::<T, 1, u8>(value_buffer, n_value_codes);
498                decode_indices!(values)
499            } else if value_width <= 2 {
500                let values = self.decode_value_buffer::<T, 2, u16>(value_buffer, n_value_codes);
501                decode_indices!(values)
502            } else if value_width <= 4 {
503                let values = self.decode_value_buffer::<T, 4, u32>(value_buffer, n_value_codes);
504                decode_indices!(values)
505            } else if value_width <= 8 {
506                let values = self.decode_value_buffer::<T, 8, u64>(value_buffer, n_value_codes);
507                decode_indices!(values)
508            } else {
509                return Err(io::Error::new(
510                    io::ErrorKind::Unsupported,
511                    "Cannot decode dictionary with value byte width greater than 8",
512                ));
513            };
514
515            Ok(values)
516        }
517    }
518
519    pub fn dictionary_encoding<T: Pod>(data: &[T]) -> Result<Vec<u8>, io::Error> {
520        let mut encoder = DictionaryEncoder::new(true);
521        encoder.encode(data)
522    }
523
524    pub fn dictionary_decoding<T: Pod>(buffer: &[u8]) -> io::Result<Vec<T>> {
525        let mut decoder = DictionaryDecoder::new(true);
526        decoder.decode(buffer)
527    }
528}
529
530#[allow(unused)]
531pub use byte_rotation::*;
532
533#[allow(unused)]
534pub use dictionary_encoding::{dictionary_decoding, dictionary_encoding};
535
536/// The kinds of data arrays found in mass spectrometry data files governed
537/// by the PSI-MS controlled vocabulary.
538#[derive(Debug, Clone, PartialEq, Hash, Eq, PartialOrd, Ord, Default)]
539#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
540pub enum ArrayType {
541    #[default]
542    Unknown,
543    MZArray,
544    IntensityArray,
545    ChargeArray,
546    SignalToNoiseArray,
547    TimeArray,
548    WavelengthArray,
549
550    IonMobilityArray,
551    MeanIonMobilityArray,
552    MeanDriftTimeArray,
553    MeanInverseReducedIonMobilityArray,
554    RawIonMobilityArray,
555    RawDriftTimeArray,
556    RawInverseReducedIonMobilityArray,
557    DeconvolutedIonMobilityArray,
558    DeconvolutedDriftTimeArray,
559    DeconvolutedInverseReducedIonMobilityArray,
560
561    ScanningQuadrupolePositionLowerBoundMZ,
562    ScanningQuadrupolePositionUpperBoundMZ,
563
564    IndexArray,
565
566    BaselineArray,
567    ResolutionArray,
568    PressureArray,
569    TemperatureArray,
570    FlowRateArray,
571    NonStandardDataArray {
572        name: Box<String>,
573    },
574}
575
576impl Display for ArrayType {
577    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
578        write!(f, "{:?}", self)
579    }
580}
581
582impl ArrayType {
583    /// Get the data type that the array is compatible with in the
584    /// `mzdata` type expectations.
585    ///
586    /// By default, the m/z array is encoded using `Float64`,
587    /// the charge state array is encoded using `Int32`, and
588    /// all other arrays are encoded using `Float32`.
589    pub const fn preferred_dtype(&self) -> BinaryDataArrayType {
590        match self {
591            ArrayType::MZArray => BinaryDataArrayType::Float64,
592            ArrayType::IntensityArray => BinaryDataArrayType::Float32,
593            ArrayType::ChargeArray => BinaryDataArrayType::Int32,
594            ArrayType::IndexArray => BinaryDataArrayType::Int64,
595            _ => BinaryDataArrayType::Float32,
596        }
597    }
598
599    /// Convert an ion mobility array to its mean variant
600    pub const fn as_mean_ion_mobility(&self) -> Option<ArrayType> {
601        Some(match self {
602            Self::RawDriftTimeArray
603            | Self::DeconvolutedDriftTimeArray
604            | Self::MeanDriftTimeArray => Self::MeanDriftTimeArray,
605            Self::RawInverseReducedIonMobilityArray
606            | Self::DeconvolutedInverseReducedIonMobilityArray
607            | Self::MeanInverseReducedIonMobilityArray => Self::MeanInverseReducedIonMobilityArray,
608            Self::RawIonMobilityArray
609            | Self::DeconvolutedIonMobilityArray
610            | Self::MeanIonMobilityArray => Self::MeanIonMobilityArray,
611            _ => return None,
612        })
613    }
614
615    /// Convert an ion mobility array to its raw variant
616    pub const fn as_raw_ion_mobility(&self) -> Option<ArrayType> {
617        Some(match self {
618            Self::RawDriftTimeArray
619            | Self::DeconvolutedDriftTimeArray
620            | Self::MeanDriftTimeArray => Self::RawDriftTimeArray,
621            Self::RawInverseReducedIonMobilityArray
622            | Self::DeconvolutedInverseReducedIonMobilityArray
623            | Self::MeanInverseReducedIonMobilityArray => Self::RawInverseReducedIonMobilityArray,
624            Self::RawIonMobilityArray
625            | Self::DeconvolutedIonMobilityArray
626            | Self::MeanIonMobilityArray => Self::RawIonMobilityArray,
627            _ => return None,
628        })
629    }
630
631    /// Convert an ion mobility array to its deconvoluted variant
632    pub const fn as_deconvoluted_ion_mobility(&self) -> Option<ArrayType> {
633        Some(match self {
634            Self::RawDriftTimeArray
635            | Self::DeconvolutedDriftTimeArray
636            | Self::MeanDriftTimeArray => Self::DeconvolutedDriftTimeArray,
637            Self::RawInverseReducedIonMobilityArray
638            | Self::DeconvolutedInverseReducedIonMobilityArray
639            | Self::MeanInverseReducedIonMobilityArray => {
640                Self::DeconvolutedInverseReducedIonMobilityArray
641            }
642            Self::RawIonMobilityArray
643            | Self::DeconvolutedIonMobilityArray
644            | Self::MeanIonMobilityArray => Self::DeconvolutedIonMobilityArray,
645            _ => return None,
646        })
647    }
648
649    /// Create a [`ArrayType::NonStandardDataArray`] with the provided name.
650    pub fn nonstandard<S: ToString>(name: S) -> ArrayType {
651        ArrayType::NonStandardDataArray {
652            name: name.to_string().into(),
653        }
654    }
655
656    /// Test if the the array describes an ion mobility quantity.
657    pub const fn is_ion_mobility(&self) -> bool {
658        matches!(
659            self,
660            Self::IonMobilityArray
661                | Self::MeanIonMobilityArray
662                | Self::MeanDriftTimeArray
663                | Self::MeanInverseReducedIonMobilityArray
664                | Self::DeconvolutedIonMobilityArray
665                | Self::DeconvolutedDriftTimeArray
666                | Self::DeconvolutedInverseReducedIonMobilityArray
667                | Self::RawIonMobilityArray
668                | Self::RawDriftTimeArray
669                | Self::RawInverseReducedIonMobilityArray,
670        )
671    }
672
673    /// Create a [`Param`] for this array type.
674    ///
675    /// If a unit is provided, that unit will be specified, otherwise a default unit may
676    /// be used instead.
677    pub fn as_param(&self, unit: Option<Unit>) -> Param {
678        const CV: ControlledVocabulary = ControlledVocabulary::MS;
679        match self {
680            ArrayType::MZArray => CV
681                .const_param_ident_unit("m/z array", 1000514, unit.unwrap_or(Unit::MZ))
682                .into(),
683            ArrayType::IntensityArray => CV
684                .const_param_ident_unit(
685                    "intensity array",
686                    1000515,
687                    unit.unwrap_or(Unit::DetectorCounts),
688                )
689                .into(),
690            ArrayType::ChargeArray => CV.const_param_ident("charge array", 1000516).into(),
691            ArrayType::TimeArray => CV
692                .const_param_ident_unit("time array", 1000595, unit.unwrap_or(Unit::Minute))
693                .into(),
694            ArrayType::WavelengthArray => CV
695                .const_param_ident_unit("wavelength array", 1000617, Unit::Nanometer)
696                .into(),
697            ArrayType::SignalToNoiseArray => CV
698                .const_param_ident("signal to noise array", 1000517)
699                .into(),
700            ArrayType::IonMobilityArray => CV
701                .const_param_ident_unit("ion mobility array", 1002893, unit.unwrap_or_default())
702                .into(),
703
704            ArrayType::RawDriftTimeArray => CV
705                .const_param_ident_unit(
706                    "raw ion mobility drift time array",
707                    1003153,
708                    unit.unwrap_or_default(),
709                )
710                .into(),
711            ArrayType::RawInverseReducedIonMobilityArray => CV
712                .const_param_ident_unit(
713                    "raw inverse reduced ion mobility array",
714                    1003008,
715                    unit.unwrap_or_default(),
716                )
717                .into(),
718            ArrayType::RawIonMobilityArray => CV
719                .const_param_ident_unit("raw ion mobility array", 1003007, unit.unwrap_or_default())
720                .into(),
721
722            ArrayType::MeanIonMobilityArray => CV
723                .const_param_ident_unit(
724                    "mean ion mobility array",
725                    1002816,
726                    unit.unwrap_or_default(),
727                )
728                .into(),
729            ArrayType::MeanDriftTimeArray => CV
730                .const_param_ident_unit(
731                    "mean ion mobility drift time array",
732                    1002477,
733                    unit.unwrap_or_default(),
734                )
735                .into(),
736            ArrayType::MeanInverseReducedIonMobilityArray => CV
737                .const_param_ident_unit(
738                    "mean inverse reduced ion mobility array",
739                    1003006,
740                    unit.unwrap_or_default(),
741                )
742                .into(),
743
744            ArrayType::DeconvolutedIonMobilityArray => CV
745                .const_param_ident_unit(
746                    "deconvoluted ion mobility array",
747                    1003154,
748                    unit.unwrap_or_default(),
749                )
750                .into(),
751            ArrayType::DeconvolutedDriftTimeArray => CV
752                .const_param_ident_unit(
753                    "deconvoluted ion mobility drift time array",
754                    1003156,
755                    unit.unwrap_or_default(),
756                )
757                .into(),
758            ArrayType::DeconvolutedInverseReducedIonMobilityArray => CV
759                .const_param_ident_unit(
760                    "deconvoluted inverse reduced ion mobility array",
761                    1003155,
762                    unit.unwrap_or_default(),
763                )
764                .into(),
765
766            ArrayType::NonStandardDataArray { name } => {
767                let mut p = CV.param_val(1000786, "non-standard data array", name.to_string());
768                p.unit = unit.unwrap_or_default();
769                p
770            }
771            ArrayType::BaselineArray => CV.const_param_ident("baseline array", 1002530).into(),
772            ArrayType::ResolutionArray => CV.const_param_ident("resolution array", 1002529).into(),
773            ArrayType::PressureArray => {
774                let mut p = CV.const_param_ident("pressure array", 1000821);
775                p.unit = unit.unwrap_or_default();
776                p.into()
777            }
778            ArrayType::TemperatureArray => {
779                let mut p = CV.const_param_ident("temperature array", 1000822);
780                p.unit = unit.unwrap_or_default();
781                p.into()
782            }
783            ArrayType::FlowRateArray => {
784                let mut p = CV.const_param_ident("flow rate array", 1000820);
785                p.unit = unit.unwrap_or_default();
786                p.into()
787            }
788            ArrayType::ScanningQuadrupolePositionLowerBoundMZ => {
789                let mut p = CV.const_param_ident("scanning quadrupole position lower bound m/z array", 1003157);
790                p.unit = unit.unwrap_or_default();
791                p.into()
792            }
793            ArrayType::ScanningQuadrupolePositionUpperBoundMZ => {
794                let mut p = CV.const_param_ident("scanning quadrupole position upper bound m/z array", 1003158);
795                p.unit = unit.unwrap_or_default();
796                p.into()
797            }
798            ArrayType::IndexArray => {
799                CV.const_param_ident("index array", 1003870).into()
800            }
801            _ => {
802                panic!("Could not determine how to name for array {}", self);
803            }
804        }
805    }
806
807    /// Create a [`ParamCow`] for this array type in a `const` context using the default unit.
808    ///
809    /// **NOTE**: If this is a [`Self::NonStandardDataArray`], this function *will* panic as
810    /// this variant requires allocation.
811    pub const fn as_param_const(&self) -> ParamCow<'static> {
812        const CV: ControlledVocabulary = ControlledVocabulary::MS;
813        match self {
814            ArrayType::MZArray => CV.const_param_ident_unit("m/z array", 1000514, Unit::MZ),
815            ArrayType::IntensityArray => {
816                CV.const_param_ident_unit("intensity array", 1000515, Unit::DetectorCounts)
817            }
818            ArrayType::ChargeArray => CV.const_param_ident("charge array", 1000516),
819            ArrayType::TimeArray => CV.const_param_ident_unit("time array", 1000595, Unit::Minute),
820            ArrayType::WavelengthArray => {
821                CV.const_param_ident_unit("wavelength array", 1000617, Unit::Nanometer)
822            }
823            ArrayType::SignalToNoiseArray => CV.const_param_ident("signal to noise array", 1000517),
824            ArrayType::IonMobilityArray => CV.const_param_ident("ion mobility array", 1002893),
825            ArrayType::RawIonMobilityArray => {
826                CV.const_param_ident("raw ion mobility array", 1003007)
827            }
828            ArrayType::MeanIonMobilityArray => {
829                CV.const_param_ident("mean ion mobility array", 1002816)
830            }
831            ArrayType::DeconvolutedIonMobilityArray => {
832                CV.const_param_ident("deconvoluted ion mobility array", 1003154)
833            }
834            ArrayType::RawDriftTimeArray => CV.const_param_ident_unit(
835                "raw ion mobility drift time array",
836                1003153,
837                Unit::Unknown,
838            ),
839            ArrayType::RawInverseReducedIonMobilityArray => CV.const_param_ident_unit(
840                "raw inverse reduced ion mobility array",
841                1003008,
842                Unit::VoltSecondPerSquareCentimeter,
843            ),
844
845            ArrayType::MeanDriftTimeArray => CV.const_param_ident_unit(
846                "mean ion mobility drift time array",
847                1002477,
848                Unit::Unknown,
849            ),
850            ArrayType::MeanInverseReducedIonMobilityArray => CV.const_param_ident_unit(
851                "mean inverse reduced ion mobility array",
852                1003006,
853                Unit::VoltSecondPerSquareCentimeter,
854            ),
855
856            ArrayType::DeconvolutedDriftTimeArray => CV.const_param_ident_unit(
857                "deconvoluted ion mobility drift time array",
858                1003156,
859                Unit::Unknown,
860            ),
861            ArrayType::DeconvolutedInverseReducedIonMobilityArray => CV.const_param_ident_unit(
862                "deconvoluted inverse reduced ion mobility array",
863                1003155,
864                Unit::VoltSecondPerSquareCentimeter,
865            ),
866
867            ArrayType::NonStandardDataArray { name: _name } => {
868                panic!(
869                    "Cannot format NonStandardDataArray in a const context, please use `as_param`"
870                );
871            }
872            ArrayType::BaselineArray => CV.const_param_ident("baseline array", 1002530),
873            ArrayType::ResolutionArray => CV.const_param_ident("resolution array", 1002529),
874            ArrayType::PressureArray => CV.const_param_ident_unit("pressure array", 1000821, Unit::Pascal),
875            ArrayType::TemperatureArray => CV.const_param_ident("temperature array", 1000822),
876            ArrayType::FlowRateArray => CV.const_param_ident_unit("flow rate array", 1000820, Unit::MicrolitersPerMinute),
877            ArrayType::ScanningQuadrupolePositionLowerBoundMZ => {
878                let mut p = CV.const_param_ident("scanning quadrupole position lower bound m/z array", 1003157);
879                p.unit = Unit::MZ;
880                p
881            }
882            ArrayType::IndexArray => {
883                CV.const_param_ident("index array", 1003870)
884            }
885            ArrayType::ScanningQuadrupolePositionUpperBoundMZ => {
886                let mut p = CV.const_param_ident("scanning quadrupole position upper bound m/z array", 1003158);
887                p.unit = Unit::MZ;
888                p
889            }
890            _ => {
891                panic!("Could not determine how to name for array");
892            }
893        }
894    }
895
896    /// Create a [`ParamCow`] for this array type in a `const` context using the provided unit.
897    ///
898    /// **NOTE**: If this is a [`Self::NonStandardDataArray`], this function *will* panic as
899    /// this variant requires allocation.
900    pub const fn as_param_with_unit_const(&self, unit: Unit) -> ParamCow<'static> {
901        const CV: ControlledVocabulary = ControlledVocabulary::MS;
902        match self {
903            ArrayType::MZArray => CV.const_param_ident_unit("m/z array", 1000514, unit),
904            ArrayType::IntensityArray => {
905                CV.const_param_ident_unit("intensity array", 1000515, unit)
906            }
907            ArrayType::ChargeArray => CV.const_param_ident_unit("charge array", 1000516, unit),
908            ArrayType::TimeArray => CV.const_param_ident_unit("time array", 1000595, unit),
909            ArrayType::RawIonMobilityArray => {
910                CV.const_param_ident_unit("raw ion mobility array", 1003007, unit)
911            }
912            ArrayType::MeanIonMobilityArray => {
913                CV.const_param_ident_unit("mean ion mobility array", 1002816, unit)
914            }
915            ArrayType::DeconvolutedIonMobilityArray => {
916                CV.const_param_ident_unit("deconvoluted ion mobility array", 1003154, unit)
917            }
918            ArrayType::NonStandardDataArray { name: _name } => {
919                panic!(
920                    "Cannot format NonStandardDataArray in a const context, please use `as_param`"
921                );
922            }
923
924            ArrayType::RawDriftTimeArray => {
925                CV.const_param_ident_unit("raw ion mobility drift time array", 1003153, unit)
926            }
927            ArrayType::RawInverseReducedIonMobilityArray => {
928                CV.const_param_ident_unit("raw inverse reduced ion mobility array", 1003008, unit)
929            }
930
931            ArrayType::MeanDriftTimeArray => {
932                CV.const_param_ident_unit("mean ion mobility drift time array", 1002477, unit)
933            }
934            ArrayType::MeanInverseReducedIonMobilityArray => {
935                CV.const_param_ident_unit("mean inverse reduced ion mobility array", 1003006, unit)
936            }
937
938            ArrayType::DeconvolutedDriftTimeArray => CV.const_param_ident_unit(
939                "deconvoluted ion mobility drift time array",
940                1003156,
941                unit,
942            ),
943            ArrayType::DeconvolutedInverseReducedIonMobilityArray => CV.const_param_ident_unit(
944                "deconvoluted inverse reduced ion mobility array",
945                1003155,
946                unit,
947            ),
948
949            ArrayType::BaselineArray => CV.const_param_ident_unit("baseline array", 1002530, unit),
950            ArrayType::ResolutionArray => {
951                CV.const_param_ident_unit("resolution array", 1002529, unit)
952            }
953            ArrayType::PressureArray => CV.const_param_ident_unit("pressure array", 1000821, unit),
954            ArrayType::TemperatureArray => {
955                CV.const_param_ident_unit("temperature array", 1000822, unit)
956            }
957            ArrayType::FlowRateArray => CV.const_param_ident_unit("flow rate array", 1000820, unit),
958            ArrayType::ScanningQuadrupolePositionLowerBoundMZ => {
959                CV.const_param_ident_unit("scanning quadrupole position lower bound m/z array", 1003157, unit)
960            }
961            ArrayType::ScanningQuadrupolePositionUpperBoundMZ => {
962                CV.const_param_ident_unit("scanning quadrupole position upper bound m/z array", 1003158, unit)
963            }
964            ArrayType::SignalToNoiseArray => {
965                CV.const_param_ident_unit("signal to noise array", 1000517, unit)
966            }
967            ArrayType::WavelengthArray => {
968                CV.const_param_ident_unit("wavelength array", 1000617, unit)
969            }
970            ArrayType::IonMobilityArray => {
971                CV.const_param_ident_unit("ion mobility array", 1002893, unit)
972            }
973            ArrayType::IndexArray => {
974                CV.const_param_ident_unit("index array", 1003870, unit)
975            }
976            _ => {
977                panic!("Could not determine how to name for array");
978            }
979        }
980    }
981
982    /// Translate from [`CURIE`] to an [`ArrayType`]
983    pub fn from_accession(x: CURIE) -> Option<Self> {
984        let tp = if x == Self::MZArray.as_param_const().curie().unwrap() {
985            Self::MZArray
986        } else if x == Self::IntensityArray.as_param_const().curie().unwrap() {
987            Self::IntensityArray
988        } else if x == Self::ChargeArray.as_param_const().curie().unwrap() {
989            Self::ChargeArray
990        } else if x == Self::SignalToNoiseArray.as_param_const().curie().unwrap() {
991            Self::SignalToNoiseArray
992        } else if x == Self::TimeArray.as_param_const().curie().unwrap() {
993            Self::TimeArray
994        } else if x == Self::WavelengthArray.as_param_const().curie().unwrap() {
995            Self::WavelengthArray
996        } else if x == Self::IonMobilityArray.as_param_const().curie().unwrap() {
997            Self::IonMobilityArray
998        } else if x == Self::MeanIonMobilityArray.as_param_const().curie().unwrap() {
999            Self::MeanIonMobilityArray
1000        } else if x == Self::MeanDriftTimeArray.as_param_const().curie().unwrap() {
1001            Self::MeanDriftTimeArray
1002        } else if x
1003            == Self::MeanInverseReducedIonMobilityArray
1004                .as_param_const()
1005                .curie()
1006                .unwrap()
1007        {
1008            Self::MeanInverseReducedIonMobilityArray
1009        } else if x == Self::RawIonMobilityArray.as_param_const().curie().unwrap() {
1010            Self::RawIonMobilityArray
1011        } else if x == Self::RawDriftTimeArray.as_param_const().curie().unwrap() {
1012            Self::RawDriftTimeArray
1013        } else if x
1014            == Self::RawInverseReducedIonMobilityArray
1015                .as_param_const()
1016                .curie()
1017                .unwrap()
1018        {
1019            Self::RawInverseReducedIonMobilityArray
1020        } else if x
1021            == Self::DeconvolutedIonMobilityArray
1022                .as_param_const()
1023                .curie()
1024                .unwrap()
1025        {
1026            Self::DeconvolutedIonMobilityArray
1027        } else if x
1028            == Self::DeconvolutedDriftTimeArray
1029                .as_param_const()
1030                .curie()
1031                .unwrap()
1032        {
1033            Self::DeconvolutedDriftTimeArray
1034        } else if x
1035            == Self::DeconvolutedInverseReducedIonMobilityArray
1036                .as_param_const()
1037                .curie()
1038                .unwrap()
1039        {
1040            Self::DeconvolutedInverseReducedIonMobilityArray
1041        } else if x == Self::BaselineArray.as_param_const().curie().unwrap() {
1042            Self::BaselineArray
1043        } else if x == Self::ResolutionArray.as_param_const().curie().unwrap() {
1044            Self::ResolutionArray
1045        } else if x == Self::PressureArray.as_param_const().curie().unwrap() {
1046            Self::PressureArray
1047        } else if x == Self::TemperatureArray.as_param_const().curie().unwrap() {
1048            Self::TemperatureArray
1049        } else if x == Self::FlowRateArray.as_param_const().curie().unwrap() {
1050            Self::FlowRateArray
1051        } else if x == Self::ScanningQuadrupolePositionLowerBoundMZ.as_param_const().curie().unwrap() {
1052            Self::ScanningQuadrupolePositionLowerBoundMZ
1053        } else if x == Self::ScanningQuadrupolePositionUpperBoundMZ.as_param_const().curie().unwrap() {
1054            Self::ScanningQuadrupolePositionUpperBoundMZ
1055        } else if x == Self::IndexArray.as_param_const().curie().unwrap() {
1056            Self::IndexArray
1057        }
1058        else if x
1059            == (Self::NonStandardDataArray {
1060                name: "".to_string().into(),
1061            })
1062            .as_param(None)
1063            .curie()
1064            .unwrap()
1065        {
1066            Self::NonStandardDataArray {
1067                name: "".to_string().into(),
1068            }
1069        } else {
1070            return None;
1071        };
1072        Some(tp)
1073    }
1074}
1075
1076/// The canonical primitive data types found in MS data file formats
1077/// supported by the PSI-MS controlled vocabulary
1078#[derive(Debug, Clone, Copy, PartialEq, Hash, Eq, Default)]
1079#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1080pub enum BinaryDataArrayType {
1081    #[default]
1082    Unknown,
1083    Float64,
1084    Float32,
1085    Int64,
1086    Int32,
1087    ASCII,
1088}
1089
1090impl Display for BinaryDataArrayType {
1091    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1092        write!(f, "{:?}", self)
1093    }
1094}
1095
1096impl BinaryDataArrayType {
1097    /// Get the size in bytes of a single value of this type
1098    pub const fn size_of(&self) -> usize {
1099        match self {
1100            BinaryDataArrayType::Unknown | BinaryDataArrayType::ASCII => 1,
1101            BinaryDataArrayType::Float32 | BinaryDataArrayType::Int32 => 4,
1102            BinaryDataArrayType::Float64 | BinaryDataArrayType::Int64 => 8,
1103        }
1104    }
1105
1106    /// Convert the data type to a [`ParamCow`] if possible.
1107    pub const fn as_param_const(&self) -> Option<ParamCow<'static>> {
1108        let name = match self {
1109            BinaryDataArrayType::Unknown => return None,
1110            BinaryDataArrayType::Float64 => "64-bit float",
1111            BinaryDataArrayType::Float32 => "32-bit float",
1112            BinaryDataArrayType::Int64 => "64-bit integer",
1113            BinaryDataArrayType::Int32 => "32-bit integer",
1114            BinaryDataArrayType::ASCII => "null-terminated ASCII string",
1115        };
1116        if let Some(curie) = self.curie() {
1117            Some(ParamCow::const_new(
1118                name,
1119                ValueRef::Empty,
1120                Some(curie.accession),
1121                Some(curie.controlled_vocabulary),
1122                Unit::Unknown,
1123            ))
1124        } else {
1125            None
1126        }
1127    }
1128
1129    /// Convert the data type to a [`CURIE`] if one exists for it
1130    pub const fn curie(&self) -> Option<CURIE> {
1131        match self {
1132            Self::Float32 => Some(curie!(MS:1000521)),
1133            Self::Float64 => Some(curie!(MS:1000523)),
1134            Self::Int32 => Some(curie!(MS:1000519)),
1135            Self::Int64 => Some(curie!(MS:1000522)),
1136            Self::ASCII => Some(curie!(MS:1001479)),
1137            _ => None,
1138        }
1139    }
1140
1141    pub fn from_accession(accession: CURIE) -> Option<Self> {
1142        match accession {
1143            x if Some(x) == Self::Float32.curie() => Some(Self::Float32),
1144            x if Some(x) == Self::Float64.curie() => Some(Self::Float64),
1145            x if Some(x) == Self::Int32.curie() => Some(Self::Int32),
1146            x if Some(x) == Self::Int64.curie() => Some(Self::Int64),
1147            x if Some(x) == Self::ASCII.curie() => Some(Self::ASCII),
1148            _ => None,
1149        }
1150    }
1151
1152    /// Byte order swap the data stored in
1153    pub fn swap_bytes(&self, data: &mut [u8]) -> Result<(), ArrayRetrievalError> {
1154        let z = self.size_of();
1155        if !(data.len() % z == 0) {
1156            return Err(ArrayRetrievalError::DataTypeSizeMismatch)
1157        }
1158        match z {
1159            1 => {
1160                data.reverse();
1161            }
1162            4 => {
1163                data.as_chunks_mut::<4>().0.into_iter().for_each(|c| {
1164                    *c = u32::from_ne_bytes(*c).swap_bytes().to_ne_bytes();
1165                });
1166            }
1167            8 => {
1168                data.as_chunks_mut::<8>().0.into_iter().for_each(|c| {
1169                    *c = u64::from_ne_bytes(*c).swap_bytes().to_ne_bytes();
1170                });
1171                // data.chunks_exact_mut(4).for_each(|c| {
1172                //     c.swap(0, 7);
1173                //     c.swap(1, 6);
1174                //     c.swap(2, 5);
1175                //     c.swap(3, 4);
1176                // });
1177            }
1178            x => {
1179                data.chunks_exact_mut(x).for_each(|c| c.reverse());
1180            }
1181        }
1182        Ok(())
1183    }
1184}
1185
1186/// The range of compression and encoding states that a raw byte buffer
1187/// might be in during different stages of decoding. Other than `Decoded`,
1188/// these states may or may not include intermediate base64 encoding.
1189#[derive(Debug, Clone, Copy, PartialEq, Hash, Default)]
1190#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1191pub enum BinaryCompressionType {
1192    #[default]
1193    NoCompression,
1194    Zlib,
1195    NumpressLinear,
1196    NumpressSLOF,
1197    NumpressPIC,
1198    NumpressLinearZlib,
1199    NumpressSLOFZlib,
1200    NumpressPICZlib,
1201    LinearPrediction,
1202    DeltaPrediction,
1203    Decoded,
1204    Zstd,
1205    ShuffleZstd,
1206    DeltaShuffleZstd,
1207    ZstdDict,
1208    NumpressLinearZstd,
1209    NumpressSLOFZstd,
1210    NumpressPICZstd,
1211}
1212
1213impl BinaryCompressionType {
1214    pub const COMPRESSION_METHODS: &[Self] = &[
1215        Self::NoCompression,
1216        Self::Zlib,
1217
1218        #[cfg(feature = "numpress")]
1219        Self::NumpressLinear,
1220        #[cfg(feature = "numpress")]
1221        Self::NumpressLinearZlib,
1222        #[cfg(feature = "numpress")]
1223        Self::NumpressSLOF,
1224        #[cfg(feature = "numpress")]
1225        Self::NumpressSLOFZlib,
1226
1227        #[cfg(feature = "zstd")]
1228        Self::Zstd,
1229        #[cfg(feature = "zstd")]
1230        Self::ShuffleZstd,
1231        #[cfg(feature = "zstd")]
1232        Self::DeltaShuffleZstd,
1233        #[cfg(feature = "zstd")]
1234        Self::ZstdDict,
1235
1236        #[cfg(all(feature = "zstd", feature = "numpress"))]
1237        Self::NumpressLinearZstd,
1238        #[cfg(all(feature = "zstd", feature = "numpress"))]
1239        Self::NumpressSLOFZstd,
1240    ];
1241
1242    /// Generate a user-understandable message about why a compression conversion operation failed
1243    pub fn unsupported_msg(&self, context: Option<&str>) -> String {
1244        match context {
1245            Some(ctx) => format!("Cannot decode array compressed with {:?} ({})", self, ctx),
1246            None => format!("Cannot decode array compressed with {:?}", self),
1247        }
1248    }
1249
1250    /// Check if the compressor handles endianess during decoding. Otherwise the data needs to be
1251    /// byteswapped to little endianess on big endian systems before use
1252    pub const fn is_endian_aware(&self) -> bool {
1253        match self {
1254            BinaryCompressionType::Zlib |
1255            BinaryCompressionType::NoCompression |
1256            BinaryCompressionType::DeltaPrediction |
1257            BinaryCompressionType::LinearPrediction |
1258            BinaryCompressionType::Zstd => false,
1259            _ => true,
1260
1261        }
1262    }
1263
1264    pub const fn accession(&self) -> Option<u32> {
1265        let acc = match self {
1266            BinaryCompressionType::NoCompression => 1000576,
1267            BinaryCompressionType::Zlib => 1000574,
1268            BinaryCompressionType::NumpressLinear => 1002312,
1269            BinaryCompressionType::NumpressSLOF => 1002314,
1270            BinaryCompressionType::NumpressPIC => 1002313,
1271            BinaryCompressionType::NumpressLinearZlib => 1002746,
1272            BinaryCompressionType::NumpressSLOFZlib => 1002748,
1273            BinaryCompressionType::NumpressPICZlib => 1002747,
1274            BinaryCompressionType::DeltaPrediction => 1003089,
1275            BinaryCompressionType::LinearPrediction => 1003090,
1276            BinaryCompressionType::NumpressSLOFZstd => 1003785,
1277            BinaryCompressionType::NumpressLinearZstd => 1003783,
1278            BinaryCompressionType::NumpressPICZstd => 1003784,
1279            BinaryCompressionType::ZstdDict => 1003782,
1280            BinaryCompressionType::Zstd => 1003780,
1281            BinaryCompressionType::ShuffleZstd => 1003781,
1282            BinaryCompressionType::DeltaShuffleZstd => 9999999,
1283            BinaryCompressionType::Decoded => return None,
1284        };
1285        Some(acc)
1286    }
1287
1288    /// Convert a [`CURIE`] into a compression method, if one exists.
1289    pub fn from_accession(accession: CURIE) -> Option<Self> {
1290        match accession {
1291            CURIE {
1292                controlled_vocabulary: ControlledVocabulary::MS,
1293                accession: 1000576,
1294            } => Some(Self::NoCompression),
1295            CURIE {
1296                controlled_vocabulary: ControlledVocabulary::MS,
1297                accession: 1000574,
1298            } => Some(BinaryCompressionType::Zlib),
1299            CURIE {
1300                controlled_vocabulary: ControlledVocabulary::MS,
1301                accession: 1002312,
1302            } => Some(BinaryCompressionType::NumpressLinear),
1303            CURIE {
1304                controlled_vocabulary: ControlledVocabulary::MS,
1305                accession: 1002314,
1306            } => Some(BinaryCompressionType::NumpressSLOF),
1307            CURIE {
1308                controlled_vocabulary: ControlledVocabulary::MS,
1309                accession: 1002313,
1310            } => Some(BinaryCompressionType::NumpressPIC),
1311            CURIE {
1312                controlled_vocabulary: ControlledVocabulary::MS,
1313                accession: 1002746,
1314            } => Some(BinaryCompressionType::NumpressLinearZlib),
1315            CURIE {
1316                controlled_vocabulary: ControlledVocabulary::MS,
1317                accession: 1002748,
1318            } => Some(BinaryCompressionType::NumpressSLOFZlib),
1319            CURIE {
1320                controlled_vocabulary: ControlledVocabulary::MS,
1321                accession: 1002747,
1322            } => Some(BinaryCompressionType::NumpressPICZlib),
1323            CURIE {
1324                controlled_vocabulary: ControlledVocabulary::MS,
1325                accession: 1003089,
1326            } => Some(BinaryCompressionType::DeltaPrediction),
1327            CURIE {
1328                controlled_vocabulary: ControlledVocabulary::MS,
1329                accession: 1003090,
1330            } => Some(BinaryCompressionType::LinearPrediction),
1331            x if x
1332                == CURIE {
1333                    controlled_vocabulary: ControlledVocabulary::MS,
1334                    accession: BinaryCompressionType::NumpressSLOFZstd.accession().unwrap(),
1335                } =>
1336            {
1337                Some(BinaryCompressionType::NumpressSLOFZstd)
1338            }
1339            x if x
1340                == CURIE {
1341                    controlled_vocabulary: ControlledVocabulary::MS,
1342                    accession: BinaryCompressionType::NumpressPICZstd.accession().unwrap(),
1343                } =>
1344            {
1345                Some(BinaryCompressionType::NumpressPICZstd)
1346            }
1347            x if x
1348                == CURIE {
1349                    controlled_vocabulary: ControlledVocabulary::MS,
1350                    accession: BinaryCompressionType::NumpressLinearZstd
1351                        .accession()
1352                        .unwrap(),
1353                } =>
1354            {
1355                Some(BinaryCompressionType::NumpressLinearZstd)
1356            }
1357            x if x
1358                == CURIE {
1359                    controlled_vocabulary: ControlledVocabulary::MS,
1360                    accession: BinaryCompressionType::ZstdDict.accession().unwrap(),
1361                } =>
1362            {
1363                Some(BinaryCompressionType::ZstdDict)
1364            }
1365            x if x
1366                == CURIE {
1367                    controlled_vocabulary: ControlledVocabulary::MS,
1368                    accession: BinaryCompressionType::Zstd.accession().unwrap(),
1369                } =>
1370            {
1371                Some(BinaryCompressionType::Zstd)
1372            }
1373            x if x
1374                == CURIE {
1375                    controlled_vocabulary: ControlledVocabulary::MS,
1376                    accession: BinaryCompressionType::ShuffleZstd.accession().unwrap(),
1377                } =>
1378            {
1379                Some(BinaryCompressionType::ShuffleZstd)
1380            }
1381            x if x
1382                == CURIE {
1383                    controlled_vocabulary: ControlledVocabulary::MS,
1384                    accession: BinaryCompressionType::DeltaShuffleZstd.accession().unwrap(),
1385                } =>
1386            {
1387                Some(BinaryCompressionType::DeltaShuffleZstd)
1388            }
1389            _ => None,
1390        }
1391    }
1392
1393    /// Convert the compression type to a [`ParamCow`].
1394    ///
1395    /// Most compression methods have a controlled vocabulary
1396    /// term.
1397    pub const fn as_param(&self) -> Option<ParamCow<'static>> {
1398        let (name, accession) = match self {
1399            BinaryCompressionType::Decoded => return None,
1400            BinaryCompressionType::NoCompression => ("no compression", self.accession()),
1401            BinaryCompressionType::Zlib => ("zlib compression", self.accession()),
1402            BinaryCompressionType::NumpressLinear => (
1403                "MS-Numpress linear prediction compression",
1404                self.accession(),
1405            ),
1406            BinaryCompressionType::NumpressSLOF => (
1407                "MS-Numpress short logged float compression",
1408                self.accession(),
1409            ),
1410            BinaryCompressionType::NumpressPIC => {
1411                ("MS-Numpress positive integer compression", self.accession())
1412            }
1413            BinaryCompressionType::NumpressLinearZlib => (
1414                "MS-Numpress linear prediction compression followed by zlib compression",
1415                self.accession(),
1416            ),
1417            BinaryCompressionType::NumpressSLOFZlib => (
1418                "MS-Numpress short logged float compression followed by zlib compression",
1419                self.accession(),
1420            ),
1421            BinaryCompressionType::NumpressPICZlib => (
1422                "MS-Numpress positive integer compression followed by zlib compression",
1423                self.accession(),
1424            ),
1425            BinaryCompressionType::DeltaPrediction => (
1426                "truncation, delta prediction and zlib compression",
1427                self.accession(),
1428            ),
1429            BinaryCompressionType::LinearPrediction => (
1430                "truncation, linear prediction and zlib compression",
1431                self.accession(),
1432            ),
1433            BinaryCompressionType::NumpressSLOFZstd => {
1434                return Some(ParamCow::const_new(
1435                    "MS-Numpress short logged float compression followed by zstd compression",
1436                    ValueRef::Empty,
1437                    self.accession(),
1438                    Some(ControlledVocabulary::MS),
1439                    Unit::Unknown,
1440                ))
1441            }
1442            BinaryCompressionType::NumpressLinearZstd => {
1443                return Some(ParamCow::const_new(
1444                    "MS-Numpress linear prediction compression followed by zstd compression",
1445                    ValueRef::Empty,
1446                    self.accession(),
1447                    Some(ControlledVocabulary::MS),
1448                    Unit::Unknown,
1449                ))
1450            }
1451            BinaryCompressionType::NumpressPICZstd => {
1452                return Some(ParamCow::const_new(
1453                    "MS-Numpress positive integer compression followed by zstd compression",
1454                    ValueRef::Empty,
1455                    self.accession(),
1456                    Some(ControlledVocabulary::MS),
1457                    Unit::Unknown,
1458                ))
1459            }
1460            BinaryCompressionType::ZstdDict => {
1461                return Some(ParamCow::const_new(
1462                    "dict-zstd compression",
1463                    ValueRef::Empty,
1464                    self.accession(),
1465                    Some(ControlledVocabulary::MS),
1466                    Unit::Unknown,
1467                ))
1468            }
1469            BinaryCompressionType::Zstd => {
1470                return Some(ParamCow::const_new(
1471                    "zstd compression",
1472                    ValueRef::Empty,
1473                    self.accession(),
1474                    Some(ControlledVocabulary::MS),
1475                    Unit::Unknown,
1476                ))
1477            }
1478            BinaryCompressionType::ShuffleZstd => {
1479                return Some(ParamCow::const_new(
1480                    "byte-shuffle-zstd compression",
1481                    ValueRef::Empty,
1482                    self.accession(),
1483                    Some(ControlledVocabulary::MS),
1484                    Unit::Unknown,
1485                ))
1486            }
1487            BinaryCompressionType::DeltaShuffleZstd => {
1488                return Some(ParamCow::const_new(
1489                    "delta-byte-shuffle-zstd compression",
1490                    ValueRef::Empty,
1491                    self.accession(),
1492                    Some(ControlledVocabulary::MS),
1493                    Unit::Unknown,
1494                ))
1495            }
1496        };
1497        Some(
1498            ControlledVocabulary::MS
1499                .const_param_ident(name, unsafe { accession.unwrap_unchecked() }),
1500        )
1501    }
1502}
1503
1504impl Display for BinaryCompressionType {
1505    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1506        write!(f, "{:?}", self)
1507    }
1508}
1509
1510/// A high level set of failure modes that an operation to retrieve a typed memory buffer
1511/// from a [`BinaryArrayMap`](crate::BinaryArrayMap) might encounter. May also be used to represented conversion
1512/// during reading or writing.
1513#[derive(Debug, Clone, Error, PartialEq)]
1514pub enum ArrayRetrievalError {
1515    #[error("Array type {0:?} not found")]
1516    NotFound(ArrayType),
1517    #[error("An error occurred while decompressing: {0}")]
1518    DecompressionError(String),
1519    #[error("The requested data type does not match the number of bytes available in the buffer")]
1520    DataTypeSizeMismatch,
1521}
1522
1523impl From<bytemuck::PodCastError> for ArrayRetrievalError {
1524    fn from(value: bytemuck::PodCastError) -> Self {
1525        match value {
1526            bytemuck::PodCastError::TargetAlignmentGreaterAndInputNotAligned => {
1527                Self::DataTypeSizeMismatch
1528            }
1529            bytemuck::PodCastError::OutputSliceWouldHaveSlop => Self::DataTypeSizeMismatch,
1530            bytemuck::PodCastError::SizeMismatch => Self::DataTypeSizeMismatch,
1531            bytemuck::PodCastError::AlignmentMismatch => Self::DataTypeSizeMismatch,
1532        }
1533    }
1534}
1535
1536impl From<ArrayRetrievalError> for io::Error {
1537    fn from(value: ArrayRetrievalError) -> Self {
1538        match value {
1539            ArrayRetrievalError::NotFound(_) => io::Error::new(io::ErrorKind::NotFound, value),
1540            ArrayRetrievalError::DecompressionError(e) => {
1541                io::Error::new(io::ErrorKind::InvalidData, e)
1542            }
1543            ArrayRetrievalError::DataTypeSizeMismatch => {
1544                io::Error::new(io::ErrorKind::InvalidData, value)
1545            }
1546        }
1547    }
1548}
1549
1550#[cfg(feature = "numpress")]
1551impl From<numpress::Error> for ArrayRetrievalError {
1552    fn from(value: numpress::Error) -> Self {
1553        ArrayRetrievalError::DecompressionError(value.to_string())
1554    }
1555}
1556
1557pub fn linear_prediction_decoding<F: Num + Copy + Mul + AddAssign>(values: &mut [F]) -> &mut [F] {
1558    if values.len() < 2 {
1559        return values;
1560    }
1561
1562    let two = F::one() + F::one();
1563
1564    let prev2 = values[1];
1565    let prev1 = values[2];
1566    let offset = values[1];
1567
1568    values
1569        .iter_mut()
1570        .skip(2)
1571        .fold((prev1, prev2), |(prev1, prev2), current| {
1572            let tmp = *current + two * prev1 - prev2 - offset;
1573            let prev1 = *current;
1574            let prev2 = prev1;
1575            *current = tmp;
1576            (prev1, prev2)
1577        });
1578
1579    for i in 0..values.len() {
1580        if i < 2 {
1581            continue;
1582        }
1583        let v = values[i] + two * values[i - 1] - values[i - 2] - values[1];
1584        values[i] = v;
1585    }
1586    values
1587}
1588
1589pub fn linear_prediction_encoding<F: Num + Copy + Mul<F> + AddAssign>(
1590    values: &mut [F],
1591) -> &mut [F] {
1592    let n = values.len();
1593    if n < 3 {
1594        return values;
1595    }
1596    let offset = values[1];
1597    let prev2 = values[0];
1598    let prev1 = values[1];
1599    let two = F::one() + F::one();
1600
1601    values
1602        .iter_mut()
1603        .fold((prev1, prev2), |(prev1, prev2), val| {
1604            *val += offset - two * prev1 + prev2;
1605            let tmp = prev1;
1606            let prev1 = *val + two * prev1 - prev2 - offset;
1607            let prev2 = tmp;
1608            (prev1, prev2)
1609        });
1610    values
1611}
1612
1613pub fn delta_decoding<F: Num + Copy + Mul + AddAssign>(values: &mut [F]) -> &mut [F] {
1614    if values.len() < 2 {
1615        return values;
1616    }
1617
1618    let offset = values[0];
1619    let prev = values[1];
1620
1621    values.iter_mut().skip(2).fold(prev, |prev, current| {
1622        *current += prev - offset;
1623        *current
1624    });
1625    values
1626}
1627
1628pub fn delta_encoding<F: Num + Copy + Mul + AddAssign>(values: &mut [F]) -> &mut [F] {
1629    let n = values.len();
1630    if n < 2 {
1631        return values;
1632    }
1633    let prev = values[0];
1634    let offset = values[0];
1635
1636    let it = values.iter_mut();
1637    it.skip(1).fold(prev, |prev, current| {
1638        let tmp = *current;
1639        *current += offset - prev;
1640        tmp
1641    });
1642    values
1643}
1644
1645#[cfg(test)]
1646mod test {
1647    use super::*;
1648
1649    #[test]
1650    fn test_dtype_size() {
1651        assert_eq!(BinaryDataArrayType::ASCII.size_of(), 1);
1652        assert_eq!(BinaryDataArrayType::Float32.size_of(), 4);
1653        assert_eq!(BinaryDataArrayType::Int32.size_of(), 4);
1654        assert_eq!(BinaryDataArrayType::Float64.size_of(), 8);
1655        assert_eq!(BinaryDataArrayType::Int64.size_of(), 8);
1656    }
1657
1658    #[test]
1659    fn test_array_type_param() {
1660        let array_types = [
1661            ArrayType::MZArray,
1662            ArrayType::IntensityArray,
1663            ArrayType::ChargeArray,
1664            ArrayType::SignalToNoiseArray,
1665            ArrayType::TimeArray,
1666            ArrayType::WavelengthArray,
1667            ArrayType::IonMobilityArray,
1668            ArrayType::MeanIonMobilityArray,
1669            ArrayType::RawIonMobilityArray,
1670            ArrayType::DeconvolutedIonMobilityArray,
1671        ];
1672
1673        for at in array_types {
1674            assert_eq!(at.as_param_const().name, at.as_param(None).name)
1675        }
1676    }
1677
1678    #[test]
1679    fn test_binary_encoding_conv() {
1680        let encodings = [
1681            BinaryCompressionType::Decoded,
1682            BinaryCompressionType::NoCompression,
1683            BinaryCompressionType::NumpressLinear,
1684            BinaryCompressionType::NumpressLinearZlib,
1685            BinaryCompressionType::NumpressPIC,
1686            BinaryCompressionType::NumpressPICZlib,
1687            BinaryCompressionType::NumpressSLOF,
1688            BinaryCompressionType::NumpressSLOFZlib,
1689            BinaryCompressionType::Zlib,
1690        ];
1691
1692        for enc in encodings {
1693            let reps = match enc {
1694                BinaryCompressionType::NoCompression => ("no compression", 1000576),
1695                BinaryCompressionType::Zlib => ("zlib compression", 1000574),
1696                BinaryCompressionType::NumpressLinear => {
1697                    ("MS-Numpress linear prediction compression", 1002312)
1698                }
1699                BinaryCompressionType::NumpressSLOF => {
1700                    ("MS-Numpress short logged float compression", 1002314)
1701                }
1702                BinaryCompressionType::NumpressPIC => {
1703                    ("MS-Numpress positive integer compression", 1002313)
1704                }
1705                BinaryCompressionType::NumpressLinearZlib => (
1706                    "MS-Numpress linear prediction compression followed by zlib compression",
1707                    1002746,
1708                ),
1709                BinaryCompressionType::NumpressPICZlib => (
1710                    "MS-Numpress positive integer compression followed by zlib compression",
1711                    1002747,
1712                ),
1713                BinaryCompressionType::NumpressSLOFZlib => (
1714                    "MS-Numpress short logged float compression followed by zlib compression",
1715                    1002748,
1716                ),
1717                _ => ("", 0),
1718            };
1719            if let Some(p) = enc.as_param() {
1720                assert_eq!(p.name, reps.0);
1721                assert_eq!(p.accession.unwrap(), reps.1);
1722            }
1723        }
1724    }
1725
1726    #[test]
1727    fn test_transpose() {
1728        let data: Vec<_> = (0..128i32).map(|i| i.pow(2u32) as f64).collect();
1729        let flip = transpose_f64(&data);
1730        let rev = reverse_transpose_f64(&flip);
1731        let rev_cast: &[f64] = bytemuck::cast_slice(&rev);
1732        assert_eq!(data, rev_cast);
1733    }
1734
1735    #[test]
1736    fn test_dict() {
1737        let data: Vec<_> = (0..127i32).map(|i| i.pow(2u32) as f64).collect();
1738        let encoded = dictionary_encoding(&data).unwrap();
1739        let decoded: Vec<f64> = dictionary_decoding(&encoded).unwrap();
1740
1741        assert_eq!(data, decoded);
1742    }
1743
1744    #[test]
1745    fn test_byteswap() {
1746        let x = 42u32;
1747        let mut bytes_of = x.to_le_bytes();
1748        BinaryDataArrayType::Int32.swap_bytes(&mut bytes_of).unwrap();
1749        let y1 = u32::from_le_bytes(bytes_of);
1750        let y2 = x.swap_bytes();
1751        assert_eq!(y1, y2);
1752
1753        bytes_of = x.to_be_bytes();
1754        BinaryDataArrayType::Int32.swap_bytes(&mut bytes_of).unwrap();
1755        let y1 = u32::from_le_bytes(bytes_of);
1756        assert_eq!(x, y1);
1757    }
1758}
mzdata_bindata/encodings.rs

mzdata_bindata/
encodings.rs
