Skip to main content

ionic/ion/decoder/decode/
arrays.rs

1use super::*;
2
3#[derive(Debug, Clone, Copy, Default)]
4pub struct ArrayAddress {
5    pub(crate) block_id: u32,
6    pub(crate) element_offset: u64,
7    pub(crate) element_count: u64,
8    pub(crate) array_type: u32,
9    pub(crate) dtype: u8,
10    pub(crate) array_filter: u8,
11    pub(crate) encoded_len: u32,
12    pub(crate) continues_previous_segment: u8,
13    pub(crate) array_cv_code: u8,
14}
15
16impl ArrayAddress {
17    pub fn array_type(&self) -> u32 {
18        self.array_type
19    }
20
21    pub fn dtype(&self) -> u8 {
22        self.dtype
23    }
24
25    pub fn array_filter(&self) -> u8 {
26        self.array_filter
27    }
28}
29
30#[derive(Debug, Clone)]
31pub struct ArrayGroup {
32    pub array_type: u32,
33    pub array_cv_code: u8,
34    pub dtype: u8,
35    pub array_filter: u8,
36    pub refs: Vec<ArrayAddress>,
37}
38
39#[derive(Clone)]
40pub(crate) struct ArrayAddressList {
41    pub(crate) len: usize,
42    pub(crate) inline: [ArrayAddress; INLINE_ARRAY_ADDRESS_CAP],
43    pub(crate) heap: Option<Vec<ArrayAddress>>,
44}
45
46impl ArrayAddressList {
47    #[inline]
48    pub(crate) fn with_capacity(capacity: usize) -> Self {
49        Self {
50            len: 0,
51            inline: [ArrayAddress::default(); INLINE_ARRAY_ADDRESS_CAP],
52            heap: (capacity > INLINE_ARRAY_ADDRESS_CAP).then(|| Vec::with_capacity(capacity)),
53        }
54    }
55
56    #[inline]
57    pub(crate) fn push(&mut self, value: ArrayAddress) {
58        if let Some(heap) = self.heap.as_mut() {
59            heap.push(value);
60            self.len = heap.len();
61            return;
62        }
63        self.inline[self.len] = value;
64        self.len += 1;
65    }
66
67    #[inline]
68    pub(crate) fn is_empty(&self) -> bool {
69        self.len == 0
70    }
71
72    #[inline]
73    pub(crate) fn as_slice(&self) -> &[ArrayAddress] {
74        match self.heap.as_deref() {
75            Some(heap) => heap,
76            None => &self.inline[..self.len],
77        }
78    }
79
80    #[inline]
81    pub(crate) fn into_vec(self) -> Vec<ArrayAddress> {
82        self.heap
83            .unwrap_or_else(|| self.inline[..self.len].to_vec())
84    }
85}
86
87#[inline]
88pub(crate) fn address_read_params(array_address: &ArrayAddress) -> (u64, u64, usize) {
89    if array_address.encoded_len > 0 {
90        (
91            array_address.element_offset,
92            array_address.encoded_len as u64,
93            1,
94        )
95    } else {
96        (
97            array_address.element_offset,
98            array_address.element_count,
99            dtype_stride(array_address.dtype),
100        )
101    }
102}
103
104#[inline]
105pub(crate) fn parse_array_address(bytes: &[u8]) -> ArrayAddress {
106    ArrayAddress {
107        element_offset: u64::from_le_bytes(bytes[0..8].try_into().unwrap()),
108        element_count: u64::from_le_bytes(bytes[8..16].try_into().unwrap()),
109        block_id: u32::from_le_bytes(bytes[16..20].try_into().unwrap()),
110        array_type: u32::from_le_bytes(bytes[20..24].try_into().unwrap()),
111        dtype: bytes[24],
112        array_filter: bytes[25],
113        encoded_len: u32::from_le_bytes(bytes[26..30].try_into().unwrap()),
114        continues_previous_segment: bytes[30],
115        array_cv_code: bytes[31],
116    }
117}
118
119#[inline]
120pub(crate) fn read_array_addresses_from_buffers(
121    entries_buf: &[u8],
122    array_addresses_buf: &[u8],
123    index: usize,
124) -> Option<ArrayAddressList> {
125    let entry_offset = index.checked_mul(INDEX_ENTRY_BYTES)?;
126    let entry_end = entry_offset.checked_add(INDEX_ENTRY_BYTES)?;
127    let entry = entries_buf.get(entry_offset..entry_end)?;
128    let ref_start = usize::try_from(u64::from_le_bytes(entry[0..8].try_into().unwrap())).ok()?;
129    let address_count =
130        usize::try_from(u64::from_le_bytes(entry[8..16].try_into().unwrap())).ok()?;
131    let max_refs = array_addresses_buf.len() / ARRAY_ADDRESS_BYTES;
132    if address_count > max_refs {
133        return None;
134    }
135    let mut refs = ArrayAddressList::with_capacity(address_count);
136    for offset in 0..address_count {
137        let pos = ref_start
138            .checked_add(offset)?
139            .checked_mul(ARRAY_ADDRESS_BYTES)?;
140        let end = pos.checked_add(ARRAY_ADDRESS_BYTES)?;
141        refs.push(parse_array_address(array_addresses_buf.get(pos..end)?));
142    }
143    Some(refs)
144}
145
146pub(crate) fn group_arrays(refs: &[ArrayAddress]) -> IonResult<Vec<ArrayGroup>> {
147    if refs.is_empty() {
148        return Ok(Vec::new());
149    }
150
151    if refs[0].continues_previous_segment != 0 {
152        return Err("array grouping: first ref must have continues_previous_segment = 0".into());
153    }
154
155    let mut groups = Vec::new();
156    let mut current_group_refs = Vec::new();
157    let mut current_type = refs[0].array_type;
158    let mut current_cv_code = refs[0].array_cv_code;
159    let mut current_dtype = refs[0].dtype;
160    let mut current_filter = refs[0].array_filter;
161
162    for address in refs {
163        if address.continues_previous_segment != 0 && address.continues_previous_segment != 1 {
164            return Err(format!(
165                "array grouping: invalid continues_previous_segment value {}, must be 0 or 1",
166                address.continues_previous_segment
167            )
168            .into());
169        }
170
171        if address.continues_previous_segment == 0 {
172            if !current_group_refs.is_empty() {
173                groups.push(ArrayGroup {
174                    array_type: current_type,
175                    array_cv_code: current_cv_code,
176                    dtype: current_dtype,
177                    array_filter: current_filter,
178                    refs: current_group_refs,
179                });
180                current_group_refs = Vec::new();
181            }
182            current_type = address.array_type;
183            current_cv_code = address.array_cv_code;
184            current_dtype = address.dtype;
185            current_filter = address.array_filter;
186        } else if address.array_type != current_type
187            || address.dtype != current_dtype
188            || address.array_filter != current_filter
189        {
190            return Err(
191                "array grouping: continuation ref has different array_type, dtype, or filter"
192                    .into(),
193            );
194        }
195
196        current_group_refs.push(*address);
197    }
198
199    if !current_group_refs.is_empty() {
200        groups.push(ArrayGroup {
201            array_type: current_type,
202            array_cv_code: current_cv_code,
203            dtype: current_dtype,
204            array_filter: current_filter,
205            refs: current_group_refs,
206        });
207    }
208
209    for group in &groups {
210        if group.refs.len() > 1 {
211            for address in &group.refs {
212                if address.encoded_len > 0 {
213                    return Err(
214                        "array grouping: multi-ref group cannot contain variable-length arrays"
215                            .into(),
216                    );
217                }
218            }
219        }
220    }
221
222    Ok(groups)
223}
224
225pub(crate) fn read_group_decoded_bytes(
226    group: &ArrayGroup,
227    container: &mut BlockReader<DefaultBlockProcessor>,
228) -> IonResult<Vec<u8>> {
229    if let [array_address] = group.refs.as_slice() {
230        let (element_offset, count, stride) = address_read_params(array_address);
231        let raw = container.get_array_bytes_from_block(
232            array_address.block_id,
233            element_offset,
234            count,
235            stride,
236            "read_group_decoded_bytes",
237        )?;
238        let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?;
239        return Ok(unfiltered.into_owned());
240    }
241
242    let mut decoded = Vec::new();
243    for array_address in &group.refs {
244        let (element_offset, count, stride) = address_read_params(array_address);
245        let raw = container.get_array_bytes_from_block(
246            array_address.block_id,
247            element_offset,
248            count,
249            stride,
250            "read_group_decoded_bytes",
251        )?;
252        let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?;
253        decoded.reserve(unfiltered.len());
254        decoded.extend_from_slice(&unfiltered);
255    }
256
257    Ok(decoded)
258}
259
260#[inline]
261pub(crate) fn dtype_stride(dtype: u8) -> usize {
262    match dtype {
263        FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
264        FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
265        FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
266        _ => 1,
267    }
268}
269
270pub(crate) fn unfilter_array_bytes(
271    raw: &[u8],
272    dtype: u8,
273    array_filter: u8,
274) -> IonResult<std::borrow::Cow<'_, [u8]>> {
275    let pk_id = PackingId::from_byte(array_filter)?;
276    match pk_id {
277        PackingId::Raw => Ok(std::borrow::Cow::Borrowed(raw)),
278        PackingId::ByteShuffle => match crate::ion::packing::Dtype::from_byte(dtype) {
279            Ok(
280                dtype_enum @ (crate::ion::packing::Dtype::F64 | crate::ion::packing::Dtype::F32),
281            ) => {
282                let mut out = Vec::new();
283                crate::ion::packing::packing_by_id(PackingId::ByteShuffle)
284                    .decode(raw, dtype_enum, &mut out)?;
285                Ok(std::borrow::Cow::Owned(out))
286            }
287            _ => Err(format!("array filter {array_filter} is not valid for dtype {dtype}").into()),
288        },
289        PackingId::DeltaShuffle => match crate::ion::packing::Dtype::from_byte(dtype) {
290            Ok(
291                dtype_enum @ (crate::ion::packing::Dtype::F64 | crate::ion::packing::Dtype::F32),
292            ) => {
293                let mut out = Vec::new();
294                crate::ion::packing::packing_by_id(PackingId::DeltaShuffle)
295                    .decode(raw, dtype_enum, &mut out)?;
296                Ok(std::borrow::Cow::Owned(out))
297            }
298            _ => Err(format!("array filter {array_filter} is not valid for dtype {dtype}").into()),
299        },
300    }
301}
302
303pub(crate) fn decode_into(
304    buf: &mut Vec<f64>,
305    raw: &[u8],
306    dtype: u8,
307    array_filter: u8,
308) -> IonResult<()> {
309    buf.clear();
310    let bytes = unfilter_array_bytes(raw, dtype, array_filter)?;
311    match dtype {
312        FILE_DTYPE_F64 => {
313            buf.reserve(bytes.len() / 8);
314            buf.extend(
315                bytes
316                    .chunks_exact(8)
317                    .map(|c| f64::from_le_bytes(c.try_into().unwrap())),
318            );
319        }
320        FILE_DTYPE_F32 => {
321            buf.reserve(bytes.len() / 4);
322            buf.extend(
323                bytes
324                    .chunks_exact(4)
325                    .map(|c| f32::from_le_bytes(c.try_into().unwrap()) as f64),
326            );
327        }
328        FILE_DTYPE_F16 => {
329            buf.reserve(bytes.len() / 2);
330            buf.extend(
331                bytes
332                    .chunks_exact(2)
333                    .map(|c| f16_bits_to_f64(u16::from_le_bytes(c.try_into().unwrap()))),
334            );
335        }
336        FILE_DTYPE_I16 => {
337            buf.reserve(bytes.len() / 2);
338            buf.extend(
339                bytes
340                    .chunks_exact(2)
341                    .map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f64),
342            );
343        }
344        FILE_DTYPE_I32 => {
345            buf.reserve(bytes.len() / 4);
346            buf.extend(
347                bytes
348                    .chunks_exact(4)
349                    .map(|c| i32::from_le_bytes(c.try_into().unwrap()) as f64),
350            );
351        }
352        FILE_DTYPE_I64 => {
353            buf.reserve(bytes.len() / 8);
354            buf.extend(
355                bytes
356                    .chunks_exact(8)
357                    .map(|c| i64::from_le_bytes(c.try_into().unwrap()) as f64),
358            );
359        }
360        _ => {
361            return Err(IonError::BadDtype {
362                dtype,
363                kind: "decode array dtype",
364            });
365        }
366    }
367    Ok(())
368}
369
370fn collect_entry_array_addresses(
371    entry_bytes: &[u8],
372    address_bytes: &[u8],
373) -> Option<Vec<ArrayAddress>> {
374    let ref_start =
375        usize::try_from(u64::from_le_bytes(entry_bytes[0..8].try_into().unwrap())).ok()?;
376    let address_count =
377        usize::try_from(u64::from_le_bytes(entry_bytes[8..16].try_into().unwrap())).ok()?;
378    let start = ref_start.checked_mul(ARRAY_ADDRESS_BYTES)?;
379    let span = address_count.checked_mul(ARRAY_ADDRESS_BYTES)?;
380    let end = start.checked_add(span)?;
381    let mut refs = Vec::with_capacity(address_count);
382    for bytes in address_bytes
383        .get(start..end)?
384        .chunks_exact(ARRAY_ADDRESS_BYTES)
385    {
386        refs.push(parse_array_address(bytes));
387    }
388    Some(refs)
389}
390
391pub(crate) fn read_scan_arrays(
392    container: &mut dyn ContainerAccess,
393    entry_bytes: &[u8],
394    address_bytes: &[u8],
395    mz: &mut Vec<f64>,
396    intensity: &mut Vec<f64>,
397) -> bool {
398    mz.clear();
399    intensity.clear();
400    let Some(refs) = collect_entry_array_addresses(entry_bytes, address_bytes) else {
401        return false;
402    };
403    let Ok(groups) = group_arrays(&refs) else {
404        return false;
405    };
406    let mut segment = Vec::new();
407    for group in &groups {
408        let target = match group.array_type {
409            ACC_MZ => &mut *mz,
410            ACC_INT => &mut *intensity,
411            _ => continue,
412        };
413        for array_address in &group.refs {
414            let (element_offset, count, stride) = address_read_params(array_address);
415            let raw = match container.get_array_bytes_from_block(
416                array_address.block_id,
417                element_offset,
418                count,
419                stride,
420                "scan",
421            ) {
422                Ok(raw) => raw,
423                Err(_) => return false,
424            };
425            if decode_into(
426                &mut segment,
427                raw,
428                array_address.dtype,
429                array_address.array_filter,
430            )
431            .is_err()
432            {
433                return false;
434            }
435            target.extend_from_slice(&segment);
436        }
437    }
438    mz.len().min(intensity.len()) > 0
439}
440
441impl IonReader {
442    pub fn spectrum_array_addresses(&self, index: usize) -> Option<Vec<ArrayAddress>> {
443        if index >= self.header.spectrum_count as usize {
444            return None;
445        }
446        read_array_addresses_from_buffers(&self.spec_entries_buf, &self.spec_array_addresses, index)
447            .map(ArrayAddressList::into_vec)
448    }
449
450    pub fn chromatogram_array_addresses(&self, index: usize) -> Option<Vec<ArrayAddress>> {
451        if index >= self.header.chrom_count as usize {
452            return None;
453        }
454        read_array_addresses_from_buffers(
455            &self.chrom_entries_buf,
456            &self.chrom_array_addresses,
457            index,
458        )
459        .map(ArrayAddressList::into_vec)
460    }
461
462    pub fn read_spectrum_values(
463        &mut self,
464        array_address: &ArrayAddress,
465        out: &mut Vec<f64>,
466    ) -> IonResult<()> {
467        let (element_offset, count, stride) = address_read_params(array_address);
468        let raw = self.spec_container.get_array_bytes_from_block(
469            array_address.block_id,
470            element_offset,
471            count,
472            stride,
473            "read_array",
474        )?;
475        decode_into(out, raw, array_address.dtype, array_address.array_filter)
476    }
477
478    pub fn read_spectrum_array(&mut self, array_address: &ArrayAddress) -> IonResult<NumericArray> {
479        let (element_offset, count, stride) = address_read_params(array_address);
480        let raw = self.spec_container.get_array_bytes_from_block(
481            array_address.block_id,
482            element_offset,
483            count,
484            stride,
485            "read_spectrum_array",
486        )?;
487        let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?;
488        super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype)
489    }
490
491    pub fn read_chromatogram_values(
492        &mut self,
493        array_address: &ArrayAddress,
494        out: &mut Vec<f64>,
495    ) -> IonResult<()> {
496        let container = self
497            .chrom_container
498            .as_mut()
499            .ok_or_else(|| IonError::from("no chromatogram container"))?;
500        let (element_offset, count, stride) = address_read_params(array_address);
501        let raw = container.get_array_bytes_from_block(
502            array_address.block_id,
503            element_offset,
504            count,
505            stride,
506            "read_chromatogram_values",
507        )?;
508        decode_into(out, raw, array_address.dtype, array_address.array_filter)
509    }
510
511    pub fn read_chromatogram_array(
512        &mut self,
513        array_address: &ArrayAddress,
514    ) -> IonResult<NumericArray> {
515        let container = self
516            .chrom_container
517            .as_mut()
518            .ok_or_else(|| IonError::from("no chromatogram container"))?;
519        let (element_offset, count, stride) = address_read_params(array_address);
520        let raw = container.get_array_bytes_from_block(
521            array_address.block_id,
522            element_offset,
523            count,
524            stride,
525            "read_chromatogram_array",
526        )?;
527        let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?;
528        super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype)
529    }
530
531    pub(crate) fn read_group_values(
532        &mut self,
533        group: &ArrayGroup,
534        out: &mut Vec<f64>,
535    ) -> IonResult<()> {
536        let Some((first, rest)) = group.refs.split_first() else {
537            out.clear();
538            return Ok(());
539        };
540        self.read_spectrum_values(first, out)?;
541        let mut window = Vec::new();
542        for array_address in rest {
543            self.read_spectrum_values(array_address, &mut window)?;
544            out.extend_from_slice(&window);
545        }
546        Ok(())
547    }
548
549    pub fn get_spectrum_array(
550        &mut self,
551        spectrum_index: usize,
552        array_type: impl Into<u32>,
553    ) -> IonResult<Vec<f64>> {
554        let array_type = array_type.into();
555        if spectrum_index >= self.header.spectrum_count as usize {
556            return Err("spectrum index out of range".into());
557        }
558
559        let Some(array_addresses) = read_array_addresses_from_buffers(
560            &self.spec_entries_buf,
561            &self.spec_array_addresses,
562            spectrum_index,
563        ) else {
564            return Ok(Vec::new());
565        };
566
567        let groups = group_arrays(array_addresses.as_slice())?;
568
569        for group in groups {
570            if group.array_type != array_type {
571                continue;
572            }
573
574            let mut values = Vec::new();
575            self.read_group_values(&group, &mut values)?;
576            return Ok(values);
577        }
578
579        Ok(Vec::new())
580    }
581}
582
583#[cfg(test)]
584mod tests {
585    use super::*;
586
587    #[test]
588    fn unfilter_array_bytes_delta_shuffle_f64_matches_cumulative_sum() {
589        let deltas: [u64; 4] = [100u64, 5, 3, 10];
590        let raw: Vec<u8> = deltas.iter().flat_map(|w| w.to_le_bytes()).collect();
591
592        let unfiltered =
593            unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::DeltaShuffle as u8).unwrap();
594
595        let mut expected_prev: u64 = 0;
596        let mut expected = Vec::new();
597        for delta in deltas {
598            expected_prev = expected_prev.wrapping_add(delta);
599            expected.extend_from_slice(&expected_prev.to_le_bytes());
600        }
601
602        assert_eq!(unfiltered.as_ref(), expected.as_slice());
603    }
604
605    #[test]
606    fn unfilter_array_bytes_delta_shuffle_rejects_misaligned_f64_input() {
607        let raw = [0u8; 7];
608        let err = unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::DeltaShuffle as u8)
609            .expect_err("7 bytes is not a multiple of the f64 word size");
610        assert!(err.contains("not a multiple of the word size"));
611    }
612
613    #[test]
614    fn unfilter_array_bytes_byte_shuffle_f64_round_trips_71() {
615        use crate::ion::packing::{PackingInput, packing_by_id};
616
617        let data: Vec<f64> = (0..600)
618            .map(|i| 250.0 + (i as f64) * 0.0137 + ((i * 5 % 11) as f64) * 0.0011)
619            .collect();
620        let raw: Vec<u8> = data.iter().flat_map(|v| v.to_le_bytes()).collect();
621
622        let mut shuffled = Vec::new();
623        packing_by_id(PackingId::ByteShuffle)
624            .encode(PackingInput::F64(&data), &mut shuffled)
625            .unwrap();
626
627        assert_ne!(
628            shuffled, raw,
629            "shuffled layout must differ from raw for varying data"
630        );
631
632        let unfiltered =
633            unfilter_array_bytes(&shuffled, FILE_DTYPE_F64, PackingId::ByteShuffle as u8).unwrap();
634
635        assert_eq!(
636            unfiltered.as_ref(),
637            raw.as_slice(),
638            "decode of ByteShuffle-tagged bytes must unshuffle back to raw"
639        );
640
641        let out: Vec<f64> = unfiltered
642            .chunks_exact(8)
643            .map(|c| f64::from_le_bytes(c.try_into().unwrap()))
644            .collect();
645        assert_eq!(out.len(), data.len());
646        for (a, b) in out.iter().zip(data.iter()) {
647            assert_eq!(a.to_bits(), b.to_bits());
648        }
649    }
650
651    #[test]
652    fn unfilter_array_bytes_delta_shuffle_rejects_unsupported_dtype() {
653        let raw = [1u8, 2, 3, 4];
654        let err = unfilter_array_bytes(&raw, FILE_DTYPE_I16, PackingId::DeltaShuffle as u8)
655            .expect_err("delta shuffle is only valid for f32/f64");
656        assert!(err.contains("is not valid for dtype"));
657    }
658
659    #[test]
660    fn shuffle_filter_on_integer_dtype_is_rejected_4() {
661        let raw = [0u8; 8];
662
663        let err = unfilter_array_bytes(&raw, FILE_DTYPE_I32, PackingId::ByteShuffle as u8)
664            .expect_err("byte shuffle is only valid for f32/f64, not i32");
665        assert!(err.contains("is not valid for dtype"));
666
667        let ok = unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::Raw as u8)
668            .expect("raw filter must still work for any dtype");
669        assert_eq!(ok.as_ref(), raw.as_slice());
670    }
671}