1use super::*;
2
3#[derive(Debug, Clone, Copy, Default)]
4pub struct ArrayAddress {
5 pub(crate) block_id: u32,
6 pub(crate) element_offset: u64,
7 pub(crate) element_count: u64,
8 pub(crate) array_type: u32,
9 pub(crate) dtype: u8,
10 pub(crate) array_filter: u8,
11 pub(crate) encoded_len: u32,
12 pub(crate) continues_previous_segment: u8,
13 pub(crate) array_cv_code: u8,
14}
15
16impl ArrayAddress {
17 pub fn array_type(&self) -> u32 {
18 self.array_type
19 }
20
21 pub fn dtype(&self) -> u8 {
22 self.dtype
23 }
24
25 pub fn array_filter(&self) -> u8 {
26 self.array_filter
27 }
28}
29
30#[derive(Debug, Clone)]
31pub struct ArrayGroup {
32 pub array_type: u32,
33 pub array_cv_code: u8,
34 pub dtype: u8,
35 pub array_filter: u8,
36 pub refs: Vec<ArrayAddress>,
37}
38
39#[derive(Clone)]
40pub(crate) struct ArrayAddressList {
41 pub(crate) len: usize,
42 pub(crate) inline: [ArrayAddress; INLINE_ARRAY_ADDRESS_CAP],
43 pub(crate) heap: Option<Vec<ArrayAddress>>,
44}
45
46impl ArrayAddressList {
47 #[inline]
48 pub(crate) fn with_capacity(capacity: usize) -> Self {
49 Self {
50 len: 0,
51 inline: [ArrayAddress::default(); INLINE_ARRAY_ADDRESS_CAP],
52 heap: (capacity > INLINE_ARRAY_ADDRESS_CAP).then(|| Vec::with_capacity(capacity)),
53 }
54 }
55
56 #[inline]
57 pub(crate) fn push(&mut self, value: ArrayAddress) {
58 if let Some(heap) = self.heap.as_mut() {
59 heap.push(value);
60 self.len = heap.len();
61 return;
62 }
63 self.inline[self.len] = value;
64 self.len += 1;
65 }
66
67 #[inline]
68 pub(crate) fn is_empty(&self) -> bool {
69 self.len == 0
70 }
71
72 #[inline]
73 pub(crate) fn as_slice(&self) -> &[ArrayAddress] {
74 match self.heap.as_deref() {
75 Some(heap) => heap,
76 None => &self.inline[..self.len],
77 }
78 }
79
80 #[inline]
81 pub(crate) fn into_vec(self) -> Vec<ArrayAddress> {
82 self.heap
83 .unwrap_or_else(|| self.inline[..self.len].to_vec())
84 }
85}
86
87#[inline]
88pub(crate) fn address_read_params(array_address: &ArrayAddress) -> (u64, u64, usize) {
89 if array_address.encoded_len > 0 {
90 (
91 array_address.element_offset,
92 array_address.encoded_len as u64,
93 1,
94 )
95 } else {
96 (
97 array_address.element_offset,
98 array_address.element_count,
99 dtype_stride(array_address.dtype),
100 )
101 }
102}
103
104#[inline]
105pub(crate) fn parse_array_address(bytes: &[u8]) -> ArrayAddress {
106 ArrayAddress {
107 element_offset: u64::from_le_bytes(bytes[0..8].try_into().unwrap()),
108 element_count: u64::from_le_bytes(bytes[8..16].try_into().unwrap()),
109 block_id: u32::from_le_bytes(bytes[16..20].try_into().unwrap()),
110 array_type: u32::from_le_bytes(bytes[20..24].try_into().unwrap()),
111 dtype: bytes[24],
112 array_filter: bytes[25],
113 encoded_len: u32::from_le_bytes(bytes[26..30].try_into().unwrap()),
114 continues_previous_segment: bytes[30],
115 array_cv_code: bytes[31],
116 }
117}
118
119#[inline]
120pub(crate) fn read_array_addresses_from_buffers(
121 entries_buf: &[u8],
122 array_addresses_buf: &[u8],
123 index: usize,
124) -> Option<ArrayAddressList> {
125 let entry_offset = index.checked_mul(INDEX_ENTRY_BYTES)?;
126 let entry_end = entry_offset.checked_add(INDEX_ENTRY_BYTES)?;
127 let entry = entries_buf.get(entry_offset..entry_end)?;
128 let ref_start = usize::try_from(u64::from_le_bytes(entry[0..8].try_into().unwrap())).ok()?;
129 let address_count =
130 usize::try_from(u64::from_le_bytes(entry[8..16].try_into().unwrap())).ok()?;
131 let max_refs = array_addresses_buf.len() / ARRAY_ADDRESS_BYTES;
132 if address_count > max_refs {
133 return None;
134 }
135 let mut refs = ArrayAddressList::with_capacity(address_count);
136 for offset in 0..address_count {
137 let pos = ref_start
138 .checked_add(offset)?
139 .checked_mul(ARRAY_ADDRESS_BYTES)?;
140 let end = pos.checked_add(ARRAY_ADDRESS_BYTES)?;
141 refs.push(parse_array_address(array_addresses_buf.get(pos..end)?));
142 }
143 Some(refs)
144}
145
146pub(crate) fn group_arrays(refs: &[ArrayAddress]) -> IonResult<Vec<ArrayGroup>> {
147 if refs.is_empty() {
148 return Ok(Vec::new());
149 }
150
151 if refs[0].continues_previous_segment != 0 {
152 return Err("array grouping: first ref must have continues_previous_segment = 0".into());
153 }
154
155 let mut groups = Vec::new();
156 let mut current_group_refs = Vec::new();
157 let mut current_type = refs[0].array_type;
158 let mut current_cv_code = refs[0].array_cv_code;
159 let mut current_dtype = refs[0].dtype;
160 let mut current_filter = refs[0].array_filter;
161
162 for address in refs {
163 if address.continues_previous_segment != 0 && address.continues_previous_segment != 1 {
164 return Err(format!(
165 "array grouping: invalid continues_previous_segment value {}, must be 0 or 1",
166 address.continues_previous_segment
167 )
168 .into());
169 }
170
171 if address.continues_previous_segment == 0 {
172 if !current_group_refs.is_empty() {
173 groups.push(ArrayGroup {
174 array_type: current_type,
175 array_cv_code: current_cv_code,
176 dtype: current_dtype,
177 array_filter: current_filter,
178 refs: current_group_refs,
179 });
180 current_group_refs = Vec::new();
181 }
182 current_type = address.array_type;
183 current_cv_code = address.array_cv_code;
184 current_dtype = address.dtype;
185 current_filter = address.array_filter;
186 } else if address.array_type != current_type
187 || address.dtype != current_dtype
188 || address.array_filter != current_filter
189 {
190 return Err(
191 "array grouping: continuation ref has different array_type, dtype, or filter"
192 .into(),
193 );
194 }
195
196 current_group_refs.push(*address);
197 }
198
199 if !current_group_refs.is_empty() {
200 groups.push(ArrayGroup {
201 array_type: current_type,
202 array_cv_code: current_cv_code,
203 dtype: current_dtype,
204 array_filter: current_filter,
205 refs: current_group_refs,
206 });
207 }
208
209 for group in &groups {
210 if group.refs.len() > 1 {
211 for address in &group.refs {
212 if address.encoded_len > 0 {
213 return Err(
214 "array grouping: multi-ref group cannot contain variable-length arrays"
215 .into(),
216 );
217 }
218 }
219 }
220 }
221
222 Ok(groups)
223}
224
225pub(crate) fn read_group_decoded_bytes(
226 group: &ArrayGroup,
227 container: &mut BlockReader<DefaultBlockProcessor>,
228) -> IonResult<Vec<u8>> {
229 if let [array_address] = group.refs.as_slice() {
230 let (element_offset, count, stride) = address_read_params(array_address);
231 let raw = container.get_array_bytes_from_block(
232 array_address.block_id,
233 element_offset,
234 count,
235 stride,
236 "read_group_decoded_bytes",
237 )?;
238 let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?;
239 return Ok(unfiltered.into_owned());
240 }
241
242 let mut decoded = Vec::new();
243 for array_address in &group.refs {
244 let (element_offset, count, stride) = address_read_params(array_address);
245 let raw = container.get_array_bytes_from_block(
246 array_address.block_id,
247 element_offset,
248 count,
249 stride,
250 "read_group_decoded_bytes",
251 )?;
252 let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?;
253 decoded.reserve(unfiltered.len());
254 decoded.extend_from_slice(&unfiltered);
255 }
256
257 Ok(decoded)
258}
259
260#[inline]
261pub(crate) fn dtype_stride(dtype: u8) -> usize {
262 match dtype {
263 FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
264 FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
265 FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
266 _ => 1,
267 }
268}
269
270pub(crate) fn unfilter_array_bytes(
271 raw: &[u8],
272 dtype: u8,
273 array_filter: u8,
274) -> IonResult<std::borrow::Cow<'_, [u8]>> {
275 let pk_id = PackingId::from_byte(array_filter)?;
276 match pk_id {
277 PackingId::Raw => Ok(std::borrow::Cow::Borrowed(raw)),
278 PackingId::ByteShuffle => match crate::ion::packing::Dtype::from_byte(dtype) {
279 Ok(
280 dtype_enum @ (crate::ion::packing::Dtype::F64 | crate::ion::packing::Dtype::F32),
281 ) => {
282 let mut out = Vec::new();
283 crate::ion::packing::packing_by_id(PackingId::ByteShuffle)
284 .decode(raw, dtype_enum, &mut out)?;
285 Ok(std::borrow::Cow::Owned(out))
286 }
287 _ => Err(format!("array filter {array_filter} is not valid for dtype {dtype}").into()),
288 },
289 PackingId::DeltaShuffle => match crate::ion::packing::Dtype::from_byte(dtype) {
290 Ok(
291 dtype_enum @ (crate::ion::packing::Dtype::F64 | crate::ion::packing::Dtype::F32),
292 ) => {
293 let mut out = Vec::new();
294 crate::ion::packing::packing_by_id(PackingId::DeltaShuffle)
295 .decode(raw, dtype_enum, &mut out)?;
296 Ok(std::borrow::Cow::Owned(out))
297 }
298 _ => Err(format!("array filter {array_filter} is not valid for dtype {dtype}").into()),
299 },
300 }
301}
302
303pub(crate) fn decode_into(
304 buf: &mut Vec<f64>,
305 raw: &[u8],
306 dtype: u8,
307 array_filter: u8,
308) -> IonResult<()> {
309 buf.clear();
310 let bytes = unfilter_array_bytes(raw, dtype, array_filter)?;
311 match dtype {
312 FILE_DTYPE_F64 => {
313 buf.reserve(bytes.len() / 8);
314 buf.extend(
315 bytes
316 .chunks_exact(8)
317 .map(|c| f64::from_le_bytes(c.try_into().unwrap())),
318 );
319 }
320 FILE_DTYPE_F32 => {
321 buf.reserve(bytes.len() / 4);
322 buf.extend(
323 bytes
324 .chunks_exact(4)
325 .map(|c| f32::from_le_bytes(c.try_into().unwrap()) as f64),
326 );
327 }
328 FILE_DTYPE_F16 => {
329 buf.reserve(bytes.len() / 2);
330 buf.extend(
331 bytes
332 .chunks_exact(2)
333 .map(|c| f16_bits_to_f64(u16::from_le_bytes(c.try_into().unwrap()))),
334 );
335 }
336 FILE_DTYPE_I16 => {
337 buf.reserve(bytes.len() / 2);
338 buf.extend(
339 bytes
340 .chunks_exact(2)
341 .map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f64),
342 );
343 }
344 FILE_DTYPE_I32 => {
345 buf.reserve(bytes.len() / 4);
346 buf.extend(
347 bytes
348 .chunks_exact(4)
349 .map(|c| i32::from_le_bytes(c.try_into().unwrap()) as f64),
350 );
351 }
352 FILE_DTYPE_I64 => {
353 buf.reserve(bytes.len() / 8);
354 buf.extend(
355 bytes
356 .chunks_exact(8)
357 .map(|c| i64::from_le_bytes(c.try_into().unwrap()) as f64),
358 );
359 }
360 _ => {
361 return Err(IonError::BadDtype {
362 dtype,
363 kind: "decode array dtype",
364 });
365 }
366 }
367 Ok(())
368}
369
370fn collect_entry_array_addresses(
371 entry_bytes: &[u8],
372 address_bytes: &[u8],
373) -> Option<Vec<ArrayAddress>> {
374 let ref_start =
375 usize::try_from(u64::from_le_bytes(entry_bytes[0..8].try_into().unwrap())).ok()?;
376 let address_count =
377 usize::try_from(u64::from_le_bytes(entry_bytes[8..16].try_into().unwrap())).ok()?;
378 let start = ref_start.checked_mul(ARRAY_ADDRESS_BYTES)?;
379 let span = address_count.checked_mul(ARRAY_ADDRESS_BYTES)?;
380 let end = start.checked_add(span)?;
381 let mut refs = Vec::with_capacity(address_count);
382 for bytes in address_bytes
383 .get(start..end)?
384 .chunks_exact(ARRAY_ADDRESS_BYTES)
385 {
386 refs.push(parse_array_address(bytes));
387 }
388 Some(refs)
389}
390
391pub(crate) fn read_scan_arrays(
392 container: &mut dyn ContainerAccess,
393 entry_bytes: &[u8],
394 address_bytes: &[u8],
395 mz: &mut Vec<f64>,
396 intensity: &mut Vec<f64>,
397) -> bool {
398 mz.clear();
399 intensity.clear();
400 let Some(refs) = collect_entry_array_addresses(entry_bytes, address_bytes) else {
401 return false;
402 };
403 let Ok(groups) = group_arrays(&refs) else {
404 return false;
405 };
406 let mut segment = Vec::new();
407 for group in &groups {
408 let target = match group.array_type {
409 ACC_MZ => &mut *mz,
410 ACC_INT => &mut *intensity,
411 _ => continue,
412 };
413 for array_address in &group.refs {
414 let (element_offset, count, stride) = address_read_params(array_address);
415 let raw = match container.get_array_bytes_from_block(
416 array_address.block_id,
417 element_offset,
418 count,
419 stride,
420 "scan",
421 ) {
422 Ok(raw) => raw,
423 Err(_) => return false,
424 };
425 if decode_into(
426 &mut segment,
427 raw,
428 array_address.dtype,
429 array_address.array_filter,
430 )
431 .is_err()
432 {
433 return false;
434 }
435 target.extend_from_slice(&segment);
436 }
437 }
438 mz.len().min(intensity.len()) > 0
439}
440
441impl IonReader {
442 pub fn spectrum_array_addresses(&self, index: usize) -> Option<Vec<ArrayAddress>> {
443 if index >= self.header.spectrum_count as usize {
444 return None;
445 }
446 read_array_addresses_from_buffers(&self.spec_entries_buf, &self.spec_array_addresses, index)
447 .map(ArrayAddressList::into_vec)
448 }
449
450 pub fn chromatogram_array_addresses(&self, index: usize) -> Option<Vec<ArrayAddress>> {
451 if index >= self.header.chrom_count as usize {
452 return None;
453 }
454 read_array_addresses_from_buffers(
455 &self.chrom_entries_buf,
456 &self.chrom_array_addresses,
457 index,
458 )
459 .map(ArrayAddressList::into_vec)
460 }
461
462 pub fn read_spectrum_values(
463 &mut self,
464 array_address: &ArrayAddress,
465 out: &mut Vec<f64>,
466 ) -> IonResult<()> {
467 let (element_offset, count, stride) = address_read_params(array_address);
468 let raw = self.spec_container.get_array_bytes_from_block(
469 array_address.block_id,
470 element_offset,
471 count,
472 stride,
473 "read_array",
474 )?;
475 decode_into(out, raw, array_address.dtype, array_address.array_filter)
476 }
477
478 pub fn read_spectrum_array(&mut self, array_address: &ArrayAddress) -> IonResult<NumericArray> {
479 let (element_offset, count, stride) = address_read_params(array_address);
480 let raw = self.spec_container.get_array_bytes_from_block(
481 array_address.block_id,
482 element_offset,
483 count,
484 stride,
485 "read_spectrum_array",
486 )?;
487 let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?;
488 super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype)
489 }
490
491 pub fn read_chromatogram_values(
492 &mut self,
493 array_address: &ArrayAddress,
494 out: &mut Vec<f64>,
495 ) -> IonResult<()> {
496 let container = self
497 .chrom_container
498 .as_mut()
499 .ok_or_else(|| IonError::from("no chromatogram container"))?;
500 let (element_offset, count, stride) = address_read_params(array_address);
501 let raw = container.get_array_bytes_from_block(
502 array_address.block_id,
503 element_offset,
504 count,
505 stride,
506 "read_chromatogram_values",
507 )?;
508 decode_into(out, raw, array_address.dtype, array_address.array_filter)
509 }
510
511 pub fn read_chromatogram_array(
512 &mut self,
513 array_address: &ArrayAddress,
514 ) -> IonResult<NumericArray> {
515 let container = self
516 .chrom_container
517 .as_mut()
518 .ok_or_else(|| IonError::from("no chromatogram container"))?;
519 let (element_offset, count, stride) = address_read_params(array_address);
520 let raw = container.get_array_bytes_from_block(
521 array_address.block_id,
522 element_offset,
523 count,
524 stride,
525 "read_chromatogram_array",
526 )?;
527 let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?;
528 super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype)
529 }
530
531 pub(crate) fn read_group_values(
532 &mut self,
533 group: &ArrayGroup,
534 out: &mut Vec<f64>,
535 ) -> IonResult<()> {
536 let Some((first, rest)) = group.refs.split_first() else {
537 out.clear();
538 return Ok(());
539 };
540 self.read_spectrum_values(first, out)?;
541 let mut window = Vec::new();
542 for array_address in rest {
543 self.read_spectrum_values(array_address, &mut window)?;
544 out.extend_from_slice(&window);
545 }
546 Ok(())
547 }
548
549 pub fn get_spectrum_array(
550 &mut self,
551 spectrum_index: usize,
552 array_type: impl Into<u32>,
553 ) -> IonResult<Vec<f64>> {
554 let array_type = array_type.into();
555 if spectrum_index >= self.header.spectrum_count as usize {
556 return Err("spectrum index out of range".into());
557 }
558
559 let Some(array_addresses) = read_array_addresses_from_buffers(
560 &self.spec_entries_buf,
561 &self.spec_array_addresses,
562 spectrum_index,
563 ) else {
564 return Ok(Vec::new());
565 };
566
567 let groups = group_arrays(array_addresses.as_slice())?;
568
569 for group in groups {
570 if group.array_type != array_type {
571 continue;
572 }
573
574 let mut values = Vec::new();
575 self.read_group_values(&group, &mut values)?;
576 return Ok(values);
577 }
578
579 Ok(Vec::new())
580 }
581}
582
583#[cfg(test)]
584mod tests {
585 use super::*;
586
587 #[test]
588 fn unfilter_array_bytes_delta_shuffle_f64_matches_cumulative_sum() {
589 let deltas: [u64; 4] = [100u64, 5, 3, 10];
590 let raw: Vec<u8> = deltas.iter().flat_map(|w| w.to_le_bytes()).collect();
591
592 let unfiltered =
593 unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::DeltaShuffle as u8).unwrap();
594
595 let mut expected_prev: u64 = 0;
596 let mut expected = Vec::new();
597 for delta in deltas {
598 expected_prev = expected_prev.wrapping_add(delta);
599 expected.extend_from_slice(&expected_prev.to_le_bytes());
600 }
601
602 assert_eq!(unfiltered.as_ref(), expected.as_slice());
603 }
604
605 #[test]
606 fn unfilter_array_bytes_delta_shuffle_rejects_misaligned_f64_input() {
607 let raw = [0u8; 7];
608 let err = unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::DeltaShuffle as u8)
609 .expect_err("7 bytes is not a multiple of the f64 word size");
610 assert!(err.contains("not a multiple of the word size"));
611 }
612
613 #[test]
614 fn unfilter_array_bytes_byte_shuffle_f64_round_trips_71() {
615 use crate::ion::packing::{PackingInput, packing_by_id};
616
617 let data: Vec<f64> = (0..600)
618 .map(|i| 250.0 + (i as f64) * 0.0137 + ((i * 5 % 11) as f64) * 0.0011)
619 .collect();
620 let raw: Vec<u8> = data.iter().flat_map(|v| v.to_le_bytes()).collect();
621
622 let mut shuffled = Vec::new();
623 packing_by_id(PackingId::ByteShuffle)
624 .encode(PackingInput::F64(&data), &mut shuffled)
625 .unwrap();
626
627 assert_ne!(
628 shuffled, raw,
629 "shuffled layout must differ from raw for varying data"
630 );
631
632 let unfiltered =
633 unfilter_array_bytes(&shuffled, FILE_DTYPE_F64, PackingId::ByteShuffle as u8).unwrap();
634
635 assert_eq!(
636 unfiltered.as_ref(),
637 raw.as_slice(),
638 "decode of ByteShuffle-tagged bytes must unshuffle back to raw"
639 );
640
641 let out: Vec<f64> = unfiltered
642 .chunks_exact(8)
643 .map(|c| f64::from_le_bytes(c.try_into().unwrap()))
644 .collect();
645 assert_eq!(out.len(), data.len());
646 for (a, b) in out.iter().zip(data.iter()) {
647 assert_eq!(a.to_bits(), b.to_bits());
648 }
649 }
650
651 #[test]
652 fn unfilter_array_bytes_delta_shuffle_rejects_unsupported_dtype() {
653 let raw = [1u8, 2, 3, 4];
654 let err = unfilter_array_bytes(&raw, FILE_DTYPE_I16, PackingId::DeltaShuffle as u8)
655 .expect_err("delta shuffle is only valid for f32/f64");
656 assert!(err.contains("is not valid for dtype"));
657 }
658
659 #[test]
660 fn shuffle_filter_on_integer_dtype_is_rejected_4() {
661 let raw = [0u8; 8];
662
663 let err = unfilter_array_bytes(&raw, FILE_DTYPE_I32, PackingId::ByteShuffle as u8)
664 .expect_err("byte shuffle is only valid for f32/f64, not i32");
665 assert!(err.contains("is not valid for dtype"));
666
667 let ok = unfilter_array_bytes(&raw, FILE_DTYPE_F64, PackingId::Raw as u8)
668 .expect("raw filter must still work for any dtype");
669 assert_eq!(ok.as_ref(), raw.as_slice());
670 }
671}