1use crate::{
2 accessions::{
3 FLOAT_32BIT, FLOAT_64BIT, HIGHEST_ION_MOBILITY, HIGHEST_OBSERVED_MZ,
4 HIGHEST_OBSERVED_WAVELENGTH, INTENSITY_ARRAY, LOWEST_ION_MOBILITY, LOWEST_OBSERVED_MZ,
5 LOWEST_OBSERVED_WAVELENGTH, NEGATIVE_SCAN, POSITION_X, POSITION_Y, POSITION_Z,
6 POSITIVE_SCAN,
7 },
8 ion::{
9 IonError, IonResult,
10 encoder::utilities::{
11 BlockWriter, CompressionMode, DefaultCompressor,
12 le_writers::{
13 write_f32_le, write_f32_slice_le, write_f64_le, write_f64_slice_le,
14 write_i16_slice_le, write_i32_slice_le, write_i64_slice_le, write_u16_slice_le,
15 },
16 meta_collector::{
17 ArrayPolicy, array_type_accession_from_binary_data_array,
18 array_type_cv_code_from_binary_data_array, parse_accession_tail_raw,
19 },
20 output::{SectionStorage, WriteBytes},
21 },
22 filter_summary::{ChromatogramSummary, SpectrumSummary},
23 packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING},
24 utilities::spectrum_source::{f16_bits_to_f64, summary_from_spectrum},
25 windowing::{WindowRange, Windowing},
26 },
27 mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum},
28};
29pub(crate) const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024;
30pub const DEFAULT_MZ_WINDOW: f64 = 20.0;
31pub(crate) const SPEC_SUMMARY_SIZE: usize = 80;
32pub(crate) const CHROM_SUMMARY_SIZE: usize = 80;
33
34pub(crate) const FILE_DTYPE_F64: u8 = Dtype::F64 as u8;
35pub(crate) const FILE_DTYPE_F32: u8 = Dtype::F32 as u8;
36pub(crate) const FILE_DTYPE_F16: u8 = Dtype::F16 as u8;
37pub(crate) const FILE_DTYPE_I16: u8 = Dtype::I16 as u8;
38pub(crate) const FILE_DTYPE_I32: u8 = Dtype::I32 as u8;
39pub(crate) const FILE_DTYPE_I64: u8 = Dtype::I64 as u8;
40
41const POLARITY_UNKNOWN: u8 = 0;
42const POLARITY_POSITIVE: u8 = 1;
43const POLARITY_NEGATIVE: u8 = 2;
44
45impl ChromatogramSummary {
46 fn unknown() -> Self {
47 Self {
48 lowest_mz: f64::NAN,
49 highest_mz: f64::NAN,
50 lowest_wavelength: f64::NAN,
51 highest_wavelength: f64::NAN,
52 lowest_ion_mobility: f64::NAN,
53 highest_ion_mobility: f64::NAN,
54 polarity: POLARITY_UNKNOWN,
55 }
56 }
57}
58
59pub(crate) fn spec_summary_from_spectrum(spectrum: &Spectrum) -> SpectrumSummary {
60 let summary = summary_from_spectrum(spectrum);
61 let (position_x, position_y, position_z) = get_spectrum_position(spectrum);
62 SpectrumSummary {
63 rt: summary.rt,
64 rt_unit: summary.rt_unit.code(),
65 base_peak_mz: summary.base_peak_mz,
66 selected_ion_mz: summary.selected_ion_mz,
67 base_peak_int: summary.base_peak_int,
68 total_ion_current: summary.total_ion_current,
69 ms_level: summary.ms_level,
70 polarity: summary.polarity,
71 position_x,
72 position_y,
73 position_z,
74 }
75}
76
77fn get_spectrum_position(spectrum: &Spectrum) -> (u32, u32, u32) {
78 let mut x = 0;
79 let mut y = 0;
80 let mut z = 0;
81 let scan_list = spectrum.scan_list.as_ref().or_else(|| {
82 spectrum
83 .spectrum_description
84 .as_ref()
85 .and_then(|description| description.scan_list.as_ref())
86 });
87 let Some(scan_list) = scan_list else {
88 return (x, y, z);
89 };
90 for scan in &scan_list.scans {
91 for param in &scan.cv_params {
92 let Some(accession) = param.accession.as_deref() else {
93 continue;
94 };
95 if !accession.starts_with("IMS:") {
96 continue;
97 }
98 let value = param
99 .value
100 .as_deref()
101 .and_then(|value| value.parse::<u32>().ok())
102 .unwrap_or(0);
103 match parse_accession_tail_raw(Some(accession)) {
104 POSITION_X => x = value,
105 POSITION_Y => y = value,
106 POSITION_Z => z = value,
107 _ => {}
108 }
109 }
110 }
111 (x, y, z)
112}
113
114#[inline]
115fn cv_f64(cv: &CvParam) -> Option<f64> {
116 cv.value.as_deref().and_then(|s| s.parse::<f64>().ok())
117}
118
119pub(crate) fn extract_chrom_summary(chrom: &Chromatogram) -> ChromatogramSummary {
120 let mut summary = ChromatogramSummary::unknown();
121 for cv in &chrom.cv_params {
122 match parse_accession_tail_raw(cv.accession.as_deref()) {
123 LOWEST_OBSERVED_MZ => {
124 if let Some(v) = cv_f64(cv) {
125 summary.lowest_mz = v;
126 }
127 }
128 HIGHEST_OBSERVED_MZ => {
129 if let Some(v) = cv_f64(cv) {
130 summary.highest_mz = v;
131 }
132 }
133 LOWEST_OBSERVED_WAVELENGTH => {
134 if let Some(v) = cv_f64(cv) {
135 summary.lowest_wavelength = v;
136 }
137 }
138 HIGHEST_OBSERVED_WAVELENGTH => {
139 if let Some(v) = cv_f64(cv) {
140 summary.highest_wavelength = v;
141 }
142 }
143 LOWEST_ION_MOBILITY => {
144 if let Some(v) = cv_f64(cv) {
145 summary.lowest_ion_mobility = v;
146 }
147 }
148 HIGHEST_ION_MOBILITY => {
149 if let Some(v) = cv_f64(cv) {
150 summary.highest_ion_mobility = v;
151 }
152 }
153 POSITIVE_SCAN => summary.polarity = POLARITY_POSITIVE,
154 NEGATIVE_SCAN => summary.polarity = POLARITY_NEGATIVE,
155 _ => {}
156 }
157 }
158 summary
159}
160
161pub(crate) fn get_supported_compression_level(compression_level: u8) -> u8 {
162 match compression_level {
163 0..=5 => 1,
164 6..=10 => 9,
165 11..=16 => 12,
166 _ => 22,
167 }
168}
169
170pub(crate) fn allow_compression_level(compression_level: u8) -> IonResult<()> {
171 if compression_level > 22 {
172 return Err(format!("compression_level must be 0-22, got {compression_level}").into());
173 }
174 Ok(())
175}
176
177#[derive(Debug, Clone, Copy)]
178pub struct WriteOptions {
179 pub compression_level: u8,
180 pub force_f32: bool,
181 pub block_size: usize,
182 pub parallel: bool,
183 pub section_storage: SectionStorage,
184 pub mz_window: f64,
185}
186
187impl Default for WriteOptions {
188 fn default() -> Self {
189 Self {
190 compression_level: 12,
191 force_f32: false,
192 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
193 parallel: true,
194 section_storage: SectionStorage::Disk,
195 mz_window: DEFAULT_MZ_WINDOW,
196 }
197 }
198}
199
200impl WriteOptions {
201 pub(crate) fn compression_is_enabled(self) -> bool {
202 self.compression_level != 0
203 }
204
205 pub(crate) fn codec_id(self) -> u8 {
206 self.compression_is_enabled() as u8
207 }
208
209 fn block_shuffle_is_enabled(self) -> bool {
210 self.compression_is_enabled()
211 }
212
213 pub(crate) fn array_filter_id(self) -> u8 {
214 if self.block_shuffle_is_enabled() {
215 PackingId::ByteShuffle as u8
216 } else {
217 PackingId::Raw as u8
218 }
219 }
220
221 pub(crate) fn compression_mode(self) -> IonResult<CompressionMode<DefaultCompressor>> {
222 if self.compression_is_enabled() {
223 Ok(CompressionMode::Compressed(DefaultCompressor::new(
224 self.compression_level as i32,
225 )?))
226 } else {
227 Ok(CompressionMode::Raw)
228 }
229 }
230
231 pub(crate) fn block_packing_id(self) -> PackingId {
232 if self.block_shuffle_is_enabled() {
233 PackingId::ByteShuffle
234 } else {
235 PackingId::Raw
236 }
237 }
238
239 pub(crate) fn array_policy(self, x_array_accession: u32) -> ArrayPolicy {
240 ArrayPolicy {
241 x_array_accession,
242 y_array_accession: INTENSITY_ARRAY,
243 force_f32: self.force_f32,
244 }
245 }
246}
247
248#[derive(Copy, Clone)]
249enum ArrayData<'a> {
250 F16(&'a [u16]),
251 F32(&'a [f32]),
252 F64(&'a [f64]),
253 I16(&'a [i16]),
254 I32(&'a [i32]),
255 I64(&'a [i64]),
256}
257
258impl<'a> ArrayData<'a> {
259 fn element_count(self) -> usize {
260 match self {
261 Self::F16(e) => e.len(),
262 Self::F32(e) => e.len(),
263 Self::F64(e) => e.len(),
264 Self::I16(e) => e.len(),
265 Self::I32(e) => e.len(),
266 Self::I64(e) => e.len(),
267 }
268 }
269
270 fn is_empty(self) -> bool {
271 self.element_count() == 0
272 }
273
274 fn slice(self, start: usize, end: usize) -> ArrayData<'a> {
275 match self {
276 Self::F16(e) => Self::F16(&e[start..end]),
277 Self::F32(e) => Self::F32(&e[start..end]),
278 Self::F64(e) => Self::F64(&e[start..end]),
279 Self::I16(e) => Self::I16(&e[start..end]),
280 Self::I32(e) => Self::I32(&e[start..end]),
281 Self::I64(e) => Self::I64(&e[start..end]),
282 }
283 }
284
285 fn value_at(self, index: usize) -> f64 {
286 match self {
287 Self::F16(e) => f16_bits_to_f64(e[index]),
288 Self::F32(e) => e[index] as f64,
289 Self::F64(e) => e[index],
290 Self::I16(e) => e[index] as f64,
291 Self::I32(e) => e[index] as f64,
292 Self::I64(e) => e[index] as f64,
293 }
294 }
295
296 fn is_monotonic_non_decreasing(self) -> bool {
297 let count = self.element_count();
298 if count < 2 {
299 return true;
300 }
301 let mut previous = self.value_at(0);
302 for index in 1..count {
303 let current = self.value_at(index);
304 if current < previous {
305 return false;
306 }
307 previous = current;
308 }
309 true
310 }
311}
312
313fn array_data_from_binary_data_array(bda: &BinaryDataArray) -> Option<ArrayData<'_>> {
314 match bda.binary.as_ref()? {
315 NumericArray::F16(e) => Some(ArrayData::F16(e)),
316 NumericArray::I16(e) => Some(ArrayData::I16(e)),
317 NumericArray::I32(e) => Some(ArrayData::I32(e)),
318 NumericArray::I64(e) => Some(ArrayData::I64(e)),
319 NumericArray::F32(e) => Some(ArrayData::F32(e)),
320 NumericArray::F64(e) => Some(ArrayData::F64(e)),
321 }
322}
323
324fn element_byte_size_for_dtype(dtype: u8) -> usize {
325 match dtype {
326 FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
327 FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
328 FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
329 _ => 1,
330 }
331}
332
333fn resolve_array_dtype(bda: &BinaryDataArray, data: ArrayData<'_>, force_f32: bool) -> u8 {
334 match data {
335 ArrayData::F16(_) => FILE_DTYPE_F16,
336 ArrayData::I16(_) => FILE_DTYPE_I16,
337 ArrayData::I32(_) => FILE_DTYPE_I32,
338 ArrayData::I64(_) => FILE_DTYPE_I64,
339 ArrayData::F32(_) | ArrayData::F64(_) => {
340 if float_data_should_be_written_as_f64(bda, data, force_f32) {
341 FILE_DTYPE_F64
342 } else {
343 FILE_DTYPE_F32
344 }
345 }
346 }
347}
348
349fn float_data_should_be_written_as_f64(
350 bda: &BinaryDataArray,
351 data: ArrayData<'_>,
352 force_f32: bool,
353) -> bool {
354 if force_f32 {
355 return false;
356 }
357 if matches!(data, ArrayData::F64(_)) {
358 return true;
359 }
360 declared_float_precision_is_64bit(bda).unwrap_or(false)
361}
362
363fn declared_float_precision_is_64bit(bda: &BinaryDataArray) -> Option<bool> {
364 if let Some(nt) = bda.numeric_type.as_ref() {
365 return match nt {
366 NumericType::Float64 => Some(true),
367 NumericType::Float32 => Some(false),
368 _ => None,
369 };
370 }
371 let (mut saw32, mut saw64) = (false, false);
372 for cv in &bda.cv_params {
373 match parse_accession_tail_raw(cv.accession.as_deref()) {
374 FLOAT_32BIT => saw32 = true,
375 FLOAT_64BIT => saw64 = true,
376 _ => {}
377 }
378 if saw32 && saw64 {
379 break;
380 }
381 }
382 match (saw32, saw64) {
383 (true, false) => Some(false),
384 (false, true) => Some(true),
385 _ => None,
386 }
387}
388
389fn validate_array_dtype(data: ArrayData<'_>, dtype: u8) -> IonResult<()> {
390 let ok = matches!(
391 (dtype, data),
392 (FILE_DTYPE_F16, ArrayData::F16(_))
393 | (FILE_DTYPE_F32, ArrayData::F32(_))
394 | (FILE_DTYPE_F32, ArrayData::F64(_))
395 | (FILE_DTYPE_F64, ArrayData::F64(_))
396 | (FILE_DTYPE_F64, ArrayData::F32(_))
397 | (FILE_DTYPE_I16, ArrayData::I16(_))
398 | (FILE_DTYPE_I32, ArrayData::I32(_))
399 | (FILE_DTYPE_I64, ArrayData::I64(_))
400 );
401 if ok {
402 Ok(())
403 } else {
404 Err(format!(
405 "write_array_data: incompatible dtype {dtype} for the given array data variant"
406 )
407 .into())
408 }
409}
410
411fn write_array_data(buf: &mut Vec<u8>, data: ArrayData<'_>, dtype: u8) {
412 match (dtype, data) {
413 (FILE_DTYPE_F16, ArrayData::F16(e)) => write_u16_slice_le(buf, e),
414 (FILE_DTYPE_F32, ArrayData::F32(e)) => write_f32_slice_le(buf, e),
415 (FILE_DTYPE_F32, ArrayData::F64(e)) => {
416 for &v in e {
417 write_f32_le(buf, v as f32);
418 }
419 }
420 (FILE_DTYPE_F64, ArrayData::F64(e)) => write_f64_slice_le(buf, e),
421 (FILE_DTYPE_F64, ArrayData::F32(e)) => {
422 for &v in e {
423 write_f64_le(buf, v as f64);
424 }
425 }
426 (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e),
427 (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e),
428 (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e),
429 _ => unreachable!("write_array_data called with unvalidated dtype/data combination"),
430 }
431}
432
433fn narrow_f64_to_f32(values: &[f64]) -> Vec<f32> {
434 values.iter().map(|&value| value as f32).collect()
435}
436
437fn widen_f32_to_f64(values: &[f32]) -> Vec<f64> {
438 values.iter().map(|&value| value as f64).collect()
439}
440
441fn encode_floats_through_packing(
442 buf: &mut Vec<u8>,
443 data: ArrayData<'_>,
444 dtype_enum: Dtype,
445 packing: &'static dyn Packing,
446) -> IonResult<()> {
447 match (dtype_enum, data) {
448 (Dtype::F64, ArrayData::F64(slice)) => packing.encode(PackingInput::F64(slice), buf),
449 (Dtype::F32, ArrayData::F32(slice)) => packing.encode(PackingInput::F32(slice), buf),
450 (Dtype::F32, ArrayData::F64(slice)) => {
451 let narrowed = narrow_f64_to_f32(slice);
452 packing.encode(PackingInput::F32(&narrowed), buf)
453 }
454 (Dtype::F64, ArrayData::F32(slice)) => {
455 let widened = widen_f32_to_f64(slice);
456 packing.encode(PackingInput::F64(&widened), buf)
457 }
458 _ => Err(IonError::from(
459 "shuffle packing requires f32 or f64 array data",
460 )),
461 }
462}
463
464pub(crate) struct EncodedArrayAddress {
465 pub(crate) element_offset: u64,
466 pub(crate) element_count: u64,
467 pub(crate) block_id: u32,
468 pub(crate) accession: u32,
469 pub(crate) dtype: u8,
470 pub(crate) array_filter: u8,
471 pub(crate) encoded_len: u32,
472 pub(crate) continues_previous_segment: u8,
473 pub(crate) array_cv_code: u8,
474}
475
476fn encode_variable_length_array(
477 output: &mut dyn WriteBytes,
478 array_type: u32,
479 data: ArrayData<'_>,
480 dtype: u8,
481 dtype_enum: Dtype,
482 packing: &'static dyn Packing,
483 container: &mut BlockWriter<DefaultCompressor>,
484) -> IonResult<(u32, u64, u32)> {
485 let mut encoded = Vec::new();
486 match packing.id() {
487 PackingId::Raw => write_array_data(&mut encoded, data, dtype),
488 _ => encode_floats_through_packing(&mut encoded, data, dtype_enum, packing)?,
489 }
490 let enc_len =
491 u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?;
492 let (bid, eoff) = container.add_item_to_box(output, array_type, encoded.len(), 1, |buf| {
493 buf.extend_from_slice(&encoded);
494 Ok(())
495 })?;
496 Ok((bid, eoff, enc_len))
497}
498
499fn write_fixed_array_payload(
500 buf: &mut Vec<u8>,
501 data: ArrayData<'_>,
502 dtype: u8,
503 dtype_enum: Dtype,
504 packing: &'static dyn Packing,
505) -> IonResult<()> {
506 match packing.id() {
507 PackingId::Raw => {
508 write_array_data(buf, data, dtype);
509 Ok(())
510 }
511 _ => encode_floats_through_packing(buf, data, dtype_enum, packing),
512 }
513}
514
515#[allow(clippy::too_many_arguments)]
516fn encode_fixed_length_array(
517 output: &mut dyn WriteBytes,
518 array_type: u32,
519 data: ArrayData<'_>,
520 dtype: u8,
521 dtype_enum: Dtype,
522 elem_bytes: usize,
523 packing: &'static dyn Packing,
524 container: &mut BlockWriter<DefaultCompressor>,
525) -> IonResult<(u32, u64, u32)> {
526 let (bid, eoff) = container.add_item_to_box(
527 output,
528 array_type,
529 data.element_count() * elem_bytes,
530 elem_bytes,
531 |buf| write_fixed_array_payload(buf, data, dtype, dtype_enum, packing),
532 )?;
533 Ok((bid, eoff, 0u32))
534}
535
536struct ArrayEncoding<'a> {
537 data: ArrayData<'a>,
538 accession: u32,
539 array_cv_code: u8,
540 dtype: u8,
541 dtype_enum: Dtype,
542 elem_bytes: usize,
543 packing: &'static dyn Packing,
544}
545
546fn resolve_array_encoding<'a>(
547 bda: &'a BinaryDataArray,
548 config: WriteOptions,
549 policy: ArrayPolicy,
550) -> IonResult<Option<ArrayEncoding<'a>>> {
551 let Some(data) = array_data_from_binary_data_array(bda) else {
552 return Ok(None);
553 };
554 if data.is_empty() {
555 return Ok(None);
556 }
557 let accession = array_type_accession_from_binary_data_array(bda);
558 let array_cv_code = array_type_cv_code_from_binary_data_array(bda);
559 let dtype = resolve_array_dtype(bda, data, policy.should_force_f32(accession));
560 validate_array_dtype(data, dtype)?;
561 let elem_bytes = element_byte_size_for_dtype(dtype);
562 let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64);
563 let requested: &'static dyn Packing = if config.compression_is_enabled() {
564 packing_for(accession, dtype_enum)
565 } else {
566 &RAW_PACKING
567 };
568 let packing: &'static dyn Packing = if requested.supports(dtype_enum) {
569 requested
570 } else {
571 &RAW_PACKING
572 };
573 Ok(Some(ArrayEncoding {
574 data,
575 accession,
576 array_cv_code,
577 dtype,
578 dtype_enum,
579 elem_bytes,
580 packing,
581 }))
582}
583
584pub(crate) fn encode_single_array(
585 output: &mut dyn WriteBytes,
586 bda: &BinaryDataArray,
587 config: WriteOptions,
588 policy: ArrayPolicy,
589 container: &mut BlockWriter<DefaultCompressor>,
590) -> IonResult<Option<EncodedArrayAddress>> {
591 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
592 return Ok(None);
593 };
594 let ArrayEncoding {
595 data,
596 accession,
597 array_cv_code,
598 dtype,
599 dtype_enum,
600 elem_bytes,
601 packing,
602 } = encoding;
603 let (block_id, element_offset, encoded_len) = if packing.is_variable_length() {
604 encode_variable_length_array(
605 output, accession, data, dtype, dtype_enum, packing, container,
606 )?
607 } else {
608 encode_fixed_length_array(
609 output, accession, data, dtype, dtype_enum, elem_bytes, packing, container,
610 )?
611 };
612 Ok(Some(EncodedArrayAddress {
613 element_offset,
614 element_count: data.element_count() as u64,
615 block_id,
616 accession,
617 dtype,
618 array_filter: packing.id() as u8,
619 encoded_len,
620 continues_previous_segment: 0,
621 array_cv_code,
622 }))
623}
624
625pub(crate) fn array_is_fixed_width_splittable(
626 bda: &BinaryDataArray,
627 config: WriteOptions,
628 policy: ArrayPolicy,
629) -> IonResult<Option<(usize, usize)>> {
630 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
631 return Ok(None);
632 };
633 if encoding.packing.is_variable_length() {
634 return Ok(None);
635 }
636 Ok(Some((encoding.data.element_count(), encoding.elem_bytes)))
637}
638
639pub(crate) fn check_spectrum_rt_order(
640 summary: &[u8],
641 spectrum_index: usize,
642 last_rt: &mut f64,
643) -> IonResult<()> {
644 let rt = f64::from_le_bytes(summary[0..8].try_into().unwrap());
645 if !rt.is_finite() {
646 return Ok(());
647 }
648 let previous = *last_rt;
649 if rt < previous {
650 return Err(format!(
651 "spectrum {spectrum_index}: retention time {rt} is below the previous spectrum's \
652 ({previous}); spectra must be stored in ascending rt order"
653 )
654 .into());
655 }
656 *last_rt = rt;
657 Ok(())
658}
659
660pub(crate) fn check_spectrum_mz_order(
661 arrays: &[BinaryDataArray],
662 spectrum_index: usize,
663) -> IonResult<()> {
664 use crate::accessions::MZ_ARRAY;
665 for bda in arrays {
666 if array_type_accession_from_binary_data_array(bda) != MZ_ARRAY {
667 continue;
668 }
669 let Some(data) = array_data_from_binary_data_array(bda) else {
670 return Ok(());
671 };
672 if !data.is_monotonic_non_decreasing() {
673 return Err(format!(
674 "spectrum {spectrum_index}: m/z array must be sorted ascending (non-decreasing) for range reads"
675 )
676 .into());
677 }
678 return Ok(());
679 }
680 Ok(())
681}
682
683pub(crate) fn window_ranges_for_item(
684 arrays: &[BinaryDataArray],
685 config: WriteOptions,
686 policy: ArrayPolicy,
687 mz_window: f64,
688) -> IonResult<Option<Vec<WindowRange>>> {
689 let windowing = Windowing::new(mz_window.round());
690 if !windowing.is_enabled() {
691 return Ok(None);
692 }
693 let Some(x_array) = arrays
694 .iter()
695 .find(|bda| array_type_accession_from_binary_data_array(bda) == policy.x_array_accession)
696 else {
697 return Ok(None);
698 };
699 let Some((x_count, _)) = array_is_fixed_width_splittable(x_array, config, policy)? else {
700 return Ok(None);
701 };
702 if x_count == 0 {
703 return Ok(None);
704 }
705 for bda in arrays {
706 match array_is_fixed_width_splittable(bda, config, policy)? {
707 Some((count, _)) if count == x_count => {}
708 _ => return Ok(None),
709 }
710 }
711 let Some(x_data) = array_data_from_binary_data_array(x_array) else {
712 return Ok(None);
713 };
714 let ranges = windowing.split_sorted(x_count, |index| x_data.value_at(index));
715 const MAX_WINDOW_INDEX: u32 = 1 << 20;
716 if ranges
717 .last()
718 .is_some_and(|range| range.window_index > MAX_WINDOW_INDEX)
719 {
720 return Err(format!(
721 "m/z window index exceeds limit {MAX_WINDOW_INDEX}; m/z value too large for window width {}",
722 mz_window.round()
723 )
724 .into());
725 }
726 Ok(Some(ranges))
727}
728
729pub(crate) fn write_array_windows(
730 output: &mut dyn WriteBytes,
731 bda: &BinaryDataArray,
732 config: WriteOptions,
733 policy: ArrayPolicy,
734 windows: &[WindowRange],
735 container: &mut BlockWriter<DefaultCompressor>,
736) -> IonResult<Vec<EncodedArrayAddress>> {
737 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
738 return Ok(Vec::new());
739 };
740 let ArrayEncoding {
741 data,
742 accession,
743 array_cv_code,
744 dtype,
745 dtype_enum,
746 elem_bytes,
747 packing,
748 } = encoding;
749
750 if packing.is_variable_length() {
751 return Err("write_array_windows: variable-length arrays cannot be windowed".into());
752 }
753
754 let mut addresses = Vec::with_capacity(windows.len());
755 for (segment_index, window) in windows.iter().enumerate() {
756 let segment_data = data.slice(window.start, window.end);
757 let element_count = window.element_count();
758 let (block_id, element_offset) = container.add_item_to_window(
759 output,
760 accession,
761 window.window_index,
762 element_count * elem_bytes,
763 elem_bytes,
764 |buf| write_fixed_array_payload(buf, segment_data, dtype, dtype_enum, packing),
765 )?;
766 addresses.push(EncodedArrayAddress {
767 element_offset,
768 element_count: element_count as u64,
769 block_id,
770 accession,
771 dtype,
772 array_filter: packing.id() as u8,
773 encoded_len: 0,
774 continues_previous_segment: (segment_index > 0) as u8,
775 array_cv_code,
776 });
777 }
778 Ok(addresses)
779}
780
781#[cfg(test)]
782mod tests {
783 use super::*;
784 use crate::{
785 ion::{
786 encoder::{ion_writer::write_mzml_to_ion, utilities::output::WriteBytes},
787 format::{FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE},
788 header::{
789 HEADER_CHROM_BLOCK_COUNT, HEADER_SPECTRUM_BLOCK_COUNT, HEADER_TARGET_BLOCK_SIZE,
790 HEADER_TARGET_MZ_WINDOW, HEADER_TOTAL_FILE_SIZE,
791 },
792 },
793 mzml::structs::MzML,
794 };
795
796 fn summary_with_rt(rt: f64) -> [u8; SPEC_SUMMARY_SIZE] {
797 let mut summary = [0u8; SPEC_SUMMARY_SIZE];
798 summary[0..8].copy_from_slice(&rt.to_le_bytes());
799 summary
800 }
801
802 #[test]
803 fn byte_shuffle_array_encode_arm_shuffles_not_raw_71() {
804 use crate::ion::packing::packing_by_id;
805
806 let data: Vec<f64> = (0..600)
807 .map(|i| 100.0 + (i as f64) * 0.0137 + ((i * 7 % 13) as f64) * 0.0009)
808 .collect();
809
810 let byte_shuffle = packing_by_id(PackingId::ByteShuffle);
811
812 let mut shuffled = Vec::new();
813 write_fixed_array_payload(
814 &mut shuffled,
815 ArrayData::F64(&data),
816 FILE_DTYPE_F64,
817 Dtype::F64,
818 byte_shuffle,
819 )
820 .unwrap();
821
822 let mut raw = Vec::new();
823 write_array_data(&mut raw, ArrayData::F64(&data), FILE_DTYPE_F64);
824
825 assert_eq!(shuffled.len(), raw.len());
826 assert_ne!(
827 shuffled, raw,
828 "ByteShuffle encode arm must shuffle, not write raw bytes"
829 );
830
831 let mut expected = Vec::new();
832 byte_shuffle
833 .encode(PackingInput::F64(&data), &mut expected)
834 .unwrap();
835 assert_eq!(
836 shuffled, expected,
837 "ByteShuffle encode arm must produce exactly the SIMD byte-transpose output"
838 );
839 }
840
841 #[test]
842 fn rt_order_allows_ascending_and_missing_but_rejects_a_drop() {
843 let mut last_rt = f64::NEG_INFINITY;
844 check_spectrum_rt_order(&summary_with_rt(1.0), 0, &mut last_rt)
845 .expect("first rt is allowed");
846 check_spectrum_rt_order(&summary_with_rt(1.0), 1, &mut last_rt)
847 .expect("equal rt is allowed");
848 check_spectrum_rt_order(&summary_with_rt(2.0), 2, &mut last_rt)
849 .expect("rising rt is allowed");
850
851 check_spectrum_rt_order(&summary_with_rt(f64::NAN), 3, &mut last_rt)
852 .expect("missing rt is allowed");
853 assert_eq!(last_rt, 2.0, "missing rt must not move the last seen rt");
854
855 let dropped = check_spectrum_rt_order(&summary_with_rt(1.5), 4, &mut last_rt);
856 let message = dropped
857 .expect_err("rt below the previous spectrum must be rejected")
858 .to_string();
859 assert!(
860 message.contains("spectrum 4"),
861 "error names the spectrum: {message}"
862 );
863 assert!(
864 message.contains("ascending rt order"),
865 "error explains the rule: {message}"
866 );
867 }
868
869 #[test]
870 fn encode_starts_with_magic_and_ends_with_trailer() {
871 let mzml = MzML::default();
872 let mut buf = Vec::new();
873 write_mzml_to_ion(
874 &mzml,
875 &WriteOptions {
876 compression_level: 0,
877 force_f32: false,
878 ..Default::default()
879 },
880 &mut buf,
881 )
882 .unwrap();
883 assert!(buf.len() >= HEADER_SIZE + FILE_TRAILER.len());
884 assert_eq!(&buf[..FILE_SIGNATURE.len()], &FILE_SIGNATURE);
885 assert_eq!(&buf[buf.len() - FILE_TRAILER.len()..], &FILE_TRAILER);
886 }
887
888 #[test]
889 fn encode_total_file_size_is_correct() {
890 let mzml = MzML::default();
891 let mut buf = Vec::new();
892 write_mzml_to_ion(
893 &mzml,
894 &WriteOptions {
895 compression_level: 0,
896 force_f32: false,
897 ..Default::default()
898 },
899 &mut buf,
900 )
901 .unwrap();
902 let total = u64::from_le_bytes(
903 buf[HEADER_TOTAL_FILE_SIZE..HEADER_TOTAL_FILE_SIZE + 8]
904 .try_into()
905 .unwrap(),
906 );
907 assert_eq!(total, buf.len() as u64);
908 }
909
910 #[test]
911 fn encode_header_target_block_size_matches_default() {
912 let mzml = MzML::default();
913 let mut buf = Vec::new();
914 write_mzml_to_ion(
915 &mzml,
916 &WriteOptions {
917 compression_level: 0,
918 force_f32: false,
919 ..Default::default()
920 },
921 &mut buf,
922 )
923 .unwrap();
924 let size = u64::from_le_bytes(
925 buf[HEADER_TARGET_BLOCK_SIZE..HEADER_TARGET_BLOCK_SIZE + 8]
926 .try_into()
927 .unwrap(),
928 );
929 assert_eq!(size, TARGET_BLOCK_UNCOMPRESSED_BYTES as u64);
930 }
931
932 #[test]
933 fn encode_header_target_mz_window_matches_default() {
934 let mzml = MzML::default();
935 let mut buf = Vec::new();
936 write_mzml_to_ion(
937 &mzml,
938 &WriteOptions {
939 compression_level: 0,
940 force_f32: false,
941 ..Default::default()
942 },
943 &mut buf,
944 )
945 .unwrap();
946 let window = u32::from_le_bytes(
947 buf[HEADER_TARGET_MZ_WINDOW..HEADER_TARGET_MZ_WINDOW + 4]
948 .try_into()
949 .unwrap(),
950 );
951 assert_eq!(window, DEFAULT_MZ_WINDOW.round() as u32);
952 }
953
954 #[test]
955 fn encode_header_crc32_is_valid() {
956 let mzml = MzML::default();
957 let mut buf = Vec::new();
958 write_mzml_to_ion(
959 &mzml,
960 &WriteOptions {
961 compression_level: 0,
962 force_f32: false,
963 ..Default::default()
964 },
965 &mut buf,
966 )
967 .unwrap();
968 let stored = u32::from_le_bytes(buf[1020..1024].try_into().unwrap());
969 assert_eq!(stored, crc32fast::hash(&buf[0..1020]));
970 }
971
972 #[test]
973 fn encode_header_block_counts_are_zero_for_empty_mzml() {
974 let mzml = MzML::default();
975 let mut buf = Vec::new();
976 write_mzml_to_ion(
977 &mzml,
978 &WriteOptions {
979 compression_level: 0,
980 force_f32: false,
981 ..Default::default()
982 },
983 &mut buf,
984 )
985 .unwrap();
986 let spec_blocks = u64::from_le_bytes(
987 buf[HEADER_SPECTRUM_BLOCK_COUNT..HEADER_SPECTRUM_BLOCK_COUNT + 8]
988 .try_into()
989 .unwrap(),
990 );
991 let chrom_blocks = u64::from_le_bytes(
992 buf[HEADER_CHROM_BLOCK_COUNT..HEADER_CHROM_BLOCK_COUNT + 8]
993 .try_into()
994 .unwrap(),
995 );
996 assert_eq!(spec_blocks, 0);
997 assert_eq!(chrom_blocks, 0);
998 }
999
1000 #[test]
1001 fn vec_patch_in_bounds() {
1002 let mut output = vec![0u8; 16];
1003 output.patch(4, &[1u8, 2, 3, 4]).unwrap();
1004 assert_eq!(&output[4..8], &[1u8, 2, 3, 4]);
1005 }
1006
1007 #[test]
1008 fn vec_patch_out_of_bounds_errors() {
1009 let mut output = vec![0u8; 4];
1010 assert!(output.patch(3, &[1u8, 2, 3]).is_err());
1011 }
1012
1013 #[test]
1014 fn declared_float_precision_prefers_numeric_type_field() {
1015 let bda = BinaryDataArray {
1016 numeric_type: Some(NumericType::Float64),
1017 ..Default::default()
1018 };
1019 assert_eq!(declared_float_precision_is_64bit(&bda), Some(true));
1020 let bda = BinaryDataArray {
1021 numeric_type: Some(NumericType::Float32),
1022 ..Default::default()
1023 };
1024 assert_eq!(declared_float_precision_is_64bit(&bda), Some(false));
1025 }
1026
1027 #[test]
1028 fn resolve_array_dtype_force_f32_overrides_f64_data() {
1029 let bda = BinaryDataArray::default();
1030 assert_eq!(
1031 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1032 FILE_DTYPE_F32
1033 );
1034 }
1035
1036 #[test]
1037 fn resolve_array_dtype_keeps_f64_data_lossless_even_when_declared_32bit() {
1038 let bda = BinaryDataArray {
1039 numeric_type: Some(NumericType::Float32),
1040 ..Default::default()
1041 };
1042 assert_eq!(
1043 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), false),
1044 FILE_DTYPE_F64
1045 );
1046 assert_eq!(
1047 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1048 FILE_DTYPE_F32
1049 );
1050 }
1051
1052 #[test]
1053 fn resolve_array_dtype_integer_types_unchanged_by_force_f32() {
1054 let bda = BinaryDataArray::default();
1055 assert_eq!(
1056 resolve_array_dtype(&bda, ArrayData::I32(&[1i32]), true),
1057 FILE_DTYPE_I32
1058 );
1059 }
1060
1061 #[test]
1062 fn element_byte_size_for_dtype_returns_correct_sizes() {
1063 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F64), 8);
1064 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F32), 4);
1065 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F16), 2);
1066 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I16), 2);
1067 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I32), 4);
1068 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I64), 8);
1069 }
1070
1071 #[test]
1072 fn encoder_config_compression_disabled_at_level_zero() {
1073 let config = WriteOptions {
1074 compression_level: 0,
1075 force_f32: false,
1076 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1077 parallel: true,
1078 section_storage: SectionStorage::Memory,
1079 mz_window: 0.0,
1080 };
1081 assert!(!config.compression_is_enabled());
1082 assert_eq!(config.codec_id(), 0);
1083 assert_eq!(config.array_filter_id(), PackingId::Raw as u8);
1084 assert!(matches!(config.block_packing_id(), PackingId::Raw));
1085 }
1086
1087 #[test]
1088 fn encoder_config_compression_enabled_at_nonzero_level() {
1089 let config = WriteOptions {
1090 compression_level: 3,
1091 force_f32: false,
1092 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1093 parallel: true,
1094 section_storage: SectionStorage::Memory,
1095 mz_window: 0.0,
1096 };
1097 assert!(config.compression_is_enabled());
1098 assert_eq!(config.codec_id(), 1);
1099 assert_eq!(config.array_filter_id(), PackingId::ByteShuffle as u8);
1100 assert!(matches!(config.block_packing_id(), PackingId::ByteShuffle));
1101 }
1102}