1use crate::{
2 accessions::{
3 FLOAT_32BIT, FLOAT_64BIT, HIGHEST_ION_MOBILITY, HIGHEST_OBSERVED_MZ,
4 HIGHEST_OBSERVED_WAVELENGTH, INTENSITY_ARRAY, LOWEST_ION_MOBILITY, LOWEST_OBSERVED_MZ,
5 LOWEST_OBSERVED_WAVELENGTH, NEGATIVE_SCAN, POSITION_X, POSITION_Y, POSITION_Z,
6 POSITIVE_SCAN,
7 },
8 ion::{
9 IonError, IonResult,
10 encoder::utilities::{
11 BlockWriter, CompressionMode, DefaultCompressor,
12 le_writers::{
13 write_f32_le, write_f32_slice_le, write_f64_le, write_f64_slice_le,
14 write_i16_slice_le, write_i32_slice_le, write_i64_slice_le, write_u16_slice_le,
15 },
16 meta_collector::{
17 ArrayPolicy, array_type_accession_from_binary_data_array,
18 array_type_cv_code_from_binary_data_array, parse_accession_tail_raw,
19 },
20 output::{SectionStorage, WriteBytes},
21 },
22 filter_summary::{ChromatogramSummary, SpectrumSummary},
23 packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING},
24 utilities::spectrum_source::{f16_bits_to_f64, summary_from_spectrum},
25 windowing::{WindowRange, Windowing},
26 },
27 mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum},
28};
29pub(crate) const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024;
30pub const DEFAULT_MZ_WINDOW: f64 = 20.0;
31pub(crate) const SPEC_SUMMARY_SIZE: usize = 80;
32pub(crate) const CHROM_SUMMARY_SIZE: usize = 80;
33
34pub(crate) const FILE_DTYPE_F64: u8 = Dtype::F64 as u8;
35pub(crate) const FILE_DTYPE_F32: u8 = Dtype::F32 as u8;
36pub(crate) const FILE_DTYPE_F16: u8 = Dtype::F16 as u8;
37pub(crate) const FILE_DTYPE_I16: u8 = Dtype::I16 as u8;
38pub(crate) const FILE_DTYPE_I32: u8 = Dtype::I32 as u8;
39pub(crate) const FILE_DTYPE_I64: u8 = Dtype::I64 as u8;
40
41const POLARITY_UNKNOWN: u8 = 0;
42const POLARITY_POSITIVE: u8 = 1;
43const POLARITY_NEGATIVE: u8 = 2;
44
45impl ChromatogramSummary {
46 fn unknown() -> Self {
47 Self {
48 lowest_mz: f64::NAN,
49 highest_mz: f64::NAN,
50 lowest_wavelength: f64::NAN,
51 highest_wavelength: f64::NAN,
52 lowest_ion_mobility: f64::NAN,
53 highest_ion_mobility: f64::NAN,
54 polarity: POLARITY_UNKNOWN,
55 }
56 }
57}
58
59pub(crate) fn spec_summary_from_spectrum(spectrum: &Spectrum) -> SpectrumSummary {
60 let summary = summary_from_spectrum(spectrum);
61 let (position_x, position_y, position_z) = get_spectrum_position(spectrum);
62 SpectrumSummary {
63 rt: summary.rt,
64 rt_unit: summary.rt_unit.code(),
65 base_peak_mz: summary.base_peak_mz,
66 selected_ion_mz: summary.selected_ion_mz,
67 base_peak_int: summary.base_peak_int,
68 total_ion_current: summary.total_ion_current,
69 ms_level: summary.ms_level,
70 polarity: summary.polarity,
71 position_x,
72 position_y,
73 position_z,
74 }
75}
76
77fn get_spectrum_position(spectrum: &Spectrum) -> (u32, u32, u32) {
78 let mut x = 0;
79 let mut y = 0;
80 let mut z = 0;
81 let scan_list = spectrum.scan_list.as_ref().or_else(|| {
82 spectrum
83 .spectrum_description
84 .as_ref()
85 .and_then(|description| description.scan_list.as_ref())
86 });
87 let Some(scan_list) = scan_list else {
88 return (x, y, z);
89 };
90 for scan in &scan_list.scans {
91 for param in &scan.cv_params {
92 let Some(accession) = param.accession.as_deref() else {
93 continue;
94 };
95 if !accession.starts_with("IMS:") {
96 continue;
97 }
98 let value = param
99 .value
100 .as_deref()
101 .and_then(|value| value.parse::<u32>().ok())
102 .unwrap_or(0);
103 match parse_accession_tail_raw(Some(accession)) {
104 POSITION_X => x = value,
105 POSITION_Y => y = value,
106 POSITION_Z => z = value,
107 _ => {}
108 }
109 }
110 }
111 (x, y, z)
112}
113
114#[inline]
115fn cv_f64(cv: &CvParam) -> Option<f64> {
116 cv.value.as_deref().and_then(|s| s.parse::<f64>().ok())
117}
118
119pub(crate) fn extract_chrom_summary(chrom: &Chromatogram) -> ChromatogramSummary {
120 let mut summary = ChromatogramSummary::unknown();
121 for cv in &chrom.cv_params {
122 match parse_accession_tail_raw(cv.accession.as_deref()) {
123 LOWEST_OBSERVED_MZ => {
124 if let Some(v) = cv_f64(cv) {
125 summary.lowest_mz = v;
126 }
127 }
128 HIGHEST_OBSERVED_MZ => {
129 if let Some(v) = cv_f64(cv) {
130 summary.highest_mz = v;
131 }
132 }
133 LOWEST_OBSERVED_WAVELENGTH => {
134 if let Some(v) = cv_f64(cv) {
135 summary.lowest_wavelength = v;
136 }
137 }
138 HIGHEST_OBSERVED_WAVELENGTH => {
139 if let Some(v) = cv_f64(cv) {
140 summary.highest_wavelength = v;
141 }
142 }
143 LOWEST_ION_MOBILITY => {
144 if let Some(v) = cv_f64(cv) {
145 summary.lowest_ion_mobility = v;
146 }
147 }
148 HIGHEST_ION_MOBILITY => {
149 if let Some(v) = cv_f64(cv) {
150 summary.highest_ion_mobility = v;
151 }
152 }
153 POSITIVE_SCAN => summary.polarity = POLARITY_POSITIVE,
154 NEGATIVE_SCAN => summary.polarity = POLARITY_NEGATIVE,
155 _ => {}
156 }
157 }
158 summary
159}
160
161pub(crate) fn get_supported_compression_level(compression_level: u8) -> u8 {
162 match compression_level {
163 0..=5 => 1,
164 6..=10 => 9,
165 11..=16 => 12,
166 _ => 22,
167 }
168}
169
170pub(crate) fn allow_compression_level(compression_level: u8) -> IonResult<()> {
171 if compression_level > 22 {
172 return Err(format!("compression_level must be 0-22, got {compression_level}").into());
173 }
174 Ok(())
175}
176
177#[derive(Debug, Clone, Copy)]
178pub struct WriteOptions {
179 pub compression_level: u8,
180 pub force_f32: bool,
181 pub block_size: usize,
182 pub parallel: bool,
183 pub section_storage: SectionStorage,
184 pub mz_window: f64,
185}
186
187impl Default for WriteOptions {
188 fn default() -> Self {
189 Self {
190 compression_level: 12,
191 force_f32: false,
192 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
193 parallel: true,
194 section_storage: SectionStorage::Disk,
195 mz_window: DEFAULT_MZ_WINDOW,
196 }
197 }
198}
199
200impl WriteOptions {
201 pub(crate) fn compression_is_enabled(self) -> bool {
202 self.compression_level != 0
203 }
204
205 pub(crate) fn codec_id(self) -> u8 {
206 self.compression_is_enabled() as u8
207 }
208
209 fn block_shuffle_is_enabled(self) -> bool {
210 self.compression_is_enabled()
211 }
212
213 pub(crate) fn array_filter_id(self) -> u8 {
214 if self.block_shuffle_is_enabled() {
215 PackingId::ByteShuffle as u8
216 } else {
217 PackingId::Raw as u8
218 }
219 }
220
221 pub(crate) fn compression_mode(self) -> IonResult<CompressionMode<DefaultCompressor>> {
222 if self.compression_is_enabled() {
223 Ok(CompressionMode::Compressed(DefaultCompressor::new(
224 self.compression_level as i32,
225 )?))
226 } else {
227 Ok(CompressionMode::Raw)
228 }
229 }
230
231 pub(crate) fn block_packing_id(self) -> PackingId {
232 if self.block_shuffle_is_enabled() {
233 PackingId::ByteShuffle
234 } else {
235 PackingId::Raw
236 }
237 }
238
239 pub(crate) fn array_policy(self, x_array_accession: u32) -> ArrayPolicy {
240 ArrayPolicy {
241 x_array_accession,
242 y_array_accession: INTENSITY_ARRAY,
243 force_f32: self.force_f32,
244 }
245 }
246}
247
248#[derive(Copy, Clone)]
249enum ArrayData<'a> {
250 F16(&'a [u16]),
251 F32(&'a [f32]),
252 F64(&'a [f64]),
253 I16(&'a [i16]),
254 I32(&'a [i32]),
255 I64(&'a [i64]),
256}
257
258impl<'a> ArrayData<'a> {
259 fn element_count(self) -> usize {
260 match self {
261 Self::F16(e) => e.len(),
262 Self::F32(e) => e.len(),
263 Self::F64(e) => e.len(),
264 Self::I16(e) => e.len(),
265 Self::I32(e) => e.len(),
266 Self::I64(e) => e.len(),
267 }
268 }
269
270 fn is_empty(self) -> bool {
271 self.element_count() == 0
272 }
273
274 fn slice(self, start: usize, end: usize) -> ArrayData<'a> {
275 match self {
276 Self::F16(e) => Self::F16(&e[start..end]),
277 Self::F32(e) => Self::F32(&e[start..end]),
278 Self::F64(e) => Self::F64(&e[start..end]),
279 Self::I16(e) => Self::I16(&e[start..end]),
280 Self::I32(e) => Self::I32(&e[start..end]),
281 Self::I64(e) => Self::I64(&e[start..end]),
282 }
283 }
284
285 fn value_at(self, index: usize) -> f64 {
286 match self {
287 Self::F16(e) => f16_bits_to_f64(e[index]),
288 Self::F32(e) => e[index] as f64,
289 Self::F64(e) => e[index],
290 Self::I16(e) => e[index] as f64,
291 Self::I32(e) => e[index] as f64,
292 Self::I64(e) => e[index] as f64,
293 }
294 }
295
296 fn is_monotonic_non_decreasing(self) -> bool {
297 let count = self.element_count();
298 if count < 2 {
299 return true;
300 }
301 let mut previous = self.value_at(0);
302 for index in 1..count {
303 let current = self.value_at(index);
304 if current < previous {
305 return false;
306 }
307 previous = current;
308 }
309 true
310 }
311}
312
313fn array_data_from_binary_data_array(bda: &BinaryDataArray) -> Option<ArrayData<'_>> {
314 match bda.binary.as_ref()? {
315 NumericArray::F16(e) => Some(ArrayData::F16(e)),
316 NumericArray::I16(e) => Some(ArrayData::I16(e)),
317 NumericArray::I32(e) => Some(ArrayData::I32(e)),
318 NumericArray::I64(e) => Some(ArrayData::I64(e)),
319 NumericArray::F32(e) => Some(ArrayData::F32(e)),
320 NumericArray::F64(e) => Some(ArrayData::F64(e)),
321 }
322}
323
324fn element_byte_size_for_dtype(dtype: u8) -> usize {
325 match dtype {
326 FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
327 FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
328 FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
329 _ => 1,
330 }
331}
332
333fn resolve_array_dtype(bda: &BinaryDataArray, data: ArrayData<'_>, force_f32: bool) -> u8 {
334 match data {
335 ArrayData::F16(_) => FILE_DTYPE_F16,
336 ArrayData::I16(_) => FILE_DTYPE_I16,
337 ArrayData::I32(_) => FILE_DTYPE_I32,
338 ArrayData::I64(_) => FILE_DTYPE_I64,
339 ArrayData::F32(_) | ArrayData::F64(_) => {
340 if float_data_should_be_written_as_f64(bda, data, force_f32) {
341 FILE_DTYPE_F64
342 } else {
343 FILE_DTYPE_F32
344 }
345 }
346 }
347}
348
349fn float_data_should_be_written_as_f64(
350 bda: &BinaryDataArray,
351 data: ArrayData<'_>,
352 force_f32: bool,
353) -> bool {
354 if force_f32 {
355 return false;
356 }
357 if matches!(data, ArrayData::F64(_)) {
358 return true;
359 }
360 declared_float_precision_is_64bit(bda).unwrap_or(false)
361}
362
363fn declared_float_precision_is_64bit(bda: &BinaryDataArray) -> Option<bool> {
364 if let Some(nt) = bda.numeric_type.as_ref() {
365 return match nt {
366 NumericType::Float64 => Some(true),
367 NumericType::Float32 => Some(false),
368 _ => None,
369 };
370 }
371 let (mut saw32, mut saw64) = (false, false);
372 for cv in &bda.cv_params {
373 match parse_accession_tail_raw(cv.accession.as_deref()) {
374 FLOAT_32BIT => saw32 = true,
375 FLOAT_64BIT => saw64 = true,
376 _ => {}
377 }
378 if saw32 && saw64 {
379 break;
380 }
381 }
382 match (saw32, saw64) {
383 (true, false) => Some(false),
384 (false, true) => Some(true),
385 _ => None,
386 }
387}
388
389fn validate_array_dtype(data: ArrayData<'_>, dtype: u8) -> IonResult<()> {
390 let ok = matches!(
391 (dtype, data),
392 (FILE_DTYPE_F16, ArrayData::F16(_))
393 | (FILE_DTYPE_F32, ArrayData::F32(_))
394 | (FILE_DTYPE_F32, ArrayData::F64(_))
395 | (FILE_DTYPE_F64, ArrayData::F64(_))
396 | (FILE_DTYPE_F64, ArrayData::F32(_))
397 | (FILE_DTYPE_I16, ArrayData::I16(_))
398 | (FILE_DTYPE_I32, ArrayData::I32(_))
399 | (FILE_DTYPE_I64, ArrayData::I64(_))
400 );
401 if ok {
402 Ok(())
403 } else {
404 Err(format!(
405 "write_array_data: incompatible dtype {dtype} for the given array data variant"
406 )
407 .into())
408 }
409}
410
411fn write_array_data(buf: &mut Vec<u8>, data: ArrayData<'_>, dtype: u8) {
412 match (dtype, data) {
413 (FILE_DTYPE_F16, ArrayData::F16(e)) => write_u16_slice_le(buf, e),
414 (FILE_DTYPE_F32, ArrayData::F32(e)) => write_f32_slice_le(buf, e),
415 (FILE_DTYPE_F32, ArrayData::F64(e)) => {
416 for &v in e {
417 write_f32_le(buf, v as f32);
418 }
419 }
420 (FILE_DTYPE_F64, ArrayData::F64(e)) => write_f64_slice_le(buf, e),
421 (FILE_DTYPE_F64, ArrayData::F32(e)) => {
422 for &v in e {
423 write_f64_le(buf, v as f64);
424 }
425 }
426 (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e),
427 (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e),
428 (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e),
429 _ => unreachable!("write_array_data called with unvalidated dtype/data combination"),
430 }
431}
432
433fn narrow_f64_to_f32(values: &[f64]) -> Vec<f32> {
434 values.iter().map(|&value| value as f32).collect()
435}
436
437fn widen_f32_to_f64(values: &[f32]) -> Vec<f64> {
438 values.iter().map(|&value| value as f64).collect()
439}
440
441fn encode_floats_through_packing(
442 buf: &mut Vec<u8>,
443 data: ArrayData<'_>,
444 dtype_enum: Dtype,
445 packing: &'static dyn Packing,
446) -> IonResult<()> {
447 match (dtype_enum, data) {
448 (Dtype::F64, ArrayData::F64(slice)) => packing.encode(PackingInput::F64(slice), buf),
449 (Dtype::F32, ArrayData::F32(slice)) => packing.encode(PackingInput::F32(slice), buf),
450 (Dtype::F32, ArrayData::F64(slice)) => {
451 let narrowed = narrow_f64_to_f32(slice);
452 packing.encode(PackingInput::F32(&narrowed), buf)
453 }
454 (Dtype::F64, ArrayData::F32(slice)) => {
455 let widened = widen_f32_to_f64(slice);
456 packing.encode(PackingInput::F64(&widened), buf)
457 }
458 _ => Err(IonError::from(
459 "shuffle packing requires f32 or f64 array data",
460 )),
461 }
462}
463
464pub(crate) struct EncodedArrayAddress {
465 pub(crate) element_offset: u64,
466 pub(crate) element_count: u64,
467 pub(crate) block_id: u32,
468 pub(crate) accession: u32,
469 pub(crate) dtype: u8,
470 pub(crate) array_filter: u8,
471 pub(crate) encoded_len: u32,
472 pub(crate) continues_previous_segment: u8,
473 pub(crate) array_cv_code: u8,
474}
475
476fn encode_variable_length_array(
477 output: &mut dyn WriteBytes,
478 array_type: u32,
479 data: ArrayData<'_>,
480 dtype: u8,
481 dtype_enum: Dtype,
482 packing: &'static dyn Packing,
483 container: &mut BlockWriter<DefaultCompressor>,
484) -> IonResult<(u32, u64, u32)> {
485 let mut encoded = Vec::new();
486 match packing.id() {
487 PackingId::Raw => write_array_data(&mut encoded, data, dtype),
488 _ => encode_floats_through_packing(&mut encoded, data, dtype_enum, packing)?,
489 }
490 let enc_len =
491 u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?;
492 let (bid, eoff) = container.add_item_to_box(output, array_type, encoded.len(), 1, |buf| {
493 buf.extend_from_slice(&encoded);
494 Ok(())
495 })?;
496 Ok((bid, eoff, enc_len))
497}
498
499fn write_fixed_array_payload(
500 buf: &mut Vec<u8>,
501 data: ArrayData<'_>,
502 dtype: u8,
503 dtype_enum: Dtype,
504 packing: &'static dyn Packing,
505) -> IonResult<()> {
506 match packing.id() {
507 PackingId::Raw => {
508 write_array_data(buf, data, dtype);
509 Ok(())
510 }
511 _ => encode_floats_through_packing(buf, data, dtype_enum, packing),
512 }
513}
514
515#[allow(clippy::too_many_arguments)]
516fn encode_fixed_length_array(
517 output: &mut dyn WriteBytes,
518 array_type: u32,
519 data: ArrayData<'_>,
520 dtype: u8,
521 dtype_enum: Dtype,
522 elem_bytes: usize,
523 packing: &'static dyn Packing,
524 container: &mut BlockWriter<DefaultCompressor>,
525) -> IonResult<(u32, u64, u32)> {
526 let (bid, eoff) = container.add_item_to_box(
527 output,
528 array_type,
529 data.element_count() * elem_bytes,
530 elem_bytes,
531 |buf| write_fixed_array_payload(buf, data, dtype, dtype_enum, packing),
532 )?;
533 Ok((bid, eoff, 0u32))
534}
535
536struct ArrayEncoding<'a> {
537 data: ArrayData<'a>,
538 accession: u32,
539 array_cv_code: u8,
540 dtype: u8,
541 dtype_enum: Dtype,
542 elem_bytes: usize,
543 packing: &'static dyn Packing,
544}
545
546fn resolve_array_encoding<'a>(
547 bda: &'a BinaryDataArray,
548 config: WriteOptions,
549 policy: ArrayPolicy,
550) -> IonResult<Option<ArrayEncoding<'a>>> {
551 let Some(data) = array_data_from_binary_data_array(bda) else {
552 return Ok(None);
553 };
554 if data.is_empty() {
555 return Ok(None);
556 }
557 let accession = array_type_accession_from_binary_data_array(bda);
558 let array_cv_code = array_type_cv_code_from_binary_data_array(bda);
559 let dtype = resolve_array_dtype(bda, data, policy.should_force_f32(accession));
560 validate_array_dtype(data, dtype)?;
561 let elem_bytes = element_byte_size_for_dtype(dtype);
562 let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64);
563 let requested: &'static dyn Packing = if config.compression_is_enabled() {
564 packing_for(accession, dtype_enum)
565 } else {
566 &RAW_PACKING
567 };
568 let packing: &'static dyn Packing = if requested.supports(dtype_enum) {
569 requested
570 } else {
571 &RAW_PACKING
572 };
573 Ok(Some(ArrayEncoding {
574 data,
575 accession,
576 array_cv_code,
577 dtype,
578 dtype_enum,
579 elem_bytes,
580 packing,
581 }))
582}
583
584pub(crate) fn encode_single_array(
585 output: &mut dyn WriteBytes,
586 bda: &BinaryDataArray,
587 config: WriteOptions,
588 policy: ArrayPolicy,
589 container: &mut BlockWriter<DefaultCompressor>,
590) -> IonResult<Option<EncodedArrayAddress>> {
591 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
592 return Ok(None);
593 };
594 let ArrayEncoding {
595 data,
596 accession,
597 array_cv_code,
598 dtype,
599 dtype_enum,
600 elem_bytes,
601 packing,
602 } = encoding;
603 let (block_id, element_offset, encoded_len) = if packing.is_variable_length() {
604 encode_variable_length_array(output, accession, data, dtype, dtype_enum, packing, container)?
605 } else {
606 encode_fixed_length_array(
607 output, accession, data, dtype, dtype_enum, elem_bytes, packing, container,
608 )?
609 };
610 Ok(Some(EncodedArrayAddress {
611 element_offset,
612 element_count: data.element_count() as u64,
613 block_id,
614 accession,
615 dtype,
616 array_filter: packing.id() as u8,
617 encoded_len,
618 continues_previous_segment: 0,
619 array_cv_code,
620 }))
621}
622
623pub(crate) fn array_is_fixed_width_splittable(
624 bda: &BinaryDataArray,
625 config: WriteOptions,
626 policy: ArrayPolicy,
627) -> IonResult<Option<(usize, usize)>> {
628 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
629 return Ok(None);
630 };
631 if encoding.packing.is_variable_length() {
632 return Ok(None);
633 }
634 Ok(Some((encoding.data.element_count(), encoding.elem_bytes)))
635}
636
637pub(crate) fn check_spectrum_rt_order(
638 summary: &[u8],
639 spectrum_index: usize,
640 last_rt: &mut f64,
641) -> IonResult<()> {
642 let rt = f64::from_le_bytes(summary[0..8].try_into().unwrap());
643 if !rt.is_finite() {
644 return Ok(());
645 }
646 let previous = *last_rt;
647 if rt < previous {
648 return Err(format!(
649 "spectrum {spectrum_index}: retention time {rt} is below the previous spectrum's \
650 ({previous}); spectra must be stored in ascending rt order"
651 )
652 .into());
653 }
654 *last_rt = rt;
655 Ok(())
656}
657
658pub(crate) fn check_spectrum_mz_order(
659 arrays: &[BinaryDataArray],
660 spectrum_index: usize,
661) -> IonResult<()> {
662 use crate::accessions::MZ_ARRAY;
663 for bda in arrays {
664 if array_type_accession_from_binary_data_array(bda) != MZ_ARRAY {
665 continue;
666 }
667 let Some(data) = array_data_from_binary_data_array(bda) else {
668 return Ok(());
669 };
670 if !data.is_monotonic_non_decreasing() {
671 return Err(format!(
672 "spectrum {spectrum_index}: m/z array must be sorted ascending (non-decreasing) for range reads"
673 )
674 .into());
675 }
676 return Ok(());
677 }
678 Ok(())
679}
680
681pub(crate) fn window_ranges_for_item(
682 arrays: &[BinaryDataArray],
683 config: WriteOptions,
684 policy: ArrayPolicy,
685 mz_window: f64,
686) -> IonResult<Option<Vec<WindowRange>>> {
687 let windowing = Windowing::new(mz_window.round());
688 if !windowing.is_enabled() {
689 return Ok(None);
690 }
691 let Some(x_array) = arrays
692 .iter()
693 .find(|bda| array_type_accession_from_binary_data_array(bda) == policy.x_array_accession)
694 else {
695 return Ok(None);
696 };
697 let Some((x_count, _)) = array_is_fixed_width_splittable(x_array, config, policy)? else {
698 return Ok(None);
699 };
700 if x_count == 0 {
701 return Ok(None);
702 }
703 for bda in arrays {
704 match array_is_fixed_width_splittable(bda, config, policy)? {
705 Some((count, _)) if count == x_count => {}
706 _ => return Ok(None),
707 }
708 }
709 let Some(x_data) = array_data_from_binary_data_array(x_array) else {
710 return Ok(None);
711 };
712 let ranges = windowing.split_sorted(x_count, |index| x_data.value_at(index));
713 const MAX_WINDOW_INDEX: u32 = 1 << 20;
714 if ranges
715 .last()
716 .is_some_and(|range| range.window_index > MAX_WINDOW_INDEX)
717 {
718 return Err(format!(
719 "m/z window index exceeds limit {MAX_WINDOW_INDEX}; m/z value too large for window width {}",
720 mz_window.round()
721 )
722 .into());
723 }
724 Ok(Some(ranges))
725}
726
727pub(crate) fn write_array_windows(
728 output: &mut dyn WriteBytes,
729 bda: &BinaryDataArray,
730 config: WriteOptions,
731 policy: ArrayPolicy,
732 windows: &[WindowRange],
733 container: &mut BlockWriter<DefaultCompressor>,
734) -> IonResult<Vec<EncodedArrayAddress>> {
735 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
736 return Ok(Vec::new());
737 };
738 let ArrayEncoding {
739 data,
740 accession,
741 array_cv_code,
742 dtype,
743 dtype_enum,
744 elem_bytes,
745 packing,
746 } = encoding;
747
748 if packing.is_variable_length() {
749 return Err("write_array_windows: variable-length arrays cannot be windowed".into());
750 }
751
752 let mut addresses = Vec::with_capacity(windows.len());
753 for (segment_index, window) in windows.iter().enumerate() {
754 let segment_data = data.slice(window.start, window.end);
755 let element_count = window.element_count();
756 let (block_id, element_offset) = container.add_item_to_window(
757 output,
758 accession,
759 window.window_index,
760 element_count * elem_bytes,
761 elem_bytes,
762 |buf| write_fixed_array_payload(buf, segment_data, dtype, dtype_enum, packing),
763 )?;
764 addresses.push(EncodedArrayAddress {
765 element_offset,
766 element_count: element_count as u64,
767 block_id,
768 accession,
769 dtype,
770 array_filter: packing.id() as u8,
771 encoded_len: 0,
772 continues_previous_segment: (segment_index > 0) as u8,
773 array_cv_code,
774 });
775 }
776 Ok(addresses)
777}
778
779#[cfg(test)]
780mod tests {
781 use super::*;
782 use crate::{
783 ion::{
784 encoder::{ion_writer::write_mzml_to_ion, utilities::output::WriteBytes},
785 format::{FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE},
786 header::{
787 HEADER_CHROM_BLOCK_COUNT, HEADER_SPECTRUM_BLOCK_COUNT, HEADER_TARGET_BLOCK_SIZE,
788 HEADER_TARGET_MZ_WINDOW, HEADER_TOTAL_FILE_SIZE,
789 },
790 },
791 mzml::structs::MzML,
792 };
793
794 fn summary_with_rt(rt: f64) -> [u8; SPEC_SUMMARY_SIZE] {
795 let mut summary = [0u8; SPEC_SUMMARY_SIZE];
796 summary[0..8].copy_from_slice(&rt.to_le_bytes());
797 summary
798 }
799
800 #[test]
801 fn byte_shuffle_array_encode_arm_shuffles_not_raw_71() {
802 use crate::ion::packing::packing_by_id;
803
804 let data: Vec<f64> = (0..600)
805 .map(|i| 100.0 + (i as f64) * 0.0137 + ((i * 7 % 13) as f64) * 0.0009)
806 .collect();
807
808 let byte_shuffle = packing_by_id(PackingId::ByteShuffle);
809
810 let mut shuffled = Vec::new();
811 write_fixed_array_payload(
812 &mut shuffled,
813 ArrayData::F64(&data),
814 FILE_DTYPE_F64,
815 Dtype::F64,
816 byte_shuffle,
817 )
818 .unwrap();
819
820 let mut raw = Vec::new();
821 write_array_data(&mut raw, ArrayData::F64(&data), FILE_DTYPE_F64);
822
823 assert_eq!(shuffled.len(), raw.len());
824 assert_ne!(
825 shuffled, raw,
826 "ByteShuffle encode arm must shuffle, not write raw bytes"
827 );
828
829 let mut expected = Vec::new();
830 byte_shuffle
831 .encode(PackingInput::F64(&data), &mut expected)
832 .unwrap();
833 assert_eq!(
834 shuffled, expected,
835 "ByteShuffle encode arm must produce exactly the SIMD byte-transpose output"
836 );
837 }
838
839 #[test]
840 fn rt_order_allows_ascending_and_missing_but_rejects_a_drop() {
841 let mut last_rt = f64::NEG_INFINITY;
842 check_spectrum_rt_order(&summary_with_rt(1.0), 0, &mut last_rt)
843 .expect("first rt is allowed");
844 check_spectrum_rt_order(&summary_with_rt(1.0), 1, &mut last_rt)
845 .expect("equal rt is allowed");
846 check_spectrum_rt_order(&summary_with_rt(2.0), 2, &mut last_rt)
847 .expect("rising rt is allowed");
848
849 check_spectrum_rt_order(&summary_with_rt(f64::NAN), 3, &mut last_rt)
850 .expect("missing rt is allowed");
851 assert_eq!(last_rt, 2.0, "missing rt must not move the last seen rt");
852
853 let dropped = check_spectrum_rt_order(&summary_with_rt(1.5), 4, &mut last_rt);
854 let message = dropped
855 .expect_err("rt below the previous spectrum must be rejected")
856 .to_string();
857 assert!(
858 message.contains("spectrum 4"),
859 "error names the spectrum: {message}"
860 );
861 assert!(
862 message.contains("ascending rt order"),
863 "error explains the rule: {message}"
864 );
865 }
866
867 #[test]
868 fn encode_starts_with_magic_and_ends_with_trailer() {
869 let mzml = MzML::default();
870 let mut buf = Vec::new();
871 write_mzml_to_ion(
872 &mzml,
873 &WriteOptions {
874 compression_level: 0,
875 force_f32: false,
876 ..Default::default()
877 },
878 &mut buf,
879 )
880 .unwrap();
881 assert!(buf.len() >= HEADER_SIZE + FILE_TRAILER.len());
882 assert_eq!(&buf[..FILE_SIGNATURE.len()], &FILE_SIGNATURE);
883 assert_eq!(&buf[buf.len() - FILE_TRAILER.len()..], &FILE_TRAILER);
884 }
885
886 #[test]
887 fn encode_total_file_size_is_correct() {
888 let mzml = MzML::default();
889 let mut buf = Vec::new();
890 write_mzml_to_ion(
891 &mzml,
892 &WriteOptions {
893 compression_level: 0,
894 force_f32: false,
895 ..Default::default()
896 },
897 &mut buf,
898 )
899 .unwrap();
900 let total = u64::from_le_bytes(
901 buf[HEADER_TOTAL_FILE_SIZE..HEADER_TOTAL_FILE_SIZE + 8]
902 .try_into()
903 .unwrap(),
904 );
905 assert_eq!(total, buf.len() as u64);
906 }
907
908 #[test]
909 fn encode_header_target_block_size_matches_default() {
910 let mzml = MzML::default();
911 let mut buf = Vec::new();
912 write_mzml_to_ion(
913 &mzml,
914 &WriteOptions {
915 compression_level: 0,
916 force_f32: false,
917 ..Default::default()
918 },
919 &mut buf,
920 )
921 .unwrap();
922 let size = u64::from_le_bytes(
923 buf[HEADER_TARGET_BLOCK_SIZE..HEADER_TARGET_BLOCK_SIZE + 8]
924 .try_into()
925 .unwrap(),
926 );
927 assert_eq!(size, TARGET_BLOCK_UNCOMPRESSED_BYTES as u64);
928 }
929
930 #[test]
931 fn encode_header_target_mz_window_matches_default() {
932 let mzml = MzML::default();
933 let mut buf = Vec::new();
934 write_mzml_to_ion(
935 &mzml,
936 &WriteOptions {
937 compression_level: 0,
938 force_f32: false,
939 ..Default::default()
940 },
941 &mut buf,
942 )
943 .unwrap();
944 let window = u32::from_le_bytes(
945 buf[HEADER_TARGET_MZ_WINDOW..HEADER_TARGET_MZ_WINDOW + 4]
946 .try_into()
947 .unwrap(),
948 );
949 assert_eq!(window, DEFAULT_MZ_WINDOW.round() as u32);
950 }
951
952 #[test]
953 fn encode_header_crc32_is_valid() {
954 let mzml = MzML::default();
955 let mut buf = Vec::new();
956 write_mzml_to_ion(
957 &mzml,
958 &WriteOptions {
959 compression_level: 0,
960 force_f32: false,
961 ..Default::default()
962 },
963 &mut buf,
964 )
965 .unwrap();
966 let stored = u32::from_le_bytes(buf[1020..1024].try_into().unwrap());
967 assert_eq!(stored, crc32fast::hash(&buf[0..1020]));
968 }
969
970 #[test]
971 fn encode_header_block_counts_are_zero_for_empty_mzml() {
972 let mzml = MzML::default();
973 let mut buf = Vec::new();
974 write_mzml_to_ion(
975 &mzml,
976 &WriteOptions {
977 compression_level: 0,
978 force_f32: false,
979 ..Default::default()
980 },
981 &mut buf,
982 )
983 .unwrap();
984 let spec_blocks = u64::from_le_bytes(
985 buf[HEADER_SPECTRUM_BLOCK_COUNT..HEADER_SPECTRUM_BLOCK_COUNT + 8]
986 .try_into()
987 .unwrap(),
988 );
989 let chrom_blocks = u64::from_le_bytes(
990 buf[HEADER_CHROM_BLOCK_COUNT..HEADER_CHROM_BLOCK_COUNT + 8]
991 .try_into()
992 .unwrap(),
993 );
994 assert_eq!(spec_blocks, 0);
995 assert_eq!(chrom_blocks, 0);
996 }
997
998 #[test]
999 fn vec_patch_in_bounds() {
1000 let mut output = vec![0u8; 16];
1001 output.patch(4, &[1u8, 2, 3, 4]).unwrap();
1002 assert_eq!(&output[4..8], &[1u8, 2, 3, 4]);
1003 }
1004
1005 #[test]
1006 fn vec_patch_out_of_bounds_errors() {
1007 let mut output = vec![0u8; 4];
1008 assert!(output.patch(3, &[1u8, 2, 3]).is_err());
1009 }
1010
1011 #[test]
1012 fn declared_float_precision_prefers_numeric_type_field() {
1013 let bda = BinaryDataArray {
1014 numeric_type: Some(NumericType::Float64),
1015 ..Default::default()
1016 };
1017 assert_eq!(declared_float_precision_is_64bit(&bda), Some(true));
1018 let bda = BinaryDataArray {
1019 numeric_type: Some(NumericType::Float32),
1020 ..Default::default()
1021 };
1022 assert_eq!(declared_float_precision_is_64bit(&bda), Some(false));
1023 }
1024
1025 #[test]
1026 fn resolve_array_dtype_force_f32_overrides_f64_data() {
1027 let bda = BinaryDataArray::default();
1028 assert_eq!(
1029 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1030 FILE_DTYPE_F32
1031 );
1032 }
1033
1034 #[test]
1035 fn resolve_array_dtype_keeps_f64_data_lossless_even_when_declared_32bit() {
1036 let bda = BinaryDataArray {
1037 numeric_type: Some(NumericType::Float32),
1038 ..Default::default()
1039 };
1040 assert_eq!(
1041 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), false),
1042 FILE_DTYPE_F64
1043 );
1044 assert_eq!(
1045 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1046 FILE_DTYPE_F32
1047 );
1048 }
1049
1050 #[test]
1051 fn resolve_array_dtype_integer_types_unchanged_by_force_f32() {
1052 let bda = BinaryDataArray::default();
1053 assert_eq!(
1054 resolve_array_dtype(&bda, ArrayData::I32(&[1i32]), true),
1055 FILE_DTYPE_I32
1056 );
1057 }
1058
1059 #[test]
1060 fn element_byte_size_for_dtype_returns_correct_sizes() {
1061 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F64), 8);
1062 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F32), 4);
1063 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F16), 2);
1064 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I16), 2);
1065 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I32), 4);
1066 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I64), 8);
1067 }
1068
1069 #[test]
1070 fn encoder_config_compression_disabled_at_level_zero() {
1071 let config = WriteOptions {
1072 compression_level: 0,
1073 force_f32: false,
1074 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1075 parallel: true,
1076 section_storage: SectionStorage::Memory,
1077 mz_window: 0.0,
1078 };
1079 assert!(!config.compression_is_enabled());
1080 assert_eq!(config.codec_id(), 0);
1081 assert_eq!(config.array_filter_id(), PackingId::Raw as u8);
1082 assert!(matches!(config.block_packing_id(), PackingId::Raw));
1083 }
1084
1085 #[test]
1086 fn encoder_config_compression_enabled_at_nonzero_level() {
1087 let config = WriteOptions {
1088 compression_level: 3,
1089 force_f32: false,
1090 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1091 parallel: true,
1092 section_storage: SectionStorage::Memory,
1093 mz_window: 0.0,
1094 };
1095 assert!(config.compression_is_enabled());
1096 assert_eq!(config.codec_id(), 1);
1097 assert_eq!(config.array_filter_id(), PackingId::ByteShuffle as u8);
1098 assert!(matches!(config.block_packing_id(), PackingId::ByteShuffle));
1099 }
1100}