1use crate::{
2 accessions::{
3 FLOAT_32BIT, FLOAT_64BIT, HIGHEST_ION_MOBILITY, HIGHEST_OBSERVED_MZ,
4 HIGHEST_OBSERVED_WAVELENGTH, INTENSITY_ARRAY, LOWEST_ION_MOBILITY, LOWEST_OBSERVED_MZ,
5 LOWEST_OBSERVED_WAVELENGTH, NEGATIVE_SCAN, POSITION_X, POSITION_Y, POSITION_Z,
6 POSITIVE_SCAN,
7 },
8 ion::{
9 IonError, IonResult,
10 encoder::utilities::{
11 BlockWriter, CompressionMode, DefaultCompressor,
12 le_writers::{
13 write_f32_le, write_f32_slice_le, write_f64_le, write_f64_slice_le,
14 write_i16_slice_le, write_i32_slice_le, write_i64_slice_le, write_u16_slice_le,
15 },
16 meta_collector::{
17 ArrayPolicy, array_type_accession_from_binary_data_array,
18 array_type_cv_code_from_binary_data_array, parse_accession_tail_raw,
19 },
20 output::SectionStorage,
21 },
22 filter_summary::{ChromatogramSummary, SpectrumSummary},
23 packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING},
24 utilities::spectrum_source::{f16_bits_to_f64, summary_from_spectrum},
25 windowing::{WindowRange, Windowing},
26 },
27 mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum},
28};
29pub const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024;
30pub const DEFAULT_MZ_WINDOW: f64 = 20.0;
31pub(crate) const SPEC_SUMMARY_SIZE: usize = 80;
32pub(crate) const CHROM_SUMMARY_SIZE: usize = 80;
33
34pub(crate) const FILE_DTYPE_F64: u8 = Dtype::F64 as u8;
35pub(crate) const FILE_DTYPE_F32: u8 = Dtype::F32 as u8;
36pub(crate) const FILE_DTYPE_F16: u8 = Dtype::F16 as u8;
37pub(crate) const FILE_DTYPE_I16: u8 = Dtype::I16 as u8;
38pub(crate) const FILE_DTYPE_I32: u8 = Dtype::I32 as u8;
39pub(crate) const FILE_DTYPE_I64: u8 = Dtype::I64 as u8;
40
41const POLARITY_UNKNOWN: u8 = 0;
42const POLARITY_POSITIVE: u8 = 1;
43const POLARITY_NEGATIVE: u8 = 2;
44
45impl ChromatogramSummary {
46 fn unknown() -> Self {
47 Self {
48 lowest_mz: f64::NAN,
49 highest_mz: f64::NAN,
50 lowest_wavelength: f64::NAN,
51 highest_wavelength: f64::NAN,
52 lowest_ion_mobility: f64::NAN,
53 highest_ion_mobility: f64::NAN,
54 polarity: POLARITY_UNKNOWN,
55 }
56 }
57}
58
59pub(crate) fn spec_summary_from_spectrum(spectrum: &Spectrum) -> SpectrumSummary {
60 let summary = summary_from_spectrum(spectrum);
61 let (position_x, position_y, position_z) = get_spectrum_position(spectrum);
62 SpectrumSummary {
63 rt: summary.rt,
64 rt_unit: summary.rt_unit.code(),
65 base_peak_mz: summary.base_peak_mz,
66 selected_ion_mz: summary.selected_ion_mz,
67 base_peak_int: summary.base_peak_int,
68 total_ion_current: summary.total_ion_current,
69 ms_level: summary.ms_level,
70 polarity: summary.polarity,
71 position_x,
72 position_y,
73 position_z,
74 }
75}
76
77fn get_spectrum_position(spectrum: &Spectrum) -> (u32, u32, u32) {
78 let mut x = 0;
79 let mut y = 0;
80 let mut z = 0;
81 let scan_list = spectrum.scan_list.as_ref().or_else(|| {
82 spectrum
83 .spectrum_description
84 .as_ref()
85 .and_then(|description| description.scan_list.as_ref())
86 });
87 let Some(scan_list) = scan_list else {
88 return (x, y, z);
89 };
90 for scan in &scan_list.scans {
91 for param in &scan.cv_params {
92 let Some(accession) = param.accession.as_deref() else {
93 continue;
94 };
95 if !accession.starts_with("IMS:") {
96 continue;
97 }
98 let value = param
99 .value
100 .as_deref()
101 .and_then(|value| value.parse::<u32>().ok())
102 .unwrap_or(0);
103 match parse_accession_tail_raw(Some(accession)) {
104 POSITION_X => x = value,
105 POSITION_Y => y = value,
106 POSITION_Z => z = value,
107 _ => {}
108 }
109 }
110 }
111 (x, y, z)
112}
113
114#[inline]
115fn cv_f64(cv: &CvParam) -> Option<f64> {
116 cv.value.as_deref().and_then(|s| s.parse::<f64>().ok())
117}
118
119pub(crate) fn extract_chrom_summary(chrom: &Chromatogram) -> ChromatogramSummary {
120 let mut summary = ChromatogramSummary::unknown();
121 for cv in &chrom.cv_params {
122 match parse_accession_tail_raw(cv.accession.as_deref()) {
123 LOWEST_OBSERVED_MZ => {
124 if let Some(v) = cv_f64(cv) {
125 summary.lowest_mz = v;
126 }
127 }
128 HIGHEST_OBSERVED_MZ => {
129 if let Some(v) = cv_f64(cv) {
130 summary.highest_mz = v;
131 }
132 }
133 LOWEST_OBSERVED_WAVELENGTH => {
134 if let Some(v) = cv_f64(cv) {
135 summary.lowest_wavelength = v;
136 }
137 }
138 HIGHEST_OBSERVED_WAVELENGTH => {
139 if let Some(v) = cv_f64(cv) {
140 summary.highest_wavelength = v;
141 }
142 }
143 LOWEST_ION_MOBILITY => {
144 if let Some(v) = cv_f64(cv) {
145 summary.lowest_ion_mobility = v;
146 }
147 }
148 HIGHEST_ION_MOBILITY => {
149 if let Some(v) = cv_f64(cv) {
150 summary.highest_ion_mobility = v;
151 }
152 }
153 POSITIVE_SCAN => summary.polarity = POLARITY_POSITIVE,
154 NEGATIVE_SCAN => summary.polarity = POLARITY_NEGATIVE,
155 _ => {}
156 }
157 }
158 summary
159}
160
161pub(crate) fn get_supported_compression_level(compression_level: u8) -> u8 {
162 match compression_level {
163 0..=5 => 1,
164 6..=10 => 9,
165 11..=16 => 12,
166 _ => 22,
167 }
168}
169
170pub(crate) fn allow_compression_level(compression_level: u8) -> IonResult<()> {
171 if compression_level > 22 {
172 return Err(format!("compression_level must be 0-22, got {compression_level}").into());
173 }
174 Ok(())
175}
176
177#[derive(Debug, Clone, Copy)]
178pub struct WriteOptions {
179 pub compression_level: u8,
180 pub force_f32: bool,
181 pub block_size: usize,
182 pub parallel: bool,
183 pub section_storage: SectionStorage,
184 pub mz_window: f64,
185}
186
187impl Default for WriteOptions {
188 fn default() -> Self {
189 Self {
190 compression_level: 12,
191 force_f32: false,
192 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
193 parallel: true,
194 section_storage: SectionStorage::Disk,
195 mz_window: DEFAULT_MZ_WINDOW,
196 }
197 }
198}
199
200impl WriteOptions {
201 pub(crate) fn compression_is_enabled(self) -> bool {
202 self.compression_level != 0
203 }
204
205 pub(crate) fn codec_id(self) -> u8 {
206 self.compression_is_enabled() as u8
207 }
208
209 fn block_shuffle_is_enabled(self) -> bool {
210 if !self.compression_is_enabled() {
211 return false;
212 }
213 !matches!(
214 std::env::var("IONIC_BLOCK_SHUFFLE").as_deref(),
215 Ok("off") | Ok("raw") | Ok("0") | Ok("none")
216 )
217 }
218
219 pub(crate) fn array_filter_id(self) -> u8 {
220 if self.block_shuffle_is_enabled() {
221 PackingId::ByteShuffle as u8
222 } else {
223 PackingId::Raw as u8
224 }
225 }
226
227 pub(crate) fn compression_mode(self) -> IonResult<CompressionMode<DefaultCompressor>> {
228 if self.compression_is_enabled() {
229 Ok(CompressionMode::Compressed(DefaultCompressor::new(
230 self.compression_level as i32,
231 )?))
232 } else {
233 Ok(CompressionMode::Raw)
234 }
235 }
236
237 pub(crate) fn block_packing_id(self) -> PackingId {
238 if self.block_shuffle_is_enabled() {
239 PackingId::ByteShuffle
240 } else {
241 PackingId::Raw
242 }
243 }
244
245 pub(crate) fn array_policy(self, x_array_accession: u32) -> ArrayPolicy {
246 ArrayPolicy {
247 x_array_accession,
248 y_array_accession: INTENSITY_ARRAY,
249 force_f32: self.force_f32,
250 }
251 }
252
253 pub fn fast() -> Self {
254 Self {
255 compression_level: 3,
256 ..Default::default()
257 }
258 }
259
260 pub fn max_ratio() -> Self {
261 Self {
262 compression_level: 22,
263 ..Default::default()
264 }
265 }
266}
267
268#[derive(Copy, Clone)]
269enum ArrayData<'a> {
270 F16(&'a [u16]),
271 F32(&'a [f32]),
272 F64(&'a [f64]),
273 I16(&'a [i16]),
274 I32(&'a [i32]),
275 I64(&'a [i64]),
276}
277
278impl<'a> ArrayData<'a> {
279 fn element_count(self) -> usize {
280 match self {
281 Self::F16(e) => e.len(),
282 Self::F32(e) => e.len(),
283 Self::F64(e) => e.len(),
284 Self::I16(e) => e.len(),
285 Self::I32(e) => e.len(),
286 Self::I64(e) => e.len(),
287 }
288 }
289
290 fn is_empty(self) -> bool {
291 self.element_count() == 0
292 }
293
294 fn slice(self, start: usize, end: usize) -> ArrayData<'a> {
295 match self {
296 Self::F16(e) => Self::F16(&e[start..end]),
297 Self::F32(e) => Self::F32(&e[start..end]),
298 Self::F64(e) => Self::F64(&e[start..end]),
299 Self::I16(e) => Self::I16(&e[start..end]),
300 Self::I32(e) => Self::I32(&e[start..end]),
301 Self::I64(e) => Self::I64(&e[start..end]),
302 }
303 }
304
305 fn value_at(self, index: usize) -> f64 {
306 match self {
307 Self::F16(e) => f16_bits_to_f64(e[index]),
308 Self::F32(e) => e[index] as f64,
309 Self::F64(e) => e[index],
310 Self::I16(e) => e[index] as f64,
311 Self::I32(e) => e[index] as f64,
312 Self::I64(e) => e[index] as f64,
313 }
314 }
315
316 fn is_monotonic_non_decreasing(self) -> bool {
317 let count = self.element_count();
318 if count < 2 {
319 return true;
320 }
321 let mut previous = self.value_at(0);
322 for index in 1..count {
323 let current = self.value_at(index);
324 if current < previous {
325 return false;
326 }
327 previous = current;
328 }
329 true
330 }
331}
332
333fn array_data_from_binary_data_array(bda: &BinaryDataArray) -> Option<ArrayData<'_>> {
334 match bda.binary.as_ref()? {
335 NumericArray::F16(e) => Some(ArrayData::F16(e)),
336 NumericArray::I16(e) => Some(ArrayData::I16(e)),
337 NumericArray::I32(e) => Some(ArrayData::I32(e)),
338 NumericArray::I64(e) => Some(ArrayData::I64(e)),
339 NumericArray::F32(e) => Some(ArrayData::F32(e)),
340 NumericArray::F64(e) => Some(ArrayData::F64(e)),
341 }
342}
343
344fn element_byte_size_for_dtype(dtype: u8) -> usize {
345 match dtype {
346 FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
347 FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
348 FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
349 _ => 1,
350 }
351}
352
353fn resolve_array_dtype(bda: &BinaryDataArray, data: ArrayData<'_>, force_f32: bool) -> u8 {
354 match data {
355 ArrayData::F16(_) => FILE_DTYPE_F16,
356 ArrayData::I16(_) => FILE_DTYPE_I16,
357 ArrayData::I32(_) => FILE_DTYPE_I32,
358 ArrayData::I64(_) => FILE_DTYPE_I64,
359 ArrayData::F32(_) | ArrayData::F64(_) => {
360 if float_data_should_be_written_as_f64(bda, data, force_f32) {
361 FILE_DTYPE_F64
362 } else {
363 FILE_DTYPE_F32
364 }
365 }
366 }
367}
368
369fn float_data_should_be_written_as_f64(
370 bda: &BinaryDataArray,
371 data: ArrayData<'_>,
372 force_f32: bool,
373) -> bool {
374 if force_f32 {
375 return false;
376 }
377 if matches!(data, ArrayData::F64(_)) {
378 return true;
379 }
380 declared_float_precision_is_64bit(bda).unwrap_or(false)
381}
382
383fn declared_float_precision_is_64bit(bda: &BinaryDataArray) -> Option<bool> {
384 if let Some(nt) = bda.numeric_type.as_ref() {
385 return match nt {
386 NumericType::Float64 => Some(true),
387 NumericType::Float32 => Some(false),
388 _ => None,
389 };
390 }
391 let (mut saw32, mut saw64) = (false, false);
392 for cv in &bda.cv_params {
393 match parse_accession_tail_raw(cv.accession.as_deref()) {
394 FLOAT_32BIT => saw32 = true,
395 FLOAT_64BIT => saw64 = true,
396 _ => {}
397 }
398 if saw32 && saw64 {
399 break;
400 }
401 }
402 match (saw32, saw64) {
403 (true, false) => Some(false),
404 (false, true) => Some(true),
405 _ => None,
406 }
407}
408
409fn validate_array_dtype(data: ArrayData<'_>, dtype: u8) -> IonResult<()> {
410 let ok = matches!(
411 (dtype, data),
412 (FILE_DTYPE_F16, ArrayData::F16(_))
413 | (FILE_DTYPE_F32, ArrayData::F32(_))
414 | (FILE_DTYPE_F32, ArrayData::F64(_))
415 | (FILE_DTYPE_F64, ArrayData::F64(_))
416 | (FILE_DTYPE_F64, ArrayData::F32(_))
417 | (FILE_DTYPE_I16, ArrayData::I16(_))
418 | (FILE_DTYPE_I32, ArrayData::I32(_))
419 | (FILE_DTYPE_I64, ArrayData::I64(_))
420 );
421 if ok {
422 Ok(())
423 } else {
424 Err(format!(
425 "write_array_data: incompatible dtype {dtype} for the given array data variant"
426 )
427 .into())
428 }
429}
430
431fn write_array_data(buf: &mut Vec<u8>, data: ArrayData<'_>, dtype: u8) {
432 match (dtype, data) {
433 (FILE_DTYPE_F16, ArrayData::F16(e)) => write_u16_slice_le(buf, e),
434 (FILE_DTYPE_F32, ArrayData::F32(e)) => write_f32_slice_le(buf, e),
435 (FILE_DTYPE_F32, ArrayData::F64(e)) => {
436 for &v in e {
437 write_f32_le(buf, v as f32);
438 }
439 }
440 (FILE_DTYPE_F64, ArrayData::F64(e)) => write_f64_slice_le(buf, e),
441 (FILE_DTYPE_F64, ArrayData::F32(e)) => {
442 for &v in e {
443 write_f64_le(buf, v as f64);
444 }
445 }
446 (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e),
447 (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e),
448 (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e),
449 _ => unreachable!("write_array_data called with unvalidated dtype/data combination"),
451 }
452}
453
454fn narrow_f64_to_f32(values: &[f64]) -> Vec<f32> {
455 values.iter().map(|&value| value as f32).collect()
456}
457
458fn widen_f32_to_f64(values: &[f32]) -> Vec<f64> {
459 values.iter().map(|&value| value as f64).collect()
460}
461
462fn encode_floats_through_packing(
463 buf: &mut Vec<u8>,
464 data: ArrayData<'_>,
465 dtype_enum: Dtype,
466 packing: &'static dyn Packing,
467) -> IonResult<()> {
468 match (dtype_enum, data) {
469 (Dtype::F64, ArrayData::F64(slice)) => packing.encode(PackingInput::F64(slice), buf),
470 (Dtype::F32, ArrayData::F32(slice)) => packing.encode(PackingInput::F32(slice), buf),
471 (Dtype::F32, ArrayData::F64(slice)) => {
472 let narrowed = narrow_f64_to_f32(slice);
473 packing.encode(PackingInput::F32(&narrowed), buf)
474 }
475 (Dtype::F64, ArrayData::F32(slice)) => {
476 let widened = widen_f32_to_f64(slice);
477 packing.encode(PackingInput::F64(&widened), buf)
478 }
479 _ => Err(IonError::from(
480 "shuffle packing requires f32 or f64 array data",
481 )),
482 }
483}
484
485pub(crate) struct EncodedArrayAddress {
486 pub(crate) element_offset: u64,
487 pub(crate) element_count: u64,
488 pub(crate) block_id: u32,
489 pub(crate) accession: u32,
490 pub(crate) dtype: u8,
491 pub(crate) array_filter: u8,
492 pub(crate) encoded_len: u32,
493 pub(crate) continues_previous_segment: u8,
494 pub(crate) array_cv_code: u8,
495}
496
497fn encode_variable_length_array(
498 array_type: u32,
499 data: ArrayData<'_>,
500 dtype: u8,
501 dtype_enum: Dtype,
502 packing: &'static dyn Packing,
503 container: &mut BlockWriter<'_, DefaultCompressor>,
504) -> IonResult<(u32, u64, u32)> {
505 let mut encoded = Vec::new();
506 match packing.id() {
507 PackingId::Raw => write_array_data(&mut encoded, data, dtype),
508 _ => encode_floats_through_packing(&mut encoded, data, dtype_enum, packing)?,
509 }
510 let enc_len =
511 u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?;
512 let (bid, eoff) = container.add_item_to_box(array_type, encoded.len(), 1, |buf| {
513 buf.extend_from_slice(&encoded);
514 Ok(())
515 })?;
516 Ok((bid, eoff, enc_len))
517}
518
519fn write_fixed_array_payload(
520 buf: &mut Vec<u8>,
521 data: ArrayData<'_>,
522 dtype: u8,
523 dtype_enum: Dtype,
524 packing: &'static dyn Packing,
525) -> IonResult<()> {
526 match packing.id() {
527 PackingId::Raw => {
528 write_array_data(buf, data, dtype);
529 Ok(())
530 }
531 _ => encode_floats_through_packing(buf, data, dtype_enum, packing),
532 }
533}
534
535fn encode_fixed_length_array(
536 array_type: u32,
537 data: ArrayData<'_>,
538 dtype: u8,
539 dtype_enum: Dtype,
540 elem_bytes: usize,
541 packing: &'static dyn Packing,
542 container: &mut BlockWriter<'_, DefaultCompressor>,
543) -> IonResult<(u32, u64, u32)> {
544 let (bid, eoff) = container.add_item_to_box(
545 array_type,
546 data.element_count() * elem_bytes,
547 elem_bytes,
548 |buf| write_fixed_array_payload(buf, data, dtype, dtype_enum, packing),
549 )?;
550 Ok((bid, eoff, 0u32))
551}
552
553struct ArrayEncoding<'a> {
554 data: ArrayData<'a>,
555 accession: u32,
556 array_cv_code: u8,
557 dtype: u8,
558 dtype_enum: Dtype,
559 elem_bytes: usize,
560 packing: &'static dyn Packing,
561}
562
563fn resolve_array_encoding<'a>(
564 bda: &'a BinaryDataArray,
565 config: WriteOptions,
566 policy: ArrayPolicy,
567) -> IonResult<Option<ArrayEncoding<'a>>> {
568 let Some(data) = array_data_from_binary_data_array(bda) else {
569 return Ok(None);
570 };
571 if data.is_empty() {
572 return Ok(None);
573 }
574 let accession = array_type_accession_from_binary_data_array(bda);
575 let array_cv_code = array_type_cv_code_from_binary_data_array(bda);
576 let dtype = resolve_array_dtype(bda, data, policy.should_force_f32(accession));
577 validate_array_dtype(data, dtype)?;
578 let elem_bytes = element_byte_size_for_dtype(dtype);
579 let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64);
580 let requested: &'static dyn Packing = if config.compression_is_enabled() {
581 packing_for(accession, dtype_enum, data.element_count())
582 } else {
583 &RAW_PACKING
584 };
585 let packing: &'static dyn Packing = if requested.supports(dtype_enum) {
586 requested
587 } else {
588 &RAW_PACKING
589 };
590 Ok(Some(ArrayEncoding {
591 data,
592 accession,
593 array_cv_code,
594 dtype,
595 dtype_enum,
596 elem_bytes,
597 packing,
598 }))
599}
600
601pub(crate) fn encode_single_array(
602 bda: &BinaryDataArray,
603 config: WriteOptions,
604 policy: ArrayPolicy,
605 container: &mut BlockWriter<'_, DefaultCompressor>,
606) -> IonResult<Option<EncodedArrayAddress>> {
607 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
608 return Ok(None);
609 };
610 let ArrayEncoding {
611 data,
612 accession,
613 array_cv_code,
614 dtype,
615 dtype_enum,
616 elem_bytes,
617 packing,
618 } = encoding;
619 let (block_id, element_offset, encoded_len) = if packing.is_variable_length() {
620 encode_variable_length_array(accession, data, dtype, dtype_enum, packing, container)?
621 } else {
622 encode_fixed_length_array(
623 accession, data, dtype, dtype_enum, elem_bytes, packing, container,
624 )?
625 };
626 Ok(Some(EncodedArrayAddress {
627 element_offset,
628 element_count: data.element_count() as u64,
629 block_id,
630 accession,
631 dtype,
632 array_filter: packing.id() as u8,
633 encoded_len,
634 continues_previous_segment: 0,
635 array_cv_code,
636 }))
637}
638
639pub(crate) fn array_is_fixed_width_splittable(
640 bda: &BinaryDataArray,
641 config: WriteOptions,
642 policy: ArrayPolicy,
643) -> IonResult<Option<(usize, usize)>> {
644 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
645 return Ok(None);
646 };
647 if encoding.packing.is_variable_length() {
648 return Ok(None);
649 }
650 Ok(Some((encoding.data.element_count(), encoding.elem_bytes)))
651}
652
653pub(crate) fn check_spectrum_rt_order(
654 summary: &[u8],
655 spectrum_index: usize,
656 last_rt: &mut f64,
657) -> IonResult<()> {
658 let rt = f64::from_le_bytes(summary[0..8].try_into().unwrap());
659 if !rt.is_finite() {
660 return Ok(());
661 }
662 let previous = *last_rt;
663 if rt < previous {
664 return Err(format!(
665 "spectrum {spectrum_index}: retention time {rt} is below the previous spectrum's \
666 ({previous}); spectra must be stored in ascending rt order"
667 )
668 .into());
669 }
670 *last_rt = rt;
671 Ok(())
672}
673
674pub(crate) fn check_spectrum_mz_order(
675 arrays: &[BinaryDataArray],
676 spectrum_index: usize,
677) -> IonResult<()> {
678 use crate::accessions::MZ_ARRAY;
679 for bda in arrays {
680 if array_type_accession_from_binary_data_array(bda) != MZ_ARRAY {
681 continue;
682 }
683 let Some(data) = array_data_from_binary_data_array(bda) else {
684 return Ok(());
685 };
686 if !data.is_monotonic_non_decreasing() {
687 return Err(format!(
688 "spectrum {spectrum_index}: m/z array must be sorted ascending (non-decreasing) for range reads"
689 )
690 .into());
691 }
692 return Ok(());
693 }
694 Ok(())
695}
696
697pub(crate) fn window_ranges_for_item(
698 arrays: &[BinaryDataArray],
699 config: WriteOptions,
700 policy: ArrayPolicy,
701 mz_window: f64,
702) -> IonResult<Option<Vec<WindowRange>>> {
703 let windowing = Windowing::new(mz_window.round());
704 if !windowing.is_enabled() {
705 return Ok(None);
706 }
707 let Some(x_array) = arrays
708 .iter()
709 .find(|bda| array_type_accession_from_binary_data_array(bda) == policy.x_array_accession)
710 else {
711 return Ok(None);
712 };
713 let Some((x_count, _)) = array_is_fixed_width_splittable(x_array, config, policy)? else {
714 return Ok(None);
715 };
716 if x_count == 0 {
717 return Ok(None);
718 }
719 for bda in arrays {
720 match array_is_fixed_width_splittable(bda, config, policy)? {
721 Some((count, _)) if count == x_count => {}
722 _ => return Ok(None),
723 }
724 }
725 let Some(x_data) = array_data_from_binary_data_array(x_array) else {
726 return Ok(None);
727 };
728 let ranges = windowing.split_sorted(x_count, |index| x_data.value_at(index));
729 const MAX_WINDOW_INDEX: u32 = 1 << 20;
730 if ranges
731 .last()
732 .is_some_and(|range| range.window_index > MAX_WINDOW_INDEX)
733 {
734 return Err(format!(
735 "m/z window index exceeds limit {MAX_WINDOW_INDEX}; m/z value too large for window width {}",
736 mz_window.round()
737 )
738 .into());
739 }
740 Ok(Some(ranges))
741}
742
743pub(crate) fn write_array_windows(
744 bda: &BinaryDataArray,
745 config: WriteOptions,
746 policy: ArrayPolicy,
747 windows: &[WindowRange],
748 container: &mut BlockWriter<'_, DefaultCompressor>,
749) -> IonResult<Vec<EncodedArrayAddress>> {
750 let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
751 return Ok(Vec::new());
752 };
753 let ArrayEncoding {
754 data,
755 accession,
756 array_cv_code,
757 dtype,
758 dtype_enum,
759 elem_bytes,
760 packing,
761 } = encoding;
762
763 if packing.is_variable_length() {
764 return Err("write_array_windows: variable-length arrays cannot be windowed".into());
765 }
766
767 let mut addresses = Vec::with_capacity(windows.len());
768 for (segment_index, window) in windows.iter().enumerate() {
769 let segment_data = data.slice(window.start, window.end);
770 let element_count = window.element_count();
771 let (block_id, element_offset) = container.add_item_to_window(
772 accession,
773 window.window_index,
774 element_count * elem_bytes,
775 elem_bytes,
776 |buf| write_fixed_array_payload(buf, segment_data, dtype, dtype_enum, packing),
777 )?;
778 addresses.push(EncodedArrayAddress {
779 element_offset,
780 element_count: element_count as u64,
781 block_id,
782 accession,
783 dtype,
784 array_filter: packing.id() as u8,
785 encoded_len: 0,
786 continues_previous_segment: (segment_index > 0) as u8,
787 array_cv_code,
788 });
789 }
790 Ok(addresses)
791}
792
793#[cfg(test)]
794mod tests {
795 use super::*;
796 use crate::{
797 ion::{
798 encoder::{ion_writer::write_mzml_to_ion, utilities::output::WriteBytes},
799 format::{FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE},
800 header::{
801 HEADER_CHROM_BLOCK_COUNT, HEADER_SPECTRUM_BLOCK_COUNT, HEADER_TARGET_BLOCK_SIZE,
802 HEADER_TARGET_MZ_WINDOW, HEADER_TOTAL_FILE_SIZE,
803 },
804 },
805 mzml::structs::MzML,
806 };
807
808 fn summary_with_rt(rt: f64) -> [u8; SPEC_SUMMARY_SIZE] {
809 let mut summary = [0u8; SPEC_SUMMARY_SIZE];
810 summary[0..8].copy_from_slice(&rt.to_le_bytes());
811 summary
812 }
813
814 #[test]
815 fn byte_shuffle_array_encode_arm_shuffles_not_raw_71() {
816 use crate::ion::packing::packing_by_id;
817
818 let data: Vec<f64> = (0..600)
819 .map(|i| 100.0 + (i as f64) * 0.0137 + ((i * 7 % 13) as f64) * 0.0009)
820 .collect();
821
822 let byte_shuffle = packing_by_id(PackingId::ByteShuffle);
823
824 let mut shuffled = Vec::new();
825 write_fixed_array_payload(
826 &mut shuffled,
827 ArrayData::F64(&data),
828 FILE_DTYPE_F64,
829 Dtype::F64,
830 byte_shuffle,
831 )
832 .unwrap();
833
834 let mut raw = Vec::new();
835 write_array_data(&mut raw, ArrayData::F64(&data), FILE_DTYPE_F64);
836
837 assert_eq!(shuffled.len(), raw.len());
838 assert_ne!(
839 shuffled, raw,
840 "ByteShuffle encode arm must shuffle, not write raw bytes"
841 );
842
843 let mut expected = Vec::new();
844 byte_shuffle
845 .encode(PackingInput::F64(&data), &mut expected)
846 .unwrap();
847 assert_eq!(
848 shuffled, expected,
849 "ByteShuffle encode arm must produce exactly the SIMD byte-transpose output"
850 );
851 }
852
853 #[test]
854 fn rt_order_allows_ascending_and_missing_but_rejects_a_drop() {
855 let mut last_rt = f64::NEG_INFINITY;
856 check_spectrum_rt_order(&summary_with_rt(1.0), 0, &mut last_rt)
857 .expect("first rt is allowed");
858 check_spectrum_rt_order(&summary_with_rt(1.0), 1, &mut last_rt)
859 .expect("equal rt is allowed");
860 check_spectrum_rt_order(&summary_with_rt(2.0), 2, &mut last_rt)
861 .expect("rising rt is allowed");
862
863 check_spectrum_rt_order(&summary_with_rt(f64::NAN), 3, &mut last_rt)
864 .expect("missing rt is allowed");
865 assert_eq!(last_rt, 2.0, "missing rt must not move the last seen rt");
866
867 let dropped = check_spectrum_rt_order(&summary_with_rt(1.5), 4, &mut last_rt);
868 let message = dropped
869 .expect_err("rt below the previous spectrum must be rejected")
870 .to_string();
871 assert!(
872 message.contains("spectrum 4"),
873 "error names the spectrum: {message}"
874 );
875 assert!(
876 message.contains("ascending rt order"),
877 "error explains the rule: {message}"
878 );
879 }
880
881 #[test]
882 fn encode_starts_with_magic_and_ends_with_trailer() {
883 let mzml = MzML::default();
884 let mut buf = Vec::new();
885 write_mzml_to_ion(
886 &mzml,
887 WriteOptions {
888 compression_level: 0,
889 force_f32: false,
890 ..Default::default()
891 },
892 &mut buf,
893 )
894 .unwrap();
895 assert!(buf.len() >= HEADER_SIZE + FILE_TRAILER.len());
896 assert_eq!(&buf[..FILE_SIGNATURE.len()], &FILE_SIGNATURE);
897 assert_eq!(&buf[buf.len() - FILE_TRAILER.len()..], &FILE_TRAILER);
898 }
899
900 #[test]
901 fn encode_total_file_size_is_correct() {
902 let mzml = MzML::default();
903 let mut buf = Vec::new();
904 write_mzml_to_ion(
905 &mzml,
906 WriteOptions {
907 compression_level: 0,
908 force_f32: false,
909 ..Default::default()
910 },
911 &mut buf,
912 )
913 .unwrap();
914 let total = u64::from_le_bytes(
915 buf[HEADER_TOTAL_FILE_SIZE..HEADER_TOTAL_FILE_SIZE + 8]
916 .try_into()
917 .unwrap(),
918 );
919 assert_eq!(total, buf.len() as u64);
920 }
921
922 #[test]
923 fn encode_header_target_block_size_matches_default() {
924 let mzml = MzML::default();
925 let mut buf = Vec::new();
926 write_mzml_to_ion(
927 &mzml,
928 WriteOptions {
929 compression_level: 0,
930 force_f32: false,
931 ..Default::default()
932 },
933 &mut buf,
934 )
935 .unwrap();
936 let size = u64::from_le_bytes(
937 buf[HEADER_TARGET_BLOCK_SIZE..HEADER_TARGET_BLOCK_SIZE + 8]
938 .try_into()
939 .unwrap(),
940 );
941 assert_eq!(size, TARGET_BLOCK_UNCOMPRESSED_BYTES as u64);
942 }
943
944 #[test]
945 fn encode_header_target_mz_window_matches_default() {
946 let mzml = MzML::default();
947 let mut buf = Vec::new();
948 write_mzml_to_ion(
949 &mzml,
950 WriteOptions {
951 compression_level: 0,
952 force_f32: false,
953 ..Default::default()
954 },
955 &mut buf,
956 )
957 .unwrap();
958 let window = u32::from_le_bytes(
959 buf[HEADER_TARGET_MZ_WINDOW..HEADER_TARGET_MZ_WINDOW + 4]
960 .try_into()
961 .unwrap(),
962 );
963 assert_eq!(window, DEFAULT_MZ_WINDOW.round() as u32);
964 }
965
966 #[test]
967 fn encode_header_crc32_is_valid() {
968 let mzml = MzML::default();
969 let mut buf = Vec::new();
970 write_mzml_to_ion(
971 &mzml,
972 WriteOptions {
973 compression_level: 0,
974 force_f32: false,
975 ..Default::default()
976 },
977 &mut buf,
978 )
979 .unwrap();
980 let stored = u32::from_le_bytes(buf[1020..1024].try_into().unwrap());
981 assert_eq!(stored, crc32fast::hash(&buf[0..1020]));
982 }
983
984 #[test]
985 fn encode_header_block_counts_are_zero_for_empty_mzml() {
986 let mzml = MzML::default();
987 let mut buf = Vec::new();
988 write_mzml_to_ion(
989 &mzml,
990 WriteOptions {
991 compression_level: 0,
992 force_f32: false,
993 ..Default::default()
994 },
995 &mut buf,
996 )
997 .unwrap();
998 let spec_blocks = u64::from_le_bytes(
999 buf[HEADER_SPECTRUM_BLOCK_COUNT..HEADER_SPECTRUM_BLOCK_COUNT + 8]
1000 .try_into()
1001 .unwrap(),
1002 );
1003 let chrom_blocks = u64::from_le_bytes(
1004 buf[HEADER_CHROM_BLOCK_COUNT..HEADER_CHROM_BLOCK_COUNT + 8]
1005 .try_into()
1006 .unwrap(),
1007 );
1008 assert_eq!(spec_blocks, 0);
1009 assert_eq!(chrom_blocks, 0);
1010 }
1011
1012 #[test]
1013 fn vec_patch_in_bounds() {
1014 let mut output = vec![0u8; 16];
1015 output.patch(4, &[1u8, 2, 3, 4]).unwrap();
1016 assert_eq!(&output[4..8], &[1u8, 2, 3, 4]);
1017 }
1018
1019 #[test]
1020 fn vec_patch_out_of_bounds_errors() {
1021 let mut output = vec![0u8; 4];
1022 assert!(output.patch(3, &[1u8, 2, 3]).is_err());
1023 }
1024
1025 #[test]
1026 fn declared_float_precision_prefers_numeric_type_field() {
1027 let bda = BinaryDataArray {
1028 numeric_type: Some(NumericType::Float64),
1029 ..Default::default()
1030 };
1031 assert_eq!(declared_float_precision_is_64bit(&bda), Some(true));
1032 let bda = BinaryDataArray {
1033 numeric_type: Some(NumericType::Float32),
1034 ..Default::default()
1035 };
1036 assert_eq!(declared_float_precision_is_64bit(&bda), Some(false));
1037 }
1038
1039 #[test]
1040 fn resolve_array_dtype_force_f32_overrides_f64_data() {
1041 let bda = BinaryDataArray::default();
1042 assert_eq!(
1043 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1044 FILE_DTYPE_F32
1045 );
1046 }
1047
1048 #[test]
1049 fn resolve_array_dtype_keeps_f64_data_lossless_even_when_declared_32bit() {
1050 let bda = BinaryDataArray {
1051 numeric_type: Some(NumericType::Float32),
1052 ..Default::default()
1053 };
1054 assert_eq!(
1055 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), false),
1056 FILE_DTYPE_F64
1057 );
1058 assert_eq!(
1059 resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1060 FILE_DTYPE_F32
1061 );
1062 }
1063
1064 #[test]
1065 fn resolve_array_dtype_integer_types_unchanged_by_force_f32() {
1066 let bda = BinaryDataArray::default();
1067 assert_eq!(
1068 resolve_array_dtype(&bda, ArrayData::I32(&[1i32]), true),
1069 FILE_DTYPE_I32
1070 );
1071 }
1072
1073 #[test]
1074 fn element_byte_size_for_dtype_returns_correct_sizes() {
1075 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F64), 8);
1076 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F32), 4);
1077 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F16), 2);
1078 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I16), 2);
1079 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I32), 4);
1080 assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I64), 8);
1081 }
1082
1083 #[test]
1084 fn encoder_config_compression_disabled_at_level_zero() {
1085 let config = WriteOptions {
1086 compression_level: 0,
1087 force_f32: false,
1088 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1089 parallel: true,
1090 section_storage: SectionStorage::Memory,
1091 mz_window: 0.0,
1092 };
1093 assert!(!config.compression_is_enabled());
1094 assert_eq!(config.codec_id(), 0);
1095 assert_eq!(config.array_filter_id(), PackingId::Raw as u8);
1096 assert!(matches!(config.block_packing_id(), PackingId::Raw));
1097 }
1098
1099 #[test]
1100 fn encoder_config_compression_enabled_at_nonzero_level() {
1101 let config = WriteOptions {
1102 compression_level: 3,
1103 force_f32: false,
1104 block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1105 parallel: true,
1106 section_storage: SectionStorage::Memory,
1107 mz_window: 0.0,
1108 };
1109 assert!(config.compression_is_enabled());
1110 assert_eq!(config.codec_id(), 1);
1111 assert_eq!(config.array_filter_id(), PackingId::ByteShuffle as u8);
1112 assert!(matches!(config.block_packing_id(), PackingId::ByteShuffle));
1113 }
1114}