Skip to main content

ionic/ion/encoder/
encode.rs

1use crate::{
2    accessions::{
3        FLOAT_32BIT, FLOAT_64BIT, HIGHEST_ION_MOBILITY, HIGHEST_OBSERVED_MZ,
4        HIGHEST_OBSERVED_WAVELENGTH, INTENSITY_ARRAY, LOWEST_ION_MOBILITY, LOWEST_OBSERVED_MZ,
5        LOWEST_OBSERVED_WAVELENGTH, NEGATIVE_SCAN, POSITION_X, POSITION_Y, POSITION_Z,
6        POSITIVE_SCAN,
7    },
8    ion::{
9        IonError, IonResult,
10        encoder::utilities::{
11            BlockWriter, CompressionMode, DefaultCompressor,
12            le_writers::{
13                write_f32_le, write_f32_slice_le, write_f64_le, write_f64_slice_le,
14                write_i16_slice_le, write_i32_slice_le, write_i64_slice_le, write_u16_slice_le,
15            },
16            meta_collector::{
17                ArrayPolicy, array_type_accession_from_binary_data_array,
18                array_type_cv_code_from_binary_data_array, parse_accession_tail_raw,
19            },
20            output::SectionStorage,
21        },
22        filter_summary::{ChromatogramSummary, SpectrumSummary},
23        packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING},
24        utilities::spectrum_source::{f16_bits_to_f64, summary_from_spectrum},
25        windowing::{WindowRange, Windowing},
26    },
27    mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum},
28};
29pub const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024;
30pub const DEFAULT_MZ_WINDOW: f64 = 20.0;
31pub(crate) const SPEC_SUMMARY_SIZE: usize = 80;
32pub(crate) const CHROM_SUMMARY_SIZE: usize = 80;
33
34pub(crate) const FILE_DTYPE_F64: u8 = Dtype::F64 as u8;
35pub(crate) const FILE_DTYPE_F32: u8 = Dtype::F32 as u8;
36pub(crate) const FILE_DTYPE_F16: u8 = Dtype::F16 as u8;
37pub(crate) const FILE_DTYPE_I16: u8 = Dtype::I16 as u8;
38pub(crate) const FILE_DTYPE_I32: u8 = Dtype::I32 as u8;
39pub(crate) const FILE_DTYPE_I64: u8 = Dtype::I64 as u8;
40
41const POLARITY_UNKNOWN: u8 = 0;
42const POLARITY_POSITIVE: u8 = 1;
43const POLARITY_NEGATIVE: u8 = 2;
44
45impl ChromatogramSummary {
46    fn unknown() -> Self {
47        Self {
48            lowest_mz: f64::NAN,
49            highest_mz: f64::NAN,
50            lowest_wavelength: f64::NAN,
51            highest_wavelength: f64::NAN,
52            lowest_ion_mobility: f64::NAN,
53            highest_ion_mobility: f64::NAN,
54            polarity: POLARITY_UNKNOWN,
55        }
56    }
57}
58
59pub(crate) fn spec_summary_from_spectrum(spectrum: &Spectrum) -> SpectrumSummary {
60    let summary = summary_from_spectrum(spectrum);
61    let (position_x, position_y, position_z) = get_spectrum_position(spectrum);
62    SpectrumSummary {
63        rt: summary.rt,
64        rt_unit: summary.rt_unit.code(),
65        base_peak_mz: summary.base_peak_mz,
66        selected_ion_mz: summary.selected_ion_mz,
67        base_peak_int: summary.base_peak_int,
68        total_ion_current: summary.total_ion_current,
69        ms_level: summary.ms_level,
70        polarity: summary.polarity,
71        position_x,
72        position_y,
73        position_z,
74    }
75}
76
77fn get_spectrum_position(spectrum: &Spectrum) -> (u32, u32, u32) {
78    let mut x = 0;
79    let mut y = 0;
80    let mut z = 0;
81    let scan_list = spectrum.scan_list.as_ref().or_else(|| {
82        spectrum
83            .spectrum_description
84            .as_ref()
85            .and_then(|description| description.scan_list.as_ref())
86    });
87    let Some(scan_list) = scan_list else {
88        return (x, y, z);
89    };
90    for scan in &scan_list.scans {
91        for param in &scan.cv_params {
92            let Some(accession) = param.accession.as_deref() else {
93                continue;
94            };
95            if !accession.starts_with("IMS:") {
96                continue;
97            }
98            let value = param
99                .value
100                .as_deref()
101                .and_then(|value| value.parse::<u32>().ok())
102                .unwrap_or(0);
103            match parse_accession_tail_raw(Some(accession)) {
104                POSITION_X => x = value,
105                POSITION_Y => y = value,
106                POSITION_Z => z = value,
107                _ => {}
108            }
109        }
110    }
111    (x, y, z)
112}
113
114#[inline]
115fn cv_f64(cv: &CvParam) -> Option<f64> {
116    cv.value.as_deref().and_then(|s| s.parse::<f64>().ok())
117}
118
119pub(crate) fn extract_chrom_summary(chrom: &Chromatogram) -> ChromatogramSummary {
120    let mut summary = ChromatogramSummary::unknown();
121    for cv in &chrom.cv_params {
122        match parse_accession_tail_raw(cv.accession.as_deref()) {
123            LOWEST_OBSERVED_MZ => {
124                if let Some(v) = cv_f64(cv) {
125                    summary.lowest_mz = v;
126                }
127            }
128            HIGHEST_OBSERVED_MZ => {
129                if let Some(v) = cv_f64(cv) {
130                    summary.highest_mz = v;
131                }
132            }
133            LOWEST_OBSERVED_WAVELENGTH => {
134                if let Some(v) = cv_f64(cv) {
135                    summary.lowest_wavelength = v;
136                }
137            }
138            HIGHEST_OBSERVED_WAVELENGTH => {
139                if let Some(v) = cv_f64(cv) {
140                    summary.highest_wavelength = v;
141                }
142            }
143            LOWEST_ION_MOBILITY => {
144                if let Some(v) = cv_f64(cv) {
145                    summary.lowest_ion_mobility = v;
146                }
147            }
148            HIGHEST_ION_MOBILITY => {
149                if let Some(v) = cv_f64(cv) {
150                    summary.highest_ion_mobility = v;
151                }
152            }
153            POSITIVE_SCAN => summary.polarity = POLARITY_POSITIVE,
154            NEGATIVE_SCAN => summary.polarity = POLARITY_NEGATIVE,
155            _ => {}
156        }
157    }
158    summary
159}
160
161pub(crate) fn get_supported_compression_level(compression_level: u8) -> u8 {
162    match compression_level {
163        0..=5 => 1,
164        6..=10 => 9,
165        11..=16 => 12,
166        _ => 22,
167    }
168}
169
170pub(crate) fn allow_compression_level(compression_level: u8) -> IonResult<()> {
171    if compression_level > 22 {
172        return Err(format!("compression_level must be 0-22, got {compression_level}").into());
173    }
174    Ok(())
175}
176
177#[derive(Debug, Clone, Copy)]
178pub struct WriteOptions {
179    pub compression_level: u8,
180    pub force_f32: bool,
181    pub block_size: usize,
182    pub parallel: bool,
183    pub section_storage: SectionStorage,
184    pub mz_window: f64,
185}
186
187impl Default for WriteOptions {
188    fn default() -> Self {
189        Self {
190            compression_level: 12,
191            force_f32: false,
192            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
193            parallel: true,
194            section_storage: SectionStorage::Disk,
195            mz_window: DEFAULT_MZ_WINDOW,
196        }
197    }
198}
199
200impl WriteOptions {
201    pub(crate) fn compression_is_enabled(self) -> bool {
202        self.compression_level != 0
203    }
204
205    pub(crate) fn codec_id(self) -> u8 {
206        self.compression_is_enabled() as u8
207    }
208
209    fn block_shuffle_is_enabled(self) -> bool {
210        if !self.compression_is_enabled() {
211            return false;
212        }
213        !matches!(
214            std::env::var("IONIC_BLOCK_SHUFFLE").as_deref(),
215            Ok("off") | Ok("raw") | Ok("0") | Ok("none")
216        )
217    }
218
219    pub(crate) fn array_filter_id(self) -> u8 {
220        if self.block_shuffle_is_enabled() {
221            PackingId::ByteShuffle as u8
222        } else {
223            PackingId::Raw as u8
224        }
225    }
226
227    pub(crate) fn compression_mode(self) -> IonResult<CompressionMode<DefaultCompressor>> {
228        if self.compression_is_enabled() {
229            Ok(CompressionMode::Compressed(DefaultCompressor::new(
230                self.compression_level as i32,
231            )?))
232        } else {
233            Ok(CompressionMode::Raw)
234        }
235    }
236
237    pub(crate) fn block_packing_id(self) -> PackingId {
238        if self.block_shuffle_is_enabled() {
239            PackingId::ByteShuffle
240        } else {
241            PackingId::Raw
242        }
243    }
244
245    pub(crate) fn array_policy(self, x_array_accession: u32) -> ArrayPolicy {
246        ArrayPolicy {
247            x_array_accession,
248            y_array_accession: INTENSITY_ARRAY,
249            force_f32: self.force_f32,
250        }
251    }
252
253    pub fn fast() -> Self {
254        Self {
255            compression_level: 3,
256            ..Default::default()
257        }
258    }
259
260    pub fn max_ratio() -> Self {
261        Self {
262            compression_level: 22,
263            ..Default::default()
264        }
265    }
266}
267
268#[derive(Copy, Clone)]
269enum ArrayData<'a> {
270    F16(&'a [u16]),
271    F32(&'a [f32]),
272    F64(&'a [f64]),
273    I16(&'a [i16]),
274    I32(&'a [i32]),
275    I64(&'a [i64]),
276}
277
278impl<'a> ArrayData<'a> {
279    fn element_count(self) -> usize {
280        match self {
281            Self::F16(e) => e.len(),
282            Self::F32(e) => e.len(),
283            Self::F64(e) => e.len(),
284            Self::I16(e) => e.len(),
285            Self::I32(e) => e.len(),
286            Self::I64(e) => e.len(),
287        }
288    }
289
290    fn is_empty(self) -> bool {
291        self.element_count() == 0
292    }
293
294    fn slice(self, start: usize, end: usize) -> ArrayData<'a> {
295        match self {
296            Self::F16(e) => Self::F16(&e[start..end]),
297            Self::F32(e) => Self::F32(&e[start..end]),
298            Self::F64(e) => Self::F64(&e[start..end]),
299            Self::I16(e) => Self::I16(&e[start..end]),
300            Self::I32(e) => Self::I32(&e[start..end]),
301            Self::I64(e) => Self::I64(&e[start..end]),
302        }
303    }
304
305    fn value_at(self, index: usize) -> f64 {
306        match self {
307            Self::F16(e) => f16_bits_to_f64(e[index]),
308            Self::F32(e) => e[index] as f64,
309            Self::F64(e) => e[index],
310            Self::I16(e) => e[index] as f64,
311            Self::I32(e) => e[index] as f64,
312            Self::I64(e) => e[index] as f64,
313        }
314    }
315
316    fn is_monotonic_non_decreasing(self) -> bool {
317        let count = self.element_count();
318        if count < 2 {
319            return true;
320        }
321        let mut previous = self.value_at(0);
322        for index in 1..count {
323            let current = self.value_at(index);
324            if current < previous {
325                return false;
326            }
327            previous = current;
328        }
329        true
330    }
331}
332
333fn array_data_from_binary_data_array(bda: &BinaryDataArray) -> Option<ArrayData<'_>> {
334    match bda.binary.as_ref()? {
335        NumericArray::F16(e) => Some(ArrayData::F16(e)),
336        NumericArray::I16(e) => Some(ArrayData::I16(e)),
337        NumericArray::I32(e) => Some(ArrayData::I32(e)),
338        NumericArray::I64(e) => Some(ArrayData::I64(e)),
339        NumericArray::F32(e) => Some(ArrayData::F32(e)),
340        NumericArray::F64(e) => Some(ArrayData::F64(e)),
341    }
342}
343
344fn element_byte_size_for_dtype(dtype: u8) -> usize {
345    match dtype {
346        FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
347        FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
348        FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
349        _ => 1,
350    }
351}
352
353fn resolve_array_dtype(bda: &BinaryDataArray, data: ArrayData<'_>, force_f32: bool) -> u8 {
354    match data {
355        ArrayData::F16(_) => FILE_DTYPE_F16,
356        ArrayData::I16(_) => FILE_DTYPE_I16,
357        ArrayData::I32(_) => FILE_DTYPE_I32,
358        ArrayData::I64(_) => FILE_DTYPE_I64,
359        ArrayData::F32(_) | ArrayData::F64(_) => {
360            if float_data_should_be_written_as_f64(bda, data, force_f32) {
361                FILE_DTYPE_F64
362            } else {
363                FILE_DTYPE_F32
364            }
365        }
366    }
367}
368
369fn float_data_should_be_written_as_f64(
370    bda: &BinaryDataArray,
371    data: ArrayData<'_>,
372    force_f32: bool,
373) -> bool {
374    if force_f32 {
375        return false;
376    }
377    if matches!(data, ArrayData::F64(_)) {
378        return true;
379    }
380    declared_float_precision_is_64bit(bda).unwrap_or(false)
381}
382
383fn declared_float_precision_is_64bit(bda: &BinaryDataArray) -> Option<bool> {
384    if let Some(nt) = bda.numeric_type.as_ref() {
385        return match nt {
386            NumericType::Float64 => Some(true),
387            NumericType::Float32 => Some(false),
388            _ => None,
389        };
390    }
391    let (mut saw32, mut saw64) = (false, false);
392    for cv in &bda.cv_params {
393        match parse_accession_tail_raw(cv.accession.as_deref()) {
394            FLOAT_32BIT => saw32 = true,
395            FLOAT_64BIT => saw64 = true,
396            _ => {}
397        }
398        if saw32 && saw64 {
399            break;
400        }
401    }
402    match (saw32, saw64) {
403        (true, false) => Some(false),
404        (false, true) => Some(true),
405        _ => None,
406    }
407}
408
409fn validate_array_dtype(data: ArrayData<'_>, dtype: u8) -> IonResult<()> {
410    let ok = matches!(
411        (dtype, data),
412        (FILE_DTYPE_F16, ArrayData::F16(_))
413            | (FILE_DTYPE_F32, ArrayData::F32(_))
414            | (FILE_DTYPE_F32, ArrayData::F64(_))
415            | (FILE_DTYPE_F64, ArrayData::F64(_))
416            | (FILE_DTYPE_F64, ArrayData::F32(_))
417            | (FILE_DTYPE_I16, ArrayData::I16(_))
418            | (FILE_DTYPE_I32, ArrayData::I32(_))
419            | (FILE_DTYPE_I64, ArrayData::I64(_))
420    );
421    if ok {
422        Ok(())
423    } else {
424        Err(format!(
425            "write_array_data: incompatible dtype {dtype} for the given array data variant"
426        )
427        .into())
428    }
429}
430
431fn write_array_data(buf: &mut Vec<u8>, data: ArrayData<'_>, dtype: u8) {
432    match (dtype, data) {
433        (FILE_DTYPE_F16, ArrayData::F16(e)) => write_u16_slice_le(buf, e),
434        (FILE_DTYPE_F32, ArrayData::F32(e)) => write_f32_slice_le(buf, e),
435        (FILE_DTYPE_F32, ArrayData::F64(e)) => {
436            for &v in e {
437                write_f32_le(buf, v as f32);
438            }
439        }
440        (FILE_DTYPE_F64, ArrayData::F64(e)) => write_f64_slice_le(buf, e),
441        (FILE_DTYPE_F64, ArrayData::F32(e)) => {
442            for &v in e {
443                write_f64_le(buf, v as f64);
444            }
445        }
446        (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e),
447        (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e),
448        (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e),
449        // SAFETY: `validate_array_dtype` is always called before this function.
450        _ => unreachable!("write_array_data called with unvalidated dtype/data combination"),
451    }
452}
453
454fn narrow_f64_to_f32(values: &[f64]) -> Vec<f32> {
455    values.iter().map(|&value| value as f32).collect()
456}
457
458fn widen_f32_to_f64(values: &[f32]) -> Vec<f64> {
459    values.iter().map(|&value| value as f64).collect()
460}
461
462fn encode_floats_through_packing(
463    buf: &mut Vec<u8>,
464    data: ArrayData<'_>,
465    dtype_enum: Dtype,
466    packing: &'static dyn Packing,
467) -> IonResult<()> {
468    match (dtype_enum, data) {
469        (Dtype::F64, ArrayData::F64(slice)) => packing.encode(PackingInput::F64(slice), buf),
470        (Dtype::F32, ArrayData::F32(slice)) => packing.encode(PackingInput::F32(slice), buf),
471        (Dtype::F32, ArrayData::F64(slice)) => {
472            let narrowed = narrow_f64_to_f32(slice);
473            packing.encode(PackingInput::F32(&narrowed), buf)
474        }
475        (Dtype::F64, ArrayData::F32(slice)) => {
476            let widened = widen_f32_to_f64(slice);
477            packing.encode(PackingInput::F64(&widened), buf)
478        }
479        _ => Err(IonError::from(
480            "shuffle packing requires f32 or f64 array data",
481        )),
482    }
483}
484
485pub(crate) struct EncodedArrayAddress {
486    pub(crate) element_offset: u64,
487    pub(crate) element_count: u64,
488    pub(crate) block_id: u32,
489    pub(crate) accession: u32,
490    pub(crate) dtype: u8,
491    pub(crate) array_filter: u8,
492    pub(crate) encoded_len: u32,
493    pub(crate) continues_previous_segment: u8,
494    pub(crate) array_cv_code: u8,
495}
496
497fn encode_variable_length_array(
498    array_type: u32,
499    data: ArrayData<'_>,
500    dtype: u8,
501    dtype_enum: Dtype,
502    packing: &'static dyn Packing,
503    container: &mut BlockWriter<'_, DefaultCompressor>,
504) -> IonResult<(u32, u64, u32)> {
505    let mut encoded = Vec::new();
506    match packing.id() {
507        PackingId::Raw => write_array_data(&mut encoded, data, dtype),
508        _ => encode_floats_through_packing(&mut encoded, data, dtype_enum, packing)?,
509    }
510    let enc_len =
511        u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?;
512    let (bid, eoff) = container.add_item_to_box(array_type, encoded.len(), 1, |buf| {
513        buf.extend_from_slice(&encoded);
514        Ok(())
515    })?;
516    Ok((bid, eoff, enc_len))
517}
518
519fn write_fixed_array_payload(
520    buf: &mut Vec<u8>,
521    data: ArrayData<'_>,
522    dtype: u8,
523    dtype_enum: Dtype,
524    packing: &'static dyn Packing,
525) -> IonResult<()> {
526    match packing.id() {
527        PackingId::Raw => {
528            write_array_data(buf, data, dtype);
529            Ok(())
530        }
531        _ => encode_floats_through_packing(buf, data, dtype_enum, packing),
532    }
533}
534
535fn encode_fixed_length_array(
536    array_type: u32,
537    data: ArrayData<'_>,
538    dtype: u8,
539    dtype_enum: Dtype,
540    elem_bytes: usize,
541    packing: &'static dyn Packing,
542    container: &mut BlockWriter<'_, DefaultCompressor>,
543) -> IonResult<(u32, u64, u32)> {
544    let (bid, eoff) = container.add_item_to_box(
545        array_type,
546        data.element_count() * elem_bytes,
547        elem_bytes,
548        |buf| write_fixed_array_payload(buf, data, dtype, dtype_enum, packing),
549    )?;
550    Ok((bid, eoff, 0u32))
551}
552
553struct ArrayEncoding<'a> {
554    data: ArrayData<'a>,
555    accession: u32,
556    array_cv_code: u8,
557    dtype: u8,
558    dtype_enum: Dtype,
559    elem_bytes: usize,
560    packing: &'static dyn Packing,
561}
562
563fn resolve_array_encoding<'a>(
564    bda: &'a BinaryDataArray,
565    config: WriteOptions,
566    policy: ArrayPolicy,
567) -> IonResult<Option<ArrayEncoding<'a>>> {
568    let Some(data) = array_data_from_binary_data_array(bda) else {
569        return Ok(None);
570    };
571    if data.is_empty() {
572        return Ok(None);
573    }
574    let accession = array_type_accession_from_binary_data_array(bda);
575    let array_cv_code = array_type_cv_code_from_binary_data_array(bda);
576    let dtype = resolve_array_dtype(bda, data, policy.should_force_f32(accession));
577    validate_array_dtype(data, dtype)?;
578    let elem_bytes = element_byte_size_for_dtype(dtype);
579    let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64);
580    let requested: &'static dyn Packing = if config.compression_is_enabled() {
581        packing_for(accession, dtype_enum, data.element_count())
582    } else {
583        &RAW_PACKING
584    };
585    let packing: &'static dyn Packing = if requested.supports(dtype_enum) {
586        requested
587    } else {
588        &RAW_PACKING
589    };
590    Ok(Some(ArrayEncoding {
591        data,
592        accession,
593        array_cv_code,
594        dtype,
595        dtype_enum,
596        elem_bytes,
597        packing,
598    }))
599}
600
601pub(crate) fn encode_single_array(
602    bda: &BinaryDataArray,
603    config: WriteOptions,
604    policy: ArrayPolicy,
605    container: &mut BlockWriter<'_, DefaultCompressor>,
606) -> IonResult<Option<EncodedArrayAddress>> {
607    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
608        return Ok(None);
609    };
610    let ArrayEncoding {
611        data,
612        accession,
613        array_cv_code,
614        dtype,
615        dtype_enum,
616        elem_bytes,
617        packing,
618    } = encoding;
619    let (block_id, element_offset, encoded_len) = if packing.is_variable_length() {
620        encode_variable_length_array(accession, data, dtype, dtype_enum, packing, container)?
621    } else {
622        encode_fixed_length_array(
623            accession, data, dtype, dtype_enum, elem_bytes, packing, container,
624        )?
625    };
626    Ok(Some(EncodedArrayAddress {
627        element_offset,
628        element_count: data.element_count() as u64,
629        block_id,
630        accession,
631        dtype,
632        array_filter: packing.id() as u8,
633        encoded_len,
634        continues_previous_segment: 0,
635        array_cv_code,
636    }))
637}
638
639pub(crate) fn array_is_fixed_width_splittable(
640    bda: &BinaryDataArray,
641    config: WriteOptions,
642    policy: ArrayPolicy,
643) -> IonResult<Option<(usize, usize)>> {
644    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
645        return Ok(None);
646    };
647    if encoding.packing.is_variable_length() {
648        return Ok(None);
649    }
650    Ok(Some((encoding.data.element_count(), encoding.elem_bytes)))
651}
652
653pub(crate) fn check_spectrum_rt_order(
654    summary: &[u8],
655    spectrum_index: usize,
656    last_rt: &mut f64,
657) -> IonResult<()> {
658    let rt = f64::from_le_bytes(summary[0..8].try_into().unwrap());
659    if !rt.is_finite() {
660        return Ok(());
661    }
662    let previous = *last_rt;
663    if rt < previous {
664        return Err(format!(
665            "spectrum {spectrum_index}: retention time {rt} is below the previous spectrum's \
666             ({previous}); spectra must be stored in ascending rt order"
667        )
668        .into());
669    }
670    *last_rt = rt;
671    Ok(())
672}
673
674pub(crate) fn check_spectrum_mz_order(
675    arrays: &[BinaryDataArray],
676    spectrum_index: usize,
677) -> IonResult<()> {
678    use crate::accessions::MZ_ARRAY;
679    for bda in arrays {
680        if array_type_accession_from_binary_data_array(bda) != MZ_ARRAY {
681            continue;
682        }
683        let Some(data) = array_data_from_binary_data_array(bda) else {
684            return Ok(());
685        };
686        if !data.is_monotonic_non_decreasing() {
687            return Err(format!(
688                "spectrum {spectrum_index}: m/z array must be sorted ascending (non-decreasing) for range reads"
689            )
690            .into());
691        }
692        return Ok(());
693    }
694    Ok(())
695}
696
697pub(crate) fn window_ranges_for_item(
698    arrays: &[BinaryDataArray],
699    config: WriteOptions,
700    policy: ArrayPolicy,
701    mz_window: f64,
702) -> IonResult<Option<Vec<WindowRange>>> {
703    let windowing = Windowing::new(mz_window.round());
704    if !windowing.is_enabled() {
705        return Ok(None);
706    }
707    let Some(x_array) = arrays
708        .iter()
709        .find(|bda| array_type_accession_from_binary_data_array(bda) == policy.x_array_accession)
710    else {
711        return Ok(None);
712    };
713    let Some((x_count, _)) = array_is_fixed_width_splittable(x_array, config, policy)? else {
714        return Ok(None);
715    };
716    if x_count == 0 {
717        return Ok(None);
718    }
719    for bda in arrays {
720        match array_is_fixed_width_splittable(bda, config, policy)? {
721            Some((count, _)) if count == x_count => {}
722            _ => return Ok(None),
723        }
724    }
725    let Some(x_data) = array_data_from_binary_data_array(x_array) else {
726        return Ok(None);
727    };
728    let ranges = windowing.split_sorted(x_count, |index| x_data.value_at(index));
729    const MAX_WINDOW_INDEX: u32 = 1 << 20;
730    if ranges
731        .last()
732        .is_some_and(|range| range.window_index > MAX_WINDOW_INDEX)
733    {
734        return Err(format!(
735            "m/z window index exceeds limit {MAX_WINDOW_INDEX}; m/z value too large for window width {}",
736            mz_window.round()
737        )
738        .into());
739    }
740    Ok(Some(ranges))
741}
742
743pub(crate) fn write_array_windows(
744    bda: &BinaryDataArray,
745    config: WriteOptions,
746    policy: ArrayPolicy,
747    windows: &[WindowRange],
748    container: &mut BlockWriter<'_, DefaultCompressor>,
749) -> IonResult<Vec<EncodedArrayAddress>> {
750    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
751        return Ok(Vec::new());
752    };
753    let ArrayEncoding {
754        data,
755        accession,
756        array_cv_code,
757        dtype,
758        dtype_enum,
759        elem_bytes,
760        packing,
761    } = encoding;
762
763    if packing.is_variable_length() {
764        return Err("write_array_windows: variable-length arrays cannot be windowed".into());
765    }
766
767    let mut addresses = Vec::with_capacity(windows.len());
768    for (segment_index, window) in windows.iter().enumerate() {
769        let segment_data = data.slice(window.start, window.end);
770        let element_count = window.element_count();
771        let (block_id, element_offset) = container.add_item_to_window(
772            accession,
773            window.window_index,
774            element_count * elem_bytes,
775            elem_bytes,
776            |buf| write_fixed_array_payload(buf, segment_data, dtype, dtype_enum, packing),
777        )?;
778        addresses.push(EncodedArrayAddress {
779            element_offset,
780            element_count: element_count as u64,
781            block_id,
782            accession,
783            dtype,
784            array_filter: packing.id() as u8,
785            encoded_len: 0,
786            continues_previous_segment: (segment_index > 0) as u8,
787            array_cv_code,
788        });
789    }
790    Ok(addresses)
791}
792
793#[cfg(test)]
794mod tests {
795    use super::*;
796    use crate::{
797        ion::{
798            encoder::{ion_writer::write_mzml_to_ion, utilities::output::WriteBytes},
799            format::{FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE},
800            header::{
801                HEADER_CHROM_BLOCK_COUNT, HEADER_SPECTRUM_BLOCK_COUNT, HEADER_TARGET_BLOCK_SIZE,
802                HEADER_TARGET_MZ_WINDOW, HEADER_TOTAL_FILE_SIZE,
803            },
804        },
805        mzml::structs::MzML,
806    };
807
808    fn summary_with_rt(rt: f64) -> [u8; SPEC_SUMMARY_SIZE] {
809        let mut summary = [0u8; SPEC_SUMMARY_SIZE];
810        summary[0..8].copy_from_slice(&rt.to_le_bytes());
811        summary
812    }
813
814    #[test]
815    fn byte_shuffle_array_encode_arm_shuffles_not_raw_71() {
816        use crate::ion::packing::packing_by_id;
817
818        let data: Vec<f64> = (0..600)
819            .map(|i| 100.0 + (i as f64) * 0.0137 + ((i * 7 % 13) as f64) * 0.0009)
820            .collect();
821
822        let byte_shuffle = packing_by_id(PackingId::ByteShuffle);
823
824        let mut shuffled = Vec::new();
825        write_fixed_array_payload(
826            &mut shuffled,
827            ArrayData::F64(&data),
828            FILE_DTYPE_F64,
829            Dtype::F64,
830            byte_shuffle,
831        )
832        .unwrap();
833
834        let mut raw = Vec::new();
835        write_array_data(&mut raw, ArrayData::F64(&data), FILE_DTYPE_F64);
836
837        assert_eq!(shuffled.len(), raw.len());
838        assert_ne!(
839            shuffled, raw,
840            "ByteShuffle encode arm must shuffle, not write raw bytes"
841        );
842
843        let mut expected = Vec::new();
844        byte_shuffle
845            .encode(PackingInput::F64(&data), &mut expected)
846            .unwrap();
847        assert_eq!(
848            shuffled, expected,
849            "ByteShuffle encode arm must produce exactly the SIMD byte-transpose output"
850        );
851    }
852
853    #[test]
854    fn rt_order_allows_ascending_and_missing_but_rejects_a_drop() {
855        let mut last_rt = f64::NEG_INFINITY;
856        check_spectrum_rt_order(&summary_with_rt(1.0), 0, &mut last_rt)
857            .expect("first rt is allowed");
858        check_spectrum_rt_order(&summary_with_rt(1.0), 1, &mut last_rt)
859            .expect("equal rt is allowed");
860        check_spectrum_rt_order(&summary_with_rt(2.0), 2, &mut last_rt)
861            .expect("rising rt is allowed");
862
863        check_spectrum_rt_order(&summary_with_rt(f64::NAN), 3, &mut last_rt)
864            .expect("missing rt is allowed");
865        assert_eq!(last_rt, 2.0, "missing rt must not move the last seen rt");
866
867        let dropped = check_spectrum_rt_order(&summary_with_rt(1.5), 4, &mut last_rt);
868        let message = dropped
869            .expect_err("rt below the previous spectrum must be rejected")
870            .to_string();
871        assert!(
872            message.contains("spectrum 4"),
873            "error names the spectrum: {message}"
874        );
875        assert!(
876            message.contains("ascending rt order"),
877            "error explains the rule: {message}"
878        );
879    }
880
881    #[test]
882    fn encode_starts_with_magic_and_ends_with_trailer() {
883        let mzml = MzML::default();
884        let mut buf = Vec::new();
885        write_mzml_to_ion(
886            &mzml,
887            WriteOptions {
888                compression_level: 0,
889                force_f32: false,
890                ..Default::default()
891            },
892            &mut buf,
893        )
894        .unwrap();
895        assert!(buf.len() >= HEADER_SIZE + FILE_TRAILER.len());
896        assert_eq!(&buf[..FILE_SIGNATURE.len()], &FILE_SIGNATURE);
897        assert_eq!(&buf[buf.len() - FILE_TRAILER.len()..], &FILE_TRAILER);
898    }
899
900    #[test]
901    fn encode_total_file_size_is_correct() {
902        let mzml = MzML::default();
903        let mut buf = Vec::new();
904        write_mzml_to_ion(
905            &mzml,
906            WriteOptions {
907                compression_level: 0,
908                force_f32: false,
909                ..Default::default()
910            },
911            &mut buf,
912        )
913        .unwrap();
914        let total = u64::from_le_bytes(
915            buf[HEADER_TOTAL_FILE_SIZE..HEADER_TOTAL_FILE_SIZE + 8]
916                .try_into()
917                .unwrap(),
918        );
919        assert_eq!(total, buf.len() as u64);
920    }
921
922    #[test]
923    fn encode_header_target_block_size_matches_default() {
924        let mzml = MzML::default();
925        let mut buf = Vec::new();
926        write_mzml_to_ion(
927            &mzml,
928            WriteOptions {
929                compression_level: 0,
930                force_f32: false,
931                ..Default::default()
932            },
933            &mut buf,
934        )
935        .unwrap();
936        let size = u64::from_le_bytes(
937            buf[HEADER_TARGET_BLOCK_SIZE..HEADER_TARGET_BLOCK_SIZE + 8]
938                .try_into()
939                .unwrap(),
940        );
941        assert_eq!(size, TARGET_BLOCK_UNCOMPRESSED_BYTES as u64);
942    }
943
944    #[test]
945    fn encode_header_target_mz_window_matches_default() {
946        let mzml = MzML::default();
947        let mut buf = Vec::new();
948        write_mzml_to_ion(
949            &mzml,
950            WriteOptions {
951                compression_level: 0,
952                force_f32: false,
953                ..Default::default()
954            },
955            &mut buf,
956        )
957        .unwrap();
958        let window = u32::from_le_bytes(
959            buf[HEADER_TARGET_MZ_WINDOW..HEADER_TARGET_MZ_WINDOW + 4]
960                .try_into()
961                .unwrap(),
962        );
963        assert_eq!(window, DEFAULT_MZ_WINDOW.round() as u32);
964    }
965
966    #[test]
967    fn encode_header_crc32_is_valid() {
968        let mzml = MzML::default();
969        let mut buf = Vec::new();
970        write_mzml_to_ion(
971            &mzml,
972            WriteOptions {
973                compression_level: 0,
974                force_f32: false,
975                ..Default::default()
976            },
977            &mut buf,
978        )
979        .unwrap();
980        let stored = u32::from_le_bytes(buf[1020..1024].try_into().unwrap());
981        assert_eq!(stored, crc32fast::hash(&buf[0..1020]));
982    }
983
984    #[test]
985    fn encode_header_block_counts_are_zero_for_empty_mzml() {
986        let mzml = MzML::default();
987        let mut buf = Vec::new();
988        write_mzml_to_ion(
989            &mzml,
990            WriteOptions {
991                compression_level: 0,
992                force_f32: false,
993                ..Default::default()
994            },
995            &mut buf,
996        )
997        .unwrap();
998        let spec_blocks = u64::from_le_bytes(
999            buf[HEADER_SPECTRUM_BLOCK_COUNT..HEADER_SPECTRUM_BLOCK_COUNT + 8]
1000                .try_into()
1001                .unwrap(),
1002        );
1003        let chrom_blocks = u64::from_le_bytes(
1004            buf[HEADER_CHROM_BLOCK_COUNT..HEADER_CHROM_BLOCK_COUNT + 8]
1005                .try_into()
1006                .unwrap(),
1007        );
1008        assert_eq!(spec_blocks, 0);
1009        assert_eq!(chrom_blocks, 0);
1010    }
1011
1012    #[test]
1013    fn vec_patch_in_bounds() {
1014        let mut output = vec![0u8; 16];
1015        output.patch(4, &[1u8, 2, 3, 4]).unwrap();
1016        assert_eq!(&output[4..8], &[1u8, 2, 3, 4]);
1017    }
1018
1019    #[test]
1020    fn vec_patch_out_of_bounds_errors() {
1021        let mut output = vec![0u8; 4];
1022        assert!(output.patch(3, &[1u8, 2, 3]).is_err());
1023    }
1024
1025    #[test]
1026    fn declared_float_precision_prefers_numeric_type_field() {
1027        let bda = BinaryDataArray {
1028            numeric_type: Some(NumericType::Float64),
1029            ..Default::default()
1030        };
1031        assert_eq!(declared_float_precision_is_64bit(&bda), Some(true));
1032        let bda = BinaryDataArray {
1033            numeric_type: Some(NumericType::Float32),
1034            ..Default::default()
1035        };
1036        assert_eq!(declared_float_precision_is_64bit(&bda), Some(false));
1037    }
1038
1039    #[test]
1040    fn resolve_array_dtype_force_f32_overrides_f64_data() {
1041        let bda = BinaryDataArray::default();
1042        assert_eq!(
1043            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1044            FILE_DTYPE_F32
1045        );
1046    }
1047
1048    #[test]
1049    fn resolve_array_dtype_keeps_f64_data_lossless_even_when_declared_32bit() {
1050        let bda = BinaryDataArray {
1051            numeric_type: Some(NumericType::Float32),
1052            ..Default::default()
1053        };
1054        assert_eq!(
1055            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), false),
1056            FILE_DTYPE_F64
1057        );
1058        assert_eq!(
1059            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1060            FILE_DTYPE_F32
1061        );
1062    }
1063
1064    #[test]
1065    fn resolve_array_dtype_integer_types_unchanged_by_force_f32() {
1066        let bda = BinaryDataArray::default();
1067        assert_eq!(
1068            resolve_array_dtype(&bda, ArrayData::I32(&[1i32]), true),
1069            FILE_DTYPE_I32
1070        );
1071    }
1072
1073    #[test]
1074    fn element_byte_size_for_dtype_returns_correct_sizes() {
1075        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F64), 8);
1076        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F32), 4);
1077        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F16), 2);
1078        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I16), 2);
1079        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I32), 4);
1080        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I64), 8);
1081    }
1082
1083    #[test]
1084    fn encoder_config_compression_disabled_at_level_zero() {
1085        let config = WriteOptions {
1086            compression_level: 0,
1087            force_f32: false,
1088            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1089            parallel: true,
1090            section_storage: SectionStorage::Memory,
1091            mz_window: 0.0,
1092        };
1093        assert!(!config.compression_is_enabled());
1094        assert_eq!(config.codec_id(), 0);
1095        assert_eq!(config.array_filter_id(), PackingId::Raw as u8);
1096        assert!(matches!(config.block_packing_id(), PackingId::Raw));
1097    }
1098
1099    #[test]
1100    fn encoder_config_compression_enabled_at_nonzero_level() {
1101        let config = WriteOptions {
1102            compression_level: 3,
1103            force_f32: false,
1104            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1105            parallel: true,
1106            section_storage: SectionStorage::Memory,
1107            mz_window: 0.0,
1108        };
1109        assert!(config.compression_is_enabled());
1110        assert_eq!(config.codec_id(), 1);
1111        assert_eq!(config.array_filter_id(), PackingId::ByteShuffle as u8);
1112        assert!(matches!(config.block_packing_id(), PackingId::ByteShuffle));
1113    }
1114}