Skip to main content

ionic/ion/encoder/
encode.rs

1use crate::{
2    accessions::{
3        FLOAT_32BIT, FLOAT_64BIT, HIGHEST_ION_MOBILITY, HIGHEST_OBSERVED_MZ,
4        HIGHEST_OBSERVED_WAVELENGTH, INTENSITY_ARRAY, LOWEST_ION_MOBILITY, LOWEST_OBSERVED_MZ,
5        LOWEST_OBSERVED_WAVELENGTH, NEGATIVE_SCAN, POSITION_X, POSITION_Y, POSITION_Z,
6        POSITIVE_SCAN,
7    },
8    ion::{
9        IonError, IonResult,
10        encoder::utilities::{
11            BlockWriter, CompressionMode, DefaultCompressor,
12            le_writers::{
13                write_f32_le, write_f32_slice_le, write_f64_le, write_f64_slice_le,
14                write_i16_slice_le, write_i32_slice_le, write_i64_slice_le, write_u16_slice_le,
15            },
16            meta_collector::{
17                ArrayPolicy, array_type_accession_from_binary_data_array,
18                array_type_cv_code_from_binary_data_array, parse_accession_tail_raw,
19            },
20            output::{SectionStorage, WriteBytes},
21        },
22        filter_summary::{ChromatogramSummary, SpectrumSummary},
23        packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING},
24        utilities::spectrum_source::{f16_bits_to_f64, summary_from_spectrum},
25        windowing::{WindowRange, Windowing},
26    },
27    mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum},
28};
29pub(crate) const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024;
30pub const DEFAULT_MZ_WINDOW: f64 = 20.0;
31pub(crate) const SPEC_SUMMARY_SIZE: usize = 80;
32pub(crate) const CHROM_SUMMARY_SIZE: usize = 80;
33
34pub(crate) const FILE_DTYPE_F64: u8 = Dtype::F64 as u8;
35pub(crate) const FILE_DTYPE_F32: u8 = Dtype::F32 as u8;
36pub(crate) const FILE_DTYPE_F16: u8 = Dtype::F16 as u8;
37pub(crate) const FILE_DTYPE_I16: u8 = Dtype::I16 as u8;
38pub(crate) const FILE_DTYPE_I32: u8 = Dtype::I32 as u8;
39pub(crate) const FILE_DTYPE_I64: u8 = Dtype::I64 as u8;
40
41const POLARITY_UNKNOWN: u8 = 0;
42const POLARITY_POSITIVE: u8 = 1;
43const POLARITY_NEGATIVE: u8 = 2;
44
45impl ChromatogramSummary {
46    fn unknown() -> Self {
47        Self {
48            lowest_mz: f64::NAN,
49            highest_mz: f64::NAN,
50            lowest_wavelength: f64::NAN,
51            highest_wavelength: f64::NAN,
52            lowest_ion_mobility: f64::NAN,
53            highest_ion_mobility: f64::NAN,
54            polarity: POLARITY_UNKNOWN,
55        }
56    }
57}
58
59pub(crate) fn spec_summary_from_spectrum(spectrum: &Spectrum) -> SpectrumSummary {
60    let summary = summary_from_spectrum(spectrum);
61    let (position_x, position_y, position_z) = get_spectrum_position(spectrum);
62    SpectrumSummary {
63        rt: summary.rt,
64        rt_unit: summary.rt_unit.code(),
65        base_peak_mz: summary.base_peak_mz,
66        selected_ion_mz: summary.selected_ion_mz,
67        base_peak_int: summary.base_peak_int,
68        total_ion_current: summary.total_ion_current,
69        ms_level: summary.ms_level,
70        polarity: summary.polarity,
71        position_x,
72        position_y,
73        position_z,
74    }
75}
76
77fn get_spectrum_position(spectrum: &Spectrum) -> (u32, u32, u32) {
78    let mut x = 0;
79    let mut y = 0;
80    let mut z = 0;
81    let scan_list = spectrum.scan_list.as_ref().or_else(|| {
82        spectrum
83            .spectrum_description
84            .as_ref()
85            .and_then(|description| description.scan_list.as_ref())
86    });
87    let Some(scan_list) = scan_list else {
88        return (x, y, z);
89    };
90    for scan in &scan_list.scans {
91        for param in &scan.cv_params {
92            let Some(accession) = param.accession.as_deref() else {
93                continue;
94            };
95            if !accession.starts_with("IMS:") {
96                continue;
97            }
98            let value = param
99                .value
100                .as_deref()
101                .and_then(|value| value.parse::<u32>().ok())
102                .unwrap_or(0);
103            match parse_accession_tail_raw(Some(accession)) {
104                POSITION_X => x = value,
105                POSITION_Y => y = value,
106                POSITION_Z => z = value,
107                _ => {}
108            }
109        }
110    }
111    (x, y, z)
112}
113
114#[inline]
115fn cv_f64(cv: &CvParam) -> Option<f64> {
116    cv.value.as_deref().and_then(|s| s.parse::<f64>().ok())
117}
118
119pub(crate) fn extract_chrom_summary(chrom: &Chromatogram) -> ChromatogramSummary {
120    let mut summary = ChromatogramSummary::unknown();
121    for cv in &chrom.cv_params {
122        match parse_accession_tail_raw(cv.accession.as_deref()) {
123            LOWEST_OBSERVED_MZ => {
124                if let Some(v) = cv_f64(cv) {
125                    summary.lowest_mz = v;
126                }
127            }
128            HIGHEST_OBSERVED_MZ => {
129                if let Some(v) = cv_f64(cv) {
130                    summary.highest_mz = v;
131                }
132            }
133            LOWEST_OBSERVED_WAVELENGTH => {
134                if let Some(v) = cv_f64(cv) {
135                    summary.lowest_wavelength = v;
136                }
137            }
138            HIGHEST_OBSERVED_WAVELENGTH => {
139                if let Some(v) = cv_f64(cv) {
140                    summary.highest_wavelength = v;
141                }
142            }
143            LOWEST_ION_MOBILITY => {
144                if let Some(v) = cv_f64(cv) {
145                    summary.lowest_ion_mobility = v;
146                }
147            }
148            HIGHEST_ION_MOBILITY => {
149                if let Some(v) = cv_f64(cv) {
150                    summary.highest_ion_mobility = v;
151                }
152            }
153            POSITIVE_SCAN => summary.polarity = POLARITY_POSITIVE,
154            NEGATIVE_SCAN => summary.polarity = POLARITY_NEGATIVE,
155            _ => {}
156        }
157    }
158    summary
159}
160
161pub(crate) fn get_supported_compression_level(compression_level: u8) -> u8 {
162    match compression_level {
163        0..=5 => 1,
164        6..=10 => 9,
165        11..=16 => 12,
166        _ => 22,
167    }
168}
169
170pub(crate) fn allow_compression_level(compression_level: u8) -> IonResult<()> {
171    if compression_level > 22 {
172        return Err(format!("compression_level must be 0-22, got {compression_level}").into());
173    }
174    Ok(())
175}
176
177#[derive(Debug, Clone, Copy)]
178pub struct WriteOptions {
179    pub compression_level: u8,
180    pub force_f32: bool,
181    pub block_size: usize,
182    pub parallel: bool,
183    pub section_storage: SectionStorage,
184    pub mz_window: f64,
185}
186
187impl Default for WriteOptions {
188    fn default() -> Self {
189        Self {
190            compression_level: 12,
191            force_f32: false,
192            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
193            parallel: true,
194            section_storage: SectionStorage::Disk,
195            mz_window: DEFAULT_MZ_WINDOW,
196        }
197    }
198}
199
200impl WriteOptions {
201    pub(crate) fn compression_is_enabled(self) -> bool {
202        self.compression_level != 0
203    }
204
205    pub(crate) fn codec_id(self) -> u8 {
206        self.compression_is_enabled() as u8
207    }
208
209    fn block_shuffle_is_enabled(self) -> bool {
210        self.compression_is_enabled()
211    }
212
213    pub(crate) fn array_filter_id(self) -> u8 {
214        if self.block_shuffle_is_enabled() {
215            PackingId::ByteShuffle as u8
216        } else {
217            PackingId::Raw as u8
218        }
219    }
220
221    pub(crate) fn compression_mode(self) -> IonResult<CompressionMode<DefaultCompressor>> {
222        if self.compression_is_enabled() {
223            Ok(CompressionMode::Compressed(DefaultCompressor::new(
224                self.compression_level as i32,
225            )?))
226        } else {
227            Ok(CompressionMode::Raw)
228        }
229    }
230
231    pub(crate) fn block_packing_id(self) -> PackingId {
232        if self.block_shuffle_is_enabled() {
233            PackingId::ByteShuffle
234        } else {
235            PackingId::Raw
236        }
237    }
238
239    pub(crate) fn array_policy(self, x_array_accession: u32) -> ArrayPolicy {
240        ArrayPolicy {
241            x_array_accession,
242            y_array_accession: INTENSITY_ARRAY,
243            force_f32: self.force_f32,
244        }
245    }
246}
247
248#[derive(Copy, Clone)]
249enum ArrayData<'a> {
250    F16(&'a [u16]),
251    F32(&'a [f32]),
252    F64(&'a [f64]),
253    I16(&'a [i16]),
254    I32(&'a [i32]),
255    I64(&'a [i64]),
256}
257
258impl<'a> ArrayData<'a> {
259    fn element_count(self) -> usize {
260        match self {
261            Self::F16(e) => e.len(),
262            Self::F32(e) => e.len(),
263            Self::F64(e) => e.len(),
264            Self::I16(e) => e.len(),
265            Self::I32(e) => e.len(),
266            Self::I64(e) => e.len(),
267        }
268    }
269
270    fn is_empty(self) -> bool {
271        self.element_count() == 0
272    }
273
274    fn slice(self, start: usize, end: usize) -> ArrayData<'a> {
275        match self {
276            Self::F16(e) => Self::F16(&e[start..end]),
277            Self::F32(e) => Self::F32(&e[start..end]),
278            Self::F64(e) => Self::F64(&e[start..end]),
279            Self::I16(e) => Self::I16(&e[start..end]),
280            Self::I32(e) => Self::I32(&e[start..end]),
281            Self::I64(e) => Self::I64(&e[start..end]),
282        }
283    }
284
285    fn value_at(self, index: usize) -> f64 {
286        match self {
287            Self::F16(e) => f16_bits_to_f64(e[index]),
288            Self::F32(e) => e[index] as f64,
289            Self::F64(e) => e[index],
290            Self::I16(e) => e[index] as f64,
291            Self::I32(e) => e[index] as f64,
292            Self::I64(e) => e[index] as f64,
293        }
294    }
295
296    fn is_monotonic_non_decreasing(self) -> bool {
297        let count = self.element_count();
298        if count < 2 {
299            return true;
300        }
301        let mut previous = self.value_at(0);
302        for index in 1..count {
303            let current = self.value_at(index);
304            if current < previous {
305                return false;
306            }
307            previous = current;
308        }
309        true
310    }
311}
312
313fn array_data_from_binary_data_array(bda: &BinaryDataArray) -> Option<ArrayData<'_>> {
314    match bda.binary.as_ref()? {
315        NumericArray::F16(e) => Some(ArrayData::F16(e)),
316        NumericArray::I16(e) => Some(ArrayData::I16(e)),
317        NumericArray::I32(e) => Some(ArrayData::I32(e)),
318        NumericArray::I64(e) => Some(ArrayData::I64(e)),
319        NumericArray::F32(e) => Some(ArrayData::F32(e)),
320        NumericArray::F64(e) => Some(ArrayData::F64(e)),
321    }
322}
323
324fn element_byte_size_for_dtype(dtype: u8) -> usize {
325    match dtype {
326        FILE_DTYPE_F16 | FILE_DTYPE_I16 => 2,
327        FILE_DTYPE_F32 | FILE_DTYPE_I32 => 4,
328        FILE_DTYPE_F64 | FILE_DTYPE_I64 => 8,
329        _ => 1,
330    }
331}
332
333fn resolve_array_dtype(bda: &BinaryDataArray, data: ArrayData<'_>, force_f32: bool) -> u8 {
334    match data {
335        ArrayData::F16(_) => FILE_DTYPE_F16,
336        ArrayData::I16(_) => FILE_DTYPE_I16,
337        ArrayData::I32(_) => FILE_DTYPE_I32,
338        ArrayData::I64(_) => FILE_DTYPE_I64,
339        ArrayData::F32(_) | ArrayData::F64(_) => {
340            if float_data_should_be_written_as_f64(bda, data, force_f32) {
341                FILE_DTYPE_F64
342            } else {
343                FILE_DTYPE_F32
344            }
345        }
346    }
347}
348
349fn float_data_should_be_written_as_f64(
350    bda: &BinaryDataArray,
351    data: ArrayData<'_>,
352    force_f32: bool,
353) -> bool {
354    if force_f32 {
355        return false;
356    }
357    if matches!(data, ArrayData::F64(_)) {
358        return true;
359    }
360    declared_float_precision_is_64bit(bda).unwrap_or(false)
361}
362
363fn declared_float_precision_is_64bit(bda: &BinaryDataArray) -> Option<bool> {
364    if let Some(nt) = bda.numeric_type.as_ref() {
365        return match nt {
366            NumericType::Float64 => Some(true),
367            NumericType::Float32 => Some(false),
368            _ => None,
369        };
370    }
371    let (mut saw32, mut saw64) = (false, false);
372    for cv in &bda.cv_params {
373        match parse_accession_tail_raw(cv.accession.as_deref()) {
374            FLOAT_32BIT => saw32 = true,
375            FLOAT_64BIT => saw64 = true,
376            _ => {}
377        }
378        if saw32 && saw64 {
379            break;
380        }
381    }
382    match (saw32, saw64) {
383        (true, false) => Some(false),
384        (false, true) => Some(true),
385        _ => None,
386    }
387}
388
389fn validate_array_dtype(data: ArrayData<'_>, dtype: u8) -> IonResult<()> {
390    let ok = matches!(
391        (dtype, data),
392        (FILE_DTYPE_F16, ArrayData::F16(_))
393            | (FILE_DTYPE_F32, ArrayData::F32(_))
394            | (FILE_DTYPE_F32, ArrayData::F64(_))
395            | (FILE_DTYPE_F64, ArrayData::F64(_))
396            | (FILE_DTYPE_F64, ArrayData::F32(_))
397            | (FILE_DTYPE_I16, ArrayData::I16(_))
398            | (FILE_DTYPE_I32, ArrayData::I32(_))
399            | (FILE_DTYPE_I64, ArrayData::I64(_))
400    );
401    if ok {
402        Ok(())
403    } else {
404        Err(format!(
405            "write_array_data: incompatible dtype {dtype} for the given array data variant"
406        )
407        .into())
408    }
409}
410
411fn write_array_data(buf: &mut Vec<u8>, data: ArrayData<'_>, dtype: u8) {
412    match (dtype, data) {
413        (FILE_DTYPE_F16, ArrayData::F16(e)) => write_u16_slice_le(buf, e),
414        (FILE_DTYPE_F32, ArrayData::F32(e)) => write_f32_slice_le(buf, e),
415        (FILE_DTYPE_F32, ArrayData::F64(e)) => {
416            for &v in e {
417                write_f32_le(buf, v as f32);
418            }
419        }
420        (FILE_DTYPE_F64, ArrayData::F64(e)) => write_f64_slice_le(buf, e),
421        (FILE_DTYPE_F64, ArrayData::F32(e)) => {
422            for &v in e {
423                write_f64_le(buf, v as f64);
424            }
425        }
426        (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e),
427        (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e),
428        (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e),
429        _ => unreachable!("write_array_data called with unvalidated dtype/data combination"),
430    }
431}
432
433fn narrow_f64_to_f32(values: &[f64]) -> Vec<f32> {
434    values.iter().map(|&value| value as f32).collect()
435}
436
437fn widen_f32_to_f64(values: &[f32]) -> Vec<f64> {
438    values.iter().map(|&value| value as f64).collect()
439}
440
441fn encode_floats_through_packing(
442    buf: &mut Vec<u8>,
443    data: ArrayData<'_>,
444    dtype_enum: Dtype,
445    packing: &'static dyn Packing,
446) -> IonResult<()> {
447    match (dtype_enum, data) {
448        (Dtype::F64, ArrayData::F64(slice)) => packing.encode(PackingInput::F64(slice), buf),
449        (Dtype::F32, ArrayData::F32(slice)) => packing.encode(PackingInput::F32(slice), buf),
450        (Dtype::F32, ArrayData::F64(slice)) => {
451            let narrowed = narrow_f64_to_f32(slice);
452            packing.encode(PackingInput::F32(&narrowed), buf)
453        }
454        (Dtype::F64, ArrayData::F32(slice)) => {
455            let widened = widen_f32_to_f64(slice);
456            packing.encode(PackingInput::F64(&widened), buf)
457        }
458        _ => Err(IonError::from(
459            "shuffle packing requires f32 or f64 array data",
460        )),
461    }
462}
463
464pub(crate) struct EncodedArrayAddress {
465    pub(crate) element_offset: u64,
466    pub(crate) element_count: u64,
467    pub(crate) block_id: u32,
468    pub(crate) accession: u32,
469    pub(crate) dtype: u8,
470    pub(crate) array_filter: u8,
471    pub(crate) encoded_len: u32,
472    pub(crate) continues_previous_segment: u8,
473    pub(crate) array_cv_code: u8,
474}
475
476fn encode_variable_length_array(
477    output: &mut dyn WriteBytes,
478    array_type: u32,
479    data: ArrayData<'_>,
480    dtype: u8,
481    dtype_enum: Dtype,
482    packing: &'static dyn Packing,
483    container: &mut BlockWriter<DefaultCompressor>,
484) -> IonResult<(u32, u64, u32)> {
485    let mut encoded = Vec::new();
486    match packing.id() {
487        PackingId::Raw => write_array_data(&mut encoded, data, dtype),
488        _ => encode_floats_through_packing(&mut encoded, data, dtype_enum, packing)?,
489    }
490    let enc_len =
491        u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?;
492    let (bid, eoff) = container.add_item_to_box(output, array_type, encoded.len(), 1, |buf| {
493        buf.extend_from_slice(&encoded);
494        Ok(())
495    })?;
496    Ok((bid, eoff, enc_len))
497}
498
499fn write_fixed_array_payload(
500    buf: &mut Vec<u8>,
501    data: ArrayData<'_>,
502    dtype: u8,
503    dtype_enum: Dtype,
504    packing: &'static dyn Packing,
505) -> IonResult<()> {
506    match packing.id() {
507        PackingId::Raw => {
508            write_array_data(buf, data, dtype);
509            Ok(())
510        }
511        _ => encode_floats_through_packing(buf, data, dtype_enum, packing),
512    }
513}
514
515#[allow(clippy::too_many_arguments)]
516fn encode_fixed_length_array(
517    output: &mut dyn WriteBytes,
518    array_type: u32,
519    data: ArrayData<'_>,
520    dtype: u8,
521    dtype_enum: Dtype,
522    elem_bytes: usize,
523    packing: &'static dyn Packing,
524    container: &mut BlockWriter<DefaultCompressor>,
525) -> IonResult<(u32, u64, u32)> {
526    let (bid, eoff) = container.add_item_to_box(
527        output,
528        array_type,
529        data.element_count() * elem_bytes,
530        elem_bytes,
531        |buf| write_fixed_array_payload(buf, data, dtype, dtype_enum, packing),
532    )?;
533    Ok((bid, eoff, 0u32))
534}
535
536struct ArrayEncoding<'a> {
537    data: ArrayData<'a>,
538    accession: u32,
539    array_cv_code: u8,
540    dtype: u8,
541    dtype_enum: Dtype,
542    elem_bytes: usize,
543    packing: &'static dyn Packing,
544}
545
546fn resolve_array_encoding<'a>(
547    bda: &'a BinaryDataArray,
548    config: WriteOptions,
549    policy: ArrayPolicy,
550) -> IonResult<Option<ArrayEncoding<'a>>> {
551    let Some(data) = array_data_from_binary_data_array(bda) else {
552        return Ok(None);
553    };
554    if data.is_empty() {
555        return Ok(None);
556    }
557    let accession = array_type_accession_from_binary_data_array(bda);
558    let array_cv_code = array_type_cv_code_from_binary_data_array(bda);
559    let dtype = resolve_array_dtype(bda, data, policy.should_force_f32(accession));
560    validate_array_dtype(data, dtype)?;
561    let elem_bytes = element_byte_size_for_dtype(dtype);
562    let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64);
563    let requested: &'static dyn Packing = if config.compression_is_enabled() {
564        packing_for(accession, dtype_enum)
565    } else {
566        &RAW_PACKING
567    };
568    let packing: &'static dyn Packing = if requested.supports(dtype_enum) {
569        requested
570    } else {
571        &RAW_PACKING
572    };
573    Ok(Some(ArrayEncoding {
574        data,
575        accession,
576        array_cv_code,
577        dtype,
578        dtype_enum,
579        elem_bytes,
580        packing,
581    }))
582}
583
584pub(crate) fn encode_single_array(
585    output: &mut dyn WriteBytes,
586    bda: &BinaryDataArray,
587    config: WriteOptions,
588    policy: ArrayPolicy,
589    container: &mut BlockWriter<DefaultCompressor>,
590) -> IonResult<Option<EncodedArrayAddress>> {
591    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
592        return Ok(None);
593    };
594    let ArrayEncoding {
595        data,
596        accession,
597        array_cv_code,
598        dtype,
599        dtype_enum,
600        elem_bytes,
601        packing,
602    } = encoding;
603    let (block_id, element_offset, encoded_len) = if packing.is_variable_length() {
604        encode_variable_length_array(output, accession, data, dtype, dtype_enum, packing, container)?
605    } else {
606        encode_fixed_length_array(
607            output, accession, data, dtype, dtype_enum, elem_bytes, packing, container,
608        )?
609    };
610    Ok(Some(EncodedArrayAddress {
611        element_offset,
612        element_count: data.element_count() as u64,
613        block_id,
614        accession,
615        dtype,
616        array_filter: packing.id() as u8,
617        encoded_len,
618        continues_previous_segment: 0,
619        array_cv_code,
620    }))
621}
622
623pub(crate) fn array_is_fixed_width_splittable(
624    bda: &BinaryDataArray,
625    config: WriteOptions,
626    policy: ArrayPolicy,
627) -> IonResult<Option<(usize, usize)>> {
628    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
629        return Ok(None);
630    };
631    if encoding.packing.is_variable_length() {
632        return Ok(None);
633    }
634    Ok(Some((encoding.data.element_count(), encoding.elem_bytes)))
635}
636
637pub(crate) fn check_spectrum_rt_order(
638    summary: &[u8],
639    spectrum_index: usize,
640    last_rt: &mut f64,
641) -> IonResult<()> {
642    let rt = f64::from_le_bytes(summary[0..8].try_into().unwrap());
643    if !rt.is_finite() {
644        return Ok(());
645    }
646    let previous = *last_rt;
647    if rt < previous {
648        return Err(format!(
649            "spectrum {spectrum_index}: retention time {rt} is below the previous spectrum's \
650             ({previous}); spectra must be stored in ascending rt order"
651        )
652        .into());
653    }
654    *last_rt = rt;
655    Ok(())
656}
657
658pub(crate) fn check_spectrum_mz_order(
659    arrays: &[BinaryDataArray],
660    spectrum_index: usize,
661) -> IonResult<()> {
662    use crate::accessions::MZ_ARRAY;
663    for bda in arrays {
664        if array_type_accession_from_binary_data_array(bda) != MZ_ARRAY {
665            continue;
666        }
667        let Some(data) = array_data_from_binary_data_array(bda) else {
668            return Ok(());
669        };
670        if !data.is_monotonic_non_decreasing() {
671            return Err(format!(
672                "spectrum {spectrum_index}: m/z array must be sorted ascending (non-decreasing) for range reads"
673            )
674            .into());
675        }
676        return Ok(());
677    }
678    Ok(())
679}
680
681pub(crate) fn window_ranges_for_item(
682    arrays: &[BinaryDataArray],
683    config: WriteOptions,
684    policy: ArrayPolicy,
685    mz_window: f64,
686) -> IonResult<Option<Vec<WindowRange>>> {
687    let windowing = Windowing::new(mz_window.round());
688    if !windowing.is_enabled() {
689        return Ok(None);
690    }
691    let Some(x_array) = arrays
692        .iter()
693        .find(|bda| array_type_accession_from_binary_data_array(bda) == policy.x_array_accession)
694    else {
695        return Ok(None);
696    };
697    let Some((x_count, _)) = array_is_fixed_width_splittable(x_array, config, policy)? else {
698        return Ok(None);
699    };
700    if x_count == 0 {
701        return Ok(None);
702    }
703    for bda in arrays {
704        match array_is_fixed_width_splittable(bda, config, policy)? {
705            Some((count, _)) if count == x_count => {}
706            _ => return Ok(None),
707        }
708    }
709    let Some(x_data) = array_data_from_binary_data_array(x_array) else {
710        return Ok(None);
711    };
712    let ranges = windowing.split_sorted(x_count, |index| x_data.value_at(index));
713    const MAX_WINDOW_INDEX: u32 = 1 << 20;
714    if ranges
715        .last()
716        .is_some_and(|range| range.window_index > MAX_WINDOW_INDEX)
717    {
718        return Err(format!(
719            "m/z window index exceeds limit {MAX_WINDOW_INDEX}; m/z value too large for window width {}",
720            mz_window.round()
721        )
722        .into());
723    }
724    Ok(Some(ranges))
725}
726
727pub(crate) fn write_array_windows(
728    output: &mut dyn WriteBytes,
729    bda: &BinaryDataArray,
730    config: WriteOptions,
731    policy: ArrayPolicy,
732    windows: &[WindowRange],
733    container: &mut BlockWriter<DefaultCompressor>,
734) -> IonResult<Vec<EncodedArrayAddress>> {
735    let Some(encoding) = resolve_array_encoding(bda, config, policy)? else {
736        return Ok(Vec::new());
737    };
738    let ArrayEncoding {
739        data,
740        accession,
741        array_cv_code,
742        dtype,
743        dtype_enum,
744        elem_bytes,
745        packing,
746    } = encoding;
747
748    if packing.is_variable_length() {
749        return Err("write_array_windows: variable-length arrays cannot be windowed".into());
750    }
751
752    let mut addresses = Vec::with_capacity(windows.len());
753    for (segment_index, window) in windows.iter().enumerate() {
754        let segment_data = data.slice(window.start, window.end);
755        let element_count = window.element_count();
756        let (block_id, element_offset) = container.add_item_to_window(
757            output,
758            accession,
759            window.window_index,
760            element_count * elem_bytes,
761            elem_bytes,
762            |buf| write_fixed_array_payload(buf, segment_data, dtype, dtype_enum, packing),
763        )?;
764        addresses.push(EncodedArrayAddress {
765            element_offset,
766            element_count: element_count as u64,
767            block_id,
768            accession,
769            dtype,
770            array_filter: packing.id() as u8,
771            encoded_len: 0,
772            continues_previous_segment: (segment_index > 0) as u8,
773            array_cv_code,
774        });
775    }
776    Ok(addresses)
777}
778
779#[cfg(test)]
780mod tests {
781    use super::*;
782    use crate::{
783        ion::{
784            encoder::{ion_writer::write_mzml_to_ion, utilities::output::WriteBytes},
785            format::{FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE},
786            header::{
787                HEADER_CHROM_BLOCK_COUNT, HEADER_SPECTRUM_BLOCK_COUNT, HEADER_TARGET_BLOCK_SIZE,
788                HEADER_TARGET_MZ_WINDOW, HEADER_TOTAL_FILE_SIZE,
789            },
790        },
791        mzml::structs::MzML,
792    };
793
794    fn summary_with_rt(rt: f64) -> [u8; SPEC_SUMMARY_SIZE] {
795        let mut summary = [0u8; SPEC_SUMMARY_SIZE];
796        summary[0..8].copy_from_slice(&rt.to_le_bytes());
797        summary
798    }
799
800    #[test]
801    fn byte_shuffle_array_encode_arm_shuffles_not_raw_71() {
802        use crate::ion::packing::packing_by_id;
803
804        let data: Vec<f64> = (0..600)
805            .map(|i| 100.0 + (i as f64) * 0.0137 + ((i * 7 % 13) as f64) * 0.0009)
806            .collect();
807
808        let byte_shuffle = packing_by_id(PackingId::ByteShuffle);
809
810        let mut shuffled = Vec::new();
811        write_fixed_array_payload(
812            &mut shuffled,
813            ArrayData::F64(&data),
814            FILE_DTYPE_F64,
815            Dtype::F64,
816            byte_shuffle,
817        )
818        .unwrap();
819
820        let mut raw = Vec::new();
821        write_array_data(&mut raw, ArrayData::F64(&data), FILE_DTYPE_F64);
822
823        assert_eq!(shuffled.len(), raw.len());
824        assert_ne!(
825            shuffled, raw,
826            "ByteShuffle encode arm must shuffle, not write raw bytes"
827        );
828
829        let mut expected = Vec::new();
830        byte_shuffle
831            .encode(PackingInput::F64(&data), &mut expected)
832            .unwrap();
833        assert_eq!(
834            shuffled, expected,
835            "ByteShuffle encode arm must produce exactly the SIMD byte-transpose output"
836        );
837    }
838
839    #[test]
840    fn rt_order_allows_ascending_and_missing_but_rejects_a_drop() {
841        let mut last_rt = f64::NEG_INFINITY;
842        check_spectrum_rt_order(&summary_with_rt(1.0), 0, &mut last_rt)
843            .expect("first rt is allowed");
844        check_spectrum_rt_order(&summary_with_rt(1.0), 1, &mut last_rt)
845            .expect("equal rt is allowed");
846        check_spectrum_rt_order(&summary_with_rt(2.0), 2, &mut last_rt)
847            .expect("rising rt is allowed");
848
849        check_spectrum_rt_order(&summary_with_rt(f64::NAN), 3, &mut last_rt)
850            .expect("missing rt is allowed");
851        assert_eq!(last_rt, 2.0, "missing rt must not move the last seen rt");
852
853        let dropped = check_spectrum_rt_order(&summary_with_rt(1.5), 4, &mut last_rt);
854        let message = dropped
855            .expect_err("rt below the previous spectrum must be rejected")
856            .to_string();
857        assert!(
858            message.contains("spectrum 4"),
859            "error names the spectrum: {message}"
860        );
861        assert!(
862            message.contains("ascending rt order"),
863            "error explains the rule: {message}"
864        );
865    }
866
867    #[test]
868    fn encode_starts_with_magic_and_ends_with_trailer() {
869        let mzml = MzML::default();
870        let mut buf = Vec::new();
871        write_mzml_to_ion(
872            &mzml,
873            &WriteOptions {
874                compression_level: 0,
875                force_f32: false,
876                ..Default::default()
877            },
878            &mut buf,
879        )
880        .unwrap();
881        assert!(buf.len() >= HEADER_SIZE + FILE_TRAILER.len());
882        assert_eq!(&buf[..FILE_SIGNATURE.len()], &FILE_SIGNATURE);
883        assert_eq!(&buf[buf.len() - FILE_TRAILER.len()..], &FILE_TRAILER);
884    }
885
886    #[test]
887    fn encode_total_file_size_is_correct() {
888        let mzml = MzML::default();
889        let mut buf = Vec::new();
890        write_mzml_to_ion(
891            &mzml,
892            &WriteOptions {
893                compression_level: 0,
894                force_f32: false,
895                ..Default::default()
896            },
897            &mut buf,
898        )
899        .unwrap();
900        let total = u64::from_le_bytes(
901            buf[HEADER_TOTAL_FILE_SIZE..HEADER_TOTAL_FILE_SIZE + 8]
902                .try_into()
903                .unwrap(),
904        );
905        assert_eq!(total, buf.len() as u64);
906    }
907
908    #[test]
909    fn encode_header_target_block_size_matches_default() {
910        let mzml = MzML::default();
911        let mut buf = Vec::new();
912        write_mzml_to_ion(
913            &mzml,
914            &WriteOptions {
915                compression_level: 0,
916                force_f32: false,
917                ..Default::default()
918            },
919            &mut buf,
920        )
921        .unwrap();
922        let size = u64::from_le_bytes(
923            buf[HEADER_TARGET_BLOCK_SIZE..HEADER_TARGET_BLOCK_SIZE + 8]
924                .try_into()
925                .unwrap(),
926        );
927        assert_eq!(size, TARGET_BLOCK_UNCOMPRESSED_BYTES as u64);
928    }
929
930    #[test]
931    fn encode_header_target_mz_window_matches_default() {
932        let mzml = MzML::default();
933        let mut buf = Vec::new();
934        write_mzml_to_ion(
935            &mzml,
936            &WriteOptions {
937                compression_level: 0,
938                force_f32: false,
939                ..Default::default()
940            },
941            &mut buf,
942        )
943        .unwrap();
944        let window = u32::from_le_bytes(
945            buf[HEADER_TARGET_MZ_WINDOW..HEADER_TARGET_MZ_WINDOW + 4]
946                .try_into()
947                .unwrap(),
948        );
949        assert_eq!(window, DEFAULT_MZ_WINDOW.round() as u32);
950    }
951
952    #[test]
953    fn encode_header_crc32_is_valid() {
954        let mzml = MzML::default();
955        let mut buf = Vec::new();
956        write_mzml_to_ion(
957            &mzml,
958            &WriteOptions {
959                compression_level: 0,
960                force_f32: false,
961                ..Default::default()
962            },
963            &mut buf,
964        )
965        .unwrap();
966        let stored = u32::from_le_bytes(buf[1020..1024].try_into().unwrap());
967        assert_eq!(stored, crc32fast::hash(&buf[0..1020]));
968    }
969
970    #[test]
971    fn encode_header_block_counts_are_zero_for_empty_mzml() {
972        let mzml = MzML::default();
973        let mut buf = Vec::new();
974        write_mzml_to_ion(
975            &mzml,
976            &WriteOptions {
977                compression_level: 0,
978                force_f32: false,
979                ..Default::default()
980            },
981            &mut buf,
982        )
983        .unwrap();
984        let spec_blocks = u64::from_le_bytes(
985            buf[HEADER_SPECTRUM_BLOCK_COUNT..HEADER_SPECTRUM_BLOCK_COUNT + 8]
986                .try_into()
987                .unwrap(),
988        );
989        let chrom_blocks = u64::from_le_bytes(
990            buf[HEADER_CHROM_BLOCK_COUNT..HEADER_CHROM_BLOCK_COUNT + 8]
991                .try_into()
992                .unwrap(),
993        );
994        assert_eq!(spec_blocks, 0);
995        assert_eq!(chrom_blocks, 0);
996    }
997
998    #[test]
999    fn vec_patch_in_bounds() {
1000        let mut output = vec![0u8; 16];
1001        output.patch(4, &[1u8, 2, 3, 4]).unwrap();
1002        assert_eq!(&output[4..8], &[1u8, 2, 3, 4]);
1003    }
1004
1005    #[test]
1006    fn vec_patch_out_of_bounds_errors() {
1007        let mut output = vec![0u8; 4];
1008        assert!(output.patch(3, &[1u8, 2, 3]).is_err());
1009    }
1010
1011    #[test]
1012    fn declared_float_precision_prefers_numeric_type_field() {
1013        let bda = BinaryDataArray {
1014            numeric_type: Some(NumericType::Float64),
1015            ..Default::default()
1016        };
1017        assert_eq!(declared_float_precision_is_64bit(&bda), Some(true));
1018        let bda = BinaryDataArray {
1019            numeric_type: Some(NumericType::Float32),
1020            ..Default::default()
1021        };
1022        assert_eq!(declared_float_precision_is_64bit(&bda), Some(false));
1023    }
1024
1025    #[test]
1026    fn resolve_array_dtype_force_f32_overrides_f64_data() {
1027        let bda = BinaryDataArray::default();
1028        assert_eq!(
1029            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1030            FILE_DTYPE_F32
1031        );
1032    }
1033
1034    #[test]
1035    fn resolve_array_dtype_keeps_f64_data_lossless_even_when_declared_32bit() {
1036        let bda = BinaryDataArray {
1037            numeric_type: Some(NumericType::Float32),
1038            ..Default::default()
1039        };
1040        assert_eq!(
1041            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), false),
1042            FILE_DTYPE_F64
1043        );
1044        assert_eq!(
1045            resolve_array_dtype(&bda, ArrayData::F64(&[1.0f64]), true),
1046            FILE_DTYPE_F32
1047        );
1048    }
1049
1050    #[test]
1051    fn resolve_array_dtype_integer_types_unchanged_by_force_f32() {
1052        let bda = BinaryDataArray::default();
1053        assert_eq!(
1054            resolve_array_dtype(&bda, ArrayData::I32(&[1i32]), true),
1055            FILE_DTYPE_I32
1056        );
1057    }
1058
1059    #[test]
1060    fn element_byte_size_for_dtype_returns_correct_sizes() {
1061        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F64), 8);
1062        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F32), 4);
1063        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_F16), 2);
1064        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I16), 2);
1065        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I32), 4);
1066        assert_eq!(element_byte_size_for_dtype(FILE_DTYPE_I64), 8);
1067    }
1068
1069    #[test]
1070    fn encoder_config_compression_disabled_at_level_zero() {
1071        let config = WriteOptions {
1072            compression_level: 0,
1073            force_f32: false,
1074            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1075            parallel: true,
1076            section_storage: SectionStorage::Memory,
1077            mz_window: 0.0,
1078        };
1079        assert!(!config.compression_is_enabled());
1080        assert_eq!(config.codec_id(), 0);
1081        assert_eq!(config.array_filter_id(), PackingId::Raw as u8);
1082        assert!(matches!(config.block_packing_id(), PackingId::Raw));
1083    }
1084
1085    #[test]
1086    fn encoder_config_compression_enabled_at_nonzero_level() {
1087        let config = WriteOptions {
1088            compression_level: 3,
1089            force_f32: false,
1090            block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES,
1091            parallel: true,
1092            section_storage: SectionStorage::Memory,
1093            mz_window: 0.0,
1094        };
1095        assert!(config.compression_is_enabled());
1096        assert_eq!(config.codec_id(), 1);
1097        assert_eq!(config.array_filter_id(), PackingId::ByteShuffle as u8);
1098        assert!(matches!(config.block_packing_id(), PackingId::ByteShuffle));
1099    }
1100}