Skip to main content

ionic/ion/decoder/decode/
to_mzml.rs

1use std::{borrow::Cow, collections::HashMap};
2
3#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
4use rayon::prelude::*;
5
6use super::*;
7use crate::ion::decoder::utilities::byte_source::SourceBytes;
8
9#[non_exhaustive]
10#[derive(Debug, Clone, PartialEq)]
11pub enum MetadatumValue {
12    Number(f64),
13    Text(String),
14    Empty,
15}
16
17#[non_exhaustive]
18#[derive(Debug, Clone, PartialEq)]
19pub struct Metadatum {
20    pub item_index: u32,
21    pub id: u32,
22    pub parent_id: u32,
23    pub tag_id: TagId,
24    pub accession: Option<String>,
25    pub unit_accession: Option<String>,
26    pub value: MetadatumValue,
27}
28
29pub(crate) struct MzmlConverter<'d> {
30    pub(crate) decoder: &'d mut IonReader,
31}
32
33impl<'d> MzmlConverter<'d> {
34    #[inline]
35    pub(crate) fn new(decoder: &'d mut IonReader) -> Self {
36        Self { decoder }
37    }
38
39    fn run_level(decoder: &IonReader) -> IonResult<MzML> {
40        let global_meta = decoder.global_metadata_rows()?;
41        let global_lookup = ChildrenLookup::new(&global_meta);
42        let meta_refs: Vec<&Metadatum> = global_meta.iter().collect();
43        let policy = DefaultMetadataPolicy;
44
45        let mut owner_rows = OwnerRows::with_capacity(global_meta.len());
46        for metadatum in &global_meta {
47            owner_rows.insert(metadatum.id, metadatum);
48        }
49
50        let run_id = global_lookup
51            .all_ids(TagId::Run)
52            .first()
53            .copied()
54            .unwrap_or(0);
55        let rows = owner_rows.get(run_id);
56
57        let mut param_buffer: Vec<&Metadatum> = Vec::new();
58        global_lookup.get_param_rows_into(&owner_rows, run_id, &policy, &mut param_buffer);
59        let (cv_params, user_params) = parse_cv_and_user_params(&param_buffer);
60
61        let source_file_ref_list = parse_run_source_file_refs(&owner_rows, &global_lookup, run_id);
62
63        Ok(MzML {
64            cv_list: parse_cv_list(&meta_refs, &global_lookup),
65            file_description: parse_file_description(&meta_refs, &global_lookup, &policy),
66            referenceable_param_group_list: parse_referenceable_param_group_list(
67                &meta_refs,
68                &global_lookup,
69                &policy,
70            ),
71            sample_list: parse_sample_list(&meta_refs, &global_lookup, &policy),
72            instrument_list: parse_instrument_list(&meta_refs, &global_lookup, &policy),
73            software_list: parse_software_list(&meta_refs, &global_lookup, &policy),
74            data_processing_list: parse_data_processing_list(&meta_refs, &global_lookup, &policy),
75            scan_settings_list: parse_scan_settings_list(&meta_refs, &global_lookup, &policy),
76            run: Run {
77                id: get_attr_text(rows, ACC_ATTR_ID).unwrap_or_default(),
78                start_time_stamp: get_attr_text(rows, ACC_ATTR_START_TIME_STAMP)
79                    .filter(|value| !value.is_empty()),
80                default_instrument_configuration_ref: get_attr_text(
81                    rows,
82                    ACC_ATTR_DEFAULT_INSTRUMENT_CONFIGURATION_REF,
83                )
84                .or_else(|| get_attr_text(rows, ACC_ATTR_INSTRUMENT_CONFIGURATION_REF)),
85                default_source_file_ref: get_attr_text(rows, ACC_ATTR_DEFAULT_SOURCE_FILE_REF),
86                sample_ref: get_attr_text(rows, ACC_ATTR_SAMPLE_REF),
87                referenceable_param_group_refs: global_lookup
88                    .ids_for(run_id, TagId::ReferenceableParamGroupRef)
89                    .iter()
90                    .filter_map(|&ref_id| {
91                        get_attr_text(owner_rows.get(ref_id), ACC_ATTR_REF)
92                            .map(|r| ReferenceableParamGroupRef { r#ref: r })
93                    })
94                    .collect(),
95                cv_params,
96                user_params,
97                source_file_ref_list,
98                spectrum_list: None,
99                chromatogram_list: None,
100            },
101        })
102    }
103
104    pub(crate) fn full(&mut self) -> IonResult<MzML> {
105        let mut mzml = Self::run_level(self.decoder)?;
106        let policy = DefaultMetadataPolicy;
107        mzml.run.spectrum_list =
108            assemble_spectrum_list(&self.decoder.spectrum_metadata_grouped()?, &policy);
109        mzml.run.chromatogram_list =
110            assemble_chromatogram_list(&self.decoder.chromatogram_metadata_grouped()?, &policy);
111
112        if let Some(spectrum_list) = mzml.run.spectrum_list.as_mut() {
113            attach_binaries(
114                self.decoder.spec_entries_buf.as_ref(),
115                self.decoder.spec_array_addresses.as_ref(),
116                &mut spectrum_list.spectra,
117                &self.decoder.spec_container,
118                "spec",
119                self.decoder.options.parallel,
120            )?;
121        }
122
123        if let (Some(chrom_list), Some(container)) = (
124            mzml.run.chromatogram_list.as_mut(),
125            self.decoder.chrom_container.as_ref(),
126        ) {
127            attach_binaries(
128                self.decoder.chrom_entries_buf.as_ref(),
129                self.decoder.chrom_array_addresses.as_ref(),
130                &mut chrom_list.chromatograms,
131                container,
132                "chrom",
133                self.decoder.options.parallel,
134            )?;
135        }
136
137        Ok(mzml)
138    }
139}
140
141fn assemble_spectrum_list(
142    groups: &[Vec<Metadatum>],
143    policy: &DefaultMetadataPolicy,
144) -> Option<SpectrumList> {
145    let mut combined: Option<SpectrumList> = None;
146    let mut index_base = 0u32;
147    for group in groups {
148        let refs: Vec<&Metadatum> = group.iter().collect();
149        let Some(list) =
150            parse_spectrum_list(&refs, &ChildrenLookup::new(group), policy, index_base)
151        else {
152            continue;
153        };
154        index_base += list.spectra.len() as u32;
155        match combined.as_mut() {
156            None => combined = Some(list),
157            Some(existing) => existing.spectra.extend(list.spectra),
158        }
159    }
160    if let Some(list) = combined.as_mut()
161        && list.count.is_none()
162    {
163        list.count = Some(list.spectra.len());
164    }
165    combined
166}
167
168fn assemble_chromatogram_list(
169    groups: &[Vec<Metadatum>],
170    policy: &DefaultMetadataPolicy,
171) -> Option<ChromatogramList> {
172    let mut combined: Option<ChromatogramList> = None;
173    let mut index_base = 0u32;
174    for group in groups {
175        let refs: Vec<&Metadatum> = group.iter().collect();
176        let Some(list) =
177            parse_chromatogram_list(&refs, &ChildrenLookup::new(group), policy, index_base)
178        else {
179            continue;
180        };
181        index_base += list.chromatograms.len() as u32;
182        match combined.as_mut() {
183            None => combined = Some(list),
184            Some(existing) => existing.chromatograms.extend(list.chromatograms),
185        }
186    }
187    if let Some(list) = combined.as_mut()
188        && list.count.is_none()
189    {
190        list.count = Some(list.chromatograms.len());
191    }
192    combined
193}
194
195pub(crate) trait BinaryArrayOwner {
196    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList>;
197}
198
199impl BinaryArrayOwner for Spectrum {
200    #[inline]
201    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
202        &mut self.binary_data_array_list
203    }
204}
205
206impl BinaryArrayOwner for Chromatogram {
207    #[inline]
208    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
209        &mut self.binary_data_array_list
210    }
211}
212
213pub(crate) fn attach_binaries<E: BinaryArrayOwner>(
214    entries_buf: &[u8],
215    array_addresses_buf: &[u8],
216    entries: &mut [E],
217    container: &BlockReader<DefaultBlockProcessor>,
218    ctx: &'static str,
219    parallel: bool,
220) -> IonResult<()> {
221    let mut refs = Vec::new();
222    let mut blocks = HashMap::new();
223    #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
224    let _ = parallel;
225
226    for index in 0..entries.len() {
227        let Some(item_refs) =
228            read_array_addresses_from_buffers(entries_buf, array_addresses_buf, index)
229        else {
230            continue;
231        };
232        if item_refs.is_empty() {
233            continue;
234        }
235        for array_address in item_refs.as_slice() {
236            let stride = if array_address.encoded_len > 0 {
237                1
238            } else {
239                dtype_stride(array_address.dtype)
240            };
241            if let Some(old) = blocks.insert(array_address.block_id, stride)
242                && old != stride
243            {
244                return Err(IonError::from(format!(
245                    "{ctx}: stride mismatch for block {} (expected {old}, got {stride})",
246                    array_address.block_id
247                )));
248            }
249        }
250        refs.push((index, item_refs));
251    }
252
253    let mut block_list: Vec<_> = blocks.into_iter().collect();
254    block_list.sort_unstable_by_key(|(block_id, _)| *block_id);
255
256    let load = |(block_id, stride): (u32, usize)| -> IonResult<(u32, SourceBytes)> {
257        Ok((block_id, container.read_block(block_id, stride, ctx)?))
258    };
259
260    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
261    let data: HashMap<u32, SourceBytes> = if parallel && block_list.len() >= 4 {
262        block_list
263            .into_par_iter()
264            .map(load)
265            .collect::<IonResult<Vec<_>>>()?
266            .into_iter()
267            .collect()
268    } else {
269        block_list
270            .into_iter()
271            .map(load)
272            .collect::<IonResult<Vec<_>>>()?
273            .into_iter()
274            .collect()
275    };
276    #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
277    let data: HashMap<u32, SourceBytes> = block_list
278        .into_iter()
279        .map(load)
280        .collect::<IonResult<Vec<_>>>()?
281        .into_iter()
282        .collect();
283
284    for (index, item_refs) in refs {
285        let groups = group_arrays(item_refs.as_slice())?;
286        let list = entries[index]
287            .binary_data_array_list_mut()
288            .get_or_insert_with(BinaryDataArrayList::default);
289
290        for group in groups {
291            if let [array_address] = group.refs.as_slice() {
292                let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
293                attach_logical_array(
294                    list,
295                    group.array_type,
296                    group.array_cv_code,
297                    group.dtype,
298                    &bytes,
299                )?;
300            } else {
301                let mut total = 0usize;
302                for array_address in &group.refs {
303                    total += checked_ref_bytes(&data, array_address, ctx)?.len();
304                }
305
306                let mut concatenated = Vec::new();
307                concatenated.reserve_exact(total);
308                for array_address in &group.refs {
309                    let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
310                    concatenated.extend_from_slice(&bytes);
311                }
312
313                attach_logical_array(
314                    list,
315                    group.array_type,
316                    group.array_cv_code,
317                    group.dtype,
318                    &concatenated,
319                )?;
320            }
321        }
322        list.count = Some(list.binary_data_arrays.len());
323    }
324
325    Ok(())
326}
327
328fn ref_byte_range(array_address: &ArrayAddress, ctx: &str) -> IonResult<(usize, usize)> {
329    let (element_offset, count, stride) = address_read_params(array_address);
330    let start = usize::try_from(element_offset)
331        .ok()
332        .and_then(|offset| offset.checked_mul(stride))
333        .ok_or_else(|| {
334            IonError::from(format!(
335                "{ctx}: item range overflow for block {}",
336                array_address.block_id
337            ))
338        })?;
339    let end = usize::try_from(count)
340        .ok()
341        .and_then(|c| c.checked_mul(stride))
342        .and_then(|len| start.checked_add(len))
343        .ok_or_else(|| {
344            IonError::from(format!(
345                "{ctx}: item range overflow for block {}",
346                array_address.block_id
347            ))
348        })?;
349    Ok((start, end))
350}
351
352fn checked_ref_bytes<'d>(
353    data: &'d HashMap<u32, SourceBytes>,
354    array_address: &ArrayAddress,
355    ctx: &str,
356) -> IonResult<&'d [u8]> {
357    let block = data.get(&array_address.block_id).ok_or_else(|| {
358        IonError::from(format!("{ctx}: missing block {}", array_address.block_id))
359    })?;
360    let (start, end) = ref_byte_range(array_address, ctx)?;
361    block.get(start..end).ok_or_else(|| {
362        IonError::from(format!(
363            "{ctx}: item range [{start}..{end}] out of bounds for block {} (len={})",
364            array_address.block_id,
365            block.len()
366        ))
367    })
368}
369
370fn unfiltered_ref_bytes<'d>(
371    data: &'d HashMap<u32, SourceBytes>,
372    group: &ArrayGroup,
373    array_address: &ArrayAddress,
374    ctx: &str,
375) -> IonResult<Cow<'d, [u8]>> {
376    let raw = checked_ref_bytes(data, array_address, ctx)?;
377    unfilter_array_bytes(raw, group.dtype, group.array_filter)
378}
379
380fn attach_logical_array(
381    binary_array_list: &mut BinaryDataArrayList,
382    array_type: u32,
383    array_cv_code: u8,
384    dtype: u8,
385    decoded_bytes: &[u8],
386) -> IonResult<()> {
387    let binary = decoded_bytes_to_binary_data(decoded_bytes, dtype)?;
388    let numeric_type = dtype_to_numeric_type(dtype)?;
389
390    let empty_index = binary_array_list
391        .binary_data_arrays
392        .iter()
393        .position(|array| binary_array_has_type(array, array_type) && array.binary.is_none());
394
395    let binary_array = if let Some(index) = empty_index {
396        &mut binary_array_list.binary_data_arrays[index]
397    } else {
398        binary_array_list
399            .binary_data_arrays
400            .push(make_binary_array_stub(array_type, array_cv_code));
401        binary_array_list.binary_data_arrays.last_mut().unwrap()
402    };
403
404    binary_array.binary = Some(binary);
405    sync_numeric_meta(binary_array, numeric_type);
406    Ok(())
407}
408
409fn raw_to_vec<T>(raw: &[u8], elem_size: usize, read: impl Fn(&[u8]) -> T) -> IonResult<Vec<T>> {
410    if !raw.len().is_multiple_of(elem_size) {
411        return Err(IonError::from(format!(
412            "array: length {} not a multiple of {elem_size}",
413            raw.len()
414        )));
415    }
416    let mut out = Vec::with_capacity(raw.len() / elem_size);
417    out.extend(raw.chunks_exact(elem_size).map(read));
418    Ok(out)
419}
420
421pub(crate) fn decoded_bytes_to_binary_data(bytes: &[u8], dtype: u8) -> IonResult<NumericArray> {
422    match dtype {
423        FILE_DTYPE_F64 => Ok(NumericArray::F64(raw_to_vec(bytes, 8, |c| {
424            f64::from_le_bytes(c.try_into().unwrap())
425        })?)),
426        FILE_DTYPE_F32 => Ok(NumericArray::F32(raw_to_vec(bytes, 4, |c| {
427            f32::from_le_bytes(c.try_into().unwrap())
428        })?)),
429        FILE_DTYPE_F16 => Ok(NumericArray::F16(raw_to_vec(bytes, 2, |c| {
430            u16::from_le_bytes(c.try_into().unwrap())
431        })?)),
432        FILE_DTYPE_I16 => Ok(NumericArray::I16(raw_to_vec(bytes, 2, |c| {
433            i16::from_le_bytes(c.try_into().unwrap())
434        })?)),
435        FILE_DTYPE_I32 => Ok(NumericArray::I32(raw_to_vec(bytes, 4, |c| {
436            i32::from_le_bytes(c.try_into().unwrap())
437        })?)),
438        FILE_DTYPE_I64 => Ok(NumericArray::I64(raw_to_vec(bytes, 8, |c| {
439            i64::from_le_bytes(c.try_into().unwrap())
440        })?)),
441        _ => Err(IonError::from(format!(
442            "unsupported dtype {dtype} in binary array"
443        ))),
444    }
445}
446
447fn dtype_to_numeric_type(dtype: u8) -> IonResult<NumericType> {
448    match dtype {
449        FILE_DTYPE_F64 => Ok(NumericType::Float64),
450        FILE_DTYPE_F32 => Ok(NumericType::Float32),
451        FILE_DTYPE_F16 => Ok(NumericType::Float16),
452        FILE_DTYPE_I16 => Ok(NumericType::Int16),
453        FILE_DTYPE_I32 => Ok(NumericType::Int32),
454        FILE_DTYPE_I64 => Ok(NumericType::Int64),
455        _ => Err(IonError::BadDtype {
456            dtype,
457            kind: "numeric type",
458        }),
459    }
460}
461
462fn make_binary_array_stub(array_type: u32, array_cv_code: u8) -> BinaryDataArray {
463    let cv_ref = crate::ion::attr_meta::cv_ref_prefix_from_code(array_cv_code).unwrap_or("MS");
464    let accession = crate::ion::attr_meta::format_accession(array_cv_code, array_type)
465        .unwrap_or_else(|| format_accession(array_type));
466    BinaryDataArray {
467        cv_params: vec![CvParam {
468            cv_ref: Some(cv_ref.to_string()),
469            accession: Some(accession),
470            name: String::new(),
471            ..Default::default()
472        }],
473        ..BinaryDataArray::default()
474    }
475}
476
477#[inline]
478fn sync_numeric_meta(binary_array: &mut BinaryDataArray, numeric_type: NumericType) {
479    let target = match numeric_type {
480        NumericType::Float16 => 1_000_520,
481        NumericType::Float32 => 1_000_521,
482        NumericType::Float64 => 1_000_523,
483        NumericType::Int16 => 1_000_518,
484        NumericType::Int32 => 1_000_519,
485        NumericType::Int64 => 1_000_522,
486    };
487    let param = CvParam {
488        cv_ref: Some("MS".into()),
489        accession: Some(format_accession(target)),
490        name: match target {
491            1_000_520 => "16-bit float",
492            1_000_521 => "32-bit float",
493            1_000_523 => "64-bit float",
494            1_000_518 => "16-bit integer",
495            1_000_519 => "32-bit integer",
496            1_000_522 => "64-bit integer",
497            _ => "numeric",
498        }
499        .into(),
500        ..Default::default()
501    };
502    match binary_array
503        .cv_params
504        .iter()
505        .position(|p| is_numeric_acc(parse_accession_tail(p.accession.as_deref())))
506    {
507        Some(position) => binary_array.cv_params[position] = param,
508        None => binary_array.cv_params.push(param),
509    }
510    binary_array.numeric_type = Some(numeric_type);
511}
512
513#[inline]
514fn is_numeric_acc(tail: AccessionTail) -> bool {
515    matches!(
516        tail.raw(),
517        INT_16BIT | INT_32BIT | INT_64BIT | FLOAT_16BIT | FLOAT_32BIT | FLOAT_64BIT
518    )
519}
520
521#[inline]
522fn binary_array_has_type(binary_array: &BinaryDataArray, array_type: u32) -> bool {
523    binary_array
524        .cv_params
525        .iter()
526        .any(|param| parse_accession_tail(param.accession.as_deref()).raw() == array_type)
527}
528
529type ListHeader<'l, L> = Option<(L, &'l [u32])>;
530
531fn list_header<L>(
532    rows: &[Metadatum],
533    parse: for<'l> fn(&OwnerRows, &'l ChildrenLookup) -> ListHeader<'l, L>,
534) -> Option<L> {
535    let mut owner_rows = OwnerRows::with_capacity(rows.len());
536    for row in rows {
537        owner_rows.insert(row.id, row);
538    }
539    parse(&owner_rows, &ChildrenLookup::new(rows)).map(|(list, _)| list)
540}
541
542fn parse_run_source_file_refs(
543    owner_rows: &OwnerRows,
544    lookup: &ChildrenLookup,
545    run_id: u32,
546) -> Option<SourceFileRefList> {
547    if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileRefList).first() {
548        let refs: Vec<_> = lookup
549            .ids_for(list_id, TagId::SourceFileRef)
550            .iter()
551            .filter_map(|&id| {
552                get_attr_text(owner_rows.get(id), ACC_ATTR_REF)
553                    .map(|value| SourceFileRef { r#ref: value })
554            })
555            .collect();
556        if !refs.is_empty() {
557            return Some(SourceFileRefList {
558                count: Some(refs.len()),
559                source_file_refs: refs,
560            });
561        }
562    }
563
564    if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileList).first() {
565        let refs: Vec<_> = lookup
566            .ids_for(list_id, TagId::SourceFile)
567            .iter()
568            .filter_map(|&id| {
569                get_attr_text(owner_rows.get(id), ACC_ATTR_ID)
570                    .map(|value| SourceFileRef { r#ref: value })
571            })
572            .collect();
573        if !refs.is_empty() {
574            return Some(SourceFileRefList {
575                count: Some(refs.len()),
576                source_file_refs: refs,
577            });
578        }
579    }
580
581    None
582}
583
584impl IonReader {
585    pub fn global_metadata(&self) -> IonResult<MzML> {
586        let mut mzml = MzmlConverter::run_level(self)?;
587        mzml.run.spectrum_list = list_header(
588            &self.spec_meta.read_first_group()?,
589            parse_spectrum_list_header,
590        );
591        mzml.run.chromatogram_list = list_header(
592            &self.chrom_meta.read_first_group()?,
593            parse_chromatogram_list_header,
594        );
595        Ok(mzml)
596    }
597
598    pub fn to_mzml(&mut self) -> IonResult<MzML> {
599        MzmlConverter::new(self).full()
600    }
601}
602
603#[cfg(test)]
604mod tests {
605    use super::*;
606
607    #[test]
608    fn f16_i16_arrays_get_real_cvparam_name_6() {
609        let mut float16_array = BinaryDataArray::default();
610        sync_numeric_meta(&mut float16_array, NumericType::Float16);
611        let float16_param = float16_array
612            .cv_params
613            .iter()
614            .find(|p| p.accession.as_deref() == Some("MS:1000520"))
615            .expect("sync_numeric_meta must add the 16-bit float precision cvParam");
616        assert_eq!(float16_param.name, "16-bit float");
617
618        let mut int16_array = BinaryDataArray::default();
619        sync_numeric_meta(&mut int16_array, NumericType::Int16);
620        let int16_param = int16_array
621            .cv_params
622            .iter()
623            .find(|p| p.accession.as_deref() == Some("MS:1000518"))
624            .expect("sync_numeric_meta must add the 16-bit integer precision cvParam");
625        assert_eq!(int16_param.name, "16-bit integer");
626    }
627}