Skip to main content

ionic/ion/decoder/decode/
to_mzml.rs

1use std::{borrow::Cow, collections::HashMap};
2
3#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
4use rayon::prelude::*;
5
6use super::*;
7use crate::ion::decoder::utilities::byte_source::SourceBytes;
8
9#[derive(Debug, Clone, PartialEq)]
10pub enum MetadatumValue {
11    Number(f64),
12    Text(String),
13    Empty,
14}
15
16#[derive(Debug, Clone, PartialEq)]
17pub struct Metadatum {
18    pub(crate) item_index: u32,
19    pub(crate) id: u32,
20    pub(crate) parent_id: u32,
21    pub(crate) tag_id: TagId,
22    pub(crate) accession: Option<String>,
23    pub(crate) unit_accession: Option<String>,
24    pub(crate) value: MetadatumValue,
25}
26
27pub(crate) struct MzmlConverter<'d> {
28    pub(crate) decoder: &'d mut IonReader,
29}
30
31impl<'d> MzmlConverter<'d> {
32    #[inline]
33    pub(crate) fn new(decoder: &'d mut IonReader) -> Self {
34        Self { decoder }
35    }
36
37    pub(crate) fn metadata_only(decoder: &IonReader) -> IonResult<MzML> {
38        let global_meta = decoder.global_metadata()?;
39        let global_lookup = ChildrenLookup::new(&global_meta);
40        let meta_refs: Vec<&Metadatum> = global_meta.iter().collect();
41        let policy = DefaultMetadataPolicy;
42
43        let mut owner_rows = OwnerRows::with_capacity(global_meta.len());
44        for metadatum in &global_meta {
45            owner_rows.insert(metadatum.id, metadatum);
46        }
47
48        let run_id = global_lookup
49            .all_ids(TagId::Run)
50            .first()
51            .copied()
52            .unwrap_or(0);
53        let rows = owner_rows.get(run_id);
54
55        let mut param_buffer: Vec<&Metadatum> = Vec::new();
56        global_lookup.get_param_rows_into(&owner_rows, run_id, &policy, &mut param_buffer);
57        let (cv_params, user_params) = parse_cv_and_user_params(&param_buffer);
58
59        let spectrum_list = assemble_spectrum_list(&decoder.spectrum_metadata_grouped()?, &policy);
60        let chromatogram_list =
61            assemble_chromatogram_list(&decoder.chromatogram_metadata_grouped()?, &policy);
62
63        let source_file_ref_list = parse_run_source_file_refs(&owner_rows, &global_lookup, run_id);
64
65        Ok(MzML {
66            cv_list: parse_cv_list(&meta_refs, &global_lookup),
67            file_description: parse_file_description(&meta_refs, &global_lookup, &policy),
68            referenceable_param_group_list: parse_referenceable_param_group_list(
69                &meta_refs,
70                &global_lookup,
71                &policy,
72            ),
73            sample_list: parse_sample_list(&meta_refs, &global_lookup, &policy),
74            instrument_list: parse_instrument_list(&meta_refs, &global_lookup, &policy),
75            software_list: parse_software_list(&meta_refs, &global_lookup, &policy),
76            data_processing_list: parse_data_processing_list(&meta_refs, &global_lookup, &policy),
77            scan_settings_list: parse_scan_settings_list(&meta_refs, &global_lookup, &policy),
78            run: Run {
79                id: get_attr_text(rows, ACC_ATTR_ID).unwrap_or_default(),
80                start_time_stamp: get_attr_text(rows, ACC_ATTR_START_TIME_STAMP)
81                    .filter(|value| !value.is_empty()),
82                default_instrument_configuration_ref: get_attr_text(
83                    rows,
84                    ACC_ATTR_DEFAULT_INSTRUMENT_CONFIGURATION_REF,
85                )
86                .or_else(|| get_attr_text(rows, ACC_ATTR_INSTRUMENT_CONFIGURATION_REF)),
87                default_source_file_ref: get_attr_text(rows, ACC_ATTR_DEFAULT_SOURCE_FILE_REF),
88                sample_ref: get_attr_text(rows, ACC_ATTR_SAMPLE_REF),
89                referenceable_param_group_refs: global_lookup
90                    .ids_for(run_id, TagId::ReferenceableParamGroupRef)
91                    .iter()
92                    .filter_map(|&ref_id| {
93                        get_attr_text(owner_rows.get(ref_id), ACC_ATTR_REF)
94                            .map(|r| ReferenceableParamGroupRef { r#ref: r })
95                    })
96                    .collect(),
97                cv_params,
98                user_params,
99                source_file_ref_list,
100                spectrum_list,
101                chromatogram_list,
102            },
103        })
104    }
105
106    pub(crate) fn full(&mut self) -> IonResult<MzML> {
107        let mut mzml = Self::metadata_only(self.decoder)?;
108
109        if let Some(spectrum_list) = mzml.run.spectrum_list.as_mut() {
110            attach_binaries(
111                self.decoder.spec_entries_buf.as_ref(),
112                self.decoder.spec_array_addresses.as_ref(),
113                &mut spectrum_list.spectra,
114                &self.decoder.spec_container,
115                "spec",
116                self.decoder.parallel,
117            )?;
118        }
119
120        if let (Some(chrom_list), Some(container)) = (
121            mzml.run.chromatogram_list.as_mut(),
122            self.decoder.chrom_container.as_ref(),
123        ) {
124            attach_binaries(
125                self.decoder.chrom_entries_buf.as_ref(),
126                self.decoder.chrom_array_addresses.as_ref(),
127                &mut chrom_list.chromatograms,
128                container,
129                "chrom",
130                self.decoder.parallel,
131            )?;
132        }
133
134        Ok(mzml)
135    }
136}
137
138fn assemble_spectrum_list(
139    groups: &[Vec<Metadatum>],
140    policy: &DefaultMetadataPolicy,
141) -> Option<SpectrumList> {
142    let mut combined: Option<SpectrumList> = None;
143    let mut index_base = 0u32;
144    for group in groups {
145        let refs: Vec<&Metadatum> = group.iter().collect();
146        let Some(list) =
147            parse_spectrum_list(&refs, &ChildrenLookup::new(group), policy, index_base)
148        else {
149            continue;
150        };
151        index_base += list.spectra.len() as u32;
152        match combined.as_mut() {
153            None => combined = Some(list),
154            Some(existing) => existing.spectra.extend(list.spectra),
155        }
156    }
157    if let Some(list) = combined.as_mut() {
158        if list.count.is_none() {
159            list.count = Some(list.spectra.len());
160        }
161    }
162    combined
163}
164
165fn assemble_chromatogram_list(
166    groups: &[Vec<Metadatum>],
167    policy: &DefaultMetadataPolicy,
168) -> Option<ChromatogramList> {
169    let mut combined: Option<ChromatogramList> = None;
170    let mut index_base = 0u32;
171    for group in groups {
172        let refs: Vec<&Metadatum> = group.iter().collect();
173        let Some(list) =
174            parse_chromatogram_list(&refs, &ChildrenLookup::new(group), policy, index_base)
175        else {
176            continue;
177        };
178        index_base += list.chromatograms.len() as u32;
179        match combined.as_mut() {
180            None => combined = Some(list),
181            Some(existing) => existing.chromatograms.extend(list.chromatograms),
182        }
183    }
184    if let Some(list) = combined.as_mut() {
185        if list.count.is_none() {
186            list.count = Some(list.chromatograms.len());
187        }
188    }
189    combined
190}
191
192pub(crate) trait BinaryArrayOwner {
193    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList>;
194}
195
196impl BinaryArrayOwner for Spectrum {
197    #[inline]
198    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
199        &mut self.binary_data_array_list
200    }
201}
202
203impl BinaryArrayOwner for Chromatogram {
204    #[inline]
205    fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
206        &mut self.binary_data_array_list
207    }
208}
209
210pub(crate) fn attach_binaries<E: BinaryArrayOwner>(
211    entries_buf: &[u8],
212    array_addresses_buf: &[u8],
213    entries: &mut [E],
214    container: &BlockReader<DefaultBlockProcessor>,
215    ctx: &'static str,
216    parallel: bool,
217) -> IonResult<()> {
218    let mut refs = Vec::new();
219    let mut blocks = HashMap::new();
220    #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
221    let _ = parallel;
222
223    for index in 0..entries.len() {
224        let Some(item_refs) =
225            read_array_addresses_from_buffers(entries_buf, array_addresses_buf, index)
226        else {
227            continue;
228        };
229        if item_refs.is_empty() {
230            continue;
231        }
232        for array_address in item_refs.as_slice() {
233            let stride = if array_address.encoded_len > 0 {
234                1
235            } else {
236                dtype_stride(array_address.dtype)
237            };
238            if let Some(old) = blocks.insert(array_address.block_id, stride)
239                && old != stride
240            {
241                return Err(IonError::from(format!(
242                    "{ctx}: stride mismatch for block {} (expected {old}, got {stride})",
243                    array_address.block_id
244                )));
245            }
246        }
247        refs.push((index, item_refs));
248    }
249
250    let mut block_list: Vec<_> = blocks.into_iter().collect();
251    block_list.sort_unstable_by_key(|(block_id, _)| *block_id);
252
253    let load = |(block_id, stride): (u32, usize)| -> IonResult<(u32, SourceBytes)> {
254        Ok((block_id, container.read_block(block_id, stride, ctx)?))
255    };
256
257    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
258    let data: HashMap<u32, SourceBytes> = if parallel && block_list.len() >= 4 {
259        block_list
260            .into_par_iter()
261            .map(load)
262            .collect::<IonResult<Vec<_>>>()?
263            .into_iter()
264            .collect()
265    } else {
266        block_list
267            .into_iter()
268            .map(load)
269            .collect::<IonResult<Vec<_>>>()?
270            .into_iter()
271            .collect()
272    };
273    #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
274    let data: HashMap<u32, SourceBytes> = block_list
275        .into_iter()
276        .map(load)
277        .collect::<IonResult<Vec<_>>>()?
278        .into_iter()
279        .collect();
280
281    for (index, item_refs) in refs {
282        let groups = group_arrays(item_refs.as_slice())?;
283        let list = entries[index]
284            .binary_data_array_list_mut()
285            .get_or_insert_with(BinaryDataArrayList::default);
286
287        for group in groups {
288            if let [array_address] = group.refs.as_slice() {
289                let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
290                attach_logical_array(
291                    list,
292                    group.array_type,
293                    group.array_cv_code,
294                    group.dtype,
295                    &bytes,
296                )?;
297            } else {
298                let mut total = 0usize;
299                for array_address in &group.refs {
300                    total += checked_ref_bytes(&data, array_address, ctx)?.len();
301                }
302
303                let mut concatenated = Vec::new();
304                concatenated.reserve_exact(total);
305                for array_address in &group.refs {
306                    let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
307                    concatenated.extend_from_slice(&bytes);
308                }
309
310                attach_logical_array(
311                    list,
312                    group.array_type,
313                    group.array_cv_code,
314                    group.dtype,
315                    &concatenated,
316                )?;
317            }
318        }
319        list.count = Some(list.binary_data_arrays.len());
320    }
321
322    Ok(())
323}
324
325fn ref_byte_range(array_address: &ArrayAddress, ctx: &str) -> IonResult<(usize, usize)> {
326    let (element_offset, count, stride) = address_read_params(array_address);
327    let start = usize::try_from(element_offset)
328        .ok()
329        .and_then(|offset| offset.checked_mul(stride))
330        .ok_or_else(|| {
331            IonError::from(format!(
332                "{ctx}: item range overflow for block {}",
333                array_address.block_id
334            ))
335        })?;
336    let end = usize::try_from(count)
337        .ok()
338        .and_then(|c| c.checked_mul(stride))
339        .and_then(|len| start.checked_add(len))
340        .ok_or_else(|| {
341            IonError::from(format!(
342                "{ctx}: item range overflow for block {}",
343                array_address.block_id
344            ))
345        })?;
346    Ok((start, end))
347}
348
349fn checked_ref_bytes<'d>(
350    data: &'d HashMap<u32, SourceBytes>,
351    array_address: &ArrayAddress,
352    ctx: &str,
353) -> IonResult<&'d [u8]> {
354    let block = data.get(&array_address.block_id).ok_or_else(|| {
355        IonError::from(format!("{ctx}: missing block {}", array_address.block_id))
356    })?;
357    let (start, end) = ref_byte_range(array_address, ctx)?;
358    block.get(start..end).ok_or_else(|| {
359        IonError::from(format!(
360            "{ctx}: item range [{start}..{end}] out of bounds for block {} (len={})",
361            array_address.block_id,
362            block.len()
363        ))
364    })
365}
366
367fn unfiltered_ref_bytes<'d>(
368    data: &'d HashMap<u32, SourceBytes>,
369    group: &ArrayGroup,
370    array_address: &ArrayAddress,
371    ctx: &str,
372) -> IonResult<Cow<'d, [u8]>> {
373    let raw = checked_ref_bytes(data, array_address, ctx)?;
374    unfilter_array_bytes(raw, group.dtype, group.array_filter)
375}
376
377pub(crate) fn attach_logical_array(
378    binary_array_list: &mut BinaryDataArrayList,
379    array_type: u32,
380    array_cv_code: u8,
381    dtype: u8,
382    decoded_bytes: &[u8],
383) -> IonResult<()> {
384    let binary = decoded_bytes_to_binary_data(decoded_bytes, dtype)?;
385    let numeric_type = dtype_to_numeric_type(dtype)?;
386
387    let empty_index = binary_array_list
388        .binary_data_arrays
389        .iter()
390        .position(|array| binary_array_has_type(array, array_type) && array.binary.is_none());
391
392    let binary_array = if let Some(index) = empty_index {
393        &mut binary_array_list.binary_data_arrays[index]
394    } else {
395        binary_array_list
396            .binary_data_arrays
397            .push(make_binary_array_stub(array_type, array_cv_code));
398        binary_array_list.binary_data_arrays.last_mut().unwrap()
399    };
400
401    binary_array.binary = Some(binary);
402    sync_numeric_meta(binary_array, numeric_type);
403    Ok(())
404}
405
406fn raw_to_vec<T>(raw: &[u8], elem_size: usize, read: impl Fn(&[u8]) -> T) -> IonResult<Vec<T>> {
407    if !raw.len().is_multiple_of(elem_size) {
408        return Err(IonError::from(format!(
409            "array: length {} not a multiple of {elem_size}",
410            raw.len()
411        )));
412    }
413    let mut out = Vec::with_capacity(raw.len() / elem_size);
414    out.extend(raw.chunks_exact(elem_size).map(read));
415    Ok(out)
416}
417
418pub(crate) fn decoded_bytes_to_binary_data(bytes: &[u8], dtype: u8) -> IonResult<NumericArray> {
419    match dtype {
420        FILE_DTYPE_F64 => Ok(NumericArray::F64(raw_to_vec(bytes, 8, |c| {
421            f64::from_le_bytes(c.try_into().unwrap())
422        })?)),
423        FILE_DTYPE_F32 => Ok(NumericArray::F32(raw_to_vec(bytes, 4, |c| {
424            f32::from_le_bytes(c.try_into().unwrap())
425        })?)),
426        FILE_DTYPE_F16 => Ok(NumericArray::F16(raw_to_vec(bytes, 2, |c| {
427            u16::from_le_bytes(c.try_into().unwrap())
428        })?)),
429        FILE_DTYPE_I16 => Ok(NumericArray::I16(raw_to_vec(bytes, 2, |c| {
430            i16::from_le_bytes(c.try_into().unwrap())
431        })?)),
432        FILE_DTYPE_I32 => Ok(NumericArray::I32(raw_to_vec(bytes, 4, |c| {
433            i32::from_le_bytes(c.try_into().unwrap())
434        })?)),
435        FILE_DTYPE_I64 => Ok(NumericArray::I64(raw_to_vec(bytes, 8, |c| {
436            i64::from_le_bytes(c.try_into().unwrap())
437        })?)),
438        _ => Err(IonError::from(format!(
439            "unsupported dtype {dtype} in binary array"
440        ))),
441    }
442}
443
444fn dtype_to_numeric_type(dtype: u8) -> IonResult<NumericType> {
445    match dtype {
446        FILE_DTYPE_F64 => Ok(NumericType::Float64),
447        FILE_DTYPE_F32 => Ok(NumericType::Float32),
448        FILE_DTYPE_F16 => Ok(NumericType::Float16),
449        FILE_DTYPE_I16 => Ok(NumericType::Int16),
450        FILE_DTYPE_I32 => Ok(NumericType::Int32),
451        FILE_DTYPE_I64 => Ok(NumericType::Int64),
452        _ => Err(IonError::BadDtype {
453            dtype,
454            kind: "numeric type",
455        }),
456    }
457}
458
459fn make_binary_array_stub(array_type: u32, array_cv_code: u8) -> BinaryDataArray {
460    let cv_ref = crate::ion::attr_meta::cv_ref_prefix_from_code(array_cv_code).unwrap_or("MS");
461    let accession = crate::ion::attr_meta::format_accession(array_cv_code, array_type)
462        .unwrap_or_else(|| format_accession(array_type));
463    BinaryDataArray {
464        cv_params: vec![CvParam {
465            cv_ref: Some(cv_ref.to_string()),
466            accession: Some(accession),
467            name: String::new(),
468            ..Default::default()
469        }],
470        ..BinaryDataArray::default()
471    }
472}
473
474#[inline]
475fn sync_numeric_meta(binary_array: &mut BinaryDataArray, numeric_type: NumericType) {
476    let target = match numeric_type {
477        NumericType::Float16 => 1_000_520,
478        NumericType::Float32 => 1_000_521,
479        NumericType::Float64 => 1_000_523,
480        NumericType::Int16 => 1_000_518,
481        NumericType::Int32 => 1_000_519,
482        NumericType::Int64 => 1_000_522,
483    };
484    let param = CvParam {
485        cv_ref: Some("MS".into()),
486        accession: Some(format_accession(target)),
487        name: match target {
488            1_000_520 => "16-bit float",
489            1_000_521 => "32-bit float",
490            1_000_523 => "64-bit float",
491            1_000_518 => "16-bit integer",
492            1_000_519 => "32-bit integer",
493            1_000_522 => "64-bit integer",
494            _ => "numeric",
495        }
496        .into(),
497        ..Default::default()
498    };
499    match binary_array
500        .cv_params
501        .iter()
502        .position(|p| is_numeric_acc(parse_accession_tail(p.accession.as_deref())))
503    {
504        Some(position) => binary_array.cv_params[position] = param,
505        None => binary_array.cv_params.push(param),
506    }
507    binary_array.numeric_type = Some(numeric_type);
508}
509
510#[inline]
511fn is_numeric_acc(tail: AccessionTail) -> bool {
512    matches!(
513        tail.raw(),
514        INT_16BIT | INT_32BIT | INT_64BIT | FLOAT_16BIT | FLOAT_32BIT | FLOAT_64BIT
515    )
516}
517
518#[inline]
519fn binary_array_has_type(binary_array: &BinaryDataArray, array_type: u32) -> bool {
520    binary_array
521        .cv_params
522        .iter()
523        .any(|param| parse_accession_tail(param.accession.as_deref()).raw() == array_type)
524}
525
526fn parse_run_source_file_refs(
527    owner_rows: &OwnerRows,
528    lookup: &ChildrenLookup,
529    run_id: u32,
530) -> Option<SourceFileRefList> {
531    if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileRefList).first() {
532        let refs: Vec<_> = lookup
533            .ids_for(list_id, TagId::SourceFileRef)
534            .iter()
535            .filter_map(|&id| {
536                get_attr_text(owner_rows.get(id), ACC_ATTR_REF)
537                    .map(|value| SourceFileRef { r#ref: value })
538            })
539            .collect();
540        if !refs.is_empty() {
541            return Some(SourceFileRefList {
542                count: Some(refs.len()),
543                source_file_refs: refs,
544            });
545        }
546    }
547
548    if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileList).first() {
549        let refs: Vec<_> = lookup
550            .ids_for(list_id, TagId::SourceFile)
551            .iter()
552            .filter_map(|&id| {
553                get_attr_text(owner_rows.get(id), ACC_ATTR_ID)
554                    .map(|value| SourceFileRef { r#ref: value })
555            })
556            .collect();
557        if !refs.is_empty() {
558            return Some(SourceFileRefList {
559                count: Some(refs.len()),
560                source_file_refs: refs,
561            });
562        }
563    }
564
565    None
566}
567
568impl IonReader {
569    pub fn metadata(&self) -> IonResult<MzML> {
570        MzmlConverter::metadata_only(self)
571    }
572
573    pub fn to_mzml(&mut self) -> IonResult<MzML> {
574        MzmlConverter::new(self).full()
575    }
576}
577
578#[cfg(test)]
579mod tests {
580    use super::*;
581
582    #[test]
583    fn f16_i16_arrays_get_real_cvparam_name_6() {
584        let mut float16_array = BinaryDataArray::default();
585        sync_numeric_meta(&mut float16_array, NumericType::Float16);
586        let float16_param = float16_array
587            .cv_params
588            .iter()
589            .find(|p| p.accession.as_deref() == Some("MS:1000520"))
590            .expect("sync_numeric_meta must add the 16-bit float precision cvParam");
591        assert_eq!(float16_param.name, "16-bit float");
592
593        let mut int16_array = BinaryDataArray::default();
594        sync_numeric_meta(&mut int16_array, NumericType::Int16);
595        let int16_param = int16_array
596            .cv_params
597            .iter()
598            .find(|p| p.accession.as_deref() == Some("MS:1000518"))
599            .expect("sync_numeric_meta must add the 16-bit integer precision cvParam");
600        assert_eq!(int16_param.name, "16-bit integer");
601    }
602}