1use std::{borrow::Cow, collections::HashMap};
2
3#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
4use rayon::prelude::*;
5
6use super::*;
7use crate::ion::decoder::utilities::byte_source::SourceBytes;
8
9#[non_exhaustive]
10#[derive(Debug, Clone, PartialEq)]
11pub enum MetadatumValue {
12 Number(f64),
13 Text(String),
14 Empty,
15}
16
17#[non_exhaustive]
18#[derive(Debug, Clone, PartialEq)]
19pub struct Metadatum {
20 pub item_index: u32,
21 pub id: u32,
22 pub parent_id: u32,
23 pub tag_id: TagId,
24 pub accession: Option<String>,
25 pub unit_accession: Option<String>,
26 pub value: MetadatumValue,
27}
28
29pub(crate) struct MzmlConverter<'d> {
30 pub(crate) decoder: &'d mut IonReader,
31}
32
33impl<'d> MzmlConverter<'d> {
34 #[inline]
35 pub(crate) fn new(decoder: &'d mut IonReader) -> Self {
36 Self { decoder }
37 }
38
39 fn run_level(decoder: &IonReader) -> IonResult<MzML> {
40 let global_meta = decoder.global_metadata_rows()?;
41 let global_lookup = ChildrenLookup::new(&global_meta);
42 let meta_refs: Vec<&Metadatum> = global_meta.iter().collect();
43 let policy = DefaultMetadataPolicy;
44
45 let mut owner_rows = OwnerRows::with_capacity(global_meta.len());
46 for metadatum in &global_meta {
47 owner_rows.insert(metadatum.id, metadatum);
48 }
49
50 let run_id = global_lookup
51 .all_ids(TagId::Run)
52 .first()
53 .copied()
54 .unwrap_or(0);
55 let rows = owner_rows.get(run_id);
56
57 let mut param_buffer: Vec<&Metadatum> = Vec::new();
58 global_lookup.get_param_rows_into(&owner_rows, run_id, &policy, &mut param_buffer);
59 let (cv_params, user_params) = parse_cv_and_user_params(¶m_buffer);
60
61 let source_file_ref_list = parse_run_source_file_refs(&owner_rows, &global_lookup, run_id);
62
63 Ok(MzML {
64 cv_list: parse_cv_list(&meta_refs, &global_lookup),
65 file_description: parse_file_description(&meta_refs, &global_lookup, &policy),
66 referenceable_param_group_list: parse_referenceable_param_group_list(
67 &meta_refs,
68 &global_lookup,
69 &policy,
70 ),
71 sample_list: parse_sample_list(&meta_refs, &global_lookup, &policy),
72 instrument_list: parse_instrument_list(&meta_refs, &global_lookup, &policy),
73 software_list: parse_software_list(&meta_refs, &global_lookup, &policy),
74 data_processing_list: parse_data_processing_list(&meta_refs, &global_lookup, &policy),
75 scan_settings_list: parse_scan_settings_list(&meta_refs, &global_lookup, &policy),
76 run: Run {
77 id: get_attr_text(rows, ACC_ATTR_ID).unwrap_or_default(),
78 start_time_stamp: get_attr_text(rows, ACC_ATTR_START_TIME_STAMP)
79 .filter(|value| !value.is_empty()),
80 default_instrument_configuration_ref: get_attr_text(
81 rows,
82 ACC_ATTR_DEFAULT_INSTRUMENT_CONFIGURATION_REF,
83 )
84 .or_else(|| get_attr_text(rows, ACC_ATTR_INSTRUMENT_CONFIGURATION_REF)),
85 default_source_file_ref: get_attr_text(rows, ACC_ATTR_DEFAULT_SOURCE_FILE_REF),
86 sample_ref: get_attr_text(rows, ACC_ATTR_SAMPLE_REF),
87 referenceable_param_group_refs: global_lookup
88 .ids_for(run_id, TagId::ReferenceableParamGroupRef)
89 .iter()
90 .filter_map(|&ref_id| {
91 get_attr_text(owner_rows.get(ref_id), ACC_ATTR_REF)
92 .map(|r| ReferenceableParamGroupRef { r#ref: r })
93 })
94 .collect(),
95 cv_params,
96 user_params,
97 source_file_ref_list,
98 spectrum_list: None,
99 chromatogram_list: None,
100 },
101 })
102 }
103
104 pub(crate) fn full(&mut self) -> IonResult<MzML> {
105 let mut mzml = Self::run_level(self.decoder)?;
106 let policy = DefaultMetadataPolicy;
107 mzml.run.spectrum_list =
108 assemble_spectrum_list(&self.decoder.spectrum_metadata_grouped()?, &policy);
109 mzml.run.chromatogram_list =
110 assemble_chromatogram_list(&self.decoder.chromatogram_metadata_grouped()?, &policy);
111
112 if let Some(spectrum_list) = mzml.run.spectrum_list.as_mut() {
113 attach_binaries(
114 self.decoder.spec_entries_buf.as_ref(),
115 self.decoder.spec_array_addresses.as_ref(),
116 &mut spectrum_list.spectra,
117 &self.decoder.spec_container,
118 "spec",
119 self.decoder.options.parallel,
120 )?;
121 }
122
123 if let (Some(chrom_list), Some(container)) = (
124 mzml.run.chromatogram_list.as_mut(),
125 self.decoder.chrom_container.as_ref(),
126 ) {
127 attach_binaries(
128 self.decoder.chrom_entries_buf.as_ref(),
129 self.decoder.chrom_array_addresses.as_ref(),
130 &mut chrom_list.chromatograms,
131 container,
132 "chrom",
133 self.decoder.options.parallel,
134 )?;
135 }
136
137 Ok(mzml)
138 }
139}
140
141fn assemble_spectrum_list(
142 groups: &[Vec<Metadatum>],
143 policy: &DefaultMetadataPolicy,
144) -> Option<SpectrumList> {
145 let mut combined: Option<SpectrumList> = None;
146 let mut index_base = 0u32;
147 for group in groups {
148 let refs: Vec<&Metadatum> = group.iter().collect();
149 let Some(list) =
150 parse_spectrum_list(&refs, &ChildrenLookup::new(group), policy, index_base)
151 else {
152 continue;
153 };
154 index_base += list.spectra.len() as u32;
155 match combined.as_mut() {
156 None => combined = Some(list),
157 Some(existing) => existing.spectra.extend(list.spectra),
158 }
159 }
160 if let Some(list) = combined.as_mut()
161 && list.count.is_none()
162 {
163 list.count = Some(list.spectra.len());
164 }
165 combined
166}
167
168fn assemble_chromatogram_list(
169 groups: &[Vec<Metadatum>],
170 policy: &DefaultMetadataPolicy,
171) -> Option<ChromatogramList> {
172 let mut combined: Option<ChromatogramList> = None;
173 let mut index_base = 0u32;
174 for group in groups {
175 let refs: Vec<&Metadatum> = group.iter().collect();
176 let Some(list) =
177 parse_chromatogram_list(&refs, &ChildrenLookup::new(group), policy, index_base)
178 else {
179 continue;
180 };
181 index_base += list.chromatograms.len() as u32;
182 match combined.as_mut() {
183 None => combined = Some(list),
184 Some(existing) => existing.chromatograms.extend(list.chromatograms),
185 }
186 }
187 if let Some(list) = combined.as_mut()
188 && list.count.is_none()
189 {
190 list.count = Some(list.chromatograms.len());
191 }
192 combined
193}
194
195pub(crate) trait BinaryArrayOwner {
196 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList>;
197}
198
199impl BinaryArrayOwner for Spectrum {
200 #[inline]
201 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
202 &mut self.binary_data_array_list
203 }
204}
205
206impl BinaryArrayOwner for Chromatogram {
207 #[inline]
208 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
209 &mut self.binary_data_array_list
210 }
211}
212
213pub(crate) fn attach_binaries<E: BinaryArrayOwner>(
214 entries_buf: &[u8],
215 array_addresses_buf: &[u8],
216 entries: &mut [E],
217 container: &BlockReader<DefaultBlockProcessor>,
218 ctx: &'static str,
219 parallel: bool,
220) -> IonResult<()> {
221 let mut refs = Vec::new();
222 let mut blocks = HashMap::new();
223 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
224 let _ = parallel;
225
226 for index in 0..entries.len() {
227 let Some(item_refs) =
228 read_array_addresses_from_buffers(entries_buf, array_addresses_buf, index)
229 else {
230 continue;
231 };
232 if item_refs.is_empty() {
233 continue;
234 }
235 for array_address in item_refs.as_slice() {
236 let stride = if array_address.encoded_len > 0 {
237 1
238 } else {
239 dtype_stride(array_address.dtype)
240 };
241 if let Some(old) = blocks.insert(array_address.block_id, stride)
242 && old != stride
243 {
244 return Err(IonError::from(format!(
245 "{ctx}: stride mismatch for block {} (expected {old}, got {stride})",
246 array_address.block_id
247 )));
248 }
249 }
250 refs.push((index, item_refs));
251 }
252
253 let mut block_list: Vec<_> = blocks.into_iter().collect();
254 block_list.sort_unstable_by_key(|(block_id, _)| *block_id);
255
256 let load = |(block_id, stride): (u32, usize)| -> IonResult<(u32, SourceBytes)> {
257 Ok((block_id, container.read_block(block_id, stride, ctx)?))
258 };
259
260 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
261 let data: HashMap<u32, SourceBytes> = if parallel && block_list.len() >= 4 {
262 block_list
263 .into_par_iter()
264 .map(load)
265 .collect::<IonResult<Vec<_>>>()?
266 .into_iter()
267 .collect()
268 } else {
269 block_list
270 .into_iter()
271 .map(load)
272 .collect::<IonResult<Vec<_>>>()?
273 .into_iter()
274 .collect()
275 };
276 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
277 let data: HashMap<u32, SourceBytes> = block_list
278 .into_iter()
279 .map(load)
280 .collect::<IonResult<Vec<_>>>()?
281 .into_iter()
282 .collect();
283
284 for (index, item_refs) in refs {
285 let groups = group_arrays(item_refs.as_slice())?;
286 let list = entries[index]
287 .binary_data_array_list_mut()
288 .get_or_insert_with(BinaryDataArrayList::default);
289
290 for group in groups {
291 if let [array_address] = group.refs.as_slice() {
292 let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
293 attach_logical_array(
294 list,
295 group.array_type,
296 group.array_cv_code,
297 group.dtype,
298 &bytes,
299 )?;
300 } else {
301 let mut total = 0usize;
302 for array_address in &group.refs {
303 total += checked_ref_bytes(&data, array_address, ctx)?.len();
304 }
305
306 let mut concatenated = Vec::new();
307 concatenated.reserve_exact(total);
308 for array_address in &group.refs {
309 let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
310 concatenated.extend_from_slice(&bytes);
311 }
312
313 attach_logical_array(
314 list,
315 group.array_type,
316 group.array_cv_code,
317 group.dtype,
318 &concatenated,
319 )?;
320 }
321 }
322 list.count = Some(list.binary_data_arrays.len());
323 }
324
325 Ok(())
326}
327
328fn ref_byte_range(array_address: &ArrayAddress, ctx: &str) -> IonResult<(usize, usize)> {
329 let (element_offset, count, stride) = address_read_params(array_address);
330 let start = usize::try_from(element_offset)
331 .ok()
332 .and_then(|offset| offset.checked_mul(stride))
333 .ok_or_else(|| {
334 IonError::from(format!(
335 "{ctx}: item range overflow for block {}",
336 array_address.block_id
337 ))
338 })?;
339 let end = usize::try_from(count)
340 .ok()
341 .and_then(|c| c.checked_mul(stride))
342 .and_then(|len| start.checked_add(len))
343 .ok_or_else(|| {
344 IonError::from(format!(
345 "{ctx}: item range overflow for block {}",
346 array_address.block_id
347 ))
348 })?;
349 Ok((start, end))
350}
351
352fn checked_ref_bytes<'d>(
353 data: &'d HashMap<u32, SourceBytes>,
354 array_address: &ArrayAddress,
355 ctx: &str,
356) -> IonResult<&'d [u8]> {
357 let block = data.get(&array_address.block_id).ok_or_else(|| {
358 IonError::from(format!("{ctx}: missing block {}", array_address.block_id))
359 })?;
360 let (start, end) = ref_byte_range(array_address, ctx)?;
361 block.get(start..end).ok_or_else(|| {
362 IonError::from(format!(
363 "{ctx}: item range [{start}..{end}] out of bounds for block {} (len={})",
364 array_address.block_id,
365 block.len()
366 ))
367 })
368}
369
370fn unfiltered_ref_bytes<'d>(
371 data: &'d HashMap<u32, SourceBytes>,
372 group: &ArrayGroup,
373 array_address: &ArrayAddress,
374 ctx: &str,
375) -> IonResult<Cow<'d, [u8]>> {
376 let raw = checked_ref_bytes(data, array_address, ctx)?;
377 unfilter_array_bytes(raw, group.dtype, group.array_filter)
378}
379
380fn attach_logical_array(
381 binary_array_list: &mut BinaryDataArrayList,
382 array_type: u32,
383 array_cv_code: u8,
384 dtype: u8,
385 decoded_bytes: &[u8],
386) -> IonResult<()> {
387 let binary = decoded_bytes_to_binary_data(decoded_bytes, dtype)?;
388 let numeric_type = dtype_to_numeric_type(dtype)?;
389
390 let empty_index = binary_array_list
391 .binary_data_arrays
392 .iter()
393 .position(|array| binary_array_has_type(array, array_type) && array.binary.is_none());
394
395 let binary_array = if let Some(index) = empty_index {
396 &mut binary_array_list.binary_data_arrays[index]
397 } else {
398 binary_array_list
399 .binary_data_arrays
400 .push(make_binary_array_stub(array_type, array_cv_code));
401 binary_array_list.binary_data_arrays.last_mut().unwrap()
402 };
403
404 binary_array.binary = Some(binary);
405 sync_numeric_meta(binary_array, numeric_type);
406 Ok(())
407}
408
409fn raw_to_vec<T>(raw: &[u8], elem_size: usize, read: impl Fn(&[u8]) -> T) -> IonResult<Vec<T>> {
410 if !raw.len().is_multiple_of(elem_size) {
411 return Err(IonError::from(format!(
412 "array: length {} not a multiple of {elem_size}",
413 raw.len()
414 )));
415 }
416 let mut out = Vec::with_capacity(raw.len() / elem_size);
417 out.extend(raw.chunks_exact(elem_size).map(read));
418 Ok(out)
419}
420
421pub(crate) fn decoded_bytes_to_binary_data(bytes: &[u8], dtype: u8) -> IonResult<NumericArray> {
422 match dtype {
423 FILE_DTYPE_F64 => Ok(NumericArray::F64(raw_to_vec(bytes, 8, |c| {
424 f64::from_le_bytes(c.try_into().unwrap())
425 })?)),
426 FILE_DTYPE_F32 => Ok(NumericArray::F32(raw_to_vec(bytes, 4, |c| {
427 f32::from_le_bytes(c.try_into().unwrap())
428 })?)),
429 FILE_DTYPE_F16 => Ok(NumericArray::F16(raw_to_vec(bytes, 2, |c| {
430 u16::from_le_bytes(c.try_into().unwrap())
431 })?)),
432 FILE_DTYPE_I16 => Ok(NumericArray::I16(raw_to_vec(bytes, 2, |c| {
433 i16::from_le_bytes(c.try_into().unwrap())
434 })?)),
435 FILE_DTYPE_I32 => Ok(NumericArray::I32(raw_to_vec(bytes, 4, |c| {
436 i32::from_le_bytes(c.try_into().unwrap())
437 })?)),
438 FILE_DTYPE_I64 => Ok(NumericArray::I64(raw_to_vec(bytes, 8, |c| {
439 i64::from_le_bytes(c.try_into().unwrap())
440 })?)),
441 _ => Err(IonError::from(format!(
442 "unsupported dtype {dtype} in binary array"
443 ))),
444 }
445}
446
447fn dtype_to_numeric_type(dtype: u8) -> IonResult<NumericType> {
448 match dtype {
449 FILE_DTYPE_F64 => Ok(NumericType::Float64),
450 FILE_DTYPE_F32 => Ok(NumericType::Float32),
451 FILE_DTYPE_F16 => Ok(NumericType::Float16),
452 FILE_DTYPE_I16 => Ok(NumericType::Int16),
453 FILE_DTYPE_I32 => Ok(NumericType::Int32),
454 FILE_DTYPE_I64 => Ok(NumericType::Int64),
455 _ => Err(IonError::BadDtype {
456 dtype,
457 kind: "numeric type",
458 }),
459 }
460}
461
462fn make_binary_array_stub(array_type: u32, array_cv_code: u8) -> BinaryDataArray {
463 let cv_ref = crate::ion::attr_meta::cv_ref_prefix_from_code(array_cv_code).unwrap_or("MS");
464 let accession = crate::ion::attr_meta::format_accession(array_cv_code, array_type)
465 .unwrap_or_else(|| format_accession(array_type));
466 BinaryDataArray {
467 cv_params: vec![CvParam {
468 cv_ref: Some(cv_ref.to_string()),
469 accession: Some(accession),
470 name: String::new(),
471 ..Default::default()
472 }],
473 ..BinaryDataArray::default()
474 }
475}
476
477#[inline]
478fn sync_numeric_meta(binary_array: &mut BinaryDataArray, numeric_type: NumericType) {
479 let target = match numeric_type {
480 NumericType::Float16 => 1_000_520,
481 NumericType::Float32 => 1_000_521,
482 NumericType::Float64 => 1_000_523,
483 NumericType::Int16 => 1_000_518,
484 NumericType::Int32 => 1_000_519,
485 NumericType::Int64 => 1_000_522,
486 };
487 let param = CvParam {
488 cv_ref: Some("MS".into()),
489 accession: Some(format_accession(target)),
490 name: match target {
491 1_000_520 => "16-bit float",
492 1_000_521 => "32-bit float",
493 1_000_523 => "64-bit float",
494 1_000_518 => "16-bit integer",
495 1_000_519 => "32-bit integer",
496 1_000_522 => "64-bit integer",
497 _ => "numeric",
498 }
499 .into(),
500 ..Default::default()
501 };
502 match binary_array
503 .cv_params
504 .iter()
505 .position(|p| is_numeric_acc(parse_accession_tail(p.accession.as_deref())))
506 {
507 Some(position) => binary_array.cv_params[position] = param,
508 None => binary_array.cv_params.push(param),
509 }
510 binary_array.numeric_type = Some(numeric_type);
511}
512
513#[inline]
514fn is_numeric_acc(tail: AccessionTail) -> bool {
515 matches!(
516 tail.raw(),
517 INT_16BIT | INT_32BIT | INT_64BIT | FLOAT_16BIT | FLOAT_32BIT | FLOAT_64BIT
518 )
519}
520
521#[inline]
522fn binary_array_has_type(binary_array: &BinaryDataArray, array_type: u32) -> bool {
523 binary_array
524 .cv_params
525 .iter()
526 .any(|param| parse_accession_tail(param.accession.as_deref()).raw() == array_type)
527}
528
529type ListHeader<'l, L> = Option<(L, &'l [u32])>;
530
531fn list_header<L>(
532 rows: &[Metadatum],
533 parse: for<'l> fn(&OwnerRows, &'l ChildrenLookup) -> ListHeader<'l, L>,
534) -> Option<L> {
535 let mut owner_rows = OwnerRows::with_capacity(rows.len());
536 for row in rows {
537 owner_rows.insert(row.id, row);
538 }
539 parse(&owner_rows, &ChildrenLookup::new(rows)).map(|(list, _)| list)
540}
541
542fn parse_run_source_file_refs(
543 owner_rows: &OwnerRows,
544 lookup: &ChildrenLookup,
545 run_id: u32,
546) -> Option<SourceFileRefList> {
547 if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileRefList).first() {
548 let refs: Vec<_> = lookup
549 .ids_for(list_id, TagId::SourceFileRef)
550 .iter()
551 .filter_map(|&id| {
552 get_attr_text(owner_rows.get(id), ACC_ATTR_REF)
553 .map(|value| SourceFileRef { r#ref: value })
554 })
555 .collect();
556 if !refs.is_empty() {
557 return Some(SourceFileRefList {
558 count: Some(refs.len()),
559 source_file_refs: refs,
560 });
561 }
562 }
563
564 if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileList).first() {
565 let refs: Vec<_> = lookup
566 .ids_for(list_id, TagId::SourceFile)
567 .iter()
568 .filter_map(|&id| {
569 get_attr_text(owner_rows.get(id), ACC_ATTR_ID)
570 .map(|value| SourceFileRef { r#ref: value })
571 })
572 .collect();
573 if !refs.is_empty() {
574 return Some(SourceFileRefList {
575 count: Some(refs.len()),
576 source_file_refs: refs,
577 });
578 }
579 }
580
581 None
582}
583
584impl IonReader {
585 pub fn global_metadata(&self) -> IonResult<MzML> {
586 let mut mzml = MzmlConverter::run_level(self)?;
587 mzml.run.spectrum_list = list_header(
588 &self.spec_meta.read_first_group()?,
589 parse_spectrum_list_header,
590 );
591 mzml.run.chromatogram_list = list_header(
592 &self.chrom_meta.read_first_group()?,
593 parse_chromatogram_list_header,
594 );
595 Ok(mzml)
596 }
597
598 pub fn to_mzml(&mut self) -> IonResult<MzML> {
599 MzmlConverter::new(self).full()
600 }
601}
602
603#[cfg(test)]
604mod tests {
605 use super::*;
606
607 #[test]
608 fn f16_i16_arrays_get_real_cvparam_name_6() {
609 let mut float16_array = BinaryDataArray::default();
610 sync_numeric_meta(&mut float16_array, NumericType::Float16);
611 let float16_param = float16_array
612 .cv_params
613 .iter()
614 .find(|p| p.accession.as_deref() == Some("MS:1000520"))
615 .expect("sync_numeric_meta must add the 16-bit float precision cvParam");
616 assert_eq!(float16_param.name, "16-bit float");
617
618 let mut int16_array = BinaryDataArray::default();
619 sync_numeric_meta(&mut int16_array, NumericType::Int16);
620 let int16_param = int16_array
621 .cv_params
622 .iter()
623 .find(|p| p.accession.as_deref() == Some("MS:1000518"))
624 .expect("sync_numeric_meta must add the 16-bit integer precision cvParam");
625 assert_eq!(int16_param.name, "16-bit integer");
626 }
627}