1use std::{borrow::Cow, collections::HashMap};
2
3#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
4use rayon::prelude::*;
5
6use super::*;
7use crate::ion::decoder::utilities::byte_source::SourceBytes;
8
9#[derive(Debug, Clone, PartialEq)]
10pub enum MetadatumValue {
11 Number(f64),
12 Text(String),
13 Empty,
14}
15
16#[derive(Debug, Clone, PartialEq)]
17pub struct Metadatum {
18 pub(crate) item_index: u32,
19 pub(crate) id: u32,
20 pub(crate) parent_id: u32,
21 pub(crate) tag_id: TagId,
22 pub(crate) accession: Option<String>,
23 pub(crate) unit_accession: Option<String>,
24 pub(crate) value: MetadatumValue,
25}
26
27pub(crate) struct MzmlConverter<'d> {
28 pub(crate) decoder: &'d mut IonReader,
29}
30
31impl<'d> MzmlConverter<'d> {
32 #[inline]
33 pub(crate) fn new(decoder: &'d mut IonReader) -> Self {
34 Self { decoder }
35 }
36
37 pub(crate) fn metadata_only(decoder: &IonReader) -> IonResult<MzML> {
38 let global_meta = decoder.global_metadata()?;
39 let global_lookup = ChildrenLookup::new(&global_meta);
40 let meta_refs: Vec<&Metadatum> = global_meta.iter().collect();
41 let policy = DefaultMetadataPolicy;
42
43 let mut owner_rows = OwnerRows::with_capacity(global_meta.len());
44 for metadatum in &global_meta {
45 owner_rows.insert(metadatum.id, metadatum);
46 }
47
48 let run_id = global_lookup
49 .all_ids(TagId::Run)
50 .first()
51 .copied()
52 .unwrap_or(0);
53 let rows = owner_rows.get(run_id);
54
55 let mut param_buffer: Vec<&Metadatum> = Vec::new();
56 global_lookup.get_param_rows_into(&owner_rows, run_id, &policy, &mut param_buffer);
57 let (cv_params, user_params) = parse_cv_and_user_params(¶m_buffer);
58
59 let spectrum_list = assemble_spectrum_list(&decoder.spectrum_metadata_grouped()?, &policy);
60 let chromatogram_list =
61 assemble_chromatogram_list(&decoder.chromatogram_metadata_grouped()?, &policy);
62
63 let source_file_ref_list = parse_run_source_file_refs(&owner_rows, &global_lookup, run_id);
64
65 Ok(MzML {
66 cv_list: parse_cv_list(&meta_refs, &global_lookup),
67 file_description: parse_file_description(&meta_refs, &global_lookup, &policy),
68 referenceable_param_group_list: parse_referenceable_param_group_list(
69 &meta_refs,
70 &global_lookup,
71 &policy,
72 ),
73 sample_list: parse_sample_list(&meta_refs, &global_lookup, &policy),
74 instrument_list: parse_instrument_list(&meta_refs, &global_lookup, &policy),
75 software_list: parse_software_list(&meta_refs, &global_lookup, &policy),
76 data_processing_list: parse_data_processing_list(&meta_refs, &global_lookup, &policy),
77 scan_settings_list: parse_scan_settings_list(&meta_refs, &global_lookup, &policy),
78 run: Run {
79 id: get_attr_text(rows, ACC_ATTR_ID).unwrap_or_default(),
80 start_time_stamp: get_attr_text(rows, ACC_ATTR_START_TIME_STAMP)
81 .filter(|value| !value.is_empty()),
82 default_instrument_configuration_ref: get_attr_text(
83 rows,
84 ACC_ATTR_DEFAULT_INSTRUMENT_CONFIGURATION_REF,
85 )
86 .or_else(|| get_attr_text(rows, ACC_ATTR_INSTRUMENT_CONFIGURATION_REF)),
87 default_source_file_ref: get_attr_text(rows, ACC_ATTR_DEFAULT_SOURCE_FILE_REF),
88 sample_ref: get_attr_text(rows, ACC_ATTR_SAMPLE_REF),
89 referenceable_param_group_refs: global_lookup
90 .ids_for(run_id, TagId::ReferenceableParamGroupRef)
91 .iter()
92 .filter_map(|&ref_id| {
93 get_attr_text(owner_rows.get(ref_id), ACC_ATTR_REF)
94 .map(|r| ReferenceableParamGroupRef { r#ref: r })
95 })
96 .collect(),
97 cv_params,
98 user_params,
99 source_file_ref_list,
100 spectrum_list,
101 chromatogram_list,
102 },
103 })
104 }
105
106 pub(crate) fn full(&mut self) -> IonResult<MzML> {
107 let mut mzml = Self::metadata_only(self.decoder)?;
108
109 if let Some(spectrum_list) = mzml.run.spectrum_list.as_mut() {
110 attach_binaries(
111 self.decoder.spec_entries_buf.as_ref(),
112 self.decoder.spec_array_addresses.as_ref(),
113 &mut spectrum_list.spectra,
114 &self.decoder.spec_container,
115 "spec",
116 self.decoder.parallel,
117 )?;
118 }
119
120 if let (Some(chrom_list), Some(container)) = (
121 mzml.run.chromatogram_list.as_mut(),
122 self.decoder.chrom_container.as_ref(),
123 ) {
124 attach_binaries(
125 self.decoder.chrom_entries_buf.as_ref(),
126 self.decoder.chrom_array_addresses.as_ref(),
127 &mut chrom_list.chromatograms,
128 container,
129 "chrom",
130 self.decoder.parallel,
131 )?;
132 }
133
134 Ok(mzml)
135 }
136}
137
138fn assemble_spectrum_list(
139 groups: &[Vec<Metadatum>],
140 policy: &DefaultMetadataPolicy,
141) -> Option<SpectrumList> {
142 let mut combined: Option<SpectrumList> = None;
143 let mut index_base = 0u32;
144 for group in groups {
145 let refs: Vec<&Metadatum> = group.iter().collect();
146 let Some(list) =
147 parse_spectrum_list(&refs, &ChildrenLookup::new(group), policy, index_base)
148 else {
149 continue;
150 };
151 index_base += list.spectra.len() as u32;
152 match combined.as_mut() {
153 None => combined = Some(list),
154 Some(existing) => existing.spectra.extend(list.spectra),
155 }
156 }
157 if let Some(list) = combined.as_mut() {
158 if list.count.is_none() {
159 list.count = Some(list.spectra.len());
160 }
161 }
162 combined
163}
164
165fn assemble_chromatogram_list(
166 groups: &[Vec<Metadatum>],
167 policy: &DefaultMetadataPolicy,
168) -> Option<ChromatogramList> {
169 let mut combined: Option<ChromatogramList> = None;
170 let mut index_base = 0u32;
171 for group in groups {
172 let refs: Vec<&Metadatum> = group.iter().collect();
173 let Some(list) =
174 parse_chromatogram_list(&refs, &ChildrenLookup::new(group), policy, index_base)
175 else {
176 continue;
177 };
178 index_base += list.chromatograms.len() as u32;
179 match combined.as_mut() {
180 None => combined = Some(list),
181 Some(existing) => existing.chromatograms.extend(list.chromatograms),
182 }
183 }
184 if let Some(list) = combined.as_mut() {
185 if list.count.is_none() {
186 list.count = Some(list.chromatograms.len());
187 }
188 }
189 combined
190}
191
192pub(crate) trait BinaryArrayOwner {
193 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList>;
194}
195
196impl BinaryArrayOwner for Spectrum {
197 #[inline]
198 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
199 &mut self.binary_data_array_list
200 }
201}
202
203impl BinaryArrayOwner for Chromatogram {
204 #[inline]
205 fn binary_data_array_list_mut(&mut self) -> &mut Option<BinaryDataArrayList> {
206 &mut self.binary_data_array_list
207 }
208}
209
210pub(crate) fn attach_binaries<E: BinaryArrayOwner>(
211 entries_buf: &[u8],
212 array_addresses_buf: &[u8],
213 entries: &mut [E],
214 container: &BlockReader<DefaultBlockProcessor>,
215 ctx: &'static str,
216 parallel: bool,
217) -> IonResult<()> {
218 let mut refs = Vec::new();
219 let mut blocks = HashMap::new();
220 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
221 let _ = parallel;
222
223 for index in 0..entries.len() {
224 let Some(item_refs) =
225 read_array_addresses_from_buffers(entries_buf, array_addresses_buf, index)
226 else {
227 continue;
228 };
229 if item_refs.is_empty() {
230 continue;
231 }
232 for array_address in item_refs.as_slice() {
233 let stride = if array_address.encoded_len > 0 {
234 1
235 } else {
236 dtype_stride(array_address.dtype)
237 };
238 if let Some(old) = blocks.insert(array_address.block_id, stride)
239 && old != stride
240 {
241 return Err(IonError::from(format!(
242 "{ctx}: stride mismatch for block {} (expected {old}, got {stride})",
243 array_address.block_id
244 )));
245 }
246 }
247 refs.push((index, item_refs));
248 }
249
250 let mut block_list: Vec<_> = blocks.into_iter().collect();
251 block_list.sort_unstable_by_key(|(block_id, _)| *block_id);
252
253 let load = |(block_id, stride): (u32, usize)| -> IonResult<(u32, SourceBytes)> {
254 Ok((block_id, container.read_block(block_id, stride, ctx)?))
255 };
256
257 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
258 let data: HashMap<u32, SourceBytes> = if parallel && block_list.len() >= 4 {
259 block_list
260 .into_par_iter()
261 .map(load)
262 .collect::<IonResult<Vec<_>>>()?
263 .into_iter()
264 .collect()
265 } else {
266 block_list
267 .into_iter()
268 .map(load)
269 .collect::<IonResult<Vec<_>>>()?
270 .into_iter()
271 .collect()
272 };
273 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
274 let data: HashMap<u32, SourceBytes> = block_list
275 .into_iter()
276 .map(load)
277 .collect::<IonResult<Vec<_>>>()?
278 .into_iter()
279 .collect();
280
281 for (index, item_refs) in refs {
282 let groups = group_arrays(item_refs.as_slice())?;
283 let list = entries[index]
284 .binary_data_array_list_mut()
285 .get_or_insert_with(BinaryDataArrayList::default);
286
287 for group in groups {
288 if let [array_address] = group.refs.as_slice() {
289 let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
290 attach_logical_array(
291 list,
292 group.array_type,
293 group.array_cv_code,
294 group.dtype,
295 &bytes,
296 )?;
297 } else {
298 let mut total = 0usize;
299 for array_address in &group.refs {
300 total += checked_ref_bytes(&data, array_address, ctx)?.len();
301 }
302
303 let mut concatenated = Vec::new();
304 concatenated.reserve_exact(total);
305 for array_address in &group.refs {
306 let bytes = unfiltered_ref_bytes(&data, &group, array_address, ctx)?;
307 concatenated.extend_from_slice(&bytes);
308 }
309
310 attach_logical_array(
311 list,
312 group.array_type,
313 group.array_cv_code,
314 group.dtype,
315 &concatenated,
316 )?;
317 }
318 }
319 list.count = Some(list.binary_data_arrays.len());
320 }
321
322 Ok(())
323}
324
325fn ref_byte_range(array_address: &ArrayAddress, ctx: &str) -> IonResult<(usize, usize)> {
326 let (element_offset, count, stride) = address_read_params(array_address);
327 let start = usize::try_from(element_offset)
328 .ok()
329 .and_then(|offset| offset.checked_mul(stride))
330 .ok_or_else(|| {
331 IonError::from(format!(
332 "{ctx}: item range overflow for block {}",
333 array_address.block_id
334 ))
335 })?;
336 let end = usize::try_from(count)
337 .ok()
338 .and_then(|c| c.checked_mul(stride))
339 .and_then(|len| start.checked_add(len))
340 .ok_or_else(|| {
341 IonError::from(format!(
342 "{ctx}: item range overflow for block {}",
343 array_address.block_id
344 ))
345 })?;
346 Ok((start, end))
347}
348
349fn checked_ref_bytes<'d>(
350 data: &'d HashMap<u32, SourceBytes>,
351 array_address: &ArrayAddress,
352 ctx: &str,
353) -> IonResult<&'d [u8]> {
354 let block = data.get(&array_address.block_id).ok_or_else(|| {
355 IonError::from(format!("{ctx}: missing block {}", array_address.block_id))
356 })?;
357 let (start, end) = ref_byte_range(array_address, ctx)?;
358 block.get(start..end).ok_or_else(|| {
359 IonError::from(format!(
360 "{ctx}: item range [{start}..{end}] out of bounds for block {} (len={})",
361 array_address.block_id,
362 block.len()
363 ))
364 })
365}
366
367fn unfiltered_ref_bytes<'d>(
368 data: &'d HashMap<u32, SourceBytes>,
369 group: &ArrayGroup,
370 array_address: &ArrayAddress,
371 ctx: &str,
372) -> IonResult<Cow<'d, [u8]>> {
373 let raw = checked_ref_bytes(data, array_address, ctx)?;
374 unfilter_array_bytes(raw, group.dtype, group.array_filter)
375}
376
377pub(crate) fn attach_logical_array(
378 binary_array_list: &mut BinaryDataArrayList,
379 array_type: u32,
380 array_cv_code: u8,
381 dtype: u8,
382 decoded_bytes: &[u8],
383) -> IonResult<()> {
384 let binary = decoded_bytes_to_binary_data(decoded_bytes, dtype)?;
385 let numeric_type = dtype_to_numeric_type(dtype)?;
386
387 let empty_index = binary_array_list
388 .binary_data_arrays
389 .iter()
390 .position(|array| binary_array_has_type(array, array_type) && array.binary.is_none());
391
392 let binary_array = if let Some(index) = empty_index {
393 &mut binary_array_list.binary_data_arrays[index]
394 } else {
395 binary_array_list
396 .binary_data_arrays
397 .push(make_binary_array_stub(array_type, array_cv_code));
398 binary_array_list.binary_data_arrays.last_mut().unwrap()
399 };
400
401 binary_array.binary = Some(binary);
402 sync_numeric_meta(binary_array, numeric_type);
403 Ok(())
404}
405
406fn raw_to_vec<T>(raw: &[u8], elem_size: usize, read: impl Fn(&[u8]) -> T) -> IonResult<Vec<T>> {
407 if !raw.len().is_multiple_of(elem_size) {
408 return Err(IonError::from(format!(
409 "array: length {} not a multiple of {elem_size}",
410 raw.len()
411 )));
412 }
413 let mut out = Vec::with_capacity(raw.len() / elem_size);
414 out.extend(raw.chunks_exact(elem_size).map(read));
415 Ok(out)
416}
417
418pub(crate) fn decoded_bytes_to_binary_data(bytes: &[u8], dtype: u8) -> IonResult<NumericArray> {
419 match dtype {
420 FILE_DTYPE_F64 => Ok(NumericArray::F64(raw_to_vec(bytes, 8, |c| {
421 f64::from_le_bytes(c.try_into().unwrap())
422 })?)),
423 FILE_DTYPE_F32 => Ok(NumericArray::F32(raw_to_vec(bytes, 4, |c| {
424 f32::from_le_bytes(c.try_into().unwrap())
425 })?)),
426 FILE_DTYPE_F16 => Ok(NumericArray::F16(raw_to_vec(bytes, 2, |c| {
427 u16::from_le_bytes(c.try_into().unwrap())
428 })?)),
429 FILE_DTYPE_I16 => Ok(NumericArray::I16(raw_to_vec(bytes, 2, |c| {
430 i16::from_le_bytes(c.try_into().unwrap())
431 })?)),
432 FILE_DTYPE_I32 => Ok(NumericArray::I32(raw_to_vec(bytes, 4, |c| {
433 i32::from_le_bytes(c.try_into().unwrap())
434 })?)),
435 FILE_DTYPE_I64 => Ok(NumericArray::I64(raw_to_vec(bytes, 8, |c| {
436 i64::from_le_bytes(c.try_into().unwrap())
437 })?)),
438 _ => Err(IonError::from(format!(
439 "unsupported dtype {dtype} in binary array"
440 ))),
441 }
442}
443
444fn dtype_to_numeric_type(dtype: u8) -> IonResult<NumericType> {
445 match dtype {
446 FILE_DTYPE_F64 => Ok(NumericType::Float64),
447 FILE_DTYPE_F32 => Ok(NumericType::Float32),
448 FILE_DTYPE_F16 => Ok(NumericType::Float16),
449 FILE_DTYPE_I16 => Ok(NumericType::Int16),
450 FILE_DTYPE_I32 => Ok(NumericType::Int32),
451 FILE_DTYPE_I64 => Ok(NumericType::Int64),
452 _ => Err(IonError::BadDtype {
453 dtype,
454 kind: "numeric type",
455 }),
456 }
457}
458
459fn make_binary_array_stub(array_type: u32, array_cv_code: u8) -> BinaryDataArray {
460 let cv_ref = crate::ion::attr_meta::cv_ref_prefix_from_code(array_cv_code).unwrap_or("MS");
461 let accession = crate::ion::attr_meta::format_accession(array_cv_code, array_type)
462 .unwrap_or_else(|| format_accession(array_type));
463 BinaryDataArray {
464 cv_params: vec![CvParam {
465 cv_ref: Some(cv_ref.to_string()),
466 accession: Some(accession),
467 name: String::new(),
468 ..Default::default()
469 }],
470 ..BinaryDataArray::default()
471 }
472}
473
474#[inline]
475fn sync_numeric_meta(binary_array: &mut BinaryDataArray, numeric_type: NumericType) {
476 let target = match numeric_type {
477 NumericType::Float16 => 1_000_520,
478 NumericType::Float32 => 1_000_521,
479 NumericType::Float64 => 1_000_523,
480 NumericType::Int16 => 1_000_518,
481 NumericType::Int32 => 1_000_519,
482 NumericType::Int64 => 1_000_522,
483 };
484 let param = CvParam {
485 cv_ref: Some("MS".into()),
486 accession: Some(format_accession(target)),
487 name: match target {
488 1_000_520 => "16-bit float",
489 1_000_521 => "32-bit float",
490 1_000_523 => "64-bit float",
491 1_000_518 => "16-bit integer",
492 1_000_519 => "32-bit integer",
493 1_000_522 => "64-bit integer",
494 _ => "numeric",
495 }
496 .into(),
497 ..Default::default()
498 };
499 match binary_array
500 .cv_params
501 .iter()
502 .position(|p| is_numeric_acc(parse_accession_tail(p.accession.as_deref())))
503 {
504 Some(position) => binary_array.cv_params[position] = param,
505 None => binary_array.cv_params.push(param),
506 }
507 binary_array.numeric_type = Some(numeric_type);
508}
509
510#[inline]
511fn is_numeric_acc(tail: AccessionTail) -> bool {
512 matches!(
513 tail.raw(),
514 INT_16BIT | INT_32BIT | INT_64BIT | FLOAT_16BIT | FLOAT_32BIT | FLOAT_64BIT
515 )
516}
517
518#[inline]
519fn binary_array_has_type(binary_array: &BinaryDataArray, array_type: u32) -> bool {
520 binary_array
521 .cv_params
522 .iter()
523 .any(|param| parse_accession_tail(param.accession.as_deref()).raw() == array_type)
524}
525
526fn parse_run_source_file_refs(
527 owner_rows: &OwnerRows,
528 lookup: &ChildrenLookup,
529 run_id: u32,
530) -> Option<SourceFileRefList> {
531 if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileRefList).first() {
532 let refs: Vec<_> = lookup
533 .ids_for(list_id, TagId::SourceFileRef)
534 .iter()
535 .filter_map(|&id| {
536 get_attr_text(owner_rows.get(id), ACC_ATTR_REF)
537 .map(|value| SourceFileRef { r#ref: value })
538 })
539 .collect();
540 if !refs.is_empty() {
541 return Some(SourceFileRefList {
542 count: Some(refs.len()),
543 source_file_refs: refs,
544 });
545 }
546 }
547
548 if let Some(&list_id) = lookup.ids_for(run_id, TagId::SourceFileList).first() {
549 let refs: Vec<_> = lookup
550 .ids_for(list_id, TagId::SourceFile)
551 .iter()
552 .filter_map(|&id| {
553 get_attr_text(owner_rows.get(id), ACC_ATTR_ID)
554 .map(|value| SourceFileRef { r#ref: value })
555 })
556 .collect();
557 if !refs.is_empty() {
558 return Some(SourceFileRefList {
559 count: Some(refs.len()),
560 source_file_refs: refs,
561 });
562 }
563 }
564
565 None
566}
567
568impl IonReader {
569 pub fn metadata(&self) -> IonResult<MzML> {
570 MzmlConverter::metadata_only(self)
571 }
572
573 pub fn to_mzml(&mut self) -> IonResult<MzML> {
574 MzmlConverter::new(self).full()
575 }
576}
577
578#[cfg(test)]
579mod tests {
580 use super::*;
581
582 #[test]
583 fn f16_i16_arrays_get_real_cvparam_name_6() {
584 let mut float16_array = BinaryDataArray::default();
585 sync_numeric_meta(&mut float16_array, NumericType::Float16);
586 let float16_param = float16_array
587 .cv_params
588 .iter()
589 .find(|p| p.accession.as_deref() == Some("MS:1000520"))
590 .expect("sync_numeric_meta must add the 16-bit float precision cvParam");
591 assert_eq!(float16_param.name, "16-bit float");
592
593 let mut int16_array = BinaryDataArray::default();
594 sync_numeric_meta(&mut int16_array, NumericType::Int16);
595 let int16_param = int16_array
596 .cv_params
597 .iter()
598 .find(|p| p.accession.as_deref() == Some("MS:1000518"))
599 .expect("sync_numeric_meta must add the 16-bit integer precision cvParam");
600 assert_eq!(int16_param.name, "16-bit integer");
601 }
602}