Skip to main content

openreadout_mzml_writer/
lib.rs

1//! Indexed mzML 1.1.0 writer.
2//!
3//! Spectra come from any [`Dataset`] that implements `read_spectrum_view`. Arrays are written
4//! base64 + zlib (m/z as 64-bit, intensity as 32-bit float). The file is written under a
5//! temporary name, read back (well-formed XML, index offsets, SHA-1 checksum, every array
6//! decoded and hash-compared to what was written), and only then renamed into place.
7//!
8//! Controlled-vocabulary accessions are those of the public PSI-MS and Unit ontologies.
9//!
10//! mzML is the open mass-spectrometry exchange format of the HUPO Proteomics Standards
11//! Initiative, read by OpenMS, ProteoWizard, pyteomics, MZmine and most analysis software.
12//!
13//! # Example
14//!
15//! ```
16//! use std::path::Path;
17//!
18//! use openreadout_core::{Dataset, Result};
19//! use openreadout_mzml_writer::{MzmlExportOptions, default_output, export_mzml};
20//!
21//! /// Write the centroided spectra of run 0 of an opened file as `<stem>.mzML`.
22//! fn to_mzml(dataset: &mut dyn Dataset, input: &Path) -> Result<()> {
23//!     let mut options = MzmlExportOptions::default();
24//!     options.centroid = true;
25//!     let report = export_mzml(dataset, input, &default_output(input), &options)?;
26//!     println!("{} spectra, SHA-1 {}", report.spectra_written, report.sha1);
27//!     Ok(())
28//! }
29//! ```
30#![forbid(unsafe_code)]
31#![warn(missing_docs)]
32
33use std::collections::BTreeSet;
34use std::fmt::Write as _;
35use std::fs::File;
36use std::io::{BufWriter, Read, Write};
37use std::path::Path;
38
39use flate2::Compression;
40use flate2::read::ZlibDecoder;
41use flate2::write::ZlibEncoder;
42use openreadout_core::{Dataset, Error, Result, Spectrum, SpectrumView};
43use rayon::prelude::*;
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46
47const CREATOR_VERSION: &str = env!("CARGO_PKG_VERSION");
48
49/// Spectra read and compressed together, at most.
50const BATCH_SPECTRA: usize = 256;
51/// A batch ends once it holds this many data points (about 100 MB decoded and compressed).
52const BATCH_POINTS: usize = 4 << 20;
53
54/// What to write.
55#[derive(Debug, Clone, Default)]
56#[non_exhaustive]
57pub struct MzmlExportOptions {
58    /// Run (spectra collection) index.
59    pub run: u32,
60    /// Write the instrument's centroid lists instead of profiles where a scan has both.
61    pub centroid: bool,
62    /// Replace an existing output file.
63    pub overwrite: bool,
64    /// Only spectra with index in `[first, last]` (zero-based, inclusive).
65    pub index_range: Option<(u64, u64)>,
66}
67
68/// Result of an mzML export.
69#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
70pub struct MzmlExportReport {
71    /// The file that was read.
72    pub input: String,
73    /// The mzML file written.
74    pub output: String,
75    /// Always `mzml`.
76    pub format: String,
77    /// Spectra written.
78    pub spectra_written: u64,
79    /// Chromatograms written (TIC, SRM and the like).
80    pub chromatograms_written: u64,
81    /// Data points (m/z-intensity pairs) written, over all spectra.
82    pub points_written: u64,
83    /// Size of the written file in bytes.
84    pub bytes_written: u64,
85    /// True when the file was read back and every array, the index and the checksum agreed.
86    pub verified: bool,
87    /// `primary` or `centroid`.
88    pub view: String,
89    /// SHA-1 of the file up to the checksum element, as recorded in it.
90    pub sha1: String,
91    /// Spectra left out because the reader refused them, such as the optical
92    /// (electromagnetic radiation) spectra of a PDA detector in an mzML file.
93    #[serde(default, skip_serializing_if = "is_zero")]
94    pub spectra_skipped: u64,
95    /// Why spectra were left out, when some were.
96    #[serde(default, skip_serializing_if = "Vec::is_empty")]
97    pub warnings: Vec<String>,
98}
99
100#[allow(clippy::trivially_copy_pass_by_ref)] // serde passes a reference
101fn is_zero(n: &u64) -> bool {
102    *n == 0
103}
104
105/// Export run `opts.run` of `ds` (read from `input`) to `output` as indexed mzML.
106pub fn export_mzml(
107    ds: &mut dyn Dataset,
108    input: &Path,
109    output: &Path,
110    opts: &MzmlExportOptions,
111) -> Result<MzmlExportReport> {
112    openreadout_core::output::refuse_dataset(output, input, &*ds)?;
113    if output.exists() && !opts.overwrite {
114        return Err(Error::Usage(format!(
115            "{} exists; pass --overwrite to replace it",
116            output.display()
117        )));
118    }
119    let info = ds.info()?;
120    // A file without spectra runs (Waters MRM) can still hold chromatograms.
121    let no_run = openreadout_core::SpectraInfo::default();
122    let run = match info.spectra.get(opts.run as usize) {
123        Some(run) => run,
124        None if info.spectra.is_empty() && opts.run == 0 && opts.index_range.is_none() => &no_run,
125        None => {
126            return Err(Error::Usage(format!(
127                "run {} does not exist ({} spectra runs in this file)",
128                opts.run,
129                info.spectra.len()
130            )));
131        }
132    };
133    let scan_count = run.scan_count;
134    // `None`: the run has no spectra, and the file holds only its chromatograms.
135    let range = match opts.index_range {
136        Some((a, b)) if a <= b && b < scan_count => Some((a, b)),
137        Some((a, b)) => {
138            return Err(Error::Usage(format!(
139                "spectrum range {a}..={b} is outside 0..{scan_count}"
140            )));
141        }
142        None if scan_count == 0 => None,
143        None => Some((0, scan_count - 1)),
144    };
145    // Without spectra the chromatograms are read first: they decide the file content and
146    // whether there is anything to write.
147    let mut early_chroms = if range.is_some() {
148        None
149    } else {
150        let chroms = chromatograms(ds, &info, true)?;
151        if chroms.is_empty() {
152            return Err(Error::Usage(
153                "the run has no spectra and no chromatograms to write as mzML; `openreadout info` lists its traces and tables, which `export --format csv` writes"
154                    .into(),
155            ));
156        }
157        Some(chroms)
158    };
159    let chrom_types: Vec<(String, String)> =
160        early_chroms.as_deref().map_or_else(Vec::new, |chroms| {
161            let mut types: Vec<(String, String)> = Vec::new();
162            for c in chroms {
163                if !types.iter().any(|(a, _)| *a == c.accession) {
164                    types.push((c.accession.clone(), c.name.clone()));
165                }
166            }
167            types
168        });
169    let view = if opts.centroid {
170        SpectrumView::Centroid
171    } else {
172        SpectrumView::Primary
173    };
174    let thermo_ids = info.format.id == "thermo-raw";
175    let agilent_ids = info.format.id == "agilent-masshunter";
176    // Waters (`function=F process=0 scan=S`) and Sciex TOF (`sample=1 period=1 cycle=C
177    // experiment=E`) spectra carry their vendor native ids; Sciex MRM groups do not follow the
178    // WIFF native-id format and are numbered by scan instead.
179    let vendor_ids = vendor_native_ids(&info, run);
180    // An mzML source keeps its spectra's ids when it names their nativeID format.
181    let source_ids = source_native_id_format(&info, run).is_some();
182    let tmp = output.with_file_name(format!(
183        ".{}.partial-{}",
184        output
185            .file_name()
186            .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string()),
187        std::process::id()
188    ));
189    // A spectrum the reader refuses as unsupported (an optical spectrum in an mzML file, say) is
190    // left out. The spectrum count comes before the spectra in the file, so when the first pass
191    // finds such spectra, the file is written again without them.
192    let mut write = |skip: &BTreeSet<u64>| -> Result<(Written, Vec<(u64, Error)>)> {
193        let f = File::create(&tmp).map_err(|e| Error::io(&tmp, e))?;
194        let mut w = CountingWriter::new(BufWriter::new(f), &tmp);
195        let (head, source_configs) = header_xml(
196            &info,
197            run,
198            input,
199            thermo_ids,
200            vendor_ids,
201            range.map(|(first, last)| last - first + 1 - skip.len() as u64),
202            &chrom_types,
203        );
204        w.write_str(&head)?;
205        let mut offsets = Vec::new();
206        let mut hashes = Vec::new();
207        let mut points = 0u64;
208        // The TIC and SRM chromatograms of the written spectra (added when the source holds none)
209        let mut derived = DerivedChromatograms::new(agilent_ids);
210        let mut refused = Vec::new();
211        if let Some((first, last)) = range {
212            // Spectra are read in batches. While the threads compress one batch, this thread
213            // reads the next, then writes the compressed batch in order, so the file does not
214            // depend on the thread count.
215            // `read_batch` returns the batch, the index to read next and the spectra it refused.
216            type Batch = (Vec<(Spectrum, String)>, u64, Vec<(u64, Error)>);
217            let mut read_batch = |from: u64| -> Result<Batch> {
218                let mut batch = Vec::new();
219                let mut refused = Vec::new();
220                let mut batch_points = 0usize;
221                let mut i = from;
222                while i <= last && batch.len() < BATCH_SPECTRA && batch_points < BATCH_POINTS {
223                    if skip.contains(&i) {
224                        i += 1;
225                        continue;
226                    }
227                    let sp = match ds.read_spectrum_view(opts.run, i, view) {
228                        Ok(sp) => sp,
229                        Err(e @ Error::Unsupported { .. }) => {
230                            refused.push((i, e));
231                            i += 1;
232                            continue;
233                        }
234                        Err(e) => return Err(e),
235                    };
236                    let id = if source_ids {
237                        sp.native_id
238                            .clone()
239                            .unwrap_or_else(|| native_id(&sp, false))
240                    } else if agilent_ids {
241                        format!("scanId={}", sp.scan_number)
242                    } else if vendor_ids {
243                        sp.native_id
244                            .clone()
245                            .unwrap_or_else(|| native_id(&sp, false))
246                    } else {
247                        native_id(&sp, thermo_ids)
248                    };
249                    batch_points = batch_points.saturating_add(sp.mz.len());
250                    batch.push((sp, id));
251                    i += 1;
252                }
253                Ok((batch, i, refused))
254            };
255            let (mut batch, mut next_from, r) = read_batch(first)?;
256            refused.extend(r);
257            let mut k = 0u64;
258            while !batch.is_empty() {
259                let (encoded, next) = rayon::join(
260                    || {
261                        batch
262                            .par_iter()
263                            .enumerate()
264                            .map(|(j, (sp, id))| {
265                                spectrum_xml(sp, k + j as u64, id, !source_configs)
266                            })
267                            .collect::<Vec<_>>()
268                    },
269                    || read_batch(next_from),
270                );
271                for ((sp, id), xml) in batch.iter().zip(encoded) {
272                    let (xml, hash) = xml?;
273                    offsets.push((id.clone(), w.pos + 6)); // after the six-space indent
274                    points += sp.mz.len() as u64;
275                    derived.add(sp);
276                    hashes.push(hash);
277                    w.write_str(&xml)?;
278                }
279                k += batch.len() as u64;
280                let r;
281                (batch, next_from, r) = next?;
282                refused.extend(r);
283            }
284            w.write_str("    </spectrumList>\n")?;
285        }
286        // The source's chromatograms (mzML inputs: TIC, SRM traces, ...), as they were read.
287        let mut chroms = match early_chroms.take() {
288            Some(chroms) => chroms,
289            None => chromatograms(ds, &info, false)?,
290        };
291        derived.append_to(&mut chroms);
292        let mut chrom_offsets = Vec::new();
293        let mut chrom_hashes = Vec::new();
294        if !chroms.is_empty() {
295            w.write_str(&format!(
296                "    <chromatogramList count=\"{}\" defaultDataProcessingRef=\"openreadout_conversion\">\n",
297                chroms.len()
298            ))?;
299            for (k, c) in chroms.iter().enumerate() {
300                chrom_offsets.push((c.id.clone(), w.pos + 6));
301                let (xml, hash) = chromatogram_xml(c, k as u64)?;
302                chrom_hashes.push(hash);
303                w.write_str(&xml)?;
304            }
305            w.write_str("    </chromatogramList>\n")?;
306        }
307        w.write_str("  </run>\n</mzML>\n")?;
308        let index_offset = w.pos + 2;
309        // An index lists at least one offset, so a file without spectra has no spectrum index.
310        let mut idx = format!(
311            "  <indexList count=\"{}\">\n",
312            usize::from(range.is_some()) + usize::from(!chroms.is_empty())
313        );
314        if range.is_some() {
315            idx.push_str("    <index name=\"spectrum\">\n");
316            for (id, off) in &offsets {
317                let _ = writeln!(idx, "      <offset idRef=\"{}\">{off}</offset>", escape(id));
318            }
319            idx.push_str("    </index>\n");
320        }
321        if !chroms.is_empty() {
322            idx.push_str("    <index name=\"chromatogram\">\n");
323            for (id, off) in &chrom_offsets {
324                let _ = writeln!(idx, "      <offset idRef=\"{}\">{off}</offset>", escape(id));
325            }
326            idx.push_str("    </index>\n");
327        }
328        idx.push_str("  </indexList>\n");
329        let _ = write!(
330            idx,
331            "  <indexListOffset>{index_offset}</indexListOffset>\n  <fileChecksum>"
332        );
333        w.write_str(&idx)?;
334        let sha = w.sha1.clone().finish_hex();
335        w.write_str(&format!("{sha}</fileChecksum>\n</indexedmzML>\n"))?;
336        w.flush()?;
337        let written = Written {
338            offsets: offsets.into_iter().map(|(_, o)| o).collect(),
339            hashes,
340            chrom_offsets: chrom_offsets.into_iter().map(|(_, o)| o).collect(),
341            chrom_hashes,
342            points,
343            sha,
344            index_offset,
345        };
346        Ok((written, refused))
347    };
348    let mut skip = BTreeSet::new();
349    let mut warnings = Vec::new();
350    let mut result = write(&skip);
351    if let Ok((_, refused)) = &mut result
352        && let Some((_, first_error)) = refused.first()
353    {
354        skip = refused.iter().map(|(i, _)| *i).collect();
355        warnings.push(format!(
356            "Left out {} spectra that the reader refused. The first: {first_error}",
357            skip.len()
358        ));
359        let range_len = range.map_or(0, |(first, last)| last - first + 1);
360        result = if skip.len() as u64 == range_len {
361            // Nothing left to write: fail with the reason the first spectrum was refused.
362            Err(refused.swap_remove(0).1)
363        } else {
364            write(&skip)
365        };
366    }
367    let written = match result {
368        Ok((written, refused)) if refused.is_empty() => written,
369        // The second pass refused a spectrum the first one read: the written count is wrong.
370        Ok((_, mut refused)) => {
371            let _ = std::fs::remove_file(&tmp);
372            return Err(refused.swap_remove(0).1);
373        }
374        Err(e) => {
375            let _ = std::fs::remove_file(&tmp);
376            return Err(e);
377        }
378    };
379    if let Err(e) = verify(&tmp, &written) {
380        if std::env::var_os("OPENREADOUT_KEEP_PARTIAL").is_none() {
381            let _ = std::fs::remove_file(&tmp);
382        }
383        return Err(Error::Other(format!(
384            "read-back verification of the mzML failed ({e}); output discarded"
385        )));
386    }
387    let bytes_written = std::fs::metadata(&tmp).map_or(0, |m| m.len());
388    if output.exists() {
389        std::fs::remove_file(output).map_err(|e| Error::io(output, e))?;
390    }
391    std::fs::rename(&tmp, output).map_err(|e| Error::io(output, e))?;
392    Ok(MzmlExportReport {
393        input: input.display().to_string(),
394        output: output.display().to_string(),
395        format: "mzml".into(),
396        spectra_written: written.hashes.len() as u64,
397        chromatograms_written: written.chrom_hashes.len() as u64,
398        points_written: written.points,
399        bytes_written,
400        verified: true,
401        view: if opts.centroid { "centroid" } else { "primary" }.into(),
402        sha1: written.sha,
403        spectra_skipped: skip.len() as u64,
404        warnings,
405    })
406}
407
408/// Default output path: the input with `.mzML`.
409pub fn default_output(input: &Path) -> std::path::PathBuf {
410    input.with_extension("mzML")
411}
412
413struct Written {
414    offsets: Vec<u64>,
415    hashes: Vec<u128>,
416    chrom_offsets: Vec<u64>,
417    chrom_hashes: Vec<u128>,
418    points: u64,
419    sha: String,
420    index_offset: u64,
421}
422
423struct CountingWriter<'a, W: Write> {
424    inner: W,
425    pos: u64,
426    sha1: Sha1,
427    path: &'a Path,
428}
429
430impl<'a, W: Write> CountingWriter<'a, W> {
431    fn new(inner: W, path: &'a Path) -> Self {
432        CountingWriter {
433            inner,
434            pos: 0,
435            sha1: Sha1::new(),
436            path,
437        }
438    }
439    fn write_str(&mut self, s: &str) -> Result<()> {
440        self.inner
441            .write_all(s.as_bytes())
442            .map_err(|e| Error::io(self.path, e))?;
443        self.sha1.update(s.as_bytes());
444        self.pos += s.len() as u64;
445        Ok(())
446    }
447    fn flush(&mut self) -> Result<()> {
448        self.inner.flush().map_err(|e| Error::io(self.path, e))
449    }
450}
451
452/// Whether the run's spectra carry vendor native ids the writer can declare: Waters always,
453/// Sciex when the run holds no MRM groups.
454fn vendor_native_ids(
455    info: &openreadout_core::FileInfo,
456    run: &openreadout_core::SpectraInfo,
457) -> bool {
458    match info.format.id.as_str() {
459        "waters-raw" => true,
460        "sciex-wiff" => run
461            .extra
462            .get("stored_spectra")
463            .and_then(|v| v.as_str())
464            .is_some_and(|v| v != "SRM"),
465        _ => false,
466    }
467}
468
469/// The nativeID format an mzML source declares for its source file (`extra.source_files[0]`
470/// terms, e.g. `MS:1000768` Thermo nativeID format), when the run is read from mzML.
471fn source_native_id_format(
472    info: &openreadout_core::FileInfo,
473    run: &openreadout_core::SpectraInfo,
474) -> Option<(String, String)> {
475    if !matches!(info.format.id.as_str(), "mzml" | "mzmlb") {
476        return None;
477    }
478    let terms = run
479        .extra
480        .get("source_files")?
481        .as_array()?
482        .first()?
483        .get("terms")?
484        .as_array()?;
485    terms.iter().find_map(|t| {
486        let name = t.get("name")?.as_str()?;
487        let acc = t.get("accession")?.as_str()?;
488        (name.ends_with("nativeID format") && acc.starts_with("MS:"))
489            .then(|| (acc.to_string(), name.to_string()))
490    })
491}
492
493fn native_id(sp: &Spectrum, thermo: bool) -> String {
494    if thermo {
495        format!(
496            "controllerType=0 controllerNumber=1 scan={}",
497            sp.scan_number
498        )
499    } else {
500        format!("scan={}", sp.scan_number)
501    }
502}
503
504/// XML attribute/text escaping.
505pub(crate) fn escape(s: &str) -> String {
506    let mut o = String::with_capacity(s.len());
507    for ch in s.chars() {
508        match ch {
509            '&' => o.push_str("&amp;"),
510            '<' => o.push_str("&lt;"),
511            '>' => o.push_str("&gt;"),
512            '"' => o.push_str("&quot;"),
513            '\'' => o.push_str("&apos;"),
514            c if (c as u32) < 0x20 && c != '\t' && c != '\n' && c != '\r' => {}
515            c => o.push(c),
516        }
517    }
518    o
519}
520
521fn cv(acc: &str, name: &str, value: &str) -> String {
522    format!(
523        "<cvParam cvRef=\"{}\" accession=\"{acc}\" name=\"{name}\" value=\"{}\"/>",
524        acc.split(':').next().unwrap_or("MS"),
525        escape(value)
526    )
527}
528
529fn cv_unit(acc: &str, name: &str, value: &str, uacc: &str, uname: &str) -> String {
530    format!(
531        "<cvParam cvRef=\"MS\" accession=\"{acc}\" name=\"{name}\" value=\"{}\" unitCvRef=\"{}\" unitAccession=\"{uacc}\" unitName=\"{uname}\"/>",
532        escape(value),
533        uacc.split(':').next().unwrap_or("MS")
534    )
535}
536
537/// Instrument configurations: one per analyzer token seen in the run (`FTMS`, `ITMS`, ...).
538fn analyzers(run: &openreadout_core::SpectraInfo) -> Vec<String> {
539    let mut v: Vec<String> = run
540        .extra
541        .get("analyzers")
542        .and_then(|a| a.as_array())
543        .map(|a| {
544            a.iter()
545                .filter_map(|x| x.as_str().map(str::to_string))
546                .collect()
547        })
548        .unwrap_or_default();
549    if v.is_empty() {
550        v.push("MS".into());
551    }
552    v
553}
554
555fn source_cv(run: &openreadout_core::SpectraInfo) -> String {
556    let src = run
557        .extra
558        .get("ion_sources")
559        .and_then(|a| a.as_array())
560        .and_then(|a| a.first())
561        .and_then(|x| x.as_str())
562        .unwrap_or("");
563    match src {
564        "ESI" => cv("MS:1000073", "electrospray ionization", ""),
565        "NSI" => cv("MS:1000398", "nanoelectrospray", ""),
566        "MALDI" => cv(
567            "MS:1000075",
568            "matrix-assisted laser desorption ionization",
569            "",
570        ),
571        "APCI" => cv("MS:1000070", "atmospheric pressure chemical ionization", ""),
572        "EI" => cv("MS:1000389", "electron ionization", ""),
573        "CI" => cv("MS:1000071", "chemical ionization", ""),
574        _ => cv("MS:1000008", "ionization type", src),
575    }
576}
577
578fn analyzer_cv(token: &str) -> (String, String) {
579    match token {
580        "FTMS" => (
581            cv("MS:1000484", "orbitrap", ""),
582            cv("MS:1000624", "inductive detector", ""),
583        ),
584        "ITMS" => (
585            cv("MS:1000083", "radial ejection linear ion trap", ""),
586            cv("MS:1000253", "electron multiplier", ""),
587        ),
588        "TQMS" | "SQMS" => (
589            cv("MS:1000081", "quadrupole", ""),
590            cv("MS:1000253", "electron multiplier", ""),
591        ),
592        "TOFMS" => (
593            cv("MS:1000084", "time-of-flight", ""),
594            cv("MS:1000253", "electron multiplier", ""),
595        ),
596        other => (
597            cv("MS:1000443", "mass analyzer type", other),
598            cv("MS:1000026", "detector type", ""),
599        ),
600    }
601}
602
603/// One instrument configuration of the source (mzML inputs): its model and serial terms
604/// `(accession, name)`, and its components `(kind, order, [(accession, name)])`.
605struct SourceConfig {
606    terms: Vec<Term>,
607    components: Vec<(String, u32, Vec<Term>)>,
608}
609
610/// A CV term: `(accession, name)`; the accession is empty when the source gave none.
611type Term = (String, String);
612
613/// The source's instrument configurations (`extra.instrument_configurations`, written by the
614/// mzML reader); empty for other inputs or when a configuration lists no components.
615fn source_configs(run: &openreadout_core::SpectraInfo) -> Vec<SourceConfig> {
616    let Some(list) = run
617        .extra
618        .get("instrument_configurations")
619        .and_then(|v| v.as_array())
620    else {
621        return Vec::new();
622    };
623    let str_of = |v: &serde_json::Value, k: &str| {
624        v.get(k)
625            .and_then(|x| x.as_str())
626            .unwrap_or_default()
627            .to_string()
628    };
629    let mut out = Vec::new();
630    for c in list {
631        let terms = c
632            .get("terms")
633            .and_then(|t| t.as_array())
634            .map(|t| {
635                t.iter()
636                    .map(|t| (str_of(t, "accession"), str_of(t, "name")))
637                    .filter(|(a, _)| !a.is_empty())
638                    .collect()
639            })
640            .unwrap_or_default();
641        let mut components = Vec::new();
642        for (k, comp) in c
643            .get("components")
644            .and_then(|v| v.as_array())
645            .into_iter()
646            .flatten()
647            .enumerate()
648        {
649            let kind = str_of(comp, "kind");
650            if !matches!(kind.as_str(), "source" | "analyzer" | "detector") {
651                continue;
652            }
653            let order = comp
654                .get("order")
655                .and_then(serde_json::Value::as_u64)
656                .and_then(|o| u32::try_from(o).ok())
657                .unwrap_or(k as u32 + 1);
658            let names = comp.get("terms").and_then(|v| v.as_array());
659            let accs = comp.get("accessions").and_then(|v| v.as_array());
660            let terms: Vec<Term> = names
661                .into_iter()
662                .flatten()
663                .enumerate()
664                .filter_map(|(i, n)| {
665                    let acc = accs
666                        .and_then(|a| a.get(i))
667                        .and_then(|a| a.as_str())
668                        .unwrap_or_default();
669                    Some((acc.to_string(), n.as_str()?.to_string()))
670                })
671                .collect();
672            components.push((kind, order, terms));
673        }
674        if components.is_empty() {
675            return Vec::new();
676        }
677        out.push(SourceConfig { terms, components });
678    }
679    out
680}
681
682/// A component term: its own accession when known, else the generic parent term of the
683/// component kind with the name as its value.
684fn component_cv(kind: &str, acc: &str, name: &str) -> String {
685    if acc.starts_with("MS:") {
686        return cv(&escape(acc), &escape(name), "");
687    }
688    match kind {
689        "source" => cv("MS:1000008", "ionization type", name),
690        "analyzer" => cv("MS:1000443", "mass analyzer type", name),
691        _ => cv("MS:1000026", "detector type", name),
692    }
693}
694
695fn header_xml(
696    info: &openreadout_core::FileInfo,
697    run: &openreadout_core::SpectraInfo,
698    input: &Path,
699    thermo: bool,
700    vendor_ids: bool,
701    count: Option<u64>,
702    chrom_types: &[(String, String)],
703) -> (String, bool) {
704    let mut levels = run.ms_levels.clone();
705    levels.sort_unstable();
706    let name = input
707        .file_name()
708        .map_or_else(String::new, |s| s.to_string_lossy().to_string());
709    let location = input
710        .parent()
711        .map(|p| format!("file://{}", p.display()))
712        .unwrap_or_default();
713    let id = input
714        .file_stem()
715        .map_or_else(|| "run".into(), |s| s.to_string_lossy().to_string());
716    let inst = run.instrument.clone().unwrap_or_default();
717    let serial = run
718        .extra
719        .get("instrument_serial")
720        .and_then(|v| v.as_str())
721        .unwrap_or("");
722    let mut s = String::new();
723    s.push_str("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
724    s.push_str("<indexedmzML xmlns=\"http://psi.hupo.org/ms/mzml\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://psi.hupo.org/ms/mzml http://psidev.info/files/ms/mzML/xsd/mzML1.1.2_idx.xsd\">\n");
725    let _ = writeln!(
726        s,
727        "<mzML xmlns=\"http://psi.hupo.org/ms/mzml\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://psi.hupo.org/ms/mzml http://psidev.info/files/ms/mzML/xsd/mzML1.1.0.xsd\" id=\"{}\" version=\"1.1.0\">",
728        escape(&id)
729    );
730    s.push_str("  <cvList count=\"2\">\n    <cv id=\"MS\" fullName=\"Proteomics Standards Initiative Mass Spectrometry Ontology\" version=\"4.1.0\" URI=\"https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/psi-ms.obo\"/>\n    <cv id=\"UO\" fullName=\"Unit Ontology\" version=\"09:04:2014\" URI=\"https://raw.githubusercontent.com/bio-ontology-research-group/unit-ontology/master/unit.obo\"/>\n  </cvList>\n");
731    s.push_str("  <fileDescription>\n    <fileContent>\n");
732    if levels.contains(&1) {
733        let _ = writeln!(s, "      {}", cv("MS:1000579", "MS1 spectrum", ""));
734    }
735    if levels.iter().any(|&l| l > 1) {
736        let _ = writeln!(s, "      {}", cv("MS:1000580", "MSn spectrum", ""));
737    }
738    for (acc, name) in chrom_types {
739        let _ = writeln!(s, "      {}", cv(&escape(acc), &escape(name), ""));
740    }
741    s.push_str("    </fileContent>\n    <sourceFileList count=\"1\">\n");
742    let _ = writeln!(
743        s,
744        "      <sourceFile id=\"RAW1\" name=\"{}\" location=\"{}\">",
745        escape(&name),
746        escape(&location)
747    );
748    if thermo {
749        let _ = writeln!(
750            s,
751            "        {}",
752            cv("MS:1000768", "Thermo nativeID format", "")
753        );
754        let _ = writeln!(s, "        {}", cv("MS:1000563", "Thermo RAW format", ""));
755    } else if info.format.id == "waters-raw" && vendor_ids {
756        let _ = writeln!(
757            s,
758            "        {}",
759            cv("MS:1000769", "Waters nativeID format", "")
760        );
761        let _ = writeln!(s, "        {}", cv("MS:1000526", "Waters raw format", ""));
762    } else if info.format.id == "sciex-wiff" && vendor_ids {
763        let _ = writeln!(
764            s,
765            "        {}",
766            cv("MS:1000770", "WIFF nativeID format", "")
767        );
768        let _ = writeln!(s, "        {}", cv("MS:1000562", "ABI WIFF format", ""));
769    } else if info.format.id == "agilent-masshunter" {
770        let _ = writeln!(
771            s,
772            "        {}",
773            cv("MS:1001508", "Agilent MassHunter nativeID format", "")
774        );
775        let _ = writeln!(
776            s,
777            "        {}",
778            cv("MS:1001509", "Agilent MassHunter format", "")
779        );
780    } else if let Some((acc, term)) = source_native_id_format(info, run) {
781        let _ = writeln!(s, "        {}", cv(&escape(&acc), &escape(&term), ""));
782    } else {
783        let _ = writeln!(
784            s,
785            "        {}",
786            cv("MS:1000776", "scan number only nativeID format", "")
787        );
788    }
789    s.push_str("      </sourceFile>\n    </sourceFileList>\n  </fileDescription>\n");
790    s.push_str("  <referenceableParamGroupList count=\"1\">\n    <referenceableParamGroup id=\"CommonInstrumentParams\">\n");
791    let model = inst.model.clone().unwrap_or_default();
792    // mzML inputs: the source's own instrument configurations, with their exact terms
793    let configs = source_configs(run);
794    let model_term = configs
795        .first()
796        .and_then(|c| {
797            c.terms
798                .iter()
799                .find(|(a, n)| a != "MS:1000529" && *n == model)
800        })
801        .filter(|_| !model.is_empty());
802    if let Some((acc, name)) = model_term {
803        let _ = writeln!(s, "      {}", cv(&escape(acc), &escape(name), ""));
804    } else if thermo {
805        let _ = writeln!(
806            s,
807            "      {}",
808            cv(
809                "MS:1000483",
810                "Thermo Fisher Scientific instrument model",
811                ""
812            )
813        );
814    } else if info.format.id == "agilent-masshunter" {
815        let _ = writeln!(
816            s,
817            "      {}",
818            cv("MS:1000490", "Agilent instrument model", "")
819        );
820    } else if info.format.id == "waters-raw" {
821        let _ = writeln!(
822            s,
823            "      {}",
824            cv("MS:1000126", "Waters instrument model", "")
825        );
826    } else if info.format.id == "sciex-wiff" {
827        let _ = writeln!(
828            s,
829            "      {}",
830            cv("MS:1000121", "SCIEX instrument model", "")
831        );
832    } else {
833        let _ = writeln!(s, "      {}", cv("MS:1000031", "instrument model", ""));
834    }
835    if !serial.is_empty() {
836        let _ = writeln!(
837            s,
838            "      {}",
839            cv("MS:1000529", "instrument serial number", serial)
840        );
841    }
842    if !model.is_empty() && model_term.is_none() {
843        let _ = writeln!(
844            s,
845            "      <userParam name=\"instrument model name\" value=\"{}\"/>",
846            escape(&model)
847        );
848    }
849    s.push_str("    </referenceableParamGroup>\n  </referenceableParamGroupList>\n");
850    s.push_str("  <softwareList count=\"2\">\n");
851    let _ = writeln!(
852        s,
853        "    <software id=\"openreadout\" version=\"{CREATOR_VERSION}\">\n      {}\n    </software>",
854        cv(
855            "MS:1000799",
856            "custom unreleased software tool",
857            "openreadout"
858        )
859    );
860    // the source's own term for its software when it has one (mzML inputs)
861    let software_term = inst.software.as_deref().and_then(|name| {
862        run.extra
863            .get("software")?
864            .as_array()?
865            .iter()
866            .find(|e| e.get("name").and_then(|n| n.as_str()) == Some(name))?
867            .get("accession")?
868            .as_str()
869            .map(|a| cv(&escape(a), &escape(name), ""))
870    });
871    let _ = writeln!(
872        s,
873        "    <software id=\"acquisition\" version=\"{}\">\n      {}\n    </software>",
874        escape(inst.software_version.as_deref().unwrap_or("unknown")),
875        software_term.unwrap_or_else(|| cv(
876            "MS:1000531",
877            "software",
878            inst.software.as_deref().unwrap_or("acquisition software")
879        ))
880    );
881    s.push_str("  </softwareList>\n");
882    if !configs.is_empty() {
883        let _ = writeln!(
884            s,
885            "  <instrumentConfigurationList count=\"{}\">",
886            configs.len()
887        );
888        for (k, c) in configs.iter().enumerate() {
889            let _ = writeln!(
890                s,
891                "    <instrumentConfiguration id=\"IC{}\">\n      <referenceableParamGroupRef ref=\"CommonInstrumentParams\"/>\n      <componentList count=\"{}\">",
892                k + 1,
893                c.components.len()
894            );
895            for (kind, order, terms) in &c.components {
896                let _ = writeln!(s, "        <{kind} order=\"{order}\">");
897                for (acc, name) in terms {
898                    let _ = writeln!(s, "          {}", component_cv(kind, acc, name));
899                }
900                let _ = writeln!(s, "        </{kind}>");
901            }
902            s.push_str("      </componentList>\n      <softwareRef ref=\"acquisition\"/>\n    </instrumentConfiguration>\n");
903        }
904        s.push_str("  </instrumentConfigurationList>\n");
905    }
906    let ans = if configs.is_empty() {
907        analyzers(run)
908    } else {
909        Vec::new()
910    };
911    if !ans.is_empty() {
912        let _ = writeln!(s, "  <instrumentConfigurationList count=\"{}\">", ans.len());
913    }
914    for (k, a) in ans.iter().enumerate() {
915        let (an, det) = analyzer_cv(a);
916        let _ = writeln!(
917            s,
918            "    <instrumentConfiguration id=\"IC{}\">\n      <referenceableParamGroupRef ref=\"CommonInstrumentParams\"/>\n      <componentList count=\"3\">\n        <source order=\"1\">\n          {}\n        </source>\n        <analyzer order=\"2\">\n          {an}\n        </analyzer>\n        <detector order=\"3\">\n          {det}\n        </detector>\n      </componentList>\n      <softwareRef ref=\"acquisition\"/>\n    </instrumentConfiguration>",
919            k + 1,
920            source_cv(run)
921        );
922    }
923    if !ans.is_empty() {
924        s.push_str("  </instrumentConfigurationList>\n");
925    }
926    let _ = writeln!(
927        s,
928        "  <dataProcessingList count=\"1\">\n    <dataProcessing id=\"openreadout_conversion\">\n      <processingMethod order=\"0\" softwareRef=\"openreadout\">\n        {}\n      </processingMethod>\n    </dataProcessing>\n  </dataProcessingList>",
929        cv("MS:1000544", "Conversion to mzML", "")
930    );
931    let start = run
932        .extra
933        .get("acquired_at")
934        .and_then(|v| v.as_str())
935        .map(|t| {
936            format!(
937                " startTimeStamp=\"{}\"",
938                escape(t.split('.').next().unwrap_or(t))
939            )
940        })
941        .unwrap_or_default();
942    // keep the source's run id (mzML inputs) when it is a valid XML ID
943    let run_id = run
944        .extra
945        .get("run_id")
946        .and_then(|v| v.as_str())
947        .filter(|r| xml_id(r) == *r)
948        .map_or_else(|| xml_id(&id), str::to_string);
949    let _ = writeln!(
950        s,
951        "  <run id=\"{}\" defaultInstrumentConfigurationRef=\"IC1\"{start} defaultSourceFileRef=\"RAW1\">",
952        escape(&run_id)
953    );
954    // A spectrumList holds at least one spectrum: a run without spectra has none.
955    if let Some(count) = count {
956        let _ = writeln!(
957            s,
958            "    <spectrumList count=\"{count}\" defaultDataProcessingRef=\"openreadout_conversion\">"
959        );
960    }
961    let _ = info;
962    (s, !configs.is_empty())
963}
964
965/// `s` as a valid XML ID (`xs:ID`): characters other than ASCII letters, digits, `_`, `-` and
966/// `.` become `_`, and an ID that does not start with a letter or `_` gets a leading `_`.
967fn xml_id(s: &str) -> String {
968    let mut id: String = s
969        .chars()
970        .map(|c| {
971            if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.') {
972                c
973            } else {
974                '_'
975            }
976        })
977        .collect();
978    if !id.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') {
979        id.insert(0, '_');
980    }
981    id
982}
983
984/// `(analyzer token, activation token, energy, windows)` parsed from a scan filter.
985type FilterParts<'a> = (Option<&'a str>, Option<String>, Option<f64>, Vec<[f64; 2]>);
986
987fn filter_parts(f: &str) -> FilterParts<'_> {
988    let analyzer = f.split_whitespace().next();
989    let mut act = None;
990    let mut energy = None;
991    for tok in f.split_whitespace() {
992        if let Some((_, rest)) = tok.split_once('@') {
993            let letters: String = rest.chars().take_while(char::is_ascii_alphabetic).collect();
994            let num: String = rest[letters.len()..]
995                .chars()
996                .take_while(|c| c.is_ascii_digit() || *c == '.')
997                .collect();
998            act = Some(letters);
999            energy = num.parse().ok();
1000        }
1001    }
1002    // `[lo-hi]`, or `[lo-hi, lo-hi, ...]` for SRM product windows.
1003    let windows = f
1004        .rfind('[')
1005        .map(|a| {
1006            let inner = &f[a + 1..f[a..].find(']').map_or(f.len(), |b| a + b)];
1007            inner
1008                .split(',')
1009                .filter_map(|w| {
1010                    let (lo, hi) = w.split_once('-')?;
1011                    Some([lo.trim().parse().ok()?, hi.trim().parse().ok()?])
1012                })
1013                .collect()
1014        })
1015        .unwrap_or_default();
1016    (analyzer, act, energy, windows)
1017}
1018
1019fn fmt_f(v: f64) -> String {
1020    let s = format!("{v}");
1021    if s.contains('.') || s.contains('e') || s.contains("inf") || s.contains("NaN") {
1022        s
1023    } else {
1024        format!("{s}.0")
1025    }
1026}
1027
1028/// `analyzer_refs`: point ion-trap (`ITMS`) scans at the second instrument configuration, as
1029/// written for scan-filter analyzers (not when the source's own configurations are copied).
1030fn spectrum_xml(
1031    sp: &Spectrum,
1032    index: u64,
1033    id: &str,
1034    analyzer_refs: bool,
1035) -> Result<(String, u128)> {
1036    if sp.mz.len() != sp.intensity.len() {
1037        return Err(Error::corrupt(
1038            "mzml",
1039            format!(
1040                "spectrum {index}: m/z array has {} values, intensity array {}",
1041                sp.mz.len(),
1042                sp.intensity.len()
1043            ),
1044        ));
1045    }
1046    let mut mz_bytes = Vec::with_capacity(sp.mz.len() * 8);
1047    for v in &sp.mz {
1048        mz_bytes.extend_from_slice(&v.to_le_bytes());
1049    }
1050    let mut in_bytes = Vec::with_capacity(sp.intensity.len() * 4);
1051    for v in &sp.intensity {
1052        in_bytes.extend_from_slice(&v.to_le_bytes());
1053    }
1054    let mut h = mz_bytes.clone();
1055    h.extend_from_slice(&in_bytes);
1056    let hash = xxhash_rust::xxh3::xxh3_128(&h);
1057    let mz_b64 = base64_encode(&zlib(&mz_bytes)?);
1058    let in_b64 = base64_encode(&zlib(&in_bytes)?);
1059    let (analyzer, act, energy, windows) = sp
1060        .scan_filter
1061        .as_deref()
1062        .map_or((None, None, None, Vec::new()), filter_parts);
1063    let has_token = |t: &str| {
1064        sp.scan_filter
1065            .as_deref()
1066            .is_some_and(|f| f.split_whitespace().any(|w| w == t))
1067    };
1068    // Readers without scan filters (mzML/mzXML inputs, timsTOF) fill the Spectrum fields instead.
1069    let act = act.or_else(|| sp.activation.as_deref().map(str::to_ascii_lowercase));
1070    let energy = energy.or(sp.collision_energy);
1071    let ic = match analyzer {
1072        Some("ITMS") if analyzer_refs => " instrumentConfigurationRef=\"IC2\"",
1073        _ => "",
1074    };
1075    let mut s = String::new();
1076    let _ = writeln!(
1077        s,
1078        "      <spectrum index=\"{index}\" id=\"{}\" defaultArrayLength=\"{}\">",
1079        escape(id),
1080        sp.mz.len()
1081    );
1082    let i = "        ";
1083    if has_token("SRM") {
1084        let _ = writeln!(s, "{i}{}", cv("MS:1000583", "SRM spectrum", ""));
1085    } else if has_token("CRM") {
1086        let _ = writeln!(s, "{i}{}", cv("MS:1000581", "CRM spectrum", ""));
1087    } else if sp.ms_level <= 1 {
1088        let _ = writeln!(s, "{i}{}", cv("MS:1000579", "MS1 spectrum", ""));
1089    } else {
1090        let _ = writeln!(s, "{i}{}", cv("MS:1000580", "MSn spectrum", ""));
1091    }
1092    let _ = writeln!(
1093        s,
1094        "{i}{}",
1095        cv("MS:1000511", "ms level", &sp.ms_level.to_string())
1096    );
1097    match sp.polarity.as_str() {
1098        "positive" => {
1099            let _ = writeln!(s, "{i}{}", cv("MS:1000130", "positive scan", ""));
1100        }
1101        "negative" => {
1102            let _ = writeln!(s, "{i}{}", cv("MS:1000129", "negative scan", ""));
1103        }
1104        _ => {}
1105    }
1106    if sp.centroided {
1107        let _ = writeln!(s, "{i}{}", cv("MS:1000127", "centroid spectrum", ""));
1108    } else {
1109        let _ = writeln!(s, "{i}{}", cv("MS:1000128", "profile spectrum", ""));
1110    }
1111    if let Some((k, &bi)) = sp
1112        .intensity
1113        .iter()
1114        .enumerate()
1115        .max_by(|a, b| a.1.total_cmp(b.1))
1116    {
1117        let _ = writeln!(
1118            s,
1119            "{i}{}",
1120            cv_unit(
1121                "MS:1000504",
1122                "base peak m/z",
1123                &fmt_f(sp.mz[k]),
1124                "MS:1000040",
1125                "m/z"
1126            )
1127        );
1128        let _ = writeln!(
1129            s,
1130            "{i}{}",
1131            cv_unit(
1132                "MS:1000505",
1133                "base peak intensity",
1134                &fmt_f(f64::from(bi)),
1135                "MS:1000131",
1136                "number of detector counts"
1137            )
1138        );
1139    }
1140    if let Some(t) = sp.total_ion_current {
1141        let _ = writeln!(s, "{i}{}", cv("MS:1000285", "total ion current", &fmt_f(t)));
1142    }
1143    if let (Some(lo), Some(hi)) = (sp.mz.first(), sp.mz.last()) {
1144        let _ = writeln!(
1145            s,
1146            "{i}{}",
1147            cv_unit(
1148                "MS:1000528",
1149                "lowest observed m/z",
1150                &fmt_f(*lo),
1151                "MS:1000040",
1152                "m/z"
1153            )
1154        );
1155        let _ = writeln!(
1156            s,
1157            "{i}{}",
1158            cv_unit(
1159                "MS:1000527",
1160                "highest observed m/z",
1161                &fmt_f(*hi),
1162                "MS:1000040",
1163                "m/z"
1164            )
1165        );
1166    }
1167    let _ = writeln!(s, "{i}<scanList count=\"1\">");
1168    let _ = writeln!(s, "{i}  {}", cv("MS:1000795", "no combination", ""));
1169    let _ = writeln!(s, "{i}  <scan{ic}>");
1170    // a spectrum whose source states no retention time gets none (never an invented 0)
1171    if let Some(rt) = sp.rt_s {
1172        let _ = writeln!(
1173            s,
1174            "{i}    {}",
1175            cv_unit(
1176                "MS:1000016",
1177                "scan start time",
1178                &fmt_f(rt / 60.0),
1179                "UO:0000031",
1180                "minute"
1181            )
1182        );
1183    }
1184    if let Some(f) = &sp.scan_filter {
1185        let _ = writeln!(s, "{i}    {}", cv("MS:1000512", "filter string", f));
1186    }
1187    if !windows.is_empty() {
1188        let _ = writeln!(s, "{i}    <scanWindowList count=\"{}\">", windows.len());
1189    }
1190    for [lo, hi] in &windows {
1191        let (lo, hi) = (*lo, *hi);
1192        let _ = writeln!(
1193            s,
1194            "{i}      <scanWindow>\n{i}        {}\n{i}        {}\n{i}      </scanWindow>",
1195            cv_unit(
1196                "MS:1000501",
1197                "scan window lower limit",
1198                &fmt_f(lo),
1199                "MS:1000040",
1200                "m/z"
1201            ),
1202            cv_unit(
1203                "MS:1000500",
1204                "scan window upper limit",
1205                &fmt_f(hi),
1206                "MS:1000040",
1207                "m/z"
1208            )
1209        );
1210    }
1211    if !windows.is_empty() {
1212        let _ = writeln!(s, "{i}    </scanWindowList>");
1213    }
1214    let _ = writeln!(s, "{i}  </scan>\n{i}</scanList>");
1215    if sp.ms_level > 1
1216        && let Some(p) = sp.precursor_mz
1217    {
1218        let _ = writeln!(s, "{i}<precursorList count=\"1\">\n{i}  <precursor>");
1219        let _ = writeln!(
1220            s,
1221            "{i}    <isolationWindow>\n{i}      {}\n{i}    </isolationWindow>",
1222            cv_unit(
1223                "MS:1000827",
1224                "isolation window target m/z",
1225                &fmt_f(p),
1226                "MS:1000040",
1227                "m/z"
1228            )
1229        );
1230        let _ = writeln!(
1231            s,
1232            "{i}    <selectedIonList count=\"1\">\n{i}      <selectedIon>\n{i}        {}",
1233            cv_unit(
1234                "MS:1000744",
1235                "selected ion m/z",
1236                &fmt_f(p),
1237                "MS:1000040",
1238                "m/z"
1239            )
1240        );
1241        if let Some(c) = sp.precursor_charge {
1242            let _ = writeln!(
1243                s,
1244                "{i}        {}",
1245                cv("MS:1000041", "charge state", &c.to_string())
1246            );
1247        }
1248        let _ = writeln!(s, "{i}      </selectedIon>\n{i}    </selectedIonList>");
1249        let _ = writeln!(s, "{i}    <activation>");
1250        let act_cv = match act.as_deref() {
1251            Some("cid") => cv("MS:1000133", "collision-induced dissociation", ""),
1252            Some("hcd") => cv("MS:1000422", "beam-type collision-induced dissociation", ""),
1253            Some("etd") => cv("MS:1000598", "electron transfer dissociation", ""),
1254            Some(other) => cv("MS:1000044", "dissociation method", other),
1255            None => cv("MS:1000044", "dissociation method", ""),
1256        };
1257        let _ = writeln!(s, "{i}      {act_cv}");
1258        if let Some(e) = energy {
1259            let _ = writeln!(
1260                s,
1261                "{i}      {}",
1262                cv_unit(
1263                    "MS:1000045",
1264                    "collision energy",
1265                    &fmt_f(e),
1266                    "UO:0000266",
1267                    "electronvolt"
1268                )
1269            );
1270        }
1271        let _ = writeln!(
1272            s,
1273            "{i}    </activation>\n{i}  </precursor>\n{i}</precursorList>"
1274        );
1275    }
1276    let _ = writeln!(s, "{i}<binaryDataArrayList count=\"2\">");
1277    let _ = writeln!(
1278        s,
1279        "{i}  <binaryDataArray encodedLength=\"{}\">\n{i}    {}\n{i}    {}\n{i}    {}\n{i}    <binary>{mz_b64}</binary>\n{i}  </binaryDataArray>",
1280        mz_b64.len(),
1281        cv("MS:1000523", "64-bit float", ""),
1282        cv("MS:1000574", "zlib compression", ""),
1283        cv_unit("MS:1000514", "m/z array", "", "MS:1000040", "m/z")
1284    );
1285    let _ = writeln!(
1286        s,
1287        "{i}  <binaryDataArray encodedLength=\"{}\">\n{i}    {}\n{i}    {}\n{i}    {}\n{i}    <binary>{in_b64}</binary>\n{i}  </binaryDataArray>",
1288        in_b64.len(),
1289        cv("MS:1000521", "32-bit float", ""),
1290        cv("MS:1000574", "zlib compression", ""),
1291        cv_unit(
1292            "MS:1000515",
1293            "intensity array",
1294            "",
1295            "MS:1000131",
1296            "number of detector counts"
1297        )
1298    );
1299    let _ = writeln!(s, "{i}</binaryDataArrayList>\n      </spectrum>");
1300    Ok((s, hash))
1301}
1302
1303fn zlib(data: &[u8]) -> Result<Vec<u8>> {
1304    let mut e = ZlibEncoder::new(Vec::new(), Compression::default());
1305    e.write_all(data)
1306        .and_then(|()| e.finish())
1307        .map_err(|err| Error::Other(format!("zlib: {err}")))
1308}
1309
1310const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
1311
1312/// Standard base64 with padding (RFC 4648 §4).
1313pub(crate) fn base64_encode(data: &[u8]) -> String {
1314    let mut out = String::with_capacity(data.len().div_ceil(3) * 4);
1315    for chunk in data.chunks(3) {
1316        let b = [
1317            chunk[0],
1318            chunk.get(1).copied().unwrap_or(0),
1319            chunk.get(2).copied().unwrap_or(0),
1320        ];
1321        let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
1322        out.push(B64[(n >> 18) as usize & 63] as char);
1323        out.push(B64[(n >> 12) as usize & 63] as char);
1324        out.push(if chunk.len() > 1 {
1325            B64[(n >> 6) as usize & 63] as char
1326        } else {
1327            '='
1328        });
1329        out.push(if chunk.len() > 2 {
1330            B64[n as usize & 63] as char
1331        } else {
1332            '='
1333        });
1334    }
1335    out
1336}
1337
1338/// Inverse of [`base64_encode`]; whitespace is ignored.
1339pub(crate) fn base64_decode(s: &str) -> Option<Vec<u8>> {
1340    // 0..=63 for the alphabet, PAD for '=', SKIP for whitespace, BAD for anything else.
1341    const PAD: u8 = 64;
1342    const SKIP: u8 = 65;
1343    const BAD: u8 = 66;
1344    const TABLE: [u8; 256] = {
1345        let mut t = [BAD; 256];
1346        let mut i = 0;
1347        while i < 64 {
1348            t[B64[i] as usize] = i as u8;
1349            i += 1;
1350        }
1351        t[b'=' as usize] = PAD;
1352        t[b' ' as usize] = SKIP;
1353        t[b'\n' as usize] = SKIP;
1354        t[b'\r' as usize] = SKIP;
1355        t[b'\t' as usize] = SKIP;
1356        t
1357    };
1358    let mut out = Vec::with_capacity(s.len() / 4 * 3);
1359    let mut quad = [0u8; 4];
1360    let mut have = 0;
1361    for &c in s.as_bytes() {
1362        let v = TABLE[usize::from(c)];
1363        match v {
1364            SKIP => continue,
1365            BAD => return None,
1366            _ => {}
1367        }
1368        quad[have] = v;
1369        have += 1;
1370        if have == 4 {
1371            have = 0;
1372            let pad = usize::from(quad[2] == PAD) + usize::from(quad[3] == PAD);
1373            let n = quad
1374                .iter()
1375                .fold(0u32, |acc, &v| (acc << 6) | u32::from(v & 63));
1376            let bytes = [(n >> 16) as u8, (n >> 8) as u8, n as u8];
1377            out.extend_from_slice(&bytes[..3 - pad.min(2)]);
1378        }
1379    }
1380    (have == 0).then_some(out)
1381}
1382
1383/// Passes reads through and feeds the first `left` bytes to a SHA-1.
1384struct HashingReader<R> {
1385    inner: R,
1386    sha: Sha1,
1387    left: u64,
1388}
1389
1390impl<R: Read> Read for HashingReader<R> {
1391    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1392        let n = self.inner.read(buf)?;
1393        let hashed = usize::try_from(self.left).map_or(n, |l| l.min(n));
1394        self.sha.update(&buf[..hashed]);
1395        self.left -= hashed as u64;
1396        Ok(n)
1397    }
1398}
1399
1400/// One spectrum or chromatogram whose arrays are checked off the reading thread.
1401struct Pending {
1402    chromatogram: bool,
1403    index: usize,
1404    /// Base64 text of each `<binary>` element; `None` for an empty element.
1405    arrays: Vec<Option<Vec<u8>>>,
1406}
1407
1408/// Check that the arrays of each item decode to the bytes whose hash was recorded, on the
1409/// rayon threads. Reports the first failing item in file order.
1410fn check_arrays(items: &[Pending], w: &Written) -> std::result::Result<(), String> {
1411    let results: Vec<std::result::Result<(), String>> = items
1412        .par_iter()
1413        .map(|p| {
1414            if !p.chromatogram && p.arrays.len() != 2 {
1415                return Err(format!("spectrum {}: {} arrays", p.index, p.arrays.len()));
1416            }
1417            let mut h = Vec::new();
1418            for a in &p.arrays {
1419                let Some(text) = a else { continue };
1420                let text = std::str::from_utf8(text).map_err(|_| "bad base64")?;
1421                let comp = base64_decode(text).ok_or("bad base64")?;
1422                ZlibDecoder::new(comp.as_slice())
1423                    .read_to_end(&mut h)
1424                    .map_err(|e| format!("zlib: {e}"))?;
1425            }
1426            let (want, what) = if p.chromatogram {
1427                (w.chrom_hashes.get(p.index), "chromatogram")
1428            } else {
1429                (w.hashes.get(p.index), "spectrum")
1430            };
1431            if Some(&xxhash_rust::xxh3::xxh3_128(&h)) != want {
1432                return Err(format!("{what} {}: arrays differ after writing", p.index));
1433            }
1434            Ok(())
1435        })
1436        .collect();
1437    results.into_iter().collect()
1438}
1439
1440/// Read the written file back and check it against what we meant to write.
1441fn verify(path: &Path, w: &Written) -> std::result::Result<(), String> {
1442    use quick_xml::Reader;
1443    use quick_xml::events::Event;
1444    use std::io::{BufReader, Seek, SeekFrom};
1445    // The file is streamed, never held in memory whole: exports of large runs are GBs.
1446    let mut file = File::open(path).map_err(|e| e.to_string())?;
1447    let len = file.metadata().map_err(|e| e.to_string())?.len();
1448    // Checksum covers everything up to and including "<fileChecksum>", which sits in the
1449    // last few hundred bytes.
1450    let marker = b"<fileChecksum>";
1451    let tail_len = len.min(4096);
1452    let mut tail = Vec::new();
1453    file.seek(SeekFrom::Start(len - tail_len))
1454        .and_then(|_| (&mut file).take(tail_len).read_to_end(&mut tail))
1455        .map_err(|e| e.to_string())?;
1456    let at = tail
1457        .windows(marker.len())
1458        .rposition(|x| x == marker)
1459        .ok_or("no fileChecksum element")?;
1460    let covered = len - tail_len + (at + marker.len()) as u64;
1461    let mut at_offset = |o: u64, want: &[u8]| -> std::result::Result<bool, String> {
1462        let mut got = vec![0u8; want.len()];
1463        file.seek(SeekFrom::Start(o)).map_err(|e| e.to_string())?;
1464        Ok(file.read_exact(&mut got).is_ok() && got == want)
1465    };
1466    for &o in &w.offsets {
1467        if !at_offset(o, b"<spectrum")? {
1468            return Err(format!(
1469                "index offset {o} does not point at a <spectrum> element"
1470            ));
1471        }
1472    }
1473    for &o in &w.chrom_offsets {
1474        if !at_offset(o, b"<chromatogram")? {
1475            return Err(format!(
1476                "index offset {o} does not point at a <chromatogram> element"
1477            ));
1478        }
1479    }
1480    if !at_offset(w.index_offset, b"<indexList")? {
1481        return Err("indexListOffset does not point at <indexList>".into());
1482    }
1483    // One pass over the file: the SHA-1 of the covered bytes, well-formed XML, and every
1484    // array decodes to the bytes we hashed.
1485    file.seek(SeekFrom::Start(0)).map_err(|e| e.to_string())?;
1486    let mut hashing = HashingReader {
1487        inner: file,
1488        sha: Sha1::new(),
1489        left: covered,
1490    };
1491    let mut reader = Reader::from_reader(BufReader::with_capacity(1 << 20, &mut hashing));
1492    let mut buf = Vec::new();
1493    let mut in_binary = false;
1494    let mut arrays: Vec<Option<Vec<u8>>> = Vec::new();
1495    let mut pending: Vec<Pending> = Vec::new();
1496    let mut pending_bytes = 0usize;
1497    let mut k = 0usize;
1498    let mut kc = 0usize;
1499    loop {
1500        match reader.read_event_into(&mut buf) {
1501            Ok(Event::Start(e)) if e.name().as_ref() == "binary" => in_binary = true,
1502            Ok(Event::End(e)) if e.name().as_ref() == "binary" => in_binary = false,
1503            Ok(Event::Empty(e)) if e.name().as_ref() == "binary" => arrays.push(None),
1504            Ok(Event::Text(t)) if in_binary => {
1505                let txt: &str = t.as_ref();
1506                pending_bytes += txt.len();
1507                arrays.push(Some(txt.as_bytes().to_vec()));
1508            }
1509            Ok(Event::End(e))
1510                if e.name().as_ref() == "spectrum" || e.name().as_ref() == "chromatogram" =>
1511            {
1512                let chromatogram = e.name().as_ref() == "chromatogram";
1513                let index = if chromatogram { &mut kc } else { &mut k };
1514                pending.push(Pending {
1515                    chromatogram,
1516                    index: *index,
1517                    arrays: std::mem::take(&mut arrays),
1518                });
1519                *index += 1;
1520                if pending.len() >= BATCH_SPECTRA || pending_bytes >= BATCH_POINTS * 4 {
1521                    check_arrays(&pending, w)?;
1522                    pending.clear();
1523                    pending_bytes = 0;
1524                }
1525            }
1526            Ok(Event::Eof) => break,
1527            Err(e) => return Err(format!("XML error at {}: {e}", reader.buffer_position())),
1528            _ => {}
1529        }
1530        buf.clear();
1531    }
1532    check_arrays(&pending, w)?;
1533    drop(reader);
1534    if hashing.left > 0 || hashing.sha.finish_hex() != w.sha {
1535        return Err("SHA-1 checksum does not match the content".into());
1536    }
1537    if k != w.hashes.len() {
1538        return Err(format!("{k} spectra read back, {} written", w.hashes.len()));
1539    }
1540    if kc != w.chrom_hashes.len() {
1541        return Err(format!(
1542            "{kc} chromatograms read back, {} written",
1543            w.chrom_hashes.len()
1544        ));
1545    }
1546    Ok(())
1547}
1548
1549/// A chromatogram to write: a source trace that is an mzML chromatogram, or a column of an MRM
1550/// table.
1551struct Chromatogram {
1552    id: String,
1553    accession: String,
1554    name: String,
1555    time_s: Vec<f64>,
1556    intensity: Vec<f64>,
1557    /// Label of the counts unit (MS:1000131): the source's, which may be the older
1558    /// "number of counts".
1559    counts_unit: &'static str,
1560    precursor_mz: Option<f64>,
1561    product_mz: Option<f64>,
1562    /// Scan polarity term, `(accession, name)`.
1563    polarity: Option<(&'static str, &'static str)>,
1564}
1565
1566/// The traces the reader marks as chromatograms (`extra.chromatogram_type_accession`, set by
1567/// the mzML reader), with their `time` and `intensity` channels. With `tables`, also the MRM
1568/// tables (Waters): their stored TIC and each `Q1 > Q3` column.
1569fn chromatograms(
1570    ds: &mut dyn Dataset,
1571    info: &openreadout_core::FileInfo,
1572    tables: bool,
1573) -> Result<Vec<Chromatogram>> {
1574    let mut out = Vec::new();
1575    for t in &info.traces {
1576        let (Some(acc), Some(name)) = (
1577            t.extra
1578                .get("chromatogram_type_accession")
1579                .and_then(|v| v.as_str()),
1580            t.extra.get("chromatogram_type").and_then(|v| v.as_str()),
1581        ) else {
1582            continue;
1583        };
1584        let col = |n: &str| t.channels.iter().position(|c| c.name == n);
1585        let (Some(ti), Some(ii)) = (col("time"), col("intensity")) else {
1586            continue;
1587        };
1588        let mut tr = ds.read_trace(t.index, 0, 0, t.sample_count)?;
1589        let (Some(time_s), Some(intensity)) = (
1590            tr.channels.get_mut(ti).map(std::mem::take),
1591            tr.channels.get_mut(ii).map(std::mem::take),
1592        ) else {
1593            continue;
1594        };
1595        let f = |k: &str| t.extra.get(k).and_then(serde_json::Value::as_f64);
1596        let counts_unit = if t.channels[ii].unit.as_deref() == Some("number of counts") {
1597            "number of counts"
1598        } else {
1599            "number of detector counts"
1600        };
1601        out.push(Chromatogram {
1602            id: t
1603                .name
1604                .clone()
1605                .unwrap_or_else(|| format!("chromatogram={}", t.index)),
1606            accession: acc.to_string(),
1607            name: name.to_string(),
1608            time_s,
1609            intensity,
1610            counts_unit,
1611            precursor_mz: f("precursor_mz"),
1612            product_mz: f("product_mz"),
1613            polarity: None,
1614        });
1615    }
1616    if tables {
1617        table_chromatograms(ds, info, &mut out)?;
1618    }
1619    Ok(out)
1620}
1621
1622/// A TIC chromatogram from every written spectrum, and one SRM chromatogram per transition of
1623/// the SRM/MRM spectra, for exports whose source stores none (vendor files; msconvert writes
1624/// both). An SRM spectrum is one whose filter names `SRM` (Thermo), whose `extra.scan_type` is
1625/// `MRM` (Sciex) or 256 (Agilent MassHunter), or whose `extra.acquisition` is `mrm`
1626/// (Shimadzu). Each of its points is one transition: Q1 is the point's own precursor
1627/// (`extra.point_precursors_mz`) or the spectrum's, Q3 the point's m/z. Transitions are told
1628/// apart by polarity, Q1, Q3 (both to 1e-4), collision energy and the method entry that
1629/// measured them (`extra.scan_method`, `event` or `experiment`: a dynamic-MRM method measures a
1630/// transition again in another time window).
1631struct DerivedChromatograms {
1632    agilent: bool,
1633    tic_time: Vec<f64>,
1634    tic: Vec<f64>,
1635    /// A spectrum without a time was written: no TIC is.
1636    tic_incomplete: bool,
1637    srm_keys: std::collections::HashMap<(u8, i64, i64, i64, i64), usize>,
1638    srm: Vec<Chromatogram>,
1639}
1640
1641impl DerivedChromatograms {
1642    fn new(agilent: bool) -> Self {
1643        DerivedChromatograms {
1644            agilent,
1645            tic_time: Vec::new(),
1646            tic: Vec::new(),
1647            tic_incomplete: false,
1648            srm_keys: std::collections::HashMap::new(),
1649            srm: Vec::new(),
1650        }
1651    }
1652
1653    fn is_srm(&self, sp: &Spectrum) -> bool {
1654        let filter = sp
1655            .scan_filter
1656            .as_deref()
1657            .is_some_and(|f| f.split_whitespace().any(|w| w == "SRM"));
1658        let scan_type = sp.extra.get("scan_type");
1659        filter
1660            || scan_type.and_then(|v| v.as_str()) == Some("MRM")
1661            || (self.agilent && scan_type.and_then(serde_json::Value::as_u64) == Some(256))
1662            || sp.extra.get("acquisition").and_then(|v| v.as_str()) == Some("mrm")
1663    }
1664
1665    fn add(&mut self, sp: &Spectrum) {
1666        let Some(t) = sp.rt_s.filter(|t| t.is_finite()) else {
1667            self.tic_incomplete = true;
1668            return;
1669        };
1670        let tic = sp
1671            .total_ion_current
1672            .unwrap_or_else(|| sp.intensity.iter().map(|&v| f64::from(v)).sum());
1673        // Spectra that split one scan (Sciex MRM: one per precursor of a cycle) share its time:
1674        // one TIC point, their sum.
1675        match (self.tic_time.last(), self.tic.last_mut()) {
1676            (Some(&last), Some(v)) if last.to_bits() == t.to_bits() => *v += tic,
1677            _ => {
1678                self.tic_time.push(t);
1679                self.tic.push(tic);
1680            }
1681        }
1682        if sp.ms_level < 2 || !self.is_srm(sp) {
1683            return;
1684        }
1685        let per_point: Option<Vec<f64>> = sp
1686            .extra
1687            .get("point_precursors_mz")
1688            .and_then(|v| v.as_array())
1689            .map(|a| a.iter().filter_map(serde_json::Value::as_f64).collect())
1690            .filter(|v: &Vec<f64>| v.len() == sp.mz.len());
1691        let (pol_code, polarity) = match sp.polarity.as_str() {
1692            "positive" => (1u8, Some(("MS:1000130", "positive scan"))),
1693            "negative" => (2, Some(("MS:1000129", "negative scan"))),
1694            _ => (0, None),
1695        };
1696        let q = |v: f64| (v * 1e4).round() as i64;
1697        let ce = sp.collision_energy.map_or(i64::MIN, q);
1698        let entry = ["scan_method", "event", "experiment"]
1699            .iter()
1700            .find_map(|k| sp.extra.get(*k).and_then(serde_json::Value::as_i64))
1701            .unwrap_or(i64::MIN);
1702        for (k, (&q3, &v)) in sp.mz.iter().zip(&sp.intensity).enumerate() {
1703            let Some(q1) = per_point
1704                .as_ref()
1705                .and_then(|p| p.get(k).copied())
1706                .or(sp.precursor_mz)
1707            else {
1708                continue;
1709            };
1710            let key = (pol_code, q(q1), q(q3), ce, entry);
1711            let next = self.srm.len();
1712            let at = *self.srm_keys.entry(key).or_insert(next);
1713            if at == next {
1714                self.srm.push(Chromatogram {
1715                    id: String::new(),
1716                    accession: "MS:1001473".into(),
1717                    name: "selected reaction monitoring chromatogram".into(),
1718                    time_s: Vec::new(),
1719                    intensity: Vec::new(),
1720                    counts_unit: "number of detector counts",
1721                    precursor_mz: Some(q1),
1722                    product_mz: Some(q3),
1723                    polarity,
1724                });
1725            }
1726            self.srm[at].time_s.push(t);
1727            self.srm[at].intensity.push(f64::from(v));
1728        }
1729    }
1730
1731    /// Add the TIC and the SRM chromatograms to `out`, unless it already holds chromatograms
1732    /// of that type (an mzML source's own).
1733    fn append_to(self, out: &mut Vec<Chromatogram>) {
1734        let has = |acc: &str| out.iter().any(|c| c.accession == acc);
1735        let (has_tic, has_srm) = (has("MS:1000235"), has("MS:1001473"));
1736        let mut ids: BTreeSet<String> = out.iter().map(|c| c.id.clone()).collect();
1737        let mut unique = |base: String| {
1738            let mut id = base.clone();
1739            let mut n = 1;
1740            while !ids.insert(id.clone()) {
1741                n += 1;
1742                id = format!("{base} ({n})");
1743            }
1744            id
1745        };
1746        if !has_tic && !self.tic_incomplete && !self.tic.is_empty() {
1747            out.push(Chromatogram {
1748                id: unique("TIC".into()),
1749                accession: "MS:1000235".into(),
1750                name: "total ion current chromatogram".into(),
1751                time_s: self.tic_time,
1752                intensity: self.tic,
1753                counts_unit: "number of detector counts",
1754                precursor_mz: None,
1755                product_mz: None,
1756                polarity: None,
1757            });
1758        }
1759        if !has_srm {
1760            for mut c in self.srm {
1761                let base = format!(
1762                    "SRM SIC Q1={} Q3={}",
1763                    id_mz(c.precursor_mz.unwrap_or(0.0)),
1764                    id_mz(c.product_mz.unwrap_or(0.0))
1765                );
1766                c.id = unique(base);
1767                out.push(c);
1768            }
1769        }
1770    }
1771}
1772
1773/// An m/z for a chromatogram id, in its shortest form: a value stored as a 32-bit float
1774/// (Thermo windows, Agilent and Sciex transitions) prints as that float does (`132.025`, not
1775/// `132.02499389648438`).
1776fn id_mz(v: f64) -> String {
1777    let f = v as f32;
1778    if f64::from(f).to_bits() == v.to_bits() {
1779        format!("{f}")
1780    } else {
1781        format!("{v}")
1782    }
1783}
1784
1785/// Q1 and Q3 of an MRM column named `Q1 > Q3`.
1786fn transition_of(name: &str) -> Option<(f64, f64)> {
1787    let (a, b) = name.split_once('>')?;
1788    Some((a.trim().parse().ok()?, b.trim().parse().ok()?))
1789}
1790
1791/// Chromatograms from the tables with a retention-time column (`rt_min`): the stored TIC
1792/// (`tic_stored`) and one SRM chromatogram per `Q1 > Q3` column.
1793fn table_chromatograms(
1794    ds: &mut dyn Dataset,
1795    info: &openreadout_core::FileInfo,
1796    out: &mut Vec<Chromatogram>,
1797) -> Result<()> {
1798    let mut ids: std::collections::HashSet<String> = out.iter().map(|c| c.id.clone()).collect();
1799    for t in &info.tables {
1800        let Some(rt) = t.columns.iter().position(|c| c.name == "rt_min") else {
1801            continue;
1802        };
1803        let wanted: Vec<(usize, Option<(f64, f64)>)> = t
1804            .columns
1805            .iter()
1806            .enumerate()
1807            .filter_map(|(k, c)| {
1808                if c.name == "tic_stored" {
1809                    Some((k, None))
1810                } else {
1811                    transition_of(&c.name).map(|q| (k, Some(q)))
1812                }
1813            })
1814            .collect();
1815        if wanted.is_empty() {
1816            continue;
1817        }
1818        let table = ds.read_table(t.index, 0, t.row_count)?;
1819        let Some(minutes) = table.columns.get(rt) else {
1820            continue;
1821        };
1822        let time_s: Vec<f64> = minutes.iter().map(|m| m * 60.0).collect();
1823        let label = t
1824            .name
1825            .clone()
1826            .unwrap_or_else(|| format!("table {}", t.index));
1827        let polarity = match t.extra.get("polarity").and_then(|v| v.as_str()) {
1828            Some("positive") => Some(("MS:1000130", "positive scan")),
1829            Some("negative") => Some(("MS:1000129", "negative scan")),
1830            _ => None,
1831        };
1832        for (k, q) in wanted {
1833            let Some(values) = table.columns.get(k) else {
1834                continue;
1835            };
1836            let (accession, name, base) = match q {
1837                None => (
1838                    "MS:1000235",
1839                    "total ion current chromatogram",
1840                    format!("TIC {label}"),
1841                ),
1842                Some(_) => (
1843                    "MS:1001473",
1844                    "selected reaction monitoring chromatogram",
1845                    format!("SRM {label} {}", t.columns[k].name),
1846                ),
1847            };
1848            // ids are unique within the file (a method can list a transition twice)
1849            let mut id = base.clone();
1850            let mut n = 1;
1851            while !ids.insert(id.clone()) {
1852                n += 1;
1853                id = format!("{base} ({n})");
1854            }
1855            out.push(Chromatogram {
1856                id,
1857                accession: accession.into(),
1858                name: name.into(),
1859                time_s: time_s.clone(),
1860                intensity: values.clone(),
1861                counts_unit: "number of detector counts",
1862                precursor_mz: q.map(|q| q.0),
1863                product_mz: q.map(|q| q.1),
1864                polarity,
1865            });
1866        }
1867    }
1868    Ok(())
1869}
1870
1871/// One `<chromatogram>` element (64-bit time in seconds and intensity, zlib) and the xxh3-128
1872/// of its decoded arrays.
1873fn chromatogram_xml(c: &Chromatogram, index: u64) -> Result<(String, u128)> {
1874    let le = |v: &[f64]| v.iter().flat_map(|x| x.to_le_bytes()).collect::<Vec<u8>>();
1875    let (tb, ib) = (le(&c.time_s), le(&c.intensity));
1876    let mut h = tb.clone();
1877    h.extend_from_slice(&ib);
1878    let hash = xxhash_rust::xxh3::xxh3_128(&h);
1879    let (t64, i64_) = (base64_encode(&zlib(&tb)?), base64_encode(&zlib(&ib)?));
1880    let i = "        ";
1881    let mut s = String::new();
1882    let _ = writeln!(
1883        s,
1884        "      <chromatogram index=\"{index}\" id=\"{}\" defaultArrayLength=\"{}\">",
1885        escape(&c.id),
1886        c.time_s.len()
1887    );
1888    let _ = writeln!(s, "{i}{}", cv(&escape(&c.accession), &escape(&c.name), ""));
1889    if let Some((acc, name)) = c.polarity {
1890        let _ = writeln!(s, "{i}{}", cv(acc, name, ""));
1891    }
1892    for (el, v) in [("precursor", c.precursor_mz), ("product", c.product_mz)] {
1893        if let Some(v) = v {
1894            // The schema requires a precursor's activation. An SRM precursor is fragmented by
1895            // collision-induced dissociation; other chromatograms do not say how.
1896            let activation = match el {
1897                "precursor" if c.accession == "MS:1001473" => format!(
1898                    "\n{i}  <activation>\n{i}    {}\n{i}  </activation>",
1899                    cv("MS:1000133", "collision-induced dissociation", "")
1900                ),
1901                "precursor" => format!("\n{i}  <activation/>"),
1902                _ => String::new(),
1903            };
1904            let _ = writeln!(
1905                s,
1906                "{i}<{el}>\n{i}  <isolationWindow>\n{i}    {}\n{i}  </isolationWindow>{activation}\n{i}</{el}>",
1907                cv_unit(
1908                    "MS:1000827",
1909                    "isolation window target m/z",
1910                    &fmt_f(v),
1911                    "MS:1000040",
1912                    "m/z"
1913                )
1914            );
1915        }
1916    }
1917    let _ = writeln!(s, "{i}<binaryDataArrayList count=\"2\">");
1918    let _ = writeln!(
1919        s,
1920        "{i}  <binaryDataArray encodedLength=\"{}\">\n{i}    {}\n{i}    {}\n{i}    {}\n{i}    <binary>{t64}</binary>\n{i}  </binaryDataArray>",
1921        t64.len(),
1922        cv("MS:1000523", "64-bit float", ""),
1923        cv("MS:1000574", "zlib compression", ""),
1924        cv_unit("MS:1000595", "time array", "", "UO:0000010", "second")
1925    );
1926    let _ = writeln!(
1927        s,
1928        "{i}  <binaryDataArray encodedLength=\"{}\">\n{i}    {}\n{i}    {}\n{i}    {}\n{i}    <binary>{i64_}</binary>\n{i}  </binaryDataArray>",
1929        i64_.len(),
1930        cv("MS:1000523", "64-bit float", ""),
1931        cv("MS:1000574", "zlib compression", ""),
1932        cv_unit(
1933            "MS:1000515",
1934            "intensity array",
1935            "",
1936            "MS:1000131",
1937            c.counts_unit
1938        )
1939    );
1940    let _ = writeln!(s, "{i}</binaryDataArrayList>\n      </chromatogram>");
1941    Ok((s, hash))
1942}
1943
1944/// SHA-1 (FIPS 180-4), used only for the mzML `fileChecksum` element.
1945#[derive(Clone)]
1946struct Sha1(sha1::Sha1);
1947
1948impl Sha1 {
1949    fn new() -> Self {
1950        Sha1(sha1::Digest::new())
1951    }
1952
1953    fn update(&mut self, data: &[u8]) {
1954        sha1::Digest::update(&mut self.0, data);
1955    }
1956
1957    fn finish_hex(self) -> String {
1958        sha1::Digest::finalize(self.0)
1959            .iter()
1960            .fold(String::with_capacity(40), |mut s, x| {
1961                let _ = write!(s, "{x:02x}");
1962                s
1963            })
1964    }
1965}
1966
1967#[cfg(test)]
1968mod tests {
1969    use super::*;
1970
1971    fn srm(rt: f64, q1: f64, points: &[(f64, f32)], tic: f64) -> Spectrum {
1972        Spectrum {
1973            ms_level: 2,
1974            rt_s: Some(rt),
1975            polarity: "negative".into(),
1976            precursor_mz: Some(q1),
1977            scan_filter: Some(format!("- c ESI SRM ms2 {q1} [..]")),
1978            total_ion_current: Some(tic),
1979            mz: points.iter().map(|p| p.0).collect(),
1980            intensity: points.iter().map(|p| p.1).collect(),
1981            ..Spectrum::default()
1982        }
1983    }
1984
1985    #[test]
1986    fn tic_and_srm_chromatograms_from_spectra() {
1987        let mut d = DerivedChromatograms::new(false);
1988        d.add(&srm(1.0, 258.0, &[(79.0, 5.0), (97.0, 7.0)], 12.0));
1989        d.add(&srm(1.0, 300.0, &[(79.0, 1.0)], 1.0)); // same scan time: one TIC point
1990        d.add(&srm(2.0, 258.0, &[(79.0, 6.0), (97.0, 8.0)], 14.0));
1991        // a Shimadzu MRM record: each point its own precursor
1992        let mut sp = srm(3.0, 300.0, &[(79.0, 2.0), (79.0, 3.0)], 5.0);
1993        sp.scan_filter = None;
1994        sp.extra
1995            .insert("acquisition".into(), serde_json::json!("mrm"));
1996        sp.extra.insert(
1997            "point_precursors_mz".into(),
1998            serde_json::json!([300.0, 258.0]),
1999        );
2000        d.add(&sp);
2001        let mut out = Vec::new();
2002        d.append_to(&mut out);
2003        let ids: Vec<&str> = out.iter().map(|c| c.id.as_str()).collect();
2004        assert_eq!(
2005            ids,
2006            [
2007                "TIC",
2008                "SRM SIC Q1=258 Q3=79",
2009                "SRM SIC Q1=258 Q3=97",
2010                "SRM SIC Q1=300 Q3=79"
2011            ]
2012        );
2013        assert_eq!(out[0].time_s, [1.0, 2.0, 3.0]);
2014        assert_eq!(out[0].intensity, [13.0, 14.0, 5.0]);
2015        assert_eq!(out[1].intensity, [5.0, 6.0, 3.0]);
2016        assert_eq!(out[3].time_s, [1.0, 3.0]);
2017        assert_eq!(out[3].intensity, [1.0, 2.0]);
2018        // a source that stores its own TIC keeps it alone
2019        let mut d = DerivedChromatograms::new(false);
2020        d.add(&srm(1.0, 258.0, &[(79.0, 5.0)], 5.0));
2021        let mut out = vec![Chromatogram {
2022            id: "TIC".into(),
2023            accession: "MS:1000235".into(),
2024            name: "total ion current chromatogram".into(),
2025            time_s: vec![1.0],
2026            intensity: vec![5.0],
2027            counts_unit: "number of detector counts",
2028            precursor_mz: None,
2029            product_mz: None,
2030            polarity: None,
2031        }];
2032        d.append_to(&mut out);
2033        assert_eq!(out.len(), 2);
2034        assert_eq!(out[1].accession, "MS:1001473");
2035    }
2036
2037    #[test]
2038    fn mismatched_spectrum_arrays_fail_without_indexing() {
2039        for (mz, intensity) in [
2040            (vec![], vec![1.0, 9.0]),
2041            (vec![100.0], vec![1.0, 9.0]),
2042            (vec![100.0], vec![]),
2043        ] {
2044            let sp = Spectrum {
2045                mz,
2046                intensity,
2047                ..Spectrum::default()
2048            };
2049            let err = spectrum_xml(&sp, 0, "scan=1", false).unwrap_err();
2050            assert_eq!(err.exit_code(), 4);
2051            assert!(err.hint().is_some());
2052        }
2053        assert!(spectrum_xml(&Spectrum::default(), 0, "scan=1", false).is_ok());
2054    }
2055
2056    #[test]
2057    fn sha1_known_vectors() {
2058        let mut s = Sha1::new();
2059        s.update(b"abc");
2060        assert_eq!(s.finish_hex(), "a9993e364706816aba3e25717850c26c9cd0d89d");
2061        let mut s = Sha1::new();
2062        s.update(b"");
2063        assert_eq!(s.finish_hex(), "da39a3ee5e6b4b0d3255bfef95601890afd80709");
2064        let mut s = Sha1::new();
2065        let msg = b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
2066        s.update(&msg[..10]);
2067        s.update(&msg[10..]);
2068        assert_eq!(s.finish_hex(), "84983e441c3bd26ebaae4aa1f95129e5e54670f1");
2069    }
2070
2071    #[test]
2072    fn base64_roundtrip() {
2073        for n in 0..20 {
2074            let data: Vec<u8> = (0..n).map(|x| (x * 37 + 11) as u8).collect();
2075            let e = base64_encode(&data);
2076            assert_eq!(base64_decode(&e).unwrap(), data);
2077        }
2078        assert_eq!(base64_encode(b"Man"), "TWFu");
2079        assert_eq!(base64_encode(b"Ma"), "TWE=");
2080    }
2081
2082    #[test]
2083    fn filter_parsing() {
2084        let (a, act, e, w) = filter_parts("FTMS + p ESI d Full ms2 84.08@cid20.00 [50.00-95.00]");
2085        assert_eq!(a, Some("FTMS"));
2086        assert_eq!(act.as_deref(), Some("cid"));
2087        assert_eq!(e, Some(20.0));
2088        assert_eq!(w, vec![[50.0, 95.0]]);
2089        let (_, act, _, w) = filter_parts("- c ESI SRM ms2 258.026 [78.942-78.944, 96.939-96.941]");
2090        assert_eq!(act, None);
2091        assert_eq!(w, vec![[78.942, 78.944], [96.939, 96.941]]);
2092    }
2093
2094    #[test]
2095    fn escaping() {
2096        assert_eq!(escape("a<b & \"c\""), "a&lt;b &amp; &quot;c&quot;");
2097    }
2098}