use miniz_oxide::inflate::TINFLStatus;
use miniz_oxide::inflate::core::{DecompressorOxide, decompress, inflate_flags};
use openreadout_core::bytes::{le_u16, le_u32};
pub const MAX_TLM_RECORD: usize = 16 << 20;
pub const TLM_FULL_SCAN: u16 = 10;
pub const TLM_SIM: u16 = 11;
pub const TLM_PRODUCT_ION_SCAN: u16 = 14;
pub const TLM_MRM: u16 = 15;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TlmEntry {
pub offset: u64,
pub size: u32,
}
#[derive(Debug, Clone, Default)]
pub struct TlmIndex {
pub entries: Vec<TlmEntry>,
pub rt_ms: Vec<u32>,
pub tic: Vec<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TlmHeader {
pub rt_ms: u32,
pub cycle_rt_ms: u32,
pub event: u32,
pub cycle: u32,
pub record: u32,
pub code: u16,
pub polarity_code: u32,
pub count: u32,
pub first_q1: u32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct TlmSpectrum {
pub ms_level: u32,
pub precursor_mz: Option<f64>,
pub centroided: bool,
pub scan_window_mz: Option<[f64; 2]>,
pub points: Vec<(f64, f64)>,
}
pub fn parse_index(rt: &[u8], index: &[u8], tic: &[u8]) -> Result<TlmIndex, String> {
if !index.len().is_multiple_of(24)
|| !rt.len().is_multiple_of(4)
|| !tic.len().is_multiple_of(8)
{
return Err(format!(
"TLM index streams of {} / {} / {} bytes are not whole records (24 / 4 / 8)",
index.len(),
rt.len(),
tic.len()
));
}
let n = index.len() / 24;
if rt.len() / 4 != n || tic.len() / 8 != n {
return Err(format!(
"TLM streams disagree: {n} index entries, {} retention times, {} TIC values",
rt.len() / 4,
tic.len() / 8
));
}
let entries = index
.as_chunks::<24>()
.0
.iter()
.map(|c| TlmEntry {
size: u32::from_le_bytes([c[0], c[1], c[2], c[3]]),
offset: u64::from(u32::from_le_bytes([c[8], c[9], c[10], c[11]])),
})
.collect();
Ok(TlmIndex {
entries,
rt_ms: rt
.as_chunks::<4>()
.0
.iter()
.map(|c| u32::from_le_bytes(*c))
.collect(),
tic: tic
.as_chunks::<8>()
.0
.iter()
.map(|c| u64::from_le_bytes(*c))
.collect(),
})
}
fn packed(raw: &[u8], e: TlmEntry) -> Result<(&[u8], usize), String> {
let start = usize::try_from(e.offset).map_err(|_| "record offset too large".to_string())?;
let end = start
.checked_add(e.size as usize)
.filter(|&x| x <= raw.len())
.ok_or_else(|| {
format!(
"record at {start} runs past MS Raw Data ({} bytes)",
raw.len()
)
})?;
let rec = &raw[start..end];
if le_u32(rec, 0) != Some(u32::MAX) {
return Err(format!("record at {start} does not start with -1"));
}
let ulen = le_u32(rec, 4).ok_or("record header cut")? as usize;
let clen = le_u32(rec, 8).ok_or("record header cut")? as usize;
let data = rec
.get(12..12usize.checked_add(clen).ok_or("record length overflows")?)
.ok_or_else(|| {
format!(
"record at {start}: {clen} packed bytes past its {} bytes",
rec.len()
)
})?;
if ulen > MAX_TLM_RECORD {
return Err(format!("record at {start} inflates to {ulen} bytes"));
}
Ok((data, ulen))
}
pub fn inflate_record(raw: &[u8], e: TlmEntry) -> Result<Vec<u8>, String> {
let (data, ulen) = packed(raw, e)?;
let out = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(data, ulen)
.map_err(|err| format!("record at {}: zlib {:?}", e.offset, err.status))?;
if out.len() != ulen {
return Err(format!(
"record at {} inflates to {} bytes, not the {ulen} it declares",
e.offset,
out.len()
));
}
Ok(out)
}
pub fn record_header(raw: &[u8], e: TlmEntry) -> Result<TlmHeader, String> {
let (data, ulen) = packed(raw, e)?;
let mut out = [0u8; 48];
let mut state = Box::new(DecompressorOxide::new());
let flags = inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER
| inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF;
let (status, _, n) = decompress(&mut state, data, &mut out, 0, flags);
let enough = n == out.len() || (matches!(status, TINFLStatus::Done) && n == ulen);
if !enough || n < 48 {
return Err(format!(
"record at {}: its header does not inflate ({status:?}, {n} bytes)",
e.offset
));
}
header(&out).ok_or_else(|| format!("record at {}: short header", e.offset))
}
pub fn header(d: &[u8]) -> Option<TlmHeader> {
Some(TlmHeader {
rt_ms: le_u32(d, 0)?,
cycle_rt_ms: le_u32(d, 4)?,
event: le_u32(d, 8)?,
cycle: le_u32(d, 12)?,
record: le_u32(d, 16)?,
code: le_u16(d, 20)?,
polarity_code: le_u32(d, 36)?,
count: le_u32(d, 40)?,
first_q1: le_u32(d, 44).unwrap_or(0),
})
}
impl TlmHeader {
pub fn precursor_mz(&self) -> Option<f64> {
matches!(self.code, TLM_MRM | TLM_PRODUCT_ION_SCAN)
.then(|| f64::from(self.first_q1) / 100.0)
}
}
pub fn ms_level(code: u16) -> Option<u32> {
match code {
TLM_SIM | TLM_FULL_SCAN => Some(1),
TLM_MRM | TLM_PRODUCT_ION_SCAN => Some(2),
_ => None,
}
}
pub fn acquisition_name(code: u16) -> String {
match code {
TLM_MRM => "mrm".into(),
TLM_SIM => "sim".into(),
TLM_PRODUCT_ION_SCAN => "product ion scan".into(),
TLM_FULL_SCAN => "full scan".into(),
c => format!("code {c}"),
}
}
pub fn decode(record: &[u8]) -> Result<TlmSpectrum, String> {
let head = header(record).ok_or("record shorter than its header")?;
let n = head.count as usize;
match head.code {
TLM_MRM | TLM_SIM => {
let end = n.checked_mul(12).and_then(|x| x.checked_add(44)).ok_or("count overflows")?;
let body = record.get(44..end).ok_or_else(|| format!("{n} ion entries past the {} record bytes", record.len()))?;
let mut pts = Vec::with_capacity(n);
let mut q1 = None;
for entry in body.as_chunks::<12>().0 {
let (precursor, product, intensity) = (
u32::from_le_bytes([entry[0], entry[1], entry[2], entry[3]]),
u32::from_le_bytes([entry[4], entry[5], entry[6], entry[7]]),
u32::from_le_bytes([entry[8], entry[9], entry[10], entry[11]]),
);
if head.code == TLM_SIM && precursor != product {
return Err(format!("SIM entry with two m/z values ({precursor}, {product})"));
}
if head.code == TLM_MRM {
match q1 {
None => q1 = Some(precursor),
Some(first) if first != precursor => {
return Err(format!("MRM record with two precursors ({first}, {precursor})"));
}
_ => {}
}
}
pts.push((f64::from(product) / 100.0, f64::from(intensity)));
}
pts.sort_by(|x, y| x.0.total_cmp(&y.0));
Ok(TlmSpectrum {
ms_level: if head.code == TLM_MRM { 2 } else { 1 },
precursor_mz: q1.map(|raw| f64::from(raw) / 100.0),
centroided: true,
scan_window_mz: None,
points: pts,
})
}
TLM_PRODUCT_ION_SCAN => Err("product-ion scan spectra are not read: their profiles could not be checked against the vendor's (docs/formats/shimadzu.md)".into()),
TLM_FULL_SCAN => Err("full-scan spectra are not read: the vendor software reports other values than the file stores at the same m/z (docs/formats/shimadzu.md)".into()),
c => Err(format!("acquisition code {c} is not read")),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rec(code: u16, body: &[u8], count: u32) -> Vec<u8> {
let mut d = Vec::new();
for v in [60_000u32, 60_000, 1, 1, 0] {
d.extend_from_slice(&v.to_le_bytes());
}
d.extend_from_slice(&code.to_le_bytes());
d.extend_from_slice(&1u16.to_le_bytes());
d.extend_from_slice(&[0; 12]);
d.extend_from_slice(&0u32.to_le_bytes());
d.extend_from_slice(&count.to_le_bytes());
d.extend_from_slice(body);
d
}
fn pack(d: &[u8]) -> Vec<u8> {
let z = miniz_oxide::deflate::compress_to_vec_zlib(d, 6);
let mut r = Vec::new();
r.extend_from_slice(&u32::MAX.to_le_bytes());
r.extend_from_slice(&(d.len() as u32).to_le_bytes());
r.extend_from_slice(&(z.len() as u32).to_le_bytes());
r.extend_from_slice(&z);
r
}
#[test]
fn mrm_sim_and_product_scans() {
let mut body = Vec::new();
for (a, b, v) in [(54420u32, 39730u32, 0u32), (54420, 32095, 10)] {
for x in [a, b, v] {
body.extend_from_slice(&x.to_le_bytes());
}
}
let d = rec(TLM_MRM, &body, 2);
let raw = pack(&d);
let e = TlmEntry {
offset: 0,
size: raw.len() as u32,
};
assert_eq!(inflate_record(&raw, e).unwrap(), d);
let h = record_header(&raw, e).unwrap();
assert_eq!((h.code, h.count, h.event, h.rt_ms), (TLM_MRM, 2, 1, 60_000));
let s = decode(&d).unwrap();
assert_eq!(s.ms_level, 2);
assert_eq!(s.precursor_mz, Some(544.2));
assert_eq!(s.points, vec![(320.95, 10.0), (397.3, 0.0)]);
let mut body = Vec::new();
for x in [104_420u32, 104_420, 3900, 3930, 5, 6, 7] {
body.extend_from_slice(&x.to_le_bytes());
}
let raw = pack(&rec(TLM_PRODUCT_ION_SCAN, &body, 3));
let h = record_header(
&raw,
TlmEntry {
offset: 0,
size: raw.len() as u32,
},
)
.unwrap();
assert_eq!(h.precursor_mz(), Some(1044.2));
assert!(decode(&rec(TLM_PRODUCT_ION_SCAN, &body, 3)).is_err());
assert!(decode(&rec(TLM_FULL_SCAN, &body, 3)).is_err());
assert!(decode(&rec(99, &[], 0)).is_err());
}
#[test]
fn broken_records_are_errors() {
assert!(parse_index(&[0; 4], &[0; 24], &[0; 16]).is_err());
assert!(parse_index(&[0; 5], &[0; 24], &[0; 8]).is_err());
let raw = pack(&rec(TLM_MRM, &[], 5));
let e = TlmEntry {
offset: 0,
size: raw.len() as u32,
};
let d = inflate_record(&raw, e).unwrap();
assert!(decode(&d).is_err()); assert!(
inflate_record(
&raw,
TlmEntry {
offset: 4,
size: 10
}
)
.is_err()
);
assert!(
inflate_record(
&raw,
TlmEntry {
offset: 0,
size: raw.len() as u32 + 1
}
)
.is_err()
);
let mut bad = raw.clone();
bad[14] ^= 0xFF;
assert!(
inflate_record(&bad, e).is_err() || decode(&inflate_record(&bad, e).unwrap()).is_err()
);
}
}