ionic-rs 0.1.2

Streamable binary format for mass-spectrometry data: lossless mzML conversion, random access, WebAssembly support
Documentation
use std::io::BufRead;

use base64::{Engine, engine::general_purpose::STANDARD};
use miniz_oxide::inflate::decompress_to_vec_zlib_with_limit;
use quick_xml::events::BytesStart;

use crate::{
    accessions::{
        ACC_COMPRESSION_ZLIB, ACC_FLOAT_16BIT_STR, ACC_FLOAT_32BIT_STR, ACC_FLOAT_64BIT_STR,
        ACC_INT_16BIT_STR, ACC_INT_32BIT_STR, ACC_INT_64BIT_STR,
    },
    ion::DecompressionLimit,
    mzml::{
        schema::TagId,
        structs::{BinaryDataArray, BinaryDataArrayList, NumericArray, NumericType},
        utilities::{
            PREALLOC_CAP, ParamCollector, ParseError, ParsingWorkspace, attr, attr_usize,
            read_base64_binary, read_cv_param, read_ref_group_ref, read_user_param,
        },
    },
};

pub(crate) fn parse_bda_list<R: BufRead>(
    ws: &mut ParsingWorkspace<R>,
    start: &BytesStart<'_>,
) -> Result<BinaryDataArrayList, ParseError> {
    let count = attr_usize(start, b"count");
    let mut list = BinaryDataArrayList {
        count,
        binary_data_arrays: Vec::with_capacity(count.unwrap_or(0).min(PREALLOC_CAP)),
    };
    ws.for_each_child(start, |ws, event| {
        let (tag, element, is_open) = event.into_parts();
        if tag != TagId::BinaryDataArray {
            return Ok(false);
        }
        if is_open {
            list.binary_data_arrays.push(parse_bda(ws, &element)?);
        } else {
            list.binary_data_arrays.push(BinaryDataArray {
                array_length: attr_usize(&element, b"arrayLength"),
                encoded_length: attr_usize(&element, b"encodedLength"),
                data_processing_ref: attr(&element, b"dataProcessingRef"),
                ..Default::default()
            });
        }
        Ok(true)
    })?;
    Ok(list)
}

pub(crate) fn parse_bda<R: BufRead>(
    ws: &mut ParsingWorkspace<R>,
    start: &BytesStart<'_>,
) -> Result<BinaryDataArray, ParseError> {
    let mut bda = BinaryDataArray {
        array_length: attr_usize(start, b"arrayLength"),
        encoded_length: attr_usize(start, b"encodedLength"),
        data_processing_ref: attr(start, b"dataProcessingRef"),
        ..Default::default()
    };
    let mut base64_bytes: Vec<u8> = Vec::new();

    ws.for_each_child(start, |ws, event| {
        let (tag, element, _) = event.into_parts();
        match tag {
            TagId::CvParam => {
                bda.receive_cv(read_cv_param(&element));
                Ok(true)
            }
            TagId::UserParam => {
                bda.receive_user(read_user_param(&element));
                Ok(true)
            }
            TagId::ReferenceableParamGroupRef => {
                bda.receive_ref_group(read_ref_group_ref(&element));
                Ok(true)
            }
            TagId::Binary => {
                if let Some(len) = bda.encoded_length {
                    base64_bytes.reserve(len.min(PREALLOC_CAP));
                }
                let closing = element.name().as_ref().to_vec();
                read_base64_binary(ws, &closing, &mut base64_bytes)?;
                Ok(true)
            }
            _ => Ok(false),
        }
    })?;

    let encoding = encoding_for_array(&bda);
    bda.numeric_type = Some(encoding.numeric_type);

    if !base64_bytes.is_empty() {
        bda.pending_zlib = encoding.is_zlib_compressed;
        bda.pending_base64 = Some(base64_bytes);
    }

    Ok(bda)
}

const MAX_NUMERIC_STRIDE_BYTES: usize = 8;
const DECOMPRESSED_SIZE_MARGIN_BYTES: usize = 1024;
const MAX_UNCOMPRESSED_SIZE_WITHOUT_DECLARED_LENGTH: usize = 1 << 31;

fn max_decompressed_size(array_length: Option<usize>) -> usize {
    match array_length {
        Some(declared_length) => declared_length
            .saturating_mul(MAX_NUMERIC_STRIDE_BYTES)
            .saturating_add(DECOMPRESSED_SIZE_MARGIN_BYTES),
        None => MAX_UNCOMPRESSED_SIZE_WITHOUT_DECLARED_LENGTH,
    }
}

pub(crate) fn finalize_bda(
    bda: &mut BinaryDataArray,
    default_array_length: Option<usize>,
) -> Result<(), ParseError> {
    let Some(base64_bytes) = bda.pending_base64.take() else {
        return Ok(());
    };
    let mut decoded = Vec::with_capacity(base64_bytes.len() / 4 * 3 + 8);
    STANDARD.decode_vec(&base64_bytes, &mut decoded)?;
    if bda.pending_zlib {
        let declared_length = bda.array_length.or(default_array_length);
        let max_size = max_decompressed_size(declared_length);
        if declared_length.is_some() {
            DecompressionLimit::default()
                .validate(decoded.len(), max_size)
                .map_err(|error| ParseError::Decompress(error.to_string()))?;
        }
        decoded = decompress_to_vec_zlib_with_limit(&decoded, max_size)
            .map_err(|e| ParseError::Decompress(format!("{e:?}")))?;
    }
    bda.binary = Some(decode_binary_data(
        bda.numeric_type.unwrap_or(NumericType::Float64),
        &decoded,
        bda.array_length,
    ));
    Ok(())
}

#[derive(Debug, Clone, Copy)]
struct BinaryArrayEncoding {
    is_zlib_compressed: bool,
    numeric_type: NumericType,
}

fn encoding_for_array(bda: &BinaryDataArray) -> BinaryArrayEncoding {
    let has = |acc: &str| {
        bda.cv_params
            .iter()
            .any(|p| p.accession.as_deref() == Some(acc))
    };
    let is_zlib_compressed = has(ACC_COMPRESSION_ZLIB);
    let (f64, f32, f16) = (
        has(ACC_FLOAT_64BIT_STR),
        has(ACC_FLOAT_32BIT_STR),
        has(ACC_FLOAT_16BIT_STR),
    );
    let (i64, i32, i16) = (
        has(ACC_INT_64BIT_STR),
        has(ACC_INT_32BIT_STR),
        has(ACC_INT_16BIT_STR),
    );

    let numeric_type = if let Some(declared) = bda.numeric_type {
        declared
    } else if f16 && !f32 && !f64 {
        NumericType::Float16
    } else if i16 && !i32 && !i64 {
        NumericType::Int16
    } else if i32 && !i64 {
        NumericType::Int32
    } else if i64 && !f64 && !f32 {
        NumericType::Int64
    } else if f64 {
        NumericType::Float64
    } else if f32 {
        NumericType::Float32
    } else if i64 {
        NumericType::Int64
    } else {
        NumericType::Float64
    };

    BinaryArrayEncoding {
        is_zlib_compressed,
        numeric_type,
    }
}

fn decode_binary_data(
    numeric_type: NumericType,
    decoded: &[u8],
    array_length: Option<usize>,
) -> NumericArray {
    match numeric_type {
        NumericType::Float64 => {
            NumericArray::F64(decode_packed_numeric_bytes(decoded, 8, array_length, |c| {
                f64::from_le_bytes(c.try_into().unwrap())
            }))
        }
        NumericType::Float32 => {
            NumericArray::F32(decode_packed_numeric_bytes(decoded, 4, array_length, |c| {
                f32::from_le_bytes(c.try_into().unwrap())
            }))
        }
        NumericType::Float16 => {
            NumericArray::F16(decode_packed_numeric_bytes(decoded, 2, array_length, |c| {
                u16::from_le_bytes(c.try_into().unwrap())
            }))
        }
        NumericType::Int64 => {
            NumericArray::I64(decode_packed_numeric_bytes(decoded, 8, array_length, |c| {
                i64::from_le_bytes(c.try_into().unwrap())
            }))
        }
        NumericType::Int32 => {
            NumericArray::I32(decode_packed_numeric_bytes(decoded, 4, array_length, |c| {
                i32::from_le_bytes(c.try_into().unwrap())
            }))
        }
        NumericType::Int16 => {
            NumericArray::I16(decode_packed_numeric_bytes(decoded, 2, array_length, |c| {
                i16::from_le_bytes(c.try_into().unwrap())
            }))
        }
    }
}

fn decode_packed_numeric_bytes<T, F>(
    bytes: &[u8],
    stride: usize,
    declared_length: Option<usize>,
    from_le: F,
) -> Vec<T>
where
    F: Fn(&[u8]) -> T,
{
    let available = (bytes.len() - bytes.len() % stride) / stride;
    let target = declared_length.unwrap_or(available).min(available);
    if target == 0 {
        return Vec::new();
    }
    let mut out = Vec::with_capacity(target);
    for chunk in bytes[..target * stride].chunks_exact(stride) {
        out.push(from_le(chunk));
    }
    out
}