ionic-rs 0.2.0

Streamable binary format for mass-spectrometry data: lossless mzML conversion, random access, WebAssembly support
Documentation
pub(crate) use crate::ion::format::{CODEC_NONE, CODEC_ZSTD};
use crate::{
    ion::{
        IonResult,
        attr_meta::format_accession,
        decoder::{
            decode::{Metadatum, MetadatumValue},
            utilities::{
                common::{
                    decompress_zstd_allow_aligned_padding, read_u32_vec, sum_string_lengths, take,
                },
                decompression_limit::DecompressionLimit,
                meta_column_layout::MetaColumnLayout,
            },
        },
        utilities::common::*,
    },
    mzml::schema::TagId,
};

#[allow(clippy::too_many_arguments)]
pub(crate) fn parse_metadata(
    bytes: &[u8],
    item_count: u64,
    meta_count: u64,
    num_count: u64,
    str_count: u64,
    compression_codec: u8,
    expected_uncompressed_bytes: usize,
    decompression_limit: DecompressionLimit,
    layout: MetaColumnLayout,
) -> IonResult<Vec<Metadatum>> {
    let owned;
    let bytes = match compression_codec {
        CODEC_NONE => bytes,
        CODEC_ZSTD => {
            owned = decompress_zstd_allow_aligned_padding(
                bytes,
                expected_uncompressed_bytes,
                decompression_limit,
            )?;
            owned.as_slice()
        }
        other => return Err(format!("unsupported compression_codec={other}").into()),
    };

    let byte_budget = bytes.len();
    let item_count = bound_count(item_count, byte_budget, "metadata item_count")?;
    let meta_count = bound_count(meta_count, byte_budget, "metadata meta_count")?;
    let num_count = bound_count(num_count, byte_budget, "metadata num_count")?;
    let str_count = bound_count(str_count, byte_budget, "metadata str_count")?;

    let mut pos = 0usize;

    let children_index_len = item_count
        .checked_add(1)
        .ok_or_else(|| crate::ion::IonError::from("metadata item_count+1 overflows usize"))?;
    let children_index = read_u32_vec(bytes, &mut pos, children_index_len)?;
    let mut metadatum_owner_ids = read_u32_vec(bytes, &mut pos, meta_count)?;
    let mut metadatum_parent_ids = read_u32_vec(bytes, &mut pos, meta_count)?;
    let metadatum_tag_ids = take(bytes, &mut pos, meta_count, "metadatum tag id")?;
    let metadatum_ref_ids = take(bytes, &mut pos, meta_count, "metadatum ref id")?;
    let metadatum_accessions = read_u32_vec(bytes, &mut pos, meta_count)?;
    let metadatum_unit_refs = take(bytes, &mut pos, meta_count, "metadatum unit ref id")?;
    let metadatum_unit_accs = read_u32_vec(bytes, &mut pos, meta_count)?;
    let value_kinds = take(bytes, &mut pos, meta_count, "metadatum value kind")?;
    let value_indices = rebuild_value_indices(value_kinds)?;

    let numeric_values = read_f64_vec(bytes, &mut pos, num_count)?;

    let string_lengths = read_u32_vec(bytes, &mut pos, str_count)?;
    let string_offsets = rebuild_string_offsets(&string_lengths)?;

    let string_bytes_needed = sum_string_lengths(&string_lengths)?;
    let string_data = take(bytes, &mut pos, string_bytes_needed, "string values")?;

    validate_trailing_bytes(bytes, pos, compression_codec, expected_uncompressed_bytes)?;
    validate_children_index(&children_index, item_count, meta_count)?;

    if layout.ids_reset {
        restore_group_unique_ids(
            &children_index,
            item_count,
            &mut metadatum_owner_ids,
            &mut metadatum_parent_ids,
        )?;
    }

    let mut out = Vec::with_capacity(meta_count);

    for item_index in 0..item_count {
        let meta_start = children_index[item_index] as usize;
        let meta_end = children_index[item_index + 1] as usize;

        for meta_index in meta_start..meta_end {
            let tag_id = TagId::from_u8(metadatum_tag_ids[meta_index]).unwrap_or(TagId::Unknown);
            let value = parse_value(
                value_kinds[meta_index],
                value_indices[meta_index],
                &numeric_values,
                &string_offsets,
                &string_lengths,
                string_data,
            )?;

            let accession = format_accession(
                metadatum_ref_ids[meta_index],
                metadatum_accessions[meta_index],
            );
            let unit_accession = format_accession(
                metadatum_unit_refs[meta_index],
                metadatum_unit_accs[meta_index],
            );

            out.push(Metadatum {
                item_index: item_index as u32,
                id: metadatum_owner_ids[meta_index],
                parent_id: metadatum_parent_ids[meta_index],
                tag_id,
                accession,
                unit_accession,
                value,
            });
        }
    }

    Ok(out)
}

#[inline]
fn validate_trailing_bytes(
    bytes: &[u8],
    pos: usize,
    compression_codec: u8,
    expected_uncompressed_bytes: usize,
) -> IonResult<()> {
    if compression_codec == CODEC_ZSTD {
        if pos != bytes.len() {
            return Err("trailing bytes in decompressed metadata section".into());
        }
    } else {
        let trailing = &bytes[pos..];
        if trailing.len() > 7 || trailing.iter().any(|&b| b != 0) {
            return Err("trailing bytes in metadata section".into());
        }
        let _ = expected_uncompressed_bytes;
    }
    Ok(())
}

#[inline]
fn validate_children_index(
    children_index: &[u32],
    item_count: usize,
    meta_count: usize,
) -> IonResult<()> {
    if children_index.is_empty() || children_index[0] != 0 {
        return Err("CI[0] must be 0".into());
    }
    if children_index[item_count] as usize != meta_count {
        return Err("CI[last] must equal meta_count".into());
    }
    let mut previous = 0u32;
    for &entry in children_index {
        if entry < previous || (entry as usize) > meta_count {
            return Err("CI is not monotonic or out of range".into());
        }
        previous = entry;
    }
    Ok(())
}

const SHARED_LIST_NODE_ID: u32 = 1;

#[inline]
fn restore_group_unique_ids(
    children_index: &[u32],
    item_count: usize,
    owner_ids: &mut [u32],
    parent_ids: &mut [u32],
) -> IonResult<()> {
    let largest_local_id = owner_ids
        .iter()
        .chain(parent_ids.iter())
        .copied()
        .max()
        .unwrap_or(0);
    let stride = largest_local_id
        .checked_add(1)
        .ok_or_else(|| crate::ion::IonError::from("metadata id remap: stride overflows u32"))?;

    for item_index in 0..item_count {
        let meta_start = children_index[item_index] as usize;
        let meta_end = children_index[item_index + 1] as usize;
        let item_base = (item_index as u32).checked_mul(stride).ok_or_else(|| {
            crate::ion::IonError::from("metadata id remap: item base overflows u32")
        })?;
        for meta_index in meta_start..meta_end {
            owner_ids[meta_index] = restore_one_id(owner_ids[meta_index], item_base)?;
            parent_ids[meta_index] = restore_one_id(parent_ids[meta_index], item_base)?;
        }
    }
    Ok(())
}

#[inline]
fn restore_one_id(local_id: u32, item_base: u32) -> IonResult<u32> {
    if local_id == SHARED_LIST_NODE_ID {
        return Ok(local_id);
    }
    item_base.checked_add(local_id).ok_or_else(|| {
        crate::ion::IonError::from("metadata id remap: group-unique id overflows u32")
    })
}

#[inline]
fn parse_value(
    value_kind: u8,
    value_index: u32,
    numeric_values: &[f64],
    string_offsets: &[u32],
    string_lengths: &[u32],
    string_data: &[u8],
) -> IonResult<MetadatumValue> {
    match value_kind {
        0 => {
            let index = value_index as usize;
            if index >= numeric_values.len() {
                return Err("numeric VI out of range".into());
            }
            Ok(MetadatumValue::Number(numeric_values[index]))
        }
        1 => {
            let index = value_index as usize;
            if index >= string_offsets.len() || index >= string_lengths.len() {
                return Err("string VI out of range".into());
            }
            let offset = string_offsets[index] as usize;
            let length = string_lengths[index] as usize;
            if offset
                .checked_add(length)
                .is_none_or(|end| end > string_data.len())
            {
                return Err("string slice out of bounds".into());
            }
            if length == 0 {
                return Ok(MetadatumValue::Text(String::new()));
            }
            let slice = &string_data[offset..offset + length];
            let text = std::str::from_utf8(slice)
                .map_err(|_| "string metadata is not valid UTF-8")?
                .to_string();
            Ok(MetadatumValue::Text(text))
        }
        2 => Ok(MetadatumValue::Empty),
        other => Err(format!("invalid value kind VK={other}").into()),
    }
}

#[inline]
fn rebuild_value_indices(value_kinds: &[u8]) -> IonResult<Vec<u32>> {
    let mut next_numeric_ordinal = 0u32;
    let mut next_string_ordinal = 0u32;
    let mut value_indices = Vec::with_capacity(value_kinds.len());
    for &kind in value_kinds {
        match kind {
            0 => {
                value_indices.push(next_numeric_ordinal);
                next_numeric_ordinal = next_numeric_ordinal.checked_add(1).ok_or_else(|| {
                    crate::ion::IonError::from("numeric value ordinal overflows u32")
                })?;
            }
            1 => {
                value_indices.push(next_string_ordinal);
                next_string_ordinal = next_string_ordinal.checked_add(1).ok_or_else(|| {
                    crate::ion::IonError::from("string value ordinal overflows u32")
                })?;
            }
            _ => value_indices.push(0),
        }
    }
    Ok(value_indices)
}

#[inline]
fn rebuild_string_offsets(string_lengths: &[u32]) -> IonResult<Vec<u32>> {
    let mut string_offsets = Vec::with_capacity(string_lengths.len());
    let mut running_offset = 0u32;
    for &length in string_lengths {
        string_offsets.push(running_offset);
        running_offset = running_offset
            .checked_add(length)
            .ok_or_else(|| crate::ion::IonError::from("string offset prefix sum overflows u32"))?;
    }
    Ok(string_offsets)
}

#[inline]
fn bound_count(count: u64, byte_budget: usize, ctx: &'static str) -> IonResult<usize> {
    if count > byte_budget as u64 {
        return Err(
            format!("{ctx}: declared count {count} exceeds byte budget {byte_budget}").into(),
        );
    }
    Ok(count as usize)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn rejects_invalid_utf8_string_value() {
        let numeric_values: [f64; 0] = [];
        let string_offsets = [0u32];
        let string_lengths = [3u32];
        let string_data: [u8; 3] = [b'A', 0xFF, b'B'];

        let result = parse_value(
            1,
            0,
            &numeric_values,
            &string_offsets,
            &string_lengths,
            &string_data,
        );

        assert!(result.is_err());
    }

    #[test]
    fn accepts_valid_multibyte_utf8_string_value() {
        let numeric_values: [f64; 0] = [];
        let text = "café";
        let string_data = text.as_bytes();
        let string_offsets = [0u32];
        let string_lengths = [string_data.len() as u32];

        let value = parse_value(
            1,
            0,
            &numeric_values,
            &string_offsets,
            &string_lengths,
            string_data,
        )
        .expect("valid UTF-8 must parse");

        match value {
            MetadatumValue::Text(t) => assert_eq!(t, text),
            other => panic!("expected Text value, got {other:?}"),
        }
    }
}