ionic-rs 0.2.1

Streamable binary format for mass-spectrometry data: lossless mzML conversion, random access, WebAssembly support
Documentation
use std::{
    borrow::Cow,
    collections::{HashMap, VecDeque},
    sync::Arc,
};

use crate::ion::{
    ByteRange, IonError, IonResult,
    decoder::{
        decode::{Metadatum, MetadatumValue, ReadOptions},
        utilities::{
            byte_source::ReadBytes,
            common::decompress_zstd_allow_aligned_padding,
            decompression_limit::DecompressionLimit,
            meta_column_layout::MetaColumnLayout,
            parse_metadata::{CODEC_NONE, CODEC_ZSTD, parse_metadata},
        },
    },
    meta_groups::{
        META_GROUP_ENTRY_SIZE, META_GROUP_HEADER_SIZE, MetaGroupEntry, MetaSection,
        group_count_for, group_of_item, item_range_of_group, meta_directory_range,
        read_group_header,
    },
};

pub(crate) struct MetaGroupReader {
    source: Arc<dyn ReadBytes>,
    section: MetaSection,
    directory: Vec<MetaGroupEntry>,
    payload_end: u64,
    compression_codec: u8,
    verify_checksums: bool,
    budget: DecompressionLimit,
    cache: GroupCache,
    layout: MetaColumnLayout,
}

#[derive(Default)]
struct MetaCounts {
    rows: u64,
    numeric: u64,
    string: u64,
}

impl MetaCounts {
    fn add(&mut self, rows: u32, numeric: u32, string: u32) {
        self.rows += rows as u64;
        self.numeric += numeric as u64;
        self.string += string as u64;
    }
}

struct CachedGroup {
    rows: Arc<[Metadatum]>,
    footprint: usize,
}

struct GroupCache {
    groups: HashMap<u64, CachedGroup>,
    order: VecDeque<u64>,
    used_bytes: usize,
    max_bytes: usize,
}

impl MetaGroupReader {
    pub(crate) fn open(
        source: Arc<dyn ReadBytes>,
        section: MetaSection,
        compression_codec: u8,
        options: &ReadOptions,
        layout: MetaColumnLayout,
    ) -> IonResult<Self> {
        if section.item_count > 0 && section.group_size == 0 {
            return Err(IonError::from("metadata groups: group size is zero"));
        }
        if section.group_count != group_count_for(section.item_count, section.group_size) {
            return Err(IonError::from(
                "metadata groups: group count does not match item count",
            ));
        }
        let directory_range = meta_directory_range(section.range, section.group_count)?;
        let directory: Vec<MetaGroupEntry> = source
            .read(directory_range)?
            .as_chunks::<META_GROUP_ENTRY_SIZE>()
            .0
            .iter()
            .map(|entry| MetaGroupEntry::read_from(entry))
            .collect();
        let mut uncompressed_sum: u64 = 0;
        for entry in &directory {
            uncompressed_sum = uncompressed_sum
                .checked_add(entry.uncompressed_size)
                .ok_or_else(|| IonError::from("metadata groups: uncompressed total overflows"))?;
        }
        if uncompressed_sum != section.totals.uncompressed {
            return Err(IonError::from(
                "metadata groups: uncompressed total does not match header",
            ));
        }
        let payload_end = directory_range.offset - section.range.offset;
        Ok(Self {
            source,
            section,
            directory,
            payload_end,
            compression_codec,
            verify_checksums: options.verify_checksums,
            budget: options.decompression_limit,
            cache: GroupCache {
                groups: HashMap::new(),
                order: VecDeque::new(),
                used_bytes: 0,
                max_bytes: options.max_cached_bytes,
            },
            layout,
        })
    }

    #[cfg(test)]
    pub(crate) fn read_all(&self) -> IonResult<Vec<Metadatum>> {
        Ok(self.read_all_grouped()?.into_iter().flatten().collect())
    }

    pub(crate) fn read_all_grouped(&self) -> IonResult<Vec<Vec<Metadatum>>> {
        let payloads = self.source.read(ByteRange {
            offset: self.section.range.offset,
            length: self.payload_end,
        })?;
        let mut groups = Vec::with_capacity(self.directory.len());
        let mut seen = MetaCounts::default();
        for (group_index, entry) in self.directory.iter().enumerate() {
            let range = self.payload_range(entry)?;
            let start = (range.offset - self.section.range.offset) as usize;
            let end = start + range.length as usize;
            let payload = payloads
                .get(start..end)
                .ok_or_else(|| IonError::from("metadata groups: payload out of bounds"))?;
            let bytes = self.decode_payload(entry, payload)?;
            let (meta_count, numeric_count, string_count) = read_group_header(&bytes)?;
            seen.add(meta_count, numeric_count, string_count);
            groups.push(self.parse_group(&bytes, group_index as u64)?);
        }
        self.check_totals(&seen)?;
        Ok(groups)
    }

    pub(crate) fn read_first_group(&self) -> IonResult<Vec<Metadatum>> {
        if self.directory.is_empty() {
            return Ok(Vec::new());
        }
        self.read_group(0)
    }

    fn check_totals(&self, seen: &MetaCounts) -> IonResult<()> {
        if seen.rows != self.section.totals.rows
            || seen.numeric != self.section.totals.numeric
            || seen.string != self.section.totals.string
        {
            return Err(IonError::from(
                "metadata groups: row totals do not match header",
            ));
        }
        Ok(())
    }

    pub(crate) fn read_item(&mut self, item_index: u64) -> IonResult<Vec<Metadatum>> {
        if item_index >= self.section.item_count {
            return Ok(Vec::new());
        }
        let group_index = group_of_item(item_index, self.section.group_size);
        let rows = self.get_cached_rows(group_index)?;
        let start = rows.partition_point(|row| (row.item_index as u64) < item_index);
        let end = rows.partition_point(|row| (row.item_index as u64) <= item_index);
        Ok(rows[start..end].to_vec())
    }

    fn get_cached_rows(&mut self, group_index: u64) -> IonResult<Arc<[Metadatum]>> {
        if let Some(group) = self.cache.groups.get(&group_index) {
            return Ok(group.rows.clone());
        }
        let rows: Arc<[Metadatum]> = Arc::from(self.read_group(group_index)?);
        let footprint = group_footprint(&rows);
        self.store_in_cache(group_index, rows.clone(), footprint);
        Ok(rows)
    }

    fn store_in_cache(&mut self, group_index: u64, rows: Arc<[Metadatum]>, footprint: usize) {
        debug_assert!(!self.cache.groups.contains_key(&group_index));
        self.cache.used_bytes += footprint;
        self.cache
            .groups
            .insert(group_index, CachedGroup { rows, footprint });
        self.cache.order.push_back(group_index);
        while self.cache.used_bytes > self.cache.max_bytes && self.cache.order.len() > 1 {
            if let Some(oldest) = self.cache.order.pop_front()
                && let Some(removed) = self.cache.groups.remove(&oldest)
            {
                self.cache.used_bytes -= removed.footprint;
            }
        }
    }

    fn read_group(&self, group_index: u64) -> IonResult<Vec<Metadatum>> {
        let entry = self
            .directory
            .get(group_index as usize)
            .ok_or_else(|| IonError::from("metadata groups: group index out of range"))?;
        let payload = self.source.read(self.payload_range(entry)?)?;
        let bytes = self.decode_payload(entry, &payload)?;
        self.parse_group(&bytes, group_index)
    }

    fn payload_range(&self, entry: &MetaGroupEntry) -> IonResult<ByteRange> {
        let end = entry
            .payload_offset
            .checked_add(entry.payload_size)
            .ok_or_else(|| IonError::from("metadata groups: payload overflows"))?;
        if end > self.payload_end {
            return Err(IonError::from(
                "metadata groups: payload overlaps the directory",
            ));
        }
        Ok(ByteRange {
            offset: self.section.range.offset + entry.payload_offset,
            length: entry.payload_size,
        })
    }

    fn decode_payload<'a>(
        &self,
        entry: &MetaGroupEntry,
        payload: &'a [u8],
    ) -> IonResult<Cow<'a, [u8]>> {
        if self.verify_checksums && crc32fast::hash(payload) != entry.checksum {
            return Err(IonError::from("metadata groups: group checksum mismatch"));
        }
        match self.compression_codec {
            CODEC_NONE => Ok(Cow::Borrowed(payload)),
            CODEC_ZSTD => {
                let size = usize::try_from(entry.uncompressed_size).map_err(|_| {
                    IonError::from("metadata groups: uncompressed size out of range")
                })?;
                let plain = decompress_zstd_allow_aligned_padding(payload, size, self.budget)?;
                Ok(Cow::Owned(plain))
            }
            other => Err(IonError::from(format!(
                "metadata groups: unsupported codec {other}"
            ))),
        }
    }

    fn parse_group(&self, bytes: &[u8], group_index: u64) -> IonResult<Vec<Metadatum>> {
        let (meta_count, numeric_count, string_count) = read_group_header(bytes)?;
        let (item_start, item_end) = item_range_of_group(
            group_index,
            self.section.group_size,
            self.section.item_count,
        );
        let mut rows = parse_metadata(
            &bytes[META_GROUP_HEADER_SIZE..],
            item_end - item_start,
            meta_count as u64,
            numeric_count as u64,
            string_count as u64,
            CODEC_NONE,
            0,
            self.budget,
            self.layout,
        )?;
        let item_base = item_start as u32;
        for row in &mut rows {
            row.item_index += item_base;
        }
        Ok(rows)
    }
}

fn group_footprint(rows: &[Metadatum]) -> usize {
    let mut total = 0;
    for row in rows {
        total += std::mem::size_of::<Metadatum>();
        if let Some(accession) = &row.accession {
            total += accession.len();
        }
        if let Some(unit) = &row.unit_accession {
            total += unit.len();
        }
        if let MetadatumValue::Text(text) = &row.value {
            total += text.len();
        }
    }
    total
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;

    use super::MetaGroupReader;
    use crate::ion::{
        ByteRange, DecompressionLimit, IonResult,
        decoder::{
            decode::ReadOptions,
            utilities::{byte_source::BytesSource, meta_column_layout::MetaColumnLayout},
        },
        format::CODEC_NONE,
        meta_groups::{MetaSection, MetaTotals},
    };

    fn no_totals() -> MetaTotals {
        MetaTotals {
            rows: 0,
            numeric: 0,
            string: 0,
            uncompressed: 0,
        }
    }

    fn new_reader(group_count: u64, group_size: u32, item_count: u64) -> IonResult<()> {
        let section = MetaSection {
            range: ByteRange {
                offset: 0,
                length: 0,
            },
            group_count,
            group_size,
            item_count,
            totals: no_totals(),
        };
        MetaGroupReader::open(
            Arc::new(BytesSource::new(Arc::from(&[][..]))),
            section,
            CODEC_NONE,
            &ReadOptions {
                max_cached_bytes: 1024,
                verify_checksums: false,
                parallel: true,
                decompression_limit: DecompressionLimit::default(),
            },
            MetaColumnLayout::new(),
        )
        .map(|_| ())
    }

    #[test]
    fn rejects_zero_group_size_when_items_exist() {
        assert!(new_reader(0, 0, 5).is_err());
    }

    #[test]
    fn rejects_group_count_that_disagrees_with_item_count() {
        assert!(new_reader(2, 8192, 5).is_err());
    }

    #[test]
    fn allows_consistent_empty_section() {
        assert!(new_reader(0, 8192, 0).is_ok());
    }
}