Skip to main content

ionic/ion/decoder/decode/
windows.rs

1use super::*;
2use crate::{
3    accessions::{INTENSITY_ARRAY, MZ_ARRAY},
4    ion::{
5        ArrayKind, ByteRangesOptions, decoder::utilities::window_directory::WindowEntry,
6        windowing::window_index,
7    },
8};
9
10fn window_span(width: f64, from: f64, to: f64, window_count: usize) -> Option<(usize, usize)> {
11    if window_count == 0 {
12        return None;
13    }
14    let low = window_index(width, from) as usize;
15    if low >= window_count {
16        return None;
17    }
18    Some((
19        low,
20        (window_index(width, to) as usize).min(window_count - 1),
21    ))
22}
23
24#[non_exhaustive]
25#[derive(Debug, Clone, PartialEq)]
26pub struct DataXY {
27    pub x: NumericArray,
28    pub y: NumericArray,
29}
30
31impl NumericArray {
32    pub fn to_f64(&self) -> Vec<f64> {
33        match self {
34            NumericArray::F64(values) => values.clone(),
35            NumericArray::F32(values) => values.iter().map(|&value| value as f64).collect(),
36            NumericArray::F16(values) => values.iter().copied().map(f16_bits_to_f64).collect(),
37            NumericArray::I16(values) => values.iter().map(|&value| value as f64).collect(),
38            NumericArray::I32(values) => values.iter().map(|&value| value as f64).collect(),
39            NumericArray::I64(values) => values.iter().map(|&value| value as f64).collect(),
40        }
41    }
42
43    pub(crate) fn extend_f64(&self, out: &mut Vec<f64>) {
44        match self {
45            NumericArray::F64(values) => out.extend_from_slice(values),
46            NumericArray::F32(values) => out.extend(values.iter().map(|&value| value as f64)),
47            NumericArray::F16(values) => out.extend(values.iter().copied().map(f16_bits_to_f64)),
48            NumericArray::I16(values) => out.extend(values.iter().map(|&value| value as f64)),
49            NumericArray::I32(values) => out.extend(values.iter().map(|&value| value as f64)),
50            NumericArray::I64(values) => out.extend(values.iter().map(|&value| value as f64)),
51        }
52    }
53}
54
55#[non_exhaustive]
56#[derive(Debug, Clone, Copy, PartialEq, Default)]
57pub enum Select {
58    #[default]
59    All,
60    Rt(Range),
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Default)]
64pub struct ScanQuery {
65    pub mz: Range,
66    pub select: Select,
67    pub ms_level: Option<u8>,
68}
69
70#[non_exhaustive]
71pub struct Window<'a> {
72    pub index: usize,
73    pub summary: &'a ScanSummary,
74    pub mz: &'a [f64],
75    pub intensity: &'a [f64],
76}
77
78fn scan_is_selected(select: &Select, summary: &ScanSummary) -> bool {
79    match select {
80        Select::All => true,
81        Select::Rt(range) => summary.rt >= range.from && summary.rt <= range.to,
82    }
83}
84
85fn scan_summary_from_record(record: &SpectrumSummary) -> ScanSummary {
86    ScanSummary {
87        rt: record.rt,
88        rt_unit: TimeUnit::from_code(record.rt_unit),
89        ms_level: record.ms_level,
90        polarity: record.polarity,
91        selected_ion_mz: record.selected_ion_mz,
92        base_peak_mz: record.base_peak_mz,
93        base_peak_int: record.base_peak_int,
94        total_ion_current: record.total_ion_current,
95        position_x: record.position_x,
96        position_y: record.position_y,
97        position_z: record.position_z,
98    }
99}
100
101#[cfg(test)]
102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub(crate) enum ItemKind {
104    Spectrum,
105    Chromatogram,
106}
107
108#[cfg(test)]
109#[derive(Debug, Clone, Copy)]
110pub(crate) struct ItemSlice {
111    pub(crate) item_index: u64,
112    pub(crate) array_address_index: u64,
113    pub(crate) intensity_address_index: u64,
114}
115
116#[derive(Debug, Clone, Copy)]
117pub(crate) struct WindowRead {
118    pub(crate) mz_address: ArrayAddress,
119    pub(crate) y_address: ArrayAddress,
120}
121
122fn empty_array(dtype: u8) -> NumericArray {
123    match dtype {
124        FILE_DTYPE_F32 => NumericArray::F32(Vec::new()),
125        FILE_DTYPE_F16 => NumericArray::F16(Vec::new()),
126        FILE_DTYPE_I16 => NumericArray::I16(Vec::new()),
127        FILE_DTYPE_I32 => NumericArray::I32(Vec::new()),
128        FILE_DTYPE_I64 => NumericArray::I64(Vec::new()),
129        _ => NumericArray::F64(Vec::new()),
130    }
131}
132
133fn value_at(array: &NumericArray, index: usize) -> f64 {
134    match array {
135        NumericArray::F64(values) => values[index],
136        NumericArray::F32(values) => values[index] as f64,
137        NumericArray::F16(values) => f16_bits_to_f64(values[index]),
138        NumericArray::I16(values) => values[index] as f64,
139        NumericArray::I32(values) => values[index] as f64,
140        NumericArray::I64(values) => values[index] as f64,
141    }
142}
143
144fn append_range(dst: &mut NumericArray, src: &NumericArray, start: usize, end: usize) {
145    match (dst, src) {
146        (NumericArray::F64(d), NumericArray::F64(s)) => d.extend_from_slice(&s[start..end]),
147        (NumericArray::F32(d), NumericArray::F32(s)) => d.extend_from_slice(&s[start..end]),
148        (NumericArray::F16(d), NumericArray::F16(s)) => d.extend_from_slice(&s[start..end]),
149        (NumericArray::I16(d), NumericArray::I16(s)) => d.extend_from_slice(&s[start..end]),
150        (NumericArray::I32(d), NumericArray::I32(s)) => d.extend_from_slice(&s[start..end]),
151        (NumericArray::I64(d), NumericArray::I64(s)) => d.extend_from_slice(&s[start..end]),
152        _ => {}
153    }
154}
155
156fn first_index_at_or_after(
157    x: &NumericArray,
158    positions: std::ops::Range<usize>,
159    target: f64,
160) -> usize {
161    let (mut low, mut high) = (positions.start, positions.end);
162    while low < high {
163        let middle = low + (high - low) / 2;
164        if value_at(x, middle) < target {
165            low = middle + 1;
166        } else {
167            high = middle;
168        }
169    }
170    low
171}
172
173fn first_index_after(x: &NumericArray, positions: std::ops::Range<usize>, target: f64) -> usize {
174    let (mut low, mut high) = (positions.start, positions.end);
175    while low < high {
176        let middle = low + (high - low) / 2;
177        if value_at(x, middle) <= target {
178            low = middle + 1;
179        } else {
180            high = middle;
181        }
182    }
183    low
184}
185
186fn range_in_sorted(x: &NumericArray, low: f64, high: f64, paired: usize) -> (usize, usize) {
187    (
188        first_index_at_or_after(x, 0..paired, low),
189        first_index_after(x, 0..paired, high),
190    )
191}
192
193fn array_run(addresses: &[ArrayAddress], kind: u32) -> Option<(usize, &[ArrayAddress])> {
194    let start = addresses.iter().position(|address| {
195        address.array_type == kind && address.continues_previous_segment == 0
196    })?;
197    let length = addresses[start + 1..]
198        .iter()
199        .take_while(|address| address.continues_previous_segment == 1)
200        .count();
201    Some((start, &addresses[start..=start + length]))
202}
203
204pub(crate) fn keep_pairs_sorted(
205    x: &NumericArray,
206    y: &NumericArray,
207    low: f64,
208    high: f64,
209    x_out: &mut NumericArray,
210    y_out: &mut NumericArray,
211) {
212    let paired = x.len().min(y.len());
213    let (start, end) = range_in_sorted(x, low, high, paired);
214    append_range(x_out, x, start, end);
215    append_range(y_out, y, start, end);
216}
217
218impl IonReader {
219    pub fn require_bounds(&mut self) -> IonResult<()> {
220        self.ensure_spec_window_directory()
221    }
222
223    fn check_window(&self, index: usize, range: Range) -> IonResult<()> {
224        if !range.from.is_finite() || !range.to.is_finite() {
225            return Err("window bounds must be finite".into());
226        }
227        if range.from > range.to {
228            return Err("window: from is greater than to".into());
229        }
230        if index >= self.header.spectrum_count as usize {
231            return Err("spectrum index out of range".into());
232        }
233        Ok(())
234    }
235
236    fn window_rows(&mut self, index: usize, mz: Range) -> IonResult<Vec<WindowEntry>> {
237        self.require_bounds()?;
238
239        let width = self.header.target_mz_window as f64;
240        let window_directory = match &self.spec_window_directory {
241            WindowDirectoryCache::Loaded(index) => index,
242            _ => return Err(IonError::MissingSpectrumBounds),
243        };
244        let Some((window_low, window_high)) =
245            window_span(width, mz.from, mz.to, window_directory.window_count())
246        else {
247            return Ok(Vec::new());
248        };
249        let mut rows = Vec::new();
250        for window in window_low..=window_high {
251            if let Some(row) = window_directory.find_in_window(window, index as u32) {
252                rows.push(row);
253            }
254        }
255        Ok(rows)
256    }
257
258    #[cfg(test)]
259    pub(crate) fn get_spectrum_mz_windows(
260        &mut self,
261        scan_index: usize,
262        mz_from: f64,
263        mz_to: f64,
264    ) -> IonResult<Vec<WindowRead>> {
265        let mz = Range {
266            from: mz_from,
267            to: mz_to,
268        };
269        self.check_window(scan_index, mz)?;
270        let rows = self.window_rows(scan_index, mz)?;
271        self.window_reads(&rows, |row| row.intensity_address)
272    }
273
274    fn window_reads(
275        &self,
276        rows: &[WindowEntry],
277        y_address: impl Fn(&WindowEntry) -> u32,
278    ) -> IonResult<Vec<WindowRead>> {
279        let mut windows = Vec::with_capacity(rows.len());
280        for row in rows {
281            windows.push(WindowRead {
282                mz_address: self.array_address_at(row.mz_address)?,
283                y_address: self.array_address_at(y_address(row))?,
284            });
285        }
286        Ok(windows)
287    }
288
289    fn array_address_at(&self, ref_index: u32) -> IonResult<ArrayAddress> {
290        let start = ref_index as usize * ARRAY_ADDRESS_BYTES;
291        let bytes = self
292            .spec_array_addresses
293            .get(start..start + ARRAY_ADDRESS_BYTES)
294            .ok_or_else(|| IonError::from("window directory: array ref index out of range"))?;
295        Ok(parse_array_address(bytes))
296    }
297
298    pub(crate) fn spec_block_byte_range(&self, block_id: u32) -> IonResult<ByteRange> {
299        self.spec_container
300            .block_byte_range(block_id)
301            .ok_or_else(|| IonError::from("spectrum block id out of range"))
302    }
303
304    pub fn eic_byte_ranges(
305        &mut self,
306        mz: Range,
307        rt: Option<Range>,
308        gap: u64,
309    ) -> IonResult<Vec<ByteRange>> {
310        if !mz.from.is_finite() || !mz.to.is_finite() {
311            return Err("eic byte ranges: mz bounds must be finite".into());
312        }
313        if mz.from > mz.to {
314            return Err("eic byte ranges: mz from is greater than to".into());
315        }
316        self.ensure_spec_window_directory()?;
317
318        let width = self.header.target_mz_window as f64;
319        let rt_is_searchable = self.spec_rt_is_finite_ascending();
320
321        let address_indices = {
322            let WindowDirectoryCache::Loaded(window_directory) = &self.spec_window_directory else {
323                return Err(IonError::MissingSpectrumBounds);
324            };
325            let Some((window_low, window_high)) =
326                window_span(width, mz.from, mz.to, window_directory.window_count())
327            else {
328                return Ok(Vec::new());
329            };
330            let mut address_indices = Vec::new();
331            for window in window_low..=window_high {
332                for position in window_directory.window_range(window) {
333                    let row = window_directory.row(position);
334                    if let Some(wanted) = rt {
335                        let found = self.spectrum_rt(row.spectrum_index as usize);
336                        if found < wanted.from || found > wanted.to {
337                            if rt_is_searchable && found > wanted.to {
338                                break;
339                            }
340                            continue;
341                        }
342                    }
343                    address_indices.push(row.mz_address);
344                    address_indices.push(row.intensity_address);
345                }
346            }
347            address_indices
348        };
349
350        let mut block_ids = Vec::with_capacity(address_indices.len());
351        for index in address_indices {
352            block_ids.push(self.array_address_at(index)?.block_id);
353        }
354        block_ids.sort_unstable();
355        block_ids.dedup();
356
357        let mut ranges = Vec::with_capacity(block_ids.len());
358        for block_id in block_ids {
359            ranges.push(self.spec_block_byte_range(block_id)?);
360        }
361        merge_ranges(&mut ranges, gap);
362        Ok(ranges)
363    }
364
365    pub fn byte_ranges(
366        &mut self,
367        scan_index: usize,
368        mz: Range,
369        options: &ByteRangesOptions,
370    ) -> IonResult<Vec<ByteRange>> {
371        self.check_window(scan_index, mz)?;
372        let rows = self.window_rows(scan_index, mz)?;
373
374        let mut block_ids = Vec::new();
375        for &array_type in &options.arrays {
376            self.add_block_ids(scan_index, array_type, &rows, &mut block_ids)?;
377        }
378        block_ids.sort_unstable();
379        block_ids.dedup();
380
381        let mut ranges = Vec::with_capacity(block_ids.len());
382        for block_id in block_ids {
383            ranges.push(self.spec_block_byte_range(block_id)?);
384        }
385        ranges.sort_unstable_by_key(|range| (range.offset, range.length));
386        Ok(ranges)
387    }
388
389    fn add_block_ids(
390        &self,
391        scan_index: usize,
392        array_type: u32,
393        rows: &[WindowEntry],
394        block_ids: &mut Vec<u32>,
395    ) -> IonResult<()> {
396        if array_type == MZ_ARRAY {
397            return self.add_window_block_ids(rows, |row| row.mz_address, block_ids);
398        }
399        if array_type == INTENSITY_ARRAY {
400            return self.add_window_block_ids(rows, |row| row.intensity_address, block_ids);
401        }
402        if let Some(shift) = self.y_shift(scan_index, array_type) {
403            return self.add_window_block_ids(
404                rows,
405                |row| row.mz_address.wrapping_add_signed(shift),
406                block_ids,
407            );
408        }
409        let Some(addresses) = read_array_addresses_from_buffers(
410            &self.spec_entries_buf,
411            &self.spec_array_addresses,
412            scan_index,
413        ) else {
414            return Ok(());
415        };
416        if let Some((_, run)) = array_run(addresses.as_slice(), array_type) {
417            block_ids.extend(run.iter().map(|address| address.block_id));
418        }
419        Ok(())
420    }
421
422    fn add_window_block_ids(
423        &self,
424        rows: &[WindowEntry],
425        address_of: impl Fn(&WindowEntry) -> u32,
426        block_ids: &mut Vec<u32>,
427    ) -> IonResult<()> {
428        for row in rows {
429            block_ids.push(self.array_address_at(address_of(row))?.block_id);
430        }
431        Ok(())
432    }
433
434    pub fn spectrum_window(
435        &mut self,
436        index: usize,
437        x_kind: impl Into<u32>,
438        y_kind: impl Into<u32>,
439        range: Range,
440    ) -> IonResult<DataXY> {
441        let (x_kind, y_kind) = (x_kind.into(), y_kind.into());
442        self.check_window(index, range)?;
443        if x_kind != MZ_ARRAY {
444            return self.filtered_window(index, x_kind, y_kind, range);
445        }
446        if y_kind == ACC_INT {
447            return self.mz_window(index, range, |row| row.intensity_address);
448        }
449        match self.y_shift(index, y_kind) {
450            Some(shift) => self.mz_window(index, range, |row| {
451                row.mz_address.wrapping_add_signed(shift)
452            }),
453            None => self.filtered_window(index, x_kind, y_kind, range),
454        }
455    }
456
457    fn mz_window(
458        &mut self,
459        index: usize,
460        mz: Range,
461        y_address: impl Fn(&WindowEntry) -> u32,
462    ) -> IonResult<DataXY> {
463        let rows = self.window_rows(index, mz)?;
464        let windows = self.window_reads(&rows, y_address)?;
465        let (mz_dtype, y_dtype) = windows
466            .first()
467            .map(|window| (window.mz_address.dtype, window.y_address.dtype))
468            .unwrap_or((FILE_DTYPE_F64, FILE_DTYPE_F64));
469        let mut mz_out = empty_array(mz_dtype);
470        let mut y_out = empty_array(y_dtype);
471        for window in windows {
472            let mz_segment = self.read_spectrum_array(&window.mz_address)?;
473            let y_segment = self.read_spectrum_array(&window.y_address)?;
474            keep_pairs_sorted(
475                &mz_segment,
476                &y_segment,
477                mz.from,
478                mz.to,
479                &mut mz_out,
480                &mut y_out,
481            );
482        }
483
484        Ok(DataXY {
485            x: mz_out,
486            y: y_out,
487        })
488    }
489
490    fn y_shift(&self, index: usize, y_kind: u32) -> Option<i32> {
491        let addresses = read_array_addresses_from_buffers(
492            &self.spec_entries_buf,
493            &self.spec_array_addresses,
494            index,
495        )?;
496        let addresses = addresses.as_slice();
497        let (mz_start, mz_run) = array_run(addresses, MZ_ARRAY)?;
498        array_run(addresses, ACC_INT)?;
499        let (y_start, y_run) = array_run(addresses, y_kind)?;
500        let same_split = mz_run
501            .iter()
502            .map(|address| address.element_count)
503            .eq(y_run.iter().map(|address| address.element_count));
504        same_split.then(|| y_start as i32 - mz_start as i32)
505    }
506
507    fn filtered_window(
508        &mut self,
509        index: usize,
510        x_kind: u32,
511        y_kind: u32,
512        range: Range,
513    ) -> IonResult<DataXY> {
514        let groups = match read_array_addresses_from_buffers(
515            &self.spec_entries_buf,
516            &self.spec_array_addresses,
517            index,
518        ) {
519            Some(addresses) => group_arrays(addresses.as_slice())?,
520            None => Vec::new(),
521        };
522        let x_group = groups.iter().find(|group| group.array_type == x_kind);
523        let y_group = groups.iter().find(|group| group.array_type == y_kind);
524        let (Some(x_group), Some(y_group)) = (x_group, y_group) else {
525            return Ok(DataXY {
526                x: empty_array(FILE_DTYPE_F64),
527                y: empty_array(FILE_DTYPE_F64),
528            });
529        };
530        let x = self.read_spectrum_group(x_group)?;
531        let y = self.read_spectrum_group(y_group)?;
532        let mut x_out = empty_array(x_group.dtype);
533        let mut y_out = empty_array(y_group.dtype);
534        for position in 0..x.len().min(y.len()) {
535            let value = value_at(&x, position);
536            if value >= range.from && value <= range.to {
537                append_range(&mut x_out, &x, position, position + 1);
538                append_range(&mut y_out, &y, position, position + 1);
539            }
540        }
541        Ok(DataXY { x: x_out, y: y_out })
542    }
543
544    fn read_spectrum_group(&mut self, group: &ArrayGroup) -> IonResult<NumericArray> {
545        let mut values = empty_array(group.dtype);
546        for address in &group.refs {
547            let segment = self.read_spectrum_array(address)?;
548            append_range(&mut values, &segment, 0, segment.len());
549        }
550        Ok(values)
551    }
552
553    pub fn scans_in(&mut self, query: &ScanQuery, mut visit: impl FnMut(&Window)) -> IonResult<()> {
554        self.require_bounds()?;
555        let count = self.header.spectrum_count as usize;
556        let mut mz_out: Vec<f64> = Vec::new();
557        let mut intensity_out: Vec<f64> = Vec::new();
558        for index in 0..count {
559            let Some(record) = self.spectrum_summary(index) else {
560                continue;
561            };
562            if let Some(level) = query.ms_level
563                && record.ms_level != level
564            {
565                continue;
566            }
567            let summary = scan_summary_from_record(&record);
568            if !scan_is_selected(&query.select, &summary) {
569                continue;
570            }
571            let Some(refs) = self.spectrum_array_addresses(index) else {
572                continue;
573            };
574            let has_mz = refs
575                .iter()
576                .any(|r| r.array_type == crate::accessions::MZ_ARRAY);
577            let has_intensity = refs.iter().any(|r| r.array_type == ACC_INT);
578            if !has_mz || !has_intensity {
579                continue;
580            }
581            let data =
582                self.spectrum_window(index, ArrayKind::Mz, ArrayKind::Intensity, query.mz)?;
583            mz_out.clear();
584            data.x.extend_f64(&mut mz_out);
585            intensity_out.clear();
586            data.y.extend_f64(&mut intensity_out);
587            let window = Window {
588                index,
589                summary: &summary,
590                mz: &mz_out,
591                intensity: &intensity_out,
592            };
593            visit(&window);
594        }
595        Ok(())
596    }
597
598    #[cfg(test)]
599    pub(crate) fn candidate_items(
600        &mut self,
601        target: ItemKind,
602        axis_accession: u32,
603        lo: f64,
604        hi: f64,
605    ) -> IonResult<Vec<ItemSlice>> {
606        let is_axis = match target {
607            ItemKind::Spectrum => axis_accession == ACC_MZ,
608            ItemKind::Chromatogram => axis_accession == crate::accessions::TIME_ARRAY,
609        };
610        if !is_axis {
611            return Ok(Vec::new());
612        }
613
614        if !lo.is_finite() || !hi.is_finite() {
615            return Err("candidate_items: m/z range must be finite".into());
616        }
617        if lo > hi {
618            return Err("candidate_items: m/z range must be ordered".into());
619        }
620
621        match target {
622            ItemKind::Spectrum => {
623                let _ = self.ensure_spec_window_directory();
624            }
625            ItemKind::Chromatogram => self.ensure_chrom_window_directory(),
626        }
627
628        let width = self.header.target_mz_window as f64;
629        let window_directory = match target {
630            ItemKind::Spectrum => &self.spec_window_directory,
631            ItemKind::Chromatogram => &self.chrom_window_directory,
632        };
633        let WindowDirectoryCache::Loaded(window_directory) = window_directory else {
634            return Ok(Vec::new());
635        };
636
637        let Some((window_low, window_high)) =
638            window_span(width, lo, hi, window_directory.window_count())
639        else {
640            return Ok(Vec::new());
641        };
642
643        let mut result = Vec::new();
644        for window in window_low..=window_high {
645            for position in window_directory.window_range(window) {
646                let row = window_directory.row(position);
647                result.push(ItemSlice {
648                    item_index: row.spectrum_index as u64,
649                    array_address_index: row.mz_address as u64,
650                    intensity_address_index: row.intensity_address as u64,
651                });
652            }
653        }
654        Ok(result)
655    }
656}
657
658#[cfg(test)]
659mod tests {
660    use super::window_span;
661
662    #[test]
663    fn window_span_clamps_to_available_windows() {
664        assert_eq!(window_span(250.0, 260.0, 600.0, 4), Some((1, 2)));
665        assert_eq!(window_span(250.0, 0.0, 100000.0, 4), Some((0, 3)));
666        assert_eq!(window_span(250.0, 100000.0, 100000.0, 4), None);
667        assert_eq!(window_span(250.0, 0.0, 100.0, 0), None);
668    }
669}