Skip to main content

openreadout_arrow/
lib.rs

1//! Parquet and Arrow IPC export of the non-image data of any OpenReadout dataset.
2//!
3//! - **Tables** (FCS events, spike and event tables, plate reads, chromatography peak tables):
4//!   one column per table column in its stored type (`uint16`, `float32`, ...); plate `well`
5//!   columns become a dictionary of well names.
6//! - **Traces** (electrophysiology sweeps, NMR FIDs and spectra, chromatograms): one row per
7//!   sample, a `sweep` column, the abscissa (`time_s`, or the spectrum's axis such as
8//!   `chemical_shift_ppm`) and one `float64` column per channel in physical units.
9//! - **Spectra** (mass spectrometry): long form, one row per point
10//!   (`scan, rt_s, ms_level, precursor_mz, mz, intensity`), plus a per-scan summary written to a
11//!   second file (`<name>.scans.parquet`).
12//!
13//! Every column carries its unit (`unit`), label and the provenance of the normalized fields it
14//! comes from in its field metadata; the file carries the source format, the instrument and a
15//! JSON copy of `info` (keys in [`meta`]). The file is written under a temporary name, read back
16//! through the Parquet or Arrow IPC reader and compared column by column (schema, metadata and
17//! an xxh3 digest of every value), and only then renamed into place.
18//!
19//! Compression is pure Rust: Snappy (Parquet default) or LZ4; Arrow IPC files are uncompressed
20//! unless LZ4 is asked for.
21//!
22//! # Example
23//!
24//! ```
25//! use std::path::Path;
26//!
27//! use openreadout_arrow::{ColumnarFormat, ColumnarOptions, export_columnar};
28//! use openreadout_core::{Dataset, Result};
29//!
30//! /// Write table 0 (e.g. the events of an FCS file) as Parquet.
31//! fn to_parquet(dataset: &mut dyn Dataset, input: &Path, output: &Path) -> Result<()> {
32//!     let mut options = ColumnarOptions::default();
33//!     options.format = ColumnarFormat::Parquet;
34//!     let report = export_columnar(dataset, input, output, &options)?;
35//!     println!("{} rows, {} columns, verified={}", report.rows_written, report.columns.len(), report.verified);
36//!     Ok(())
37//! }
38//! ```
39#![forbid(unsafe_code)]
40#![warn(missing_docs)]
41
42pub mod meta;
43mod source;
44mod verify;
45
46use std::fs::File;
47use std::io::BufWriter;
48use std::path::{Path, PathBuf};
49
50pub use arrow_array;
51use arrow_array::RecordBatch;
52pub use arrow_schema;
53use arrow_schema::SchemaRef;
54use openreadout_core::model::FileInfo;
55use openreadout_core::{Dataset, Error, Result};
56use schemars::JsonSchema;
57use serde::{Deserialize, Serialize};
58
59use source::{BatchSource, SpectraSource, TableSource, TraceSource};
60use verify::Digest;
61
62/// The container written.
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
64#[non_exhaustive]
65pub enum ColumnarFormat {
66    /// Apache Parquet (`.parquet`).
67    #[default]
68    Parquet,
69    /// Arrow IPC file format, also known as Feather v2 (`.arrow`).
70    ArrowIpc,
71}
72
73impl ColumnarFormat {
74    /// The name used in reports and on the command line: `parquet` or `arrow`.
75    pub fn id(self) -> &'static str {
76        match self {
77            ColumnarFormat::Parquet => "parquet",
78            ColumnarFormat::ArrowIpc => "arrow",
79        }
80    }
81    /// File extension without the dot.
82    pub fn extension(self) -> &'static str {
83        self.id()
84    }
85}
86
87/// Column compression. Every codec here is implemented in pure Rust.
88#[derive(Debug, Clone, Copy, PartialEq, Eq)]
89#[non_exhaustive]
90pub enum ColumnarCompression {
91    /// No compression.
92    None,
93    /// Snappy (Parquet only; the Parquet default).
94    Snappy,
95    /// LZ4 (Parquet `LZ4_RAW`, Arrow IPC `LZ4_FRAME`).
96    Lz4,
97}
98
99impl ColumnarCompression {
100    /// Lowercase name for reports.
101    pub fn id(self) -> &'static str {
102        match self {
103            ColumnarCompression::None => "none",
104            ColumnarCompression::Snappy => "snappy",
105            ColumnarCompression::Lz4 => "lz4",
106        }
107    }
108}
109
110/// Which part of the file to export.
111#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
112#[non_exhaustive]
113pub enum ColumnarSelection {
114    /// The spectra of run 0 when the file holds spectra and no images, else table 0, else
115    /// trace 0.
116    #[default]
117    Auto,
118    /// Table `N` (see `info` → `tables[]`).
119    Table(u32),
120    /// Trace `N` (see `info` → `traces[]`).
121    Trace(u32),
122    /// The spectra of run `N` (see `info` → `spectra[]`).
123    Spectra(u32),
124}
125
126/// What to write.
127#[derive(Debug, Clone, Default)]
128#[non_exhaustive]
129pub struct ColumnarOptions {
130    /// Parquet or Arrow IPC.
131    pub format: ColumnarFormat,
132    /// `None`: Snappy for Parquet, no compression for Arrow IPC.
133    pub compression: Option<ColumnarCompression>,
134    /// The table, trace or spectra run.
135    pub select: ColumnarSelection,
136    /// Traces: only this sweep (default: every sweep, with a `sweep` column).
137    pub sweep: Option<u32>,
138    /// Tables: rows `[first, last]`; traces: samples `[first, last]` of each sweep (zero-based,
139    /// inclusive; `None` = to the end).
140    pub rows: Option<(u64, Option<u64>)>,
141    /// Spectra: the instrument's centroid lists instead of profiles where a scan has both.
142    pub centroid: bool,
143    /// Replace an existing output file.
144    pub overwrite: bool,
145}
146
147/// Output of a Parquet or Arrow IPC export.
148#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
149pub struct ColumnarExportReport {
150    /// The input file.
151    pub input: String,
152    /// The file written.
153    pub output: String,
154    /// `parquet` or `arrow`.
155    pub format: String,
156    /// What was exported: `table`, `trace` or `spectra`.
157    pub kind: String,
158    /// Table, trace or spectra-run index.
159    pub index: u32,
160    /// Rows written (table rows, trace samples over all sweeps, or spectrum points).
161    pub rows_written: u64,
162    /// Column names, in order.
163    pub columns: Vec<String>,
164    /// `none`, `snappy` or `lz4`.
165    pub compression: String,
166    /// Size of the written file in bytes.
167    pub bytes_written: u64,
168    /// True when the file was read back and its schema, metadata and every value matched.
169    pub verified: bool,
170    /// Traces: the sweeps written.
171    #[serde(default, skip_serializing_if = "Option::is_none")]
172    pub sweeps: Option<Vec<u32>>,
173    /// Tables and traces: the first row (sample of each sweep) written, zero-based.
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub first_row: Option<u64>,
176    /// Spectra: spectra (scans) written.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub spectra_written: Option<u64>,
179    /// Spectra: the per-scan summary file (`<name>.scans.parquet`).
180    #[serde(default, skip_serializing_if = "Option::is_none")]
181    pub summary_output: Option<String>,
182    /// Spectra: size of the summary file in bytes.
183    #[serde(default, skip_serializing_if = "Option::is_none")]
184    pub summary_bytes: Option<u64>,
185}
186
187/// The resolved export target.
188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189enum Target {
190    Table(u32),
191    Trace(u32),
192    Spectra(u32),
193}
194
195fn resolve(info: &FileInfo, sel: ColumnarSelection) -> Result<Target> {
196    let unsupported = |what: &str| {
197        Error::unsupported(
198            "export",
199            format!("{what} of a {} file", info.format.name),
200            "Parquet/Arrow export writes tables, traces and mass spectra; this file holds images: use `--format ome-tiff` or `--format ome-zarr`.",
201        )
202    };
203    Ok(match sel {
204        ColumnarSelection::Auto => {
205            if info.images.is_empty() && !info.spectra.is_empty() {
206                Target::Spectra(info.spectra[0].index)
207            } else if let Some(t) = info.tables.first() {
208                Target::Table(t.index)
209            } else if let Some(t) = info.traces.first() {
210                Target::Trace(t.index)
211            } else if !info.spectra.is_empty() {
212                Target::Spectra(info.spectra[0].index)
213            } else {
214                return Err(unsupported("Parquet/Arrow export"));
215            }
216        }
217        ColumnarSelection::Table(i) => {
218            if info.tables.is_empty() {
219                return Err(if info.traces.is_empty() && info.spectra.is_empty() {
220                    unsupported("table export")
221                } else {
222                    Error::Usage(format!(
223                        "this file has no tables; it holds {} (use {})",
224                        if info.traces.is_empty() {
225                            "spectra"
226                        } else {
227                            "traces"
228                        },
229                        if info.traces.is_empty() {
230                            "--spectra"
231                        } else {
232                            "--trace N"
233                        }
234                    ))
235                });
236            }
237            if !info.tables.iter().any(|t| t.index == i) {
238                return Err(Error::Usage(format!(
239                    "--table {i} out of range (file has {} tables)",
240                    info.tables.len()
241                )));
242            }
243            Target::Table(i)
244        }
245        ColumnarSelection::Trace(i) => {
246            if info.traces.is_empty() {
247                return Err(if info.tables.is_empty() && info.spectra.is_empty() {
248                    unsupported("trace export")
249                } else {
250                    Error::Usage(format!(
251                        "this file has no traces; it holds {} (use {})",
252                        if info.tables.is_empty() {
253                            "spectra"
254                        } else {
255                            "tables"
256                        },
257                        if info.tables.is_empty() {
258                            "--spectra"
259                        } else {
260                            "--table N"
261                        }
262                    ))
263                });
264            }
265            if !info.traces.iter().any(|t| t.index == i) {
266                return Err(Error::Usage(format!(
267                    "--trace {i} out of range (file has {} traces)",
268                    info.traces.len()
269                )));
270            }
271            Target::Trace(i)
272        }
273        ColumnarSelection::Spectra(r) => {
274            if info.spectra.is_empty() {
275                return Err(if info.images.is_empty() {
276                    Error::Usage(
277                        "this file holds no mass spectra; export a table (--table N) or a trace (--trace N)".into(),
278                    )
279                } else {
280                    unsupported("spectra export")
281                });
282            }
283            Target::Spectra(r)
284        }
285    })
286}
287
288/// Default output path next to `input` (or `base`, its mirror under a batch output directory):
289/// `<stem>.parquet`, `<stem>.table1.parquet`, `<stem>.trace2.sweep0.parquet`,
290/// `<stem>.run1.parquet`, ... (`.arrow` for Arrow IPC).
291pub fn default_output(base: &Path, info: &FileInfo, opts: &ColumnarOptions) -> PathBuf {
292    let stem = base
293        .file_stem()
294        .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string());
295    let ext = opts.format.extension();
296    let mid = match resolve(info, opts.select) {
297        Ok(Target::Table(t)) if t > 0 => format!(".table{t}"),
298        Ok(Target::Trace(t)) => match (t, opts.sweep) {
299            (0, None) => String::new(),
300            (0, Some(s)) => format!(".sweep{s}"),
301            (t, None) => format!(".trace{t}"),
302            (t, Some(s)) => format!(".trace{t}.sweep{s}"),
303        },
304        Ok(Target::Spectra(r)) if r > 0 => format!(".run{r}"),
305        _ => String::new(),
306    };
307    base.with_file_name(format!("{stem}{mid}.{ext}"))
308}
309
310/// The per-scan summary written beside a spectra export: `X.parquet` → `X.scans.parquet`.
311pub fn summary_output(output: &Path) -> PathBuf {
312    let name = output
313        .file_name()
314        .map_or_else(|| "export".into(), |n| n.to_string_lossy().to_string());
315    let (stem, ext) = match name.rsplit_once('.') {
316        Some((s, e)) if !s.is_empty() => (s.to_string(), format!(".{e}")),
317        _ => (name.clone(), String::new()),
318    };
319    output.with_file_name(format!("{stem}.scans{ext}"))
320}
321
322fn source_name(input: &Path) -> String {
323    input.file_name().map_or_else(
324        || input.display().to_string(),
325        |n| n.to_string_lossy().to_string(),
326    )
327}
328
329/// The batch source of one export.
330enum Src {
331    Table(TableSource),
332    Trace(TraceSource),
333    Spectra(Box<SpectraSource>),
334}
335
336impl Src {
337    fn get(&mut self) -> &mut dyn BatchSource {
338        match self {
339            Src::Table(s) => s,
340            Src::Trace(s) => s,
341            Src::Spectra(s) => s.as_mut(),
342        }
343    }
344    fn spectra(&self) -> Option<&SpectraSource> {
345        match self {
346            Src::Spectra(s) => Some(s),
347            _ => None,
348        }
349    }
350}
351
352/// Build the batch source of an export.
353fn build(ds: &mut dyn Dataset, input: &Path, opts: &ColumnarOptions) -> Result<(Src, Target)> {
354    let mut file_info = ds.info()?;
355    let prov = ds.provenance();
356    let name = source_name(input);
357    // The embedded `info` names the source by its file name, not the local path it was read from.
358    file_info.path.clone_from(&name);
359    let info = openreadout_core::experiment::InfoOutput::new(&*ds, file_info);
360    let target = resolve(&info, opts.select)?;
361    Ok(match target {
362        Target::Table(i) => {
363            if opts.sweep.is_some() {
364                return Err(Error::Usage(
365                    "--sweep applies to traces; a table is selected with --table and --rows".into(),
366                ));
367            }
368            let t = info
369                .tables
370                .iter()
371                .find(|t| t.index == i)
372                .cloned()
373                .unwrap_or_default();
374            let total = t.row_count;
375            let (first, last) = opts.rows.unwrap_or((0, None));
376            if first > total || (first == total && total > 0) {
377                return Err(Error::Usage(format!(
378                    "--rows starts at {first} but table {i} has {total} rows"
379                )));
380            }
381            let end = last.map_or(total, |l| l.saturating_add(1).min(total));
382            (
383                Src::Table(TableSource::new(&info, &prov, &t, (first, end), &name)),
384                target,
385            )
386        }
387        Target::Trace(i) => {
388            let t = info
389                .traces
390                .iter()
391                .find(|t| t.index == i)
392                .cloned()
393                .unwrap_or_default();
394            let sweeps: Vec<u32> = match opts.sweep {
395                Some(s) if s >= t.sweep_count => {
396                    return Err(Error::Usage(format!(
397                        "--sweep {s} out of range (trace {i} has {} sweeps)",
398                        t.sweep_count
399                    )));
400                }
401                Some(s) => vec![s],
402                None => (0..t.sweep_count).collect(),
403            };
404            let rows = opts.rows.unwrap_or((0, None));
405            (
406                Src::Trace(TraceSource::new(&info, &prov, &t, &sweeps, rows, &name)),
407                target,
408            )
409        }
410        Target::Spectra(r) => {
411            if opts.sweep.is_some() || opts.rows.is_some() {
412                return Err(Error::Usage(
413                    "--sweep and --rows do not apply to spectra; every scan of the run is written"
414                        .into(),
415                ));
416            }
417            let s = SpectraSource::new(&info, &prov, r, opts.centroid, &name)?;
418            (Src::Spectra(Box::new(s)), target)
419        }
420    })
421}
422
423enum Sink {
424    Parquet(parquet::arrow::ArrowWriter<BufWriter<File>>),
425    Ipc(arrow_ipc::writer::FileWriter<BufWriter<File>>),
426}
427
428impl Sink {
429    fn create(
430        path: &Path,
431        format: ColumnarFormat,
432        compression: ColumnarCompression,
433        schema: &SchemaRef,
434    ) -> Result<Self> {
435        let f = BufWriter::new(File::create(path).map_err(|e| Error::io(path, e))?);
436        let err = |e: String| Error::Other(format!("{}: {e}", path.display()));
437        Ok(match format {
438            ColumnarFormat::Parquet => {
439                use parquet::basic::Compression as C;
440                use parquet::file::metadata::KeyValue;
441                // Non-Arrow Parquet readers see the file-level metadata as key/value pairs too.
442                let kv: Vec<KeyValue> = schema
443                    .metadata()
444                    .iter()
445                    .map(|(k, v)| KeyValue::new(k.clone(), v.clone()))
446                    .collect();
447                let props = parquet::file::properties::WriterProperties::builder()
448                    .set_compression(match compression {
449                        ColumnarCompression::None => C::UNCOMPRESSED,
450                        ColumnarCompression::Snappy => C::SNAPPY,
451                        ColumnarCompression::Lz4 => C::LZ4_RAW,
452                    })
453                    .set_max_row_group_row_count(Some(1 << 20))
454                    .set_max_row_group_bytes(Some(128 << 20))
455                    .set_key_value_metadata(Some(kv))
456                    .build();
457                Sink::Parquet(
458                    parquet::arrow::ArrowWriter::try_new(f, schema.clone(), Some(props))
459                        .map_err(|e| err(e.to_string()))?,
460                )
461            }
462            ColumnarFormat::ArrowIpc => {
463                let c = match compression {
464                    ColumnarCompression::None => None,
465                    ColumnarCompression::Lz4 => Some(arrow_ipc::CompressionType::LZ4_FRAME),
466                    ColumnarCompression::Snappy => {
467                        return Err(Error::Usage(
468                            "Arrow IPC files support --compression none or lz4, not snappy".into(),
469                        ));
470                    }
471                };
472                let o = arrow_ipc::writer::IpcWriteOptions::default()
473                    .try_with_compression(c)
474                    .map_err(|e| err(e.to_string()))?;
475                Sink::Ipc(
476                    arrow_ipc::writer::FileWriter::try_new_with_options(f, schema, o)
477                        .map_err(|e| err(e.to_string()))?,
478                )
479            }
480        })
481    }
482
483    fn write(&mut self, b: &RecordBatch) -> Result<()> {
484        match self {
485            Sink::Parquet(w) => w.write(b).map_err(|e| Error::Other(e.to_string())),
486            Sink::Ipc(w) => w.write(b).map_err(|e| Error::Other(e.to_string())),
487        }
488    }
489
490    fn close(self) -> Result<()> {
491        use std::io::Write as _;
492        let mut inner = match self {
493            Sink::Parquet(w) => w.into_inner().map_err(|e| Error::Other(e.to_string()))?,
494            Sink::Ipc(w) => w.into_inner().map_err(|e| Error::Other(e.to_string()))?,
495        };
496        inner
497            .flush()
498            .map_err(|e| Error::Other(format!("flush: {e}")))?;
499        let f = inner
500            .into_inner()
501            .map_err(|e| Error::Other(format!("flush: {e}")))?;
502        f.sync_all().map_err(|e| Error::Other(format!("sync: {e}")))
503    }
504}
505
506fn temp_path(output: &Path) -> PathBuf {
507    let name = output
508        .file_name()
509        .map_or_else(|| "export".into(), |n| n.to_string_lossy().to_string());
510    output.with_file_name(format!(".{name}.partial-{}", std::process::id()))
511}
512
513/// Write `batches` (already hashed into `digest`) to `tmp`, then read it back and compare.
514fn write_one(
515    tmp: &Path,
516    format: ColumnarFormat,
517    compression: ColumnarCompression,
518    schema: &SchemaRef,
519    produce: &mut dyn FnMut(&mut Sink, &mut Digest) -> Result<()>,
520) -> Result<(Digest, u64)> {
521    let mut sink = Sink::create(tmp, format, compression, schema)?;
522    let mut digest = Digest::default();
523    produce(&mut sink, &mut digest)?;
524    sink.close()?;
525    verify::read_back(tmp, format, schema, &digest)?;
526    let bytes = std::fs::metadata(tmp).map_err(|e| Error::io(tmp, e))?.len();
527    Ok((digest, bytes))
528}
529
530/// How much an export with `opts` reads, for progress reports: the rows of a table, the
531/// samples of a trace over the selected sweeps, or the spectra of a run.
532pub fn export_size(ds: &mut dyn Dataset, input: &Path, opts: &ColumnarOptions) -> Result<u64> {
533    let (src, _) = build(ds, input, opts)?;
534    Ok(match src {
535        Src::Spectra(s) => s.spectrum_count(),
536        mut other => other.get().total_rows(),
537    })
538}
539
540/// Export one table, trace or spectra run of `ds` to `output` as Parquet or Arrow IPC. Spectra
541/// also write the per-scan summary to [`summary_output`]`(output)`. Everything is written under
542/// temporary names, read back and verified, and only then renamed into place.
543pub fn export_columnar(
544    ds: &mut dyn Dataset,
545    input: &Path,
546    output: &Path,
547    opts: &ColumnarOptions,
548) -> Result<ColumnarExportReport> {
549    let compression = opts.compression.unwrap_or(match opts.format {
550        ColumnarFormat::Parquet => ColumnarCompression::Snappy,
551        ColumnarFormat::ArrowIpc => ColumnarCompression::None,
552    });
553    openreadout_core::output::refuse_dataset(output, input, &*ds)?;
554    let (mut src, target) = build(ds, input, opts)?;
555    let summary = src.spectra().map(|_| summary_output(output));
556    for out in std::iter::once(output).chain(summary.as_deref()) {
557        openreadout_core::output::refuse_inputs(out, [input])?;
558        if out.exists() && !opts.overwrite {
559            return Err(Error::Usage(format!(
560                "{} exists; pass --overwrite to replace it",
561                out.display()
562            )));
563        }
564    }
565    let schema = src.get().schema();
566    let tmp = temp_path(output);
567    let tmp_summary = summary.as_deref().map(temp_path);
568    let cleanup = |paths: &[Option<&Path>]| {
569        if std::env::var_os("OPENREADOUT_KEEP_PARTIAL").is_none() {
570            for p in paths.iter().flatten() {
571                std::fs::remove_file(p).ok();
572            }
573        }
574    };
575    let mut summary_batch = None;
576    let result = (|| -> Result<(Digest, u64, Option<u64>)> {
577        let (digest, bytes) =
578            write_one(&tmp, opts.format, compression, &schema, &mut |sink, d| {
579                while let Some(b) = src.get().next_batch(ds)? {
580                    d.update(&b)?;
581                    sink.write(&b)?;
582                }
583                Ok(())
584            })?;
585        let mut summary_bytes = None;
586        if let Some(tmp_s) = &tmp_summary {
587            // The summary rows were gathered by the spectra source while it streamed.
588            let s = src
589                .spectra()
590                .ok_or_else(|| Error::Other("internal: not a spectra export".into()))?;
591            let batch = s.summary_batch()?;
592            let sschema = s.summary_schema();
593            let (_, b) = write_one(tmp_s, opts.format, compression, &sschema, &mut |sink, d| {
594                d.update(&batch)?;
595                sink.write(&batch)
596            })?;
597            summary_bytes = Some(b);
598            summary_batch = Some(s.spectra_read());
599        }
600        Ok((digest, bytes, summary_bytes))
601    })();
602    let (digest, bytes, summary_bytes) = match result {
603        Ok(v) => v,
604        Err(e) => {
605            cleanup(&[Some(&tmp), tmp_summary.as_deref()]);
606            return Err(e);
607        }
608    };
609    if let (Some(tmp_s), Some(out_s)) = (&tmp_summary, &summary) {
610        std::fs::rename(tmp_s, out_s).map_err(|e| Error::io(out_s, e))?;
611    }
612    std::fs::rename(&tmp, output).map_err(|e| Error::io(output, e))?;
613    let (kind, index) = match target {
614        Target::Table(i) => ("table", i),
615        Target::Trace(i) => ("trace", i),
616        Target::Spectra(i) => ("spectra", i),
617    };
618    let sweeps = match target {
619        Target::Trace(_) => {
620            let info = ds.info()?;
621            let t = info.traces.iter().find(|t| t.index == index);
622            Some(match opts.sweep {
623                Some(s) => vec![s],
624                None => (0..t.map_or(0, |t| t.sweep_count)).collect(),
625            })
626        }
627        _ => None,
628    };
629    Ok(ColumnarExportReport {
630        input: input.display().to_string(),
631        output: output.display().to_string(),
632        format: opts.format.id().into(),
633        kind: kind.into(),
634        index,
635        rows_written: digest.rows,
636        columns: schema.fields().iter().map(|f| f.name().clone()).collect(),
637        compression: compression.id().into(),
638        bytes_written: bytes,
639        verified: true,
640        sweeps,
641        first_row: match target {
642            Target::Spectra(_) => None,
643            _ => Some(opts.rows.map_or(0, |r| r.0)),
644        },
645        spectra_written: summary_batch,
646        summary_output: summary.map(|p| p.display().to_string()),
647        summary_bytes,
648    })
649}
650
651/// An export read into memory: the schema and its record batches (and, for spectra, the
652/// per-scan summary). For in-process consumers such as the Python bindings (`File.to_arrow()`).
653#[derive(Debug, Clone)]
654pub struct ColumnarData {
655    /// Schema with the same field and file metadata as the exported files.
656    pub schema: SchemaRef,
657    /// The rows, in batches of at most 65,536 rows (262,144 points for spectra).
658    pub batches: Vec<RecordBatch>,
659    /// Spectra only: the per-scan summary.
660    pub summary: Option<RecordBatch>,
661}
662
663/// Read one table, trace or spectra run into Arrow record batches, exactly as
664/// [`export_columnar`] would write them. `max_rows` caps the rows read (an error when the
665/// selection is larger: pass `rows`/`sweep` to narrow it).
666pub fn read_columnar(
667    ds: &mut dyn Dataset,
668    input: &Path,
669    opts: &ColumnarOptions,
670    max_rows: Option<u64>,
671) -> Result<ColumnarData> {
672    let (mut src, _) = build(ds, input, opts)?;
673    let is_spectra = src.spectra().is_some();
674    if let Some(cap) = max_rows
675        && !is_spectra
676        && src.get().total_rows() > cap
677    {
678        return Err(Error::Usage(format!(
679            "the selection has {} rows, more than the limit of {cap}; narrow it with rows or sweep",
680            src.get().total_rows()
681        )));
682    }
683    let mut batches = Vec::new();
684    let mut rows = 0u64;
685    while let Some(b) = src.get().next_batch(ds)? {
686        rows += b.num_rows() as u64;
687        if let Some(cap) = max_rows
688            && rows > cap
689        {
690            return Err(Error::Usage(format!(
691                "more than {cap} points; export to a file instead (`openreadout export --format parquet`)"
692            )));
693        }
694        batches.push(b);
695    }
696    let summary = match src.spectra() {
697        Some(s) => Some(s.summary_batch()?),
698        None => None,
699    };
700    Ok(ColumnarData {
701        schema: src.get().schema(),
702        batches,
703        summary,
704    })
705}
706
707/// Serialize record batches as an Arrow IPC *stream* (for consumers that take bytes, e.g.
708/// `pyarrow.ipc.open_stream`).
709pub fn ipc_stream_bytes(schema: &SchemaRef, batches: &[RecordBatch]) -> Result<Vec<u8>> {
710    let mut buf = Vec::new();
711    {
712        let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut buf, schema)
713            .map_err(|e| Error::Other(e.to_string()))?;
714        for b in batches {
715            w.write(b).map_err(|e| Error::Other(e.to_string()))?;
716        }
717        w.finish().map_err(|e| Error::Other(e.to_string()))?;
718    }
719    Ok(buf)
720}
721
722/// Read a Parquet or Arrow IPC file (e.g. one this crate wrote) into memory.
723pub fn read_file(path: &Path) -> Result<(SchemaRef, Vec<RecordBatch>)> {
724    let head = {
725        use std::io::Read as _;
726        let mut f = File::open(path).map_err(|e| Error::io(path, e))?;
727        let mut b = [0u8; 6];
728        f.read_exact(&mut b).map_err(|e| Error::io(path, e))?;
729        b
730    };
731    let open = || File::open(path).map_err(|e| Error::io(path, e));
732    let err = |e: String| Error::corrupt("parquet", format!("{}: {e}", path.display()));
733    if &head[..4] == b"PAR1" {
734        let b = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(open()?)
735            .map_err(|e| err(e.to_string()))?;
736        let schema = b.schema().clone();
737        let batches = b
738            .build()
739            .map_err(|e| err(e.to_string()))?
740            .collect::<std::result::Result<Vec<_>, _>>()
741            .map_err(|e| err(e.to_string()))?;
742        Ok((schema, batches))
743    } else if &head == b"ARROW1" {
744        let r = arrow_ipc::reader::FileReader::try_new_buffered(open()?, None)
745            .map_err(|e| err(e.to_string()))?;
746        let schema = r.schema();
747        let batches = r
748            .collect::<std::result::Result<Vec<_>, _>>()
749            .map_err(|e| err(e.to_string()))?;
750        Ok((schema, batches))
751    } else {
752        Err(Error::UnknownFormat {
753            path: path.to_path_buf(),
754        })
755    }
756}