1#![forbid(unsafe_code)]
40#![warn(missing_docs)]
41
42pub mod meta;
43mod source;
44mod verify;
45
46use std::fs::File;
47use std::io::BufWriter;
48use std::path::{Path, PathBuf};
49
50pub use arrow_array;
51use arrow_array::RecordBatch;
52pub use arrow_schema;
53use arrow_schema::SchemaRef;
54use openreadout_core::model::FileInfo;
55use openreadout_core::{Dataset, Error, Result};
56use schemars::JsonSchema;
57use serde::{Deserialize, Serialize};
58
59use source::{BatchSource, SpectraSource, TableSource, TraceSource};
60use verify::Digest;
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
64#[non_exhaustive]
65pub enum ColumnarFormat {
66 #[default]
68 Parquet,
69 ArrowIpc,
71}
72
73impl ColumnarFormat {
74 pub fn id(self) -> &'static str {
76 match self {
77 ColumnarFormat::Parquet => "parquet",
78 ColumnarFormat::ArrowIpc => "arrow",
79 }
80 }
81 pub fn extension(self) -> &'static str {
83 self.id()
84 }
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq)]
89#[non_exhaustive]
90pub enum ColumnarCompression {
91 None,
93 Snappy,
95 Lz4,
97}
98
99impl ColumnarCompression {
100 pub fn id(self) -> &'static str {
102 match self {
103 ColumnarCompression::None => "none",
104 ColumnarCompression::Snappy => "snappy",
105 ColumnarCompression::Lz4 => "lz4",
106 }
107 }
108}
109
110#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
112#[non_exhaustive]
113pub enum ColumnarSelection {
114 #[default]
117 Auto,
118 Table(u32),
120 Trace(u32),
122 Spectra(u32),
124}
125
126#[derive(Debug, Clone, Default)]
128#[non_exhaustive]
129pub struct ColumnarOptions {
130 pub format: ColumnarFormat,
132 pub compression: Option<ColumnarCompression>,
134 pub select: ColumnarSelection,
136 pub sweep: Option<u32>,
138 pub rows: Option<(u64, Option<u64>)>,
141 pub centroid: bool,
143 pub overwrite: bool,
145}
146
147#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
149pub struct ColumnarExportReport {
150 pub input: String,
152 pub output: String,
154 pub format: String,
156 pub kind: String,
158 pub index: u32,
160 pub rows_written: u64,
162 pub columns: Vec<String>,
164 pub compression: String,
166 pub bytes_written: u64,
168 pub verified: bool,
170 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub sweeps: Option<Vec<u32>>,
173 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub first_row: Option<u64>,
176 #[serde(default, skip_serializing_if = "Option::is_none")]
178 pub spectra_written: Option<u64>,
179 #[serde(default, skip_serializing_if = "Option::is_none")]
181 pub summary_output: Option<String>,
182 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub summary_bytes: Option<u64>,
185}
186
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189enum Target {
190 Table(u32),
191 Trace(u32),
192 Spectra(u32),
193}
194
195fn resolve(info: &FileInfo, sel: ColumnarSelection) -> Result<Target> {
196 let unsupported = |what: &str| {
197 Error::unsupported(
198 "export",
199 format!("{what} of a {} file", info.format.name),
200 "Parquet/Arrow export writes tables, traces and mass spectra; this file holds images: use `--format ome-tiff` or `--format ome-zarr`.",
201 )
202 };
203 Ok(match sel {
204 ColumnarSelection::Auto => {
205 if info.images.is_empty() && !info.spectra.is_empty() {
206 Target::Spectra(info.spectra[0].index)
207 } else if let Some(t) = info.tables.first() {
208 Target::Table(t.index)
209 } else if let Some(t) = info.traces.first() {
210 Target::Trace(t.index)
211 } else if !info.spectra.is_empty() {
212 Target::Spectra(info.spectra[0].index)
213 } else {
214 return Err(unsupported("Parquet/Arrow export"));
215 }
216 }
217 ColumnarSelection::Table(i) => {
218 if info.tables.is_empty() {
219 return Err(if info.traces.is_empty() && info.spectra.is_empty() {
220 unsupported("table export")
221 } else {
222 Error::Usage(format!(
223 "this file has no tables; it holds {} (use {})",
224 if info.traces.is_empty() {
225 "spectra"
226 } else {
227 "traces"
228 },
229 if info.traces.is_empty() {
230 "--spectra"
231 } else {
232 "--trace N"
233 }
234 ))
235 });
236 }
237 if !info.tables.iter().any(|t| t.index == i) {
238 return Err(Error::Usage(format!(
239 "--table {i} out of range (file has {} tables)",
240 info.tables.len()
241 )));
242 }
243 Target::Table(i)
244 }
245 ColumnarSelection::Trace(i) => {
246 if info.traces.is_empty() {
247 return Err(if info.tables.is_empty() && info.spectra.is_empty() {
248 unsupported("trace export")
249 } else {
250 Error::Usage(format!(
251 "this file has no traces; it holds {} (use {})",
252 if info.tables.is_empty() {
253 "spectra"
254 } else {
255 "tables"
256 },
257 if info.tables.is_empty() {
258 "--spectra"
259 } else {
260 "--table N"
261 }
262 ))
263 });
264 }
265 if !info.traces.iter().any(|t| t.index == i) {
266 return Err(Error::Usage(format!(
267 "--trace {i} out of range (file has {} traces)",
268 info.traces.len()
269 )));
270 }
271 Target::Trace(i)
272 }
273 ColumnarSelection::Spectra(r) => {
274 if info.spectra.is_empty() {
275 return Err(if info.images.is_empty() {
276 Error::Usage(
277 "this file holds no mass spectra; export a table (--table N) or a trace (--trace N)".into(),
278 )
279 } else {
280 unsupported("spectra export")
281 });
282 }
283 Target::Spectra(r)
284 }
285 })
286}
287
288pub fn default_output(base: &Path, info: &FileInfo, opts: &ColumnarOptions) -> PathBuf {
292 let stem = base
293 .file_stem()
294 .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string());
295 let ext = opts.format.extension();
296 let mid = match resolve(info, opts.select) {
297 Ok(Target::Table(t)) if t > 0 => format!(".table{t}"),
298 Ok(Target::Trace(t)) => match (t, opts.sweep) {
299 (0, None) => String::new(),
300 (0, Some(s)) => format!(".sweep{s}"),
301 (t, None) => format!(".trace{t}"),
302 (t, Some(s)) => format!(".trace{t}.sweep{s}"),
303 },
304 Ok(Target::Spectra(r)) if r > 0 => format!(".run{r}"),
305 _ => String::new(),
306 };
307 base.with_file_name(format!("{stem}{mid}.{ext}"))
308}
309
310pub fn summary_output(output: &Path) -> PathBuf {
312 let name = output
313 .file_name()
314 .map_or_else(|| "export".into(), |n| n.to_string_lossy().to_string());
315 let (stem, ext) = match name.rsplit_once('.') {
316 Some((s, e)) if !s.is_empty() => (s.to_string(), format!(".{e}")),
317 _ => (name.clone(), String::new()),
318 };
319 output.with_file_name(format!("{stem}.scans{ext}"))
320}
321
322fn source_name(input: &Path) -> String {
323 input.file_name().map_or_else(
324 || input.display().to_string(),
325 |n| n.to_string_lossy().to_string(),
326 )
327}
328
329enum Src {
331 Table(TableSource),
332 Trace(TraceSource),
333 Spectra(Box<SpectraSource>),
334}
335
336impl Src {
337 fn get(&mut self) -> &mut dyn BatchSource {
338 match self {
339 Src::Table(s) => s,
340 Src::Trace(s) => s,
341 Src::Spectra(s) => s.as_mut(),
342 }
343 }
344 fn spectra(&self) -> Option<&SpectraSource> {
345 match self {
346 Src::Spectra(s) => Some(s),
347 _ => None,
348 }
349 }
350}
351
352fn build(ds: &mut dyn Dataset, input: &Path, opts: &ColumnarOptions) -> Result<(Src, Target)> {
354 let mut file_info = ds.info()?;
355 let prov = ds.provenance();
356 let name = source_name(input);
357 file_info.path.clone_from(&name);
359 let info = openreadout_core::experiment::InfoOutput::new(&*ds, file_info);
360 let target = resolve(&info, opts.select)?;
361 Ok(match target {
362 Target::Table(i) => {
363 if opts.sweep.is_some() {
364 return Err(Error::Usage(
365 "--sweep applies to traces; a table is selected with --table and --rows".into(),
366 ));
367 }
368 let t = info
369 .tables
370 .iter()
371 .find(|t| t.index == i)
372 .cloned()
373 .unwrap_or_default();
374 let total = t.row_count;
375 let (first, last) = opts.rows.unwrap_or((0, None));
376 if first > total || (first == total && total > 0) {
377 return Err(Error::Usage(format!(
378 "--rows starts at {first} but table {i} has {total} rows"
379 )));
380 }
381 let end = last.map_or(total, |l| l.saturating_add(1).min(total));
382 (
383 Src::Table(TableSource::new(&info, &prov, &t, (first, end), &name)),
384 target,
385 )
386 }
387 Target::Trace(i) => {
388 let t = info
389 .traces
390 .iter()
391 .find(|t| t.index == i)
392 .cloned()
393 .unwrap_or_default();
394 let sweeps: Vec<u32> = match opts.sweep {
395 Some(s) if s >= t.sweep_count => {
396 return Err(Error::Usage(format!(
397 "--sweep {s} out of range (trace {i} has {} sweeps)",
398 t.sweep_count
399 )));
400 }
401 Some(s) => vec![s],
402 None => (0..t.sweep_count).collect(),
403 };
404 let rows = opts.rows.unwrap_or((0, None));
405 (
406 Src::Trace(TraceSource::new(&info, &prov, &t, &sweeps, rows, &name)),
407 target,
408 )
409 }
410 Target::Spectra(r) => {
411 if opts.sweep.is_some() || opts.rows.is_some() {
412 return Err(Error::Usage(
413 "--sweep and --rows do not apply to spectra; every scan of the run is written"
414 .into(),
415 ));
416 }
417 let s = SpectraSource::new(&info, &prov, r, opts.centroid, &name)?;
418 (Src::Spectra(Box::new(s)), target)
419 }
420 })
421}
422
423enum Sink {
424 Parquet(parquet::arrow::ArrowWriter<BufWriter<File>>),
425 Ipc(arrow_ipc::writer::FileWriter<BufWriter<File>>),
426}
427
428impl Sink {
429 fn create(
430 path: &Path,
431 format: ColumnarFormat,
432 compression: ColumnarCompression,
433 schema: &SchemaRef,
434 ) -> Result<Self> {
435 let f = BufWriter::new(File::create(path).map_err(|e| Error::io(path, e))?);
436 let err = |e: String| Error::Other(format!("{}: {e}", path.display()));
437 Ok(match format {
438 ColumnarFormat::Parquet => {
439 use parquet::basic::Compression as C;
440 use parquet::file::metadata::KeyValue;
441 let kv: Vec<KeyValue> = schema
443 .metadata()
444 .iter()
445 .map(|(k, v)| KeyValue::new(k.clone(), v.clone()))
446 .collect();
447 let props = parquet::file::properties::WriterProperties::builder()
448 .set_compression(match compression {
449 ColumnarCompression::None => C::UNCOMPRESSED,
450 ColumnarCompression::Snappy => C::SNAPPY,
451 ColumnarCompression::Lz4 => C::LZ4_RAW,
452 })
453 .set_max_row_group_row_count(Some(1 << 20))
454 .set_max_row_group_bytes(Some(128 << 20))
455 .set_key_value_metadata(Some(kv))
456 .build();
457 Sink::Parquet(
458 parquet::arrow::ArrowWriter::try_new(f, schema.clone(), Some(props))
459 .map_err(|e| err(e.to_string()))?,
460 )
461 }
462 ColumnarFormat::ArrowIpc => {
463 let c = match compression {
464 ColumnarCompression::None => None,
465 ColumnarCompression::Lz4 => Some(arrow_ipc::CompressionType::LZ4_FRAME),
466 ColumnarCompression::Snappy => {
467 return Err(Error::Usage(
468 "Arrow IPC files support --compression none or lz4, not snappy".into(),
469 ));
470 }
471 };
472 let o = arrow_ipc::writer::IpcWriteOptions::default()
473 .try_with_compression(c)
474 .map_err(|e| err(e.to_string()))?;
475 Sink::Ipc(
476 arrow_ipc::writer::FileWriter::try_new_with_options(f, schema, o)
477 .map_err(|e| err(e.to_string()))?,
478 )
479 }
480 })
481 }
482
483 fn write(&mut self, b: &RecordBatch) -> Result<()> {
484 match self {
485 Sink::Parquet(w) => w.write(b).map_err(|e| Error::Other(e.to_string())),
486 Sink::Ipc(w) => w.write(b).map_err(|e| Error::Other(e.to_string())),
487 }
488 }
489
490 fn close(self) -> Result<()> {
491 use std::io::Write as _;
492 let mut inner = match self {
493 Sink::Parquet(w) => w.into_inner().map_err(|e| Error::Other(e.to_string()))?,
494 Sink::Ipc(w) => w.into_inner().map_err(|e| Error::Other(e.to_string()))?,
495 };
496 inner
497 .flush()
498 .map_err(|e| Error::Other(format!("flush: {e}")))?;
499 let f = inner
500 .into_inner()
501 .map_err(|e| Error::Other(format!("flush: {e}")))?;
502 f.sync_all().map_err(|e| Error::Other(format!("sync: {e}")))
503 }
504}
505
506fn temp_path(output: &Path) -> PathBuf {
507 let name = output
508 .file_name()
509 .map_or_else(|| "export".into(), |n| n.to_string_lossy().to_string());
510 output.with_file_name(format!(".{name}.partial-{}", std::process::id()))
511}
512
513fn write_one(
515 tmp: &Path,
516 format: ColumnarFormat,
517 compression: ColumnarCompression,
518 schema: &SchemaRef,
519 produce: &mut dyn FnMut(&mut Sink, &mut Digest) -> Result<()>,
520) -> Result<(Digest, u64)> {
521 let mut sink = Sink::create(tmp, format, compression, schema)?;
522 let mut digest = Digest::default();
523 produce(&mut sink, &mut digest)?;
524 sink.close()?;
525 verify::read_back(tmp, format, schema, &digest)?;
526 let bytes = std::fs::metadata(tmp).map_err(|e| Error::io(tmp, e))?.len();
527 Ok((digest, bytes))
528}
529
530pub fn export_size(ds: &mut dyn Dataset, input: &Path, opts: &ColumnarOptions) -> Result<u64> {
533 let (src, _) = build(ds, input, opts)?;
534 Ok(match src {
535 Src::Spectra(s) => s.spectrum_count(),
536 mut other => other.get().total_rows(),
537 })
538}
539
540pub fn export_columnar(
544 ds: &mut dyn Dataset,
545 input: &Path,
546 output: &Path,
547 opts: &ColumnarOptions,
548) -> Result<ColumnarExportReport> {
549 let compression = opts.compression.unwrap_or(match opts.format {
550 ColumnarFormat::Parquet => ColumnarCompression::Snappy,
551 ColumnarFormat::ArrowIpc => ColumnarCompression::None,
552 });
553 openreadout_core::output::refuse_dataset(output, input, &*ds)?;
554 let (mut src, target) = build(ds, input, opts)?;
555 let summary = src.spectra().map(|_| summary_output(output));
556 for out in std::iter::once(output).chain(summary.as_deref()) {
557 openreadout_core::output::refuse_inputs(out, [input])?;
558 if out.exists() && !opts.overwrite {
559 return Err(Error::Usage(format!(
560 "{} exists; pass --overwrite to replace it",
561 out.display()
562 )));
563 }
564 }
565 let schema = src.get().schema();
566 let tmp = temp_path(output);
567 let tmp_summary = summary.as_deref().map(temp_path);
568 let cleanup = |paths: &[Option<&Path>]| {
569 if std::env::var_os("OPENREADOUT_KEEP_PARTIAL").is_none() {
570 for p in paths.iter().flatten() {
571 std::fs::remove_file(p).ok();
572 }
573 }
574 };
575 let mut summary_batch = None;
576 let result = (|| -> Result<(Digest, u64, Option<u64>)> {
577 let (digest, bytes) =
578 write_one(&tmp, opts.format, compression, &schema, &mut |sink, d| {
579 while let Some(b) = src.get().next_batch(ds)? {
580 d.update(&b)?;
581 sink.write(&b)?;
582 }
583 Ok(())
584 })?;
585 let mut summary_bytes = None;
586 if let Some(tmp_s) = &tmp_summary {
587 let s = src
589 .spectra()
590 .ok_or_else(|| Error::Other("internal: not a spectra export".into()))?;
591 let batch = s.summary_batch()?;
592 let sschema = s.summary_schema();
593 let (_, b) = write_one(tmp_s, opts.format, compression, &sschema, &mut |sink, d| {
594 d.update(&batch)?;
595 sink.write(&batch)
596 })?;
597 summary_bytes = Some(b);
598 summary_batch = Some(s.spectra_read());
599 }
600 Ok((digest, bytes, summary_bytes))
601 })();
602 let (digest, bytes, summary_bytes) = match result {
603 Ok(v) => v,
604 Err(e) => {
605 cleanup(&[Some(&tmp), tmp_summary.as_deref()]);
606 return Err(e);
607 }
608 };
609 if let (Some(tmp_s), Some(out_s)) = (&tmp_summary, &summary) {
610 std::fs::rename(tmp_s, out_s).map_err(|e| Error::io(out_s, e))?;
611 }
612 std::fs::rename(&tmp, output).map_err(|e| Error::io(output, e))?;
613 let (kind, index) = match target {
614 Target::Table(i) => ("table", i),
615 Target::Trace(i) => ("trace", i),
616 Target::Spectra(i) => ("spectra", i),
617 };
618 let sweeps = match target {
619 Target::Trace(_) => {
620 let info = ds.info()?;
621 let t = info.traces.iter().find(|t| t.index == index);
622 Some(match opts.sweep {
623 Some(s) => vec![s],
624 None => (0..t.map_or(0, |t| t.sweep_count)).collect(),
625 })
626 }
627 _ => None,
628 };
629 Ok(ColumnarExportReport {
630 input: input.display().to_string(),
631 output: output.display().to_string(),
632 format: opts.format.id().into(),
633 kind: kind.into(),
634 index,
635 rows_written: digest.rows,
636 columns: schema.fields().iter().map(|f| f.name().clone()).collect(),
637 compression: compression.id().into(),
638 bytes_written: bytes,
639 verified: true,
640 sweeps,
641 first_row: match target {
642 Target::Spectra(_) => None,
643 _ => Some(opts.rows.map_or(0, |r| r.0)),
644 },
645 spectra_written: summary_batch,
646 summary_output: summary.map(|p| p.display().to_string()),
647 summary_bytes,
648 })
649}
650
651#[derive(Debug, Clone)]
654pub struct ColumnarData {
655 pub schema: SchemaRef,
657 pub batches: Vec<RecordBatch>,
659 pub summary: Option<RecordBatch>,
661}
662
663pub fn read_columnar(
667 ds: &mut dyn Dataset,
668 input: &Path,
669 opts: &ColumnarOptions,
670 max_rows: Option<u64>,
671) -> Result<ColumnarData> {
672 let (mut src, _) = build(ds, input, opts)?;
673 let is_spectra = src.spectra().is_some();
674 if let Some(cap) = max_rows
675 && !is_spectra
676 && src.get().total_rows() > cap
677 {
678 return Err(Error::Usage(format!(
679 "the selection has {} rows, more than the limit of {cap}; narrow it with rows or sweep",
680 src.get().total_rows()
681 )));
682 }
683 let mut batches = Vec::new();
684 let mut rows = 0u64;
685 while let Some(b) = src.get().next_batch(ds)? {
686 rows += b.num_rows() as u64;
687 if let Some(cap) = max_rows
688 && rows > cap
689 {
690 return Err(Error::Usage(format!(
691 "more than {cap} points; export to a file instead (`openreadout export --format parquet`)"
692 )));
693 }
694 batches.push(b);
695 }
696 let summary = match src.spectra() {
697 Some(s) => Some(s.summary_batch()?),
698 None => None,
699 };
700 Ok(ColumnarData {
701 schema: src.get().schema(),
702 batches,
703 summary,
704 })
705}
706
707pub fn ipc_stream_bytes(schema: &SchemaRef, batches: &[RecordBatch]) -> Result<Vec<u8>> {
710 let mut buf = Vec::new();
711 {
712 let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut buf, schema)
713 .map_err(|e| Error::Other(e.to_string()))?;
714 for b in batches {
715 w.write(b).map_err(|e| Error::Other(e.to_string()))?;
716 }
717 w.finish().map_err(|e| Error::Other(e.to_string()))?;
718 }
719 Ok(buf)
720}
721
722pub fn read_file(path: &Path) -> Result<(SchemaRef, Vec<RecordBatch>)> {
724 let head = {
725 use std::io::Read as _;
726 let mut f = File::open(path).map_err(|e| Error::io(path, e))?;
727 let mut b = [0u8; 6];
728 f.read_exact(&mut b).map_err(|e| Error::io(path, e))?;
729 b
730 };
731 let open = || File::open(path).map_err(|e| Error::io(path, e));
732 let err = |e: String| Error::corrupt("parquet", format!("{}: {e}", path.display()));
733 if &head[..4] == b"PAR1" {
734 let b = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(open()?)
735 .map_err(|e| err(e.to_string()))?;
736 let schema = b.schema().clone();
737 let batches = b
738 .build()
739 .map_err(|e| err(e.to_string()))?
740 .collect::<std::result::Result<Vec<_>, _>>()
741 .map_err(|e| err(e.to_string()))?;
742 Ok((schema, batches))
743 } else if &head == b"ARROW1" {
744 let r = arrow_ipc::reader::FileReader::try_new_buffered(open()?, None)
745 .map_err(|e| err(e.to_string()))?;
746 let schema = r.schema();
747 let batches = r
748 .collect::<std::result::Result<Vec<_>, _>>()
749 .map_err(|e| err(e.to_string()))?;
750 Ok((schema, batches))
751 } else {
752 Err(Error::UnknownFormat {
753 path: path.to_path_buf(),
754 })
755 }
756}