Skip to main content

openreadout_plate/
lib.rs

1//! Reader for microplate-reader exports: the text, CSV and XLSX files that plate-reader
2//! software writes for spreadsheets.
3//!
4//! Dialects (docs/formats/plate-readers.md): Agilent BioTek Gen5, Molecular Devices SoftMax
5//! Pro (text exports and the binary `.pda`/`.sda` documents), BMG LABTECH MARS and SMART
6//! Control, PerkinElmer/Revvity EnVision and Kaleido, Tecan i-control and Magellan, Thermo
7//! SkanIt, and generic plate matrices. Provenance: docs/provenance/plate-readers.md.
8//! Every plate read is exposed as one long-form table
9//! (`well,row,col,read,wavelength_nm,time_s,value`), and [`export_asm`] writes a file as
10//! Allotrope Simple Model plate-reader JSON.
11//!
12//! # Example
13//!
14//! Every reader implements [`openreadout_core::FormatReader`]: `descriptor`
15//! says what it reads, `sniff` recognises a file from its first bytes, and `open` returns a
16//! [`openreadout_core::Dataset`] with a header-only `info()` summary and lazy data
17//! access.
18//!
19//! ```
20//! use openreadout_core::FormatReader;
21//! use openreadout_plate::PlateReader;
22//!
23//! let format = PlateReader.descriptor();
24//! assert_eq!(format.id, openreadout_plate::FORMAT_ID);
25//! println!("{} ({}): .{}", format.name, format.family, format.extensions.join(", ."));
26//! ```
27//!
28//! Reading a file:
29//!
30//! ```no_run
31//! use std::path::Path;
32//!
33//! use openreadout_core::FormatReader;
34//! use openreadout_plate::PlateReader;
35//!
36//! let mut dataset = PlateReader.open(Path::new("absorbance.xlsx"))?;
37//! let info = dataset.info()?; // headers only
38//! let table = &info.tables[0];
39//! println!("{} row(s) x {} column(s)", table.row_count, table.columns.len());
40//! // The first 100 rows, column-major (`columns[c][r]`).
41//! let rows = dataset.read_table(0, 0, 100)?;
42//! for (column, values) in table.columns.iter().zip(&rows.columns) {
43//!     println!("{}: {:?}", column.name, values.first());
44//! }
45//! # Ok::<(), openreadout_core::Error>(())
46//! ```
47//!
48//! Applications that accept any instrument file usually register every reader in an
49//! [`openreadout_core::Registry`] and let it detect the format; that is what the
50//! `openreadout` command-line tool does.
51//!
52//! # API stability
53//!
54//! The supported API is what this page documents. The parser modules are public only so that
55//! tests and fuzz targets can reach them: they are hidden from this documentation and may change
56//! in any release.
57#![forbid(unsafe_code)]
58#![warn(missing_docs)]
59
60mod asm;
61mod assurance;
62mod dataset;
63mod datetime;
64mod grid;
65mod model;
66mod sheet;
67mod text;
68mod vendors;
69
70use std::path::Path;
71
72use openreadout_core::model::{DetectConfidence, FormatDescriptor};
73use openreadout_core::reader::{Dataset, Detection, FormatReader, has_extension};
74use openreadout_core::source::{Fs, Input};
75use openreadout_core::{Error, Result};
76
77pub use asm::{ASM_MANIFEST, AsmExportReport, export_asm};
78pub use dataset::{PlateDataset, TABLE_COLUMNS};
79
80/// Format id used on the command line and in JSON.
81pub const FORMAT_ID: &str = "plate";
82
83/// Largest export read into memory (plate-reader exports are kilobytes to a few megabytes).
84const MAX_FILE_BYTES: u64 = 256 << 20;
85
86const TEXT_EXTENSIONS: [&str; 5] = ["txt", "csv", "tsv", "tab", "asc"];
87const BOOK_EXTENSIONS: [&str; 5] = ["xlsx", "xlsm", "xls", "xlsb", "ods"];
88/// SoftMax Pro binary documents (`.pda` SoftMax Pro 5; `.sda`/`.pda` SoftMax Pro 6/7).
89const BINARY_EXTENSIONS: [&str; 2] = ["pda", "sda"];
90/// Gen5 experiment (compound file) and protocol files.
91const GEN5_EXTENSIONS: [&str; 2] = ["xpt", "prt"];
92
93/// The plate-reader export reader.
94#[derive(Debug, Default, Clone, Copy)]
95pub struct PlateReader;
96
97/// Which dialect a text head looks like, if any.
98fn sniff_text(text: &str) -> Option<model::Kind> {
99    use model::Kind;
100    if vendors::softmax::sniff(text) {
101        Some(Kind::SoftMaxPro)
102    } else if vendors::gen5::sniff(text) {
103        Some(Kind::Gen5)
104    } else if vendors::bmg::sniff(text) {
105        Some(Kind::BmgMars)
106    } else if vendors::envision::sniff_envision(text) {
107        Some(Kind::EnVision)
108    } else if vendors::envision::sniff_kaleido(text) {
109        Some(Kind::Kaleido)
110    } else if vendors::tecan::sniff_icontrol(text) {
111        Some(Kind::TecanIControl)
112    } else if vendors::incucyte::sniff(text) {
113        Some(Kind::Incucyte)
114    } else {
115        None
116    }
117}
118
119fn sniff_book(book: &sheet::Book) -> Option<model::Kind> {
120    use model::Kind;
121    let first_text = |f: &dyn Fn(&str) -> bool| {
122        book.sheets.iter().any(|s| {
123            let head: String = (0..s.rows.len().min(80))
124                .map(|r| {
125                    s.rows[r]
126                        .iter()
127                        .map(sheet::Cell::text)
128                        .collect::<Vec<_>>()
129                        .join("\t")
130                })
131                .collect::<Vec<_>>()
132                .join("\n");
133            f(&head)
134        })
135    };
136    if vendors::tecan::is_icontrol_book(book) {
137        Some(Kind::TecanIControl)
138    } else if vendors::skanit::is_skanit_book(book) {
139        Some(Kind::SkanIt)
140    } else if vendors::tecan::is_magellan_book(book) {
141        Some(Kind::TecanMagellan)
142    } else if vendors::bmg::is_smart_control(book) {
143        Some(Kind::BmgSmartControl)
144    } else if first_text(&vendors::gen5::sniff) {
145        Some(Kind::Gen5)
146    } else if vendors::bmg::is_mars_book(book) {
147        Some(Kind::BmgMars)
148    } else if vendors::generic::sniff(book) {
149        Some(Kind::Generic)
150    } else {
151        None
152    }
153}
154
155/// Read and parse an export.
156pub(crate) fn load(fs: &Fs, path: &Path) -> Result<model::Export> {
157    let meta = fs.metadata(path).map_err(|e| Error::io(path, e))?;
158    if meta.len() > MAX_FILE_BYTES {
159        return Err(Error::unsupported(
160            FORMAT_ID,
161            format!(
162                "a {} MiB export (larger than {} MiB)",
163                meta.len() >> 20,
164                MAX_FILE_BYTES >> 20
165            ),
166            "Plate-reader exports are small; this file is probably not one. Split it or open it with the vendor software.",
167        ));
168    }
169    let bytes = fs.read(path).map_err(|e| Error::io(path, e))?;
170    if let Some(ex) = binary_document(path, &bytes)? {
171        return Ok(ex);
172    }
173    let file_name = path
174        .file_name()
175        .map_or_else(String::new, |n| n.to_string_lossy().into_owned());
176    let mut ex = if sheet::is_workbook(&bytes) {
177        let book = if fs.is_local() {
178            drop(bytes);
179            sheet::workbook_book(path)?
180        } else {
181            sheet::workbook_book_bytes(path, bytes)?
182        };
183        let kind = sniff_book(&book).ok_or_else(|| Error::UnknownFormat {
184            path: path.to_path_buf(),
185        })?;
186        let mut ex = parse_kind(kind, &book, &file_name);
187        note_unread_sheets(&book, &mut ex);
188        ex
189    } else {
190        let book = sheet::text_book(&bytes);
191        let (text, _) = text::decode(&bytes[..bytes.len().min(1 << 20)]);
192        let kind = sniff_text(&text)
193            .or_else(|| vendors::generic::sniff(&book).then_some(model::Kind::Generic))
194            .ok_or_else(|| Error::UnknownFormat {
195                path: path.to_path_buf(),
196            })?;
197        parse_kind(kind, &book, &file_name)
198    };
199    ex.finish();
200    if ex.blocks.iter().all(|b| b.obs.is_empty()) && ex.kind == model::Kind::Incucyte {
201        return Err(Error::unsupported(
202            FORMAT_ID,
203            "an Incucyte export without a column per well (group means, standard deviations or per-image values)",
204            "Export the metric with one column per well (Incucyte: Export Data, values per well) and read that file.",
205        ));
206    }
207    if ex.blocks.iter().all(|b| b.obs.is_empty()) && ex.kind == model::Kind::Generic {
208        return Err(Error::UnknownFormat {
209            path: path.to_path_buf(),
210        });
211    }
212    Ok(ex)
213}
214
215/// SoftMax Pro binary documents: decoded when their layout is one the corpus validates,
216/// otherwise refused (exit 6). `Ok(None)` for files that are not binary documents.
217fn binary_document(path: &Path, bytes: &[u8]) -> Result<Option<model::Export>> {
218    let gen5_hint = "Export the plate from Gen5 as text or Excel (File > Export, or a Power Export) and read that file.";
219    if vendors::gen5_xpt::sniff(bytes, has_extension(path, &["xpt"])) {
220        let mut ex = vendors::gen5_xpt::parse(bytes).map_err(|why| {
221            Error::unsupported(
222                FORMAT_ID,
223                format!("Gen5 experiment file that is not decoded ({why})"),
224                gen5_hint,
225            )
226        })?;
227        ex.finish();
228        if ex.blocks.iter().all(|b| b.obs.is_empty()) {
229            // say why: the reads that were refused and the reason for each
230            let why = ex
231                .sections
232                .get("refused_reads")
233                .and_then(serde_json::Value::as_array)
234                .map(|r| {
235                    r.iter()
236                        .filter_map(serde_json::Value::as_str)
237                        .collect::<Vec<_>>()
238                        .join("; ")
239                })
240                .filter(|w| !w.is_empty());
241            return Err(Error::unsupported(
242                FORMAT_ID,
243                match why {
244                    Some(w) => {
245                        format!("Gen5 experiment file whose plate data are not decoded ({w})")
246                    }
247                    None => "Gen5 experiment file whose plate data are not decoded".into(),
248                },
249                gen5_hint,
250            ));
251        }
252        return Ok(Some(ex));
253    }
254    if has_extension(path, &["prt"]) && vendors::gen5_xpt::is_protocol(bytes) {
255        return Err(Error::unsupported(
256            FORMAT_ID,
257            "Gen5 protocol file (it holds the procedure, no plate data)",
258            "Open the experiment (.xpt) run from this protocol, or its text/Excel export.",
259        ));
260    }
261    let hint = "Export the plate data from SoftMax Pro as text (File > Export, .txt, Plate or Time format) and read that file.";
262    let ex = if vendors::softmax_sda::sniff(bytes) {
263        vendors::softmax_sda::parse(bytes)
264    } else if vendors::softmax_pda::sniff(bytes) {
265        vendors::softmax_pda::parse(bytes)
266    } else if has_extension(path, &BINARY_EXTENSIONS) && !sheet::is_workbook(bytes) {
267        let (text, _) = text::decode(&bytes[..bytes.len().min(4096)]);
268        if vendors::softmax::sniff(&text) {
269            return Ok(None); // a text export saved with a binary extension
270        }
271        return Err(Error::unsupported(
272            FORMAT_ID,
273            "SoftMax Pro binary document of an unrecognised layout",
274            hint,
275        ));
276    } else {
277        return Ok(None);
278    };
279    let Some(mut ex) = ex else {
280        return Err(Error::unsupported(
281            FORMAT_ID,
282            "SoftMax Pro binary document of an unrecognised layout",
283            hint,
284        ));
285    };
286    ex.finish();
287    if ex.blocks.iter().all(|b| b.obs.is_empty()) {
288        let why = ex
289            .sections
290            .get("refused_plates")
291            .and_then(serde_json::Value::as_array)
292            .map(|a| {
293                a.iter()
294                    .filter_map(serde_json::Value::as_str)
295                    .collect::<Vec<_>>()
296                    .join("; ")
297            })
298            .filter(|s| !s.is_empty())
299            .unwrap_or_else(|| "no plate section holds data".into());
300        return Err(Error::unsupported(
301            FORMAT_ID,
302            format!("SoftMax Pro document whose plate data are not decoded ({why})"),
303            hint,
304        ));
305    }
306    Ok(Some(ex))
307}
308
309/// Report the non-blank worksheets of a workbook that the dialect did not read. A sheet that is
310/// an export on its own (any dialect but the generic matrix scan) holds plate data the file
311/// does not return; any other sheet (notes or calculations added to the workbook) is only named.
312fn note_unread_sheets(book: &sheet::Book, ex: &mut model::Export) {
313    use openreadout_core::model::Finding;
314    let read: std::collections::BTreeSet<&str> = ex
315        .sheets_read
316        .iter()
317        .map(String::as_str)
318        .chain(ex.blocks.iter().filter_map(|b| b.sheet.as_deref()))
319        .collect();
320    let unread: Vec<(String, Option<model::Kind>)> = book
321        .sheets
322        .iter()
323        .filter(|s| !read.contains(s.name.as_str()))
324        .filter(|s| (0..s.rows.len()).any(|r| !s.row_is_blank(r)))
325        .map(|s| {
326            let alone = sheet::Book {
327                sheets: vec![s.clone()],
328                container: book.container.clone(),
329            };
330            let kind = sniff_book(&alone).filter(|k| *k != model::Kind::Generic);
331            (s.name.clone(), kind)
332        })
333        .collect();
334    for (name, kind) in &unread {
335        match kind {
336            Some(k) => ex.findings.push(Finding::warning(
337                "worksheet_not_read",
338                format!(
339                    "worksheet {name:?} looks like a separate `{}` export and was not read; its values are not in the tables (save that sheet as its own file and read it)",
340                    k.id()
341                ),
342            )),
343            None => ex.findings.push(Finding::info(
344                "worksheet_not_read",
345                format!(
346                    "worksheet {name:?} holds no plate export this reader recognises and was not read (open it in a spreadsheet program if it matters)"
347                ),
348            )),
349        }
350    }
351    if !unread.is_empty() {
352        ex.notes.push(format!(
353            "worksheets not read: {} (`check` says why)",
354            unread
355                .iter()
356                .map(|(n, _)| format!("{n:?}"))
357                .collect::<Vec<_>>()
358                .join(", ")
359        ));
360    }
361    ex.unread_sheets = unread;
362}
363
364fn parse_kind(kind: model::Kind, book: &sheet::Book, file_name: &str) -> model::Export {
365    use model::Kind;
366    match kind {
367        Kind::Gen5 => vendors::gen5::parse(book, file_name),
368        Kind::SoftMaxPro => vendors::softmax::parse(book),
369        Kind::BmgMars => vendors::bmg::parse(book, false),
370        Kind::BmgSmartControl => vendors::bmg::parse(book, true),
371        Kind::EnVision => vendors::envision::parse_envision(book),
372        Kind::Kaleido => vendors::envision::parse_kaleido(book),
373        Kind::TecanIControl => vendors::tecan::parse_icontrol(book),
374        Kind::TecanMagellan => vendors::tecan::parse_magellan(book),
375        Kind::SkanIt => vendors::skanit::parse(book),
376        Kind::Incucyte => vendors::incucyte::parse(book),
377        Kind::Generic => vendors::generic::parse(book),
378    }
379}
380
381impl FormatReader for PlateReader {
382    fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
383        Some(&assurance::PLATE)
384    }
385
386    fn descriptor(&self) -> FormatDescriptor {
387        FormatDescriptor {
388            id: FORMAT_ID.into(),
389            name: "Plate-reader exports".into(),
390            vendor: "Agilent BioTek, Molecular Devices, BMG LABTECH, Revvity/PerkinElmer, Tecan, Thermo Fisher Scientific".into(),
391            extensions: TEXT_EXTENSIONS
392                .iter()
393                .chain(BOOK_EXTENSIONS.iter())
394                .chain(BINARY_EXTENSIONS.iter())
395                .chain(GEN5_EXTENSIONS.iter())
396                .map(|s| (*s).to_string())
397                .collect(),
398            family: "plate-reader".into(),
399            can_read: true,
400            can_write: false,
401            confidence: assurance::PLATE.confidence,
402            known_gaps: vec![
403                "SoftMax Pro binary documents: SoftMax Pro 5 .pda plate sections (absorbance, endpoint or kinetic, one wavelength, 96 wells; 4 documents from 2 labs validated value-for-value against their text exports) and SoftMax Pro 6/7 .sda endpoint plates (absorbance, fluorescence, luminescence; one wavelength; 6 documents validated) are decoded; kinetic, spectrum and well-scan sections of 6/7 documents, several wavelengths, other plate sizes and cuvette sets are refused (exit 6: export them as text)".into(),
404                "Gen5 experiment files (.xpt): endpoint and kinetic reads decoded from the plate archives (validated on Gen5 2.08, 3.04, 3.11 and 3.15 files from three labs against their exports); the detection mode comes from the read name (number, ex,em, Lum), other names are `unknown`; flagged cells (OVRFLW etc.) are NaN without their text; area scans, spectra, well scans and protocol files (.prt) are refused".into(),
405                "Gen5 area scans and multi-plate kinetic exports, SoftMax Pro well scans and fluorescence spectra are not decoded".into(),
406                "Values calculated by the vendor software are kept as `calculated` reads; curve fits and group tables are kept verbatim only (the plate-reader assays of `openreadout analyze` recompute standard curves, IC50s and kinetics from the reads). SoftMax Pro 5 document templates give the layout (samples, groups, `Blank` group as blanks); SoftMax Pro text-export group tables and SoftMax Pro 6/7 group sections are not used as layouts, and document-stored reduced values are not read".into(),
407                "Day/month order of slash dates is assumed month-first when ambiguous (flagged)".into(),
408            ],
409        }
410    }
411
412    fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
413        if vendors::softmax_sda::sniff_document(head)
414            || (vendors::softmax_sda::sniff(head) && has_extension(path, &BINARY_EXTENSIONS))
415        {
416            return Some(Detection {
417                format_id: FORMAT_ID,
418                confidence: DetectConfidence::Definite,
419                note: Some("SoftMax Pro 6/7 binary document".into()),
420            });
421        }
422        if vendors::gen5_xpt::sniff(head, has_extension(path, &["xpt"])) {
423            return Some(Detection {
424                format_id: FORMAT_ID,
425                confidence: DetectConfidence::Definite,
426                note: Some("Gen5 experiment file (compound file)".into()),
427            });
428        }
429        if has_extension(path, &["prt"]) && vendors::gen5_xpt::is_protocol(head) {
430            return Some(Detection {
431                format_id: FORMAT_ID,
432                confidence: DetectConfidence::Likely,
433                note: Some("Gen5 protocol file: holds no plate data (refused on open)".into()),
434            });
435        }
436        if vendors::softmax_pda::sniff(head) {
437            return Some(Detection {
438                format_id: FORMAT_ID,
439                confidence: DetectConfidence::Definite,
440                note: Some("SoftMax Pro 5 binary document".into()),
441            });
442        }
443        if has_extension(path, &BINARY_EXTENSIONS) && !head.is_empty() && head.contains(&0) {
444            return Some(Detection {
445                format_id: FORMAT_ID,
446                confidence: DetectConfidence::ExtensionOnly,
447                note: Some(
448                    "SoftMax Pro binary document of an unrecognised layout: refused on open".into(),
449                ),
450            });
451        }
452        if sheet::is_workbook(head) {
453            return has_extension(path, &BOOK_EXTENSIONS).then(|| Detection {
454                format_id: FORMAT_ID,
455                confidence: DetectConfidence::Likely,
456                note: Some(
457                    "spreadsheet workbook; the exporter is identified from its sheets on open"
458                        .into(),
459                ),
460            });
461        }
462        if head.is_empty()
463            || head.iter().take(4096).filter(|b| **b == 0).count() > 64
464                && !head.starts_with(&[0xFF, 0xFE])
465                && !head.starts_with(&[0xFE, 0xFF])
466        {
467            return None;
468        }
469        let (text, _) = text::decode(head);
470        if let Some(kind) = sniff_text(&text) {
471            return Some(Detection {
472                format_id: FORMAT_ID,
473                confidence: DetectConfidence::Definite,
474                note: Some(format!("{} export", kind.id())),
475            });
476        }
477        if has_extension(path, &TEXT_EXTENSIONS) && vendors::generic::sniff(&sheet::text_book(head))
478        {
479            return Some(Detection {
480                format_id: FORMAT_ID,
481                confidence: DetectConfidence::Likely,
482                note: Some("plate matrix (8×12, 16×24, …) with no known exporter signature".into()),
483            });
484        }
485        None
486    }
487
488    fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
489        Ok(Box::new(PlateDataset::open(path)?))
490    }
491
492    fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
493        Ok(Box::new(PlateDataset::open_input(input)?))
494    }
495
496    fn reads_any_source(&self) -> bool {
497        true
498    }
499}