1#![forbid(unsafe_code)]
58#![warn(missing_docs)]
59
60mod asm;
61mod assurance;
62mod dataset;
63mod datetime;
64mod grid;
65mod model;
66mod sheet;
67mod text;
68mod vendors;
69
70use std::path::Path;
71
72use openreadout_core::model::{DetectConfidence, FormatDescriptor};
73use openreadout_core::reader::{Dataset, Detection, FormatReader, has_extension};
74use openreadout_core::source::{Fs, Input};
75use openreadout_core::{Error, Result};
76
77pub use asm::{ASM_MANIFEST, AsmExportReport, export_asm};
78pub use dataset::{PlateDataset, TABLE_COLUMNS};
79
80pub const FORMAT_ID: &str = "plate";
82
83const MAX_FILE_BYTES: u64 = 256 << 20;
85
86const TEXT_EXTENSIONS: [&str; 5] = ["txt", "csv", "tsv", "tab", "asc"];
87const BOOK_EXTENSIONS: [&str; 5] = ["xlsx", "xlsm", "xls", "xlsb", "ods"];
88const BINARY_EXTENSIONS: [&str; 2] = ["pda", "sda"];
90const GEN5_EXTENSIONS: [&str; 2] = ["xpt", "prt"];
92
93#[derive(Debug, Default, Clone, Copy)]
95pub struct PlateReader;
96
97fn sniff_text(text: &str) -> Option<model::Kind> {
99 use model::Kind;
100 if vendors::softmax::sniff(text) {
101 Some(Kind::SoftMaxPro)
102 } else if vendors::gen5::sniff(text) {
103 Some(Kind::Gen5)
104 } else if vendors::bmg::sniff(text) {
105 Some(Kind::BmgMars)
106 } else if vendors::envision::sniff_envision(text) {
107 Some(Kind::EnVision)
108 } else if vendors::envision::sniff_kaleido(text) {
109 Some(Kind::Kaleido)
110 } else if vendors::tecan::sniff_icontrol(text) {
111 Some(Kind::TecanIControl)
112 } else if vendors::incucyte::sniff(text) {
113 Some(Kind::Incucyte)
114 } else {
115 None
116 }
117}
118
119fn sniff_book(book: &sheet::Book) -> Option<model::Kind> {
120 use model::Kind;
121 let first_text = |f: &dyn Fn(&str) -> bool| {
122 book.sheets.iter().any(|s| {
123 let head: String = (0..s.rows.len().min(80))
124 .map(|r| {
125 s.rows[r]
126 .iter()
127 .map(sheet::Cell::text)
128 .collect::<Vec<_>>()
129 .join("\t")
130 })
131 .collect::<Vec<_>>()
132 .join("\n");
133 f(&head)
134 })
135 };
136 if vendors::tecan::is_icontrol_book(book) {
137 Some(Kind::TecanIControl)
138 } else if vendors::skanit::is_skanit_book(book) {
139 Some(Kind::SkanIt)
140 } else if vendors::tecan::is_magellan_book(book) {
141 Some(Kind::TecanMagellan)
142 } else if vendors::bmg::is_smart_control(book) {
143 Some(Kind::BmgSmartControl)
144 } else if first_text(&vendors::gen5::sniff) {
145 Some(Kind::Gen5)
146 } else if vendors::bmg::is_mars_book(book) {
147 Some(Kind::BmgMars)
148 } else if vendors::generic::sniff(book) {
149 Some(Kind::Generic)
150 } else {
151 None
152 }
153}
154
155pub(crate) fn load(fs: &Fs, path: &Path) -> Result<model::Export> {
157 let meta = fs.metadata(path).map_err(|e| Error::io(path, e))?;
158 if meta.len() > MAX_FILE_BYTES {
159 return Err(Error::unsupported(
160 FORMAT_ID,
161 format!(
162 "a {} MiB export (larger than {} MiB)",
163 meta.len() >> 20,
164 MAX_FILE_BYTES >> 20
165 ),
166 "Plate-reader exports are small; this file is probably not one. Split it or open it with the vendor software.",
167 ));
168 }
169 let bytes = fs.read(path).map_err(|e| Error::io(path, e))?;
170 if let Some(ex) = binary_document(path, &bytes)? {
171 return Ok(ex);
172 }
173 let file_name = path
174 .file_name()
175 .map_or_else(String::new, |n| n.to_string_lossy().into_owned());
176 let mut ex = if sheet::is_workbook(&bytes) {
177 let book = if fs.is_local() {
178 drop(bytes);
179 sheet::workbook_book(path)?
180 } else {
181 sheet::workbook_book_bytes(path, bytes)?
182 };
183 let kind = sniff_book(&book).ok_or_else(|| Error::UnknownFormat {
184 path: path.to_path_buf(),
185 })?;
186 let mut ex = parse_kind(kind, &book, &file_name);
187 note_unread_sheets(&book, &mut ex);
188 ex
189 } else {
190 let book = sheet::text_book(&bytes);
191 let (text, _) = text::decode(&bytes[..bytes.len().min(1 << 20)]);
192 let kind = sniff_text(&text)
193 .or_else(|| vendors::generic::sniff(&book).then_some(model::Kind::Generic))
194 .ok_or_else(|| Error::UnknownFormat {
195 path: path.to_path_buf(),
196 })?;
197 parse_kind(kind, &book, &file_name)
198 };
199 ex.finish();
200 if ex.blocks.iter().all(|b| b.obs.is_empty()) && ex.kind == model::Kind::Incucyte {
201 return Err(Error::unsupported(
202 FORMAT_ID,
203 "an Incucyte export without a column per well (group means, standard deviations or per-image values)",
204 "Export the metric with one column per well (Incucyte: Export Data, values per well) and read that file.",
205 ));
206 }
207 if ex.blocks.iter().all(|b| b.obs.is_empty()) && ex.kind == model::Kind::Generic {
208 return Err(Error::UnknownFormat {
209 path: path.to_path_buf(),
210 });
211 }
212 Ok(ex)
213}
214
215fn binary_document(path: &Path, bytes: &[u8]) -> Result<Option<model::Export>> {
218 let gen5_hint = "Export the plate from Gen5 as text or Excel (File > Export, or a Power Export) and read that file.";
219 if vendors::gen5_xpt::sniff(bytes, has_extension(path, &["xpt"])) {
220 let mut ex = vendors::gen5_xpt::parse(bytes).map_err(|why| {
221 Error::unsupported(
222 FORMAT_ID,
223 format!("Gen5 experiment file that is not decoded ({why})"),
224 gen5_hint,
225 )
226 })?;
227 ex.finish();
228 if ex.blocks.iter().all(|b| b.obs.is_empty()) {
229 let why = ex
231 .sections
232 .get("refused_reads")
233 .and_then(serde_json::Value::as_array)
234 .map(|r| {
235 r.iter()
236 .filter_map(serde_json::Value::as_str)
237 .collect::<Vec<_>>()
238 .join("; ")
239 })
240 .filter(|w| !w.is_empty());
241 return Err(Error::unsupported(
242 FORMAT_ID,
243 match why {
244 Some(w) => {
245 format!("Gen5 experiment file whose plate data are not decoded ({w})")
246 }
247 None => "Gen5 experiment file whose plate data are not decoded".into(),
248 },
249 gen5_hint,
250 ));
251 }
252 return Ok(Some(ex));
253 }
254 if has_extension(path, &["prt"]) && vendors::gen5_xpt::is_protocol(bytes) {
255 return Err(Error::unsupported(
256 FORMAT_ID,
257 "Gen5 protocol file (it holds the procedure, no plate data)",
258 "Open the experiment (.xpt) run from this protocol, or its text/Excel export.",
259 ));
260 }
261 let hint = "Export the plate data from SoftMax Pro as text (File > Export, .txt, Plate or Time format) and read that file.";
262 let ex = if vendors::softmax_sda::sniff(bytes) {
263 vendors::softmax_sda::parse(bytes)
264 } else if vendors::softmax_pda::sniff(bytes) {
265 vendors::softmax_pda::parse(bytes)
266 } else if has_extension(path, &BINARY_EXTENSIONS) && !sheet::is_workbook(bytes) {
267 let (text, _) = text::decode(&bytes[..bytes.len().min(4096)]);
268 if vendors::softmax::sniff(&text) {
269 return Ok(None); }
271 return Err(Error::unsupported(
272 FORMAT_ID,
273 "SoftMax Pro binary document of an unrecognised layout",
274 hint,
275 ));
276 } else {
277 return Ok(None);
278 };
279 let Some(mut ex) = ex else {
280 return Err(Error::unsupported(
281 FORMAT_ID,
282 "SoftMax Pro binary document of an unrecognised layout",
283 hint,
284 ));
285 };
286 ex.finish();
287 if ex.blocks.iter().all(|b| b.obs.is_empty()) {
288 let why = ex
289 .sections
290 .get("refused_plates")
291 .and_then(serde_json::Value::as_array)
292 .map(|a| {
293 a.iter()
294 .filter_map(serde_json::Value::as_str)
295 .collect::<Vec<_>>()
296 .join("; ")
297 })
298 .filter(|s| !s.is_empty())
299 .unwrap_or_else(|| "no plate section holds data".into());
300 return Err(Error::unsupported(
301 FORMAT_ID,
302 format!("SoftMax Pro document whose plate data are not decoded ({why})"),
303 hint,
304 ));
305 }
306 Ok(Some(ex))
307}
308
309fn note_unread_sheets(book: &sheet::Book, ex: &mut model::Export) {
313 use openreadout_core::model::Finding;
314 let read: std::collections::BTreeSet<&str> = ex
315 .sheets_read
316 .iter()
317 .map(String::as_str)
318 .chain(ex.blocks.iter().filter_map(|b| b.sheet.as_deref()))
319 .collect();
320 let unread: Vec<(String, Option<model::Kind>)> = book
321 .sheets
322 .iter()
323 .filter(|s| !read.contains(s.name.as_str()))
324 .filter(|s| (0..s.rows.len()).any(|r| !s.row_is_blank(r)))
325 .map(|s| {
326 let alone = sheet::Book {
327 sheets: vec![s.clone()],
328 container: book.container.clone(),
329 };
330 let kind = sniff_book(&alone).filter(|k| *k != model::Kind::Generic);
331 (s.name.clone(), kind)
332 })
333 .collect();
334 for (name, kind) in &unread {
335 match kind {
336 Some(k) => ex.findings.push(Finding::warning(
337 "worksheet_not_read",
338 format!(
339 "worksheet {name:?} looks like a separate `{}` export and was not read; its values are not in the tables (save that sheet as its own file and read it)",
340 k.id()
341 ),
342 )),
343 None => ex.findings.push(Finding::info(
344 "worksheet_not_read",
345 format!(
346 "worksheet {name:?} holds no plate export this reader recognises and was not read (open it in a spreadsheet program if it matters)"
347 ),
348 )),
349 }
350 }
351 if !unread.is_empty() {
352 ex.notes.push(format!(
353 "worksheets not read: {} (`check` says why)",
354 unread
355 .iter()
356 .map(|(n, _)| format!("{n:?}"))
357 .collect::<Vec<_>>()
358 .join(", ")
359 ));
360 }
361 ex.unread_sheets = unread;
362}
363
364fn parse_kind(kind: model::Kind, book: &sheet::Book, file_name: &str) -> model::Export {
365 use model::Kind;
366 match kind {
367 Kind::Gen5 => vendors::gen5::parse(book, file_name),
368 Kind::SoftMaxPro => vendors::softmax::parse(book),
369 Kind::BmgMars => vendors::bmg::parse(book, false),
370 Kind::BmgSmartControl => vendors::bmg::parse(book, true),
371 Kind::EnVision => vendors::envision::parse_envision(book),
372 Kind::Kaleido => vendors::envision::parse_kaleido(book),
373 Kind::TecanIControl => vendors::tecan::parse_icontrol(book),
374 Kind::TecanMagellan => vendors::tecan::parse_magellan(book),
375 Kind::SkanIt => vendors::skanit::parse(book),
376 Kind::Incucyte => vendors::incucyte::parse(book),
377 Kind::Generic => vendors::generic::parse(book),
378 }
379}
380
381impl FormatReader for PlateReader {
382 fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
383 Some(&assurance::PLATE)
384 }
385
386 fn descriptor(&self) -> FormatDescriptor {
387 FormatDescriptor {
388 id: FORMAT_ID.into(),
389 name: "Plate-reader exports".into(),
390 vendor: "Agilent BioTek, Molecular Devices, BMG LABTECH, Revvity/PerkinElmer, Tecan, Thermo Fisher Scientific".into(),
391 extensions: TEXT_EXTENSIONS
392 .iter()
393 .chain(BOOK_EXTENSIONS.iter())
394 .chain(BINARY_EXTENSIONS.iter())
395 .chain(GEN5_EXTENSIONS.iter())
396 .map(|s| (*s).to_string())
397 .collect(),
398 family: "plate-reader".into(),
399 can_read: true,
400 can_write: false,
401 confidence: assurance::PLATE.confidence,
402 known_gaps: vec![
403 "SoftMax Pro binary documents: SoftMax Pro 5 .pda plate sections (absorbance, endpoint or kinetic, one wavelength, 96 wells; 4 documents from 2 labs validated value-for-value against their text exports) and SoftMax Pro 6/7 .sda endpoint plates (absorbance, fluorescence, luminescence; one wavelength; 6 documents validated) are decoded; kinetic, spectrum and well-scan sections of 6/7 documents, several wavelengths, other plate sizes and cuvette sets are refused (exit 6: export them as text)".into(),
404 "Gen5 experiment files (.xpt): endpoint and kinetic reads decoded from the plate archives (validated on Gen5 2.08, 3.04, 3.11 and 3.15 files from three labs against their exports); the detection mode comes from the read name (number, ex,em, Lum), other names are `unknown`; flagged cells (OVRFLW etc.) are NaN without their text; area scans, spectra, well scans and protocol files (.prt) are refused".into(),
405 "Gen5 area scans and multi-plate kinetic exports, SoftMax Pro well scans and fluorescence spectra are not decoded".into(),
406 "Values calculated by the vendor software are kept as `calculated` reads; curve fits and group tables are kept verbatim only (the plate-reader assays of `openreadout analyze` recompute standard curves, IC50s and kinetics from the reads). SoftMax Pro 5 document templates give the layout (samples, groups, `Blank` group as blanks); SoftMax Pro text-export group tables and SoftMax Pro 6/7 group sections are not used as layouts, and document-stored reduced values are not read".into(),
407 "Day/month order of slash dates is assumed month-first when ambiguous (flagged)".into(),
408 ],
409 }
410 }
411
412 fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
413 if vendors::softmax_sda::sniff_document(head)
414 || (vendors::softmax_sda::sniff(head) && has_extension(path, &BINARY_EXTENSIONS))
415 {
416 return Some(Detection {
417 format_id: FORMAT_ID,
418 confidence: DetectConfidence::Definite,
419 note: Some("SoftMax Pro 6/7 binary document".into()),
420 });
421 }
422 if vendors::gen5_xpt::sniff(head, has_extension(path, &["xpt"])) {
423 return Some(Detection {
424 format_id: FORMAT_ID,
425 confidence: DetectConfidence::Definite,
426 note: Some("Gen5 experiment file (compound file)".into()),
427 });
428 }
429 if has_extension(path, &["prt"]) && vendors::gen5_xpt::is_protocol(head) {
430 return Some(Detection {
431 format_id: FORMAT_ID,
432 confidence: DetectConfidence::Likely,
433 note: Some("Gen5 protocol file: holds no plate data (refused on open)".into()),
434 });
435 }
436 if vendors::softmax_pda::sniff(head) {
437 return Some(Detection {
438 format_id: FORMAT_ID,
439 confidence: DetectConfidence::Definite,
440 note: Some("SoftMax Pro 5 binary document".into()),
441 });
442 }
443 if has_extension(path, &BINARY_EXTENSIONS) && !head.is_empty() && head.contains(&0) {
444 return Some(Detection {
445 format_id: FORMAT_ID,
446 confidence: DetectConfidence::ExtensionOnly,
447 note: Some(
448 "SoftMax Pro binary document of an unrecognised layout: refused on open".into(),
449 ),
450 });
451 }
452 if sheet::is_workbook(head) {
453 return has_extension(path, &BOOK_EXTENSIONS).then(|| Detection {
454 format_id: FORMAT_ID,
455 confidence: DetectConfidence::Likely,
456 note: Some(
457 "spreadsheet workbook; the exporter is identified from its sheets on open"
458 .into(),
459 ),
460 });
461 }
462 if head.is_empty()
463 || head.iter().take(4096).filter(|b| **b == 0).count() > 64
464 && !head.starts_with(&[0xFF, 0xFE])
465 && !head.starts_with(&[0xFE, 0xFF])
466 {
467 return None;
468 }
469 let (text, _) = text::decode(head);
470 if let Some(kind) = sniff_text(&text) {
471 return Some(Detection {
472 format_id: FORMAT_ID,
473 confidence: DetectConfidence::Definite,
474 note: Some(format!("{} export", kind.id())),
475 });
476 }
477 if has_extension(path, &TEXT_EXTENSIONS) && vendors::generic::sniff(&sheet::text_book(head))
478 {
479 return Some(Detection {
480 format_id: FORMAT_ID,
481 confidence: DetectConfidence::Likely,
482 note: Some("plate matrix (8×12, 16×24, …) with no known exporter signature".into()),
483 });
484 }
485 None
486 }
487
488 fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
489 Ok(Box::new(PlateDataset::open(path)?))
490 }
491
492 fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
493 Ok(Box::new(PlateDataset::open_input(input)?))
494 }
495
496 fn reads_any_source(&self) -> bool {
497 true
498 }
499}