Skip to main content

openreadout_hcs/
lib.rs

1//! Clean-room readers for high-content screening (HCS) plates.
2//!
3//! Three plate-imager families, each an index file plus one TIFF per plane:
4//!
5//! - **Revvity/PerkinElmer Harmony** exports of the Opera Phenix, Operetta and Operetta CLS
6//!   (format id `opera-harmony`): `Images/Index.idx.xml` (also `Index.xml`, `Index.ref.xml`), and
7//!   Columbus exports (`ImageIndex.ColumbusIDX.xml` with multi-page TIFF or `.flex` planes).
8//! - **Molecular Devices ImageXpress / MetaXpress** (format id `imagexpress`): an `.HTD` plate
9//!   description, or a plate folder without one.
10//! - **Yokogawa CellVoyager** CV7000/CV8000/CQ1 (format id `cellvoyager`):
11//!   `MeasurementData.mlf` with `MeasurementDetail.mrf`, the `.mes` setting and `.wpi`/`.wpp`.
12//!
13//! Every field of view of every well is one image (`images[i].extra.well`, `field`); the plate
14//! (id, type, wells, completeness of the copy on disk) is reported by
15//! [`openreadout_core::Dataset::plate`]. `info` reads the index and lists the plate folder
16//! once; it opens at most one plane file (for the sample type), however many the plate has.
17//!
18//! Layout and vocabulary: `docs/formats/hcs.md`, `docs/formats/opera-harmony.md`,
19//! `docs/formats/imagexpress.md`, `docs/formats/cellvoyager.md`. Provenance:
20//! `docs/provenance/opera-harmony.md`, `imagexpress.md`, `cellvoyager.md`.
21//!
22//! # Example
23//!
24//! ```
25//! use openreadout_core::FormatReader;
26//! use openreadout_hcs::HarmonyReader;
27//!
28//! let format = HarmonyReader.descriptor();
29//! assert_eq!(format.id, openreadout_hcs::HARMONY_FORMAT_ID);
30//! println!("{} ({}): {}", format.name, format.family, format.extensions.join(", "));
31//! ```
32//!
33//! Reading a plate:
34//!
35//! ```no_run
36//! use std::path::Path;
37//!
38//! use openreadout_core::{FormatReader, PlaneIndex};
39//! use openreadout_hcs::HarmonyReader;
40//!
41//! let mut dataset = HarmonyReader.open(Path::new("Measurement 1/Images/Index.idx.xml"))?;
42//! let plate = dataset.plate().expect("a plate");
43//! println!("{} wells imaged, {} planes missing", plate.wells.len(), plate.planes_missing);
44//! let well = plate.well("C05").expect("C05 was imaged");
45//! // First field of view of well C05, channel 0.
46//! let plane = dataset.read_plane(well.images[0], PlaneIndex { c: 0, z: 0, t: 0 })?;
47//! assert_eq!(plane.data.len(), plane.expected_len());
48//! # Ok::<(), openreadout_core::Error>(())
49//! ```
50//!
51//! # API stability
52//!
53//! The supported API is what this page documents. The parser modules are public only so that
54//! tests and fuzz targets can reach them: they are hidden from this documentation and may change
55//! in any release.
56#![forbid(unsafe_code)]
57#![warn(missing_docs)]
58
59mod assurance;
60#[doc(hidden)]
61pub mod cellvoyager;
62#[doc(hidden)]
63pub mod dataset;
64#[doc(hidden)]
65pub mod flex;
66#[doc(hidden)]
67pub mod harmony;
68#[doc(hidden)]
69pub mod imagexpress;
70#[doc(hidden)]
71pub mod model;
72#[doc(hidden)]
73pub mod planes;
74
75use std::path::{Path, PathBuf};
76
77use openreadout_core::model::{DetectConfidence, FormatDescriptor};
78use openreadout_core::reader::{Dataset, Detection, FormatReader, has_extension};
79use openreadout_core::source::{Fs, Input};
80use openreadout_core::{Error, Result};
81
82pub use cellvoyager::CELLVOYAGER_FORMAT_ID;
83#[doc(hidden)]
84pub use dataset::HcsDataset;
85pub use harmony::HARMONY_FORMAT_ID;
86pub use imagexpress::IMAGEXPRESS_FORMAT_ID;
87
88/// Format family of the three readers.
89pub const FAMILY: &str = "microscopy";
90
91fn small_head(fs: &Fs, path: &Path) -> Vec<u8> {
92    use std::io::Read;
93    let mut head = vec![0u8; 4096];
94    let n = fs
95        .open(path)
96        .and_then(|mut f| f.read(&mut head))
97        .unwrap_or(0);
98    head.truncate(n);
99    head
100}
101
102fn definite(format_id: &'static str, note: Option<String>) -> Detection {
103    Detection {
104        format_id,
105        confidence: DetectConfidence::Definite,
106        note,
107    }
108}
109
110/// Revvity/PerkinElmer Harmony exports (Opera Phenix, Operetta, Operetta CLS).
111#[derive(Debug, Default, Clone, Copy)]
112pub struct HarmonyReader;
113
114impl HarmonyReader {
115    fn index_of(input: &Input) -> Option<PathBuf> {
116        let fs = input.fs();
117        let p = input.path();
118        if fs.is_dir(p) {
119            harmony::find_index(fs, p)
120        } else {
121            Some(p.to_path_buf())
122        }
123    }
124}
125
126impl FormatReader for HarmonyReader {
127    fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
128        Some(&assurance::OPERA_HARMONY)
129    }
130
131    fn descriptor(&self) -> FormatDescriptor {
132        FormatDescriptor {
133            id: HARMONY_FORMAT_ID.into(),
134            name: "Revvity/PerkinElmer Harmony and Columbus exports (Opera Phenix, Operetta, Opera)".into(),
135            vendor: "Revvity (PerkinElmer)".into(),
136            extensions: vec!["idx.xml".into(), "xml".into(), "flex".into()],
137            family: FAMILY.into(),
138            can_read: true,
139            can_write: false,
140            confidence: assurance::OPERA_HARMONY.confidence,
141            known_gaps: vec![
142                "Harmony 4/5 `Index.idx.xml` with TIFF planes, Columbus `ImageIndex.ColumbusIDX.xml` with multi-page TIFF or `.flex` planes, and standalone Opera `.flex` files or measurement folders are read; the older `Index.ref.xml` layout with gzip-compressed planes (`.tiff.gz`) is not".into(),
143                "Standalone `.flex`: time series (Kinetic) and compressed pages are not validated on a public file; one well per file, pages as the Flex XML's Image@BufferNo say".into(),
144                "FLIM planes (several FlimID values) and sequence/fk/fl variants in file names are not told apart".into(),
145                "Flat-field profiles and skew-crop parameters are listed by `info --view full`, not applied".into(),
146                "The Z step is the PositionZ difference of the first field's first two planes".into(),
147            ],
148        }
149    }
150
151    fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
152        if flex::is_flex_name(path) && openreadout_tiff::looks_like_tiff(head) {
153            return Some(definite(
154                HARMONY_FORMAT_ID,
155                Some("Opera .flex file (one well; the Flex XML describes its pages); open the measurement folder for the whole plate".into()),
156            ));
157        }
158        (has_extension(path, &["xml"]) && harmony::looks_like_harmony(head)).then(|| {
159            definite(
160                HARMONY_FORMAT_ID,
161                Some("Harmony plate index; its TIFF planes are read as one plate".into()),
162            )
163        })
164    }
165
166    fn sniff_input(&self, head: &[u8], input: &Input) -> Option<Detection> {
167        let fs = input.fs();
168        let p = input.path();
169        if fs.is_dir(p) {
170            let Some(idx) = harmony::find_index(fs, p) else {
171                let n = flex::flex_files(fs, p).len();
172                return (n > 0).then(|| {
173                    definite(
174                        HARMONY_FORMAT_ID,
175                        Some(format!("Opera measurement folder of {n} .flex file(s)")),
176                    )
177                });
178            };
179            return harmony::looks_like_harmony(&small_head(fs, &idx)).then(|| {
180                definite(
181                    HARMONY_FORMAT_ID,
182                    Some(format!("Harmony measurement folder ({})", idx.display())),
183                )
184            });
185        }
186        self.sniff(head, p)
187    }
188
189    fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
190        self.open_input(&Input::local(path))
191    }
192
193    fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
194        let fs = input.fs().clone();
195        let p = input.path();
196        if flex::is_flex_name(p) || (fs.is_dir(p) && harmony::find_index(&fs, p).is_none()) {
197            let root = if fs.is_dir(p) {
198                p.to_path_buf()
199            } else {
200                p.parent().map(Path::to_path_buf).unwrap_or_default()
201            };
202            let plate = dataset::cached(HARMONY_FORMAT_ID, &fs, p, &root, || {
203                flex::parse(&fs, p).map(|pl| dataset::finish(pl, &fs, |_| false))
204            })?;
205            return Ok(Box::new(HcsDataset::new(plate, fs, self.descriptor())));
206        }
207        let idx = Self::index_of(input).ok_or_else(|| {
208            Error::unsupported(
209                HARMONY_FORMAT_ID,
210                "a folder without Index.idx.xml",
211                "Open the Harmony measurement folder, its Images folder or the Index.idx.xml inside it.",
212            )
213        })?;
214        let root = idx.parent().map(Path::to_path_buf).unwrap_or_default();
215        let plate = dataset::cached(HARMONY_FORMAT_ID, &fs, &idx, &root, || {
216            harmony::parse(&fs, &idx).map(|p| dataset::finish(p, &fs, harmony::is_plane_name))
217        })?;
218        Ok(Box::new(HcsDataset::new(plate, fs, self.descriptor())))
219    }
220
221    fn reads_any_source(&self) -> bool {
222        true
223    }
224}
225
226/// Molecular Devices ImageXpress / MetaXpress plates.
227#[derive(Debug, Default, Clone, Copy)]
228pub struct ImageXpressReader;
229
230impl FormatReader for ImageXpressReader {
231    fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
232        Some(&assurance::IMAGEXPRESS)
233    }
234
235    fn descriptor(&self) -> FormatDescriptor {
236        FormatDescriptor {
237            id: IMAGEXPRESS_FORMAT_ID.into(),
238            name: "Molecular Devices ImageXpress / MetaXpress plate (HTD + TIFF)".into(),
239            vendor: "Molecular Devices".into(),
240            extensions: vec!["htd".into()],
241            family: FAMILY.into(),
242            can_read: true,
243            can_write: false,
244            confidence: assurance::IMAGEXPRESS.confidence,
245            known_gaps: vec![
246                "Z series (ZStep_<n> folders) and time series (TimePoint_<n> folders) follow the folder names; Z projections and stitched montages are not handled specially".into(),
247                "A plate folder without its HTD is read from the file names: missing planes cannot be detected there".into(),
248                "Excitation wavelengths and per-channel colours are not recorded in the files read; channel names come from the HTD (WaveName) or the plane files' illumination setting".into(),
249                "Cellomics/CellWorX variants of the HTD (`.pnl`) are not read".into(),
250            ],
251        }
252    }
253
254    fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
255        (has_extension(path, &["htd"]) || imagexpress::looks_like_htd(head))
256            .then_some(())
257            .filter(|()| imagexpress::looks_like_htd(head))
258            .map(|()| {
259                definite(
260                    IMAGEXPRESS_FORMAT_ID,
261                    Some(
262                        "MetaXpress plate description; its TIFF planes are read as one plate"
263                            .into(),
264                    ),
265                )
266            })
267    }
268
269    fn sniff_input(&self, head: &[u8], input: &Input) -> Option<Detection> {
270        let fs = input.fs();
271        let p = input.path();
272        if fs.is_dir(p) {
273            if let Some(h) = imagexpress::find_htd(fs, p) {
274                return imagexpress::looks_like_htd(&small_head(fs, &h)).then(|| {
275                    definite(
276                        IMAGEXPRESS_FORMAT_ID,
277                        Some(format!("ImageXpress plate folder ({})", h.display())),
278                    )
279                });
280            }
281            // a plate folder without its HTD: MetaXpress file names and plane headers
282            return imagexpress::plate_folder_without_htd(fs, p).map(|n| Detection {
283                format_id: IMAGEXPRESS_FORMAT_ID,
284                confidence: DetectConfidence::Likely,
285                note: Some(format!(
286                    "folder of {n} MetaXpress plane files (<plate>_<well>_s<site>_w<wave>.tif with MetaMorph headers) without an HTD"
287                )),
288            });
289        }
290        self.sniff(head, p)
291    }
292
293    fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
294        self.open_input(&Input::local(path))
295    }
296
297    fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
298        let fs = input.fs().clone();
299        let p = input.path().to_path_buf();
300        let root = if fs.is_dir(&p) {
301            p.clone()
302        } else {
303            p.parent().map(Path::to_path_buf).unwrap_or_default()
304        };
305        let plate = dataset::cached(IMAGEXPRESS_FORMAT_ID, &fs, &p, &root, || {
306            imagexpress::parse(&fs, &p).map(|pl| dataset::finish(pl, &fs, |_| false))
307        })?;
308        Ok(Box::new(HcsDataset::new(plate, fs, self.descriptor())))
309    }
310
311    fn reads_any_source(&self) -> bool {
312        true
313    }
314}
315
316/// Yokogawa CellVoyager measurements (CV7000, CV8000, CQ1).
317#[derive(Debug, Default, Clone, Copy)]
318pub struct CellVoyagerReader;
319
320impl CellVoyagerReader {
321    fn mlf_of(input: &Input) -> Option<PathBuf> {
322        let fs = input.fs();
323        let p = input.path();
324        if fs.is_dir(p) {
325            return cellvoyager::find_index(fs, p);
326        }
327        let name = p.file_name()?.to_string_lossy().to_ascii_lowercase();
328        if model::has_extension(&name, &["mlf"]) {
329            return Some(p.to_path_buf());
330        }
331        cellvoyager::find_index(fs, p.parent()?)
332    }
333}
334
335impl FormatReader for CellVoyagerReader {
336    fn assurance(&self) -> Option<&'static openreadout_core::assurance::AssuranceProfile> {
337        Some(&assurance::CELLVOYAGER)
338    }
339
340    fn descriptor(&self) -> FormatDescriptor {
341        FormatDescriptor {
342            id: CELLVOYAGER_FORMAT_ID.into(),
343            name: "Yokogawa CellVoyager measurement (CV7000, CV8000, CQ1)".into(),
344            vendor: "Yokogawa".into(),
345            extensions: vec!["mlf".into(), "mrf".into(), "wpi".into(), "mes".into()],
346            family: FAMILY.into(),
347            can_read: true,
348            can_write: false,
349            confidence: assurance::CELLVOYAGER.confidence,
350            known_gaps: vec![
351                "Tiled fields (several PartialTileIndex values) are not stitched".into(),
352                "The objective's numerical aperture is not recorded in the files read".into(),
353                "Excitation is reported only for channels with a single laser; the emission band comes from the filter name (BP<centre>/<width>)".into(),
354                "Correction files (shading, geometry, crosstalk) are listed, not applied".into(),
355            ],
356        }
357    }
358
359    fn sniff(&self, head: &[u8], path: &Path) -> Option<Detection> {
360        (has_extension(path, &["mlf", "mrf", "wpi", "mes"]) && cellvoyager::looks_like_cellvoyager(head))
361            .then(|| definite(CELLVOYAGER_FORMAT_ID, Some("CellVoyager measurement; MeasurementData.mlf and its TIFF planes are read as one plate".into())))
362    }
363
364    fn sniff_input(&self, head: &[u8], input: &Input) -> Option<Detection> {
365        let fs = input.fs();
366        let p = input.path();
367        if fs.is_dir(p) {
368            let mlf = cellvoyager::find_index(fs, p)?;
369            return cellvoyager::looks_like_cellvoyager(&small_head(fs, &mlf)).then(|| {
370                definite(
371                    CELLVOYAGER_FORMAT_ID,
372                    Some(format!(
373                        "CellVoyager measurement folder ({})",
374                        mlf.display()
375                    )),
376                )
377            });
378        }
379        let d = self.sniff(head, p)?;
380        // `.mrf`/`.wpi`/`.mes` stand for the measurement only when its record file is there
381        if has_extension(p, &["mlf"]) || Self::mlf_of(input).is_some() {
382            Some(d)
383        } else {
384            None
385        }
386    }
387
388    fn open(&self, path: &Path) -> Result<Box<dyn Dataset>> {
389        self.open_input(&Input::local(path))
390    }
391
392    fn open_input(&self, input: &Input) -> Result<Box<dyn Dataset>> {
393        let fs = input.fs().clone();
394        let mlf = Self::mlf_of(input).ok_or_else(|| {
395            Error::unsupported(
396                CELLVOYAGER_FORMAT_ID,
397                "a CellVoyager file without MeasurementData.mlf next to it",
398                "Open the measurement folder or its MeasurementData.mlf (the record of every image).",
399            )
400        })?;
401        let root = mlf.parent().map(Path::to_path_buf).unwrap_or_default();
402        let plate = dataset::cached(CELLVOYAGER_FORMAT_ID, &fs, &mlf, &root, || {
403            cellvoyager::parse(&fs, &mlf)
404                .map(|p| dataset::finish(p, &fs, cellvoyager::is_plane_name))
405        })?;
406        Ok(Box::new(HcsDataset::new(plate, fs, self.descriptor())))
407    }
408
409    fn reads_any_source(&self) -> bool {
410        true
411    }
412}