Skip to main content

openreadout_omezarr/
lib.rs

1//! OME-Zarr writer: OME-NGFF 0.5 on Zarr v3 (via the pure-Rust `zarrs` crate).
2//!
3//! Every image becomes a `multiscales` group with 5-D arrays in `t, c, z, y, x` order, one
4//! array per resolution level (2×2 mean pyramid), plus `omero` channel display metadata.
5//! A single exported image is written at the root of the store, with an `OME/` group holding
6//! `METADATA.ome.xml` (objective, instrument and channel details OME-NGFF has no place for);
7//! several images become a `bioformats2raw.layout` collection (`0/`, `1/`, ... plus an `OME/`
8//! group listing the series and holding `METADATA.ome.xml`). A multi-well plate (a high-content screening
9//! plate, [`openreadout_core::plate`]) becomes an OME-NGFF HCS plate: `<row>/<column>/<field>`
10//! image groups under `plate` and `well` metadata.
11//!
12//! The store is written to a temporary directory next to the output, every level-0 plane is
13//! read back through `zarrs` and hash-compared with the source plane, and only then is the
14//! directory renamed into place. The source file is never opened for writing.
15//!
16//! OME-Zarr (OME-NGFF) is the cloud-friendly successor of OME-TIFF: a directory of small
17//! compressed chunks plus JSON metadata, readable by napari, `ome-zarr-py`, bioio and web
18//! viewers.
19//!
20//! # Example
21//!
22//! ```
23//! use std::path::Path;
24//!
25//! use openreadout_core::{Dataset, Result};
26//! use openreadout_omezarr::{ZarrExportOptions, default_output, export_ome_zarr};
27//!
28//! /// Write an opened file next to its input as `<stem>.ome.zarr`, with 256-pixel chunks.
29//! fn to_ome_zarr(dataset: &mut dyn Dataset, input: &Path) -> Result<()> {
30//!     let mut options = ZarrExportOptions::default();
31//!     options.chunk = 256;
32//!     let report = export_ome_zarr(dataset, input, &default_output(input), &options)?;
33//!     println!("{} planes in {}", report.planes_written, report.output);
34//!     Ok(())
35//! }
36//! ```
37#![forbid(unsafe_code)]
38#![warn(missing_docs)]
39
40mod ngff;
41mod pixels;
42mod tiled;
43
44use std::path::{Path, PathBuf};
45use std::sync::Arc;
46
47use openreadout_core::model::{FileInfo, ImageInfo};
48use openreadout_core::parallel::{PlaneRequest, ReadContext, read_in_order};
49use openreadout_core::plate::PlateSummary;
50use openreadout_core::reader::PlaneIndex;
51use openreadout_core::select::Selection;
52use openreadout_core::{Dataset, Error, PixelType, Result};
53pub use openreadout_ometiff::PyramidMode;
54use openreadout_ometiff::{Codec, ExportReport, WrittenImage, build_ome_xml_metadata_only};
55use schemars::JsonSchema;
56use serde::{Deserialize, Serialize};
57use zarrs::array::codec::GzipCodec;
58use zarrs::array::{
59    Array, ArrayBuilder, ArrayBytes, ArrayMetadataOptions, ArraySubset, BytesToBytesCodecTraits,
60    CodecOptions, DataType, data_type,
61};
62use zarrs::filesystem::FilesystemStore;
63use zarrs::group::GroupBuilder;
64
65use crate::ngff::ChannelRange;
66
67/// Default chunk edge (y and x) in pixels.
68pub const DEFAULT_CHUNK: u32 = 512;
69/// gzip level used for the `deflate` codec.
70const GZIP_LEVEL: u32 = 5;
71
72const CREATOR: &str = concat!("openreadout ", env!("CARGO_PKG_VERSION"));
73
74/// OME-Zarr export options.
75#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
76#[non_exhaustive]
77pub struct ZarrExportOptions {
78    /// Only this image index (default: all).
79    pub image: Option<u32>,
80    /// Plane selection strings (`c=0`, `z=1-3`, `t=0,2`).
81    pub select: Vec<String>,
82    /// `none` or `deflate` (stored with the Zarr `gzip` codec). `lzw` is rejected.
83    pub codec: Codec,
84    /// Replace an existing output directory.
85    pub overwrite: bool,
86    /// Embed the vendor metadata tree (as JSON) in `OME/METADATA.ome.xml` (not for plates).
87    pub embed_vendor: bool,
88    /// Chunk edge in pixels for y and x (chunks are `1×1×1×chunk×chunk`).
89    pub chunk: u32,
90    /// Resolution levels including full resolution. `None`: add 2× levels while a level is
91    /// larger than 1024 px in either dimension (mean pyramids; source pyramids: all of them).
92    pub levels: Option<u32>,
93    /// Export this source pyramid level as the full resolution (0 = the source's full
94    /// resolution).
95    pub level: u32,
96    /// Export only this rectangle of every plane, in the pixel coordinates of `level`.
97    pub region: Option<openreadout_core::Region>,
98    /// Where the lower levels come from. `Auto`: the source's own pyramid when it has one and
99    /// the whole image is exported from level 0, else 2 × 2 means.
100    pub pyramid: PyramidMode,
101    /// Multi-well plates: write the OME-NGFF HCS layout (`plate`, `well` and field groups)
102    /// when every image is exported. `false` writes a `bioformats2raw.layout` collection.
103    pub plate: bool,
104    /// Multi-well plates: only the images of these wells (`C05`, `c5`); empty = every well.
105    pub wells: Vec<String>,
106    /// Leave out images whose selected planes include missing files (a partial copy of a
107    /// plate) instead of failing; they are listed in `images_skipped`.
108    pub skip_incomplete: bool,
109}
110
111impl Default for ZarrExportOptions {
112    fn default() -> Self {
113        ZarrExportOptions {
114            image: None,
115            select: Vec::new(),
116            codec: Codec::Deflate,
117            overwrite: false,
118            embed_vendor: false,
119            chunk: DEFAULT_CHUNK,
120            levels: None,
121            level: 0,
122            region: None,
123            pyramid: PyramidMode::Auto,
124            plate: true,
125            wells: Vec::new(),
126            skip_incomplete: false,
127        }
128    }
129}
130
131fn zerr(what: &str, e: impl std::fmt::Display) -> Error {
132    Error::Other(format!("OME-Zarr {what}: {e}"))
133}
134
135fn zarr_data_type(p: PixelType) -> Result<DataType> {
136    Ok(match p {
137        PixelType::Int8 => data_type::int8(),
138        PixelType::Int16 => data_type::int16(),
139        PixelType::Int32 => data_type::int32(),
140        PixelType::Uint8 => data_type::uint8(),
141        PixelType::Uint16 => data_type::uint16(),
142        PixelType::Uint32 => data_type::uint32(),
143        PixelType::Float => data_type::float32(),
144        PixelType::Double => data_type::float64(),
145        other => return Err(unsupported_pixel_type(other)),
146    })
147}
148
149fn unsupported_pixel_type(p: PixelType) -> Error {
150    Error::unsupported(
151        "ome-zarr export",
152        format!("pixel type {}", p.ome_name()),
153        "Neither OME-Zarr nor OME-TIFF export writes 64-bit integer or complex samples. \
154         `openreadout planes FILE --dump-dir DIR` writes each plane's raw little-endian samples \
155         instead.",
156    )
157}
158
159fn zarr_array_builder(
160    shape: Vec<u64>,
161    chunks: Vec<u64>,
162    p: PixelType,
163    codec: Codec,
164) -> Result<ArrayBuilder> {
165    let mut b = match p {
166        PixelType::Int8 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i8),
167        PixelType::Int16 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i16),
168        PixelType::Int32 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i32),
169        PixelType::Uint8 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u8),
170        PixelType::Uint16 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u16),
171        PixelType::Uint32 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u32),
172        PixelType::Float => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0.0f32),
173        PixelType::Double => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0.0f64),
174        other => return Err(unsupported_pixel_type(other)),
175    };
176    b.dimension_names(Some(ngff::AXES));
177    match codec {
178        Codec::None => {}
179        Codec::Deflate => {
180            let gz: Arc<dyn BytesToBytesCodecTraits> =
181                Arc::new(GzipCodec::new(GZIP_LEVEL).map_err(|e| zerr("gzip codec", e))?);
182            b.bytes_to_bytes_codecs(vec![gz]);
183        }
184        // `Lzw`, and any codec added to the OME-TIFF writer later.
185        other => {
186            return Err(Error::Usage(format!(
187                "OME-Zarr export supports compression none or deflate (written as the Zarr gzip codec), not {}",
188                format!("{other:?}").to_lowercase()
189            )));
190        }
191    }
192    Ok(b)
193}
194
195/// Chunk write options. Chunks that hold only the fill value (zero) are stored too: `zarrs`
196/// leaves them out by default, and a store whose last chunks are missing looks like an
197/// acquisition still being written to readers that watch stores grow, OpenReadout's among
198/// them, for a few minutes after the export.
199fn write_options() -> CodecOptions {
200    CodecOptions::default().with_store_empty_chunks(true)
201}
202
203/// Plane bytes are little-endian; `zarrs` array bytes are native-endian. Swapping is its
204/// own inverse, so this also converts native bytes back to little-endian.
205fn le_to_native(data: &[u8], bytes_per_sample: usize) -> std::borrow::Cow<'_, [u8]> {
206    if cfg!(target_endian = "big") && bytes_per_sample > 1 {
207        let mut v = data.to_vec();
208        for s in v.chunks_exact_mut(bytes_per_sample) {
209            s.reverse();
210        }
211        std::borrow::Cow::Owned(v)
212    } else {
213        std::borrow::Cow::Borrowed(data)
214    }
215}
216
217/// One image as it is laid out in the store.
218struct Plan<'a> {
219    info: &'a ImageInfo,
220    /// Group path inside the store (`/` or `/<n>`).
221    group: String,
222    c_map: Vec<u32>,
223    z_map: Vec<u32>,
224    t_map: Vec<u32>,
225    levels: Vec<(u32, u32)>,
226    /// Plate layout: (row name, column name) of the image's well.
227    well: Option<(String, String)>,
228}
229
230impl Plan<'_> {
231    fn spp(&self) -> usize {
232        self.info.samples_per_pixel.max(1) as usize
233    }
234    fn zarr_c(&self) -> usize {
235        self.c_map.len() * self.spp()
236    }
237    fn array_path(&self, level: usize) -> String {
238        if self.group == "/" {
239            format!("/{level}")
240        } else {
241            format!("{}/{level}", self.group)
242        }
243    }
244}
245
246/// The plans, and the images left out because their selected planes include missing files.
247fn plans<'a>(
248    info: &'a FileInfo,
249    opts: &ZarrExportOptions,
250    layout: Option<&PlateSummary>,
251    refused: &[u32],
252) -> Result<(Vec<Plan<'a>>, Vec<u32>)> {
253    let sel = Selection::parse(&opts.select)?;
254    // images of the requested wells, and each image's well
255    let mut well_of: std::collections::HashMap<u32, (String, String)> =
256        std::collections::HashMap::new();
257    if let Some(l) = layout {
258        for w in &l.wells {
259            for &i in &w.images {
260                well_of.insert(i, (w.row.clone(), w.column.to_string()));
261            }
262        }
263    }
264    let mut wanted_wells = std::collections::HashSet::new();
265    for w in &opts.wells {
266        let Some(l) = layout else {
267            return Err(Error::Usage(
268                "--well applies to multi-well plates; this file is not one".into(),
269            ));
270        };
271        let pw = l
272            .well(w)
273            .ok_or_else(|| Error::Usage(format!("well '{w}' was not imaged on this plate")))?;
274        wanted_wells.extend(pw.images.iter().copied());
275    }
276    let mut out = Vec::new();
277    let mut skipped = Vec::new();
278    let mut incomplete = Vec::new();
279    for im in &info.images {
280        if opts.image.is_some_and(|i| i != im.index) || refused.contains(&im.index) {
281            continue;
282        }
283        if !opts.wells.is_empty() && !wanted_wells.contains(&im.index) {
284            continue;
285        }
286        let (missing, all_missing) = openreadout_core::plate::missing_planes(im);
287        let needs_missing = all_missing || missing.iter().any(|&(c, z, t)| sel.contains(c, z, t));
288        if needs_missing {
289            if opts.skip_incomplete {
290                skipped.push(im.index);
291                continue;
292            }
293            incomplete.push(im.index);
294        }
295        // Per-plane lists below are sized by the declared dimensions; refuse absurd ones.
296        openreadout_core::limits::checked_plane_count("ome-zarr", im.size_c, im.size_z, im.size_t)?;
297        let pick = |n: u32, s: &[u32]| -> Vec<u32> {
298            (0..n).filter(|v| s.is_empty() || s.contains(v)).collect()
299        };
300        let c_map = pick(im.size_c, &sel.c);
301        let z_map = pick(im.size_z, &sel.z);
302        let t_map = pick(im.size_t, &sel.t);
303        if c_map.is_empty() || z_map.is_empty() || t_map.is_empty() {
304            continue;
305        }
306        // Fail before writing anything when a plane could not be held in memory (the
307        // streaming writer, used for such planes, reads them block by block).
308        if !tiled::needs_tiled(im, opts) {
309            openreadout_core::pixel::plane_bytes_checked(
310                "ome-zarr",
311                im.size_x,
312                im.size_y,
313                im.samples_per_pixel.max(1) as usize * im.pixel_type.bytes_per_sample(),
314            )?;
315        }
316        out.push(Plan {
317            info: im,
318            group: String::new(),
319            c_map,
320            z_map,
321            t_map,
322            levels: ngff::plan_levels(
323                im.size_x,
324                im.size_y,
325                if opts.pyramid == PyramidMode::None {
326                    Some(1)
327                } else {
328                    opts.levels
329                },
330            ),
331            well: well_of.get(&im.index).cloned(),
332        });
333    }
334    if !incomplete.is_empty() {
335        let names: Vec<String> = incomplete
336            .iter()
337            .take(6)
338            .map(|i| {
339                info.images
340                    .iter()
341                    .find(|im| im.index == *i)
342                    .and_then(|im| im.name.clone())
343                    .unwrap_or_else(|| format!("image {i}"))
344            })
345            .collect();
346        return Err(Error::Unsupported {
347            format: "ome-zarr export",
348            feature: format!(
349                "{} image(s) whose plane files are missing ({}{})",
350                incomplete.len(),
351                names.join(", "),
352                if incomplete.len() > 6 { ", ..." } else { "" }
353            ),
354            hint: Some(
355                "This copy of the plate is incomplete (`openreadout check` lists the missing files). Pass --skip-incomplete to export only the complete fields, or choose wells with --well.".into(),
356            ),
357        });
358    }
359    if out.is_empty() && !skipped.is_empty() {
360        return Err(Error::Usage(format!(
361            "every selected image has missing plane files ({} skipped); nothing to export",
362            skipped.len()
363        )));
364    }
365    if out.is_empty() {
366        return Err(Error::Usage(match opts.image {
367            Some(i) if !info.images.iter().any(|im| im.index == i) => format!(
368                "image {i} does not exist (the file has {} images)",
369                info.images.len()
370            ),
371            _ => "selection matches no planes".into(),
372        }));
373    }
374    let plate_mode = opts.plate && opts.image.is_none() && out.iter().all(|p| p.well.is_some());
375    if plate_mode {
376        let mut k: std::collections::HashMap<(String, String), usize> =
377            std::collections::HashMap::new();
378        for p in &mut out {
379            if let Some((r, c)) = &p.well {
380                let n = k.entry((r.clone(), c.clone())).or_insert(0);
381                p.group = format!("/{r}/{c}/{n}");
382                *n += 1;
383            }
384        }
385    } else {
386        let single = out.len() == 1;
387        for (n, p) in out.iter_mut().enumerate() {
388            p.well = None;
389            p.group = if single { "/".into() } else { format!("/{n}") };
390        }
391    }
392    Ok((out, skipped))
393}
394
395/// Total size of the regular files under `dir`.
396fn dir_size(dir: &Path) -> u64 {
397    let mut total = 0;
398    let Ok(rd) = std::fs::read_dir(dir) else {
399        return 0;
400    };
401    for e in rd.flatten() {
402        match e.file_type() {
403            Ok(t) if t.is_dir() => total += dir_size(&e.path()),
404            Ok(t) if t.is_file() => total += e.metadata().map_or(0, |m| m.len()),
405            _ => {}
406        }
407    }
408    total
409}
410
411/// Refuse outputs that would clobber the input or something that is not a Zarr store.
412fn check_output(input: &Path, output: &Path, overwrite: bool) -> Result<()> {
413    let canon = |p: &Path| std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf());
414    let inp = canon(input);
415    if output.exists() {
416        let out = canon(output);
417        if inp == out || inp.starts_with(&out) {
418            return Err(Error::Usage(format!(
419                "{} contains the input file; raw files are never modified",
420                output.display()
421            )));
422        }
423        if !overwrite {
424            return Err(Error::Usage(format!(
425                "{} exists; pass --overwrite to replace it",
426                output.display()
427            )));
428        }
429        if output.is_dir() && !output.join("zarr.json").is_file() {
430            return Err(Error::Usage(format!(
431                "{} is a directory that is not a Zarr v3 store (no zarr.json); refusing to replace it",
432                output.display()
433            )));
434        }
435    }
436    Ok(())
437}
438
439/// Export `ds` (already opened from `input`) to an OME-Zarr store (a directory) at `output`.
440pub fn export_ome_zarr(
441    ds: &mut dyn Dataset,
442    input: &Path,
443    output: &Path,
444    opts: &ZarrExportOptions,
445) -> Result<ExportReport> {
446    export_ome_zarr_with(ds, input, output, opts, &ReadContext::default())
447}
448
449/// [`export_ome_zarr`] with parallel plane decoding and progress reporting (planes are
450/// decoded by worker threads and written in the same order whatever the thread count).
451pub fn export_ome_zarr_with(
452    ds: &mut dyn Dataset,
453    input: &Path,
454    output: &Path,
455    opts: &ZarrExportOptions,
456    ctx: &ReadContext<'_>,
457) -> Result<ExportReport> {
458    openreadout_core::output::refuse_dataset(output, input, &*ds)?;
459    if opts.chunk == 0 {
460        return Err(Error::Usage("chunk size must be at least 1".into()));
461    }
462    if opts.levels == Some(0) {
463        return Err(Error::Usage(
464            "levels must be at least 1 (1 = full resolution only)".into(),
465        ));
466    }
467    if opts.codec == Codec::Lzw {
468        zarr_array_builder(vec![1], vec![1], PixelType::Uint8, opts.codec)?;
469    }
470    check_output(input, output, opts.overwrite)?;
471    let info = ds.info()?;
472    if info.images.is_empty() {
473        return Err(openreadout_ometiff::no_images_error(
474            &info,
475            "ome-zarr export",
476        ));
477    }
478    let layout = openreadout_core::plate::plate_layout(ds, &info);
479    let refused = openreadout_ometiff::refused_images(ds, &info, opts.image);
480    let (plans, mut images_skipped) = plans(&info, opts, layout.as_ref(), &refused)?;
481    images_skipped.extend(refused);
482    images_skipped.sort_unstable();
483
484    let tmp = output.with_file_name(format!(
485        ".{}.partial-{}",
486        output
487            .file_name()
488            .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string()),
489        std::process::id()
490    ));
491    if tmp.exists() {
492        std::fs::remove_dir_all(&tmp).map_err(|e| Error::io(&tmp, e))?;
493    }
494    let keep_partial = std::env::var_os("OPENREADOUT_KEEP_PARTIAL").is_some();
495    let streamed = plans.iter().any(|p| tiled::needs_tiled(p.info, opts));
496    let result = if streamed {
497        tiled::write_store_tiled(ds, &info, &plans, &tmp, opts, ctx, layout.as_ref())
498    } else {
499        write_store(ds, &info, &plans, &tmp, opts, ctx, layout.as_ref()).and_then(
500            |(planes, hashes, xml)| {
501                verify(&tmp, &plans, &hashes)?;
502                Ok((planes, xml, Vec::new()))
503            },
504        )
505    };
506    let (planes_written, ome_xml_bytes, resolutions) = match result {
507        Ok(v) => v,
508        Err(e) => {
509            if !keep_partial {
510                let _ = std::fs::remove_dir_all(&tmp);
511            }
512            return Err(e);
513        }
514    };
515    let bytes_written = dir_size(&tmp);
516    if output.exists() {
517        if output.is_dir() {
518            std::fs::remove_dir_all(output).map_err(|e| Error::io(output, e))?;
519        } else {
520            std::fs::remove_file(output).map_err(|e| Error::io(output, e))?;
521        }
522    }
523    std::fs::rename(&tmp, output).map_err(|e| Error::io(output, e))?;
524    Ok(ExportReport {
525        input: input.display().to_string(),
526        output: output.display().to_string(),
527        format: "ome-zarr".into(),
528        images_written: plans.len() as u32,
529        planes_written,
530        bytes_written,
531        verified: true,
532        codec: opts.codec,
533        ome_xml_bytes,
534        resolutions,
535        layout: plans
536            .first()
537            .is_some_and(|p| p.well.is_some())
538            .then(|| "plate".to_string()),
539        images_skipped,
540    })
541}
542
543/// Source hash of one exported plane, keyed by its position in the store.
544struct PlaneRecord {
545    image: usize,
546    t: u64,
547    c: u64,
548    z: u64,
549    xxh3: String,
550}
551
552type Written = (u64, Vec<PlaneRecord>, u64);
553
554fn write_store(
555    ds: &mut dyn Dataset,
556    info: &FileInfo,
557    plans: &[Plan<'_>],
558    dir: &Path,
559    opts: &ZarrExportOptions,
560    ctx: &ReadContext<'_>,
561    layout: Option<&PlateSummary>,
562) -> Result<Written> {
563    let total: u64 = plans
564        .iter()
565        .map(|p| (p.t_map.len() * p.z_map.len() * p.c_map.len()) as u64)
566        .sum();
567    std::fs::create_dir_all(dir).map_err(|e| Error::io(dir, e))?;
568    let store = Arc::new(FilesystemStore::new(dir).map_err(|e| zerr("store", e))?);
569    let meta_opts = ArrayMetadataOptions::default().with_include_zarrs_metadata(false);
570    let mut records = Vec::new();
571    let mut count = 0u64;
572    let format_id: &'static str = match info.format.id.as_str() {
573        "czi" => "czi",
574        "nd2" => "nd2",
575        "lif" => "lif",
576        _ => "source",
577    };
578
579    for (pi, plan) in plans.iter().enumerate() {
580        let im = plan.info;
581        let spp = plan.spp();
582        let bps = im.pixel_type.bytes_per_sample();
583        let arrays: Vec<Array<FilesystemStore>> = plan
584            .levels
585            .iter()
586            .enumerate()
587            .map(|(l, &(w, h))| {
588                let shape = vec![
589                    plan.t_map.len() as u64,
590                    plan.zarr_c() as u64,
591                    plan.z_map.len() as u64,
592                    u64::from(h),
593                    u64::from(w),
594                ];
595                let chunks = vec![
596                    1,
597                    1,
598                    1,
599                    u64::from(opts.chunk.min(h).max(1)),
600                    u64::from(opts.chunk.min(w).max(1)),
601                ];
602                let a = zarr_array_builder(shape, chunks, im.pixel_type, opts.codec)?
603                    .build(store.clone(), &plan.array_path(l))
604                    .map_err(|e| zerr("array", e))?;
605                a.store_metadata_opt(&meta_opts)
606                    .map_err(|e| zerr("array metadata", e))?;
607                Ok(a)
608            })
609            .collect::<Result<_>>()?;
610        let mut ranges = vec![ChannelRange::EMPTY; plan.zarr_c()];
611        // (ti, zi, ci) of each request, in the store's t, z, c order.
612        let mut slots = Vec::new();
613        let mut requests = Vec::new();
614        for (ti, &t) in plan.t_map.iter().enumerate() {
615            for (zi, &z) in plan.z_map.iter().enumerate() {
616                for (ci, &c) in plan.c_map.iter().enumerate() {
617                    slots.push((ti, zi, ci));
618                    requests.push(PlaneRequest {
619                        image: im.index,
620                        index: PlaneIndex { c, z, t },
621                        level: 0,
622                        region: None,
623                    });
624                }
625            }
626        }
627        let plane_bytes = u64::from(im.size_x)
628            * u64::from(im.size_y)
629            * spp as u64
630            * im.pixel_type.bytes_per_sample() as u64;
631        read_in_order(
632            ds,
633            ctx,
634            &requests,
635            plane_bytes,
636            &|r, plane| {
637                let PlaneIndex { c, z, t } = r.index;
638                if plane.data.len() != plane.expected_len()
639                    || plane.width != im.size_x
640                    || plane.height != im.size_y
641                    || plane.pixel_type != im.pixel_type
642                    || plane.samples_per_pixel.max(1) as usize != spp
643                {
644                    return Err(Error::corrupt(
645                        format_id,
646                        format!(
647                            "image {} plane c={c} z={z} t={t} is {}x{} {} x{} ({} bytes), expected {}x{} {} x{}",
648                            im.index,
649                            plane.width,
650                            plane.height,
651                            plane.pixel_type.ome_name(),
652                            plane.samples_per_pixel,
653                            plane.data.len(),
654                            im.size_x,
655                            im.size_y,
656                            im.pixel_type.ome_name(),
657                            spp
658                        ),
659                    ));
660                }
661                let h = plane.xxh3_hex();
662                Ok((plane, h))
663            },
664            &mut |i, (plane, xxh3)| {
665                let (ti, zi, ci) = slots[i];
666                records.push(PlaneRecord {
667                    image: pi,
668                    t: ti as u64,
669                    c: (ci * spp) as u64,
670                    z: zi as u64,
671                    xxh3,
672                });
673                for (s, mut data) in pixels::deinterleave(&plane.data, spp, bps)
674                    .into_iter()
675                    .enumerate()
676                {
677                    let zc = (ci * spp + s) as u64;
678                    ranges[ci * spp + s] =
679                        pixels::union(ranges[ci * spp + s], pixels::range(&data, im.pixel_type));
680                    for (l, a) in arrays.iter().enumerate() {
681                        let (w, h) = plan.levels[l];
682                        if l > 0 {
683                            let (pw, ph) = plan.levels[l - 1];
684                            data = pixels::downsample_2x(&data, pw, ph, im.pixel_type);
685                        }
686                        let subset = ArraySubset::new_with_ranges(&[
687                            ti as u64..ti as u64 + 1,
688                            zc..zc + 1,
689                            zi as u64..zi as u64 + 1,
690                            0..u64::from(h),
691                            0..u64::from(w),
692                        ]);
693                        a.store_array_subset_opt(
694                            &subset,
695                            ArrayBytes::new_flen(le_to_native(&data, bps)),
696                            &write_options(),
697                        )
698                        .map_err(|e| zerr("chunk write", e))?;
699                    }
700                }
701                count += 1;
702                ctx.report(count, total);
703                Ok(())
704            },
705        )?;
706        let default_z = (plan.z_map.len() / 2) as u32;
707        let attrs = ngff::image_attributes(
708            ngff::multiscales(im, &plan.levels, CREATOR),
709            ngff::omero(im, &plan.c_map, &ranges, default_z),
710        );
711        write_group(&store, &plan.group, attrs)?;
712    }
713
714    let xml_len = write_collection(ds, info, plans, dir, &store, opts, layout)?;
715    Ok((count, records, xml_len))
716}
717
718/// The metadata of a store that is not a plate: for several images the collection root and
719/// the `OME` group listing the series; for every store `OME/METADATA.ome.xml`, which carries
720/// what OME-NGFF has no place for (objective, instrument, acquisition mode, exposures).
721/// Returns the XML's size (0 for plates).
722fn write_collection(
723    ds: &mut dyn Dataset,
724    info: &FileInfo,
725    plans: &[Plan<'_>],
726    dir: &Path,
727    store: &Arc<FilesystemStore>,
728    opts: &ZarrExportOptions,
729    layout: Option<&PlateSummary>,
730) -> Result<u64> {
731    if plans.first().is_some_and(|p| p.well.is_some()) {
732        write_plate_groups(store, plans, layout)?;
733        return Ok(0);
734    }
735    if plans.len() > 1 {
736        write_group(store, "/", ngff::collection_root_attributes())?;
737        let series: Vec<String> = plans
738            .iter()
739            .map(|p| p.group.trim_start_matches('/').to_string())
740            .collect();
741        write_group(store, "/OME", ngff::series_attributes(&series))?;
742    } else {
743        // A single image stays at the root (readers open it as a plain image); the `OME`
744        // group only holds the metadata document.
745        write_group(store, "/OME", serde_json::json!({}))?;
746    }
747    let written: Vec<WrittenImage<'_>> = plans
748        .iter()
749        .map(|p| {
750            // Describe the selected planes (exported index space) and where each came
751            // from, so the metadata document carries per-plane `Plane` elements.
752            let mut planes = Vec::new();
753            let mut source_planes = Vec::new();
754            for (ti, &t) in p.t_map.iter().enumerate() {
755                for (zi, &z) in p.z_map.iter().enumerate() {
756                    for (ci, &c) in p.c_map.iter().enumerate() {
757                        planes.push((ci as u32, zi as u32, ti as u32));
758                        source_planes.push((c, z, t));
759                    }
760                }
761            }
762            WrittenImage {
763                info: p.info,
764                first_ifd: 0,
765                planes,
766                size_c: p.c_map.len() as u32,
767                size_z: p.z_map.len() as u32,
768                size_t: p.t_map.len() as u32,
769                c_map: p.c_map.clone(),
770                source_planes,
771                frames: ds
772                    .frames(p.info.index, None)
773                    .map(|(_, r)| r)
774                    .unwrap_or_default(),
775            }
776        })
777        .collect();
778    let vendor_json = if opts.embed_vendor {
779        ds.vendor_metadata()
780            .ok()
781            .filter(|v| !v.is_null())
782            .and_then(|v| serde_json::to_string(&v).ok())
783            .filter(|s| s.len() < 8 << 20)
784    } else {
785        None
786    };
787    let xml = build_ome_xml_metadata_only(info, &written, CREATOR, vendor_json.as_deref())
788        .map_err(Error::Other)?;
789    let p = dir.join("OME").join("METADATA.ome.xml");
790    std::fs::write(&p, xml.as_bytes()).map_err(|e| Error::io(&p, e))?;
791    Ok(xml.len() as u64)
792}
793
794/// The `plate` root, one group per row and the `well` groups of an HCS plate store.
795fn write_plate_groups(
796    store: &Arc<FilesystemStore>,
797    plans: &[Plan<'_>],
798    layout: Option<&PlateSummary>,
799) -> Result<()> {
800    // wells in plate order with their field paths
801    let mut wells: std::collections::BTreeMap<(u32, u32), (String, String, Vec<String>)> =
802        std::collections::BTreeMap::new();
803    for p in plans {
804        let Some((r, c)) = &p.well else { continue };
805        let (ri, ci) = openreadout_core::plate::parse_well(&format!("{r}{c}"))
806            .ok_or_else(|| Error::Other(format!("plate export: bad well {r}{c}")))?;
807        let field = p.group.rsplit('/').next().unwrap_or("0").to_string();
808        wells
809            .entry((ri, ci))
810            .or_insert_with(|| (r.clone(), c.clone(), Vec::new()))
811            .2
812            .push(field);
813    }
814    let rows = layout
815        .map_or(0, |l| l.rows)
816        .max(wells.keys().map(|k| k.0 + 1).max().unwrap_or(0));
817    let cols = layout
818        .map_or(0, |l| l.columns)
819        .max(wells.keys().map(|k| k.1 + 1).max().unwrap_or(0));
820    let row_names: Vec<String> = (0..rows).map(openreadout_core::plate::row_name).collect();
821    let col_names: Vec<String> = (1..=cols).map(|c| c.to_string()).collect();
822    let name = layout.and_then(|l| l.id.clone());
823    let field_count = wells.values().map(|w| w.2.len()).max().unwrap_or(0);
824    let attrs = ngff::plate_attributes(
825        name.as_deref(),
826        &row_names,
827        &col_names,
828        &wells
829            .iter()
830            .map(|(&(ri, ci), (r, c, _))| (format!("{r}/{c}"), ri, ci))
831            .collect::<Vec<_>>(),
832        field_count,
833    );
834    write_group(store, "/", attrs)?;
835    let used_rows: std::collections::BTreeSet<&String> = wells.values().map(|w| &w.0).collect();
836    for r in used_rows {
837        write_group(store, &format!("/{r}"), serde_json::json!({}))?;
838    }
839    for (r, c, fields) in wells.values() {
840        write_group(store, &format!("/{r}/{c}"), ngff::well_attributes(fields))?;
841    }
842    Ok(())
843}
844
845fn write_group(store: &Arc<FilesystemStore>, path: &str, attrs: serde_json::Value) -> Result<()> {
846    let serde_json::Value::Object(map) = attrs else {
847        return Err(Error::Other("group attributes must be an object".into()));
848    };
849    let g = GroupBuilder::new()
850        .attributes(map)
851        .build(store.clone(), path)
852        .map_err(|e| zerr("group", e))?;
853    g.store_metadata().map_err(|e| zerr("group metadata", e))
854}
855
856/// Re-open the written store from disk and compare every level-0 plane with its source hash.
857fn verify(dir: &Path, plans: &[Plan<'_>], records: &[PlaneRecord]) -> Result<()> {
858    let store = Arc::new(FilesystemStore::new(dir).map_err(|e| zerr("store", e))?);
859    let arrays: Vec<Array<FilesystemStore>> = plans
860        .iter()
861        .map(|p| Array::open(store.clone(), &p.array_path(0)).map_err(|e| zerr("read-back", e)))
862        .collect::<Result<_>>()?;
863    let mut bad = 0usize;
864    for r in records {
865        let plan = &plans[r.image];
866        let spp = plan.spp() as u64;
867        let bps = plan.info.pixel_type.bytes_per_sample();
868        let (w, h) = plan.levels[0];
869        let mut samples = Vec::with_capacity(spp as usize);
870        for s in 0..spp {
871            let subset = ArraySubset::new_with_ranges(&[
872                r.t..r.t + 1,
873                r.c + s..r.c + s + 1,
874                r.z..r.z + 1,
875                0..u64::from(h),
876                0..u64::from(w),
877            ]);
878            let bytes: ArrayBytes<'static> = arrays[r.image]
879                .retrieve_array_subset(&subset)
880                .map_err(|e| zerr("read-back", e))?;
881            let raw = bytes.into_fixed().map_err(|e| zerr("read-back", e))?;
882            samples.push(le_to_native(&raw, bps).into_owned());
883        }
884        let data = pixels::interleave(&samples, bps);
885        if format!("{:032x}", xxhash_rust::xxh3::xxh3_128(&data)) != r.xxh3 {
886            bad += 1;
887        }
888    }
889    if bad > 0 {
890        return Err(Error::Other(format!(
891            "read-back verification failed: {bad} of {} planes differ after writing; output discarded",
892            records.len()
893        )));
894    }
895    Ok(())
896}
897
898/// Default output path for `input`: same directory, `<stem>.ome.zarr`.
899pub fn default_output(input: &Path) -> PathBuf {
900    let stem = input
901        .file_stem()
902        .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string());
903    input.with_file_name(format!("{stem}.ome.zarr"))
904}
905
906#[cfg(test)]
907mod tests {
908    use super::*;
909    use openreadout_core::model::{CheckReport, FormatDescriptor, LsEntry};
910    use openreadout_core::{Confidence, Plane, ProvenanceMap};
911
912    struct Ramp(ImageInfo);
913
914    impl Dataset for Ramp {
915        fn info(&self) -> Result<FileInfo> {
916            Ok(FileInfo {
917                path: "ramp".into(),
918                size_bytes: 0,
919                format: FormatDescriptor {
920                    id: "ramp".into(),
921                    name: "Ramp".into(),
922                    vendor: String::new(),
923                    extensions: vec![],
924                    family: "microscopy".into(),
925                    can_read: true,
926                    can_write: false,
927                    confidence: Confidence::Low,
928                    known_gaps: vec![],
929                },
930                format_version: None,
931                images: vec![self.0.clone()],
932                tables: vec![],
933                spectra: vec![],
934                traces: Vec::new(),
935                plane_count: 1,
936                notes: vec![],
937            })
938        }
939        fn vendor_metadata(&self) -> Result<serde_json::Value> {
940            Ok(serde_json::Value::Null)
941        }
942        fn provenance(&self) -> ProvenanceMap {
943            ProvenanceMap::new()
944        }
945        fn entries(&self) -> Result<Vec<LsEntry>> {
946            Ok(vec![])
947        }
948        fn read_plane(&mut self, _: u32, _: PlaneIndex) -> Result<Plane> {
949            Ok(Plane {
950                width: 4,
951                height: 2,
952                pixel_type: PixelType::Uint8,
953                samples_per_pixel: 1,
954                data: (1..=8).collect(),
955            })
956        }
957        fn check(&mut self) -> Result<CheckReport> {
958            Ok(CheckReport::new("ramp", "ramp"))
959        }
960    }
961
962    #[test]
963    fn verification_detects_a_changed_chunk() {
964        let dir = tempfile::tempdir().unwrap();
965        let store = dir.path().join("s");
966        let mut ds = Ramp(ImageInfo::new(0, 4, 2, PixelType::Uint8).finish());
967        let info = ds.info().unwrap();
968        let opts = ZarrExportOptions {
969            codec: Codec::None,
970            ..Default::default()
971        };
972        let (plans, _) = plans(&info, &opts, None, &[]).unwrap();
973        let (n, records, _) = write_store(
974            &mut ds,
975            &info,
976            &plans,
977            &store,
978            &opts,
979            &ReadContext::default(),
980            None,
981        )
982        .unwrap();
983        assert_eq!(n, 1);
984        verify(&store, &plans, &records).unwrap();
985        // flip one sample in the (uncompressed) level-0 chunk
986        let chunk = store.join("0/c/0/0/0/0/0");
987        let mut bytes = std::fs::read(&chunk).unwrap();
988        bytes[3] ^= 0xFF;
989        std::fs::write(&chunk, bytes).unwrap();
990        let e = verify(&store, &plans, &records).unwrap_err();
991        assert!(e.to_string().contains("1 of 1 planes differ"), "{e}");
992    }
993}