1#![forbid(unsafe_code)]
38#![warn(missing_docs)]
39
40mod ngff;
41mod pixels;
42mod tiled;
43
44use std::path::{Path, PathBuf};
45use std::sync::Arc;
46
47use openreadout_core::model::{FileInfo, ImageInfo};
48use openreadout_core::parallel::{PlaneRequest, ReadContext, read_in_order};
49use openreadout_core::plate::PlateSummary;
50use openreadout_core::reader::PlaneIndex;
51use openreadout_core::select::Selection;
52use openreadout_core::{Dataset, Error, PixelType, Result};
53pub use openreadout_ometiff::PyramidMode;
54use openreadout_ometiff::{Codec, ExportReport, WrittenImage, build_ome_xml_metadata_only};
55use schemars::JsonSchema;
56use serde::{Deserialize, Serialize};
57use zarrs::array::codec::GzipCodec;
58use zarrs::array::{
59 Array, ArrayBuilder, ArrayBytes, ArrayMetadataOptions, ArraySubset, BytesToBytesCodecTraits,
60 CodecOptions, DataType, data_type,
61};
62use zarrs::filesystem::FilesystemStore;
63use zarrs::group::GroupBuilder;
64
65use crate::ngff::ChannelRange;
66
67pub const DEFAULT_CHUNK: u32 = 512;
69const GZIP_LEVEL: u32 = 5;
71
72const CREATOR: &str = concat!("openreadout ", env!("CARGO_PKG_VERSION"));
73
74#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
76#[non_exhaustive]
77pub struct ZarrExportOptions {
78 pub image: Option<u32>,
80 pub select: Vec<String>,
82 pub codec: Codec,
84 pub overwrite: bool,
86 pub embed_vendor: bool,
88 pub chunk: u32,
90 pub levels: Option<u32>,
93 pub level: u32,
96 pub region: Option<openreadout_core::Region>,
98 pub pyramid: PyramidMode,
101 pub plate: bool,
104 pub wells: Vec<String>,
106 pub skip_incomplete: bool,
109}
110
111impl Default for ZarrExportOptions {
112 fn default() -> Self {
113 ZarrExportOptions {
114 image: None,
115 select: Vec::new(),
116 codec: Codec::Deflate,
117 overwrite: false,
118 embed_vendor: false,
119 chunk: DEFAULT_CHUNK,
120 levels: None,
121 level: 0,
122 region: None,
123 pyramid: PyramidMode::Auto,
124 plate: true,
125 wells: Vec::new(),
126 skip_incomplete: false,
127 }
128 }
129}
130
131fn zerr(what: &str, e: impl std::fmt::Display) -> Error {
132 Error::Other(format!("OME-Zarr {what}: {e}"))
133}
134
135fn zarr_data_type(p: PixelType) -> Result<DataType> {
136 Ok(match p {
137 PixelType::Int8 => data_type::int8(),
138 PixelType::Int16 => data_type::int16(),
139 PixelType::Int32 => data_type::int32(),
140 PixelType::Uint8 => data_type::uint8(),
141 PixelType::Uint16 => data_type::uint16(),
142 PixelType::Uint32 => data_type::uint32(),
143 PixelType::Float => data_type::float32(),
144 PixelType::Double => data_type::float64(),
145 other => return Err(unsupported_pixel_type(other)),
146 })
147}
148
149fn unsupported_pixel_type(p: PixelType) -> Error {
150 Error::unsupported(
151 "ome-zarr export",
152 format!("pixel type {}", p.ome_name()),
153 "Neither OME-Zarr nor OME-TIFF export writes 64-bit integer or complex samples. \
154 `openreadout planes FILE --dump-dir DIR` writes each plane's raw little-endian samples \
155 instead.",
156 )
157}
158
159fn zarr_array_builder(
160 shape: Vec<u64>,
161 chunks: Vec<u64>,
162 p: PixelType,
163 codec: Codec,
164) -> Result<ArrayBuilder> {
165 let mut b = match p {
166 PixelType::Int8 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i8),
167 PixelType::Int16 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i16),
168 PixelType::Int32 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0i32),
169 PixelType::Uint8 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u8),
170 PixelType::Uint16 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u16),
171 PixelType::Uint32 => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0u32),
172 PixelType::Float => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0.0f32),
173 PixelType::Double => ArrayBuilder::new(shape, chunks, zarr_data_type(p)?, 0.0f64),
174 other => return Err(unsupported_pixel_type(other)),
175 };
176 b.dimension_names(Some(ngff::AXES));
177 match codec {
178 Codec::None => {}
179 Codec::Deflate => {
180 let gz: Arc<dyn BytesToBytesCodecTraits> =
181 Arc::new(GzipCodec::new(GZIP_LEVEL).map_err(|e| zerr("gzip codec", e))?);
182 b.bytes_to_bytes_codecs(vec![gz]);
183 }
184 other => {
186 return Err(Error::Usage(format!(
187 "OME-Zarr export supports compression none or deflate (written as the Zarr gzip codec), not {}",
188 format!("{other:?}").to_lowercase()
189 )));
190 }
191 }
192 Ok(b)
193}
194
195fn write_options() -> CodecOptions {
200 CodecOptions::default().with_store_empty_chunks(true)
201}
202
203fn le_to_native(data: &[u8], bytes_per_sample: usize) -> std::borrow::Cow<'_, [u8]> {
206 if cfg!(target_endian = "big") && bytes_per_sample > 1 {
207 let mut v = data.to_vec();
208 for s in v.chunks_exact_mut(bytes_per_sample) {
209 s.reverse();
210 }
211 std::borrow::Cow::Owned(v)
212 } else {
213 std::borrow::Cow::Borrowed(data)
214 }
215}
216
217struct Plan<'a> {
219 info: &'a ImageInfo,
220 group: String,
222 c_map: Vec<u32>,
223 z_map: Vec<u32>,
224 t_map: Vec<u32>,
225 levels: Vec<(u32, u32)>,
226 well: Option<(String, String)>,
228}
229
230impl Plan<'_> {
231 fn spp(&self) -> usize {
232 self.info.samples_per_pixel.max(1) as usize
233 }
234 fn zarr_c(&self) -> usize {
235 self.c_map.len() * self.spp()
236 }
237 fn array_path(&self, level: usize) -> String {
238 if self.group == "/" {
239 format!("/{level}")
240 } else {
241 format!("{}/{level}", self.group)
242 }
243 }
244}
245
246fn plans<'a>(
248 info: &'a FileInfo,
249 opts: &ZarrExportOptions,
250 layout: Option<&PlateSummary>,
251 refused: &[u32],
252) -> Result<(Vec<Plan<'a>>, Vec<u32>)> {
253 let sel = Selection::parse(&opts.select)?;
254 let mut well_of: std::collections::HashMap<u32, (String, String)> =
256 std::collections::HashMap::new();
257 if let Some(l) = layout {
258 for w in &l.wells {
259 for &i in &w.images {
260 well_of.insert(i, (w.row.clone(), w.column.to_string()));
261 }
262 }
263 }
264 let mut wanted_wells = std::collections::HashSet::new();
265 for w in &opts.wells {
266 let Some(l) = layout else {
267 return Err(Error::Usage(
268 "--well applies to multi-well plates; this file is not one".into(),
269 ));
270 };
271 let pw = l
272 .well(w)
273 .ok_or_else(|| Error::Usage(format!("well '{w}' was not imaged on this plate")))?;
274 wanted_wells.extend(pw.images.iter().copied());
275 }
276 let mut out = Vec::new();
277 let mut skipped = Vec::new();
278 let mut incomplete = Vec::new();
279 for im in &info.images {
280 if opts.image.is_some_and(|i| i != im.index) || refused.contains(&im.index) {
281 continue;
282 }
283 if !opts.wells.is_empty() && !wanted_wells.contains(&im.index) {
284 continue;
285 }
286 let (missing, all_missing) = openreadout_core::plate::missing_planes(im);
287 let needs_missing = all_missing || missing.iter().any(|&(c, z, t)| sel.contains(c, z, t));
288 if needs_missing {
289 if opts.skip_incomplete {
290 skipped.push(im.index);
291 continue;
292 }
293 incomplete.push(im.index);
294 }
295 openreadout_core::limits::checked_plane_count("ome-zarr", im.size_c, im.size_z, im.size_t)?;
297 let pick = |n: u32, s: &[u32]| -> Vec<u32> {
298 (0..n).filter(|v| s.is_empty() || s.contains(v)).collect()
299 };
300 let c_map = pick(im.size_c, &sel.c);
301 let z_map = pick(im.size_z, &sel.z);
302 let t_map = pick(im.size_t, &sel.t);
303 if c_map.is_empty() || z_map.is_empty() || t_map.is_empty() {
304 continue;
305 }
306 if !tiled::needs_tiled(im, opts) {
309 openreadout_core::pixel::plane_bytes_checked(
310 "ome-zarr",
311 im.size_x,
312 im.size_y,
313 im.samples_per_pixel.max(1) as usize * im.pixel_type.bytes_per_sample(),
314 )?;
315 }
316 out.push(Plan {
317 info: im,
318 group: String::new(),
319 c_map,
320 z_map,
321 t_map,
322 levels: ngff::plan_levels(
323 im.size_x,
324 im.size_y,
325 if opts.pyramid == PyramidMode::None {
326 Some(1)
327 } else {
328 opts.levels
329 },
330 ),
331 well: well_of.get(&im.index).cloned(),
332 });
333 }
334 if !incomplete.is_empty() {
335 let names: Vec<String> = incomplete
336 .iter()
337 .take(6)
338 .map(|i| {
339 info.images
340 .iter()
341 .find(|im| im.index == *i)
342 .and_then(|im| im.name.clone())
343 .unwrap_or_else(|| format!("image {i}"))
344 })
345 .collect();
346 return Err(Error::Unsupported {
347 format: "ome-zarr export",
348 feature: format!(
349 "{} image(s) whose plane files are missing ({}{})",
350 incomplete.len(),
351 names.join(", "),
352 if incomplete.len() > 6 { ", ..." } else { "" }
353 ),
354 hint: Some(
355 "This copy of the plate is incomplete (`openreadout check` lists the missing files). Pass --skip-incomplete to export only the complete fields, or choose wells with --well.".into(),
356 ),
357 });
358 }
359 if out.is_empty() && !skipped.is_empty() {
360 return Err(Error::Usage(format!(
361 "every selected image has missing plane files ({} skipped); nothing to export",
362 skipped.len()
363 )));
364 }
365 if out.is_empty() {
366 return Err(Error::Usage(match opts.image {
367 Some(i) if !info.images.iter().any(|im| im.index == i) => format!(
368 "image {i} does not exist (the file has {} images)",
369 info.images.len()
370 ),
371 _ => "selection matches no planes".into(),
372 }));
373 }
374 let plate_mode = opts.plate && opts.image.is_none() && out.iter().all(|p| p.well.is_some());
375 if plate_mode {
376 let mut k: std::collections::HashMap<(String, String), usize> =
377 std::collections::HashMap::new();
378 for p in &mut out {
379 if let Some((r, c)) = &p.well {
380 let n = k.entry((r.clone(), c.clone())).or_insert(0);
381 p.group = format!("/{r}/{c}/{n}");
382 *n += 1;
383 }
384 }
385 } else {
386 let single = out.len() == 1;
387 for (n, p) in out.iter_mut().enumerate() {
388 p.well = None;
389 p.group = if single { "/".into() } else { format!("/{n}") };
390 }
391 }
392 Ok((out, skipped))
393}
394
395fn dir_size(dir: &Path) -> u64 {
397 let mut total = 0;
398 let Ok(rd) = std::fs::read_dir(dir) else {
399 return 0;
400 };
401 for e in rd.flatten() {
402 match e.file_type() {
403 Ok(t) if t.is_dir() => total += dir_size(&e.path()),
404 Ok(t) if t.is_file() => total += e.metadata().map_or(0, |m| m.len()),
405 _ => {}
406 }
407 }
408 total
409}
410
411fn check_output(input: &Path, output: &Path, overwrite: bool) -> Result<()> {
413 let canon = |p: &Path| std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf());
414 let inp = canon(input);
415 if output.exists() {
416 let out = canon(output);
417 if inp == out || inp.starts_with(&out) {
418 return Err(Error::Usage(format!(
419 "{} contains the input file; raw files are never modified",
420 output.display()
421 )));
422 }
423 if !overwrite {
424 return Err(Error::Usage(format!(
425 "{} exists; pass --overwrite to replace it",
426 output.display()
427 )));
428 }
429 if output.is_dir() && !output.join("zarr.json").is_file() {
430 return Err(Error::Usage(format!(
431 "{} is a directory that is not a Zarr v3 store (no zarr.json); refusing to replace it",
432 output.display()
433 )));
434 }
435 }
436 Ok(())
437}
438
439pub fn export_ome_zarr(
441 ds: &mut dyn Dataset,
442 input: &Path,
443 output: &Path,
444 opts: &ZarrExportOptions,
445) -> Result<ExportReport> {
446 export_ome_zarr_with(ds, input, output, opts, &ReadContext::default())
447}
448
449pub fn export_ome_zarr_with(
452 ds: &mut dyn Dataset,
453 input: &Path,
454 output: &Path,
455 opts: &ZarrExportOptions,
456 ctx: &ReadContext<'_>,
457) -> Result<ExportReport> {
458 openreadout_core::output::refuse_dataset(output, input, &*ds)?;
459 if opts.chunk == 0 {
460 return Err(Error::Usage("chunk size must be at least 1".into()));
461 }
462 if opts.levels == Some(0) {
463 return Err(Error::Usage(
464 "levels must be at least 1 (1 = full resolution only)".into(),
465 ));
466 }
467 if opts.codec == Codec::Lzw {
468 zarr_array_builder(vec![1], vec![1], PixelType::Uint8, opts.codec)?;
469 }
470 check_output(input, output, opts.overwrite)?;
471 let info = ds.info()?;
472 if info.images.is_empty() {
473 return Err(openreadout_ometiff::no_images_error(
474 &info,
475 "ome-zarr export",
476 ));
477 }
478 let layout = openreadout_core::plate::plate_layout(ds, &info);
479 let refused = openreadout_ometiff::refused_images(ds, &info, opts.image);
480 let (plans, mut images_skipped) = plans(&info, opts, layout.as_ref(), &refused)?;
481 images_skipped.extend(refused);
482 images_skipped.sort_unstable();
483
484 let tmp = output.with_file_name(format!(
485 ".{}.partial-{}",
486 output
487 .file_name()
488 .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string()),
489 std::process::id()
490 ));
491 if tmp.exists() {
492 std::fs::remove_dir_all(&tmp).map_err(|e| Error::io(&tmp, e))?;
493 }
494 let keep_partial = std::env::var_os("OPENREADOUT_KEEP_PARTIAL").is_some();
495 let streamed = plans.iter().any(|p| tiled::needs_tiled(p.info, opts));
496 let result = if streamed {
497 tiled::write_store_tiled(ds, &info, &plans, &tmp, opts, ctx, layout.as_ref())
498 } else {
499 write_store(ds, &info, &plans, &tmp, opts, ctx, layout.as_ref()).and_then(
500 |(planes, hashes, xml)| {
501 verify(&tmp, &plans, &hashes)?;
502 Ok((planes, xml, Vec::new()))
503 },
504 )
505 };
506 let (planes_written, ome_xml_bytes, resolutions) = match result {
507 Ok(v) => v,
508 Err(e) => {
509 if !keep_partial {
510 let _ = std::fs::remove_dir_all(&tmp);
511 }
512 return Err(e);
513 }
514 };
515 let bytes_written = dir_size(&tmp);
516 if output.exists() {
517 if output.is_dir() {
518 std::fs::remove_dir_all(output).map_err(|e| Error::io(output, e))?;
519 } else {
520 std::fs::remove_file(output).map_err(|e| Error::io(output, e))?;
521 }
522 }
523 std::fs::rename(&tmp, output).map_err(|e| Error::io(output, e))?;
524 Ok(ExportReport {
525 input: input.display().to_string(),
526 output: output.display().to_string(),
527 format: "ome-zarr".into(),
528 images_written: plans.len() as u32,
529 planes_written,
530 bytes_written,
531 verified: true,
532 codec: opts.codec,
533 ome_xml_bytes,
534 resolutions,
535 layout: plans
536 .first()
537 .is_some_and(|p| p.well.is_some())
538 .then(|| "plate".to_string()),
539 images_skipped,
540 })
541}
542
543struct PlaneRecord {
545 image: usize,
546 t: u64,
547 c: u64,
548 z: u64,
549 xxh3: String,
550}
551
552type Written = (u64, Vec<PlaneRecord>, u64);
553
554fn write_store(
555 ds: &mut dyn Dataset,
556 info: &FileInfo,
557 plans: &[Plan<'_>],
558 dir: &Path,
559 opts: &ZarrExportOptions,
560 ctx: &ReadContext<'_>,
561 layout: Option<&PlateSummary>,
562) -> Result<Written> {
563 let total: u64 = plans
564 .iter()
565 .map(|p| (p.t_map.len() * p.z_map.len() * p.c_map.len()) as u64)
566 .sum();
567 std::fs::create_dir_all(dir).map_err(|e| Error::io(dir, e))?;
568 let store = Arc::new(FilesystemStore::new(dir).map_err(|e| zerr("store", e))?);
569 let meta_opts = ArrayMetadataOptions::default().with_include_zarrs_metadata(false);
570 let mut records = Vec::new();
571 let mut count = 0u64;
572 let format_id: &'static str = match info.format.id.as_str() {
573 "czi" => "czi",
574 "nd2" => "nd2",
575 "lif" => "lif",
576 _ => "source",
577 };
578
579 for (pi, plan) in plans.iter().enumerate() {
580 let im = plan.info;
581 let spp = plan.spp();
582 let bps = im.pixel_type.bytes_per_sample();
583 let arrays: Vec<Array<FilesystemStore>> = plan
584 .levels
585 .iter()
586 .enumerate()
587 .map(|(l, &(w, h))| {
588 let shape = vec![
589 plan.t_map.len() as u64,
590 plan.zarr_c() as u64,
591 plan.z_map.len() as u64,
592 u64::from(h),
593 u64::from(w),
594 ];
595 let chunks = vec![
596 1,
597 1,
598 1,
599 u64::from(opts.chunk.min(h).max(1)),
600 u64::from(opts.chunk.min(w).max(1)),
601 ];
602 let a = zarr_array_builder(shape, chunks, im.pixel_type, opts.codec)?
603 .build(store.clone(), &plan.array_path(l))
604 .map_err(|e| zerr("array", e))?;
605 a.store_metadata_opt(&meta_opts)
606 .map_err(|e| zerr("array metadata", e))?;
607 Ok(a)
608 })
609 .collect::<Result<_>>()?;
610 let mut ranges = vec![ChannelRange::EMPTY; plan.zarr_c()];
611 let mut slots = Vec::new();
613 let mut requests = Vec::new();
614 for (ti, &t) in plan.t_map.iter().enumerate() {
615 for (zi, &z) in plan.z_map.iter().enumerate() {
616 for (ci, &c) in plan.c_map.iter().enumerate() {
617 slots.push((ti, zi, ci));
618 requests.push(PlaneRequest {
619 image: im.index,
620 index: PlaneIndex { c, z, t },
621 level: 0,
622 region: None,
623 });
624 }
625 }
626 }
627 let plane_bytes = u64::from(im.size_x)
628 * u64::from(im.size_y)
629 * spp as u64
630 * im.pixel_type.bytes_per_sample() as u64;
631 read_in_order(
632 ds,
633 ctx,
634 &requests,
635 plane_bytes,
636 &|r, plane| {
637 let PlaneIndex { c, z, t } = r.index;
638 if plane.data.len() != plane.expected_len()
639 || plane.width != im.size_x
640 || plane.height != im.size_y
641 || plane.pixel_type != im.pixel_type
642 || plane.samples_per_pixel.max(1) as usize != spp
643 {
644 return Err(Error::corrupt(
645 format_id,
646 format!(
647 "image {} plane c={c} z={z} t={t} is {}x{} {} x{} ({} bytes), expected {}x{} {} x{}",
648 im.index,
649 plane.width,
650 plane.height,
651 plane.pixel_type.ome_name(),
652 plane.samples_per_pixel,
653 plane.data.len(),
654 im.size_x,
655 im.size_y,
656 im.pixel_type.ome_name(),
657 spp
658 ),
659 ));
660 }
661 let h = plane.xxh3_hex();
662 Ok((plane, h))
663 },
664 &mut |i, (plane, xxh3)| {
665 let (ti, zi, ci) = slots[i];
666 records.push(PlaneRecord {
667 image: pi,
668 t: ti as u64,
669 c: (ci * spp) as u64,
670 z: zi as u64,
671 xxh3,
672 });
673 for (s, mut data) in pixels::deinterleave(&plane.data, spp, bps)
674 .into_iter()
675 .enumerate()
676 {
677 let zc = (ci * spp + s) as u64;
678 ranges[ci * spp + s] =
679 pixels::union(ranges[ci * spp + s], pixels::range(&data, im.pixel_type));
680 for (l, a) in arrays.iter().enumerate() {
681 let (w, h) = plan.levels[l];
682 if l > 0 {
683 let (pw, ph) = plan.levels[l - 1];
684 data = pixels::downsample_2x(&data, pw, ph, im.pixel_type);
685 }
686 let subset = ArraySubset::new_with_ranges(&[
687 ti as u64..ti as u64 + 1,
688 zc..zc + 1,
689 zi as u64..zi as u64 + 1,
690 0..u64::from(h),
691 0..u64::from(w),
692 ]);
693 a.store_array_subset_opt(
694 &subset,
695 ArrayBytes::new_flen(le_to_native(&data, bps)),
696 &write_options(),
697 )
698 .map_err(|e| zerr("chunk write", e))?;
699 }
700 }
701 count += 1;
702 ctx.report(count, total);
703 Ok(())
704 },
705 )?;
706 let default_z = (plan.z_map.len() / 2) as u32;
707 let attrs = ngff::image_attributes(
708 ngff::multiscales(im, &plan.levels, CREATOR),
709 ngff::omero(im, &plan.c_map, &ranges, default_z),
710 );
711 write_group(&store, &plan.group, attrs)?;
712 }
713
714 let xml_len = write_collection(ds, info, plans, dir, &store, opts, layout)?;
715 Ok((count, records, xml_len))
716}
717
718fn write_collection(
723 ds: &mut dyn Dataset,
724 info: &FileInfo,
725 plans: &[Plan<'_>],
726 dir: &Path,
727 store: &Arc<FilesystemStore>,
728 opts: &ZarrExportOptions,
729 layout: Option<&PlateSummary>,
730) -> Result<u64> {
731 if plans.first().is_some_and(|p| p.well.is_some()) {
732 write_plate_groups(store, plans, layout)?;
733 return Ok(0);
734 }
735 if plans.len() > 1 {
736 write_group(store, "/", ngff::collection_root_attributes())?;
737 let series: Vec<String> = plans
738 .iter()
739 .map(|p| p.group.trim_start_matches('/').to_string())
740 .collect();
741 write_group(store, "/OME", ngff::series_attributes(&series))?;
742 } else {
743 write_group(store, "/OME", serde_json::json!({}))?;
746 }
747 let written: Vec<WrittenImage<'_>> = plans
748 .iter()
749 .map(|p| {
750 let mut planes = Vec::new();
753 let mut source_planes = Vec::new();
754 for (ti, &t) in p.t_map.iter().enumerate() {
755 for (zi, &z) in p.z_map.iter().enumerate() {
756 for (ci, &c) in p.c_map.iter().enumerate() {
757 planes.push((ci as u32, zi as u32, ti as u32));
758 source_planes.push((c, z, t));
759 }
760 }
761 }
762 WrittenImage {
763 info: p.info,
764 first_ifd: 0,
765 planes,
766 size_c: p.c_map.len() as u32,
767 size_z: p.z_map.len() as u32,
768 size_t: p.t_map.len() as u32,
769 c_map: p.c_map.clone(),
770 source_planes,
771 frames: ds
772 .frames(p.info.index, None)
773 .map(|(_, r)| r)
774 .unwrap_or_default(),
775 }
776 })
777 .collect();
778 let vendor_json = if opts.embed_vendor {
779 ds.vendor_metadata()
780 .ok()
781 .filter(|v| !v.is_null())
782 .and_then(|v| serde_json::to_string(&v).ok())
783 .filter(|s| s.len() < 8 << 20)
784 } else {
785 None
786 };
787 let xml = build_ome_xml_metadata_only(info, &written, CREATOR, vendor_json.as_deref())
788 .map_err(Error::Other)?;
789 let p = dir.join("OME").join("METADATA.ome.xml");
790 std::fs::write(&p, xml.as_bytes()).map_err(|e| Error::io(&p, e))?;
791 Ok(xml.len() as u64)
792}
793
794fn write_plate_groups(
796 store: &Arc<FilesystemStore>,
797 plans: &[Plan<'_>],
798 layout: Option<&PlateSummary>,
799) -> Result<()> {
800 let mut wells: std::collections::BTreeMap<(u32, u32), (String, String, Vec<String>)> =
802 std::collections::BTreeMap::new();
803 for p in plans {
804 let Some((r, c)) = &p.well else { continue };
805 let (ri, ci) = openreadout_core::plate::parse_well(&format!("{r}{c}"))
806 .ok_or_else(|| Error::Other(format!("plate export: bad well {r}{c}")))?;
807 let field = p.group.rsplit('/').next().unwrap_or("0").to_string();
808 wells
809 .entry((ri, ci))
810 .or_insert_with(|| (r.clone(), c.clone(), Vec::new()))
811 .2
812 .push(field);
813 }
814 let rows = layout
815 .map_or(0, |l| l.rows)
816 .max(wells.keys().map(|k| k.0 + 1).max().unwrap_or(0));
817 let cols = layout
818 .map_or(0, |l| l.columns)
819 .max(wells.keys().map(|k| k.1 + 1).max().unwrap_or(0));
820 let row_names: Vec<String> = (0..rows).map(openreadout_core::plate::row_name).collect();
821 let col_names: Vec<String> = (1..=cols).map(|c| c.to_string()).collect();
822 let name = layout.and_then(|l| l.id.clone());
823 let field_count = wells.values().map(|w| w.2.len()).max().unwrap_or(0);
824 let attrs = ngff::plate_attributes(
825 name.as_deref(),
826 &row_names,
827 &col_names,
828 &wells
829 .iter()
830 .map(|(&(ri, ci), (r, c, _))| (format!("{r}/{c}"), ri, ci))
831 .collect::<Vec<_>>(),
832 field_count,
833 );
834 write_group(store, "/", attrs)?;
835 let used_rows: std::collections::BTreeSet<&String> = wells.values().map(|w| &w.0).collect();
836 for r in used_rows {
837 write_group(store, &format!("/{r}"), serde_json::json!({}))?;
838 }
839 for (r, c, fields) in wells.values() {
840 write_group(store, &format!("/{r}/{c}"), ngff::well_attributes(fields))?;
841 }
842 Ok(())
843}
844
845fn write_group(store: &Arc<FilesystemStore>, path: &str, attrs: serde_json::Value) -> Result<()> {
846 let serde_json::Value::Object(map) = attrs else {
847 return Err(Error::Other("group attributes must be an object".into()));
848 };
849 let g = GroupBuilder::new()
850 .attributes(map)
851 .build(store.clone(), path)
852 .map_err(|e| zerr("group", e))?;
853 g.store_metadata().map_err(|e| zerr("group metadata", e))
854}
855
856fn verify(dir: &Path, plans: &[Plan<'_>], records: &[PlaneRecord]) -> Result<()> {
858 let store = Arc::new(FilesystemStore::new(dir).map_err(|e| zerr("store", e))?);
859 let arrays: Vec<Array<FilesystemStore>> = plans
860 .iter()
861 .map(|p| Array::open(store.clone(), &p.array_path(0)).map_err(|e| zerr("read-back", e)))
862 .collect::<Result<_>>()?;
863 let mut bad = 0usize;
864 for r in records {
865 let plan = &plans[r.image];
866 let spp = plan.spp() as u64;
867 let bps = plan.info.pixel_type.bytes_per_sample();
868 let (w, h) = plan.levels[0];
869 let mut samples = Vec::with_capacity(spp as usize);
870 for s in 0..spp {
871 let subset = ArraySubset::new_with_ranges(&[
872 r.t..r.t + 1,
873 r.c + s..r.c + s + 1,
874 r.z..r.z + 1,
875 0..u64::from(h),
876 0..u64::from(w),
877 ]);
878 let bytes: ArrayBytes<'static> = arrays[r.image]
879 .retrieve_array_subset(&subset)
880 .map_err(|e| zerr("read-back", e))?;
881 let raw = bytes.into_fixed().map_err(|e| zerr("read-back", e))?;
882 samples.push(le_to_native(&raw, bps).into_owned());
883 }
884 let data = pixels::interleave(&samples, bps);
885 if format!("{:032x}", xxhash_rust::xxh3::xxh3_128(&data)) != r.xxh3 {
886 bad += 1;
887 }
888 }
889 if bad > 0 {
890 return Err(Error::Other(format!(
891 "read-back verification failed: {bad} of {} planes differ after writing; output discarded",
892 records.len()
893 )));
894 }
895 Ok(())
896}
897
898pub fn default_output(input: &Path) -> PathBuf {
900 let stem = input
901 .file_stem()
902 .map_or_else(|| "export".into(), |s| s.to_string_lossy().to_string());
903 input.with_file_name(format!("{stem}.ome.zarr"))
904}
905
906#[cfg(test)]
907mod tests {
908 use super::*;
909 use openreadout_core::model::{CheckReport, FormatDescriptor, LsEntry};
910 use openreadout_core::{Confidence, Plane, ProvenanceMap};
911
912 struct Ramp(ImageInfo);
913
914 impl Dataset for Ramp {
915 fn info(&self) -> Result<FileInfo> {
916 Ok(FileInfo {
917 path: "ramp".into(),
918 size_bytes: 0,
919 format: FormatDescriptor {
920 id: "ramp".into(),
921 name: "Ramp".into(),
922 vendor: String::new(),
923 extensions: vec![],
924 family: "microscopy".into(),
925 can_read: true,
926 can_write: false,
927 confidence: Confidence::Low,
928 known_gaps: vec![],
929 },
930 format_version: None,
931 images: vec![self.0.clone()],
932 tables: vec![],
933 spectra: vec![],
934 traces: Vec::new(),
935 plane_count: 1,
936 notes: vec![],
937 })
938 }
939 fn vendor_metadata(&self) -> Result<serde_json::Value> {
940 Ok(serde_json::Value::Null)
941 }
942 fn provenance(&self) -> ProvenanceMap {
943 ProvenanceMap::new()
944 }
945 fn entries(&self) -> Result<Vec<LsEntry>> {
946 Ok(vec![])
947 }
948 fn read_plane(&mut self, _: u32, _: PlaneIndex) -> Result<Plane> {
949 Ok(Plane {
950 width: 4,
951 height: 2,
952 pixel_type: PixelType::Uint8,
953 samples_per_pixel: 1,
954 data: (1..=8).collect(),
955 })
956 }
957 fn check(&mut self) -> Result<CheckReport> {
958 Ok(CheckReport::new("ramp", "ramp"))
959 }
960 }
961
962 #[test]
963 fn verification_detects_a_changed_chunk() {
964 let dir = tempfile::tempdir().unwrap();
965 let store = dir.path().join("s");
966 let mut ds = Ramp(ImageInfo::new(0, 4, 2, PixelType::Uint8).finish());
967 let info = ds.info().unwrap();
968 let opts = ZarrExportOptions {
969 codec: Codec::None,
970 ..Default::default()
971 };
972 let (plans, _) = plans(&info, &opts, None, &[]).unwrap();
973 let (n, records, _) = write_store(
974 &mut ds,
975 &info,
976 &plans,
977 &store,
978 &opts,
979 &ReadContext::default(),
980 None,
981 )
982 .unwrap();
983 assert_eq!(n, 1);
984 verify(&store, &plans, &records).unwrap();
985 let chunk = store.join("0/c/0/0/0/0/0");
987 let mut bytes = std::fs::read(&chunk).unwrap();
988 bytes[3] ^= 0xFF;
989 std::fs::write(&chunk, bytes).unwrap();
990 let e = verify(&store, &plans, &records).unwrap_err();
991 assert!(e.to_string().contains("1 of 1 planes differ"), "{e}");
992 }
993}