data_beans/
import.rs

1//! Build a backend from the formats single-cell tools write.
2//!
3//! * [`from_mtx`]: a MatrixMarket file with its feature and barcode TSVs,
4//!   such as a 10x Cell Ranger MEX triplet ([`MtxImport`]).
5//! * [`from_h5ad`] (feature `hdf5`): an AnnData `.h5ad` file
6//!   ([`H5adImport`]), its `obs` written beside the backend as
7//!   `{stem}.cell_metadata.tsv.gz`.
8//! * [`from_10x_matrix`] (feature `hdf5`): a Cell Ranger feature-barcode
9//!   matrix `.h5` ([`TenxMatrixImport`]).
10//! * [`from_10x_molecule`] (feature `hdf5`): a Cell Ranger
11//!   `molecule_info.h5`, counted as Cell Ranger counts it
12//!   ([`TenxMoleculeImport`]).
13//!
14//! Each writes the backend [`Output`] names, squeezed when it asks, and
15//! returns the path of the file written: `{output}.zarr.zip` by default. The
16//! `stage_*` functions stop short of the last step and return a [`Staged`]
17//! build, for a caller that looks at the backend before keeping it (as the
18//! CLI's import preview does). These are what `data-beans from-mtx`,
19//! `from-h5ad`, `from-10x-matrix` and `from-10x-molecule` run.
20//!
21//! ```no_run
22//! use data_beans::import::{self, MtxImport, Output};
23//!
24//! let mtx = MtxImport {
25//!     row: Some("filtered/features.tsv.gz".into()),
26//!     col: Some("filtered/barcodes.tsv.gz".into()),
27//!     ..MtxImport::new("filtered/matrix.mtx.gz")
28//! };
29//! let backend = import::from_mtx(&mtx, &Output::new("pbmc"))?; // "pbmc.zarr.zip"
30//!
31//! #[cfg(feature = "hdf5")]
32//! let backend = import::from_h5ad(&import::H5adImport::new("pbmc.h5ad"), &Output::new("pbmc"))?;
33//! # anyhow::Ok(())
34//! ```
35
36#[cfg(feature = "hdf5")]
37mod h5ad;
38mod mtx;
39#[cfg(feature = "hdf5")]
40mod tenx_matrix;
41#[cfg(feature = "hdf5")]
42mod tenx_molecule;
43
44#[cfg(feature = "hdf5")]
45pub use h5ad::{from_h5ad, stage_h5ad, H5adImport};
46pub use mtx::{from_mtx, stage_mtx, MtxImport};
47#[cfg(feature = "hdf5")]
48pub use tenx_matrix::{from_10x_matrix, stage_10x_matrix, TenxMatrixImport};
49#[cfg(feature = "hdf5")]
50pub use tenx_molecule::{from_10x_molecule, stage_10x_molecule, TenxMoleculeImport};
51
52use crate::sparse_io::{open_sparse_matrix_by_path, SparseIoBackend};
53use log::info;
54
55/// Where an import writes its backend.
56#[derive(Clone, Debug)]
57pub struct Output {
58    /// The output's header or name: `{output}.zarr.zip`, `{output}.zarr` or
59    /// `{output}.h5`, a backend suffix it already has kept. Files written
60    /// beside the backend (metadata tables) are named by its stem.
61    pub output: Box<str>,
62    pub backend: SparseIoBackend,
63    /// Zip a zarr backend into `.zarr.zip`; else keep the `.zarr` directory.
64    pub zip: bool,
65    /// Drop rows and columns with too few non-zeros once it is written.
66    pub squeeze: Option<Squeeze>,
67}
68
69impl Output {
70    /// `{output}.zarr.zip`, not squeezed: the CLI's defaults.
71    pub fn new(output: &str) -> Self {
72        Self {
73            output: output.into(),
74            backend: SparseIoBackend::Zarr,
75            zip: true,
76            squeeze: None,
77        }
78    }
79
80    /// Clear what a previous run left at the working file, and stage the
81    /// build there.
82    fn stage(&self) -> anyhow::Result<(Staged, SparseIoBackend)> {
83        let (effective, backend, working) =
84            crate::zarr_io::prepare_output(&self.output, self.backend.clone(), self.zip)?;
85        let squeeze = self.squeeze;
86        Ok((
87            Staged {
88                effective,
89                working,
90                squeeze,
91            },
92            backend,
93        ))
94    }
95}
96
97/// A squeeze: the rows and columns with fewer non-zeros than these go.
98#[derive(Clone, Copy, Debug)]
99pub struct Squeeze {
100    pub rows: usize,
101    pub columns: usize,
102    /// Cells per job; `None` scales it by the number of features.
103    pub block_size: Option<usize>,
104}
105
106impl Default for Squeeze {
107    /// Rows and columns without a single non-zero go.
108    fn default() -> Self {
109        Self {
110            rows: 1,
111            columns: 1,
112            block_size: None,
113        }
114    }
115}
116
117impl Squeeze {
118    /// Squeeze the backend `backend_file` in place.
119    pub fn apply(&self, backend_file: &str) -> anyhow::Result<()> {
120        crate::qc::squeeze_in_place(backend_file, self.rows, self.columns, self.block_size)
121    }
122}
123
124/// A backend written but not yet in place: for a `.zarr.zip` output, the
125/// `.zarr` directory it is zipped from. [`finish`](Self::finish) puts it in
126/// place; [`discard`](Self::discard) deletes it.
127#[derive(Debug)]
128#[must_use = "a staged backend is only in place once finished"]
129pub struct Staged {
130    /// The output as named, `.zarr.zip` added under zip.
131    effective: Box<str>,
132    /// The file the backend was written to.
133    working: Box<str>,
134    /// The squeeze [`Output`] asked for, applied when it is finished.
135    squeeze: Option<Squeeze>,
136}
137
138impl Staged {
139    /// The backend as written, to open and read before it is finished.
140    pub fn working(&self) -> &str {
141        &self.working
142    }
143
144    /// Squeeze the backend as its [`Output`] asked, and put it in place;
145    /// the path of the file to open it by.
146    pub fn finish(self) -> anyhow::Result<Box<str>> {
147        if let Some(squeeze) = &self.squeeze {
148            squeeze.apply(&self.working)?;
149        }
150        let found = crate::zarr_io::finalize_output(&self.working, &self.effective)?;
151        info!("done");
152        Ok(found.into())
153    }
154
155    /// Delete the backend, kept nowhere.
156    pub fn discard(self) -> anyhow::Result<()> {
157        remove_build(&self.working)
158    }
159}
160
161/// Delete the backend written at `working`, and the stage it was built in
162/// once nothing else is built there.
163pub(crate) fn remove_build(working: &str) -> anyhow::Result<()> {
164    info!("deleting {working}");
165    open_sparse_matrix_by_path(working)?.remove_backend_file()?;
166    crate::zarr_io::leave_stage(working);
167    Ok(())
168}
169
170/// Log how many rows of each feature type `label` lists.
171pub(crate) fn log_feature_type_histogram(label: &str, row_types: &[Box<str>]) {
172    let mut counts: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new();
173    for t in row_types {
174        *counts.entry(t.as_ref()).or_insert(0) += 1;
175    }
176    info!("Feature types in {}: {:?}", label, counts);
177}
178
179#[cfg(test)]
180mod tests;
data_beans/import.rs

data_beans/
import.rs
