Trait SparseIo 

pub trait SparseIo: Sync + Send {
    type IndexIter: IntoIterator<Item = usize> + FromIterator<usize>;

    // Required methods
    fn read_triplets_by_rows(
        &self,
        rows: Self::IndexIter,
    ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>;
    fn read_triplets_by_columns(
        &self,
        columns: Self::IndexIter,
    ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>;
    fn read_triplets_by_single_column(
        &self,
        col: usize,
    ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>;
    fn to_mtx_file(&self, mtx_file: &str) -> Result<()>;
    fn num_rows(&self) -> Option<usize>;
    fn num_columns(&self) -> Option<usize>;
    fn num_non_zeros(&self) -> Option<usize>;
    fn reopen_backend(&mut self) -> Result<()>;
    fn column_indptr(&self) -> &[u64];
    fn register_row_names_file(&mut self, row_name_file: &str);
    fn register_column_names_file(&mut self, column_name_file: &str);
    fn register_row_names_vec(&mut self, rows: &[Box<str>]);
    fn register_column_names_vec(&mut self, columns: &[Box<str>]);
    fn register_names_file(
        &mut self,
        key: &str,
        name_file: &str,
        name_columns: Range<usize>,
        name_sep: &str,
    ) -> Result<()>;
    fn register_names_vec(
        &mut self,
        key: &str,
        names: &[Box<str>],
    ) -> Result<()>;
    fn row_names(&self) -> Result<Vec<Box<str>>>;
    fn column_names(&self) -> Result<Vec<Box<str>>>;
    fn retrieve_registered_names(&self, key: &str) -> Result<Vec<Box<str>>>;
    fn remove_backend_file(&self) -> Result<()>;
    fn initialize_backend(&mut self) -> Result<()>;
    fn record_mtx_shape(
        &mut self,
        mtx_shape: Option<(usize, usize, usize)>,
    ) -> Result<()>;
    fn record_csr_dataset_backend(
        &mut self,
        csr_cols: &[u64],
        csr_vals: &[f32],
        csr_rowptr: &[u64],
    ) -> Result<()>;
    fn record_csc_dataset_backend(
        &mut self,
        csc_rows: &[u64],
        csc_vals: &[f32],
        csc_colptr: &[u64],
    ) -> Result<()>;
    fn cs_create(&mut self, key: CsKey, len: usize) -> Result<()>;
    fn cs_write_u64(
        &mut self,
        key: CsKey,
        offset: u64,
        data: &[u64],
    ) -> Result<()>;
    fn cs_write_f32(
        &mut self,
        key: CsKey,
        offset: u64,
        data: &[f32],
    ) -> Result<()>;
    fn read_row_indptr(&mut self) -> Result<()>;
    fn read_column_indptr(&mut self) -> Result<()>;
    fn preload_columns(&mut self) -> Result<()>;
    fn clean_preloaded_columns(&mut self);
    fn preload_rows(&mut self) -> Result<()>;
    fn clean_preloaded_rows(&mut self);
    fn get_backend_file_name(&self) -> &str;
    fn backend_type(&self) -> SparseIoBackend;

    // Provided methods
    fn read_columns_ndarray(
        &self,
        columns: Self::IndexIter,
    ) -> Result<Array2<f32>> { ... }
    fn read_columns_tensor(&self, columns: Self::IndexIter) -> Result<Tensor> { ... }
    fn read_columns_dmatrix(
        &self,
        columns: Self::IndexIter,
    ) -> Result<DMatrix<f32>> { ... }
    fn read_columns_csr(
        &self,
        columns: Self::IndexIter,
    ) -> Result<CsrMatrix<f32>> { ... }
    fn read_columns_csc(
        &self,
        columns: Self::IndexIter,
    ) -> Result<CscMatrix<f32>> { ... }
    fn csc_column_arrays(&self) -> Option<(&[u64], &[u64], &[f32])> { ... }
    fn read_rows_ndarray(&self, rows: Self::IndexIter) -> Result<Array2<f32>> { ... }
    fn read_rows_tensor(&self, rows: Self::IndexIter) -> Result<Tensor> { ... }
    fn read_rows_dmatrix(&self, rows: Self::IndexIter) -> Result<DMatrix<f32>> { ... }
    fn read_rows_csr(&self, rows: Self::IndexIter) -> Result<CsrMatrix<f32>> { ... }
    fn read_rows_csc(&self, rows: Self::IndexIter) -> Result<CscMatrix<f32>> { ... }
    fn import_mtx_file(
        &mut self,
        mtx_file: &str,
        index_by_row: bool,
    ) -> Result<()> { ... }
    fn import_dmatrix_by_row(&mut self, matrix: &DMatrix<f32>) -> Result<()> { ... }
    fn import_dmatrix_by_col(&mut self, matrix: &DMatrix<f32>) -> Result<()> { ... }
    fn import_ndarray_by_row(&mut self, array: &Array2<f32>) -> Result<()> { ... }
    fn import_ndarray_by_col(&mut self, array: &Array2<f32>) -> Result<()> { ... }
    fn column_nnz(&self, col: usize) -> Option<u64> { ... }
    fn subset_columns_rows(
        &mut self,
        columns: Option<&Vec<usize>>,
        rows: Option<&Vec<usize>>,
    ) -> Result<()> { ... }
    fn reorder_rows(&mut self, row_names_order: &[Box<str>]) -> Result<()> { ... }
    fn record_triplets_by_row(
        &mut self,
        row_col_val_triplets: &mut Vec<(u64, u64, f32)>,
    ) -> Result<()> { ... }
    fn record_triplets_by_col(
        &mut self,
        row_col_val_triplets: &mut Vec<(u64, u64, f32)>,
    ) -> Result<()> { ... }
    fn begin_streaming_csc(
        &mut self,
        shape: (usize, usize, usize),
    ) -> Result<()> { ... }
    fn append_csc_slab(
        &mut self,
        col_offset: u64,
        nnz_offset: u64,
        local_colptr: &[u64],
        row_indices: &[u64],
        values: &[f32],
    ) -> Result<()> { ... }
    fn finalize_streaming_csc(&mut self) -> Result<()> { ... }
    fn begin_streaming_csr(
        &mut self,
        shape: (usize, usize, usize),
    ) -> Result<()> { ... }
    fn append_csr_slab(
        &mut self,
        row_offset: u64,
        nnz_offset: u64,
        local_rowptr: &[u64],
        col_indices: &[u64],
        values: &[f32],
    ) -> Result<()> { ... }
    fn finalize_streaming_csr(&mut self) -> Result<()> { ... }
    fn build_csr_from_csc_streaming(&mut self) -> Result<()> { ... }
}

Required Associated Types

type IndexIter: IntoIterator<Item = usize> + FromIterator<usize>

Required Methods

fn read_triplets_by_rows( &self, rows: Self::IndexIter, ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>

Read rows within the range and return a vector of triplets (row, column, value)

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_triplets_by_columns( &self, columns: Self::IndexIter, ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>

Read columns within the range and return a vector of triplets (row, col, value)

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_triplets_by_single_column( &self, col: usize, ) -> Result<(usize, usize, Vec<(u64, u64, f32)>)>

Read columns within the range and return a vector of triplets (row, col, value)

• col : usize

fn to_mtx_file(&self, mtx_file: &str) -> Result<()>

Export the data to a mtx file. This will take time.

• mtx_file: mtx file to be written

fn num_rows(&self) -> Option<usize>

Number of rows in the underlying data matrix

fn num_columns(&self) -> Option<usize>

Number of columns in the underlying data matrix

fn num_non_zeros(&self) -> Option<usize>

Number of non-zero elements

fn reopen_backend(&mut self) -> Result<()>

Re-open handles on the CURRENT backend path after its contents were replaced from outside (a finished temp file renamed into place). The zarr store is path-addressed so this is a cache refresh; hdf5 holds an open file handle that would otherwise point at the deleted inode.

fn column_indptr(&self) -> &[u64]

The resident by-column indptr, loaded at open(). Empty when the backend carries no /by_column/indptr array — read_column_indptr silently does nothing on that failure, and the accessors below must report that absence rather than read zeros out of it.

fn register_row_names_file(&mut self, row_name_file: &str)

Set row names for the matrix

• row_name_file: a file each line contains row name words

fn register_column_names_file(&mut self, column_name_file: &str)

Set column names for the matrix

• column_name_file: a file each line contains column name words

fn register_row_names_vec(&mut self, rows: &[Box<str>])

Set row names for the matrix

• rows: a vector of row names

fn register_column_names_vec(&mut self, columns: &[Box<str>])

Set column names for the matrix

• columns: a vector of column names

fn register_names_file( &mut self, key: &str, name_file: &str, name_columns: Range<usize>, name_sep: &str, ) -> Result<()>

Add arbitrary names (a vector of strings)

• group_name: group name
• name_file: a file each line contains name words
• name_columns: range of columns to be used for name
• name_sep: separator for name columns

fn register_names_vec(&mut self, key: &str, names: &[Box<str>]) -> Result<()>

Add arbitrary names (a vector of strings)

• group_name: group name
• names: a file each line contains name words

fn row_names(&self) -> Result<Vec<Box<str>>>

fn column_names(&self) -> Result<Vec<Box<str>>>

fn retrieve_registered_names(&self, key: &str) -> Result<Vec<Box<str>>>

Get back the registered names

• key: key for the registered names

fn remove_backend_file(&self) -> Result<()>

Remove backend file

fn initialize_backend(&mut self) -> Result<()>

Initialize backend

fn record_mtx_shape( &mut self, mtx_shape: Option<(usize, usize, usize)>, ) -> Result<()>

fn record_csr_dataset_backend( &mut self, csr_cols: &[u64], csr_vals: &[f32], csr_rowptr: &[u64], ) -> Result<()>

CSR data structure in Zarr backend

    └── by_row
        ├── data
        ├── indices (column indices)
        └── isndptr (row pointers)

fn record_csc_dataset_backend( &mut self, csc_rows: &[u64], csc_vals: &[f32], csc_colptr: &[u64], ) -> Result<()>

Helper function to add CSC dataset to HDF5 backend

Helper function to record the CSC dataset
    ├── by_column
    │   ├── data
    │   ├── indices (row indices)
    │   └── indptr (column pointers)

fn cs_create(&mut self, key: CsKey, len: usize) -> Result<()>

Create a fixed-size 1-D backend dataset of len elements for the given CSC/CSR slot. No data is written yet.

fn cs_write_u64(&mut self, key: CsKey, offset: u64, data: &[u64]) -> Result<()>

Write a u64 slab at offset in the specified dataset. Used for CSC/CSR indices and indptr.

fn cs_write_f32(&mut self, key: CsKey, offset: u64, data: &[f32]) -> Result<()>

Write an f32 slab at offset in the specified dataset. Used for CSC/CSR data.

fn read_row_indptr(&mut self) -> Result<()>

preload row index pointers

fn read_column_indptr(&mut self) -> Result<()>

preload column index pointers

fn preload_columns(&mut self) -> Result<()>

preload all the columns for faster processing

fn clean_preloaded_columns(&mut self)

unload the memory

fn preload_rows(&mut self) -> Result<()>

preload all the rows for faster processing

fn clean_preloaded_rows(&mut self)

unload the row memory

fn get_backend_file_name(&self) -> &str

backend file name

fn backend_type(&self) -> SparseIoBackend

backend file type

Provided Methods

fn read_columns_ndarray(&self, columns: Self::IndexIter) -> Result<Array2<f32>>

Read columns within the range and return dense ndarray::Array2

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_columns_tensor(&self, columns: Self::IndexIter) -> Result<Tensor>

Read columns within the range and return dense candle_core::Tensor

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_columns_dmatrix(&self, columns: Self::IndexIter) -> Result<DMatrix<f32>>

Read columns within the range and return dense nalgebrea::DMatrix

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_columns_csr(&self, columns: Self::IndexIter) -> Result<CsrMatrix<f32>>

Read columns within the range and return sparse CsrMatrix

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_columns_csc(&self, columns: Self::IndexIter) -> Result<CscMatrix<f32>>

Read columns within the range and return sparse CsrMatrix

• columns : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn csc_column_arrays(&self) -> Option<(&[u64], &[u64], &[f32])>

Zero-copy view of preloaded column-major CSC arrays as (indptr, indices, data). Returns None when the backend has not preloaded columns or doesn’t support direct array access. Callers (e.g. SparseIoVec::read_columns_csc) use this to skip the triplet roundtrip when columns are already in memory.

fn read_rows_ndarray(&self, rows: Self::IndexIter) -> Result<Array2<f32>>

Read rows within the range and return dense ndarray::Array2

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_rows_tensor(&self, rows: Self::IndexIter) -> Result<Tensor>

Read rows within the range and return dense candle_core::Tensor

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_rows_dmatrix(&self, rows: Self::IndexIter) -> Result<DMatrix<f32>>

Read rows within the range and return dense nalgebra::DMatrix

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_rows_csr(&self, rows: Self::IndexIter) -> Result<CsrMatrix<f32>>

Read rows within the range and return sparse CsrMatrix

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn read_rows_csc(&self, rows: Self::IndexIter) -> Result<CscMatrix<f32>>

Read rows within the range and return sparse CscMatrix

• rows : range e.g., 0..3 -> [0, 1, 2] or vec![0, 1, 2]

fn import_mtx_file(&mut self, mtx_file: &str, index_by_row: bool) -> Result<()>

Read an mtx file once and populate the backend: the column (CSC) index always, the row (CSR) index as well when index_by_row. Both are streamed out of the same triplet vector, so the file is inflated once and the triplets are the only full-size structure alive.

• mtx_file: mtx file to be read into the backend

fn import_dmatrix_by_row(&mut self, matrix: &DMatrix<f32>) -> Result<()>

Add dmatrix to zarr backend by row (CSR format)

• array - 2D array to be added to the backend

fn import_dmatrix_by_col(&mut self, matrix: &DMatrix<f32>) -> Result<()>

Add dmatrix to zarr backend by column (CSC format)

• array - 2D array to be added to the backend

fn import_ndarray_by_row(&mut self, array: &Array2<f32>) -> Result<()>

Add ndarray to zarr backend by row (CSR format)

• array - 2D array to be added to the backend

fn import_ndarray_by_col(&mut self, array: &Array2<f32>) -> Result<()>

Add ndarray to zarr backend by column (CSC format)

• array - 2D array to be added to the backend

fn column_nnz(&self, col: usize) -> Option<u64>

Exact nnz of one column, from the resident indptr — no I/O.

None for an out-of-range column or when the indptr is absent. This is what lets a streaming writer declare a column subset’s total nnz up front without a counting pass over the data.

fn subset_columns_rows( &mut self, columns: Option<&Vec<usize>>, rows: Option<&Vec<usize>>, ) -> Result<()>

Select the columns of the data and create a new backend file

• columns: columns to be subsetted
• rows: if something, subset the rows

fn reorder_rows(&mut self, row_names_order: &[Box<str>]) -> Result<()>

Reposition rows in a new order specified by remap

• row_names_order - a vector of row names in the new order

fn record_triplets_by_row( &mut self, row_col_val_triplets: &mut Vec<(u64, u64, f32)>, ) -> Result<()>

Stream the triplets out as CSR slabs; the row-major twin of record_triplets_by_col.

fn record_triplets_by_col( &mut self, row_col_val_triplets: &mut Vec<(u64, u64, f32)>, ) -> Result<()>

Stream the triplets out as CSC slabs.

After the one in-place sort the triplet vector is the only full-size structure alive; the slab staging buffers are bounded by [SLAB_NNZ], and the backend’s streaming audits check the column tiling and the appended count on the way through.

fn begin_streaming_csc(&mut self, shape: (usize, usize, usize)) -> Result<()>

Begin a streaming CSC build for a sparse matrix of known shape. Pre-creates /by_column/{data, indices, indptr} at their final sizes so subsequent append_csc_slab calls write into disjoint hyperslabs without further allocation.

fn append_csc_slab( &mut self, col_offset: u64, nnz_offset: u64, local_colptr: &[u64], row_indices: &[u64], values: &[f32], ) -> Result<()>

Append one contiguous CSC column band.

• col_offset — global column index where this band starts
• nnz_offset — global nnz offset where this band’s values land
• local_colptr — length batch_ncol, values in [0, batch_nnz], will be shifted by nnz_offset before writing
• row_indices — length batch_nnz
• values — length batch_nnz

fn finalize_streaming_csc(&mut self) -> Result<()>

Finalize CSC streaming by writing the final indptr sentinel at position ncol, equal to the total nnz.

fn begin_streaming_csr(&mut self, shape: (usize, usize, usize)) -> Result<()>

Begin a streaming CSR build for a sparse matrix of known shape; the row-major twin of begin_streaming_csc.

fn append_csr_slab( &mut self, row_offset: u64, nnz_offset: u64, local_rowptr: &[u64], col_indices: &[u64], values: &[f32], ) -> Result<()>

Append one contiguous CSR row band; the row-major twin of append_csc_slab, with the same audits.

• row_offset — global row index where this band starts
• nnz_offset — global nnz offset where this band’s values land
• local_rowptr — length batch_nrow, values in [0, batch_nnz], will be shifted by nnz_offset before writing
• col_indices — length batch_nnz
• values — length batch_nnz

fn finalize_streaming_csr(&mut self) -> Result<()>

Finalize CSR streaming: write the indptr sentinel at position nrow, load the row index, and check the appended count against the declared nnz – the one violation the written indptr cannot reveal.

fn build_csr_from_csc_streaming(&mut self) -> Result<()>

Build /by_row/{data, indices, indptr} by transposing the already- written CSC data on disk. Uses two passes over CSC with bounded auxiliary memory (~24 B × nrow plus one row-band worth of CSR).

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl SparseIo for SparseMtxData

type IndexIter = Vec<usize>