Struct ChunkedDataset 

pub struct ChunkedDataset { /* private fields */ }

Chunked dataset manager

Implementations

impl ChunkedDataset

pub fn from_arrays( x: &Array2<Float>, y: &Array1<Float>, config: ChunkedProcessingConfig, ) -> Result<Self>

Create a new chunked dataset from arrays

pub fn from_files( _data_files: Vec<PathBuf>, _config: ChunkedProcessingConfig, ) -> Result<Self>

Create chunked dataset from files

pub fn get_chunk( &mut self, chunk_id: usize, ) -> Result<(&Array2<Float>, &Array1<Float>)>

Get a chunk by ID

pub fn chunk_iter(&mut self) -> ChunkIterator<'_>

Get chunk iterator

pub fn n_chunks(&self) -> usize

Get number of chunks

pub fn n_samples(&self) -> usize

Get total number of samples

pub fn n_features(&self) -> usize

Get number of features

pub fn process_chunks<F, R>(&mut self, processor: F) -> Result<Vec<R>>

where F: FnMut(usize, &Array2<Float>, &Array1<Float>) -> Result<R>,

Process chunks with a given function

pub fn compute_stats(&mut self) -> Result<ChunkedDatasetStats>

Compute statistics across all chunks

Trait Implementations

impl Drop for ChunkedDataset

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.