Struct MemoryMappedArray 

pub struct MemoryMappedArray<A>
where
    A: Clone + Copy + 'static + Send + Sync,
{
    pub shape: Vec<usize>,
    pub file_path: PathBuf,
    pub mode: AccessMode,
    pub offset: usize,
    pub size: usize,
    /* private fields */
}

Memory-mapped array that efficiently maps file data directly into memory

Fields

shape: Vec<usize>

The shape of the array

file_path: PathBuf

The path to the mapped file

mode: AccessMode

The access mode

offset: usize

The offset in the file where the data starts (in bytes)

size: usize

The total number of elements

Implementations

impl<A> MemoryMappedArray<A>

where A: Clone + Copy + 'static + Send + Sync,

pub fn clone_ref(&self) -> CoreResult<Self>

Create a new reference to the same memory-mapped file

pub fn as_slice(&self) -> &[A]

Get the underlying slice of data

pub fn path(filepath: &Path, shape: &[usize]) -> Result<Self, CoreError>

Open an existing memory-mapped array file

pub fn new<S, D>( data: Option<&ArrayBase<S, D>>, file_path: &Path, mode: AccessMode, offset: usize, ) -> Result<Self, CoreError>

where S: Data<Elem = A>, D: Dimension,

Create a new memory-mapped array from an existing array

Arguments

• data - The source array to map to a file
• file_path - The path to the file to create or open
• mode - The access mode
• offset - The offset in the file where the data should start (in bytes)

Returns

A new MemoryMappedArray instance

pub fn new_temp<S, D>( data: &ArrayBase<S, D>, mode: AccessMode, offset: usize, ) -> Result<Self, CoreError>

where S: Data<Elem = A>, D: Dimension,

Create a new memory-mapped array with a temporary file

Arguments

• data - The source array to map to a temporary file
• mode - The access mode
• offset - The offset in the file where the data should start (in bytes)

Returns

A new MemoryMappedArray instance backed by a temporary file

pub fn as_array<D>(&self) -> Result<Array<A, D>, CoreError>

where D: Dimension,

Get a view of the array data as an ndarray Array with the given dimension

Returns

An ndarray Array view of the memory-mapped data

pub fn as_array_mut<D>(&mut self) -> Result<ArrayViewMut<'_, A, D>, CoreError>

where D: Dimension,

Get a mutable view of the array data as an ndarray ArrayViewMut with the given dimension

Returns

A mutable ndarray ArrayViewMut of the memory-mapped data

Errors

Returns an error if the array is in read-only mode

pub fn flush(&mut self) -> Result<(), CoreError>

Flush changes to disk if the array is writable

Returns

Ok(()) if the flush succeeded, or an error

pub fn reload(&mut self) -> Result<(), CoreError>

Reload the memory mapping from disk

This function is useful when changes have been made to the underlying file by other processes or by direct file I/O operations.

Returns

Ok(()) if the reload succeeded, or an error

pub fn is_temp(&self) -> bool

Check if the array is temporary

Returns

true if the array is backed by a temporary file, false otherwise

pub fn as_bytes(&self) -> Result<&[u8], CoreError>

Get a view of the memory-mapped data as bytes

Returns

A byte slice view of the memory-mapped data

pub fn as_bytes_mut(&mut self) -> Result<&mut [u8], CoreError>

Get a mutable view of the memory-mapped data as bytes

Returns

A mutable byte slice view of the memory-mapped data

impl<A: ZeroCopySerializable> MemoryMappedArray<A>

pub fn save_array<S, D>( data: &ArrayBase<S, D>, file_path: impl AsRef<Path>, metadata: Option<Value>, ) -> CoreResult<Self>

where S: Data<Elem = A>, D: Dimension,

Create a new memory-mapped array from an existing array and save with zero-copy serialization.

This method provides a convenient way to convert a standard ndarray to a memory-mapped array with zero-copy serialization in a single operation. It’s particularly useful for initializing memory-mapped arrays with data.

Arguments

• data - The source ndarray to be converted and saved
• filepath - Path where the memory-mapped array will be saved
• metadata - Optional metadata to include with the array

Returns

A new memory-mapped array with read-write access to the saved data

Example

// Create an ndarray
let data = Array2::<f64>::from_shape_fn((100, 100), |(i, j)| (i * 100 + j) as f64);

// Create metadata
let metadata = json!({"description": "Temperature data", "units": "Celsius"});

// Convert to memory-mapped array and save
let mmap = MemoryMappedArray::<f64>::save_array(&data, "temperature.bin", Some(metadata))?;

pub fn open_zero_copy( filepath: impl AsRef<Path>, mode: AccessMode, ) -> CoreResult<Self>

Open a zero-copy serialized memory-mapped array from a file.

This is a convenient wrapper around the load_zero_copy method with a more intuitive name. It memory-maps a file containing a previously serialized array, providing efficient access to the data.

Arguments

• filepath - Path to the file containing the serialized array
• mode - Access mode (ReadOnly, ReadWrite, or CopyOnWrite)

Returns

A memory-mapped array or an error with context

Example

// Open a memory-mapped array with read-only access
let array = MemoryMappedArray::<f64>::open_zero_copy("data/temperature.bin", AccessMode::ReadOnly)?;

// Access the array
let ndarray = array.readonlyarray::<scirs2_core::ndarray::Ix2>()?;
println!("First value: {}", ndarray[[0, 0]]);

pub fn read_metadata(filepath: impl AsRef<Path>) -> CoreResult<Value>

Read the metadata from a zero-copy serialized file without loading the entire array.

This method efficiently extracts just the metadata from a file without memory-mapping the entire array data. This is useful for checking array properties or file information before deciding whether to load the full array.

Arguments

• filepath - Path to the file containing the serialized array

Returns

The metadata as a JSON value or an empty JSON object if no metadata was stored

Example

// Read metadata without loading the array
let metadata = MemoryMappedArray::<f64>::read_metadata("data/large_dataset.bin")?;

// Check properties
if let Some(created) = metadata.get("created") {
    println!("Dataset created on: {}", created);
}

if let Some(dimensions) = metadata.get("dimensions") {
    println!("Dataset dimensions: {}", dimensions);
}

pub fn readonlyarray<D>(&self) -> CoreResult<Array<A, D>>

where D: Dimension,

Get a read-only view of the array as an ndarray Array.

This method provides a convenient way to access the memory-mapped array as a standard ndarray Array with the specified dimensionality.

Type Parameters

• D - The dimensionality for the returned array (e.g., Ix1, Ix2, IxDyn)

Returns

A read-only ndarray Array view of the memory-mapped data

Example

let array = MemoryMappedArray::<f64>::open_zero_copy("matrix.bin", AccessMode::ReadOnly)?;

// Access as a 2D ndarray
let ndarray = array.readonlyarray::<Ix2>()?;

// Now you can use all the ndarray methods
let sum = ndarray.sum();
let mean = ndarray.mean_or(0.0);
println!("Matrix sum: {}, mean: {}", sum, mean);

pub fn update_metadata( file_path: impl AsRef<Path>, metadata: Value, ) -> CoreResult<()>

Update metadata in a zero-copy serialized file without rewriting the entire array.

This method efficiently updates just the metadata portion of a serialized array file without touching the actual array data. When possible, it performs the update in place to avoid creating a new file.

Arguments

• filepath - Path to the file containing the serialized array
• metadata - The new metadata to store (as JSON)

Returns

CoreResult<()> indicating success or an error with context

Behavior

• If the new metadata is the same size or smaller than the original, the update is done in-place
• If the new metadata is larger, the entire file is rewritten to maintain proper alignment

Example

// Add processing information to the metadata
let updated_metadata = json!({
    "description": "Temperature dataset",
    "processed": true,
    "processing_date": "2023-05-21",
    "normalization_applied": true,
    "outliers_removed": 12
});

// Update the metadata without affecting the array data
MemoryMappedArray::<f64>::update_metadata("data/temperature.bin", updated_metadata)?;

Trait Implementations

impl<A: Clone + Copy + 'static + Send + Sync> AdaptiveChunking<A> for MemoryMappedArray<A>

fn adaptive_chunking( &self, params: AdaptiveChunkingParams, ) -> CoreResult<AdaptiveChunkingResult>

Calculate an optimal chunking strategy based on array characteristics.

fn process_chunks_adaptive<F, R>( &self, params: AdaptiveChunkingParams, f: F, ) -> CoreResult<Vec<R>>

where F: Fn(&[A], usize) -> R,

Process chunks using an automatically determined optimal chunking strategy.

fn process_chunks_mut_adaptive<F>( &mut self, params: AdaptiveChunkingParams, f: F, ) -> CoreResult<()>

where F: Fn(&mut [A], usize),

Process chunks mutably using an automatically determined optimal chunking strategy.

fn process_chunks_parallel_adaptive<F, R>( &self, params: AdaptiveChunkingParams, f: F, ) -> CoreResult<Vec<R>>

where F: Fn(&[A], usize) -> R + Send + Sync, R: Send, A: Send + Sync,

Process chunks in parallel using an automatically determined optimal chunking strategy.

impl<A: Clone + Copy + 'static + Send + Sync + Zero> ArithmeticOps<A> for MemoryMappedArray<A>

fn add(&self, other: &Self) -> CoreResult<MemoryMappedArray<A>>

where A: Add<Output = A>,

Adds two arrays element-wise.

fn sub(&self, other: &Self) -> CoreResult<MemoryMappedArray<A>>

where A: Sub<Output = A>,

Subtracts another array from this one element-wise.

fn mul(&self, other: &Self) -> CoreResult<MemoryMappedArray<A>>

where A: Mul<Output = A>,

Multiplies two arrays element-wise.

fn div(&self, other: &Self) -> CoreResult<MemoryMappedArray<A>>

where A: Div<Output = A>,

Divides this array by another element-wise.

impl<A: Clone + Copy + 'static + Send + Sync + Zero> BroadcastOps<A> for MemoryMappedArray<A>

fn broadcast_op<F>( &self, other: &Self, f: F, ) -> CoreResult<MemoryMappedArray<A>>

where F: Fn(A, A) -> A + Send + Sync,

Broadcasts an operation between two arrays of compatible shapes.

impl<A> Clone for MemoryMappedArray<A>

where A: Clone + Copy + 'static + Send + Sync,

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<A> DataExchange<A> for MemoryMappedArray<A>

where A: Clone + Copy + 'static + Send + Sync + Debug,

fn export_data( &self, interface: &ZeroCopyInterface, name: &str, ) -> CoreResult<DataId>

Export data to the zero-copy interface

fn from_interface(interface: &ZeroCopyInterface, name: &str) -> CoreResult<Self>

Import data from the zero-copy interface

impl<A> Debug for MemoryMappedArray<A>

where A: Clone + Copy + 'static + Send + Sync + Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<A> Drop for MemoryMappedArray<A>

where A: Clone + Copy + 'static + Send + Sync,

fn drop(&mut self)

Executes the destructor for this type.

impl<A: Clone + Copy + 'static + Send + Sync> MemoryMappedChunkIter<A> for MemoryMappedArray<A>

fn chunks(&self, strategy: ChunkingStrategy) -> ChunkIter<'_, A>

Create an iterator over chunks of the array (for 1D arrays only).

impl<A: Clone + Copy + 'static + Send + Sync> MemoryMappedChunks<A> for MemoryMappedArray<A>

fn chunk_count(&self, strategy: ChunkingStrategy) -> usize

Get the number of chunks for the given chunking strategy.

fn process_chunks<F, R>(&self, strategy: ChunkingStrategy, f: F) -> Vec<R>

where F: Fn(&[A], usize) -> R,

Process each chunk with a function and collect the results.

fn process_chunks_mut<F>(&mut self, strategy: ChunkingStrategy, f: F)

where F: Fn(&mut [A], usize),

Process each chunk with a mutable function that modifies the data in-place.

impl<A: Clone + Copy + 'static + Send + Sync> MemoryMappedChunksParallel<A> for MemoryMappedArray<A>

Available on crate feature parallel only.

fn process_chunks_parallel<F, R>( &self, strategy: ChunkingStrategy, f: F, ) -> Vec<R>

where F: Fn(&[A], usize) -> R + Send + Sync, R: Send,

Process chunks in parallel and collect the results.

fn process_chunks_mut_parallel<F>(&mut self, strategy: ChunkingStrategy, f: F)

where F: Fn(&mut [A], usize) + Send + Sync,

Process chunks in parallel with a mutable function.

impl<A: Clone + Copy + 'static + Send + Sync> MemoryMappedSlicing<A> for MemoryMappedArray<A>

fn slice<I, E>(&self, sliceinfo: I) -> CoreResult<MemoryMappedSlice<A, E>>

where I: SliceArg<E>, E: Dimension,

Creates a slice of the memory-mapped array using standard slice syntax.

fn slice_1d( &self, range: impl RangeBounds<usize>, ) -> CoreResult<MemoryMappedSlice<A, Ix1>>

Creates a 1D slice using a range.

fn slice_2d( &self, row_range: impl RangeBounds<usize>, col_range: impl RangeBounds<usize>, ) -> CoreResult<MemoryMappedSlice<A, Ix2>>

Creates a 2D slice using ranges for each dimension.

impl<A: Clone + Copy + 'static + Send + Sync + Zero> ZeroCopyOps<A> for MemoryMappedArray<A>

fn map_zero_copy<F>(&self, f: F) -> CoreResult<MemoryMappedArray<A>>

where F: Fn(A) -> A + Send + Sync,

Maps a function over each element of the array without loading the entire array.

fn reduce_zero_copy<F>(&self, init: A, f: F) -> CoreResult<A>

where F: Fn(A, A) -> A + Send + Sync,

Reduces the array to a single value by applying a binary operation.

fn combine_zero_copy<F>( &self, other: &Self, f: F, ) -> CoreResult<MemoryMappedArray<A>>

where F: Fn(A, A) -> A + Send + Sync,

Performs a binary operation between two memory-mapped arrays element-wise.

fn filter_zero_copy<F>(&self, predicate: F) -> CoreResult<Vec<A>>

where F: Fn(&A) -> bool + Send + Sync,

Filters elements based on a predicate function.

fn max_zero_copy(&self) -> CoreResult<A>

where A: PartialOrd,

Returns the maximum element in the array.

fn min_zero_copy(&self) -> CoreResult<A>

where A: PartialOrd,

Returns the minimum element in the array.

fn sum_zero_copy(&self) -> CoreResult<A>

where A: Add<Output = A> + From<u8>,

Calculates the sum of all elements in the array.

fn product_zero_copy(&self) -> CoreResult<A>

where A: Mul<Output = A> + From<u8>,

Calculates the product of all elements in the array.

fn mean_zero_copy(&self) -> CoreResult<A>

where A: Add<Output = A> + Div<Output = A> + From<u8> + From<usize>,

Calculates the mean of all elements in the array.

impl<A: ZeroCopySerializable> ZeroCopySerialization<A> for MemoryMappedArray<A>

fn save_zero_copy( &self, path: impl AsRef<Path>, metadata: Option<Value>, ) -> CoreResult<()>

Save the array to a file with zero-copy serialization.

fn load_zero_copy( path: impl AsRef<Path>, mode: AccessMode, ) -> CoreResult<MemoryMappedArray<A>>

Load an array from a file with zero-copy deserialization.

fn as_bytes_slice(&self) -> CoreResult<&[u8]>

Get the raw byte representation of the array.

fn as_bytes_slice_mut(&mut self) -> CoreResult<&mut [u8]>

Get the mutable raw byte representation of the array.