Struct SuperArray 

pub struct SuperArray { /* private fields */ }

SuperArray

Higher-order container for multiple immutable FieldArray segments.

Description

• Stores an ordered sequence of FieldArray chunks, each with identical field metadata.
• Equivalent to Apache Arrow’s ChunkedArray when sent over FFI, where it is treated as a single logical column.
• It can also serve as an unbounded or continuously growing collection of segments, making it useful for streaming ingestion and partitioned storage.
• Chunk lengths may vary without restriction.

Example

// Create from multiple chunks with matching metadata
let sa = SuperArray::from_chunks(vec![fa("col", &[1, 2], 0), fa("col", &[3], 0)]);

assert_eq!(sa.len(), 3);         // total rows across chunks
assert_eq!(sa.n_chunks(), 2);    // number of chunks
assert_eq!(sa.field().name, "col");

Implementations

impl SuperArray

pub fn new() -> Self

Constructs an empty ChunkedArray.

pub fn from_field_array_chunks(chunks: Vec<FieldArray>) -> Self

Constructs a ChunkedArray from FieldArray chunks. Panics if chunks is empty or metadata/type/nullable mismatch is found.

Examples found in repository

examples/broadcasting/test_broadcasting.rs (line 369)

fn test_super_array_broadcasting() {
    #[cfg(feature = "chunked")]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("│  Operation: SuperArray{{[1,2],[3,4]}} * SuperArray{{[2,2],[2,2]}}");
        println!("│  Expected:  SuperArray{{[2,4],[6,8]}}");

        // Create chunked arrays (multiple field array chunks)
        let chunk1_a = Array::from_int32(IntegerArray::from_slice(&vec64![1, 2]));
        let chunk2_a = Array::from_int32(IntegerArray::from_slice(&vec64![3, 4]));
        let fa1 = minarrow::FieldArray::from_arr("chunk1", chunk1_a);
        let fa2 = minarrow::FieldArray::from_arr("chunk1", chunk2_a);
        let super_arr1 = SuperArray::from_field_array_chunks(vec![fa1, fa2]);

        let chunk1_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let chunk2_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let fa3 = minarrow::FieldArray::from_arr("chunk1", chunk1_b);
        let fa4 = minarrow::FieldArray::from_arr("chunk1", chunk2_b);
        let super_arr2 = SuperArray::from_field_array_chunks(vec![fa3, fa4]);

        match Value::SuperArray(Arc::new(super_arr1)) * Value::SuperArray(Arc::new(super_arr2)) {
            Ok(Value::SuperArray(result)) => {
                println!("│  Result with {} chunks:", result.len());
                for i in 0..result.len() {
                    if let Some(fa) = result.chunk(i) {
                        if let Array::NumericArray(NumericArray::Int32(arr)) = &fa.array {
                            println!("│    Chunk {}: {:?}", i, arr.data.as_slice());
                        }
                    }
                }
                println!("└─ ✓ Passed\n");
            }
            Ok(other) => println!("└─ ✗ Error: Unexpected result type {:?}\n", other),
            Err(e) => println!("└─ ✗ Error: {:?}\n", e),
        }
    }

    #[cfg(not(feature = "chunked"))]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("└─ ⊘ Skipped (chunked feature not enabled)\n");
    }
}

pub fn from_chunks(chunks: Vec<FieldArray>) -> Self

Construct from Vec<FieldArray>.

pub fn from_slices(slices: &[ArrayV], field: Arc<Field>) -> Self

Materialises a ChunkedArray from an existing slice of ArrayView tuples, using the provided field metadata (applied to all slices).

Panics if the slice list is empty, or if any slice’s type or nullability does not match the provided field.

pub fn slice(&self, offset: usize, len: usize) -> SuperArrayV

Returns a zero-copy view of this chunked array over the window [offset..offset+len).

If the chunks are fragmented in memory, access patterns may result in degraded cache locality and reduced SIMD optimisation.

pub fn copy_to_array(&self) -> Array

Materialises a contiguous Array holding all rows.

pub fn field(&self) -> &Field

Returns the field metadata from the first chunk (guaranteed by constructor).

pub fn arrow_type(&self) -> ArrowType

Returns the Arrow physical type.

pub fn is_nullable(&self) -> bool

Returns the nullability flag.

pub fn n_chunks(&self) -> usize

Returns the number of logical chunks.

pub fn len(&self) -> usize

Returns total logical length (sum of all chunk lengths).

Examples found in repository

examples/broadcasting/test_broadcasting.rs (line 379)

fn test_super_array_broadcasting() {
    #[cfg(feature = "chunked")]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("│  Operation: SuperArray{{[1,2],[3,4]}} * SuperArray{{[2,2],[2,2]}}");
        println!("│  Expected:  SuperArray{{[2,4],[6,8]}}");

        // Create chunked arrays (multiple field array chunks)
        let chunk1_a = Array::from_int32(IntegerArray::from_slice(&vec64![1, 2]));
        let chunk2_a = Array::from_int32(IntegerArray::from_slice(&vec64![3, 4]));
        let fa1 = minarrow::FieldArray::from_arr("chunk1", chunk1_a);
        let fa2 = minarrow::FieldArray::from_arr("chunk1", chunk2_a);
        let super_arr1 = SuperArray::from_field_array_chunks(vec![fa1, fa2]);

        let chunk1_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let chunk2_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let fa3 = minarrow::FieldArray::from_arr("chunk1", chunk1_b);
        let fa4 = minarrow::FieldArray::from_arr("chunk1", chunk2_b);
        let super_arr2 = SuperArray::from_field_array_chunks(vec![fa3, fa4]);

        match Value::SuperArray(Arc::new(super_arr1)) * Value::SuperArray(Arc::new(super_arr2)) {
            Ok(Value::SuperArray(result)) => {
                println!("│  Result with {} chunks:", result.len());
                for i in 0..result.len() {
                    if let Some(fa) = result.chunk(i) {
                        if let Array::NumericArray(NumericArray::Int32(arr)) = &fa.array {
                            println!("│    Chunk {}: {:?}", i, arr.data.as_slice());
                        }
                    }
                }
                println!("└─ ✓ Passed\n");
            }
            Ok(other) => println!("└─ ✗ Error: Unexpected result type {:?}\n", other),
            Err(e) => println!("└─ ✗ Error: {:?}\n", e),
        }
    }

    #[cfg(not(feature = "chunked"))]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("└─ ⊘ Skipped (chunked feature not enabled)\n");
    }
}

pub fn is_empty(&self) -> bool

Returns true if the array has no chunks or all chunks are empty.

pub fn chunks(&self) -> &[FieldArray]

Returns a read-only reference to all underlying chunks.

pub fn chunks_mut(&mut self) -> &mut [FieldArray]

Returns a mutable reference to the underlying chunks.

pub fn chunk(&self, idx: usize) -> Option<&FieldArray>

Returns a reference to a specific chunk, if it exists.

Examples found in repository

examples/broadcasting/test_broadcasting.rs (line 381)

fn test_super_array_broadcasting() {
    #[cfg(feature = "chunked")]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("│  Operation: SuperArray{{[1,2],[3,4]}} * SuperArray{{[2,2],[2,2]}}");
        println!("│  Expected:  SuperArray{{[2,4],[6,8]}}");

        // Create chunked arrays (multiple field array chunks)
        let chunk1_a = Array::from_int32(IntegerArray::from_slice(&vec64![1, 2]));
        let chunk2_a = Array::from_int32(IntegerArray::from_slice(&vec64![3, 4]));
        let fa1 = minarrow::FieldArray::from_arr("chunk1", chunk1_a);
        let fa2 = minarrow::FieldArray::from_arr("chunk1", chunk2_a);
        let super_arr1 = SuperArray::from_field_array_chunks(vec![fa1, fa2]);

        let chunk1_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let chunk2_b = Array::from_int32(IntegerArray::from_slice(&vec64![2, 2]));
        let fa3 = minarrow::FieldArray::from_arr("chunk1", chunk1_b);
        let fa4 = minarrow::FieldArray::from_arr("chunk1", chunk2_b);
        let super_arr2 = SuperArray::from_field_array_chunks(vec![fa3, fa4]);

        match Value::SuperArray(Arc::new(super_arr1)) * Value::SuperArray(Arc::new(super_arr2)) {
            Ok(Value::SuperArray(result)) => {
                println!("│  Result with {} chunks:", result.len());
                for i in 0..result.len() {
                    if let Some(fa) = result.chunk(i) {
                        if let Array::NumericArray(NumericArray::Int32(arr)) = &fa.array {
                            println!("│    Chunk {}: {:?}", i, arr.data.as_slice());
                        }
                    }
                }
                println!("└─ ✓ Passed\n");
            }
            Ok(other) => println!("└─ ✗ Error: Unexpected result type {:?}\n", other),
            Err(e) => println!("└─ ✗ Error: {:?}\n", e),
        }
    }

    #[cfg(not(feature = "chunked"))]
    {
        println!("┌─ Test 11: SuperArray (Chunked) Broadcasting");
        println!("└─ ⊘ Skipped (chunked feature not enabled)\n");
    }
}

pub fn push(&mut self, chunk: FieldArray)

Validates and appends a new chunk.

Panics

If the chunk does not match the expected type or nullability.

Trait Implementations

impl Add for SuperArray

Available on crate feature chunked only.

type Output = Result<SuperArray, MinarrowError>

The resulting type after applying the + operator.

fn add(self, rhs: Self) -> Self::Output

Performs the + operation.

impl ByteSize for SuperArray

Available on crate feature chunked only.

fn est_bytes(&self) -> usize

Returns the estimated byte size of this object in memory.

impl Clone for SuperArray

fn clone(&self) -> SuperArray

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Concatenate for SuperArray

fn concat(self, other: Self) -> Result<Self, MinarrowError>

Concatenates two SuperArrays by appending all chunks from other to self.

Requirements

• Both SuperArrays must have the same field metadata (name, type, nullability)

Returns

A new SuperArray containing all chunks from self followed by all chunks from other

Errors

• IncompatibleTypeError if field metadata doesn’t match

impl Debug for SuperArray

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for SuperArray

fn default() -> Self

Returns the “default value” for a type.

impl Display for SuperArray

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Div for SuperArray

Available on crate feature chunked only.

type Output = Result<SuperArray, MinarrowError>

The resulting type after applying the / operator.

fn div(self, rhs: Self) -> Self::Output

Performs the / operation.

impl From<FieldArray> for SuperArray

fn from(field_array: FieldArray) -> Self

Converts to this type from the input type.

impl From<SuperArray> for SuperArrayV

SuperArray -> SuperArrayV conversion

fn from(super_array: SuperArray) -> Self

Converts to this type from the input type.

impl From<SuperArray> for Value

Available on crate feature chunked only.

fn from(v: SuperArray) -> Self

Converts to this type from the input type.

impl FromIterator<FieldArray> for SuperArray

fn from_iter<T: IntoIterator<Item = FieldArray>>(iter: T) -> Self

Creates a value from an iterator.

impl Mul for SuperArray

Available on crate feature chunked only.

type Output = Result<SuperArray, MinarrowError>

The resulting type after applying the * operator.

fn mul(self, rhs: Self) -> Self::Output

Performs the * operation.

impl PartialEq for SuperArray

fn eq(&self, other: &SuperArray) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Rem for SuperArray

Available on crate feature chunked only.

type Output = Result<SuperArray, MinarrowError>

The resulting type after applying the % operator.

fn rem(self, rhs: Self) -> Self::Output

Performs the % operation.

impl Shape for SuperArray

fn shape(&self) -> ShapeDim

Returns arbitrary Shape dimension for any data shape

fn shape_1d(&self) -> usize

Returns the first dimension shape

fn shape_2d(&self) -> (usize, usize)

Returns the first and second dimension shapes

fn shape_3d(&self) -> (usize, usize, usize)

Returns the first, second and third dimension shapes

fn shape_4d(&self) -> (usize, usize, usize, usize)

Returns the first, second, third and fourth dimension shapes

impl Sub for SuperArray

Available on crate feature chunked only.

type Output = Result<SuperArray, MinarrowError>

The resulting type after applying the - operator.

fn sub(self, rhs: Self) -> Self::Output

Performs the - operation.

impl TryFrom<Value> for SuperArray

Available on crate feature chunked only.

type Error = MinarrowError

The type returned in the event of a conversion error.

fn try_from(v: Value) -> Result<Self, Self::Error>

Performs the conversion.

impl StructuralPartialEq for SuperArray