Struct ArrayBuilder 

pub struct ArrayBuilder { /* private fields */ }

An Array builder.

ArrayBuilder is initialised from an array shape, data type, chunk grid, and fill value.

• The default array-to-bytes codec is dependent on the data type:
  • bytes for fixed-length data types,
  • vlen-utf8 for the string variable-length data type,
  • vlen-bytes for the bytes variable-length data type, and
  • vlen for any other variable-length data type.
• Array-to-array and bytes-to-bytes codecs are empty by default.
• The default chunk key encoding is default with the / chunk key separator.
• Attributes, storage transformers, and dimension names are empty.

Use the methods in the array builder to change the configuration away from these defaults, and then build the array at a path of some storage with ArrayBuilder::build.

build does not modify the store! Array metadata has to be explicitly written with Array::store_metadata.

Simple Example

This array is uncompressed, and has no dimension names or attributes.

use zarrs::array::{ArrayBuilder, DataType, FillValue, ZARR_NAN_F32};
let mut array = ArrayBuilder::new(
    vec![8, 8], // array shape
    vec![4, 4], // regular chunk shape
    DataType::Float32, // data type
    f32::NAN, // fill value
)
.build(store.clone(), "/group/array")?;
array.store_metadata()?; // write metadata to the store

Advanced Example

The array is compressed with the zstd codec, dimension names are set, and the experimental rectangular chunk grid is used.

This example uses alternative types to specify the array shape, data type, chunk grid, and fill value. In general you don’t want to use strings, prefer concrete types (like RectangularChunkGridConfiguration, for example).

use zarrs::array::{ArrayBuilder, DataType, FillValue, ZARR_NAN_F32};
let mut array = ArrayBuilder::new(
    [8, 8],
    r#"{"name":"rectangular","configuration":{"chunk_shape": [[1, 2, 5], 4]}}"#,
    "float32",
    "NaN",
)
.bytes_to_bytes_codecs(vec![
    #[cfg(feature = "zstd")]
    Arc::new(zarrs::array::codec::ZstdCodec::new(5, false)),
])
.dimension_names(Some(["y", "x"]))
.build(store.clone(), "/group/array")?;
array.store_metadata()?; // write metadata to the store

Implementations

impl ArrayBuilder

pub fn new( shape: impl Into<ArrayShape>, chunk_grid_metadata: impl Into<ArrayBuilderChunkGridMetadata>, data_type: impl Into<ArrayBuilderDataType>, fill_value: impl Into<ArrayBuilderFillValue>, ) -> Self

Create a new array builder for an array at path from an array shape and chunk grid metadata.

The length of the array shape must match the dimensionality of the intended array. Some chunk grids (e.g. regular) support all zero shape, indicating the shape is unbounded.

Examples found in repository

examples/custom_data_type_variable_size.rs (lines 158-163)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![3, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeVariableSize)),
        [],
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeVariableSizeElement::from(Some(1.0)),
        CustomDataTypeVariableSizeElement::from(None),
        CustomDataTypeVariableSizeElement::from(Some(3.0)),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeVariableSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeVariableSizeElement::from(Some(1.0)));
    assert_eq!(data[1], CustomDataTypeVariableSizeElement::from(None));
    assert_eq!(data[2], CustomDataTypeVariableSizeElement::from(Some(3.0)));
    assert_eq!(data[3], CustomDataTypeVariableSizeElement::from(None));

    println!("{data:#?}");
}

examples/custom_data_type_fixed_size.rs (lines 273-278)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![2, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFixedSize)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
        CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFixedSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
    assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
    assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
    assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });

    println!("{data:#?}");
}

examples/custom_data_type_uint12.rs (lines 209-214)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        vec![5, 1],    // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_float8_e3m4.rs (lines 221-226)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFloat8e3m4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFloat8e3m4Element::from(2.34),
        CustomDataTypeFloat8e3m4Element::from(3.45),
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NAN),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFloat8e3m4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!(
            "float8_e3m4: {:08b} f32: {}",
            f.to_ne_bytes()[0],
            f.as_f32()
        );
    }

    assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
    assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
    assert_eq!(
        data[2],
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
    );
    assert_eq!(
        data[3],
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
    );
    assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
    assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
}

examples/custom_data_type_uint4.rs (lines 207-212)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

examples/array_write_read_string.rs (lines 55-60)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;
    use zarrs::{array::DataType, array_subset::ArraySubset, storage::store};

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![4, 4], // array shape
        vec![2, 2], // regular chunk shape
        DataType::String,
        "_",
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    array.store_chunk_ndarray(
        &[0, 0],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["a", "bb", "ccc", "dddd"]).unwrap(),
    )?;
    array.store_chunk_ndarray(
        &[0, 1],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["4444", "333", "22", "1"]).unwrap(),
    )?;
    let subset_all = array.subset_all();
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    let ndarray_subset: Array2<&str> = array![["!", "@@"], ["###", "$$$$"]];
    array.store_array_subset_ndarray(
        ArraySubset::new_with_ranges(&[1..3, 1..3]).start(),
        ndarray_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_array_subset [1..3, 1..3]:\nndarray::ArrayD<String>\n{data_all}");

    // Retrieve bytes directly, convert into a single string allocation, create a &str ndarray
    // TODO: Add a convenience function for this?
    let data_all = array.retrieve_array_subset(&subset_all)?;
    let (bytes, offsets) = data_all.into_variable()?;
    let string = String::from_utf8(bytes.into_owned())?;
    let elements = offsets
        .iter()
        .tuple_windows()
        .map(|(&curr, &next)| &string[curr..next])
        .collect::<Vec<&str>>();
    let ndarray = ArrayD::<&str>::from_shape_vec(subset_all.shape_usize(), elements)?;
    println!("ndarray::ArrayD<&str>:\n{ndarray}");

    Ok(())
}

Additional examples can be found in:

• examples/rectangular_array_write_read.rs
• examples/array_write_read.rs
• examples/sharded_array_write_read.rs
• examples/array_write_read_ndarray.rs
• examples/async_array_write_read.rs

pub fn new_with_chunk_grid( chunk_grid: impl Into<ArrayBuilderChunkGrid>, data_type: impl Into<ArrayBuilderDataType>, fill_value: impl Into<ArrayBuilderFillValue>, ) -> Self

Create a new array builder with a concrete chunk grid (with an associated array shape).

pub fn from_array<T: ?Sized>(array: &Array<T>) -> Self

Create a new builder copying the configuration of an existing array.

pub fn shape(&mut self, shape: impl Into<ArrayShape>) -> &mut Self

Set the shape.

pub fn data_type( &mut self, data_type: impl Into<ArrayBuilderDataType>, ) -> &mut Self

Set the data type.

pub fn chunk_grid_metadata( &mut self, chunk_grid_metadata: impl Into<ArrayBuilderChunkGridMetadata>, ) -> &mut Self

Set the chunk grid metadata.

pub fn chunk_grid( &mut self, chunk_grid: impl Into<ArrayBuilderChunkGrid>, ) -> &mut Self

Set the chunk grid. This may also change the array shape.

pub fn fill_value( &mut self, fill_value: impl Into<ArrayBuilderFillValue>, ) -> &mut Self

Set the fill value.

pub fn chunk_key_encoding( &mut self, chunk_key_encoding: ChunkKeyEncoding, ) -> &mut Self

Set the chunk key encoding.

If left unmodified, the array will use default chunk key encoding with the / chunk key separator.

pub fn chunk_key_encoding_default_separator( &mut self, separator: ChunkKeySeparator, ) -> &mut Self

Set the chunk key encoding to default with separator.

If left unmodified, the array will use default chunk key encoding with the / chunk key separator.

pub fn array_to_array_codecs( &mut self, array_to_array_codecs: Vec<Arc<dyn ArrayToArrayCodecTraits>>, ) -> &mut Self

Set the array-to-array codecs.

If left unmodified, the array will have no array-to-array codecs.

Examples found in repository

examples/custom_data_type_variable_size.rs (lines 164-169)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![3, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeVariableSize)),
        [],
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeVariableSizeElement::from(Some(1.0)),
        CustomDataTypeVariableSizeElement::from(None),
        CustomDataTypeVariableSizeElement::from(Some(3.0)),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeVariableSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeVariableSizeElement::from(Some(1.0)));
    assert_eq!(data[1], CustomDataTypeVariableSizeElement::from(None));
    assert_eq!(data[2], CustomDataTypeVariableSizeElement::from(Some(3.0)));
    assert_eq!(data[3], CustomDataTypeVariableSizeElement::from(None));

    println!("{data:#?}");
}

examples/custom_data_type_fixed_size.rs (lines 279-284)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![2, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFixedSize)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
        CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFixedSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
    assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
    assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
    assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });

    println!("{data:#?}");
}

examples/custom_data_type_uint12.rs (lines 215-220)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        vec![5, 1],    // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_float8_e3m4.rs (lines 227-232)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFloat8e3m4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFloat8e3m4Element::from(2.34),
        CustomDataTypeFloat8e3m4Element::from(3.45),
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NAN),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFloat8e3m4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!(
            "float8_e3m4: {:08b} f32: {}",
            f.to_ne_bytes()[0],
            f.as_f32()
        );
    }

    assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
    assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
    assert_eq!(
        data[2],
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
    );
    assert_eq!(
        data[3],
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
    );
    assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
    assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
}

examples/custom_data_type_uint4.rs (lines 213-218)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

pub fn array_to_array_codecs_named( &mut self, array_to_array_codecs: Vec<impl Into<NamedArrayToArrayCodec>>, ) -> &mut Self

Set the array-to-array codecs with non-default names.

If left unmodified, the array will have no array-to-array codecs.

pub fn array_to_bytes_codec( &mut self, array_to_bytes_codec: Arc<dyn ArrayToBytesCodecTraits>, ) -> &mut Self

Set the array-to-bytes codec.

If left unmodified, the array will default to using the bytes codec with native endian encoding.

Examples found in repository

examples/custom_data_type_uint12.rs (line 221)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        vec![5, 1],    // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_uint4.rs (line 219)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

examples/sharded_array_write_read.rs (line 75)

fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use zarrs::{
        array::{
            codec::{self, array_to_bytes::sharding::ShardingCodecBuilder},
            DataType,
        },
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use std::sync::Arc;

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/sharded_array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    // Create an array
    let array_path = "/group/array";
    let shard_shape = vec![4, 8];
    let inner_chunk_shape = vec![4, 4];
    let mut sharding_codec_builder =
        ShardingCodecBuilder::new(inner_chunk_shape.as_slice().try_into()?);
    sharding_codec_builder.bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ]);
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        shard_shape,
        DataType::UInt16,
        0u16,
    )
    .array_to_bytes_codec(Arc::new(sharding_codec_builder.build()))
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // The array metadata is
    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Use default codec options (concurrency etc)
    let options = CodecOptions::default();

    // Write some shards (in parallel)
    (0..2).into_par_iter().try_for_each(|s| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![s, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices)? {
            let chunk_array = ndarray::ArrayD::<u16>::from_shape_fn(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                |ij| {
                    (s * chunk_shape[0].get() * chunk_shape[1].get()
                        + ij[0] as u64 * chunk_shape[1].get()
                        + ij[1] as u64) as u16
                },
            );
            array.store_chunk_ndarray(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    // Read the whole array
    let data_all = array.retrieve_array_subset_ndarray::<u16>(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a shard back from the store
    let shard_indices = vec![1, 0];
    let data_shard = array.retrieve_chunk_ndarray::<u16>(&shard_indices)?;
    println!("Shard [1,0] is:\n{data_shard}\n");

    // Read an inner chunk from the store
    let subset_chunk_1_0 = ArraySubset::new_with_ranges(&[4..8, 0..4]);
    let data_chunk = array.retrieve_array_subset_ndarray::<u16>(&subset_chunk_1_0)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2 = array.retrieve_array_subset_ndarray::<u16>(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Decode inner chunks
    // In some cases, it might be preferable to decode inner chunks in a shard directly.
    // If using the partial decoder, then the shard index will only be read once from the store.
    let partial_decoder = array.partial_decoder(&[0, 0])?;
    println!("Decoded inner chunks:");
    for inner_chunk_subset in [
        ArraySubset::new_with_start_shape(vec![0, 0], inner_chunk_shape.clone())?,
        ArraySubset::new_with_start_shape(vec![0, 4], inner_chunk_shape.clone())?,
    ] {
        println!("{inner_chunk_subset}");
        let decoded_inner_chunk_bytes =
            partial_decoder.partial_decode(&inner_chunk_subset, &options)?;
        let ndarray = bytes_to_ndarray::<u16>(
            &inner_chunk_shape,
            decoded_inner_chunk_bytes.into_fixed()?.into_owned(),
        )?;
        println!("{ndarray}\n");
    }

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{}", tree);

    println!(
        "The keys in the store are:\n[{}]",
        store.list().unwrap_or_default().iter().format(", ")
    );

    Ok(())
}

pub fn array_to_bytes_codec_named( &mut self, array_to_bytes_codec: impl Into<NamedArrayToBytesCodec>, ) -> &mut Self

Set the array-to-bytes codec with non-default names.

If left unmodified, the array will default to using the bytes codec with native endian encoding.

pub fn bytes_to_bytes_codecs( &mut self, bytes_to_bytes_codecs: Vec<Arc<dyn BytesToBytesCodecTraits>>, ) -> &mut Self

Set the bytes-to-bytes codecs.

If left unmodified, the array will have no bytes-to-bytes codecs.

Examples found in repository

examples/custom_data_type_variable_size.rs (lines 170-175)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![3, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeVariableSize)),
        [],
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeVariableSizeElement::from(Some(1.0)),
        CustomDataTypeVariableSizeElement::from(None),
        CustomDataTypeVariableSizeElement::from(Some(3.0)),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeVariableSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeVariableSizeElement::from(Some(1.0)));
    assert_eq!(data[1], CustomDataTypeVariableSizeElement::from(None));
    assert_eq!(data[2], CustomDataTypeVariableSizeElement::from(Some(3.0)));
    assert_eq!(data[3], CustomDataTypeVariableSizeElement::from(None));

    println!("{data:#?}");
}

examples/custom_data_type_fixed_size.rs (lines 285-290)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![2, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFixedSize)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
        CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFixedSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
    assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
    assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
    assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });

    println!("{data:#?}");
}

examples/custom_data_type_uint12.rs (lines 222-227)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        vec![5, 1],    // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_float8_e3m4.rs (lines 233-238)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFloat8e3m4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFloat8e3m4Element::from(2.34),
        CustomDataTypeFloat8e3m4Element::from(3.45),
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NAN),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFloat8e3m4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!(
            "float8_e3m4: {:08b} f32: {}",
            f.to_ne_bytes()[0],
            f.as_f32()
        );
    }

    assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
    assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
    assert_eq!(
        data[2],
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
    );
    assert_eq!(
        data[3],
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
    );
    assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
    assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
}

examples/custom_data_type_uint4.rs (lines 220-225)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

examples/rectangular_array_write_read.rs (lines 70-73)

fn rectangular_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use zarrs::{
        array::{codec, DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        MetadataV3::new_with_configuration(
            "rectangular",
            RectangularChunkGridConfiguration {
                chunk_shape: vec![[1, 2, 3, 2].try_into()?, 4.try_into()?], // chunk sizes
            },
        ),
        DataType::Float32,
        ZARR_NAN_F32,
    )
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ])
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // Write some chunks (in parallel)
    (0..4).into_par_iter().try_for_each(|i| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![i, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices)? {
            let chunk_array = ndarray::ArrayD::<f32>::from_elem(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                i as f32,
            );
            array.store_chunk_ndarray(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write a subset spanning multiple chunks, including updating chunks already written
    array.store_array_subset_ndarray(
        &[3, 3], // start
        ndarray::ArrayD::<f32>::from_shape_vec(
            vec![3, 3],
            vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
        )?,
    )?;

    // Store elements directly, in this case set the 7th column to 123.0
    array.store_array_subset_elements::<f32>(
        &ArraySubset::new_with_ranges(&[0..8, 6..7]),
        &[123.0; 8],
    )?;

    // Store elements directly in a chunk, in this case set the last row of the bottom right chunk
    array.store_chunk_subset_elements::<f32>(
        // chunk indices
        &[3, 1],
        // subset within chunk
        &ArraySubset::new_with_ranges(&[1..2, 0..4]),
        &[-4.0; 4],
    )?;

    // Read the whole array
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a chunk back from the store
    let chunk_indices = vec![1, 0];
    let data_chunk = array.retrieve_chunk_ndarray::<f32>(&chunk_indices)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2 = array.retrieve_array_subset_ndarray::<f32>(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{tree}");

    Ok(())
}

pub fn bytes_to_bytes_codecs_named( &mut self, bytes_to_bytes_codecs: Vec<impl Into<NamedBytesToBytesCodec>>, ) -> &mut Self

Set the bytes-to-bytes codecs with non-default names.

If left unmodified, the array will have no bytes-to-bytes codecs.

pub fn attributes(&mut self, attributes: Map<String, Value>) -> &mut Self

Set the user defined attributes.

If left unmodified, the user defined attributes of the array will be empty.

pub fn attributes_mut(&mut self) -> &mut Map<String, Value>

Return a mutable reference to the attributes.

pub fn additional_fields( &mut self, additional_fields: AdditionalFieldsV3, ) -> &mut Self

Set the additional fields.

Set additional fields not defined in the Zarr specification. Use this cautiously. In general, store user defined attributes using ArrayBuilder::attributes.

zarrs and other implementations are expected to error when opening an array with unsupported additional fields, unless they are a JSON object containing "must_understand": false.

pub fn dimension_names<I, D>(&mut self, dimension_names: Option<I>) -> &mut Self

where I: IntoIterator<Item = D>, D: IntoDimensionName,

Set the dimension names.

If left unmodified, all dimension names are “unnamed”.

Examples found in repository

examples/array_write_read_string.rs (line 62)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;
    use zarrs::{array::DataType, array_subset::ArraySubset, storage::store};

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![4, 4], // array shape
        vec![2, 2], // regular chunk shape
        DataType::String,
        "_",
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    array.store_chunk_ndarray(
        &[0, 0],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["a", "bb", "ccc", "dddd"]).unwrap(),
    )?;
    array.store_chunk_ndarray(
        &[0, 1],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["4444", "333", "22", "1"]).unwrap(),
    )?;
    let subset_all = array.subset_all();
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    let ndarray_subset: Array2<&str> = array![["!", "@@"], ["###", "$$$$"]];
    array.store_array_subset_ndarray(
        ArraySubset::new_with_ranges(&[1..3, 1..3]).start(),
        ndarray_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_array_subset [1..3, 1..3]:\nndarray::ArrayD<String>\n{data_all}");

    // Retrieve bytes directly, convert into a single string allocation, create a &str ndarray
    // TODO: Add a convenience function for this?
    let data_all = array.retrieve_array_subset(&subset_all)?;
    let (bytes, offsets) = data_all.into_variable()?;
    let string = String::from_utf8(bytes.into_owned())?;
    let elements = offsets
        .iter()
        .tuple_windows()
        .map(|(&curr, &next)| &string[curr..next])
        .collect::<Vec<&str>>();
    let ndarray = ArrayD::<&str>::from_shape_vec(subset_all.shape_usize(), elements)?;
    println!("ndarray::ArrayD<&str>:\n{ndarray}");

    Ok(())
}

examples/rectangular_array_write_read.rs (line 74)

fn rectangular_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use zarrs::{
        array::{codec, DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        MetadataV3::new_with_configuration(
            "rectangular",
            RectangularChunkGridConfiguration {
                chunk_shape: vec![[1, 2, 3, 2].try_into()?, 4.try_into()?], // chunk sizes
            },
        ),
        DataType::Float32,
        ZARR_NAN_F32,
    )
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ])
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // Write some chunks (in parallel)
    (0..4).into_par_iter().try_for_each(|i| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![i, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices)? {
            let chunk_array = ndarray::ArrayD::<f32>::from_elem(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                i as f32,
            );
            array.store_chunk_ndarray(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write a subset spanning multiple chunks, including updating chunks already written
    array.store_array_subset_ndarray(
        &[3, 3], // start
        ndarray::ArrayD::<f32>::from_shape_vec(
            vec![3, 3],
            vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
        )?,
    )?;

    // Store elements directly, in this case set the 7th column to 123.0
    array.store_array_subset_elements::<f32>(
        &ArraySubset::new_with_ranges(&[0..8, 6..7]),
        &[123.0; 8],
    )?;

    // Store elements directly in a chunk, in this case set the last row of the bottom right chunk
    array.store_chunk_subset_elements::<f32>(
        // chunk indices
        &[3, 1],
        // subset within chunk
        &ArraySubset::new_with_ranges(&[1..2, 0..4]),
        &[-4.0; 4],
    )?;

    // Read the whole array
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a chunk back from the store
    let chunk_indices = vec![1, 0];
    let data_chunk = array.retrieve_chunk_ndarray::<f32>(&chunk_indices)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2 = array.retrieve_array_subset_ndarray::<f32>(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{tree}");

    Ok(())
}

examples/array_write_read.rs (line 65)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;
    use zarrs::{
        array::{DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        vec![4, 4], // regular chunk shape
        DataType::Float32,
        ZARR_NAN_F32,
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    (0..2).into_par_iter().try_for_each(|i| {
        let chunk_indices: Vec<u64> = vec![0, i];
        let chunk_subset = array.chunk_grid().subset(&chunk_indices)?.ok_or_else(|| {
            zarrs::array::ArrayError::InvalidChunkGridIndicesError(chunk_indices.to_vec())
        })?;
        array.store_chunk_elements(
            &chunk_indices,
            &vec![i as f32 * 0.1; chunk_subset.num_elements() as usize],
        )
    })?;

    let subset_all = array.subset_all();
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all:+4.1}\n");

    // Store multiple chunks
    array.store_chunks_elements::<f32>(
        &ArraySubset::new_with_ranges(&[1..2, 0..2]),
        &[
            //
            1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
            //
            1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
        ],
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunks [1..2, 0..2]:\n{data_all:+4.1}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    array.store_array_subset_elements::<f32>(
        &ArraySubset::new_with_ranges(&[3..6, 3..6]),
        &[-3.3, -3.4, -3.5, -4.3, -4.4, -4.5, -5.3, -5.4, -5.5],
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_array_subset [3..6, 3..6]:\n{data_all:+4.1}\n");

    // Store array subset
    array.store_array_subset_elements::<f32>(
        &ArraySubset::new_with_ranges(&[0..8, 6..7]),
        &[-0.6, -1.6, -2.6, -3.6, -4.6, -5.6, -6.6, -7.6],
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_array_subset [0..8, 6..7]:\n{data_all:+4.1}\n");

    // Store chunk subset
    array.store_chunk_subset_elements::<f32>(
        // chunk indices
        &[1, 1],
        // subset within chunk
        &ArraySubset::new_with_ranges(&[3..4, 0..4]),
        &[-7.4, -7.5, -7.6, -7.7],
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunk_subset [3..4, 0..4] of chunk [1, 1]:\n{data_all:+4.1}\n");

    // Erase a chunk
    array.erase_chunk(&[0, 0])?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("erase_chunk [0, 0]:\n{data_all:+4.1}\n");

    // Read a chunk
    let chunk_indices = vec![0, 1];
    let data_chunk = array.retrieve_chunk_ndarray::<f32>(&chunk_indices)?;
    println!("retrieve_chunk [0, 1]:\n{data_chunk:+4.1}\n");

    // Read chunks
    let chunks = ArraySubset::new_with_ranges(&[0..2, 1..2]);
    let data_chunks = array.retrieve_chunks_ndarray::<f32>(&chunks)?;
    println!("retrieve_chunks [0..2, 1..2]:\n{data_chunks:+4.1}\n");

    // Retrieve an array subset
    let subset = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_subset = array.retrieve_array_subset_ndarray::<f32>(&subset)?;
    println!("retrieve_array_subset [2..6, 3..5]:\n{data_subset:+4.1}\n");

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("hierarchy_tree:\n{}", tree);

    Ok(())
}

examples/sharded_array_write_read.rs (line 76)

fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use zarrs::{
        array::{
            codec::{self, array_to_bytes::sharding::ShardingCodecBuilder},
            DataType,
        },
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use std::sync::Arc;

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/sharded_array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    // Create an array
    let array_path = "/group/array";
    let shard_shape = vec![4, 8];
    let inner_chunk_shape = vec![4, 4];
    let mut sharding_codec_builder =
        ShardingCodecBuilder::new(inner_chunk_shape.as_slice().try_into()?);
    sharding_codec_builder.bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ]);
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        shard_shape,
        DataType::UInt16,
        0u16,
    )
    .array_to_bytes_codec(Arc::new(sharding_codec_builder.build()))
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // The array metadata is
    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Use default codec options (concurrency etc)
    let options = CodecOptions::default();

    // Write some shards (in parallel)
    (0..2).into_par_iter().try_for_each(|s| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![s, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices)? {
            let chunk_array = ndarray::ArrayD::<u16>::from_shape_fn(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                |ij| {
                    (s * chunk_shape[0].get() * chunk_shape[1].get()
                        + ij[0] as u64 * chunk_shape[1].get()
                        + ij[1] as u64) as u16
                },
            );
            array.store_chunk_ndarray(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    // Read the whole array
    let data_all = array.retrieve_array_subset_ndarray::<u16>(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a shard back from the store
    let shard_indices = vec![1, 0];
    let data_shard = array.retrieve_chunk_ndarray::<u16>(&shard_indices)?;
    println!("Shard [1,0] is:\n{data_shard}\n");

    // Read an inner chunk from the store
    let subset_chunk_1_0 = ArraySubset::new_with_ranges(&[4..8, 0..4]);
    let data_chunk = array.retrieve_array_subset_ndarray::<u16>(&subset_chunk_1_0)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2 = array.retrieve_array_subset_ndarray::<u16>(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Decode inner chunks
    // In some cases, it might be preferable to decode inner chunks in a shard directly.
    // If using the partial decoder, then the shard index will only be read once from the store.
    let partial_decoder = array.partial_decoder(&[0, 0])?;
    println!("Decoded inner chunks:");
    for inner_chunk_subset in [
        ArraySubset::new_with_start_shape(vec![0, 0], inner_chunk_shape.clone())?,
        ArraySubset::new_with_start_shape(vec![0, 4], inner_chunk_shape.clone())?,
    ] {
        println!("{inner_chunk_subset}");
        let decoded_inner_chunk_bytes =
            partial_decoder.partial_decode(&inner_chunk_subset, &options)?;
        let ndarray = bytes_to_ndarray::<u16>(
            &inner_chunk_shape,
            decoded_inner_chunk_bytes.into_fixed()?.into_owned(),
        )?;
        println!("{ndarray}\n");
    }

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{}", tree);

    println!(
        "The keys in the store are:\n[{}]",
        store.list().unwrap_or_default().iter().format(", ")
    );

    Ok(())
}

examples/array_write_read_ndarray.rs (line 66)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;
    use zarrs::{
        array::{DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
        storage::store,
    };

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        vec![4, 4], // regular chunk shape
        DataType::Float32,
        ZARR_NAN_F32,
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    (0..2).into_par_iter().try_for_each(|i| {
        let chunk_indices: Vec<u64> = vec![0, i];
        let chunk_subset = array.chunk_grid().subset(&chunk_indices)?.ok_or_else(|| {
            zarrs::array::ArrayError::InvalidChunkGridIndicesError(chunk_indices.to_vec())
        })?;
        array.store_chunk_ndarray(
            &chunk_indices,
            ArrayD::<f32>::from_shape_vec(
                chunk_subset.shape_usize(),
                vec![i as f32 * 0.1; chunk_subset.num_elements() as usize],
            )
            .unwrap(),
        )
    })?;

    let subset_all = array.subset_all();
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all:+4.1}\n");

    // Store multiple chunks
    let ndarray_chunks: Array2<f32> = array![
        [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,],
        [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,],
        [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,],
        [1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,],
    ];
    array.store_chunks_ndarray(&ArraySubset::new_with_ranges(&[1..2, 0..2]), ndarray_chunks)?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunks [1..2, 0..2]:\n{data_all:+4.1}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    let ndarray_subset: Array2<f32> =
        array![[-3.3, -3.4, -3.5,], [-4.3, -4.4, -4.5,], [-5.3, -5.4, -5.5],];
    array.store_array_subset_ndarray(
        ArraySubset::new_with_ranges(&[3..6, 3..6]).start(),
        ndarray_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_array_subset [3..6, 3..6]:\n{data_all:+4.1}\n");

    // Store array subset
    let ndarray_subset: Array2<f32> = array![
        [-0.6],
        [-1.6],
        [-2.6],
        [-3.6],
        [-4.6],
        [-5.6],
        [-6.6],
        [-7.6],
    ];
    array.store_array_subset_ndarray(
        ArraySubset::new_with_ranges(&[0..8, 6..7]).start(),
        ndarray_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_array_subset [0..8, 6..7]:\n{data_all:+4.1}\n");

    // Store chunk subset
    let ndarray_chunk_subset: Array2<f32> = array![[-7.4, -7.5, -7.6, -7.7],];
    array.store_chunk_subset_ndarray(
        // chunk indices
        &[1, 1],
        // subset within chunk
        ArraySubset::new_with_ranges(&[3..4, 0..4]).start(),
        ndarray_chunk_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("store_chunk_subset [3..4, 0..4] of chunk [1, 1]:\n{data_all:+4.1}\n");

    // Erase a chunk
    array.erase_chunk(&[0, 0])?;
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&subset_all)?;
    println!("erase_chunk [0, 0]:\n{data_all:+4.1}\n");

    // Read a chunk
    let chunk_indices = vec![0, 1];
    let data_chunk = array.retrieve_chunk_ndarray::<f32>(&chunk_indices)?;
    println!("retrieve_chunk [0, 1]:\n{data_chunk:+4.1}\n");

    // Read chunks
    let chunks = ArraySubset::new_with_ranges(&[0..2, 1..2]);
    let data_chunks = array.retrieve_chunks_ndarray::<f32>(&chunks)?;
    println!("retrieve_chunks [0..2, 1..2]:\n{data_chunks:+4.1}\n");

    // Retrieve an array subset
    let subset = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_subset = array.retrieve_array_subset_ndarray::<f32>(&subset)?;
    println!("retrieve_array_subset [2..6, 3..5]:\n{data_subset:+4.1}\n");

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("hierarchy_tree:\n{}", tree);

    Ok(())
}

examples/async_array_write_read.rs (line 62)

async fn async_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use futures::StreamExt;
    use std::sync::Arc;
    use zarrs::{
        array::{DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
    };

    // Create a store
    let mut store: AsyncReadableWritableListableStorage = Arc::new(
        zarrs_object_store::AsyncObjectStore::new(object_store::memory::InMemory::new()),
    );
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .async_store_metadata()
        .await?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.async_store_metadata().await?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        vec![4, 4], // regular chunk shape
        DataType::Float32,
        ZARR_NAN_F32,
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build_arc(store.clone(), array_path)?;

    // Write array metadata to store
    array.async_store_metadata().await?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    let store_chunk = |i: u64| {
        let array = array.clone();
        async move {
            let chunk_indices: Vec<u64> = vec![0, i];
            let chunk_subset = array.chunk_grid().subset(&chunk_indices)?.ok_or_else(|| {
                zarrs::array::ArrayError::InvalidChunkGridIndicesError(chunk_indices.to_vec())
            })?;
            array
                .async_store_chunk_elements(
                    &chunk_indices,
                    &vec![i as f32 * 0.1; chunk_subset.num_elements() as usize],
                )
                .await
        }
    };
    futures::stream::iter(0..2)
        .map(Ok)
        .try_for_each_concurrent(None, store_chunk)
        .await?;

    let subset_all = array.subset_all();
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunk [0, 0] and [0, 1]:\n{data_all:+4.1}\n");

    // Store multiple chunks
    array
        .async_store_chunks_elements::<f32>(
            &ArraySubset::new_with_ranges(&[1..2, 0..2]),
            &[
                //
                1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
                //
                1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
            ],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunks [1..2, 0..2]:\n{data_all:+4.1}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    array
        .async_store_array_subset_elements::<f32>(
            &ArraySubset::new_with_ranges(&[3..6, 3..6]),
            &[-3.3, -3.4, -3.5, -4.3, -4.4, -4.5, -5.3, -5.4, -5.5],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_array_subset [3..6, 3..6]:\n{data_all:+4.1}\n");

    // Store array subset
    array
        .async_store_array_subset_elements::<f32>(
            &ArraySubset::new_with_ranges(&[0..8, 6..7]),
            &[-0.6, -1.6, -2.6, -3.6, -4.6, -5.6, -6.6, -7.6],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_array_subset [0..8, 6..7]:\n{data_all:+4.1}\n");

    // Store chunk subset
    array
        .async_store_chunk_subset_elements::<f32>(
            // chunk indices
            &[1, 1],
            // subset within chunk
            &ArraySubset::new_with_ranges(&[3..4, 0..4]),
            &[-7.4, -7.5, -7.6, -7.7],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunk_subset [3..4, 0..4] of chunk [1, 1]:\n{data_all:+4.1}\n");

    // Erase a chunk
    array.async_erase_chunk(&[0, 0]).await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_erase_chunk [0, 0]:\n{data_all:+4.1}\n");

    // Read a chunk
    let chunk_indices = vec![0, 1];
    let data_chunk = array
        .async_retrieve_chunk_ndarray::<f32>(&chunk_indices)
        .await?;
    println!("async_retrieve_chunk [0, 1]:\n{data_chunk:+4.1}\n");

    // Read chunks
    let chunks = ArraySubset::new_with_ranges(&[0..2, 1..2]);
    let data_chunks = array.async_retrieve_chunks_ndarray::<f32>(&chunks).await?;
    println!("async_retrieve_chunks [0..2, 1..2]:\n{data_chunks:+4.1}\n");

    // Retrieve an array subset
    let subset = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_subset = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset)
        .await?;
    println!("async_retrieve_array_subset [2..6, 3..5]:\n{data_subset:+4.1}\n");

    // Show the hierarchy
    let node = Node::async_open(store, "/").await.unwrap();
    let tree = node.hierarchy_tree();
    println!("hierarchy_tree:\n{}", tree);

    Ok(())
}

pub fn storage_transformers( &mut self, storage_transformers: StorageTransformerChain, ) -> &mut Self

Set the storage transformers.

If left unmodified, there are no storage transformers.

pub fn build_metadata(&self) -> Result<ArrayMetadataV3, ArrayCreateError>

Get the metadata of an array that would be created with the current builder state.

Errors

Returns an ArrayCreateError if this metadata is invalid/unsupported by zarrs.

pub fn build<TStorage: ?Sized>( &self, storage: Arc<TStorage>, path: &str, ) -> Result<Array<TStorage>, ArrayCreateError>

Build into an Array.

Errors

Returns ArrayCreateError if there is an error creating the array. This can be due to a storage error, an invalid path, or a problem with array configuration.

Examples found in repository

examples/custom_data_type_variable_size.rs (line 177)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![3, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeVariableSize)),
        [],
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeVariableSizeElement::from(Some(1.0)),
        CustomDataTypeVariableSizeElement::from(None),
        CustomDataTypeVariableSizeElement::from(Some(3.0)),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeVariableSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeVariableSizeElement::from(Some(1.0)));
    assert_eq!(data[1], CustomDataTypeVariableSizeElement::from(None));
    assert_eq!(data[2], CustomDataTypeVariableSizeElement::from(Some(3.0)));
    assert_eq!(data[3], CustomDataTypeVariableSizeElement::from(None));

    println!("{data:#?}");
}

examples/custom_data_type_fixed_size.rs (line 292)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        vec![2, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFixedSize)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
        CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFixedSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
    assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
    assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
    assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });

    println!("{data:#?}");
}

examples/custom_data_type_uint12.rs (line 229)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        vec![5, 1],    // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_float8_e3m4.rs (line 240)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeFloat8e3m4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFloat8e3m4Element::from(2.34),
        CustomDataTypeFloat8e3m4Element::from(3.45),
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NAN),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFloat8e3m4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!(
            "float8_e3m4: {:08b} f32: {}",
            f.to_ne_bytes()[0],
            f.as_f32()
        );
    }

    assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
    assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
    assert_eq!(
        data[2],
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
    );
    assert_eq!(
        data[3],
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
    );
    assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
    assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
}

examples/custom_data_type_uint4.rs (line 227)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        vec![5, 1], // regular chunk shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

examples/array_write_read_string.rs (line 64)

fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use std::sync::Arc;
    use zarrs::{array::DataType, array_subset::ArraySubset, storage::store};

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    // let mut store: ReadableWritableListableStorage = Arc::new(
    //     zarrs::filesystem::FilesystemStore::new("zarrs/tests/data/array_write_read.zarr")?,
    // );
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![4, 4], // array shape
        vec![2, 2], // regular chunk shape
        DataType::String,
        "_",
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    array.store_chunk_ndarray(
        &[0, 0],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["a", "bb", "ccc", "dddd"]).unwrap(),
    )?;
    array.store_chunk_ndarray(
        &[0, 1],
        ArrayD::<&str>::from_shape_vec(vec![2, 2], vec!["4444", "333", "22", "1"]).unwrap(),
    )?;
    let subset_all = array.subset_all();
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_chunk [0, 0] and [0, 1]:\n{data_all}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    let ndarray_subset: Array2<&str> = array![["!", "@@"], ["###", "$$$$"]];
    array.store_array_subset_ndarray(
        ArraySubset::new_with_ranges(&[1..3, 1..3]).start(),
        ndarray_subset,
    )?;
    let data_all = array.retrieve_array_subset_ndarray::<String>(&subset_all)?;
    println!("store_array_subset [1..3, 1..3]:\nndarray::ArrayD<String>\n{data_all}");

    // Retrieve bytes directly, convert into a single string allocation, create a &str ndarray
    // TODO: Add a convenience function for this?
    let data_all = array.retrieve_array_subset(&subset_all)?;
    let (bytes, offsets) = data_all.into_variable()?;
    let string = String::from_utf8(bytes.into_owned())?;
    let elements = offsets
        .iter()
        .tuple_windows()
        .map(|(&curr, &next)| &string[curr..next])
        .collect::<Vec<&str>>();
    let ndarray = ArrayD::<&str>::from_shape_vec(subset_all.shape_usize(), elements)?;
    println!("ndarray::ArrayD<&str>:\n{ndarray}");

    Ok(())
}

Additional examples can be found in:

• examples/rectangular_array_write_read.rs
• examples/array_write_read.rs
• examples/sharded_array_write_read.rs
• examples/array_write_read_ndarray.rs

pub fn build_arc<TStorage: ?Sized>( &self, storage: Arc<TStorage>, path: &str, ) -> Result<Arc<Array<TStorage>>, ArrayCreateError>

Build into an Arc<Array>.

Errors

Returns ArrayCreateError if there is an error creating the array. This can be due to a storage error, an invalid path, or a problem with array configuration.

Examples found in repository

examples/async_array_write_read.rs (line 64)

async fn async_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use futures::StreamExt;
    use std::sync::Arc;
    use zarrs::{
        array::{DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        node::Node,
    };

    // Create a store
    let mut store: AsyncReadableWritableListableStorage = Arc::new(
        zarrs_object_store::AsyncObjectStore::new(object_store::memory::InMemory::new()),
    );
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .async_store_metadata()
        .await?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.async_store_metadata().await?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        vec![4, 4], // regular chunk shape
        DataType::Float32,
        ZARR_NAN_F32,
    )
    // .bytes_to_bytes_codecs(vec![]) // uncompressed
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build_arc(store.clone(), array_path)?;

    // Write array metadata to store
    array.async_store_metadata().await?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write some chunks
    let store_chunk = |i: u64| {
        let array = array.clone();
        async move {
            let chunk_indices: Vec<u64> = vec![0, i];
            let chunk_subset = array.chunk_grid().subset(&chunk_indices)?.ok_or_else(|| {
                zarrs::array::ArrayError::InvalidChunkGridIndicesError(chunk_indices.to_vec())
            })?;
            array
                .async_store_chunk_elements(
                    &chunk_indices,
                    &vec![i as f32 * 0.1; chunk_subset.num_elements() as usize],
                )
                .await
        }
    };
    futures::stream::iter(0..2)
        .map(Ok)
        .try_for_each_concurrent(None, store_chunk)
        .await?;

    let subset_all = array.subset_all();
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunk [0, 0] and [0, 1]:\n{data_all:+4.1}\n");

    // Store multiple chunks
    array
        .async_store_chunks_elements::<f32>(
            &ArraySubset::new_with_ranges(&[1..2, 0..2]),
            &[
                //
                1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
                //
                1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1, 1.0, 1.0, 1.0, 1.0, 1.1, 1.1, 1.1, 1.1,
            ],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunks [1..2, 0..2]:\n{data_all:+4.1}\n");

    // Write a subset spanning multiple chunks, including updating chunks already written
    array
        .async_store_array_subset_elements::<f32>(
            &ArraySubset::new_with_ranges(&[3..6, 3..6]),
            &[-3.3, -3.4, -3.5, -4.3, -4.4, -4.5, -5.3, -5.4, -5.5],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_array_subset [3..6, 3..6]:\n{data_all:+4.1}\n");

    // Store array subset
    array
        .async_store_array_subset_elements::<f32>(
            &ArraySubset::new_with_ranges(&[0..8, 6..7]),
            &[-0.6, -1.6, -2.6, -3.6, -4.6, -5.6, -6.6, -7.6],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_array_subset [0..8, 6..7]:\n{data_all:+4.1}\n");

    // Store chunk subset
    array
        .async_store_chunk_subset_elements::<f32>(
            // chunk indices
            &[1, 1],
            // subset within chunk
            &ArraySubset::new_with_ranges(&[3..4, 0..4]),
            &[-7.4, -7.5, -7.6, -7.7],
        )
        .await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_store_chunk_subset [3..4, 0..4] of chunk [1, 1]:\n{data_all:+4.1}\n");

    // Erase a chunk
    array.async_erase_chunk(&[0, 0]).await?;
    let data_all = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset_all)
        .await?;
    println!("async_erase_chunk [0, 0]:\n{data_all:+4.1}\n");

    // Read a chunk
    let chunk_indices = vec![0, 1];
    let data_chunk = array
        .async_retrieve_chunk_ndarray::<f32>(&chunk_indices)
        .await?;
    println!("async_retrieve_chunk [0, 1]:\n{data_chunk:+4.1}\n");

    // Read chunks
    let chunks = ArraySubset::new_with_ranges(&[0..2, 1..2]);
    let data_chunks = array.async_retrieve_chunks_ndarray::<f32>(&chunks).await?;
    println!("async_retrieve_chunks [0..2, 1..2]:\n{data_chunks:+4.1}\n");

    // Retrieve an array subset
    let subset = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_subset = array
        .async_retrieve_array_subset_ndarray::<f32>(&subset)
        .await?;
    println!("async_retrieve_array_subset [2..6, 3..5]:\n{data_subset:+4.1}\n");

    // Show the hierarchy
    let node = Node::async_open(store, "/").await.unwrap();
    let tree = node.hierarchy_tree();
    println!("hierarchy_tree:\n{}", tree);

    Ok(())
}

Trait Implementations

impl Debug for ArrayBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.