Struct GzipCodec

pub struct GzipCodec { /* private fields */ }

Available on crate feature gzip only.

A gzip codec implementation.

Implementations

impl GzipCodec

pub fn new(compression_level: u32) -> Result<Self, GzipCompressionLevelError>

Create a new gzip codec.

Errors

Returns GzipCompressionLevelError if compression_level is not valid.

Examples found in repository

examples/custom_data_type_variable_size.rs (line 170)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        DataType::Extension(Arc::new(CustomDataTypeVariableSize)),
        vec![3, 1].try_into().unwrap(), // regular chunk shape
        FillValue::from(vec![]),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeVariableSizeElement::from(Some(1.0)),
        CustomDataTypeVariableSizeElement::from(None),
        CustomDataTypeVariableSizeElement::from(Some(3.0)),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeVariableSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeVariableSizeElement::from(Some(1.0)));
    assert_eq!(data[1], CustomDataTypeVariableSizeElement::from(None));
    assert_eq!(data[2], CustomDataTypeVariableSizeElement::from(Some(3.0)));
    assert_eq!(data[3], CustomDataTypeVariableSizeElement::from(None));

    println!("{data:#?}");
}

examples/custom_data_type_fixed_size.rs (line 287)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFixedSizeElement { x: 1, y: 2.3 };
    let array = ArrayBuilder::new(
        vec![4, 1], // array shape
        DataType::Extension(Arc::new(CustomDataTypeFixedSize)),
        vec![2, 1].try_into().unwrap(), // regular chunk shape
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFixedSizeElement { x: 3, y: 4.5 },
        CustomDataTypeFixedSizeElement { x: 6, y: 7.8 },
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFixedSizeElement>(&array.subset_all())
        .unwrap();

    assert_eq!(data[0], CustomDataTypeFixedSizeElement { x: 3, y: 4.5 });
    assert_eq!(data[1], CustomDataTypeFixedSizeElement { x: 6, y: 7.8 });
    assert_eq!(data[2], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });
    assert_eq!(data[3], CustomDataTypeFixedSizeElement { x: 1, y: 2.3 });

    println!("{data:#?}");
}

examples/custom_data_type_uint12.rs (line 224)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt12Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![4096, 1], // array shape
        DataType::Extension(Arc::new(CustomDataTypeUInt12)),
        vec![5, 1].try_into().unwrap(), // regular chunk shape
        FillValue::new(fill_value.to_le_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data: Vec<CustomDataTypeUInt12Element> = (0..4096)
        .into_iter()
        .map(|i| CustomDataTypeUInt12Element::try_from(i).unwrap())
        .collect();

    array
        .store_array_subset_elements(&array.subset_all(), &data)
        .unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(&array.subset_all())
        .unwrap();

    for i in 0usize..4096 {
        let element = CustomDataTypeUInt12Element::try_from(i as u64).unwrap();
        assert_eq!(data[i], element);
        let element_pd = array
            .retrieve_array_subset_elements::<CustomDataTypeUInt12Element>(
                &ArraySubset::new_with_ranges(&[(i as u64)..i as u64 + 1, 0..1]),
            )
            .unwrap()[0];
        assert_eq!(element_pd, element);
    }
}

examples/custom_data_type_float8_e3m4.rs (line 235)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeFloat8e3m4Element::from(1.23);
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        DataType::Extension(Arc::new(CustomDataTypeFloat8e3m4)),
        vec![5, 1].try_into().unwrap(), // regular chunk shape
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeFloat8e3m4Element::from(2.34),
        CustomDataTypeFloat8e3m4Element::from(3.45),
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY),
        CustomDataTypeFloat8e3m4Element::from(f32::NAN),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeFloat8e3m4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!(
            "float8_e3m4: {:08b} f32: {}",
            f.to_ne_bytes()[0],
            f.as_f32()
        );
    }

    assert_eq!(data[0], CustomDataTypeFloat8e3m4Element::from(2.34));
    assert_eq!(data[1], CustomDataTypeFloat8e3m4Element::from(3.45));
    assert_eq!(
        data[2],
        CustomDataTypeFloat8e3m4Element::from(f32::INFINITY)
    );
    assert_eq!(
        data[3],
        CustomDataTypeFloat8e3m4Element::from(f32::NEG_INFINITY)
    );
    assert_eq!(data[4], CustomDataTypeFloat8e3m4Element::from(f32::NAN));
    assert_eq!(data[5], CustomDataTypeFloat8e3m4Element::from(1.23));
}

examples/custom_data_type_uint4.rs (line 222)

fn main() {
    let store = std::sync::Arc::new(MemoryStore::default());
    let array_path = "/array";
    let fill_value = CustomDataTypeUInt4Element::try_from(15).unwrap();
    let array = ArrayBuilder::new(
        vec![6, 1], // array shape
        DataType::Extension(Arc::new(CustomDataTypeUInt4)),
        vec![5, 1].try_into().unwrap(), // regular chunk shape
        FillValue::new(fill_value.to_ne_bytes().to_vec()),
    )
    .array_to_array_codecs(vec![
        #[cfg(feature = "transpose")]
        Arc::new(zarrs::array::codec::TransposeCodec::new(
            zarrs::array::codec::array_to_array::transpose::TransposeOrder::new(&[1, 0]).unwrap(),
        )),
    ])
    .array_to_bytes_codec(Arc::new(zarrs::array::codec::PackBitsCodec::default()))
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(zarrs::array::codec::GzipCodec::new(5).unwrap()),
        #[cfg(feature = "crc32c")]
        Arc::new(zarrs::array::codec::Crc32cCodec::new()),
    ])
    // .storage_transformers(vec![].into())
    .build(store, array_path)
    .unwrap();
    println!("{}", array.metadata().to_string_pretty());

    let data = [
        CustomDataTypeUInt4Element::try_from(1).unwrap(),
        CustomDataTypeUInt4Element::try_from(2).unwrap(),
        CustomDataTypeUInt4Element::try_from(3).unwrap(),
        CustomDataTypeUInt4Element::try_from(4).unwrap(),
        CustomDataTypeUInt4Element::try_from(5).unwrap(),
    ];
    array.store_chunk_elements(&[0, 0], &data).unwrap();

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(&array.subset_all())
        .unwrap();

    for f in &data {
        println!("uint4: {:08b} u8: {}", f.as_u8(), f.as_u8());
    }

    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(1).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[2], CustomDataTypeUInt4Element::try_from(3).unwrap());
    assert_eq!(data[3], CustomDataTypeUInt4Element::try_from(4).unwrap());
    assert_eq!(data[4], CustomDataTypeUInt4Element::try_from(5).unwrap());
    assert_eq!(data[5], CustomDataTypeUInt4Element::try_from(15).unwrap());

    let data = array
        .retrieve_array_subset_elements::<CustomDataTypeUInt4Element>(
            &ArraySubset::new_with_ranges(&[1..3, 0..1]),
        )
        .unwrap();
    assert_eq!(data[0], CustomDataTypeUInt4Element::try_from(2).unwrap());
    assert_eq!(data[1], CustomDataTypeUInt4Element::try_from(3).unwrap());
}

examples/rectangular_array_write_read.rs (line 70)

fn rectangular_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
    use rayon::prelude::{IntoParallelIterator, ParallelIterator};
    use zarrs::array::ChunkGrid;
    use zarrs::{
        array::{chunk_grid::RectangularChunkGrid, codec, FillValue},
        node::Node,
    };
    use zarrs::{
        array::{DataType, ZARR_NAN_F32},
        array_subset::ArraySubset,
        storage::store,
    };

    // Create a store
    // let path = tempfile::TempDir::new()?;
    // let mut store: ReadableWritableListableStorage =
    //     Arc::new(zarrs::filesystem::FilesystemStore::new(path.path())?);
    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
        if arg1 == "--usage-log" {
            let log_writer = Arc::new(std::sync::Mutex::new(
                // std::io::BufWriter::new(
                std::io::stdout(),
                //    )
            ));
            store = Arc::new(UsageLogStorageAdapter::new(store, log_writer, || {
                chrono::Utc::now().format("[%T%.3f] ").to_string()
            }));
        }
    }

    // Create the root group
    zarrs::group::GroupBuilder::new()
        .build(store.clone(), "/")?
        .store_metadata()?;

    // Create a group with attributes
    let group_path = "/group";
    let mut group = zarrs::group::GroupBuilder::new().build(store.clone(), group_path)?;
    group
        .attributes_mut()
        .insert("foo".into(), serde_json::Value::String("bar".into()));
    group.store_metadata()?;

    println!(
        "The group metadata is:\n{}\n",
        group.metadata().to_string_pretty()
    );

    // Create an array
    let array_path = "/group/array";
    let array = zarrs::array::ArrayBuilder::new(
        vec![8, 8], // array shape
        DataType::Float32,
        ChunkGrid::new(RectangularChunkGrid::new(&[
            [1, 2, 3, 2].try_into()?,
            4.try_into()?,
        ])),
        FillValue::from(ZARR_NAN_F32),
    )
    .bytes_to_bytes_codecs(vec![
        #[cfg(feature = "gzip")]
        Arc::new(codec::GzipCodec::new(5)?),
    ])
    .dimension_names(["y", "x"].into())
    // .storage_transformers(vec![].into())
    .build(store.clone(), array_path)?;

    // Write array metadata to store
    array.store_metadata()?;

    // Write some chunks (in parallel)
    (0..4).into_par_iter().try_for_each(|i| {
        let chunk_grid = array.chunk_grid();
        let chunk_indices = vec![i, 0];
        if let Some(chunk_shape) = chunk_grid.chunk_shape(&chunk_indices, array.shape())? {
            let chunk_array = ndarray::ArrayD::<f32>::from_elem(
                chunk_shape
                    .iter()
                    .map(|u| u.get() as usize)
                    .collect::<Vec<_>>(),
                i as f32,
            );
            array.store_chunk_ndarray(&chunk_indices, chunk_array)
        } else {
            Err(zarrs::array::ArrayError::InvalidChunkGridIndicesError(
                chunk_indices.to_vec(),
            ))
        }
    })?;

    println!(
        "The array metadata is:\n{}\n",
        array.metadata().to_string_pretty()
    );

    // Write a subset spanning multiple chunks, including updating chunks already written
    array.store_array_subset_ndarray(
        &[3, 3], // start
        ndarray::ArrayD::<f32>::from_shape_vec(
            vec![3, 3],
            vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
        )?,
    )?;

    // Store elements directly, in this case set the 7th column to 123.0
    array.store_array_subset_elements::<f32>(
        &ArraySubset::new_with_ranges(&[0..8, 6..7]),
        &[123.0; 8],
    )?;

    // Store elements directly in a chunk, in this case set the last row of the bottom right chunk
    array.store_chunk_subset_elements::<f32>(
        // chunk indices
        &[3, 1],
        // subset within chunk
        &ArraySubset::new_with_ranges(&[1..2, 0..4]),
        &[-4.0; 4],
    )?;

    // Read the whole array
    let data_all = array.retrieve_array_subset_ndarray::<f32>(&array.subset_all())?;
    println!("The whole array is:\n{data_all}\n");

    // Read a chunk back from the store
    let chunk_indices = vec![1, 0];
    let data_chunk = array.retrieve_chunk_ndarray::<f32>(&chunk_indices)?;
    println!("Chunk [1,0] is:\n{data_chunk}\n");

    // Read the central 4x2 subset of the array
    let subset_4x2 = ArraySubset::new_with_ranges(&[2..6, 3..5]); // the center 4x2 region
    let data_4x2 = array.retrieve_array_subset_ndarray::<f32>(&subset_4x2)?;
    println!("The middle 4x2 subset is:\n{data_4x2}\n");

    // Show the hierarchy
    let node = Node::open(&store, "/").unwrap();
    let tree = node.hierarchy_tree();
    println!("The Zarr hierarchy tree is:\n{tree}");

    Ok(())
}

Additional examples can be found in:

• examples/sharded_array_write_read.rs

pub fn new_with_configuration( configuration: &GzipCodecConfiguration, ) -> Result<Self, PluginCreateError>

Create a new gzip codec from configuration.

Errors

Returns an error if the configuration is not supported.

Trait Implementations

impl BytesToBytesCodecTraits for GzipCodec

fn into_dyn(self: Arc<Self>) -> Arc<dyn BytesToBytesCodecTraits>

Return a dynamic version of the codec.

fn recommended_concurrency( &self, _decoded_representation: &BytesRepresentation, ) -> Result<RecommendedConcurrency, CodecError>

Return the maximum internal concurrency supported for the requested decoded representation.

fn encode<'a>( &self, decoded_value: RawBytes<'a>, _options: &CodecOptions, ) -> Result<RawBytes<'a>, CodecError>

Encode chunk bytes.

fn decode<'a>( &self, encoded_value: RawBytes<'a>, _decoded_representation: &BytesRepresentation, _options: &CodecOptions, ) -> Result<RawBytes<'a>, CodecError>

Decode chunk bytes.

fn encoded_representation( &self, decoded_representation: &BytesRepresentation, ) -> BytesRepresentation

Returns the size of the encoded representation given a size of the decoded representation.

fn partial_decoder( self: Arc<Self>, input_handle: Arc<dyn BytesPartialDecoderTraits>, decoded_representation: &BytesRepresentation, options: &CodecOptions, ) -> Result<Arc<dyn BytesPartialDecoderTraits>, CodecError>

Initialises a partial decoder.

fn partial_encoder( self: Arc<Self>, input_handle: Arc<dyn BytesPartialDecoderTraits>, output_handle: Arc<dyn BytesPartialEncoderTraits>, decoded_representation: &BytesRepresentation, options: &CodecOptions, ) -> Result<Arc<dyn BytesPartialEncoderTraits>, CodecError>

Initialise a partial encoder.

fn async_partial_decoder<'life0, 'life1, 'async_trait>( self: Arc<Self>, input_handle: Arc<dyn AsyncBytesPartialDecoderTraits>, decoded_representation: &'life0 BytesRepresentation, options: &'life1 CodecOptions, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn AsyncBytesPartialDecoderTraits>, CodecError>> + Send + 'async_trait>>

where Self: Sync + Send + 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Available on crate feature async only.

Initialises an asynchronous partial decoder.

fn async_partial_encoder<'life0, 'life1, 'async_trait>( self: Arc<Self>, input_handle: Arc<dyn AsyncBytesPartialDecoderTraits>, output_handle: Arc<dyn AsyncBytesPartialEncoderTraits>, decoded_representation: &'life0 BytesRepresentation, options: &'life1 CodecOptions, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn AsyncBytesPartialEncoderTraits>, CodecError>> + Send + 'async_trait>>

where Self: Sync + Send + 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Available on crate feature async only.

Initialise an asynchronous partial encoder.

impl Clone for GzipCodec

fn clone(&self) -> GzipCodec

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl CodecTraits for GzipCodec

fn identifier(&self) -> &str

Unique identifier for the codec.

fn configuration_opt( &self, _name: &str, _options: &CodecMetadataOptions, ) -> Option<Configuration>

Create the codec configuration.

fn partial_decoder_should_cache_input(&self) -> bool

Indicates if the input to a codecs partial decoder should be cached for optimal performance. If true, a cache may be inserted before it in a CodecChain partial decoder.

fn partial_decoder_decodes_all(&self) -> bool

Indicates if a partial decoder decodes all bytes from its input handle and its output should be cached for optimal performance. If true, a cache will be inserted at some point after it in a CodecChain partial decoder.

fn default_name(&self) -> String

The default name of the codec.

fn configuration(&self, name: &str) -> Option<Configuration>

Create the codec configuration with default options.

impl Debug for GzipCodec

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.