mod blosc_codec;
mod blosc_configuration;
mod blosc_partial_decoder;
use std::{
ffi::c_int,
ffi::{c_char, c_void},
};
pub use blosc_codec::BloscCodec;
pub use blosc_configuration::{BloscCodecConfiguration, BloscCodecConfigurationV1};
use blosc_sys::{
blosc_cbuffer_validate, blosc_compress_ctx, blosc_decompress_ctx, BLOSC_BITSHUFFLE,
BLOSC_BLOSCLZ_COMPNAME, BLOSC_LZ4HC_COMPNAME, BLOSC_LZ4_COMPNAME, BLOSC_MAX_OVERHEAD,
BLOSC_NOSHUFFLE, BLOSC_SHUFFLE, BLOSC_SNAPPY_COMPNAME, BLOSC_ZLIB_COMPNAME,
BLOSC_ZSTD_COMPNAME,
};
use serde::{Deserialize, Serialize};
use thiserror::Error;
#[derive(Debug, Error)]
#[error("blosc error")]
struct BloscError;
#[derive(Serialize, Copy, Clone, Debug, Eq, PartialEq)]
pub struct BloscCompressionLevel(u8);
impl TryFrom<u8> for BloscCompressionLevel {
type Error = u8;
fn try_from(level: u8) -> Result<Self, Self::Error> {
if level <= 9 {
Ok(Self(level))
} else {
Err(level)
}
}
}
impl<'de> serde::Deserialize<'de> for BloscCompressionLevel {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let level = u8::deserialize(d)?;
if level <= 9 {
Ok(Self(level))
} else {
Err(serde::de::Error::custom("clevel must be between 0 and 9"))
}
}
}
#[derive(Serialize, Deserialize, Copy, Clone, Debug, Eq, PartialEq)]
#[serde(rename_all = "lowercase")]
#[repr(u32)]
pub enum BloscShuffleMode {
NoShuffle = BLOSC_NOSHUFFLE,
Shuffle = BLOSC_SHUFFLE,
BitShuffle = BLOSC_BITSHUFFLE,
}
#[derive(Serialize, Deserialize, Copy, Clone, Debug, Eq, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum BloscCompressor {
BloscLZ,
LZ4,
LZ4HC,
Snappy,
Zlib,
Zstd,
}
impl BloscCompressor {
fn as_cstr(&self) -> *const u8 {
match self {
BloscCompressor::BloscLZ => BLOSC_BLOSCLZ_COMPNAME.as_ptr(),
BloscCompressor::LZ4 => BLOSC_LZ4_COMPNAME.as_ptr(),
BloscCompressor::LZ4HC => BLOSC_LZ4HC_COMPNAME.as_ptr(),
BloscCompressor::Snappy => BLOSC_SNAPPY_COMPNAME.as_ptr(),
BloscCompressor::Zlib => BLOSC_ZLIB_COMPNAME.as_ptr(),
BloscCompressor::Zstd => BLOSC_ZSTD_COMPNAME.as_ptr(),
}
}
}
fn compress_bytes(
src: &[u8],
clevel: BloscCompressionLevel,
shuffle_mode: BloscShuffleMode,
typesize: usize,
compressor: BloscCompressor,
blocksize: usize,
) -> Result<Vec<u8>, BloscError> {
let destsize = src.len() + BLOSC_MAX_OVERHEAD as usize;
let mut dest: Vec<u8> = Vec::with_capacity(destsize);
let destsize = unsafe {
blosc_compress_ctx(
c_int::from(clevel.0),
shuffle_mode as c_int,
typesize,
src.len(),
src.as_ptr().cast::<c_void>(),
dest.as_mut_ptr().cast::<c_void>(),
destsize,
compressor.as_cstr().cast::<c_char>(),
blocksize,
1,
)
};
if destsize > 0 {
unsafe {
#[allow(clippy::cast_sign_loss)]
dest.set_len(destsize as usize);
}
dest.shrink_to_fit();
Ok(dest)
} else {
Err(BloscError)
}
}
fn decompress_bytes(src: &[u8]) -> Result<Vec<u8>, BloscError> {
let mut destsize: usize = 0;
let bytes_safe: bool = unsafe {
blosc_cbuffer_validate(
src.as_ptr().cast::<c_void>(),
src.len(),
std::ptr::addr_of_mut!(destsize),
)
} == 0;
if bytes_safe {
let mut dest: Vec<u8> = Vec::with_capacity(destsize);
let destsize = unsafe {
blosc_decompress_ctx(
src.as_ptr().cast::<c_void>(),
dest.as_mut_ptr().cast::<c_void>(),
destsize,
1,
)
};
if destsize <= 0 {
Err(BloscError)
} else {
unsafe {
#[allow(clippy::cast_sign_loss)]
dest.set_len(destsize as usize);
}
dest.shrink_to_fit();
Ok(dest)
}
} else {
Err(BloscError)
}
}
#[cfg(test)]
mod tests {
use crate::{
array::{
codec::BytesToBytesCodecTraits, ArrayRepresentation, BytesRepresentation, DataType,
FillValue,
},
array_subset::ArraySubset,
byte_range::ByteRange,
};
use super::*;
const JSON_VALID1: &'static str = r#"
{
"cname": "lz4",
"clevel": 5,
"shuffle": "shuffle",
"typesize": 4,
"blocksize": 0
}"#;
const JSON_VALID2: &'static str = r#"
{
"cname": "lz4",
"clevel": 4,
"shuffle": "bitshuffle",
"typesize": 4,
"blocksize": 0
}"#;
#[test]
fn codec_blosc_round_trip1() {
let elements: Vec<u16> = (0..32).collect();
let bytes = safe_transmute::transmute_to_bytes(&elements).to_vec();
let bytes_representation = BytesRepresentation::KnownSize(bytes.len());
let codec_configuration: BloscCodecConfiguration =
serde_json::from_str(JSON_VALID1).unwrap();
let codec = BloscCodec::new_with_configuration(&codec_configuration).unwrap();
let encoded = codec.encode(bytes.clone()).unwrap();
let decoded = codec
.decode(encoded.clone(), &bytes_representation)
.unwrap();
assert_eq!(bytes, decoded);
}
#[test]
fn codec_blosc_round_trip2() {
let elements: Vec<u16> = (0..32).collect();
let bytes = safe_transmute::transmute_to_bytes(&elements).to_vec();
let bytes_representation = BytesRepresentation::KnownSize(bytes.len());
let codec_configuration: BloscCodecConfiguration =
serde_json::from_str(JSON_VALID2).unwrap();
let codec = BloscCodec::new_with_configuration(&codec_configuration).unwrap();
let encoded = codec.encode(bytes.clone()).unwrap();
let decoded = codec
.decode(encoded.clone(), &bytes_representation)
.unwrap();
assert_eq!(bytes, decoded);
}
#[test]
fn codec_blosc_partial_decode() {
let array_representation =
ArrayRepresentation::new(vec![2, 2, 2], DataType::UInt16, FillValue::from(0u16))
.unwrap();
let bytes_representation = BytesRepresentation::KnownSize(array_representation.size());
let elements: Vec<u16> = (0..array_representation.num_elements() as u16).collect();
let bytes = safe_transmute::transmute_to_bytes(&elements).to_vec();
let codec_configuration: BloscCodecConfiguration =
serde_json::from_str(JSON_VALID2).unwrap();
let codec = BloscCodec::new_with_configuration(&codec_configuration).unwrap();
let encoded = codec.encode(bytes.clone()).unwrap();
let decoded_regions: Vec<ByteRange> =
ArraySubset::new_with_start_shape(vec![0, 1, 0], vec![2, 1, 1])
.unwrap()
.byte_ranges(
array_representation.shape(),
array_representation.element_size(),
)
.unwrap();
let input_handle = Box::new(std::io::Cursor::new(encoded));
let partial_decoder = codec.partial_decoder(input_handle);
let decoded = partial_decoder
.partial_decode(&bytes_representation, &decoded_regions)
.unwrap();
let decoded: Vec<u16> = decoded
.into_iter()
.flatten()
.collect::<Vec<_>>()
.chunks(std::mem::size_of::<u16>())
.map(|b| u16::from_ne_bytes(b.try_into().unwrap()))
.collect();
let answer: Vec<u16> = vec![2, 6];
assert_eq!(answer, decoded);
}
}