use hdf5_pure::{
AttrValue, Datatype, DatatypeByteOrder, FileBuilder, FileSpaceStrategy, RepackOptions,
ScaleOffset, repack,
};
fn tmp(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(name)
}
#[test]
fn drops_object_and_shrinks_file() {
let src = tmp("hdf5_pure_repack_drop_src.h5");
let dst = tmp("hdf5_pure_repack_drop_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("keep").with_i32_data(&[1, 2, 3]);
b.create_dataset("bulk").with_f64_data(&vec![9.0; 4096]);
b.write(&src).unwrap();
let src_size = std::fs::metadata(&src).unwrap().len();
repack(&src, &dst, &RepackOptions::new().drop_path("bulk")).unwrap();
let dst_size = std::fs::metadata(&dst).unwrap().len();
assert!(
dst_size < src_size,
"dropping the bulk dataset should shrink the file (src {src_size}, dst {dst_size})"
);
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.root().datasets().unwrap(), vec!["keep".to_string()]);
assert_eq!(
f.dataset("keep").unwrap().read_i32().unwrap(),
vec![1, 2, 3]
);
assert!(f.dataset("bulk").is_err());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn pure_compaction_preserves_everything() {
let src = tmp("hdf5_pure_repack_compact_src.h5");
let dst = tmp("hdf5_pure_repack_compact_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("ints").with_i64_data(&[10, 20, 30, 40]);
b.create_dataset("floats").with_f64_data(&[1.5, 2.5, 3.5]);
b.set_attr("title", AttrValue::String("experiment".to_string()));
let mut g = b.create_group("grp");
g.create_dataset("inner").with_f32_data(&[0.25, 0.5]);
g.set_attr("units", AttrValue::AsciiString("m/s".to_string()));
b.add_group(g.finish());
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(
f.dataset("ints").unwrap().read_i64().unwrap(),
vec![10, 20, 30, 40]
);
assert_eq!(
f.dataset("floats").unwrap().read_f64().unwrap(),
vec![1.5, 2.5, 3.5]
);
assert_eq!(
f.dataset("grp/inner").unwrap().read_f32().unwrap(),
vec![0.25, 0.5]
);
let root_attrs = f.root().attrs().unwrap();
assert_eq!(
root_attrs.get("title"),
Some(&AttrValue::String("experiment".to_string()))
);
let grp_attrs = f.group("grp").unwrap().attrs().unwrap();
assert_eq!(
grp_attrs.get("units"),
Some(&AttrValue::String("m/s".to_string()))
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_v1_symbol_table_source_with_attributes() {
let dst = tmp("hdf5_pure_repack_v1_attrs_dst.h5");
let src = "tests/fixtures/attrs.h5";
let source = hdf5_pure::File::open(src).unwrap();
let src_data = source.dataset("data").unwrap().read_f64().unwrap();
let src_data_attrs = source.dataset("data").unwrap().attrs().unwrap();
let src_root_attrs = source.root().attrs().unwrap();
assert!(!src_data_attrs.is_empty() && !src_root_attrs.is_empty());
repack(src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), src_data);
assert_eq!(f.dataset("data").unwrap().attrs().unwrap(), src_data_attrs);
assert_eq!(f.root().attrs().unwrap(), src_root_attrs);
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_v1_nested_symbol_table_groups() {
let dst = tmp("hdf5_pure_repack_v1_groups_dst.h5");
let src = "tests/fixtures/two_groups.h5";
repack(src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let mut groups = f.root().groups().unwrap();
groups.sort();
assert_eq!(groups, vec!["group1".to_string(), "group2".to_string()]);
assert_eq!(
f.dataset("group1/values").unwrap().read_i32().unwrap(),
vec![10, 20, 30]
);
assert_eq!(
f.group("group2").unwrap().datasets().unwrap(),
vec!["temps".to_string()]
);
std::fs::remove_file(&dst).ok();
}
#[test]
fn carries_dataset_attributes() {
let src = tmp("hdf5_pure_repack_dsattr_src.h5");
let dst = tmp("hdf5_pure_repack_dsattr_dst.h5");
let mut b = FileBuilder::new();
let ds = b.create_dataset("signal");
ds.with_f64_data(&[1.0, 2.0, 3.0]);
ds.set_attr("sample_rate", AttrValue::F64(44100.0));
ds.set_attr("label", AttrValue::String("voltage".to_string()));
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let attrs = f.dataset("signal").unwrap().attrs().unwrap();
assert_eq!(attrs.get("sample_rate"), Some(&AttrValue::F64(44100.0)));
assert_eq!(
attrs.get("label"),
Some(&AttrValue::String("voltage".to_string()))
);
assert_eq!(
f.dataset("signal").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn drops_whole_group_subtree() {
let src = tmp("hdf5_pure_repack_dropgrp_src.h5");
let dst = tmp("hdf5_pure_repack_dropgrp_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("root_ds").with_i32_data(&[1]);
let mut g = b.create_group("doomed");
g.create_dataset("a").with_i32_data(&[1, 2]);
g.create_dataset("b").with_i32_data(&[3, 4]);
b.add_group(g.finish());
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new().drop_path("/doomed")).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.root().datasets().unwrap(), vec!["root_ds".to_string()]);
assert!(f.group("doomed").is_err());
assert!(f.dataset("doomed/a").is_err());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn preserves_chunked_and_compressed_dataset() {
let src = tmp("hdf5_pure_repack_chunk_src.h5");
let dst = tmp("hdf5_pure_repack_chunk_dst.h5");
let data: Vec<f64> = (0..2048).map(|i| i as f64 * 0.5).collect();
let mut b = FileBuilder::new();
b.create_dataset("keep").with_i32_data(&[7]);
b.create_dataset("comp")
.with_f64_data(&data)
.with_chunks(&[256])
.with_shuffle()
.with_deflate(6);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.dataset("comp").unwrap().read_f64().unwrap(), data);
assert_eq!(f.dataset("keep").unwrap().read_i32().unwrap(), vec![7]);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn preserves_multidim_and_maxshape() {
let src = tmp("hdf5_pure_repack_md_src.h5");
let dst = tmp("hdf5_pure_repack_md_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("grid")
.with_f64_data(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
.with_shape(&[2, 3])
.with_maxshape(&[u64::MAX, 3])
.with_chunks(&[1, 3]);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let ds = f.dataset("grid").unwrap();
assert_eq!(ds.shape().unwrap(), vec![2, 3]);
assert_eq!(ds.read_f64().unwrap(), vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn roundtrips_integer_scale_offset() {
let data: Vec<i32> = (0..4096).map(|i| i % 8).collect();
let so_src = tmp("hdf5_pure_repack_so_src.h5");
let so_dst = tmp("hdf5_pure_repack_so_dst.h5");
let plain_src = tmp("hdf5_pure_repack_soplain_src.h5");
let plain_dst = tmp("hdf5_pure_repack_soplain_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("vals")
.with_i32_data(&data)
.with_chunks(&[512])
.with_scale_offset(ScaleOffset::Integer(0));
b.write(&so_src).unwrap();
let mut p = FileBuilder::new();
p.create_dataset("vals")
.with_i32_data(&data)
.with_chunks(&[512]);
p.write(&plain_src).unwrap();
repack(&so_src, &so_dst, &RepackOptions::new()).unwrap();
repack(&plain_src, &plain_dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&so_dst).unwrap();
assert_eq!(f.dataset("vals").unwrap().read_i32().unwrap(), data);
let so_size = std::fs::metadata(&so_dst).unwrap().len();
let plain_size = std::fs::metadata(&plain_dst).unwrap().len();
assert!(
so_size < plain_size,
"repacked scale-offset file ({so_size}) should be smaller than the unfiltered repack ({plain_size}), proving the filter was re-applied"
);
for p in [so_src, so_dst, plain_src, plain_dst] {
std::fs::remove_file(p).ok();
}
}
#[test]
fn roundtrips_lossy_float_scale_offset_verbatim() {
let src = tmp("hdf5_pure_repack_fso_src.h5");
let dst = tmp("hdf5_pure_repack_fso_dst.h5");
let data: Vec<f64> = (0..1024).map(|i| (i as f64) * 0.01).collect();
let mut b = FileBuilder::new();
b.create_dataset("vals")
.with_f64_data(&data)
.with_chunks(&[256])
.with_scale_offset(ScaleOffset::FloatDScale(3));
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let src_vals = hdf5_pure::File::open(&src)
.unwrap()
.dataset("vals")
.unwrap()
.read_f64()
.unwrap();
let dst_f = hdf5_pure::File::open(&dst).unwrap();
let dst_ds = dst_f.dataset("vals").unwrap();
assert_eq!(
dst_ds.read_f64().unwrap(),
src_vals,
"verbatim chunk copy must reproduce the (lossy-rounded) values exactly"
);
assert!(
dst_ds.chunk_cache_stats().index_loaded(),
"repacked dataset should still be chunked"
);
assert!(
std::fs::metadata(&dst).unwrap().len() < 1024 * 8,
"scale-offset filter should keep the repacked file below the raw data size"
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn roundtrips_opaque_and_bitfield_datatypes() {
let src = tmp("hdf5_pure_repack_dt_src.h5");
let dst = tmp("hdf5_pure_repack_dt_dst.h5");
let opaque_dt = Datatype::Opaque {
size: 4,
tag: b"rgba".to_vec(),
};
let opaque_raw: Vec<u8> = vec![0xDE, 0xAD, 0xBE, 0xEF, 1, 2, 3, 4, 9, 8, 7, 6];
let bitfield_dt = Datatype::BitField {
size: 2,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 12,
};
let bitfield_raw: Vec<u8> = vec![0x34, 0x12, 0xFF, 0x0F];
let mut b = FileBuilder::new();
b.create_dataset("blob")
.with_raw_data(opaque_dt.clone(), opaque_raw.clone(), 3);
b.create_dataset("flags")
.with_raw_data(bitfield_dt.clone(), bitfield_raw.clone(), 2);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let blob = f.dataset("blob").unwrap();
assert_eq!(blob.datatype().unwrap(), opaque_dt);
assert_eq!(blob.read_raw().unwrap(), opaque_raw);
let flags = f.dataset("flags").unwrap();
assert_eq!(flags.datatype().unwrap(), bitfield_dt);
assert_eq!(flags.read_raw().unwrap(), bitfield_raw);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn preserves_file_space_strategy() {
let src = tmp("hdf5_pure_repack_fss_src.h5");
let dst = tmp("hdf5_pure_repack_fss_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("keep").with_i32_data(&[1, 2, 3]);
b.create_dataset("drop_me").with_f64_data(&vec![0.0; 1000]);
b.with_file_space_strategy(FileSpaceStrategy::Page, false, 4)
.with_file_space_page_size(8192);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new().drop_path("drop_me")).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.file_space_strategy(), Some(FileSpaceStrategy::Page));
let info = f.file_space_info().unwrap();
assert_eq!(info.page_size, 8192);
assert_eq!(info.threshold, 4);
assert!(!info.persist);
assert_eq!(
f.dataset("keep").unwrap().read_i32().unwrap(),
vec![1, 2, 3]
);
assert!(f.dataset("drop_me").is_err());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn rejects_nonexistent_drop_path() {
let src = tmp("hdf5_pure_repack_baddrop_src.h5");
let dst = tmp("hdf5_pure_repack_baddrop_dst.h5");
let mut b = FileBuilder::new();
b.create_dataset("present").with_i32_data(&[1]);
b.write(&src).unwrap();
let err = repack(&src, &dst, &RepackOptions::new().drop_path("absent")).unwrap_err();
assert!(
matches!(err, hdf5_pure::Error::RepackUnsupported(_)),
"expected RepackUnsupported, got {err:?}"
);
assert!(
!dst.exists(),
"dst must not be created when the repack fails"
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn verbatim_chunk_copy_preserves_compressed_bytes() {
let src = tmp("hdf5_pure_repack_verbatim_src.h5");
let dst = tmp("hdf5_pure_repack_verbatim_dst.h5");
let data: Vec<i32> = (0..4096).map(|i| i % 4).collect();
let mut b = FileBuilder::new();
b.create_dataset("vals")
.with_i32_data(&data)
.with_chunks(&[512])
.with_shuffle()
.with_deflate(6);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let ds = f.dataset("vals").unwrap();
assert_eq!(
ds.read_i32().unwrap(),
data,
"values must round-trip exactly"
);
assert!(
ds.chunk_cache_stats().index_loaded(),
"repacked dataset must still be chunked"
);
assert!(
std::fs::metadata(&dst).unwrap().len() < 4096 * 4,
"deflate filter must survive (file smaller than raw element bytes)"
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_multichunk_2d_fixed_array() {
let src = tmp("hdf5_pure_repack_fa_src.h5");
let dst = tmp("hdf5_pure_repack_fa_dst.h5");
let data: Vec<f64> = (0..16).map(|i| i as f64 * 1.5).collect();
let mut b = FileBuilder::new();
b.create_dataset("grid")
.with_f64_data(&data)
.with_shape(&[4, 4])
.with_chunks(&[2, 2])
.with_deflate(4);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let ds = f.dataset("grid").unwrap();
assert_eq!(ds.shape().unwrap(), vec![4, 4]);
assert_eq!(ds.read_f64().unwrap(), data);
assert!(ds.chunk_cache_stats().index_loaded());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_resizable_extensible_array() {
let src = tmp("hdf5_pure_repack_ea_src.h5");
let dst = tmp("hdf5_pure_repack_ea_dst.h5");
let data: Vec<i64> = (0..1000).collect();
let mut b = FileBuilder::new();
b.create_dataset("series")
.with_i64_data(&data)
.with_shape(&[1000])
.with_maxshape(&[u64::MAX])
.with_chunks(&[128])
.with_deflate(3);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let ds = f.dataset("series").unwrap();
assert_eq!(ds.read_i64().unwrap(), data);
assert_eq!(ds.shape().unwrap(), vec![1000]);
assert!(ds.chunk_cache_stats().index_loaded());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[cfg(feature = "zfp")]
#[test]
fn roundtrips_zfp_verbatim() {
let src = tmp("hdf5_pure_repack_zfp_src.h5");
let dst = tmp("hdf5_pure_repack_zfp_dst.h5");
let data: Vec<f64> = (0..1024).map(|i| (i as f64).sin()).collect();
let mut b = FileBuilder::new();
b.create_dataset("vals")
.with_f64_data(&data)
.with_chunks(&[256])
.with_zfp(32.0);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let src_vals = hdf5_pure::File::open(&src)
.unwrap()
.dataset("vals")
.unwrap()
.read_f64()
.unwrap();
let dst_f = hdf5_pure::File::open(&dst).unwrap();
let dst_ds = dst_f.dataset("vals").unwrap();
assert_eq!(
dst_ds.read_f64().unwrap(),
src_vals,
"verbatim ZFP chunk copy must reproduce the stored values exactly"
);
assert!(
dst_ds.chunk_cache_stats().index_loaded(),
"repacked ZFP dataset must still be chunked"
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_single_chunk_filtered_verbatim() {
let src = tmp("hdf5_pure_repack_single_src.h5");
let dst = tmp("hdf5_pure_repack_single_dst.h5");
let data: Vec<i32> = (0..256).map(|i| i % 5).collect();
let mut b = FileBuilder::new();
b.create_dataset("vals")
.with_i32_data(&data)
.with_chunks(&[256]) .with_deflate(6);
b.write(&src).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
let ds = f.dataset("vals").unwrap();
assert_eq!(ds.read_i32().unwrap(), data);
assert!(ds.chunk_cache_stats().index_loaded());
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}
#[test]
fn repacks_chunked_dataset_from_a_userblock_file() {
let src = tmp("hdf5_pure_repack_ub_src.h5");
let dst = tmp("hdf5_pure_repack_ub_dst.h5");
let data: Vec<f64> = (0..1000).map(|i| i as f64 * 0.25).collect();
let mut b = FileBuilder::new();
b.with_userblock(512);
b.create_dataset("chk")
.with_f64_data(&data)
.with_shape(&[1000])
.with_deflate(6);
b.create_dataset("plain").with_f64_data(&[1.0, 2.0, 3.0]);
std::fs::write(&src, b.finish().unwrap()).unwrap();
repack(&src, &dst, &RepackOptions::new()).unwrap();
let f = hdf5_pure::File::open(&dst).unwrap();
assert_eq!(f.dataset("chk").unwrap().read_f64().unwrap(), data);
assert_eq!(
f.dataset("plain").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
std::fs::remove_file(&src).ok();
std::fs::remove_file(&dst).ok();
}