use std::collections::HashMap;
use std::path::Path;
use crate::attribute::{Attribute, AttributeValue, Attributes};
use crate::dimension::{Dimension, Dimensions};
use crate::error::{NetCdfError, Result};
use crate::metadata::{CfMetadata, NetCdfMetadata, NetCdfVersion};
use crate::variable::{DataType, Variable, Variables};
use oxinetcdf::{ByteOrder, Dtype, NcAttribute, NcFile, NcGroup};
#[cfg(feature = "netcdf3")]
use std::cell::RefCell;
pub struct NetCdfReader {
metadata: NetCdfMetadata,
nc4: Option<Nc4Backend>,
#[cfg(feature = "netcdf3")]
file_nc3: Option<RefCell<netcdf3::FileReader>>,
}
struct Nc4Backend {
file: NcFile,
var_paths: HashMap<String, String>,
}
impl std::fmt::Debug for NetCdfReader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NetCdfReader")
.field("metadata", &self.metadata)
.finish_non_exhaustive()
}
}
impl NetCdfReader {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
#[cfg(feature = "netcdf3")]
{
if let Ok(file) = netcdf3::FileReader::open(path) {
return Self::from_netcdf3(file);
}
}
match NcFile::open(path) {
Ok(file) => Self::from_oxinetcdf(file),
Err(e) => Err(NetCdfError::InvalidFormat(format!(
"file is not a readable NetCDF-3 or NetCDF-4/HDF5 file: {e}"
))),
}
}
fn from_oxinetcdf(file: NcFile) -> Result<Self> {
let root = file.root_group().map_err(map_nc_err)?;
let mut metadata = build_metadata_from_group(&root)?;
metadata.parse_cf_metadata();
let var_paths = root
.variables
.iter()
.map(|v| (v.name.clone(), v.h5_path.clone()))
.collect();
Ok(Self {
metadata,
nc4: Some(Nc4Backend { file, var_paths }),
#[cfg(feature = "netcdf3")]
file_nc3: None,
})
}
#[cfg(feature = "netcdf3")]
pub fn from_netcdf3(file: netcdf3::FileReader) -> Result<Self> {
let metadata = Self::read_metadata_nc3(&file)?;
Ok(Self {
metadata,
nc4: None,
file_nc3: Some(RefCell::new(file)),
})
}
#[must_use]
pub const fn metadata(&self) -> &NetCdfMetadata {
&self.metadata
}
#[must_use]
pub fn version(&self) -> NetCdfVersion {
self.metadata.version()
}
#[must_use]
pub fn dimensions(&self) -> &Dimensions {
self.metadata.dimensions()
}
#[must_use]
pub fn variables(&self) -> &Variables {
self.metadata.variables()
}
#[must_use]
pub fn global_attributes(&self) -> &Attributes {
self.metadata.global_attributes()
}
#[must_use]
pub fn cf_metadata(&self) -> Option<&CfMetadata> {
self.metadata.cf_metadata()
}
#[cfg(feature = "netcdf3")]
fn read_metadata_nc3(file: &netcdf3::FileReader) -> Result<NetCdfMetadata> {
use crate::nc3_compat;
let mut metadata = NetCdfMetadata::new_classic();
let dataset = file.data_set();
let dimensions = nc3_compat::read_dimensions(dataset)?;
for dimension in dimensions {
metadata.dimensions_mut().add(dimension)?;
}
for attr_name in dataset.get_global_attr_names() {
if let Some(attr) = nc3_compat::read_global_attribute(dataset, &attr_name)? {
metadata.global_attributes_mut().add(attr)?;
}
}
for var_name in dataset.get_var_names() {
let var = nc3_compat::read_variable(dataset, &var_name)?;
metadata.variables_mut().add(var)?;
}
metadata.parse_cf_metadata();
Ok(metadata)
}
#[cfg(feature = "netcdf3")]
fn convert_datatype_nc3(nc3_type: netcdf3::DataType) -> Result<DataType> {
use netcdf3::DataType as Nc3Type;
match nc3_type {
Nc3Type::I8 => Ok(DataType::I8),
Nc3Type::I16 => Ok(DataType::I16),
Nc3Type::I32 => Ok(DataType::I32),
Nc3Type::F32 => Ok(DataType::F32),
Nc3Type::F64 => Ok(DataType::F64),
Nc3Type::U8 => Ok(DataType::Char), }
}
pub fn read_f32(&self, var_name: &str) -> Result<Vec<f32>> {
if let Some(nc4) = &self.nc4 {
return nc4.read_f32(var_name);
}
#[cfg(feature = "netcdf3")]
if let Some(ref file_cell) = self.file_nc3 {
return Self::read_f32_nc3(&mut file_cell.borrow_mut(), var_name);
}
Err(NetCdfError::FeatureNotEnabled {
feature: "netcdf reader".to_string(),
message: "No reader backend available".to_string(),
})
}
pub fn read_f64(&self, var_name: &str) -> Result<Vec<f64>> {
if let Some(nc4) = &self.nc4 {
return nc4.read_f64(var_name);
}
#[cfg(feature = "netcdf3")]
if let Some(ref file_cell) = self.file_nc3 {
return Self::read_f64_nc3(&mut file_cell.borrow_mut(), var_name);
}
Err(NetCdfError::FeatureNotEnabled {
feature: "netcdf reader".to_string(),
message: "No reader backend available".to_string(),
})
}
pub fn read_i32(&self, var_name: &str) -> Result<Vec<i32>> {
if let Some(nc4) = &self.nc4 {
return nc4.read_i32(var_name);
}
#[cfg(feature = "netcdf3")]
if let Some(ref file_cell) = self.file_nc3 {
return Self::read_i32_nc3(&mut file_cell.borrow_mut(), var_name);
}
Err(NetCdfError::FeatureNotEnabled {
feature: "netcdf reader".to_string(),
message: "No reader backend available".to_string(),
})
}
#[cfg(feature = "netcdf3")]
fn read_f32_nc3(file: &mut netcdf3::FileReader, var_name: &str) -> Result<Vec<f32>> {
let dataset = file.data_set();
let var_info = dataset
.get_var(var_name)
.ok_or_else(|| NetCdfError::VariableNotFound {
name: var_name.to_string(),
})?;
use netcdf3::DataType as Nc3Type;
let data_type = var_info.data_type();
if data_type != Nc3Type::F32 {
return Err(NetCdfError::DataTypeMismatch {
expected: "F32".to_string(),
found: format!("{:?}", data_type),
});
}
let data = file.read_var_f32(var_name)?;
Ok(data)
}
#[cfg(feature = "netcdf3")]
fn read_f64_nc3(file: &mut netcdf3::FileReader, var_name: &str) -> Result<Vec<f64>> {
let dataset = file.data_set();
let var_info = dataset
.get_var(var_name)
.ok_or_else(|| NetCdfError::VariableNotFound {
name: var_name.to_string(),
})?;
use netcdf3::DataType as Nc3Type;
let data_type = var_info.data_type();
if data_type != Nc3Type::F64 {
return Err(NetCdfError::DataTypeMismatch {
expected: "F64".to_string(),
found: format!("{:?}", data_type),
});
}
let data = file.read_var_f64(var_name)?;
Ok(data)
}
#[cfg(feature = "netcdf3")]
fn read_i32_nc3(file: &mut netcdf3::FileReader, var_name: &str) -> Result<Vec<i32>> {
let dataset = file.data_set();
let var_info = dataset
.get_var(var_name)
.ok_or_else(|| NetCdfError::VariableNotFound {
name: var_name.to_string(),
})?;
use netcdf3::DataType as Nc3Type;
let data_type = var_info.data_type();
if data_type != Nc3Type::I32 {
return Err(NetCdfError::DataTypeMismatch {
expected: "I32".to_string(),
found: format!("{:?}", data_type),
});
}
let data = file.read_var_i32(var_name)?;
Ok(data)
}
}
impl Nc4Backend {
fn path(&self, var_name: &str) -> Result<&str> {
self.var_paths
.get(var_name)
.map(String::as_str)
.ok_or_else(|| NetCdfError::VariableNotFound {
name: var_name.to_string(),
})
}
fn read_f32(&self, var_name: &str) -> Result<Vec<f32>> {
let path = self.path(var_name)?;
let ds = self
.file
.h5()
.dataset(path)
.map_err(|e| NetCdfError::Io(format!("read '{var_name}': {e}")))?;
ds.as_f32().map_err(|e| NetCdfError::DataTypeMismatch {
expected: "f32".to_string(),
found: format!("{e}"),
})
}
fn read_f64(&self, var_name: &str) -> Result<Vec<f64>> {
let path = self.path(var_name)?;
let ds = self
.file
.h5()
.dataset(path)
.map_err(|e| NetCdfError::Io(format!("read '{var_name}': {e}")))?;
ds.as_f64().map_err(|e| NetCdfError::DataTypeMismatch {
expected: "f64".to_string(),
found: format!("{e}"),
})
}
fn read_i32(&self, var_name: &str) -> Result<Vec<i32>> {
let path = self.path(var_name)?;
let ds = self
.file
.h5()
.dataset(path)
.map_err(|e| NetCdfError::Io(format!("read '{var_name}': {e}")))?;
ds.as_i32().map_err(|e| NetCdfError::DataTypeMismatch {
expected: "i32".to_string(),
found: format!("{e}"),
})
}
}
fn map_nc_err(e: oxinetcdf::NcError) -> NetCdfError {
match e {
oxinetcdf::NcError::VariableNotFound(name) => NetCdfError::VariableNotFound { name },
oxinetcdf::NcError::Unsupported(msg) => NetCdfError::Other(format!("unsupported: {msg}")),
other => NetCdfError::InvalidFormat(other.to_string()),
}
}
fn build_metadata_from_group(root: &NcGroup) -> Result<NetCdfMetadata> {
let version = if root.attrs.iter().any(|a| a.name == "_nc3_strict") {
NetCdfVersion::NetCdf4Classic
} else {
NetCdfVersion::NetCdf4
};
let mut metadata = NetCdfMetadata::new(version);
for d in &root.dimensions {
add_dimension(&mut metadata, &d.name, d.len, d.is_unlimited)?;
}
for v in &root.variables {
let Some(data_type) = dtype_to_datatype(&v.dtype) else {
continue;
};
for axis in &v.dims {
if !metadata.dimensions().contains(&axis.name) {
add_dimension(&mut metadata, &axis.name, axis.len, axis.is_unlimited)?;
}
}
let dim_names: Vec<String> = v.dims.iter().map(|a| a.name.clone()).collect();
let mut variable = Variable::new(&v.name, data_type, dim_names)?;
variable.set_coordinate(v.is_coordinate);
for attr in &v.attrs {
if let Some(value) = nc_attr_to_value(attr)
&& let Ok(a) = Attribute::new(attr.name.clone(), value)
{
variable.attributes_mut().set(a);
}
}
let _ = metadata.variables_mut().add(variable);
}
for attr in &root.attrs {
if let Some(value) = nc_attr_to_value(attr)
&& let Ok(a) = Attribute::new(attr.name.clone(), value)
{
let _ = metadata.global_attributes_mut().add(a);
}
}
Ok(metadata)
}
fn add_dimension(
metadata: &mut NetCdfMetadata,
name: &str,
len: u64,
unlimited: bool,
) -> Result<()> {
if metadata.dimensions().contains(name) {
return Ok(());
}
let len = usize::try_from(len).map_err(|_| NetCdfError::InvalidShape {
message: format!("dimension '{name}' length {len} exceeds usize"),
})?;
let dim = if unlimited {
Dimension::new_unlimited(name, len)?
} else {
Dimension::new(name, len)?
};
let _ = metadata.dimensions_mut().add(dim);
Ok(())
}
fn dtype_to_datatype(dtype: &Dtype) -> Option<DataType> {
Some(match dtype {
Dtype::Float { size: 4, .. } => DataType::F32,
Dtype::Float { size: 8, .. } => DataType::F64,
Dtype::Int {
size: 1,
signed: true,
..
} => DataType::I8,
Dtype::Int {
size: 1,
signed: false,
..
} => DataType::U8,
Dtype::Int {
size: 2,
signed: true,
..
} => DataType::I16,
Dtype::Int {
size: 2,
signed: false,
..
} => DataType::U16,
Dtype::Int {
size: 4,
signed: true,
..
} => DataType::I32,
Dtype::Int {
size: 4,
signed: false,
..
} => DataType::U32,
Dtype::Int {
size: 8,
signed: true,
..
} => DataType::I64,
Dtype::Int {
size: 8,
signed: false,
..
} => DataType::U64,
Dtype::String {
fixed_len: Some(1), ..
} => DataType::Char,
Dtype::String { .. } => DataType::String,
_ => return None,
})
}
fn nc_attr_to_value(attr: &NcAttribute) -> Option<AttributeValue> {
let raw = attr.raw();
let value = match &raw.dtype {
Dtype::String { .. } => AttributeValue::Text(attr.as_text().ok()?),
Dtype::Float { size: 4, order } => AttributeValue::F32(decode_f32(&raw.data, order)),
Dtype::Float { size: 8, order } => AttributeValue::F64(decode_f64(&raw.data, order)),
Dtype::Int {
size: 1,
signed: true,
..
} => AttributeValue::I8(raw.data.iter().map(|&b| b as i8).collect()),
Dtype::Int {
size: 1,
signed: false,
..
} => AttributeValue::U8(raw.data.clone()),
Dtype::Int {
size: 2,
signed: true,
order,
} => AttributeValue::I16(decode_chunks(
&raw.data,
order,
i16::from_le_bytes,
i16::from_be_bytes,
)),
Dtype::Int {
size: 2,
signed: false,
order,
} => AttributeValue::U16(decode_chunks(
&raw.data,
order,
u16::from_le_bytes,
u16::from_be_bytes,
)),
Dtype::Int {
size: 4,
signed: true,
order,
} => AttributeValue::I32(decode_chunks(
&raw.data,
order,
i32::from_le_bytes,
i32::from_be_bytes,
)),
Dtype::Int {
size: 4,
signed: false,
order,
} => AttributeValue::U32(decode_chunks(
&raw.data,
order,
u32::from_le_bytes,
u32::from_be_bytes,
)),
Dtype::Int {
size: 8,
signed: true,
order,
} => AttributeValue::I64(decode_chunks(
&raw.data,
order,
i64::from_le_bytes,
i64::from_be_bytes,
)),
Dtype::Int {
size: 8,
signed: false,
order,
} => AttributeValue::U64(decode_chunks(
&raw.data,
order,
u64::from_le_bytes,
u64::from_be_bytes,
)),
_ => return None,
};
Some(value)
}
fn decode_chunks<T, const N: usize>(
data: &[u8],
order: &ByteOrder,
from_le: fn([u8; N]) -> T,
from_be: fn([u8; N]) -> T,
) -> Vec<T> {
data.chunks_exact(N)
.filter_map(|c| <[u8; N]>::try_from(c).ok())
.map(|arr| match order {
ByteOrder::Little => from_le(arr),
ByteOrder::Big => from_be(arr),
})
.collect()
}
fn decode_f32(data: &[u8], order: &ByteOrder) -> Vec<f32> {
decode_chunks(data, order, f32::from_le_bytes, f32::from_be_bytes)
}
fn decode_f64(data: &[u8], order: &ByteOrder) -> Vec<f64> {
decode_chunks(data, order, f64::from_le_bytes, f64::from_be_bytes)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use oxinetcdf::{NcFileWriter, NcType, VarOrGroup};
fn write_real_nc4_fixture(tag: &str) -> std::path::PathBuf {
let path = std::env::temp_dir().join(format!(
"oxigdal_nc4_reader_{}_{}.nc",
tag,
std::process::id()
));
let mut nc = NcFileWriter::new();
let lat = nc.def_dim("lat", 3).expect("def lat");
let lon = nc.def_dim("lon", 4).expect("def lon");
let temp = nc
.def_var("temp", &[lat, lon], NcType::Float64)
.expect("def temp");
let data: Vec<f64> = (0..12).map(|i| i as f64 * 0.5).collect();
nc.put_var_f64(temp, &data).expect("put temp");
nc.put_att_str(VarOrGroup::Var(temp), "units", "kelvin")
.expect("units");
nc.put_att_str(VarOrGroup::Var(temp), "standard_name", "air_temperature")
.expect("standard_name");
nc.put_att_str(VarOrGroup::Root, "Conventions", "CF-1.8")
.expect("conventions");
nc.put_att_str(VarOrGroup::Root, "title", "Reader Fixture")
.expect("title");
nc.close(&path).expect("close fixture");
path
}
#[test]
fn test_open_real_netcdf4_reads_dimensions_and_variables() {
let path = write_real_nc4_fixture("dims");
let reader = NetCdfReader::open(&path).expect("open real NetCDF-4");
assert!(reader.version().is_netcdf4());
assert_eq!(reader.dimensions().len(), 2);
assert_eq!(reader.dimensions().get("lat").expect("lat dim").len(), 3);
assert_eq!(reader.dimensions().get("lon").expect("lon dim").len(), 4);
let temp = reader.variables().get("temp").expect("temp var");
assert_eq!(temp.data_type(), DataType::F64);
assert_eq!(temp.dimension_names(), &["lat", "lon"]);
let _ = std::fs::remove_file(&path);
}
#[test]
fn test_open_real_netcdf4_reads_variable_data() {
let path = write_real_nc4_fixture("data");
let reader = NetCdfReader::open(&path).expect("open real NetCDF-4");
let data = reader.read_f64("temp").expect("read temp");
assert_eq!(data.len(), 12);
for (i, &v) in data.iter().enumerate() {
assert!((v - i as f64 * 0.5).abs() < 1e-12);
}
let _ = std::fs::remove_file(&path);
}
#[test]
fn test_open_real_netcdf4_reads_attributes_and_cf() {
let path = write_real_nc4_fixture("attrs");
let reader = NetCdfReader::open(&path).expect("open real NetCDF-4");
let cf = reader.cf_metadata().expect("cf metadata");
assert!(cf.is_cf_compliant());
assert_eq!(cf.conventions.as_deref(), Some("CF-1.8"));
assert_eq!(cf.title.as_deref(), Some("Reader Fixture"));
let temp = reader.variables().get("temp").expect("temp var");
let units = temp.attributes().get("units").expect("units attr");
assert_eq!(units.value().as_text().expect("text"), "kelvin");
assert!(temp.attributes().get("standard_name").is_some());
assert!(temp.attributes().get("DIMENSION_LIST").is_none());
assert!(temp.attributes().get("CLASS").is_none());
let _ = std::fs::remove_file(&path);
}
#[test]
fn test_open_missing_variable_errors() {
let path = write_real_nc4_fixture("missing");
let reader = NetCdfReader::open(&path).expect("open real NetCDF-4");
let result = reader.read_f64("does_not_exist");
assert!(matches!(result, Err(NetCdfError::VariableNotFound { .. })));
let _ = std::fs::remove_file(&path);
}
#[test]
fn test_open_garbage_file_errors() {
let temp_dir = std::env::temp_dir();
let temp_file = temp_dir.join(format!("oxigdal_reader_garbage_{}.bin", std::process::id()));
std::fs::write(&temp_file, b"this is not any netcdf file").expect("write temp");
let result = NetCdfReader::open(&temp_file);
assert!(result.is_err());
assert!(!matches!(result, Err(NetCdfError::NetCdf4NotAvailable)));
let _ = std::fs::remove_file(&temp_file);
}
#[test]
fn test_data_type_conversion() {
#[cfg(feature = "netcdf3")]
{
use netcdf3::DataType as Nc3Type;
assert_eq!(
NetCdfReader::convert_datatype_nc3(Nc3Type::F32).expect("convert F32"),
DataType::F32
);
assert_eq!(
NetCdfReader::convert_datatype_nc3(Nc3Type::F64).expect("convert F64"),
DataType::F64
);
assert_eq!(
NetCdfReader::convert_datatype_nc3(Nc3Type::I32).expect("convert I32"),
DataType::I32
);
}
}
}