use std::collections::HashSet;
use std::path::Path;
use netcdf_reader::{NcFile, NcType, NcVariable};
use crate::error::{Error, Result};
use super::{ArrayData, Coord, DType, NativeDataset, NativeField, Reader, Records, Selection};
#[derive(Debug, Default, Clone, Copy)]
pub struct NetcdfReader;
impl NetcdfReader {
pub fn new() -> Self {
Self
}
}
impl Reader for NetcdfReader {
fn formats(&self) -> &'static [&'static str] {
&["netcdf"]
}
fn extensions(&self) -> &'static [&'static str] {
&["nc", "nc4", "cdf"]
}
fn read_native(
&self,
blob_path: &Path,
variables: &[String],
select: &Selection,
) -> Result<NativeDataset> {
self.read_native_records(blob_path, variables, select, None)
}
fn read_native_records(
&self,
blob_path: &Path,
variables: &[String],
select: &Selection,
records: Option<&Records>,
) -> Result<NativeDataset> {
let file = NcFile::open(blob_path).map_err(fmt_err)?;
let sel = dim_selection(&file, select)?;
let sel = merge_records(&file, sel, records)?;
decode(&file, variables, sel.as_ref())
}
fn supports_records(&self) -> bool {
true
}
fn dim_length(&self, blob_path: &Path, dim: &str) -> Result<Option<usize>> {
let file = NcFile::open(blob_path).map_err(fmt_err)?;
Ok(nc_dim_length(&file, dim)?)
}
fn supports_selection(&self) -> bool {
true
}
}
type DimSelection = std::collections::HashMap<String, Vec<usize>>;
fn is_time_dim(vars: &[NcVariable], dim: &str) -> bool {
for v in vars {
if v.name() == dim {
if let Some(units) = att_text(v, "units") {
if units
.split_whitespace()
.any(|w| w.eq_ignore_ascii_case("since"))
{
return true;
}
}
}
}
dim.eq_ignore_ascii_case("time")
}
fn dim_selection(file: &NcFile, select: &Selection) -> Result<Option<DimSelection>> {
let axes = match select {
Selection::All => return Ok(None),
Selection::Orthogonal(axes) => axes,
};
let vars: Vec<NcVariable> = file.variables().map_err(fmt_err)?.to_vec();
let mut bydim: Vec<(String, &super::AxisSelect, usize)> = Vec::new();
let mut matched = false;
for var in &vars {
if classify(var).is_none() {
continue;
}
let (dims, shape) = field_dims(&vars, var);
if dims.len() != axes.len() {
continue;
}
matched = true;
for ((axis, name), len) in axes.iter().zip(dims.iter()).zip(shape.iter()) {
match bydim.iter().find(|(seen_name, _, _)| seen_name == name) {
Some((_, seen, _)) if *seen != axis => {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"select is ambiguous: dimension '{}' is asked for two different \
selectors by two rank-{} variables in this blob; a netcdf \
`select` is positional over file-order dims and must agree",
name,
axes.len()
),
})
}
Some(_) => {}
None => bydim.push((name.clone(), axis, *len)),
}
}
}
if !matched {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"select has {} axes but no variable in the blob has rank {}; a netcdf \
`select` is positional over the file-order dims of the arrays it applies to",
axes.len(),
axes.len()
),
});
}
let mut out = DimSelection::new();
for (name, axis, len) in bydim {
if matches!(axis, super::AxisSelect::All) {
continue;
}
if is_time_dim(&vars, &name) {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"select asks for a subset of the time dimension '{name}'; record \
selection is the Provider's (it owns the cadence: \
records_per_sample), not the reader's — the time axis of a netcdf \
`select` must be \"all\""
),
});
}
out.insert(name, axis.resolve_in(len, "netcdf")?);
}
Ok(if out.is_empty() { None } else { Some(out) })
}
fn nc_dim_length(file: &NcFile, dim: &str) -> Result<Option<usize>> {
let dims = file.dimensions().map_err(fmt_err)?;
Ok(dims
.iter()
.find(|d| d.name == dim)
.map(|d| d.size as usize))
}
fn merge_records(
file: &NcFile,
sel: Option<DimSelection>,
records: Option<&Records>,
) -> Result<Option<DimSelection>> {
let Some(records) = records else {
return Ok(sel);
};
let dim = records.dim.as_str();
let Some(len) = nc_dim_length(file, dim)? else {
let mut present: Vec<String> = file
.dimensions()
.map_err(fmt_err)?
.iter()
.map(|d| d.name.clone())
.collect();
present.sort_unstable();
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"records names dimension '{dim}', which the blob does not have; \
present: {present:?}"
),
});
};
if records.indices.is_empty() {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"records asks for no records of '{dim}'; an empty selection is an \
error, not a whole-axis read"
),
});
}
let mut bad: Vec<usize> = records.indices.iter().copied().filter(|i| *i >= len).collect();
bad.sort_unstable();
bad.dedup();
if !bad.is_empty() {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"records index/indices {:?} are outside 0:{} for dimension '{dim}' \
(the reader does not wrap: locating a cadence tick inside its file \
is the Provider's job)",
bad,
len - 1
),
});
}
let mut out = sel.unwrap_or_default();
if out.contains_key(dim) {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"select and records both narrow dimension '{dim}'; a record \
selection is the Provider's and a `select` must leave the record \
axis \"all\""
),
});
}
out.insert(dim.to_string(), records.indices.clone());
Ok(Some(out))
}
fn decode(
file: &NcFile,
variables: &[String],
sel: Option<&DimSelection>,
) -> Result<NativeDataset> {
let vars: Vec<NcVariable> = file.variables().map_err(fmt_err)?.to_vec();
let want: HashSet<&str> = variables.iter().map(String::as_str).collect();
if !want.is_empty() {
let mut present: Vec<&str> = vars
.iter()
.filter(|v| !v.is_coordinate_variable())
.map(NcVariable::name)
.collect();
present.sort_unstable();
let mut missing: Vec<&str> = want
.iter()
.copied()
.filter(|n| !present.contains(n))
.collect();
if !missing.is_empty() {
missing.sort_unstable();
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"requested variables not in blob: {missing:?}; \
present data variables: {present:?}"
),
});
}
}
let mut out = NativeDataset::default();
for var in &vars {
let is_coord = var.is_coordinate_variable();
if !is_coord && !want.is_empty() && !want.contains(var.name()) {
continue;
}
let Some(class) = classify(var) else {
if want.contains(var.name()) {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"requested variable {:?} has netcdf type {:?}, which has no \
native array reading (compound/opaque/enum/vlen variables \
are not decoded)",
var.name(),
var.dtype()
),
});
}
continue;
};
let field = decode_field(file, &vars, var, class, sel)?;
if is_coord {
out.coords.insert(
var.name().to_string(),
Coord {
field,
units: att_text(var, "units"),
calendar: att_text(var, "calendar"),
},
);
} else {
out.variables.insert(var.name().to_string(), field);
}
}
Ok(out)
}
enum FieldClass {
Float,
Int32,
Int64,
Text,
}
fn classify(var: &NcVariable) -> Option<FieldClass> {
if matches!(var.dtype(), NcType::Char | NcType::String) {
return Some(FieldClass::Text);
}
if var.attribute("scale_factor").is_some() || var.attribute("add_offset").is_some() {
return Some(FieldClass::Float);
}
match var.dtype() {
NcType::Float | NcType::Double => Some(FieldClass::Float),
NcType::Byte | NcType::Short | NcType::Int | NcType::UByte | NcType::UShort => {
Some(FieldClass::Int32)
}
NcType::UInt | NcType::Int64 | NcType::UInt64 => Some(FieldClass::Int64),
_ => None,
}
}
fn stacks_last_dimension(vars: &[NcVariable], var: &NcVariable) -> bool {
let var_dims = var.dimensions();
let Some(last) = var_dims.last().map(|d| d.name.as_str()) else {
return false;
};
if vars.iter().any(|v| v.name() == last) {
return false;
}
vars.iter().all(|v| {
let dims = v.dimensions();
!dims.iter().any(|d| d.name == last)
|| (matches!(v.dtype(), NcType::Char)
&& dims.last().map(|d| d.name.as_str()) == Some(last))
})
}
fn consumed_length_dim<'a>(vars: &[NcVariable], var: &'a NcVariable) -> Option<&'a str> {
if !matches!(var.dtype(), NcType::Char) || !stacks_last_dimension(vars, var) {
return None;
}
var.dimensions().last().map(|d| d.name.as_str())
}
fn field_dims(vars: &[NcVariable], var: &NcVariable) -> (Vec<String>, Vec<usize>) {
let mut dims: Vec<String> = var.dimensions().iter().map(|d| d.name.clone()).collect();
let mut shape: Vec<usize> = var.dimensions().iter().map(|d| d.size as usize).collect();
if consumed_length_dim(vars, var).is_some() {
dims.pop();
shape.pop();
}
(dims, shape)
}
struct Slab {
info: netcdf_reader::NcSliceInfo,
slab_shape: Vec<usize>,
take: Vec<Vec<usize>>,
needs_gather: bool,
}
fn progression(idxs: &[usize]) -> (u64, bool) {
if idxs.len() <= 1 {
return (1, false);
}
if idxs[1] <= idxs[0] {
return (1, true); }
let step = idxs[1] - idxs[0];
for w in idxs.windows(2) {
if w[1] <= w[0] || w[1] - w[0] != step {
return (1, true);
}
}
(step as u64, false)
}
fn plan_slab(var: &NcVariable, sel: Option<&DimSelection>) -> Option<Slab> {
let sel = sel?;
let dims = var.dimensions();
if !dims.iter().any(|d| sel.contains_key(&d.name)) {
return None;
}
let mut selections = Vec::with_capacity(dims.len());
let mut slab_shape = Vec::with_capacity(dims.len());
let mut take = Vec::with_capacity(dims.len());
let mut needs_gather = false;
for d in dims.iter() {
let len = d.size as usize;
match sel.get(&d.name) {
None => {
selections.push(netcdf_reader::NcSliceInfoElem::Slice {
start: 0,
end: len as u64,
step: 1,
});
slab_shape.push(len);
take.push((0..len).collect());
}
Some(idxs) if idxs.is_empty() => {
selections.push(netcdf_reader::NcSliceInfoElem::Slice {
start: 0,
end: 0,
step: 1,
});
slab_shape.push(0);
take.push(Vec::new());
}
Some(idxs) => {
let lo = *idxs.iter().min().unwrap_or(&0);
let hi = *idxs.iter().max().unwrap_or(&0);
let (step, gather) = progression(idxs);
let step = if gather { 1 } else { step };
selections.push(netcdf_reader::NcSliceInfoElem::Slice {
start: lo as u64,
end: (hi + 1) as u64,
step,
});
let count = (hi - lo) / (step as usize) + 1;
slab_shape.push(count);
take.push(idxs.iter().map(|g| (g - lo) / (step as usize)).collect());
needs_gather |= gather;
}
}
}
Some(Slab {
info: netcdf_reader::NcSliceInfo { selections },
slab_shape,
take,
needs_gather,
})
}
fn gather<T: Clone>(flat: &[T], slab_shape: &[usize], take: &[Vec<usize>]) -> Vec<T> {
let out_shape: Vec<usize> = take.iter().map(Vec::len).collect();
let n: usize = out_shape.iter().product();
let mut src_strides = vec![1usize; slab_shape.len()];
for i in (0..slab_shape.len().saturating_sub(1)).rev() {
src_strides[i] = src_strides[i + 1] * slab_shape[i + 1];
}
let mut out = Vec::with_capacity(n);
let mut idx = vec![0usize; out_shape.len()];
for _ in 0..n {
let mut off = 0;
for (ax, &k) in idx.iter().enumerate() {
off += take[ax][k] * src_strides[ax];
}
out.push(flat[off].clone());
for ax in (0..idx.len()).rev() {
idx[ax] += 1;
if idx[ax] < out_shape[ax] {
break;
}
idx[ax] = 0;
}
}
out
}
fn read_values(
file: &NcFile,
var: &NcVariable,
slab: Option<&Slab>,
unpacked: bool,
) -> Result<(Vec<f64>, Vec<usize>)> {
let name = var.name();
if let Some(slab) = slab {
let out_shape: Vec<usize> = slab.take.iter().map(Vec::len).collect();
if out_shape.contains(&0) {
return Ok((Vec::new(), out_shape));
}
}
let Some(slab) = slab else {
let arr = if unpacked {
file.read_variable_unpacked_masked(name).map_err(fmt_err)?
} else {
file.read_variable_as_f64(name).map_err(fmt_err)?
};
let shape: Vec<usize> = var.dimensions().iter().map(|d| d.size as usize).collect();
return Ok((arr.iter().copied().collect(), shape));
};
let arr = if unpacked {
file.read_variable_slice_unpacked_masked(name, &slab.info)
.map_err(fmt_err)?
} else {
file.read_variable_slice_as_f64(name, &slab.info)
.map_err(fmt_err)?
};
let flat: Vec<f64> = arr.iter().copied().collect();
let out_shape: Vec<usize> = slab.take.iter().map(Vec::len).collect();
if slab.needs_gather {
Ok((gather(&flat, &slab.slab_shape, &slab.take), out_shape))
} else {
Ok((flat, out_shape))
}
}
fn decode_field(
file: &NcFile,
vars: &[NcVariable],
var: &NcVariable,
class: FieldClass,
sel: Option<&DimSelection>,
) -> Result<NativeField> {
let (dims, shape) = field_dims(vars, var);
match class {
FieldClass::Text => decode_text_field(file, vars, var, dims, shape, sel),
FieldClass::Float => {
let (data, shape) = read_values(file, var, plan_slab(var, sel).as_ref(), true)?;
Ok(NativeField {
dtype: DType::Float64,
dims,
shape,
data: ArrayData::F64(data),
fill_value: None, })
}
FieldClass::Int32 => {
let (raw, shape) = read_values(file, var, plan_slab(var, sel).as_ref(), false)?;
Ok(NativeField {
dtype: DType::Int32,
dims,
shape,
data: ArrayData::I32(raw.iter().map(|&v| v as i32).collect()),
fill_value: int_fill(var),
})
}
FieldClass::Int64 => {
let (raw, shape) = read_values(file, var, plan_slab(var, sel).as_ref(), false)?;
Ok(NativeField {
dtype: DType::Int64,
dims,
shape,
data: ArrayData::I64(raw.iter().map(|&v| v as i64).collect()),
fill_value: int_fill(var),
})
}
}
}
fn decode_text_field(
file: &NcFile,
vars: &[NcVariable],
var: &NcVariable,
dims: Vec<String>,
shape: Vec<usize>,
sel: Option<&DimSelection>,
) -> Result<NativeField> {
let name = var.name();
let groups = file.read_variable_as_strings(name).map_err(fmt_err)?;
let values = if consumed_length_dim(vars, var).is_some() {
groups
} else if matches!(var.dtype(), NcType::String) {
groups
} else {
let group_len = if shape.len() >= 2 {
shape[shape.len() - 1]
} else {
shape.iter().product::<usize>().max(1)
};
let mut out: Vec<String> = Vec::with_capacity(group_len * groups.len());
for group in &groups {
let mut n = 0usize;
for ch in group.chars() {
out.push(if ch == '\0' {
String::new()
} else {
ch.to_string()
});
n += 1;
}
if n > group_len {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"char variable {name:?} is not one byte per character \
({n} characters in a {group_len}-cell run); a non-ASCII \
char array on a shared dimension has no single native \
reading"
),
});
}
for _ in n..group_len {
out.push(String::new());
}
}
out
};
let expected: usize = shape.iter().product();
if values.len() != expected {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"text variable {name:?} decoded to {} strings, but its shape \
{shape:?} holds {expected}",
values.len()
),
});
}
let (values, shape) = select_text(vars, var, &dims, shape, values, sel)?;
Ok(NativeField {
dtype: DType::Str,
dims,
shape,
data: ArrayData::Str(values),
fill_value: None,
})
}
fn select_text(
vars: &[NcVariable],
var: &NcVariable,
dims: &[String],
shape: Vec<usize>,
values: Vec<String>,
sel: Option<&DimSelection>,
) -> Result<(Vec<String>, Vec<usize>)> {
let Some(sel) = sel else {
return Ok((values, shape));
};
if let Some(strlen) = consumed_length_dim(vars, var) {
if sel.contains_key(strlen) {
return Err(Error::Format {
format: "netcdf".to_string(),
detail: format!(
"select asks for a subset of '{strlen}', which is the string \
LENGTH of the char variable {:?}, not an axis of it; the \
decoded field has dims {dims:?}, and selecting along a string \
length would truncate every string rather than choose cells",
var.name()
),
});
}
}
if !dims.iter().any(|d| sel.contains_key(d)) {
return Ok((values, shape));
}
let take: Vec<Vec<usize>> = dims
.iter()
.zip(shape.iter())
.map(|(d, &len)| match sel.get(d) {
Some(idxs) => idxs.clone(),
None => (0..len).collect(),
})
.collect();
let out_shape: Vec<usize> = take.iter().map(Vec::len).collect();
if out_shape.contains(&0) {
return Ok((Vec::new(), out_shape));
}
Ok((gather(&values, &shape, &take), out_shape))
}
fn int_fill(var: &NcVariable) -> Option<f64> {
att_f64(var, "_FillValue").or_else(|| att_f64(var, "missing_value"))
}
fn att_f64(var: &NcVariable, name: &str) -> Option<f64> {
var.attribute(name).and_then(|a| a.value.as_f64())
}
fn att_text(var: &NcVariable, name: &str) -> Option<String> {
var.attribute(name).and_then(|a| a.value.as_string())
}
fn fmt_err(e: netcdf_reader::Error) -> Error {
Error::Format {
format: "netcdf".to_string(),
detail: e.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::AxisSelect;
use std::io::Write;
fn read_bytes(bytes: &[u8]) -> Result<NativeDataset> {
read_bytes_projected(bytes, &[])
}
fn read_bytes_projected(bytes: &[u8], variables: &[String]) -> Result<NativeDataset> {
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(bytes).unwrap();
f.flush().unwrap();
NetcdfReader::new().read_native(f.path(), variables, &Selection::All)
}
fn read_bytes_select(bytes: &[u8], axes: Vec<AxisSelect>) -> Result<NativeDataset> {
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(bytes).unwrap();
f.flush().unwrap();
NetcdfReader::new().read_native(f.path(), &[], &Selection::Orthogonal(axes))
}
fn open_blob(bytes: &[u8]) -> (tempfile::NamedTempFile, NcFile) {
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(bytes).unwrap();
f.flush().unwrap();
let file = NcFile::open(f.path()).expect("a readable blob");
(f, file)
}
fn decode_with_dim_selection(
bytes: &[u8],
sel: &[(&str, Vec<usize>)],
) -> Result<NativeDataset> {
let (_keep, file) = open_blob(bytes);
let map: DimSelection = sel
.iter()
.map(|(d, idx)| ((*d).to_string(), idx.clone()))
.collect();
decode(&file, &[], Some(&map))
}
fn value_of(ds: &NativeDataset) -> (Vec<String>, Vec<usize>, Vec<f64>) {
let f = ds.variables.get("value").expect("a `value` field");
let ArrayData::F64(vals) = &f.data else {
panic!(
"a float variable must carry ArrayData::F64, got {:?}",
f.data
)
};
(f.dims.clone(), f.shape.clone(), vals.clone())
}
fn label_of(ds: &NativeDataset) -> (Vec<String>, Vec<usize>, Vec<String>) {
let f = ds.variables.get("label").expect("a `label` field");
assert_eq!(f.dtype, DType::Str, "a text variable is a `string` field");
let ArrayData::Str(vals) = &f.data else {
panic!(
"a text variable must carry ArrayData::Str, got {:?}",
f.data
)
};
(f.dims.clone(), f.shape.clone(), vals.clone())
}
const CHAR_VAR_CDF1: &[u8] = b"\
\x43\x44\x46\x01\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x01\
\x00\x00\x00\x01\x6e\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x02\x00\x00\x00\x05\
\x76\x61\x6c\x75\x65\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0c\
\x00\x00\x00\x7c\x00\x00\x00\x05\x6c\x61\x62\x65\x6c\x00\x00\x00\
\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x88\x3f\x80\x00\x00\
\x40\x00\x00\x00\x40\x40\x00\x00\x61\x62\x63\x00";
const STRING_ROWS_CDF1: &[u8] = b"\
\x43\x44\x46\x01\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x02\
\x00\x00\x00\x01\x6e\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x06\
\x73\x74\x72\x6c\x65\x6e\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x02\x00\x00\x00\x05\
\x76\x61\x6c\x75\x65\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x10\
\x00\x00\x00\x90\x00\x00\x00\x05\x6c\x61\x62\x65\x6c\x00\x00\x00\
\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x10\x00\x00\x00\xa0\
\x3f\x80\x00\x00\x40\x00\x00\x00\x40\x40\x00\x00\x40\x80\x00\x00\
\x61\x62\x00\x00\x63\x64\x20\x20\x65\x66\x67\x68\x00\x00\x00\x00";
const SCALAR_STRING_CDF1: &[u8] = b"\
\x43\x44\x46\x01\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x02\
\x00\x00\x00\x01\x6e\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x06\
\x73\x74\x72\x6c\x65\x6e\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x02\x00\x00\x00\x05\
\x76\x61\x6c\x75\x65\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0c\
\x00\x00\x00\x8c\x00\x00\x00\x05\x6c\x61\x62\x65\x6c\x00\x00\x00\
\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\
\x00\x00\x00\x02\x00\x00\x00\x08\x00\x00\x00\x98\x3f\x80\x00\x00\
\x40\x00\x00\x00\x40\x40\x00\x00\x68\x69\x00\x00\x00\x00\x00\x00";
const CHAR_HOLES_CDF1: &[u8] = b"\
\x43\x44\x46\x01\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x01\
\x00\x00\x00\x01\x6e\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x02\x00\x00\x00\x05\
\x76\x61\x6c\x75\x65\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0c\
\x00\x00\x00\x7c\x00\x00\x00\x05\x6c\x61\x62\x65\x6c\x00\x00\x00\
\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\
\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x88\x3f\x80\x00\x00\
\x40\x00\x00\x00\x40\x40\x00\x00\x61\x00\x63\x00";
#[test]
fn a_char_variable_on_a_shared_axis_is_one_string_per_cell() {
let all = read_bytes(CHAR_VAR_CDF1).expect("plain decode");
let (dims, shape, vals) = label_of(&all);
assert_eq!(dims, ["n"]);
assert_eq!(shape, [3]);
assert_eq!(vals, ["a", "b", "c"]);
}
#[test]
fn a_nul_cell_in_a_char_axis_is_the_empty_string() {
let all = read_bytes(CHAR_HOLES_CDF1).expect("plain decode");
let (dims, shape, vals) = label_of(&all);
assert_eq!(dims, ["n"]);
assert_eq!(shape, [3]);
assert_eq!(vals, ["a", "", "c"]);
}
#[test]
fn a_private_last_dimension_is_the_string_length_not_an_axis() {
let all = read_bytes(STRING_ROWS_CDF1).expect("plain decode");
let (dims, shape, vals) = label_of(&all);
assert_eq!(dims, ["n"], "the strlen dimension is consumed");
assert_eq!(shape, [4]);
assert_eq!(vals, ["ab", "cd ", "efgh", ""]);
}
#[test]
fn a_one_dimensional_char_on_a_private_dimension_is_a_scalar_string() {
let all = read_bytes(SCALAR_STRING_CDF1).expect("plain decode");
let (dims, shape, vals) = label_of(&all);
assert!(dims.is_empty(), "a scalar string has no dims, got {dims:?}");
assert!(
shape.is_empty(),
"a scalar string has no shape, got {shape:?}"
);
assert_eq!(vals, ["hi"]);
}
#[test]
fn read_everything_returns_the_text_variable_alongside_the_numeric_one() {
for blob in [CHAR_VAR_CDF1, STRING_ROWS_CDF1, SCALAR_STRING_CDF1] {
let all = read_bytes(blob).expect("plain decode");
let mut names: Vec<&str> = all.variables.keys().map(String::as_str).collect();
names.sort_unstable();
assert_eq!(names, ["label", "value"], "read-everything must keep both");
}
}
#[test]
fn the_projection_selects_a_text_variable_like_any_other() {
let one = read_bytes_projected(STRING_ROWS_CDF1, &["label".to_string()])
.expect("a requested text variable now decodes");
assert_eq!(one.variables.keys().collect::<Vec<_>>(), ["label"]);
assert_eq!(label_of(&one).2, ["ab", "cd ", "efgh", ""]);
let other = read_bytes_projected(STRING_ROWS_CDF1, &["value".to_string()])
.expect("projected decode");
assert_eq!(other.variables.keys().collect::<Vec<_>>(), ["value"]);
let err = read_bytes_projected(STRING_ROWS_CDF1, &["nope".to_string()])
.expect_err("an absent name must still be an error");
let msg = err.to_string();
assert!(
msg.contains("nope"),
"error must name the absent one: {msg}"
);
assert!(
msg.contains("label"),
"error must list what IS present: {msg}"
);
}
#[test]
fn a_select_windows_a_char_variable_on_its_shared_axis() {
let ds = read_bytes_select(CHAR_VAR_CDF1, vec![AxisSelect::Indices(vec![2, 0])])
.expect("a select over the shared axis");
let (dims, shape, vals) = label_of(&ds);
assert_eq!(dims, ["n"], "a windowed axis is never dropped");
assert_eq!(shape, [2]);
assert_eq!(vals, ["c", "a"], "in the order asked for, not sorted");
let (vdims, vshape, vvals) = value_of(&ds);
assert_eq!(vdims, ["n"]);
assert_eq!(vshape, [2]);
assert_eq!(
vvals,
[3.0, 1.0],
"the numeric neighbour loses the same rows"
);
}
#[test]
fn a_select_windows_the_strings_not_their_characters() {
let ds = read_bytes_select(STRING_ROWS_CDF1, vec![AxisSelect::Indices(vec![2, 0])])
.expect("a select over `n`");
let (dims, shape, vals) = label_of(&ds);
assert_eq!(dims, ["n"], "`strlen` is still consumed under a select");
assert_eq!(shape, [2]);
assert_eq!(
vals,
["efgh", "ab"],
"whole strings, in the order asked for"
);
assert_eq!(value_of(&ds).2, [3.0, 1.0]);
}
#[test]
fn a_consumed_string_length_never_makes_an_axis_count_match() {
let err = read_bytes_select(
STRING_ROWS_CDF1,
vec![AxisSelect::All, AxisSelect::Indices(vec![0])],
)
.expect_err("a char variable's consumed dimension must not answer for rank 2");
let msg = err.to_string();
assert!(
msg.contains("no variable in the blob has rank 2"),
"the error must say the axis count matched nothing: {msg}"
);
}
#[test]
fn a_consumed_string_length_is_not_an_axis_a_selector_can_bind_to() {
let ds = read_bytes_select(SCALAR_STRING_CDF1, vec![AxisSelect::Indices(vec![0, 2])])
.expect("a 1-axis select binds `n`, the only rank-1 field's dim");
let (dims, shape, vals) = label_of(&ds);
assert!(dims.is_empty(), "a scalar string stays scalar: {dims:?}");
assert!(shape.is_empty(), "…and shapeless: {shape:?}");
assert_eq!(vals, ["hi"], "the whole string, not a selected character");
assert_eq!(value_of(&ds).2, [1.0, 3.0], "`n` was windowed as asked");
}
#[test]
fn a_selection_naming_a_consumed_string_length_is_an_error() {
let err = decode_with_dim_selection(STRING_ROWS_CDF1, &[("strlen", vec![0, 1])])
.expect_err("a string length is not a selectable axis");
let msg = err.to_string();
assert!(msg.contains("strlen"), "the error must name it: {msg}");
assert!(msg.contains("label"), "…and the variable: {msg}");
assert!(
msg.contains("string") && msg.contains("truncate"),
"…and say why it is refused rather than applied: {msg}"
);
}
#[test]
fn an_empty_axis_is_legal_for_a_text_field_too() {
let ds = read_bytes_select(
STRING_ROWS_CDF1,
vec![AxisSelect::Range {
start: 1,
stop: 1,
step: 1,
}],
)
.expect("an empty half-open range is a legal zero-length axis");
let (dims, shape, vals) = label_of(&ds);
assert_eq!(dims, ["n"], "a zero-length axis is KEPT in dims");
assert_eq!(shape, [0]);
assert!(vals.is_empty(), "shape 0 must mean no cells, got {vals:?}");
let (vdims, vshape, vvals) = value_of(&ds);
assert_eq!(vdims, ["n"]);
assert_eq!(vshape, [0]);
assert!(vvals.is_empty(), "the numeric field agrees: {vvals:?}");
}
#[test]
fn field_dims_is_the_identity_for_a_non_text_variable() {
for blob in [
CHAR_VAR_CDF1,
CHAR_HOLES_CDF1,
STRING_ROWS_CDF1,
SCALAR_STRING_CDF1,
] {
let (_keep, file) = open_blob(blob);
let vars: Vec<NcVariable> = file.variables().unwrap().to_vec();
let mut checked = 0;
for var in &vars {
if matches!(var.dtype(), NcType::Char | NcType::String) {
continue;
}
let (dims, shape) = field_dims(&vars, var);
let on_disk: Vec<String> =
var.dimensions().iter().map(|d| d.name.clone()).collect();
let sizes: Vec<usize> = var.dimensions().iter().map(|d| d.size as usize).collect();
assert_eq!(dims, on_disk, "{} lost or gained a dim", var.name());
assert_eq!(shape, sizes, "{} lost or gained a length", var.name());
assert!(
consumed_length_dim(&vars, var).is_none(),
"{} is not a char array and consumes nothing",
var.name()
);
checked += 1;
}
assert!(checked > 0, "every fixture has a numeric variable to check");
}
}
#[test]
fn rejects_bad_magic() {
let err = read_bytes(b"NOPE, not a netcdf file at all").unwrap_err();
assert!(matches!(err, Error::Format { .. }));
}
#[test]
fn truncated_classic_file_is_an_error_not_a_panic() {
let err = read_bytes(b"CDF\x01\x00\x00").unwrap_err();
assert!(matches!(err, Error::Format { .. }));
}
fn put_u32(v: &mut Vec<u8>, x: u32) {
v.extend_from_slice(&x.to_be_bytes());
}
fn put_name(v: &mut Vec<u8>, s: &str) {
put_u32(v, s.len() as u32);
v.extend_from_slice(s.as_bytes());
while v.len() % 4 != 0 {
v.push(0);
}
}
struct VarSpec {
name: &'static str,
dimids: Vec<u32>,
attr: Option<(&'static str, &'static str)>,
data: Vec<f64>,
}
fn build_cdf1(dims: &[(&str, u32)], vars: &[VarSpec]) -> Vec<u8> {
let mut h: Vec<u8> = Vec::new();
h.extend_from_slice(b"CDF\x01");
put_u32(&mut h, 0); put_u32(&mut h, 10); put_u32(&mut h, dims.len() as u32);
for (name, len) in dims {
put_name(&mut h, name);
put_u32(&mut h, *len);
}
put_u32(&mut h, 0); put_u32(&mut h, 0);
put_u32(&mut h, 11); put_u32(&mut h, vars.len() as u32);
let mut begin_at: Vec<usize> = Vec::new();
for v in vars {
put_name(&mut h, v.name);
put_u32(&mut h, v.dimids.len() as u32);
for d in &v.dimids {
put_u32(&mut h, *d);
}
match v.attr {
None => {
put_u32(&mut h, 0); put_u32(&mut h, 0);
}
Some((an, av)) => {
put_u32(&mut h, 12); put_u32(&mut h, 1);
put_name(&mut h, an);
put_u32(&mut h, 2); put_u32(&mut h, av.len() as u32);
h.extend_from_slice(av.as_bytes());
while h.len() % 4 != 0 {
h.push(0);
}
}
}
put_u32(&mut h, 6); put_u32(&mut h, (v.data.len() * 8) as u32); begin_at.push(h.len());
put_u32(&mut h, 0); }
let mut offset = h.len() as u32;
for (v, at) in vars.iter().zip(&begin_at) {
h[*at..*at + 4].copy_from_slice(&offset.to_be_bytes());
offset += (v.data.len() * 8) as u32;
}
for v in vars {
for x in &v.data {
h.extend_from_slice(&x.to_be_bytes());
}
}
h
}
const NREC: usize = 4;
const NLAT: usize = 2;
const NLON: usize = 3;
fn t2m_at(r: usize, i: usize, j: usize) -> f64 {
(100 * (r + 1) + 10 * i + j) as f64
}
fn cadence_cdf1() -> Vec<u8> {
let mut t2m = Vec::new();
for r in 0..NREC {
for i in 0..NLAT {
for j in 0..NLON {
t2m.push(t2m_at(r, i, j));
}
}
}
let mut orog = Vec::new();
for i in 0..NLAT {
for j in 0..NLON {
orog.push(7.0 + i as f64 + j as f64);
}
}
build_cdf1(
&[("time", NREC as u32), ("lat", NLAT as u32), ("lon", NLON as u32)],
&[
VarSpec {
name: "time",
dimids: vec![0],
attr: Some(("units", "hours since 2018-11-09 00:00:00")),
data: (0..NREC).map(|r| r as f64).collect(),
},
VarSpec {
name: "lat",
dimids: vec![1],
attr: None,
data: vec![40.0, 41.0],
},
VarSpec {
name: "lon",
dimids: vec![2],
attr: None,
data: vec![-90.0, -89.0, -88.0],
},
VarSpec {
name: "t2m",
dimids: vec![0, 1, 2],
attr: None,
data: t2m,
},
VarSpec {
name: "orog",
dimids: vec![1, 2],
attr: None,
data: orog,
},
],
)
}
fn read_cadence(
variables: &[String],
select: &Selection,
records: Option<&Records>,
) -> Result<NativeDataset> {
let bytes = cadence_cdf1();
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(&bytes).unwrap();
f.flush().unwrap();
NetcdfReader::new().read_native_records(f.path(), variables, select, records)
}
fn field_of(ds: &NativeDataset, name: &str) -> (Vec<String>, Vec<usize>, Vec<f64>) {
let f = ds
.variables
.get(name)
.unwrap_or_else(|| panic!("a `{name}` field"));
let ArrayData::F64(vals) = &f.data else {
panic!("expected F64, got {:?}", f.data)
};
(f.dims.clone(), f.shape.clone(), vals.clone())
}
fn coord_of(ds: &NativeDataset, name: &str) -> Vec<f64> {
let c = ds
.coords
.get(name)
.unwrap_or_else(|| panic!("a `{name}` coord"));
let ArrayData::F64(vals) = &c.field.data else {
panic!("expected F64, got {:?}", c.field.data)
};
vals.clone()
}
#[test]
fn whole_file_decode_is_the_pinned_fixture() {
let ds = read_cadence(&[], &Selection::All, None).expect("the cadence fixture decodes");
let (dims, shape, vals) = field_of(&ds, "t2m");
assert_eq!(dims, vec!["time", "lat", "lon"]);
assert_eq!(shape, vec![NREC, NLAT, NLON]);
assert_eq!(
vals,
vec![
100.0, 101.0, 102.0, 110.0, 111.0, 112.0, 200.0, 201.0, 202.0, 210.0, 211.0, 212.0, 300.0, 301.0, 302.0, 310.0, 311.0, 312.0, 400.0, 401.0, 402.0, 410.0, 411.0, 412.0, ]
);
assert_eq!(coord_of(&ds, "time"), vec![0.0, 1.0, 2.0, 3.0]);
assert_eq!(coord_of(&ds, "lat"), vec![40.0, 41.0]);
assert_eq!(
ds.coords.get("time").unwrap().units.as_deref(),
Some("hours since 2018-11-09 00:00:00")
);
}
#[test]
fn record_pushdown_equals_the_whole_file_read() {
let whole = read_cadence(&[], &Selection::All, None).expect("whole-file read");
let (_, _, whole_t2m) = field_of(&whole, "t2m");
let whole_time = coord_of(&whole, "time");
let cells = NLAT * NLON;
let mut sweeps: Vec<Vec<usize>> = Vec::new();
for a in 0..NREC {
sweeps.push(vec![a]); for b in 0..NREC {
sweeps.push(vec![a, b]); }
}
sweeps.push((0..NREC).collect()); sweeps.push(vec![3, 1, 2, 0]);
for idx in sweeps {
let got = read_cadence(&[], &Selection::All, Some(&Records::new("time", idx.clone())))
.unwrap_or_else(|e| panic!("records {idx:?}: {e}"));
let (dims, shape, vals) = field_of(&got, "t2m");
assert_eq!(dims, vec!["time", "lat", "lon"], "records {idx:?}");
assert_eq!(shape, vec![idx.len(), NLAT, NLON], "records {idx:?}");
let want: Vec<f64> = idx
.iter()
.flat_map(|r| whole_t2m[r * cells..(r + 1) * cells].to_vec())
.collect();
assert_eq!(vals, want, "records {idx:?}: pushed != whole-file gathered");
let want_time: Vec<f64> = idx.iter().map(|r| whole_time[*r]).collect();
assert_eq!(coord_of(&got, "time"), want_time, "records {idx:?}");
assert_eq!(coord_of(&got, "lat"), coord_of(&whole, "lat"));
assert_eq!(coord_of(&got, "lon"), coord_of(&whole, "lon"));
assert_eq!(field_of(&got, "orog"), field_of(&whole, "orog"));
}
}
#[test]
fn records_are_absolute_ordered_and_may_repeat() {
let one = read_cadence(&[], &Selection::All, Some(&Records::new("time", vec![2])))
.expect("one record");
assert_eq!(
field_of(&one, "t2m").2,
vec![300.0, 301.0, 302.0, 310.0, 311.0, 312.0]
);
assert_eq!(coord_of(&one, "time"), vec![2.0]);
let rev = read_cadence(&[], &Selection::All, Some(&Records::new("time", vec![3, 1])))
.expect("a reversed pair");
assert_eq!(
field_of(&rev, "t2m").2,
vec![400.0, 401.0, 402.0, 410.0, 411.0, 412.0, 200.0, 201.0, 202.0, 210.0, 211.0, 212.0]
);
assert_eq!(coord_of(&rev, "time"), vec![3.0, 1.0]);
let dup = read_cadence(&[], &Selection::All, Some(&Records::new("time", vec![3, 3])))
.expect("the degenerate end-of-data bracket");
let (_, shape, vals) = field_of(&dup, "t2m");
assert_eq!(shape, vec![2, NLAT, NLON]);
assert_eq!(
vals,
vec![400.0, 401.0, 402.0, 410.0, 411.0, 412.0, 400.0, 401.0, 402.0, 410.0, 411.0, 412.0]
);
assert_eq!(coord_of(&dup, "time"), vec![3.0, 3.0]);
}
#[test]
fn records_compose_with_a_projection_and_a_spatial_select() {
let sel = Selection::Orthogonal(vec![
AxisSelect::All, AxisSelect::Indices(vec![1]),
AxisSelect::Range {
start: 0,
stop: 3,
step: 2,
},
]);
let got = read_cadence(
&["t2m".to_string()],
&sel,
Some(&Records::new("time", vec![2, 0])),
)
.expect("all three narrowings together");
assert_eq!(got.variables.keys().collect::<Vec<_>>(), vec!["t2m"]);
let (dims, shape, vals) = field_of(&got, "t2m");
assert_eq!(dims, vec!["time", "lat", "lon"]);
assert_eq!(shape, vec![2, 1, 2]);
assert_eq!(
vals,
vec![
t2m_at(2, 1, 0),
t2m_at(2, 1, 2),
t2m_at(0, 1, 0),
t2m_at(0, 1, 2)
]
);
assert_eq!(coord_of(&got, "time"), vec![2.0, 0.0]);
assert_eq!(coord_of(&got, "lat"), vec![41.0]);
assert_eq!(coord_of(&got, "lon"), vec![-90.0, -88.0]);
}
#[test]
fn a_select_still_may_not_narrow_the_time_axis() {
let sel = Selection::Orthogonal(vec![
AxisSelect::Indices(vec![0]),
AxisSelect::All,
AxisSelect::All,
]);
let err = read_cadence(&[], &sel, None).unwrap_err();
assert!(
format!("{err}").contains("record selection is the Provider's"),
"{err}"
);
let err = read_cadence(&[], &sel, Some(&Records::new("time", vec![0]))).unwrap_err();
assert!(format!("{err}").contains("must be \"all\""), "{err}");
}
#[test]
fn a_bad_records_is_an_error_not_a_wrapped_or_widened_read() {
for bad in [vec![4], vec![0, 9], vec![usize::MAX]] {
let err = read_cadence(&[], &Selection::All, Some(&Records::new("time", bad.clone())))
.unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("outside 0:3"), "records {bad:?}: {msg}");
}
let err = read_cadence(&[], &Selection::All, Some(&Records::new("nope", vec![0])))
.unwrap_err();
assert!(format!("{err}").contains("which the blob does not have"), "{err}");
let err =
read_cadence(&[], &Selection::All, Some(&Records::new("time", vec![]))).unwrap_err();
assert!(format!("{err}").contains("empty selection is an error"), "{err}");
let sel = Selection::Orthogonal(vec![
AxisSelect::All,
AxisSelect::Indices(vec![0]),
AxisSelect::All,
]);
let err = read_cadence(&[], &sel, Some(&Records::new("lat", vec![0]))).unwrap_err();
assert!(format!("{err}").contains("both narrow dimension"), "{err}");
}
#[test]
fn dim_length_reads_the_header_and_is_declared() {
let bytes = cadence_cdf1();
let mut f = tempfile::NamedTempFile::new().unwrap();
f.write_all(&bytes).unwrap();
f.flush().unwrap();
let r = NetcdfReader::new();
assert_eq!(r.dim_length(f.path(), "time").unwrap(), Some(NREC));
assert_eq!(r.dim_length(f.path(), "lon").unwrap(), Some(NLON));
assert_eq!(r.dim_length(f.path(), "no-such-dim").unwrap(), None);
assert!(r.supports_records());
let other = crate::format::Ff10Reader::new();
assert!(!other.supports_records());
assert_eq!(other.dim_length(f.path(), "time").unwrap(), None);
let err = other
.read_native_records(
f.path(),
&[],
&Selection::All,
Some(&Records::new("time", vec![0])),
)
.unwrap_err();
assert!(
format!("{err}").contains("does not honour a `records` pushdown"),
"{err}"
);
}
}