mod lazy;
pub use lazy::NpzChunkReader;
use crate::error::{Error, Result};
use crate::formats::npy;
use crate::options::{ReadOptions, WriteOptions};
use crate::value::{Dict, Value};
use crate::zip::{ZipReader, ZipWriter};
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::Path;
pub fn read_bytes(buf: &[u8], opts: &ReadOptions) -> Result<Value> {
let z = ZipReader::new(buf)?;
let mut d = Dict::new();
for e in z.entries() {
if !e.name.ends_with(".npy") {
continue;
}
let name = e.name.trim_end_matches(".npy").to_string();
let raw = z.read(e)?;
nest(&mut d, &name, npy::read_bytes(&raw, opts)?);
}
Ok(Value::Dict(d))
}
fn nest(root: &mut Dict, path: &str, v: Value) {
match path.split_once('/') {
None => {
root.insert(Value::Str(path.to_string()), v);
}
Some((head, tail)) => {
let key = Value::Str(head.to_string());
if !matches!(root.get(head), Some(Value::Dict(_))) {
root.insert(key.clone(), Value::Dict(Dict::new()));
}
if let Some(Value::Dict(child)) = root.get_mut(head) {
nest(child, tail, v);
}
}
}
}
pub fn read_path(path: &Path, opts: &ReadOptions) -> Result<Value> {
read_bytes(&std::fs::read(path)?, opts)
}
pub fn write_path(path: &Path, value: &Value, opts: &WriteOptions) -> Result<()> {
let mut members = Vec::new();
flatten(value, String::new(), &mut members)?;
let mut z = ZipWriter::new(Vec::new());
for (name, v) in members {
z.add(&format!("{name}.npy"), &npy::write_bytes(v, opts)?, opts.compression)?;
}
let image = z.finish()?;
let mut file = BufWriter::new(File::create(path)?);
file.write_all(&image)?;
file.flush()?;
Ok(())
}
fn flatten<'a>(v: &'a Value, prefix: String, out: &mut Vec<(String, &'a Value)>) -> Result<()> {
let join = |p: &str, k: &str| if p.is_empty() { k.to_string() } else { format!("{p}/{k}") };
match v {
Value::Dict(d) => {
for (k, val) in d.iter() {
let name =
k.as_str().ok_or_else(|| Error::invalid("npz member names must be strings"))?;
if name.contains('/') {
return Err(Error::invalid(format!(
"`/` is the nesting separator and cannot appear in the key `{name}`"
)));
}
flatten(val, join(&prefix, name), out)?;
}
Ok(())
}
Value::List(items) | Value::Tuple(items) if prefix.is_empty() => {
for (i, val) in items.iter().enumerate() {
flatten(val, format!("arr_{i}"), out)?;
}
Ok(())
}
single => {
out.push((if prefix.is_empty() { "arr_0".into() } else { prefix }, single));
Ok(())
}
}
}