use std::path::{Path, PathBuf};
use flodl::tensor::{DType, Result, Tensor, TensorError};
use crate::graph::network::Network;
struct TensorBlob {
name: String,
dtype: &'static str,
shape: Vec<i64>,
data: Vec<u8>,
}
fn collect_tensors(net: &Network) -> Result<Vec<TensorBlob>> {
let mut blobs = Vec::new();
for (idx, layer) in net.layers.iter().enumerate() {
push_linear(&mut blobs, &format!("node{idx}"), layer)?;
}
for (node, ports) in net.port_projections.iter().enumerate() {
for (port, proj) in ports.iter().enumerate() {
if let Some(linear) = proj {
push_linear(&mut blobs, &format!("proj{node}_{port}_0"), linear)?;
}
}
}
Ok(blobs)
}
fn push_linear(blobs: &mut Vec<TensorBlob>, prefix: &str, layer: &flodl::nn::Linear) -> Result<()> {
let w = layer.weight.variable.data();
blobs.push(TensorBlob {
name: format!("{prefix}.weight"),
dtype: dtype_tag(w.dtype()),
shape: w.shape(),
data: w.to_blob()?,
});
if let Some(b) = &layer.bias {
let bt = b.variable.data();
blobs.push(TensorBlob {
name: format!("{prefix}.bias"),
dtype: dtype_tag(bt.dtype()),
shape: bt.shape(),
data: bt.to_blob()?,
});
}
Ok(())
}
fn dtype_tag(dt: DType) -> &'static str {
match dt {
DType::Float32 => "F32",
DType::Float64 => "F64",
other => unreachable!("unexpected parameter dtype {other:?}"),
}
}
fn serialize(blobs: &[TensorBlob]) -> Result<Vec<u8>> {
let mut offset = 0usize;
let mut entries = Vec::with_capacity(blobs.len());
let mut data = Vec::new();
for b in blobs {
let end = offset + b.data.len();
entries.push(format!(
"\"{}\":{{\"dtype\":\"{}\",\"shape\":{:?},\"data_offsets\":[{},{}]}}",
b.name, b.dtype, b.shape, offset, end
));
data.extend_from_slice(&b.data);
offset = end;
}
let header = format!("{{{}}}", entries.join(","));
let len = header.len();
let padded = len.div_ceil(8) * 8;
let mut header_bytes = header.into_bytes();
header_bytes.resize(padded, b' ');
let mut out = Vec::with_capacity(8 + padded + data.len());
out.extend_from_slice(&(padded as u64).to_le_bytes());
out.extend_from_slice(&header_bytes);
out.extend_from_slice(&data);
Ok(out)
}
pub fn export_safetensors(net: &Network, path: &Path) -> Result<PathBuf> {
let blobs = collect_tensors(net)?;
if blobs.is_empty() {
return Err(TensorError::new(
"network has no parameters — nothing to export",
));
}
let bytes = serialize(&blobs)?;
std::fs::write(path, bytes)
.map_err(|e| TensorError::new(&format!("failed to write {}: {e}", path.display())))?;
Ok(path.to_path_buf())
}
pub fn load_safetensors(net: &mut Network, path: &Path) -> Result<()> {
let raw = std::fs::read(path)
.map_err(|e| TensorError::new(&format!("failed to read {}: {e}", path.display())))?;
if raw.len() < 8 {
return Err(TensorError::new("safetensors file too short"));
}
let hdr_len = u64::from_le_bytes(raw[..8].try_into().unwrap()) as usize;
let header: serde_json::Map<String, serde_json::Value> =
serde_json::from_slice(&raw[8..8 + hdr_len])
.map_err(|e| TensorError::new(&format!("bad safetensors header: {e}")))?;
let data_start = 8 + hdr_len;
let mut copied = 0usize;
for (idx, layer) in net.layers.iter_mut().enumerate() {
copied += load_linear(&header, &raw[data_start..], &format!("node{idx}"), layer)?;
}
for (node, ports) in net.port_projections.iter_mut().enumerate() {
for (port, proj) in ports.iter_mut().enumerate() {
if let Some(linear) = proj {
copied += load_linear(
&header,
&raw[data_start..],
&format!("proj{node}_{port}_0"),
linear,
)?;
}
}
}
use flodl::nn::Module;
let expected = net.parameters().len();
if copied != expected {
return Err(TensorError::new(&format!(
"incomplete safetensors: {} copied {copied} tensor(s) but the network has {expected} — \
refusing a partial load (missing weights would silently stay at their fresh init)",
path.display()
)));
}
Ok(())
}
fn load_linear(
header: &serde_json::Map<String, serde_json::Value>,
data: &[u8],
prefix: &str,
layer: &mut flodl::nn::Linear,
) -> Result<usize> {
let mut copied = 0usize;
let w = header
.get(&format!("{prefix}.weight"))
.ok_or_else(|| TensorError::new(&format!("safetensors missing {prefix}.weight")))?;
copy_entry(w, data, &mut layer.weight.variable)?;
copied += 1;
if let Some(bias) = &mut layer.bias {
let b = header
.get(&format!("{prefix}.bias"))
.ok_or_else(|| TensorError::new(&format!("safetensors missing {prefix}.bias")))?;
copy_entry(b, data, &mut bias.variable)?;
copied += 1;
}
Ok(copied)
}
fn copy_entry(
entry: &serde_json::Value,
data: &[u8],
variable: &mut flodl::Variable,
) -> Result<()> {
let shape: Vec<usize> = entry["shape"]
.as_array()
.ok_or_else(|| TensorError::new("bad tensor entry"))?
.iter()
.map(|v| v.as_u64().unwrap_or(0) as usize)
.collect();
let [a, b] = [
entry["data_offsets"][0].as_u64().unwrap_or(0) as usize,
entry["data_offsets"][1].as_u64().unwrap_or(0) as usize,
];
let blob = data
.get(a..b)
.ok_or_else(|| TensorError::new("tensor offsets out of bounds"))?;
let shape_i64: Vec<i64> = shape.iter().map(|&d| d as i64).collect();
let t = Tensor::from_blob(blob, &shape_i64, DType::Float32, flodl::tensor::Device::CPU)?;
let cur = variable.data();
if cur.shape() != t.shape() {
return Err(TensorError::new(&format!(
"shape mismatch: file {:?} vs network {:?}",
t.shape(),
cur.shape()
)));
}
let t = t.to_device(cur.device())?;
variable.set_data(t);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::network::Network;
use crate::graph::node::Node;
use crate::graph::topology::Topology;
fn small_topo() -> Topology {
let mut graph = Topology::new(42, None);
graph.nodes.push(Node::new_input(0, 2));
graph.nodes.push(Node::new_hidden(1, 1, 1));
graph.nodes.push(Node::new_output(2, 1, 1));
graph.refresh_labels();
graph.finalize();
graph
}
#[test]
fn safetensors_roundtrip_header_and_blobs() {
let graph = small_topo();
let net = Network::build(&graph, flodl::tensor::Device::CPU).unwrap();
let dir = std::env::temp_dir().join(format!(
"gras_st_test_{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("test.safetensors");
export_safetensors(&net, &path).unwrap();
let raw = std::fs::read(&path).unwrap();
let hdr_len = u64::from_le_bytes(raw[..8].try_into().unwrap()) as usize;
assert_eq!(hdr_len % 8, 0, "header must be 8-aligned");
assert!(8 + hdr_len < raw.len());
let header: serde_json::Map<String, serde_json::Value> =
serde_json::from_slice(&raw[8..8 + hdr_len]).unwrap();
assert_eq!(header.len(), 2 * net.layers.len(), "weight+bias per layer");
for (name, entry) in &header {
assert!(name.starts_with("node"), "unexpected tensor name {name}");
assert_eq!(entry["dtype"].as_str().unwrap(), "F32");
let [a, b] = [
entry["data_offsets"][0].as_u64().unwrap() as usize,
entry["data_offsets"][1].as_u64().unwrap() as usize,
];
assert!(b > a);
assert!(8 + hdr_len + b <= raw.len(), "blob out of bounds");
let shape: Vec<usize> = entry["shape"]
.as_array()
.unwrap()
.iter()
.map(|v| v.as_u64().unwrap() as usize)
.collect();
assert_eq!(shape.iter().product::<usize>() * 4, b - a);
}
assert!(header.contains_key("node0.weight"));
assert!(header.contains_key("node1.weight"));
let mut net2 = Network::build(&graph, flodl::tensor::Device::CPU).unwrap();
load_safetensors(&mut net2, &path).unwrap();
for (l1, l2) in net.layers.iter().zip(net2.layers.iter()) {
assert_eq!(
l1.weight.variable.data().to_blob().unwrap(),
l2.weight.variable.data().to_blob().unwrap()
);
}
std::fs::remove_dir_all(&dir).ok();
}
fn bridged_topo() -> Topology {
use crate::graph::topology::{Connection, Port};
let mut graph = Topology::new(7, None);
let mut input_node = Node::new_input(0, 4);
input_node.hidden_dim = Some(128); graph.nodes.push(input_node);
let mut h1 = Node::new_hidden(1, 1, 1);
h1.hidden_dim = Some(32); graph.nodes.push(h1);
graph.nodes.push(Node::new_output(2, 1, 1));
graph.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
});
graph.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
});
graph.refresh_labels();
graph.finalize();
graph
}
#[test]
fn safetensors_roundtrip_includes_port_projections() {
let graph = bridged_topo();
let net = Network::build(&graph, flodl::tensor::Device::CPU).unwrap();
let dir = std::env::temp_dir().join(format!(
"gras_st_proj_test_{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("bridged.safetensors");
use flodl::nn::Module;
let expected = net.parameters().len();
export_safetensors(&net, &path).unwrap();
let raw = std::fs::read(&path).unwrap();
let hdr_len = u64::from_le_bytes(raw[..8].try_into().unwrap()) as usize;
let header: serde_json::Map<String, serde_json::Value> =
serde_json::from_slice(&raw[8..8 + hdr_len]).unwrap();
assert_eq!(
header.len(),
expected,
"file must carry one entry per parameter tensor (bridges included)"
);
assert!(
header.keys().any(|k| k.starts_with("proj")),
"a bridged net's file must contain proj* entries; got {:?}",
header.keys().collect::<Vec<_>>()
);
let mut net2 = Network::build(&graph, flodl::tensor::Device::CPU).unwrap();
load_safetensors(&mut net2, &path).unwrap();
for (a, b) in net.parameters().iter().zip(net2.parameters().iter()) {
assert_eq!(
a.variable.data().to_blob().unwrap(),
b.variable.data().to_blob().unwrap(),
"parameter mismatch after round-trip"
);
}
let truncated = dir.join("partial.safetensors");
let only_header_len = 8 + hdr_len;
std::fs::write(&truncated, &raw[..only_header_len]).unwrap();
let mut net3 = Network::build(&graph, flodl::tensor::Device::CPU).unwrap();
assert!(
load_safetensors(&mut net3, &truncated).is_err(),
"a partial safetensors must be rejected, not partially loaded"
);
std::fs::remove_dir_all(&dir).ok();
}
}