use std::collections::HashMap;
use flodl::nn::{Linear, Module, Parameter};
use flodl::{DType, Device, Tensor, Variable};
use tracing::debug;
use crate::utils::error::NetworkError;
use crate::graph::inventory::{LayerInventory, PieceKind};
use crate::graph::node::Node;
use crate::graph::topology::{CombineOp, Connection, Port, Topology};
use crate::utils::graph_utils::build_node_sources;
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct NetworkOptions {
pub device: Device,
pub dtype: DType,
pub topology_seed: usize,
pub dropout_prob: f32,
}
impl Default for NetworkOptions {
fn default() -> Self {
NetworkOptions {
device: Device::CPU,
dtype: DType::Float32,
topology_seed: 0,
dropout_prob: 0.0,
}
}
}
impl serde::Serialize for NetworkOptions {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::ser::SerializeStruct;
let mut st = serializer.serialize_struct("NetworkOptions", 4)?;
let device = match self.device {
Device::CPU => "CPU".to_string(),
Device::CUDA(n) => format!("CUDA({n})"),
};
st.serialize_field("device", &device)?;
let dtype = match self.dtype {
DType::Float16 => "Float16",
DType::BFloat16 => "BFloat16",
DType::Float32 => "Float32",
DType::Float64 => "Float64",
DType::Int32 => "Int32",
DType::Int64 => "Int64",
};
st.serialize_field("dtype", dtype)?;
st.serialize_field("topology_seed", &(self.topology_seed as u64))?;
st.serialize_field("dropout_prob", &self.dropout_prob)?;
st.end()
}
}
pub struct Network {
pub(crate) name: String,
pub input_dim: usize,
pub hidden_dim: usize,
pub layers: Vec<Linear>,
pub(crate) connections: Vec<Connection>,
pub nodes: Vec<Node>,
pub node_dims: Vec<(usize, usize)>,
pub(crate) node_sources: Vec<Vec<Option<Port>>>,
pub output_node: usize,
pub(crate) port_projections: Vec<Vec<Option<Linear>>>,
pub(crate) dropout_layers: Vec<Option<flodl::nn::Dropout>>,
pub(crate) port_masks: Vec<Vec<Option<Tensor>>>,
}
struct BuildPlan {
rng: Option<fastrand::Rng>,
node_sources: Vec<Vec<Option<crate::graph::topology::Port>>>,
node_dims: Vec<(usize, usize)>,
}
impl Network {
pub fn build(graph: &Topology, device: Device) -> flodl::tensor::Result<Self> {
Self::build_with_options(
graph,
&NetworkOptions {
device,
dtype: DType::Float32,
topology_seed: graph
.options
.weight_seed
.unwrap_or(graph.options.topology_seed),
dropout_prob: graph.options.dropout_prob,
},
)
.inspect(|net| {
net.eval();
})
}
pub fn build_with_options(
graph: &Topology,
opts: &NetworkOptions,
) -> flodl::tensor::Result<Self> {
Self::build_inner(graph, opts, None)
}
pub fn build_with_inventory(
graph: &Topology,
device: Device,
inventory: &mut LayerInventory,
) -> flodl::tensor::Result<Self> {
inventory.begin_net();
Self::build_inner(
graph,
&NetworkOptions {
device,
dtype: DType::Float32,
topology_seed: graph
.options
.weight_seed
.unwrap_or(graph.options.topology_seed),
dropout_prob: graph.options.dropout_prob,
},
Some(inventory),
)
.inspect(|net| {
net.eval();
})
}
fn build_inner(
graph: &Topology,
opts: &NetworkOptions,
mut inventory: Option<&mut LayerInventory>,
) -> flodl::tensor::Result<Self> {
graph.validate().map_err(NetworkError::InvalidTopology)?;
let plan = Self::prepare_build(graph, opts);
let (mut rng, node_sources, node_dims) = (plan.rng, plan.node_sources, plan.node_dims);
let layers = Self::build_linear_layers(
graph,
&node_dims,
opts.device,
opts.dtype,
rng.as_mut(),
inventory.as_deref_mut(),
)?;
let port_projections = Self::bridge_diff_dims(
&node_sources,
&node_dims,
opts.device,
opts.dtype,
rng.as_mut(),
inventory.as_deref_mut(),
)?;
let dropout_layers = Self::build_dropout_layers(graph, opts);
let port_masks = Self::build_port_masks(graph, opts)?;
Self::assemble(
graph,
&node_dims,
&node_sources,
layers,
port_projections,
dropout_layers,
port_masks,
)
}
fn prepare_build(graph: &Topology, opts: &NetworkOptions) -> BuildPlan {
let node_inputs: Vec<usize> = graph.nodes.iter().map(|n| n.num_inputs).collect();
let node_sources = build_node_sources(&graph.connections, &node_inputs);
let node_dims = graph.node_dims();
let rng = Some(fastrand::Rng::with_seed(opts.topology_seed as u64));
debug!(
"Network::build -- graph id={} nodes={} wires={} input_dim={}",
graph.id,
graph.nodes.len(),
graph.connections.len(),
graph.options.input_dim.unwrap_or(1)
);
BuildPlan {
rng,
node_sources,
node_dims,
}
}
fn build_linear_layers(
graph: &Topology,
node_dims: &[(usize, usize)],
device: Device,
dtype: DType,
mut rng: Option<&mut fastrand::Rng>,
mut inventory: Option<&mut LayerInventory>,
) -> flodl::tensor::Result<Vec<Linear>> {
graph
.nodes
.iter()
.zip(node_dims)
.map(|(_node, &(in_dim, out_dim))| {
if let Some(inv) = inventory.as_deref_mut() {
let vals = inv.piece(PieceKind::Layer, in_dim, out_dim);
Self::linear_from_piece(vals, in_dim as i64, out_dim as i64, device, dtype)
} else {
Self::create_linear_with_seed(
in_dim as i64,
out_dim as i64,
device,
dtype,
rng.as_deref_mut(),
)
}
})
.collect()
}
fn bridge_diff_dims(
node_sources: &[Vec<Option<Port>>],
node_dims: &[(usize, usize)],
device: Device,
dtype: DType,
mut rng: Option<&mut fastrand::Rng>,
mut inventory: Option<&mut LayerInventory>,
) -> flodl::tensor::Result<Vec<Vec<Option<Linear>>>> {
node_sources
.iter()
.enumerate()
.map(|(node_id, ports)| {
let in_dim = node_dims[node_id].0;
ports
.iter()
.map(|source| {
let Some(source) = source else {
return Ok(None);
};
let src_dim = node_dims[source.node].1;
if src_dim == in_dim {
Ok(None)
} else if let Some(inv) = inventory.as_deref_mut() {
let vals = inv.piece(PieceKind::Bridge, src_dim, in_dim);
Ok(Some(Self::linear_from_piece(
vals,
src_dim as i64,
in_dim as i64,
device,
dtype,
)?))
} else {
Ok(Some(Self::create_linear_with_seed(
src_dim as i64,
in_dim as i64,
device,
dtype,
rng.as_deref_mut(),
)?))
}
})
.collect()
})
.collect()
}
fn build_dropout_layers(
graph: &Topology,
opts: &NetworkOptions,
) -> Vec<Option<flodl::nn::Dropout>> {
graph
.nodes
.iter()
.map(|node| {
if node.kind == crate::graph::node::NodeKind::Hidden && opts.dropout_prob > 0.0 {
Some(flodl::nn::Dropout::new(opts.dropout_prob as f64))
} else {
None
}
})
.collect()
}
fn build_port_masks(
graph: &Topology,
opts: &NetworkOptions,
) -> flodl::tensor::Result<Vec<Vec<Option<Tensor>>>> {
let d = graph.options.input_dim.unwrap_or(1) as i64;
graph
.nodes
.iter()
.map(|node| {
(0..node.num_outputs.max(1))
.map(|p| {
let Some(mask) = node.port_feature_mask(p) else {
return Ok(None);
};
let t = match opts.dtype {
DType::Float64 => Tensor::from_f64(
&mask
.iter()
.map(|b| if *b == 0 { 0.0 } else { 1.0 })
.collect::<Vec<f64>>(),
&[1, d],
opts.device,
)?,
_ => Tensor::from_f32(
&mask
.iter()
.map(|b| if *b == 0 { 0.0 } else { 1.0 })
.collect::<Vec<f32>>(),
&[1, d],
opts.device,
)?,
};
Ok(Some(t))
})
.collect::<flodl::tensor::Result<Vec<_>>>()
})
.collect()
}
fn assemble(
graph: &Topology,
node_dims: &[(usize, usize)],
node_sources: &[Vec<Option<crate::graph::topology::Port>>],
layers: Vec<Linear>,
port_projections: Vec<Vec<Option<Linear>>>,
dropout_layers: Vec<Option<flodl::nn::Dropout>>,
port_masks: Vec<Vec<Option<Tensor>>>,
) -> flodl::tensor::Result<Network> {
let topo = &graph.options;
let output_node = graph.nodes.len() - 1;
let name = format!("network_{}_{}", graph.id, fastrand::u64(..));
debug!(
"Network::build -- {} port_proj_levels, dims={:?}",
port_projections.len(),
node_dims
.iter()
.map(|&(i, o)| format!("{i}->{o}"))
.collect::<Vec<_>>()
);
Ok(Network {
name,
input_dim: topo.input_dim.unwrap_or(1),
hidden_dim: node_dims.iter().map(|&(_, out)| out).max().unwrap_or(8),
layers,
connections: graph.connections.clone(),
nodes: graph.nodes.clone(),
node_dims: node_dims.to_vec(),
node_sources: node_sources.to_vec(),
output_node,
port_projections,
dropout_layers,
port_masks,
})
}
fn linear_from_piece(
vals: &[f64],
in_dim: i64,
out_dim: i64,
device: Device,
dtype: DType,
) -> flodl::tensor::Result<Linear> {
let n = (out_dim * in_dim) as usize;
let need = n + out_dim as usize;
if vals.len() < need {
return Err(flodl::tensor::TensorError::new(&format!(
"layer inventory: piece holds {} values, {need} needed for {in_dim}->{out_dim}",
vals.len()
)));
}
let bound = 1.0 / (in_dim as f64).sqrt();
let w: Vec<f64> = vals[..n].iter().map(|v| v * bound).collect();
let b: Vec<f64> = vals[n..need].iter().map(|v| v * bound).collect();
let (weight, bias) = match dtype {
DType::Float64 => (
Tensor::from_f64(&w, &[out_dim, in_dim], device)?,
Tensor::from_f64(&b, &[out_dim], device)?,
),
_ => {
let w32: Vec<f32> = w.iter().map(|&v| v as f32).collect();
let b32: Vec<f32> = b.iter().map(|&v| v as f32).collect();
(
Tensor::from_f32(&w32, &[out_dim, in_dim], device)?,
Tensor::from_f32(&b32, &[out_dim], device)?,
)
}
};
Ok(Linear {
weight: Parameter::new(weight, "weight"),
bias: Some(Parameter::new(bias, "bias")),
})
}
fn create_linear_with_seed(
in_dim: i64,
out_dim: i64,
device: Device,
dtype: DType,
rng: Option<&mut fastrand::Rng>,
) -> flodl::tensor::Result<Linear> {
let Some(rng) = rng else {
return Linear::on_device(in_dim, out_dim, device);
};
let n = (out_dim * in_dim) as usize;
let bound = 1.0 / (in_dim as f64).sqrt();
let w = match dtype {
DType::Float64 => {
let data: Vec<f64> = (0..n).map(|_| (rng.f64() * 2.0 - 1.0) * bound).collect();
Tensor::from_f64(&data, &[out_dim, in_dim], device)?
}
_ => {
let data: Vec<f32> = (0..n)
.map(|_| ((rng.f64() * 2.0 - 1.0) * bound) as f32)
.collect();
Tensor::from_f32(&data, &[out_dim, in_dim], device)?
}
};
let b = match dtype {
DType::Float64 => {
let data: Vec<f64> = (0..out_dim as usize)
.map(|_| (rng.f64() * 2.0 - 1.0) * bound)
.collect();
Tensor::from_f64(&data, &[out_dim], device)?
}
_ => {
let data: Vec<f32> = (0..out_dim as usize)
.map(|_| ((rng.f64() * 2.0 - 1.0) * bound) as f32)
.collect();
Tensor::from_f32(&data, &[out_dim], device)?
}
};
Ok(Linear {
weight: Parameter::new(w, "weight"),
bias: Some(Parameter::new(b, "bias")),
})
}
}
impl Network {
fn gather_port_tensors(
&self,
port_outputs: &HashMap<(usize, usize), Variable>,
node_id: usize,
) -> flodl::tensor::Result<Vec<Variable>> {
let mut out = Vec::new();
for (port_idx, source) in self.node_sources[node_id].iter().enumerate() {
let Some(p) = source else {
continue;
};
let mut t = port_outputs
.get(&(p.node, p.index))
.cloned()
.ok_or_else(|| {
flodl::tensor::TensorError::new(&format!(
"forward: source n{}o{} not yet computed (topological order violated?)",
p.node, p.index
))
})?;
if let Some(proj) = self.port_projections[node_id]
.get(port_idx)
.and_then(|p| p.as_ref())
{
t = proj.forward(&t)?;
}
out.push(t);
}
Ok(out)
}
fn combine_inputs(
&self,
ports: Vec<Variable>,
node_id: usize,
) -> flodl::tensor::Result<Variable> {
let n = ports.len();
let mut iter = ports.into_iter();
let Some(mut result) = iter.next() else {
let dim = self.node_dims[node_id].1 as i64;
let opts = flodl::TensorOptions {
dtype: flodl::DType::Float32,
device: Device::CPU,
};
return Ok(Variable::new(
flodl::Tensor::zeros(&[1, dim], opts).unwrap(),
false,
));
};
let op = self.nodes[node_id].combine_op.unwrap_or(CombineOp::Add);
for t in iter {
result = match op {
CombineOp::Add | CombineOp::Mean => result.add(&t)?,
CombineOp::Multiply => result.mul(&t)?,
CombineOp::Subtract => result.sub(&t)?,
CombineOp::Divide => crate::graph::node::safe_divide(&result, &t)?,
CombineOp::Max => result.maximum(&t)?,
CombineOp::Min => result.minimum(&t)?,
};
}
match op {
CombineOp::Mean if n > 1 => result.mul_scalar(1.0 / n as f64),
_ => Ok(result),
}
}
fn standardize(&self, node_id: usize, x: Variable) -> flodl::tensor::Result<Variable> {
match self.nodes[node_id].standardize {
Some(op) => op.apply(&x),
None => Ok(x),
}
}
fn apply_dropout(&self, node_id: usize, x: Variable) -> flodl::tensor::Result<Variable> {
match &self.dropout_layers[node_id] {
Some(dropout) => dropout.forward(&x),
None => Ok(x),
}
}
pub fn to_json(&self) -> flodl::tensor::Result<String> {
crate::spec::NetworkFacts::from_network(self).to_json()
}
pub fn topology_blueprint(&self) -> crate::graph::topology::Topology {
let mut topo = crate::graph::topology::Topology::new(
0,
Some(crate::graph::topology::TopologyOptions {
input_dim: Some(self.input_dim),
output_dim: Some(self.node_dims[self.output_node].1),
..Default::default()
}),
);
topo.nodes = self.nodes.clone();
topo.connections = self.connections.clone();
topo
}
pub fn clone_weights_from(&mut self, other: &Network) -> flodl::tensor::Result<()> {
if self.layers.len() != other.layers.len() {
return Err(flodl::tensor::TensorError::new(&format!(
"clone_weights_from: architecture mismatch ({} vs {} layers)",
self.layers.len(),
other.layers.len()
)));
}
for (dst, src) in self.layers.iter_mut().zip(other.layers.iter()) {
let sw = src.weight.variable.data();
let dw = dst.weight.variable.data();
if dw.shape() != sw.shape() {
return Err(flodl::tensor::TensorError::new(&format!(
"clone_weights_from: layer weight shape mismatch {:?} vs {:?}",
dw.shape(),
sw.shape()
)));
}
dst.weight.variable.set_data(sw.clone());
match (&mut dst.bias, &src.bias) {
(Some(db), Some(sb)) => {
let sbt = sb.variable.data();
db.variable.set_data(sbt.clone());
}
(None, None) => {}
_ => {
return Err(flodl::tensor::TensorError::new(
"clone_weights_from: bias presence mismatch",
));
}
}
}
for (dst_ports, src_ports) in self
.port_projections
.iter_mut()
.zip(other.port_projections.iter())
{
for (dst_proj, src_proj) in dst_ports.iter_mut().zip(src_ports.iter()) {
match (dst_proj, src_proj) {
(Some(dp), Some(sp)) => {
let sw = sp.weight.variable.data();
dp.weight.variable.set_data(sw.clone());
match (&mut dp.bias, &sp.bias) {
(Some(db), Some(sb)) => {
let sbt = sb.variable.data();
db.variable.set_data(sbt.clone());
}
(None, None) => {}
_ => {
return Err(flodl::tensor::TensorError::new(
"clone_weights_from: projection bias mismatch",
));
}
}
}
(None, None) => {}
_ => {
return Err(flodl::tensor::TensorError::new(
"clone_weights_from: projection presence mismatch",
));
}
}
}
}
Ok(())
}
pub fn transplant_weights_from(
&mut self,
donor: &Network,
map: &[(usize, usize)],
) -> flodl::tensor::Result<usize> {
let mut copied = 0usize;
for &(child, parent) in map {
if child >= self.layers.len() || parent >= donor.layers.len() {
continue;
}
let (child_dims, donor_dims) = (self.node_dims[child], donor.node_dims[parent]);
if child_dims.1 != donor_dims.1 {
continue;
}
{
let (dst, src) = (&mut self.layers[child], &donor.layers[parent]);
let (dw, sw) = (dst.weight.variable.data(), src.weight.variable.data());
if dw.shape() == sw.shape() {
dst.weight.variable.set_data(sw);
} else {
let (c_in, c_out) = (child_dims.0, child_dims.1);
let d_in = donor_dims.0;
let (merged, donor_w) = (dw.to_f32_vec()?, sw.to_f32_vec()?);
if merged.len() != c_in * c_out || donor_w.len() != d_in * c_out {
continue;
}
let merged = Self::merge_input_columns(merged, c_in, &donor_w, d_in);
let device = dst.weight.variable.data().device();
let merged = Tensor::from_f32(&merged, &[c_out as i64, c_in as i64], device)?;
dst.weight.variable.set_data(merged);
}
if let (Some(db), Some(sb)) = (&mut dst.bias, &src.bias) {
let sbt = sb.variable.data();
if db.variable.data().shape() == sbt.shape() {
db.variable.set_data(sbt);
}
}
}
let donor_ports = &donor.port_projections[parent];
for (dp, sp) in self.port_projections[child]
.iter_mut()
.zip(donor_ports.iter())
{
let (Some(dp), Some(sp)) = (dp, sp) else {
continue;
};
let (dpt, spt) = (dp.weight.variable.data(), sp.weight.variable.data());
if dpt.shape() != spt.shape() {
continue;
}
dp.weight.variable.set_data(spt);
if let (Some(db), Some(sb)) = (&mut dp.bias, &sp.bias) {
let sbt = sb.variable.data();
if db.variable.data().shape() == sbt.shape() {
db.variable.set_data(sbt);
}
}
}
copied += 1;
}
Ok(copied)
}
fn merge_input_columns(
mut child: Vec<f32>,
c_in: usize,
donor: &[f32],
d_in: usize,
) -> Vec<f32> {
let out = child.len() / c_in.max(1);
for row in 0..out {
for col in 0..c_in.min(d_in) {
child[row * c_in + col] = donor[row * d_in + col];
}
}
child
}
pub fn export_weights(&self) -> flodl::tensor::Result<Vec<f32>> {
let mut out = Vec::new();
let mut count = 0usize;
let push_tensor = |t: &flodl::tensor::Tensor,
out: &mut Vec<f32>,
count: &mut usize|
-> flodl::tensor::Result<()> {
let v = t.to_f32_vec()?;
*count += 1;
out.extend_from_slice(&v);
Ok(())
};
for layer in &self.layers {
push_tensor(&layer.weight.variable.data(), &mut out, &mut count)?;
if let Some(b) = &layer.bias {
push_tensor(&b.variable.data(), &mut out, &mut count)?;
}
}
for ports in &self.port_projections {
for p in ports.iter().flatten() {
push_tensor(&p.weight.variable.data(), &mut out, &mut count)?;
if let Some(b) = &p.bias {
push_tensor(&b.variable.data(), &mut out, &mut count)?;
}
}
}
out.splice(0..0, [count as f32, out.len() as f32]);
Ok(out)
}
pub fn import_weights(&mut self, blob: &[f32]) -> flodl::tensor::Result<()> {
let Some(&[count, total]) = blob.first_chunk::<2>() else {
return Err(flodl::tensor::TensorError::new(
"import_weights: blob too small for header",
));
};
let count = count as usize;
let total = total as usize;
if blob.len() - 2 != total {
return Err(flodl::tensor::TensorError::new(&format!(
"import_weights: header says {} values, blob holds {}",
total,
blob.len() - 2
)));
}
let mut cursor = 2usize;
let mut seen = 0usize;
let load = |var: &flodl::Variable,
blob: &[f32],
cursor: &mut usize,
seen: &mut usize|
-> flodl::tensor::Result<()> {
let data = var.data();
let shape = data.shape();
let n: usize = shape.iter().product::<i64>() as usize;
let chunk = blob
.get(*cursor..*cursor + n)
.ok_or_else(|| flodl::tensor::TensorError::new("import_weights: blob truncated"))?;
let fresh = flodl::tensor::Tensor::from_f32(chunk, &shape, data.device())?;
var.set_data(fresh);
*cursor += n;
*seen += 1;
Ok(())
};
for layer in &self.layers {
load(&layer.weight.variable, blob, &mut cursor, &mut seen)?;
if let Some(b) = &layer.bias {
load(&b.variable, blob, &mut cursor, &mut seen)?;
}
}
for ports in &self.port_projections {
for p in ports.iter().flatten() {
load(&p.weight.variable, blob, &mut cursor, &mut seen)?;
if let Some(b) = &p.bias {
load(&b.variable, blob, &mut cursor, &mut seen)?;
}
}
}
if seen != count {
return Err(flodl::tensor::TensorError::new(&format!(
"import_weights: architecture mismatch — blob holds {count} tensors, net has {seen}"
)));
}
Ok(())
}
}
impl Module for Network {
fn forward(&self, input: &Variable) -> flodl::tensor::Result<Variable> {
let mut port_outputs: HashMap<(usize, usize), Variable> = HashMap::new();
for node_id in 0..self.layers.len() {
let base = if self.nodes[node_id].kind == crate::graph::node::NodeKind::Input {
self.layers[node_id].forward(input)?
} else {
let ports = self.gather_port_tensors(&port_outputs, node_id)?;
let combined = self.combine_inputs(ports, node_id)?;
let projected = self.layers[node_id].forward(&combined)?;
let standardized = self.standardize(node_id, projected)?;
self.apply_dropout(node_id, standardized)?
};
if self.nodes[node_id].kind == crate::graph::node::NodeKind::Input {
for o in 0..self.nodes[node_id].num_outputs.max(1) {
let out = match self.port_masks[node_id].get(o).and_then(|m| m.as_ref()) {
Some(mask) => {
let gated = Variable::new(input.data().mul(mask)?, false);
self.layers[node_id].forward(&gated)?
}
None => base.clone(),
};
port_outputs.insert((node_id, o), out);
}
} else {
for o in 0..self.nodes[node_id].num_outputs.max(1) {
let act = self.nodes[node_id].port_activation(o);
let out = act.apply(&base)?;
port_outputs.insert((node_id, o), out);
}
}
}
let out_node = self.output_node;
Ok(port_outputs
.get(&(out_node, 0))
.cloned()
.expect("output node port 0 must exist"))
}
fn parameters(&self) -> Vec<Parameter> {
let mut params: Vec<Parameter> = Vec::new();
for layer in &self.layers {
params.extend(layer.parameters());
}
for ports in &self.port_projections {
for p in ports.iter().flatten() {
params.extend(p.parameters());
}
}
params
}
fn name(&self) -> &str {
&self.name
}
fn set_training(&self, training: bool) {
for d in self.dropout_layers.iter().flatten() {
d.set_training(training);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::node::Activation;
use flodl::{DType, Tensor, TensorOptions};
fn cpu_opts() -> TensorOptions {
TensorOptions {
dtype: DType::Float32,
device: Device::CPU,
}
}
fn tiny_graph(topology_seed: usize, weight_seed: Option<usize>) -> Topology {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 2));
graph.nodes.push(Node::new_hidden(1, 3, 2));
graph.nodes.push(Node::new_output(2, 2, 1));
graph.options.topology_seed = topology_seed;
graph.options.weight_seed = weight_seed;
graph.refresh_labels();
graph.finalize();
graph
}
#[test]
fn a_shared_weight_seed_starts_equal_architectures_equal() {
let a = Network::build(&tiny_graph(7, Some(42)), Device::CPU).unwrap();
let b = Network::build(&tiny_graph(999, Some(42)), Device::CPU).unwrap();
assert_eq!(a.export_weights().unwrap(), b.export_weights().unwrap());
let c = Network::build(&tiny_graph(7, Some(43)), Device::CPU).unwrap();
assert_ne!(a.export_weights().unwrap(), c.export_weights().unwrap());
}
#[test]
fn a_missing_weight_seed_falls_back_to_the_topology_seed() {
let a = Network::build(&tiny_graph(7, None), Device::CPU).unwrap();
let b = Network::build(&tiny_graph(7, None), Device::CPU).unwrap();
assert_eq!(a.export_weights().unwrap(), b.export_weights().unwrap());
let c = Network::build(&tiny_graph(8, None), Device::CPU).unwrap();
assert_ne!(a.export_weights().unwrap(), c.export_weights().unwrap());
}
#[test]
fn transplant_copies_the_paired_nodes() {
let donor = Network::build(&tiny_graph(7, Some(1)), Device::CPU).unwrap();
let mut child = Network::build(&tiny_graph(7, Some(2)), Device::CPU).unwrap();
assert_ne!(
donor.export_weights().unwrap(),
child.export_weights().unwrap()
);
let map: Vec<(usize, usize)> = (0..child.layers.len()).map(|i| (i, i)).collect();
assert_eq!(
child.transplant_weights_from(&donor, &map).unwrap(),
map.len()
);
assert_eq!(
child.export_weights().unwrap(),
donor.export_weights().unwrap()
);
}
#[test]
fn merging_input_columns_keeps_the_childs_extra_inputs() {
let child = vec![10.0, 11.0, 12.0, 20.0, 21.0, 22.0];
let donor = vec![1.0, 2.0, 3.0, 4.0];
assert_eq!(
Network::merge_input_columns(child, 3, &donor, 2),
vec![1.0, 2.0, 12.0, 3.0, 4.0, 22.0]
);
}
#[test]
fn same_gene_ignores_input_width_but_not_output_width() {
let n = Node::new_hidden(1, 2, 2);
assert!(crate::engine::child::same_gene(&n, (4, 16), &n, (64, 16)));
assert!(!crate::engine::child::same_gene(&n, (4, 16), &n, (64, 32)));
}
fn rand_input(batch: i64, input_dim: usize) -> Variable {
Variable::new(
Tensor::randn(&[batch, input_dim as i64], cpu_opts()).unwrap(),
false,
)
}
#[test]
fn test_network_build_and_forward() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 2));
graph.nodes.push(Node::new_hidden(1, 3, 2));
graph.nodes.push(Node::new_output(2, 2, 1));
graph.refresh_labels();
graph.finalize();
assert!(!graph.connections.is_empty());
let module = Network::build(&graph, Device::CPU).unwrap();
let batch = 4i64;
let input = rand_input(batch, graph.options.input_dim.unwrap_or(1));
let output = module.forward(&input).unwrap();
assert_eq!(
output.shape(),
&[batch, graph.options.output_dim.unwrap_or(1) as i64]
);
assert!(module.parameters().len() >= graph.nodes.len() * 2);
}
#[test]
fn test_network_forward_mean() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 2));
graph.nodes.push(Node::new_hidden(1, 3, 2));
graph.nodes.push(Node::new_output(2, 2, 1));
graph.nodes[1].combine_op = Some(CombineOp::Mean);
graph.finalize();
let module = Network::build(&graph, Device::CPU).unwrap();
let batch = 2i64;
let input = rand_input(batch, graph.options.input_dim.unwrap_or(1));
let output = module.forward(&input).unwrap();
assert_eq!(
output.shape(),
&[batch, graph.options.output_dim.unwrap_or(1) as i64]
);
}
#[test]
fn test_divide_combine_with_dropout_stays_finite() -> flodl::tensor::Result<()> {
use flodl::Optimizer;
let mut graph = Topology::new(0, None);
graph.options.dropout_prob = 0.25;
graph.options.input_dim = Some(8);
graph.options.output_dim = Some(4);
graph.nodes.push(Node::new_input(0, 2));
let mut den = Node::new_hidden(1, 1, 1);
den.hidden_dim = Some(16);
den.combine_op = Some(CombineOp::Add);
graph.nodes.push(den);
let mut div = Node::new_hidden(2, 2, 1);
div.hidden_dim = Some(16);
div.combine_op = Some(CombineOp::Divide);
graph.nodes.push(div);
graph.nodes.push(Node::new_output(3, 1, 1));
graph.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
});
graph.connections.push(Connection {
from: Port { node: 0, index: 1 },
to: Port { node: 2, index: 0 },
});
graph.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 1 },
});
graph.connections.push(Connection {
from: Port { node: 2, index: 0 },
to: Port { node: 3, index: 0 },
});
graph.finalize();
let net = Network::build(&graph, Device::CPU).unwrap();
let mut optimizer = flodl::nn::Adam::new(&net.parameters(), 1e-3);
for step in 0..50 {
net.train();
let input = rand_input(4, 8);
let loss = net.forward(&input)?;
assert!(
loss.data().to_f32_vec()?.iter().all(|x| x.is_finite()),
"forward produced non-finite output at step {step}"
);
optimizer.zero_grad();
let scalar = loss.mean()?; scalar.set_requires_grad(true)?;
scalar.backward()?;
for p in net.parameters() {
if let Some(g) = p.variable.grad() {
assert!(
g.to_f32_vec()?.iter().all(|x| x.is_finite()),
"non-finite gradient at step {step}"
);
}
}
optimizer.step()?;
net.eval();
}
Ok(())
}
#[test]
fn test_network_forward_orphans() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_hidden(0, 2, 1));
graph.nodes.push(Node::new_hidden(1, 1, 1));
graph.finalize();
let module = Network::build(&graph, Device::CPU).unwrap();
let batch = 2i64;
let input = rand_input(batch, graph.options.input_dim.unwrap_or(1));
let output = module.forward(&input).unwrap();
assert_eq!(
output.shape(),
&[batch, graph.options.output_dim.unwrap_or(1) as i64]
);
}
#[test]
fn test_network_forward_activation_and_node_dim() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
let mut wide = Node::new_hidden(1, 1, 1);
wide.hidden_dim = Some(32);
wide.activation = Activation::ReLU;
graph.nodes.push(wide);
graph.nodes.push(Node::new_output(2, 1, 1));
graph.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
});
graph.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
});
let module = Network::build(&graph, Device::CPU).unwrap();
assert_eq!(module.node_dims[0].1, 8); assert_eq!(module.node_dims[1].0, 8); assert_eq!(module.node_dims[1].1, 32); assert_eq!(module.nodes[1].activation, Activation::ReLU);
assert_eq!(module.node_dims[2].0, 32); assert_eq!(module.node_dims[2].1, 8);
let batch = 3i64;
let input = rand_input(batch, graph.options.input_dim.unwrap_or(1));
let output = module.forward(&input).unwrap();
assert_eq!(output.shape(), &[batch, 8]);
}
#[test]
fn test_network_build_rejects_invalid_graph() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
graph.nodes.push(Node::new_hidden(1, 1, 1));
graph.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 0, index: 0 }, });
assert!(Network::build(&graph, Device::CPU).is_err());
}
#[test]
fn test_network_to_json_facts() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
let mut wide = Node::new_hidden(1, 1, 1);
wide.hidden_dim = Some(32);
wide.activation = Activation::ReLU;
graph.nodes.push(wide);
graph.nodes.push(Node::new_output(2, 1, 1));
graph.finalize();
let module = Network::build(&graph, Device::CPU).unwrap();
let json = module.to_json().unwrap();
let v: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(v["num_nodes"], 3);
assert_eq!(v["num_wires"], module.connections.len() as u64);
assert_eq!(v["output_node"], module.output_node as u64);
assert_eq!(v["param_tensors"], 2 * 3);
let expected: i64 = module.parameters().iter().map(|p| p.variable.numel()).sum();
assert_eq!(v["param_elements"], expected);
assert_eq!(v["node_dims"][1][1], 32);
assert_eq!(v["orphan_counts"][0], graph.orphan_counts().0 as u64);
assert_eq!(v["orphan_counts"][1], graph.orphan_counts().1 as u64);
assert_eq!(v["kind_counts"][0], 1); assert_eq!(v["kind_counts"][1], 1); assert_eq!(v["kind_counts"][2], 1); let acts = v["activation_counts"].as_array().unwrap();
assert!(acts.contains(&serde_json::json!(["ReLU", 1])));
assert!(acts.contains(&serde_json::json!(["Identity", 2])));
assert_eq!(v["depths"][0], 0); assert!(v["depths"][2].as_u64().unwrap() >= v["depths"][1].as_u64().unwrap());
}
#[test]
fn legacy_stacked_blueprint_runs_identically_to_a_native_one_wire_graph() {
let opts = crate::graph::topology::TopologyOptions {
input_dim: Some(2),
output_dim: Some(1),
..Default::default()
};
let mut native = Topology::new(9, Some(opts));
native.nodes.push(Node::new_input(0, 2));
native.nodes.push(Node::new_hidden(1, 2, 1));
native.nodes.push(Node::new_output(2, 1, 1));
native.connections = vec![
Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
},
Connection {
from: Port { node: 0, index: 1 },
to: Port { node: 1, index: 1 },
},
Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
},
];
let mut legacy = Topology::new(9, Some(opts));
legacy.nodes.push(Node::new_input(0, 2));
legacy.nodes.push(Node::new_hidden(1, 1, 1)); legacy.nodes.push(Node::new_output(2, 1, 1));
legacy.connections = vec![
Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
},
Connection {
from: Port { node: 0, index: 1 },
to: Port { node: 1, index: 0 },
},
Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
},
];
let legacy = Topology::from_json(&legacy.to_json().unwrap()).unwrap();
let a = Network::build(&native, Device::CPU).unwrap();
let b = Network::build(&legacy, Device::CPU).unwrap();
let input = Variable::new(
Tensor::from_f32(&[0.5, -1.5], &[1, 2], Device::CPU).unwrap(),
false,
);
let out_a = a.forward(&input).unwrap().data().to_f32_vec().unwrap();
let out_b = b.forward(&input).unwrap().data().to_f32_vec().unwrap();
assert_eq!(out_a, out_b);
}
#[test]
fn test_per_port_activations_give_distinct_signals() {
let opts = crate::graph::topology::TopologyOptions {
min_hidden_outputs_per_node: 2,
max_hidden_outputs_per_node: 2,
input_dim: Some(8),
output_dim: Some(1),
..Default::default()
};
let mut g = Topology::new(0, Some(opts));
g.nodes.push(Node::new_input(0, 1));
let mut n1 = Node::new_hidden(1, 1, 2).with_activation(Activation::ReLU);
n1.hidden_dim = Some(4);
g.nodes.push(n1);
g.nodes.push(Node::new_output(2, 2, 1));
g.finalize(); g.nodes[1].num_outputs = 2;
g.connections.retain(|c| c.to.node != 2);
g.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
});
g.connections.push(Connection {
from: Port { node: 1, index: 1 },
to: Port { node: 2, index: 1 },
});
g.nodes[1].port_activations = Some(vec![Activation::ReLU, Activation::Tanh]);
g.validate().unwrap();
let net = Network::build(&g, Device::CPU).unwrap();
let input = Variable::new(
Tensor::from_f32(
&[-1.0, -2.0, 0.5, 3.0, -1.0, -2.0, 0.5, 3.0],
&[1, 8],
Device::CPU,
)
.unwrap(),
false,
);
let out = net.forward(&input).unwrap();
assert_eq!(out.shape(), &[1, 1]);
let sample = Variable::new(
Tensor::from_f32(&[-1.0, -2.0, 0.5, 3.0], &[1, 4], Device::CPU).unwrap(),
false,
);
let p0 = Activation::ReLU.apply(&sample).unwrap();
let p1 = Activation::Tanh.apply(&sample).unwrap();
let d0 = p0.data().to_f32_vec().unwrap();
let d1 = p1.data().to_f32_vec().unwrap();
assert_ne!(
d0, d1,
"port 0 (ReLU) and port 1 (Tanh) must differ on negative input"
);
}
#[test]
fn test_unique_names() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
graph.nodes.push(Node::new_output(1, 1, 1));
graph.finalize();
let a = Network::build(&graph, Device::CPU).unwrap();
let b = Network::build(&graph, Device::CPU).unwrap();
assert_ne!(a.name(), b.name());
assert!(a.name().starts_with("network_0_"));
}
#[test]
fn test_seeded_build_is_deterministic() {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
graph.nodes.push(Node::new_hidden(1, 1, 1));
graph.nodes.push(Node::new_output(2, 1, 1));
graph.finalize();
graph.validate().unwrap();
let seeded = |seed: usize| {
Network::build_with_options(
&graph,
&NetworkOptions {
topology_seed: seed,
..Default::default()
},
)
.unwrap()
};
let flat = |net: &Network| -> Vec<f32> {
net.parameters()
.iter()
.flat_map(|p| p.variable.data().to_f32_vec().unwrap())
.collect()
};
let a = flat(&seeded(42));
let b = flat(&seeded(42));
assert_eq!(a.len(), b.len());
assert!(
a.iter().zip(&b).all(|(x, y)| (x - y).abs() < 1e-6),
"same seed must reproduce every weight"
);
let c = flat(&seeded(43));
assert!(
a.iter().zip(&c).any(|(x, y)| (x - y).abs() > 1e-6),
"different seed must change at least one weight"
);
let out = seeded(42)
.forward(&rand_input(2, graph.options.input_dim.unwrap_or(1)))
.unwrap();
assert_eq!(
out.shape(),
&[2, graph.options.output_dim.unwrap_or(1) as i64]
);
}
}
#[cfg(test)]
mod var_dim_test {
use super::*;
use flodl::{DType, Tensor, TensorOptions};
use proptest::prelude::*;
#[test]
fn test_variable_hidden_dim_forward() {
let mut g = Topology::new(0, None);
g.nodes.push(Node::new_input(0, 1));
let mut h = Node::new_hidden(1, 1, 1);
h.hidden_dim = Some(4);
g.nodes.push(h);
g.nodes.push(Node::new_output(2, 1, 1));
g.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 1, index: 0 },
});
g.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 0 },
});
g.finalize();
g.validate().unwrap();
let net = Network::build(&g, Device::CPU).unwrap();
let input = Variable::new(
Tensor::randn(
&[2, 1],
TensorOptions {
dtype: DType::Float32,
device: Device::CPU,
},
)
.unwrap(),
false,
);
let out = net.forward(&input).unwrap();
println!("output shape: {:?}", out.shape());
assert_eq!(out.shape(), &[2, 1]);
}
#[test]
fn test_variable_hidden_dim_fan_in() {
let mut g = Topology::new(0, None);
g.nodes.push(Node::new_input(0, 1));
let mut wide = Node::new_input(1, 1);
wide.hidden_dim = Some(16);
g.nodes.push(wide);
let mut h = Node::new_hidden(2, 2, 1);
h.hidden_dim = Some(8);
g.nodes.push(h);
g.nodes.push(Node::new_output(3, 1, 1));
g.connections.push(Connection {
from: Port { node: 0, index: 0 },
to: Port { node: 2, index: 0 },
});
g.connections.push(Connection {
from: Port { node: 1, index: 0 },
to: Port { node: 2, index: 1 },
});
g.connections.push(Connection {
from: Port { node: 2, index: 0 },
to: Port { node: 3, index: 0 },
});
g.finalize();
g.validate().unwrap();
let net = Network::build(&g, Device::CPU).unwrap();
let input = Variable::new(
Tensor::randn(
&[2, 1],
TensorOptions {
dtype: DType::Float32,
device: Device::CPU,
},
)
.unwrap(),
false,
);
let out = net.forward(&input).unwrap();
println!("fan-in output shape: {:?}", out.shape());
assert_eq!(out.shape(), &[2, 1]);
}
proptest! {
#[test]
fn prop_build_succeeds_for_valid_topology(
topo in crate::graph::topology::test_strategies::topology_strategy()
) {
let net = Network::build(&topo, Device::CPU);
prop_assert!(net.is_ok(), "build failed: {:?}", net.err());
let net = net.unwrap();
prop_assert!(!net.parameters().is_empty(), "no parameters");
}
#[test]
fn prop_output_shape_matches_output_dim(
topo in crate::graph::topology::test_strategies::topology_strategy()
) {
let net = Network::build(&topo, Device::CPU).unwrap();
let bs = 4;
let input = Variable::new(
Tensor::randn(&[bs as i64, topo.options.input_dim.unwrap_or(1) as i64],
TensorOptions { dtype: DType::Float32, device: Device::CPU }).unwrap(),
false,
);
let out = net.forward(&input).unwrap();
prop_assert_eq!(out.shape()[0], bs as i64);
prop_assert_eq!(out.shape()[1], topo.options.output_dim.unwrap_or(1) as i64);
}
}
#[test]
fn test_feature_mask_hides_a_feature_from_every_port() {
let opts = crate::graph::topology::TopologyOptions {
input_dim: Some(4),
output_dim: Some(1),
..Default::default()
};
let mut graph = Topology::new(7, Some(opts));
graph.create_random_hidden_nodes(3);
graph.finalize();
let input_idx = graph
.nodes
.iter()
.position(|n| n.kind == crate::graph::node::NodeKind::Input)
.unwrap();
let n_ports = graph.nodes[input_idx].num_outputs.max(1);
graph.nodes[input_idx].feature_masks = Some(vec![vec![0u8, 1, 1, 1]; n_ports]);
let net = Network::build(&graph, Device::CPU).unwrap();
let a = Variable::new(
Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[1, 4], Device::CPU).unwrap(),
false,
);
let b = Variable::new(
Tensor::from_f32(&[9.0, 2.0, 3.0, 4.0], &[1, 4], Device::CPU).unwrap(),
false,
);
let spread = |net: &Network| {
net.forward(&a)
.unwrap()
.data()
.sub(&net.forward(&b).unwrap().data())
.unwrap()
.abs()
.unwrap()
.sum()
.unwrap()
.item()
.unwrap() as f32
};
let masked_spread = spread(&net);
assert!(
masked_spread < 1e-6,
"a feature masked out of every port must not reach the output (spread {masked_spread})"
);
graph.nodes[input_idx].feature_masks = None;
let unmasked = Network::build(&graph, Device::CPU).unwrap();
let unmasked_spread = spread(&unmasked);
assert!(
unmasked_spread > 1e-6,
"control: without masks the same net DOES see feature 0 (spread {unmasked_spread})"
);
}
}