use flodl::Variable;
use serde::{Deserialize, Serialize};
pub const DIVIDE_EPS: f64 = 1e-3;
pub fn safe_divide(n: &Variable, d: &Variable) -> flodl::tensor::Result<Variable> {
let d_abs = d.abs()?;
let clamped = d_abs.clamp_min(DIVIDE_EPS)?;
let ratio = d.div(&clamped)?;
n.mul(&ratio)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum CombineOp {
Add,
Mean,
Multiply,
Subtract,
Divide,
Max,
Min,
}
impl CombineOp {
pub fn apply(&self, tensors: &[Variable]) -> flodl::tensor::Result<Variable> {
match self {
CombineOp::Add => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.add(t)?;
}
Ok(result)
}
CombineOp::Mean => {
let sum = CombineOp::Add.apply(tensors)?;
sum.mul_scalar(1.0 / tensors.len() as f64)
}
CombineOp::Multiply => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.mul(t)?;
}
Ok(result)
}
CombineOp::Subtract => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.sub(t)?;
}
Ok(result)
}
CombineOp::Divide => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = safe_divide(&result, t)?;
}
Ok(result)
}
CombineOp::Max => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.maximum(t)?;
}
Ok(result)
}
CombineOp::Min => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.minimum(t)?;
}
Ok(result)
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub enum Activation {
#[default]
Identity,
ReLU,
GeLU,
SiLU,
SELU,
Tanh,
Sigmoid,
Mish,
LeakyReLU,
ELU,
GeluTanh,
Softplus,
HardSwish,
HardSigmoid,
Sin,
Cos,
Softmax,
LogSoftmax,
}
impl Activation {
pub fn apply(&self, x: &Variable) -> flodl::tensor::Result<Variable> {
match self {
Activation::Identity => Ok(x.clone()),
Activation::ReLU => x.relu(),
Activation::GeLU => x.gelu(),
Activation::SiLU => x.silu(),
Activation::SELU => x.selu(),
Activation::Tanh => x.tanh(),
Activation::Sigmoid => x.sigmoid(),
Activation::Mish => x.mish(),
Activation::LeakyReLU => x.leaky_relu(0.01),
Activation::ELU => x.elu(1.0),
Activation::GeluTanh => x.gelu_tanh(),
Activation::Softplus => x.softplus(1.0, 20.0),
Activation::HardSwish => x.hardswish(),
Activation::HardSigmoid => x.hardsigmoid(),
Activation::Sin => x.sin(),
Activation::Cos => x.cos(),
Activation::Softmax => x.softmax(-1),
Activation::LogSoftmax => x.log_softmax(-1),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum StandardizeOp {
#[default]
Identity,
LayerNorm,
RmsNorm,
InstanceNorm,
}
impl StandardizeOp {
pub fn apply(&self, x: &Variable) -> flodl::tensor::Result<Variable> {
match self {
StandardizeOp::Identity => Ok(x.clone()),
StandardizeOp::LayerNorm => {
let mean = x.mean_dim(-1, true)?; let centered = x.sub(&mean)?;
let var = centered.mul(¢ered)?.mean_dim(-1, true)?; let std = var.add_scalar(1e-5)?.sqrt()?; let normed = centered.div(&std)?;
Ok(normed)
}
StandardizeOp::RmsNorm => {
let ms = x.mul(x)?.mean_dim(-1, true)?; let rms = ms.add_scalar(1e-5)?.sqrt()?;
Ok(x.div(&rms)?)
}
StandardizeOp::InstanceNorm => {
let mean = x.mean_dim(-1, true)?;
let centered = x.sub(&mean)?;
let var = centered.mul(¢ered)?.mean_dim(-1, true)?;
let std = var.add_scalar(1e-5)?.sqrt()?;
Ok(centered.div(&std)?)
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct Node {
pub id: usize, pub num_inputs: usize, pub num_outputs: usize, pub kind: NodeKind, pub hidden_dim: Option<usize>,
pub activation: Activation,
#[serde(default)]
pub port_activations: Option<Vec<Activation>>,
#[serde(default)]
pub combine_op: Option<CombineOp>,
#[serde(default)]
pub standardize: Option<StandardizeOp>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum NodeKind {
Input, Hidden, Output, }
impl Node {
pub fn new_input(id: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs: 0,
num_outputs,
kind: NodeKind::Input,
hidden_dim: None,
activation: Activation::Identity,
port_activations: None,
combine_op: None,
standardize: None,
}
}
pub fn new_hidden(id: usize, num_inputs: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs,
num_outputs,
kind: NodeKind::Hidden,
hidden_dim: None,
activation: Activation::Identity,
port_activations: None,
combine_op: None,
standardize: None,
}
}
pub fn new_output(id: usize, num_inputs: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs,
num_outputs,
kind: NodeKind::Output,
hidden_dim: None,
activation: Activation::Identity,
port_activations: None,
combine_op: None,
standardize: None,
}
}
pub fn with_activation(mut self, activation: Activation) -> Self {
self.activation = activation;
self
}
pub fn with_combine_op(mut self, combine_op: CombineOp) -> Self {
self.combine_op = Some(combine_op);
self
}
pub fn with_hidden_dim(mut self, hidden_dim: usize) -> Self {
self.hidden_dim = Some(hidden_dim);
self
}
pub fn with_port_activations(mut self, ports: Vec<Activation>) -> Self {
self.port_activations = if ports.is_empty() { None } else { Some(ports) };
self
}
pub fn port_activation(&self, i: usize) -> Activation {
self.port_activations
.as_ref()
.and_then(|v| v.get(i).copied())
.unwrap_or(self.activation)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::topology::{Topology, TopologyOptions};
use proptest::prelude::*;
#[test]
fn new_pool_ops_are_numerically_sound() {
let x = flodl::Tensor::from_f32(
&[1.0, 2.0, 3.0, 4.0, -1.0, 0.0, 1.0, 2.0],
&[2, 4],
flodl::Device::CPU,
)
.unwrap();
let x = flodl::Variable::new(x, false);
let s = Activation::Softmax.apply(&x).unwrap();
let sums = s.data().sum().unwrap().item().unwrap() as f32;
assert!(
(sums - 2.0).abs() < 1e-4,
"softmax rows must sum to 1 each (got {sums})"
);
let ls = Activation::LogSoftmax.apply(&x).unwrap();
let recon = ls.data().exp().unwrap().sum().unwrap().item().unwrap() as f32;
assert!(
(recon - 2.0).abs() < 1e-4,
"exp(log_softmax) must sum to 1 per row (got {recon})"
);
let r = StandardizeOp::RmsNorm.apply(&x).unwrap();
let ms = r
.data()
.mul(&r.data())
.unwrap()
.mean()
.unwrap()
.item()
.unwrap() as f32;
let rms = ms.sqrt();
assert!(
(rms - 1.0).abs() < 1e-3,
"rmsnorm output RMS ≈ 1 (got {rms})"
);
let i = StandardizeOp::InstanceNorm.apply(&x).unwrap();
let m = i.data().mean().unwrap().item().unwrap() as f32;
assert!(m.abs() < 1e-5, "instancenorm per-row mean ≈ 0 (got {m})");
}
fn node_strategy() -> impl Strategy<Value = Node> {
(0usize..100, 0usize..8, 0usize..8, 0usize..3).prop_map(
|(id, num_inputs, num_outputs, kind)| Node {
id,
num_inputs,
num_outputs,
kind: match kind {
0 => NodeKind::Input,
1 => NodeKind::Hidden,
_ => NodeKind::Output,
},
hidden_dim: None,
activation: Activation::Identity,
port_activations: None,
combine_op: None,
standardize: None,
},
)
}
#[test]
fn test_new_node_inputs() {
let node: Node = Node::new_input(1, 2);
assert_eq!(node.num_inputs, 0);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.hidden_dim, None);
assert_eq!(node.activation, Activation::Identity);
}
#[test]
fn test_new_node_hidden() {
let node: Node = Node::new_hidden(1, 3, 2);
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.activation, Activation::Identity);
}
#[test]
fn test_node_builders() {
let node = Node::new_hidden(1, 3, 2)
.with_activation(Activation::GeLU)
.with_hidden_dim(32);
assert_eq!(node.activation, Activation::GeLU);
assert_eq!(node.hidden_dim, Some(32));
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.kind, NodeKind::Hidden);
let wide = Node::new_hidden(1, 3, 2)
.with_hidden_dim(16)
.with_activation(Activation::ReLU);
assert_eq!(wide.hidden_dim, Some(16));
assert_eq!(wide.activation, Activation::ReLU);
}
#[test]
fn test_new_node_outputs() {
let node: Node = Node::new_output(1, 3, 2);
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.hidden_dim, None);
}
#[test]
fn test_activation_default_and_display() {
assert_eq!(Activation::default(), Activation::Identity);
assert_eq!(Activation::ReLU.to_string(), "relu");
assert_eq!(Activation::GeLU.to_string(), "gelu");
assert_eq!(Activation::SELU.to_string(), "selu");
assert_eq!(Activation::LeakyReLU.to_string(), "leaky_relu");
assert_eq!(Activation::ELU.to_string(), "elu");
assert_eq!(Activation::GeluTanh.to_string(), "gelu_tanh");
assert_eq!(Activation::Softplus.to_string(), "softplus");
assert_eq!(Activation::HardSwish.to_string(), "hardswish");
assert_eq!(Activation::HardSigmoid.to_string(), "hardsigmoid");
}
proptest! {
#[test]
fn prop_node_builders_preserve_identity(
node in node_strategy(),
hidden_dim in 1usize..128,
relu in any::<bool>(),
) {
let activation = if relu { Activation::ReLU } else { Activation::GeLU };
let built = node
.clone()
.with_hidden_dim(hidden_dim)
.with_activation(activation);
prop_assert_eq!(built.id, node.id);
prop_assert_eq!(built.kind, node.kind);
prop_assert_eq!(built.num_inputs, node.num_inputs);
prop_assert_eq!(built.num_outputs, node.num_outputs);
prop_assert_eq!(built.activation, activation);
prop_assert_eq!(built.hidden_dim, Some(hidden_dim));
let bad = node.with_hidden_dim(0);
prop_assert_eq!(bad.hidden_dim, Some(0));
}
#[test]
fn prop_random_hidden_node_respects_port_ranges(
min_inputs in 1usize..5,
min_outputs in 1usize..5,
span_in in 0usize..4,
span_out in 0usize..4,
) {
let opts = TopologyOptions {
topology_seed: 16,
min_hidden_num_nodes: 2,
max_hidden_num_nodes: 5,
min_hidden_inputs_per_node: min_inputs,
max_hidden_inputs_per_node: min_inputs + span_in,
min_hidden_outputs_per_node: min_outputs,
max_hidden_outputs_per_node: min_outputs + span_out,
input_dim: Some(4),
output_dim: Some(2),
dropout_prob: 0.0,
};
let mut graph = Topology::new(0, Some(opts));
graph.create_random_hidden_node();
let node = &graph.nodes[0];
prop_assert!(node.id == 0);
prop_assert!(node.kind == NodeKind::Hidden);
prop_assert!(
node.num_inputs >= min_inputs && node.num_inputs <= min_inputs + span_in,
"num_inputs {} outside [{}, {}]",
node.num_inputs,
min_inputs,
min_inputs + span_in
);
prop_assert!(
node.num_outputs >= min_outputs && node.num_outputs <= min_outputs + span_out,
"num_outputs {} outside [{}, {}]",
node.num_outputs,
min_outputs,
min_outputs + span_out
);
}
}
}