use flodl::Variable;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum CombineOp {
Add,
Mean,
Multiply,
Subtract,
Divide,
Max,
Min,
}
impl CombineOp {
pub fn apply(&self, tensors: &[Variable]) -> flodl::tensor::Result<Variable> {
match self {
CombineOp::Add => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.add(t)?;
}
Ok(result)
}
CombineOp::Mean => {
let sum = CombineOp::Add.apply(tensors)?;
sum.mul_scalar(1.0 / tensors.len() as f64)
}
CombineOp::Multiply => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.mul(t)?;
}
Ok(result)
}
CombineOp::Subtract => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.sub(t)?;
}
Ok(result)
}
CombineOp::Divide => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.div(t)?;
}
Ok(result)
}
CombineOp::Max => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.maximum(t)?;
}
Ok(result)
}
CombineOp::Min => {
let mut result = tensors[0].clone();
for t in &tensors[1..] {
result = result.minimum(t)?;
}
Ok(result)
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub enum Activation {
#[default]
Identity,
ReLU,
GeLU,
SiLU,
SELU,
Tanh,
Sigmoid,
Mish,
LeakyReLU,
ELU,
GeluTanh,
Softplus,
HardSwish,
HardSigmoid,
Sin,
Cos,
}
impl Activation {
pub fn apply(&self, x: &Variable) -> flodl::tensor::Result<Variable> {
match self {
Activation::Identity => Ok(x.clone()),
Activation::ReLU => x.relu(),
Activation::GeLU => x.gelu(),
Activation::SiLU => x.silu(),
Activation::SELU => x.selu(),
Activation::Tanh => x.tanh(),
Activation::Sigmoid => x.sigmoid(),
Activation::Mish => x.mish(),
Activation::LeakyReLU => x.leaky_relu(0.01),
Activation::ELU => x.elu(1.0),
Activation::GeluTanh => x.gelu_tanh(),
Activation::Softplus => x.softplus(1.0, 20.0),
Activation::HardSwish => x.hardswish(),
Activation::HardSigmoid => x.hardsigmoid(),
Activation::Sin => x.sin(),
Activation::Cos => x.cos(),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum StandardizeOp {
#[default]
Identity,
LayerNorm,
}
impl StandardizeOp {
pub fn apply(&self, x: &Variable) -> flodl::tensor::Result<Variable> {
match self {
StandardizeOp::Identity => Ok(x.clone()),
StandardizeOp::LayerNorm => {
let mean = x.mean_dim(-1, true)?; let centered = x.sub(&mean)?;
let var = centered.mul(¢ered)?.mean_dim(-1, true)?; let std = var.add_scalar(1e-5)?.sqrt()?; let normed = centered.div(&std)?;
Ok(normed)
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct Node {
pub id: usize, pub num_inputs: usize, pub num_outputs: usize, pub kind: NodeKind, pub hidden_dim: Option<usize>,
pub activation: Activation,
#[serde(default)]
pub combine_op: Option<CombineOp>,
#[serde(default)]
pub standardize: Option<StandardizeOp>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum NodeKind {
Input, Hidden, Output, }
impl Node {
pub fn new_input(id: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs: 0,
num_outputs,
kind: NodeKind::Input,
hidden_dim: None,
activation: Activation::Identity,
combine_op: None,
standardize: None,
}
}
pub fn new_hidden(id: usize, num_inputs: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs,
num_outputs,
kind: NodeKind::Hidden,
hidden_dim: None,
activation: Activation::Identity,
combine_op: None,
standardize: None,
}
}
pub fn new_output(id: usize, num_inputs: usize, num_outputs: usize) -> Self {
Node {
id,
num_inputs,
num_outputs,
kind: NodeKind::Output,
hidden_dim: None,
activation: Activation::Identity,
combine_op: None,
standardize: None,
}
}
pub fn with_activation(mut self, activation: Activation) -> Self {
self.activation = activation;
self
}
pub fn with_combine_op(mut self, combine_op: CombineOp) -> Self {
self.combine_op = Some(combine_op);
self
}
pub fn with_hidden_dim(mut self, hidden_dim: usize) -> Self {
self.hidden_dim = Some(hidden_dim);
self
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::topology::{Topology, TopologyOptions};
use proptest::prelude::*;
fn node_strategy() -> impl Strategy<Value = Node> {
(0usize..100, 0usize..8, 0usize..8, 0usize..3).prop_map(
|(id, num_inputs, num_outputs, kind)| Node {
id,
num_inputs,
num_outputs,
kind: match kind {
0 => NodeKind::Input,
1 => NodeKind::Hidden,
_ => NodeKind::Output,
},
hidden_dim: None,
activation: Activation::Identity,
combine_op: None,
standardize: None,
},
)
}
#[test]
fn test_new_node_inputs() {
let node: Node = Node::new_input(1, 2);
assert_eq!(node.num_inputs, 0);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.hidden_dim, None);
assert_eq!(node.activation, Activation::Identity);
}
#[test]
fn test_new_node_hidden() {
let node: Node = Node::new_hidden(1, 3, 2);
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.activation, Activation::Identity);
}
#[test]
fn test_node_builders() {
let node = Node::new_hidden(1, 3, 2)
.with_activation(Activation::GeLU)
.with_hidden_dim(32);
assert_eq!(node.activation, Activation::GeLU);
assert_eq!(node.hidden_dim, Some(32));
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.kind, NodeKind::Hidden);
let wide = Node::new_hidden(1, 3, 2)
.with_hidden_dim(16)
.with_activation(Activation::ReLU);
assert_eq!(wide.hidden_dim, Some(16));
assert_eq!(wide.activation, Activation::ReLU);
}
#[test]
fn test_new_node_outputs() {
let node: Node = Node::new_output(1, 3, 2);
assert_eq!(node.num_inputs, 3);
assert_eq!(node.num_outputs, 2);
assert_eq!(node.hidden_dim, None);
}
#[test]
fn test_activation_default_and_display() {
assert_eq!(Activation::default(), Activation::Identity);
assert_eq!(Activation::ReLU.to_string(), "relu");
assert_eq!(Activation::GeLU.to_string(), "gelu");
assert_eq!(Activation::SELU.to_string(), "selu");
assert_eq!(Activation::LeakyReLU.to_string(), "leaky_relu");
assert_eq!(Activation::ELU.to_string(), "elu");
assert_eq!(Activation::GeluTanh.to_string(), "gelu_tanh");
assert_eq!(Activation::Softplus.to_string(), "softplus");
assert_eq!(Activation::HardSwish.to_string(), "hardswish");
assert_eq!(Activation::HardSigmoid.to_string(), "hardsigmoid");
}
proptest! {
#[test]
fn prop_node_builders_preserve_identity(
node in node_strategy(),
hidden_dim in 1usize..128,
relu in any::<bool>(),
) {
let activation = if relu { Activation::ReLU } else { Activation::GeLU };
let built = node
.clone()
.with_hidden_dim(hidden_dim)
.with_activation(activation);
prop_assert_eq!(built.id, node.id);
prop_assert_eq!(built.kind, node.kind);
prop_assert_eq!(built.num_inputs, node.num_inputs);
prop_assert_eq!(built.num_outputs, node.num_outputs);
prop_assert_eq!(built.activation, activation);
prop_assert_eq!(built.hidden_dim, Some(hidden_dim));
let bad = node.with_hidden_dim(0);
prop_assert_eq!(bad.hidden_dim, Some(0));
}
#[test]
fn prop_random_hidden_node_respects_port_ranges(
min_inputs in 1usize..5,
min_outputs in 1usize..5,
span_in in 0usize..4,
span_out in 0usize..4,
) {
let opts = TopologyOptions {
seed: 16,
min_hidden_num_nodes: 2,
max_hidden_num_nodes: 5,
min_hidden_inputs_per_node: min_inputs,
max_hidden_inputs_per_node: min_inputs + span_in,
min_hidden_outputs_per_node: min_outputs,
max_hidden_outputs_per_node: min_outputs + span_out,
input_dim: 1,
hidden_dim: 8,
output_dim: 1,
dropout_prob: 0.0,
};
let mut graph = Topology::new(0, Some(opts));
graph.create_random_hidden_node();
let node = &graph.nodes[0];
prop_assert!(node.id == 0);
prop_assert!(node.kind == NodeKind::Hidden);
prop_assert!(
node.num_inputs >= min_inputs && node.num_inputs <= min_inputs + span_in,
"num_inputs {} outside [{}, {}]",
node.num_inputs,
min_inputs,
min_inputs + span_in
);
prop_assert!(
node.num_outputs >= min_outputs && node.num_outputs <= min_outputs + span_out,
"num_outputs {} outside [{}, {}]",
node.num_outputs,
min_outputs,
min_outputs + span_out
);
}
}
}