use std::collections::{HashMap, HashSet};
use fandhe_ai_autodiff::nn::Module as AutodiffModule;
use fandhe_ai_autodiff::nn::RequiresGradSnapshot;
use crate::{AutodiffError, TapeRef, Tensor, Var};
pub trait Module {
fn forward<'t>(&self, tape: TapeRef<'t>, input: &Var<'t>) -> Result<Var<'t>, AutodiffError>;
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
Vec::new()
}
fn set_parameter(&mut self, name: &str, _value: Tensor<f32>) -> Result<(), AutodiffError> {
Err(AutodiffError::InvalidArgument(format!(
"Module::set_parameter: no parameter named `{name}` (default fail-safe; this \
Module does not override set_parameter)"
)))
}
fn state_dict(&self) -> HashMap<String, Tensor<f32>> {
self.named_parameters()
.into_iter()
.map(|(name, tensor)| (name, tensor.clone()))
.collect()
}
fn load_state_dict(
&mut self,
state: HashMap<String, Tensor<f32>>,
) -> Result<(), AutodiffError> {
AutodiffModule::load_state_dict(&mut FacadeModuleAdapter(self), state)
}
fn set_training(&mut self, _training: bool) {}
fn training(&self) -> bool {
true
}
fn set_requires_grad(&mut self, _requires_grad: bool) -> Result<(), AutodiffError> {
let param_count = self.named_parameters().len();
if param_count == 0 {
Ok(())
} else {
Err(AutodiffError::InvalidArgument(format!(
"Module::set_requires_grad: this Module has {param_count} named parameter(s) \
but does not override set_requires_grad (fail-closed default; freezing would \
silently be a no-op)"
)))
}
}
fn freeze(&mut self) -> Result<(), AutodiffError> {
self.set_requires_grad(false)
}
fn requires_grad(&self) -> bool {
true
}
fn children(&self) -> Vec<(String, &dyn Module)> {
Vec::new()
}
fn named_modules(&self) -> Vec<(String, &dyn Module)> {
let mut out = Vec::new();
let root_key: NodeKey = (self as *const Self as *const (), self.type_name());
let mut ancestors: Vec<NodeKey> = vec![root_key];
let mut visited: HashSet<NodeKey> = HashSet::new();
for (name, child) in self.children() {
collect_named_modules(name, child, &mut ancestors, &mut visited, &mut out);
}
out
}
fn parameter_count(&self) -> usize {
self.named_parameters()
.into_iter()
.fold(0usize, |acc, (_, tensor)| {
acc.saturating_add(tensor.numel())
})
}
fn type_name(&self) -> &'static str {
std::any::type_name::<Self>()
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
Vec::new()
}
}
pub(super) type NodeKey = (*const (), &'static str);
fn collect_named_modules<'a>(
name: String,
child: &'a dyn Module,
ancestors: &mut Vec<NodeKey>,
visited: &mut HashSet<NodeKey>,
out: &mut Vec<(String, &'a dyn Module)>,
) {
let key: NodeKey = (child as *const dyn Module as *const (), child.type_name());
if ancestors.contains(&key) {
return;
}
let is_zst = std::mem::size_of_val(child) == 0;
if !is_zst && !visited.insert(key) {
return;
}
out.push((name.clone(), child));
ancestors.push(key);
for (descendant_name, descendant) in child.children() {
collect_named_modules(
format!("{name}.{descendant_name}"),
descendant,
ancestors,
visited,
out,
);
}
ancestors.pop();
}
pub(crate) struct FacadeModuleAdapter<P>(pub(crate) P);
impl<P> AutodiffModule for FacadeModuleAdapter<P>
where
P: std::ops::DerefMut,
P::Target: Module,
{
fn forward<'t>(
&self,
tape: &'t fandhe_ai_autodiff::Tape,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Module::forward(&*self.0, TapeRef::from_autodiff(tape), input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
Module::named_parameters(&*self.0)
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
Module::set_parameter(&mut *self.0, name, value)
}
fn set_training(&mut self, training: bool) {
Module::set_training(&mut *self.0, training);
}
fn training(&self) -> bool {
Module::training(&*self.0)
}
fn set_requires_grad(&mut self, requires_grad: bool) -> Result<(), AutodiffError> {
Module::set_requires_grad(&mut *self.0, requires_grad)
}
fn requires_grad(&self) -> bool {
Module::requires_grad(&*self.0)
}
fn requires_grad_snapshot(&mut self) -> Result<RequiresGradSnapshot, AutodiffError> {
crate::nn::container::snapshot_requires_grad(&mut *self.0)
.map(|snap| RequiresGradSnapshot::Opaque(Box::new(snap)))
}
fn restore_requires_grad_snapshot(
&mut self,
snapshot: &RequiresGradSnapshot,
) -> Result<(), AutodiffError> {
let inner = match snapshot {
RequiresGradSnapshot::Opaque(any) => {
any.downcast_ref::<crate::nn::container::RequiresGradSnapshot>()
}
_ => None,
};
match inner {
Some(snap) => crate::nn::container::restore_requires_grad(&mut *self.0, snap),
None => Err(AutodiffError::InvalidArgument(
"FacadeModuleAdapter::restore_requires_grad_snapshot: snapshot was not \
produced by a FacadeModuleAdapter (structure changed during rollback)"
.to_string(),
)),
}
}
fn type_name(&self) -> &'static str {
Module::type_name(&*self.0)
}
fn supports_forward_host(&self) -> bool {
false
}
fn forward_host(
&self,
_ops: &dyn fandhe_ai_tensor_core::BackendOps,
_input: &Tensor<f32>,
) -> Result<Tensor<f32>, AutodiffError> {
Err(AutodiffError::InvalidArgument(
"FacadeModuleAdapter::forward_host: facade nn::Module does not support the host \
(tape-free) inference path; supports_forward_host() is false and must be honored"
.to_string(),
))
}
}
#[cfg(test)]
mod tests {
use super::*;
struct TwoParam {
a: Tensor<f32>,
b: Tensor<f32>,
fail_b: bool,
}
impl Module for TwoParam {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
let a = tape.var(&self.a);
input.add(&a)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("a".into(), &self.a), ("b".into(), &self.b)]
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
match name {
"a" => self.a = value,
"b" if self.fail_b => {
return Err(AutodiffError::InvalidArgument("b failed".into()));
}
"b" => self.b = value,
_ => return Err(AutodiffError::InvalidArgument(format!("unknown {name}"))),
}
Ok(())
}
}
struct Stateless;
impl Module for Stateless {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
}
fn t(v: f32) -> Tensor<f32> {
Tensor::from_slice(&[v, v], &[2]).expect("tensor")
}
fn bits(x: &Tensor<f32>) -> Vec<u32> {
x.as_slice()
.expect("host tensor")
.iter()
.map(|v| v.to_bits())
.collect()
}
#[test]
fn bridge_forward_matches_direct_forward() {
let mut layer = TwoParam {
a: t(1.0),
b: t(2.0),
fail_b: false,
};
let ta = crate::tape();
let xa = ta.var(&t(3.0));
let direct = layer.forward(TapeRef::from(&ta), &xa).expect("direct");
let tb = crate::tape();
let xb = tb.var(&t(3.0));
let bridge = FacadeModuleAdapter(&mut layer);
let via = AutodiffModule::forward(&bridge, &tb.0, &xb).expect("bridge");
assert_eq!(bits(&direct.value()), bits(&via.value()));
}
#[test]
fn default_set_parameter_message_matches_autodiff() {
let mine = Stateless
.set_parameter("w", t(0.0))
.expect_err("default is Err")
.to_string();
let mut relu = fandhe_ai_autodiff::nn::activation::Relu;
let theirs = AutodiffModule::set_parameter(&mut relu, "w", t(0.0))
.expect_err("default is Err")
.to_string();
assert_eq!(mine, theirs);
}
#[test]
fn load_state_dict_errors_and_rollback_via_bridge() {
let mut layer = TwoParam {
a: t(1.0),
b: t(2.0),
fail_b: true,
};
let mut good = layer.state_dict();
good.insert("a".into(), t(9.0));
let err = layer.load_state_dict(good).expect_err("b fails");
assert!(err.to_string().contains("b failed"), "{err}");
assert_eq!(bits(&layer.a), bits(&t(1.0)), "a is rolled back");
let mut missing = layer.state_dict();
missing.remove("b");
let e = layer.load_state_dict(missing).expect_err("missing");
assert!(e.to_string().contains("missing keys"), "{e}");
}
use fandhe_ai_autodiff::nn::{Linear, Sequential};
struct Scale {
a: Tensor<f32>,
training: bool,
}
impl Scale {
fn new() -> Self {
Self {
a: Tensor::from_slice(&[0.5, -1.5, 2.0], &[3]).expect("tensor"),
training: true,
}
}
}
impl Module for Scale {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
let a = tape.var(&self.a);
input.mul(&a)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("a".into(), &self.a)]
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
if name != "a" {
return Err(AutodiffError::InvalidArgument(format!("unknown {name}")));
}
self.a = value;
Ok(())
}
fn set_training(&mut self, training: bool) {
self.training = training;
}
fn training(&self) -> bool {
self.training
}
}
fn input() -> Tensor<f32> {
Tensor::from_slice(&[0.1, 0.2, -0.3, 0.4, -0.5, 0.6], &[2, 3]).expect("tensor")
}
fn adapter() -> FacadeModuleAdapter<Box<dyn Module>> {
FacadeModuleAdapter(Box::new(Scale::new()) as Box<dyn Module>)
}
type Snapshot = (Vec<u32>, Option<Vec<u32>>, Vec<Option<Vec<u32>>>, usize);
fn snapshot<'t>(tape: &'t crate::Tape, x: &Var<'t>, out: &Var<'t>) -> Snapshot {
let loss = out.sum(None).expect("sum");
let grads = tape.backward(&loss).expect("backward");
let leaves = (0..tape.leaf_count())
.map(|i| {
let v = tape.leaf(i).expect("leaf");
grads.get(&v).expect("get").map(bits)
})
.collect();
(
bits(&out.value()),
grads.get(x).expect("get").map(bits),
leaves,
tape.0.len(),
)
}
#[test]
fn adapter_in_autodiff_sequential_matches_manual_chain() {
for linear_first in [true, false] {
let linear = Linear::new(3, 3, true, 7).expect("linear");
let reference = Scale::new();
let tape = crate::tape();
let x = tape.var(&input());
let mut h = x;
if linear_first {
h = linear.bind(&tape.0).forward(&h).expect("linear");
h = Module::forward(&reference, TapeRef::from(&tape), &h).expect("scale");
} else {
h = Module::forward(&reference, TapeRef::from(&tape), &h).expect("scale");
h = linear.bind(&tape.0).forward(&h).expect("linear");
}
let expected = snapshot(&tape, &x, &h);
let mut seq = Sequential::new();
let linear2 = Linear::new(3, 3, true, 7).expect("linear");
if linear_first {
seq.push(Box::new(linear2));
seq.push(Box::new(adapter()));
} else {
seq.push(Box::new(adapter()));
seq.push(Box::new(linear2));
}
let tape2 = crate::tape();
let x2 = tape2.var(&input());
let out = AutodiffModule::forward(&seq, &tape2.0, &x2).expect("seq forward");
let actual = snapshot(&tape2, &x2, &out);
assert_eq!(expected, actual, "linear_first={linear_first}");
}
}
#[test]
fn adapter_state_dict_semantics_match_facade_layer() {
let mut ad = adapter();
let direct = Scale::new();
let via = AutodiffModule::state_dict(&ad);
let want = Module::state_dict(&direct);
assert_eq!(via.len(), want.len());
assert_eq!(bits(&via["a"]), bits(&want["a"]));
let mut seq = Sequential::new();
seq.push(Box::new(Linear::new(3, 3, false, 1).expect("linear")));
seq.push(Box::new(adapter()));
let mut sd = AutodiffModule::state_dict(&seq);
let new_a = Tensor::from_slice(&[9.0, 8.0, 7.0], &[3]).expect("tensor");
sd.insert("1.a".into(), new_a.clone());
AutodiffModule::load_state_dict(&mut seq, sd).expect("load");
assert_eq!(bits(&AutodiffModule::state_dict(&seq)["1.a"]), bits(&new_a));
let mut direct_mut = Scale::new();
let e_direct = Module::load_state_dict(&mut direct_mut, HashMap::new())
.expect_err("missing")
.to_string();
let e_adapter = AutodiffModule::load_state_dict(&mut ad, HashMap::new())
.expect_err("missing")
.to_string();
assert_eq!(e_direct, e_adapter);
}
#[test]
fn adapter_propagates_training_flag() {
let mut seq = Sequential::new();
seq.push(Box::new(adapter()));
assert!(AutodiffModule::training(&seq));
AutodiffModule::set_training(&mut seq, false);
assert!(!AutodiffModule::training(&seq));
assert!(!seq.layers()[0].training());
AutodiffModule::set_training(&mut seq, true);
assert!(seq.layers()[0].training());
}
#[test]
fn default_set_requires_grad_message_matches_autodiff() {
struct AdParams(Tensor<f32>);
impl AutodiffModule for AdParams {
fn forward<'t>(
&self,
_tape: &'t fandhe_ai_autodiff::Tape,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.0)]
}
}
struct FacParams(Tensor<f32>);
impl Module for FacParams {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.0)]
}
}
let mine = FacParams(t(1.0))
.set_requires_grad(false)
.expect_err("fail-closed")
.to_string();
let theirs = AutodiffModule::set_requires_grad(&mut AdParams(t(1.0)), false)
.expect_err("fail-closed")
.to_string();
assert_eq!(mine, theirs);
}
#[test]
fn default_requires_grad_for_stateless_layer() {
let mut s = Stateless;
assert!(s.set_requires_grad(false).is_ok());
assert!(s.freeze().is_ok());
assert!(s.requires_grad());
let mut p = TwoParam {
a: t(1.0),
b: t(2.0),
fail_b: false,
};
assert!(matches!(p.freeze(), Err(AutodiffError::InvalidArgument(_))));
}
struct Freezable {
a: Tensor<f32>,
rg: bool,
}
impl Module for Freezable {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
let a = if self.rg {
tape.var(&self.a)
} else {
tape.var_no_grad(&self.a)
};
input.mul(&a)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("a".into(), &self.a)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.rg = v;
Ok(())
}
fn requires_grad(&self) -> bool {
self.rg
}
}
#[test]
fn adapter_delegates_set_requires_grad_and_requires_grad() {
let mut seq = Sequential::new();
seq.push(Box::new(FacadeModuleAdapter(Box::new(Freezable {
a: Tensor::from_slice(&[1.0, 2.0, 3.0], &[3]).expect("tensor"),
rg: true,
}) as Box<dyn Module>)));
assert!(AutodiffModule::requires_grad(&seq));
AutodiffModule::freeze(&mut seq).expect("freeze");
assert!(!AutodiffModule::requires_grad(&seq));
assert!(!seq.layers()[0].requires_grad());
AutodiffModule::set_requires_grad(&mut seq, true).expect("unfreeze");
assert!(AutodiffModule::requires_grad(&seq));
let mut bad = Sequential::new();
bad.push(Box::new(adapter()));
assert!(AutodiffModule::freeze(&mut bad).is_err());
}
#[test]
fn adapter_delegates_type_name_but_not_children() {
let ad = adapter();
assert_eq!(
AutodiffModule::type_name(&ad),
Module::type_name(&Scale::new())
);
assert!(AutodiffModule::children(&ad).is_empty());
assert_eq!(
AutodiffModule::parameter_count(&ad),
Module::parameter_count(&Scale::new())
);
let mut seq = Sequential::new();
seq.push(Box::new(adapter()));
seq.push(Box::new(adapter()));
let names: Vec<String> = AutodiffModule::named_modules(&seq)
.into_iter()
.map(|(n, _)| n)
.collect();
assert_eq!(names, vec!["0".to_string(), "1".to_string()]);
}
#[test]
fn adapter_host_inference_is_fail_closed() {
let ad = adapter();
assert!(!AutodiffModule::supports_forward_host(&ad));
let ops = fandhe_ai_backend_cpu::CpuBackendOps::new();
let e = AutodiffModule::forward_host(&ad, &ops, &input()).expect_err("must be Err");
assert!(matches!(e, AutodiffError::InvalidArgument(_)), "{e}");
let mut seq = Sequential::new();
seq.push(Box::new(adapter()));
let e = AutodiffModule::forward_host(&seq, &ops, &input()).expect_err("must be Err");
assert!(matches!(e, AutodiffError::InvalidArgument(_)), "{e}");
assert!(AutodiffModule::as_linear(&ad).is_none());
assert!(!AutodiffModule::is_pooling(&ad));
}
}