use std::collections::HashSet;
use crate::nn::Module;
use crate::nn::module::NodeKey;
use crate::{AutodiffError, TapeRef, Tensor, Var};
pub(crate) enum RequiresGradSnapshot {
Leaf(bool),
Nested {
own: bool,
children: Vec<(ChildIdentity, RequiresGradSnapshot)>,
},
}
type ChildIdentity = (String, &'static str, *const (), usize);
pub(crate) fn snapshot_requires_grad<M: Module + ?Sized>(
module: &mut M,
) -> Result<RequiresGradSnapshot, AutodiffError> {
let identities: Vec<ChildIdentity> = module
.children()
.into_iter()
.map(|(n, c)| {
(
n,
c.type_name(),
c as *const dyn Module as *const (),
std::mem::size_of_val(c),
)
})
.collect();
let type_name = module.type_name();
let nested = {
let children = module.children_mut();
check_children_consistent(type_name, &identities, &children)?;
children
.into_iter()
.zip(identities)
.map(|((_, child), identity)| Ok((identity, snapshot_requires_grad(child)?)))
.collect::<Result<Vec<_>, AutodiffError>>()?
};
if nested.is_empty() {
Ok(RequiresGradSnapshot::Leaf(module.requires_grad()))
} else {
Ok(RequiresGradSnapshot::Nested {
own: module.requires_grad(),
children: nested,
})
}
}
fn check_children_consistent(
type_name: &str,
expected: &[ChildIdentity],
children_mut: &[(String, &mut dyn Module)],
) -> Result<(), AutodiffError> {
let consistent = expected.len() == children_mut.len()
&& expected
.iter()
.zip(children_mut)
.all(|((n, ty, addr, size), (m, c))| {
n == m
&& *ty == c.type_name()
&& *addr == (&**c as *const dyn Module as *const ())
&& *size == std::mem::size_of_val(&**c)
});
if consistent {
Ok(())
} else {
Err(AutodiffError::InvalidArgument(format!(
"set_requires_grad: `{type_name}` exposes children {:?} (types {:?}) via `children` \
but {:?} (types {:?}) via `children_mut` (a count, name, type, address or size \
mismatch); implement `children_mut` consistently with `children` (same count, \
names, order, types and child references) so that freezing can be rolled back \
per leaf",
expected.iter().map(|(n, _, _, _)| n).collect::<Vec<_>>(),
expected.iter().map(|(_, t, _, _)| t).collect::<Vec<_>>(),
children_mut.iter().map(|(n, _)| n).collect::<Vec<_>>(),
children_mut
.iter()
.map(|(_, c)| c.type_name())
.collect::<Vec<_>>()
)))
}
}
fn child_matches(expected: &ChildIdentity, name: &str, child: &dyn Module) -> bool {
let (exp_name, exp_type, exp_addr, exp_size) = expected;
exp_name == name
&& *exp_type == child.type_name()
&& *exp_addr == (child as *const dyn Module as *const ())
&& *exp_size == std::mem::size_of_val(child)
}
pub(crate) fn restore_requires_grad<M: Module + ?Sized>(
module: &mut M,
snapshot: &RequiresGradSnapshot,
) -> Result<(), AutodiffError> {
match snapshot {
RequiresGradSnapshot::Leaf(value) => {
let now = module.children().len();
if now != 0 {
return Err(AutodiffError::InvalidArgument(format!(
"restore_requires_grad: a leaf at snapshot time now exposes {now} \
child(ren); `set_requires_grad` was not called to avoid propagating to \
unknown children, so the module may be left in a partially applied state"
)));
}
module.set_requires_grad(*value)
}
RequiresGradSnapshot::Nested {
own,
children: expected,
} => {
let mut failures: Vec<String> = Vec::new();
let references_intact = {
let children = module.children_mut();
children.len() == expected.len()
&& children
.iter()
.zip(expected)
.all(|((name, child), (exp, _))| child_matches(exp, name, &**child))
};
if references_intact {
if let Err(e) = module.set_requires_grad(*own) {
failures.push(format!("self ({e})"));
}
} else {
failures.push(
"self (not restored: the children no longer match the snapshot in count, \
name, type, address or size, so `set_requires_grad` was skipped to avoid \
propagating to a different child)"
.to_string(),
);
}
let children = module.children_mut();
if children.len() != expected.len() {
failures.push(format!(
"child count changed (snapshot {}, actual {}); only matching positions \
were restored",
expected.len(),
children.len()
));
}
for (pos, (name, child)) in children.into_iter().enumerate() {
let Some((exp, snap)) = expected.get(pos) else {
break;
};
if !child_matches(exp, &name, &*child) {
failures.push(format!(
"child `{}` (its name, type, address or size no longer matches the \
snapshot [expected type `{}`, found `{}` at `{name}`], so it was not \
restored)",
exp.0,
exp.1,
child.type_name()
));
continue;
}
if let Err(e) = restore_requires_grad(child, snap) {
failures.push(format!("child `{name}` ({e})"));
}
}
if failures.is_empty() {
Ok(())
} else {
Err(AutodiffError::InvalidArgument(format!(
"restore_requires_grad: rollback incomplete, the module may be left in a \
partially applied state: {}",
failures.join(", ")
)))
}
}
}
}
fn set_requires_grad_with_rollback(
owner: &str,
mut children: Vec<&mut dyn Module>,
requires_grad: bool,
) -> Result<(), AutodiffError> {
let previous: Vec<RequiresGradSnapshot> = children
.iter_mut()
.map(|m| snapshot_requires_grad(&mut **m))
.collect::<Result<_, _>>()?;
for index in 0..children.len() {
if let Err(err) = children[index].set_requires_grad(requires_grad) {
let mut failures: Vec<String> = Vec::new();
for i in (0..=index).rev() {
if let Err(e) = restore_requires_grad(&mut *children[i], &previous[i]) {
failures.push(format!("module {i} ({e})"));
}
}
if !failures.is_empty() {
return Err(AutodiffError::InvalidArgument(format!(
"{owner}::set_requires_grad: failed to apply to module {index} \
({err}), and rollback also failed for: {}; the {owner} may now be \
left in a partially applied state",
failures.join(", ")
)));
}
return Err(err);
}
}
Ok(())
}
fn aggregate_requires_grad<'a>(children: impl Iterator<Item = &'a dyn Module>) -> bool {
let mut saw = false;
for m in children {
if m.parameter_count() == 0 {
continue;
}
saw = true;
if m.requires_grad() {
return true;
}
}
!saw
}
pub struct ModuleList {
modules: Vec<Box<dyn Module>>,
training: bool,
}
impl Default for ModuleList {
fn default() -> Self {
Self::new()
}
}
impl ModuleList {
pub fn new() -> Self {
ModuleList {
modules: Vec::new(),
training: true,
}
}
pub fn push(&mut self, module: Box<dyn Module>) {
self.modules.push(module);
}
pub fn len(&self) -> usize {
self.modules.len()
}
pub fn is_empty(&self) -> bool {
self.modules.is_empty()
}
pub fn get(&self, index: usize) -> Option<&dyn Module> {
self.modules.get(index).map(|m| m.as_ref())
}
pub fn get_mut(&mut self, index: usize) -> Option<&mut (dyn Module + 'static)> {
self.modules.get_mut(index).map(|m| m.as_mut())
}
pub fn iter(&self) -> impl Iterator<Item = &Box<dyn Module>> {
self.modules.iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Box<dyn Module>> {
self.modules.iter_mut()
}
}
impl FromIterator<Box<dyn Module>> for ModuleList {
fn from_iter<I: IntoIterator<Item = Box<dyn Module>>>(iter: I) -> Self {
ModuleList {
modules: iter.into_iter().collect(),
training: true,
}
}
}
impl Module for ModuleList {
fn forward<'t>(&self, _tape: TapeRef<'t>, _input: &Var<'t>) -> Result<Var<'t>, AutodiffError> {
Err(AutodiffError::InvalidArgument(
"ModuleList has no forward (nn.ModuleList is a holder, not a callable module)"
.to_string(),
))
}
fn children(&self) -> Vec<(String, &dyn Module)> {
self.modules
.iter()
.enumerate()
.map(|(i, m)| (i.to_string(), m.as_ref()))
.collect()
}
fn set_training(&mut self, training: bool) {
self.training = training;
for module in &mut self.modules {
module.set_training(training);
}
}
fn training(&self) -> bool {
self.training
}
fn set_requires_grad(&mut self, requires_grad: bool) -> Result<(), AutodiffError> {
let children: Vec<&mut dyn Module> = self
.modules
.iter_mut()
.map(|m| -> &mut dyn Module { m.as_mut() })
.collect();
set_requires_grad_with_rollback("ModuleList", children, requires_grad)
}
fn requires_grad(&self) -> bool {
aggregate_requires_grad(self.modules.iter().map(|m| m.as_ref()))
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
self.modules
.iter_mut()
.enumerate()
.map(|(i, m)| -> (String, &mut dyn Module) { (i.to_string(), m.as_mut()) })
.collect()
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
let mut out = Vec::new();
for (index, module) in self.modules.iter().enumerate() {
for (name, tensor) in module.named_parameters() {
out.push((format!("{index}.{name}"), tensor));
}
}
out
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
let (index_str, rest) = name.split_once('.').ok_or_else(|| {
AutodiffError::InvalidArgument(format!(
"ModuleList::set_parameter: no parameter named `{name}` (expected `{{index}}.{{name}}`)"
))
})?;
let index: usize = index_str.parse().map_err(|_| {
AutodiffError::InvalidArgument(format!(
"ModuleList::set_parameter: no parameter named `{name}` (`{index_str}` is not a valid module index)"
))
})?;
let len = self.modules.len();
match self.modules.get_mut(index) {
Some(module) => module.set_parameter(rest, value),
None => Err(AutodiffError::InvalidArgument(format!(
"ModuleList::set_parameter: no parameter named `{name}` (index {index} out of range; ModuleList has {len} modules)"
))),
}
}
}
pub struct Sequential {
inner: ModuleList,
}
impl Default for Sequential {
fn default() -> Self {
Self::new()
}
}
impl Sequential {
pub fn new() -> Self {
Sequential {
inner: ModuleList::new(),
}
}
pub fn push(&mut self, module: Box<dyn Module>) {
self.inner.push(module);
}
#[allow(clippy::should_implement_trait)]
pub fn add<M: Module + 'static>(mut self, module: M) -> Self {
self.inner.push(Box::new(module));
self
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
pub fn layers(&self) -> &[Box<dyn Module>] {
&self.inner.modules
}
pub fn layers_mut(&mut self) -> &mut [Box<dyn Module>] {
&mut self.inner.modules
}
}
impl From<ModuleList> for Sequential {
fn from(inner: ModuleList) -> Self {
Sequential { inner }
}
}
impl Module for Sequential {
fn forward<'t>(&self, tape: TapeRef<'t>, input: &Var<'t>) -> Result<Var<'t>, AutodiffError> {
let mut current = *input;
for layer in &self.inner.modules {
current = layer.forward(tape, ¤t)?;
}
Ok(current)
}
fn children(&self) -> Vec<(String, &dyn Module)> {
self.inner.children()
}
fn set_training(&mut self, training: bool) {
self.inner.set_training(training);
}
fn training(&self) -> bool {
self.inner.training()
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.inner.named_parameters()
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
self.inner.set_parameter(name, value)
}
fn set_requires_grad(&mut self, requires_grad: bool) -> Result<(), AutodiffError> {
self.inner.set_requires_grad(requires_grad)
}
fn requires_grad(&self) -> bool {
self.inner.requires_grad()
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
self.inner.children_mut()
}
}
pub struct ModuleDict {
modules: Vec<(String, Box<dyn Module>)>,
training: bool,
}
impl Default for ModuleDict {
fn default() -> Self {
Self::new()
}
}
fn validate_module_dict_key(key: &str) -> Result<(), AutodiffError> {
if key.is_empty() {
return Err(AutodiffError::InvalidArgument(
"ModuleDict: key must not be empty".to_string(),
));
}
if key.contains('.') {
return Err(AutodiffError::InvalidArgument(format!(
"ModuleDict: key `{key}` must not contain `.` (reserved for named_modules/\
set_parameter path separation)"
)));
}
Ok(())
}
impl ModuleDict {
pub fn new() -> Self {
ModuleDict {
modules: Vec::new(),
training: true,
}
}
pub fn from_pairs(pairs: Vec<(String, Box<dyn Module>)>) -> Result<Self, AutodiffError> {
let mut dict = ModuleDict::new();
for (key, module) in pairs {
dict.insert(key, module)?;
}
Ok(dict)
}
pub fn insert(
&mut self,
key: impl Into<String>,
module: Box<dyn Module>,
) -> Result<Option<Box<dyn Module>>, AutodiffError> {
let key = key.into();
validate_module_dict_key(&key)?;
if let Some(slot) = self.modules.iter_mut().find(|(k, _)| *k == key) {
Ok(Some(std::mem::replace(&mut slot.1, module)))
} else {
self.modules.push((key, module));
Ok(None)
}
}
pub fn remove(&mut self, key: &str) -> Option<Box<dyn Module>> {
let index = self.modules.iter().position(|(k, _)| k == key)?;
Some(self.modules.remove(index).1)
}
pub fn get(&self, key: &str) -> Option<&dyn Module> {
self.modules
.iter()
.find(|(k, _)| k == key)
.map(|(_, m)| m.as_ref())
}
pub fn get_mut(&mut self, key: &str) -> Option<&mut (dyn Module + 'static)> {
self.modules
.iter_mut()
.find(|(k, _)| k == key)
.map(|(_, m)| m.as_mut())
}
pub fn contains_key(&self, key: &str) -> bool {
self.modules.iter().any(|(k, _)| k == key)
}
pub fn keys(&self) -> impl Iterator<Item = &str> {
self.modules.iter().map(|(k, _)| k.as_str())
}
pub fn iter(&self) -> impl Iterator<Item = (&str, &dyn Module)> {
self.modules.iter().map(|(k, m)| (k.as_str(), m.as_ref()))
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = (&str, &mut (dyn Module + 'static))> {
self.modules
.iter_mut()
.map(|(k, m)| (k.as_str(), m.as_mut()))
}
pub fn len(&self) -> usize {
self.modules.len()
}
pub fn is_empty(&self) -> bool {
self.modules.is_empty()
}
}
impl Module for ModuleDict {
fn forward<'t>(&self, _tape: TapeRef<'t>, _input: &Var<'t>) -> Result<Var<'t>, AutodiffError> {
Err(AutodiffError::InvalidArgument(
"ModuleDict has no forward (nn.ModuleDict is a holder, not a callable module)"
.to_string(),
))
}
fn set_training(&mut self, training: bool) {
self.training = training;
for (_, module) in &mut self.modules {
module.set_training(training);
}
}
fn training(&self) -> bool {
self.training
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
let mut out = Vec::new();
for (key, module) in &self.modules {
for (name, tensor) in module.named_parameters() {
out.push((format!("{key}.{name}"), tensor));
}
}
out
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
let (key, rest) = name.split_once('.').ok_or_else(|| {
AutodiffError::InvalidArgument(format!(
"ModuleDict::set_parameter: no parameter named `{name}` (expected `{{key}}.{{name}}`)"
))
})?;
match self.get_mut(key) {
Some(module) => module.set_parameter(rest, value),
None => Err(AutodiffError::InvalidArgument(format!(
"ModuleDict::set_parameter: no parameter named `{name}` (no child module with key `{key}`)"
))),
}
}
fn children(&self) -> Vec<(String, &dyn Module)> {
self.modules
.iter()
.map(|(key, module)| (key.clone(), module.as_ref()))
.collect()
}
fn set_requires_grad(&mut self, requires_grad: bool) -> Result<(), AutodiffError> {
let children: Vec<&mut dyn Module> = self
.modules
.iter_mut()
.map(|(_, m)| -> &mut dyn Module { m.as_mut() })
.collect();
set_requires_grad_with_rollback("ModuleDict", children, requires_grad)
}
fn requires_grad(&self) -> bool {
aggregate_requires_grad(self.modules.iter().map(|(_, m)| m.as_ref()))
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
self.modules
.iter_mut()
.map(|(key, module)| -> (String, &mut dyn Module) { (key.clone(), module.as_mut()) })
.collect()
}
}
fn short_type_name(full: &str) -> &str {
let without_generics = full.split('<').next().unwrap_or(full);
without_generics
.rsplit("::")
.next()
.unwrap_or(without_generics)
}
pub fn summary(module: &dyn Module) -> String {
let mut out = String::new();
let mut ancestors: Vec<NodeKey> =
vec![(module as *const dyn Module as *const (), module.type_name())];
let mut visited: HashSet<NodeKey> = HashSet::new();
write_module(&mut out, None, module, 0, &mut ancestors, &mut visited);
out.push_str(&format!("Submodules: {}\n", module.named_modules().len()));
out.push_str(&format!("Total parameters: {}\n", module.parameter_count()));
out
}
fn write_module(
out: &mut String,
name: Option<&str>,
module: &dyn Module,
depth: usize,
ancestors: &mut Vec<NodeKey>,
visited: &mut HashSet<NodeKey>,
) {
let indent = " ".repeat(depth);
let type_name = short_type_name(module.type_name());
let children = module.children();
if children.is_empty() {
match name {
Some(name) => out.push_str(&format!(
"{indent}({name}): {type_name} [params: {}]\n",
module.parameter_count()
)),
None => out.push_str(&format!(
"{type_name} [params: {}]\n",
module.parameter_count()
)),
}
return;
}
match name {
Some(name) => out.push_str(&format!("{indent}({name}): {type_name}(\n")),
None => out.push_str(&format!("{type_name}(\n")),
}
for (child_name, child) in &children {
let key: NodeKey = (*child as *const dyn Module as *const (), child.type_name());
if ancestors.contains(&key) {
continue;
}
let is_zst = std::mem::size_of_val(*child) == 0;
if !is_zst && !visited.insert(key) {
continue;
}
ancestors.push(key);
write_module(out, Some(child_name), *child, depth + 1, ancestors, visited);
ancestors.pop();
}
out.push_str(&format!(
"{indent}) [params: {}]\n",
module.parameter_count()
));
}
#[cfg(test)]
mod tests {
use super::*;
use fandhe_ai_autodiff::nn::Module as AutodiffModule;
struct BuiltinLayer<L: AutodiffModule>(L);
impl<L: AutodiffModule> Module for BuiltinLayer<L> {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
self.0.forward(tape.0, input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.0.named_parameters()
}
fn set_parameter(&mut self, name: &str, value: Tensor<f32>) -> Result<(), AutodiffError> {
self.0.set_parameter(name, value)
}
fn type_name(&self) -> &'static str {
self.0.type_name()
}
}
fn linear() -> fandhe_ai_autodiff::nn::Linear {
fandhe_ai_autodiff::nn::Linear::new(3, 4, true, 7).expect("linear")
}
fn pair() -> (ModuleList, fandhe_ai_autodiff::nn::ModuleList) {
let mut mine = ModuleList::new();
mine.push(Box::new(BuiltinLayer(linear())));
let mut theirs = fandhe_ai_autodiff::nn::ModuleList::new();
theirs.push(Box::new(linear()));
(mine, theirs)
}
#[test]
fn names_and_errors_match_autodiff_container() {
let (mut mine, mut theirs) = pair();
let n1: Vec<String> = mine
.named_parameters()
.into_iter()
.map(|(n, _)| n)
.collect();
let n2: Vec<String> = theirs
.named_parameters()
.into_iter()
.map(|(n, _)| n)
.collect();
assert_eq!(n1, n2);
let t = crate::tape();
let x = t.var(&Tensor::from_slice(&[0.0f32; 3], &[1, 3]).expect("x"));
let e1 = mine
.forward(TapeRef::from(&t), &x)
.expect_err("Err")
.to_string();
let e2 = theirs.forward(&t.0, &x).expect_err("Err").to_string();
assert_eq!(e1, e2);
let v = Tensor::from_slice(&[0.0f32], &[1]).expect("v");
for key in ["weight", "x.weight", "9.weight", "0.nope"] {
let a = mine
.set_parameter(key, v.clone())
.expect_err("Err")
.to_string();
let b = theirs
.set_parameter(key, v.clone())
.expect_err("Err")
.to_string();
assert_eq!(a, b, "key={key}");
}
}
use std::cell::RefCell;
use std::rc::Rc;
type Calls = Rc<RefCell<Vec<bool>>>;
struct Fz {
p: Tensor<f32>,
rg: bool,
fail_apply: bool,
fail_restore: bool,
}
impl Fz {
fn new() -> Self {
Fz {
p: Tensor::from_slice(&[1.0f32, 2.0], &[2]).expect("p"),
rg: true,
fail_apply: false,
fail_restore: false,
}
}
}
impl Module for Fz {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.p)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
if (!v && self.fail_apply) || (v && self.fail_restore) {
self.rg = false;
return Err(AutodiffError::InvalidArgument("child failed".into()));
}
self.rg = v;
Ok(())
}
fn requires_grad(&self) -> bool {
self.rg
}
}
struct Ident;
impl Module for Ident {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
}
fn list_of(children: Vec<Box<dyn Module>>) -> ModuleList {
children.into_iter().collect()
}
#[test]
fn freeze_propagates_and_aggregates() {
let mut l = list_of(vec![
Box::new(Fz::new()),
Box::new(Ident),
Box::new(Fz::new()),
]);
assert!(l.requires_grad());
l.freeze().expect("freeze");
assert!(!l.requires_grad(), "無状態の子は集約から除外");
assert!(!l.get(0).expect("0").requires_grad());
assert!(!l.get(2).expect("2").requires_grad());
l.get_mut(0)
.expect("0")
.set_requires_grad(true)
.expect("ok");
assert!(l.requires_grad());
assert!(ModuleList::new().requires_grad());
assert!(list_of(vec![Box::new(Ident)]).requires_grad());
}
#[test]
fn rollback_restores_failed_child_itself() {
let mut failing = Fz::new();
failing.fail_apply = true;
let mut l = list_of(vec![Box::new(Fz::new()), Box::new(failing)]);
let e = l.freeze().expect_err("2 番目が失敗");
assert!(e.to_string().contains("child failed"), "{e}");
assert!(l.get(0).expect("0").requires_grad());
assert!(l.get(1).expect("1").requires_grad(), "失敗した子自身も戻る");
}
#[test]
fn rollback_calls_setter_even_when_trait_getter_looks_unchanged() {
struct Sticky(Calls, Tensor<f32>, bool);
impl Module for Sticky {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.1)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.0.borrow_mut().push(v);
if self.2 && !v {
return Err(AutodiffError::InvalidArgument("fail".into()));
}
Ok(())
}
}
let c: Calls = Rc::new(RefCell::new(Vec::new()));
let p = Tensor::from_slice(&[1.0f32], &[1]).expect("p");
let mut l = list_of(vec![
Box::new(Sticky(c.clone(), p.clone(), false)),
Box::new(Sticky(c.clone(), p, true)),
]);
l.freeze().expect_err("fail");
assert_eq!(*c.borrow(), vec![false, false, true, true]);
}
#[test]
fn rollback_failures_are_aggregated_and_continue() {
let mut a = Fz::new();
a.fail_restore = true;
let mut b = Fz::new();
b.fail_apply = true;
let mut l = list_of(vec![Box::new(a), Box::new(Fz::new()), Box::new(b)]);
let e = l.freeze().expect_err("3 番目が失敗").to_string();
assert!(e.contains("rollback also failed for"), "{e}");
assert!(e.contains("partially applied state"), "{e}");
assert!(e.contains("module 0"), "{e}");
assert!(
l.get(1).expect("1").requires_grad(),
"他の子の復元は続行される"
);
}
#[test]
fn sequential_delegates_and_nested_uniform_rollback() {
let mut s = Sequential::new();
s.push(Box::new(Fz::new()));
s.freeze().expect("freeze");
assert!(!s.requires_grad());
let mut failing = Fz::new();
failing.fail_apply = true;
let inner = list_of(vec![Box::new(Fz::new()), Box::new(Fz::new())]);
let mut outer = Sequential::new();
outer.push(Box::new(inner));
outer.push(Box::new(failing));
outer.freeze().expect_err("後続が失敗");
assert!(outer.requires_grad());
assert!(outer.layers()[0].requires_grad());
}
fn leaf_states(m: &dyn Module) -> Vec<bool> {
let children = m.children();
if children.is_empty() {
vec![m.requires_grad()]
} else {
children
.into_iter()
.flat_map(|(_, c)| leaf_states(c))
.collect()
}
}
fn fz(rg: bool) -> Fz {
let mut f = Fz::new();
f.rg = rg;
f
}
fn failing_fz() -> Fz {
let mut f = Fz::new();
f.fail_apply = true;
f
}
#[test]
fn nested_module_list_mixed_state_restored_per_leaf_on_failure() {
let inner = list_of(vec![Box::new(fz(false)), Box::new(fz(true))]);
let mut outer = list_of(vec![Box::new(inner), Box::new(failing_fz())]);
let before = leaf_states(&outer);
assert_eq!(before, vec![false, true, true]);
outer.freeze().expect_err("後続が失敗");
assert_eq!(leaf_states(&outer), before);
}
#[test]
fn nested_sequential_mixed_state_restored_per_leaf_on_failure() {
let mut deep = Sequential::new();
deep.push(Box::new(fz(true)));
deep.push(Box::new(fz(false)));
let inner = list_of(vec![Box::new(fz(false)), Box::new(deep)]);
let mut outer = Sequential::new();
outer.push(Box::new(inner));
outer.push(Box::new(failing_fz()));
let before = leaf_states(&outer);
assert_eq!(before, vec![false, true, false, true]);
outer.freeze().expect_err("後続が失敗");
assert_eq!(leaf_states(&outer), before);
}
#[test]
fn nested_module_dict_mixed_state_restored_per_leaf_on_failure() {
let mut inner = ModuleDict::new();
inner.insert("a", Box::new(fz(false))).expect("k");
inner.insert("b", Box::new(fz(true))).expect("k");
let mut outer = ModuleDict::new();
outer.insert("in", Box::new(inner)).expect("k");
outer.insert("bad", Box::new(failing_fz())).expect("k");
let before = leaf_states(&outer);
assert_eq!(before, vec![false, true, true]);
outer.freeze().expect_err("後続が失敗");
assert_eq!(leaf_states(&outer), before);
}
#[test]
fn module_dict_freeze_propagates_and_aggregates() {
let mut d = ModuleDict::new();
d.insert("a", Box::new(Fz::new())).expect("k");
d.insert("n", Box::new(Ident)).expect("k");
d.insert("l", Box::new(list_of(vec![Box::new(fz(true))])))
.expect("k");
assert!(d.requires_grad());
d.freeze().expect("freeze");
assert_eq!(leaf_states(&d), vec![false, true, false]);
assert!(!d.requires_grad());
d.set_requires_grad(true).expect("unfreeze");
assert!(d.requires_grad());
assert!(ModuleDict::new().requires_grad());
}
fn adapter_leaf_states(m: &mut dyn fandhe_ai_autodiff::nn::Module) -> Vec<bool> {
fn flat(s: &super::RequiresGradSnapshot, out: &mut Vec<bool>) {
match s {
super::RequiresGradSnapshot::Leaf(v) => out.push(*v),
super::RequiresGradSnapshot::Nested { children: c, .. } => {
c.iter().for_each(|(_, x)| flat(x, out))
}
}
}
let snap = m.requires_grad_snapshot().expect("snapshot");
let fandhe_ai_autodiff::nn::RequiresGradSnapshot::Opaque(any) = snap else {
panic!("アダプタは Opaque を返すはず");
};
let inner = any
.downcast_ref::<super::RequiresGradSnapshot>()
.expect("facade スナップショット");
let mut out = Vec::new();
flat(inner, &mut out);
out
}
fn adapted(
m: impl Module + 'static,
) -> crate::nn::module::FacadeModuleAdapter<Box<dyn Module>> {
crate::nn::module::FacadeModuleAdapter(Box::new(m) as Box<dyn Module>)
}
fn mixed_facade_container() -> ModuleList {
let inner = list_of(vec![Box::new(fz(false)), Box::new(fz(true))]);
list_of(vec![Box::new(fz(false)), Box::new(inner)])
}
#[test]
fn adapter_in_autodiff_module_list_restores_mixed_facade_container_per_leaf() {
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(mixed_facade_container())));
outer.push(Box::new(adapted(failing_fz())));
let before = adapter_leaf_states(outer.get_mut(0).expect("0"));
assert_eq!(before, vec![false, false, true]);
fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false).expect_err("失敗");
assert_eq!(adapter_leaf_states(outer.get_mut(0).expect("0")), before);
}
#[test]
fn adapter_in_autodiff_sequential_restores_mixed_facade_container_per_leaf() {
let mut inner = fandhe_ai_autodiff::nn::ModuleList::new();
inner.push(Box::new(adapted(mixed_facade_container())));
let mut outer = fandhe_ai_autodiff::nn::Sequential::new();
outer.push(Box::new(inner));
outer.push(Box::new(adapted(failing_fz())));
let list = outer.layers_mut()[0]
.as_module_list_mut()
.expect("ModuleList");
let before = adapter_leaf_states(list.get_mut(0).expect("0"));
fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false).expect_err("失敗");
let list = outer.layers_mut()[0]
.as_module_list_mut()
.expect("ModuleList");
assert_eq!(adapter_leaf_states(list.get_mut(0).expect("0")), before);
}
#[test]
fn adapter_in_autodiff_module_dict_restores_mixed_facade_container_per_leaf() {
let mut outer = fandhe_ai_autodiff::nn::ModuleDict::new();
outer
.insert("m", Box::new(adapted(mixed_facade_container())))
.expect("k");
outer
.insert("bad", Box::new(adapted(failing_fz())))
.expect("k");
let before = adapter_leaf_states(outer.get_mut("m").expect("m"));
fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false).expect_err("失敗");
assert_eq!(adapter_leaf_states(outer.get_mut("m").expect("m")), before);
}
struct SelfFlagComp {
own: Rc<std::cell::Cell<bool>>,
a: Fz,
b: Fz,
}
impl Module for SelfFlagComp {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("a.p".into(), &self.a.p), ("b.p".into(), &self.b.p)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.own.set(v);
self.a.set_requires_grad(v)?;
self.b.set_requires_grad(v)
}
fn requires_grad(&self) -> bool {
self.own.get()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
vec![("a".into(), &self.a), ("b".into(), &self.b)]
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
vec![("a".into(), &mut self.a), ("b".into(), &mut self.b)]
}
}
#[test]
fn adapter_restores_composite_own_flag_after_later_failure() {
let own = Rc::new(std::cell::Cell::new(true));
let mut a = Fz::new();
a.rg = false;
let comp = SelfFlagComp {
own: Rc::clone(&own),
a,
b: Fz::new(),
};
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(comp)));
outer.push(Box::new(adapted(failing_fz())));
let before = adapter_leaf_states(outer.get_mut(0).expect("0"));
assert_eq!(before, vec![false, true]);
fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false).expect_err("失敗");
assert!(own.get(), "自身のフラグが呼び出し前へ戻る");
assert_eq!(adapter_leaf_states(outer.get_mut(0).expect("0")), before);
}
#[test]
fn facade_list_restores_composite_own_flag_after_failure() {
let own = Rc::new(std::cell::Cell::new(true));
let mut a = Fz::new();
a.rg = false;
let mut bad = Fz::new();
bad.fail_apply = true;
let comp = SelfFlagComp {
own: Rc::clone(&own),
a,
b: bad,
};
let mut l = list_of(vec![Box::new(comp)]);
l.freeze().expect_err("複合層内部の失敗");
assert!(own.get(), "自身のフラグが呼び出し前へ戻る");
assert_eq!(leaf_states_of(&l), vec![false, true]);
}
fn leaf_states_of(m: &dyn Module) -> Vec<bool> {
let c = m.children();
if c.is_empty() {
vec![m.requires_grad()]
} else {
c.into_iter().flat_map(|(_, x)| leaf_states_of(x)).collect()
}
}
struct Comp {
a: Fz,
b: Fz,
}
impl Module for Comp {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("a.p".into(), &self.a.p), ("b.p".into(), &self.b.p)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.a.set_requires_grad(v)?;
self.b.set_requires_grad(v)
}
fn requires_grad(&self) -> bool {
self.a.requires_grad() || self.b.requires_grad()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
vec![("a".into(), &self.a), ("b".into(), &self.b)]
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
vec![("a".into(), &mut self.a), ("b".into(), &mut self.b)]
}
}
struct ChildrenOnly(Comp);
impl Module for ChildrenOnly {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
self.0.forward(tape, input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.0.named_parameters()
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.0.set_requires_grad(v)
}
fn requires_grad(&self) -> bool {
self.0.requires_grad()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
self.0.children()
}
}
fn mixed_comp() -> Comp {
Comp {
a: fz(false),
b: fz(true),
}
}
#[test]
fn adapter_in_autodiff_module_list_restores_user_composite_per_leaf() {
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(mixed_comp())));
outer.push(Box::new(adapted(failing_fz())));
let before = adapter_leaf_states(outer.get_mut(0).expect("0"));
assert_eq!(before, vec![false, true]);
fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false).expect_err("失敗");
assert_eq!(adapter_leaf_states(outer.get_mut(0).expect("0")), before);
}
#[test]
fn adapter_in_autodiff_module_list_rejects_children_only_composite_before_mutation() {
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(fz(true))));
outer.push(Box::new(adapted(ChildrenOnly(mixed_comp()))));
let e = fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false)
.expect_err("事前検査で拒否");
assert!(matches!(e, AutodiffError::InvalidArgument(_)), "{e}");
assert!(e.to_string().contains("children_mut"), "{e}");
assert_eq!(
adapter_leaf_states(outer.get_mut(0).expect("0")),
vec![true]
);
}
#[test]
fn adapter_in_autodiff_module_list_detects_child_swapped_during_freeze() {
use std::cell::Cell;
use std::rc::Rc;
struct Sl {
p: Tensor<f32>,
rg: Rc<Cell<bool>>,
}
impl Module for Sl {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.p)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.rg.set(v);
Ok(())
}
fn requires_grad(&self) -> bool {
self.rg.get()
}
}
struct Sw {
a: Sl,
b: Sl,
switched: bool,
}
impl Module for Sw {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.a.named_parameters()
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.switched = true;
if !v {
self.b.set_requires_grad(false)?;
self.a.set_requires_grad(false)?;
}
Ok(())
}
fn requires_grad(&self) -> bool {
self.a.requires_grad() || self.b.requires_grad()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
let c = if self.switched { &self.b } else { &self.a };
vec![("x".into(), c)]
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
let c = if self.switched {
&mut self.b
} else {
&mut self.a
};
vec![("x".into(), c)]
}
}
let mk = |v: bool| Sl {
p: Tensor::from_slice(&[1.0f32, 2.0], &[2]).expect("p"),
rg: Rc::new(Cell::new(v)),
};
let b = mk(false);
let b_rg = Rc::clone(&b.rg);
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(Sw {
a: mk(true),
b,
switched: false,
})));
outer.push(Box::new(adapted(failing_fz())));
let e = fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false)
.expect_err("後続子が失敗しロールバックも不完全");
let msg = e.to_string();
assert!(msg.contains("partially applied"), "{msg}");
assert!(msg.contains("child failed"), "元エラーも含む: {msg}");
assert!(!b_rg.get(), "別の子 b へスナップショットの値を適用しない");
}
#[test]
fn adapter_in_autodiff_module_list_skips_own_setter_when_child_swapped() {
use std::cell::Cell;
use std::rc::Rc;
struct Sl {
p: Tensor<f32>,
rg: Rc<Cell<bool>>,
}
impl Module for Sl {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
vec![("p".into(), &self.p)]
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.rg.set(v);
Ok(())
}
fn requires_grad(&self) -> bool {
self.rg.get()
}
}
struct Sw {
a: Sl,
b: Sl,
switched: bool,
setter_calls: Rc<Cell<usize>>,
}
impl Module for Sw {
fn forward<'t>(
&self,
_tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
Ok(*input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.a.named_parameters()
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.setter_calls.set(self.setter_calls.get() + 1);
self.switched = true;
self.a.set_requires_grad(v)?;
self.b.set_requires_grad(v)
}
fn requires_grad(&self) -> bool {
self.a.requires_grad() || self.b.requires_grad()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
let c = if self.switched { &self.b } else { &self.a };
vec![("x".into(), c)]
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
let c = if self.switched {
&mut self.b
} else {
&mut self.a
};
vec![("x".into(), c)]
}
}
let mk = |v: bool| Sl {
p: Tensor::from_slice(&[1.0f32, 2.0], &[2]).expect("p"),
rg: Rc::new(Cell::new(v)),
};
let b = mk(true);
let b_rg = Rc::clone(&b.rg);
let setter_calls = Rc::new(Cell::new(0usize));
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(Sw {
a: mk(true),
b,
switched: false,
setter_calls: Rc::clone(&setter_calls),
})));
outer.push(Box::new(adapted(failing_fz())));
let e = fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false)
.expect_err("後続子が失敗しロールバックも不完全");
let msg = e.to_string();
assert!(msg.contains("partially applied"), "{msg}");
assert!(msg.contains("child failed"), "元エラーも含む: {msg}");
assert_eq!(setter_calls.get(), 1, "復元では自身の setter を呼ばない");
assert!(!b_rg.get(), "b は復元値 true で書き換えられない");
}
#[test]
fn adapter_in_autodiff_module_list_rejects_swapped_children_mut_before_mutation() {
struct Swapped(Comp);
impl Module for Swapped {
fn forward<'t>(
&self,
tape: TapeRef<'t>,
input: &Var<'t>,
) -> Result<Var<'t>, AutodiffError> {
self.0.forward(tape, input)
}
fn named_parameters(&self) -> Vec<(String, &Tensor<f32>)> {
self.0.named_parameters()
}
fn set_requires_grad(&mut self, v: bool) -> Result<(), AutodiffError> {
self.0.set_requires_grad(v)
}
fn requires_grad(&self) -> bool {
self.0.requires_grad()
}
fn children(&self) -> Vec<(String, &dyn Module)> {
vec![("x".into(), &self.0.a)]
}
fn children_mut(&mut self) -> Vec<(String, &mut dyn Module)> {
vec![("x".into(), &mut self.0.b)]
}
}
let mut outer = fandhe_ai_autodiff::nn::ModuleList::new();
outer.push(Box::new(adapted(fz(true))));
outer.push(Box::new(adapted(Swapped(mixed_comp()))));
let e = fandhe_ai_autodiff::nn::Module::set_requires_grad(&mut outer, false)
.expect_err("事前検査で拒否");
assert!(matches!(e, AutodiffError::InvalidArgument(_)), "{e}");
assert!(e.to_string().contains("children_mut"), "{e}");
assert_eq!(
adapter_leaf_states(outer.get_mut(0).expect("0")),
vec![true]
);
}
#[test]
fn adapter_restore_rejects_foreign_snapshot() {
use fandhe_ai_autodiff::nn::{Module as Ad, RequiresGradSnapshot as Snap};
let mut a = adapted(fz(true));
assert!(a.restore_requires_grad_snapshot(&Snap::Leaf(true)).is_err());
let foreign = Snap::Opaque(Box::new(7u8));
let e = a
.restore_requires_grad_snapshot(&foreign)
.expect_err("型違い");
assert!(e.to_string().contains("not produced by"), "{e}");
let _ = Ad::requires_grad(&a);
}
#[test]
fn restore_shape_mismatch_is_fail_closed() {
let mut l = list_of(vec![Box::new(fz(true))]);
let snap = RequiresGradSnapshot::Nested {
own: true,
children: vec![
(
("0".to_string(), "", std::ptr::null(), 0),
RequiresGradSnapshot::Leaf(true),
),
(
("1".to_string(), "", std::ptr::null(), 0),
RequiresGradSnapshot::Leaf(true),
),
],
};
let e = restore_requires_grad(&mut l, &snap).expect_err("不一致");
assert!(e.to_string().contains("child count changed"), "{e}");
let mut leaf = fz(true);
let e = restore_requires_grad(&mut leaf, &snap).expect_err("葉");
assert!(e.to_string().contains("child count changed"), "{e}");
}
fn relu_box() -> Box<dyn Module> {
Box::new(BuiltinLayer(fandhe_ai_autodiff::nn::activation::Relu))
}
fn dict_pair() -> (ModuleDict, fandhe_ai_autodiff::nn::ModuleDict) {
let mut mine = ModuleDict::new();
mine.insert("enc", Box::new(BuiltinLayer(linear())))
.expect("k");
mine.insert("act", relu_box()).expect("k");
let mut theirs = fandhe_ai_autodiff::nn::ModuleDict::new();
theirs.insert("enc", Box::new(linear())).expect("k");
theirs
.insert("act", Box::new(fandhe_ai_autodiff::nn::activation::Relu))
.expect("k");
(mine, theirs)
}
#[test]
fn module_dict_matches_autodiff_module_dict() {
let (mut mine, mut theirs) = dict_pair();
let names = |v: Vec<(String, &Tensor<f32>)>| -> Vec<String> {
v.into_iter().map(|(n, _)| n).collect()
};
assert_eq!(
names(mine.named_parameters()),
names(theirs.named_parameters())
);
let t = crate::tape();
let x = t.var(&Tensor::from_slice(&[0.0f32; 3], &[1, 3]).expect("x"));
let e1 = mine
.forward(TapeRef::from(&t), &x)
.expect_err("Err")
.to_string();
let e2 = theirs.forward(&t.0, &x).expect_err("Err").to_string();
assert_eq!(e1, e2);
let v = Tensor::from_slice(&[0.0f32], &[1]).expect("v");
for key in ["weight", "x.weight", "enc.nope"] {
let a = mine
.set_parameter(key, v.clone())
.expect_err("Err")
.to_string();
let b = theirs
.set_parameter(key, v.clone())
.expect_err("Err")
.to_string();
assert_eq!(a, b, "key={key}");
}
for key in ["", "a.b"] {
let a = mine.insert(key, relu_box()).err().expect("Err").to_string();
let b = theirs
.insert(key, Box::new(fandhe_ai_autodiff::nn::activation::Relu))
.err()
.expect("Err")
.to_string();
assert_eq!(a, b, "key={key:?}");
}
}
#[test]
fn summary_matches_autodiff_summary() {
let (mine, theirs) = dict_pair();
assert_eq!(summary(&mine), fandhe_ai_autodiff::nn::summary(&theirs));
let mut seq = Sequential::new();
seq.push(Box::new(BuiltinLayer(linear())));
seq.push(relu_box());
seq.push(Box::new(mine));
let mut aseq = fandhe_ai_autodiff::nn::Sequential::new();
aseq.push(Box::new(linear()));
aseq.push(Box::new(fandhe_ai_autodiff::nn::activation::Relu));
aseq.push(Box::new(theirs));
assert_eq!(summary(&seq), fandhe_ai_autodiff::nn::summary(&aseq));
}
}