use crate::err::Error;
use crate::err::Result;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Fan {
pub fan_in: usize,
pub fan_out: usize,
}
impl Fan {
pub const fn new(fan_in: usize, fan_out: usize) -> Self {
Self { fan_in, fan_out }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ParameterRole {
Weight,
Bias,
Scale,
Offset,
Other,
}
#[derive(Debug, Clone, Copy)]
pub struct InitContext {
pub role: ParameterRole,
pub fan: Option<Fan>,
}
impl InitContext {
pub const fn new(role: ParameterRole) -> Self {
Self { role, fan: None }
}
pub const fn with_fan(mut self, fan_in: usize, fan_out: usize) -> Self {
self.fan = Some(Fan { fan_in, fan_out });
self
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum InitPlan {
Zeros,
Ones,
Constant(f64),
Uniform {
low: f64,
high: f64,
},
Normal {
mean: f64,
std: f64,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Default)]
pub enum Init {
#[default]
Zeros,
Ones,
Rand,
Randn,
Constant(f64),
Uniform {
bound: f64,
},
KaimingUniform {
a: f64,
},
KaimingNormal {
a: f64,
},
XavierUniform,
XavierNormal,
}
impl Init {
pub fn plan(self, context: InitContext) -> Result<InitPlan> {
match self {
Init::Zeros => Ok(InitPlan::Zeros),
Init::Ones => Ok(InitPlan::Ones),
Init::Rand => Ok(InitPlan::Uniform {
low: 0.0,
high: 1.0,
}),
Init::Randn => Ok(InitPlan::Normal {
mean: 0.0,
std: 1.0,
}),
Init::Constant(val) => Ok(InitPlan::Constant(val)),
Init::Uniform { bound } => Ok(InitPlan::Uniform {
low: -bound,
high: bound,
}),
Init::KaimingUniform { a } => {
let fan = context.fan.ok_or_else(|| Error::ShapeMismatch {
op: "Init::plan",
expected: vec![],
got: vec![],
msg: alloc::string::String::from("KaimingUniform requires fan context"),
})?;
let fan_val = fan.fan_in;
let std = f64::sqrt(2.0 / ((1.0 + a * a) * fan_val as f64));
let bound = f64::sqrt(3.0) * std;
Ok(InitPlan::Uniform {
low: -bound,
high: bound,
})
}
Init::KaimingNormal { a } => {
let fan = context.fan.ok_or_else(|| Error::ShapeMismatch {
op: "Init::plan",
expected: vec![],
got: vec![],
msg: alloc::string::String::from("KaimingNormal requires fan context"),
})?;
let std = f64::sqrt(2.0 / ((1.0 + a * a) * fan.fan_in as f64));
Ok(InitPlan::Normal { mean: 0.0, std })
}
Init::XavierUniform => {
let fan = context.fan.ok_or_else(|| Error::ShapeMismatch {
op: "Init::plan",
expected: vec![],
got: vec![],
msg: alloc::string::String::from("XavierUniform requires fan context"),
})?;
let bound = f64::sqrt(6.0 / (fan.fan_in as f64 + fan.fan_out as f64));
Ok(InitPlan::Uniform {
low: -bound,
high: bound,
})
}
Init::XavierNormal => {
let fan = context.fan.ok_or_else(|| Error::ShapeMismatch {
op: "Init::plan",
expected: vec![],
got: vec![],
msg: alloc::string::String::from("XavierNormal requires fan context"),
})?;
let std = f64::sqrt(2.0 / (fan.fan_in as f64 + fan.fan_out as f64));
Ok(InitPlan::Normal { mean: 0.0, std })
}
}
}
}
pub fn zeros() -> Init {
Init::Zeros
}
pub fn ones() -> Init {
Init::Ones
}
pub fn rand() -> Init {
Init::Rand
}
pub fn randn() -> Init {
Init::Randn
}
pub fn normal() -> Init {
Init::Randn
}
pub fn constant(value: f64) -> Init {
Init::Constant(value)
}
pub fn uniform(bound: f64) -> Init {
Init::Uniform { bound }
}
pub fn kaiming_uniform() -> Init {
Init::KaimingUniform { a: f64::sqrt(5.0) }
}
pub fn kaiming_uniform_with_a(a: f64) -> Init {
Init::KaimingUniform { a }
}
pub fn kaiming_normal() -> Init {
Init::KaimingNormal { a: f64::sqrt(5.0) }
}
pub fn kaiming_normal_with_a(a: f64) -> Init {
Init::KaimingNormal { a }
}
pub fn xavier_uniform() -> Init {
Init::XavierUniform
}
pub fn xavier_normal() -> Init {
Init::XavierNormal
}