use crate::tensor::Tensor;
use mathverse_core::error::MathResult;
pub fn relu(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| x.max(0.0)).collect() }
}
pub fn relu_grad(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| if x > 0.0 { 1.0 } else { 0.0 }).collect() }
}
pub fn leaky_relu(t: &Tensor, slope: f64) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| if x > 0.0 { x } else { slope * x }).collect() }
}
pub fn leaky_relu_grad(t: &Tensor, slope: f64) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| if x > 0.0 { 1.0 } else { slope }).collect() }
}
pub fn prelu(t: &Tensor, alpha: &Tensor) -> MathResult<Tensor> {
let target = crate::tensor::broadcast_shapes(&t.shape, &alpha.shape)?;
let a = t.broadcast_to(&target)?;
let b = alpha.broadcast_to(&target)?;
let data: Vec<f64> = a.data.iter().zip(&b.data)
.map(|(&x, &a)| if x > 0.0 { x } else { a * x })
.collect();
Ok(Tensor { shape: target, data })
}
pub fn elu(t: &Tensor, alpha: f64) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter()
.map(|&x| if x > 0.0 { x } else { alpha * (x.exp() - 1.0) })
.collect() }
}
pub fn selu(t: &Tensor) -> Tensor {
let lambda = 1.0507009873554805;
let alpha = 1.6732632423543772;
Tensor { shape: t.shape.clone(), data: t.data.iter()
.map(|&x| if x > 0.0 { lambda * x } else { lambda * alpha * (x.exp() - 1.0) })
.collect() }
}
pub fn gelu(t: &Tensor) -> Tensor {
let c = (2.0 / std::f64::consts::PI).sqrt();
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| {
let inner = c * (x + 0.044715 * x.powi(3));
0.5 * x * (1.0 + inner.tanh())
}).collect() }
}
pub fn gelu_grad(t: &Tensor) -> Tensor {
let c = (2.0 / std::f64::consts::PI).sqrt();
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| {
let inner = c * (x + 0.044715 * x.powi(3));
let tanh_val = inner.tanh();
let sech2 = 1.0 - tanh_val * tanh_val;
let d_inner = c * (1.0 + 3.0 * 0.044715 * x * x);
0.5 * (1.0 + tanh_val) + 0.5 * x * sech2 * d_inner
}).collect() }
}
pub fn sigmoid(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| {
if x >= 0.0 { 1.0 / (1.0 + (-x).exp()) }
else { let e = x.exp(); e / (1.0 + e) }
}).collect() }
}
pub fn sigmoid_grad(t: &Tensor) -> Tensor {
let s = sigmoid(t);
Tensor { shape: t.shape.clone(), data: s.data.iter().map(|&v| v * (1.0 - v)).collect() }
}
pub fn tanh(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|x| x.tanh()).collect() }
}
pub fn tanh_grad(t: &Tensor) -> Tensor {
let h = tanh(t);
Tensor { shape: h.shape.clone(), data: h.data.iter().map(|&v| 1.0 - v * v).collect() }
}
pub fn swish(t: &Tensor) -> Tensor {
let s = sigmoid(t);
Tensor { shape: t.shape.clone(), data: t.data.iter().zip(&s.data).map(|(&x, &s)| x * s).collect() }
}
pub fn mish(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| {
let sp = (1.0 + x.exp()).ln();
x * sp.tanh()
}).collect() }
}
pub fn softplus(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| {
if x > 20.0 { x } else if x < -20.0 { 0.0 } else { (1.0 + x.exp()).ln() }
}).collect() }
}
pub fn softsign(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| x / (1.0 + x.abs())).collect() }
}
pub fn hard_sigmoid(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|&x| (x / 6.0 + 0.5).clamp(0.0, 1.0)).collect() }
}
pub fn hard_tanh(t: &Tensor) -> Tensor {
Tensor { shape: t.shape.clone(), data: t.data.iter().map(|x| x.clamp(-1.0, 1.0)).collect() }
}
pub fn identity(t: &Tensor) -> Tensor { t.clone() }
pub fn softmax(t: &Tensor, axis: usize) -> MathResult<Tensor> {
if axis >= t.shape.len() {
return Err(mathverse_core::error::MathError::InvalidArgument("axis out of range"));
}
let axis_size = t.shape[axis];
let outer: usize = t.shape[..axis].iter().product();
let inner: usize = t.shape[axis + 1..].iter().product();
let mut out = vec![0.0; t.numel()];
for i in 0..outer {
for j in 0..inner {
let mut max_val = f64::NEG_INFINITY;
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
if t.data[idx] > max_val { max_val = t.data[idx]; }
}
let mut sum = 0.0;
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
let e = (t.data[idx] - max_val).exp();
out[idx] = e;
sum += e;
}
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
out[idx] /= sum;
}
}
}
Ok(Tensor { shape: t.shape.clone(), data: out })
}
pub fn log_softmax(t: &Tensor, axis: usize) -> MathResult<Tensor> {
if axis >= t.shape.len() {
return Err(mathverse_core::error::MathError::InvalidArgument("axis out of range"));
}
let axis_size = t.shape[axis];
let outer: usize = t.shape[..axis].iter().product();
let inner: usize = t.shape[axis + 1..].iter().product();
let mut out = vec![0.0; t.numel()];
for i in 0..outer {
for j in 0..inner {
let mut max_val = f64::NEG_INFINITY;
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
if t.data[idx] > max_val { max_val = t.data[idx]; }
}
let mut sum_exp = 0.0;
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
sum_exp += (t.data[idx] - max_val).exp();
}
let log_sum = sum_exp.ln();
for k in 0..axis_size {
let idx = i * axis_size * inner + k * inner + j;
out[idx] = t.data[idx] - max_val - log_sum;
}
}
}
Ok(Tensor { shape: t.shape.clone(), data: out })
}
#[cfg(test)]
mod tests {
use super::*;
const E: f64 = 1e-6;
#[test]
fn relu_test() {
let t = Tensor::new(&[5], &[-2.0, -1.0, 0.0, 1.0, 2.0]).unwrap();
let r = relu(&t);
assert_eq!(r.data, vec![0.0, 0.0, 0.0, 1.0, 2.0]);
let g = relu_grad(&t);
assert_eq!(g.data, vec![0.0, 0.0, 0.0, 1.0, 1.0]);
}
#[test]
fn sigmoid_test() {
let t = Tensor::new(&[3], &[-1.0, 0.0, 1.0]).unwrap();
let s = sigmoid(&t);
assert!((s.data[1] - 0.5).abs() < E);
assert!((s.data[0] + s.data[2] - 1.0).abs() < E);
}
#[test]
fn softmax_test() {
let t = Tensor::new(&[2, 3], &[1.0, 2.0, 3.0, 1.0, 2.0, 3.0]).unwrap();
let s = softmax(&t, 1).unwrap();
let row0: f64 = (0..3).map(|j| s.data[j]).sum();
assert!((row0 - 1.0).abs() < E);
assert!(s.data[2] > s.data[0]);
}
#[test]
fn softmax_stability() {
let t = Tensor::new(&[3], &[1000.0, 1001.0, 1002.0]).unwrap();
let s = softmax(&t, 0).unwrap();
let sum: f64 = s.data.iter().sum();
assert!((sum - 1.0).abs() < E);
assert!(s.data[2] > s.data[0]);
}
#[test]
fn gelu_test() {
let t = Tensor::new(&[1], &[0.0]).unwrap();
let g = gelu(&t);
assert!(g.data[0].abs() < E); }
#[test]
fn tanh_test() {
let t = Tensor::new(&[1], &[0.0]).unwrap();
let h = tanh(&t);
assert!(h.data[0].abs() < E);
}
#[test]
fn softplus_test() {
let t = Tensor::new(&[1], &[0.0]).unwrap();
let s = softplus(&t);
assert!((s.data[0] - 2.0_f64.ln()).abs() < E); }
#[test]
fn swish_test() {
let t = Tensor::new(&[1], &[0.0]).unwrap();
let s = swish(&t);
assert!(s.data[0].abs() < E); }
#[test]
fn mish_test() {
let t = Tensor::new(&[1], &[0.0]).unwrap();
let m = mish(&t);
assert!(m.data[0].abs() < E); }
#[test]
fn log_softmax_test() {
let t = Tensor::new(&[3], &[1.0, 2.0, 3.0]).unwrap();
let ls = log_softmax(&t, 0).unwrap();
let sum: f64 = ls.data.iter().map(|x| x.exp()).sum();
assert!((sum - 1.0).abs() < E);
}
}