use super::kernel::{elementwise_unary, elementwise_unary_assign};
use crate::backend_ops::{BackendOps, UnaryOp};
use coeus_core::{Float, Scalar};
use coeus_tensor::Tensor;
#[inline]
pub fn relu<T: Scalar, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Relu).expect("relu")
}
#[inline]
pub fn sigmoid<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Sigmoid).expect("sigmoid")
}
#[inline]
pub fn tanh<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Tanh).expect("tanh")
}
#[inline]
pub fn gelu<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Gelu).expect("gelu")
}
#[inline]
pub fn relu_assign<T: Scalar, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Relu)
}
#[inline]
pub fn sigmoid_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Sigmoid)
}
#[inline]
pub fn tanh_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Tanh)
}
#[inline]
pub fn gelu_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Gelu)
}
#[inline]
pub fn silu<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Silu).expect("silu")
}
#[inline]
pub fn silu_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Silu)
}
#[inline]
pub fn mish<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Mish).expect("mish")
}
#[inline]
pub fn mish_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Mish)
}
#[inline]
pub fn elu<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Elu).expect("elu")
}
#[inline]
pub fn elu_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Elu)
}
#[inline]
pub fn softplus<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::Softplus).expect("softplus")
}
#[inline]
pub fn softplus_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::Softplus)
}
#[inline]
pub fn gelu_tanh<T: Float, B: BackendOps<T>>(input: &Tensor<T, B>, backend: &B) -> Tensor<T, B> {
elementwise_unary(input, backend, UnaryOp::GeluTanh).expect("gelu_tanh")
}
#[inline]
pub fn gelu_tanh_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
elementwise_unary_assign(input, backend, UnaryOp::GeluTanh)
}
#[inline]
pub fn leaky_relu<T: Float, B: BackendOps<T>>(
input: &Tensor<T, B>,
backend: &B,
negative_slope: f64,
) -> Tensor<T, B> {
elementwise_unary(
input,
backend,
UnaryOp::LeakyRelu(f64::to_bits(negative_slope)),
)
.expect("leaky_relu")
}
#[inline]
pub fn leaky_relu_assign<T: Float, B: BackendOps<T>>(
input: &mut Tensor<T, B>,
backend: &B,
negative_slope: f64,
) -> Result<(), B::Error> {
elementwise_unary_assign(
input,
backend,
UnaryOp::LeakyRelu(f64::to_bits(negative_slope)),
)
}
mod gated;
mod softmax;
pub use gated::glu;
pub use softmax::{causal_softmax, log_softmax_axis, masked_softmax};
#[cfg(test)]
mod tests {
use super::*;
use coeus_core::SequentialBackend;
fn assert_close(actual: &[f64], expected: &[f64], label: &str) {
assert_eq!(actual.len(), expected.len(), "{label} length mismatch");
for (index, (&got, &want)) in actual.iter().zip(expected).enumerate() {
assert!(
(got - want).abs() <= 1e-12,
"{label}[{index}] = {got}, expected {want}"
);
}
}
#[test]
fn masked_softmax_excludes_masked_lanes() {
let backend = SequentialBackend::new();
let input =
Tensor::<f64, SequentialBackend>::from_slice([2, 3], &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
let mask =
Tensor::<f64, SequentialBackend>::from_slice([2, 3], &[1.0, 0.0, 1.0, 0.0, 1.0, 1.0]);
let out =
masked_softmax(&input, &mask, 1, &backend).expect("valid masked softmax test input");
let exp_1 = 1.0f64.exp();
let exp_3 = 3.0f64.exp();
let exp_5 = 5.0f64.exp();
let exp_6 = 6.0f64.exp();
let expected = [
exp_1 / (exp_1 + exp_3),
0.0,
exp_3 / (exp_1 + exp_3),
0.0,
exp_5 / (exp_5 + exp_6),
exp_6 / (exp_5 + exp_6),
];
assert_eq!(out.shape(), &[2, 3]);
assert_close(out.as_slice(), &expected, "masked_softmax");
}
#[test]
fn masked_softmax_all_masked_row_is_zero() {
let backend = SequentialBackend::new();
let input = Tensor::<f64, SequentialBackend>::from_slice([1, 3], &[1.0, 2.0, 3.0]);
let mask = Tensor::<f64, SequentialBackend>::zeros([1, 3]);
let out =
masked_softmax(&input, &mask, 1, &backend).expect("valid masked softmax test input");
assert_eq!(out.as_slice(), &[0.0, 0.0, 0.0]);
}
#[test]
fn causal_softmax_masks_future_columns() {
let backend = SequentialBackend::new();
let input = Tensor::<f64, SequentialBackend>::from_slice(
[1, 3, 3],
&[1.0, 9.0, 9.0, 1.0, 2.0, 9.0, 1.0, 2.0, 3.0],
);
let out = causal_softmax(&input, 2, &backend).expect("valid causal softmax test input");
let exp_1 = 1.0f64.exp();
let exp_2 = 2.0f64.exp();
let exp_3 = 3.0f64.exp();
let expected = [
1.0,
0.0,
0.0,
exp_1 / (exp_1 + exp_2),
exp_2 / (exp_1 + exp_2),
0.0,
exp_1 / (exp_1 + exp_2 + exp_3),
exp_2 / (exp_1 + exp_2 + exp_3),
exp_3 / (exp_1 + exp_2 + exp_3),
];
assert_close(out.as_slice(), &expected, "causal_softmax");
}
#[test]
fn glu_splits_axis_and_gates_first_half() {
let backend = SequentialBackend::new();
let input = Tensor::<f64, SequentialBackend>::from_slice([2], &[2.0, 4.0]);
let out = glu(&input, 0, &backend).expect("valid GLU test input");
let expected = 2.0 / (1.0 + (-4.0f64).exp());
assert_eq!(out.shape(), &[1]);
assert!(
(out.as_slice()[0] - expected).abs() <= 1e-12,
"glu output {} vs {expected}",
out.as_slice()[0]
);
}
}