vectra 0.2.4

A multi-dimensional array library for Rust, similar to NumPy
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

//! Machine learning activation functions and operations.
//!
//! This module provides common activation functions used in neural networks
//! and machine learning applications.

use std::cmp::Ordering;

use num_traits::{Float, Zero};

use crate::{CmpExt, NumExt, core::Array};

impl<const D: usize, T: Zero + Clone + CmpExt> Array<D, T> {
    /// Apply the ReLU (Rectified Linear Unit) activation function element-wise.
    ///
    /// ReLU(x) = max(0, x)
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![-1, 0, 1, 2], [2, 2]);
    /// let result = arr.relu();
    /// ```
    pub fn relu(&self) -> Self {
        self.map(|x| {
            let zero = T::zero();

            match x.cmp_ext(&zero) {
                Ordering::Greater => x.clone(),
                _ => zero,
            }
        })
    }
}

impl<const D: usize> Array<D, f32> {
    /// Apply the GELU (Gaussian Error Linear Unit) activation function element-wise.
    ///
    /// GELU(x) = 0.5 * x * (1 + tanh(√(2/π) * (x + 0.044715 * x³)))
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![-1.0, 0.0, 1.0, 2.0], [2, 2]);
    /// let result = arr.gelu();
    /// ```
    pub fn gelu(&self) -> Self {
        self.map(|x| 0.5 * x * (1.0 + (0.7978845608 * (x + 0.044715 * x.powi(3))).tanh()))
    }
}

impl<const D: usize> Array<D, f64> {
    /// Apply the GELU (Gaussian Error Linear Unit) activation function element-wise.
    ///
    /// GELU(x) = 0.5 * x * (1 + tanh(√(2/π) * (x + 0.044715 * x³)))
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![-1.0, 0.0, 1.0, 2.0], [2, 2]);
    /// let result = arr.gelu();
    /// ```
    pub fn gelu(&self) -> Self {
        self.map(|x| 0.5 * x * (1.0 + (0.7978845608 * (x + 0.044715 * x.powi(3))).tanh()))
    }
}

impl<const D: usize, T: NumExt + Float> Array<D, T> {
    /// Apply the sigmoid activation function element-wise.
    ///
    /// sigmoid(x) = 1 / (1 + e^(-x))
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![-1.0, 0.0, 1.0, 2.0], [2, 2]);
    /// let result = arr.sigmoid();
    /// ```
    pub fn sigmoid(&self) -> Self {
        (-self).exp().add_scalar(T::one()).recip()
    }

    /// Apply the softmax function along the last dimension.
    ///
    /// softmax(x_i) = e^(x_i - max(x)) / Σ(e^(x_j - max(x)))
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![1.0, 2.0, 3.0, 4.0], [2, 2]);
    /// let result = arr.softmax();
    /// ```
    pub fn softmax(&self) -> Array<D, T> {
        let a = self.max_axis((D - 1) as isize);
        let a = (self - &a).exp();
        let a_t = a.sum_axis((D - 1) as isize);

        &a / &a_t
    }

    /// Apply RMS (Root Mean Square) normalization along the last dimension.
    ///
    /// RMS_norm(x) = x / sqrt(mean(x²) + eps)
    ///
    /// # Arguments
    ///
    /// * `eps` - Small epsilon value to prevent division by zero
    ///
    /// # Examples
    ///
    /// ```rust
    /// use vectra::prelude::*;
    ///
    /// let arr = Array::from_vec(vec![1.0, 2.0, 3.0, 4.0], [2, 2]);
    /// let result = arr.rms_norm(1e-8);
    /// ```
    pub fn rms_norm(&self, eps: T) -> Self {
        let a = self.map(|x| x.powi(2));
        let a_t = a.sum_axis((D - 1) as isize);

        &a / &a_t.sqrt().add_scalar(eps)
    }
}