Struct Swish

pub struct Swish { /* private fields */ }

Swish activation function.

Swish is defined as: f(x) = x * sigmoid(β * x)

where β is a trainable parameter. When β = 1, it’s called Swish-1 and is commonly used. This activation was introduced in “Searching for Activation Functions” by Ramachandran et al.

Examples

use scirs2_neural::activations::Swish;
use scirs2_neural::activations::Activation;
use ndarray::Array;

let swish = Swish::new(1.0);
let input = Array::from_vec(vec![1.0, -1.0, 2.0, -2.0]).into_dyn();
let output = swish.forward(&input).unwrap();

Implementations

impl Swish

pub fn new(beta: f64) -> Self

Create a new Swish activation function with given beta parameter.

Arguments

• beta - Parameter controlling the shape of the Swish function. β = 1.0 gives the standard Swish-1 function.

Examples found in repository

examples/activations_example.rs (line 19)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Activation Functions Demonstration");

    // Create a set of input values
    let x_values: Vec<f64> = (-50..=50).map(|i| i as f64 / 10.0).collect();
    let x = Array1::from(x_values.clone());
    let x_dyn = x.clone().into_dyn();

    // Initialize all activation functions
    let relu = ReLU::new();
    let leaky_relu = ReLU::leaky(0.1);
    let sigmoid = Sigmoid::new();
    let tanh = Tanh::new();
    let gelu = GELU::new();
    let gelu_fast = GELU::fast();
    let swish = Swish::new(1.0);
    let mish = Mish::new();

    // Compute outputs for each activation function
    let relu_output = relu.forward(&x_dyn)?;
    let leaky_relu_output = leaky_relu.forward(&x_dyn)?;
    let sigmoid_output = sigmoid.forward(&x_dyn)?;
    let tanh_output = tanh.forward(&x_dyn)?;
    let gelu_output = gelu.forward(&x_dyn)?;
    let gelu_fast_output = gelu_fast.forward(&x_dyn)?;
    let swish_output = swish.forward(&x_dyn)?;
    let mish_output = mish.forward(&x_dyn)?;

    // Print sample values for each activation
    println!("Sample activation values for input x = -2.0, -1.0, 0.0, 1.0, 2.0:");

    let indices = [5, 40, 50, 60, 95]; // Corresponding to x = -2, -1, 0, 1, 2

    println!(
        "| {:<10} | {:<10} | {:<10} | {:<10} | {:<10} | {:<10} |",
        "x", "-2.0", "-1.0", "0.0", "1.0", "2.0"
    );
    println!(
        "|{:-<12}|{:-<12}|{:-<12}|{:-<12}|{:-<12}|{:-<12}|",
        "", "", "", "", "", ""
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "ReLU",
        relu_output[[indices[0]]],
        relu_output[[indices[1]]],
        relu_output[[indices[2]]],
        relu_output[[indices[3]]],
        relu_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "LeakyReLU",
        leaky_relu_output[[indices[0]]],
        leaky_relu_output[[indices[1]]],
        leaky_relu_output[[indices[2]]],
        leaky_relu_output[[indices[3]]],
        leaky_relu_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "Sigmoid",
        sigmoid_output[[indices[0]]],
        sigmoid_output[[indices[1]]],
        sigmoid_output[[indices[2]]],
        sigmoid_output[[indices[3]]],
        sigmoid_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "Tanh",
        tanh_output[[indices[0]]],
        tanh_output[[indices[1]]],
        tanh_output[[indices[2]]],
        tanh_output[[indices[3]]],
        tanh_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "GELU",
        gelu_output[[indices[0]]],
        gelu_output[[indices[1]]],
        gelu_output[[indices[2]]],
        gelu_output[[indices[3]]],
        gelu_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "GELU Fast",
        gelu_fast_output[[indices[0]]],
        gelu_fast_output[[indices[1]]],
        gelu_fast_output[[indices[2]]],
        gelu_fast_output[[indices[3]]],
        gelu_fast_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "Swish",
        swish_output[[indices[0]]],
        swish_output[[indices[1]]],
        swish_output[[indices[2]]],
        swish_output[[indices[3]]],
        swish_output[[indices[4]]]
    );

    println!(
        "| {:<10} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} | {:<10.6} |",
        "Mish",
        mish_output[[indices[0]]],
        mish_output[[indices[1]]],
        mish_output[[indices[2]]],
        mish_output[[indices[3]]],
        mish_output[[indices[4]]]
    );

    // Now test the backward pass with some dummy gradient output
    println!("\nTesting backward pass...");

    // Create a dummy gradient output
    let dummy_grad = Array1::<f64>::ones(x.len()).into_dyn();

    // Compute gradients for each activation function
    let _relu_grad = relu.backward(&dummy_grad, &relu_output)?;
    let _leaky_relu_grad = leaky_relu.backward(&dummy_grad, &leaky_relu_output)?;
    let _sigmoid_grad = sigmoid.backward(&dummy_grad, &sigmoid_output)?;
    let _tanh_grad = tanh.backward(&dummy_grad, &tanh_output)?;
    let _gelu_grad = gelu.backward(&dummy_grad, &gelu_output)?;
    let _gelu_fast_grad = gelu_fast.backward(&dummy_grad, &gelu_fast_output)?;
    let _swish_grad = swish.backward(&dummy_grad, &swish_output)?;
    let _mish_grad = mish.backward(&dummy_grad, &mish_output)?;

    println!("Backward pass completed successfully.");

    // Test with matrix input instead of vector
    println!("\nTesting with matrix input...");

    // Create a 3x4 matrix
    let mut matrix = Array2::<f64>::zeros((3, 4));
    for i in 0..3 {
        for j in 0..4 {
            matrix[[i, j]] = -2.0 + (i as f64 * 4.0 + j as f64) * 0.5;
        }
    }

    // Print input matrix
    println!("Input matrix:");
    for i in 0..3 {
        print!("[ ");
        for j in 0..4 {
            print!("{:6.2} ", matrix[[i, j]]);
        }
        println!("]");
    }

    // Apply GELU activation to the matrix
    let gelu_matrix_output = gelu.forward(&matrix.into_dyn())?;

    // Print output matrix
    println!("\nAfter GELU activation:");
    for i in 0..3 {
        print!("[ ");
        for j in 0..4 {
            print!("{:6.2} ", gelu_matrix_output[[i, j]]);
        }
        println!("]");
    }

    println!("\nActivation functions demonstration completed successfully!");

    // Note about visualization
    println!("\nFor visualization of activation functions:");
    println!("1. You can use external plotting libraries like plotly or matplotlib");
    println!("2. To visualize these functions, you would plot the x_values against");
    println!("   the output values for each activation function");
    println!("3. The data from this example can be exported for plotting as needed");

    // Example of how to access the data for plotting
    println!("\nExample data points for plotting ReLU:");
    for i in 0..5 {
        let idx = i * 20; // Sample every 20th point
        if idx < x_values.len() {
            println!(
                "x: {:.2}, y: {:.6}",
                x_values[idx],
                convert_to_vec(&relu_output)[idx]
            );
        }
    }

    Ok(())
}

Trait Implementations

impl<F: Float + Debug> Activation<F> for Swish

fn forward(&self, input: &Array<F, IxDyn>) -> Result<Array<F, IxDyn>>

Apply the activation function to the input

fn backward( &self, grad_output: &Array<F, IxDyn>, output: &Array<F, IxDyn>, ) -> Result<Array<F, IxDyn>>

Compute the derivative of the activation function with respect to the input

impl Clone for Swish

fn clone(&self) -> Swish

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Swish

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Swish

fn default() -> Self

Returns the “default value” for a type.

impl Copy for Swish