Struct LayerNorm

pub struct LayerNorm<F: Float + Debug> { /* private fields */ }

Layer Normalization layer

Implements layer normalization as described in “Layer Normalization” by Ba, Kiros, and Hinton. It normalizes the inputs across the last dimension and applies learnable scale and shift parameters.

Examples

use scirs2_neural::layers::{LayerNorm, Layer};
use ndarray::{Array, Array3};
use rand::rngs::SmallRng;
use rand::SeedableRng;

// Create a layer normalization layer for a 64-dimensional feature space
let mut rng = SmallRng::seed_from_u64(42);
let layer_norm = LayerNorm::new(64, 1e-5, &mut rng).unwrap();

// Forward pass with a batch of 2 samples, sequence length 3
let batch_size = 2;
let seq_len = 3;
let d_model = 64;
let input = Array3::<f64>::from_elem((batch_size, seq_len, d_model), 0.1).into_dyn();
let output = layer_norm.forward(&input).unwrap();

// Output shape should match input shape
assert_eq!(output.shape(), input.shape());

Implementations

impl<F: Float + Debug + ScalarOperand + 'static> LayerNorm<F>

pub fn new<R: Rng>( normalized_shape: usize, eps: f64, _rng: &mut R, ) -> Result<Self>

Create a new layer normalization layer

Arguments

• normalized_shape - Shape of the input features to normalize over
• eps - Small constant added for numerical stability
• rng - Random number generator for initialization

Returns

• A new layer normalization layer

Examples found in repository

examples/model_serialization_example.rs (line 37)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Model Serialization Example");

    // Initialize random number generator
    let mut rng = SmallRng::seed_from_u64(42);

    // 1. Create a simple neural network model
    let mut model = Sequential::new();

    // Add layers
    let input_dim = 784; // MNIST image size: 28x28 = 784
    let hidden_dim_1 = 256;
    let hidden_dim_2 = 128;
    let output_dim = 10; // 10 classes for digits 0-9

    // Input layer to first hidden layer
    let dense1 = Dense::new(input_dim, hidden_dim_1, Some("relu"), &mut rng)?;
    model.add_layer(dense1);

    // Dropout for regularization
    let dropout1 = Dropout::new(0.2, &mut rng)?;
    model.add_layer(dropout1);

    // First hidden layer to second hidden layer
    let dense2 = Dense::new(hidden_dim_1, hidden_dim_2, Some("relu"), &mut rng)?;
    model.add_layer(dense2);

    // Layer normalization
    let layer_norm = LayerNorm::new(hidden_dim_2, 1e-5, &mut rng)?;
    model.add_layer(layer_norm);

    // Second hidden layer to output layer
    let dense3 = Dense::new(hidden_dim_2, output_dim, Some("softmax"), &mut rng)?;
    model.add_layer(dense3);

    println!(
        "Created a neural network with {} layers",
        model.num_layers()
    );

    // 2. Test the model with some dummy input
    let batch_size = 2;
    let input = Array2::<f32>::from_elem((batch_size, input_dim), 0.1);
    let output = model.forward(&input.clone().into_dyn())?;

    println!("Model output shape: {:?}", output.shape());
    println!("First few output values:");
    for i in 0..batch_size {
        print!("Sample {}: [ ", i);
        for j in 0..5 {
            // Print first 5 values
            print!("{:.6} ", output[[i, j]]);
        }
        println!("... ]");
    }

    // 3. Save the model to a file
    let model_path = Path::new("mnist_model.json");
    serialization::save_model(&model, model_path, SerializationFormat::JSON)?;

    println!("\nModel saved to {}", model_path.display());

    // 4. Load the model from the file
    let loaded_model = serialization::load_model::<f32, _>(model_path, SerializationFormat::JSON)?;

    println!(
        "Model loaded successfully with {} layers",
        loaded_model.num_layers()
    );

    // 5. Test the loaded model with the same input
    let loaded_output = loaded_model.forward(&input.into_dyn())?;

    println!("\nLoaded model output shape: {:?}", loaded_output.shape());
    println!("First few output values:");
    for i in 0..batch_size {
        print!("Sample {}: [ ", i);
        for j in 0..5 {
            // Print first 5 values
            print!("{:.6} ", loaded_output[[i, j]]);
        }
        println!("... ]");
    }

    // 6. Compare original and loaded model outputs
    let mut max_diff = 0.0;
    for i in 0..batch_size {
        for j in 0..output_dim {
            let diff = (output[[i, j]] - loaded_output[[i, j]]).abs();
            if diff > max_diff {
                max_diff = diff;
            }
        }
    }

    println!(
        "\nMaximum difference between original and loaded model outputs: {:.6}",
        max_diff
    );

    if max_diff < 1e-6 {
        println!("Models are identical! Serialization and deserialization worked correctly.");
    } else {
        println!("Warning: There are differences between the original and loaded models.");
        println!(
            "This might be due to numerical precision issues or a problem with serialization."
        );
    }

    Ok(())
}

pub fn normalized_shape(&self) -> usize

Get the normalized shape

pub fn eps(&self) -> f64

Get the epsilon value

Trait Implementations

impl<F: Float + Debug + ScalarOperand + 'static> Clone for LayerNorm<F>

fn clone(&self) -> Self

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<F: Debug + Float + Debug> Debug for LayerNorm<F>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<F: Float + Debug + ScalarOperand + 'static> Layer<F> for LayerNorm<F>

fn as_any(&self) -> &dyn Any

Get the layer as a dyn Any for downcasting

fn as_any_mut(&mut self) -> &mut dyn Any

Get the layer as a mutable dyn Any for downcasting

fn forward(&self, input: &Array<F, IxDyn>) -> Result<Array<F, IxDyn>>

Forward pass of the layer

fn backward( &self, input: &Array<F, IxDyn>, grad_output: &Array<F, IxDyn>, ) -> Result<Array<F, IxDyn>>

Backward pass of the layer to compute gradients

fn update(&mut self, learning_rate: F) -> Result<()>

Update the layer parameters with the given gradients

fn params(&self) -> Vec<Array<F, IxDyn>>

Get the parameters of the layer

fn gradients(&self) -> Vec<Array<F, IxDyn>>

Get the gradients of the layer parameters

fn set_gradients(&mut self, _gradients: &[Array<F, IxDyn>]) -> Result<()>

Set the gradients of the layer parameters

fn set_params(&mut self, _params: &[Array<F, IxDyn>]) -> Result<()>

Set the parameters of the layer

fn set_training(&mut self, _training: bool)

Set the layer to training mode (true) or evaluation mode (false)

fn is_training(&self) -> bool

Get the current training mode

fn layer_type(&self) -> &str

Get the type of the layer (e.g., “Dense”, “Conv2D”)

fn parameter_count(&self) -> usize

Get the number of trainable parameters in this layer

fn layer_description(&self) -> String

Get a detailed description of this layer

impl<F: Float + Debug + ScalarOperand + 'static> ParamLayer<F> for LayerNorm<F>

fn get_parameters(&self) -> Vec<&Array<F, IxDyn>>

Get the parameters of the layer as a vector of arrays

fn get_gradients(&self) -> Vec<&Array<F, IxDyn>>

Get the gradients of the parameters

fn set_parameters(&mut self, params: Vec<Array<F, IxDyn>>) -> Result<()>

Set the parameters of the layer