Struct Dropout

pub struct Dropout<F: Float + Debug + Send + Sync> { /* private fields */ }

Dropout layer

During training, randomly sets input elements to zero with probability p. During inference, scales the output by 1/(1-p) to maintain the expected value.

Examples

use scirs2_neural::layers::{Dropout, Layer};
use ndarray::{Array, Array2};
use rand::rngs::SmallRng;
use rand::SeedableRng;

// Create a dropout layer with 0.5 dropout probability
let mut rng = SmallRng::seed_from_u64(42);
let dropout = Dropout::new(0.5, &mut rng).unwrap();

// Forward pass with a batch of 2 samples, 10 features
let batch_size = 2;
let features = 10;
let input = Array2::<f64>::from_elem((batch_size, features), 1.0).into_dyn();

// Forward pass in training mode (some values will be dropped)
let output = dropout.forward(&input).unwrap();

// Output shape should match input shape
assert_eq!(output.shape(), input.shape());

Implementations

impl<F: Float + Debug + ScalarOperand + Send + Sync + 'static> Dropout<F>

pub fn new<R: Rng + 'static + Clone + Send + Sync>( p: f64, rng: &mut R, ) -> Result<Self>

Create a new dropout layer

Arguments

• p - Dropout probability (0.0 to 1.0)
• rng - Random number generator

Returns

• A new dropout layer

Examples found in repository

examples/model_evaluation_example.rs (line 54)

    fn build(&self) -> Result<Self::Model> {
        let mut model = Sequential::new();

        let mut rng = SmallRng::seed_from_u64(42);
        model.add(Dense::<F>::new(
            self.input_dim,
            self.hidden_dim,
            Some("relu"),
            &mut rng,
        )?);

        model.add(Dropout::<F>::new(0.2, &mut rng)?);

        model.add(Dense::<F>::new(
            self.hidden_dim,
            self.output_dim,
            None,
            &mut rng,
        )?);

        Ok(model)
    }

examples/image_classification_complete.rs (line 165)

fn build_cnn_model(
    input_channels: usize,
    num_classes: usize,
    rng: &mut SmallRng,
) -> Result<Sequential<f32>> {
    let mut model = Sequential::new();

    // First convolutional block
    model.add(Dense::new(
        input_channels * 32 * 32,
        512,
        Some("relu"),
        rng,
    )?);
    model.add(Dropout::new(0.25, rng)?);

    // Hidden layers
    model.add(Dense::new(512, 256, Some("relu"), rng)?);
    model.add(Dropout::new(0.25, rng)?);

    model.add(Dense::new(256, 128, Some("relu"), rng)?);
    model.add(Dropout::new(0.25, rng)?);

    // Output layer
    model.add(Dense::new(128, num_classes, Some("softmax"), rng)?);

    Ok(model)
}

examples/model_architecture_visualization.rs (line 98)

fn create_mlp_model(rng: &mut SmallRng) -> Result<Sequential<f32>> {
    let mut model = Sequential::new();

    // Hidden layers with decreasing sizes
    let dense1 = Dense::new(784, 512, Some("relu"), rng)?;
    model.add_layer(dense1);

    let dropout1 = Dropout::new(0.2, rng)?;
    model.add_layer(dropout1);

    let dense2 = Dense::new(512, 256, Some("relu"), rng)?;
    model.add_layer(dense2);

    let dense3 = Dense::new(256, 128, Some("relu"), rng)?;
    model.add_layer(dense3);

    let dropout2 = Dropout::new(0.3, rng)?;
    model.add_layer(dropout2);

    // Output layer
    let dense4 = Dense::new(128, 10, Some("softmax"), rng)?;
    model.add_layer(dense4);

    Ok(model)
}

// Create a simple CNN model for MNIST
fn create_cnn_model(rng: &mut SmallRng) -> Result<Sequential<f32>> {
    let mut model = Sequential::new();

    // First convolutional block
    let conv1 = Conv2D::new(
        1,                      // input channels
        32,                     // output channels
        (3, 3),                 // kernel size
        (1, 1),                 // stride
        PaddingMode::Custom(1), // padding mode
        rng,
    )?;
    model.add_layer(conv1);

    let batch_norm1 = BatchNorm::new(32, 0.99, 1e-5, rng)?;
    model.add_layer(batch_norm1);

    // Second convolutional block
    let conv2 = Conv2D::new(
        32,                     // input channels
        64,                     // output channels
        (3, 3),                 // kernel size
        (2, 2),                 // stride (downsampling)
        PaddingMode::Custom(1), // padding mode
        rng,
    )?;
    model.add_layer(conv2);

    let batch_norm2 = BatchNorm::new(64, 0.99, 1e-5, rng)?;
    model.add_layer(batch_norm2);

    let dropout1 = Dropout::new(0.25, rng)?;
    model.add_layer(dropout1);

    // Flatten for fully connected layers - commented out as Flatten is not available
    // let flatten = Flatten::new()?;
    // model.add_layer(flatten);

    // Dense layers
    let dense1 = Dense::new(64 * 14 * 14, 128, Some("relu"), rng)?;
    model.add_layer(dense1);

    let dropout2 = Dropout::new(0.5, rng)?;
    model.add_layer(dropout2);

    // Output layer
    let dense2 = Dense::new(128, 10, Some("softmax"), rng)?;
    model.add_layer(dense2);

    Ok(model)
}

examples/improved_model_serialization.rs (line 26)

fn create_xor_model(rng: &mut SmallRng) -> Result<Sequential<f32>> {
    let mut model = Sequential::new();

    // XOR problem requires a hidden layer
    let input_dim = 2;
    let hidden_dim = 4;
    let output_dim = 1;

    // Input to hidden layer with ReLU activation
    let dense1 = Dense::new(input_dim, hidden_dim, Some("relu"), rng)?;
    model.add_layer(dense1);

    // Optional dropout for regularization (low rate as XOR is small)
    let dropout = Dropout::new(0.1, rng)?;
    model.add_layer(dropout);

    // Hidden to output layer with sigmoid activation (binary output)
    let dense2 = Dense::new(hidden_dim, output_dim, Some("sigmoid"), rng)?;
    model.add_layer(dense2);

    Ok(model)
}

examples/advanced_training_example.rs (line 37)

fn create_regression_model<
    F: Float + Debug + ScalarOperand + Send + Sync + FromPrimitive + 'static,
>(
    input_dim: usize,
    hidden_dim: usize,
    output_dim: usize,
) -> Result<Sequential<F>> {
    let mut model = Sequential::new();

    // Create RNG for initializing layers
    let mut rng = SmallRng::seed_from_u64(42);

    // First dense layer with ReLU activation
    let dense1 = Dense::new(input_dim, hidden_dim, Some("relu"), &mut rng)?;
    model.add(dense1);

    // Dropout layer for regularization
    let dropout1 = Dropout::new(0.2, &mut rng)?;
    model.add(dropout1);

    // Second dense layer with ReLU activation
    let dense2 = Dense::new(hidden_dim, hidden_dim / 2, Some("relu"), &mut rng)?;
    model.add(dense2);

    // Another dropout layer
    let dropout2 = Dropout::new(0.2, &mut rng)?;
    model.add(dropout2);

    // Output layer with no activation
    let dense3 = Dense::new(hidden_dim / 2, output_dim, None, &mut rng)?;
    model.add(dense3);

    Ok(model)
}

examples/text_classification_complete.rs (line 336)

fn build_text_model(
    _vocab_size: usize,
    embedding_dim: usize,
    hidden_dim: usize,
    num_classes: usize,
    max_length: usize,
    rng: &mut SmallRng,
) -> StdResult<Sequential<f32>> {
    let mut model = Sequential::new();

    // Note: This is a simplified version since our Sequential model expects specific layer types
    // In a real implementation, we'd need specialized text layers

    // Flatten the input tokens and use dense layers to simulate embedding and LSTM processing
    let input_size = max_length; // Each token position

    // Simulate embedding layer with dense transformation
    model.add(Dense::new(
        input_size,
        embedding_dim * 2,
        Some("relu"),
        rng,
    )?);
    model.add(Dropout::new(0.1, rng)?);

    // Simulate LSTM processing with dense layers
    model.add(Dense::new(
        embedding_dim * 2,
        hidden_dim,
        Some("tanh"),
        rng,
    )?);
    model.add(Dropout::new(0.2, rng)?);

    // Add attention-like mechanism with another dense layer
    model.add(Dense::new(hidden_dim, hidden_dim / 2, Some("relu"), rng)?);
    model.add(Dropout::new(0.2, rng)?);

    // Classification head
    model.add(Dense::new(
        hidden_dim / 2,
        num_classes,
        Some("softmax"),
        rng,
    )?);

    Ok(model)
}

/// Build an advanced text model with attention
fn build_attention_text_model(
    _vocab_size: usize,
    embedding_dim: usize,
    hidden_dim: usize,
    num_classes: usize,
    max_length: usize,
    rng: &mut SmallRng,
) -> StdResult<Sequential<f32>> {
    let mut model = Sequential::new();

    // Simplified attention-based model using dense layers
    let input_size = max_length;

    // Multi-layer processing to simulate transformer-like behavior
    model.add(Dense::new(input_size, embedding_dim, Some("relu"), rng)?);
    model.add(Dropout::new(0.1, rng)?);

    // Simulate self-attention with dense layers
    model.add(Dense::new(embedding_dim, hidden_dim, Some("relu"), rng)?);
    model.add(Dropout::new(0.1, rng)?);

    // Second attention layer
    model.add(Dense::new(hidden_dim, hidden_dim, Some("relu"), rng)?);
    model.add(Dropout::new(0.1, rng)?);

    // Global pooling simulation
    model.add(Dense::new(hidden_dim, hidden_dim / 2, Some("relu"), rng)?);
    model.add(Dropout::new(0.2, rng)?);

    // Classification head
    model.add(Dense::new(
        hidden_dim / 2,
        num_classes,
        Some("softmax"),
        rng,
    )?);

    Ok(model)
}

Additional examples can be found in:

• examples/model_visualization_cnn.rs
• examples/generative_models_complete.rs
• examples/object_detection_complete.rs
• examples/model_serialization_example.rs
• examples/dropout_example.rs
• examples/advanced_optimizers_example.rs

pub fn set_training(&mut self, training: bool)

Set the training mode

In training mode, elements are randomly dropped. In inference mode, all elements are kept but scaled.

Examples found in repository

examples/dropout_example.rs (line 163)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Dropout Example");

    // Initialize random number generator with a fixed seed for reproducibility
    let mut rng = SmallRng::seed_from_u64(42);

    // 1. Example: Simple dropout on a vector
    println!("\nExample 1: Simple dropout on a vector");

    // Create a vector of ones
    let n_features = 20;
    let input = Array2::<f32>::from_elem((1, n_features), 1.0);

    // Create a dropout layer with dropout probability 0.5
    let dropout = Dropout::new(0.5, &mut rng)?;

    // Apply dropout in training mode
    let output = dropout.forward(&input.into_dyn())?;

    // Count elements that were dropped
    let mut dropped_count = 0;
    for i in 0..n_features {
        let val = output.slice(ndarray::s![0, i]).into_scalar();
        if *val == 0.0 {
            dropped_count += 1;
        }
    }

    // Print results
    println!("Input: Array of {} ones", n_features);
    println!(
        "Output after dropout (p=0.5): {} elements dropped",
        dropped_count
    );
    println!(
        "Dropout rate: {:.2}%",
        (dropped_count as f32 / n_features as f32) * 100.0
    );

    // 2. Example: Using dropout in a neural network to prevent overfitting
    println!("\nExample 2: Using dropout in a simple neural network");

    // Create a simple neural network with dropout
    // 1. Create dense layer (input -> hidden)
    let input_dim = 10;
    let hidden_dim = 100;
    let output_dim = 2;
    let dense1 = Dense::new(input_dim, hidden_dim, Some("relu"), &mut rng)?;

    // 2. Create dropout layer (after first dense layer)
    let dropout1 = Dropout::new(0.2, &mut rng)?;

    // 3. Create second dense layer (hidden -> output)
    let dense2 = Dense::new(hidden_dim, output_dim, None, &mut rng)?;

    // Generate a random input
    let batch_size = 16;
    let mut input_data = Array2::<f32>::zeros((batch_size, input_dim));
    for i in 0..batch_size {
        for j in 0..input_dim {
            input_data[[i, j]] = rng.random_range(-1.0..1.0);
        }
    }

    // Forward pass through the network with dropout
    println!("Running forward pass with dropout...");
    let hidden_output = dense1.forward(&input_data.clone().into_dyn())?;
    let hidden_dropped = dropout1.forward(&hidden_output)?;
    let final_output = dense2.forward(&hidden_dropped)?;

    println!("Input shape: {:?}", input_data.shape());
    println!("Hidden layer shape: {:?}", hidden_output.shape());
    println!("Output shape: {:?}", final_output.shape());

    // 3. Example: Dropout in a CNN architecture
    println!("\nExample 3: Dropout in a CNN architecture");

    // Create a sample CNN architecture with dropout

    // 1. Create convolutional layer
    let in_channels = 3; // RGB input
    let out_channels = 16;
    let conv1 = Conv2D::new(
        in_channels,
        out_channels,
        (3, 3),
        (1, 1),
        PaddingMode::Same,
        &mut rng,
    )?;

    // 2. Create batch normalization layer
    let bn1 = BatchNorm::new(out_channels, 0.9, 1e-5, &mut rng)?;

    // 3. Create dropout layer for spatial dropout (dropping entire feature maps)
    let dropout_conv = Dropout::new(0.25, &mut rng)?;

    // 4. Create flattened dense layer
    let height = 32;
    let width = 32;
    let flattened_size = out_channels * height * width;
    let dense3 = Dense::new(flattened_size, output_dim, None, &mut rng)?;

    // Generate random input image
    let mut image_input = Array4::<f32>::zeros((batch_size, in_channels, height, width));
    for n in 0..batch_size {
        for c in 0..in_channels {
            for h in 0..height {
                for w in 0..width {
                    image_input[[n, c, h, w]] = rng.random_range(-1.0..1.0);
                }
            }
        }
    }

    // Forward pass through CNN with dropout
    println!("Running forward pass through CNN with dropout...");
    let conv_output = conv1.forward(&image_input.clone().into_dyn())?;
    let bn_output = bn1.forward(&conv_output)?;
    let dropout_output = dropout_conv.forward(&bn_output)?;

    // Reshape for dense layer (flatten spatial and channel dimensions)
    let mut flattened = Array2::<f32>::zeros((batch_size, flattened_size));
    for n in 0..batch_size {
        let mut idx = 0;
        for c in 0..out_channels {
            for h in 0..height {
                for w in 0..width {
                    flattened[[n, idx]] =
                        *dropout_output.slice(ndarray::s![n, c, h, w]).into_scalar();
                    idx += 1;
                }
            }
        }
    }

    let cnn_output = dense3.forward(&flattened.into_dyn())?;

    println!("CNN output shape: {:?}", cnn_output.shape());

    // 4. Example: Switching between training and inference modes
    println!("\nExample 4: Switching between training and inference modes");

    // Create dropout layer with p=0.5
    let mut switchable_dropout = Dropout::new(0.5, &mut rng)?;

    // Create some input data
    let test_input = Array2::<f32>::from_elem((1, 10), 1.0);

    // Training mode (default)
    let training_output = switchable_dropout.forward(&test_input.clone().into_dyn())?;

    // Count non-zero elements in training mode
    let training_nonzero = training_output.iter().filter(|&&x| x != 0.0).count();

    // Switch to inference mode
    switchable_dropout.set_training(false);
    let inference_output = switchable_dropout.forward(&test_input.clone().into_dyn())?;

    // Count non-zero elements in inference mode
    let inference_nonzero = inference_output.iter().filter(|&&x| x != 0.0).count();

    println!("Training mode: {} of 10 elements kept", training_nonzero);
    println!("Inference mode: {} of 10 elements kept", inference_nonzero);

    // In inference mode, all elements should be preserved
    assert_eq!(inference_nonzero, 10);

    println!("\nDropout implementation demonstration completed successfully!");

    Ok(())
}

pub fn p(&self) -> f64

Get the dropout probability

pub fn is_training(&self) -> bool

Get the training mode

Trait Implementations

impl<F: Float + Debug + Send + Sync> Clone for Dropout<F>

fn clone(&self) -> Self

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<F: Float + Debug + Send + Sync> Debug for Dropout<F>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<F: Float + Debug + ScalarOperand + Send + Sync + 'static> Layer<F> for Dropout<F>

fn as_any(&self) -> &dyn Any

Get the layer as a dyn Any for downcasting

fn as_any_mut(&mut self) -> &mut dyn Any

Get the layer as a mutable dyn Any for downcasting

fn forward(&self, input: &Array<F, IxDyn>) -> Result<Array<F, IxDyn>>

Forward pass of the layer

fn backward( &self, _input: &Array<F, IxDyn>, grad_output: &Array<F, IxDyn>, ) -> Result<Array<F, IxDyn>>

Backward pass of the layer to compute gradients

fn update(&mut self, _learning_rate: F) -> Result<()>

Update the layer parameters with the given gradients

fn layer_type(&self) -> &str

Get the type of the layer (e.g., “Dense”, “Conv2D”)

fn parameter_count(&self) -> usize

Get the number of trainable parameters in this layer

fn layer_description(&self) -> String

Get a detailed description of this layer

fn set_training(&mut self, training: bool)

Set the layer to training mode (true) or evaluation mode (false)

fn is_training(&self) -> bool

Get the current training mode

fn params(&self) -> Vec<Array<F, IxDyn>>

Get the parameters of the layer

fn gradients(&self) -> Vec<Array<F, IxDyn>>

Get the gradients of the layer parameters

fn set_gradients(&mut self, _gradients: &[Array<F, IxDyn>]) -> Result<()>

Set the gradients of the layer parameters

fn set_params(&mut self, _params: &[Array<F, IxDyn>]) -> Result<()>

Set the parameters of the layer