Struct CrossEntropyLoss

pub struct CrossEntropyLoss { /* private fields */ }

Cross-entropy loss function.

The cross-entropy loss is defined as: L = -sum(y_true * log(y_pred)) where y_true are target probabilities and y_pred are predicted probabilities.

It is commonly used for classification problems.

Examples

use scirs2_neural::losses::CrossEntropyLoss;
use scirs2_neural::losses::Loss;
use ndarray::{Array, arr1, arr2};

let ce = CrossEntropyLoss::new(1e-10);

// One-hot encoded targets and softmax'd predictions for a 3-class problem
let predictions = arr2(&[
    [0.7, 0.2, 0.1],  // First sample, class probabilities
    [0.3, 0.6, 0.1]   // Second sample, class probabilities
]).into_dyn();

let targets = arr2(&[
    [1.0, 0.0, 0.0],  // First sample, true class is 0
    [0.0, 1.0, 0.0]   // Second sample, true class is 1
]).into_dyn();

// Forward pass to calculate loss
let loss = ce.forward(&predictions, &targets).unwrap();

// Backward pass to calculate gradients
let gradients = ce.backward(&predictions, &targets).unwrap();

Implementations

impl CrossEntropyLoss

pub fn new(epsilon: f64) -> Self

Create a new cross-entropy loss function

Arguments

• epsilon - Small value to add to predictions to avoid log(0)

Examples found in repository

examples/object_detection_complete.rs (line 324)

    pub fn new(classification_weight: f32, regression_weight: f32) -> Self {
        Self {
            classification_loss: CrossEntropyLoss::new(1e-7),
            regression_loss: MeanSquaredError,
            classification_weight,
            regression_weight,
        }
    }

examples/improved_model_serialization.rs (line 75)

fn train_model(
    model: &mut Sequential<f32>,
    x: &Array2<f32>,
    y: &Array2<f32>,
    epochs: usize,
) -> Result<()> {
    println!("Training XOR model...");

    // Setup loss function and optimizer
    let loss_fn = CrossEntropyLoss::new(1e-10);
    let mut optimizer = Adam::new(0.01, 0.9, 0.999, 1e-8);

    // Train for specified number of epochs
    for epoch in 0..epochs {
        // Convert to dynamic dimension arrays
        let x_dyn = x.clone().into_dyn();
        let y_dyn = y.clone().into_dyn();

        // Perform a training step
        let loss = model.train_batch(&x_dyn, &y_dyn, &loss_fn, &mut optimizer)?;

        // Print progress every 100 epochs
        if epoch % 100 == 0 || epoch == epochs - 1 {
            println!("Epoch {}/{}: loss = {:.6}", epoch + 1, epochs, loss);
        }
    }

    println!("Training completed.");
    Ok(())
}

examples/generative_models_complete.rs (line 786)

fn train_gan_model() -> StdResult<()> {
    println!("⚔️ Starting GAN Training");

    let mut rng = SmallRng::seed_from_u64(42);
    let config = GenerativeConfig::default();

    // Create models
    println!("🏗️ Building GAN models...");
    let generator = GANGenerator::new(config.clone(), &mut rng)?;
    let discriminator = GANDiscriminator::new(config.clone(), &mut rng)?;
    println!("✅ GAN models created");

    // Create dataset
    let mut dataset = GenerativeDataset::new(config.clone(), 456);

    // Create loss functions
    let _adversarial_loss = CrossEntropyLoss::new(1e-7);

    println!("📊 GAN training configuration:");
    println!("   - Generator latent dim: {}", config.latent_dim);
    println!("   - Discriminator architecture: {:?}", config.hidden_dims);

    // Training loop (simplified)
    let num_epochs = 15;
    let batch_size = 4;

    for epoch in 0..num_epochs {
        println!("\n📈 Epoch {}/{}", epoch + 1, num_epochs);

        let mut d_loss_total = 0.0;
        let mut g_loss_total = 0.0;
        let num_batches = 8;

        for batch_idx in 0..num_batches {
            // Train Discriminator
            let real_images = dataset.generate_batch(batch_size);
            let real_images_dyn = real_images.into_dyn();

            // Generate fake images
            let mut noise = Array2::<f32>::zeros((batch_size, config.latent_dim));
            for elem in noise.iter_mut() {
                *elem = rng.random_range(-1.0..1.0);
            }
            let noise_dyn = noise.into_dyn();
            let fake_images = generator.forward(&noise_dyn)?;

            // Discriminator predictions
            let real_pred = discriminator.forward(&real_images_dyn)?;
            let fake_pred = discriminator.forward(&fake_images)?;

            // Simplified loss calculation (normally would use proper labels)
            let mut d_loss_real = 0.0f32;
            let mut d_loss_fake = 0.0f32;

            for &pred in real_pred.iter() {
                d_loss_real += -(1.0f32).ln() - pred; // Log loss for real=1
            }

            for &pred in fake_pred.iter() {
                d_loss_fake += -(1.0 - pred).ln(); // Log loss for fake=0
            }

            let d_loss = (d_loss_real + d_loss_fake) / (batch_size * 2) as f32;
            d_loss_total += d_loss;

            // Train Generator (simplified)
            let fake_pred_for_g = discriminator.forward(&fake_images)?;
            let mut g_loss = 0.0f32;

            for &pred in fake_pred_for_g.iter() {
                g_loss += -(1.0f32).ln() - pred; // Want discriminator to output 1 for fake
            }
            g_loss /= batch_size as f32;
            g_loss_total += g_loss;

            if batch_idx % 4 == 0 {
                print!(
                    "🔄 Batch {}/{} - D Loss: {:.4}, G Loss: {:.4}        \r",
                    batch_idx + 1,
                    num_batches,
                    d_loss,
                    g_loss
                );
            }
        }

        let avg_d_loss = d_loss_total / num_batches as f32;
        let avg_g_loss = g_loss_total / num_batches as f32;

        println!(
            "✅ Epoch {} - D Loss: {:.4}, G Loss: {:.4}",
            epoch + 1,
            avg_d_loss,
            avg_g_loss
        );

        // Generate samples every few epochs
        if (epoch + 1) % 5 == 0 {
            println!("🎲 Generating samples...");

            let mut sample_noise = Array2::<f32>::zeros((4, config.latent_dim));
            for elem in sample_noise.iter_mut() {
                *elem = rng.random_range(-1.0..1.0);
            }
            let sample_noise_dyn = sample_noise.into_dyn();
            let generated = generator.forward(&sample_noise_dyn)?;

            println!("📊 Generated {} samples", generated.shape()[0]);
        }
    }

    println!("\n🎉 GAN training completed!");
    Ok(())
}

examples/loss_functions_example.rs (line 29)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Loss functions example");

    // Mean Squared Error example
    println!("\n--- Mean Squared Error Example ---");
    let mse = MeanSquaredError::new();

    // Create sample data for regression
    let predictions = Array::from_vec(vec![1.0, 2.0, 3.0]).into_dyn();
    let targets = Array::from_vec(vec![1.5, 1.8, 2.5]).into_dyn();

    // Calculate loss
    let loss = mse.forward(&predictions, &targets)?;
    println!("Predictions: {:?}", predictions);
    println!("Targets: {:?}", targets);
    println!("MSE Loss: {:.4}", loss);

    // Calculate gradients
    let gradients = mse.backward(&predictions, &targets)?;
    println!("MSE Gradients: {:?}", gradients);

    // Cross-Entropy Loss example
    println!("\n--- Cross-Entropy Loss Example ---");
    let ce = CrossEntropyLoss::new(1e-10);

    // Create sample data for multi-class classification
    let predictions = Array::from_shape_vec(IxDyn(&[2, 3]), vec![0.7, 0.2, 0.1, 0.3, 0.6, 0.1])?;
    let targets = Array::from_shape_vec(IxDyn(&[2, 3]), vec![1.0, 0.0, 0.0, 0.0, 1.0, 0.0])?;

    // Calculate loss
    let loss = ce.forward(&predictions, &targets)?;
    println!("Predictions (probabilities):");
    println!("{:?}", predictions);
    println!("Targets (one-hot):");
    println!("{:?}", targets);
    println!("Cross-Entropy Loss: {:.4}", loss);

    // Calculate gradients
    let gradients = ce.backward(&predictions, &targets)?;
    println!("Cross-Entropy Gradients:");
    println!("{:?}", gradients);

    // Focal Loss example
    println!("\n--- Focal Loss Example ---");
    let focal = FocalLoss::new(2.0, Some(0.25), 1e-10);

    // Create sample data for imbalanced classification
    let predictions = Array::from_shape_vec(IxDyn(&[2, 3]), vec![0.7, 0.2, 0.1, 0.3, 0.6, 0.1])?;
    let targets = Array::from_shape_vec(IxDyn(&[2, 3]), vec![1.0, 0.0, 0.0, 0.0, 1.0, 0.0])?;

    // Calculate loss
    let loss = focal.forward(&predictions, &targets)?;
    println!("Predictions (probabilities):");
    println!("{:?}", predictions);
    println!("Targets (one-hot):");
    println!("{:?}", targets);
    println!("Focal Loss (gamma=2.0, alpha=0.25): {:.4}", loss);

    // Calculate gradients
    let gradients = focal.backward(&predictions, &targets)?;
    println!("Focal Loss Gradients:");
    println!("{:?}", gradients);

    // Contrastive Loss example
    println!("\n--- Contrastive Loss Example ---");
    let contrastive = ContrastiveLoss::new(1.0);

    // Create sample data for similarity learning
    // Embedding pairs (batch_size x 2 x embedding_dim)
    let embeddings = Array::from_shape_vec(
        IxDyn(&[2, 2, 3]),
        vec![
            0.1, 0.2, 0.3, // First pair, first embedding
            0.1, 0.3, 0.3, // First pair, second embedding (similar)
            0.5, 0.5, 0.5, // Second pair, first embedding
            0.9, 0.8, 0.7, // Second pair, second embedding (dissimilar)
        ],
    )?;

    // Labels: 1 for similar pairs, 0 for dissimilar
    let labels = Array::from_shape_vec(IxDyn(&[2, 1]), vec![1.0, 0.0])?;

    // Calculate loss
    let loss = contrastive.forward(&embeddings, &labels)?;
    println!("Embeddings (batch_size x 2 x embedding_dim):");
    println!("{:?}", embeddings);
    println!("Labels (1 for similar, 0 for dissimilar):");
    println!("{:?}", labels);
    println!("Contrastive Loss (margin=1.0): {:.4}", loss);

    // Calculate gradients
    let gradients = contrastive.backward(&embeddings, &labels)?;
    println!("Contrastive Loss Gradients (first few):");
    println!("{:?}", gradients.slice(ndarray::s![0, .., 0]));

    // Triplet Loss example
    println!("\n--- Triplet Loss Example ---");
    let triplet = TripletLoss::new(0.5);

    // Create sample data for triplet learning
    // Embedding triplets (batch_size x 3 x embedding_dim)
    let embeddings = Array::from_shape_vec(
        IxDyn(&[2, 3, 3]),
        vec![
            0.1, 0.2, 0.3, // First triplet, anchor
            0.1, 0.3, 0.3, // First triplet, positive
            0.5, 0.5, 0.5, // First triplet, negative
            0.6, 0.6, 0.6, // Second triplet, anchor
            0.5, 0.6, 0.6, // Second triplet, positive
            0.1, 0.1, 0.1, // Second triplet, negative
        ],
    )?;

    // Dummy labels (not used by triplet loss)
    let dummy_labels = Array::zeros(IxDyn(&[2, 1]));

    // Calculate loss
    let loss = triplet.forward(&embeddings, &dummy_labels)?;
    println!("Embeddings (batch_size x 3 x embedding_dim):");
    println!("  - First dimension: batch size");
    println!("  - Second dimension: [anchor, positive, negative]");
    println!("  - Third dimension: embedding components");
    println!("{:?}", embeddings);
    println!("Triplet Loss (margin=0.5): {:.4}", loss);

    // Calculate gradients
    let gradients = triplet.backward(&embeddings, &dummy_labels)?;
    println!("Triplet Loss Gradients (first few):");
    println!("{:?}", gradients.slice(ndarray::s![0, .., 0]));

    Ok(())
}

examples/semantic_segmentation_complete.rs (line 634)

fn train_segmentation_model() -> StdResult<()> {
    println!("🎨 Starting Semantic Segmentation Training");

    let mut rng = SmallRng::seed_from_u64(42);
    let config = SegmentationConfig::default();

    println!("🚀 Starting model training...");

    // Create model
    println!("🏗️ Building U-Net segmentation model...");
    let model = UNetModel::new(config.clone(), &mut rng)?;
    println!("✅ Model created with {} classes", config.num_classes);

    // Create dataset
    let mut dataset = SegmentationDataset::new(config.clone(), 123);

    // Create loss function
    let loss_fn = CrossEntropyLoss::new(1e-7);

    // Create metrics
    let metrics = SegmentationMetrics::new(config.num_classes);

    println!("📊 Training configuration:");
    println!("   - Input size: {:?}", config.input_size);
    println!("   - Number of classes: {}", config.num_classes);
    println!("   - Encoder channels: {:?}", config.encoder_channels);
    println!("   - Decoder channels: {:?}", config.decoder_channels);
    println!("   - Skip connections: {}", config.skip_connections);

    // Training loop
    let num_epochs = 15;
    let batch_size = 2; // Small batch size due to memory constraints
    let _learning_rate = 0.001;

    for epoch in 0..num_epochs {
        println!("\n📈 Epoch {}/{}", epoch + 1, num_epochs);

        let mut epoch_loss = 0.0;
        let num_batches = 10; // Small number of batches for demo

        for batch_idx in 0..num_batches {
            // Generate training batch
            let (images, masks) = dataset.generate_batch(batch_size);
            let images_dyn = images.into_dyn();

            // Forward pass
            let logits = model.forward(&images_dyn)?;

            // Prepare targets
            let targets = masks_to_targets(&masks, config.num_classes);

            // Compute loss
            let batch_loss = loss_fn.forward(&logits, &targets)?;
            epoch_loss += batch_loss;

            if batch_idx % 5 == 0 {
                print!(
                    "🔄 Batch {}/{} - Loss: {:.4}                \r",
                    batch_idx + 1,
                    num_batches,
                    batch_loss
                );
            }
        }

        let avg_loss = epoch_loss / num_batches as f32;
        println!(
            "✅ Epoch {} completed - Average Loss: {:.4}",
            epoch + 1,
            avg_loss
        );

        // Evaluation every few epochs
        if (epoch + 1) % 5 == 0 {
            println!("🔍 Running evaluation...");

            // Generate validation batch
            let (val_images, val_masks) = dataset.generate_batch(batch_size);
            let val_images_dyn = val_images.into_dyn();

            // Get predictions
            let val_logits = model.forward(&val_images_dyn)?;
            let predictions = logits_to_predictions(&val_logits);

            // Calculate metrics
            let pixel_acc = metrics.pixel_accuracy(&predictions, &val_masks);
            let miou = metrics.mean_iou(&predictions, &val_masks);

            println!("📊 Validation metrics:");
            println!("   - Pixel Accuracy: {:.4}", pixel_acc);
            println!("   - Mean IoU: {:.4}", miou);

            // Print class-wise IoU
            println!("   - Class-wise IoU:");
            for class_id in 0..config.num_classes {
                let class_iou = metrics.class_iou(&predictions, &val_masks, class_id);
                if !class_iou.is_nan() {
                    println!("     Class {}: {:.4}", class_id, class_iou);
                }
            }
        }
    }

    println!("\n🎉 Semantic segmentation training completed!");

    // Final evaluation
    println!("🔬 Final evaluation...");
    let (test_images, test_masks) = dataset.generate_batch(4);
    let test_images_dyn = test_images.into_dyn();

    let test_logits = model.forward(&test_images_dyn)?;
    let final_predictions = logits_to_predictions(&test_logits);

    let final_pixel_acc = metrics.pixel_accuracy(&final_predictions, &test_masks);
    let final_miou = metrics.mean_iou(&final_predictions, &test_masks);

    println!("📈 Final metrics:");
    println!("   - Pixel Accuracy: {:.4}", final_pixel_acc);
    println!("   - Mean IoU: {:.4}", final_miou);

    // Confusion matrix
    let confusion = metrics.confusion_matrix(&final_predictions, &test_masks);
    println!("   - Confusion Matrix:");
    for i in 0..config.num_classes {
        print!("     [");
        for j in 0..config.num_classes {
            print!("{:4}", confusion[[i, j]]);
        }
        println!("]");
    }

    // Performance analysis
    println!("\n📊 Model Analysis:");
    println!("   - Architecture: U-Net with skip connections");
    println!(
        "   - Parameters: ~{:.1}K (estimated)",
        (config.encoder_channels.iter().sum::<usize>()
            + config.decoder_channels.iter().sum::<usize>())
            / 1000
    );
    println!("   - Memory efficient: ✅ (skip connections preserve spatial info)");
    println!("   - JIT optimized: ✅");

    Ok(())
}

examples/text_classification_complete.rs (line 537)

fn train_text_classifier() -> StdResult<()> {
    println!("📝 Starting Text Classification Training Example");
    println!("{}", "=".repeat(60));

    let mut rng = SmallRng::seed_from_u64(42);

    // Dataset parameters
    let num_samples = 800;
    let num_classes = 3;
    let max_length = 20;
    let embedding_dim = 64;
    let hidden_dim = 128;

    println!("📊 Dataset Configuration:");
    println!("   - Samples: {}", num_samples);
    println!(
        "   - Classes: {} (Positive, Negative, Neutral)",
        num_classes
    );
    println!("   - Max sequence length: {}", max_length);
    println!("   - Embedding dimension: {}", embedding_dim);

    // Create synthetic text dataset
    println!("\n🔄 Creating synthetic text dataset...");
    let dataset = TextDataset::create_synthetic_dataset(num_samples, num_classes, max_length);
    let (train_dataset, val_dataset) = dataset.train_val_split(0.2);

    println!("   - Vocabulary size: {}", dataset.vocab.vocab_size);
    println!("   - Training samples: {}", train_dataset.len());
    println!("   - Validation samples: {}", val_dataset.len());

    // Show some example texts
    println!("\n📄 Sample texts:");
    for i in 0..3.min(train_dataset.texts.len()) {
        println!(
            "   [Class {}]: {}",
            train_dataset.labels[i], train_dataset.texts[i]
        );
    }

    // Build model
    println!("\n🏗️  Building text classification model...");
    let model = build_text_model(
        dataset.vocab.vocab_size,
        embedding_dim,
        hidden_dim,
        num_classes,
        max_length,
        &mut rng,
    )?;

    let total_params: usize = model.params().iter().map(|p| p.len()).sum();
    println!("   - Model layers: {}", model.len());
    println!("   - Total parameters: {}", total_params);

    // Training configuration
    let config = TrainingConfig {
        batch_size: 16,
        epochs: 30,
        learning_rate: 0.001,
        shuffle: true,
        verbose: 1,
        validation: Some(ValidationSettings {
            enabled: true,
            validation_split: 0.2,
            batch_size: 32,
            num_workers: 0,
        }),
        gradient_accumulation: None,
        mixed_precision: None,
        num_workers: 0,
    };

    println!("\n⚙️  Training Configuration:");
    println!("   - Batch size: {}", config.batch_size);
    println!("   - Learning rate: {}", config.learning_rate);
    println!("   - Epochs: {}", config.epochs);

    // Set up training
    let loss_fn = CrossEntropyLoss::new(1e-7);
    let optimizer = Adam::new(config.learning_rate as f32, 0.9, 0.999, 1e-8);

    let mut trainer = Trainer::new(model, optimizer, loss_fn, config);

    // Train the model
    println!("\n🏋️  Starting training...");
    println!("{}", "-".repeat(40));

    let training_session = trainer.train(&train_dataset, Some(&val_dataset))?;

    println!("\n✅ Training completed!");
    println!("   - Epochs trained: {}", training_session.epochs_trained);

    // Evaluate model
    println!("\n📊 Final Evaluation:");
    let val_metrics = trainer.validate(&val_dataset)?;

    for (metric, value) in &val_metrics {
        println!("   - {}: {:.4}", metric, value);
    }

    // Test on sample texts
    println!("\n🔍 Sample Predictions:");
    let sample_indices = vec![0, 1, 2, 3, 4];

    // Manually collect batch since get_batch is not part of Dataset trait
    let mut batch_tokens = Vec::new();
    let mut batch_targets = Vec::new();

    for &idx in &sample_indices {
        let (tokens, targets) = val_dataset.get(idx)?;
        batch_tokens.push(tokens);
        batch_targets.push(targets);
    }

    // Concatenate into batch arrays
    let sample_tokens = ndarray::concatenate(
        ndarray::Axis(0),
        &batch_tokens.iter().map(|a| a.view()).collect::<Vec<_>>(),
    )?;
    let sample_targets = ndarray::concatenate(
        ndarray::Axis(0),
        &batch_targets.iter().map(|a| a.view()).collect::<Vec<_>>(),
    )?;

    let model = trainer.get_model();
    let predictions = model.forward(&sample_tokens)?;

    let class_names = ["Positive", "Negative", "Neutral"];

    for i in 0..sample_indices.len().min(val_dataset.texts.len()) {
        let pred_row = predictions.slice(s![i, ..]);
        let target_row = sample_targets.slice(s![i, ..]);

        let pred_class = pred_row
            .iter()
            .enumerate()
            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())
            .map(|(i, _)| i)
            .unwrap_or(0);

        let true_class = target_row
            .iter()
            .enumerate()
            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap())
            .map(|(i, _)| i)
            .unwrap_or(0);

        let confidence = pred_row[pred_class];

        if sample_indices[i] < val_dataset.texts.len() {
            println!("   Text: \"{}\"", val_dataset.texts[sample_indices[i]]);
            println!(
                "   Predicted: {} (confidence: {:.3})",
                class_names[pred_class], confidence
            );
            println!("   Actual: {}", class_names[true_class]);
            println!();
        }
    }

    // Calculate detailed metrics
    let detailed_metrics = calculate_text_metrics(&predictions, &sample_targets);
    println!("📈 Detailed Metrics:");
    for (metric, value) in &detailed_metrics {
        println!("   - {}: {:.4}", metric, value);
    }

    Ok(())
}

Additional examples can be found in:

• examples/image_classification_complete.rs
• examples/advanced_optimizers_example.rs
• examples/training_loop_example.rs

Trait Implementations

impl Clone for CrossEntropyLoss

fn clone(&self) -> CrossEntropyLoss

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CrossEntropyLoss

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for CrossEntropyLoss

fn default() -> Self

Returns the “default value” for a type.

impl<F: Float + Debug> Loss<F> for CrossEntropyLoss

fn forward( &self, predictions: &Array<F, IxDyn>, targets: &Array<F, IxDyn>, ) -> Result<F>

Calculate the loss between predictions and targets

fn backward( &self, predictions: &Array<F, IxDyn>, targets: &Array<F, IxDyn>, ) -> Result<Array<F, IxDyn>>

Calculate the gradient of the loss with respect to the predictions

impl Copy for CrossEntropyLoss