Struct ValidationHandler

pub struct ValidationHandler<F: Float + Debug + ScalarOperand + Display + FromPrimitive + Send + Sync> {
    pub config: ValidationConfig,
    /* private fields */
}

Validation handler for model validation during training

Fields

config: ValidationConfig

Configuration for validation

Implementations

impl<F: Float + Debug + ScalarOperand + Display + FromPrimitive + Send + Sync> ValidationHandler<F>

pub fn new(config: ValidationConfig) -> Result<Self>

Create a new validation handler

Examples found in repository

examples/model_evaluation_example.rs (line 257)

fn main() -> Result<()> {
    println!("Model Evaluation Framework Example");
    println!("---------------------------------");

    // 1. Basic evaluation
    println!("\n1. Basic Evaluation:");

    // Generate synthetic regression dataset
    let dataset = generate_regression_dataset::<f32>(1000, 5)?;

    // Split into train, validation, and test sets
    let n_samples = dataset.len();
    let train_size = n_samples * 6 / 10;
    let val_size = n_samples * 2 / 10;
    let _test_size = n_samples - train_size - val_size;

    let mut indices: Vec<usize> = (0..n_samples).collect();
    use rand::seq::SliceRandom;
    let mut shuffle_rng = SmallRng::seed_from_u64(44);
    indices.shuffle(&mut shuffle_rng);

    let train_indices = indices[0..train_size].to_vec();
    let val_indices = indices[train_size..train_size + val_size].to_vec();
    let test_indices = indices[train_size + val_size..].to_vec();

    println!(
        "Dataset splits: Train={}, Validation={}, Test={}",
        train_indices.len(),
        val_indices.len(),
        test_indices.len()
    );

    // Create subset datasets
    let _train_dataset = SubsetDataset::new(dataset.clone(), train_indices.clone())?;
    let val_dataset = SubsetDataset::new(dataset.clone(), val_indices.clone())?;
    let test_dataset = SubsetDataset::new(dataset.clone(), test_indices.clone())?;

    // Build a simple model
    let model_builder = SimpleModelBuilder::<f32>::new(5, 32, 1);
    let mut model = model_builder.build()?;

    // Create loss function
    let loss_fn = MeanSquaredError::new();

    // Create evaluator
    let eval_config = EvaluationConfig {
        batch_size: 32,
        shuffle: false,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::MeanAbsoluteError,
            MetricType::RSquared,
        ],
        steps: None,
        verbose: 1,
    };

    let mut evaluator = Evaluator::new(eval_config)?;

    // Evaluate model on validation set
    println!("\nEvaluating model on validation set:");
    let val_metrics = evaluator.evaluate(&mut model, &val_dataset, Some(&loss_fn))?;

    println!("Validation metrics:");
    for (name, value) in &val_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // 2. Validation with early stopping
    println!("\n2. Validation with Early Stopping:");

    // Configure early stopping
    let early_stopping_config = EarlyStoppingConfig {
        monitor: "val_loss".to_string(),
        min_delta: 0.001,
        patience: 5,
        restore_best_weights: true,
        mode: EarlyStoppingMode::Min,
    };

    let validation_config = ValidationConfig {
        batch_size: 32,
        shuffle: false,
        num_workers: 0,
        steps: None,
        metrics: vec![MetricType::Loss, MetricType::MeanSquaredError],
        verbose: 1,
        early_stopping: Some(early_stopping_config),
    };

    let mut validation_handler = ValidationHandler::new(validation_config)?;

    // Simulate training loop with validation
    println!("\nSimulating training loop with validation:");
    let num_epochs = 10;

    for epoch in 0..num_epochs {
        println!("Epoch {}/{}", epoch + 1, num_epochs);

        // Simulate training step (not actually training the model)
        println!("Training...");

        // Validate model
        let (val_metrics, should_stop) =
            validation_handler.validate(&mut model, &val_dataset, Some(&loss_fn), epoch)?;

        println!("Validation metrics:");
        for (name, value) in &val_metrics {
            println!("  {}: {:.4}", name, value);
        }

        if should_stop {
            println!("Early stopping triggered!");
            break;
        }
    }

    // 3. Cross-validation
    println!("\n3. Cross-Validation:");

    // Configure cross-validation
    let cv_config = CrossValidationConfig {
        strategy: CrossValidationStrategy::KFold(5),
        shuffle: true,
        random_seed: Some(42),
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::RSquared,
        ],
        verbose: 1,
    };

    let mut cross_validator = CrossValidator::new(cv_config)?;

    // Perform cross-validation
    println!("\nPerforming 5-fold cross-validation:");
    let cv_results = cross_validator.cross_validate(&model_builder, &dataset, Some(&loss_fn))?;

    println!("Cross-validation results:");
    for (name, values) in &cv_results {
        // Calculate mean and std
        let sum: f32 = values.iter().sum();
        let mean = sum / values.len() as f32;

        let variance_sum: f32 = values.iter().map(|&x| (x - mean).powi(2)).sum();
        let std = (variance_sum / values.len() as f32).sqrt();

        println!("  {}: {:.4} ± {:.4}", name, mean, std);
    }

    // 4. Test set evaluation
    println!("\n4. Test Set Evaluation:");

    // Configure test evaluator
    let test_config = TestConfig {
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::MeanAbsoluteError,
            MetricType::RSquared,
        ],
        steps: None,
        verbose: 1,
        generate_predictions: true,
        save_outputs: false,
    };

    let mut test_evaluator = TestEvaluator::new(test_config)?;

    // Evaluate model on test set
    println!("\nEvaluating model on test set:");
    let test_metrics = test_evaluator.evaluate(&mut model, &test_dataset, Some(&loss_fn))?;

    println!("Test metrics:");
    for (name, value) in &test_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // 5. Classification example
    println!("\n5. Classification Example:");

    // Generate synthetic classification dataset
    let n_classes = 3;
    let class_dataset = generate_classification_dataset::<f32>(1000, 5, n_classes)?;

    // Split dataset
    let _class_train_dataset = SubsetDataset::new(class_dataset.clone(), train_indices.clone())?;
    let class_test_dataset = SubsetDataset::new(class_dataset.clone(), test_indices.clone())?;

    // Build classification model
    let class_model_builder = SimpleModelBuilder::<f32>::new(5, 32, n_classes);
    let mut class_model = class_model_builder.build()?;

    // Configure test evaluator for classification
    let class_test_config = TestConfig {
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Accuracy,
            MetricType::Precision,
            MetricType::Recall,
            MetricType::F1Score,
        ],
        steps: None,
        verbose: 1,
        generate_predictions: true,
        save_outputs: false,
    };

    let mut class_test_evaluator = TestEvaluator::new(class_test_config)?;

    // Evaluate classification model
    println!("\nEvaluating classification model:");
    let class_metrics =
        class_test_evaluator.evaluate(&mut class_model, &class_test_dataset, None)?;

    println!("Classification metrics:");
    for (name, value) in &class_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // Generate classification report
    println!("\nClassification Report:");
    match class_test_evaluator.classification_report() {
        Ok(report) => println!("{}", report),
        Err(e) => println!("Could not generate classification report: {}", e),
    }

    // Generate confusion matrix
    println!("\nConfusion Matrix:");
    match class_test_evaluator.confusion_matrix() {
        Ok(cm) => println!("{}", cm),
        Err(e) => println!("Could not generate confusion matrix: {}", e),
    }

    println!("\nModel Evaluation Example Completed Successfully!");

    Ok(())
}

pub fn validate<L: Layer<F>>( &mut self, model: &mut L, dataset: &dyn Dataset<F>, loss_fn: Option<&dyn Loss<F>>, epoch: usize, ) -> Result<(HashMap<String, F>, bool)>

Validate a model on a dataset

Examples found in repository

examples/model_evaluation_example.rs (line 271)

fn main() -> Result<()> {
    println!("Model Evaluation Framework Example");
    println!("---------------------------------");

    // 1. Basic evaluation
    println!("\n1. Basic Evaluation:");

    // Generate synthetic regression dataset
    let dataset = generate_regression_dataset::<f32>(1000, 5)?;

    // Split into train, validation, and test sets
    let n_samples = dataset.len();
    let train_size = n_samples * 6 / 10;
    let val_size = n_samples * 2 / 10;
    let _test_size = n_samples - train_size - val_size;

    let mut indices: Vec<usize> = (0..n_samples).collect();
    use rand::seq::SliceRandom;
    let mut shuffle_rng = SmallRng::seed_from_u64(44);
    indices.shuffle(&mut shuffle_rng);

    let train_indices = indices[0..train_size].to_vec();
    let val_indices = indices[train_size..train_size + val_size].to_vec();
    let test_indices = indices[train_size + val_size..].to_vec();

    println!(
        "Dataset splits: Train={}, Validation={}, Test={}",
        train_indices.len(),
        val_indices.len(),
        test_indices.len()
    );

    // Create subset datasets
    let _train_dataset = SubsetDataset::new(dataset.clone(), train_indices.clone())?;
    let val_dataset = SubsetDataset::new(dataset.clone(), val_indices.clone())?;
    let test_dataset = SubsetDataset::new(dataset.clone(), test_indices.clone())?;

    // Build a simple model
    let model_builder = SimpleModelBuilder::<f32>::new(5, 32, 1);
    let mut model = model_builder.build()?;

    // Create loss function
    let loss_fn = MeanSquaredError::new();

    // Create evaluator
    let eval_config = EvaluationConfig {
        batch_size: 32,
        shuffle: false,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::MeanAbsoluteError,
            MetricType::RSquared,
        ],
        steps: None,
        verbose: 1,
    };

    let mut evaluator = Evaluator::new(eval_config)?;

    // Evaluate model on validation set
    println!("\nEvaluating model on validation set:");
    let val_metrics = evaluator.evaluate(&mut model, &val_dataset, Some(&loss_fn))?;

    println!("Validation metrics:");
    for (name, value) in &val_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // 2. Validation with early stopping
    println!("\n2. Validation with Early Stopping:");

    // Configure early stopping
    let early_stopping_config = EarlyStoppingConfig {
        monitor: "val_loss".to_string(),
        min_delta: 0.001,
        patience: 5,
        restore_best_weights: true,
        mode: EarlyStoppingMode::Min,
    };

    let validation_config = ValidationConfig {
        batch_size: 32,
        shuffle: false,
        num_workers: 0,
        steps: None,
        metrics: vec![MetricType::Loss, MetricType::MeanSquaredError],
        verbose: 1,
        early_stopping: Some(early_stopping_config),
    };

    let mut validation_handler = ValidationHandler::new(validation_config)?;

    // Simulate training loop with validation
    println!("\nSimulating training loop with validation:");
    let num_epochs = 10;

    for epoch in 0..num_epochs {
        println!("Epoch {}/{}", epoch + 1, num_epochs);

        // Simulate training step (not actually training the model)
        println!("Training...");

        // Validate model
        let (val_metrics, should_stop) =
            validation_handler.validate(&mut model, &val_dataset, Some(&loss_fn), epoch)?;

        println!("Validation metrics:");
        for (name, value) in &val_metrics {
            println!("  {}: {:.4}", name, value);
        }

        if should_stop {
            println!("Early stopping triggered!");
            break;
        }
    }

    // 3. Cross-validation
    println!("\n3. Cross-Validation:");

    // Configure cross-validation
    let cv_config = CrossValidationConfig {
        strategy: CrossValidationStrategy::KFold(5),
        shuffle: true,
        random_seed: Some(42),
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::RSquared,
        ],
        verbose: 1,
    };

    let mut cross_validator = CrossValidator::new(cv_config)?;

    // Perform cross-validation
    println!("\nPerforming 5-fold cross-validation:");
    let cv_results = cross_validator.cross_validate(&model_builder, &dataset, Some(&loss_fn))?;

    println!("Cross-validation results:");
    for (name, values) in &cv_results {
        // Calculate mean and std
        let sum: f32 = values.iter().sum();
        let mean = sum / values.len() as f32;

        let variance_sum: f32 = values.iter().map(|&x| (x - mean).powi(2)).sum();
        let std = (variance_sum / values.len() as f32).sqrt();

        println!("  {}: {:.4} ± {:.4}", name, mean, std);
    }

    // 4. Test set evaluation
    println!("\n4. Test Set Evaluation:");

    // Configure test evaluator
    let test_config = TestConfig {
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Loss,
            MetricType::MeanSquaredError,
            MetricType::MeanAbsoluteError,
            MetricType::RSquared,
        ],
        steps: None,
        verbose: 1,
        generate_predictions: true,
        save_outputs: false,
    };

    let mut test_evaluator = TestEvaluator::new(test_config)?;

    // Evaluate model on test set
    println!("\nEvaluating model on test set:");
    let test_metrics = test_evaluator.evaluate(&mut model, &test_dataset, Some(&loss_fn))?;

    println!("Test metrics:");
    for (name, value) in &test_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // 5. Classification example
    println!("\n5. Classification Example:");

    // Generate synthetic classification dataset
    let n_classes = 3;
    let class_dataset = generate_classification_dataset::<f32>(1000, 5, n_classes)?;

    // Split dataset
    let _class_train_dataset = SubsetDataset::new(class_dataset.clone(), train_indices.clone())?;
    let class_test_dataset = SubsetDataset::new(class_dataset.clone(), test_indices.clone())?;

    // Build classification model
    let class_model_builder = SimpleModelBuilder::<f32>::new(5, 32, n_classes);
    let mut class_model = class_model_builder.build()?;

    // Configure test evaluator for classification
    let class_test_config = TestConfig {
        batch_size: 32,
        num_workers: 0,
        metrics: vec![
            MetricType::Accuracy,
            MetricType::Precision,
            MetricType::Recall,
            MetricType::F1Score,
        ],
        steps: None,
        verbose: 1,
        generate_predictions: true,
        save_outputs: false,
    };

    let mut class_test_evaluator = TestEvaluator::new(class_test_config)?;

    // Evaluate classification model
    println!("\nEvaluating classification model:");
    let class_metrics =
        class_test_evaluator.evaluate(&mut class_model, &class_test_dataset, None)?;

    println!("Classification metrics:");
    for (name, value) in &class_metrics {
        println!("  {}: {:.4}", name, value);
    }

    // Generate classification report
    println!("\nClassification Report:");
    match class_test_evaluator.classification_report() {
        Ok(report) => println!("{}", report),
        Err(e) => println!("Could not generate classification report: {}", e),
    }

    // Generate confusion matrix
    println!("\nConfusion Matrix:");
    match class_test_evaluator.confusion_matrix() {
        Ok(cm) => println!("{}", cm),
        Err(e) => println!("Could not generate confusion matrix: {}", e),
    }

    println!("\nModel Evaluation Example Completed Successfully!");

    Ok(())
}

pub fn has_early_stopping(&self) -> bool

Check if early stopping is enabled

pub fn get_early_stopping_state(&self) -> Option<&EarlyStoppingState<F>>

Get the current early stopping state

pub fn reset_early_stopping(&mut self)

Reset early stopping state

Trait Implementations

impl<F: Debug + Float + Debug + ScalarOperand + Display + FromPrimitive + Send + Sync> Debug for ValidationHandler<F>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.