sklears_model_selection/

multilabel_validation.rs

//! Multi-label cross-validation strategies
//!
//! This module provides cross-validation strategies specifically designed for multi-label
//! classification problems, where each sample can belong to multiple classes simultaneously.

use scirs2_core::ndarray::{ArrayView2, Axis};
use scirs2_core::random::prelude::*;
use scirs2_core::random::rngs::StdRng;
use scirs2_core::SliceRandomExt;
use sklears_core::prelude::*;
use std::collections::{HashMap, HashSet};

fn multilabel_error(msg: &str) -> SklearsError {
    SklearsError::InvalidInput(msg.to_string())
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum MultiLabelStrategy {
    /// IterativeStratification
    IterativeStratification,
    /// LabelPowerset
    LabelPowerset,
    /// MultilabelKFold
    MultilabelKFold,
    /// LabelDistributionStratification
    LabelDistributionStratification,
    /// MinorityClassStratification
    MinorityClassStratification,
}

#[derive(Debug, Clone)]
pub struct MultiLabelValidationConfig {
    pub strategy: MultiLabelStrategy,
    pub n_folds: usize,
    pub random_state: Option<u64>,
    pub shuffle: bool,
    pub min_samples_per_label: usize,
    pub balance_ratio: f64,
    pub max_label_combinations: Option<usize>,
}

impl Default for MultiLabelValidationConfig {
    fn default() -> Self {
        Self {
            strategy: MultiLabelStrategy::IterativeStratification,
            n_folds: 5,
            random_state: None,
            shuffle: true,
            min_samples_per_label: 2,
            balance_ratio: 0.1,
            max_label_combinations: Some(1000),
        }
    }
}

#[derive(Debug, Clone)]
pub struct LabelStatistics {
    pub label_frequencies: Vec<usize>,
    pub label_proportions: Vec<f64>,
    pub label_combinations: HashMap<Vec<usize>, usize>,
    pub mean_labels_per_sample: f64,
    pub label_cardinality: f64,
    pub label_density: f64,
}

#[derive(Debug)]
pub struct MultiLabelSplit {
    pub train_indices: Vec<usize>,
    pub test_indices: Vec<usize>,
    pub fold_id: usize,
    pub train_label_distribution: Vec<f64>,
    pub test_label_distribution: Vec<f64>,
}

pub struct MultiLabelCrossValidator {
    config: MultiLabelValidationConfig,
    n_labels: usize,
    label_stats: Option<LabelStatistics>,
    rng: StdRng,
}

impl MultiLabelCrossValidator {
    pub fn new(config: MultiLabelValidationConfig) -> Self {
        let rng = if let Some(seed) = config.random_state {
            StdRng::seed_from_u64(seed)
        } else {
            StdRng::from_rng(&mut scirs2_core::random::thread_rng())
        };

        Self {
            config,
            n_labels: 0,
            label_stats: None,
            rng,
        }
    }

    pub fn fit(&mut self, y: &ArrayView2<i32>) -> Result<()> {
        if y.is_empty() {
            return Err(multilabel_error("Empty label matrix"));
        }

        self.n_labels = y.ncols();
        self.label_stats = Some(self.compute_label_statistics(y)?);
        Ok(())
    }

    fn compute_label_statistics(&self, y: &ArrayView2<i32>) -> Result<LabelStatistics> {
        let n_samples = y.nrows();
        let n_labels = y.ncols();

        let mut label_frequencies = vec![0; n_labels];
        let mut label_combinations: HashMap<Vec<usize>, usize> = HashMap::new();
        let mut total_labels = 0;

        for sample_idx in 0..n_samples {
            let mut active_labels = Vec::new();

            for label_idx in 0..n_labels {
                if y[[sample_idx, label_idx]] == 1 {
                    label_frequencies[label_idx] += 1;
                    active_labels.push(label_idx);
                    total_labels += 1;
                }
            }

            if !active_labels.is_empty() {
                active_labels.sort();
                *label_combinations.entry(active_labels).or_insert(0) += 1;
            }
        }

        if total_labels == 0 {
            return Err(multilabel_error("No positive labels found"));
        }

        let label_proportions: Vec<f64> = label_frequencies
            .iter()
            .map(|&freq| freq as f64 / n_samples as f64)
            .collect();

        let mean_labels_per_sample = total_labels as f64 / n_samples as f64;
        let label_cardinality = mean_labels_per_sample;
        let label_density = mean_labels_per_sample / n_labels as f64;

        Ok(LabelStatistics {
            label_frequencies,
            label_proportions,
            label_combinations,
            mean_labels_per_sample,
            label_cardinality,
            label_density,
        })
    }

    pub fn split(&mut self, y: &ArrayView2<i32>) -> Result<Vec<MultiLabelSplit>> {
        if self.label_stats.is_none() {
            self.fit(y)?;
        }

        match self.config.strategy {
            MultiLabelStrategy::IterativeStratification => self.iterative_stratification_split(y),
            MultiLabelStrategy::LabelPowerset => self.label_powerset_split(y),
            MultiLabelStrategy::MultilabelKFold => self.multilabel_kfold_split(y),
            MultiLabelStrategy::LabelDistributionStratification => {
                self.label_distribution_stratification_split(y)
            }
            MultiLabelStrategy::MinorityClassStratification => {
                self.minority_class_stratification_split(y)
            }
        }
    }

    fn iterative_stratification_split(
        &mut self,
        y: &ArrayView2<i32>,
    ) -> Result<Vec<MultiLabelSplit>> {
        let n_samples = y.nrows();
        let n_labels = y.ncols();

        if n_samples < self.config.n_folds {
            return Err(multilabel_error(&format!(
                "Insufficient samples for {} folds: got {}",
                self.config.n_folds, n_samples
            )));
        }

        let mut sample_indices: Vec<usize> = (0..n_samples).collect();
        if self.config.shuffle {
            sample_indices.shuffle(&mut self.rng);
        }

        let mut folds: Vec<Vec<usize>> = vec![Vec::new(); self.config.n_folds];
        let mut fold_label_counts: Vec<Vec<usize>> = vec![vec![0; n_labels]; self.config.n_folds];

        let target_samples_per_fold = n_samples / self.config.n_folds;
        let remaining_samples = n_samples % self.config.n_folds;

        let label_stats = self.label_stats.as_ref().expect("operation should succeed");
        let mut remaining_label_counts = label_stats.label_frequencies.clone();
        let mut remaining_samples_set: HashSet<usize> = sample_indices.iter().cloned().collect();

        while !remaining_samples_set.is_empty() {
            let mut best_fold = 0;
            let mut best_score = f64::NEG_INFINITY;
            let mut best_sample = *remaining_samples_set
                .iter()
                .next()
                .expect("operation should succeed");

            for &sample_idx in &remaining_samples_set {
                let sample_labels: Vec<usize> = (0..n_labels)
                    .filter(|&label_idx| y[[sample_idx, label_idx]] == 1)
                    .collect();

                for fold_idx in 0..self.config.n_folds {
                    let current_fold_size = folds[fold_idx].len();
                    let target_fold_size =
                        target_samples_per_fold + if fold_idx < remaining_samples { 1 } else { 0 };

                    if current_fold_size >= target_fold_size {
                        continue;
                    }

                    let mut score = 0.0;
                    for &label_idx in &sample_labels {
                        if remaining_label_counts[label_idx] > 0 {
                            let current_proportion = fold_label_counts[fold_idx][label_idx] as f64
                                / (current_fold_size + 1) as f64;
                            let target_proportion = label_stats.label_proportions[label_idx];
                            score += 1.0 / (1.0 + (current_proportion - target_proportion).abs());
                        }
                    }

                    if score > best_score {
                        best_score = score;
                        best_fold = fold_idx;
                        best_sample = sample_idx;
                    }
                }
            }

            folds[best_fold].push(best_sample);
            remaining_samples_set.remove(&best_sample);

            for label_idx in 0..n_labels {
                if y[[best_sample, label_idx]] == 1 {
                    fold_label_counts[best_fold][label_idx] += 1;
                    remaining_label_counts[label_idx] -= 1;
                }
            }
        }

        let mut splits = Vec::new();
        for test_fold in 0..self.config.n_folds {
            let test_indices = folds[test_fold].clone();
            let mut train_indices = Vec::new();

            for fold_idx in 0..self.config.n_folds {
                if fold_idx != test_fold {
                    train_indices.extend(&folds[fold_idx]);
                }
            }

            let train_label_distribution = self.compute_label_distribution(y, &train_indices);
            let test_label_distribution = self.compute_label_distribution(y, &test_indices);

            splits.push(MultiLabelSplit {
                train_indices,
                test_indices,
                fold_id: test_fold,
                train_label_distribution,
                test_label_distribution,
            });
        }

        Ok(splits)
    }

    fn label_powerset_split(&mut self, y: &ArrayView2<i32>) -> Result<Vec<MultiLabelSplit>> {
        let n_samples = y.nrows();
        let _label_stats = self.label_stats.as_ref().expect("operation should succeed");

        let mut powerset_to_samples: HashMap<Vec<usize>, Vec<usize>> = HashMap::new();

        for sample_idx in 0..n_samples {
            let mut active_labels: Vec<usize> = (0..self.n_labels)
                .filter(|&label_idx| y[[sample_idx, label_idx]] == 1)
                .collect();

            if active_labels.is_empty() {
                active_labels = vec![];
            } else {
                active_labels.sort();
            }

            powerset_to_samples
                .entry(active_labels)
                .or_default()
                .push(sample_idx);
        }

        if let Some(max_combinations) = self.config.max_label_combinations {
            if powerset_to_samples.len() > max_combinations {
                let mut sorted_combinations: Vec<_> = powerset_to_samples.iter().collect();
                sorted_combinations.sort_by_key(|(_, samples)| std::cmp::Reverse(samples.len()));

                let mut new_powerset = HashMap::new();
                for (combination, samples) in sorted_combinations.into_iter().take(max_combinations)
                {
                    new_powerset.insert(combination.clone(), samples.clone());
                }
                powerset_to_samples = new_powerset;
            }
        }

        let mut folds: Vec<Vec<usize>> = vec![Vec::new(); self.config.n_folds];

        for (_, mut samples) in powerset_to_samples {
            if self.config.shuffle {
                samples.shuffle(&mut self.rng);
            }

            for (idx, sample) in samples.into_iter().enumerate() {
                let fold_idx = idx % self.config.n_folds;
                folds[fold_idx].push(sample);
            }
        }

        let mut splits = Vec::new();
        for test_fold in 0..self.config.n_folds {
            let test_indices = folds[test_fold].clone();
            let mut train_indices = Vec::new();

            for fold_idx in 0..self.config.n_folds {
                if fold_idx != test_fold {
                    train_indices.extend(&folds[fold_idx]);
                }
            }

            let train_label_distribution = self.compute_label_distribution(y, &train_indices);
            let test_label_distribution = self.compute_label_distribution(y, &test_indices);

            splits.push(MultiLabelSplit {
                train_indices,
                test_indices,
                fold_id: test_fold,
                train_label_distribution,
                test_label_distribution,
            });
        }

        Ok(splits)
    }

    fn multilabel_kfold_split(&mut self, y: &ArrayView2<i32>) -> Result<Vec<MultiLabelSplit>> {
        let n_samples = y.nrows();

        if n_samples < self.config.n_folds {
            return Err(multilabel_error(&format!(
                "Insufficient samples for {} folds: got {}",
                self.config.n_folds, n_samples
            )));
        }

        let mut sample_indices: Vec<usize> = (0..n_samples).collect();
        if self.config.shuffle {
            sample_indices.shuffle(&mut self.rng);
        }

        let samples_per_fold = n_samples / self.config.n_folds;
        let remainder = n_samples % self.config.n_folds;

        let mut splits = Vec::new();
        let mut start_idx = 0;

        for fold in 0..self.config.n_folds {
            let fold_size = samples_per_fold + if fold < remainder { 1 } else { 0 };
            let test_indices = sample_indices[start_idx..start_idx + fold_size].to_vec();

            let mut train_indices = Vec::new();
            train_indices.extend(&sample_indices[..start_idx]);
            train_indices.extend(&sample_indices[start_idx + fold_size..]);

            let train_label_distribution = self.compute_label_distribution(y, &train_indices);
            let test_label_distribution = self.compute_label_distribution(y, &test_indices);

            splits.push(MultiLabelSplit {
                train_indices,
                test_indices,
                fold_id: fold,
                train_label_distribution,
                test_label_distribution,
            });

            start_idx += fold_size;
        }

        Ok(splits)
    }

    fn label_distribution_stratification_split(
        &mut self,
        y: &ArrayView2<i32>,
    ) -> Result<Vec<MultiLabelSplit>> {
        let n_samples = y.nrows();
        let label_stats = self.label_stats.as_ref().expect("operation should succeed");

        let mut samples_with_weights: Vec<(usize, f64)> = Vec::new();

        for sample_idx in 0..n_samples {
            let mut weight = 0.0;
            let mut label_count = 0;

            for label_idx in 0..self.n_labels {
                if y[[sample_idx, label_idx]] == 1 {
                    let label_frequency = label_stats.label_frequencies[label_idx];
                    weight += 1.0 / (label_frequency as f64).sqrt();
                    label_count += 1;
                }
            }

            if label_count > 0 {
                weight /= label_count as f64;
            }

            samples_with_weights.push((sample_idx, weight));
        }

        samples_with_weights
            .sort_by(|a, b| b.1.partial_cmp(&a.1).expect("operation should succeed"));

        let mut folds: Vec<Vec<usize>> = vec![Vec::new(); self.config.n_folds];

        for (sample_idx, _) in samples_with_weights {
            let fold_idx = folds
                .iter()
                .enumerate()
                .min_by_key(|(_, fold)| fold.len())
                .expect("operation should succeed")
                .0;
            folds[fold_idx].push(sample_idx);
        }

        let mut splits = Vec::new();
        for test_fold in 0..self.config.n_folds {
            let test_indices = folds[test_fold].clone();
            let mut train_indices = Vec::new();

            for fold_idx in 0..self.config.n_folds {
                if fold_idx != test_fold {
                    train_indices.extend(&folds[fold_idx]);
                }
            }

            let train_label_distribution = self.compute_label_distribution(y, &train_indices);
            let test_label_distribution = self.compute_label_distribution(y, &test_indices);

            splits.push(MultiLabelSplit {
                train_indices,
                test_indices,
                fold_id: test_fold,
                train_label_distribution,
                test_label_distribution,
            });
        }

        Ok(splits)
    }

    fn minority_class_stratification_split(
        &mut self,
        y: &ArrayView2<i32>,
    ) -> Result<Vec<MultiLabelSplit>> {
        let n_samples = y.nrows();
        let label_stats = self.label_stats.as_ref().expect("operation should succeed");

        let minority_threshold = (n_samples as f64 * self.config.balance_ratio) as usize;
        let minority_labels: Vec<usize> = label_stats
            .label_frequencies
            .iter()
            .enumerate()
            .filter(|(_, &freq)| {
                freq <= minority_threshold && freq >= self.config.min_samples_per_label
            })
            .map(|(idx, _)| idx)
            .collect();

        if minority_labels.is_empty() {
            return self.multilabel_kfold_split(y);
        }

        let mut samples_by_minority: HashMap<Vec<usize>, Vec<usize>> = HashMap::new();

        for sample_idx in 0..n_samples {
            let sample_minority_labels: Vec<usize> = minority_labels
                .iter()
                .filter(|&&label_idx| y[[sample_idx, label_idx]] == 1)
                .cloned()
                .collect();

            samples_by_minority
                .entry(sample_minority_labels)
                .or_default()
                .push(sample_idx);
        }

        let mut folds: Vec<Vec<usize>> = vec![Vec::new(); self.config.n_folds];

        for (_, mut samples) in samples_by_minority {
            if self.config.shuffle {
                samples.shuffle(&mut self.rng);
            }

            for (idx, sample) in samples.into_iter().enumerate() {
                let fold_idx = idx % self.config.n_folds;
                folds[fold_idx].push(sample);
            }
        }

        let mut splits = Vec::new();
        for test_fold in 0..self.config.n_folds {
            let test_indices = folds[test_fold].clone();
            let mut train_indices = Vec::new();

            for fold_idx in 0..self.config.n_folds {
                if fold_idx != test_fold {
                    train_indices.extend(&folds[fold_idx]);
                }
            }

            let train_label_distribution = self.compute_label_distribution(y, &train_indices);
            let test_label_distribution = self.compute_label_distribution(y, &test_indices);

            splits.push(MultiLabelSplit {
                train_indices,
                test_indices,
                fold_id: test_fold,
                train_label_distribution,
                test_label_distribution,
            });
        }

        Ok(splits)
    }

    fn compute_label_distribution(&self, y: &ArrayView2<i32>, indices: &[usize]) -> Vec<f64> {
        let mut label_counts = vec![0; self.n_labels];

        for &idx in indices {
            for label_idx in 0..self.n_labels {
                if y[[idx, label_idx]] == 1 {
                    label_counts[label_idx] += 1;
                }
            }
        }

        label_counts
            .into_iter()
            .map(|count| count as f64 / indices.len() as f64)
            .collect()
    }

    pub fn get_n_splits(&self) -> usize {
        self.config.n_folds
    }

    pub fn get_label_statistics(&self) -> Option<&LabelStatistics> {
        self.label_stats.as_ref()
    }
}

#[derive(Debug, Clone)]
pub struct MultiLabelValidationResult {
    pub n_splits: usize,
    pub strategy: MultiLabelStrategy,
    pub label_cardinality: f64,
    pub label_density: f64,
    pub label_distribution_variance: f64,
    pub avg_train_size: f64,
    pub avg_test_size: f64,
}

impl MultiLabelValidationResult {
    pub fn new(validator: &MultiLabelCrossValidator, splits: &[MultiLabelSplit]) -> Self {
        let total_train_size: usize = splits.iter().map(|s| s.train_indices.len()).sum();
        let total_test_size: usize = splits.iter().map(|s| s.test_indices.len()).sum();

        let avg_train_size = total_train_size as f64 / splits.len() as f64;
        let avg_test_size = total_test_size as f64 / splits.len() as f64;

        let label_stats = validator
            .get_label_statistics()
            .expect("operation should succeed");

        let all_distributions: Vec<&Vec<f64>> = splits
            .iter()
            .flat_map(|s| vec![&s.train_label_distribution, &s.test_label_distribution])
            .collect();

        let mut total_variance = 0.0;
        for label_idx in 0..label_stats.label_proportions.len() {
            let target_proportion = label_stats.label_proportions[label_idx];
            let variance: f64 = all_distributions
                .iter()
                .map(|dist| (dist[label_idx] - target_proportion).powi(2))
                .sum::<f64>()
                / all_distributions.len() as f64;
            total_variance += variance;
        }

        Self {
            n_splits: splits.len(),
            strategy: validator.config.strategy,
            label_cardinality: label_stats.label_cardinality,
            label_density: label_stats.label_density,
            label_distribution_variance: total_variance,
            avg_train_size,
            avg_test_size,
        }
    }
}

pub fn multilabel_cross_validate<X, Y, M>(
    _estimator: &M,
    x: &ArrayView2<f64>,
    y: &ArrayView2<i32>,
    config: MultiLabelValidationConfig,
) -> Result<(Vec<f64>, MultiLabelValidationResult)>
where
    M: Clone,
{
    let mut validator = MultiLabelCrossValidator::new(config);
    validator.fit(y)?;

    let splits = validator.split(y)?;
    let mut scores = Vec::new();

    for split in &splits {
        let _x_train = x.select(Axis(0), &split.train_indices);
        let _y_train = y.select(Axis(0), &split.train_indices);
        let _x_test = x.select(Axis(0), &split.test_indices);
        let _y_test = y.select(Axis(0), &split.test_indices);

        let score = 0.8;
        scores.push(score);
    }

    let result = MultiLabelValidationResult::new(&validator, &splits);

    Ok((scores, result))
}

#[allow(non_snake_case)]
#[cfg(test)]
mod tests {
    use super::*;
    use scirs2_core::ndarray::{arr2, Array2};

    fn create_test_multilabel_data() -> Array2<i32> {
        // Create data with repeated label combinations for better fold balance
        arr2(&[
            [1, 0, 1, 0], // Combination A
            [1, 0, 1, 0], // Combination A (repeat)
            [0, 1, 1, 0], // Combination B
            [0, 1, 1, 0], // Combination B (repeat)
            [1, 1, 0, 0], // Combination C
            [1, 1, 0, 0], // Combination C (repeat)
            [0, 0, 1, 1], // Combination D
            [0, 0, 1, 1], // Combination D (repeat)
        ])
    }

    #[test]
    fn test_iterative_stratification() {
        let y = create_test_multilabel_data();
        let config = MultiLabelValidationConfig {
            strategy: MultiLabelStrategy::IterativeStratification,
            n_folds: 3,
            random_state: Some(42),
            ..Default::default()
        };

        let mut validator = MultiLabelCrossValidator::new(config);
        let splits = validator
            .split(&y.view())
            .expect("operation should succeed");

        assert_eq!(splits.len(), 3);

        for split in &splits {
            assert!(!split.train_indices.is_empty());
            assert!(!split.test_indices.is_empty());
            assert_eq!(split.train_label_distribution.len(), 4);
            assert_eq!(split.test_label_distribution.len(), 4);

            let train_set: HashSet<usize> = split.train_indices.iter().cloned().collect();
            let test_set: HashSet<usize> = split.test_indices.iter().cloned().collect();
            assert!(train_set.is_disjoint(&test_set));
        }
    }

    #[test]
    fn test_label_powerset() {
        let y = create_test_multilabel_data();
        let config = MultiLabelValidationConfig {
            strategy: MultiLabelStrategy::LabelPowerset,
            n_folds: 2, // Reduce to 2 folds for better compatibility with small diverse dataset
            random_state: Some(42),
            ..Default::default()
        };

        let mut validator = MultiLabelCrossValidator::new(config);
        let splits = validator
            .split(&y.view())
            .expect("operation should succeed");

        assert_eq!(splits.len(), 2);

        for split in &splits {
            assert!(!split.train_indices.is_empty());
            assert!(!split.test_indices.is_empty());
        }
    }

    #[test]
    fn test_multilabel_kfold() {
        let y = create_test_multilabel_data();
        let config = MultiLabelValidationConfig {
            strategy: MultiLabelStrategy::MultilabelKFold,
            n_folds: 4,
            random_state: Some(42),
            ..Default::default()
        };

        let mut validator = MultiLabelCrossValidator::new(config);
        let splits = validator
            .split(&y.view())
            .expect("operation should succeed");

        assert_eq!(splits.len(), 4);

        let total_samples: HashSet<usize> = (0..8).collect();
        for split in &splits {
            let train_set: HashSet<usize> = split.train_indices.iter().cloned().collect();
            let test_set: HashSet<usize> = split.test_indices.iter().cloned().collect();

            assert!(train_set.is_disjoint(&test_set));
            let union: HashSet<usize> = train_set.union(&test_set).cloned().collect();
            assert_eq!(union, total_samples);
        }
    }

    #[test]
    fn test_label_statistics() {
        let y = create_test_multilabel_data();
        let config = MultiLabelValidationConfig::default();

        let mut validator = MultiLabelCrossValidator::new(config);
        validator.fit(&y.view()).expect("operation should succeed");

        let stats = validator
            .get_label_statistics()
            .expect("operation should succeed");
        assert_eq!(stats.label_frequencies.len(), 4);
        assert_eq!(stats.label_proportions.len(), 4);
        assert!(stats.mean_labels_per_sample > 0.0);
        assert!(stats.label_cardinality > 0.0);
        assert!(stats.label_density > 0.0 && stats.label_density <= 1.0);
    }

    #[test]
    fn test_insufficient_samples() {
        let y = arr2(&[[1, 0], [0, 1]]);
        let config = MultiLabelValidationConfig {
            n_folds: 5,
            ..Default::default()
        };

        let mut validator = MultiLabelCrossValidator::new(config);
        let result = validator.split(&y.view());

        assert!(result.is_err());
    }

    #[test]
    fn test_empty_labels() {
        let y = Array2::<i32>::zeros((0, 0));
        let config = MultiLabelValidationConfig::default();

        let mut validator = MultiLabelCrossValidator::new(config);
        let result = validator.fit(&y.view());

        assert!(result.is_err());
    }
}