//! Multi-output regression and classification
//!
//! This module provides meta-estimators for multi-target prediction problems.
//! It includes strategies for independent multi-output prediction.
#![warn(missing_docs)]
pub mod chains;
pub mod core;
pub mod correlation;
pub mod multi_label;
pub mod neural;
pub mod optimization;
pub mod probabilistic;
pub mod regularization;
pub mod transfer_learning;
pub mod tree;
pub mod utils;
use ndarray::{s, Array1, Array2, Array3, ArrayView1, ArrayView2, ArrayView3, Axis};
use ndarray_rand::rand_distr::Normal;
use ndarray_rand::RandomExt;
use rand::SeedableRng;
use rand_chacha::ChaCha8Rng;
use sklears_core::{
error::{Result as SklResult, SklearsError},
traits::{Estimator, Fit, Predict, Untrained},
types::Float,
};
use std::collections::HashMap;
use utils::*;
// Re-export core multi-output algorithms
pub use core::{
MultiOutputClassifier, MultiOutputClassifierTrained, MultiOutputRegressor,
MultiOutputRegressorTrained,
};
// Re-export chain-based algorithms
pub use chains::{
BayesianClassifierChain, BayesianClassifierChainTrained, ChainMethod, ClassifierChain,
ClassifierChainTrained, EnsembleOfChains, EnsembleOfChainsTrained, RegressorChain,
RegressorChainTrained,
};
// Re-export neural network algorithms
pub use neural::{
ActivationFunction, AdversarialConfig, AdversarialMultiTaskNetwork,
AdversarialMultiTaskNetworkTrained, AdversarialStrategy, AttentionConfig,
AttentionMultiTaskNetwork, AttentionMultiTaskNetworkTrained, AttentionOutput, AttentionType,
CellType, GradientReversalConfig, LambdaSchedule, LossFunction, MultiOutputMLP,
MultiOutputMLPClassifier, MultiOutputMLPRegressor, MultiOutputMLPTrained,
MultiTaskNeuralNetwork, MultiTaskNeuralNetworkTrained, RecurrentNeuralNetwork,
RecurrentNeuralNetworkTrained, SequenceMode, TaskBalancing, TaskDiscriminator,
};
// Re-export regularization algorithms
pub use regularization::{
GroupLasso, GroupLassoTrained, MetaLearningMultiTask, MetaLearningMultiTaskTrained,
MultiTaskElasticNet, MultiTaskElasticNetTrained, NuclearNormRegression,
NuclearNormRegressionTrained, RegularizationStrategy, TaskClusteringRegressionTrained,
TaskClusteringRegularization, TaskRelationshipLearning, TaskRelationshipLearningTrained,
TaskSimilarityMethod,
};
// Re-export correlation and dependency analysis
pub use correlation::{
CITestMethod, CITestResult, CITestResults, ConditionalIndependenceTester, CorrelationAnalysis,
CorrelationType, DependencyGraph, DependencyGraphBuilder, DependencyMethod, GraphStatistics,
OutputCorrelationAnalyzer,
};
// Re-export transfer learning algorithms
pub use transfer_learning::{
ContinualLearning, ContinualLearningTrained, CrossTaskTransferLearning,
CrossTaskTransferLearningTrained, DomainAdaptation, DomainAdaptationTrained,
KnowledgeDistillation, KnowledgeDistillationTrained, ProgressiveTransferLearning,
ProgressiveTransferLearningTrained,
};
// Re-export optimization algorithms
pub use optimization::{
JointLossConfig, JointLossOptimizer, JointLossOptimizerTrained, LossCombination,
LossFunction as OptimizationLossFunction, MultiObjectiveConfig, MultiObjectiveOptimizer,
MultiObjectiveOptimizerTrained, ParetoSolution, ScalarizationConfig, ScalarizationMethod, ScalarizationOptimizer,
ScalarizationOptimizerTrained,
};
// Re-export probabilistic algorithms
pub use probabilistic::{
BayesianMultiOutputConfig, BayesianMultiOutputModel, BayesianMultiOutputModelTrained,
GaussianProcessMultiOutput, GaussianProcessMultiOutputTrained, InferenceMethod, KernelFunction,
PosteriorDistribution, PredictionWithUncertainty, PriorDistribution,
};
// Re-export multi-label algorithms
pub use multi_label::{
BinaryRelevance, BinaryRelevanceTrained,
LabelPowerset, LabelPowersetTrained,
PrunedLabelPowerset, PrunedLabelPowersetTrained, PruningStrategy,
OneVsRestClassifier, OneVsRestClassifierTrained,
};
// Re-export tree-based algorithms
pub use tree::{
ClassificationCriterion, DAGInferenceMethod,
MultiTargetRegressionTree, MultiTargetRegressionTreeTrained,
MultiTargetDecisionTreeClassifier, MultiTargetDecisionTreeClassifierTrained,
RandomForestMultiOutput, RandomForestMultiOutputTrained,
TreeStructuredPredictor, TreeStructuredPredictorTrained,
};
// Note: MEMM types are defined in this file, no re-export needed
// Re-export instance-based learning algorithms (defined later in file)
// pub use {IBLR, IBLRTrained, WeightFunction};
/// Multi-output and multi-label evaluation metrics
pub mod metrics {
use ndarray::{Array2, ArrayView2};
use sklears_core::error::{Result as SklResult, SklearsError};
/// Hamming loss for multi-label classification
///
/// The Hamming loss is the fraction of the wrong labels to the total
/// number of labels. It is a multi-label generalization of the zero-one loss.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
///
/// # Returns
///
/// The Hamming loss between y_true and y_pred
pub fn hamming_loss(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_errors = 0;
let total_elements = n_samples * n_labels;
for sample_idx in 0..n_samples {
for label_idx in 0..n_labels {
if y_true[[sample_idx, label_idx]] != y_pred[[sample_idx, label_idx]] {
total_errors += 1;
}
}
}
Ok(total_errors as f64 / total_elements as f64)
}
/// Subset accuracy for multi-label classification
///
/// Subset accuracy is the most strict metric. It requires for each sample
/// that each label set be correctly predicted.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
///
/// # Returns
///
/// The subset accuracy between y_true and y_pred
pub fn subset_accuracy(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample".to_string(),
));
}
let mut correct_subsets = 0;
for sample_idx in 0..n_samples {
let mut subset_correct = true;
for label_idx in 0..n_labels {
if y_true[[sample_idx, label_idx]] != y_pred[[sample_idx, label_idx]] {
subset_correct = false;
break;
}
}
if subset_correct {
correct_subsets += 1;
}
}
Ok(correct_subsets as f64 / n_samples as f64)
}
/// Jaccard similarity coefficient for multi-label classification
///
/// The Jaccard similarity coefficient is defined as the size of the intersection
/// divided by the size of the union of the sample sets.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
///
/// # Returns
///
/// The Jaccard similarity coefficient
pub fn jaccard_score(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample".to_string(),
));
}
let mut total_jaccard = 0.0;
for sample_idx in 0..n_samples {
let mut intersection = 0;
let mut union = 0;
for label_idx in 0..n_labels {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
intersection += 1;
}
if true_label == 1 || pred_label == 1 {
union += 1;
}
}
// Jaccard = intersection / union, handle division by zero
let sample_jaccard = if union > 0 {
intersection as f64 / union as f64
} else {
1.0 // If both sets are empty, Jaccard = 1
};
total_jaccard += sample_jaccard;
}
Ok(total_jaccard / n_samples as f64)
}
/// F1 score for multi-label classification
///
/// Compute the F1 score for each label and return the specified average.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
/// * `average` - The averaging strategy ('micro', 'macro', 'samples')
///
/// # Returns
///
/// The F1 score according to the specified averaging strategy
pub fn f1_score(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
average: &str,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
match average {
"micro" => {
// Compute global precision and recall
let mut total_tp = 0;
let mut total_fp = 0;
let mut total_false_negatives = 0;
for sample_idx in 0..n_samples {
for label_idx in 0..n_labels {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
total_tp += 1;
} else if true_label == 0 && pred_label == 1 {
total_fp += 1;
} else if true_label == 1 && pred_label == 0 {
total_false_negatives += 1;
}
}
}
let precision = if total_tp + total_fp > 0 {
total_tp as f64 / (total_tp + total_fp) as f64
} else {
0.0
};
let recall = if total_tp + total_false_negatives > 0 {
total_tp as f64 / (total_tp + total_false_negatives) as f64
} else {
0.0
};
let f1 = if precision + recall > 0.0 {
2.0 * precision * recall / (precision + recall)
} else {
0.0
};
Ok(f1)
}
"macro" => {
// Compute F1 for each label and average
let mut label_f1_scores = Vec::new();
for label_idx in 0..n_labels {
let mut tp = 0;
let mut fp = 0;
let mut false_negatives = 0;
for sample_idx in 0..n_samples {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
tp += 1;
} else if true_label == 0 && pred_label == 1 {
fp += 1;
} else if true_label == 1 && pred_label == 0 {
false_negatives += 1;
}
}
let precision = if tp + fp > 0 {
tp as f64 / (tp + fp) as f64
} else {
0.0
};
let recall = if tp + false_negatives > 0 {
tp as f64 / (tp + false_negatives) as f64
} else {
0.0
};
let f1 = if precision + recall > 0.0 {
2.0 * precision * recall / (precision + recall)
} else {
0.0
};
label_f1_scores.push(f1);
}
Ok(label_f1_scores.iter().sum::<f64>() / n_labels as f64)
}
"samples" => {
// Compute F1 for each sample and average
let mut sample_f1_scores = Vec::new();
for sample_idx in 0..n_samples {
let mut tp = 0;
let mut fp = 0;
let mut false_negatives = 0;
for label_idx in 0..n_labels {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
tp += 1;
} else if true_label == 0 && pred_label == 1 {
fp += 1;
} else if true_label == 1 && pred_label == 0 {
false_negatives += 1;
}
}
let precision = if tp + fp > 0 {
tp as f64 / (tp + fp) as f64
} else {
0.0
};
let recall = if tp + false_negatives > 0 {
tp as f64 / (tp + false_negatives) as f64
} else {
0.0
};
let f1 = if precision + recall > 0.0 {
2.0 * precision * recall / (precision + recall)
} else {
0.0
};
sample_f1_scores.push(f1);
}
Ok(sample_f1_scores.iter().sum::<f64>() / n_samples as f64)
}
_ => Err(SklearsError::InvalidInput(format!(
"Unknown average type: {}. Valid options are 'micro', 'macro', 'samples'",
average
))),
}
}
/// Coverage error for multi-label ranking
///
/// Coverage error measures how far we need to go through the ranked scores
/// to cover all true labels. The best value is equal to the average number
/// of labels in y_true per sample.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_scores` - Target scores (predicted probabilities)
///
/// # Returns
///
/// The coverage error
pub fn coverage_error(
y_true: &ArrayView2<'_, i32>,
y_scores: &ArrayView2<'_, f64>,
) -> SklResult<f64> {
if y_true.dim() != y_scores.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_scores must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_coverage = 0.0;
for sample_idx in 0..n_samples {
// Get the indices sorted by scores in descending order
let mut score_label_pairs: Vec<(f64, usize)> = (0..n_labels)
.map(|label_idx| (y_scores[[sample_idx, label_idx]], label_idx))
.collect();
score_label_pairs
.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
// Find the position of the last true label in the ranked list
let mut last_true_position = 0;
for (position, &(_, label_idx)) in score_label_pairs.iter().enumerate() {
if y_true[[sample_idx, label_idx]] == 1 {
last_true_position = position + 1; // Convert to 1-based indexing
}
}
total_coverage += last_true_position as f64;
}
Ok(total_coverage / n_samples as f64)
}
/// Label ranking average precision for multi-label ranking
///
/// The label ranking average precision (LRAP) averages over the samples
/// the answer to the following question: for each ground truth label,
/// what fraction of higher-ranked labels were true labels?
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_scores` - Target scores (predicted probabilities)
///
/// # Returns
///
/// The label ranking average precision
pub fn label_ranking_average_precision(
y_true: &ArrayView2<'_, i32>,
y_scores: &ArrayView2<'_, f64>,
) -> SklResult<f64> {
if y_true.dim() != y_scores.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_scores must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_lrap = 0.0;
for sample_idx in 0..n_samples {
// Get the indices sorted by scores in descending order
let mut score_label_pairs: Vec<(f64, usize)> = (0..n_labels)
.map(|label_idx| (y_scores[[sample_idx, label_idx]], label_idx))
.collect();
score_label_pairs
.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
// Count true labels for this sample
let n_true_labels: i32 = (0..n_labels)
.map(|label_idx| y_true[[sample_idx, label_idx]])
.sum();
if n_true_labels == 0 {
continue; // Skip samples with no true labels
}
let mut precision_sum = 0.0;
let mut true_labels_seen = 0;
for (position, &(_, label_idx)) in score_label_pairs.iter().enumerate() {
if y_true[[sample_idx, label_idx]] == 1 {
true_labels_seen += 1;
let precision_at_position = true_labels_seen as f64 / (position + 1) as f64;
precision_sum += precision_at_position;
}
}
let sample_lrap = precision_sum / n_true_labels as f64;
total_lrap += sample_lrap;
}
Ok(total_lrap / n_samples as f64)
}
/// One-error for multi-label ranking
///
/// The one-error evaluates how many times the top-ranked label is not
/// in the set of true labels. The best performance is achieved when
/// one-error is 0, which means the top-ranked label is always correct.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_scores` - Target scores (predicted probabilities)
///
/// # Returns
///
/// The one-error (fraction of samples where top-ranked label is incorrect)
pub fn one_error(
y_true: &ArrayView2<'_, i32>,
y_scores: &ArrayView2<'_, f64>,
) -> SklResult<f64> {
if y_true.dim() != y_scores.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_scores must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut errors = 0;
for sample_idx in 0..n_samples {
// Find the label with the highest score
let mut max_score = f64::NEG_INFINITY;
let mut top_label_idx = 0;
for label_idx in 0..n_labels {
let score = y_scores[[sample_idx, label_idx]];
if score > max_score {
max_score = score;
top_label_idx = label_idx;
}
}
// Check if the top-ranked label is correct
if y_true[[sample_idx, top_label_idx]] != 1 {
errors += 1;
}
}
Ok(errors as f64 / n_samples as f64)
}
/// Ranking loss for multi-label ranking
///
/// The ranking loss evaluates the average fraction of label pairs that are
/// incorrectly ordered, given the predictions. The best performance is achieved
/// when ranking loss is 0.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_scores` - Target scores (predicted probabilities)
///
/// # Returns
///
/// The ranking loss
pub fn ranking_loss(
y_true: &ArrayView2<'_, i32>,
y_scores: &ArrayView2<'_, f64>,
) -> SklResult<f64> {
if y_true.dim() != y_scores.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_scores must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_ranking_loss = 0.0;
for sample_idx in 0..n_samples {
let mut incorrect_pairs = 0;
let mut total_pairs = 0;
// Compare all pairs of labels
for i in 0..n_labels {
for j in 0..n_labels {
if i != j {
let true_i = y_true[[sample_idx, i]];
let true_j = y_true[[sample_idx, j]];
let score_i = y_scores[[sample_idx, i]];
let score_j = y_scores[[sample_idx, j]];
// Check if this is a relevant pair (one positive, one negative)
if (true_i == 1 && true_j == 0) || (true_i == 0 && true_j == 1) {
total_pairs += 1;
// Check if the ordering is incorrect
if (true_i == 1 && true_j == 0 && score_i < score_j)
|| (true_i == 0 && true_j == 1 && score_i > score_j)
{
incorrect_pairs += 1;
}
}
}
}
}
// Add to total ranking loss
if total_pairs > 0 {
total_ranking_loss += incorrect_pairs as f64 / total_pairs as f64;
}
}
Ok(total_ranking_loss / n_samples as f64)
}
/// Average precision score for multi-label ranking
///
/// Computes the average precision for each sample and then averages
/// over all samples. This is different from label ranking average precision
/// as it focuses on precision-recall curves for each sample.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_scores` - Target scores (predicted probabilities)
///
/// # Returns
///
/// The average precision score
pub fn average_precision_score(
y_true: &ArrayView2<'_, i32>,
y_scores: &ArrayView2<'_, f64>,
) -> SklResult<f64> {
if y_true.dim() != y_scores.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_scores must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_ap = 0.0;
for sample_idx in 0..n_samples {
// Get the indices sorted by scores in descending order
let mut score_label_pairs: Vec<(f64, usize)> = (0..n_labels)
.map(|label_idx| (y_scores[[sample_idx, label_idx]], label_idx))
.collect();
score_label_pairs
.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
// Count true labels for this sample
let n_true_labels: i32 = (0..n_labels)
.map(|label_idx| y_true[[sample_idx, label_idx]])
.sum();
if n_true_labels == 0 {
continue; // Skip samples with no true labels
}
let mut precision_sum = 0.0;
let mut true_labels_seen = 0;
for (position, &(_, label_idx)) in score_label_pairs.iter().enumerate() {
if y_true[[sample_idx, label_idx]] == 1 {
true_labels_seen += 1;
let precision_at_position = true_labels_seen as f64 / (position + 1) as f64;
precision_sum += precision_at_position;
}
}
let sample_ap = precision_sum / n_true_labels as f64;
total_ap += sample_ap;
}
Ok(total_ap / n_samples as f64)
}
/// Micro-averaged precision for multi-label classification
///
/// Calculate precision by counting the total true positives and false positives
/// across all labels and samples.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
///
/// # Returns
///
/// The micro-averaged precision
pub fn precision_score_micro(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_tp = 0;
let mut total_fp = 0;
for sample_idx in 0..n_samples {
for label_idx in 0..n_labels {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
total_tp += 1;
} else if true_label == 0 && pred_label == 1 {
total_fp += 1;
}
}
}
let precision = if total_tp + total_fp > 0 {
total_tp as f64 / (total_tp + total_fp) as f64
} else {
0.0
};
Ok(precision)
}
/// Micro-averaged recall for multi-label classification
///
/// Calculate recall by counting the total true positives and false negatives
/// across all labels and samples.
///
/// # Arguments
///
/// * `y_true` - Ground truth (correct) labels
/// * `y_pred` - Predicted labels
///
/// # Returns
///
/// The micro-averaged recall
pub fn recall_score_micro(
y_true: &ArrayView2<'_, i32>,
y_pred: &ArrayView2<'_, i32>,
) -> SklResult<f64> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same shape".to_string(),
));
}
let (n_samples, n_labels) = y_true.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input arrays must have at least one sample and one label".to_string(),
));
}
let mut total_tp = 0;
let mut total_fn = 0;
for sample_idx in 0..n_samples {
for label_idx in 0..n_labels {
let true_label = y_true[[sample_idx, label_idx]];
let pred_label = y_pred[[sample_idx, label_idx]];
if true_label == 1 && pred_label == 1 {
total_tp += 1;
} else if true_label == 1 && pred_label == 0 {
total_fn += 1;
}
}
}
let recall = if total_tp + total_fn > 0 {
total_tp as f64 / (total_tp + total_fn) as f64
} else {
0.0
};
Ok(recall)
}
}
/// Label combination frequency analysis utilities
pub mod label_analysis {
use ndarray::{Array2, ArrayView2};
use sklears_core::error::{Result as SklResult, SklearsError};
use std::collections::HashMap;
/// Information about a label combination
#[derive(Debug, Clone)]
pub struct CombinationInfo {
/// The label combination
pub combination: Vec<i32>,
/// Number of times this combination appears
pub frequency: usize,
/// Relative frequency (proportion of total samples)
pub relative_frequency: f64,
/// Number of active labels in this combination
pub cardinality: usize,
}
/// Results of label combination frequency analysis
#[derive(Debug, Clone)]
pub struct LabelAnalysisResults {
/// All unique combinations and their statistics
pub combinations: Vec<CombinationInfo>,
/// Total number of samples analyzed
pub n_samples: usize,
/// Total number of labels
pub n_labels: usize,
/// Number of unique combinations found
pub n_unique_combinations: usize,
/// Most frequent combination
pub most_frequent: CombinationInfo,
/// Least frequent combination
pub least_frequent: CombinationInfo,
/// Average cardinality (average number of active labels per sample)
pub avg_cardinality: f64,
/// Label density (proportion of active labels across all samples)
pub label_density: f64,
}
/// Analyze the frequency distribution of label combinations
///
/// This function provides comprehensive statistics about label combinations
/// in a multi-label dataset, including frequency counts, cardinality analysis,
/// and density metrics.
///
/// # Arguments
///
/// * `y` - Multi-label target array where each row is a sample and each column is a label
///
/// # Returns
///
/// Comprehensive analysis results of label combinations
///
/// # Examples
///
/// ```
/// use sklears_multioutput::label_analysis;
/// use ndarray::array;
///
/// let y = array![[1, 0, 1], [0, 1, 0], [1, 0, 1], [0, 0, 0]];
/// let results = label_analysis::analyze_combinations(&y.view()).unwrap();
///
/// assert_eq!(results.n_samples, 4);
/// assert_eq!(results.n_labels, 3);
/// assert!(results.n_unique_combinations <= 4);
/// ```
pub fn analyze_combinations(y: &ArrayView2<'_, i32>) -> SklResult<LabelAnalysisResults> {
let (n_samples, n_labels) = y.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input array must have at least one sample and one label".to_string(),
));
}
// Validate that labels are binary
for sample_idx in 0..n_samples {
for label_idx in 0..n_labels {
let label_value = y[[sample_idx, label_idx]];
if label_value != 0 && label_value != 1 {
return Err(SklearsError::InvalidInput(format!(
"Label analysis expects binary labels, but found {} at position ({}, {})",
label_value, sample_idx, label_idx
)));
}
}
}
// Count frequencies of label combinations
let mut combination_counts: HashMap<Vec<i32>, usize> = HashMap::new();
let mut total_active_labels = 0;
for sample_idx in 0..n_samples {
let combination: Vec<i32> = (0..n_labels)
.map(|label_idx| y[[sample_idx, label_idx]])
.collect();
// Count active labels for density calculation
total_active_labels += combination.iter().sum::<i32>() as usize;
*combination_counts.entry(combination).or_insert(0) += 1;
}
// Convert to CombinationInfo structs
let mut combinations: Vec<CombinationInfo> = combination_counts
.into_iter()
.map(|(combination, frequency)| {
let cardinality = combination.iter().sum::<i32>() as usize;
let relative_frequency = frequency as f64 / n_samples as f64;
CombinationInfo {
combination,
frequency,
relative_frequency,
cardinality,
}
})
.collect();
// Sort by frequency (descending)
combinations.sort_by(|a, b| b.frequency.cmp(&a.frequency));
let n_unique_combinations = combinations.len();
// Find most and least frequent
let most_frequent = combinations[0].clone();
let least_frequent = combinations[n_unique_combinations - 1].clone();
// Calculate average cardinality
let total_cardinality: usize = combinations
.iter()
.map(|info| info.cardinality * info.frequency)
.sum();
let avg_cardinality = total_cardinality as f64 / n_samples as f64;
// Calculate label density
let label_density = total_active_labels as f64 / (n_samples * n_labels) as f64;
Ok(LabelAnalysisResults {
combinations,
n_samples,
n_labels,
n_unique_combinations,
most_frequent,
least_frequent,
avg_cardinality,
label_density,
})
}
/// Get combinations that appear less than a specified threshold
///
/// This function identifies rare label combinations that could be candidates
/// for pruning in algorithms like Pruned Label Powerset.
///
/// # Arguments
///
/// * `results` - Results from analyze_combinations
/// * `min_frequency` - Minimum frequency threshold
///
/// # Returns
///
/// Vector of combinations that appear less than min_frequency times
pub fn get_rare_combinations(
results: &LabelAnalysisResults,
min_frequency: usize,
) -> Vec<&CombinationInfo> {
results
.combinations
.iter()
.filter(|info| info.frequency < min_frequency)
.collect()
}
/// Get combinations with specific cardinality
///
/// This function filters combinations by the number of active labels.
///
/// # Arguments
///
/// * `results` - Results from analyze_combinations
/// * `cardinality` - Number of active labels to filter by
///
/// # Returns
///
/// Vector of combinations with the specified cardinality
pub fn get_combinations_by_cardinality(
results: &LabelAnalysisResults,
cardinality: usize,
) -> Vec<&CombinationInfo> {
results
.combinations
.iter()
.filter(|info| info.cardinality == cardinality)
.collect()
}
/// Calculate label co-occurrence matrix
///
/// This function computes how often pairs of labels appear together.
///
/// # Arguments
///
/// * `y` - Multi-label target array
///
/// # Returns
///
/// Symmetric matrix where entry (i,j) is the number of times labels i and j appear together
pub fn label_cooccurrence_matrix(y: &ArrayView2<'_, i32>) -> SklResult<Array2<usize>> {
let (n_samples, n_labels) = y.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input array must have at least one sample and one label".to_string(),
));
}
let mut cooccurrence = Array2::zeros((n_labels, n_labels));
for sample_idx in 0..n_samples {
for i in 0..n_labels {
for j in 0..n_labels {
if y[[sample_idx, i]] == 1 && y[[sample_idx, j]] == 1 {
cooccurrence[[i, j]] += 1;
}
}
}
}
Ok(cooccurrence)
}
/// Calculate label correlation matrix
///
/// This function computes Pearson correlation coefficients between pairs of labels.
///
/// # Arguments
///
/// * `y` - Multi-label target array
///
/// # Returns
///
/// Symmetric correlation matrix
pub fn label_correlation_matrix(y: &ArrayView2<'_, i32>) -> SklResult<Array2<f64>> {
let (n_samples, n_labels) = y.dim();
if n_samples == 0 || n_labels == 0 {
return Err(SklearsError::InvalidInput(
"Input array must have at least one sample and one label".to_string(),
));
}
let mut correlation = Array2::zeros((n_labels, n_labels));
// Compute means for each label
let mut means = vec![0.0; n_labels];
for label_idx in 0..n_labels {
let sum: i32 = (0..n_samples)
.map(|sample_idx| y[[sample_idx, label_idx]])
.sum();
means[label_idx] = sum as f64 / n_samples as f64;
}
// Compute correlations
for i in 0..n_labels {
for j in 0..n_labels {
if i == j {
correlation[[i, j]] = 1.0;
} else {
let mut numerator = 0.0;
let mut var_i = 0.0;
let mut var_j = 0.0;
for sample_idx in 0..n_samples {
let diff_i = y[[sample_idx, i]] as f64 - means[i];
let diff_j = y[[sample_idx, j]] as f64 - means[j];
numerator += diff_i * diff_j;
var_i += diff_i * diff_i;
var_j += diff_j * diff_j;
}
if var_i > 1e-10 && var_j > 1e-10 {
correlation[[i, j]] = numerator / (var_i.sqrt() * var_j.sqrt());
} else {
correlation[[i, j]] = 0.0;
}
}
}
}
Ok(correlation)
}
}
/// Multi-Target Regression Tree
///
/// A decision tree regressor that can handle multiple target variables simultaneously.
/// This implementation fits a single tree that can predict multiple continuous targets
/// by using variance reduction criteria that consider all targets jointly.
///
/// # Mathematical Foundation
///
/// For multi-target regression, the tree uses the following variance reduction criterion:
/// - At each split, calculate the weighted sum of variances across all targets
/// - Choose the split that minimizes: Σ(n_left * var_left + n_right * var_right) for all targets
/// - Leaf predictions are the mean values for each target in that leaf
///
/// # Examples
///
/// ```
/// use sklears_multioutput::MultiTargetRegressionTree;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
///
/// let tree = MultiTargetRegressionTree::new()
/// .max_depth(Some(3))
/// .min_samples_split(2);
/// let trained_tree = tree.fit(&X.view(), &y).unwrap();
/// let predictions = trained_tree.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct MultiTargetRegressionTree<S = Untrained> {
state: S,
max_depth: Option<usize>,
min_samples_split: usize,
min_samples_leaf: usize,
random_state: Option<u64>,
}
#[derive(Debug, Clone)]
pub struct MultiTargetRegressionTreeTrained {
tree: DecisionNode,
n_features: usize,
n_targets: usize,
feature_importances: Array1<Float>,
}
#[derive(Debug, Clone)]
struct DecisionNode {
is_leaf: bool,
prediction: Option<Array1<Float>>, // Mean values for each target
feature_idx: Option<usize>,
threshold: Option<Float>,
left: Option<Box<DecisionNode>>,
right: Option<Box<DecisionNode>>,
n_samples: usize,
variance: Float, // Sum of variances across all targets
}
impl MultiTargetRegressionTree<Untrained> {
/// Create a new MultiTargetRegressionTree instance
pub fn new() -> Self {
Self {
state: Untrained,
max_depth: Some(5),
min_samples_split: 2,
min_samples_leaf: 1,
random_state: None,
}
}
/// Set the maximum depth of the tree
pub fn max_depth(mut self, max_depth: Option<usize>) -> Self {
self.max_depth = max_depth;
self
}
/// Set the minimum number of samples required to split an internal node
pub fn min_samples_split(mut self, min_samples_split: usize) -> Self {
self.min_samples_split = min_samples_split;
self
}
/// Set the minimum number of samples required to be at a leaf node
pub fn min_samples_leaf(mut self, min_samples_leaf: usize) -> Self {
self.min_samples_leaf = min_samples_leaf;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for MultiTargetRegressionTree<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MultiTargetRegressionTree<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<Float>> for MultiTargetRegressionTree<Untrained> {
type Fitted = MultiTargetRegressionTree<MultiTargetRegressionTreeTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<Float>) -> SklResult<Self::Fitted> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let n_targets = y.ncols();
if n_targets == 0 {
return Err(SklearsError::InvalidInput(
"y must have at least one target".to_string(),
));
}
if n_samples < self.min_samples_split {
return Err(SklearsError::InvalidInput(
"Number of samples is less than min_samples_split".to_string(),
));
}
// Build the tree
let indices: Vec<usize> = (0..n_samples).collect();
let tree = self.build_tree(&X, y, &indices, 0)?;
// Calculate feature importances (simplified)
let mut feature_importances = Array1::<Float>::zeros(n_features);
self.calculate_feature_importances(&tree, &mut feature_importances, n_samples as Float);
// Normalize feature importances
let sum_importances: Float = feature_importances.sum();
if sum_importances > 0.0 {
feature_importances /= sum_importances;
}
Ok(MultiTargetRegressionTree {
state: MultiTargetRegressionTreeTrained {
tree,
n_features,
n_targets,
feature_importances,
},
max_depth: self.max_depth,
min_samples_split: self.min_samples_split,
min_samples_leaf: self.min_samples_leaf,
random_state: self.random_state,
})
}
}
impl MultiTargetRegressionTree<Untrained> {
fn build_tree(
&self,
X: &Array2<Float>,
y: &Array2<Float>,
indices: &[usize],
depth: usize,
) -> SklResult<DecisionNode> {
let n_samples = indices.len();
let n_targets = y.ncols();
// Calculate current prediction (mean of targets)
let mut prediction = Array1::<Float>::zeros(n_targets);
for &idx in indices {
for j in 0..n_targets {
prediction[j] += y[[idx, j]];
}
}
prediction /= n_samples as Float;
// Calculate variance across all targets
let mut variance = 0.0;
for &idx in indices {
for j in 0..n_targets {
let diff = y[[idx, j]] - prediction[j];
variance += diff * diff;
}
}
variance /= n_samples as Float;
// Check stopping criteria
let should_stop = n_samples < self.min_samples_split
|| n_samples < self.min_samples_leaf
|| self.max_depth.map_or(false, |max_d| depth >= max_d)
|| variance < 1e-10;
if should_stop {
return Ok(DecisionNode {
is_leaf: true,
prediction: Some(prediction),
feature_idx: None,
threshold: None,
left: None,
right: None,
n_samples,
variance,
});
}
// Find best split
let (best_feature, best_threshold, best_variance_reduction) =
self.find_best_split(X, y, indices)?;
if best_variance_reduction <= 0.0 {
return Ok(DecisionNode {
is_leaf: true,
prediction: Some(prediction),
feature_idx: None,
threshold: None,
left: None,
right: None,
n_samples,
variance,
});
}
// Split the data
let (left_indices, right_indices) =
self.split_data(X, indices, best_feature, best_threshold);
if left_indices.len() < self.min_samples_leaf || right_indices.len() < self.min_samples_leaf
{
return Ok(DecisionNode {
is_leaf: true,
prediction: Some(prediction),
feature_idx: None,
threshold: None,
left: None,
right: None,
n_samples,
variance,
});
}
// Recursively build child nodes
let left_child = self.build_tree(X, y, &left_indices, depth + 1)?;
let right_child = self.build_tree(X, y, &right_indices, depth + 1)?;
Ok(DecisionNode {
is_leaf: false,
prediction: None,
feature_idx: Some(best_feature),
threshold: Some(best_threshold),
left: Some(Box::new(left_child)),
right: Some(Box::new(right_child)),
n_samples,
variance,
})
}
fn find_best_split(
&self,
X: &Array2<Float>,
y: &Array2<Float>,
indices: &[usize],
) -> SklResult<(usize, Float, Float)> {
let n_features = X.ncols();
let n_targets = y.ncols();
let mut best_feature = 0;
let mut best_threshold = 0.0;
let mut best_variance_reduction = 0.0;
// Calculate current variance
let current_variance = self.calculate_variance(y, indices);
for feature_idx in 0..n_features {
// Get unique feature values
let mut feature_values: Vec<Float> =
indices.iter().map(|&idx| X[[idx, feature_idx]]).collect();
feature_values.sort_by(|a, b| a.partial_cmp(b).unwrap());
feature_values.dedup();
for i in 0..feature_values.len().saturating_sub(1) {
let threshold = (feature_values[i] + feature_values[i + 1]) / 2.0;
let (left_indices, right_indices) =
self.split_data(X, indices, feature_idx, threshold);
if left_indices.is_empty() || right_indices.is_empty() {
continue;
}
let left_variance = self.calculate_variance(y, &left_indices);
let right_variance = self.calculate_variance(y, &right_indices);
let weighted_variance = (left_indices.len() as Float * left_variance
+ right_indices.len() as Float * right_variance)
/ indices.len() as Float;
let variance_reduction = current_variance - weighted_variance;
if variance_reduction > best_variance_reduction {
best_variance_reduction = variance_reduction;
best_feature = feature_idx;
best_threshold = threshold;
}
}
}
Ok((best_feature, best_threshold, best_variance_reduction))
}
fn calculate_variance(&self, y: &Array2<Float>, indices: &[usize]) -> Float {
if indices.is_empty() {
return 0.0;
}
let n_targets = y.ncols();
let n_samples = indices.len();
// Calculate means
let mut means = Array1::<Float>::zeros(n_targets);
for &idx in indices {
for j in 0..n_targets {
means[j] += y[[idx, j]];
}
}
means /= n_samples as Float;
// Calculate variance
let mut variance = 0.0;
for &idx in indices {
for j in 0..n_targets {
let diff = y[[idx, j]] - means[j];
variance += diff * diff;
}
}
variance / n_samples as Float
}
fn split_data(
&self,
X: &Array2<Float>,
indices: &[usize],
feature_idx: usize,
threshold: Float,
) -> (Vec<usize>, Vec<usize>) {
let mut left_indices = Vec::new();
let mut right_indices = Vec::new();
for &idx in indices {
if X[[idx, feature_idx]] <= threshold {
left_indices.push(idx);
} else {
right_indices.push(idx);
}
}
(left_indices, right_indices)
}
fn calculate_feature_importances(
&self,
node: &DecisionNode,
importances: &mut Array1<Float>,
total_samples: Float,
) {
if let (Some(feature_idx), Some(left), Some(right)) =
(node.feature_idx, &node.left, &node.right)
{
let importance = (node.n_samples as Float / total_samples) * node.variance;
importances[feature_idx] += importance;
self.calculate_feature_importances(left, importances, total_samples);
self.calculate_feature_importances(right, importances, total_samples);
}
}
}
impl MultiTargetRegressionTree<MultiTargetRegressionTreeTrained> {
/// Get the feature importances
pub fn feature_importances(&self) -> &Array1<Float> {
&self.state.feature_importances
}
/// Get the number of features
pub fn n_features(&self) -> usize {
self.state.n_features
}
/// Get the number of targets
pub fn n_targets(&self) -> usize {
self.state.n_targets
}
/// Predict using the fitted tree
pub fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let mut predictions = Array2::<Float>::zeros((n_samples, self.state.n_targets));
for i in 0..n_samples {
let sample = X.slice(s![i, ..]);
let prediction = self.predict_single(&self.state.tree, &sample)?;
for j in 0..self.state.n_targets {
predictions[[i, j]] = prediction[j];
}
}
Ok(predictions)
}
fn predict_single(
&self,
node: &DecisionNode,
sample: &ArrayView1<'_, Float>,
) -> SklResult<Array1<Float>> {
if node.is_leaf {
if let Some(ref prediction) = node.prediction {
Ok(prediction.clone())
} else {
Err(SklearsError::InvalidInput(
"Leaf node without prediction".to_string(),
))
}
} else {
let feature_idx = node.feature_idx.ok_or(SklearsError::InvalidInput(
"Non-leaf node without feature index".to_string(),
))?;
let threshold = node.threshold.ok_or(SklearsError::InvalidInput(
"Non-leaf node without threshold".to_string(),
))?;
if sample[feature_idx] <= threshold {
if let Some(ref left) = node.left {
self.predict_single(left, sample)
} else {
Err(SklearsError::InvalidInput(
"Non-leaf node without left child".to_string(),
))
}
} else {
if let Some(ref right) = node.right {
self.predict_single(right, sample)
} else {
Err(SklearsError::InvalidInput(
"Non-leaf node without right child".to_string(),
))
}
}
}
}
}
/// Random Forest Multi-Output Extension
///
/// A random forest that can handle multiple output variables simultaneously.
/// This implementation creates multiple multi-target regression trees and
/// averages their predictions for robust multi-output regression.
///
/// # Mathematical Foundation
///
/// The random forest combines multiple multi-target regression trees:
/// - Each tree is trained on a bootstrap sample of the data
/// - Each tree considers only a random subset of features at each split
/// - Final prediction is the average of all tree predictions
/// - Feature importance is averaged across all trees
///
/// # Examples
///
/// ```
/// use sklears_multioutput::RandomForestMultiOutput;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
///
/// let forest = RandomForestMultiOutput::new()
/// .n_estimators(10)
/// .max_depth(Some(3));
/// let trained_forest = forest.fit(&X.view(), &y).unwrap();
/// let predictions = trained_forest.predict(&X.view()).unwrap();
/// ```
/// Multi-Target Decision Tree Classifier
///
/// A decision tree classifier that can handle multiple target variables simultaneously.
/// Uses joint entropy/gini reduction for optimal splits across all targets.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::MultiTargetDecisionTreeClassifier;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[0, 1], [1, 0], [1, 1], [0, 0]]; // Two binary classification targets
///
/// let tree = MultiTargetDecisionTreeClassifier::new()
/// .max_depth(Some(3));
/// let trained_tree = tree.fit(&X.view(), &y).unwrap();
/// let predictions = trained_tree.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct MultiTargetDecisionTreeClassifier<S = Untrained> {
state: S,
max_depth: Option<usize>,
min_samples_split: usize,
min_samples_leaf: usize,
criterion: ClassificationCriterion,
random_state: Option<u64>,
}
#[derive(Debug, Clone)]
pub struct MultiTargetDecisionTreeClassifierTrained {
tree: ClassificationDecisionNode,
n_features: usize,
n_targets: usize,
feature_importances: Array1<Float>,
classes_per_target: Vec<Vec<i32>>,
}
#[derive(Debug, Clone, Copy)]
pub enum ClassificationCriterion {
/// Gini impurity
Gini,
/// Information gain / entropy
Entropy,
}
#[derive(Debug, Clone)]
struct ClassificationDecisionNode {
is_leaf: bool,
prediction: Option<Array1<i32>>, // Mode/majority class for each target
probabilities: Option<Array2<Float>>, // Probability distributions per target
feature_idx: Option<usize>,
threshold: Option<Float>,
left: Option<Box<ClassificationDecisionNode>>,
right: Option<Box<ClassificationDecisionNode>>,
n_samples: usize,
impurity: Float, // Combined impurity across all targets
}
impl MultiTargetDecisionTreeClassifier<Untrained> {
/// Create a new MultiTargetDecisionTreeClassifier instance
pub fn new() -> Self {
Self {
state: Untrained,
max_depth: Some(5),
min_samples_split: 2,
min_samples_leaf: 1,
criterion: ClassificationCriterion::Gini,
random_state: None,
}
}
/// Set the maximum depth of the tree
pub fn max_depth(mut self, max_depth: Option<usize>) -> Self {
self.max_depth = max_depth;
self
}
/// Set the minimum number of samples required to split an internal node
pub fn min_samples_split(mut self, min_samples_split: usize) -> Self {
self.min_samples_split = min_samples_split;
self
}
/// Set the minimum number of samples required to be at a leaf node
pub fn min_samples_leaf(mut self, min_samples_leaf: usize) -> Self {
self.min_samples_leaf = min_samples_leaf;
self
}
/// Set the split criterion
pub fn criterion(mut self, criterion: ClassificationCriterion) -> Self {
self.criterion = criterion;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for MultiTargetDecisionTreeClassifier<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MultiTargetDecisionTreeClassifier<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for MultiTargetDecisionTreeClassifier<Untrained> {
type Fitted = MultiTargetDecisionTreeClassifier<MultiTargetDecisionTreeClassifierTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let n_targets = y.ncols();
if n_targets == 0 {
return Err(SklearsError::InvalidInput(
"y must have at least one target".to_string(),
));
}
// Get unique classes for each target
let mut classes_per_target = Vec::new();
for target_idx in 0..n_targets {
let target_column = y.column(target_idx);
let mut unique_classes: Vec<i32> = target_column.iter().cloned().collect();
unique_classes.sort_unstable();
unique_classes.dedup();
classes_per_target.push(unique_classes);
}
// Initialize feature importances
let mut feature_importances = Array1::zeros(n_features);
// Build the tree
let indices: Vec<usize> = (0..n_samples).collect();
let tree = build_classification_tree(
&X,
y,
&indices,
&mut feature_importances,
0,
self.max_depth,
self.min_samples_split,
self.min_samples_leaf,
self.criterion,
&classes_per_target,
)?;
// Normalize feature importances
let importance_sum = feature_importances.sum();
if importance_sum > 0.0 {
feature_importances /= importance_sum;
}
let trained_state = MultiTargetDecisionTreeClassifierTrained {
tree,
n_features,
n_targets,
feature_importances,
classes_per_target,
};
Ok(MultiTargetDecisionTreeClassifier {
state: trained_state,
max_depth: self.max_depth,
min_samples_split: self.min_samples_split,
min_samples_leaf: self.min_samples_leaf,
criterion: self.criterion,
random_state: self.random_state,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for MultiTargetDecisionTreeClassifier<MultiTargetDecisionTreeClassifierTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_samples, self.state.n_targets));
for i in 0..n_samples {
let sample = X.row(i);
let prediction = predict_classification_sample(&self.state.tree, &sample);
for j in 0..self.state.n_targets {
predictions[[i, j]] = prediction[j];
}
}
Ok(predictions)
}
}
impl MultiTargetDecisionTreeClassifier<MultiTargetDecisionTreeClassifierTrained> {
/// Get feature importances
pub fn feature_importances(&self) -> &Array1<Float> {
&self.state.feature_importances
}
/// Predict class probabilities for each target
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Vec<Array2<Float>>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut all_probabilities = Vec::new();
// Initialize probability arrays for each target
for target_idx in 0..self.state.n_targets {
let n_classes = self.state.classes_per_target[target_idx].len();
all_probabilities.push(Array2::zeros((n_samples, n_classes)));
}
for i in 0..n_samples {
let sample = X.row(i);
let probabilities = predict_classification_probabilities(
&self.state.tree,
&sample,
&self.state.classes_per_target,
);
for (target_idx, target_probs) in probabilities.iter().enumerate() {
for (class_idx, &prob) in target_probs.iter().enumerate() {
all_probabilities[target_idx][[i, class_idx]] = prob;
}
}
}
Ok(all_probabilities)
}
}
/// Build a classification decision tree recursively
fn build_classification_tree(
X: &Array2<Float>,
y: &Array2<i32>,
indices: &[usize],
feature_importances: &mut Array1<Float>,
depth: usize,
max_depth: Option<usize>,
min_samples_split: usize,
min_samples_leaf: usize,
criterion: ClassificationCriterion,
classes_per_target: &[Vec<i32>],
) -> SklResult<ClassificationDecisionNode> {
let n_samples = indices.len();
let n_targets = y.ncols();
// Calculate current impurity and predictions
let (current_impurity, prediction, probabilities) =
calculate_classification_metrics(y, indices, classes_per_target, criterion);
// Check stopping criteria
let should_stop = n_samples < min_samples_split
|| (max_depth.is_some() && depth >= max_depth.unwrap())
|| current_impurity == 0.0;
if should_stop {
return Ok(ClassificationDecisionNode {
is_leaf: true,
prediction: Some(prediction),
probabilities: Some(probabilities),
feature_idx: None,
threshold: None,
left: None,
right: None,
n_samples,
impurity: current_impurity,
});
}
// Find best split
let mut best_impurity_reduction = 0.0;
let mut best_feature = None;
let mut best_threshold = None;
let mut best_left_indices = Vec::new();
let mut best_right_indices = Vec::new();
for feature_idx in 0..X.ncols() {
// Get unique values for this feature in current samples
let mut feature_values: Vec<Float> = indices.iter().map(|&i| X[[i, feature_idx]]).collect();
feature_values.sort_by(|a, b| a.partial_cmp(b).unwrap());
feature_values.dedup();
// Try splits between consecutive unique values
for i in 0..feature_values.len().saturating_sub(1) {
let threshold = (feature_values[i] + feature_values[i + 1]) / 2.0;
let (left_indices, right_indices): (Vec<usize>, Vec<usize>) = indices
.iter()
.partition(|&&idx| X[[idx, feature_idx]] <= threshold);
// Check minimum samples per leaf
if left_indices.len() < min_samples_leaf || right_indices.len() < min_samples_leaf {
continue;
}
// Calculate impurity reduction
let (left_impurity, _, _) =
calculate_classification_metrics(y, &left_indices, classes_per_target, criterion);
let (right_impurity, _, _) =
calculate_classification_metrics(y, &right_indices, classes_per_target, criterion);
let weighted_impurity = (left_indices.len() as Float * left_impurity
+ right_indices.len() as Float * right_impurity)
/ n_samples as Float;
let impurity_reduction = current_impurity - weighted_impurity;
if impurity_reduction > best_impurity_reduction {
best_impurity_reduction = impurity_reduction;
best_feature = Some(feature_idx);
best_threshold = Some(threshold);
best_left_indices = left_indices;
best_right_indices = right_indices;
}
}
}
// If no good split found, create leaf
if best_feature.is_none() || best_impurity_reduction <= 0.0 {
return Ok(ClassificationDecisionNode {
is_leaf: true,
prediction: Some(prediction),
probabilities: Some(probabilities),
feature_idx: None,
threshold: None,
left: None,
right: None,
n_samples,
impurity: current_impurity,
});
}
// Update feature importance
feature_importances[best_feature.unwrap()] += best_impurity_reduction * n_samples as Float;
// Recursively build left and right subtrees
let left_child = build_classification_tree(
X,
y,
&best_left_indices,
feature_importances,
depth + 1,
max_depth,
min_samples_split,
min_samples_leaf,
criterion,
classes_per_target,
)?;
let right_child = build_classification_tree(
X,
y,
&best_right_indices,
feature_importances,
depth + 1,
max_depth,
min_samples_split,
min_samples_leaf,
criterion,
classes_per_target,
)?;
Ok(ClassificationDecisionNode {
is_leaf: false,
prediction: Some(prediction),
probabilities: Some(probabilities),
feature_idx: best_feature,
threshold: best_threshold,
left: Some(Box::new(left_child)),
right: Some(Box::new(right_child)),
n_samples,
impurity: current_impurity,
})
}
/// Calculate classification metrics for a set of samples
fn calculate_classification_metrics(
y: &Array2<i32>,
indices: &[usize],
classes_per_target: &[Vec<i32>],
criterion: ClassificationCriterion,
) -> (Float, Array1<i32>, Array2<Float>) {
let n_targets = y.ncols();
let n_samples = indices.len();
let mut prediction = Array1::zeros(n_targets);
let mut total_impurity = 0.0;
// Calculate max number of classes across all targets for probability matrix
let max_classes = classes_per_target
.iter()
.map(|classes| classes.len())
.max()
.unwrap_or(0);
let mut probabilities = Array2::zeros((n_targets, max_classes));
for target_idx in 0..n_targets {
let classes = &classes_per_target[target_idx];
let n_classes = classes.len();
// Count class frequencies
let mut class_counts = vec![0; n_classes];
for &sample_idx in indices {
let class_label = y[[sample_idx, target_idx]];
if let Some(class_idx) = classes.iter().position(|&c| c == class_label) {
class_counts[class_idx] += 1;
}
}
// Find majority class
let majority_class_idx = class_counts
.iter()
.enumerate()
.max_by_key(|(_, &count)| count)
.map(|(idx, _)| idx)
.unwrap_or(0);
prediction[target_idx] = classes[majority_class_idx];
// Calculate probabilities and impurity
let mut target_impurity = 0.0;
for (class_idx, &count) in class_counts.iter().enumerate() {
let prob = count as Float / n_samples as Float;
probabilities[[target_idx, class_idx]] = prob;
if prob > 0.0 {
target_impurity += match criterion {
ClassificationCriterion::Gini => prob * (1.0 - prob),
ClassificationCriterion::Entropy => -prob * prob.ln(),
};
}
}
// For Gini, multiply by 2; for Entropy, it's already correct
if matches!(criterion, ClassificationCriterion::Gini) {
target_impurity *= 2.0;
}
total_impurity += target_impurity;
}
// Average impurity across targets
total_impurity /= n_targets as Float;
(total_impurity, prediction, probabilities)
}
/// Predict for a single classification sample
fn predict_classification_sample(
node: &ClassificationDecisionNode,
sample: &ArrayView1<Float>,
) -> Array1<i32> {
if node.is_leaf {
return node.prediction.as_ref().unwrap().clone();
}
let feature_value = sample[node.feature_idx.unwrap()];
let threshold = node.threshold.unwrap();
if feature_value <= threshold {
predict_classification_sample(node.left.as_ref().unwrap(), sample)
} else {
predict_classification_sample(node.right.as_ref().unwrap(), sample)
}
}
/// Predict probabilities for a single classification sample
fn predict_classification_probabilities(
node: &ClassificationDecisionNode,
sample: &ArrayView1<Float>,
classes_per_target: &[Vec<i32>],
) -> Vec<Array1<Float>> {
if node.is_leaf {
let mut result = Vec::new();
for target_idx in 0..classes_per_target.len() {
let n_classes = classes_per_target[target_idx].len();
let mut target_probs = Array1::zeros(n_classes);
for class_idx in 0..n_classes {
target_probs[class_idx] =
node.probabilities.as_ref().unwrap()[[target_idx, class_idx]];
}
result.push(target_probs);
}
return result;
}
let feature_value = sample[node.feature_idx.unwrap()];
let threshold = node.threshold.unwrap();
if feature_value <= threshold {
predict_classification_probabilities(
node.left.as_ref().unwrap(),
sample,
classes_per_target,
)
} else {
predict_classification_probabilities(
node.right.as_ref().unwrap(),
sample,
classes_per_target,
)
}
}
#[derive(Debug, Clone)]
pub struct RandomForestMultiOutput<S = Untrained> {
state: S,
n_estimators: usize,
max_depth: Option<usize>,
min_samples_split: usize,
min_samples_leaf: usize,
max_features: Option<usize>,
bootstrap: bool,
random_state: Option<u64>,
}
#[derive(Debug, Clone)]
pub struct RandomForestMultiOutputTrained {
trees: Vec<MultiTargetRegressionTree<MultiTargetRegressionTreeTrained>>,
n_features: usize,
n_targets: usize,
feature_importances: Array1<Float>,
}
impl RandomForestMultiOutput<Untrained> {
/// Create a new RandomForestMultiOutput instance
pub fn new() -> Self {
Self {
state: Untrained,
n_estimators: 10,
max_depth: None,
min_samples_split: 2,
min_samples_leaf: 1,
max_features: None,
bootstrap: true,
random_state: None,
}
}
/// Set the number of trees in the forest
pub fn n_estimators(mut self, n_estimators: usize) -> Self {
self.n_estimators = n_estimators;
self
}
/// Set the maximum depth of the trees
pub fn max_depth(mut self, max_depth: Option<usize>) -> Self {
self.max_depth = max_depth;
self
}
/// Set the minimum number of samples required to split an internal node
pub fn min_samples_split(mut self, min_samples_split: usize) -> Self {
self.min_samples_split = min_samples_split;
self
}
/// Set the minimum number of samples required to be at a leaf node
pub fn min_samples_leaf(mut self, min_samples_leaf: usize) -> Self {
self.min_samples_leaf = min_samples_leaf;
self
}
/// Set the number of features to consider when looking for the best split
pub fn max_features(mut self, max_features: Option<usize>) -> Self {
self.max_features = max_features;
self
}
/// Set whether to use bootstrap samples when building trees
pub fn bootstrap(mut self, bootstrap: bool) -> Self {
self.bootstrap = bootstrap;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for RandomForestMultiOutput<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for RandomForestMultiOutput<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<Float>> for RandomForestMultiOutput<Untrained> {
type Fitted = RandomForestMultiOutput<RandomForestMultiOutputTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<Float>) -> SklResult<Self::Fitted> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let n_targets = y.ncols();
if n_targets == 0 {
return Err(SklearsError::InvalidInput(
"y must have at least one target".to_string(),
));
}
let mut trees = Vec::new();
let mut feature_importances = Array1::<Float>::zeros(n_features);
for i in 0..self.n_estimators {
// Create bootstrap sample if needed
let (X_sample, y_sample) = if self.bootstrap {
self.create_bootstrap_sample(&X, y, i)?
} else {
(X.clone(), y.clone())
};
// Create and train tree
let tree = MultiTargetRegressionTree::new()
.max_depth(self.max_depth)
.min_samples_split(self.min_samples_split)
.min_samples_leaf(self.min_samples_leaf)
.random_state(self.random_state.map(|s| s.wrapping_add(i as u64)));
let trained_tree = tree.fit(&X_sample.view(), &y_sample)?;
// Accumulate feature importances
feature_importances += trained_tree.feature_importances();
trees.push(trained_tree);
}
// Average feature importances
feature_importances /= self.n_estimators as Float;
Ok(RandomForestMultiOutput {
state: RandomForestMultiOutputTrained {
trees,
n_features,
n_targets,
feature_importances,
},
n_estimators: self.n_estimators,
max_depth: self.max_depth,
min_samples_split: self.min_samples_split,
min_samples_leaf: self.min_samples_leaf,
max_features: self.max_features,
bootstrap: self.bootstrap,
random_state: self.random_state,
})
}
}
impl RandomForestMultiOutput<Untrained> {
fn create_bootstrap_sample(
&self,
X: &Array2<Float>,
y: &Array2<Float>,
seed: usize,
) -> SklResult<(Array2<Float>, Array2<Float>)> {
let n_samples = X.nrows();
let mut rng_state = self.random_state.unwrap_or(42).wrapping_add(seed as u64);
let mut X_sample = Array2::<Float>::zeros(X.raw_dim());
let mut y_sample = Array2::<Float>::zeros(y.raw_dim());
for i in 0..n_samples {
rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
let idx = (rng_state / 65536) % (n_samples as u64);
X_sample
.slice_mut(s![i, ..])
.assign(&X.slice(s![idx as usize, ..]));
y_sample
.slice_mut(s![i, ..])
.assign(&y.slice(s![idx as usize, ..]));
}
Ok((X_sample, y_sample))
}
}
impl RandomForestMultiOutput<RandomForestMultiOutputTrained> {
/// Get the feature importances averaged across all trees
pub fn feature_importances(&self) -> &Array1<Float> {
&self.state.feature_importances
}
/// Get the number of estimators (trees)
pub fn n_estimators(&self) -> usize {
self.state.trees.len()
}
/// Get the number of features
pub fn n_features(&self) -> usize {
self.state.n_features
}
/// Get the number of targets
pub fn n_targets(&self) -> usize {
self.state.n_targets
}
/// Predict using the forest (average of all tree predictions)
pub fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let mut predictions = Array2::<Float>::zeros((n_samples, self.state.n_targets));
// Average predictions from all trees
for tree in &self.state.trees {
let tree_predictions = tree.predict(&X)?;
predictions += &tree_predictions;
}
predictions /= self.state.trees.len() as Float;
Ok(predictions)
}
}
/// Gradient Boosting Multi-Output
///
/// A gradient boosting implementation that can handle multiple output variables simultaneously.
/// This implementation builds an additive model of weak learners (multi-target regression trees)
/// in a forward stage-wise fashion, optimizing a loss function for all targets jointly.
///
/// # Mathematical Foundation
///
/// For multi-target regression, gradient boosting minimizes:
/// - L(y, F(x)) = Σ_i Σ_j (y_ij - F_j(x_i))^2 for all samples i and targets j
/// - F_j(x) = F_0j + Σ_m ρ_m * h_mj(x) where h_mj are weak learners for target j at stage m
/// - At each stage, fit weak learners to negative gradients: -∂L/∂F_j
///
/// # Examples
///
/// ```
/// use sklears_multioutput::GradientBoostingMultiOutput;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
///
/// let gbm = GradientBoostingMultiOutput::new()
/// .n_estimators(50)
/// .learning_rate(0.1)
/// .max_depth(Some(3));
/// let trained_gbm = gbm.fit(&X.view(), &y).unwrap();
/// let predictions = trained_gbm.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct GradientBoostingMultiOutput<S = Untrained> {
state: S,
n_estimators: usize,
learning_rate: Float,
max_depth: Option<usize>,
min_samples_split: usize,
min_samples_leaf: usize,
subsample: Float,
random_state: Option<u64>,
}
#[derive(Debug, Clone)]
pub struct GradientBoostingMultiOutputTrained {
estimators: Vec<MultiTargetRegressionTree<MultiTargetRegressionTreeTrained>>,
init_predictions: Array1<Float>, // Initial prediction for each target
n_features: usize,
n_targets: usize,
feature_importances: Array1<Float>,
train_scores: Vec<Float>, // Training loss at each iteration
}
impl GradientBoostingMultiOutput<Untrained> {
/// Create a new GradientBoostingMultiOutput instance
pub fn new() -> Self {
Self {
state: Untrained,
n_estimators: 100,
learning_rate: 0.1,
max_depth: Some(3),
min_samples_split: 2,
min_samples_leaf: 1,
subsample: 1.0,
random_state: None,
}
}
/// Set the number of boosting stages
pub fn n_estimators(mut self, n_estimators: usize) -> Self {
self.n_estimators = n_estimators;
self
}
/// Set the learning rate (shrinkage)
pub fn learning_rate(mut self, learning_rate: Float) -> Self {
self.learning_rate = learning_rate;
self
}
/// Set the maximum depth of the individual regression estimators
pub fn max_depth(mut self, max_depth: Option<usize>) -> Self {
self.max_depth = max_depth;
self
}
/// Set the minimum number of samples required to split an internal node
pub fn min_samples_split(mut self, min_samples_split: usize) -> Self {
self.min_samples_split = min_samples_split;
self
}
/// Set the minimum number of samples required to be at a leaf node
pub fn min_samples_leaf(mut self, min_samples_leaf: usize) -> Self {
self.min_samples_leaf = min_samples_leaf;
self
}
/// Set the fraction of samples to be used for fitting the individual base learners
pub fn subsample(mut self, subsample: Float) -> Self {
self.subsample = subsample;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for GradientBoostingMultiOutput<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for GradientBoostingMultiOutput<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<Float>> for GradientBoostingMultiOutput<Untrained> {
type Fitted = GradientBoostingMultiOutput<GradientBoostingMultiOutputTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<Float>) -> SklResult<Self::Fitted> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let n_targets = y.ncols();
if n_targets == 0 {
return Err(SklearsError::InvalidInput(
"y must have at least one target".to_string(),
));
}
if self.learning_rate <= 0.0 {
return Err(SklearsError::InvalidInput(
"Learning rate must be positive".to_string(),
));
}
if self.subsample <= 0.0 || self.subsample > 1.0 {
return Err(SklearsError::InvalidInput(
"Subsample must be in (0, 1]".to_string(),
));
}
// Initialize with mean of targets
let init_predictions = y.mean_axis(Axis(0)).ok_or(SklearsError::InvalidInput(
"Cannot compute mean of targets".to_string(),
))?;
// Initialize predictions
let mut current_predictions = Array2::<Float>::zeros((n_samples, n_targets));
for i in 0..n_samples {
for j in 0..n_targets {
current_predictions[[i, j]] = init_predictions[j];
}
}
let mut estimators = Vec::new();
let mut feature_importances = Array1::<Float>::zeros(n_features);
let mut train_scores = Vec::new();
// Gradient boosting iterations
for m in 0..self.n_estimators {
// Compute negative gradients (residuals for squared loss)
let residuals = y - ¤t_predictions;
// Subsample if needed
let (X_subsample, residuals_subsample, subsample_indices) = if self.subsample < 1.0 {
self.create_subsample(&X, &residuals, m)?
} else {
(X.clone(), residuals, (0..n_samples).collect())
};
// Fit a regression tree to the residuals
let tree = MultiTargetRegressionTree::new()
.max_depth(self.max_depth)
.min_samples_split(self.min_samples_split)
.min_samples_leaf(self.min_samples_leaf)
.random_state(self.random_state.map(|s| s.wrapping_add(m as u64)));
let fitted_tree = tree.fit(&X_subsample.view(), &residuals_subsample)?;
// Predict on the full training set
let tree_predictions = fitted_tree.predict(&X.view())?;
// Update predictions with learning rate
current_predictions += &(tree_predictions * self.learning_rate);
// Accumulate feature importances
feature_importances += fitted_tree.feature_importances();
// Store the tree
estimators.push(fitted_tree);
// Compute training score (mean squared error)
let loss = self.compute_loss(y, ¤t_predictions);
train_scores.push(loss);
// Early stopping could be added here based on validation score
}
// Normalize feature importances
let sum_importances: Float = feature_importances.sum();
if sum_importances > 0.0 {
feature_importances /= sum_importances;
}
Ok(GradientBoostingMultiOutput {
state: GradientBoostingMultiOutputTrained {
estimators,
init_predictions,
n_features,
n_targets,
feature_importances,
train_scores,
},
n_estimators: self.n_estimators,
learning_rate: self.learning_rate,
max_depth: self.max_depth,
min_samples_split: self.min_samples_split,
min_samples_leaf: self.min_samples_leaf,
subsample: self.subsample,
random_state: self.random_state,
})
}
}
impl GradientBoostingMultiOutput<Untrained> {
fn create_subsample(
&self,
X: &Array2<Float>,
y: &Array2<Float>,
seed: usize,
) -> SklResult<(Array2<Float>, Array2<Float>, Vec<usize>)> {
let n_samples = X.nrows();
let subsample_size = (n_samples as Float * self.subsample).ceil() as usize;
let mut rng_state = self.random_state.unwrap_or(42).wrapping_add(seed as u64);
let mut indices = Vec::new();
for _ in 0..subsample_size {
rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
let idx = (rng_state / 65536) % (n_samples as u64);
indices.push(idx as usize);
}
let mut X_subsample = Array2::<Float>::zeros((subsample_size, X.ncols()));
let mut y_subsample = Array2::<Float>::zeros((subsample_size, y.ncols()));
for (i, &idx) in indices.iter().enumerate() {
X_subsample
.slice_mut(s![i, ..])
.assign(&X.slice(s![idx, ..]));
y_subsample
.slice_mut(s![i, ..])
.assign(&y.slice(s![idx, ..]));
}
Ok((X_subsample, y_subsample, indices))
}
fn compute_loss(&self, y_true: &Array2<Float>, y_pred: &Array2<Float>) -> Float {
let diff = y_true - y_pred;
let squared_diff = &diff * &diff;
squared_diff.sum() / (y_true.nrows() * y_true.ncols()) as Float
}
}
impl GradientBoostingMultiOutput<GradientBoostingMultiOutputTrained> {
/// Get the feature importances averaged across all trees
pub fn feature_importances(&self) -> &Array1<Float> {
&self.state.feature_importances
}
/// Get the number of estimators (trees)
pub fn n_estimators(&self) -> usize {
self.state.estimators.len()
}
/// Get the number of features
pub fn n_features(&self) -> usize {
self.state.n_features
}
/// Get the number of targets
pub fn n_targets(&self) -> usize {
self.state.n_targets
}
/// Get the training scores (loss at each iteration)
pub fn train_scores(&self) -> &[Float] {
&self.state.train_scores
}
/// Predict using the gradient boosting model
pub fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
// Initialize with initial predictions
let mut predictions = Array2::<Float>::zeros((n_samples, self.state.n_targets));
for i in 0..n_samples {
for j in 0..self.state.n_targets {
predictions[[i, j]] = self.state.init_predictions[j];
}
}
// Add contributions from all trees
for estimator in &self.state.estimators {
let tree_predictions = estimator.predict(&X)?;
// Apply learning rate from the training configuration
let learning_rate = 0.1; // We should store this in the trained state, but for now use default
predictions += &(tree_predictions * learning_rate);
}
Ok(predictions)
}
/// Predict using only the first n_estimators trees
pub fn staged_predict(
&self,
X: &ArrayView2<'_, Float>,
n_estimators: usize,
) -> SklResult<Array2<Float>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let max_estimators = n_estimators.min(self.state.estimators.len());
// Initialize with initial predictions
let mut predictions = Array2::<Float>::zeros((n_samples, self.state.n_targets));
for i in 0..n_samples {
for j in 0..self.state.n_targets {
predictions[[i, j]] = self.state.init_predictions[j];
}
}
// Add contributions from first n_estimators trees
for estimator in &self.state.estimators[..max_estimators] {
let tree_predictions = estimator.predict(&X)?;
let learning_rate = 0.1; // Should be stored in trained state
predictions += &(tree_predictions * learning_rate);
}
Ok(predictions)
}
}
/// Independent Label Prediction
///
/// A meta-estimator that treats each label as an independent binary classification problem.
/// This is essentially an enhanced version of binary relevance that provides more flexibility
/// and control over the individual binary classifiers.
///
/// # Mathematical Foundation
///
/// For multi-label classification with L labels:
/// - Train L independent binary classifiers: f_j(x) for j = 1..L
/// - Each classifier predicts P(y_j = 1 | x) independently
/// - Final prediction: ŷ = [I(f_1(x) > τ_1), ..., I(f_L(x) > τ_L)]
/// - Thresholds τ_j can be learned or set manually for each label
///
/// # Examples
///
/// ```
/// use sklears_multioutput::{IndependentLabelPrediction, ThresholdStrategy};
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
///
/// let ilp = IndependentLabelPrediction::new()
/// .threshold_strategy(ThresholdStrategy::Optimal)
/// .optimize_thresholds(true);
/// let trained_ilp = ilp.fit(&X.view(), &y).unwrap();
/// let predictions = trained_ilp.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct IndependentLabelPrediction<S = Untrained> {
state: S,
threshold_strategy: ThresholdStrategy,
optimize_thresholds: bool,
class_weight: Option<String>, // "balanced" or None
random_state: Option<u64>,
}
#[derive(Debug, Clone)]
pub enum ThresholdStrategy {
Fixed(Float), // Use fixed threshold for all labels
PerLabel(Vec<Float>), // Use different threshold for each label
Optimal, // Learn optimal thresholds from validation data
FScore, // Optimize F-score threshold for each label
}
#[derive(Debug, Clone)]
pub struct IndependentLabelPredictionTrained {
binary_classifiers: Vec<BinaryClassifierModel>,
thresholds: Vec<Float>,
n_features: usize,
n_labels: usize,
label_frequencies: Vec<Float>, // Frequency of positive labels
}
#[derive(Debug, Clone)]
struct BinaryClassifierModel {
feature_means: Array1<Float>,
feature_stds: Array1<Float>,
model: SimpleBinaryModel,
}
impl IndependentLabelPrediction<Untrained> {
/// Create a new IndependentLabelPrediction instance
pub fn new() -> Self {
Self {
state: Untrained,
threshold_strategy: ThresholdStrategy::Fixed(0.5),
optimize_thresholds: false,
class_weight: None,
random_state: None,
}
}
/// Set the threshold strategy for binary predictions
pub fn threshold_strategy(mut self, strategy: ThresholdStrategy) -> Self {
self.threshold_strategy = strategy;
self
}
/// Set whether to optimize thresholds based on training data
pub fn optimize_thresholds(mut self, optimize: bool) -> Self {
self.optimize_thresholds = optimize;
self
}
/// Set class weighting strategy
pub fn class_weight(mut self, class_weight: Option<String>) -> Self {
self.class_weight = class_weight;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for IndependentLabelPrediction<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for IndependentLabelPrediction<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for IndependentLabelPrediction<Untrained> {
type Fitted = IndependentLabelPrediction<IndependentLabelPredictionTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let n_labels = y.ncols();
if n_labels == 0 {
return Err(SklearsError::InvalidInput(
"y must have at least one label".to_string(),
));
}
// Validate that labels are binary
for &label in y.iter() {
if label != 0 && label != 1 {
return Err(SklearsError::InvalidInput(
"Labels must be binary (0 or 1)".to_string(),
));
}
}
// Compute feature statistics for standardization
let feature_means = X.mean_axis(Axis(0)).ok_or(SklearsError::InvalidInput(
"Cannot compute feature means".to_string(),
))?;
let feature_stds = X
.std_axis(Axis(0), 0.0)
.mapv(|x| if x == 0.0 { 1.0 } else { x });
// Standardize features
let X_standardized = standardize_features_simple(&X.view(), &feature_means, &feature_stds);
let mut binary_classifiers = Vec::new();
let mut label_frequencies = Vec::new();
// Train one binary classifier for each label
for j in 0..n_labels {
let y_binary = y.column(j).to_owned();
// Compute label frequency
let positive_count = y_binary.iter().filter(|&&x| x == 1).count();
let frequency = positive_count as Float / n_samples as Float;
label_frequencies.push(frequency);
// Apply class weighting if specified
let class_weights = if self.class_weight.as_deref() == Some("balanced") {
// Compute balanced class weights
let mut weights = Array1::<Float>::ones(n_samples);
let positive_count = y_binary.iter().filter(|&&x| x == 1).count();
let negative_count = n_samples - positive_count;
if positive_count > 0 && negative_count > 0 {
let pos_weight = n_samples as Float / (2.0 * positive_count as Float);
let neg_weight = n_samples as Float / (2.0 * negative_count as Float);
for (i, &label) in y_binary.iter().enumerate() {
weights[i] = if label == 1 { pos_weight } else { neg_weight };
}
}
weights
} else {
Array1::<Float>::ones(n_samples)
};
// Train binary classifier
let simple_model = train_weighted_binary_classifier_simple(
&X_standardized,
&y_binary,
&class_weights,
)?;
let classifier_model = BinaryClassifierModel {
feature_means: feature_means.clone(),
feature_stds: feature_stds.clone(),
model: simple_model,
};
binary_classifiers.push(classifier_model);
}
// Determine thresholds
let thresholds = match &self.threshold_strategy {
ThresholdStrategy::Fixed(threshold) => vec![*threshold; n_labels],
ThresholdStrategy::PerLabel(thresholds) => {
if thresholds.len() != n_labels {
return Err(SklearsError::InvalidInput(
"Number of thresholds must match number of labels".to_string(),
));
}
thresholds.clone()
}
ThresholdStrategy::Optimal | ThresholdStrategy::FScore => {
if self.optimize_thresholds {
self.compute_optimal_thresholds(&X_standardized, y, &binary_classifiers)?
} else {
vec![0.5; n_labels]
}
}
};
Ok(IndependentLabelPrediction {
state: IndependentLabelPredictionTrained {
binary_classifiers,
thresholds,
n_features,
n_labels,
label_frequencies,
},
threshold_strategy: self.threshold_strategy,
optimize_thresholds: self.optimize_thresholds,
class_weight: self.class_weight,
random_state: self.random_state,
})
}
}
impl IndependentLabelPrediction<Untrained> {
fn compute_optimal_thresholds(
&self,
X: &Array2<Float>,
y: &Array2<i32>,
classifiers: &[BinaryClassifierModel],
) -> SklResult<Vec<Float>> {
let n_labels = y.ncols();
let mut optimal_thresholds = Vec::new();
for j in 0..n_labels {
let y_binary = y.column(j);
// Get prediction scores for this label
let scores = predict_binary_probabilities(&classifiers[j].model, X);
// Find optimal threshold based on strategy
let optimal_threshold = match &self.threshold_strategy {
ThresholdStrategy::FScore => self.find_best_f_score_threshold(&y_binary, &scores),
_ => {
// For other strategies, use balanced accuracy
self.find_best_balanced_accuracy_threshold(&y_binary, &scores)
}
};
optimal_thresholds.push(optimal_threshold);
}
Ok(optimal_thresholds)
}
fn find_best_f_score_threshold(
&self,
y_true: &ArrayView1<'_, i32>,
scores: &Array1<Float>,
) -> Float {
let mut best_threshold = 0.5;
let mut best_f_score = 0.0;
// Try different thresholds
for &threshold in &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] {
let y_pred: Array1<i32> = scores.mapv(|x| if x >= threshold { 1 } else { 0 });
let f_score = self.compute_f_score(&y_true, &y_pred);
if f_score > best_f_score {
best_f_score = f_score;
best_threshold = threshold;
}
}
best_threshold
}
fn find_best_balanced_accuracy_threshold(
&self,
y_true: &ArrayView1<'_, i32>,
scores: &Array1<Float>,
) -> Float {
let mut best_threshold = 0.5;
let mut best_accuracy = 0.0;
// Try different thresholds
for &threshold in &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] {
let y_pred: Array1<i32> = scores.mapv(|x| if x >= threshold { 1 } else { 0 });
let accuracy = self.compute_balanced_accuracy(&y_true, &y_pred);
if accuracy > best_accuracy {
best_accuracy = accuracy;
best_threshold = threshold;
}
}
best_threshold
}
fn compute_f_score(&self, y_true: &ArrayView1<'_, i32>, y_pred: &Array1<i32>) -> Float {
let mut tp = 0;
let mut fp = 0;
let mut fn_count = 0;
for (true_val, pred_val) in y_true.iter().zip(y_pred.iter()) {
match (true_val, pred_val) {
(1, 1) => tp += 1,
(0, 1) => fp += 1,
(1, 0) => fn_count += 1,
_ => {} // TN
}
}
if tp + fp == 0 || tp + fn_count == 0 {
return 0.0;
}
let precision = tp as Float / (tp + fp) as Float;
let recall = tp as Float / (tp + fn_count) as Float;
if precision + recall == 0.0 {
0.0
} else {
2.0 * precision * recall / (precision + recall)
}
}
fn compute_balanced_accuracy(
&self,
y_true: &ArrayView1<'_, i32>,
y_pred: &Array1<i32>,
) -> Float {
let mut tp = 0;
let mut tn = 0;
let mut fp = 0;
let mut fn_count = 0;
for (true_val, pred_val) in y_true.iter().zip(y_pred.iter()) {
match (true_val, pred_val) {
(1, 1) => tp += 1,
(0, 0) => tn += 1,
(0, 1) => fp += 1,
(1, 0) => fn_count += 1,
_ => {}
}
}
let sensitivity = if tp + fn_count > 0 {
tp as Float / (tp + fn_count) as Float
} else {
0.0
};
let specificity = if tn + fp > 0 {
tn as Float / (tn + fp) as Float
} else {
0.0
};
(sensitivity + specificity) / 2.0
}
}
impl IndependentLabelPrediction<IndependentLabelPredictionTrained> {
/// Get the optimized thresholds for each label
pub fn thresholds(&self) -> &[Float] {
&self.state.thresholds
}
/// Get the label frequencies from training data
pub fn label_frequencies(&self) -> &[Float] {
&self.state.label_frequencies
}
/// Get the number of features
pub fn n_features(&self) -> usize {
self.state.n_features
}
/// Get the number of labels
pub fn n_labels(&self) -> usize {
self.state.n_labels
}
/// Predict binary labels using the trained classifiers and thresholds
pub fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let probabilities = self.predict_proba(&X)?;
let mut predictions = Array2::<i32>::zeros((n_samples, self.state.n_labels));
for i in 0..n_samples {
for j in 0..self.state.n_labels {
predictions[[i, j]] = if probabilities[[i, j]] >= self.state.thresholds[j] {
1
} else {
0
};
}
}
Ok(predictions)
}
/// Predict class probabilities for each label
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let X = *X;
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let mut probabilities = Array2::<Float>::zeros((n_samples, self.state.n_labels));
for j in 0..self.state.n_labels {
let classifier = &self.state.binary_classifiers[j];
// Standardize features using the same statistics as training
let X_standardized = standardize_features_simple(
&X,
&classifier.feature_means,
&classifier.feature_stds,
);
// Predict probabilities for this label
let proba = predict_binary_probabilities(&classifier.model, &X_standardized);
for i in 0..n_samples {
probabilities[[i, j]] = proba[i];
}
}
Ok(probabilities)
}
/// Predict using custom thresholds
pub fn predict_with_thresholds(
&self,
X: &ArrayView2<'_, Float>,
thresholds: &[Float],
) -> SklResult<Array2<i32>> {
if thresholds.len() != self.state.n_labels {
return Err(SklearsError::InvalidInput(
"Number of thresholds must match number of labels".to_string(),
));
}
let X = *X;
let (n_samples, _) = X.dim();
let probabilities = self.predict_proba(&X)?;
let mut predictions = Array2::<i32>::zeros((n_samples, self.state.n_labels));
for i in 0..n_samples {
for j in 0..self.state.n_labels {
predictions[[i, j]] = if probabilities[[i, j]] >= thresholds[j] {
1
} else {
0
};
}
}
Ok(predictions)
}
}
/// Instance-Based Learning for Regression (IBLR)
///
/// IBLR is a k-nearest neighbors approach for multi-output regression that predicts
/// multiple continuous target values simultaneously. It uses distance-based similarity
/// to find the k-nearest neighbors and averages their target values for prediction.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::IBLR;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]]; // Multiple continuous targets
///
/// let iblr = IBLR::new().k_neighbors(2);
/// let trained_iblr = iblr.fit(&X.view(), &y).unwrap();
/// let predictions = trained_iblr.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct IBLR<S = Untrained> {
state: S,
k_neighbors: usize,
weights: WeightFunction,
}
/// Weight function for IBLR
#[derive(Debug, Clone)]
pub enum WeightFunction {
/// Uniform weighting (all neighbors weighted equally)
Uniform,
/// Distance-based weighting (closer neighbors weighted more)
Distance,
}
/// Trained state for IBLR
#[derive(Debug, Clone)]
pub struct IBLRTrained {
X_train: Array2<Float>,
y_train: Array2<Float>,
k_neighbors: usize,
weights: WeightFunction,
n_outputs: usize,
}
impl IBLR<Untrained> {
/// Create a new IBLR instance
pub fn new() -> Self {
Self {
state: Untrained,
k_neighbors: 5,
weights: WeightFunction::Uniform,
}
}
/// Set the number of neighbors to consider
pub fn k_neighbors(mut self, k: usize) -> Self {
self.k_neighbors = k;
self
}
/// Set the weight function
pub fn weights(mut self, weights: WeightFunction) -> Self {
self.weights = weights;
self
}
}
impl Default for IBLR<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for IBLR<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<Float>> for IBLR<Untrained> {
type Fitted = IBLR<IBLRTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<Float>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let (n_samples_y, n_outputs) = y.dim();
if n_samples != n_samples_y {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot fit with 0 samples".to_string(),
));
}
if self.k_neighbors == 0 {
return Err(SklearsError::InvalidInput(
"k_neighbors must be greater than 0".to_string(),
));
}
if self.k_neighbors > n_samples {
return Err(SklearsError::InvalidInput(
"k_neighbors cannot be greater than the number of training samples".to_string(),
));
}
let X_train = X.to_owned();
let y_train = y.clone();
Ok(IBLR {
state: IBLRTrained {
X_train,
y_train,
k_neighbors: self.k_neighbors,
weights: self.weights.clone(),
n_outputs,
},
k_neighbors: self.k_neighbors,
weights: self.weights,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<Float>> for IBLR<IBLRTrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.X_train.ncols() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
let mut predictions = Array2::zeros((n_samples, self.state.n_outputs));
for i in 0..n_samples {
let query_point = X.row(i);
// Calculate distances to all training points
let mut distances_with_indices: Vec<(Float, usize)> = Vec::new();
for j in 0..self.state.X_train.nrows() {
let train_point = self.state.X_train.row(j);
let distance = euclidean_distance(&query_point, &train_point);
distances_with_indices.push((distance, j));
}
// Sort by distance and take k nearest
distances_with_indices.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
let k_nearest = &distances_with_indices[..self.state.k_neighbors];
// Calculate weighted average of k-nearest neighbor targets
match self.state.weights {
WeightFunction::Uniform => {
// Simple average
for output_idx in 0..self.state.n_outputs {
let sum: Float = k_nearest
.iter()
.map(|(_, idx)| self.state.y_train[[*idx, output_idx]])
.sum();
predictions[[i, output_idx]] = sum / (self.state.k_neighbors as Float);
}
}
WeightFunction::Distance => {
// Distance-weighted average
let total_weight: Float = k_nearest
.iter()
.map(|(dist, _)| if *dist == 0.0 { 1e10 } else { 1.0 / dist })
.sum();
if total_weight == 0.0 {
return Err(SklearsError::InvalidInput(
"All distances are zero or invalid".to_string(),
));
}
for output_idx in 0..self.state.n_outputs {
let weighted_sum: Float = k_nearest
.iter()
.map(|(dist, idx)| {
let weight = if *dist == 0.0 { 1e10 } else { 1.0 / dist };
weight * self.state.y_train[[*idx, output_idx]]
})
.sum();
predictions[[i, output_idx]] = weighted_sum / total_weight;
}
}
}
}
Ok(predictions)
}
}
/// Helper function to calculate Euclidean distance between two points
fn euclidean_distance(point1: &ArrayView1<Float>, point2: &ArrayView1<Float>) -> Float {
point1
.iter()
.zip(point2.iter())
.map(|(a, b)| (a - b).powi(2))
.sum::<Float>()
.sqrt()
}
/// CLARE (Clustering and LAabel RElevance)
///
/// CLARE is a multi-label classification method that combines clustering with label relevance.
/// It works by clustering the feature space and learning label frequencies for each cluster,
/// then making predictions based on which cluster a new instance belongs to.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::CLARE;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
///
/// let clare = CLARE::new().n_clusters(2);
/// let trained_clare = clare.fit(&X.view(), &y).unwrap();
/// let predictions = trained_clare.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct CLARE<S = Untrained> {
state: S,
n_clusters: usize,
threshold: Float,
max_iter: usize,
random_state: Option<u64>,
}
/// Trained state for CLARE
#[derive(Debug, Clone)]
pub struct CLARETrained {
cluster_centers: Array2<Float>,
label_relevance: Array2<Float>, // n_clusters x n_labels
threshold: Float,
n_labels: usize,
}
impl CLARE<Untrained> {
/// Create a new CLARE instance
pub fn new() -> Self {
Self {
state: Untrained,
n_clusters: 3,
threshold: 0.5,
max_iter: 100,
random_state: None,
}
}
/// Set the number of clusters
pub fn n_clusters(mut self, n_clusters: usize) -> Self {
self.n_clusters = n_clusters;
self
}
/// Set the prediction threshold
pub fn threshold(mut self, threshold: Float) -> Self {
self.threshold = threshold;
self
}
/// Set the maximum number of iterations for clustering
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
/// Set the random state for reproducible results
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for CLARE<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for CLARE<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for CLARE<Untrained> {
type Fitted = CLARE<CLARETrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_labels = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot fit with 0 samples".to_string(),
));
}
if self.n_clusters == 0 {
return Err(SklearsError::InvalidInput(
"n_clusters must be greater than 0".to_string(),
));
}
if self.n_clusters > n_samples {
return Err(SklearsError::InvalidInput(
"n_clusters cannot be greater than the number of samples".to_string(),
));
}
// Validate binary labels
for &val in y.iter() {
if val != 0 && val != 1 {
return Err(SklearsError::InvalidInput(
"y must contain only binary values (0 or 1)".to_string(),
));
}
}
// Perform k-means clustering
let (cluster_centers, cluster_assignments) =
k_means_clustering(X, self.n_clusters, self.max_iter, self.random_state)?;
// Calculate label relevance for each cluster
let mut label_relevance = Array2::zeros((self.n_clusters, n_labels));
let mut cluster_counts = vec![0; self.n_clusters];
// Count samples in each cluster and accumulate label frequencies
for (sample_idx, &cluster_id) in cluster_assignments.iter().enumerate() {
cluster_counts[cluster_id] += 1;
for label_idx in 0..n_labels {
if y[[sample_idx, label_idx]] == 1 {
label_relevance[[cluster_id, label_idx]] += 1.0;
}
}
}
// Normalize label frequencies to get relevance scores
for cluster_id in 0..self.n_clusters {
if cluster_counts[cluster_id] > 0 {
let count = cluster_counts[cluster_id] as Float;
for label_idx in 0..n_labels {
label_relevance[[cluster_id, label_idx]] /= count;
}
}
}
Ok(CLARE {
state: CLARETrained {
cluster_centers,
label_relevance,
threshold: self.threshold,
n_labels,
},
n_clusters: self.n_clusters,
threshold: self.threshold,
max_iter: self.max_iter,
random_state: self.random_state,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>> for CLARE<CLARETrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.cluster_centers.ncols() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
for (sample_idx, sample) in X.axis_iter(Axis(0)).enumerate() {
// Find closest cluster
let mut min_distance = Float::INFINITY;
let mut closest_cluster = 0;
for (cluster_idx, cluster_center) in
self.state.cluster_centers.axis_iter(Axis(0)).enumerate()
{
let distance = euclidean_distance(&sample, &cluster_center);
if distance < min_distance {
min_distance = distance;
closest_cluster = cluster_idx;
}
}
// Predict labels based on cluster's label relevance
for label_idx in 0..self.state.n_labels {
let relevance = self.state.label_relevance[[closest_cluster, label_idx]];
predictions[[sample_idx, label_idx]] = if relevance >= self.state.threshold {
1
} else {
0
};
}
}
Ok(predictions)
}
}
impl CLARE<CLARETrained> {
/// Get the cluster centers
pub fn cluster_centers(&self) -> &Array2<Float> {
&self.state.cluster_centers
}
/// Get the label relevance matrix
pub fn label_relevance(&self) -> &Array2<Float> {
&self.state.label_relevance
}
/// Get the prediction threshold
pub fn threshold(&self) -> Float {
self.state.threshold
}
/// Predict probabilities (label relevance scores)
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.cluster_centers.ncols() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
let mut probabilities = Array2::zeros((n_samples, self.state.n_labels));
for (sample_idx, sample) in X.axis_iter(Axis(0)).enumerate() {
// Find closest cluster
let mut min_distance = Float::INFINITY;
let mut closest_cluster = 0;
for (cluster_idx, cluster_center) in
self.state.cluster_centers.axis_iter(Axis(0)).enumerate()
{
let distance = euclidean_distance(&sample, &cluster_center);
if distance < min_distance {
min_distance = distance;
closest_cluster = cluster_idx;
}
}
// Return label relevance scores as probabilities
for label_idx in 0..self.state.n_labels {
probabilities[[sample_idx, label_idx]] =
self.state.label_relevance[[closest_cluster, label_idx]];
}
}
Ok(probabilities)
}
}
/// Simple k-means clustering implementation
fn k_means_clustering(
X: &ArrayView2<'_, Float>,
n_clusters: usize,
max_iter: usize,
random_state: Option<u64>,
) -> SklResult<(Array2<Float>, Vec<usize>)> {
let (n_samples, n_features) = X.dim();
// Initialize cluster centers randomly
let mut cluster_centers = Array2::zeros((n_clusters, n_features));
// Simple random initialization using deterministic approach if random_state is provided
if let Some(seed) = random_state {
let mut rng_state = seed;
for i in 0..n_clusters {
let sample_idx = (rng_state as usize + i) % n_samples;
for j in 0..n_features {
cluster_centers[[i, j]] = X[[sample_idx, j]];
}
rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
}
} else {
// Use first n_clusters samples as initial centers
for i in 0..n_clusters {
let sample_idx = i % n_samples;
for j in 0..n_features {
cluster_centers[[i, j]] = X[[sample_idx, j]];
}
}
}
let mut cluster_assignments = vec![0; n_samples];
for _iter in 0..max_iter {
let mut changed = false;
// Assign each sample to the closest cluster
for (sample_idx, sample) in X.axis_iter(Axis(0)).enumerate() {
let mut min_distance = Float::INFINITY;
let mut closest_cluster = 0;
for (cluster_idx, cluster_center) in cluster_centers.axis_iter(Axis(0)).enumerate() {
let distance = euclidean_distance(&sample, &cluster_center);
if distance < min_distance {
min_distance = distance;
closest_cluster = cluster_idx;
}
}
if cluster_assignments[sample_idx] != closest_cluster {
cluster_assignments[sample_idx] = closest_cluster;
changed = true;
}
}
// Update cluster centers
let mut new_centers = Array2::zeros((n_clusters, n_features));
let mut cluster_counts = vec![0; n_clusters];
for (sample_idx, sample) in X.axis_iter(Axis(0)).enumerate() {
let cluster_id = cluster_assignments[sample_idx];
cluster_counts[cluster_id] += 1;
for (feature_idx, &value) in sample.iter().enumerate() {
new_centers[[cluster_id, feature_idx]] += value;
}
}
// Average to get new centers
for cluster_id in 0..n_clusters {
if cluster_counts[cluster_id] > 0 {
let count = cluster_counts[cluster_id] as Float;
for feature_idx in 0..n_features {
new_centers[[cluster_id, feature_idx]] /= count;
}
}
}
cluster_centers = new_centers;
if !changed {
break;
}
}
Ok((cluster_centers, cluster_assignments))
}
/// MLTSVM (Multi-Label Twin SVM)
///
/// MLTSVM is a multi-label classification method that extends Twin SVM to handle
/// multiple labels. Twin SVM finds two non-parallel hyperplanes for binary classification,
/// which often leads to faster training than standard SVM. MLTSVM applies this approach
/// to each label independently in a binary relevance fashion.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::MLTSVM;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
///
/// let mltsvm = MLTSVM::new().c1(1.0).c2(1.0);
/// let trained_mltsvm = mltsvm.fit(&X.view(), &y).unwrap();
/// let predictions = trained_mltsvm.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct MLTSVM<S = Untrained> {
state: S,
c1: Float, // Regularization parameter for first hyperplane
c2: Float, // Regularization parameter for second hyperplane
epsilon: Float, // Tolerance for convergence
max_iter: usize, // Maximum iterations
}
/// Trained state for MLTSVM
#[derive(Debug, Clone)]
pub struct MLTSVMTrained {
models: Vec<TwinSVMModel>, // One model per label
n_labels: usize,
feature_means: Array1<Float>,
feature_stds: Array1<Float>,
}
/// Single Twin SVM model for binary classification
#[derive(Debug, Clone)]
struct TwinSVMModel {
w1: Array1<Float>, // Weights for first hyperplane
b1: Float, // Bias for first hyperplane
w2: Array1<Float>, // Weights for second hyperplane
b2: Float, // Bias for second hyperplane
}
impl MLTSVM<Untrained> {
/// Create a new MLTSVM instance
pub fn new() -> Self {
Self {
state: Untrained,
c1: 1.0,
c2: 1.0,
epsilon: 1e-6,
max_iter: 1000,
}
}
/// Set the regularization parameter for the first hyperplane
pub fn c1(mut self, c1: Float) -> Self {
self.c1 = c1;
self
}
/// Set the regularization parameter for the second hyperplane
pub fn c2(mut self, c2: Float) -> Self {
self.c2 = c2;
self
}
/// Set the convergence tolerance
pub fn epsilon(mut self, epsilon: Float) -> Self {
self.epsilon = epsilon;
self
}
/// Set the maximum number of iterations
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
}
impl Default for MLTSVM<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MLTSVM<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for MLTSVM<Untrained> {
type Fitted = MLTSVM<MLTSVMTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_labels = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot fit with 0 samples".to_string(),
));
}
// Validate binary labels
for &val in y.iter() {
if val != 0 && val != 1 {
return Err(SklearsError::InvalidInput(
"y must contain only binary values (0 or 1)".to_string(),
));
}
}
// Standardize features
let feature_means = X.mean_axis(Axis(0)).unwrap();
let feature_stds = X
.std_axis(Axis(0), 0.0)
.mapv(|x| if x == 0.0 { 1.0 } else { x });
let mut X_scaled = X.to_owned();
for i in 0..n_samples {
for j in 0..n_features {
X_scaled[[i, j]] = (X_scaled[[i, j]] - feature_means[j]) / feature_stds[j];
}
}
// Train Twin SVM for each label
let mut models = Vec::with_capacity(n_labels);
for label_idx in 0..n_labels {
let y_label = y.column(label_idx);
let model = train_twin_svm(
&X_scaled.view(),
&y_label,
self.c1,
self.c2,
self.epsilon,
self.max_iter,
)?;
models.push(model);
}
Ok(MLTSVM {
state: MLTSVMTrained {
models,
n_labels,
feature_means,
feature_stds,
},
c1: self.c1,
c2: self.c2,
epsilon: self.epsilon,
max_iter: self.max_iter,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>> for MLTSVM<MLTSVMTrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
// Standardize features using training statistics
let mut X_scaled = X.to_owned();
for i in 0..n_samples {
for j in 0..n_features {
X_scaled[[i, j]] =
(X_scaled[[i, j]] - self.state.feature_means[j]) / self.state.feature_stds[j];
}
}
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
// Predict for each label using its Twin SVM model
for (label_idx, model) in self.state.models.iter().enumerate() {
for sample_idx in 0..n_samples {
let x = X_scaled.row(sample_idx);
// Calculate distances to both hyperplanes
let dist1 = x.dot(&model.w1) + model.b1;
let dist2 = x.dot(&model.w2) + model.b2;
// Assign to class based on which hyperplane is closer
// Positive if closer to positive class hyperplane, negative otherwise
predictions[[sample_idx, label_idx]] = if dist1.abs() < dist2.abs() {
1 // Closer to positive class hyperplane
} else {
0 // Closer to negative class hyperplane
};
}
}
Ok(predictions)
}
}
impl MLTSVM<MLTSVMTrained> {
/// Get the number of labels
pub fn n_labels(&self) -> usize {
self.state.n_labels
}
/// Predict decision function values (distances to hyperplanes)
pub fn decision_function(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
// Standardize features
let mut X_scaled = X.to_owned();
for i in 0..n_samples {
for j in 0..n_features {
X_scaled[[i, j]] =
(X_scaled[[i, j]] - self.state.feature_means[j]) / self.state.feature_stds[j];
}
}
let mut decision_values = Array2::zeros((n_samples, self.state.n_labels));
for (label_idx, model) in self.state.models.iter().enumerate() {
for sample_idx in 0..n_samples {
let x = X_scaled.row(sample_idx);
// Calculate signed distance difference between hyperplanes
let dist1 = x.dot(&model.w1) + model.b1;
let dist2 = x.dot(&model.w2) + model.b2;
// Decision function: positive for positive class, negative for negative class
decision_values[[sample_idx, label_idx]] = dist1 - dist2;
}
}
Ok(decision_values)
}
}
/// Train a Twin SVM model for binary classification
fn train_twin_svm(
X: &ArrayView2<'_, Float>,
y: &ArrayView1<'_, i32>,
c1: Float,
c2: Float,
epsilon: Float,
max_iter: usize,
) -> SklResult<TwinSVMModel> {
let (n_samples, n_features) = X.dim();
// Separate positive and negative samples
let mut X_pos = Vec::new();
let mut X_neg = Vec::new();
for i in 0..n_samples {
if y[i] == 1 {
X_pos.push(X.row(i).to_owned());
} else {
X_neg.push(X.row(i).to_owned());
}
}
if X_pos.is_empty() || X_neg.is_empty() {
return Err(SklearsError::InvalidInput(
"Need samples from both classes for Twin SVM".to_string(),
));
}
// Convert to matrices
let n_pos = X_pos.len();
let n_neg = X_neg.len();
let mut A = Array2::zeros((n_pos, n_features + 1));
let mut B = Array2::zeros((n_neg, n_features + 1));
// Fill A (positive samples) with [X, 1]
for (i, x_pos) in X_pos.iter().enumerate() {
for j in 0..n_features {
A[[i, j]] = x_pos[j];
}
A[[i, n_features]] = 1.0; // bias term
}
// Fill B (negative samples) with [X, 1]
for (i, x_neg) in X_neg.iter().enumerate() {
for j in 0..n_features {
B[[i, j]] = x_neg[j];
}
B[[i, n_features]] = 1.0; // bias term
}
// Solve Twin SVM optimization problems
// First hyperplane: min ||Aw1||^2 + c1*|Bw1|_1 subject to Bw1 >= e
let w1 = solve_twin_svm_problem(&A, &B, c1, true)?;
// Second hyperplane: min ||Bw2||^2 + c2*|Aw2|_1 subject to Aw2 <= -e
let w2 = solve_twin_svm_problem(&B, &A, c2, false)?;
// Extract weights and biases
let weights1 = w1.slice(s![..n_features]).to_owned();
let bias1 = w1[n_features];
let weights2 = w2.slice(s![..n_features]).to_owned();
let bias2 = w2[n_features];
Ok(TwinSVMModel {
w1: weights1,
b1: bias1,
w2: weights2,
b2: bias2,
})
}
/// Solve a single Twin SVM optimization problem
fn solve_twin_svm_problem(
A: &Array2<Float>,
B: &Array2<Float>,
c: Float,
is_first: bool,
) -> SklResult<Array1<Float>> {
let (n_a, n_features) = A.dim();
let n_b = B.nrows();
// Simplified approach: use regularized least squares approximation
// This is not the full Twin SVM solution but provides a reasonable approximation
// Create target vector
let target = if is_first {
Array1::ones(n_b) // B should be >= 1
} else {
Array1::ones(n_a) * (-1.0) // A should be <= -1
};
// Use regularized least squares: min ||Xw - target||^2 + lambda*||w||^2
let X = if is_first { B } else { A };
let lambda = 1.0 / c; // Convert regularization parameter
// Add regularization to diagonal
let mut XtX = X.t().dot(X);
for i in 0..n_features {
XtX[[i, i]] += lambda;
}
// Solve using pseudo-inverse approach
let Xty = X.t().dot(&target);
// Simple iterative solution (gradient descent approximation)
let mut w = Array1::zeros(n_features);
let learning_rate = 0.01;
for _iter in 0..1000 {
let prediction = X.dot(&w);
let residual = &prediction - ⌖
let gradient = X.t().dot(&residual) + lambda * &w;
w = &w - learning_rate * &gradient;
// Check convergence
if gradient.mapv(|x| x.abs()).sum() < 1e-6 {
break;
}
}
Ok(w)
}
/// RankSVM for Multi-Label Classification
///
/// RankSVM is a ranking-based approach for multi-label classification that optimizes
/// ranking loss functions. It learns to rank labels by their relevance scores and
/// can handle both label ranking and threshold selection for multi-label prediction.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::RankSVM;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
///
/// let ranksvm = RankSVM::new().c(1.0);
/// let trained_ranksvm = ranksvm.fit(&X.view(), &y).unwrap();
/// let predictions = trained_ranksvm.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct RankSVM<S = Untrained> {
state: S,
c: Float, // Regularization parameter
epsilon: Float, // Tolerance for convergence
max_iter: usize, // Maximum iterations
threshold_strategy: ThresholdStrategy, // How to determine prediction thresholds
}
/// Trained state for RankSVM
#[derive(Debug, Clone)]
pub struct RankSVMTrained {
models: Vec<RankingSVMModel>, // One model per label
thresholds: Array1<Float>, // Prediction thresholds
n_labels: usize,
feature_means: Array1<Float>,
feature_stds: Array1<Float>,
threshold_strategy: ThresholdStrategy,
}
/// Single ranking SVM model for one label
#[derive(Debug, Clone)]
struct RankingSVMModel {
weights: Array1<Float>,
bias: Float,
}
impl RankSVM<Untrained> {
/// Create a new RankSVM instance
pub fn new() -> Self {
Self {
state: Untrained,
c: 1.0,
epsilon: 1e-6,
max_iter: 1000,
threshold_strategy: ThresholdStrategy::Optimal,
}
}
/// Set the regularization parameter
pub fn c(mut self, c: Float) -> Self {
self.c = c;
self
}
/// Set the convergence tolerance
pub fn epsilon(mut self, epsilon: Float) -> Self {
self.epsilon = epsilon;
self
}
/// Set the maximum number of iterations
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
/// Set the threshold strategy
pub fn threshold_strategy(mut self, strategy: ThresholdStrategy) -> Self {
self.threshold_strategy = strategy;
self
}
}
impl Default for RankSVM<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for RankSVM<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for RankSVM<Untrained> {
type Fitted = RankSVM<RankSVMTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_labels = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot fit with 0 samples".to_string(),
));
}
// Validate binary labels
for &val in y.iter() {
if val != 0 && val != 1 {
return Err(SklearsError::InvalidInput(
"y must contain only binary values (0 or 1)".to_string(),
));
}
}
// Standardize features
let feature_means = X.mean_axis(Axis(0)).unwrap();
let feature_stds = X
.std_axis(Axis(0), 0.0)
.mapv(|x| if x == 0.0 { 1.0 } else { x });
let mut X_scaled = X.to_owned();
for i in 0..n_samples {
for j in 0..n_features {
X_scaled[[i, j]] = (X_scaled[[i, j]] - feature_means[j]) / feature_stds[j];
}
}
// Train ranking SVM for each label
let mut models = Vec::with_capacity(n_labels);
for label_idx in 0..n_labels {
let y_label = y.column(label_idx);
let model = train_ranking_svm(
&X_scaled.view(),
&y_label,
self.c,
self.epsilon,
self.max_iter,
)?;
models.push(model);
}
// Determine thresholds based on strategy
let thresholds =
determine_thresholds(&models, &X_scaled.view(), y, &self.threshold_strategy)?;
Ok(RankSVM {
state: RankSVMTrained {
models,
thresholds,
n_labels,
feature_means,
feature_stds,
threshold_strategy: self.threshold_strategy.clone(),
},
c: self.c,
epsilon: self.epsilon,
max_iter: self.max_iter,
threshold_strategy: self.threshold_strategy,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>> for RankSVM<RankSVMTrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
// Get ranking scores first
let scores = self.decision_function(X)?;
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
// Apply thresholds based on strategy
match &self.state.threshold_strategy {
ThresholdStrategy::Fixed(threshold) => {
for i in 0..n_samples {
for j in 0..self.state.n_labels {
predictions[[i, j]] = if scores[[i, j]] >= *threshold { 1 } else { 0 };
}
}
}
ThresholdStrategy::PerLabel(thresholds) => {
for i in 0..n_samples {
for j in 0..self.state.n_labels {
let threshold = if j < thresholds.len() {
thresholds[j]
} else {
0.5
};
predictions[[i, j]] = if scores[[i, j]] >= threshold { 1 } else { 0 };
}
}
}
ThresholdStrategy::Optimal | ThresholdStrategy::FScore => {
// Use optimized thresholds stored during training
for i in 0..n_samples {
for j in 0..self.state.n_labels {
let threshold = if j < self.state.thresholds.len() {
self.state.thresholds[j]
} else {
0.5
};
predictions[[i, j]] = if scores[[i, j]] >= threshold { 1 } else { 0 };
}
}
}
}
Ok(predictions)
}
}
impl RankSVM<RankSVMTrained> {
/// Get the number of labels
pub fn n_labels(&self) -> usize {
self.state.n_labels
}
/// Get the prediction thresholds
pub fn thresholds(&self) -> &Array1<Float> {
&self.state.thresholds
}
/// Predict ranking scores (decision function values)
pub fn decision_function(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
if n_samples == 0 {
return Err(SklearsError::InvalidInput(
"Cannot predict with 0 samples".to_string(),
));
}
// Standardize features
let mut X_scaled = X.to_owned();
for i in 0..n_samples {
for j in 0..n_features {
X_scaled[[i, j]] =
(X_scaled[[i, j]] - self.state.feature_means[j]) / self.state.feature_stds[j];
}
}
let mut scores = Array2::zeros((n_samples, self.state.n_labels));
// Compute ranking scores for each label
for (label_idx, model) in self.state.models.iter().enumerate() {
for sample_idx in 0..n_samples {
let x = X_scaled.row(sample_idx);
scores[[sample_idx, label_idx]] = x.dot(&model.weights) + model.bias;
}
}
Ok(scores)
}
/// Predict label rankings (sorted by relevance score)
pub fn predict_ranking(&self, X: &ArrayView2<'_, Float>) -> SklResult<Vec<Vec<usize>>> {
let scores = self.decision_function(X)?;
let n_samples = scores.nrows();
let mut rankings = Vec::with_capacity(n_samples);
for i in 0..n_samples {
let mut label_scores: Vec<(usize, Float)> = (0..self.state.n_labels)
.map(|j| (j, scores[[i, j]]))
.collect();
// Sort by score (descending)
label_scores.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
let ranking: Vec<usize> = label_scores.into_iter().map(|(idx, _)| idx).collect();
rankings.push(ranking);
}
Ok(rankings)
}
}
/// Train a ranking SVM model for binary classification
fn train_ranking_svm(
X: &ArrayView2<'_, Float>,
y: &ArrayView1<'_, i32>,
c: Float,
epsilon: Float,
max_iter: usize,
) -> SklResult<RankingSVMModel> {
let (n_samples, n_features) = X.dim();
// Check if we have both positive and negative samples
let has_positive = y.iter().any(|&label| label == 1);
let has_negative = y.iter().any(|&label| label == 0);
if !has_positive || !has_negative {
// If only one class, create a simple model
let weights = Array1::zeros(n_features);
let bias = if has_positive { 1.0 } else { -1.0 };
return Ok(RankingSVMModel { weights, bias });
}
// Use regularized logistic regression as a ranking approximation
// This provides a ranking-based score while being computationally tractable
let mut weights = Array1::zeros(n_features);
let mut bias = 0.0;
let learning_rate = 0.01;
let lambda = 1.0 / c; // Regularization parameter
for _iter in 0..max_iter {
let mut gradient_w = Array1::zeros(n_features);
let mut gradient_b = 0.0;
// Calculate gradients using logistic loss
for i in 0..n_samples {
let x_i = X.row(i);
let y_i = if y[i] == 1 { 1.0 } else { -1.0 };
let score = x_i.dot(&weights) + bias;
let prob = 1.0 / (1.0 + (-y_i * score).exp());
let factor = y_i * (1.0 - prob);
gradient_w = gradient_w + factor * &x_i.to_owned();
gradient_b += factor;
}
// Add regularization to weights gradient
gradient_w = gradient_w - lambda * &weights;
// Update parameters
weights = &weights + learning_rate * &gradient_w;
bias += learning_rate * gradient_b;
// Check convergence
let gradient_norm = gradient_w.mapv(|x| x.abs()).sum() + gradient_b.abs();
if gradient_norm < epsilon {
break;
}
}
Ok(RankingSVMModel { weights, bias })
}
/// Determine prediction thresholds based on strategy
fn determine_thresholds(
models: &[RankingSVMModel],
X: &ArrayView2<'_, Float>,
y: &Array2<i32>,
strategy: &ThresholdStrategy,
) -> SklResult<Array1<Float>> {
let n_labels = models.len();
let n_samples = X.nrows();
match strategy {
ThresholdStrategy::Fixed(threshold) => Ok(Array1::from_elem(n_labels, *threshold)),
ThresholdStrategy::PerLabel(thresholds) => {
if thresholds.len() == n_labels {
Ok(Array1::from_vec(thresholds.clone()))
} else {
// If provided thresholds don't match label count, compute optimal per-label thresholds
let mut computed_thresholds = Array1::zeros(n_labels);
for (label_idx, model) in models.iter().enumerate() {
let mut scores = Vec::new();
let mut labels = Vec::new();
for sample_idx in 0..n_samples {
let x = X.row(sample_idx);
let score = x.dot(&model.weights) + model.bias;
scores.push(score);
labels.push(y[[sample_idx, label_idx]]);
}
computed_thresholds[label_idx] = find_optimal_threshold(&scores, &labels)?;
}
Ok(computed_thresholds)
}
}
ThresholdStrategy::Optimal => {
// Find global threshold that optimizes F1 score across all labels
let mut all_scores = Vec::new();
let mut all_labels = Vec::new();
for (label_idx, model) in models.iter().enumerate() {
for sample_idx in 0..n_samples {
let x = X.row(sample_idx);
let score = x.dot(&model.weights) + model.bias;
all_scores.push(score);
all_labels.push(y[[sample_idx, label_idx]]);
}
}
let optimal_threshold = find_optimal_threshold(&all_scores, &all_labels)?;
Ok(Array1::from_elem(n_labels, optimal_threshold))
}
ThresholdStrategy::FScore => {
// Compute F-score optimized thresholds per label
let mut thresholds = Array1::zeros(n_labels);
for (label_idx, model) in models.iter().enumerate() {
let mut scores = Vec::new();
let mut labels = Vec::new();
for sample_idx in 0..n_samples {
let x = X.row(sample_idx);
let score = x.dot(&model.weights) + model.bias;
scores.push(score);
labels.push(y[[sample_idx, label_idx]]);
}
thresholds[label_idx] = find_optimal_threshold(&scores, &labels)?;
}
Ok(thresholds)
}
}
}
/// Find optimal threshold that maximizes F1 score
fn find_optimal_threshold(scores: &[Float], labels: &[i32]) -> SklResult<Float> {
if scores.len() != labels.len() {
return Err(SklearsError::InvalidInput(
"Scores and labels must have same length".to_string(),
));
}
// Create threshold candidates from unique scores
let mut unique_scores: Vec<Float> = scores.iter().cloned().collect();
unique_scores.sort_by(|a, b| a.partial_cmp(b).unwrap());
unique_scores.dedup();
if unique_scores.is_empty() {
return Ok(0.0);
}
let mut best_threshold = 0.0;
let mut best_f1 = 0.0;
// Try each unique score as a threshold
for &threshold in &unique_scores {
let mut tp = 0;
let mut fp = 0;
let mut fn_ = 0;
for (i, &score) in scores.iter().enumerate() {
let prediction = if score >= threshold { 1 } else { 0 };
let label = labels[i];
match (prediction, label) {
(1, 1) => tp += 1,
(1, 0) => fp += 1,
(0, 1) => fn_ += 1,
_ => {}
}
}
let precision = if tp + fp > 0 {
tp as Float / (tp + fp) as Float
} else {
0.0
};
let recall = if tp + fn_ > 0 {
tp as Float / (tp + fn_) as Float
} else {
0.0
};
let f1 = if precision + recall > 0.0 {
2.0 * precision * recall / (precision + recall)
} else {
0.0
};
if f1 > best_f1 {
best_f1 = f1;
best_threshold = threshold;
}
}
Ok(best_threshold)
}
#[cfg(test)]
mod tests {
use super::*;
use ndarray::array;
#[test]
fn test_multi_output_classifier() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let moc = MultiOutputClassifier::new();
let fitted = moc.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_targets(), 2);
assert_eq!(fitted.classes().len(), 2);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are valid (within the classes for each target)
for target_idx in 0..2 {
let target_classes = &fitted.classes()[target_idx];
for sample_idx in 0..4 {
let pred = predictions[[sample_idx, target_idx]];
assert!(target_classes.contains(&pred));
}
}
}
#[test]
fn test_multi_output_regressor() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [2.0, 1.5], [1.0, 1.5]];
let mor = MultiOutputRegressor::new();
let fitted = mor.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_targets(), 2);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Predictions should be finite numbers
for pred in predictions.iter() {
assert!(pred.is_finite());
}
}
#[test]
fn test_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[0, 1], [1, 0], [0, 1]]; // Wrong number of rows
let moc = MultiOutputClassifier::new();
assert!(moc.fit(&X.view(), &y).is_err());
}
#[test]
fn test_empty_targets() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = Array2::<i32>::zeros((2, 0)); // No targets
let moc = MultiOutputClassifier::new();
assert!(moc.fit(&X.view(), &y).is_err());
}
#[test]
fn test_prediction_shape_mismatch() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[0, 1], [1, 0]];
let moc = MultiOutputClassifier::new();
let fitted = moc.fit(&X.view(), &y).unwrap();
// Test with wrong number of features
let X_wrong = array![[1.0, 2.0, 3.0]]; // 3 features instead of 2
assert!(fitted.predict(&X_wrong.view()).is_err());
}
#[test]
fn test_classifier_chain() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let cc = ClassifierChain::new();
let fitted = cc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.n_targets(), 2);
assert_eq!(fitted.chain_order(), &[0, 1]); // Default order
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are valid
for sample_idx in 0..4 {
for target_idx in 0..2 {
let pred = predictions[[sample_idx, target_idx]];
// Predictions should be either 0 or 1 for binary classification
assert!(pred == 0 || pred == 1);
}
}
}
#[test]
fn test_classifier_chain_custom_order() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[0, 1], [1, 0], [1, 1]];
let cc = ClassifierChain::new().order(vec![1, 0]); // Reverse order
let fitted = cc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.chain_order(), &[1, 0]);
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
}
#[test]
fn test_classifier_chain_invalid_order() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[0, 1], [1, 0]];
let cc = ClassifierChain::new().order(vec![0, 1, 2]); // Too many indices
assert!(cc.fit_simple(&X.view(), &y).is_err());
}
#[test]
fn test_classifier_chain_monte_carlo() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let cc = ClassifierChain::new();
let fitted = cc.fit_simple(&X.view(), &y).unwrap();
// Test Monte Carlo predictions with probabilities
let mc_probs = fitted
.predict_monte_carlo(&X.view(), 100, Some(42))
.unwrap();
assert_eq!(mc_probs.dim(), (4, 2));
// All probabilities should be between 0 and 1
for prob in mc_probs.iter() {
assert!(*prob >= 0.0 && *prob <= 1.0);
}
// Test Monte Carlo predictions with labels
let mc_labels = fitted
.predict_monte_carlo_labels(&X.view(), 100, Some(42))
.unwrap();
assert_eq!(mc_labels.dim(), (4, 2));
// All predictions should be binary (0 or 1)
for pred in mc_labels.iter() {
assert!(*pred == 0 || *pred == 1);
}
// Test reproducibility with same random state
let mc_probs2 = fitted
.predict_monte_carlo(&X.view(), 100, Some(42))
.unwrap();
for (i, (&prob1, &prob2)) in mc_probs.iter().zip(mc_probs2.iter()).enumerate() {
assert!(
(prob1 - prob2).abs() < 1e-10,
"Probabilities should be identical with same random state at index {}",
i
);
}
}
#[test]
fn test_classifier_chain_monte_carlo_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[0, 1], [1, 0]];
let cc = ClassifierChain::new();
let fitted = cc.fit_simple(&X.view(), &y).unwrap();
// Test with zero samples
assert!(fitted.predict_monte_carlo(&X.view(), 0, None).is_err());
// Test with wrong number of features
let X_wrong = array![[1.0, 2.0, 3.0]]; // 3 features instead of 2
assert!(fitted
.predict_monte_carlo(&X_wrong.view(), 10, None)
.is_err());
}
#[test]
fn test_regressor_chain() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [2.0, 1.5], [1.0, 1.5]];
let rc = RegressorChain::new();
let fitted = rc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.n_targets(), 2);
assert_eq!(fitted.chain_order(), &[0, 1]); // Default order
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Predictions should be finite numbers
for pred in predictions.iter() {
assert!(pred.is_finite());
}
}
#[test]
fn test_regressor_chain_custom_order() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [2.0, 1.5]];
let rc = RegressorChain::new().order(vec![1, 0]); // Reverse order
let fitted = rc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.chain_order(), &[1, 0]);
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
// Predictions should be finite numbers
for pred in predictions.iter() {
assert!(pred.is_finite());
}
}
#[test]
fn test_regressor_chain_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [2.0, 1.5]]; // Wrong number of rows
let rc = RegressorChain::new();
assert!(rc.fit_simple(&X.view(), &y).is_err());
}
#[test]
fn test_regressor_chain_invalid_order() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.5, 2.5], [2.5, 3.5]];
let rc = RegressorChain::new().order(vec![0, 1, 2]); // Too many indices
assert!(rc.fit_simple(&X.view(), &y).is_err());
}
#[test]
fn test_binary_relevance() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
let br = BinaryRelevance::new();
let fitted = br.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 2);
assert_eq!(fitted.classes().len(), 2);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are binary (0 or 1)
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
// Test probability predictions
let probabilities = fitted.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (4, 2));
// Check that probabilities are in [0, 1]
for prob in probabilities.iter() {
assert!(*prob >= 0.0 && *prob <= 1.0);
}
}
#[test]
fn test_binary_relevance_single_label() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1], [0], [1]]; // Single binary label
let br = BinaryRelevance::new();
let fitted = br.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 1);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 1));
// Check that predictions are binary
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
}
#[test]
fn test_binary_relevance_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let br = BinaryRelevance::new();
assert!(br.fit(&X.view(), &y).is_err());
}
#[test]
fn test_binary_relevance_non_binary_labels() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[0, 1], [1, 2], [2, 0]]; // Non-binary labels
let br = BinaryRelevance::new();
assert!(br.fit(&X.view(), &y).is_err());
}
#[test]
fn test_binary_relevance_predict_shape_mismatch() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1]];
let br = BinaryRelevance::new();
let fitted = br.fit(&X.view(), &y).unwrap();
// Test with wrong number of features
let X_wrong = array![[1.0, 2.0, 3.0]]; // 3 features instead of 2
assert!(fitted.predict(&X_wrong.view()).is_err());
assert!(fitted.predict_proba(&X_wrong.view()).is_err());
}
#[test]
fn test_label_powerset() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary combinations
let lp = LabelPowerset::new();
let fitted = lp.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 2);
assert_eq!(fitted.n_classes(), 4); // 4 unique combinations: [1,0], [0,1], [1,1], [0,0]
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are binary (0 or 1)
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
// Test decision function
let scores = fitted.decision_function(&X.view()).unwrap();
assert_eq!(scores.dim(), (4, 4)); // 4 samples, 4 classes
// Scores should be finite
for score in scores.iter() {
assert!(score.is_finite());
}
}
#[test]
fn test_label_powerset_simple_case() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1, 0], [0, 1], [1, 0]]; // Only 2 unique combinations
let lp = LabelPowerset::new();
let fitted = lp.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 2);
assert_eq!(fitted.n_classes(), 2); // Only 2 unique combinations: [1,0], [0,1]
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
// Check that predictions are binary
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
}
#[test]
fn test_label_powerset_single_label() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1], [0], [1]]; // Single binary label
let lp = LabelPowerset::new();
let fitted = lp.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 1);
assert_eq!(fitted.n_classes(), 2); // 2 unique combinations: [1], [0]
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 1));
// Check that predictions are binary
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
}
#[test]
fn test_label_powerset_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let lp = LabelPowerset::new();
assert!(lp.fit(&X.view(), &y).is_err());
}
#[test]
fn test_label_powerset_non_binary_labels() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[0, 1], [1, 2], [2, 0]]; // Non-binary labels
let lp = LabelPowerset::new();
assert!(lp.fit(&X.view(), &y).is_err());
}
#[test]
fn test_label_powerset_predict_shape_mismatch() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1]];
let lp = LabelPowerset::new();
let fitted = lp.fit(&X.view(), &y).unwrap();
// Test with wrong number of features
let X_wrong = array![[1.0, 2.0, 3.0]]; // 3 features instead of 2
assert!(fitted.predict(&X_wrong.view()).is_err());
assert!(fitted.decision_function(&X_wrong.view()).is_err());
}
#[test]
fn test_label_powerset_all_same_combination() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1, 0], [1, 0], [1, 0]]; // All samples have same label combination
let lp = LabelPowerset::new();
let fitted = lp.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 2);
assert_eq!(fitted.n_classes(), 1); // Only 1 unique combination
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
// All predictions should be [1, 0]
for sample_idx in 0..3 {
assert_eq!(predictions[[sample_idx, 0]], 1);
assert_eq!(predictions[[sample_idx, 1]], 0);
}
}
#[test]
fn test_pruned_label_powerset_default_strategy() {
// Test data with some rare combinations
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[1.0, 1.0],
[2.0, 2.0],
[3.0, 3.0],
[1.5, 2.5],
[2.5, 1.5]
];
let y = array![
[1, 0],
[0, 1],
[1, 1],
[0, 0], // Frequent combinations
[1, 0],
[0, 1],
[1, 0],
[0, 1], // More frequent ones
];
let plp = PrunedLabelPowerset::new()
.min_frequency(2)
.strategy(PruningStrategy::DefaultMapping(vec![0, 0]));
let fitted = plp.fit(&X.view(), &y).unwrap();
// Should have pruned to only frequent combinations
assert!(fitted.n_frequent_classes() <= 4); // At most [1,0], [0,1], [1,1], [0,0]
assert_eq!(fitted.min_frequency(), 2);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (8, 2));
// All predictions should be binary
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
}
#[test]
fn test_pruned_label_powerset_similarity_strategy() {
// Test with similarity mapping strategy
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[1.0, 1.0],
[2.0, 2.0],
[3.0, 3.0]
];
let y = array![
[1, 0],
[1, 0],
[1, 0], // Frequent: [1, 0] appears 3 times
[0, 1],
[0, 1], // Frequent: [0, 1] appears 2 times
[1, 1] // Rare: [1, 1] appears 1 time
];
let plp = PrunedLabelPowerset::new()
.min_frequency(2)
.strategy(PruningStrategy::SimilarityMapping);
let fitted = plp.fit(&X.view(), &y).unwrap();
// Should have only 2 frequent combinations: [1,0] and [0,1]
assert_eq!(fitted.n_frequent_classes(), 2);
// The rare combination [1,1] should be mapped to one of the frequent ones
let mapping = fitted.combination_mapping();
let rare_combo = vec![1, 1];
assert!(mapping.contains_key(&rare_combo));
// The mapped combination should be one of the frequent ones
let mapped = mapping.get(&rare_combo).unwrap();
assert!(mapped == &vec![1, 0] || mapped == &vec![0, 1]);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// All predictions should be binary
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
}
#[test]
fn test_pruned_label_powerset_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[0, 1], [1, 0]];
// Test with minimum frequency that results in no frequent combinations
let plp = PrunedLabelPowerset::new().min_frequency(5); // Too high
assert!(plp.fit(&X.view(), &y).is_err());
// Test with invalid default combination length
let plp =
PrunedLabelPowerset::new().strategy(PruningStrategy::DefaultMapping(vec![0, 1, 0])); // 3 elements for 2 labels
assert!(plp.fit(&X.view(), &y).is_err());
// Test with non-binary labels
let y_bad = array![[2, 1], [1, 0]]; // Contains non-binary value
let plp = PrunedLabelPowerset::new();
assert!(plp.fit(&X.view(), &y_bad).is_err());
}
#[test]
fn test_pruned_label_powerset_edge_cases() {
// Test with minimal data that meets frequency requirement
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [1, 0]]; // Same combination twice
let plp = PrunedLabelPowerset::new().min_frequency(2);
let fitted = plp.fit(&X.view(), &y).unwrap();
// Should have at least 1 combination, possibly 2 if default is added
assert!(fitted.n_frequent_classes() >= 1);
assert!(fitted.frequent_combinations().len() >= 1);
// The frequent combinations should include [1, 0]
assert!(fitted.frequent_combinations().contains(&vec![1, 0]));
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (2, 2));
// All predictions should be [1, 0]
for sample_idx in 0..2 {
assert_eq!(predictions[[sample_idx, 0]], 1);
assert_eq!(predictions[[sample_idx, 1]], 0);
}
}
#[test]
fn test_metrics_hamming_loss() {
let y_true = array![[1, 0, 1], [0, 1, 0], [1, 1, 1]];
let y_pred = array![[1, 0, 0], [0, 1, 1], [1, 0, 1]]; // 3 errors out of 9
let loss = metrics::hamming_loss(&y_true.view(), &y_pred.view()).unwrap();
assert!((loss - 3.0 / 9.0).abs() < 1e-10);
}
#[test]
fn test_metrics_subset_accuracy() {
let y_true = array![[1, 0, 1], [0, 1, 0], [1, 1, 1]];
let y_pred = array![[1, 0, 1], [0, 1, 1], [1, 0, 1]]; // Only first subset matches
let accuracy = metrics::subset_accuracy(&y_true.view(), &y_pred.view()).unwrap();
assert!((accuracy - 1.0 / 3.0).abs() < 1e-10);
}
#[test]
fn test_metrics_jaccard_score() {
let y_true = array![[1, 0, 1], [0, 1, 0]];
let y_pred = array![[1, 0, 0], [0, 1, 1]];
let score = metrics::jaccard_score(&y_true.view(), &y_pred.view()).unwrap();
// Sample 1: intersection=1, union=2, jaccard=0.5
// Sample 2: intersection=1, union=2, jaccard=0.5
// Average: 0.5
assert!((score - 0.5).abs() < 1e-10);
}
#[test]
fn test_metrics_f1_score_micro() {
let y_true = array![[1, 0, 1], [0, 1, 0], [1, 1, 1]];
let y_pred = array![[1, 0, 0], [0, 1, 1], [1, 0, 1]];
let f1 = metrics::f1_score(&y_true.view(), &y_pred.view(), "micro").unwrap();
// TP=4, FP=1, FN=2
// Precision = 4/5 = 0.8, Recall = 4/6 = 0.6667
// F1 = 2 * 0.8 * 0.6667 / (0.8 + 0.6667) = 0.727
assert!((f1 - 0.7272727272727273).abs() < 1e-10);
}
#[test]
fn test_metrics_f1_score_macro() {
let y_true = array![[1, 0], [0, 1], [1, 1]];
let y_pred = array![[1, 0], [0, 1], [1, 0]]; // Perfect for label 0, imperfect for label 1
let f1 = metrics::f1_score(&y_true.view(), &y_pred.view(), "macro").unwrap();
// Label 0: TP=2, FP=0, FN=0 -> F1=1.0
// Label 1: TP=1, FP=0, FN=1 -> Precision=1.0, Recall=0.5, F1=0.667
// Macro average: (1.0 + 0.667) / 2 = 0.833
assert!((f1 - 0.8333333333333334).abs() < 1e-10);
}
#[test]
fn test_metrics_f1_score_samples() {
let y_true = array![[1, 0], [0, 1], [1, 1]];
let y_pred = array![[1, 0], [0, 1], [1, 0]];
let f1 = metrics::f1_score(&y_true.view(), &y_pred.view(), "samples").unwrap();
// Sample 0: TP=1, FP=0, FN=0 -> F1=1.0
// Sample 1: TP=1, FP=0, FN=0 -> F1=1.0
// Sample 2: TP=1, FP=0, FN=1 -> Precision=1.0, Recall=0.5, F1=0.667
// Average: (1.0 + 1.0 + 0.667) / 3 = 0.889
assert!((f1 - 0.8888888888888888).abs() < 1e-10);
}
#[test]
fn test_metrics_coverage_error() {
let y_true = array![[1, 0, 1], [0, 1, 0]];
let y_scores = array![[0.9, 0.1, 0.8], [0.2, 0.9, 0.3]];
let coverage = metrics::coverage_error(&y_true.view(), &y_scores.view()).unwrap();
// Sample 0: sorted scores [0.9, 0.8, 0.1] -> labels [0, 2, 1]
// true labels are at positions 1 and 2, so coverage = 2
// Sample 1: sorted scores [0.9, 0.3, 0.2] -> labels [1, 2, 0]
// true label is at position 1, so coverage = 1
// Average: (2 + 1) / 2 = 1.5
assert!((coverage - 1.5).abs() < 1e-10);
}
#[test]
fn test_metrics_label_ranking_average_precision() {
let y_true = array![[1, 0, 1], [0, 1, 0]];
let y_scores = array![[0.9, 0.1, 0.8], [0.2, 0.9, 0.3]];
let lrap =
metrics::label_ranking_average_precision(&y_true.view(), &y_scores.view()).unwrap();
// Sample 0: sorted scores [0.9, 0.8, 0.1] -> labels [0, 2, 1]
// true labels: 0 (pos 1), 2 (pos 2)
// precision at pos 1: 1/1=1.0, precision at pos 2: 2/2=1.0
// LRAP = (1.0 + 1.0) / 2 = 1.0
// Sample 1: sorted scores [0.9, 0.3, 0.2] -> labels [1, 2, 0]
// true label: 1 (pos 1)
// precision at pos 1: 1/1=1.0
// LRAP = 1.0 / 1 = 1.0
// Average: (1.0 + 1.0) / 2 = 1.0
assert!((lrap - 1.0).abs() < 1e-10);
}
#[test]
fn test_metrics_invalid_shapes() {
let y_true = array![[1, 0], [0, 1]];
let y_pred = array![[1, 0, 1]]; // Wrong shape
assert!(metrics::hamming_loss(&y_true.view(), &y_pred.view()).is_err());
assert!(metrics::subset_accuracy(&y_true.view(), &y_pred.view()).is_err());
assert!(metrics::jaccard_score(&y_true.view(), &y_pred.view()).is_err());
assert!(metrics::f1_score(&y_true.view(), &y_pred.view(), "micro").is_err());
}
#[test]
fn test_metrics_invalid_f1_average() {
let y_true = array![[1, 0], [0, 1]];
let y_pred = array![[1, 0], [0, 1]];
assert!(metrics::f1_score(&y_true.view(), &y_pred.view(), "invalid").is_err());
}
#[test]
fn test_ensemble_of_chains() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let eoc = EnsembleOfChains::new().n_chains(3).random_state(42);
let fitted = eoc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.n_chains(), 3);
assert_eq!(fitted.n_targets(), 2);
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are binary (0 or 1)
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
// Test probability predictions
let probabilities = fitted.predict_proba_simple(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (4, 2));
// Check that probabilities are in [0, 1]
for prob in probabilities.iter() {
assert!(*prob >= 0.0 && *prob <= 1.0);
}
}
#[test]
fn test_ensemble_of_chains_single_chain() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1]];
let eoc = EnsembleOfChains::new().n_chains(1);
let fitted = eoc.fit_simple(&X.view(), &y).unwrap();
assert_eq!(fitted.n_chains(), 1);
let predictions = fitted.predict_simple(&X.view()).unwrap();
assert_eq!(predictions.dim(), (2, 2));
}
#[test]
fn test_ensemble_of_chains_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let eoc = EnsembleOfChains::new();
assert!(eoc.fit_simple(&X.view(), &y).is_err());
}
#[test]
fn test_one_vs_rest_classifier() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [1.0, 1.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
let ovr = OneVsRestClassifier::new();
let fitted = ovr.fit(&X.view(), &y).unwrap();
assert_eq!(fitted.n_labels(), 2);
assert_eq!(fitted.classes().len(), 2);
let predictions = fitted.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are binary (0 or 1)
for pred in predictions.iter() {
assert!(*pred == 0 || *pred == 1);
}
// Test probability predictions
let probabilities = fitted.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (4, 2));
// Check that probabilities are in [0, 1]
for prob in probabilities.iter() {
assert!(*prob >= 0.0 && *prob <= 1.0);
}
// Test decision function
let scores = fitted.decision_function(&X.view()).unwrap();
assert_eq!(scores.dim(), (4, 2));
// Scores should be finite
for score in scores.iter() {
assert!(score.is_finite());
}
}
#[test]
fn test_one_vs_rest_classifier_invalid_input() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let ovr = OneVsRestClassifier::new();
assert!(ovr.fit(&X.view(), &y).is_err());
}
#[test]
fn test_metrics_one_error() {
let y_true = array![[1, 0, 0], [0, 1, 0], [0, 0, 1]];
let y_scores = array![[0.9, 0.1, 0.05], [0.1, 0.8, 0.1], [0.05, 0.1, 0.85]];
let one_err = metrics::one_error(&y_true.view(), &y_scores.view()).unwrap();
// All top-ranked labels are correct, so one-error should be 0
assert!((one_err - 0.0).abs() < 1e-10);
}
#[test]
fn test_metrics_one_error_with_errors() {
let y_true = array![[1, 0], [0, 1]];
let y_scores = array![[0.3, 0.7], [0.6, 0.4]]; // Top predictions are wrong
let one_err = metrics::one_error(&y_true.view(), &y_scores.view()).unwrap();
// Both samples have incorrect top predictions, so one-error should be 1.0
assert!((one_err - 1.0).abs() < 1e-10);
}
#[test]
fn test_metrics_ranking_loss() {
let y_true = array![[1, 0], [0, 1]];
let y_scores = array![[0.8, 0.2], [0.3, 0.7]]; // Correct ordering
let ranking_loss = metrics::ranking_loss(&y_true.view(), &y_scores.view()).unwrap();
// Perfect ranking, so loss should be 0
assert!((ranking_loss - 0.0).abs() < 1e-10);
}
#[test]
fn test_metrics_ranking_loss_with_errors() {
let y_true = array![[1, 0], [0, 1]];
let y_scores = array![[0.2, 0.8], [0.7, 0.3]]; // Incorrect ordering
let ranking_loss = metrics::ranking_loss(&y_true.view(), &y_scores.view()).unwrap();
// All pairs are incorrectly ordered, so loss should be 1.0
assert!((ranking_loss - 1.0).abs() < 1e-10);
}
#[test]
fn test_metrics_average_precision_score() {
let y_true = array![[1, 0, 1], [0, 1, 0]];
let y_scores = array![[0.9, 0.1, 0.8], [0.2, 0.9, 0.3]];
let ap_score = metrics::average_precision_score(&y_true.view(), &y_scores.view()).unwrap();
// With perfect ranking for both samples, AP should be 1.0
assert!((ap_score - 1.0).abs() < 1e-10);
}
#[test]
fn test_metrics_precision_recall_micro() {
let y_true = array![[1, 0, 1], [0, 1, 0], [1, 1, 1]];
let y_pred = array![[1, 0, 0], [0, 1, 1], [1, 0, 1]];
let precision = metrics::precision_score_micro(&y_true.view(), &y_pred.view()).unwrap();
let recall = metrics::recall_score_micro(&y_true.view(), &y_pred.view()).unwrap();
// TP=4, FP=1, FN=2
// Precision = 4/5 = 0.8, Recall = 4/6 = 0.6667
assert!((precision - 0.8).abs() < 1e-10);
assert!((recall - 0.6666666666666666).abs() < 1e-10);
}
#[test]
fn test_metrics_invalid_shapes_new_metrics() {
let y_true = array![[1, 0], [0, 1]];
let y_pred = array![[1, 0, 1]]; // Wrong shape
let y_scores = array![[0.8, 0.2, 0.1]]; // Wrong shape
assert!(metrics::one_error(&y_true.view(), &y_scores.view()).is_err());
assert!(metrics::ranking_loss(&y_true.view(), &y_scores.view()).is_err());
assert!(metrics::average_precision_score(&y_true.view(), &y_scores.view()).is_err());
assert!(metrics::precision_score_micro(&y_true.view(), &y_pred.view()).is_err());
assert!(metrics::recall_score_micro(&y_true.view(), &y_pred.view()).is_err());
}
#[test]
fn test_label_analysis_basic() {
let y = array![
[1, 0, 1], // Cardinality 2
[0, 1, 0], // Cardinality 1
[1, 0, 1], // Cardinality 2 (duplicate)
[0, 0, 0], // Cardinality 0
[1, 1, 1], // Cardinality 3
];
let results = label_analysis::analyze_combinations(&y.view()).unwrap();
assert_eq!(results.n_samples, 5);
assert_eq!(results.n_labels, 3);
assert_eq!(results.n_unique_combinations, 4); // [1,0,1], [0,1,0], [0,0,0], [1,1,1]
// Check that [1,0,1] is most frequent (appears twice)
assert_eq!(results.most_frequent.combination, vec![1, 0, 1]);
assert_eq!(results.most_frequent.frequency, 2);
assert!((results.most_frequent.relative_frequency - 0.4).abs() < 1e-10);
assert_eq!(results.most_frequent.cardinality, 2);
// Average cardinality should be (2+1+2+0+3)/5 = 1.6
assert!((results.avg_cardinality - 1.6).abs() < 1e-10);
// Label density: total active labels = 8, total possible = 15, density = 8/15
assert!((results.label_density - 8.0 / 15.0).abs() < 1e-10);
}
#[test]
fn test_label_analysis_utility_functions() {
let y = array![
[1, 0],
[1, 0],
[1, 0], // Frequent: [1, 0] appears 3 times
[0, 1],
[0, 1], // Frequent: [0, 1] appears 2 times
[1, 1] // Rare: [1, 1] appears 1 time
];
let results = label_analysis::analyze_combinations(&y.view()).unwrap();
// Test get_rare_combinations
let rare = label_analysis::get_rare_combinations(&results, 2);
assert_eq!(rare.len(), 1);
assert_eq!(rare[0].combination, vec![1, 1]);
assert_eq!(rare[0].frequency, 1);
// Test get_combinations_by_cardinality
let cardinality_1 = label_analysis::get_combinations_by_cardinality(&results, 1);
assert_eq!(cardinality_1.len(), 2); // [1, 0] and [0, 1]
let cardinality_2 = label_analysis::get_combinations_by_cardinality(&results, 2);
assert_eq!(cardinality_2.len(), 1); // [1, 1]
assert_eq!(cardinality_2[0].combination, vec![1, 1]);
}
#[test]
fn test_label_cooccurrence_matrix() {
let y = array![
[1, 1, 0], // Labels 0 and 1 co-occur
[1, 0, 1], // Labels 0 and 2 co-occur
[0, 1, 1], // Labels 1 and 2 co-occur
[1, 1, 1], // All labels co-occur
];
let cooccurrence = label_analysis::label_cooccurrence_matrix(&y.view()).unwrap();
assert_eq!(cooccurrence.dim(), (3, 3));
// Label 0 appears with itself in samples 0, 1, 3 = 3 times
assert_eq!(cooccurrence[[0, 0]], 3);
// Label 1 appears with itself in samples 0, 2, 3 = 3 times
assert_eq!(cooccurrence[[1, 1]], 3);
// Label 2 appears with itself in samples 1, 2, 3 = 3 times
assert_eq!(cooccurrence[[2, 2]], 3);
// Labels 0 and 1 co-occur in samples 0, 3 = 2 times
assert_eq!(cooccurrence[[0, 1]], 2);
assert_eq!(cooccurrence[[1, 0]], 2);
// Labels 0 and 2 co-occur in samples 1, 3 = 2 times
assert_eq!(cooccurrence[[0, 2]], 2);
assert_eq!(cooccurrence[[2, 0]], 2);
// Labels 1 and 2 co-occur in samples 2, 3 = 2 times
assert_eq!(cooccurrence[[1, 2]], 2);
assert_eq!(cooccurrence[[2, 1]], 2);
}
#[test]
fn test_label_correlation_matrix() {
let y = array![[1, 1, 0], [1, 0, 1], [0, 1, 1], [0, 0, 0],];
let correlation = label_analysis::label_correlation_matrix(&y.view()).unwrap();
assert_eq!(correlation.dim(), (3, 3));
// Diagonal should be 1.0 (perfect self-correlation)
assert!((correlation[[0, 0]] - 1.0).abs() < 1e-10);
assert!((correlation[[1, 1]] - 1.0).abs() < 1e-10);
assert!((correlation[[2, 2]] - 1.0).abs() < 1e-10);
// Matrix should be symmetric
for i in 0..3 {
for j in 0..3 {
assert!((correlation[[i, j]] - correlation[[j, i]]).abs() < 1e-10);
}
}
// All correlations should be between -1 and 1
for i in 0..3 {
for j in 0..3 {
assert!(correlation[[i, j]] >= -1.0 && correlation[[i, j]] <= 1.0);
}
}
}
#[test]
fn test_label_analysis_invalid_input() {
// Test with non-binary labels
let y_bad = array![[2, 1], [1, 0]]; // Contains non-binary value
assert!(label_analysis::analyze_combinations(&y_bad.view()).is_err());
// Test with empty array
let y_empty = Array2::<i32>::zeros((0, 2));
assert!(label_analysis::analyze_combinations(&y_empty.view()).is_err());
let y_no_labels = Array2::<i32>::zeros((2, 0));
assert!(label_analysis::analyze_combinations(&y_no_labels.view()).is_err());
// Test cooccurrence matrix with empty data
assert!(label_analysis::label_cooccurrence_matrix(&y_empty.view()).is_err());
assert!(label_analysis::label_correlation_matrix(&y_empty.view()).is_err());
}
#[test]
fn test_label_analysis_edge_cases() {
// Test with single sample
let y_single = array![[1, 0, 1]];
let results = label_analysis::analyze_combinations(&y_single.view()).unwrap();
assert_eq!(results.n_samples, 1);
assert_eq!(results.n_unique_combinations, 1);
assert_eq!(results.most_frequent.combination, vec![1, 0, 1]);
assert_eq!(results.least_frequent.combination, vec![1, 0, 1]);
assert_eq!(results.avg_cardinality, 2.0);
// Test with all zeros
let y_zeros = array![[0, 0], [0, 0]];
let results = label_analysis::analyze_combinations(&y_zeros.view()).unwrap();
assert_eq!(results.avg_cardinality, 0.0);
assert_eq!(results.label_density, 0.0);
// Test with all ones
let y_ones = array![[1, 1], [1, 1]];
let results = label_analysis::analyze_combinations(&y_ones.view()).unwrap();
assert_eq!(results.avg_cardinality, 2.0);
assert_eq!(results.label_density, 1.0);
}
#[test]
fn test_iblr_basic_functionality() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]]; // Multi-output regression targets
let iblr = IBLR::new().k_neighbors(2);
let trained_iblr = iblr.fit(&X.view(), &y).unwrap();
let predictions = trained_iblr.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Check that predictions are reasonable (close to actual values)
for i in 0..4 {
for j in 0..2 {
assert!((predictions[[i, j]] - y[[i, j]]).abs() < 2.0); // Allow some tolerance
}
}
}
#[test]
fn test_iblr_configuration() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
// Test different k values
let iblr1 = IBLR::new().k_neighbors(1);
let iblr2 = IBLR::new().k_neighbors(3);
let trained1 = iblr1.fit(&X.view(), &y).unwrap();
let trained2 = iblr2.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
assert_eq!(pred1.dim(), (3, 2));
assert_eq!(pred2.dim(), (3, 2));
// Test weight functions
let iblr_uniform = IBLR::new().k_neighbors(2).weights(WeightFunction::Uniform);
let iblr_distance = IBLR::new().k_neighbors(2).weights(WeightFunction::Distance);
let trained_uniform = iblr_uniform.fit(&X.view(), &y).unwrap();
let trained_distance = iblr_distance.fit(&X.view(), &y).unwrap();
let pred_uniform = trained_uniform.predict(&X.view()).unwrap();
let pred_distance = trained_distance.predict(&X.view()).unwrap();
assert_eq!(pred_uniform.dim(), (3, 2));
assert_eq!(pred_distance.dim(), (3, 2));
}
#[test]
fn test_iblr_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]]; // Mismatched samples
let iblr = IBLR::new();
assert!(iblr.fit(&X.view(), &y).is_err());
// Test k_neighbors validation
let y_valid = array![[1.0, 2.0], [2.0, 3.0]];
let iblr_zero_k = IBLR::new().k_neighbors(0);
assert!(iblr_zero_k.fit(&X.view(), &y_valid).is_err());
let iblr_large_k = IBLR::new().k_neighbors(5); // More than samples
assert!(iblr_large_k.fit(&X.view(), &y_valid).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1.0, 2.0], [2.0, 3.0]];
let iblr_for_predict = IBLR::new().k_neighbors(2);
let trained = iblr_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_wrong_features = array![[1.0, 2.0, 3.0]]; // Extra feature
assert!(trained.predict(&X_wrong_features.view()).is_err());
// Test empty data
let X_empty = Array2::<Float>::zeros((0, 2));
let y_empty = Array2::<Float>::zeros((0, 2));
assert!(IBLR::new().fit(&X_empty.view(), &y_empty).is_err());
}
#[test]
fn test_iblr_weight_functions() {
let X = array![[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [2.0, 2.0]];
let y = array![[1.0, 1.0], [2.0, 1.0], [1.0, 2.0], [3.0, 3.0]];
// Test uniform weighting
let iblr_uniform = IBLR::new().k_neighbors(3).weights(WeightFunction::Uniform);
let trained_uniform = iblr_uniform.fit(&X.view(), &y).unwrap();
let pred_uniform = trained_uniform.predict(&X.view()).unwrap();
// Test distance weighting
let iblr_distance = IBLR::new().k_neighbors(3).weights(WeightFunction::Distance);
let trained_distance = iblr_distance.fit(&X.view(), &y).unwrap();
let pred_distance = trained_distance.predict(&X.view()).unwrap();
// Predictions should be reasonable for both
assert_eq!(pred_uniform.dim(), (4, 2));
assert_eq!(pred_distance.dim(), (4, 2));
// For training points, predictions should be reasonable (not exact with k=3)
for j in 0..2 {
assert!((pred_uniform[[0, j]] - y[[0, j]]).abs() < 1.0); // More reasonable tolerance
assert!((pred_distance[[0, j]] - y[[0, j]]).abs() < 1.0); // More reasonable tolerance
}
}
#[test]
fn test_iblr_single_neighbor() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]];
let y = array![[10.0, 20.0], [20.0, 30.0], [30.0, 40.0]];
let iblr = IBLR::new().k_neighbors(1);
let trained = iblr.fit(&X.view(), &y).unwrap();
// Test prediction on training data (should be exact for k=1)
let predictions = trained.predict(&X.view()).unwrap();
for i in 0..3 {
for j in 0..2 {
assert!((predictions[[i, j]] - y[[i, j]]).abs() < 1e-10);
}
}
}
#[test]
fn test_iblr_interpolation() {
let X = array![[0.0, 0.0], [1.0, 1.0], [2.0, 2.0]];
let y = array![[0.0, 0.0], [1.0, 1.0], [2.0, 2.0]];
let iblr = IBLR::new().k_neighbors(2);
let trained = iblr.fit(&X.view(), &y).unwrap();
// Test prediction at midpoint
let X_test = array![[0.5, 0.5]];
let prediction = trained.predict(&X_test.view()).unwrap();
// Should be close to interpolated value [0.5, 0.5]
assert!((prediction[[0, 0]] - 0.5).abs() < 0.1);
assert!((prediction[[0, 1]] - 0.5).abs() < 0.1);
}
#[test]
fn test_clare_basic_functionality() {
let X = array![[1.0, 1.0], [1.5, 1.5], [5.0, 5.0], [5.5, 5.5]];
let y = array![[1, 0], [1, 0], [0, 1], [0, 1]]; // Two clear clusters with different label patterns
let clare = CLARE::new().n_clusters(2).random_state(Some(42));
let trained_clare = clare.fit(&X.view(), &y).unwrap();
let predictions = trained_clare.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Verify cluster centers were learned
assert_eq!(trained_clare.cluster_centers().dim(), (2, 2));
assert_eq!(trained_clare.label_relevance().dim(), (2, 2));
}
#[test]
fn test_clare_configuration() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test different configurations
let clare1 = CLARE::new().n_clusters(2).threshold(0.3);
let clare2 = CLARE::new().n_clusters(3).max_iter(50);
let clare3 = CLARE::new().random_state(Some(123));
let trained1 = clare1.fit(&X.view(), &y).unwrap();
let trained2 = clare2.fit(&X.view(), &y).unwrap();
let trained3 = clare3.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
let pred3 = trained3.predict(&X.view()).unwrap();
assert_eq!(pred1.dim(), (4, 2));
assert_eq!(pred2.dim(), (4, 2));
assert_eq!(pred3.dim(), (4, 2));
// Test accessors
assert_eq!(trained1.threshold(), 0.3);
assert_eq!(trained1.cluster_centers().dim(), (2, 2));
assert_eq!(trained2.cluster_centers().dim(), (3, 2));
}
#[test]
fn test_clare_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Mismatched samples
let clare = CLARE::new();
assert!(clare.fit(&X.view(), &y).is_err());
// Test n_clusters validation
let y_valid = array![[1, 0], [0, 1]];
let clare_zero_clusters = CLARE::new().n_clusters(0);
assert!(clare_zero_clusters.fit(&X.view(), &y_valid).is_err());
let clare_too_many_clusters = CLARE::new().n_clusters(5); // More than samples
assert!(clare_too_many_clusters.fit(&X.view(), &y_valid).is_err());
// Test non-binary labels
let y_non_binary = array![[1, 2], [0, 1]]; // Contains 2
assert!(CLARE::new().fit(&X.view(), &y_non_binary).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1, 0], [0, 1]];
let clare_for_predict = CLARE::new().n_clusters(2);
let trained = clare_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_wrong_features = array![[1.0, 2.0, 3.0]]; // Extra feature
assert!(trained.predict(&X_wrong_features.view()).is_err());
// Test empty data
let X_empty = Array2::<Float>::zeros((0, 2));
let y_empty = Array2::<i32>::zeros((0, 2));
assert!(CLARE::new().fit(&X_empty.view(), &y_empty).is_err());
}
#[test]
fn test_clare_predict_proba() {
let X = array![[1.0, 1.0], [1.2, 1.2], [5.0, 5.0], [5.2, 5.2]];
let y = array![[1, 0], [1, 0], [0, 1], [0, 1]];
let clare = CLARE::new().n_clusters(2).random_state(Some(42));
let trained_clare = clare.fit(&X.view(), &y).unwrap();
// Test predict_proba
let probabilities = trained_clare.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (4, 2));
// All probabilities should be between 0 and 1
for prob in probabilities.iter() {
assert!(*prob >= 0.0 && *prob <= 1.0);
}
}
#[test]
fn test_clare_clustering_consistency() {
let X = array![
[0.0, 0.0],
[0.1, 0.1], // First cluster
[5.0, 5.0],
[5.1, 5.1] // Second cluster
];
let y = array![
[1, 0],
[1, 0], // First cluster: always label 0 active
[0, 1],
[0, 1] // Second cluster: always label 1 active
];
let clare = CLARE::new()
.n_clusters(2)
.threshold(0.5)
.random_state(Some(42));
let trained_clare = clare.fit(&X.view(), &y).unwrap();
let predictions = trained_clare.predict(&X.view()).unwrap();
// With clear clustering, predictions should be good
let probabilities = trained_clare.predict_proba(&X.view()).unwrap();
// First two samples should have high probability for label 0
for i in 0..2 {
assert!(probabilities[[i, 0]] > 0.8); // High prob for label 0
assert!(probabilities[[i, 1]] < 0.2); // Low prob for label 1
}
// Last two samples should have high probability for label 1
for i in 2..4 {
assert!(probabilities[[i, 0]] < 0.2); // Low prob for label 0
assert!(probabilities[[i, 1]] > 0.8); // High prob for label 1
}
}
#[test]
fn test_clare_single_cluster() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]];
// Use only 1 cluster
let clare = CLARE::new().n_clusters(1);
let trained_clare = clare.fit(&X.view(), &y).unwrap();
let predictions = trained_clare.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert_eq!(trained_clare.cluster_centers().dim(), (1, 2));
// With 1 cluster, all samples should get same prediction
// (based on average label frequency)
for i in 1..3 {
for j in 0..2 {
assert_eq!(predictions[[0, j]], predictions[[i, j]]);
}
}
}
#[test]
fn test_clare_reproducibility() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Train two models with same random state
let clare1 = CLARE::new().n_clusters(2).random_state(Some(42));
let trained1 = clare1.fit(&X.view(), &y).unwrap();
let clare2 = CLARE::new().n_clusters(2).random_state(Some(42));
let trained2 = clare2.fit(&X.view(), &y).unwrap();
// Should produce same cluster centers
let centers1 = trained1.cluster_centers();
let centers2 = trained2.cluster_centers();
for i in 0..centers1.nrows() {
for j in 0..centers1.ncols() {
assert!((centers1[[i, j]] - centers2[[i, j]]).abs() < 1e-10);
}
}
// Should produce same predictions
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
for i in 0..pred1.nrows() {
for j in 0..pred1.ncols() {
assert_eq!(pred1[[i, j]], pred2[[i, j]]);
}
}
}
#[test]
fn test_mltsvm_basic_functionality() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
let mltsvm = MLTSVM::new().c1(1.0).c2(1.0);
let trained_mltsvm = mltsvm.fit(&X.view(), &y).unwrap();
let predictions = trained_mltsvm.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
assert_eq!(trained_mltsvm.n_labels(), 2);
// All predictions should be binary (0 or 1)
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
}
#[test]
fn test_mltsvm_configuration() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test different configurations
let mltsvm1 = MLTSVM::new().c1(0.5).c2(1.5);
let mltsvm2 = MLTSVM::new().epsilon(1e-8).max_iter(500);
let trained1 = mltsvm1.fit(&X.view(), &y).unwrap();
let trained2 = mltsvm2.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
assert_eq!(pred1.dim(), (4, 2));
assert_eq!(pred2.dim(), (4, 2));
}
#[test]
fn test_mltsvm_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Mismatched samples
let mltsvm = MLTSVM::new();
assert!(mltsvm.fit(&X.view(), &y).is_err());
// Test non-binary labels
let y_non_binary = array![[1, 2], [0, 1]]; // Contains 2
assert!(MLTSVM::new().fit(&X.view(), &y_non_binary).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 2.0]];
let y_train = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let mltsvm_for_predict = MLTSVM::new();
let trained = mltsvm_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_wrong_features = array![[1.0, 2.0, 3.0]]; // Extra feature
assert!(trained.predict(&X_wrong_features.view()).is_err());
// Test empty data
let X_empty = Array2::<Float>::zeros((0, 2));
let y_empty = Array2::<i32>::zeros((0, 2));
assert!(MLTSVM::new().fit(&X_empty.view(), &y_empty).is_err());
}
#[test]
fn test_mltsvm_decision_function() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]];
let y = array![[1, 0], [1, 0], [0, 1], [0, 1]];
let mltsvm = MLTSVM::new();
let trained_mltsvm = mltsvm.fit(&X.view(), &y).unwrap();
// Test decision function
let decision_values = trained_mltsvm.decision_function(&X.view()).unwrap();
assert_eq!(decision_values.dim(), (4, 2));
// Decision values should be real numbers (no constraints on range)
// Just check that we get reasonable outputs
for &val in decision_values.iter() {
assert!(val.is_finite());
}
}
#[test]
fn test_mltsvm_separable_data() {
let X = array![
[0.0, 0.0],
[0.5, 0.5], // Negative class cluster
[3.0, 3.0],
[3.5, 3.5] // Positive class cluster
];
let y = array![
[0, 1],
[0, 1], // First label: negative, Second label: positive
[1, 0],
[1, 0] // First label: positive, Second label: negative
];
let mltsvm = MLTSVM::new().c1(1.0).c2(1.0);
let trained_mltsvm = mltsvm.fit(&X.view(), &y).unwrap();
let predictions = trained_mltsvm.predict(&X.view()).unwrap();
// With linearly separable data, MLTSVM should perform well
let mut correct_predictions = 0;
let total_predictions = predictions.len();
for i in 0..predictions.nrows() {
for j in 0..predictions.ncols() {
if predictions[[i, j]] == y[[i, j]] {
correct_predictions += 1;
}
}
}
let accuracy = correct_predictions as Float / total_predictions as Float;
// Should get reasonably good accuracy on separable data
assert!(accuracy >= 0.5); // At least better than random
}
#[test]
fn test_mltsvm_feature_scaling() {
// Test with features of very different scales
let X = array![
[1000.0, 0.001],
[2000.0, 0.002],
[3000.0, 0.003],
[4000.0, 0.004]
];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let mltsvm = MLTSVM::new();
let trained_mltsvm = mltsvm.fit(&X.view(), &y).unwrap();
let predictions = trained_mltsvm.predict(&X.view()).unwrap();
// Should handle feature scaling internally
assert_eq!(predictions.dim(), (4, 2));
// All predictions should be binary
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
}
#[test]
fn test_mltsvm_consistency() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Train the same model multiple times (deterministic should give same results)
let mltsvm1 = MLTSVM::new().c1(1.0).c2(1.0);
let trained1 = mltsvm1.fit(&X.view(), &y).unwrap();
let mltsvm2 = MLTSVM::new().c1(1.0).c2(1.0);
let trained2 = mltsvm2.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
// Should be deterministic (same predictions)
for i in 0..pred1.nrows() {
for j in 0..pred1.ncols() {
assert_eq!(pred1[[i, j]], pred2[[i, j]]);
}
}
}
#[test]
fn test_ranksvm_basic_functionality() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]]; // Multi-label binary
let ranksvm = RankSVM::new().c(1.0);
let trained_ranksvm = ranksvm.fit(&X.view(), &y).unwrap();
let predictions = trained_ranksvm.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
assert_eq!(trained_ranksvm.n_labels(), 2);
// All predictions should be binary (0 or 1)
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
}
#[test]
fn test_ranksvm_threshold_strategies() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test different threshold strategies
let ranksvm1 = RankSVM::new().threshold_strategy(ThresholdStrategy::Fixed(0.5));
let ranksvm2 = RankSVM::new().threshold_strategy(ThresholdStrategy::Optimal);
let ranksvm3 =
RankSVM::new().threshold_strategy(ThresholdStrategy::PerLabel(vec![0.3, 0.7]));
let ranksvm4 = RankSVM::new().threshold_strategy(ThresholdStrategy::FScore);
let trained1 = ranksvm1.fit(&X.view(), &y).unwrap();
let trained2 = ranksvm2.fit(&X.view(), &y).unwrap();
let trained3 = ranksvm3.fit(&X.view(), &y).unwrap();
let trained4 = ranksvm4.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
let pred3 = trained3.predict(&X.view()).unwrap();
let pred4 = trained4.predict(&X.view()).unwrap();
assert_eq!(pred1.dim(), (4, 2));
assert_eq!(pred2.dim(), (4, 2));
assert_eq!(pred3.dim(), (4, 2));
assert_eq!(pred4.dim(), (4, 2));
// Test threshold accessors
assert_eq!(trained1.thresholds().len(), 2);
assert_eq!(trained2.thresholds().len(), 2);
assert_eq!(trained3.thresholds().len(), 2);
}
#[test]
fn test_ranksvm_decision_function() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]];
let y = array![[1, 0], [1, 0], [0, 1], [0, 1]];
let ranksvm = RankSVM::new();
let trained_ranksvm = ranksvm.fit(&X.view(), &y).unwrap();
// Test decision function
let scores = trained_ranksvm.decision_function(&X.view()).unwrap();
assert_eq!(scores.dim(), (4, 2));
// Scores should be real numbers
for &score in scores.iter() {
assert!(score.is_finite());
}
}
#[test]
fn test_ranksvm_predict_ranking() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]];
let y = array![[1, 0, 0], [0, 1, 0], [0, 0, 1]]; // Three labels, one active per sample
let ranksvm = RankSVM::new();
let trained_ranksvm = ranksvm.fit(&X.view(), &y).unwrap();
// Test ranking prediction
let rankings = trained_ranksvm.predict_ranking(&X.view()).unwrap();
assert_eq!(rankings.len(), 3); // 3 samples
for ranking in &rankings {
assert_eq!(ranking.len(), 3); // 3 labels
// Should contain all label indices
let mut sorted_ranking = ranking.clone();
sorted_ranking.sort();
assert_eq!(sorted_ranking, vec![0, 1, 2]);
}
}
#[test]
fn test_ranksvm_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Mismatched samples
let ranksvm = RankSVM::new();
assert!(ranksvm.fit(&X.view(), &y).is_err());
// Test non-binary labels
let y_non_binary = array![[1, 2], [0, 1]]; // Contains 2
assert!(RankSVM::new().fit(&X.view(), &y_non_binary).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 2.0]];
let y_train = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let ranksvm_for_predict = RankSVM::new();
let trained = ranksvm_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_wrong_features = array![[1.0, 2.0, 3.0]]; // Extra feature
assert!(trained.predict(&X_wrong_features.view()).is_err());
assert!(trained.decision_function(&X_wrong_features.view()).is_err());
assert!(trained.predict_ranking(&X_wrong_features.view()).is_err());
// Test empty data
let X_empty = Array2::<Float>::zeros((0, 2));
let y_empty = Array2::<i32>::zeros((0, 2));
assert!(RankSVM::new().fit(&X_empty.view(), &y_empty).is_err());
}
#[test]
fn test_ranksvm_configuration() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test different configurations
let ranksvm1 = RankSVM::new().c(0.5).epsilon(1e-8);
let ranksvm2 = RankSVM::new().max_iter(500);
let trained1 = ranksvm1.fit(&X.view(), &y).unwrap();
let trained2 = ranksvm2.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
assert_eq!(pred1.dim(), (4, 2));
assert_eq!(pred2.dim(), (4, 2));
}
#[test]
fn test_ranksvm_ranking_consistency() {
let X = array![[0.0, 0.0], [1.0, 1.0], [2.0, 2.0]];
let y = array![
[0, 0, 1], // Last label should rank highest
[0, 1, 0], // Middle label should rank highest
[1, 0, 0] // First label should rank highest
];
let ranksvm = RankSVM::new().threshold_strategy(ThresholdStrategy::FScore);
let trained_ranksvm = ranksvm.fit(&X.view(), &y).unwrap();
let rankings = trained_ranksvm.predict_ranking(&X.view()).unwrap();
let scores = trained_ranksvm.decision_function(&X.view()).unwrap();
// Check that rankings are consistent with scores
for i in 0..3 {
let ranking = &rankings[i];
// First ranked label should have highest score
let top_label = ranking[0];
for j in 1..ranking.len() {
let other_label = ranking[j];
assert!(scores[[i, top_label]] >= scores[[i, other_label]]);
}
}
}
#[test]
fn test_ranksvm_single_class_handling() {
let X = array![[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]];
// Test with all positive for one label, mixed for other
let y = array![[1, 0], [1, 1], [1, 0]]; // First label: all positive, second label: mixed
let ranksvm = RankSVM::new();
let trained = ranksvm.fit(&X.view(), &y).unwrap();
let predictions = trained.predict(&X.view()).unwrap();
let scores = trained.decision_function(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert_eq!(scores.dim(), (3, 2));
// All scores should be finite
for &score in scores.iter() {
assert!(score.is_finite());
}
}
#[test]
fn test_ranksvm_reproducibility() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Train two models with same configuration
let ranksvm1 = RankSVM::new().c(1.0).epsilon(1e-6);
let trained1 = ranksvm1.fit(&X.view(), &y).unwrap();
let ranksvm2 = RankSVM::new().c(1.0).epsilon(1e-6);
let trained2 = ranksvm2.fit(&X.view(), &y).unwrap();
let pred1 = trained1.predict(&X.view()).unwrap();
let pred2 = trained2.predict(&X.view()).unwrap();
let scores1 = trained1.decision_function(&X.view()).unwrap();
let scores2 = trained2.decision_function(&X.view()).unwrap();
// Should be deterministic (same predictions and scores)
for i in 0..pred1.nrows() {
for j in 0..pred1.ncols() {
assert_eq!(pred1[[i, j]], pred2[[i, j]]);
assert!((scores1[[i, j]] - scores2[[i, j]]).abs() < 1e-10);
}
}
}
}
/// Calibrated Binary Relevance Method
///
/// Enhanced binary relevance that applies probability calibration to improve
/// prediction reliability and provide confidence estimates.
#[derive(Debug, Clone)]
pub struct CalibratedBinaryRelevance<S = Untrained> {
state: S,
calibration_method: CalibrationMethod,
}
/// Calibration methods for probability calibration
#[derive(Debug, Clone, Copy)]
pub enum CalibrationMethod {
/// Platt scaling using sigmoid function
Sigmoid,
/// Isotonic regression for non-parametric calibration
Isotonic,
}
/// Trained state for CalibratedBinaryRelevance
#[derive(Debug, Clone)]
pub struct CalibratedBinaryRelevanceTrained {
pub base_models: BinaryRelevanceTrained,
pub calibration_params: Vec<(f64, f64)>, // (slope, intercept) for sigmoid calibration
}
impl CalibratedBinaryRelevance<Untrained> {
/// Create a new CalibratedBinaryRelevance instance
pub fn new() -> Self {
Self {
state: Untrained,
calibration_method: CalibrationMethod::Sigmoid,
}
}
/// Set the calibration method
pub fn calibration_method(mut self, method: CalibrationMethod) -> Self {
self.calibration_method = method;
self
}
}
impl Default for CalibratedBinaryRelevance<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for CalibratedBinaryRelevance<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for CalibratedBinaryRelevance<Untrained> {
type Fitted = CalibratedBinaryRelevance<CalibratedBinaryRelevanceTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
// First fit base binary relevance model
let base_model = BinaryRelevance::new().fit(X, y)?;
// For simplicity, use default calibration parameters
// In a full implementation, these would be learned from validation data
let n_labels = y.ncols();
let calibration_params = vec![(1.0, 0.0); n_labels]; // No calibration for now
Ok(CalibratedBinaryRelevance {
state: CalibratedBinaryRelevanceTrained {
base_models: base_model.state,
calibration_params,
},
calibration_method: self.calibration_method,
})
}
}
impl CalibratedBinaryRelevance<CalibratedBinaryRelevanceTrained> {
/// Predict probabilities for each label
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<f64>> {
let X = X.to_owned();
let (n_samples, n_features) = X.dim();
if n_features != self.state.base_models.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
let mut probabilities = Array2::zeros((n_samples, self.state.base_models.n_labels));
// Get probabilities from each binary classifier
for label_idx in 0..self.state.base_models.n_labels {
let (slope, intercept) = self.state.calibration_params[label_idx];
if let Some((weights, bias)) = self.state.base_models.binary_classifiers.get(&label_idx)
{
for (sample_idx, sample) in X.axis_iter(Axis(0)).enumerate() {
// Calculate linear prediction
let mut linear_score = *bias;
for feature_idx in 0..n_features {
linear_score += weights[feature_idx] * sample[feature_idx] as f64;
}
// Apply calibration
let calibrated_score = slope * linear_score + intercept;
let probability: f64 = 1.0 / (1.0 + (-calibrated_score).exp());
probabilities[[sample_idx, label_idx]] = probability.clamp(0.0, 1.0);
}
}
}
Ok(probabilities)
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for CalibratedBinaryRelevance<CalibratedBinaryRelevanceTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let probabilities = self.predict_proba(X)?;
let (n_samples, n_labels) = probabilities.dim();
let mut predictions = Array2::zeros((n_samples, n_labels));
for i in 0..n_samples {
for j in 0..n_labels {
predictions[[i, j]] = if probabilities[[i, j]] > 0.5 { 1 } else { 0 };
}
}
Ok(predictions)
}
}
/// Random Label Combinations Method
///
/// Generates random label combinations for evaluation and testing purposes.
/// Useful for creating synthetic multi-label datasets with controlled characteristics.
pub struct RandomLabelCombinations {
pub n_labels: usize,
pub n_samples: usize,
pub density: f64,
pub random_state: Option<u64>,
}
impl RandomLabelCombinations {
/// Create a new RandomLabelCombinations instance
pub fn new(n_labels: usize, n_samples: usize) -> Self {
Self {
n_labels,
n_samples,
density: 0.5,
random_state: None,
}
}
/// Set the label density (probability of each label being active)
pub fn density(mut self, density: f64) -> Self {
self.density = density.clamp(0.0, 1.0);
self
}
/// Set the random state for reproducible generation
pub fn random_state(mut self, seed: u64) -> Self {
self.random_state = Some(seed);
self
}
/// Generate random label combinations
pub fn generate(&self) -> SklResult<Array2<i32>> {
if self.n_labels == 0 || self.n_samples == 0 {
return Err(SklearsError::InvalidInput(
"n_labels and n_samples must be positive".to_string(),
));
}
let mut labels = Array2::zeros((self.n_samples, self.n_labels));
// Simple pseudo-random generation based on sample and label indices
let seed = self.random_state.unwrap_or(42);
for i in 0..self.n_samples {
for j in 0..self.n_labels {
// Simple hash-based pseudo-random number generation
let hash = ((i as u64 * 1103515245 + j as u64 * 12345 + seed) % 2147483647) as f64
/ 2147483647.0;
labels[[i, j]] = if hash < self.density { 1 } else { 0 };
}
}
Ok(labels)
}
/// Generate balanced random combinations ensuring minimum and maximum cardinality
pub fn generate_balanced(
&self,
min_cardinality: usize,
max_cardinality: usize,
) -> SklResult<Array2<i32>> {
if min_cardinality > max_cardinality {
return Err(SklearsError::InvalidInput(
"min_cardinality must be <= max_cardinality".to_string(),
));
}
if max_cardinality > self.n_labels {
return Err(SklearsError::InvalidInput(
"max_cardinality cannot exceed n_labels".to_string(),
));
}
let mut labels = Array2::zeros((self.n_samples, self.n_labels));
let seed = self.random_state.unwrap_or(42);
for i in 0..self.n_samples {
// Determine cardinality for this sample
let range = max_cardinality - min_cardinality + 1;
let hash = (i as u64 * 1103515245 + seed) % 2147483647;
let cardinality = min_cardinality + (hash as usize % range);
// Randomly select which labels to activate
let mut active_labels = Vec::new();
for j in 0..self.n_labels {
let label_hash = ((i as u64 * 12345 + j as u64 * 6789 + seed) % 2147483647) as f64
/ 2147483647.0;
active_labels.push((j, label_hash));
}
// Sort by hash and take the first 'cardinality' labels
active_labels.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
for k in 0..cardinality {
labels[[i, active_labels[k].0]] = 1;
}
}
Ok(labels)
}
}
/// ML-kNN: Multi-Label k-Nearest Neighbors
///
/// ML-kNN is an adaptation of the k-nearest neighbors algorithm for multi-label classification.
/// It uses the maximum a posteriori (MAP) principle to determine the label set for a test instance
/// based on the labels of its k nearest neighbors.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::MLkNN;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5], [0.5, 2.5]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
///
/// let model = MLkNN::new().k(3).smoothing(1.0);
/// let trained_model = model.fit(&X.view(), &y).unwrap();
/// let predictions = trained_model.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct MLkNN<S = Untrained> {
state: S,
k: usize,
smoothing: Float,
}
/// Trained state for ML-kNN
#[derive(Debug, Clone)]
pub struct MLkNNTrained {
X_train: Array2<Float>,
y_train: Array2<i32>,
k: usize,
smoothing: Float,
n_labels: usize,
prior_prob_true: Array1<Float>,
prior_prob_false: Array1<Float>,
cond_prob: Array2<Float>, // [label][count] = probability
}
impl MLkNN<Untrained> {
/// Create a new ML-kNN instance
pub fn new() -> Self {
Self {
state: Untrained,
k: 10,
smoothing: 1.0,
}
}
/// Set the number of nearest neighbors
pub fn k(mut self, k: usize) -> Self {
self.k = k;
self
}
/// Set the smoothing parameter for Laplace smoothing
pub fn smoothing(mut self, smoothing: Float) -> Self {
self.smoothing = smoothing;
self
}
}
impl Default for MLkNN<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MLkNN<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for MLkNN<Untrained> {
type Fitted = MLkNN<MLkNNTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let X_train = X.to_owned();
let y_train = y.clone();
let (n_samples, n_features) = X_train.dim();
let n_labels = y_train.ncols();
if n_samples != y_train.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples <= self.k {
return Err(SklearsError::InvalidInput(
"Number of samples must be greater than k".to_string(),
));
}
// Validate binary labels
for &val in y_train.iter() {
if val != 0 && val != 1 {
return Err(SklearsError::InvalidInput(
"y must contain only binary values (0 or 1)".to_string(),
));
}
}
// Compute prior probabilities for each label
let mut prior_prob_true = Array1::zeros(n_labels);
let mut prior_prob_false = Array1::zeros(n_labels);
for j in 0..n_labels {
let positive_count = y_train.column(j).iter().filter(|&&x| x == 1).count() as Float;
let total_count = n_samples as Float;
// Apply Laplace smoothing
prior_prob_true[j] =
(positive_count + self.smoothing) / (total_count + 2.0 * self.smoothing);
prior_prob_false[j] = (total_count - positive_count + self.smoothing)
/ (total_count + 2.0 * self.smoothing);
}
// Compute conditional probabilities P(Cj=1|Hj=c) for each label j and count c
let mut cond_prob = Array2::zeros((n_labels, self.k + 1));
for i in 0..n_samples {
let xi = X_train.row(i);
let yi = y_train.row(i);
// Find k nearest neighbors (excluding self)
let mut distances: Vec<(usize, Float)> = Vec::new();
for (idx, neighbor) in X_train.axis_iter(Axis(0)).enumerate() {
if idx != i {
let dist = euclidean_distance(&xi, &neighbor);
distances.push((idx, dist));
}
}
distances.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
let neighbors: Vec<usize> =
distances.iter().take(self.k).map(|&(idx, _)| idx).collect();
// For each label, count how many neighbors have it
for j in 0..n_labels {
let neighbor_count = neighbors
.iter()
.filter(|&&neighbor_idx| y_train[[neighbor_idx, j]] == 1)
.count();
// Update conditional probability statistics
if yi[j] == 1 {
// If current instance has label j, increment count for this neighbor_count
cond_prob[[j, neighbor_count]] += 1.0;
}
}
}
// Apply Laplace smoothing to conditional probabilities
for j in 0..n_labels {
let label_positive_count =
y_train.column(j).iter().filter(|&&x| x == 1).count() as Float;
for c in 0..=self.k {
cond_prob[[j, c]] = (cond_prob[[j, c]] + self.smoothing)
/ (label_positive_count + (self.k + 1) as Float * self.smoothing);
}
}
let trained_state = MLkNNTrained {
X_train,
y_train,
k: self.k,
smoothing: self.smoothing,
n_labels,
prior_prob_true,
prior_prob_false,
cond_prob,
};
Ok(MLkNN {
state: trained_state,
k: self.k,
smoothing: self.smoothing,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>> for MLkNN<MLkNNTrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.X_train.ncols() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
for i in 0..n_samples {
let xi = X.row(i);
// Find k nearest neighbors in training data
let mut distances: Vec<(usize, Float)> = Vec::new();
for (idx, neighbor) in self.state.X_train.axis_iter(Axis(0)).enumerate() {
let dist = euclidean_distance(&xi, &neighbor);
distances.push((idx, dist));
}
distances.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
let neighbors: Vec<usize> = distances
.iter()
.take(self.state.k)
.map(|&(idx, _)| idx)
.collect();
// For each label, apply MAP principle
for j in 0..self.state.n_labels {
let neighbor_count = neighbors
.iter()
.filter(|&&neighbor_idx| self.state.y_train[[neighbor_idx, j]] == 1)
.count();
// Compute posterior probabilities using Bayes' theorem
let p_label_true_given_neighbors =
self.state.cond_prob[[j, neighbor_count]] * self.state.prior_prob_true[j];
let p_label_false_given_neighbors = (1.0
- self.state.cond_prob[[j, neighbor_count]])
* self.state.prior_prob_false[j];
// Apply MAP: choose label with higher posterior probability
if p_label_true_given_neighbors > p_label_false_given_neighbors {
predictions[[i, j]] = 1;
} else {
predictions[[i, j]] = 0;
}
}
}
Ok(predictions)
}
}
impl MLkNN<MLkNNTrained> {
/// Predict class probabilities for multi-label classification
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.X_train.ncols() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut probabilities = Array2::zeros((n_samples, self.state.n_labels));
for i in 0..n_samples {
let xi = X.row(i);
// Find k nearest neighbors in training data
let mut distances: Vec<(usize, Float)> = Vec::new();
for (idx, neighbor) in self.state.X_train.axis_iter(Axis(0)).enumerate() {
let dist = euclidean_distance(&xi, &neighbor);
distances.push((idx, dist));
}
distances.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
let neighbors: Vec<usize> = distances
.iter()
.take(self.state.k)
.map(|&(idx, _)| idx)
.collect();
// For each label, compute probability
for j in 0..self.state.n_labels {
let neighbor_count = neighbors
.iter()
.filter(|&&neighbor_idx| self.state.y_train[[neighbor_idx, j]] == 1)
.count();
// Compute posterior probabilities
let p_label_true_given_neighbors =
self.state.cond_prob[[j, neighbor_count]] * self.state.prior_prob_true[j];
let p_label_false_given_neighbors = (1.0
- self.state.cond_prob[[j, neighbor_count]])
* self.state.prior_prob_false[j];
// Normalize probabilities
let total_prob = p_label_true_given_neighbors + p_label_false_given_neighbors;
if total_prob > 0.0 {
probabilities[[i, j]] = p_label_true_given_neighbors / total_prob;
} else {
probabilities[[i, j]] = self.state.prior_prob_true[j];
}
}
}
Ok(probabilities)
}
}
/// Cost-Sensitive Binary Relevance
///
/// This method extends Binary Relevance to handle cost-sensitive multi-label classification.
/// It allows specifying different costs for false positives and false negatives for each label,
/// which is crucial for imbalanced datasets or applications where different types of errors
/// have different consequences.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::{CostSensitiveBinaryRelevance, CostMatrix, CalibrationMethod};
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5], [0.5, 2.5]];
/// let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
///
/// // Higher cost for false negatives on label 0, higher cost for false positives on label 1
/// let cost_matrix = CostMatrix::from_fp_fn_costs(
/// vec![1.0, 3.0], // false positive costs
/// vec![2.0, 1.0] // false negative costs
/// ).unwrap();
///
/// let model = CostSensitiveBinaryRelevance::new()
/// .cost_matrix(cost_matrix)
/// .calibration_method(CalibrationMethod::Sigmoid);
/// let trained_model = model.fit(&X.view(), &y).unwrap();
/// let predictions = trained_model.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct CostSensitiveBinaryRelevance<S = Untrained> {
state: S,
cost_matrix: Option<CostMatrix>,
calibration_method: CalibrationMethod,
}
/// Cost matrix for cost-sensitive learning
#[derive(Debug, Clone)]
pub struct CostMatrix {
/// Cost of false positives for each label
pub fp_costs: Vec<Float>,
/// Cost of false negatives for each label
pub fn_costs: Vec<Float>,
}
impl CostMatrix {
/// Create a cost matrix from false positive and false negative costs
pub fn from_fp_fn_costs(fp_costs: Vec<Float>, fn_costs: Vec<Float>) -> SklResult<Self> {
if fp_costs.len() != fn_costs.len() {
return Err(SklearsError::InvalidInput(
"fp_costs and fn_costs must have the same length".to_string(),
));
}
if fp_costs.iter().any(|&c| c <= 0.0) || fn_costs.iter().any(|&c| c <= 0.0) {
return Err(SklearsError::InvalidInput(
"All costs must be positive".to_string(),
));
}
Ok(Self { fp_costs, fn_costs })
}
/// Create a balanced cost matrix with equal costs for all labels
pub fn balanced(n_labels: usize) -> Self {
Self {
fp_costs: vec![1.0; n_labels],
fn_costs: vec![1.0; n_labels],
}
}
/// Get the optimal threshold for label j based on cost ratio
pub fn get_threshold(&self, j: usize) -> Float {
let cost_ratio = self.fp_costs[j] / (self.fp_costs[j] + self.fn_costs[j]);
cost_ratio
}
}
/// Trained state for Cost-Sensitive Binary Relevance
#[derive(Debug, Clone)]
pub struct CostSensitiveBinaryRelevanceTrained {
n_labels: usize,
binary_models: Vec<SimpleBinaryModel>,
cost_matrix: CostMatrix,
feature_means: Array1<Float>,
feature_stds: Array1<Float>,
}
/// Simple binary model for cost-sensitive learning
#[derive(Debug, Clone)]
pub struct SimpleBinaryModel {
weights: Array1<Float>,
bias: Float,
}
impl CostSensitiveBinaryRelevance<Untrained> {
/// Create a new Cost-Sensitive Binary Relevance instance
pub fn new() -> Self {
Self {
state: Untrained,
cost_matrix: None,
calibration_method: CalibrationMethod::Sigmoid,
}
}
/// Set the cost matrix
pub fn cost_matrix(mut self, cost_matrix: CostMatrix) -> Self {
self.cost_matrix = Some(cost_matrix);
self
}
/// Set the calibration method
pub fn calibration_method(mut self, method: CalibrationMethod) -> Self {
self.calibration_method = method;
self
}
}
impl Default for CostSensitiveBinaryRelevance<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for CostSensitiveBinaryRelevance<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for CostSensitiveBinaryRelevance<Untrained> {
type Fitted = CostSensitiveBinaryRelevance<CostSensitiveBinaryRelevanceTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_labels = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
// Validate binary labels
for &val in y.iter() {
if val != 0 && val != 1 {
return Err(SklearsError::InvalidInput(
"y must contain only binary values (0 or 1)".to_string(),
));
}
}
// Set default balanced cost matrix if none provided
let cost_matrix = self
.cost_matrix
.unwrap_or_else(|| CostMatrix::balanced(n_labels));
if cost_matrix.fp_costs.len() != n_labels || cost_matrix.fn_costs.len() != n_labels {
return Err(SklearsError::InvalidInput(
"Cost matrix must have same number of labels as y".to_string(),
));
}
// Compute feature statistics for standardization
let feature_means = X.mean_axis(Axis(0)).unwrap();
let feature_stds = X
.std_axis(Axis(0), 0.0)
.mapv(|x| if x == 0.0 { 1.0 } else { x });
// Standardize features
let X_standardized = standardize_features_simple(X, &feature_means, &feature_stds);
let mut binary_models = Vec::new();
// Train one cost-sensitive binary classifier for each label
for j in 0..n_labels {
let y_binary = y.column(j).to_owned();
// Apply cost-sensitive sampling/weighting
let class_weights = compute_cost_sensitive_weights(&y_binary, &cost_matrix, j);
// Train binary classifier with cost-sensitive weights
let model = train_weighted_binary_classifier_simple(
&X_standardized,
&y_binary,
&class_weights,
)?;
binary_models.push(model);
}
let trained_state = CostSensitiveBinaryRelevanceTrained {
n_labels,
binary_models,
cost_matrix,
feature_means,
feature_stds,
};
Ok(CostSensitiveBinaryRelevance {
state: trained_state,
cost_matrix: None,
calibration_method: self.calibration_method,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for CostSensitiveBinaryRelevance<CostSensitiveBinaryRelevanceTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
// Standardize features using training statistics
let X_standardized =
standardize_features_simple(X, &self.state.feature_means, &self.state.feature_stds);
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
for j in 0..self.state.n_labels {
// Get model probabilities
let probabilities =
predict_binary_probabilities(&self.state.binary_models[j], &X_standardized);
// Apply cost-sensitive threshold
let threshold = self.state.cost_matrix.get_threshold(j);
for i in 0..n_samples {
predictions[[i, j]] = if probabilities[i] > threshold { 1 } else { 0 };
}
}
Ok(predictions)
}
}
impl CostSensitiveBinaryRelevance<CostSensitiveBinaryRelevanceTrained> {
/// Predict class probabilities
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.feature_means.len() {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
// Standardize features using training statistics
let X_standardized =
standardize_features_simple(X, &self.state.feature_means, &self.state.feature_stds);
let mut probabilities = Array2::zeros((n_samples, self.state.n_labels));
for j in 0..self.state.n_labels {
let label_probabilities =
predict_binary_probabilities(&self.state.binary_models[j], &X_standardized);
for i in 0..n_samples {
probabilities[[i, j]] = label_probabilities[i];
}
}
Ok(probabilities)
}
/// Get the cost matrix used during training
pub fn cost_matrix(&self) -> &CostMatrix {
&self.state.cost_matrix
}
}
/// Compute cost-sensitive class weights for a binary classification problem
fn compute_cost_sensitive_weights(
y_binary: &Array1<i32>,
cost_matrix: &CostMatrix,
label_idx: usize,
) -> Array1<Float> {
let n_samples = y_binary.len();
let mut weights = Array1::ones(n_samples);
let fp_cost = cost_matrix.fp_costs[label_idx];
let fn_cost = cost_matrix.fn_costs[label_idx];
// Count positive and negative samples
let n_positive = y_binary.iter().filter(|&&x| x == 1).count() as Float;
let n_negative = (n_samples as Float) - n_positive;
if n_positive > 0.0 && n_negative > 0.0 {
// Weight positive samples by false negative cost
let positive_weight = fn_cost / n_positive;
// Weight negative samples by false positive cost
let negative_weight = fp_cost / n_negative;
for (i, &label) in y_binary.iter().enumerate() {
weights[i] = if label == 1 {
positive_weight
} else {
negative_weight
};
}
}
weights
}
/// Standardize features using mean and standard deviation
fn standardize_features_simple(
X: &ArrayView2<'_, Float>,
means: &Array1<Float>,
stds: &Array1<Float>,
) -> Array2<Float> {
let mut X_standardized = X.to_owned();
for (i, mut row) in X_standardized.axis_iter_mut(Axis(0)).enumerate() {
for (j, feature) in row.iter_mut().enumerate() {
*feature = (*feature - means[j]) / stds[j];
}
}
X_standardized
}
/// Train a weighted binary classifier
fn train_weighted_binary_classifier_simple(
X: &Array2<Float>,
y: &Array1<i32>,
weights: &Array1<Float>,
) -> SklResult<SimpleBinaryModel> {
let (n_samples, n_features) = X.dim();
// Simple weighted logistic regression implementation
let mut w = Array1::zeros(n_features);
let mut b = 0.0;
let learning_rate = 0.01;
let max_iterations = 1000;
let tolerance = 1e-6;
for _iteration in 0..max_iterations {
let mut gradient_w = Array1::zeros(n_features);
let mut gradient_b = 0.0;
let mut total_weight = 0.0;
for i in 0..n_samples {
let xi = X.row(i);
let yi = y[i] as Float;
let weight = weights[i];
// Compute prediction (logistic function)
let linear_pred = xi.dot(&w) + b;
let prob = 1.0 / (1.0 + (-linear_pred).exp());
// Compute weighted gradients
let error = prob - yi;
gradient_w += &(weight * error * &xi);
gradient_b += weight * error;
total_weight += weight;
}
// Update parameters
let old_w = w.clone();
let old_b = b;
if total_weight > 0.0 {
w -= &(learning_rate * gradient_w / total_weight);
b -= learning_rate * gradient_b / total_weight;
}
// Check for convergence
let w_change = (&w - &old_w).mapv(|x| x * x).sum().sqrt();
let b_change = (b - old_b).abs();
if w_change < tolerance && b_change < tolerance {
break;
}
}
Ok(SimpleBinaryModel {
weights: w,
bias: b,
})
}
/// Predict probabilities using a simple binary model
fn predict_binary_probabilities(model: &SimpleBinaryModel, X: &Array2<Float>) -> Array1<Float> {
let mut probabilities = Array1::zeros(X.nrows());
for (i, xi) in X.axis_iter(Axis(0)).enumerate() {
let linear_pred = xi.dot(&model.weights) + model.bias;
let prob = 1.0 / (1.0 + (-linear_pred).exp());
probabilities[i] = prob;
}
probabilities
}
#[cfg(test)]
mod new_method_tests {
use super::*;
use ndarray::array;
#[test]
fn test_calibrated_binary_relevance() {
let X = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5], [0.5, 2.5]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let model = CalibratedBinaryRelevance::new().calibration_method(CalibrationMethod::Sigmoid);
let trained_model = model.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Test probability prediction
let probabilities = trained_model.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (4, 2));
// Check that probabilities are in valid range
for &prob in probabilities.iter() {
assert!(prob >= 0.0 && prob <= 1.0);
}
}
#[test]
fn test_random_label_combinations() {
let generator = RandomLabelCombinations::new(5, 10)
.density(0.3)
.random_state(42);
let labels = generator.generate().unwrap();
assert_eq!(labels.dim(), (10, 5));
// Check that all values are binary
for &val in labels.iter() {
assert!(val == 0 || val == 1);
}
// Test balanced generation
let balanced_labels = generator.generate_balanced(1, 3).unwrap();
assert_eq!(balanced_labels.dim(), (10, 5));
// Check that each sample has between 1 and 3 active labels
for i in 0..10 {
let cardinality = balanced_labels.row(i).iter().sum::<i32>();
assert!(cardinality >= 1 && cardinality <= 3);
}
}
#[test]
fn test_random_label_combinations_edge_cases() {
let generator = RandomLabelCombinations::new(3, 5);
// Test invalid min > max cardinality
assert!(generator.generate_balanced(3, 2).is_err());
// Test max cardinality > n_labels
assert!(generator.generate_balanced(1, 5).is_err());
// Test zero labels or samples
let empty_generator = RandomLabelCombinations::new(0, 5);
assert!(empty_generator.generate().is_err());
let empty_samples = RandomLabelCombinations::new(3, 0);
assert!(empty_samples.generate().is_err());
}
#[test]
fn test_mlknn_basic_functionality() {
let X = array![
[1.0, 2.0],
[2.0, 1.0],
[1.5, 1.5],
[0.5, 2.5],
[2.5, 0.5],
[1.0, 1.0]
];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0], [0, 1], [1, 0]];
let model = MLkNN::new().k(3).smoothing(1.0);
let trained_model = model.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are binary
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
// Test probability prediction
let probabilities = trained_model.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (6, 2));
// Check that probabilities are in valid range
for &prob in probabilities.iter() {
assert!(prob >= 0.0 && prob <= 1.0);
}
}
#[test]
fn test_mlknn_error_handling() {
let X = array![[1.0, 2.0], [2.0, 1.0]];
let y = array![[1, 0], [0, 1]];
// Test k too large
let model = MLkNN::new().k(5);
assert!(model.fit(&X.view(), &y).is_err());
// Test non-binary labels
let y_invalid = array![[1, 2], [0, 1]];
let model = MLkNN::new().k(1);
assert!(model.fit(&X.view(), &y_invalid).is_err());
// Test mismatched dimensions
let X_train = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5]];
let y_train = array![[1, 0], [0, 1], [1, 1]];
let model = MLkNN::new().k(2);
let trained_model = model.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_model.predict(&X_test.view()).is_err());
assert!(trained_model.predict_proba(&X_test.view()).is_err());
}
#[test]
fn test_mlknn_configuration() {
let model = MLkNN::new().k(5).smoothing(0.5);
assert_eq!(model.k, 5);
assert_eq!(model.smoothing, 0.5);
let default_model = MLkNN::default();
assert_eq!(default_model.k, 10);
assert_eq!(default_model.smoothing, 1.0);
}
#[test]
fn test_cost_sensitive_binary_relevance_basic() {
let X = array![
[1.0, 2.0],
[2.0, 1.0],
[1.5, 1.5],
[0.5, 2.5],
[2.5, 0.5],
[1.0, 1.0]
];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0], [0, 1], [1, 0]];
// Create cost matrix with higher cost for false negatives on label 0
let cost_matrix = CostMatrix::from_fp_fn_costs(
vec![1.0, 1.0], // false positive costs
vec![3.0, 1.0], // false negative costs (higher for label 0)
)
.unwrap();
let model = CostSensitiveBinaryRelevance::new()
.cost_matrix(cost_matrix)
.calibration_method(CalibrationMethod::Sigmoid);
let trained_model = model.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are binary
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
// Test probability prediction
let probabilities = trained_model.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (6, 2));
// Check that probabilities are in valid range
for &prob in probabilities.iter() {
assert!(prob >= 0.0 && prob <= 1.0);
}
// Test cost matrix access
let retrieved_cost_matrix = trained_model.cost_matrix();
assert_eq!(retrieved_cost_matrix.fp_costs[0], 1.0);
assert_eq!(retrieved_cost_matrix.fn_costs[0], 3.0);
}
#[test]
fn test_cost_matrix_creation_and_validation() {
// Test successful creation
let cost_matrix =
CostMatrix::from_fp_fn_costs(vec![1.0, 2.0, 3.0], vec![2.0, 1.0, 1.5]).unwrap();
assert_eq!(cost_matrix.fp_costs.len(), 3);
assert_eq!(cost_matrix.fn_costs.len(), 3);
// Test balanced cost matrix
let balanced = CostMatrix::balanced(5);
assert_eq!(balanced.fp_costs, vec![1.0; 5]);
assert_eq!(balanced.fn_costs, vec![1.0; 5]);
// Test threshold calculation
let threshold_0 = cost_matrix.get_threshold(0);
let expected_threshold_0 = 1.0 / (1.0 + 2.0); // fp_cost / (fp_cost + fn_cost)
assert!((threshold_0 - expected_threshold_0).abs() < 1e-10);
// Test invalid inputs
assert!(CostMatrix::from_fp_fn_costs(vec![1.0, 2.0], vec![1.0]).is_err()); // Different lengths
assert!(CostMatrix::from_fp_fn_costs(vec![0.0, 1.0], vec![1.0, 1.0]).is_err()); // Zero cost
assert!(CostMatrix::from_fp_fn_costs(vec![-1.0, 1.0], vec![1.0, 1.0]).is_err());
// Negative cost
}
#[test]
fn test_cost_sensitive_binary_relevance_error_handling() {
let X = array![[1.0, 2.0], [2.0, 1.0]];
let y = array![[1, 0], [0, 1]];
// Test non-binary labels
let y_invalid = array![[1, 2], [0, 1]];
let model = CostSensitiveBinaryRelevance::new();
assert!(model.fit(&X.view(), &y_invalid).is_err());
// Test mismatched cost matrix dimensions
let wrong_cost_matrix = CostMatrix::from_fp_fn_costs(
vec![1.0, 2.0, 3.0], // 3 labels
vec![1.0, 1.0, 1.0],
)
.unwrap();
let model = CostSensitiveBinaryRelevance::new().cost_matrix(wrong_cost_matrix);
assert!(model.fit(&X.view(), &y).is_err()); // y has only 2 labels
// Test mismatched dimensions during prediction
let model = CostSensitiveBinaryRelevance::new();
let trained_model = model.fit(&X.view(), &y).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_model.predict(&X_test.view()).is_err());
assert!(trained_model.predict_proba(&X_test.view()).is_err());
}
#[test]
fn test_cost_sensitive_binary_relevance_default_behavior() {
let X = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5], [0.5, 2.5]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test with default balanced cost matrix
let model = CostSensitiveBinaryRelevance::new();
let trained_model = model.fit(&X.view(), &y).unwrap();
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Test default configuration
let default_model = CostSensitiveBinaryRelevance::default();
let trained_default = default_model.fit(&X.view(), &y).unwrap();
let default_predictions = trained_default.predict(&X.view()).unwrap();
assert_eq!(default_predictions.dim(), (4, 2));
// Cost matrix should be balanced
let cost_matrix = trained_model.cost_matrix();
assert_eq!(cost_matrix.fp_costs, vec![1.0, 1.0]);
assert_eq!(cost_matrix.fn_costs, vec![1.0, 1.0]);
}
#[test]
fn test_bayesian_classifier_chain_basic() {
let X = array![
[1.0, 2.0],
[2.0, 1.0],
[1.5, 1.5],
[0.5, 2.5],
[2.5, 0.5],
[1.0, 1.0]
];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0], [0, 1], [1, 0]];
let model = BayesianClassifierChain::new()
.n_samples(50)
.prior_strength(1.0);
let trained_model = model.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are binary
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
// Test uncertainty prediction
let uncertainties = trained_model.predict_uncertainty(&X.view()).unwrap();
assert_eq!(uncertainties.dim(), (6, 2));
// Check that uncertainties are non-negative
for &uncertainty in uncertainties.iter() {
assert!(uncertainty >= 0.0);
}
// Test order access
let order = trained_model.order();
assert_eq!(order.len(), 2);
}
#[test]
fn test_bayesian_classifier_chain_custom_order() {
let X = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5]];
let y = array![[1, 0, 1], [0, 1, 0], [1, 1, 1]];
let model = BayesianClassifierChain::new()
.order(vec![2, 0, 1])
.n_samples(30)
.prior_strength(0.5);
let trained_model = model.fit(&X.view(), &y).unwrap();
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 3));
// Check that the order is preserved
assert_eq!(trained_model.order(), &[2, 0, 1]);
}
#[test]
fn test_bayesian_classifier_chain_error_handling() {
let X = array![[1.0, 2.0], [2.0, 1.0]];
let y = array![[1, 0], [0, 1]];
// Test non-binary labels
let y_invalid = array![[1, 2], [0, 1]];
let model = BayesianClassifierChain::new();
assert!(model.fit(&X.view(), &y_invalid).is_err());
// Test invalid order
let model_invalid_order = BayesianClassifierChain::new().order(vec![0, 1, 2]); // 3 labels but y only has 2
assert!(model_invalid_order.fit(&X.view(), &y).is_err());
// Test mismatched dimensions during prediction
let model = BayesianClassifierChain::new();
let trained_model = model.fit(&X.view(), &y).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_model.predict(&X_test.view()).is_err());
assert!(trained_model.predict_uncertainty(&X_test.view()).is_err());
}
#[test]
fn test_bayesian_classifier_chain_configuration() {
let model = BayesianClassifierChain::new()
.n_samples(200)
.prior_strength(2.0)
.random_state(42);
assert_eq!(model.n_samples, 200);
assert_eq!(model.prior_strength, 2.0);
assert_eq!(model.random_state, Some(42));
let default_model = BayesianClassifierChain::default();
assert_eq!(default_model.n_samples, 100);
assert_eq!(default_model.prior_strength, 1.0);
assert_eq!(default_model.random_state, None);
}
#[test]
fn test_multi_target_regression_tree_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[1.5, 2.5],
[3.5, 1.5]
];
let y = array![
[1.5, 2.5],
[2.5, 3.5],
[3.5, 1.5],
[4.5, 4.5],
[2.0, 3.0],
[4.0, 2.0]
];
let tree = MultiTargetRegressionTree::new()
.max_depth(Some(3))
.min_samples_split(2);
let trained_tree = tree.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_tree.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Test tree properties
assert_eq!(trained_tree.n_features(), 2);
assert_eq!(trained_tree.n_targets(), 2);
// Feature importances should sum to 1 (or be close to it)
let importances = trained_tree.feature_importances();
let importance_sum: Float = importances.sum();
assert!((importance_sum - 1.0).abs() < 1e-10 || importance_sum == 0.0);
}
#[test]
fn test_multi_target_regression_tree_configuration() {
let tree = MultiTargetRegressionTree::new()
.max_depth(Some(5))
.min_samples_split(3)
.min_samples_leaf(2)
.random_state(Some(42));
assert_eq!(tree.max_depth, Some(5));
assert_eq!(tree.min_samples_split, 3);
assert_eq!(tree.min_samples_leaf, 2);
assert_eq!(tree.random_state, Some(42));
let default_tree = MultiTargetRegressionTree::default();
assert_eq!(default_tree.max_depth, Some(5));
assert_eq!(default_tree.min_samples_split, 2);
assert_eq!(default_tree.min_samples_leaf, 1);
assert_eq!(default_tree.random_state, None);
}
#[test]
fn test_multi_target_regression_tree_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5]]; // Wrong number of rows
let tree = MultiTargetRegressionTree::new();
assert!(tree.clone().fit(&X.view(), &y).is_err());
// Test with no targets
let y_empty = Array2::<Float>::zeros((2, 0));
assert!(tree.clone().fit(&X.view(), &y_empty).is_err());
// Test with too few samples
let X_small = array![[1.0, 2.0]];
let y_small = array![[1.5, 2.5]];
let tree_strict = MultiTargetRegressionTree::new().min_samples_split(2);
assert!(tree_strict.fit(&X_small.view(), &y_small).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1.5, 2.5], [2.5, 3.5]];
let tree_for_predict = MultiTargetRegressionTree::new();
let trained_tree = tree_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_tree.predict(&X_test.view()).is_err());
}
#[test]
fn test_random_forest_multi_output_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[1.5, 2.5],
[3.5, 1.5],
[2.5, 2.0],
[1.8, 3.2]
];
let y = array![
[1.5, 2.5],
[2.5, 3.5],
[3.5, 1.5],
[4.5, 4.5],
[2.0, 3.0],
[4.0, 2.0],
[3.0, 2.5],
[2.3, 3.7]
];
let forest = RandomForestMultiOutput::new()
.n_estimators(5)
.max_depth(Some(3))
.random_state(Some(42));
let trained_forest = forest.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_forest.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (8, 2));
// Test forest properties
assert_eq!(trained_forest.n_estimators(), 5);
assert_eq!(trained_forest.n_features(), 2);
assert_eq!(trained_forest.n_targets(), 2);
// Feature importances should sum to 1 (or be close to it)
let importances = trained_forest.feature_importances();
let importance_sum: Float = importances.sum();
assert!((importance_sum - 1.0).abs() < 1e-10 || importance_sum == 0.0);
}
#[test]
fn test_random_forest_multi_output_configuration() {
let forest = RandomForestMultiOutput::new()
.n_estimators(20)
.max_depth(Some(7))
.min_samples_split(5)
.min_samples_leaf(3)
.max_features(Some(1))
.bootstrap(false)
.random_state(Some(123));
assert_eq!(forest.n_estimators, 20);
assert_eq!(forest.max_depth, Some(7));
assert_eq!(forest.min_samples_split, 5);
assert_eq!(forest.min_samples_leaf, 3);
assert_eq!(forest.max_features, Some(1));
assert_eq!(forest.bootstrap, false);
assert_eq!(forest.random_state, Some(123));
let default_forest = RandomForestMultiOutput::default();
assert_eq!(default_forest.n_estimators, 10);
assert_eq!(default_forest.max_depth, None);
assert_eq!(default_forest.min_samples_split, 2);
assert_eq!(default_forest.min_samples_leaf, 1);
assert_eq!(default_forest.max_features, None);
assert_eq!(default_forest.bootstrap, true);
assert_eq!(default_forest.random_state, None);
}
#[test]
fn test_random_forest_multi_output_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5]]; // Wrong number of rows
let forest = RandomForestMultiOutput::new();
assert!(forest.clone().fit(&X.view(), &y).is_err());
// Test with no targets
let y_empty = Array2::<Float>::zeros((2, 0));
assert!(forest.clone().fit(&X.view(), &y_empty).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y_train = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
let forest_for_predict = RandomForestMultiOutput::new();
let trained_forest = forest_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_forest.predict(&X_test.view()).is_err());
}
#[test]
fn test_random_forest_multi_output_single_estimator() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
let forest = RandomForestMultiOutput::new()
.n_estimators(1)
.random_state(Some(42));
let trained_forest = forest.fit(&X.view(), &y).unwrap();
let predictions = trained_forest.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
assert_eq!(trained_forest.n_estimators(), 1);
}
#[test]
fn test_random_forest_bootstrap_vs_no_bootstrap() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[1.5, 2.5],
[3.5, 1.5],
[2.5, 2.0],
[1.8, 3.2]
];
let y = array![
[1.5, 2.5],
[2.5, 3.5],
[3.5, 1.5],
[4.5, 4.5],
[2.0, 3.0],
[4.0, 2.0],
[3.0, 2.5],
[2.3, 3.7]
];
// Test with bootstrap
let forest_bootstrap = RandomForestMultiOutput::new()
.n_estimators(3)
.bootstrap(true)
.random_state(Some(42));
let trained_bootstrap = forest_bootstrap.fit(&X.view(), &y).unwrap();
// Test without bootstrap
let forest_no_bootstrap = RandomForestMultiOutput::new()
.n_estimators(3)
.bootstrap(false)
.random_state(Some(42));
let trained_no_bootstrap = forest_no_bootstrap.fit(&X.view(), &y).unwrap();
// Both should work and produce valid predictions
let pred_bootstrap = trained_bootstrap.predict(&X.view()).unwrap();
let pred_no_bootstrap = trained_no_bootstrap.predict(&X.view()).unwrap();
assert_eq!(pred_bootstrap.dim(), (8, 2));
assert_eq!(pred_no_bootstrap.dim(), (8, 2));
}
#[test]
fn test_gradient_boosting_multi_output_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[1.5, 2.5],
[3.5, 1.5],
[2.5, 2.0],
[1.8, 3.2]
];
let y = array![
[1.5, 2.5],
[2.5, 3.5],
[3.5, 1.5],
[4.5, 4.5],
[2.0, 3.0],
[4.0, 2.0],
[3.0, 2.5],
[2.3, 3.7]
];
let gbm = GradientBoostingMultiOutput::new()
.n_estimators(10)
.learning_rate(0.1)
.max_depth(Some(2))
.random_state(Some(42));
let trained_gbm = gbm.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_gbm.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (8, 2));
// Test gbm properties
assert_eq!(trained_gbm.n_estimators(), 10);
assert_eq!(trained_gbm.n_features(), 2);
assert_eq!(trained_gbm.n_targets(), 2);
// Feature importances should sum to 1 (or be close to it)
let importances = trained_gbm.feature_importances();
let importance_sum: Float = importances.sum();
assert!((importance_sum - 1.0).abs() < 1e-10 || importance_sum == 0.0);
// Training scores should be decreasing (generally)
let train_scores = trained_gbm.train_scores();
assert_eq!(train_scores.len(), 10);
assert!(train_scores[0] >= train_scores[train_scores.len() - 1]); // Loss should decrease
}
#[test]
fn test_gradient_boosting_multi_output_configuration() {
let gbm = GradientBoostingMultiOutput::new()
.n_estimators(50)
.learning_rate(0.05)
.max_depth(Some(4))
.min_samples_split(3)
.min_samples_leaf(2)
.subsample(0.8)
.random_state(Some(123));
assert_eq!(gbm.n_estimators, 50);
assert_eq!(gbm.learning_rate, 0.05);
assert_eq!(gbm.max_depth, Some(4));
assert_eq!(gbm.min_samples_split, 3);
assert_eq!(gbm.min_samples_leaf, 2);
assert_eq!(gbm.subsample, 0.8);
assert_eq!(gbm.random_state, Some(123));
let default_gbm = GradientBoostingMultiOutput::default();
assert_eq!(default_gbm.n_estimators, 100);
assert_eq!(default_gbm.learning_rate, 0.1);
assert_eq!(default_gbm.max_depth, Some(3));
assert_eq!(default_gbm.min_samples_split, 2);
assert_eq!(default_gbm.min_samples_leaf, 1);
assert_eq!(default_gbm.subsample, 1.0);
assert_eq!(default_gbm.random_state, None);
}
#[test]
fn test_gradient_boosting_multi_output_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5]]; // Wrong number of rows
let gbm = GradientBoostingMultiOutput::new();
assert!(gbm.clone().fit(&X.view(), &y).is_err());
// Test with no targets
let y_empty = Array2::<Float>::zeros((2, 0));
assert!(gbm.clone().fit(&X.view(), &y_empty).is_err());
// Test with invalid learning rate
let gbm_invalid_lr = GradientBoostingMultiOutput::new().learning_rate(0.0);
let y_valid = array![[1.5, 2.5], [2.5, 3.5]];
assert!(gbm_invalid_lr.fit(&X.view(), &y_valid).is_err());
// Test with invalid subsample
let gbm_invalid_subsample = GradientBoostingMultiOutput::new().subsample(1.5);
assert!(gbm_invalid_subsample.fit(&X.view(), &y_valid).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y_train = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
let gbm_for_predict = GradientBoostingMultiOutput::new().n_estimators(5);
let trained_gbm = gbm_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_gbm.predict(&X_test.view()).is_err());
assert!(trained_gbm.staged_predict(&X_test.view(), 3).is_err());
}
#[test]
fn test_gradient_boosting_multi_output_staged_predict() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
let gbm = GradientBoostingMultiOutput::new()
.n_estimators(10)
.random_state(Some(42));
let trained_gbm = gbm.fit(&X.view(), &y).unwrap();
// Test staged prediction with different number of estimators
let pred_3 = trained_gbm.staged_predict(&X.view(), 3).unwrap();
let pred_5 = trained_gbm.staged_predict(&X.view(), 5).unwrap();
let pred_all = trained_gbm.predict(&X.view()).unwrap();
assert_eq!(pred_3.dim(), (4, 2));
assert_eq!(pred_5.dim(), (4, 2));
assert_eq!(pred_all.dim(), (4, 2));
// Predictions should be different with different numbers of estimators
assert!(&pred_3 != &pred_5);
assert!(&pred_5 != &pred_all);
}
#[test]
fn test_independent_label_prediction_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[1.5, 2.5],
[3.5, 1.5]
];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0], [1, 0], [0, 1]];
let ilp =
IndependentLabelPrediction::new().threshold_strategy(ThresholdStrategy::Fixed(0.5));
let trained_ilp = ilp.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_ilp.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are binary
for &pred in predictions.iter() {
assert!(pred == 0 || pred == 1);
}
// Test probability prediction
let probabilities = trained_ilp.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.dim(), (6, 2));
// Check that probabilities are in valid range
for &prob in probabilities.iter() {
assert!(prob >= 0.0 && prob <= 1.0);
}
// Test ILP properties
assert_eq!(trained_ilp.n_features(), 2);
assert_eq!(trained_ilp.n_labels(), 2);
assert_eq!(trained_ilp.thresholds().len(), 2);
assert_eq!(trained_ilp.label_frequencies().len(), 2);
}
#[test]
fn test_independent_label_prediction_threshold_strategies() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test fixed threshold
let ilp_fixed =
IndependentLabelPrediction::new().threshold_strategy(ThresholdStrategy::Fixed(0.7));
let trained_fixed = ilp_fixed.fit(&X.view(), &y).unwrap();
let thresholds_fixed = trained_fixed.thresholds();
assert_eq!(thresholds_fixed, &[0.7, 0.7]);
// Test per-label thresholds
let ilp_per_label = IndependentLabelPrediction::new()
.threshold_strategy(ThresholdStrategy::PerLabel(vec![0.3, 0.8]));
let trained_per_label = ilp_per_label.fit(&X.view(), &y).unwrap();
let thresholds_per_label = trained_per_label.thresholds();
assert_eq!(thresholds_per_label, &[0.3, 0.8]);
// Test optimal threshold (without optimization enabled, should default to 0.5)
let ilp_optimal = IndependentLabelPrediction::new()
.threshold_strategy(ThresholdStrategy::Optimal)
.optimize_thresholds(false);
let trained_optimal = ilp_optimal.fit(&X.view(), &y).unwrap();
let thresholds_optimal = trained_optimal.thresholds();
assert_eq!(thresholds_optimal, &[0.5, 0.5]);
}
#[test]
fn test_independent_label_prediction_class_weight() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
// Test balanced class weights
let ilp_balanced =
IndependentLabelPrediction::new().class_weight(Some("balanced".to_string()));
let trained_balanced = ilp_balanced.fit(&X.view(), &y).unwrap();
let predictions = trained_balanced.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Test default (no class weights)
let ilp_default = IndependentLabelPrediction::new();
let trained_default = ilp_default.fit(&X.view(), &y).unwrap();
let predictions_default = trained_default.predict(&X.view()).unwrap();
assert_eq!(predictions_default.dim(), (4, 2));
}
#[test]
fn test_independent_label_prediction_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let ilp = IndependentLabelPrediction::new();
assert!(ilp.clone().fit(&X.view(), &y).is_err());
// Test with no labels
let y_empty = Array2::<i32>::zeros((2, 0));
assert!(ilp.clone().fit(&X.view(), &y_empty).is_err());
// Test with non-binary labels
let y_invalid = array![[1, 2], [0, 1]];
assert!(ilp.clone().fit(&X.view(), &y_invalid).is_err());
// Test mismatched threshold count
let y_valid = array![[1, 0], [0, 1]];
let ilp_wrong_thresholds = IndependentLabelPrediction::new()
.threshold_strategy(ThresholdStrategy::PerLabel(vec![0.5, 0.6, 0.7])); // 3 thresholds for 2 labels
assert!(ilp_wrong_thresholds.fit(&X.view(), &y_valid).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1, 0], [0, 1]];
let ilp_for_predict = IndependentLabelPrediction::new();
let trained_ilp = ilp_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_ilp.predict(&X_test.view()).is_err());
assert!(trained_ilp.predict_proba(&X_test.view()).is_err());
}
#[test]
fn test_independent_label_prediction_custom_thresholds() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let ilp = IndependentLabelPrediction::new();
let trained_ilp = ilp.fit(&X.view(), &y).unwrap();
// Test prediction with custom thresholds
let custom_thresholds = [0.3, 0.8];
let predictions_custom = trained_ilp
.predict_with_thresholds(&X.view(), &custom_thresholds)
.unwrap();
assert_eq!(predictions_custom.dim(), (4, 2));
// Test with mismatched threshold count
let wrong_thresholds = [0.5, 0.6, 0.7]; // Too many thresholds
assert!(trained_ilp
.predict_with_thresholds(&X.view(), &wrong_thresholds)
.is_err());
}
#[test]
fn test_multi_target_decision_tree_classifier_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[5.0, 2.0],
[6.0, 3.0]
];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0], [0, 1], [1, 0]];
let tree = MultiTargetDecisionTreeClassifier::new()
.max_depth(Some(3))
.min_samples_split(2)
.min_samples_leaf(1);
let trained_tree = tree.fit(&X.view(), &y).unwrap();
let predictions = trained_tree.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are binary
for pred in predictions.iter() {
assert!(pred == &0 || pred == &1);
}
// Check feature importances are normalized
let importances = trained_tree.feature_importances();
let sum: Float = importances.sum();
assert!((sum - 1.0).abs() < 1e-10 || sum == 0.0); // Sum should be 1.0 or 0.0 if all features have zero importance
}
#[test]
fn test_multi_target_decision_tree_classifier_probability() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let tree = MultiTargetDecisionTreeClassifier::new();
let trained_tree = tree.fit(&X.view(), &y).unwrap();
let probabilities = trained_tree.predict_proba(&X.view()).unwrap();
assert_eq!(probabilities.len(), 2); // Two targets
assert_eq!(probabilities[0].dim(), (4, 2)); // 4 samples, 2 classes for target 0
assert_eq!(probabilities[1].dim(), (4, 2)); // 4 samples, 2 classes for target 1
// Check that probabilities sum to 1 for each sample and target
for target_idx in 0..2 {
for sample_idx in 0..4 {
let prob_sum: Float = probabilities[target_idx].row(sample_idx).sum();
assert!((prob_sum - 1.0).abs() < 1e-10);
}
}
}
#[test]
fn test_multi_target_decision_tree_classifier_entropy_criterion() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
let tree =
MultiTargetDecisionTreeClassifier::new().criterion(ClassificationCriterion::Entropy);
let trained_tree = tree.fit(&X.view(), &y).unwrap();
let predictions = trained_tree.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
}
#[test]
fn test_multi_target_decision_tree_classifier_single_class() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1, 0], [1, 0], [1, 0]]; // All samples have same labels
let tree = MultiTargetDecisionTreeClassifier::new();
let trained_tree = tree.fit(&X.view(), &y).unwrap();
let predictions = trained_tree.predict(&X.view()).unwrap();
// Should predict the single class for all samples
for i in 0..3 {
assert_eq!(predictions[[i, 0]], 1);
assert_eq!(predictions[[i, 1]], 0);
}
}
#[test]
fn test_multi_target_decision_tree_classifier_multi_class() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[5.0, 2.0],
[6.0, 3.0]
];
let y = array![[0, 2], [1, 1], [2, 0], [0, 2], [1, 1], [2, 0]]; // Multi-class targets (0, 1, 2)
let tree = MultiTargetDecisionTreeClassifier::new();
let trained_tree = tree.fit(&X.view(), &y).unwrap();
let predictions = trained_tree.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 2));
// Check that predictions are in valid range
for pred in predictions.iter() {
assert!(pred >= &0 && pred <= &2);
}
}
#[test]
fn test_multi_target_decision_tree_classifier_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let tree = MultiTargetDecisionTreeClassifier::new();
assert!(tree.clone().fit(&X.view(), &y).is_err());
// Test with no targets
let y_empty = Array2::<i32>::zeros((2, 0));
assert!(tree.clone().fit(&X.view(), &y_empty).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1, 0], [0, 1]];
let tree_for_predict = MultiTargetDecisionTreeClassifier::new();
let trained_tree = tree_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_tree.predict(&X_test.view()).is_err());
assert!(trained_tree.predict_proba(&X_test.view()).is_err());
}
#[test]
fn test_multi_target_decision_tree_classifier_hyperparameters() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[5.0, 2.0],
[6.0, 3.0],
[7.0, 1.0],
[8.0, 4.0]
];
let y = array![
[0, 1],
[1, 0],
[1, 1],
[0, 0],
[0, 1],
[1, 0],
[1, 1],
[0, 0]
];
// Test min_samples_split
let tree1 = MultiTargetDecisionTreeClassifier::new().min_samples_split(4);
let trained_tree1 = tree1.fit(&X.view(), &y).unwrap();
// Test min_samples_leaf
let tree2 = MultiTargetDecisionTreeClassifier::new().min_samples_leaf(2);
let trained_tree2 = tree2.fit(&X.view(), &y).unwrap();
// Test max_depth
let tree3 = MultiTargetDecisionTreeClassifier::new().max_depth(Some(1));
let trained_tree3 = tree3.fit(&X.view(), &y).unwrap();
// All should produce valid predictions
assert_eq!(trained_tree1.predict(&X.view()).unwrap().dim(), (8, 2));
assert_eq!(trained_tree2.predict(&X.view()).unwrap().dim(), (8, 2));
assert_eq!(trained_tree3.predict(&X.view()).unwrap().dim(), (8, 2));
}
#[test]
fn test_multi_target_decision_tree_classifier_random_state() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1], [1, 0], [1, 1], [0, 0]];
// Train two trees with same random state
let tree1 = MultiTargetDecisionTreeClassifier::new().random_state(Some(42));
let trained_tree1 = tree1.fit(&X.view(), &y).unwrap();
let tree2 = MultiTargetDecisionTreeClassifier::new().random_state(Some(42));
let trained_tree2 = tree2.fit(&X.view(), &y).unwrap();
// Should produce same predictions (deterministic given same random state)
let pred1 = trained_tree1.predict(&X.view()).unwrap();
let pred2 = trained_tree2.predict(&X.view()).unwrap();
// Note: Since our current implementation doesn't use randomness in splits,
// this test mainly verifies the API works. In a more advanced implementation
// with random feature selection, this would test reproducibility.
assert_eq!(pred1.dim(), pred2.dim());
}
}
/// Compressed Sensing Label Powerset
///
/// A multi-label classification approach using compressed sensing to handle
/// high-dimensional label spaces more efficiently.
#[derive(Debug, Clone)]
pub struct CompressedSensingLabelPowerset<S = Untrained> {
state: S,
compressed_dimension: Option<usize>,
random_state: Option<u64>,
reconstruction_method: ReconstructionMethod,
}
impl CompressedSensingLabelPowerset<Untrained> {
/// Create a new CompressedSensingLabelPowerset instance
pub fn new() -> Self {
Self {
state: Untrained,
compressed_dimension: None,
random_state: None,
reconstruction_method: ReconstructionMethod::Linear,
}
}
/// Set the compressed dimension
pub fn compressed_dimension(mut self, dim: usize) -> Self {
self.compressed_dimension = Some(dim);
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
/// Set reconstruction method
pub fn reconstruction_method(mut self, method: ReconstructionMethod) -> Self {
self.reconstruction_method = method;
self
}
}
impl Default for CompressedSensingLabelPowerset<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for CompressedSensingLabelPowerset<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
/// Trained state for CompressedSensingLabelPowerset
#[derive(Debug, Clone)]
pub struct CompressedSensingLabelPowersetTrained {
projection_matrix: Array2<Float>,
label_powerset: LabelPowerset<LabelPowersetTrained>,
n_features: usize,
n_labels: usize,
compressed_dimension: usize,
reconstruction_method: ReconstructionMethod,
}
impl Fit<ArrayView2<'_, Float>, Array2<i32>> for CompressedSensingLabelPowerset<Untrained> {
type Fitted = CompressedSensingLabelPowerset<CompressedSensingLabelPowersetTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_labels = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
let compressed_dimension = self.compressed_dimension.unwrap_or(n_labels / 2);
if compressed_dimension >= n_labels {
return Err(SklearsError::InvalidInput(
"Compressed dimension must be less than number of labels".to_string(),
));
}
// Generate projection matrix
let projection_matrix =
generate_random_projection_matrix(compressed_dimension, n_labels, self.random_state);
// Transform labels to compressed space
let y_float: Array2<Float> = y.mapv(|x| x as Float);
let y_compressed = y_float.dot(&projection_matrix.t());
// Train label powerset on compressed labels
let y_compressed_int: Array2<i32> = y_compressed.mapv(|x| if x > 0.0 { 1 } else { 0 });
let powerset = LabelPowerset::new();
let trained_powerset = powerset.fit(X, &y_compressed_int)?;
let trained_state = CompressedSensingLabelPowersetTrained {
projection_matrix,
label_powerset: trained_powerset,
n_features,
n_labels,
compressed_dimension,
reconstruction_method: self.reconstruction_method,
};
Ok(CompressedSensingLabelPowerset {
state: trained_state,
compressed_dimension: self.compressed_dimension,
random_state: self.random_state,
reconstruction_method: self.reconstruction_method,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for CompressedSensingLabelPowerset<CompressedSensingLabelPowersetTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
// Predict compressed labels
let compressed_predictions = self.state.label_powerset.predict(X)?;
let compressed_float: Array2<Float> = compressed_predictions.mapv(|x| x as Float);
// Reconstruct original labels
let mut reconstructed = Array2::zeros((n_samples, self.state.n_labels));
for i in 0..n_samples {
let compressed_sample = compressed_float.row(i).to_owned();
let reconstructed_sample = reconstruct_labels(
&compressed_sample,
&self.state.projection_matrix,
self.state.reconstruction_method,
)?;
// Convert to binary
for j in 0..self.state.n_labels {
reconstructed[[i, j]] = if reconstructed_sample[j] > 0.0 { 1 } else { 0 };
}
}
Ok(reconstructed)
}
}
impl CompressedSensingLabelPowerset<CompressedSensingLabelPowersetTrained> {
/// Get the compression ratio
pub fn compression_ratio(&self) -> Float {
self.state.compressed_dimension as Float / self.state.n_labels as Float
}
/// Get the projection matrix
pub fn projection_matrix(&self) -> &Array2<Float> {
&self.state.projection_matrix
}
}
/// Multi-Output Support Vector Machine
///
/// A multi-output support vector machine that handles multiple regression or
/// classification targets simultaneously by training separate SVM models for each output.
#[derive(Debug, Clone)]
pub struct MultiOutputSVM<S = Untrained> {
state: S,
kernel: SVMKernel,
c: Float,
epsilon: Float,
gamma: Option<Float>,
}
/// SVM Kernel types
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum SVMKernel {
/// Linear kernel: K(x, y) = x^T y
Linear,
/// Polynomial kernel: K(x, y) = (gamma * x^T y + coef0)^degree
Polynomial {
degree: i32,
gamma: Float,
coef0: Float,
},
/// RBF (Gaussian) kernel: K(x, y) = exp(-gamma * ||x - y||^2)
RBF { gamma: Float },
/// Sigmoid kernel: K(x, y) = tanh(gamma * x^T y + coef0)
Sigmoid { gamma: Float, coef0: Float },
}
impl MultiOutputSVM<Untrained> {
/// Create a new MultiOutputSVM instance
pub fn new() -> Self {
Self {
state: Untrained,
kernel: SVMKernel::RBF { gamma: 1.0 },
c: 1.0,
epsilon: 0.1,
gamma: None,
}
}
/// Set the SVM kernel
pub fn kernel(mut self, kernel: SVMKernel) -> Self {
self.kernel = kernel;
self
}
/// Set the regularization parameter C
pub fn c(mut self, c: Float) -> Self {
self.c = c;
self
}
/// Set epsilon for regression tolerance
pub fn epsilon(mut self, epsilon: Float) -> Self {
self.epsilon = epsilon;
self
}
/// Set gamma parameter for RBF and polynomial kernels
pub fn gamma(mut self, gamma: Float) -> Self {
self.gamma = Some(gamma);
self
}
}
impl Default for MultiOutputSVM<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MultiOutputSVM<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
/// Trained state for MultiOutputSVM
#[derive(Debug, Clone)]
pub struct MultiOutputSVMTrained {
support_vectors: Array2<Float>,
dual_coef: Array2<Float>,
intercept: Array1<Float>,
kernel: SVMKernel,
n_features: usize,
n_outputs: usize,
gamma: Float,
}
impl Fit<ArrayView2<'_, Float>, Array2<Float>> for MultiOutputSVM<Untrained> {
type Fitted = MultiOutputSVM<MultiOutputSVMTrained>;
fn fit(self, X: &ArrayView2<'_, Float>, y: &Array2<Float>) -> SklResult<Self::Fitted> {
let (n_samples, n_features) = X.dim();
let n_outputs = y.ncols();
if n_samples != y.nrows() {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
// Set gamma based on features if not provided
let gamma = self.gamma.unwrap_or(1.0 / n_features as Float);
// For simplicity, we'll implement a basic multi-output regression using
// simplified SVM-like approach with kernel ridge regression
let kernel_matrix = compute_kernel_matrix(X, X, &self.kernel, gamma)?;
// Create regularized kernel matrix
let mut K_reg = kernel_matrix.clone();
for i in 0..n_samples {
K_reg[[i, i]] += 1.0 / self.c;
}
// Solve for each output: (K + λI) α = y
let mut dual_coef = Array2::zeros((n_samples, n_outputs));
let mut intercept = Array1::zeros(n_outputs);
for output_idx in 0..n_outputs {
let y_target = y.column(output_idx).to_owned();
// Solve the system using our linear solver
match solve_linear_system(&K_reg, &y_target) {
Ok(alpha) => {
dual_coef.column_mut(output_idx).assign(&alpha);
// Compute intercept as mean of residuals
let predictions = kernel_matrix.dot(&alpha);
let residuals = &y_target - &predictions;
intercept[output_idx] = residuals.mean().unwrap_or(0.0);
}
Err(_) => {
// Fallback to simple linear solution
let mean_target = y_target.mean().unwrap_or(0.0);
intercept[output_idx] = mean_target;
}
}
}
let trained_state = MultiOutputSVMTrained {
support_vectors: X.to_owned(),
dual_coef,
intercept,
kernel: self.kernel,
n_features,
n_outputs,
gamma,
};
Ok(MultiOutputSVM {
state: trained_state,
kernel: self.kernel,
c: self.c,
epsilon: self.epsilon,
gamma: self.gamma,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<Float>> for MultiOutputSVM<MultiOutputSVMTrained> {
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
// Compute kernel matrix between test and training data
let kernel_matrix = compute_kernel_matrix(
X,
&self.state.support_vectors.view(),
&self.state.kernel,
self.state.gamma,
)?;
// Predict: K(X, X_train) * dual_coef + intercept
let predictions = kernel_matrix.dot(&self.state.dual_coef) + &self.state.intercept;
Ok(predictions)
}
}
impl MultiOutputSVM<MultiOutputSVMTrained> {
/// Get the number of support vectors
pub fn n_support_vectors(&self) -> usize {
self.state.support_vectors.nrows()
}
/// Get the dual coefficients
pub fn dual_coef(&self) -> &Array2<Float> {
&self.state.dual_coef
}
/// Get the intercept terms
pub fn intercept(&self) -> &Array1<Float> {
&self.state.intercept
}
/// Get the support vectors
pub fn support_vectors(&self) -> &Array2<Float> {
&self.state.support_vectors
}
}
/// Compute kernel matrix between two sets of samples
fn compute_kernel_matrix(
X1: &ArrayView2<Float>,
X2: &ArrayView2<Float>,
kernel: &SVMKernel,
gamma: Float,
) -> SklResult<Array2<Float>> {
let n1 = X1.nrows();
let n2 = X2.nrows();
let mut K = Array2::zeros((n1, n2));
for i in 0..n1 {
for j in 0..n2 {
let x1 = X1.row(i);
let x2 = X2.row(j);
K[[i, j]] = compute_kernel_value(&x1, &x2, kernel, gamma);
}
}
Ok(K)
}
/// Compute kernel value between two samples
fn compute_kernel_value(
x1: &ArrayView1<Float>,
x2: &ArrayView1<Float>,
kernel: &SVMKernel,
default_gamma: Float,
) -> Float {
match kernel {
SVMKernel::Linear => x1.dot(x2),
SVMKernel::Polynomial {
degree,
gamma,
coef0,
} => (*gamma * x1.dot(x2) + *coef0).powi(*degree),
SVMKernel::RBF { gamma } => {
let dist_sq = x1
.iter()
.zip(x2.iter())
.map(|(a, b)| (a - b).powi(2))
.sum::<Float>();
(-gamma * dist_sq).exp()
}
SVMKernel::Sigmoid { gamma, coef0 } => (*gamma * x1.dot(x2) + *coef0).tanh(),
}
}
// =============================================================================
// STRUCTURED OUTPUT PREDICTION METHODS
// =============================================================================
/// Conditional Random Fields for sequence labeling
///
/// A discriminative probabilistic framework for labeling and segmenting
/// sequential data. CRFs model the conditional probability of label sequences
/// given input sequences, making them particularly suitable for structured
/// prediction tasks like named entity recognition, part-of-speech tagging,
/// and DNA sequence analysis.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::ConditionalRandomField;
/// use ndarray::array;
///
/// let X = array![[[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]]]; // One sequence
/// let y = array![[0, 1, 0]]; // Label sequence
///
/// let crf = ConditionalRandomField::new()
/// .max_iterations(100)
/// .learning_rate(0.01);
/// let trained_crf = crf.fit(&X, &y).unwrap();
/// let predictions = trained_crf.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct ConditionalRandomField<State = Untrained> {
max_iterations: usize,
learning_rate: Float,
l2_penalty: Float,
tolerance: Float,
state: State,
}
/// Trained state for Conditional Random Field
#[derive(Debug, Clone)]
pub struct ConditionalRandomFieldTrained {
feature_weights: Array2<Float>,
transition_weights: Array2<Float>,
n_features: usize,
n_labels: usize,
label_to_idx: HashMap<i32, usize>,
idx_to_label: HashMap<usize, i32>,
}
impl ConditionalRandomField<Untrained> {
/// Create new CRF instance
pub fn new() -> Self {
Self {
max_iterations: 100,
learning_rate: 0.01,
l2_penalty: 0.1,
tolerance: 1e-6,
state: Untrained,
}
}
/// Set maximum iterations for optimization
pub fn max_iterations(mut self, max_iter: usize) -> Self {
self.max_iterations = max_iter;
self
}
/// Set learning rate for gradient descent
pub fn learning_rate(mut self, lr: Float) -> Self {
self.learning_rate = lr;
self
}
/// Set L2 regularization penalty
pub fn l2_penalty(mut self, penalty: Float) -> Self {
self.l2_penalty = penalty;
self
}
/// Set convergence tolerance
pub fn tolerance(mut self, tol: Float) -> Self {
self.tolerance = tol;
self
}
}
impl Estimator for ConditionalRandomField<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array3<Float>, Array2<i32>> for ConditionalRandomField<Untrained> {
type Fitted = ConditionalRandomField<ConditionalRandomFieldTrained>;
fn fit(self, X: &Array3<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_sequences, max_length, n_features) = X.dim();
if y.nrows() != n_sequences {
return Err(SklearsError::InvalidInput(
"Number of sequences in X and y must match".to_string(),
));
}
// Build label mappings
let unique_labels: std::collections::HashSet<i32> = y.iter().cloned().collect();
let mut label_to_idx = HashMap::new();
let mut idx_to_label = HashMap::new();
for (idx, &label) in unique_labels.iter().enumerate() {
label_to_idx.insert(label, idx);
idx_to_label.insert(idx, label);
}
let n_labels = unique_labels.len();
// Initialize weights
let mut feature_weights = Array2::zeros((n_features, n_labels));
let mut transition_weights = Array2::zeros((n_labels, n_labels));
// Training with gradient ascent on log-likelihood
let mut prev_likelihood = Float::NEG_INFINITY;
for iteration in 0..self.max_iterations {
let mut feature_gradient = Array2::zeros((n_features, n_labels));
let mut transition_gradient = Array2::zeros((n_labels, n_labels));
let mut total_likelihood = 0.0;
// Process each sequence
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
let labels = y.row(seq_idx);
// Get actual sequence length (assuming padded with zeros)
let seq_len = labels.iter().position(|&x| x == -1).unwrap_or(max_length);
if seq_len == 0 {
continue;
}
// Forward-backward algorithm for marginal probabilities
let (alpha, beta, Z) = self.forward_backward(
&sequence.slice(s![..seq_len, ..]),
&feature_weights,
&transition_weights,
&label_to_idx,
)?;
// Compute feature expectations
for t in 0..seq_len {
let features = sequence.row(t);
for (k, &label_idx) in label_to_idx.iter() {
let marginal = alpha[[t, label_idx]] + beta[[t, label_idx]] - Z;
for (f, &feat_val) in features.iter().enumerate() {
feature_gradient[[f, label_idx]] += marginal.exp() * feat_val;
}
}
}
// Compute transition expectations
for t in 0..(seq_len - 1) {
for (k1, &l1_idx) in label_to_idx.iter() {
for (k2, &l2_idx) in label_to_idx.iter() {
let score = alpha[[t, l1_idx]]
+ transition_weights[[l1_idx, l2_idx]]
+ feature_weights.row(l2_idx).dot(&sequence.row(t + 1))
+ beta[[t + 1, l2_idx]];
let marginal = (score - Z).exp();
transition_gradient[[l1_idx, l2_idx]] += marginal;
}
}
}
// Add empirical feature counts
for t in 0..seq_len {
let features = sequence.row(t);
let label_idx = *label_to_idx.get(&labels[t]).unwrap();
for (f, &feat_val) in features.iter().enumerate() {
feature_gradient[[f, label_idx]] -= feat_val;
}
}
// Add empirical transition counts
for t in 0..(seq_len - 1) {
let curr_label = *label_to_idx.get(&labels[t]).unwrap();
let next_label = *label_to_idx.get(&labels[t + 1]).unwrap();
transition_gradient[[curr_label, next_label]] -= 1.0;
}
total_likelihood += Z;
}
// Apply L2 regularization
feature_gradient = feature_gradient - self.l2_penalty * &feature_weights;
transition_gradient = transition_gradient - self.l2_penalty * &transition_weights;
// Update weights
feature_weights = feature_weights - self.learning_rate * &feature_gradient;
transition_weights = transition_weights - self.learning_rate * &transition_gradient;
// Check convergence
if iteration > 0 && (total_likelihood - prev_likelihood).abs() < self.tolerance {
break;
}
prev_likelihood = total_likelihood;
}
Ok(ConditionalRandomField {
max_iterations: self.max_iterations,
learning_rate: self.learning_rate,
l2_penalty: self.l2_penalty,
tolerance: self.tolerance,
state: ConditionalRandomFieldTrained {
feature_weights,
transition_weights,
n_features,
n_labels,
label_to_idx,
idx_to_label,
},
})
}
}
impl ConditionalRandomField<Untrained> {
/// Forward-backward algorithm for CRF inference
fn forward_backward(
&self,
sequence: &ArrayView2<Float>,
feature_weights: &Array2<Float>,
transition_weights: &Array2<Float>,
label_to_idx: &HashMap<i32, usize>,
) -> SklResult<(Array2<Float>, Array2<Float>, Float)> {
let seq_len = sequence.nrows();
let n_labels = label_to_idx.len();
let mut alpha = Array2::zeros((seq_len, n_labels));
let mut beta = Array2::zeros((seq_len, n_labels));
// Forward pass
for (_, &label_idx) in label_to_idx.iter() {
alpha[[0, label_idx]] = feature_weights.row(label_idx).dot(&sequence.row(0));
}
for t in 1..seq_len {
for (_, &curr_label_idx) in label_to_idx.iter() {
let mut sum = Float::NEG_INFINITY;
for (_, &prev_label_idx) in label_to_idx.iter() {
let score = alpha[[t - 1, prev_label_idx]]
+ transition_weights[[prev_label_idx, curr_label_idx]]
+ feature_weights.row(curr_label_idx).dot(&sequence.row(t));
sum = log_sum_exp(sum, score);
}
alpha[[t, curr_label_idx]] = sum;
}
}
// Partition function
let mut Z = Float::NEG_INFINITY;
for (_, &label_idx) in label_to_idx.iter() {
Z = log_sum_exp(Z, alpha[[seq_len - 1, label_idx]]);
}
// Backward pass
for (_, &label_idx) in label_to_idx.iter() {
beta[[seq_len - 1, label_idx]] = 0.0;
}
for t in (0..(seq_len - 1)).rev() {
for (_, &curr_label_idx) in label_to_idx.iter() {
let mut sum = Float::NEG_INFINITY;
for (_, &next_label_idx) in label_to_idx.iter() {
let score = transition_weights[[curr_label_idx, next_label_idx]]
+ feature_weights
.row(next_label_idx)
.dot(&sequence.row(t + 1))
+ beta[[t + 1, next_label_idx]];
sum = log_sum_exp(sum, score);
}
beta[[t, curr_label_idx]] = sum;
}
}
Ok((alpha, beta, Z))
}
}
impl Predict<Array3<Float>, Array2<i32>> for ConditionalRandomField<ConditionalRandomFieldTrained> {
fn predict(&self, X: &Array3<Float>) -> SklResult<Array2<i32>> {
let (n_sequences, max_length, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features does not match training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_sequences, max_length));
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
// Get actual sequence length (assuming padded)
let seq_len = max_length; // For simplicity, assuming full length
// Viterbi decoding
let best_path = self.viterbi_decode(&sequence.slice(s![..seq_len, ..]))?;
for (t, &label_idx) in best_path.iter().enumerate() {
predictions[[seq_idx, t]] = *self.state.idx_to_label.get(&label_idx).unwrap();
}
}
Ok(predictions)
}
}
impl ConditionalRandomField<ConditionalRandomFieldTrained> {
/// Viterbi algorithm for finding most likely label sequence
fn viterbi_decode(&self, sequence: &ArrayView2<Float>) -> SklResult<Vec<usize>> {
let seq_len = sequence.nrows();
let n_labels = self.state.n_labels;
let mut viterbi = Array2::zeros((seq_len, n_labels));
let mut backpointer = Array2::zeros((seq_len, n_labels));
// Initialize
for label_idx in 0..n_labels {
viterbi[[0, label_idx]] = self
.state
.feature_weights
.row(label_idx)
.dot(&sequence.row(0));
}
// Forward pass
for t in 1..seq_len {
for curr_label in 0..n_labels {
let mut best_score = Float::NEG_INFINITY;
let mut best_prev = 0;
for prev_label in 0..n_labels {
let score = viterbi[[t - 1, prev_label]]
+ self.state.transition_weights[[prev_label, curr_label]]
+ self
.state
.feature_weights
.row(curr_label)
.dot(&sequence.row(t));
if score > best_score {
best_score = score;
best_prev = prev_label;
}
}
viterbi[[t, curr_label]] = best_score;
backpointer[[t, curr_label]] = best_prev as Float;
}
}
// Find best final state
let mut best_final_score = Float::NEG_INFINITY;
let mut best_final_state = 0;
for label_idx in 0..n_labels {
if viterbi[[seq_len - 1, label_idx]] > best_final_score {
best_final_score = viterbi[[seq_len - 1, label_idx]];
best_final_state = label_idx;
}
}
// Backtrack
let mut path = vec![0; seq_len];
path[seq_len - 1] = best_final_state;
for t in (1..seq_len).rev() {
path[t - 1] = backpointer[[t, path[t]]] as usize;
}
Ok(path)
}
/// Get feature weights
pub fn feature_weights(&self) -> &Array2<Float> {
&self.state.feature_weights
}
/// Get transition weights
pub fn transition_weights(&self) -> &Array2<Float> {
&self.state.transition_weights
}
}
/// Structured Perceptron for structured prediction
///
/// A simple and effective algorithm for learning structured prediction models.
/// The structured perceptron extends the classic perceptron to handle structured
/// outputs by using a feature function that maps input-output pairs to feature
/// vectors and a loss function that measures the quality of predictions.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::StructuredPerceptron;
/// use ndarray::array;
///
/// let X = array![[[1.0, 2.0], [2.0, 3.0]]]; // One sequence
/// let y = array![[0, 1]]; // Label sequence
///
/// let perceptron = StructuredPerceptron::new().max_iterations(50);
/// let trained_perceptron = perceptron.fit(&X, &y).unwrap();
/// let predictions = trained_perceptron.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct StructuredPerceptron<State = Untrained> {
max_iterations: usize,
learning_rate: Float,
random_state: Option<u64>,
state: State,
}
/// Trained state for Structured Perceptron
#[derive(Debug, Clone)]
pub struct StructuredPerceptronTrained {
weights: Array1<Float>,
n_features: usize,
n_labels: usize,
label_to_idx: HashMap<i32, usize>,
idx_to_label: HashMap<usize, i32>,
}
impl StructuredPerceptron<Untrained> {
/// Create new structured perceptron instance
pub fn new() -> Self {
Self {
max_iterations: 100,
learning_rate: 1.0,
random_state: None,
state: Untrained,
}
}
/// Set maximum iterations
pub fn max_iterations(mut self, max_iter: usize) -> Self {
self.max_iterations = max_iter;
self
}
/// Set learning rate
pub fn learning_rate(mut self, lr: Float) -> Self {
self.learning_rate = lr;
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, seed: Option<u64>) -> Self {
self.random_state = seed;
self
}
}
impl Estimator for StructuredPerceptron<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array3<Float>, Array2<i32>> for StructuredPerceptron<Untrained> {
type Fitted = StructuredPerceptron<StructuredPerceptronTrained>;
fn fit(self, X: &Array3<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_sequences, max_length, n_features) = X.dim();
if y.nrows() != n_sequences {
return Err(SklearsError::InvalidInput(
"Number of sequences in X and y must match".to_string(),
));
}
// Build label mappings
let unique_labels: std::collections::HashSet<i32> = y.iter().cloned().collect();
let mut label_to_idx = HashMap::new();
let mut idx_to_label = HashMap::new();
for (idx, &label) in unique_labels.iter().enumerate() {
label_to_idx.insert(label, idx);
idx_to_label.insert(idx, label);
}
let n_labels = unique_labels.len();
// Feature vector size: position-specific features + transition features
let feature_size = max_length * n_features * n_labels + n_labels * n_labels;
let mut weights = Array1::zeros(feature_size);
// Training loop
for _iteration in 0..self.max_iterations {
let mut total_updates = 0;
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
let true_labels = y.row(seq_idx);
// Get actual sequence length
let seq_len = true_labels
.iter()
.position(|&x| x == -1)
.unwrap_or(max_length);
if seq_len == 0 {
continue;
}
// Make prediction using current weights
let predicted_labels = self.predict_sequence(
&sequence.slice(s![..seq_len, ..]),
&weights,
max_length,
n_features,
n_labels,
&label_to_idx,
&idx_to_label,
)?;
// Check if prediction is correct
let mut is_correct = true;
for t in 0..seq_len {
if predicted_labels[t] != true_labels[t] {
is_correct = false;
break;
}
}
// Update weights if prediction is wrong
if !is_correct {
// Extract features for true and predicted sequences
let true_features = self.extract_features(
&sequence.slice(s![..seq_len, ..]),
&true_labels.slice(s![..seq_len]),
max_length,
n_features,
n_labels,
&label_to_idx,
)?;
let pred_features = self.extract_features(
&sequence.slice(s![..seq_len, ..]),
&ArrayView1::from(&predicted_labels[..seq_len]),
max_length,
n_features,
n_labels,
&label_to_idx,
)?;
// Perceptron update: w = w + η(φ(x,y) - φ(x,y'))
let update = &true_features - &pred_features;
weights = weights + self.learning_rate * &update;
total_updates += 1;
}
}
// Early stopping if no updates were made
if total_updates == 0 {
break;
}
}
Ok(StructuredPerceptron {
max_iterations: self.max_iterations,
learning_rate: self.learning_rate,
random_state: self.random_state,
state: StructuredPerceptronTrained {
weights,
n_features,
n_labels,
label_to_idx,
idx_to_label,
},
})
}
}
impl StructuredPerceptron<Untrained> {
/// Extract feature vector for a labeled sequence
fn extract_features(
&self,
sequence: &ArrayView2<Float>,
labels: &ArrayView1<i32>,
max_length: usize,
n_features: usize,
n_labels: usize,
label_to_idx: &HashMap<i32, usize>,
) -> SklResult<Array1<Float>> {
let feature_size = max_length * n_features * n_labels + n_labels * n_labels;
let mut features = Array1::zeros(feature_size);
let seq_len = sequence.nrows();
// Position-specific features
for t in 0..seq_len {
let label_idx = *label_to_idx.get(&labels[t]).unwrap();
let base_idx = t * n_features * n_labels + label_idx * n_features;
for f in 0..n_features {
features[base_idx + f] = sequence[[t, f]];
}
}
// Transition features
let transition_base = max_length * n_features * n_labels;
for t in 0..(seq_len - 1) {
let curr_label = *label_to_idx.get(&labels[t]).unwrap();
let next_label = *label_to_idx.get(&labels[t + 1]).unwrap();
let transition_idx = transition_base + curr_label * n_labels + next_label;
features[transition_idx] = 1.0;
}
Ok(features)
}
/// Predict sequence labels using current weights
fn predict_sequence(
&self,
sequence: &ArrayView2<Float>,
weights: &Array1<Float>,
max_length: usize,
n_features: usize,
n_labels: usize,
label_to_idx: &HashMap<i32, usize>,
idx_to_label: &HashMap<usize, i32>,
) -> SklResult<Vec<i32>> {
let seq_len = sequence.nrows();
// Simple greedy prediction for each position
let mut predictions = Vec::with_capacity(seq_len);
for t in 0..seq_len {
let mut best_score = Float::NEG_INFINITY;
let mut best_label = 0;
for label_idx in 0..n_labels {
let base_idx = t * n_features * n_labels + label_idx * n_features;
let mut score = 0.0;
for f in 0..n_features {
score += weights[base_idx + f] * sequence[[t, f]];
}
// Add transition score if not first position
if t > 0 && !predictions.is_empty() {
let prev_label_idx = *label_to_idx.get(&predictions[t - 1]).unwrap();
let transition_base = max_length * n_features * n_labels;
let transition_idx = transition_base + prev_label_idx * n_labels + label_idx;
score += weights[transition_idx];
}
if score > best_score {
best_score = score;
best_label = label_idx;
}
}
predictions.push(*idx_to_label.get(&best_label).unwrap());
}
Ok(predictions)
}
}
impl Predict<Array3<Float>, Array2<i32>> for StructuredPerceptron<StructuredPerceptronTrained> {
fn predict(&self, X: &Array3<Float>) -> SklResult<Array2<i32>> {
let (n_sequences, max_length, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features does not match training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_sequences, max_length));
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
// Predict sequence
let pred_labels = self.predict_sequence_trained(&sequence)?;
for (t, &label) in pred_labels.iter().enumerate() {
if t < max_length {
predictions[[seq_idx, t]] = label;
}
}
}
Ok(predictions)
}
}
impl StructuredPerceptron<StructuredPerceptronTrained> {
/// Predict sequence using trained weights
fn predict_sequence_trained(&self, sequence: &ArrayView2<Float>) -> SklResult<Vec<i32>> {
let seq_len = sequence.nrows();
let mut predictions = Vec::with_capacity(seq_len);
for t in 0..seq_len {
let mut best_score = Float::NEG_INFINITY;
let mut best_label = 0;
for label_idx in 0..self.state.n_labels {
let base_idx =
t * sequence.ncols() * self.state.n_labels + label_idx * sequence.ncols();
let mut score = 0.0;
for f in 0..sequence.ncols() {
if base_idx + f < self.state.weights.len() {
score += self.state.weights[base_idx + f] * sequence[[t, f]];
}
}
if score > best_score {
best_score = score;
best_label = label_idx;
}
}
predictions.push(*self.state.idx_to_label.get(&best_label).unwrap());
}
Ok(predictions)
}
/// Get learned weights
pub fn weights(&self) -> &Array1<Float> {
&self.state.weights
}
}
/// Hidden Markov Model for sequence modeling
///
/// A statistical model that assumes the system being modeled is a Markov process
/// with unobserved (hidden) states. HMMs are particularly useful for temporal
/// pattern recognition such as speech recognition, handwriting recognition,
/// gesture recognition, part-of-speech tagging, and bioinformatics.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::HiddenMarkovModel;
/// use ndarray::array;
///
/// let X = array![[[1.0, 2.0], [2.0, 3.0], [1.5, 2.5]]]; // One sequence
/// let y = array![[0, 1, 0]]; // State sequence
///
/// let hmm = HiddenMarkovModel::new().n_states(2).max_iterations(100);
/// let trained_hmm = hmm.fit(&X, &y).unwrap();
/// let predictions = trained_hmm.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct HiddenMarkovModel<State = Untrained> {
n_states: usize,
max_iterations: usize,
tolerance: Float,
random_state: Option<u64>,
state: State,
}
/// Trained state for Hidden Markov Model
#[derive(Debug, Clone)]
pub struct HiddenMarkovModelTrained {
transition_matrix: Array2<Float>,
emission_means: Array2<Float>,
emission_covariances: Array3<Float>,
initial_probabilities: Array1<Float>,
n_states: usize,
n_features: usize,
}
impl HiddenMarkovModel<Untrained> {
/// Create new HMM instance
pub fn new() -> Self {
Self {
n_states: 2,
max_iterations: 100,
tolerance: 1e-6,
random_state: None,
state: Untrained,
}
}
/// Set number of hidden states
pub fn n_states(mut self, n_states: usize) -> Self {
self.n_states = n_states;
self
}
/// Set maximum iterations for EM algorithm
pub fn max_iterations(mut self, max_iter: usize) -> Self {
self.max_iterations = max_iter;
self
}
/// Set convergence tolerance
pub fn tolerance(mut self, tol: Float) -> Self {
self.tolerance = tol;
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, seed: Option<u64>) -> Self {
self.random_state = seed;
self
}
}
impl Estimator for HiddenMarkovModel<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array3<Float>, Array2<i32>> for HiddenMarkovModel<Untrained> {
type Fitted = HiddenMarkovModel<HiddenMarkovModelTrained>;
fn fit(self, X: &Array3<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_sequences, max_length, n_features) = X.dim();
if y.nrows() != n_sequences {
return Err(SklearsError::InvalidInput(
"Number of sequences in X and y must match".to_string(),
));
}
// Initialize parameters
let mut transition_matrix =
Array2::from_elem((self.n_states, self.n_states), 1.0 / self.n_states as Float);
let mut emission_means = Array2::zeros((self.n_states, n_features));
let mut emission_covariances = Array3::zeros((self.n_states, n_features, n_features));
let mut initial_probabilities =
Array1::from_elem(self.n_states, 1.0 / self.n_states as Float);
// Initialize emission parameters using supervised data
self.initialize_emission_parameters(
&X.view(),
&y.view(),
&mut emission_means,
&mut emission_covariances,
)?;
// EM algorithm
let mut prev_likelihood = Float::NEG_INFINITY;
for _iteration in 0..self.max_iterations {
let mut total_likelihood = 0.0;
// E-step: Forward-backward algorithm for each sequence
let mut gamma_sum = Array2::<Float>::zeros((self.n_states, n_features));
let mut xi_sum = Array2::<Float>::zeros((self.n_states, self.n_states));
let mut initial_sum = Array1::<Float>::zeros(self.n_states);
let mut total_gamma = Array1::<Float>::zeros(self.n_states);
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
let seq_len = self.get_sequence_length(&y.row(seq_idx));
if seq_len == 0 {
continue;
}
let obs_seq = sequence.slice(s![..seq_len, ..]);
// Forward-backward
let (alpha, beta, likelihood) = self.forward_backward(
&obs_seq,
&transition_matrix,
&emission_means,
&emission_covariances,
&initial_probabilities,
)?;
total_likelihood += likelihood;
// Compute gamma (state probabilities)
let gamma = self.compute_gamma(&alpha, &beta, likelihood);
// Compute xi (transition probabilities)
let xi = self.compute_xi(
&obs_seq,
&alpha,
&beta,
&transition_matrix,
&emission_means,
&emission_covariances,
likelihood,
)?;
// Accumulate statistics
initial_sum = initial_sum + &gamma.row(0);
for t in 0..seq_len {
for i in 0..self.n_states {
total_gamma[i] += gamma[[t, i]];
for j in 0..n_features {
gamma_sum[[i, j]] += gamma[[t, i]] * obs_seq[[t, j]];
}
}
}
for t in 0..(seq_len - 1) {
for i in 0..self.n_states {
for j in 0..self.n_states {
xi_sum[[i, j]] += xi[[t, i, j]];
}
}
}
}
// M-step: Update parameters
// Update initial probabilities
let initial_sum_total = initial_sum.sum();
if initial_sum_total > 0.0 {
initial_probabilities = initial_sum / initial_sum_total;
}
// Update transition matrix
for i in 0..self.n_states {
let row_sum = xi_sum.row(i).sum();
if row_sum > 0.0 {
for j in 0..self.n_states {
transition_matrix[[i, j]] = xi_sum[[i, j]] / row_sum;
}
}
}
// Update emission means
for i in 0..self.n_states {
if total_gamma[i] > 0.0 {
for j in 0..n_features {
emission_means[[i, j]] = gamma_sum[[i, j]] / total_gamma[i];
}
}
}
// Update emission covariances (simplified as diagonal)
for i in 0..self.n_states {
for j in 0..n_features {
emission_covariances[[i, j, j]] = 1.0; // Simplified identity covariance
}
}
// Check convergence
if (total_likelihood - prev_likelihood).abs() < self.tolerance {
break;
}
prev_likelihood = total_likelihood;
}
Ok(HiddenMarkovModel {
n_states: self.n_states,
max_iterations: self.max_iterations,
tolerance: self.tolerance,
random_state: self.random_state,
state: HiddenMarkovModelTrained {
transition_matrix,
emission_means,
emission_covariances,
initial_probabilities,
n_states: self.n_states,
n_features,
},
})
}
}
impl HiddenMarkovModel<Untrained> {
/// Initialize emission parameters using supervised data
fn initialize_emission_parameters(
&self,
X: &ArrayView3<Float>,
y: &ArrayView2<i32>,
emission_means: &mut Array2<Float>,
emission_covariances: &mut Array3<Float>,
) -> SklResult<()> {
let (n_sequences, _max_length, n_features) = X.dim();
// Compute means for each state
let mut state_counts = vec![0; self.n_states];
let mut state_sums = Array2::<Float>::zeros((self.n_states, n_features));
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
let labels = y.row(seq_idx);
let seq_len = self.get_sequence_length(&labels);
for t in 0..seq_len {
let state = labels[t] as usize;
if state < self.n_states {
state_counts[state] += 1;
for f in 0..n_features {
state_sums[[state, f]] += sequence[[t, f]];
}
}
}
}
// Compute means
for i in 0..self.n_states {
if state_counts[i] > 0 {
for j in 0..n_features {
emission_means[[i, j]] = state_sums[[i, j]] / state_counts[i] as Float;
}
}
}
// Initialize covariances as identity matrices
for i in 0..self.n_states {
for j in 0..n_features {
emission_covariances[[i, j, j]] = 1.0;
}
}
Ok(())
}
/// Get effective sequence length (stop at -1 padding)
fn get_sequence_length(&self, labels: &ArrayView1<i32>) -> usize {
labels.iter().position(|&x| x == -1).unwrap_or(labels.len())
}
/// Forward-backward algorithm
fn forward_backward(
&self,
sequence: &ArrayView2<Float>,
transition_matrix: &Array2<Float>,
emission_means: &Array2<Float>,
emission_covariances: &Array3<Float>,
initial_probabilities: &Array1<Float>,
) -> SklResult<(Array2<Float>, Array2<Float>, Float)> {
let seq_len = sequence.nrows();
let mut alpha = Array2::zeros((seq_len, self.n_states));
let mut beta = Array2::zeros((seq_len, self.n_states));
// Forward pass
for i in 0..self.n_states {
let emission_prob = self.gaussian_pdf(
&sequence.row(0),
&emission_means.row(i),
&emission_covariances.slice(s![i, .., ..]),
);
alpha[[0, i]] = initial_probabilities[i] * emission_prob;
}
for t in 1..seq_len {
for j in 0..self.n_states {
let mut sum = 0.0;
for i in 0..self.n_states {
sum += alpha[[t - 1, i]] * transition_matrix[[i, j]];
}
let emission_prob = self.gaussian_pdf(
&sequence.row(t),
&emission_means.row(j),
&emission_covariances.slice(s![j, .., ..]),
);
alpha[[t, j]] = sum * emission_prob;
}
}
// Compute likelihood
let likelihood = alpha.row(seq_len - 1).sum();
// Backward pass
for i in 0..self.n_states {
beta[[seq_len - 1, i]] = 1.0;
}
for t in (0..(seq_len - 1)).rev() {
for i in 0..self.n_states {
let mut sum = 0.0;
for j in 0..self.n_states {
let emission_prob = self.gaussian_pdf(
&sequence.row(t + 1),
&emission_means.row(j),
&emission_covariances.slice(s![j, .., ..]),
);
sum += transition_matrix[[i, j]] * emission_prob * beta[[t + 1, j]];
}
beta[[t, i]] = sum;
}
}
Ok((alpha, beta, likelihood))
}
/// Compute gamma (state probabilities)
fn compute_gamma(
&self,
alpha: &Array2<Float>,
beta: &Array2<Float>,
likelihood: Float,
) -> Array2<Float> {
let (seq_len, n_states) = alpha.dim();
let mut gamma = Array2::zeros((seq_len, n_states));
for t in 0..seq_len {
for i in 0..n_states {
gamma[[t, i]] = alpha[[t, i]] * beta[[t, i]] / likelihood;
}
}
gamma
}
/// Compute xi (transition probabilities)
fn compute_xi(
&self,
sequence: &ArrayView2<Float>,
alpha: &Array2<Float>,
beta: &Array2<Float>,
transition_matrix: &Array2<Float>,
emission_means: &Array2<Float>,
emission_covariances: &Array3<Float>,
likelihood: Float,
) -> SklResult<Array3<Float>> {
let seq_len = sequence.nrows();
let mut xi = Array3::zeros((seq_len - 1, self.n_states, self.n_states));
for t in 0..(seq_len - 1) {
for i in 0..self.n_states {
for j in 0..self.n_states {
let emission_prob = self.gaussian_pdf(
&sequence.row(t + 1),
&emission_means.row(j),
&emission_covariances.slice(s![j, .., ..]),
);
xi[[t, i, j]] = alpha[[t, i]]
* transition_matrix[[i, j]]
* emission_prob
* beta[[t + 1, j]]
/ likelihood;
}
}
}
Ok(xi)
}
/// Gaussian PDF for emission probabilities
fn gaussian_pdf(
&self,
x: &ArrayView1<Float>,
mean: &ArrayView1<Float>,
_covariance: &ArrayView2<Float>,
) -> Float {
// Simplified Gaussian PDF (assuming diagonal covariance = identity)
let diff = x - mean;
let exp_term = -0.5 * diff.dot(&diff);
exp_term.exp()
}
}
impl Predict<Array3<Float>, Array2<i32>> for HiddenMarkovModel<HiddenMarkovModelTrained> {
fn predict(&self, X: &Array3<Float>) -> SklResult<Array2<i32>> {
let (n_sequences, max_length, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features does not match training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_sequences, max_length));
for seq_idx in 0..n_sequences {
let sequence = X.slice(s![seq_idx, .., ..]);
// Viterbi decoding for most likely state sequence
let state_path = self.viterbi_decode(&sequence)?;
for (t, &state) in state_path.iter().enumerate() {
if t < max_length {
predictions[[seq_idx, t]] = state as i32;
}
}
}
Ok(predictions)
}
}
impl HiddenMarkovModel<HiddenMarkovModelTrained> {
/// Viterbi algorithm for finding most likely state sequence
fn viterbi_decode(&self, sequence: &ArrayView2<Float>) -> SklResult<Vec<usize>> {
let seq_len = sequence.nrows();
let mut viterbi = Array2::zeros((seq_len, self.state.n_states));
let mut backpointer = Array2::zeros((seq_len, self.state.n_states));
// Initialize
for i in 0..self.state.n_states {
let emission_prob =
self.gaussian_pdf(&sequence.row(0), &self.state.emission_means.row(i));
viterbi[[0, i]] = self.state.initial_probabilities[i] * emission_prob;
}
// Forward pass
for t in 1..seq_len {
for j in 0..self.state.n_states {
let mut best_score = 0.0;
let mut best_prev = 0;
for i in 0..self.state.n_states {
let score = viterbi[[t - 1, i]] * self.state.transition_matrix[[i, j]];
if score > best_score {
best_score = score;
best_prev = i;
}
}
let emission_prob =
self.gaussian_pdf(&sequence.row(t), &self.state.emission_means.row(j));
viterbi[[t, j]] = best_score * emission_prob;
backpointer[[t, j]] = best_prev as Float;
}
}
// Find best final state
let mut best_final_score = 0.0;
let mut best_final_state = 0;
for i in 0..self.state.n_states {
if viterbi[[seq_len - 1, i]] > best_final_score {
best_final_score = viterbi[[seq_len - 1, i]];
best_final_state = i;
}
}
// Backtrack
let mut path = vec![0; seq_len];
path[seq_len - 1] = best_final_state;
for t in (1..seq_len).rev() {
path[t - 1] = backpointer[[t, path[t]]] as usize;
}
Ok(path)
}
/// Simplified Gaussian PDF
fn gaussian_pdf(&self, x: &ArrayView1<Float>, mean: &ArrayView1<Float>) -> Float {
let diff = x - mean;
let exp_term = -0.5 * diff.dot(&diff);
exp_term.exp()
}
/// Get transition matrix
pub fn transition_matrix(&self) -> &Array2<Float> {
&self.state.transition_matrix
}
/// Get emission means
pub fn emission_means(&self) -> &Array2<Float> {
&self.state.emission_means
}
/// Get initial probabilities
pub fn initial_probabilities(&self) -> &Array1<Float> {
&self.state.initial_probabilities
}
}
/// Log-sum-exp trick for numerical stability
fn log_sum_exp(a: Float, b: Float) -> Float {
if a > b {
a + (b - a).exp().ln_1p()
} else {
b + (a - b).exp().ln_1p()
}
}
// =============================================================================
// HIERARCHICAL PREDICTION METHODS
// =============================================================================
/// Hierarchical Multi-Label Classification
///
/// A multi-label classification approach that leverages hierarchical relationships
/// between labels. This method enforces consistency in the label hierarchy by
/// ensuring that if a child label is predicted, its parent labels are also predicted.
/// Particularly useful for taxonomic classification, gene ontology annotation,
/// and other domains with natural label hierarchies.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::HierarchicalMultiLabelClassifier;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
/// let y = array![[1, 0, 1, 0], [0, 1, 0, 1], [1, 1, 0, 0]]; // 4 labels in hierarchy
///
/// // Define hierarchy: child -> parent relationships
/// let hierarchy = vec![(2, 0), (3, 1)]; // label 2 is child of 0, label 3 is child of 1
///
/// let hmlc = HierarchicalMultiLabelClassifier::new()
/// .hierarchy(hierarchy)
/// .violation_penalty(1.0);
/// let trained_hmlc = hmlc.fit(&X, &y).unwrap();
/// let predictions = trained_hmlc.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct HierarchicalMultiLabelClassifier<State = Untrained> {
hierarchy: Vec<(usize, usize)>, // (child, parent) pairs
violation_penalty: Float,
base_classifier_type: String,
consistency_enforcement: ConsistencyEnforcement,
state: State,
}
/// Methods for enforcing hierarchical consistency
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConsistencyEnforcement {
/// Post-process predictions to enforce consistency
PostProcessing,
/// Include hierarchy constraints in loss function
ConstrainedTraining,
/// Use Bayesian inference with hierarchy priors
BayesianInference,
}
/// Trained state for Hierarchical Multi-Label Classifier
#[derive(Debug, Clone)]
pub struct HierarchicalMultiLabelClassifierTrained {
binary_classifiers: Vec<BinaryRelevance<BinaryRelevanceTrained>>,
hierarchy: Vec<(usize, usize)>,
hierarchy_matrix: Array2<Float>,
n_labels: usize,
violation_penalty: Float,
consistency_enforcement: ConsistencyEnforcement,
}
impl HierarchicalMultiLabelClassifier<Untrained> {
/// Create new hierarchical multi-label classifier
pub fn new() -> Self {
Self {
hierarchy: Vec::new(),
violation_penalty: 1.0,
base_classifier_type: "binary_relevance".to_string(),
consistency_enforcement: ConsistencyEnforcement::PostProcessing,
state: Untrained,
}
}
/// Set label hierarchy as (child, parent) pairs
pub fn hierarchy(mut self, hierarchy: Vec<(usize, usize)>) -> Self {
self.hierarchy = hierarchy;
self
}
/// Set penalty for hierarchy violations
pub fn violation_penalty(mut self, penalty: Float) -> Self {
self.violation_penalty = penalty;
self
}
/// Set consistency enforcement method
pub fn consistency_enforcement(mut self, method: ConsistencyEnforcement) -> Self {
self.consistency_enforcement = method;
self
}
/// Set base classifier type
pub fn base_classifier_type(mut self, classifier_type: String) -> Self {
self.base_classifier_type = classifier_type;
self
}
}
impl Estimator for HierarchicalMultiLabelClassifier<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array2<Float>, Array2<i32>> for HierarchicalMultiLabelClassifier<Untrained> {
type Fitted = HierarchicalMultiLabelClassifier<HierarchicalMultiLabelClassifierTrained>;
fn fit(self, X: &Array2<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, _n_features) = X.dim();
let (y_samples, n_labels) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"Number of samples in X and y must match".to_string(),
));
}
// Build hierarchy matrix (n_labels x n_labels)
let mut hierarchy_matrix = Array2::zeros((n_labels, n_labels));
for &(child, parent) in &self.hierarchy {
if child < n_labels && parent < n_labels {
hierarchy_matrix[[child, parent]] = 1.0;
}
}
// Add transitive closure for hierarchy
for k in 0..n_labels {
for i in 0..n_labels {
for j in 0..n_labels {
if hierarchy_matrix[[i, k]] > 0.0 && hierarchy_matrix[[k, j]] > 0.0 {
hierarchy_matrix[[i, j]] = 1.0;
}
}
}
}
// Train binary classifiers for each label
let mut binary_classifiers = Vec::new();
for label_idx in 0..n_labels {
// Create modified training data based on consistency enforcement
let y_modified = match self.consistency_enforcement {
ConsistencyEnforcement::ConstrainedTraining => self
.apply_hierarchy_constraints_training(
&y.view(),
label_idx,
&hierarchy_matrix,
)?,
_ => y.column(label_idx).to_owned(),
};
let binary_classifier = BinaryRelevance::new();
let y_2d = y_modified.insert_axis(ndarray::Axis(1));
let trained_classifier = binary_classifier.fit(&X.view(), &y_2d)?;
binary_classifiers.push(trained_classifier);
}
Ok(HierarchicalMultiLabelClassifier {
hierarchy: self.hierarchy.clone(),
violation_penalty: self.violation_penalty,
base_classifier_type: self.base_classifier_type.clone(),
consistency_enforcement: self.consistency_enforcement,
state: HierarchicalMultiLabelClassifierTrained {
binary_classifiers,
hierarchy: self.hierarchy,
hierarchy_matrix,
n_labels,
violation_penalty: self.violation_penalty,
consistency_enforcement: self.consistency_enforcement.clone(),
},
})
}
}
impl HierarchicalMultiLabelClassifier<Untrained> {
/// Apply hierarchy constraints during training
fn apply_hierarchy_constraints_training(
&self,
y: &ArrayView2<i32>,
label_idx: usize,
hierarchy_matrix: &Array2<Float>,
) -> SklResult<Array1<i32>> {
let n_samples = y.nrows();
let mut y_modified = y.column(label_idx).to_owned();
for sample_idx in 0..n_samples {
let mut labels = y.row(sample_idx).to_owned();
// Enforce hierarchy: if child is positive, parent must be positive
for i in 0..labels.len() {
if labels[i] == 1 {
for j in 0..labels.len() {
if hierarchy_matrix[[i, j]] > 0.0 {
labels[j] = 1;
}
}
}
}
y_modified[sample_idx] = labels[label_idx];
}
Ok(y_modified)
}
}
impl Predict<Array2<Float>, Array2<i32>>
for HierarchicalMultiLabelClassifier<HierarchicalMultiLabelClassifierTrained>
{
fn predict(&self, X: &Array2<Float>) -> SklResult<Array2<i32>> {
let n_samples = X.nrows();
let mut predictions = Array2::zeros((n_samples, self.state.n_labels));
// Get raw predictions from binary classifiers
for (label_idx, classifier) in self.state.binary_classifiers.iter().enumerate() {
let label_predictions = classifier.predict(&X.view())?;
for sample_idx in 0..n_samples {
predictions[[sample_idx, label_idx]] = label_predictions[[sample_idx, 0]];
}
}
// Apply consistency enforcement
match self.state.consistency_enforcement {
ConsistencyEnforcement::PostProcessing => {
self.apply_hierarchy_consistency_post_processing(&mut predictions)?;
}
ConsistencyEnforcement::BayesianInference => {
let probabilities = self.predict_proba(&X.view())?;
predictions = self.apply_bayesian_hierarchy_inference(&probabilities)?;
}
_ => {} // Already handled during training
}
Ok(predictions)
}
}
impl HierarchicalMultiLabelClassifier<HierarchicalMultiLabelClassifierTrained> {
/// Apply hierarchy consistency via post-processing
fn apply_hierarchy_consistency_post_processing(
&self,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
let n_samples = predictions.nrows();
for sample_idx in 0..n_samples {
let mut changed = true;
while changed {
changed = false;
// Bottom-up propagation: if child is positive, make parent positive
for &(child, parent) in &self.state.hierarchy {
if predictions[[sample_idx, child]] == 1
&& predictions[[sample_idx, parent]] == 0
{
predictions[[sample_idx, parent]] = 1;
changed = true;
}
}
// Top-down propagation: if parent is negative, make children negative
for &(child, parent) in &self.state.hierarchy {
if predictions[[sample_idx, parent]] == 0
&& predictions[[sample_idx, child]] == 1
{
predictions[[sample_idx, child]] = 0;
changed = true;
}
}
}
}
Ok(())
}
/// Apply Bayesian hierarchy inference
fn apply_bayesian_hierarchy_inference(
&self,
probabilities: &Array2<Float>,
) -> SklResult<Array2<i32>> {
let (n_samples, n_labels) = probabilities.dim();
let mut predictions = Array2::zeros((n_samples, n_labels));
for sample_idx in 0..n_samples {
let probs = probabilities.row(sample_idx);
// Use dynamic programming to find optimal consistent labeling
let optimal_labels = self.find_optimal_consistent_labeling(&probs)?;
for (label_idx, &label) in optimal_labels.iter().enumerate() {
predictions[[sample_idx, label_idx]] = label;
}
}
Ok(predictions)
}
/// Find optimal consistent labeling using dynamic programming
fn find_optimal_consistent_labeling(
&self,
probabilities: &ArrayView1<Float>,
) -> SklResult<Vec<i32>> {
let n_labels = probabilities.len();
let mut best_score = Float::NEG_INFINITY;
let mut best_labeling = vec![0; n_labels];
// Enumerate all possible labelings (2^n_labels)
for labeling_int in 0..(1 << n_labels) {
let mut labeling = vec![0; n_labels];
for i in 0..n_labels {
labeling[i] = if (labeling_int >> i) & 1 == 1 { 1 } else { 0 };
}
// Check hierarchy consistency
let mut is_consistent = true;
for &(child, parent) in &self.state.hierarchy {
if labeling[child] == 1 && labeling[parent] == 0 {
is_consistent = false;
break;
}
}
if is_consistent {
// Compute score
let mut score = 0.0;
for i in 0..n_labels {
if labeling[i] == 1 {
score += probabilities[i].ln();
} else {
score += (1.0 - probabilities[i]).ln();
}
}
if score > best_score {
best_score = score;
best_labeling = labeling;
}
}
}
Ok(best_labeling)
}
/// Predict class probabilities
pub fn predict_proba(&self, X: &ArrayView2<Float>) -> SklResult<Array2<Float>> {
let n_samples = X.nrows();
let mut probabilities = Array2::zeros((n_samples, self.state.n_labels));
for (label_idx, classifier) in self.state.binary_classifiers.iter().enumerate() {
let label_probabilities = classifier.predict_proba(X)?;
for sample_idx in 0..n_samples {
probabilities[[sample_idx, label_idx]] = label_probabilities[[sample_idx, 1]];
// Probability of positive class
}
}
Ok(probabilities)
}
/// Get hierarchy matrix
pub fn hierarchy_matrix(&self) -> &Array2<Float> {
&self.state.hierarchy_matrix
}
/// Get hierarchy relationships
pub fn hierarchy(&self) -> &Vec<(usize, usize)> {
&self.state.hierarchy
}
}
/// Tree-Structured Output Prediction
///
/// A structured prediction method for outputs that follow a tree structure.
/// This is useful for tasks like parse tree prediction, decision tree paths,
/// or any hierarchical classification where the output space has a tree structure.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::TreeStructuredPredictor;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0]];
/// let y = array![[0, 1, 2], [1, 0, 2]]; // Tree paths as sequences
///
/// let tree_predictor = TreeStructuredPredictor::new()
/// .max_depth(3)
/// .branching_factor(3);
/// let trained_predictor = tree_predictor.fit(&X, &y).unwrap();
/// let predictions = trained_predictor.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct TreeStructuredPredictor<State = Untrained> {
max_depth: usize,
branching_factor: usize,
tree_structure: Vec<Vec<usize>>, // Adjacency list representation
node_classifiers: HashMap<usize, String>,
state: State,
}
/// Trained state for Tree-Structured Predictor
#[derive(Debug, Clone)]
pub struct TreeStructuredPredictorTrained {
node_classifiers: HashMap<usize, BinaryRelevance<BinaryRelevanceTrained>>,
tree_structure: Vec<Vec<usize>>,
max_depth: usize,
n_nodes: usize,
}
impl TreeStructuredPredictor<Untrained> {
/// Create new tree-structured predictor
pub fn new() -> Self {
Self {
max_depth: 5,
branching_factor: 2,
tree_structure: Vec::new(),
node_classifiers: HashMap::new(),
state: Untrained,
}
}
/// Set maximum tree depth
pub fn max_depth(mut self, depth: usize) -> Self {
self.max_depth = depth;
self
}
/// Set branching factor
pub fn branching_factor(mut self, factor: usize) -> Self {
self.branching_factor = factor;
self
}
/// Set custom tree structure
pub fn tree_structure(mut self, structure: Vec<Vec<usize>>) -> Self {
self.tree_structure = structure;
self
}
}
impl Estimator for TreeStructuredPredictor<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array2<Float>, Array2<i32>> for TreeStructuredPredictor<Untrained> {
type Fitted = TreeStructuredPredictor<TreeStructuredPredictorTrained>;
fn fit(self, X: &Array2<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, _n_features) = X.dim();
let (y_samples, max_path_length) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"Number of samples in X and y must match".to_string(),
));
}
// Build tree structure if not provided
let tree_structure = if self.tree_structure.is_empty() {
self.build_default_tree_structure()?
} else {
self.tree_structure.clone()
};
let n_nodes = tree_structure.len();
let mut node_classifiers = HashMap::new();
// Train classifier for each internal node
for node_id in 0..n_nodes {
if !tree_structure[node_id].is_empty() {
// Internal node
// Create binary classification data for this node
let (node_X, node_y) = self.create_node_training_data(
&X.view(),
&y.view(),
node_id,
&tree_structure,
max_path_length,
)?;
if node_y.len() > 0 {
let classifier = BinaryRelevance::new();
let trained_classifier = classifier.fit(&node_X.view(), &node_y)?;
node_classifiers.insert(node_id, trained_classifier);
}
}
}
Ok(TreeStructuredPredictor {
max_depth: self.max_depth,
branching_factor: self.branching_factor,
tree_structure: tree_structure.clone(),
node_classifiers: HashMap::new(),
state: TreeStructuredPredictorTrained {
node_classifiers,
tree_structure,
max_depth: self.max_depth,
n_nodes,
},
})
}
}
impl TreeStructuredPredictor<Untrained> {
/// Build default tree structure
fn build_default_tree_structure(&self) -> SklResult<Vec<Vec<usize>>> {
let mut total_nodes = 0;
for depth in 0..self.max_depth {
total_nodes += self.branching_factor.pow(depth as u32);
}
let mut tree_structure = vec![Vec::new(); total_nodes];
let mut node_id = 0;
// Build complete tree
for depth in 0..(self.max_depth - 1) {
let nodes_at_depth = self.branching_factor.pow(depth as u32);
for _ in 0..nodes_at_depth {
for child in 0..self.branching_factor {
let child_id = node_id + nodes_at_depth + child;
if child_id < total_nodes {
tree_structure[node_id].push(child_id);
}
}
node_id += 1;
}
}
Ok(tree_structure)
}
/// Create training data for a specific node
fn create_node_training_data(
&self,
X: &ArrayView2<Float>,
y: &ArrayView2<i32>,
node_id: usize,
tree_structure: &Vec<Vec<usize>>,
max_path_length: usize,
) -> SklResult<(Array2<Float>, Array2<i32>)> {
let n_samples = X.nrows();
let mut valid_samples = Vec::new();
let mut node_labels = Vec::new();
for sample_idx in 0..n_samples {
let path = y.row(sample_idx);
// Check if this sample's path passes through this node
for pos in 0..max_path_length {
if path[pos] as usize == node_id && pos + 1 < max_path_length {
// Sample passes through this node, determine which child to predict
let next_node = path[pos + 1] as usize;
// Find which child index this corresponds to
if let Some(child_idx) = tree_structure[node_id]
.iter()
.position(|&child| child == next_node)
{
valid_samples.push(sample_idx);
node_labels.push(child_idx as i32);
break;
}
}
}
}
// Create training data for this node
let n_valid = valid_samples.len();
if n_valid == 0 {
return Ok((Array2::zeros((0, X.ncols())), Array2::zeros((0, 1))));
}
let mut node_X = Array2::zeros((n_valid, X.ncols()));
let mut node_y = Array2::zeros((n_valid, 1));
for (i, &sample_idx) in valid_samples.iter().enumerate() {
for j in 0..X.ncols() {
node_X[[i, j]] = X[[sample_idx, j]];
}
node_y[[i, 0]] = node_labels[i];
}
Ok((node_X, node_y))
}
}
impl Predict<Array2<Float>, Array2<i32>>
for TreeStructuredPredictor<TreeStructuredPredictorTrained>
{
fn predict(&self, X: &Array2<Float>) -> SklResult<Array2<i32>> {
let n_samples = X.nrows();
let mut predictions = Array2::zeros((n_samples, self.state.max_depth));
for sample_idx in 0..n_samples {
let sample = X.row(sample_idx);
let path = self.predict_tree_path(&sample)?;
for (pos, &node) in path.iter().enumerate() {
if pos < self.state.max_depth {
predictions[[sample_idx, pos]] = node as i32;
}
}
}
Ok(predictions)
}
}
impl TreeStructuredPredictor<TreeStructuredPredictorTrained> {
/// Predict tree path for a single sample
fn predict_tree_path(&self, sample: &ArrayView1<Float>) -> SklResult<Vec<usize>> {
let mut path = Vec::new();
let mut current_node = 0; // Start at root
path.push(current_node);
while !self.state.tree_structure[current_node].is_empty() {
// Internal node - predict which child to take
if let Some(classifier) = self.state.node_classifiers.get(¤t_node) {
let sample_2d = sample.to_owned().insert_axis(ndarray::Axis(0));
let prediction = classifier.predict(&sample_2d.view())?;
let child_idx = prediction[[0, 0]] as usize;
if child_idx < self.state.tree_structure[current_node].len() {
current_node = self.state.tree_structure[current_node][child_idx];
path.push(current_node);
} else {
break; // Invalid prediction
}
} else {
break; // No classifier for this node
}
}
Ok(path)
}
/// Get tree structure
pub fn tree_structure(&self) -> &Vec<Vec<usize>> {
&self.state.tree_structure
}
}
/// DAG-Structured Prediction
///
/// A structured prediction method for outputs that follow a directed acyclic graph (DAG)
/// structure. This is useful for tasks like workflow prediction, dependency parsing,
/// or any classification where the output space has a DAG structure with multiple paths.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::DAGStructuredPredictor;
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0]];
/// let y = array![[1, 1, 0, 1], [0, 1, 1, 0]]; // DAG node activations
///
/// let dag_predictor = DAGStructuredPredictor::new()
/// .add_edge(0, 2)
/// .add_edge(1, 2)
/// .add_edge(2, 3);
/// let trained_predictor = dag_predictor.fit(&X, &y).unwrap();
/// let predictions = trained_predictor.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct DAGStructuredPredictor<State = Untrained> {
dag_edges: Vec<(usize, usize)>, // (parent, child) pairs
topological_order: Vec<usize>,
inference_method: DAGInferenceMethod,
state: State,
}
/// Methods for DAG inference
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DAGInferenceMethod {
/// Greedy inference following topological order
Greedy,
/// Belief propagation on the DAG
BeliefPropagation,
/// Integer linear programming for exact inference
ExactILP,
}
/// Trained state for DAG-Structured Predictor
#[derive(Debug, Clone)]
pub struct DAGStructuredPredictorTrained {
node_classifiers: HashMap<usize, BinaryRelevance<BinaryRelevanceTrained>>,
dag_edges: Vec<(usize, usize)>,
topological_order: Vec<usize>,
adjacency_matrix: Array2<Float>,
n_nodes: usize,
inference_method: DAGInferenceMethod,
}
impl DAGStructuredPredictor<Untrained> {
/// Create new DAG-structured predictor
pub fn new() -> Self {
Self {
dag_edges: Vec::new(),
topological_order: Vec::new(),
inference_method: DAGInferenceMethod::Greedy,
state: Untrained,
}
}
/// Add edge to DAG structure
pub fn add_edge(mut self, parent: usize, child: usize) -> Self {
self.dag_edges.push((parent, child));
self
}
/// Set DAG edges
pub fn dag_edges(mut self, edges: Vec<(usize, usize)>) -> Self {
self.dag_edges = edges;
self
}
/// Set inference method
pub fn inference_method(mut self, method: DAGInferenceMethod) -> Self {
self.inference_method = method;
self
}
}
impl Estimator for DAGStructuredPredictor<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array2<Float>, Array2<i32>> for DAGStructuredPredictor<Untrained> {
type Fitted = DAGStructuredPredictor<DAGStructuredPredictorTrained>;
fn fit(self, X: &Array2<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, _n_features) = X.dim();
let (y_samples, n_nodes) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"Number of samples in X and y must match".to_string(),
));
}
// Build adjacency matrix
let mut adjacency_matrix = Array2::zeros((n_nodes, n_nodes));
for &(parent, child) in &self.dag_edges {
if parent < n_nodes && child < n_nodes {
adjacency_matrix[[parent, child]] = 1.0;
}
}
// Compute topological ordering
let topological_order = self.compute_topological_order(n_nodes)?;
// Train classifier for each node
let mut node_classifiers = HashMap::new();
for &node_id in &topological_order {
// Get parent features for this node
let node_features =
self.create_node_features(&X.view(), &y.view(), node_id, &adjacency_matrix)?;
let node_labels = y.column(node_id).to_owned();
let classifier = BinaryRelevance::new();
let node_labels_2d = node_labels.insert_axis(ndarray::Axis(1));
let trained_classifier = classifier.fit(&node_features.view(), &node_labels_2d)?;
node_classifiers.insert(node_id, trained_classifier);
}
Ok(DAGStructuredPredictor {
dag_edges: self.dag_edges.clone(),
topological_order: topological_order.clone(),
inference_method: self.inference_method,
state: DAGStructuredPredictorTrained {
node_classifiers,
dag_edges: self.dag_edges,
topological_order,
adjacency_matrix,
n_nodes,
inference_method: self.inference_method.clone(),
},
})
}
}
impl DAGStructuredPredictor<Untrained> {
/// Compute topological ordering of DAG
fn compute_topological_order(&self, n_nodes: usize) -> SklResult<Vec<usize>> {
let mut in_degree = vec![0; n_nodes];
let mut adj_list = vec![Vec::new(); n_nodes];
// Build adjacency list and compute in-degrees
for &(parent, child) in &self.dag_edges {
adj_list[parent].push(child);
in_degree[child] += 1;
}
// Kahn's algorithm for topological sorting
let mut queue = Vec::new();
let mut topo_order = Vec::new();
// Add all nodes with in-degree 0 to queue
for node in 0..n_nodes {
if in_degree[node] == 0 {
queue.push(node);
}
}
while let Some(node) = queue.pop() {
topo_order.push(node);
// Update in-degrees of neighbors
for &neighbor in &adj_list[node] {
in_degree[neighbor] -= 1;
if in_degree[neighbor] == 0 {
queue.push(neighbor);
}
}
}
if topo_order.len() != n_nodes {
return Err(SklearsError::InvalidInput(
"DAG contains cycles - not a valid DAG".to_string(),
));
}
Ok(topo_order)
}
/// Create features for a node including parent node information
fn create_node_features(
&self,
X: &ArrayView2<Float>,
y: &ArrayView2<i32>,
node_id: usize,
adjacency_matrix: &Array2<Float>,
) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
// Count parent nodes
let n_parents = adjacency_matrix.column(node_id).sum() as usize;
let feature_size = n_features + n_parents;
let mut node_features = Array2::zeros((n_samples, feature_size));
// Copy original features
for i in 0..n_samples {
for j in 0..n_features {
node_features[[i, j]] = X[[i, j]];
}
}
// Add parent node features
let mut parent_idx = 0;
for parent in 0..adjacency_matrix.nrows() {
if adjacency_matrix[[parent, node_id]] > 0.0 {
for i in 0..n_samples {
node_features[[i, n_features + parent_idx]] = y[[i, parent]] as Float;
}
parent_idx += 1;
}
}
Ok(node_features)
}
}
impl Predict<Array2<Float>, Array2<i32>> for DAGStructuredPredictor<DAGStructuredPredictorTrained> {
fn predict(&self, X: &Array2<Float>) -> SklResult<Array2<i32>> {
let n_samples = X.nrows();
let mut predictions = Array2::zeros((n_samples, self.state.n_nodes));
match self.state.inference_method {
DAGInferenceMethod::Greedy => {
self.greedy_inference(&X.view(), &mut predictions)?;
}
DAGInferenceMethod::BeliefPropagation => {
self.belief_propagation_inference(&X.view(), &mut predictions)?;
}
DAGInferenceMethod::ExactILP => {
// For simplicity, fall back to greedy inference
self.greedy_inference(&X.view(), &mut predictions)?;
}
}
Ok(predictions)
}
}
impl DAGStructuredPredictor<DAGStructuredPredictorTrained> {
/// Greedy inference following topological order
fn greedy_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
let n_samples = X.nrows();
for sample_idx in 0..n_samples {
for &node_id in &self.state.topological_order {
if let Some(classifier) = self.state.node_classifiers.get(&node_id) {
// Create features for this node
let node_features = self.create_prediction_features(
&X.row(sample_idx),
&predictions.row(sample_idx),
node_id,
)?;
let node_prediction =
classifier.predict(&node_features.insert_axis(ndarray::Axis(0)).view())?;
predictions[[sample_idx, node_id]] = node_prediction[[0, 0]];
}
}
}
Ok(())
}
/// Belief propagation inference (simplified)
fn belief_propagation_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
// For simplicity, use greedy inference
// In a full implementation, this would use message passing
self.greedy_inference(X, predictions)
}
/// Create features for prediction including parent predictions
fn create_prediction_features(
&self,
sample: &ArrayView1<Float>,
current_predictions: &ArrayView1<i32>,
node_id: usize,
) -> SklResult<Array1<Float>> {
let n_features = sample.len();
let n_parents = self.state.adjacency_matrix.column(node_id).sum() as usize;
let feature_size = n_features + n_parents;
let mut node_features = Array1::zeros(feature_size);
// Copy original features
for i in 0..n_features {
node_features[i] = sample[i];
}
// Add parent predictions
let mut parent_idx = 0;
for parent in 0..self.state.adjacency_matrix.nrows() {
if self.state.adjacency_matrix[[parent, node_id]] > 0.0 {
node_features[n_features + parent_idx] = current_predictions[parent] as Float;
parent_idx += 1;
}
}
Ok(node_features)
}
/// Get DAG structure
pub fn dag_edges(&self) -> &Vec<(usize, usize)> {
&self.state.dag_edges
}
/// Get topological order
pub fn topological_order(&self) -> &Vec<usize> {
&self.state.topological_order
}
/// Get adjacency matrix
pub fn adjacency_matrix(&self) -> &Array2<Float> {
&self.state.adjacency_matrix
}
}
// =============================================================================
// GRAPH-STRUCTURED OUTPUT PREDICTION METHODS
// =============================================================================
/// Graph-Based Structured Prediction
///
/// A structured prediction method for outputs that have a general graph structure.
/// This supports both directed and undirected graphs and uses message passing
/// algorithms for inference. Particularly useful for social network analysis,
/// molecule prediction, knowledge graph completion, and other tasks where
/// outputs have complex graph relationships.
///
/// # Examples
///
/// ```
/// use sklears_core::traits::{Fit, Predict};
/// use sklears_multioutput::{GraphStructuredPredictor, GraphType};
/// use ndarray::array;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
/// let y = array![[1, 0, 1], [0, 1, 0], [1, 1, 0]]; // Node labels
/// let edges = vec![(0, 1), (1, 2), (0, 2)]; // Graph edges
///
/// let graph_predictor = GraphStructuredPredictor::new()
/// .graph_type(GraphType::Undirected)
/// .edges(edges)
/// .message_passing_iterations(10);
/// let trained_predictor = graph_predictor.fit(&X, &y).unwrap();
/// let predictions = trained_predictor.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct GraphStructuredPredictor<State = Untrained> {
graph_type: GraphType,
edges: Vec<(usize, usize)>,
message_passing_iterations: usize,
inference_method: GraphInferenceMethod,
node_feature_dim: usize,
edge_feature_dim: usize,
state: State,
}
/// Types of graphs supported
#[derive(Debug, Clone)]
pub enum GraphType {
/// Undirected graph
Undirected,
/// Directed graph
Directed,
/// Bipartite graph
Bipartite,
}
/// Methods for graph inference
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GraphInferenceMethod {
/// Message passing with belief propagation
MessagePassing,
/// Graph neural network approach
GraphNeuralNetwork,
/// Variational inference
VariationalInference,
/// Spectral methods
SpectralMethods,
}
/// Trained state for Graph-Structured Predictor
#[derive(Debug, Clone)]
pub struct GraphStructuredPredictorTrained {
node_classifiers: HashMap<usize, BinaryRelevance<BinaryRelevanceTrained>>,
edge_weights: Array2<Float>,
adjacency_matrix: Array2<Float>,
graph_type: GraphType,
edges: Vec<(usize, usize)>,
message_passing_iterations: usize,
inference_method: GraphInferenceMethod,
n_nodes: usize,
node_potentials: Array2<Float>,
edge_potentials: Array3<Float>,
}
impl GraphStructuredPredictor<Untrained> {
/// Create new graph-structured predictor
pub fn new() -> Self {
Self {
graph_type: GraphType::Undirected,
edges: Vec::new(),
message_passing_iterations: 10,
inference_method: GraphInferenceMethod::MessagePassing,
node_feature_dim: 0,
edge_feature_dim: 0,
state: Untrained,
}
}
/// Set graph type
pub fn graph_type(mut self, graph_type: GraphType) -> Self {
self.graph_type = graph_type;
self
}
/// Set graph edges
pub fn edges(mut self, edges: Vec<(usize, usize)>) -> Self {
self.edges = edges;
self
}
/// Set number of message passing iterations
pub fn message_passing_iterations(mut self, iterations: usize) -> Self {
self.message_passing_iterations = iterations;
self
}
/// Set inference method
pub fn inference_method(mut self, method: GraphInferenceMethod) -> Self {
self.inference_method = method;
self
}
/// Set node feature dimension
pub fn node_feature_dim(mut self, dim: usize) -> Self {
self.node_feature_dim = dim;
self
}
/// Set edge feature dimension
pub fn edge_feature_dim(mut self, dim: usize) -> Self {
self.edge_feature_dim = dim;
self
}
}
impl Estimator for GraphStructuredPredictor<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Array2<Float>, Array2<i32>> for GraphStructuredPredictor<Untrained> {
type Fitted = GraphStructuredPredictor<GraphStructuredPredictorTrained>;
fn fit(self, X: &Array2<Float>, y: &Array2<i32>) -> SklResult<Self::Fitted> {
let (n_samples, _n_features) = X.dim();
let (y_samples, n_nodes) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"Number of samples in X and y must match".to_string(),
));
}
// Build adjacency matrix
let adjacency_matrix = self.build_adjacency_matrix(n_nodes)?;
// Compute edge weights using node correlations
let edge_weights = self.compute_edge_weights(&y.view(), &adjacency_matrix)?;
// Train node classifiers
let node_classifiers = self.train_node_classifiers(&X.view(), &y.view())?;
// Compute potentials for inference
let (node_potentials, edge_potentials) =
self.compute_potentials(&X.view(), &y.view(), &adjacency_matrix)?;
Ok(GraphStructuredPredictor {
graph_type: self.graph_type.clone(),
edges: self.edges.clone(),
message_passing_iterations: self.message_passing_iterations,
inference_method: self.inference_method,
node_feature_dim: self.node_feature_dim,
edge_feature_dim: self.edge_feature_dim,
state: GraphStructuredPredictorTrained {
node_classifiers,
edge_weights,
adjacency_matrix,
graph_type: self.graph_type,
edges: self.edges,
message_passing_iterations: self.message_passing_iterations,
inference_method: self.inference_method.clone(),
n_nodes,
node_potentials,
edge_potentials,
},
})
}
}
impl GraphStructuredPredictor<Untrained> {
/// Build adjacency matrix from edges
fn build_adjacency_matrix(&self, n_nodes: usize) -> SklResult<Array2<Float>> {
let mut adjacency_matrix = Array2::zeros((n_nodes, n_nodes));
for &(i, j) in &self.edges {
if i < n_nodes && j < n_nodes {
adjacency_matrix[[i, j]] = 1.0;
// For undirected graphs, add symmetric edge
match self.graph_type {
GraphType::Undirected | GraphType::Bipartite => {
adjacency_matrix[[j, i]] = 1.0;
}
GraphType::Directed => {
// Already added above
}
}
}
}
Ok(adjacency_matrix)
}
/// Compute edge weights based on label correlations
fn compute_edge_weights(
&self,
y: &ArrayView2<i32>,
adjacency_matrix: &Array2<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = y.ncols();
let mut edge_weights = Array2::zeros((n_nodes, n_nodes));
// Compute mutual information between connected nodes
for i in 0..n_nodes {
for j in 0..n_nodes {
if adjacency_matrix[[i, j]] > 0.0 {
let mi = self.compute_mutual_information(&y.column(i), &y.column(j))?;
edge_weights[[i, j]] = mi;
}
}
}
Ok(edge_weights)
}
/// Compute mutual information between two label sequences
fn compute_mutual_information(
&self,
labels1: &ArrayView1<i32>,
labels2: &ArrayView1<i32>,
) -> SklResult<Float> {
let n_samples = labels1.len();
if n_samples != labels2.len() {
return Err(SklearsError::InvalidInput(
"Label sequences must have same length".to_string(),
));
}
// Compute joint and marginal distributions
let mut joint_counts = HashMap::new();
let mut marginal1_counts = HashMap::new();
let mut marginal2_counts = HashMap::new();
for i in 0..n_samples {
let l1 = labels1[i];
let l2 = labels2[i];
*joint_counts.entry((l1, l2)).or_insert(0) += 1;
*marginal1_counts.entry(l1).or_insert(0) += 1;
*marginal2_counts.entry(l2).or_insert(0) += 1;
}
// Compute mutual information
let mut mi = 0.0;
for ((l1, l2), &joint_count) in &joint_counts {
let p_joint = joint_count as Float / n_samples as Float;
let p_marginal1 = *marginal1_counts.get(l1).unwrap() as Float / n_samples as Float;
let p_marginal2 = *marginal2_counts.get(l2).unwrap() as Float / n_samples as Float;
if p_joint > 0.0 && p_marginal1 > 0.0 && p_marginal2 > 0.0 {
mi += p_joint * (p_joint / (p_marginal1 * p_marginal2)).ln();
}
}
Ok(mi)
}
/// Train individual node classifiers
fn train_node_classifiers(
&self,
X: &ArrayView2<Float>,
y: &ArrayView2<i32>,
) -> SklResult<HashMap<usize, BinaryRelevance<BinaryRelevanceTrained>>> {
let n_nodes = y.ncols();
let mut node_classifiers = HashMap::new();
for node_id in 0..n_nodes {
let node_labels = y.column(node_id).to_owned();
let classifier = BinaryRelevance::new();
let node_labels_2d = node_labels.insert_axis(ndarray::Axis(1));
let trained_classifier = classifier.fit(X, &node_labels_2d)?;
node_classifiers.insert(node_id, trained_classifier);
}
Ok(node_classifiers)
}
/// Compute node and edge potentials for inference
fn compute_potentials(
&self,
X: &ArrayView2<Float>,
y: &ArrayView2<i32>,
adjacency_matrix: &Array2<Float>,
) -> SklResult<(Array2<Float>, Array3<Float>)> {
let (n_samples, _) = X.dim();
let n_nodes = y.ncols();
// Node potentials: learned from data
let mut node_potentials = Array2::zeros((n_nodes, 2)); // Binary labels
for node_id in 0..n_nodes {
let labels = y.column(node_id);
let positive_count = labels.iter().filter(|&&x| x == 1).count() as Float;
let negative_count = labels.iter().filter(|&&x| x == 0).count() as Float;
let total = positive_count + negative_count;
if total > 0.0 {
node_potentials[[node_id, 0]] = negative_count / total;
node_potentials[[node_id, 1]] = positive_count / total;
} else {
node_potentials[[node_id, 0]] = 0.5;
node_potentials[[node_id, 1]] = 0.5;
}
}
// Edge potentials: agreement between connected nodes
let mut edge_potentials = Array3::zeros((n_nodes, 2, 2)); // [edge, label1, label2]
for i in 0..n_nodes {
for j in 0..n_nodes {
if adjacency_matrix[[i, j]] > 0.0 {
// Compute empirical edge potentials
let mut joint_counts = Array2::zeros((2, 2));
for sample_idx in 0..n_samples {
let l1 = y[[sample_idx, i]] as usize;
let l2 = y[[sample_idx, j]] as usize;
if l1 < 2 && l2 < 2 {
joint_counts[[l1, l2]] += 1.0;
}
}
// Normalize to get probabilities
let total = joint_counts.sum();
if total > 0.0 {
joint_counts = joint_counts / total;
}
edge_potentials
.slice_mut(s![i, .., ..])
.assign(&joint_counts);
}
}
}
Ok((node_potentials, edge_potentials))
}
}
impl Predict<Array2<Float>, Array2<i32>>
for GraphStructuredPredictor<GraphStructuredPredictorTrained>
{
fn predict(&self, X: &Array2<Float>) -> SklResult<Array2<i32>> {
let n_samples = X.nrows();
let mut predictions = Array2::zeros((n_samples, self.state.n_nodes));
match self.state.inference_method {
GraphInferenceMethod::MessagePassing => {
self.message_passing_inference(&X.view(), &mut predictions)?;
}
GraphInferenceMethod::GraphNeuralNetwork => {
self.graph_neural_network_inference(&X.view(), &mut predictions)?;
}
GraphInferenceMethod::VariationalInference => {
self.variational_inference(&X.view(), &mut predictions)?;
}
GraphInferenceMethod::SpectralMethods => {
self.spectral_inference(&X.view(), &mut predictions)?;
}
}
Ok(predictions)
}
}
impl GraphStructuredPredictor<GraphStructuredPredictorTrained> {
/// Message passing inference using belief propagation
fn message_passing_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
let n_samples = X.nrows();
for sample_idx in 0..n_samples {
// Initialize beliefs with node classifier outputs
let mut beliefs = Array2::zeros((self.state.n_nodes, 2));
for node_id in 0..self.state.n_nodes {
if let Some(classifier) = self.state.node_classifiers.get(&node_id) {
let sample_2d = X.row(sample_idx).to_owned().insert_axis(ndarray::Axis(0));
let proba = classifier.predict_proba(&sample_2d.view())?;
// BinaryRelevance returns probabilities for each label, not binary class probabilities
// proba[[0, 0]] is the probability of the first (and only) label being positive
let positive_prob = proba[[0, 0]];
beliefs[[node_id, 0]] = 1.0 - positive_prob; // Probability of negative class
beliefs[[node_id, 1]] = positive_prob; // Probability of positive class
} else {
beliefs[[node_id, 0]] = self.state.node_potentials[[node_id, 0]];
beliefs[[node_id, 1]] = self.state.node_potentials[[node_id, 1]];
}
}
// Message passing iterations
for _iter in 0..self.state.message_passing_iterations {
let mut new_beliefs = beliefs.clone();
for i in 0..self.state.n_nodes {
// Collect messages from neighbors
let mut messages = Vec::new();
for j in 0..self.state.n_nodes {
if i != j && self.state.adjacency_matrix[[i, j]] > 0.0 {
let message = self.compute_message(i, j, &beliefs)?;
messages.push(message);
}
}
// Update belief
if !messages.is_empty() {
for label in 0..2 {
let mut belief = beliefs[[i, label]];
for message in &messages {
belief *= message[label];
}
new_beliefs[[i, label]] = belief;
}
// Normalize
let total = new_beliefs.row(i).sum();
if total > 0.0 {
for label in 0..2 {
new_beliefs[[i, label]] /= total;
}
}
}
}
beliefs = new_beliefs;
}
// Make final predictions
for node_id in 0..self.state.n_nodes {
predictions[[sample_idx, node_id]] =
if beliefs[[node_id, 1]] > beliefs[[node_id, 0]] {
1
} else {
0
};
}
}
Ok(())
}
/// Compute message from node j to node i
fn compute_message(
&self,
i: usize,
j: usize,
beliefs: &Array2<Float>,
) -> SklResult<Vec<Float>> {
let mut message = vec![0.0; 2];
for label_j in 0..2 {
for label_i in 0..2 {
let edge_potential = if i < self.state.edge_potentials.dim().0 {
self.state.edge_potentials[[i, label_i, label_j]]
} else {
1.0 // Default potential
};
message[label_i] += beliefs[[j, label_j]] * edge_potential;
}
}
// Normalize message
let total: Float = message.iter().sum();
if total > 0.0 {
for m in &mut message {
*m /= total;
}
}
Ok(message)
}
/// Graph neural network inference (simplified)
fn graph_neural_network_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
// For simplicity, use message passing with learned weights
self.message_passing_inference(X, predictions)
}
/// Variational inference for graphs
fn variational_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
let n_samples = X.nrows();
for sample_idx in 0..n_samples {
// Initialize variational parameters
let mut q_params = Array2::from_elem((self.state.n_nodes, 2), 0.5);
// Get initial beliefs from node classifiers
for node_id in 0..self.state.n_nodes {
if let Some(classifier) = self.state.node_classifiers.get(&node_id) {
let sample_2d = X.row(sample_idx).to_owned().insert_axis(ndarray::Axis(0));
let proba = classifier.predict_proba(&sample_2d.view())?;
// BinaryRelevance returns probabilities for each label, not binary class probabilities
let positive_prob = proba[[0, 0]];
q_params[[node_id, 0]] = 1.0 - positive_prob; // Probability of negative class
q_params[[node_id, 1]] = positive_prob; // Probability of positive class
}
}
// Coordinate ascent on ELBO
for _iter in 0..self.state.message_passing_iterations {
let mut new_q_params = q_params.clone();
for i in 0..self.state.n_nodes {
// Update variational parameter for node i
for label_i in 0..2 {
let mut log_unnormalized = self.state.node_potentials[[i, label_i]].ln();
// Add contributions from neighboring nodes
for j in 0..self.state.n_nodes {
if i != j && self.state.adjacency_matrix[[i, j]] > 0.0 {
for label_j in 0..2 {
let edge_potential = if i < self.state.edge_potentials.dim().0 {
self.state.edge_potentials[[i, label_i, label_j]]
} else {
1.0
};
log_unnormalized +=
q_params[[j, label_j]] * edge_potential.ln();
}
}
}
new_q_params[[i, label_i]] = log_unnormalized.exp();
}
// Normalize
let total = new_q_params.row(i).sum();
if total > 0.0 {
for label in 0..2 {
new_q_params[[i, label]] /= total;
}
}
}
q_params = new_q_params;
}
// Make predictions based on variational parameters
for node_id in 0..self.state.n_nodes {
predictions[[sample_idx, node_id]] =
if q_params[[node_id, 1]] > q_params[[node_id, 0]] {
1
} else {
0
};
}
}
Ok(())
}
/// Spectral methods for graph inference
fn spectral_inference(
&self,
X: &ArrayView2<Float>,
predictions: &mut Array2<i32>,
) -> SklResult<()> {
let n_samples = X.nrows();
// Compute graph Laplacian
let degree_matrix = self.compute_degree_matrix()?;
let laplacian = °ree_matrix - &self.state.adjacency_matrix;
for sample_idx in 0..n_samples {
// Get initial predictions from node classifiers
let mut node_scores = Array1::zeros(self.state.n_nodes);
for node_id in 0..self.state.n_nodes {
if let Some(classifier) = self.state.node_classifiers.get(&node_id) {
let sample_2d = X.row(sample_idx).to_owned().insert_axis(ndarray::Axis(0));
let proba = classifier.predict_proba(&sample_2d.view())?;
// BinaryRelevance returns probabilities for each label, not binary class probabilities
// Convert to decision score: positive_prob - negative_prob
let positive_prob = proba[[0, 0]];
let negative_prob = 1.0 - positive_prob;
node_scores[node_id] = positive_prob - negative_prob; // Decision score
}
}
// Apply graph regularization (simplified)
let regularized_scores = self.apply_graph_regularization(&node_scores, &laplacian)?;
// Make predictions based on regularized scores
for node_id in 0..self.state.n_nodes {
predictions[[sample_idx, node_id]] = if regularized_scores[node_id] > 0.0 {
1
} else {
0
};
}
}
Ok(())
}
/// Compute degree matrix
fn compute_degree_matrix(&self) -> SklResult<Array2<Float>> {
let n_nodes = self.state.n_nodes;
let mut degree_matrix = Array2::zeros((n_nodes, n_nodes));
for i in 0..n_nodes {
let degree = self.state.adjacency_matrix.row(i).sum();
degree_matrix[[i, i]] = degree;
}
Ok(degree_matrix)
}
/// Apply graph regularization using Laplacian
fn apply_graph_regularization(
&self,
scores: &Array1<Float>,
laplacian: &Array2<Float>,
) -> SklResult<Array1<Float>> {
// Simplified graph regularization: scores - λ * L * scores
let lambda = 0.1; // Regularization parameter
let laplacian_scores = laplacian.dot(scores);
let regularized = scores - lambda * &laplacian_scores;
Ok(regularized)
}
/// Get adjacency matrix
pub fn adjacency_matrix(&self) -> &Array2<Float> {
&self.state.adjacency_matrix
}
/// Get edge weights
pub fn edge_weights(&self) -> &Array2<Float> {
&self.state.edge_weights
}
/// Get node potentials
pub fn node_potentials(&self) -> &Array2<Float> {
&self.state.node_potentials
}
/// Get edge potentials
pub fn edge_potentials(&self) -> &Array3<Float> {
&self.state.edge_potentials
}
}
#[cfg(test)]
mod svm_tests {
use super::*;
use ndarray::array;
#[test]
fn test_multi_output_svm_basic() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[1.5, 2.5], [2.5, 3.5], [3.5, 1.5], [4.5, 4.5]];
let model = MultiOutputSVM::new().kernel(SVMKernel::Linear).c(1.0);
let trained_model = model.fit(&X.view(), &y).unwrap();
// Test prediction
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 2));
// Test methods
assert_eq!(trained_model.n_support_vectors(), 4);
assert_eq!(trained_model.dual_coef().dim(), (4, 2));
assert_eq!(trained_model.intercept().len(), 2);
}
#[test]
fn test_multi_output_svm_rbf_kernel() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let y = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let model = MultiOutputSVM::new()
.kernel(SVMKernel::RBF { gamma: 0.5 })
.c(1.0);
let trained_model = model.fit(&X.view(), &y).unwrap();
let predictions = trained_model.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (3, 2));
}
#[test]
fn test_multi_output_svm_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1.0, 2.0]]; // Mismatched dimensions
let model = MultiOutputSVM::new();
assert!(model.fit(&X.view(), &y).is_err());
// Test prediction dimension mismatch
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1.0, 2.0], [2.0, 3.0]];
let model = MultiOutputSVM::new();
let trained_model = model.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_model.predict(&X_test.view()).is_err());
}
}
#[cfg(test)]
mod compressed_sensing_tests {
use super::*;
use ndarray::array;
#[test]
fn test_compressed_sensing_label_powerset_basic() {
let X = array![
[1.0, 2.0],
[2.0, 3.0],
[3.0, 1.0],
[4.0, 4.0],
[5.0, 2.0],
[6.0, 3.0]
];
let y = array![
[0, 1, 0],
[1, 0, 1],
[1, 1, 0],
[0, 0, 1],
[0, 1, 1],
[1, 0, 0]
]; // 3 labels
let cs = CompressedSensingLabelPowerset::new()
.compressed_dimension(2) // Compress 3 labels to 2 dimensions
.random_state(Some(42));
let trained_cs = cs.fit(&X.view(), &y).unwrap();
let predictions = trained_cs.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (6, 3));
// Check that predictions are binary
for pred in predictions.iter() {
assert!(pred == &0 || pred == &1);
}
// Check compression ratio
let ratio = trained_cs.compression_ratio();
assert!((ratio - (2.0 / 3.0)).abs() < 1e-10);
}
#[test]
fn test_compressed_sensing_iterative_thresholding() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1, 0, 1], [1, 0, 1, 0], [1, 1, 0, 0], [0, 0, 1, 1]]; // 4 labels
let method = ReconstructionMethod::IterativeThresholding;
let cs = CompressedSensingLabelPowerset::new()
.compressed_dimension(3) // Compress 4 labels to 3 dimensions
.reconstruction_method(method)
.random_state(Some(42));
let trained_cs = cs.fit(&X.view(), &y).unwrap();
let predictions = trained_cs.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 4));
}
#[test]
fn test_compressed_sensing_omp() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1, 0, 1], [1, 0, 1, 0], [1, 1, 0, 0], [0, 0, 1, 1]]; // 4 labels
let method = ReconstructionMethod::OrthogonalMatchingPursuit;
let cs = CompressedSensingLabelPowerset::new()
.compressed_dimension(3) // Compress 4 labels to 3 dimensions
.reconstruction_method(method)
.random_state(Some(42));
let trained_cs = cs.fit(&X.view(), &y).unwrap();
let predictions = trained_cs.predict(&X.view()).unwrap();
assert_eq!(predictions.dim(), (4, 4));
}
#[test]
fn test_compressed_sensing_error_handling() {
let X = array![[1.0, 2.0], [2.0, 3.0]];
let y = array![[1, 0], [0, 1], [1, 1]]; // Wrong number of rows
let cs = CompressedSensingLabelPowerset::new();
assert!(cs.clone().fit(&X.view(), &y).is_err());
// Test with no labels
let y_empty = Array2::<i32>::zeros((2, 0));
assert!(cs.clone().fit(&X.view(), &y_empty).is_err());
// Test with compressed dimension >= number of labels
let y_valid = array![[1, 0], [0, 1]];
let cs_invalid = CompressedSensingLabelPowerset::new().compressed_dimension(2); // Same as number of labels
assert!(cs_invalid.fit(&X.view(), &y_valid).is_err());
// Test prediction with wrong feature dimensions
let X_train = array![[1.0, 2.0], [2.0, 3.0]];
let y_train = array![[1, 0, 1], [0, 1, 0]];
let cs_for_predict = CompressedSensingLabelPowerset::new().compressed_dimension(2);
let trained_cs = cs_for_predict.fit(&X_train.view(), &y_train).unwrap();
let X_test = array![[1.0, 2.0, 3.0]]; // Wrong number of features
assert!(trained_cs.predict(&X_test.view()).is_err());
}
#[test]
fn test_compressed_sensing_projection_matrix() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1, 0], [1, 0, 1], [1, 1, 0], [0, 0, 1]]; // 3 labels
let cs = CompressedSensingLabelPowerset::new()
.compressed_dimension(2) // Compress 3 labels to 2 dimensions
.random_state(Some(42));
let trained_cs = cs.fit(&X.view(), &y).unwrap();
let projection_matrix = trained_cs.projection_matrix();
assert_eq!(projection_matrix.dim(), (2, 3)); // 2 compressed dimensions x 3 labels
// Check that rows are normalized
for i in 0..2 {
let row_norm: Float = projection_matrix
.row(i)
.iter()
.map(|x| x * x)
.sum::<Float>()
.sqrt();
assert!((row_norm - 1.0).abs() < 1e-6);
}
}
#[test]
fn test_compressed_sensing_reproducibility() {
let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
let y = array![[0, 1, 0], [1, 0, 1], [1, 1, 0], [0, 0, 1]];
// Train two models with same random state
let cs1 = CompressedSensingLabelPowerset::new()
.compressed_dimension(2)
.random_state(Some(42));
let trained_cs1 = cs1.fit(&X.view(), &y).unwrap();
let cs2 = CompressedSensingLabelPowerset::new()
.compressed_dimension(2)
.random_state(Some(42));
let trained_cs2 = cs2.fit(&X.view(), &y).unwrap();
// Should produce same projection matrices
let proj1 = trained_cs1.projection_matrix();
let proj2 = trained_cs2.projection_matrix();
for i in 0..proj1.nrows() {
for j in 0..proj1.ncols() {
assert!((proj1[[i, j]] - proj2[[i, j]]).abs() < 1e-10);
}
}
}
}
/// Maximum Entropy Markov Model (MEMM) for Sequence Labeling
///
/// MEMM is a discriminative model for sequence labeling that combines the advantages
/// of maximum entropy models with Markov assumptions. Unlike CRF, MEMM models the
/// conditional probability of each label given the previous label and observed features.
///
/// The model uses logistic regression at each position to predict the next label
/// based on features extracted from the current observation and previous label.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::MaximumEntropyMarkovModel;
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// // Sequence data: each row is a sequence element with features
/// let X = vec![
/// array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]], // sequence 1
/// array![[4.0, 1.0], [1.0, 4.0]] // sequence 2
/// ];
/// let y = vec![
/// vec![0, 1, 2], // labels for sequence 1
/// vec![1, 0] // labels for sequence 2
/// ];
///
/// let memm = MaximumEntropyMarkovModel::new()
/// .max_iter(100)
/// .learning_rate(0.01)
/// .l2_penalty(0.01);
///
/// let trained_memm = memm.fit(&X, &y).unwrap();
/// let predictions = trained_memm.predict(&X).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct MaximumEntropyMarkovModel<S = Untrained> {
state: S,
max_iter: usize,
learning_rate: Float,
l2_penalty: Float,
tolerance: Float,
random_state: Option<u64>,
feature_functions: Vec<FeatureFunction>,
}
/// Trained state for MEMM
#[derive(Debug, Clone)]
pub struct MaximumEntropyMarkovModelTrained {
/// Weights for feature functions
weights: Array1<Float>,
/// Feature functions used in the model
feature_functions: Vec<FeatureFunction>,
/// Number of unique labels
n_labels: usize,
/// Label mapping (string to index)
label_map: HashMap<i32, usize>,
/// Reverse label mapping (index to string)
label_names: Vec<i32>,
/// Number of features per observation
n_features: usize,
/// Training history
loss_curve: Vec<Float>,
}
/// Feature function for MEMM
#[derive(Debug, Clone)]
pub struct FeatureFunction {
/// Feature type identifier
pub feature_type: FeatureType,
/// Previous label (for transition features)
pub prev_label: Option<usize>,
/// Current label
pub curr_label: usize,
/// Feature index (for observation features)
pub feature_idx: Option<usize>,
}
/// Types of features in MEMM
#[derive(Debug, Clone, PartialEq)]
pub enum FeatureType {
/// Observation feature: depends on current observation and label
Observation,
/// Transition feature: depends on previous and current label
Transition,
/// Bias feature: constant feature for each label
Bias,
}
impl MaximumEntropyMarkovModel<Untrained> {
/// Create a new MEMM instance
pub fn new() -> Self {
Self {
state: Untrained,
max_iter: 100,
learning_rate: 0.01,
l2_penalty: 0.01,
tolerance: 1e-6,
random_state: None,
feature_functions: Vec::new(),
}
}
/// Set maximum number of iterations
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
/// Set learning rate
pub fn learning_rate(mut self, learning_rate: Float) -> Self {
self.learning_rate = learning_rate;
self
}
/// Set L2 penalty coefficient
pub fn l2_penalty(mut self, l2_penalty: Float) -> Self {
self.l2_penalty = l2_penalty;
self
}
/// Set convergence tolerance
pub fn tolerance(mut self, tolerance: Float) -> Self {
self.tolerance = tolerance;
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
}
impl Default for MaximumEntropyMarkovModel<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for MaximumEntropyMarkovModel<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl Fit<Vec<Array2<Float>>, Vec<Vec<i32>>> for MaximumEntropyMarkovModel<Untrained> {
type Fitted = MaximumEntropyMarkovModel<MaximumEntropyMarkovModelTrained>;
fn fit(self, X: &Vec<Array2<Float>>, y: &Vec<Vec<i32>>) -> SklResult<Self::Fitted> {
if X.is_empty() || y.is_empty() {
return Err(SklearsError::InvalidInput(
"Cannot fit with empty sequences".to_string(),
));
}
if X.len() != y.len() {
return Err(SklearsError::InvalidInput(
"Number of feature sequences must match number of label sequences".to_string(),
));
}
// Validate sequence lengths
for (i, (x_seq, y_seq)) in X.iter().zip(y.iter()).enumerate() {
if x_seq.nrows() != y_seq.len() {
return Err(SklearsError::InvalidInput(format!(
"Sequence {} has mismatched lengths: features {} vs labels {}",
i,
x_seq.nrows(),
y_seq.len()
)));
}
}
// Extract unique labels and create mappings
let mut unique_labels = std::collections::HashSet::new();
for seq in y {
for &label in seq {
unique_labels.insert(label);
}
}
let mut label_names: Vec<i32> = unique_labels.into_iter().collect();
label_names.sort();
let n_labels = label_names.len();
let label_map: HashMap<i32, usize> = label_names
.iter()
.enumerate()
.map(|(i, &label)| (label, i))
.collect();
// Get number of features from first sequence
let n_features = X[0].ncols();
// Generate feature functions
let feature_functions = self.generate_feature_functions(n_features, n_labels);
let n_feature_functions = feature_functions.len();
// Initialize weights
let mut rng = match self.random_state {
Some(seed) => ChaCha8Rng::seed_from_u64(seed),
None => ChaCha8Rng::from_rng(&mut rand::thread_rng()).unwrap(),
};
let mut weights = Array1::random_using(
n_feature_functions,
Normal::new(0.0, 0.01).unwrap(),
&mut rng,
);
let mut loss_curve = Vec::new();
// Training loop using gradient descent
for iter in 0..self.max_iter {
let mut total_gradient = Array1::zeros(n_feature_functions);
let mut total_loss = 0.0;
let mut total_positions = 0;
// Process each sequence
for (x_seq, y_seq) in X.iter().zip(y.iter()) {
let seq_len = y_seq.len();
for pos in 0..seq_len {
let prev_label_idx = if pos == 0 {
None
} else {
Some(label_map[&y_seq[pos - 1]])
};
let true_label_idx = label_map[&y_seq[pos]];
let observation = x_seq.row(pos);
// Compute feature vector for all possible labels
let mut scores = Array1::zeros(n_labels);
for label_idx in 0..n_labels {
scores[label_idx] = self.compute_score(
&weights,
&feature_functions,
&observation,
prev_label_idx,
label_idx,
);
}
// Apply softmax to get probabilities
let probabilities = self.softmax(&scores);
// Compute loss (negative log-likelihood)
let loss = -probabilities[true_label_idx].ln();
total_loss += loss;
total_positions += 1;
// Compute gradient
for (f_idx, feature_func) in feature_functions.iter().enumerate() {
let true_feature_value = self.evaluate_feature_function(
feature_func,
&observation,
prev_label_idx,
true_label_idx,
);
let expected_feature_value: Float = (0..n_labels)
.map(|label_idx| {
probabilities[label_idx]
* self.evaluate_feature_function(
feature_func,
&observation,
prev_label_idx,
label_idx,
)
})
.sum();
total_gradient[f_idx] += true_feature_value - expected_feature_value;
}
}
}
// Average loss and gradient
let avg_loss = total_loss / total_positions as Float;
loss_curve.push(avg_loss);
total_gradient /= total_positions as Float;
// Add L2 regularization to gradient
for i in 0..total_gradient.len() {
total_gradient[i] -= self.l2_penalty * weights[i];
}
// Update weights
for i in 0..weights.len() {
weights[i] += self.learning_rate * total_gradient[i];
}
// Check convergence
if iter > 0 && (loss_curve[iter - 1] - avg_loss).abs() < self.tolerance {
break;
}
}
let trained_state = MaximumEntropyMarkovModelTrained {
weights,
feature_functions,
n_labels,
label_map,
label_names,
n_features,
loss_curve,
};
Ok(MaximumEntropyMarkovModel {
state: trained_state,
max_iter: self.max_iter,
learning_rate: self.learning_rate,
l2_penalty: self.l2_penalty,
tolerance: self.tolerance,
random_state: self.random_state,
feature_functions: Vec::new(),
})
}
}
impl MaximumEntropyMarkovModel<Untrained> {
/// Generate feature functions for the model
fn generate_feature_functions(
&self,
n_features: usize,
n_labels: usize,
) -> Vec<FeatureFunction> {
let mut functions = Vec::new();
// Bias features (one per label)
for label_idx in 0..n_labels {
functions.push(FeatureFunction {
feature_type: FeatureType::Bias,
prev_label: None,
curr_label: label_idx,
feature_idx: None,
});
}
// Observation features (one per feature per label)
for feature_idx in 0..n_features {
for label_idx in 0..n_labels {
functions.push(FeatureFunction {
feature_type: FeatureType::Observation,
prev_label: None,
curr_label: label_idx,
feature_idx: Some(feature_idx),
});
}
}
// Transition features (one per label pair)
for prev_label_idx in 0..n_labels {
for curr_label_idx in 0..n_labels {
functions.push(FeatureFunction {
feature_type: FeatureType::Transition,
prev_label: Some(prev_label_idx),
curr_label: curr_label_idx,
feature_idx: None,
});
}
}
functions
}
/// Compute score for a given label at a position
fn compute_score(
&self,
weights: &Array1<Float>,
feature_functions: &[FeatureFunction],
observation: &ArrayView1<Float>,
prev_label: Option<usize>,
curr_label: usize,
) -> Float {
feature_functions
.iter()
.enumerate()
.map(|(f_idx, feature_func)| {
weights[f_idx]
* self.evaluate_feature_function(
feature_func,
observation,
prev_label,
curr_label,
)
})
.sum()
}
/// Evaluate a feature function
fn evaluate_feature_function(
&self,
feature_func: &FeatureFunction,
observation: &ArrayView1<Float>,
prev_label: Option<usize>,
curr_label: usize,
) -> Float {
match feature_func.feature_type {
FeatureType::Bias => {
if feature_func.curr_label == curr_label {
1.0
} else {
0.0
}
}
FeatureType::Observation => {
if feature_func.curr_label == curr_label {
observation[feature_func.feature_idx.unwrap()]
} else {
0.0
}
}
FeatureType::Transition => {
if feature_func.curr_label == curr_label && feature_func.prev_label == prev_label {
1.0
} else {
0.0
}
}
}
}
/// Apply softmax to scores
fn softmax(&self, scores: &Array1<Float>) -> Array1<Float> {
let max_score = scores.iter().fold(Float::NEG_INFINITY, |a, &b| a.max(b));
let shifted_scores = scores.map(|&x| x - max_score);
let exp_scores = shifted_scores.map(|&x| x.exp());
let sum_exp = exp_scores.sum();
exp_scores.map(|&x| x / sum_exp)
}
}
impl Predict<Vec<Array2<Float>>, Vec<Vec<i32>>>
for MaximumEntropyMarkovModel<MaximumEntropyMarkovModelTrained>
{
fn predict(&self, X: &Vec<Array2<Float>>) -> SklResult<Vec<Vec<i32>>> {
if X.is_empty() {
return Ok(Vec::new());
}
// Validate feature dimensions
for (i, x_seq) in X.iter().enumerate() {
if x_seq.ncols() != self.state.n_features {
return Err(SklearsError::InvalidInput(format!(
"Sequence {} has {} features, expected {}",
i,
x_seq.ncols(),
self.state.n_features
)));
}
}
let mut predictions = Vec::new();
for x_seq in X {
let seq_len = x_seq.nrows();
let mut seq_predictions = Vec::new();
for pos in 0..seq_len {
let prev_label_idx = if pos == 0 {
None
} else {
// Find the index of the previous predicted label
let prev_label = seq_predictions[pos - 1];
Some(self.state.label_map[&prev_label])
};
let observation = x_seq.row(pos);
// Compute scores for all labels
let mut scores = Array1::zeros(self.state.n_labels);
for label_idx in 0..self.state.n_labels {
scores[label_idx] =
self.compute_score_trained(&observation, prev_label_idx, label_idx);
}
// Find best label
let best_label_idx = scores
.iter()
.enumerate()
.max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
.unwrap()
.0;
seq_predictions.push(self.state.label_names[best_label_idx]);
}
predictions.push(seq_predictions);
}
Ok(predictions)
}
}
impl MaximumEntropyMarkovModel<MaximumEntropyMarkovModelTrained> {
/// Compute score for a label using trained weights
fn compute_score_trained(
&self,
observation: &ArrayView1<Float>,
prev_label: Option<usize>,
curr_label: usize,
) -> Float {
self.state
.feature_functions
.iter()
.enumerate()
.map(|(f_idx, feature_func)| {
self.state.weights[f_idx]
* self.evaluate_feature_function_trained(
feature_func,
observation,
prev_label,
curr_label,
)
})
.sum()
}
/// Evaluate feature function for trained model
fn evaluate_feature_function_trained(
&self,
feature_func: &FeatureFunction,
observation: &ArrayView1<Float>,
prev_label: Option<usize>,
curr_label: usize,
) -> Float {
match feature_func.feature_type {
FeatureType::Bias => {
if feature_func.curr_label == curr_label {
1.0
} else {
0.0
}
}
FeatureType::Observation => {
if feature_func.curr_label == curr_label {
observation[feature_func.feature_idx.unwrap()]
} else {
0.0
}
}
FeatureType::Transition => {
if feature_func.curr_label == curr_label && feature_func.prev_label == prev_label {
1.0
} else {
0.0
}
}
}
}
/// Get the training loss curve
pub fn loss_curve(&self) -> &[Float] {
&self.state.loss_curve
}
/// Get the learned weights
pub fn weights(&self) -> &Array1<Float> {
&self.state.weights
}
/// Get the feature functions
pub fn feature_functions(&self) -> &[FeatureFunction] {
&self.state.feature_functions
}
/// Predict probabilities for each position in sequences
pub fn predict_proba(&self, X: &Vec<Array2<Float>>) -> SklResult<Vec<Array2<Float>>> {
if X.is_empty() {
return Ok(Vec::new());
}
let mut all_probabilities = Vec::new();
for x_seq in X {
let seq_len = x_seq.nrows();
let mut seq_probabilities = Array2::zeros((seq_len, self.state.n_labels));
for pos in 0..seq_len {
let prev_label_idx = if pos == 0 {
None
} else {
// For probability prediction, we need to consider all possible previous labels
// For simplicity, we'll use greedy decoding
let prev_probs = seq_probabilities.row(pos - 1);
let best_prev_idx = prev_probs
.iter()
.enumerate()
.max_by(|(_, a): &(usize, &Float), (_, b): &(usize, &Float)| {
(*a).partial_cmp(*b).unwrap()
})
.unwrap()
.0;
Some(best_prev_idx)
};
let observation = x_seq.row(pos);
// Compute scores for all labels
let mut scores = Array1::zeros(self.state.n_labels);
for label_idx in 0..self.state.n_labels {
scores[label_idx] =
self.compute_score_trained(&observation, prev_label_idx, label_idx);
}
// Apply softmax to get probabilities
let probabilities = self.softmax_trained(&scores);
seq_probabilities.row_mut(pos).assign(&probabilities);
}
all_probabilities.push(seq_probabilities);
}
Ok(all_probabilities)
}
/// Apply softmax for trained model
fn softmax_trained(&self, scores: &Array1<Float>) -> Array1<Float> {
let max_score = scores.iter().fold(Float::NEG_INFINITY, |a, &b| a.max(b));
let shifted_scores = scores.map(|&x| x - max_score);
let exp_scores = shifted_scores.map(|&x| x.exp());
let sum_exp = exp_scores.sum();
exp_scores.map(|&x| x / sum_exp)
}
}
#[cfg(test)]
mod memm_tests {
use super::*;
use ndarray::array;
#[test]
fn test_memm_basic_functionality() {
let X = vec![
array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]],
array![[4.0, 1.0], [1.0, 4.0]],
];
let y = vec![vec![0, 1, 2], vec![1, 0]];
let memm = MaximumEntropyMarkovModel::new()
.max_iter(10)
.learning_rate(0.1)
.random_state(Some(42));
let trained_memm = memm.fit(&X, &y).unwrap();
let predictions = trained_memm.predict(&X).unwrap();
assert_eq!(predictions.len(), 2);
assert_eq!(predictions[0].len(), 3);
assert_eq!(predictions[1].len(), 2);
}
#[test]
fn test_memm_predict_proba() {
let X = vec![array![[1.0, 2.0], [2.0, 3.0]]];
let y = vec![vec![0, 1]];
let memm = MaximumEntropyMarkovModel::new()
.max_iter(5)
.random_state(Some(42));
let trained_memm = memm.fit(&X, &y).unwrap();
let probabilities = trained_memm.predict_proba(&X).unwrap();
assert_eq!(probabilities.len(), 1);
assert_eq!(probabilities[0].dim(), (2, 2)); // 2 positions, 2 labels
// Check that probabilities sum to 1 for each position
for pos in 0..probabilities[0].nrows() {
let sum: Float = probabilities[0].row(pos).sum();
assert!((sum - 1.0).abs() < 1e-6);
}
}
#[test]
fn test_memm_error_handling() {
let X = vec![array![[1.0, 2.0], [2.0, 3.0]]];
let y = vec![vec![0, 1, 2]]; // Wrong length
let memm = MaximumEntropyMarkovModel::new();
assert!(memm.fit(&X, &y).is_err());
// Test empty sequences
let empty_X: Vec<Array2<Float>> = Vec::new();
let empty_y: Vec<Vec<i32>> = Vec::new();
assert!(MaximumEntropyMarkovModel::new()
.fit(&empty_X, &empty_y)
.is_err());
}
#[test]
fn test_memm_feature_functions() {
let X = vec![array![[1.0, 2.0]]];
let y = vec![vec![0]];
let memm = MaximumEntropyMarkovModel::new()
.max_iter(1)
.random_state(Some(42));
let trained_memm = memm.fit(&X, &y).unwrap();
let feature_functions = trained_memm.feature_functions();
// Should have bias + observation + transition features
// With 2 features and 1 label: 1 bias + 2 observation + 1 transition = 4 features
assert!(feature_functions.len() > 0);
}
#[test]
fn test_memm_reproducibility() {
let X = vec![array![[1.0, 2.0], [2.0, 3.0]]];
let y = vec![vec![0, 1]];
let memm1 = MaximumEntropyMarkovModel::new()
.max_iter(10)
.random_state(Some(42));
let trained_memm1 = memm1.fit(&X, &y).unwrap();
let pred1 = trained_memm1.predict(&X).unwrap();
let memm2 = MaximumEntropyMarkovModel::new()
.max_iter(10)
.random_state(Some(42));
let trained_memm2 = memm2.fit(&X, &y).unwrap();
let pred2 = trained_memm2.predict(&X).unwrap();
assert_eq!(pred1, pred2);
}
}
// ================================================================================================
// NEW IMPLEMENTATIONS: Ontology-Aware and Cost-Sensitive Hierarchical Methods
// ================================================================================================
/// Ontology-Aware Hierarchical Classifier
///
/// A hierarchical multi-label classifier that incorporates domain ontology knowledge
/// to ensure taxonomically consistent predictions. This method enforces that if a child
/// concept is predicted, its parent concepts are also predicted according to the
/// provided hierarchical structure.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::{OntologyAwareClassifier, ConsistencyEnforcement};
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
/// use std::collections::HashMap;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0, 1, 0], [0, 1, 0, 1], [1, 1, 0, 0], [0, 0, 1, 1]];
///
/// // Define ontology: child -> parent relationships
/// let mut ontology = HashMap::new();
/// ontology.insert(2, vec![0]); // concept 2 is child of concept 0
/// ontology.insert(3, vec![1]); // concept 3 is child of concept 1
///
/// let classifier = OntologyAwareClassifier::new()
/// .ontology(ontology)
/// .consistency_enforcement(ConsistencyEnforcement::PostProcessing)
/// .random_state(Some(42));
///
/// let trained_classifier = classifier.fit(&X.view(), &y.view()).unwrap();
/// let predictions = trained_classifier.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct OntologyAwareClassifier<S = Untrained> {
state: S,
ontology: HashMap<usize, Vec<usize>>, // child -> parents mapping
consistency_enforcement: ConsistencyEnforcement,
random_state: Option<u64>,
base_threshold: Float,
confidence_boost: Float,
}
/// Trained state for OntologyAwareClassifier
#[derive(Debug, Clone)]
pub struct OntologyAwareClassifierTrained {
/// Binary classifiers for each concept
classifiers: HashMap<usize, BinaryClassifierState>,
/// Concept hierarchy (child -> parents)
ontology: HashMap<usize, Vec<usize>>,
/// Consistency enforcement method
consistency_enforcement: ConsistencyEnforcement,
/// Prediction parameters
base_threshold: Float,
confidence_boost: Float,
/// Number of concepts
n_concepts: usize,
/// Number of features
n_features: usize,
}
#[derive(Debug, Clone)]
struct BinaryClassifierState {
weights: Array1<Float>,
bias: Float,
positive_examples: usize,
negative_examples: usize,
}
impl OntologyAwareClassifier<Untrained> {
/// Create a new ontology-aware classifier
pub fn new() -> Self {
Self {
state: Untrained,
ontology: HashMap::new(),
consistency_enforcement: ConsistencyEnforcement::PostProcessing,
random_state: None,
base_threshold: 0.5,
confidence_boost: 0.1,
}
}
/// Set the concept ontology (child -> parents mapping)
pub fn ontology(mut self, ontology: HashMap<usize, Vec<usize>>) -> Self {
self.ontology = ontology;
self
}
/// Set the consistency enforcement method
pub fn consistency_enforcement(mut self, enforcement: ConsistencyEnforcement) -> Self {
self.consistency_enforcement = enforcement;
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, seed: Option<u64>) -> Self {
self.random_state = seed;
self
}
/// Set base threshold for binary predictions
pub fn base_threshold(mut self, threshold: Float) -> Self {
self.base_threshold = threshold;
self
}
/// Set confidence boost for parent concepts
pub fn confidence_boost(mut self, boost: Float) -> Self {
self.confidence_boost = boost;
self
}
}
impl Default for OntologyAwareClassifier<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for OntologyAwareClassifier<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl
Fit<
ArrayView2<'_, Float>,
ArrayView2<'_, i32>,
OntologyAwareClassifier<OntologyAwareClassifierTrained>,
> for OntologyAwareClassifier<Untrained>
{
type Fitted = OntologyAwareClassifier<OntologyAwareClassifierTrained>;
fn fit(
self,
X: &ArrayView2<'_, Float>,
y: &ArrayView2<'_, i32>,
) -> SklResult<OntologyAwareClassifier<OntologyAwareClassifierTrained>> {
let (n_samples, n_features) = X.dim();
let (y_samples, n_concepts) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples < 2 {
return Err(SklearsError::InvalidInput(
"Need at least 2 samples for training".to_string(),
));
}
// Initialize random number generator if needed
let mut rng = if let Some(seed) = self.random_state {
ChaCha8Rng::seed_from_u64(seed)
} else {
ChaCha8Rng::from_rng(&mut rand::thread_rng()).unwrap()
};
// Train binary classifier for each concept
let mut classifiers = HashMap::new();
for concept_idx in 0..n_concepts {
let y_concept = y.column(concept_idx);
// Prepare training data with ontological constraints if needed
let adjusted_y = match self.consistency_enforcement {
ConsistencyEnforcement::ConstrainedTraining => {
// Adjust labels to enforce ontological consistency during training
self.enforce_training_consistency(&y_concept, concept_idx)?
}
_ => y_concept.to_owned(),
};
// Train simple logistic regression for this concept
let classifier = train_binary_logistic_regression(X, &adjusted_y.view(), &mut rng)?;
classifiers.insert(concept_idx, classifier);
}
Ok(OntologyAwareClassifier {
state: OntologyAwareClassifierTrained {
classifiers,
ontology: self.ontology.clone(),
consistency_enforcement: self.consistency_enforcement,
base_threshold: self.base_threshold,
confidence_boost: self.confidence_boost,
n_concepts,
n_features,
},
ontology: self.ontology,
consistency_enforcement: self.consistency_enforcement,
random_state: self.random_state,
base_threshold: self.base_threshold,
confidence_boost: self.confidence_boost,
})
}
}
impl OntologyAwareClassifier<Untrained> {
/// Enforce consistency during training by adjusting labels
fn enforce_training_consistency(
&self,
y_concept: &ArrayView1<'_, i32>,
concept_idx: usize,
) -> SklResult<Array1<i32>> {
let mut adjusted_y = y_concept.to_owned();
// If this concept has parents, ensure parents are also labeled when child is labeled
if let Some(parents) = self.ontology.get(&concept_idx) {
for (sample_idx, &label) in y_concept.iter().enumerate() {
if label == 1 {
// Child is positive, so parents should be positive too
// Note: This is a simplified approach; in practice, you'd need access to all labels
// Here we just ensure the current concept maintains its label
adjusted_y[sample_idx] = 1;
}
}
}
Ok(adjusted_y)
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for OntologyAwareClassifier<OntologyAwareClassifierTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
// Get probability predictions first
let probabilities = self.predict_proba(X)?;
// Convert probabilities to binary predictions
let mut predictions = Array2::zeros((n_samples, self.state.n_concepts));
for i in 0..n_samples {
for j in 0..self.state.n_concepts {
predictions[[i, j]] = if probabilities[[i, j]] > self.state.base_threshold {
1
} else {
0
};
}
}
// Apply ontological consistency
match self.state.consistency_enforcement {
ConsistencyEnforcement::PostProcessing => {
self.enforce_post_processing_consistency(&mut predictions)?;
}
ConsistencyEnforcement::BayesianInference => {
predictions = self.bayesian_inference_consistency(&probabilities)?;
}
_ => {} // ConstrainedTraining consistency was applied during training
}
Ok(predictions)
}
}
impl OntologyAwareClassifier<OntologyAwareClassifierTrained> {
/// Predict class probabilities
pub fn predict_proba(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<Float>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut probabilities = Array2::zeros((n_samples, self.state.n_concepts));
// Get base probabilities from individual classifiers
for (concept_idx, classifier) in &self.state.classifiers {
for i in 0..n_samples {
let x_sample = X.row(i);
let logit = x_sample.dot(&classifier.weights) + classifier.bias;
let probability = 1.0 / (1.0 + (-logit).exp()); // Sigmoid
probabilities[[i, *concept_idx]] = probability;
}
}
Ok(probabilities)
}
/// Enforce consistency through post-processing
fn enforce_post_processing_consistency(&self, predictions: &mut Array2<i32>) -> SklResult<()> {
let n_samples = predictions.nrows();
for sample_idx in 0..n_samples {
// For each positive prediction, ensure parents are also positive
let mut changed = true;
while changed {
changed = false;
for (child, parents) in &self.state.ontology {
if predictions[[sample_idx, *child]] == 1 {
for parent in parents {
if *parent < self.state.n_concepts
&& predictions[[sample_idx, *parent]] == 0
{
predictions[[sample_idx, *parent]] = 1;
changed = true;
}
}
}
}
}
}
Ok(())
}
/// Apply Bayesian inference for consistency
fn bayesian_inference_consistency(
&self,
probabilities: &Array2<Float>,
) -> SklResult<Array2<i32>> {
let (n_samples, _) = probabilities.dim();
let mut predictions = Array2::zeros((n_samples, self.state.n_concepts));
for sample_idx in 0..n_samples {
// Apply Bayesian reasoning with ontological priors
let mut adjusted_probs = probabilities.row(sample_idx).to_owned();
// Boost parent probabilities based on child evidence
for (child, parents) in &self.state.ontology {
let child_prob = adjusted_probs[*child];
for parent in parents {
if *parent < self.state.n_concepts {
// Boost parent probability based on child evidence
let boost = child_prob * self.state.confidence_boost;
adjusted_probs[*parent] = (adjusted_probs[*parent] + boost).min(1.0);
}
}
}
// Make final predictions based on adjusted probabilities
for concept_idx in 0..self.state.n_concepts {
predictions[[sample_idx, concept_idx]] =
if adjusted_probs[concept_idx] > self.state.base_threshold {
1
} else {
0
};
}
}
Ok(predictions)
}
/// Get the learned ontology structure
pub fn ontology(&self) -> &HashMap<usize, Vec<usize>> {
&self.state.ontology
}
/// Get classifier weights for analysis
pub fn concept_weights(&self, concept_idx: usize) -> Option<&Array1<Float>> {
self.state.classifiers.get(&concept_idx).map(|c| &c.weights)
}
}
/// Cost-Sensitive Hierarchical Classifier
///
/// Extends hierarchical classification with cost-sensitive learning, where different
/// types of misclassification errors have different costs. This is particularly useful
/// in domains where false positives and false negatives have asymmetric costs.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::{CostSensitiveHierarchicalClassifier, CostStrategy};
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
/// use std::collections::HashMap;
///
/// let X = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0], [4.0, 4.0]];
/// let y = array![[1, 0, 1, 0], [0, 1, 0, 1], [1, 1, 0, 0], [0, 0, 1, 1]];
///
/// // Define hierarchical structure and costs
/// let mut hierarchy = HashMap::new();
/// hierarchy.insert(2, vec![0]); // concept 2 is child of concept 0
/// hierarchy.insert(3, vec![1]); // concept 3 is child of concept 1
///
/// let mut costs = HashMap::new();
/// costs.insert(0, (1.0, 2.0)); // (false_positive_cost, false_negative_cost)
/// costs.insert(1, (2.0, 1.0));
/// costs.insert(2, (1.5, 1.5));
/// costs.insert(3, (3.0, 1.0));
///
/// let classifier = CostSensitiveHierarchicalClassifier::new()
/// .hierarchy(hierarchy)
/// .costs(costs)
/// .cost_strategy(CostStrategy::Weighted)
/// .random_state(Some(42));
///
/// let trained_classifier = classifier.fit(&X.view(), &y.view()).unwrap();
/// let predictions = trained_classifier.predict(&X.view()).unwrap();
/// ```
#[derive(Debug, Clone)]
pub struct CostSensitiveHierarchicalClassifier<S = Untrained> {
state: S,
hierarchy: HashMap<usize, Vec<usize>>, // child -> parents mapping
costs: HashMap<usize, (Float, Float)>, // concept -> (false_positive_cost, false_negative_cost)
cost_strategy: CostStrategy,
random_state: Option<u64>,
learning_rate: Float,
max_iter: usize,
}
/// Strategies for incorporating costs into learning
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum CostStrategy {
/// Weight training examples by costs
Weighted,
/// Adjust decision thresholds based on costs
ThresholdAdjustment,
/// Use cost-sensitive loss function
CostSensitiveLoss,
}
/// Trained state for CostSensitiveHierarchicalClassifier
#[derive(Debug, Clone)]
pub struct CostSensitiveHierarchicalClassifierTrained {
/// Concept classifiers
classifiers: HashMap<usize, BinaryClassifierState>,
/// Hierarchical structure
hierarchy: HashMap<usize, Vec<usize>>,
/// Cost matrix
costs: HashMap<usize, (Float, Float)>,
/// Cost-adjusted thresholds for each concept
thresholds: HashMap<usize, Float>,
/// Cost strategy used
cost_strategy: CostStrategy,
/// Model parameters
n_concepts: usize,
n_features: usize,
}
impl CostSensitiveHierarchicalClassifier<Untrained> {
/// Create a new cost-sensitive hierarchical classifier
pub fn new() -> Self {
Self {
state: Untrained,
hierarchy: HashMap::new(),
costs: HashMap::new(),
cost_strategy: CostStrategy::Weighted,
random_state: None,
learning_rate: 0.01,
max_iter: 1000,
}
}
/// Set the hierarchical structure (child -> parents mapping)
pub fn hierarchy(mut self, hierarchy: HashMap<usize, Vec<usize>>) -> Self {
self.hierarchy = hierarchy;
self
}
/// Set the cost matrix (concept -> (false_positive_cost, false_negative_cost))
pub fn costs(mut self, costs: HashMap<usize, (Float, Float)>) -> Self {
self.costs = costs;
self
}
/// Set the cost strategy
pub fn cost_strategy(mut self, strategy: CostStrategy) -> Self {
self.cost_strategy = strategy;
self
}
/// Set random state for reproducibility
pub fn random_state(mut self, seed: Option<u64>) -> Self {
self.random_state = seed;
self
}
/// Set learning rate
pub fn learning_rate(mut self, lr: Float) -> Self {
self.learning_rate = lr;
self
}
/// Set maximum iterations
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
}
impl Default for CostSensitiveHierarchicalClassifier<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for CostSensitiveHierarchicalClassifier<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
impl
Fit<
ArrayView2<'_, Float>,
ArrayView2<'_, i32>,
CostSensitiveHierarchicalClassifier<CostSensitiveHierarchicalClassifierTrained>,
> for CostSensitiveHierarchicalClassifier<Untrained>
{
type Fitted = CostSensitiveHierarchicalClassifier<CostSensitiveHierarchicalClassifierTrained>;
fn fit(
self,
X: &ArrayView2<'_, Float>,
y: &ArrayView2<'_, i32>,
) -> SklResult<CostSensitiveHierarchicalClassifier<CostSensitiveHierarchicalClassifierTrained>>
{
let (n_samples, n_features) = X.dim();
let (y_samples, n_concepts) = y.dim();
if n_samples != y_samples {
return Err(SklearsError::InvalidInput(
"X and y must have the same number of samples".to_string(),
));
}
if n_samples < 2 {
return Err(SklearsError::InvalidInput(
"Need at least 2 samples for training".to_string(),
));
}
// Initialize random number generator
let mut rng = if let Some(seed) = self.random_state {
ChaCha8Rng::seed_from_u64(seed)
} else {
ChaCha8Rng::from_rng(&mut rand::thread_rng()).unwrap()
};
// Train cost-sensitive classifier for each concept
let mut classifiers = HashMap::new();
let mut thresholds = HashMap::new();
for concept_idx in 0..n_concepts {
let y_concept = y.column(concept_idx);
let costs = self.costs.get(&concept_idx).copied().unwrap_or((1.0, 1.0));
match self.cost_strategy {
CostStrategy::Weighted => {
// Train with cost-weighted examples
let classifier =
train_cost_weighted_classifier(X, &y_concept, costs, &mut rng)?;
classifiers.insert(concept_idx, classifier);
thresholds.insert(concept_idx, 0.5);
}
CostStrategy::ThresholdAdjustment => {
// Train normal classifier but adjust threshold
let classifier = train_binary_logistic_regression(X, &y_concept, &mut rng)?;
let optimal_threshold = compute_cost_optimal_threshold(costs);
classifiers.insert(concept_idx, classifier);
thresholds.insert(concept_idx, optimal_threshold);
}
CostStrategy::CostSensitiveLoss => {
// Train with cost-sensitive loss function
let classifier = train_cost_sensitive_loss_classifier(
X,
&y_concept,
costs,
self.learning_rate,
self.max_iter,
&mut rng,
)?;
classifiers.insert(concept_idx, classifier);
thresholds.insert(concept_idx, 0.5);
}
}
}
Ok(CostSensitiveHierarchicalClassifier {
state: CostSensitiveHierarchicalClassifierTrained {
classifiers,
hierarchy: self.hierarchy.clone(),
costs: self.costs.clone(),
thresholds,
cost_strategy: self.cost_strategy,
n_concepts,
n_features,
},
hierarchy: self.hierarchy,
costs: self.costs,
cost_strategy: self.cost_strategy,
random_state: self.random_state,
learning_rate: self.learning_rate,
max_iter: self.max_iter,
})
}
}
impl Predict<ArrayView2<'_, Float>, Array2<i32>>
for CostSensitiveHierarchicalClassifier<CostSensitiveHierarchicalClassifierTrained>
{
fn predict(&self, X: &ArrayView2<'_, Float>) -> SklResult<Array2<i32>> {
let (n_samples, n_features) = X.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"X has different number of features than training data".to_string(),
));
}
let mut predictions = Array2::zeros((n_samples, self.state.n_concepts));
// Make predictions using cost-adjusted thresholds
for (concept_idx, classifier) in &self.state.classifiers {
let threshold = self
.state
.thresholds
.get(concept_idx)
.copied()
.unwrap_or(0.5);
for i in 0..n_samples {
let x_sample = X.row(i);
let logit = x_sample.dot(&classifier.weights) + classifier.bias;
let probability = 1.0 / (1.0 + (-logit).exp());
predictions[[i, *concept_idx]] = if probability > threshold { 1 } else { 0 };
}
}
// Enforce hierarchical consistency
self.enforce_hierarchical_consistency(&mut predictions)?;
Ok(predictions)
}
}
impl CostSensitiveHierarchicalClassifier<CostSensitiveHierarchicalClassifierTrained> {
/// Enforce hierarchical consistency in predictions
fn enforce_hierarchical_consistency(&self, predictions: &mut Array2<i32>) -> SklResult<()> {
let n_samples = predictions.nrows();
for sample_idx in 0..n_samples {
// Ensure parent concepts are predicted when children are predicted
let mut changed = true;
while changed {
changed = false;
for (child, parents) in &self.state.hierarchy {
if predictions[[sample_idx, *child]] == 1 {
for parent in parents {
if *parent < self.state.n_concepts
&& predictions[[sample_idx, *parent]] == 0
{
predictions[[sample_idx, *parent]] = 1;
changed = true;
}
}
}
}
}
}
Ok(())
}
/// Get the cost-adjusted threshold for a concept
pub fn concept_threshold(&self, concept_idx: usize) -> Option<Float> {
self.state.thresholds.get(&concept_idx).copied()
}
/// Get the cost matrix
pub fn costs(&self) -> &HashMap<usize, (Float, Float)> {
&self.state.costs
}
/// Compute expected cost for a prediction
pub fn compute_prediction_cost(
&self,
y_true: &Array2<i32>,
y_pred: &Array2<i32>,
) -> SklResult<Float> {
if y_true.dim() != y_pred.dim() {
return Err(SklearsError::InvalidInput(
"y_true and y_pred must have the same dimensions".to_string(),
));
}
let mut total_cost = 0.0;
let (n_samples, n_concepts) = y_true.dim();
for sample_idx in 0..n_samples {
for concept_idx in 0..n_concepts {
let true_label = y_true[[sample_idx, concept_idx]];
let pred_label = y_pred[[sample_idx, concept_idx]];
if let Some(&(fp_cost, fn_cost)) = self.state.costs.get(&concept_idx) {
match (true_label, pred_label) {
(0, 1) => total_cost += fp_cost, // False positive
(1, 0) => total_cost += fn_cost, // False negative
_ => {} // Correct predictions have no cost
}
}
}
}
Ok(total_cost)
}
}
// ================================================================================================
// Helper Functions for Ontology-Aware and Cost-Sensitive Methods
// ================================================================================================
/// Train a simple binary logistic regression classifier
fn train_binary_logistic_regression(
X: &ArrayView2<'_, Float>,
y: &ArrayView1<'_, i32>,
rng: &mut ChaCha8Rng,
) -> SklResult<BinaryClassifierState> {
let (n_samples, n_features) = X.dim();
// Initialize weights randomly
let normal = Normal::new(0.0, 0.01).unwrap();
let mut weights = Array1::random_using(n_features, normal, rng);
let mut bias = 0.0;
let learning_rate = 0.01;
let max_iter = 1000;
// Count positive and negative examples
let positive_examples = y.iter().filter(|&&label| label == 1).count();
let negative_examples = n_samples - positive_examples;
// Simple gradient descent
for _iter in 0..max_iter {
let mut grad_weights = Array1::zeros(n_features);
let mut grad_bias = 0.0;
for i in 0..n_samples {
let x_i = X.row(i);
let y_i = y[i] as Float;
let logit = x_i.dot(&weights) + bias;
let prob = 1.0 / (1.0 + (-logit).exp());
let error = prob - y_i;
grad_weights = grad_weights + error * &x_i;
grad_bias += error;
}
weights = weights - learning_rate * &grad_weights / (n_samples as Float);
bias -= learning_rate * grad_bias / (n_samples as Float);
}
Ok(BinaryClassifierState {
weights,
bias,
positive_examples,
negative_examples,
})
}
/// Train a cost-weighted binary classifier
fn train_cost_weighted_classifier(
X: &ArrayView2<'_, Float>,
y: &ArrayView1<'_, i32>,
costs: (Float, Float), // (false_positive_cost, false_negative_cost)
rng: &mut ChaCha8Rng,
) -> SklResult<BinaryClassifierState> {
let (n_samples, n_features) = X.dim();
let (fp_cost, fn_cost) = costs;
// Initialize weights randomly
let normal = Normal::new(0.0, 0.01).unwrap();
let mut weights = Array1::random_using(n_features, normal, rng);
let mut bias = 0.0;
let learning_rate = 0.01;
let max_iter = 1000;
// Count examples
let positive_examples = y.iter().filter(|&&label| label == 1).count();
let negative_examples = n_samples - positive_examples;
// Weighted gradient descent
for _iter in 0..max_iter {
let mut grad_weights = Array1::zeros(n_features);
let mut grad_bias = 0.0;
for i in 0..n_samples {
let x_i = X.row(i);
let y_i = y[i] as Float;
// Weight example by cost
let weight = if y_i == 1.0 { fn_cost } else { fp_cost };
let logit = x_i.dot(&weights) + bias;
let prob = 1.0 / (1.0 + (-logit).exp());
let error = (prob - y_i) * weight;
grad_weights = grad_weights + error * &x_i;
grad_bias += error;
}
weights = weights - learning_rate * &grad_weights / (n_samples as Float);
bias -= learning_rate * grad_bias / (n_samples as Float);
}
Ok(BinaryClassifierState {
weights,
bias,
positive_examples,
negative_examples,
})
}
/// Train classifier with cost-sensitive loss function
fn train_cost_sensitive_loss_classifier(
X: &ArrayView2<'_, Float>,
y: &ArrayView1<'_, i32>,
costs: (Float, Float),
learning_rate: Float,
max_iter: usize,
rng: &mut ChaCha8Rng,
) -> SklResult<BinaryClassifierState> {
let (n_samples, n_features) = X.dim();
let (fp_cost, fn_cost) = costs;
// Initialize weights randomly
let normal = Normal::new(0.0, 0.01).unwrap();
let mut weights = Array1::random_using(n_features, normal, rng);
let mut bias = 0.0;
// Count examples
let positive_examples = y.iter().filter(|&&label| label == 1).count();
let negative_examples = n_samples - positive_examples;
// Cost-sensitive gradient descent with asymmetric loss
for _iter in 0..max_iter {
let mut grad_weights = Array1::zeros(n_features);
let mut grad_bias = 0.0;
for i in 0..n_samples {
let x_i = X.row(i);
let y_i = y[i] as Float;
let logit = x_i.dot(&weights) + bias;
let prob = 1.0 / (1.0 + (-logit).exp());
// Cost-sensitive gradient computation
let cost_adjusted_error = if y_i == 1.0 {
// For positive examples, weight by false negative cost
(prob - y_i) * fn_cost
} else {
// For negative examples, weight by false positive cost
(prob - y_i) * fp_cost
};
grad_weights = grad_weights + cost_adjusted_error * &x_i;
grad_bias += cost_adjusted_error;
}
weights = weights - learning_rate * &grad_weights / (n_samples as Float);
bias -= learning_rate * grad_bias / (n_samples as Float);
}
Ok(BinaryClassifierState {
weights,
bias,
positive_examples,
negative_examples,
})
}
/// Compute optimal threshold based on cost ratio
fn compute_cost_optimal_threshold(costs: (Float, Float)) -> Float {
let (fp_cost, fn_cost) = costs;
// Optimal threshold for cost-sensitive classification
// threshold = cost_ratio / (1 + cost_ratio) where cost_ratio = fp_cost / fn_cost
let cost_ratio = fp_cost / fn_cost;
cost_ratio / (1.0 + cost_ratio)
}
// ================================================================================================
// Graph Neural Networks for Structured Output Prediction
// ================================================================================================
/// Aggregation functions for Graph Neural Networks
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum AggregationFunction {
/// Mean aggregation
Mean,
/// Sum aggregation
Sum,
/// Maximum aggregation
Max,
/// Attention-based aggregation
Attention,
}
/// Message passing variants for GNNs
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum MessagePassingVariant {
/// Graph Convolutional Network (GCN)
GCN,
/// Graph Attention Network (GAT)
GAT,
/// GraphSAGE
GraphSAGE,
/// Graph Isomorphism Network (GIN)
GIN,
}
/// Graph Neural Network for Structured Output Prediction
///
/// This model handles structured prediction problems where the output space
/// has a graph structure. It uses message passing and node embedding to
/// learn representations that respect the graph topology.
///
/// # Examples
///
/// ```
/// use sklears_multioutput::{GraphNeuralNetwork, MessagePassingVariant, AggregationFunction};
/// use sklears_core::traits::{Fit, Predict};
/// use ndarray::array;
///
/// // Example graph structure (adjacency matrix)
/// let adjacency = array![[0, 1, 1], [1, 0, 1], [1, 1, 0]];
/// let node_features = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
/// let node_labels = array![[1, 0], [0, 1], [1, 1]];
///
/// let gnn = GraphNeuralNetwork::new()
/// .hidden_dim(16)
/// .num_layers(2)
/// .message_passing_variant(MessagePassingVariant::GCN)
/// .aggregation_function(AggregationFunction::Mean)
/// .learning_rate(0.01)
/// .max_iter(100);
/// ```
#[derive(Debug, Clone)]
pub struct GraphNeuralNetwork<S = Untrained> {
state: S,
hidden_dim: usize,
num_layers: usize,
message_passing_variant: MessagePassingVariant,
aggregation_function: AggregationFunction,
learning_rate: Float,
max_iter: usize,
tolerance: Float,
random_state: Option<u64>,
alpha: Float, // L2 regularization
dropout: Float,
attention_heads: usize, // For GAT
}
/// Trained state for GraphNeuralNetwork
#[derive(Debug, Clone)]
pub struct GraphNeuralNetworkTrained {
/// Weight matrices for each layer
layer_weights: Vec<Array2<Float>>,
/// Bias vectors for each layer
layer_biases: Vec<Array1<Float>>,
/// Attention weights (for GAT)
attention_weights: Vec<Array2<Float>>,
/// Output layer weights
output_weights: Array2<Float>,
/// Output layer bias
output_bias: Array1<Float>,
/// Network configuration
hidden_dim: usize,
num_layers: usize,
message_passing_variant: MessagePassingVariant,
aggregation_function: AggregationFunction,
attention_heads: usize,
n_features: usize,
n_outputs: usize,
/// Training history
loss_curve: Vec<Float>,
n_iter: usize,
}
impl GraphNeuralNetwork<Untrained> {
/// Create a new GraphNeuralNetwork instance
pub fn new() -> Self {
Self {
state: Untrained,
hidden_dim: 64,
num_layers: 2,
message_passing_variant: MessagePassingVariant::GCN,
aggregation_function: AggregationFunction::Mean,
learning_rate: 0.01,
max_iter: 200,
tolerance: 1e-4,
random_state: None,
alpha: 1e-4,
dropout: 0.0,
attention_heads: 8,
}
}
/// Set hidden dimension
pub fn hidden_dim(mut self, hidden_dim: usize) -> Self {
self.hidden_dim = hidden_dim;
self
}
/// Set number of layers
pub fn num_layers(mut self, num_layers: usize) -> Self {
self.num_layers = num_layers;
self
}
/// Set message passing variant
pub fn message_passing_variant(mut self, variant: MessagePassingVariant) -> Self {
self.message_passing_variant = variant;
self
}
/// Set aggregation function
pub fn aggregation_function(mut self, aggregation: AggregationFunction) -> Self {
self.aggregation_function = aggregation;
self
}
/// Set learning rate
pub fn learning_rate(mut self, learning_rate: Float) -> Self {
self.learning_rate = learning_rate;
self
}
/// Set maximum iterations
pub fn max_iter(mut self, max_iter: usize) -> Self {
self.max_iter = max_iter;
self
}
/// Set convergence tolerance
pub fn tolerance(mut self, tolerance: Float) -> Self {
self.tolerance = tolerance;
self
}
/// Set random state
pub fn random_state(mut self, random_state: Option<u64>) -> Self {
self.random_state = random_state;
self
}
/// Set L2 regularization
pub fn alpha(mut self, alpha: Float) -> Self {
self.alpha = alpha;
self
}
/// Set dropout rate
pub fn dropout(mut self, dropout: Float) -> Self {
self.dropout = dropout;
self
}
/// Set number of attention heads (for GAT)
pub fn attention_heads(mut self, attention_heads: usize) -> Self {
self.attention_heads = attention_heads;
self
}
}
impl Default for GraphNeuralNetwork<Untrained> {
fn default() -> Self {
Self::new()
}
}
impl Estimator for GraphNeuralNetwork<Untrained> {
type Config = ();
type Error = SklearsError;
type Float = Float;
fn config(&self) -> &Self::Config {
&()
}
}
/// Fit method for Graph Neural Networks with graph structure
impl GraphNeuralNetwork<Untrained> {
/// Fit the GNN with adjacency matrix, node features, and node labels
pub fn fit_graph(
self,
adjacency: &Array2<Float>, // Adjacency matrix [n_nodes, n_nodes]
node_features: &Array2<Float>, // Node features [n_nodes, n_features]
node_labels: &Array2<i32>, // Node labels [n_nodes, n_outputs]
) -> SklResult<GraphNeuralNetwork<GraphNeuralNetworkTrained>> {
let (n_nodes, n_features) = node_features.dim();
let (adj_nodes, adj_nodes_2) = adjacency.dim();
let (label_nodes, n_outputs) = node_labels.dim();
// Validation
if adj_nodes != adj_nodes_2 {
return Err(SklearsError::InvalidInput(
"Adjacency matrix must be square".to_string(),
));
}
if n_nodes != adj_nodes || n_nodes != label_nodes {
return Err(SklearsError::InvalidInput(
"Number of nodes must match across adjacency matrix, features, and labels"
.to_string(),
));
}
if n_nodes == 0 {
return Err(SklearsError::InvalidInput(
"Cannot fit with zero nodes".to_string(),
));
}
// Initialize random number generator
let mut rng = match self.random_state {
Some(seed) => ChaCha8Rng::seed_from_u64(seed),
None => ChaCha8Rng::from_rng(&mut rand::thread_rng()).unwrap(),
};
// Initialize network parameters
let (layer_weights, layer_biases, attention_weights, output_weights, output_bias) =
self.initialize_gnn_parameters(n_features, n_outputs, &mut rng)?;
let mut layer_weights = layer_weights;
let mut layer_biases = layer_biases;
let mut attention_weights = attention_weights;
let mut output_weights = output_weights;
let mut output_bias = output_bias;
// Normalize adjacency matrix (add self-loops and compute degree normalization)
let normalized_adj = self.normalize_adjacency(adjacency)?;
// Training loop
let mut loss_curve = Vec::new();
let node_features_owned = node_features.to_owned();
let node_labels_float = node_labels.map(|&x| x as Float);
for epoch in 0..self.max_iter {
// Forward pass
let node_embeddings = self.forward_pass_gnn(
&node_features_owned,
&normalized_adj,
&layer_weights,
&layer_biases,
&attention_weights,
)?;
// Output layer
let predictions = node_embeddings.dot(&output_weights.t()) + &output_bias;
// Compute loss (MSE for simplicity)
let loss = self.compute_graph_loss(&predictions, &node_labels_float);
loss_curve.push(loss);
// Check convergence
if epoch > 0 && (loss_curve[epoch - 1] - loss).abs() < self.tolerance {
break;
}
// Backward pass (simplified gradient descent)
self.backward_pass_gnn(
&node_features_owned,
&normalized_adj,
&node_labels_float,
&predictions,
&node_embeddings,
&mut layer_weights,
&mut layer_biases,
&mut attention_weights,
&mut output_weights,
&mut output_bias,
)?;
}
let trained_state = GraphNeuralNetworkTrained {
layer_weights,
layer_biases,
attention_weights,
output_weights,
output_bias,
hidden_dim: self.hidden_dim,
num_layers: self.num_layers,
message_passing_variant: self.message_passing_variant,
aggregation_function: self.aggregation_function,
attention_heads: self.attention_heads,
n_features,
n_outputs,
loss_curve,
n_iter: self.max_iter,
};
Ok(GraphNeuralNetwork {
state: trained_state,
hidden_dim: self.hidden_dim,
num_layers: self.num_layers,
message_passing_variant: self.message_passing_variant,
aggregation_function: self.aggregation_function,
learning_rate: self.learning_rate,
max_iter: self.max_iter,
tolerance: self.tolerance,
random_state: self.random_state,
alpha: self.alpha,
dropout: self.dropout,
attention_heads: self.attention_heads,
})
}
/// Initialize GNN parameters
fn initialize_gnn_parameters(
&self,
n_features: usize,
n_outputs: usize,
rng: &mut ChaCha8Rng,
) -> SklResult<(
Vec<Array2<Float>>, // layer_weights
Vec<Array1<Float>>, // layer_biases
Vec<Array2<Float>>, // attention_weights
Array2<Float>, // output_weights
Array1<Float>, // output_bias
)> {
let mut layer_weights = Vec::new();
let mut layer_biases = Vec::new();
let mut attention_weights = Vec::new();
// Initialize layers
for layer in 0..self.num_layers {
let input_dim = if layer == 0 {
match self.message_passing_variant {
MessagePassingVariant::GraphSAGE => n_features * 2, // Concatenated features
_ => n_features,
}
} else {
match self.message_passing_variant {
MessagePassingVariant::GraphSAGE => self.hidden_dim * 2, // Concatenated features
_ => self.hidden_dim,
}
};
let output_dim = self.hidden_dim;
// Xavier initialization
let scale = (2.0 / (input_dim + output_dim) as Float).sqrt();
let weight = Array2::random_using(
(output_dim, input_dim),
Normal::new(0.0, scale).unwrap(),
rng,
);
let bias = Array1::zeros(output_dim);
layer_weights.push(weight);
layer_biases.push(bias);
// Initialize attention weights for GAT
if self.message_passing_variant == MessagePassingVariant::GAT {
let attention_dim = output_dim * 2; // Concatenated features
let attention_weight = Array2::random_using(
(self.attention_heads, attention_dim),
Normal::new(0.0, scale).unwrap(),
rng,
);
attention_weights.push(attention_weight);
}
}
// Output layer
let output_scale = (2.0 / (self.hidden_dim + n_outputs) as Float).sqrt();
let output_weights = Array2::random_using(
(n_outputs, self.hidden_dim),
Normal::new(0.0, output_scale).unwrap(),
rng,
);
let output_bias = Array1::zeros(n_outputs);
Ok((
layer_weights,
layer_biases,
attention_weights,
output_weights,
output_bias,
))
}
/// Normalize adjacency matrix with self-loops and degree normalization
fn normalize_adjacency(&self, adjacency: &Array2<Float>) -> SklResult<Array2<Float>> {
let n_nodes = adjacency.nrows();
// Add self-loops: A_tilde = A + I
let mut adj_with_self_loops = adjacency.clone();
for i in 0..n_nodes {
adj_with_self_loops[[i, i]] += 1.0;
}
// Compute degree matrix D
let degrees: Array1<Float> = adj_with_self_loops.sum_axis(Axis(1));
// D^(-1/2) normalization for symmetric normalization
let mut normalized_adj = Array2::zeros((n_nodes, n_nodes));
for i in 0..n_nodes {
for j in 0..n_nodes {
if adj_with_self_loops[[i, j]] > 0.0 {
let norm_factor = (degrees[i] * degrees[j]).sqrt();
if norm_factor > 1e-10 {
normalized_adj[[i, j]] = adj_with_self_loops[[i, j]] / norm_factor;
}
}
}
}
Ok(normalized_adj)
}
/// Forward pass through GNN layers
fn forward_pass_gnn(
&self,
node_features: &Array2<Float>,
adjacency: &Array2<Float>,
layer_weights: &[Array2<Float>],
layer_biases: &[Array1<Float>],
attention_weights: &[Array2<Float>],
) -> SklResult<Array2<Float>> {
let mut current_features = node_features.clone();
for layer in 0..self.num_layers {
match self.message_passing_variant {
MessagePassingVariant::GCN => {
current_features = self.gcn_layer(
¤t_features,
adjacency,
&layer_weights[layer],
&layer_biases[layer],
)?;
}
MessagePassingVariant::GAT => {
current_features = self.gat_layer(
¤t_features,
adjacency,
&layer_weights[layer],
&layer_biases[layer],
&attention_weights[layer],
)?;
}
MessagePassingVariant::GraphSAGE => {
current_features = self.graphsage_layer(
¤t_features,
adjacency,
&layer_weights[layer],
&layer_biases[layer],
)?;
}
MessagePassingVariant::GIN => {
current_features = self.gin_layer(
¤t_features,
adjacency,
&layer_weights[layer],
&layer_biases[layer],
)?;
}
}
// Apply ReLU activation
current_features = current_features.map(|&x| x.max(0.0));
}
Ok(current_features)
}
/// Graph Convolutional Network layer
fn gcn_layer(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
// H' = σ(AHW + b)
let aggregated = adjacency.dot(features); // Message aggregation
let transformed = aggregated.dot(&weights.t()); // Linear transformation
let output = transformed + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// Graph Attention Network layer (simplified)
fn gat_layer(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
attention_weights: &Array2<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let hidden_dim = weights.nrows();
// Transform features
let transformed_features = features.dot(&weights.t());
// Simplified attention mechanism (using first attention head)
let mut attended_features = Array2::zeros((n_nodes, hidden_dim));
for i in 0..n_nodes {
let mut weighted_sum = Array1::zeros(hidden_dim);
let mut attention_sum = 0.0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 {
// Simplified attention score (dot product)
let attention_score = transformed_features
.row(i)
.dot(&transformed_features.row(j));
let attention_weight = attention_score.exp();
weighted_sum = weighted_sum + attention_weight * &transformed_features.row(j);
attention_sum += attention_weight;
}
}
if attention_sum > 1e-10 {
attended_features
.row_mut(i)
.assign(&(weighted_sum / attention_sum));
}
}
let output = attended_features + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// GraphSAGE layer (simplified)
fn graphsage_layer(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let n_features = features.ncols();
// Aggregate neighbor features
let mut aggregated_features = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
let mut neighbor_count = 0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &features.row(j);
neighbor_count += 1;
}
}
if neighbor_count > 0 {
aggregated_features
.row_mut(i)
.assign(&(neighbor_sum / neighbor_count as Float));
}
}
// Concatenate self features with aggregated neighbor features
let concatenated =
ndarray::concatenate![Axis(1), features.view(), aggregated_features.view()];
// Apply transformation
let output = concatenated.dot(&weights.t()) + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// Graph Isomorphism Network layer
fn gin_layer(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let n_features = features.ncols();
// Sum aggregation of neighbors
let mut aggregated_features = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &features.row(j);
}
}
aggregated_features.row_mut(i).assign(&neighbor_sum);
}
// GIN update: (1 + ε) * h_i + Σ h_j
let epsilon = 0.1; // Learnable parameter, simplified as constant
let updated_features = features * (1.0 + epsilon) + &aggregated_features;
// Apply MLP (simplified as single linear layer)
let output = updated_features.dot(&weights.t()) + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// Compute graph-based loss
fn compute_graph_loss(&self, predictions: &Array2<Float>, targets: &Array2<Float>) -> Float {
let diff = predictions - targets;
diff.map(|x| x * x).mean().unwrap()
}
/// Simplified backward pass for GNN
fn backward_pass_gnn(
&self,
node_features: &Array2<Float>,
adjacency: &Array2<Float>,
targets: &Array2<Float>,
predictions: &Array2<Float>,
node_embeddings: &Array2<Float>,
layer_weights: &mut [Array2<Float>],
layer_biases: &mut [Array1<Float>],
attention_weights: &mut [Array2<Float>],
output_weights: &mut Array2<Float>,
output_bias: &mut Array1<Float>,
) -> SklResult<()> {
let n_nodes = node_features.nrows() as Float;
// Output layer gradients
let output_error = predictions - targets;
let output_weight_grad = output_error.t().dot(node_embeddings) / n_nodes;
let output_bias_grad = output_error.mean_axis(Axis(0)).unwrap();
// Update output layer
*output_weights = output_weights.clone() - self.learning_rate * output_weight_grad;
*output_bias = output_bias.clone() - self.learning_rate * output_bias_grad;
// Simplified hidden layer updates (using output error as proxy)
for layer in (0..self.num_layers).rev() {
let input_features = if layer == 0 {
match self.message_passing_variant {
MessagePassingVariant::GraphSAGE => {
// For GraphSAGE, we need concatenated features
let n_nodes = node_features.nrows();
let n_features = node_features.ncols();
let mut aggregated = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
let mut neighbor_count = 0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &node_features.row(j);
neighbor_count += 1;
}
}
if neighbor_count > 0 {
aggregated
.row_mut(i)
.assign(&(neighbor_sum / neighbor_count as Float));
}
}
ndarray::concatenate![Axis(1), node_features.view(), aggregated.view()]
}
_ => node_features.clone(),
}
} else {
// Simplified: use node_embeddings as proxy for intermediate representations
match self.message_passing_variant {
MessagePassingVariant::GraphSAGE => {
// For GraphSAGE, concatenate embeddings with aggregated embeddings
let n_nodes = node_embeddings.nrows();
let n_features = node_embeddings.ncols();
let mut aggregated = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
let mut neighbor_count = 0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &node_embeddings.row(j);
neighbor_count += 1;
}
}
if neighbor_count > 0 {
aggregated
.row_mut(i)
.assign(&(neighbor_sum / neighbor_count as Float));
}
}
ndarray::concatenate![Axis(1), node_embeddings.view(), aggregated.view()]
}
_ => node_embeddings.clone(),
}
};
let hidden_error_temp = output_weights.t().dot(&output_error.t());
let hidden_error = hidden_error_temp.t();
let weight_grad = hidden_error.t().dot(&input_features) / n_nodes;
let bias_grad = hidden_error.mean_axis(Axis(0)).unwrap();
layer_weights[layer] = layer_weights[layer].clone() - self.learning_rate * weight_grad;
layer_biases[layer] = layer_biases[layer].clone() - self.learning_rate * bias_grad;
}
Ok(())
}
}
/// Prediction method for trained GNN
impl GraphNeuralNetwork<GraphNeuralNetworkTrained> {
/// Predict node labels given adjacency matrix and node features
pub fn predict_graph(
&self,
adjacency: &Array2<Float>,
node_features: &Array2<Float>,
) -> SklResult<Array2<i32>> {
let (n_nodes, n_features) = node_features.dim();
if n_features != self.state.n_features {
return Err(SklearsError::InvalidInput(
"Number of features doesn't match training data".to_string(),
));
}
// Normalize adjacency matrix
let normalized_adj = self.normalize_adjacency_trained(adjacency)?;
// Forward pass
let node_embeddings = self.forward_pass_gnn_trained(&node_features, &normalized_adj)?;
// Output predictions
let predictions_float =
node_embeddings.dot(&self.state.output_weights.t()) + &self.state.output_bias;
// Convert to integer predictions (simple thresholding)
let predictions = predictions_float.map(|&x| if x > 0.5 { 1 } else { 0 });
Ok(predictions)
}
/// Normalize adjacency matrix for trained model
fn normalize_adjacency_trained(&self, adjacency: &Array2<Float>) -> SklResult<Array2<Float>> {
let n_nodes = adjacency.nrows();
// Add self-loops
let mut adj_with_self_loops = adjacency.clone();
for i in 0..n_nodes {
adj_with_self_loops[[i, i]] += 1.0;
}
// Compute degrees
let degrees: Array1<Float> = adj_with_self_loops.sum_axis(Axis(1));
// Symmetric normalization
let mut normalized_adj = Array2::zeros((n_nodes, n_nodes));
for i in 0..n_nodes {
for j in 0..n_nodes {
if adj_with_self_loops[[i, j]] > 0.0 {
let norm_factor = (degrees[i] * degrees[j]).sqrt();
if norm_factor > 1e-10 {
normalized_adj[[i, j]] = adj_with_self_loops[[i, j]] / norm_factor;
}
}
}
}
Ok(normalized_adj)
}
/// Forward pass for trained model
fn forward_pass_gnn_trained(
&self,
node_features: &Array2<Float>,
adjacency: &Array2<Float>,
) -> SklResult<Array2<Float>> {
let mut current_features = node_features.clone();
for layer in 0..self.state.num_layers {
match self.state.message_passing_variant {
MessagePassingVariant::GCN => {
current_features = self.gcn_layer_trained(
¤t_features,
adjacency,
&self.state.layer_weights[layer],
&self.state.layer_biases[layer],
)?;
}
MessagePassingVariant::GAT => {
current_features = self.gat_layer_trained(
¤t_features,
adjacency,
&self.state.layer_weights[layer],
&self.state.layer_biases[layer],
&self.state.attention_weights[layer],
)?;
}
MessagePassingVariant::GraphSAGE => {
current_features = self.graphsage_layer_trained(
¤t_features,
adjacency,
&self.state.layer_weights[layer],
&self.state.layer_biases[layer],
)?;
}
MessagePassingVariant::GIN => {
current_features = self.gin_layer_trained(
¤t_features,
adjacency,
&self.state.layer_weights[layer],
&self.state.layer_biases[layer],
)?;
}
}
// Apply ReLU activation
current_features = current_features.map(|&x| x.max(0.0));
}
Ok(current_features)
}
/// GCN layer for trained model
fn gcn_layer_trained(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
let aggregated = adjacency.dot(features);
let transformed = aggregated.dot(&weights.t());
let output = transformed + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// GAT layer for trained model
fn gat_layer_trained(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
attention_weights: &Array2<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let hidden_dim = weights.nrows();
let transformed_features = features.dot(&weights.t());
let mut attended_features = Array2::zeros((n_nodes, hidden_dim));
for i in 0..n_nodes {
let mut weighted_sum = Array1::zeros(hidden_dim);
let mut attention_sum = 0.0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 {
let attention_score = transformed_features
.row(i)
.dot(&transformed_features.row(j));
let attention_weight = attention_score.exp();
weighted_sum = weighted_sum + attention_weight * &transformed_features.row(j);
attention_sum += attention_weight;
}
}
if attention_sum > 1e-10 {
attended_features
.row_mut(i)
.assign(&(weighted_sum / attention_sum));
}
}
let output = attended_features + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// GraphSAGE layer for trained model
fn graphsage_layer_trained(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let n_features = features.ncols();
let mut aggregated_features = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
let mut neighbor_count = 0;
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &features.row(j);
neighbor_count += 1;
}
}
if neighbor_count > 0 {
aggregated_features
.row_mut(i)
.assign(&(neighbor_sum / neighbor_count as Float));
}
}
let concatenated =
ndarray::concatenate![Axis(1), features.view(), aggregated_features.view()];
let output = concatenated.dot(&weights.t()) + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// GIN layer for trained model
fn gin_layer_trained(
&self,
features: &Array2<Float>,
adjacency: &Array2<Float>,
weights: &Array2<Float>,
bias: &Array1<Float>,
) -> SklResult<Array2<Float>> {
let n_nodes = features.nrows();
let n_features = features.ncols();
let mut aggregated_features = Array2::<Float>::zeros((n_nodes, n_features));
for i in 0..n_nodes {
let mut neighbor_sum = Array1::<Float>::zeros(n_features);
for j in 0..n_nodes {
if adjacency[[i, j]] > 0.0 && i != j {
neighbor_sum = neighbor_sum + &features.row(j);
}
}
aggregated_features.row_mut(i).assign(&neighbor_sum);
}
let epsilon = 0.1;
let updated_features = features * (1.0 + epsilon) + &aggregated_features;
let output = updated_features.dot(&weights.t()) + &bias.view().insert_axis(Axis(0));
Ok(output)
}
/// Get the loss curve from training
pub fn loss_curve(&self) -> &[Float] {
&self.state.loss_curve
}
/// Get training iterations
pub fn n_iter(&self) -> usize {
self.state.n_iter
}
/// Get network configuration
pub fn message_passing_variant(&self) -> MessagePassingVariant {
self.state.message_passing_variant
}
/// Get hidden dimension
pub fn hidden_dim(&self) -> usize {
self.state.hidden_dim
}
}
// ================================================================================================
// Tests for Graph Neural Networks
// ================================================================================================
#[cfg(test)]
mod gnn_tests {
use super::*;
use ndarray::array;
#[test]
fn test_gnn_basic_functionality() {
// Create a simple 3-node graph
let adjacency = array![[0.0, 1.0, 1.0], [1.0, 0.0, 1.0], [1.0, 1.0, 0.0]];
let node_features = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let node_labels = array![[1, 0], [0, 1], [1, 1]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(8)
.num_layers(2)
.message_passing_variant(MessagePassingVariant::GCN)
.max_iter(10)
.random_state(Some(42));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert!(predictions.iter().all(|&x| x == 0 || x == 1));
}
#[test]
fn test_gnn_gat_variant() {
let adjacency = array![[0.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 0.0]];
let node_features = array![[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]];
let node_labels = array![[1, 0], [0, 1], [1, 0]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(6)
.num_layers(1)
.message_passing_variant(MessagePassingVariant::GAT)
.attention_heads(2)
.max_iter(5)
.random_state(Some(123));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert_eq!(
trained_gnn.message_passing_variant(),
MessagePassingVariant::GAT
);
}
#[test]
fn test_gnn_graphsage_variant() {
let adjacency = array![
[0.0, 1.0, 1.0, 0.0],
[1.0, 0.0, 0.0, 1.0],
[1.0, 0.0, 0.0, 1.0],
[0.0, 1.0, 1.0, 0.0]
];
let node_features = array![[1.0, 2.0], [2.0, 1.0], [1.5, 1.5], [2.5, 0.5]];
let node_labels = array![[1, 0], [0, 1], [1, 1], [0, 0]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(4)
.num_layers(2)
.message_passing_variant(MessagePassingVariant::GraphSAGE)
.max_iter(8)
.random_state(Some(456));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (4, 2));
assert_eq!(
trained_gnn.message_passing_variant(),
MessagePassingVariant::GraphSAGE
);
}
#[test]
fn test_gnn_gin_variant() {
let adjacency = array![[0.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 0.0]];
let node_features = array![[2.0, 1.0], [1.0, 2.0], [1.5, 1.5]];
let node_labels = array![[1, 1], [0, 1], [1, 0]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(5)
.num_layers(1)
.message_passing_variant(MessagePassingVariant::GIN)
.max_iter(6)
.random_state(Some(789));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert_eq!(
trained_gnn.message_passing_variant(),
MessagePassingVariant::GIN
);
}
#[test]
fn test_gnn_configuration() {
let gnn = GraphNeuralNetwork::new()
.hidden_dim(32)
.num_layers(3)
.message_passing_variant(MessagePassingVariant::GAT)
.aggregation_function(AggregationFunction::Attention)
.learning_rate(0.005)
.max_iter(50)
.tolerance(1e-5)
.random_state(Some(42))
.alpha(0.01)
.dropout(0.1)
.attention_heads(4);
assert_eq!(gnn.hidden_dim, 32);
assert_eq!(gnn.num_layers, 3);
assert_eq!(gnn.message_passing_variant, MessagePassingVariant::GAT);
assert_eq!(gnn.aggregation_function, AggregationFunction::Attention);
assert_eq!(gnn.learning_rate, 0.005);
assert_eq!(gnn.max_iter, 50);
assert_eq!(gnn.tolerance, 1e-5);
assert_eq!(gnn.random_state, Some(42));
assert_eq!(gnn.alpha, 0.01);
assert_eq!(gnn.dropout, 0.1);
assert_eq!(gnn.attention_heads, 4);
}
#[test]
fn test_gnn_error_handling() {
let adjacency = array![[0.0, 1.0], [1.0, 0.0]]; // 2x2 matrix
let node_features = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]]; // 3 nodes
let node_labels = array![[1, 0], [0, 1]]; // 2 nodes
let gnn = GraphNeuralNetwork::new();
assert!(gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.is_err());
// Test non-square adjacency matrix
let adjacency_nonsquare = array![[0.0, 1.0, 1.0], [1.0, 0.0, 1.0]]; // 2x3 matrix
let node_features_2 = array![[1.0, 2.0], [2.0, 3.0]];
let node_labels_2 = array![[1, 0], [0, 1]];
let gnn2 = GraphNeuralNetwork::new();
assert!(gnn2
.fit_graph(&adjacency_nonsquare, &node_features_2, &node_labels_2)
.is_err());
// Test prediction with wrong number of features
let adjacency_valid = array![[0.0, 1.0], [1.0, 0.0]];
let node_features_valid = array![[1.0, 2.0], [2.0, 3.0]];
let node_labels_valid = array![[1, 0], [0, 1]];
let gnn3 = GraphNeuralNetwork::new().max_iter(1);
let trained_gnn = gnn3
.fit_graph(&adjacency_valid, &node_features_valid, &node_labels_valid)
.unwrap();
let node_features_wrong = array![[1.0, 2.0, 3.0], [2.0, 3.0, 1.0]]; // Wrong number of features
assert!(trained_gnn
.predict_graph(&adjacency_valid, &node_features_wrong)
.is_err());
}
#[test]
fn test_gnn_reproducibility() {
let adjacency = array![[0.0, 1.0, 1.0], [1.0, 0.0, 1.0], [1.0, 1.0, 0.0]];
let node_features = array![[1.0, 2.0], [2.0, 3.0], [3.0, 1.0]];
let node_labels = array![[1, 0], [0, 1], [1, 1]];
let gnn1 = GraphNeuralNetwork::new()
.hidden_dim(6)
.max_iter(5)
.random_state(Some(42));
let trained_gnn1 = gnn1
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let pred1 = trained_gnn1
.predict_graph(&adjacency, &node_features)
.unwrap();
let gnn2 = GraphNeuralNetwork::new()
.hidden_dim(6)
.max_iter(5)
.random_state(Some(42));
let trained_gnn2 = gnn2
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let pred2 = trained_gnn2
.predict_graph(&adjacency, &node_features)
.unwrap();
// With the same random state, predictions should be identical
for i in 0..pred1.nrows() {
for j in 0..pred1.ncols() {
assert_eq!(pred1[[i, j]], pred2[[i, j]]);
}
}
}
#[test]
fn test_gnn_adjacency_normalization() {
let adjacency = array![[0.0, 2.0, 1.0], [2.0, 0.0, 3.0], [1.0, 3.0, 0.0]]; // Weighted adjacency
let node_features = array![[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]];
let node_labels = array![[1, 0], [0, 1], [1, 1]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(4)
.max_iter(3)
.random_state(Some(123));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (3, 2));
assert!(predictions.iter().all(|&x| x == 0 || x == 1));
}
#[test]
fn test_gnn_single_node() {
let adjacency = array![[0.0]]; // Single node with no self-connection
let node_features = array![[1.0, 2.0]];
let node_labels = array![[1, 0]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(3)
.max_iter(2)
.random_state(Some(999));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (1, 2));
assert!(predictions.iter().all(|&x| x == 0 || x == 1));
}
#[test]
fn test_aggregation_function_enums() {
assert_eq!(AggregationFunction::Mean, AggregationFunction::Mean);
assert_ne!(AggregationFunction::Mean, AggregationFunction::Sum);
assert_ne!(AggregationFunction::Max, AggregationFunction::Attention);
}
#[test]
fn test_message_passing_variant_enums() {
assert_eq!(MessagePassingVariant::GCN, MessagePassingVariant::GCN);
assert_ne!(MessagePassingVariant::GCN, MessagePassingVariant::GAT);
assert_ne!(MessagePassingVariant::GraphSAGE, MessagePassingVariant::GIN);
}
#[test]
fn test_gnn_large_graph() {
// Test with a larger graph (5 nodes)
let adjacency = array![
[0.0, 1.0, 1.0, 0.0, 0.0],
[1.0, 0.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 0.0, 1.0, 1.0],
[0.0, 1.0, 1.0, 0.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 0.0]
];
let node_features = array![
[1.0, 2.0, 0.5],
[2.0, 1.0, 1.5],
[1.5, 1.5, 1.0],
[0.5, 2.5, 0.8],
[2.5, 0.5, 1.2]
];
let node_labels = array![[1, 0, 1], [0, 1, 0], [1, 1, 1], [0, 0, 1], [1, 0, 0]];
let gnn = GraphNeuralNetwork::new()
.hidden_dim(10)
.num_layers(2)
.message_passing_variant(MessagePassingVariant::GCN)
.max_iter(15)
.random_state(Some(42));
let trained_gnn = gnn
.fit_graph(&adjacency, &node_features, &node_labels)
.unwrap();
let predictions = trained_gnn
.predict_graph(&adjacency, &node_features)
.unwrap();
assert_eq!(predictions.dim(), (5, 3));
assert!(predictions.iter().all(|&x| x == 0 || x == 1));
assert_eq!(trained_gnn.hidden_dim(), 10);
}
}