gbrt_rs/objective/

binary_classification.rs

//! Binary Classification Objectives for Gradient Boosting
//! 
//! This module provides objective functions for binary classification tasks,
//! where the target variable is in {0, 1}. The primary implementation is
//! [`LogLossObjective`], which optimizes the binary cross-entropy (logistic loss).
//! 
//! # Mathematical Background
//! 
//! Binary classification in gradient boosting works by:
//! 1. Computing raw predictions (log-odds)
//! 2. Applying sigmoid transformation to get probabilities
//! 3. Using log loss to measure prediction quality
//! 4. Computing gradients and Hessians for boosting iterations
//! 
//! # Log Loss Formula
//! 
//! For true label `y ∈ {0, 1}` and predicted probability `p ∈ (0, 1)`:
//! 
//! ```text
//! L(y, p) = -[y·log(p) + (1-y)·log(1-p)]
//! ```
//! 
//! The raw prediction (before sigmoid) represents log-odds:
//! 
//! ```text
//! log(p/(1-p)) = raw_prediction
//! p = 1/(1 + exp(-raw_prediction))  // sigmoid function
//! ```

use super::{Objective, ObjectiveError, ObjectiveResult};
use serde::{Deserialize, Serialize};
use std::fmt;

/// Extended trait for binary classification objectives.
/// 
/// This trait extends the base [`Objective`] trait with methods specific to
/// binary classification, such as probability-to-class conversion and accuracy
/// metrics.
/// 
/// # Type Conversions
/// 
/// - Raw predictions are log-odds (any real number)
/// - Transformed predictions are probabilities in (0, 1)
/// - Class predictions are 0.0 or 1.0 based on threshold
pub trait BinaryClassificationObjective: Objective {
    /// Converts predicted probabilities to class labels (0.0 or 1.0).
    /// 
    /// # Parameters
    /// - `probabilities`: Slice of probability values in [0, 1]
    /// - `threshold`: Decision threshold (default: 0.5)
    /// 
    /// # Returns
    /// Vector of class predictions (0.0 or 1.0)
    /// 
    /// # Panics
    /// Panics if threshold is not in [0, 1]
    fn predict_classes(&self, probabilities: &[f64], threshold: f64) -> Vec<f64>;
    
    /// Computes classification accuracy.
    /// 
    /// # Parameters
    /// - `y_true`: True binary labels (0.0 or 1.0)
    /// - `y_pred_proba`: Predicted probabilities
    /// - `threshold`: Decision threshold for classification
    /// 
    /// # Returns
    /// Accuracy score in [0, 1]
    fn accuracy(&self, y_true: &[f64], y_pred_proba: &[f64], threshold: f64) -> ObjectiveResult<f64>;
    
    /// Computes binary cross-entropy loss.
    /// 
    /// This is the same as [`Objective::loss`] but operates on probabilities
    /// instead of raw predictions.
    fn binary_cross_entropy(&self, y_true: &[f64], y_pred_proba: &[f64]) -> ObjectiveResult<f64>;
}

/// Binary logistic loss objective for binary classification.
/// 
/// This objective implements the log loss (binary cross-entropy) function with
/// numerical stability improvements. It uses epsilon-clipping to avoid log(0)
/// and provides sigmoid transformations for probability calculations.
/// 
/// # Mathematical Details
/// 
/// - **Sigmoid**: `σ(x) = 1/(1 + exp(-x))`
/// - **Loss**: `L(y, p) = -[y·log(p) + (1-y)·log(1-p)]` where p = σ(raw)
/// - **Gradient**: `∂L/∂raw = p - y`
/// - **Hessian**: `∂²L/∂raw² = p·(1-p)`
/// - **Initial Prediction**: log-odds of positive class proportion
/// 
/// # Numerical Stability
/// 
/// Probabilities are clipped to `[epsilon, 1-epsilon]` where `epsilon = 1e-15`
/// by default. This prevents infinite loss values while maintaining precision.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LogLossObjective {
    epsilon: f64,
}

impl LogLossObjective {
    /// Creates a new LogLossObjective with default epsilon (1e-15).
    pub fn new() -> Self {
        Self { epsilon: 1e-15 }
    }
    
    /// Sets custom epsilon for numerical stability.
    ///
    /// # Parameters
    /// - `epsilon`: Clipping threshold (default: 1e-15)
    ///
    /// # Panics
    /// Model training may produce NaN if epsilon is too small (≤ 1e-20)
    pub fn with_epsilon(mut self, epsilon: f64) -> Self {
        self.epsilon = epsilon;
        self
    }
    
    /// Sigmoid activation function.
    /// 
    /// Maps log-odds to probability in (0, 1).
    fn sigmoid(&self, x: f64) -> f64 {
        1.0 / (1.0 + (-x).exp())
    }
    
    /// Clips probability to avoid numerical instability in log calculations.
    fn clip_probability(&self, p: f64) -> f64 {
        p.max(self.epsilon).min(1.0 - self.epsilon)
    }
    
    /// Converts log-odds to probability using sigmoid.
    fn log_odds_to_probability(&self, log_odds: f64) -> f64 {
        self.sigmoid(log_odds)
    }
}

impl Default for LogLossObjective {
    fn default() -> Self {
        Self::new()
    }
}

impl Objective for LogLossObjective {
    /// Computes binary cross-entropy loss.
    fn loss(&self, y_true: &[f64], y_pred: &[f64]) -> ObjectiveResult<f64> {
        self.binary_cross_entropy(y_true, y_pred)
    }

    /// Computes gradient of loss with respect to raw predictions.
    /// 
    /// Gradient = σ(raw) - y, where y ∈ {0, 1}
    fn gradient(&self, y_true: &[f64], y_pred: &[f64]) -> ObjectiveResult<Vec<f64>> {
        if y_true.len() != y_pred.len() {
            return Err(ObjectiveError::InvalidInput(
                "Targets and predictions must have the same length".to_string()
            ));
        }
        
        Ok(y_true.iter()
            .zip(y_pred.iter())
            .map(|(&y, &y_hat)| {
                let p = self.sigmoid(y_hat);
                p - y
            })
            .collect())
    }
    
    /// Computes Hessian (second derivative) of loss.
    /// 
    /// Hessian = p·(1-p) where p = σ(raw)
    fn hessian(&self, _y_true: &[f64], y_pred: &[f64]) -> ObjectiveResult<Vec<f64>> {
        Ok(y_pred.iter()
            .map(|&y_hat| {
                let p = self.sigmoid(y_hat);
                p * (1.0 - p)
            })
            .collect())
    }
    
    /// Computes gradient and Hessian in a single pass for efficiency.
    fn gradient_hessian(&self, y_true: &[f64], y_pred: &[f64]) -> ObjectiveResult<(Vec<f64>, Vec<f64>)> {
        let gradient = self.gradient(y_true, y_pred)?;
        
        let hessian = y_pred.iter()
            .map(|&y_hat| {
                let p = self.sigmoid(y_hat);
                p * (1.0 - p)
            })
            .collect();
        
        Ok((gradient, hessian))
    }
    
    /// Transforms raw predictions to probabilities using sigmoid.
    fn transform(&self, y_pred: &[f64]) -> Vec<f64> {
        y_pred.iter()
            .map(|&x| self.log_odds_to_probability(x))
            .collect()
    }
    
    fn name(&self) -> &str {
        "log_loss"
    }
    
    fn is_regression(&self) -> bool {
        false
    }
    
    fn is_classification(&self) -> bool {
        true
    }
    
    /// Computes optimal initial prediction as log-odds of positive class proportion.
    fn initial_prediction(&self, y_true: &[f64]) -> ObjectiveResult<f64> {
        if y_true.is_empty() {
            return Err(ObjectiveError::InvalidInput("Empty targets".to_string()));
        }
        
        // For classification, use log-odds of positive class proportion
        let positive_count = y_true.iter().filter(|&&y| y == 1.0).count() as f64;
        let positive_ratio = positive_count / y_true.len() as f64;
        let positive_ratio_clamped = positive_ratio.max(self.epsilon).min(1.0 - self.epsilon);
        
        Ok((positive_ratio_clamped / (1.0 - positive_ratio_clamped)).ln())
    }
    
    /// Validates that targets are binary (0.0 or 1.0).
    fn validate_targets(&self, y_true: &[f64]) -> ObjectiveResult<()> {
        if y_true.is_empty() {
            return Err(ObjectiveError::InvalidInput("Empty targets".to_string()));
        }
        
        // Check that targets are either 0 or 1
        for &y in y_true {
            if y != 0.0 && y != 1.0 {
                return Err(ObjectiveError::InvalidInput(
                    format!("Binary classification targets must be 0 or 1, got {}", y)
                ));
            }
        }
        
        Ok(())
    }
}

impl BinaryClassificationObjective for LogLossObjective {
    /// Converts probabilities to binary class predictions using threshold.
    fn predict_classes(&self, probabilities: &[f64], threshold: f64) -> Vec<f64> {
        if !(0.0..=1.0).contains(&threshold) {
            panic!("Threshold must be between 0 and 1");
        }
        
        probabilities.iter()
            .map(|&p| if p >= threshold { 1.0 } else { 0.0 })
            .collect()
    }
   
    /// Computes classification accuracy.
    fn accuracy(&self, y_true: &[f64], y_pred_proba: &[f64], threshold: f64) -> ObjectiveResult<f64> {
        if y_true.len() != y_pred_proba.len() {
            return Err(ObjectiveError::InvalidInput(
                "Targets and predictions must have the same length".to_string()
            ));
        }
        
        self.validate_targets(y_true)?;
        
        let predictions = self.predict_classes(y_pred_proba, threshold);
        let correct = y_true.iter()
            .zip(predictions.iter())
            .filter(|&(&true_val, &pred_val)| (true_val - pred_val).abs() < 1e-10)
            .count();
        
        Ok(correct as f64 / y_true.len() as f64)
    }
    
    /// Computes binary cross-entropy loss with epsilon-clipping for stability.
    fn binary_cross_entropy(&self, y_true: &[f64], y_pred_proba: &[f64]) -> ObjectiveResult<f64> {
        if y_true.len() != y_pred_proba.len() {
            return Err(ObjectiveError::InvalidInput(
                "Targets and predictions must have the same length".to_string()
            ));
        }
        
        self.validate_targets(y_true)?;
        
        let loss: f64 = y_true.iter()
            .zip(y_pred_proba.iter())
            .map(|(&y, &p)| {
                let p_clipped = self.clip_probability(p);
                - (y * p_clipped.ln() + (1.0 - y) * (1.0 - p_clipped).ln())
            })
            .sum();
        
        Ok(loss / y_true.len() as f64)
    }
}

impl fmt::Display for LogLossObjective {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "LogLoss(epsilon={})", self.epsilon)
    }
}