miniboosts 0.2.1

MiniBoosts: A collection of boosting algorithms written in Rust 🦀
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140


/// This trait defines the loss functions.
pub trait LossFunction {
    /// Loss value for a single point.
    fn eval_at_point(&self, prediction: f64, true_value: f64) -> f64;


    /// Loss value for a single point.
    fn eval(&self, predictions: &[f64], target: &[f64]) -> f64 {
        let n_items = predictions.len();

        assert_eq!(n_items, target.len());


        predictions.into_iter()
            .zip(target)
            .map(|(&p, &y)| self.eval_at_point(p, y))
            .sum::<f64>()
            / n_items as f64
    }

    /// Gradient vector at current point.
    fn gradient(&self, predictions: &[f64], target: &[f64]) -> Vec<f64>;


    /// Best coffecient for the newly-attained hypothesis.
    fn best_coefficient(
        &self, 
        residuals: &[f64],
        predictions: &[f64],
    ) -> f64;
}


/// Some well-known loss functions.
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum GBMLoss {
    /// `L1`-loss.
    /// This loss function is also known as
    /// **Least Absolute Deviation (LAD)**.
    L1,

    /// `L2`-loss.
    /// This loss function is also known as
    /// **Mean Squared Error (MSE)**.
    L2,
}


impl LossFunction for GBMLoss {
    fn eval_at_point(&self, prediction: f64, true_value: f64) -> f64 {
        match self {
            Self::L1 => (prediction - true_value).abs(),
            Self::L2 => (prediction - true_value).powi(2),
        }
    }


    fn gradient(&self, predictions: &[f64], target: &[f64]) -> Vec<f64>
    {
        let n_sample = predictions.len() as f64;
        assert_eq!(n_sample as usize, target.len());


        match self {
            Self::L1 => {
                predictions.into_iter()
                    .zip(target)
                    .map(|(p, y)| (p - y).signum() / n_sample)
                    .collect()
            },
            Self::L2 => {
                predictions.into_iter()
                    .zip(target)
                    .map(|(p, y)| 2.0 * (p - y) / n_sample)
                    .collect()
            }
        }
    }


    fn best_coefficient(
        &self, 
        residuals: &[f64],
        predictions: &[f64],
    ) -> f64
    {
        match self {
            Self::L1 => {
                let mut items = residuals.into_iter()
                    .zip(predictions)
                    .filter_map(|(&r, &p)| 
                        if p == 0.0 { None } else { Some((p.abs(), r / p)) }
                    )
                    .collect::<Vec<_>>();

                weighted_median(&mut items[..])
            },
            Self::L2 => {
                let r_sum = residuals.into_iter().sum::<f64>();
                let p_sum = predictions.into_iter().sum::<f64>();

                assert!(p_sum != 0.0);

                r_sum / p_sum
            },
        }
    }
}


/// Returns a median of the given array
fn weighted_median(items: &mut [(f64, f64)]) -> f64 {
    let n_items = items.len();

    assert!(n_items > 0);

    if n_items == 1 {
        return items[0].1;
    }
    items.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());

    let total_weight = items.iter()
        .map(|(w, _)| *w)
        .sum::<f64>();


    let mut partial_sum = 0.0_f64;
    let mut iter = items.into_iter();
    while let Some((w, x)) = iter.next() {
        partial_sum += *w;
        if partial_sum >= 0.5 * total_weight {
            return *x;
        }
    }

    unreachable!()
}