#[inline]
pub(crate) fn soft_threshold(value: f64, threshold: f64) -> f64 {
debug_assert!(threshold >= 0.0);
if value.is_nan() {
return value;
}
if value > threshold {
value - threshold
} else if value < -threshold {
value + threshold
} else {
0.0
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct ElasticNetPenalty {
l1: f64,
l2: f64,
}
impl ElasticNetPenalty {
pub(crate) fn new(alpha: f64, l1_ratio: f64) -> Self {
debug_assert!(alpha >= 0.0 && (0.0..=1.0).contains(&l1_ratio));
Self {
l1: alpha * l1_ratio,
l2: alpha * (1.0 - l1_ratio),
}
}
#[cfg(test)]
pub(crate) fn value(&self, coefficients: &[f64]) -> f64 {
crate::numeric::sum_in_order(coefficients.iter().map(|&coefficient| {
self.l1 * coefficient.abs() + 0.5 * self.l2 * coefficient * coefficient
}))
}
#[inline]
pub(crate) fn coordinate_minimizer(&self, target: f64, curvature: f64) -> f64 {
debug_assert!(curvature >= 0.0);
let denominator = curvature + self.l2;
if denominator <= 0.0 {
return 0.0;
}
soft_threshold(target, self.l1) / denominator
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn thresholding_shrinks_toward_zero_and_stops_there() {
assert_eq!(soft_threshold(5.0, 2.0), 3.0);
assert_eq!(soft_threshold(-5.0, 2.0), -3.0);
assert_eq!(soft_threshold(1.0, 2.0), 0.0);
assert_eq!(soft_threshold(-1.0, 2.0), 0.0);
assert_eq!(soft_threshold(7.5, 0.0), 7.5);
assert_eq!(soft_threshold(-7.5, 0.0), -7.5);
}
#[test]
fn a_removed_coefficient_is_a_positively_signed_zero() {
for &(value, threshold) in &[
(0.0_f64, 1.0_f64),
(-0.0, 1.0),
(2.0, 2.0),
(-2.0, 2.0),
(-1.5, 3.0),
(0.0, 0.0),
(-0.0, 0.0),
] {
let result = soft_threshold(value, threshold);
assert_eq!(result, 0.0, "value {value} threshold {threshold}");
assert!(
result.is_sign_positive(),
"value {value} threshold {threshold} produced a negative zero"
);
}
assert!((-2.0_f64 + 2.0).is_sign_positive() || soft_threshold(-2.0, 2.0) == 0.0);
}
#[test]
fn thresholding_is_continuous_monotone_and_a_contraction() {
let threshold = 1.25_f64;
let mut previous = f64::NEG_INFINITY;
let mut last_value = f64::NEG_INFINITY;
for step in -400..=400 {
let value = f64::from(step) / 100.0;
let result = soft_threshold(value, threshold);
assert!(result >= previous, "monotonicity at {value}");
assert!(result.abs() <= value.abs(), "expansion at {value}");
assert!(
result == 0.0 || result.signum() == value.signum(),
"sign flip at {value}"
);
if last_value.is_finite() {
assert!(
(result - previous).abs() <= (value - last_value).abs() + 1.0e-15,
"not a contraction at {value}"
);
}
previous = result;
last_value = value;
}
}
#[test]
fn a_non_finite_input_propagates_rather_than_reading_as_removed() {
assert!(soft_threshold(f64::NAN, 1.0).is_nan());
assert_eq!(soft_threshold(f64::INFINITY, 1.0), f64::INFINITY);
assert_eq!(soft_threshold(f64::NEG_INFINITY, 1.0), f64::NEG_INFINITY);
}
#[test]
fn the_two_terms_are_split_the_way_the_contract_documents() {
let penalty = ElasticNetPenalty::new(2.0, 0.25);
assert_eq!(penalty, ElasticNetPenalty { l1: 0.5, l2: 1.5 });
assert_eq!(ElasticNetPenalty::new(3.0, 1.0).l2, 0.0);
assert_eq!(ElasticNetPenalty::new(3.0, 0.0).l1, 0.0);
assert_eq!(
ElasticNetPenalty::new(0.0, 0.5),
ElasticNetPenalty::new(0.0, 1.0)
);
}
#[test]
fn the_penalty_value_matches_its_written_form() {
let penalty = ElasticNetPenalty::new(2.0, 0.25);
let coefficients = [1.0_f64, -2.0, 0.0, 0.5];
let expected = coefficients
.iter()
.map(|value| 0.5 * value.abs() + 0.5 * 1.5 * value * value)
.sum::<f64>();
assert!((penalty.value(&coefficients) - expected).abs() <= 1.0e-15);
assert_eq!(penalty.value(&[]), 0.0);
}
#[test]
fn the_coordinate_minimizer_is_the_exact_argmin_of_its_own_one_variable_problem() {
let penalty = ElasticNetPenalty::new(0.8, 0.6);
for target_step in -30..=30 {
for curvature_step in 1..=12 {
let target = f64::from(target_step) / 5.0;
let curvature = f64::from(curvature_step) / 4.0;
let objective = |value: f64| {
0.5 * curvature * value * value - target * value
+ penalty.value(std::slice::from_ref(&value))
};
let minimizer = penalty.coordinate_minimizer(target, curvature);
let best = objective(minimizer);
for offset in -2000..=2000 {
let candidate = minimizer + f64::from(offset) / 1000.0;
assert!(
objective(candidate) >= best - 1.0e-12,
"target {target} curvature {curvature}: {candidate} beats {minimizer}"
);
}
}
}
}
#[test]
fn a_coordinate_with_no_curvature_and_no_l2_term_stays_at_zero() {
let lasso = ElasticNetPenalty::new(1.0, 1.0);
assert_eq!(lasso.coordinate_minimizer(0.0, 0.0), 0.0);
assert_eq!(lasso.coordinate_minimizer(5.0, 0.0), 0.0);
assert!(lasso.coordinate_minimizer(5.0, 0.0).is_sign_positive());
let elastic = ElasticNetPenalty::new(1.0, 0.5);
assert_eq!(elastic.coordinate_minimizer(5.0, 0.0), (5.0 - 0.5) / 0.5);
}
#[test]
fn a_large_enough_l1_term_removes_every_coordinate() {
let penalty = ElasticNetPenalty::new(1.0e6, 1.0);
for target_step in -100..=100 {
let target = f64::from(target_step);
let value = penalty.coordinate_minimizer(target, 1.0);
assert_eq!(value, 0.0, "target {target}");
assert!(value.is_sign_positive());
}
}
}