rustyqlib/core/optimization/
bfgs.rs

1//! BFGS quasi-Newton: builds an inverse-Hessian approximation from
2//! gradient differences, giving superlinear convergence on smooth
3//! problems. The default choice for unconstrained smooth calibration.
4
5use super::line_search::backtracking;
6use super::numerics::{dot, norm_inf, numeric_gradient};
7use super::{OptimConfig, OptimResult};
8
9/// Minimize `f` from `x0` by BFGS. `grad` falls back to central finite
10/// differences when absent. The inverse-Hessian update is skipped
11/// whenever the curvature condition `s.y > 0` fails, which keeps the
12/// approximation positive definite under the Armijo-only line search.
13pub fn bfgs(
14    cfg: &OptimConfig,
15    f: &dyn Fn(&[f64]) -> f64,
16    grad: Option<&dyn Fn(&[f64]) -> Vec<f64>>,
17    x0: &[f64],
18) -> OptimResult {
19    let g_of = |x: &[f64]| match grad {
20        Some(g) => g(x),
21        None => numeric_gradient(f, x),
22    };
23    let n = x0.len();
24    let mut x = x0.to_vec();
25    let mut fx = f(&x);
26    let mut g = g_of(&x);
27    // h = approximate inverse Hessian, dense n x n, initialized to I
28    let mut h: Vec<Vec<f64>> = (0..n)
29        .map(|i| (0..n).map(|j| if i == j { 1.0 } else { 0.0 }).collect())
30        .collect();
31
32    for it in 0..cfg.max_iter {
33        if norm_inf(&g) <= cfg.tol {
34            return OptimResult { x, value: fx, iterations: it, converged: true };
35        }
36        // d = -H g
37        let mut dir: Vec<f64> = (0..n).map(|i| -dot(&h[i], &g)).collect();
38        let mut slope = dot(&g, &dir);
39        if slope >= 0.0 {
40            // numerical breakdown: reset the approximation
41            for (i, row) in h.iter_mut().enumerate() {
42                for (j, v) in row.iter_mut().enumerate() {
43                    *v = if i == j { 1.0 } else { 0.0 };
44                }
45            }
46            dir = g.iter().map(|gi| -gi).collect();
47            slope = dot(&g, &dir);
48        }
49        let (x_new, f_new) = match backtracking(f, &x, fx, &dir, slope, 1.0) {
50            Some(step) => step,
51            None => return OptimResult { x, value: fx, iterations: it, converged: false },
52        };
53        let g_new = g_of(&x_new);
54        let s: Vec<f64> = x_new.iter().zip(&x).map(|(a, b)| a - b).collect();
55        let y: Vec<f64> = g_new.iter().zip(&g).map(|(a, b)| a - b).collect();
56        let sy = dot(&s, &y);
57        if sy > 1e-12 {
58            // H <- (I - s y^T / sy) H (I - y s^T / sy) + s s^T / sy
59            let rho = 1.0 / sy;
60            let hy: Vec<f64> = (0..n).map(|i| dot(&h[i], &y)).collect();
61            let yhy = dot(&y, &hy);
62            for i in 0..n {
63                for j in 0..n {
64                    h[i][j] += -rho * (s[i] * hy[j] + hy[i] * s[j])
65                        + rho * rho * yhy * s[i] * s[j]
66                        + rho * s[i] * s[j];
67                }
68            }
69        }
70        x = x_new;
71        fx = f_new;
72        g = g_new;
73    }
74    OptimResult { x, value: fx, iterations: cfg.max_iter, converged: norm_inf(&g) <= cfg.tol }
75}
76
77#[cfg(test)]
78mod tests {
79    use super::*;
80
81    fn rosenbrock(x: &[f64]) -> f64 {
82        (1.0 - x[0]).powi(2) + 100.0 * (x[1] - x[0] * x[0]).powi(2)
83    }
84
85    #[test]
86    fn minimizes_rosenbrock_quickly() {
87        let f = |x: &[f64]| rosenbrock(x);
88        let r = bfgs(&OptimConfig::new(1e-8, 500), &f, None, &[-1.2, 1.0]);
89        assert!((r.x[0] - 1.0).abs() < 1e-5 && (r.x[1] - 1.0).abs() < 1e-5, "{r:?}");
90        assert!(r.iterations < 200, "took {} iterations", r.iterations);
91    }
92
93    #[test]
94    fn superlinear_beats_conjugate_gradient_on_rosenbrock() {
95        let f = |x: &[f64]| rosenbrock(x);
96        let cfg = OptimConfig::new(1e-6, 20_000);
97        let b = bfgs(&cfg, &f, None, &[-1.2, 1.0]);
98        let cg = super::super::conjugate_gradient::conjugate_gradient(&cfg, &f, None, &[-1.2, 1.0]);
99        assert!(b.iterations < cg.iterations, "bfgs {} vs cg {}", b.iterations, cg.iterations);
100    }
101
102    #[test]
103    fn four_dimensional_quadratic_converges() {
104        let f = |x: &[f64]| {
105            x.iter().enumerate().map(|(i, xi)| (i + 1) as f64 * (xi - i as f64).powi(2)).sum()
106        };
107        let r = bfgs(&OptimConfig::default(), &f, None, &[5.0; 4]);
108        assert!(r.converged, "{r:?}");
109        for (i, xi) in r.x.iter().enumerate() {
110            assert!((xi - i as f64).abs() < 1e-6, "{:?}", r.x);
111        }
112    }
113}
rustyqlib/core/optimization/bfgs.rs

rustyqlib/core/optimization/
bfgs.rs
