1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
//! Adapts unconstrained solvers to the [`Optimizer`] type slot.
//!
//! Does not implement L-BFGS, Nelder–Mead, or a trust-region method. [`Lbfgs`],
//! [`NelderMead`], and [`TrustRegion`] map user-unit [`crate::Interval`] through a logit so
//! the solver stays unconstrained. Positive intervals (`lo > 0`) use a log-uniform logit,
//! matching restart sampling. The unconstrained coordinate is scaled so the Jacobian is 1
//! at the interval midpoint. When a gradient solver asks for cost and gradient at the same
//! point, one [`crate::Differentiable::value_and_gradient_into`] call fills both.
//! [`TrustRegion`] also maps the analytic Hessian to logit coordinates. [`NelderMead`]
//! evaluates [`crate::Objective::value`] only. [`Adam`] is a mini-batch loop for
//! [`crate::Svgp`] and does not implement [`Optimizer`]. [`FastSimulatedAnnealing`] is a
//! homemade value-only solver that walks log-`θ` with Cauchy / Metropolis steps instead of
//! a logit map. The first evaluation and each restart use [`crate::Objective::value`]; each
//! coordinate step uses [`crate::Objective::value_at_changes`].
use NonZeroU32;
pub use Adam;
pub use ;
pub use Lbfgs;
pub use ;
pub use NelderMead;
pub use TrustRegion;
use crateGprError;
/// Represents the result of [`Optimizer::minimize`].
///
/// See the example on [`Optimizer`].
/// Represents the hyperparameter optimizer.
///
/// `P` is the objective this algorithm can minimize. [`Lbfgs`]
/// requires [`crate::Differentiable`], [`TrustRegion`]
/// [`crate::TwiceDifferentiable`], and [`NelderMead`] and
/// [`FastSimulatedAnnealing`] only [`crate::Objective`]. Every one searches
/// inside [`crate::Objective::fill_intervals`].
///
/// A user optimizer implements this generically over the capability it
/// needs. It can read the intervals and call a built-in optimizer on the
/// same objective.
///
/// # Examples
///
/// ```rust
/// use gprx::kernel::{KernelSpec, RbfKernel};
/// use gprx::{
/// Differentiable, GaussianLikelihood, Gpr, GprError, Interval, Lbfgs, NelderMead,
/// OptResult, Optimizer,
/// };
///
/// /// A short Nelder–Mead pass, then L-BFGS from its result.
/// #[derive(Clone, Debug)]
/// struct Polish;
///
/// impl<P: Differentiable> Optimizer<P> for Polish {
/// // Neither stage reports changed coordinates, so the fit rebuilds the
/// // whole kernel at each step (the default, written out here).
/// const USES_CHANGE_INDICES: bool = false;
///
/// fn minimize(&self, objective: &mut P, init: &[f64]) -> Result<OptResult, GprError> {
/// let mut intervals = vec![Interval::DEFAULT_POSITIVE; objective.num_params()];
/// objective.fill_intervals(&mut intervals)?;
/// let coarse = NelderMead::new()
/// .with_max_iterations(20)
/// .minimize(objective, init)?;
/// let fine = Lbfgs::new().minimize(objective, &coarse.params)?;
/// // Keep the better point; count the iterations of both stages.
/// let best = if fine.value <= coarse.value { fine.params } else { coarse.params };
/// Ok(OptResult {
/// params: best,
/// value: fine.value.min(coarse.value),
/// iterations: coarse.iterations + fine.iterations,
/// })
/// }
/// }
///
/// # fn main() -> Result<(), GprError> {
/// let fitted = Gpr::new(
/// KernelSpec::from(RbfKernel::new(1.0)?),
/// GaussianLikelihood::new(0.1)?,
/// )
/// .with_optimizer(Polish)
/// .fit(&[0.0, 1.0, 2.0, 3.0], 4, 1, &[0.0, 0.8, 0.9, 0.1])
/// .map_err(|(_, e)| e)?;
/// assert!(fitted.neg_log_marginal_likelihood()?.is_finite());
/// # Ok(())
/// # }
/// ```
/// Represents no hyperparameter search: the model keeps the `θ` it was built with.
///
/// It is not an [`Optimizer`]. A trainer whose optimizer slot is `Fixed`
/// factors instead of fitting: [`crate::Gpr<Fixed>::factor`],
/// [`crate::Sgpr<Fixed>::factor`], and [`crate::Svgp<Fixed>::factor`]
/// (`Fixed` is the default slot of [`crate::Svgp`]). On the exact models,
/// [`crate::FittedGpr`] and [`crate::OnlineGpr`] with this slot refactor at
/// their current `θ` on `refit`; the sparse models have no `refit` for
/// `Fixed`. A loaded model carries this slot.
///
/// See the example on [`crate::Gpr::factor`].
;
/// Checks that `init` has the objective's `n` parameters.
/// The restart loop every built-in optimizer shares.
///
/// Checks `init`'s length, reads the intervals, runs once from
/// `first_start(init)`, then once from each start `sample` draws with the
/// restart seed. `run` receives `None` for the first run and `Some(k)` for
/// restart `k` (1-based), and adds its result to the best so far. The first
/// run's error is returned; a restart's error only drops that restart. Fails
/// when no run produced a result.
Sized>