1use ndarray::{Array1, Array2, ArrayView2, Axis};
9use solow_core::{Error, Result};
10
11#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
13#[derive(Copy, Clone, Debug, PartialEq, Eq)]
14pub enum PowerMethod {
15 YeoJohnson,
17 BoxCox,
19}
20
21#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
23#[derive(Clone, Debug, PartialEq)]
24pub struct PowerTransformer {
25 pub lambdas: Array1<f64>,
27 pub method: PowerMethod,
29 pub standardize: bool,
31 pub post_mean: Array1<f64>,
33 pub post_std: Array1<f64>,
35}
36
37impl PowerTransformer {
38 pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
40 Self::fit_with(x, PowerMethod::YeoJohnson, true)
41 }
42
43 pub fn fit_with(
45 x: ArrayView2<'_, f64>,
46 method: PowerMethod,
47 standardize: bool,
48 ) -> Result<Self> {
49 if x.nrows() == 0 || x.ncols() == 0 {
50 return Err(Error::Value(
51 "PowerTransformer::fit_with: x must be non-empty".into(),
52 ));
53 }
54 if matches!(method, PowerMethod::BoxCox) {
55 for &v in x.iter() {
56 if v <= 0.0 {
57 return Err(Error::Value(
58 "PowerTransformer::fit_with: Box-Cox requires strictly positive inputs"
59 .into(),
60 ));
61 }
62 }
63 }
64 let d = x.ncols();
65 let mut lambdas = Array1::<f64>::zeros(d);
66 for j in 0..d {
67 let col: Vec<f64> = x.column(j).iter().copied().collect();
68 lambdas[j] = fit_lambda(&col, method);
69 }
70 let tr = transform_matrix(x, &lambdas, method);
72 let (post_mean, post_std) = if standardize {
73 column_stats(&tr)
74 } else {
75 (Array1::<f64>::zeros(d), Array1::<f64>::ones(d))
76 };
77 Ok(Self {
78 lambdas,
79 method,
80 standardize,
81 post_mean,
82 post_std,
83 })
84 }
85
86 pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
88 if x.ncols() != self.lambdas.len() {
89 return Err(Error::Shape(format!(
90 "PowerTransformer::transform: expected {} columns, got {}",
91 self.lambdas.len(),
92 x.ncols()
93 )));
94 }
95 let mut tr = transform_matrix(x, &self.lambdas, self.method);
96 if self.standardize {
97 for (j, mut col) in tr.axis_iter_mut(Axis(1)).enumerate() {
98 for v in col.iter_mut() {
99 *v = (*v - self.post_mean[j]) / self.post_std[j];
100 }
101 }
102 }
103 Ok(tr)
104 }
105
106 pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
108 let pt = Self::fit(x)?;
109 let tr = pt.transform(x)?;
110 Ok((pt, tr))
111 }
112}
113
114fn transform_matrix(
115 x: ArrayView2<'_, f64>,
116 lambdas: &Array1<f64>,
117 method: PowerMethod,
118) -> Array2<f64> {
119 let (n, d) = (x.nrows(), x.ncols());
120 let mut out = Array2::<f64>::zeros((n, d));
121 for i in 0..n {
122 for j in 0..d {
123 out[[i, j]] = apply(x[[i, j]], lambdas[j], method);
124 }
125 }
126 out
127}
128
129fn apply(v: f64, lambda: f64, method: PowerMethod) -> f64 {
130 match method {
131 PowerMethod::YeoJohnson => {
132 if v >= 0.0 {
133 if lambda.abs() < 1e-9 {
134 (1.0 + v).ln()
135 } else {
136 ((1.0 + v).powf(lambda) - 1.0) / lambda
137 }
138 } else if (lambda - 2.0).abs() < 1e-9 {
139 -(1.0 - v).ln()
140 } else {
141 -(((1.0 - v).powf(2.0 - lambda) - 1.0) / (2.0 - lambda))
142 }
143 }
144 PowerMethod::BoxCox => {
145 if lambda.abs() < 1e-9 {
146 v.ln()
147 } else {
148 (v.powf(lambda) - 1.0) / lambda
149 }
150 }
151 }
152}
153
154fn fit_lambda(col: &[f64], method: PowerMethod) -> f64 {
155 let grid_coarse: Vec<f64> = (-40..=40).map(|i| i as f64 * 0.1).collect(); let (l_coarse, _) = grid_search(col, &grid_coarse, method);
159 let grid_fine: Vec<f64> = (0..=200)
160 .map(|i| l_coarse - 0.2 + i as f64 * 0.002)
161 .collect();
162 let (best_l, _) = grid_search(col, &grid_fine, method);
163 best_l
164}
165
166fn grid_search(col: &[f64], grid: &[f64], method: PowerMethod) -> (f64, f64) {
167 let mut best_l = 0.0_f64;
168 let mut best_ll = f64::NEG_INFINITY;
169 for &l in grid {
170 let ll = yeo_johnson_log_likelihood(col, l, method);
171 if ll > best_ll {
172 best_ll = ll;
173 best_l = l;
174 }
175 }
176 (best_l, best_ll)
177}
178
179fn yeo_johnson_log_likelihood(col: &[f64], lambda: f64, method: PowerMethod) -> f64 {
180 let n = col.len() as f64;
181 let mut log_j = 0.0_f64;
183 let mut transformed = Vec::with_capacity(col.len());
184 for &v in col {
185 transformed.push(apply(v, lambda, method));
186 log_j += log_derivative(v, lambda, method);
187 }
188 let mean: f64 = transformed.iter().sum::<f64>() / n;
190 let var: f64 = transformed.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / n;
191 if var <= 0.0 {
192 return f64::NEG_INFINITY;
193 }
194 -0.5 * n * var.ln() + log_j
195}
196
197fn log_derivative(v: f64, lambda: f64, method: PowerMethod) -> f64 {
198 match method {
199 PowerMethod::YeoJohnson => {
200 if v >= 0.0 {
201 (lambda - 1.0) * (1.0 + v).ln()
202 } else {
203 (1.0 - lambda) * (1.0 - v).ln()
204 }
205 }
206 PowerMethod::BoxCox => (lambda - 1.0) * v.ln(),
207 }
208}
209
210fn column_stats(x: &Array2<f64>) -> (Array1<f64>, Array1<f64>) {
211 let (n, d) = x.dim();
212 let mut mean = Array1::<f64>::zeros(d);
213 for j in 0..d {
214 for i in 0..n {
215 mean[j] += x[[i, j]];
216 }
217 mean[j] /= n as f64;
218 }
219 let mut std = Array1::<f64>::zeros(d);
220 for j in 0..d {
221 for i in 0..n {
222 std[j] += (x[[i, j]] - mean[j]).powi(2);
223 }
224 std[j] = (std[j] / (n as f64 - 1.0).max(1.0)).sqrt().max(1e-12);
225 }
226 (mean, std)
227}
228
229#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
235#[derive(Copy, Clone, Debug, PartialEq, Eq)]
236pub enum QuantileOutput {
237 Uniform,
239 Normal,
241}
242
243#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
245#[derive(Clone, Debug, PartialEq)]
246pub struct QuantileTransformer {
247 pub quantiles: Vec<Vec<f64>>,
249 pub references: Vec<f64>,
252 pub output: QuantileOutput,
254 pub n_quantiles: usize,
256}
257
258impl QuantileTransformer {
259 pub fn fit(x: ArrayView2<'_, f64>) -> Result<Self> {
262 Self::fit_with(x, 1000.min(x.nrows()), QuantileOutput::Uniform)
263 }
264
265 pub fn fit_with(
267 x: ArrayView2<'_, f64>,
268 n_quantiles: usize,
269 output: QuantileOutput,
270 ) -> Result<Self> {
271 if x.nrows() == 0 || x.ncols() == 0 {
272 return Err(Error::Value(
273 "QuantileTransformer::fit_with: x must be non-empty".into(),
274 ));
275 }
276 if n_quantiles < 2 {
277 return Err(Error::Value(format!(
278 "QuantileTransformer::fit_with: n_quantiles must be ≥ 2 (got {n_quantiles})"
279 )));
280 }
281 let d = x.ncols();
282 let mut quantiles = Vec::with_capacity(d);
284 for j in 0..d {
285 let mut col: Vec<f64> = x.column(j).iter().copied().collect();
286 col.sort_by(|a, b| a.partial_cmp(b).unwrap());
287 let mut q = Vec::with_capacity(n_quantiles);
288 for k in 0..n_quantiles {
289 let p = k as f64 / (n_quantiles - 1) as f64;
290 q.push(quantile_sorted(&col, p));
291 }
292 quantiles.push(q);
293 }
294 let references: Vec<f64> = (0..n_quantiles)
296 .map(|k| {
297 let p = k as f64 / (n_quantiles - 1) as f64;
298 match output {
299 QuantileOutput::Uniform => p,
300 QuantileOutput::Normal => normal_inverse_cdf(p.clamp(1e-6, 1.0 - 1e-6)),
301 }
302 })
303 .collect();
304 Ok(Self {
305 quantiles,
306 references,
307 output,
308 n_quantiles,
309 })
310 }
311
312 pub fn transform(&self, x: ArrayView2<'_, f64>) -> Result<Array2<f64>> {
314 if x.ncols() != self.quantiles.len() {
315 return Err(Error::Shape(format!(
316 "QuantileTransformer::transform: expected {} cols, got {}",
317 self.quantiles.len(),
318 x.ncols()
319 )));
320 }
321 let (n, d) = (x.nrows(), x.ncols());
322 let mut out = Array2::<f64>::zeros((n, d));
323 for j in 0..d {
324 let q = &self.quantiles[j];
325 for i in 0..n {
326 out[[i, j]] = interp(x[[i, j]], q, &self.references);
327 }
328 }
329 Ok(out)
330 }
331
332 pub fn fit_transform(x: ArrayView2<'_, f64>) -> Result<(Self, Array2<f64>)> {
334 let qt = Self::fit(x)?;
335 let tr = qt.transform(x)?;
336 Ok((qt, tr))
337 }
338}
339
340fn quantile_sorted(sorted: &[f64], q: f64) -> f64 {
341 let n = sorted.len();
342 if n == 0 {
343 return f64::NAN;
344 }
345 let h = (n as f64 - 1.0) * q;
346 let lo = h.floor() as usize;
347 let hi = (lo + 1).min(n - 1);
348 let frac = h - lo as f64;
349 (1.0 - frac) * sorted[lo] + frac * sorted[hi]
350}
351
352fn interp(v: f64, xs: &[f64], ys: &[f64]) -> f64 {
353 let n = xs.len();
354 if v <= xs[0] {
355 return ys[0];
356 }
357 if v >= xs[n - 1] {
358 return ys[n - 1];
359 }
360 let mut lo = 0usize;
361 let mut hi = n - 1;
362 while lo + 1 < hi {
363 let mid = (lo + hi) / 2;
364 if xs[mid] <= v {
365 lo = mid;
366 } else {
367 hi = mid;
368 }
369 }
370 let (x0, x1) = (xs[lo], xs[hi]);
371 let (y0, y1) = (ys[lo], ys[hi]);
372 if (x1 - x0).abs() < 1e-300 {
373 return y1;
374 }
375 y0 + (y1 - y0) * (v - x0) / (x1 - x0)
376}
377
378fn normal_inverse_cdf(p: f64) -> f64 {
379 let p_low = 0.02425;
382 let p_high = 1.0 - p_low;
383 let a = [
384 -3.969_683_028_665_376e1,
385 2.209_460_984_245_205e2,
386 -2.759_285_104_469_687e2,
387 1.383_577_518_672_690e2,
388 -3.066_479_806_614_716e1,
389 2.506_628_277_459_239,
390 ];
391 let b = [
392 -5.447_609_879_822_406e1,
393 1.615_858_368_580_409e2,
394 -1.556_989_798_598_866e2,
395 6.680_131_188_771_972e1,
396 -1.328_068_155_288_572e1,
397 ];
398 let c = [
399 -7.784_894_002_430_293e-3,
400 -3.223_964_580_411_365e-1,
401 -2.400_758_277_161_838,
402 -2.549_732_539_343_734,
403 4.374_664_141_464_968,
404 2.938_163_982_698_783,
405 ];
406 let d = [
407 7.784_695_709_041_462e-3,
408 3.224_671_290_700_398e-1,
409 2.445_134_137_142_996,
410 3.754_408_661_907_416,
411 ];
412 if p < p_low {
413 let q = (-2.0 * p.ln()).sqrt();
414 (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
415 / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1.0)
416 } else if p <= p_high {
417 let q = p - 0.5;
418 let r = q * q;
419 (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q
420 / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1.0)
421 } else {
422 let q = (-2.0 * (1.0 - p).ln()).sqrt();
423 -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
424 / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1.0)
425 }
426}
427
428#[cfg(test)]
429mod tests {
430 use super::*;
431 use ndarray::array;
432
433 #[test]
434 fn yeo_johnson_reduces_skew() {
435 let x = array![
437 [1.0],
438 [2.0],
439 [3.0],
440 [4.0],
441 [5.0],
442 [10.0],
443 [20.0],
444 [50.0],
445 [100.0],
446 [200.0]
447 ];
448 let (pt, tr) = PowerTransformer::fit_transform(x.view()).unwrap();
449 assert!(pt.lambdas[0].is_finite());
451 let mean: f64 = tr.column(0).iter().sum::<f64>() / tr.nrows() as f64;
452 assert!(mean.abs() < 1e-6);
453 }
454
455 #[test]
456 fn quantile_transformer_uniform_maps_to_grid() {
457 let x = array![[1.0], [2.0], [3.0], [4.0], [5.0]];
458 let (_, tr) = QuantileTransformer::fit_transform(x.view()).unwrap();
459 let mut mn = f64::INFINITY;
461 let mut mx = f64::NEG_INFINITY;
462 for &v in tr.iter() {
463 if v < mn {
464 mn = v;
465 }
466 if v > mx {
467 mx = v;
468 }
469 }
470 assert!(mn >= 0.0);
471 assert!(mx <= 1.0);
472 assert!(mx - mn > 0.5);
473 }
474}