Skip to main content

simply_simd/register/
math.rs

1use num_traits::NumCast;
2
3use crate::architectures::interface::*;
4use crate::register::Simd;
5use crate::simd_types::*;
6
7const EXP_COEFFS_F32: [f32; 8] = [
8    0.0001984127,
9    0.0013888889,
10    0.008333334,
11    0.041666668,
12    0.16666667,
13    0.5,
14    1.0,
15    1.0,
16];
17
18// ln2^n / n!
19const EXP2_COEFFS_F32: [f32; 8] = [
20    1.525_273_4e-5,
21    1.540_353e-4,
22    1.333_355_8e-3,
23    9.618_129e-3,
24    5.550_411e-2,
25    2.402_265e-1,
26    std::f32::consts::LN_2,
27    1.0,
28];
29
30const LOG2_E_F32: f32 = std::f32::consts::LOG2_E;
31const LN2_HI_F32: f32 = 6.931_457_5e-1;
32const LN2_LO_F32: f32 = 1.428_606_8e-6;
33const MAGIC_F32: f32 = 12582912.0;
34
35const EXP_COEFFS_F64: [f64; 14] = [
36    1.6059043836821613e-10,
37    2.08767569878681e-9,
38    2.505210838544172e-8,
39    2.755731922398589e-7,
40    2.7557319223985893e-6,
41    2.48015873015873e-5,
42    0.0001984126984126984,
43    0.001388888888888889,
44    0.008333333333333333,
45    0.041666666666666664,
46    0.16666666666666666,
47    0.5,
48    1.0,
49    1.0,
50];
51
52const EXP2_COEFFS_F64: [f64; 14] = [
53    1.3691488853904e-12,
54    2.567843599348e-11,
55    4.445538271870811e-10,
56    7.054911620801123e-9,
57    1.0178086009239699e-7,
58    1.3215486790144307e-6,
59    1.525273380405984e-5,
60    1.5403530393381606e-4,
61    1.3333558146428443e-3,
62    9.618129107628477e-3,
63    5.550410866482158e-2,
64    0.24022650695910072,
65    std::f64::consts::LN_2,
66    1.0,
67];
68
69const LOG2_E_F64: f64 = std::f64::consts::LOG2_E;
70const LN2_HI_F64: f64 = 6.931471803691238e-1;
71const LN2_LO_F64: f64 = 1.908214929270588e-10;
72const MAGIC_F64: f64 = 6755399441055744.0;
73
74// Q(f) ~= log2(1+f)/f on [sqrt(0.5)-1, sqrt(2)-1], degree 9, max rel err 4.7e-9
75const LOG2_Q_F32: [f32; 10] = [
76    -1.0749789e-01,
77    1.8476033e-01,
78    -1.9138838e-01,
79    2.0485765e-01,
80    -2.3960812e-01,
81    2.8855255e-01,
82    -3.6069655e-01,
83    4.8089853e-01,
84    -7.2134733e-01,
85    std::f32::consts::LOG2_E,
86];
87
88const fn log2_coeffs<const N: usize>() -> [f64; N] {
89    let mut out = [0.0; N];
90    let mut n = 0;
91    while n < N {
92        out[N - 1 - n] = 2.0 * std::f64::consts::LOG2_E / (2 * n + 1) as f64;
93        n += 1;
94    }
95    out
96}
97
98const LOG2_COEFFS_F64: [f64; 11] = log2_coeffs::<11>();
99// const LOG2_COEFFS_F32: [f32; 5] = {
100//     let c = log2_coeffs::<5>();
101//     let mut out = [0.0f32; 5];
102//     let mut i = 0;
103//     while i < 5 {
104//         out[i] = c[i] as f32;
105//         i += 1;
106//     }
107//     out
108// };
109
110// TODO: Explore using a LUT?
111impl<T: SimdFloat, A: Arch> Simd<T, A> {
112    #[inline]
113    pub fn exp(self) -> Self {
114        match T::TYPE {
115            SimdType::F64 => {
116                let log2_e = Simd::splat(NumCast::from(LOG2_E_F64).unwrap());
117                let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F64).unwrap());
118                let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F64).unwrap());
119                let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
120
121                let lower_bound = Simd::splat(NumCast::from(-708.0).unwrap());
122                let upper_bound = Simd::splat(NumCast::from(709.0).unwrap());
123
124                let k = (self * log2_e).round();
125
126                let r = (-k).mul_add(ln2_hi, self);
127                let r = (-k).mul_add(ln2_lo, r);
128
129                let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F64[0]).unwrap());
130                for x in EXP_COEFFS_F64.iter().skip(1) {
131                    let x = Simd::splat(NumCast::from(*x).unwrap());
132                    p = p.mul_add(r, x)
133                }
134
135                let add_bits = Simd::splat(NumCast::from(1023).unwrap());
136                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
137                let scale: Self = ((kb + add_bits) << 52).raw_cast();
138
139                let res = p * scale;
140
141                let is_zero = self.simd_lt(lower_bound);
142                let is_inf = self.simd_gt(upper_bound);
143
144                let zero = Simd::zero();
145                let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
146
147                let res = is_zero.select(zero, res);
148                is_inf.select(inf, res)
149            }
150            SimdType::F32 => {
151                let log2_e = Simd::splat(NumCast::from(LOG2_E_F32).unwrap());
152                let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F32).unwrap());
153                let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F32).unwrap());
154                let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
155
156                let lower_bound = Simd::splat(NumCast::from(-87.0).unwrap());
157                let upper_bound = Simd::splat(NumCast::from(88.0).unwrap());
158
159                let k = (self * log2_e).round();
160
161                let r = (-k).mul_add(ln2_hi, self);
162                let r = (-k).mul_add(ln2_lo, r);
163
164                let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F32[0]).unwrap());
165                for x in EXP_COEFFS_F32.iter().skip(1) {
166                    let x = Simd::splat(NumCast::from(*x).unwrap());
167                    p = p.mul_add(r, x)
168                }
169
170                let add_bits = Simd::splat(NumCast::from(127).unwrap());
171                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
172                let scale: Self = ((kb + add_bits) << 23).raw_cast();
173
174                let res = p * scale;
175
176                let is_zero = self.simd_lt(lower_bound);
177                let is_inf = self.simd_gt(upper_bound);
178
179                let zero = Simd::zero();
180                let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
181
182                let res = is_zero.select(zero, res);
183                is_inf.select(inf, res)
184            }
185            _ => unreachable!(),
186        }
187    }
188
189    /// # Safety
190    ///
191    /// Does not check for the 0.0 or INF cases. Assumes the given values
192    /// do not result in 0.0 or INF.
193    #[inline(always)]
194    pub unsafe fn exp_unchecked(self) -> Self {
195        match T::TYPE {
196            SimdType::F64 => {
197                let log2_e = Simd::splat(NumCast::from(LOG2_E_F64).unwrap());
198                let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F64).unwrap());
199                let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F64).unwrap());
200                let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
201
202                let k = (self * log2_e).round();
203
204                let r = (-k).mul_add(ln2_hi, self);
205                let r = (-k).mul_add(ln2_lo, r);
206
207                let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F64[0]).unwrap());
208                for x in EXP_COEFFS_F64.iter().skip(1) {
209                    let x = Simd::splat(NumCast::from(*x).unwrap());
210                    p = p.mul_add(r, x)
211                }
212
213                let add_bits = Simd::splat(NumCast::from(1023).unwrap());
214                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
215                let scale: Self = ((kb + add_bits) << 52).raw_cast();
216
217                p * scale
218            }
219            SimdType::F32 => {
220                let log2_e = Simd::splat(NumCast::from(LOG2_E_F32).unwrap());
221                let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F32).unwrap());
222                let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F32).unwrap());
223                let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
224
225                let k = (self * log2_e).round();
226
227                let r = (-k).mul_add(ln2_hi, self);
228                let r = (-k).mul_add(ln2_lo, r);
229
230                let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F32[0]).unwrap());
231                for x in EXP_COEFFS_F32.iter().skip(1) {
232                    let x = Simd::splat(NumCast::from(*x).unwrap());
233                    p = p.mul_add(r, x)
234                }
235
236                let add_bits = Simd::splat(NumCast::from(127).unwrap());
237                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
238                let scale: Self = ((kb + add_bits) << 23).raw_cast();
239
240                p * scale
241            }
242            _ => unreachable!(),
243        }
244    }
245
246    #[inline]
247    pub fn exp2(self) -> Self {
248        match T::TYPE {
249            SimdType::F64 => {
250                let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
251
252                let lower_bound = Simd::splat(NumCast::from(-1022.0).unwrap());
253                let upper_bound = Simd::splat(NumCast::from(1024.0).unwrap());
254
255                let k = self.round();
256
257                let r = self - k;
258
259                let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F64[0]).unwrap());
260                for x in EXP2_COEFFS_F64.iter().skip(1) {
261                    let x = Simd::splat(NumCast::from(*x).unwrap());
262                    p = p.mul_add(r, x)
263                }
264
265                let add_bits = Simd::splat(NumCast::from(1023).unwrap());
266                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
267                let scale: Self = ((kb + add_bits) << 52).raw_cast();
268
269                let res = p * scale;
270
271                let is_zero = self.simd_lt(lower_bound);
272                let is_inf = self.simd_gt(upper_bound);
273
274                let zero = Simd::zero();
275                let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
276
277                let res = is_zero.select(zero, res);
278                is_inf.select(inf, res)
279            }
280            SimdType::F32 => {
281                let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
282
283                let lower_bound = Simd::splat(NumCast::from(-126.0).unwrap());
284                let upper_bound = Simd::splat(NumCast::from(128.0).unwrap());
285
286                let k = self.round();
287
288                let r = self - k;
289
290                let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F32[0]).unwrap());
291                for x in EXP2_COEFFS_F32.iter().skip(1) {
292                    let x = Simd::splat(NumCast::from(*x).unwrap());
293                    p = p.mul_add(r, x)
294                }
295
296                let add_bits = Simd::splat(NumCast::from(127).unwrap());
297                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
298                let scale: Self = ((kb + add_bits) << 23).raw_cast();
299
300                let res = p * scale;
301
302                let is_zero = self.simd_lt(lower_bound);
303                let is_inf = self.simd_gt(upper_bound);
304
305                let zero = Simd::zero();
306                let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
307
308                let res = is_zero.select(zero, res);
309                is_inf.select(inf, res)
310            }
311            _ => unreachable!(),
312        }
313    }
314
315    /// # Safety
316    ///
317    /// Does not check for the 0.0 or INF cases. Assumes the given values
318    /// do not result in 0.0 or INF.
319    #[inline]
320    pub unsafe fn exp2_unchecked(self) -> Self {
321        match T::TYPE {
322            SimdType::F64 => {
323                let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
324
325                let k = self.round();
326
327                let r = self - k;
328
329                let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F64[0]).unwrap());
330                for x in EXP2_COEFFS_F64.iter().skip(1) {
331                    let x = Simd::splat(NumCast::from(*x).unwrap());
332                    p = p.mul_add(r, x)
333                }
334
335                let add_bits = Simd::splat(NumCast::from(1023).unwrap());
336                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
337                let scale: Self = ((kb + add_bits) << 52).raw_cast();
338
339                p * scale
340            }
341            SimdType::F32 => {
342                let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
343
344                let k = self.round();
345
346                let r = self - k;
347
348                let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F32[0]).unwrap());
349                for x in EXP2_COEFFS_F32.iter().skip(1) {
350                    let x = Simd::splat(NumCast::from(*x).unwrap());
351                    p = p.mul_add(r, x)
352                }
353
354                let add_bits = Simd::splat(NumCast::from(127).unwrap());
355                let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
356                let scale: Self = ((kb + add_bits) << 23).raw_cast();
357
358                p * scale
359            }
360            _ => unreachable!(),
361        }
362    }
363
364    #[inline]
365    pub fn log2(self) -> Self {
366        match T::TYPE {
367            SimdType::F32 => {
368                let zero = Simd::zero();
369                let one = Simd::splat(NumCast::from(1.0f32).unwrap());
370
371                let min_normal = Simd::splat(NumCast::from(f32::MIN_POSITIVE).unwrap());
372                let is_sub = self.simd_lt(min_normal);
373                let x = is_sub.select(
374                    self * Simd::splat(NumCast::from(16777216.0f32).unwrap()),
375                    self,
376                );
377
378                let bits: Simd<T::UintType, A> = x.raw_cast();
379                let b = bits + Simd::splat(NumCast::from(0x004A_FB0Du32).unwrap());
380
381                let e_raw: Self =
382                    ((b >> 23) | Simd::splat(NumCast::from(0x4B00_0000u32).unwrap())).raw_cast();
383                let e = e_raw - Simd::splat(NumCast::from(8388608.0f32 + 127.0).unwrap());
384                let e = is_sub.select(e - Simd::splat(NumCast::from(24.0f32).unwrap()), e);
385
386                let m: Self = ((b & Simd::splat(NumCast::from(0x007F_FFFFu32).unwrap()))
387                    + Simd::splat(NumCast::from(0x3F35_04F3u32).unwrap()))
388                .raw_cast();
389
390                let f = m - one;
391                let mut q = Simd::splat(NumCast::from(LOG2_Q_F32[0]).unwrap());
392                for c in LOG2_Q_F32.iter().skip(1) {
393                    q = q.mul_add(f, Simd::splat(NumCast::from(*c).unwrap()));
394                }
395                let res = f.mul_add(q, e);
396
397                let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
398                let res = self.simd_eq(zero).select(-inf, res);
399                let res = self.simd_eq(inf).select(inf, res);
400                let res = self
401                    .simd_lt(zero)
402                    .select(Simd::splat(NumCast::from(f32::NAN).unwrap()), res);
403                self.simd_ne(self).select(self, res)
404            }
405            SimdType::F64 => {
406                let zero = Simd::zero();
407                let one = Simd::splat(NumCast::from(1.0f64).unwrap());
408
409                let min_normal = Simd::splat(NumCast::from(f64::MIN_POSITIVE).unwrap());
410                let is_sub = self.simd_lt(min_normal);
411                let x = is_sub.select(
412                    self * Simd::splat(NumCast::from(18014398509481984.0f64).unwrap()),
413                    self,
414                );
415
416                let bits: Simd<T::UintType, A> = x.raw_cast();
417
418                let e_raw: Self = ((bits >> 52)
419                    | Simd::splat(NumCast::from(0x4330_0000_0000_0000u64).unwrap()))
420                .raw_cast();
421                let e = e_raw - Simd::splat(NumCast::from(4503599627370496.0f64 + 1023.0).unwrap());
422
423                let m: Self = ((bits
424                    & Simd::splat(NumCast::from(0x000F_FFFF_FFFF_FFFFu64).unwrap()))
425                    | Simd::splat(NumCast::from(0x3FF0_0000_0000_0000u64).unwrap()))
426                .raw_cast();
427
428                let big = m.simd_gt(Simd::splat(
429                    NumCast::from(std::f64::consts::SQRT_2).unwrap(),
430                ));
431                let m = big.select(m * Simd::splat(NumCast::from(0.5f64).unwrap()), m);
432                let e = big.select(e + one, e);
433                let e = is_sub.select(e - Simd::splat(NumCast::from(54.0f64).unwrap()), e);
434
435                let f = m - one;
436                let s = f / (f + Simd::splat(NumCast::from(2.0f64).unwrap()));
437                let s2 = s * s;
438
439                let mut p = Simd::splat(NumCast::from(LOG2_COEFFS_F64[0]).unwrap());
440                for c in LOG2_COEFFS_F64.iter().skip(1) {
441                    p = p.mul_add(s2, Simd::splat(NumCast::from(*c).unwrap()));
442                }
443                let res = s.mul_add(p, e);
444
445                let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
446                let res = self.simd_eq(zero).select(-inf, res);
447                let res = self.simd_eq(inf).select(inf, res);
448                let res = self
449                    .simd_lt(zero)
450                    .select(Simd::splat(NumCast::from(f64::NAN).unwrap()), res);
451                self.simd_ne(self).select(self, res)
452            }
453            _ => unreachable!(),
454        }
455    }
456
457    /// # Safety
458    ///
459    /// Does not check for the -INF, INF, or NaN cases. Assumes the given values
460    /// do not result in 0.0 or INF.
461    #[inline]
462    pub unsafe fn log2_unchecked(self) -> Self {
463        match T::TYPE {
464            SimdType::F32 => {
465                let one = Simd::splat(NumCast::from(1.0f32).unwrap());
466
467                let min_normal = Simd::splat(NumCast::from(f32::MIN_POSITIVE).unwrap());
468                let is_sub = self.simd_lt(min_normal);
469                let x = is_sub.select(
470                    self * Simd::splat(NumCast::from(16777216.0f32).unwrap()),
471                    self,
472                );
473
474                let bits: Simd<T::UintType, A> = x.raw_cast();
475                let b = bits + Simd::splat(NumCast::from(0x004A_FB0Du32).unwrap());
476
477                let e_raw: Self =
478                    ((b >> 23) | Simd::splat(NumCast::from(0x4B00_0000u32).unwrap())).raw_cast();
479                let e = e_raw - Simd::splat(NumCast::from(8388608.0f32 + 127.0).unwrap());
480                let e = is_sub.select(e - Simd::splat(NumCast::from(24.0f32).unwrap()), e);
481
482                let m: Self = ((b & Simd::splat(NumCast::from(0x007F_FFFFu32).unwrap()))
483                    + Simd::splat(NumCast::from(0x3F35_04F3u32).unwrap()))
484                .raw_cast();
485
486                let f = m - one;
487                let mut q = Simd::splat(NumCast::from(LOG2_Q_F32[0]).unwrap());
488                for c in LOG2_Q_F32.iter().skip(1) {
489                    q = q.mul_add(f, Simd::splat(NumCast::from(*c).unwrap()));
490                }
491                f.mul_add(q, e)
492            }
493            SimdType::F64 => {
494                let one = Simd::splat(NumCast::from(1.0f64).unwrap());
495
496                let min_normal = Simd::splat(NumCast::from(f64::MIN_POSITIVE).unwrap());
497                let is_sub = self.simd_lt(min_normal);
498                let x = is_sub.select(
499                    self * Simd::splat(NumCast::from(18014398509481984.0f64).unwrap()),
500                    self,
501                );
502
503                let bits: Simd<T::UintType, A> = x.raw_cast();
504
505                let e_raw: Self = ((bits >> 52)
506                    | Simd::splat(NumCast::from(0x4330_0000_0000_0000u64).unwrap()))
507                .raw_cast();
508                let e = e_raw - Simd::splat(NumCast::from(4503599627370496.0f64 + 1023.0).unwrap());
509
510                let m: Self = ((bits
511                    & Simd::splat(NumCast::from(0x000F_FFFF_FFFF_FFFFu64).unwrap()))
512                    | Simd::splat(NumCast::from(0x3FF0_0000_0000_0000u64).unwrap()))
513                .raw_cast();
514
515                let big = m.simd_gt(Simd::splat(
516                    NumCast::from(std::f64::consts::SQRT_2).unwrap(),
517                ));
518                let m = big.select(m * Simd::splat(NumCast::from(0.5f64).unwrap()), m);
519                let e = big.select(e + one, e);
520                let e = is_sub.select(e - Simd::splat(NumCast::from(54.0f64).unwrap()), e);
521
522                let f = m - one;
523                let s = f / (f + Simd::splat(NumCast::from(2.0f64).unwrap()));
524                let s2 = s * s;
525
526                let mut p = Simd::splat(NumCast::from(LOG2_COEFFS_F64[0]).unwrap());
527                for c in LOG2_COEFFS_F64.iter().skip(1) {
528                    p = p.mul_add(s2, Simd::splat(NumCast::from(*c).unwrap()));
529                }
530                s.mul_add(p, e)
531            }
532            _ => unreachable!(),
533        }
534    }
535
536    pub fn pow(self, y: Self) -> Self {
537        let zero = Simd::zero();
538        let one = Simd::splat(NumCast::from(1.0f32).unwrap());
539        let half = Simd::splat(NumCast::from(0.5f32).unwrap());
540        let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
541        let nan = Simd::splat(NumCast::from(f32::NAN).unwrap());
542
543        let res = (y * self.abs().log2()).exp2();
544
545        let neg_x = self.simd_lt(zero);
546        let is_int = y.simd_eq(y.round());
547        let yh = y * half;
548        let is_odd = yh.simd_ne(yh.round());
549        let signed = is_odd.select(-res, res);
550        let res = neg_x.select(is_int.select(signed, nan), res);
551
552        let res = y.simd_eq(zero).select(one, res);
553        let res = self.simd_eq(one).select(one, res);
554        let abs_x_is_one = self.abs().simd_eq(one);
555        let y_is_inf = y.abs().simd_eq(inf);
556        abs_x_is_one.select(y_is_inf.select(one, res), res)
557    }
558
559    /// # Safety
560    ///
561    /// - `self` must be positive, normal, and finite.
562    /// - `y` must be finite.
563    /// - Result must be representable as an f32.
564    pub unsafe fn pow_unchecked(self, y: Self) -> Self {
565        unsafe { (y * self.log2_unchecked()).exp2_unchecked() }
566    }
567}