1use num_traits::NumCast;
2
3use crate::architectures::interface::*;
4use crate::register::Simd;
5use crate::simd_types::*;
6
7const EXP_COEFFS_F32: [f32; 8] = [
8 0.0001984127,
9 0.0013888889,
10 0.008333334,
11 0.041666668,
12 0.16666667,
13 0.5,
14 1.0,
15 1.0,
16];
17
18const EXP2_COEFFS_F32: [f32; 8] = [
20 1.525_273_4e-5,
21 1.540_353e-4,
22 1.333_355_8e-3,
23 9.618_129e-3,
24 5.550_411e-2,
25 2.402_265e-1,
26 std::f32::consts::LN_2,
27 1.0,
28];
29
30const LOG2_E_F32: f32 = std::f32::consts::LOG2_E;
31const LN2_HI_F32: f32 = 6.931_457_5e-1;
32const LN2_LO_F32: f32 = 1.428_606_8e-6;
33const MAGIC_F32: f32 = 12582912.0;
34
35const EXP_COEFFS_F64: [f64; 14] = [
36 1.6059043836821613e-10,
37 2.08767569878681e-9,
38 2.505210838544172e-8,
39 2.755731922398589e-7,
40 2.7557319223985893e-6,
41 2.48015873015873e-5,
42 0.0001984126984126984,
43 0.001388888888888889,
44 0.008333333333333333,
45 0.041666666666666664,
46 0.16666666666666666,
47 0.5,
48 1.0,
49 1.0,
50];
51
52const EXP2_COEFFS_F64: [f64; 14] = [
53 1.3691488853904e-12,
54 2.567843599348e-11,
55 4.445538271870811e-10,
56 7.054911620801123e-9,
57 1.0178086009239699e-7,
58 1.3215486790144307e-6,
59 1.525273380405984e-5,
60 1.5403530393381606e-4,
61 1.3333558146428443e-3,
62 9.618129107628477e-3,
63 5.550410866482158e-2,
64 0.24022650695910072,
65 std::f64::consts::LN_2,
66 1.0,
67];
68
69const LOG2_E_F64: f64 = std::f64::consts::LOG2_E;
70const LN2_HI_F64: f64 = 6.931471803691238e-1;
71const LN2_LO_F64: f64 = 1.908214929270588e-10;
72const MAGIC_F64: f64 = 6755399441055744.0;
73
74const LOG2_Q_F32: [f32; 10] = [
76 -1.0749789e-01,
77 1.8476033e-01,
78 -1.9138838e-01,
79 2.0485765e-01,
80 -2.3960812e-01,
81 2.8855255e-01,
82 -3.6069655e-01,
83 4.8089853e-01,
84 -7.2134733e-01,
85 std::f32::consts::LOG2_E,
86];
87
88const fn log2_coeffs<const N: usize>() -> [f64; N] {
89 let mut out = [0.0; N];
90 let mut n = 0;
91 while n < N {
92 out[N - 1 - n] = 2.0 * std::f64::consts::LOG2_E / (2 * n + 1) as f64;
93 n += 1;
94 }
95 out
96}
97
98const LOG2_COEFFS_F64: [f64; 11] = log2_coeffs::<11>();
99impl<T: SimdFloat, A: Arch> Simd<T, A> {
112 #[inline]
113 pub fn exp(self) -> Self {
114 match T::TYPE {
115 SimdType::F64 => {
116 let log2_e = Simd::splat(NumCast::from(LOG2_E_F64).unwrap());
117 let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F64).unwrap());
118 let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F64).unwrap());
119 let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
120
121 let lower_bound = Simd::splat(NumCast::from(-708.0).unwrap());
122 let upper_bound = Simd::splat(NumCast::from(709.0).unwrap());
123
124 let k = (self * log2_e).round();
125
126 let r = (-k).mul_add(ln2_hi, self);
127 let r = (-k).mul_add(ln2_lo, r);
128
129 let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F64[0]).unwrap());
130 for x in EXP_COEFFS_F64.iter().skip(1) {
131 let x = Simd::splat(NumCast::from(*x).unwrap());
132 p = p.mul_add(r, x)
133 }
134
135 let add_bits = Simd::splat(NumCast::from(1023).unwrap());
136 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
137 let scale: Self = ((kb + add_bits) << 52).raw_cast();
138
139 let res = p * scale;
140
141 let is_zero = self.simd_lt(lower_bound);
142 let is_inf = self.simd_gt(upper_bound);
143
144 let zero = Simd::zero();
145 let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
146
147 let res = is_zero.select(zero, res);
148 is_inf.select(inf, res)
149 }
150 SimdType::F32 => {
151 let log2_e = Simd::splat(NumCast::from(LOG2_E_F32).unwrap());
152 let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F32).unwrap());
153 let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F32).unwrap());
154 let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
155
156 let lower_bound = Simd::splat(NumCast::from(-87.0).unwrap());
157 let upper_bound = Simd::splat(NumCast::from(88.0).unwrap());
158
159 let k = (self * log2_e).round();
160
161 let r = (-k).mul_add(ln2_hi, self);
162 let r = (-k).mul_add(ln2_lo, r);
163
164 let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F32[0]).unwrap());
165 for x in EXP_COEFFS_F32.iter().skip(1) {
166 let x = Simd::splat(NumCast::from(*x).unwrap());
167 p = p.mul_add(r, x)
168 }
169
170 let add_bits = Simd::splat(NumCast::from(127).unwrap());
171 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
172 let scale: Self = ((kb + add_bits) << 23).raw_cast();
173
174 let res = p * scale;
175
176 let is_zero = self.simd_lt(lower_bound);
177 let is_inf = self.simd_gt(upper_bound);
178
179 let zero = Simd::zero();
180 let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
181
182 let res = is_zero.select(zero, res);
183 is_inf.select(inf, res)
184 }
185 _ => unreachable!(),
186 }
187 }
188
189 #[inline(always)]
194 pub unsafe fn exp_unchecked(self) -> Self {
195 match T::TYPE {
196 SimdType::F64 => {
197 let log2_e = Simd::splat(NumCast::from(LOG2_E_F64).unwrap());
198 let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F64).unwrap());
199 let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F64).unwrap());
200 let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
201
202 let k = (self * log2_e).round();
203
204 let r = (-k).mul_add(ln2_hi, self);
205 let r = (-k).mul_add(ln2_lo, r);
206
207 let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F64[0]).unwrap());
208 for x in EXP_COEFFS_F64.iter().skip(1) {
209 let x = Simd::splat(NumCast::from(*x).unwrap());
210 p = p.mul_add(r, x)
211 }
212
213 let add_bits = Simd::splat(NumCast::from(1023).unwrap());
214 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
215 let scale: Self = ((kb + add_bits) << 52).raw_cast();
216
217 p * scale
218 }
219 SimdType::F32 => {
220 let log2_e = Simd::splat(NumCast::from(LOG2_E_F32).unwrap());
221 let ln2_hi = Simd::splat(NumCast::from(LN2_HI_F32).unwrap());
222 let ln2_lo = Simd::splat(NumCast::from(LN2_LO_F32).unwrap());
223 let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
224
225 let k = (self * log2_e).round();
226
227 let r = (-k).mul_add(ln2_hi, self);
228 let r = (-k).mul_add(ln2_lo, r);
229
230 let mut p = Simd::splat(NumCast::from(EXP_COEFFS_F32[0]).unwrap());
231 for x in EXP_COEFFS_F32.iter().skip(1) {
232 let x = Simd::splat(NumCast::from(*x).unwrap());
233 p = p.mul_add(r, x)
234 }
235
236 let add_bits = Simd::splat(NumCast::from(127).unwrap());
237 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
238 let scale: Self = ((kb + add_bits) << 23).raw_cast();
239
240 p * scale
241 }
242 _ => unreachable!(),
243 }
244 }
245
246 #[inline]
247 pub fn exp2(self) -> Self {
248 match T::TYPE {
249 SimdType::F64 => {
250 let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
251
252 let lower_bound = Simd::splat(NumCast::from(-1022.0).unwrap());
253 let upper_bound = Simd::splat(NumCast::from(1024.0).unwrap());
254
255 let k = self.round();
256
257 let r = self - k;
258
259 let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F64[0]).unwrap());
260 for x in EXP2_COEFFS_F64.iter().skip(1) {
261 let x = Simd::splat(NumCast::from(*x).unwrap());
262 p = p.mul_add(r, x)
263 }
264
265 let add_bits = Simd::splat(NumCast::from(1023).unwrap());
266 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
267 let scale: Self = ((kb + add_bits) << 52).raw_cast();
268
269 let res = p * scale;
270
271 let is_zero = self.simd_lt(lower_bound);
272 let is_inf = self.simd_gt(upper_bound);
273
274 let zero = Simd::zero();
275 let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
276
277 let res = is_zero.select(zero, res);
278 is_inf.select(inf, res)
279 }
280 SimdType::F32 => {
281 let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
282
283 let lower_bound = Simd::splat(NumCast::from(-126.0).unwrap());
284 let upper_bound = Simd::splat(NumCast::from(128.0).unwrap());
285
286 let k = self.round();
287
288 let r = self - k;
289
290 let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F32[0]).unwrap());
291 for x in EXP2_COEFFS_F32.iter().skip(1) {
292 let x = Simd::splat(NumCast::from(*x).unwrap());
293 p = p.mul_add(r, x)
294 }
295
296 let add_bits = Simd::splat(NumCast::from(127).unwrap());
297 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
298 let scale: Self = ((kb + add_bits) << 23).raw_cast();
299
300 let res = p * scale;
301
302 let is_zero = self.simd_lt(lower_bound);
303 let is_inf = self.simd_gt(upper_bound);
304
305 let zero = Simd::zero();
306 let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
307
308 let res = is_zero.select(zero, res);
309 is_inf.select(inf, res)
310 }
311 _ => unreachable!(),
312 }
313 }
314
315 #[inline]
320 pub unsafe fn exp2_unchecked(self) -> Self {
321 match T::TYPE {
322 SimdType::F64 => {
323 let magic = Simd::splat(NumCast::from(MAGIC_F64).unwrap());
324
325 let k = self.round();
326
327 let r = self - k;
328
329 let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F64[0]).unwrap());
330 for x in EXP2_COEFFS_F64.iter().skip(1) {
331 let x = Simd::splat(NumCast::from(*x).unwrap());
332 p = p.mul_add(r, x)
333 }
334
335 let add_bits = Simd::splat(NumCast::from(1023).unwrap());
336 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
337 let scale: Self = ((kb + add_bits) << 52).raw_cast();
338
339 p * scale
340 }
341 SimdType::F32 => {
342 let magic = Simd::splat(NumCast::from(MAGIC_F32).unwrap());
343
344 let k = self.round();
345
346 let r = self - k;
347
348 let mut p = Simd::splat(NumCast::from(EXP2_COEFFS_F32[0]).unwrap());
349 for x in EXP2_COEFFS_F32.iter().skip(1) {
350 let x = Simd::splat(NumCast::from(*x).unwrap());
351 p = p.mul_add(r, x)
352 }
353
354 let add_bits = Simd::splat(NumCast::from(127).unwrap());
355 let kb: Simd<T::UintType, A> = (k + magic).raw_cast();
356 let scale: Self = ((kb + add_bits) << 23).raw_cast();
357
358 p * scale
359 }
360 _ => unreachable!(),
361 }
362 }
363
364 #[inline]
365 pub fn log2(self) -> Self {
366 match T::TYPE {
367 SimdType::F32 => {
368 let zero = Simd::zero();
369 let one = Simd::splat(NumCast::from(1.0f32).unwrap());
370
371 let min_normal = Simd::splat(NumCast::from(f32::MIN_POSITIVE).unwrap());
372 let is_sub = self.simd_lt(min_normal);
373 let x = is_sub.select(
374 self * Simd::splat(NumCast::from(16777216.0f32).unwrap()),
375 self,
376 );
377
378 let bits: Simd<T::UintType, A> = x.raw_cast();
379 let b = bits + Simd::splat(NumCast::from(0x004A_FB0Du32).unwrap());
380
381 let e_raw: Self =
382 ((b >> 23) | Simd::splat(NumCast::from(0x4B00_0000u32).unwrap())).raw_cast();
383 let e = e_raw - Simd::splat(NumCast::from(8388608.0f32 + 127.0).unwrap());
384 let e = is_sub.select(e - Simd::splat(NumCast::from(24.0f32).unwrap()), e);
385
386 let m: Self = ((b & Simd::splat(NumCast::from(0x007F_FFFFu32).unwrap()))
387 + Simd::splat(NumCast::from(0x3F35_04F3u32).unwrap()))
388 .raw_cast();
389
390 let f = m - one;
391 let mut q = Simd::splat(NumCast::from(LOG2_Q_F32[0]).unwrap());
392 for c in LOG2_Q_F32.iter().skip(1) {
393 q = q.mul_add(f, Simd::splat(NumCast::from(*c).unwrap()));
394 }
395 let res = f.mul_add(q, e);
396
397 let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
398 let res = self.simd_eq(zero).select(-inf, res);
399 let res = self.simd_eq(inf).select(inf, res);
400 let res = self
401 .simd_lt(zero)
402 .select(Simd::splat(NumCast::from(f32::NAN).unwrap()), res);
403 self.simd_ne(self).select(self, res)
404 }
405 SimdType::F64 => {
406 let zero = Simd::zero();
407 let one = Simd::splat(NumCast::from(1.0f64).unwrap());
408
409 let min_normal = Simd::splat(NumCast::from(f64::MIN_POSITIVE).unwrap());
410 let is_sub = self.simd_lt(min_normal);
411 let x = is_sub.select(
412 self * Simd::splat(NumCast::from(18014398509481984.0f64).unwrap()),
413 self,
414 );
415
416 let bits: Simd<T::UintType, A> = x.raw_cast();
417
418 let e_raw: Self = ((bits >> 52)
419 | Simd::splat(NumCast::from(0x4330_0000_0000_0000u64).unwrap()))
420 .raw_cast();
421 let e = e_raw - Simd::splat(NumCast::from(4503599627370496.0f64 + 1023.0).unwrap());
422
423 let m: Self = ((bits
424 & Simd::splat(NumCast::from(0x000F_FFFF_FFFF_FFFFu64).unwrap()))
425 | Simd::splat(NumCast::from(0x3FF0_0000_0000_0000u64).unwrap()))
426 .raw_cast();
427
428 let big = m.simd_gt(Simd::splat(
429 NumCast::from(std::f64::consts::SQRT_2).unwrap(),
430 ));
431 let m = big.select(m * Simd::splat(NumCast::from(0.5f64).unwrap()), m);
432 let e = big.select(e + one, e);
433 let e = is_sub.select(e - Simd::splat(NumCast::from(54.0f64).unwrap()), e);
434
435 let f = m - one;
436 let s = f / (f + Simd::splat(NumCast::from(2.0f64).unwrap()));
437 let s2 = s * s;
438
439 let mut p = Simd::splat(NumCast::from(LOG2_COEFFS_F64[0]).unwrap());
440 for c in LOG2_COEFFS_F64.iter().skip(1) {
441 p = p.mul_add(s2, Simd::splat(NumCast::from(*c).unwrap()));
442 }
443 let res = s.mul_add(p, e);
444
445 let inf = Simd::splat(NumCast::from(f64::INFINITY).unwrap());
446 let res = self.simd_eq(zero).select(-inf, res);
447 let res = self.simd_eq(inf).select(inf, res);
448 let res = self
449 .simd_lt(zero)
450 .select(Simd::splat(NumCast::from(f64::NAN).unwrap()), res);
451 self.simd_ne(self).select(self, res)
452 }
453 _ => unreachable!(),
454 }
455 }
456
457 #[inline]
462 pub unsafe fn log2_unchecked(self) -> Self {
463 match T::TYPE {
464 SimdType::F32 => {
465 let one = Simd::splat(NumCast::from(1.0f32).unwrap());
466
467 let min_normal = Simd::splat(NumCast::from(f32::MIN_POSITIVE).unwrap());
468 let is_sub = self.simd_lt(min_normal);
469 let x = is_sub.select(
470 self * Simd::splat(NumCast::from(16777216.0f32).unwrap()),
471 self,
472 );
473
474 let bits: Simd<T::UintType, A> = x.raw_cast();
475 let b = bits + Simd::splat(NumCast::from(0x004A_FB0Du32).unwrap());
476
477 let e_raw: Self =
478 ((b >> 23) | Simd::splat(NumCast::from(0x4B00_0000u32).unwrap())).raw_cast();
479 let e = e_raw - Simd::splat(NumCast::from(8388608.0f32 + 127.0).unwrap());
480 let e = is_sub.select(e - Simd::splat(NumCast::from(24.0f32).unwrap()), e);
481
482 let m: Self = ((b & Simd::splat(NumCast::from(0x007F_FFFFu32).unwrap()))
483 + Simd::splat(NumCast::from(0x3F35_04F3u32).unwrap()))
484 .raw_cast();
485
486 let f = m - one;
487 let mut q = Simd::splat(NumCast::from(LOG2_Q_F32[0]).unwrap());
488 for c in LOG2_Q_F32.iter().skip(1) {
489 q = q.mul_add(f, Simd::splat(NumCast::from(*c).unwrap()));
490 }
491 f.mul_add(q, e)
492 }
493 SimdType::F64 => {
494 let one = Simd::splat(NumCast::from(1.0f64).unwrap());
495
496 let min_normal = Simd::splat(NumCast::from(f64::MIN_POSITIVE).unwrap());
497 let is_sub = self.simd_lt(min_normal);
498 let x = is_sub.select(
499 self * Simd::splat(NumCast::from(18014398509481984.0f64).unwrap()),
500 self,
501 );
502
503 let bits: Simd<T::UintType, A> = x.raw_cast();
504
505 let e_raw: Self = ((bits >> 52)
506 | Simd::splat(NumCast::from(0x4330_0000_0000_0000u64).unwrap()))
507 .raw_cast();
508 let e = e_raw - Simd::splat(NumCast::from(4503599627370496.0f64 + 1023.0).unwrap());
509
510 let m: Self = ((bits
511 & Simd::splat(NumCast::from(0x000F_FFFF_FFFF_FFFFu64).unwrap()))
512 | Simd::splat(NumCast::from(0x3FF0_0000_0000_0000u64).unwrap()))
513 .raw_cast();
514
515 let big = m.simd_gt(Simd::splat(
516 NumCast::from(std::f64::consts::SQRT_2).unwrap(),
517 ));
518 let m = big.select(m * Simd::splat(NumCast::from(0.5f64).unwrap()), m);
519 let e = big.select(e + one, e);
520 let e = is_sub.select(e - Simd::splat(NumCast::from(54.0f64).unwrap()), e);
521
522 let f = m - one;
523 let s = f / (f + Simd::splat(NumCast::from(2.0f64).unwrap()));
524 let s2 = s * s;
525
526 let mut p = Simd::splat(NumCast::from(LOG2_COEFFS_F64[0]).unwrap());
527 for c in LOG2_COEFFS_F64.iter().skip(1) {
528 p = p.mul_add(s2, Simd::splat(NumCast::from(*c).unwrap()));
529 }
530 s.mul_add(p, e)
531 }
532 _ => unreachable!(),
533 }
534 }
535
536 pub fn pow(self, y: Self) -> Self {
537 let zero = Simd::zero();
538 let one = Simd::splat(NumCast::from(1.0f32).unwrap());
539 let half = Simd::splat(NumCast::from(0.5f32).unwrap());
540 let inf = Simd::splat(NumCast::from(f32::INFINITY).unwrap());
541 let nan = Simd::splat(NumCast::from(f32::NAN).unwrap());
542
543 let res = (y * self.abs().log2()).exp2();
544
545 let neg_x = self.simd_lt(zero);
546 let is_int = y.simd_eq(y.round());
547 let yh = y * half;
548 let is_odd = yh.simd_ne(yh.round());
549 let signed = is_odd.select(-res, res);
550 let res = neg_x.select(is_int.select(signed, nan), res);
551
552 let res = y.simd_eq(zero).select(one, res);
553 let res = self.simd_eq(one).select(one, res);
554 let abs_x_is_one = self.abs().simd_eq(one);
555 let y_is_inf = y.abs().simd_eq(inf);
556 abs_x_is_one.select(y_is_inf.select(one, res), res)
557 }
558
559 pub unsafe fn pow_unchecked(self, y: Self) -> Self {
565 unsafe { (y * self.log2_unchecked()).exp2_unchecked() }
566 }
567}