#[cfg(test)]
mod expr_tests;
mod frame;
pub(crate) mod node;
pub use node::Kind as FpKind;
#[cfg(test)]
mod reference;
mod soft;
pub(crate) mod syntax;
#[cfg(test)]
pub(crate) mod tests;
use core::cmp::Ordering;
use core::fmt;
use crate::{BitVec, Width, WidthError};
use frame::{Frame, MID_PRECISION, Mid, NARROW_PRECISION, SMALL_PRECISION, Wide};
use soft::Fmt;
macro_rules! frames_for {
($p:expr, $S:ident => $body:expr) => {{
let p: u32 = $p;
if p <= SMALL_PRECISION {
#[allow(dead_code)]
type $S = u64;
$body
} else if p <= NARROW_PRECISION {
#[allow(dead_code)]
type $S = u128;
$body
} else if p <= MID_PRECISION {
#[allow(dead_code)]
type $S = Mid;
$body
} else {
#[allow(dead_code)]
type $S = Wide;
$body
}
}};
}
macro_rules! frames {
($fmt:expr, $S:ident => $body:expr) => {
frames_for!($fmt.sb(), $S => $body)
};
}
macro_rules! frames_short {
($fmt:expr, $S:ident => $body:expr) => {
if $fmt.width().bits() <= 64 && $fmt.sb() <= 60 {
#[allow(dead_code)]
type $S = u64;
$body
} else {
frames!($fmt, $S => $body)
}
};
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct FpFormat {
packed: u16,
}
impl FpFormat {
pub const F16: FpFormat = FpFormat::pack(5, 11);
pub const BF16: FpFormat = FpFormat::pack(8, 8);
pub const F32: FpFormat = FpFormat::pack(8, 24);
pub const F64: FpFormat = FpFormat::pack(11, 53);
pub const F128: FpFormat = FpFormat::pack(15, 113);
pub const F256: FpFormat = FpFormat::pack(19, 237);
pub const X87: FpFormat = FpFormat::pack(15, 64);
const fn pack(eb: u32, sb: u32) -> FpFormat {
FpFormat {
packed: (eb << 9 | sb) as u16,
}
}
pub const NAMED: [(FpFormat, &'static str); 6] = [
(FpFormat::F16, "f16"),
(FpFormat::BF16, "bf16"),
(FpFormat::F32, "f32"),
(FpFormat::F64, "f64"),
(FpFormat::F128, "f128"),
(FpFormat::F256, "f256"),
];
pub const fn new(eb: u32, sb: u32) -> Result<FpFormat, WidthError> {
if eb < 2 || eb > 31 || sb < 2 || sb > Width::MAX_BITS as u32 - eb {
return Err(WidthError::FpFormat { eb, sb });
}
Ok(FpFormat::pack(eb, sb))
}
pub const fn eb(self) -> u32 {
(self.packed >> 9) as u32
}
pub const fn sb(self) -> u32 {
(self.packed & 0x1ff) as u32
}
pub fn width(self) -> Width {
Width::new((self.eb() + self.sb()) as u16).expect("a valid format fits in 512 bits")
}
pub fn name(self) -> Option<&'static str> {
Self::NAMED
.iter()
.find(|(f, _)| *f == self)
.map(|&(_, n)| n)
}
pub fn from_name(name: &str) -> Option<FpFormat> {
Self::NAMED
.iter()
.find(|&&(_, n)| n == name)
.map(|&(f, _)| f)
}
fn fmt(self) -> Fmt {
Fmt::new(self.eb(), self.sb())
}
fn check(self, v: &BitVec) -> Result<(), WidthError> {
let (left, right) = (self.width().bits(), v.width().bits());
if left == right {
Ok(())
} else {
Err(WidthError::Mismatch { left, right })
}
}
pub fn nan(self) -> BitVec {
if let Some(w) = self.word() {
return w.value(w.inf | 1 << (self.sb() - 2));
}
frames!(self, S => store::<S>(self.width(), soft::nan::<S>(&self.fmt())))
}
pub fn inf(self, negative: bool) -> BitVec {
if let Some(w) = self.word() {
return w.value(w.inf | if negative { w.sign } else { 0 });
}
frames!(self, S => store::<S>(self.width(), soft::inf::<S>(&self.fmt(), negative)))
}
pub fn zero(self, negative: bool) -> BitVec {
if let Some(w) = self.word() {
return w.value(if negative { w.sign } else { 0 });
}
frames!(self, S => store::<S>(self.width(), soft::zero::<S>(&self.fmt(), negative)))
}
fn word(self) -> Option<Word> {
let w = self.eb() + self.sb();
(w <= 64).then(|| {
let sign = 1u64 << (w - 1);
Word {
width: self.width(),
sign,
inf: ((1u64 << self.eb()) - 1) << (self.sb() - 1),
}
})
}
pub fn add(self, rm: RoundingMode, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames_short!(f, S => {
let r = soft::add::<S>(&f.fmt(), rm, load(x), load(y));
store(f.width(), r)
})
})
}
pub fn sub(self, rm: RoundingMode, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames_short!(f, S => {
let r = soft::sub::<S>(&f.fmt(), rm, load(x), load(y));
store(f.width(), r)
})
})
}
pub fn mul(self, rm: RoundingMode, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames!(f, S => {
let r = soft::mul::<S>(&f.fmt(), rm, load(x), load(y));
store(f.width(), r)
})
})
}
pub fn div(self, rm: RoundingMode, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames!(f, S => {
let r = soft::div::<S>(&f.fmt(), rm, load(x), load(y));
store(f.width(), r)
})
})
}
pub fn fma(
self,
rm: RoundingMode,
a: &BitVec,
b: &BitVec,
c: &BitVec,
) -> Result<BitVec, WidthError> {
self.check(a)?;
self.check(b)?;
self.check(c)?;
Ok(frames!(self, S => {
let r = soft::fma::<S>(&self.fmt(), rm, load(a), load(b), load(c));
store(self.width(), r)
}))
}
pub fn sqrt(self, rm: RoundingMode, a: &BitVec) -> Result<BitVec, WidthError> {
self.check(a)?;
Ok(frames!(self, S => store(self.width(), soft::sqrt::<S>(&self.fmt(), rm, load(a)))))
}
pub fn rem(self, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames!(f, S => {
store(f.width(), soft::rem::<S>(&f.fmt(), load(x), load(y)))
})
})
}
pub fn round_to_integral(self, rm: RoundingMode, a: &BitVec) -> Result<BitVec, WidthError> {
self.check(a)?;
Ok(frames_short!(self, S => {
store(self.width(), soft::round_to_integral::<S>(&self.fmt(), rm, load(a)))
}))
}
pub fn min(self, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames_short!(f, S => {
store(f.width(), soft::min::<S>(&f.fmt(), load(x), load(y)))
})
})
}
pub fn max(self, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
self.bin(a, b, |f, x, y| {
frames_short!(f, S => {
store(f.width(), soft::max::<S>(&f.fmt(), load(x), load(y)))
})
})
}
pub fn neg(self, a: &BitVec) -> Result<BitVec, WidthError> {
self.check(a)?;
Ok(frames!(self, S => store(self.width(), soft::neg::<S>(&self.fmt(), load(a)))))
}
pub fn abs(self, a: &BitVec) -> Result<BitVec, WidthError> {
self.check(a)?;
let top = self.width().bits() - 1;
Ok(frames!(self, S => store(self.width(), load::<S>(a).low(u32::from(top)))))
}
pub fn copysign(self, a: &BitVec, b: &BitVec) -> Result<BitVec, WidthError> {
let abs = self.abs(a)?;
self.check(b)?;
if b.msb() { self.neg(&abs) } else { Ok(abs) }
}
pub fn compare(self, a: &BitVec, b: &BitVec) -> Result<Option<Ordering>, WidthError> {
self.check(a)?;
self.check(b)?;
if let Some(w) = self.word() {
return Ok(w.compare(a.limbs()[0], b.limbs()[0]));
}
Ok(frames_short!(self, S => soft::compare::<S>(&self.fmt(), load(a), load(b))))
}
pub fn cmp(self, op: FpCmpOp, a: &BitVec, b: &BitVec) -> Result<bool, WidthError> {
let o = self.compare(a, b)?;
Ok(match (op, o) {
(_, None) => false,
(FpCmpOp::Eq, Some(o)) => o == Ordering::Equal,
(FpCmpOp::Lt, Some(o)) => o == Ordering::Less,
(FpCmpOp::Le, Some(o)) => o != Ordering::Greater,
(FpCmpOp::Gt, Some(o)) => o == Ordering::Greater,
(FpCmpOp::Ge, Some(o)) => o != Ordering::Less,
})
}
pub fn test(self, t: FpTest, a: &BitVec) -> Result<bool, WidthError> {
self.check(a)?;
let (eb, sb) = (self.eb(), self.sb());
let (e_bits, t_zero) = match self.word() {
Some(_) => {
let x = a.limbs()[0];
(
(x >> (sb - 1)) & ((1 << eb) - 1),
x & ((1 << (sb - 1)) - 1) == 0,
)
}
None => (
frames!(self, S => {
let mut l = [0u64; 1];
load::<S>(a).shr(sb - 1).low(eb).write_limbs(&mut l);
l[0]
}),
frames!(self, S => load::<S>(a).low(sb - 1).is_zero()),
),
};
let e_max = (1u64 << eb) - 1;
let nan = e_bits == e_max && !t_zero;
Ok(match t {
FpTest::Nan => nan,
FpTest::Infinite => e_bits == e_max && t_zero,
FpTest::Zero => e_bits == 0 && t_zero,
FpTest::Subnormal => e_bits == 0 && !t_zero,
FpTest::Normal => e_bits != 0 && e_bits != e_max,
FpTest::Negative => a.msb() && !nan,
FpTest::Positive => !a.msb() && !nan,
})
}
pub fn convert(self, to: FpFormat, rm: RoundingMode, a: &BitVec) -> Result<BitVec, WidthError> {
self.check(a)?;
let (from_f, to_f) = (self.fmt(), to.fmt());
if self.width().bits() <= 64 && to.width().bits() <= 64 && self.sb().max(to.sb()) <= 60 {
return Ok(store(
to.width(),
soft::convert::<u64>(&from_f, &to_f, rm, load(a)),
));
}
Ok(frames_for!(self.sb().max(to.sb()), S => {
store(to.width(), soft::convert::<S>(&from_f, &to_f, rm, load(a)))
}))
}
pub fn from_sint(self, rm: RoundingMode, x: &BitVec) -> BitVec {
let negative = x.msb();
let mag = if negative {
BitVec::un_unchecked(crate::UnOp::Neg, x)
} else {
*x
};
self.round_magnitude(rm, negative, &mag)
}
pub fn from_uint(self, rm: RoundingMode, x: &BitVec) -> BitVec {
self.round_magnitude(rm, false, x)
}
fn round_magnitude(self, rm: RoundingMode, negative: bool, mag: &BitVec) -> BitVec {
let f = self.fmt();
let wide = Wide::from_limbs(mag.limbs());
if wide.is_zero() {
return self.zero(false);
}
let keep = self.sb() + 3;
let (m, e) = match wide.bits().checked_sub(keep) {
Some(extra) if extra > 0 => (wide.shr_jam(extra), extra),
_ => (wide, 0),
};
let mut l = [0u64; 9];
m.write_limbs(&mut l);
frames!(self, S => {
let r = soft::round::<S>(&f, rm, negative, S::from_limbs(&l), i64::from(e), false);
store(self.width(), r)
})
}
pub(crate) fn round_rational(
self,
rm: RoundingMode,
negative: bool,
num: &[u64],
den: &[u64],
) -> Option<BitVec> {
let (n, d) = (Wide::from_limbs(num), Wide::from_limbs(den));
if d.is_zero() {
return None;
}
if n.is_zero() {
return Some(self.zero(false));
}
let k = i64::from(self.sb() + 3) - (i64::from(n.bits()) - i64::from(d.bits()));
let capacity = 1_200;
let (q, r) = if k >= 0 {
if i64::from(n.bits()) + k > capacity {
return None;
}
n.shl(k as u32).divrem(d)
} else {
if i64::from(d.bits()) - k > capacity {
return None;
}
n.divrem(d.shl((-k) as u32))
};
let mut l = [0u64; 20];
q.write_limbs(&mut l);
let f = self.fmt();
Some(frames!(self, S => {
let r = soft::round::<S>(&f, rm, negative, S::from_limbs(&l), -k, !r.is_zero());
store(self.width(), r)
}))
}
pub fn to_sint(self, rm: RoundingMode, a: &BitVec, width: Width) -> Result<BitVec, WidthError> {
self.to_integer(rm, a, width, true)
}
pub fn to_uint(self, rm: RoundingMode, a: &BitVec, width: Width) -> Result<BitVec, WidthError> {
self.to_integer(rm, a, width, false)
}
fn to_integer(
self,
rm: RoundingMode,
a: &BitVec,
width: Width,
signed: bool,
) -> Result<BitVec, WidthError> {
self.check(a)?;
let n = u32::from(width.bits());
let (max, min) = if signed {
(BitVec::smax(width), BitVec::smin(width))
} else {
(BitVec::ones(width), BitVec::zero(width))
};
let v = soft::to_integer::<Wide>(&self.fmt(), rm, load(a), n + 1);
Ok(match v {
soft::IntVal::Nan => BitVec::zero(width),
soft::IntVal::PosInf | soft::IntVal::Huge { sign: false } => max,
soft::IntVal::NegInf | soft::IntVal::Huge { sign: true } => min,
soft::IntVal::Int { sign, mag } => {
let mut l = [0u64; 9];
mag.write_limbs(&mut l);
if !sign {
let limit = if signed { n - 1 } else { n };
if mag.bits() > limit {
max
} else {
BitVec::wrapping_from_limbs(width, &l)
}
} else if mag.is_zero() {
BitVec::zero(width)
} else if !signed || mag > Wide::pow2(n - 1) {
min
} else {
let m = BitVec::wrapping_from_limbs(width, &l);
BitVec::un_unchecked(crate::UnOp::Neg, &m)
}
}
})
}
fn bin(
self,
a: &BitVec,
b: &BitVec,
k: impl FnOnce(FpFormat, &BitVec, &BitVec) -> BitVec,
) -> Result<BitVec, WidthError> {
self.check(a)?;
self.check(b)?;
Ok(k(self, a, b))
}
}
#[derive(Clone, Copy)]
struct Word {
width: Width,
sign: u64,
inf: u64,
}
impl Word {
fn value(self, x: u64) -> BitVec {
BitVec::from_canonical_u64(self.width, x)
}
fn compare(self, x: u64, y: u64) -> Option<Ordering> {
let abs = self.sign - 1;
let (ax, ay) = (x & abs, y & abs);
if ax > self.inf || ay > self.inf {
return None;
}
if ax == 0 && ay == 0 {
return Some(Ordering::Equal);
}
let key = |v: u64, a: u64| {
if v & self.sign != 0 {
abs - a
} else {
self.sign | a
}
};
Some(key(x, ax).cmp(&key(y, ay)))
}
}
impl fmt::Debug for FpFormat {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.name() {
Some(n) => write!(f, "{n}"),
None => write!(f, "fp<{}, {}>", self.eb(), self.sb()),
}
}
}
impl fmt::Display for FpFormat {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(self, f)
}
}
fn load<S: Frame>(v: &BitVec) -> S {
S::from_limbs(v.limbs())
}
fn store<S: Frame>(w: Width, x: S) -> BitVec {
let mut l = [0u64; 8];
x.write_limbs(&mut l);
BitVec::wrapping_from_limbs(w, &l)
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
#[non_exhaustive]
pub enum FpOp {
Add(RoundingMode),
Mul(RoundingMode),
Div(RoundingMode),
Fma(RoundingMode),
Sqrt(RoundingMode),
Rem,
RoundToIntegral(RoundingMode),
Min,
Max,
Eq,
Lt,
Le,
Convert {
to: FpFormat,
rm: RoundingMode,
},
FromSInt(RoundingMode),
FromUInt(RoundingMode),
ToSInt(RoundingMode, Width),
ToUInt(RoundingMode, Width),
}
impl FpOp {
pub fn rounding_mode(self) -> Option<RoundingMode> {
match self {
FpOp::Add(rm)
| FpOp::Mul(rm)
| FpOp::Div(rm)
| FpOp::Fma(rm)
| FpOp::Sqrt(rm)
| FpOp::RoundToIntegral(rm)
| FpOp::Convert { rm, .. }
| FpOp::FromSInt(rm)
| FpOp::FromUInt(rm)
| FpOp::ToSInt(rm, _)
| FpOp::ToUInt(rm, _) => Some(rm),
FpOp::Rem | FpOp::Min | FpOp::Max | FpOp::Eq | FpOp::Lt | FpOp::Le => None,
}
}
pub fn arity(self) -> usize {
node::Kind::of_op(self).arity()
}
}
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
pub struct FpArgs {
args: [crate::Expr; 3],
len: u8,
}
impl FpArgs {
pub(crate) fn new(args: &[crate::Expr]) -> FpArgs {
let mut a = [args[0]; 3];
a[..args.len()].copy_from_slice(args);
FpArgs {
args: a,
len: args.len() as u8,
}
}
pub fn as_slice(&self) -> &[crate::Expr] {
&self.args[..usize::from(self.len)]
}
}
pub(crate) fn eval(d: &node::Desc, args: &[BitVec]) -> BitVec {
let f = d.format;
let a = &args[0];
let r = match d.op {
FpOp::Add(rm) => f.add(rm, a, &args[1]),
FpOp::Mul(rm) => f.mul(rm, a, &args[1]),
FpOp::Div(rm) => f.div(rm, a, &args[1]),
FpOp::Fma(rm) => f.fma(rm, a, &args[1], &args[2]),
FpOp::Sqrt(rm) => f.sqrt(rm, a),
FpOp::Rem => f.rem(a, &args[1]),
FpOp::RoundToIntegral(rm) => f.round_to_integral(rm, a),
FpOp::Min => f.min(a, &args[1]),
FpOp::Max => f.max(a, &args[1]),
FpOp::Eq => f.cmp(FpCmpOp::Eq, a, &args[1]).map(BitVec::from_bool),
FpOp::Lt => f.cmp(FpCmpOp::Lt, a, &args[1]).map(BitVec::from_bool),
FpOp::Le => f.cmp(FpCmpOp::Le, a, &args[1]).map(BitVec::from_bool),
FpOp::Convert { to, rm } => f.convert(to, rm, a),
FpOp::FromSInt(rm) => Ok(f.from_sint(rm, a)),
FpOp::FromUInt(rm) => Ok(f.from_uint(rm, a)),
FpOp::ToSInt(rm, w) => f.to_sint(rm, a, w),
FpOp::ToUInt(rm, w) => f.to_uint(rm, a, w),
};
r.expect("operand widths are checked when the node is built")
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
#[non_exhaustive]
pub enum RoundingMode {
Rne,
Rna,
Rtp,
Rtn,
Rtz,
}
impl RoundingMode {
pub const ALL: [RoundingMode; 5] = [
RoundingMode::Rne,
RoundingMode::Rna,
RoundingMode::Rtp,
RoundingMode::Rtn,
RoundingMode::Rtz,
];
pub const fn name(self) -> &'static str {
match self {
RoundingMode::Rne => "rne",
RoundingMode::Rna => "rna",
RoundingMode::Rtp => "rtp",
RoundingMode::Rtn => "rtn",
RoundingMode::Rtz => "rtz",
}
}
pub fn from_name(name: &str) -> Option<RoundingMode> {
Self::ALL.into_iter().find(|m| m.name() == name)
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
#[non_exhaustive]
pub enum FpCmpOp {
Eq,
Lt,
Le,
Gt,
Ge,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
#[non_exhaustive]
pub enum FpTest {
Nan,
Infinite,
Zero,
Subnormal,
Normal,
Negative,
Positive,
}
pub fn x87_load(x: &BitVec) -> Result<BitVec, WidthError> {
if x.width().bits() != 80 {
return Err(WidthError::Mismatch {
left: 80,
right: x.width().bits(),
});
}
let bits = x.to_u128().expect("80 bits");
let sign = bits >> 79 & 1;
let e = (bits >> 64) & 0x7fff;
let i = (bits >> 63) & 1;
let frac = bits & ((1u128 << 63) - 1);
let w79 = FpFormat::X87.width();
let out = |s: u128, e: u128, f: u128| BitVec::wrapping_from_u128(w79, s << 78 | e << 63 | f);
Ok(match (e, i) {
(0, 0) => out(sign, 0, frac),
(0, _) => out(sign, 1, frac),
(_, 1) => out(sign, e, frac),
_ => FpFormat::X87.nan(),
})
}
pub fn x87_store(a: &BitVec) -> Result<BitVec, WidthError> {
FpFormat::X87.check(a)?;
let bits = a.to_u128().expect("79 bits");
let sign = bits >> 78 & 1;
let e = (bits >> 63) & 0x7fff;
let frac = bits & ((1u128 << 63) - 1);
let i = u128::from(e != 0);
Ok(BitVec::wrapping_from_u128(
Width::new(80).expect("80 bits"),
sign << 79 | e << 64 | i << 63 | frac,
))
}
impl BitVec {
pub fn from_f32(v: f32) -> BitVec {
BitVec::wrapping_from_u64(Width::W32, u64::from(v.to_bits()))
}
pub fn from_f64(v: f64) -> BitVec {
BitVec::wrapping_from_u64(Width::W64, v.to_bits())
}
pub fn to_f32(&self) -> Option<f32> {
(self.width() == Width::W32).then(|| f32::from_bits(self.limbs()[0] as u32))
}
pub fn to_f64(&self) -> Option<f64> {
(self.width() == Width::W64).then(|| f64::from_bits(self.limbs()[0]))
}
}