mod arith;
mod float;
use std::fmt::{self, Display, Formatter, Write};
use anyhow::Result;
use rust_decimal::Decimal;
use crate::expr::Error;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LexDecimal {
NaN,
Infinite {
negative: bool,
},
Finite {
negative: bool,
coefficient: u128,
exponent: i32,
},
}
impl LexDecimal {
pub const ZERO: LexDecimal = LexDecimal::Finite {
negative: false,
coefficient: 0,
exponent: 0,
};
pub fn from_i64(v: i64) -> Self {
LexDecimal::Finite {
negative: v < 0,
coefficient: v.unsigned_abs() as u128,
exponent: 0,
}
}
pub fn from_decimal(v: Decimal) -> Self {
let mantissa = v.mantissa();
LexDecimal::Finite {
negative: mantissa < 0,
coefficient: mantissa.unsigned_abs(),
exponent: -(v.scale() as i32),
}
}
pub fn from_f64(v: f64) -> Self {
if v.is_nan() {
LexDecimal::NaN
} else if v.is_infinite() {
LexDecimal::Infinite {
negative: v < 0.0,
}
} else if v == 0.0 {
LexDecimal::Finite {
negative: v.is_sign_negative(),
coefficient: 0,
exponent: 0,
}
} else {
let (negative, coefficient, exponent) = float::from_f64(v);
LexDecimal::Finite {
negative,
coefficient,
exponent,
}
}
}
pub fn is_finite(&self) -> bool {
matches!(self, LexDecimal::Finite { .. })
}
pub fn is_nan(&self) -> bool {
matches!(self, LexDecimal::NaN)
}
pub fn to_decimal(&self) -> Result<Decimal> {
Ok(Decimal::from_str_radix(&self.to_string(), 10)?)
}
pub fn to_i64_positional(&self) -> Result<i64> {
let refused = || anyhow::Error::new(Error::Serialization(format!("{self} is not an i64")));
let LexDecimal::Finite {
negative,
coefficient,
exponent,
} = *self
else {
return Err(refused());
};
let zeros =
u32::try_from(exponent).ok().filter(|zeros| *zeros <= 15).ok_or_else(refused)?;
let magnitude = coefficient.checked_mul(10u128.pow(zeros)).ok_or_else(refused)?;
let magnitude = i128::try_from(magnitude).map_err(|_| refused())?;
i64::try_from(if negative {
-magnitude
} else {
magnitude
})
.map_err(|_| refused())
}
pub fn to_decimal_exact(&self) -> Result<Decimal> {
const MAX_MANTISSA: u128 = (1 << 96) - 1;
const MAX_SCALE: i32 = 28;
let unrepresentable =
|| anyhow::Error::new(Error::Serialization(format!("{self} is not a Decimal")));
let LexDecimal::Finite {
negative,
coefficient,
exponent,
} = *self
else {
return Err(unrepresentable());
};
if coefficient == 0 {
return Ok(Decimal::ZERO);
}
let mut mantissa = coefficient;
let mut exponent = exponent;
while exponent > 0 {
mantissa = mantissa.checked_mul(10).ok_or_else(unrepresentable)?;
exponent -= 1;
}
while exponent < -MAX_SCALE && mantissa % 10 == 0 {
mantissa /= 10;
exponent += 1;
}
if exponent < -MAX_SCALE || mantissa > MAX_MANTISSA {
return Err(unrepresentable());
}
let mantissa = mantissa as i128;
Ok(Decimal::from_i128_with_scale(
if negative {
-mantissa
} else {
mantissa
},
(-exponent) as u32,
))
}
pub fn to_decimal_rounded(&self) -> Option<Decimal> {
const MANTISSA_LIMIT: u128 = 1 << 96;
const MAX_SCALE: u32 = 28;
const MIN_LEADING_EXPONENT: i32 = -6;
let LexDecimal::Finite {
negative,
coefficient: mantissa,
exponent,
} = *self
else {
return None;
};
if mantissa == 0 {
return Some(Decimal::ZERO);
}
let digits_count = mantissa.ilog10() as i32 + 1;
if exponent + digits_count - 1 < MIN_LEADING_EXPONENT {
return None;
}
let (kept, scale) = if exponent > 0 {
let mut kept = mantissa;
for _ in 0..exponent {
kept = kept.checked_mul(10).filter(|kept| *kept < MANTISSA_LIMIT)?;
}
(kept, 0)
} else {
let fraction = exponent.unsigned_abs();
let pow10 = |n: u32| 10u128.checked_pow(n);
let mut dropped = fraction.saturating_sub(MAX_SCALE);
while mantissa / pow10(dropped)? >= MANTISSA_LIMIT {
dropped += 1;
}
if dropped > fraction {
return None;
}
let mut kept = mantissa / pow10(dropped)?;
let mut scale = fraction - dropped;
if dropped > 0 && mantissa / pow10(dropped - 1)? % 10 >= 5 {
kept += 1;
if kept >= MANTISSA_LIMIT {
if scale == 0 {
return None;
}
kept = (kept + 4) / 10;
scale -= 1;
}
}
(kept, scale)
};
let kept = kept as i128;
Decimal::try_from_i128_with_scale(
if negative {
-kept
} else {
kept
},
scale,
)
.ok()
}
pub fn to_f64(&self) -> f64 {
match *self {
LexDecimal::NaN => f64::NAN,
LexDecimal::Infinite {
negative,
} => {
if negative {
f64::NEG_INFINITY
} else {
f64::INFINITY
}
}
LexDecimal::Finite {
negative,
coefficient,
exponent,
} => float::to_f64(negative, coefficient, exponent),
}
}
}
impl Display for LexDecimal {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match *self {
LexDecimal::NaN => f.write_str("NaN"),
LexDecimal::Infinite {
negative,
} => f.write_str(if negative {
"-Inf"
} else {
"Inf"
}),
LexDecimal::Finite {
negative,
coefficient,
exponent,
} => {
let digits = coefficient.to_string();
let len = digits.len() as i64;
let scale = -(exponent as i64);
if negative {
f.write_char('-')?;
}
let leading_zeros = if scale >= 0 {
(scale - len).max(0)
} else {
0
};
let trailing_zeros = if scale < 0 {
-scale
} else {
0
};
if leading_zeros > 5 {
let (first, rest) = digits.split_at(1);
f.write_str(first)?;
if !rest.is_empty() {
f.write_char('.')?;
f.write_str(rest)?;
}
write!(f, "E{:+}", len - scale - 1)
} else if trailing_zeros > 15 {
write!(f, "{digits}e{:+}", -scale)
} else if scale <= 0 {
f.write_str(&digits)?;
for _ in 0..trailing_zeros {
f.write_char('0')?;
}
Ok(())
} else if scale < len {
let (int, frac) = digits.split_at((len - scale) as usize);
write!(f, "{int}.{frac}")
} else {
f.write_str("0.")?;
for _ in 0..leading_zeros {
f.write_char('0')?;
}
f.write_str(&digits)
}
}
}
}
}
pub struct DecimalLexEncoder;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum KeyDecimal {
Lex(LexDecimal),
Wide {
negative: bool,
digits: Vec<u8>,
exponent: i64,
},
}
struct FiniteHeader {
negative: bool,
scale: i64,
two_byte: bool,
digits_at: usize,
}
impl DecimalLexEncoder {
const EXP_BIAS: i32 = 6144;
const MAX_DECODED_SCALE: i32 = 6177;
const MIN_TWO_BYTE_SCALE: i64 = -5378;
const MAX_TWO_BYTE_SCALE: i64 = Self::MAX_DECODED_SCALE as i64;
const LOW_ESCAPE: u8 = 0x01;
const HIGH_ESCAPE: u8 = 0xFF;
const WIDE_SCALE_DIGITS: usize = 8;
const WIDE_SCALE_LIMIT: u64 = 255u64.pow(8);
const MAX_WIDE_DIGITS: usize = 3 * surrealdb_types::bigdecimal::MAX_COEFFICIENT_BYTES;
const FINITE_NEGATIVE_MARKER: u8 = 0x40;
const FINITE_POSITIVE_MARKER: u8 = 0xA0;
const ZERO_MARKER: u8 = 0x80;
const INFINITE_NEGATIVE_MARKER: u8 = 0x20;
const INFINITE_POSITIVE_MAKER: u8 = 0xC0;
const NAN_MARKER: u8 = 0xFF;
pub fn encode(dec: LexDecimal) -> Vec<u8> {
let mut out = Vec::with_capacity(24);
Self::encode_into(dec, &mut out);
out
}
pub fn encode_into(dec: LexDecimal, out: &mut Vec<u8>) {
let (negative, coefficient, exponent) = match dec {
LexDecimal::NaN => {
out.extend_from_slice(&[Self::NAN_MARKER, 0x00]);
return;
}
LexDecimal::Infinite {
negative: true,
} => {
out.extend_from_slice(&[Self::INFINITE_NEGATIVE_MARKER, 0x00]);
return;
}
LexDecimal::Infinite {
negative: false,
} => {
out.extend_from_slice(&[Self::INFINITE_POSITIVE_MAKER, 0x00]);
return;
}
LexDecimal::Finite {
coefficient: 0,
..
} => {
out.extend_from_slice(&[Self::ZERO_MARKER, 0x00]);
return;
}
LexDecimal::Finite {
negative,
coefficient,
exponent,
} => (negative, coefficient, exponent),
};
let mut digits = [0u8; 39];
let digits = coefficient_digits(coefficient, &mut digits);
let scale = i64::from(exponent) + (digits.len() as i64 - 1);
Self::encode_finite(negative, digits, scale, out);
}
pub fn encode_finite(negative: bool, digits: &[u8], scale: i64, out: &mut Vec<u8>) {
out.reserve(11 + digits.len().div_ceil(2) + 1);
if negative {
out.push(Self::FINITE_NEGATIVE_MARKER);
Self::encode_scale(true, scale, out);
Self::pack_digits(digits, 0xFF, out);
} else {
out.push(Self::FINITE_POSITIVE_MARKER);
Self::encode_scale(false, scale, out);
Self::pack_digits(digits, 0x00, out);
}
out.push(0x00);
}
fn encode_scale(negative: bool, scale: i64, out: &mut Vec<u8>) {
if (Self::MIN_TWO_BYTE_SCALE..=Self::MAX_TWO_BYTE_SCALE).contains(&scale) {
let biased = (scale + i64::from(Self::EXP_BIAS)) as u16;
let biased = if negative {
0xFFFF - biased
} else {
biased
};
out.extend([(biased / 255) as u8 + 1, (biased % 255) as u8 + 1]);
return;
}
let above = scale > Self::MAX_TWO_BYTE_SCALE;
let scale = i128::from(scale);
let ordinal = if above {
scale - i128::from(Self::MAX_TWO_BYTE_SCALE) - 1
} else {
i128::from(Self::WIDE_SCALE_LIMIT)
- 1 - (i128::from(Self::MIN_TWO_BYTE_SCALE) - 1 - scale)
};
let ordinal = ordinal as u64;
let (escape, ordinal) = match (negative, above) {
(false, true) => (Self::HIGH_ESCAPE, ordinal),
(false, false) => (Self::LOW_ESCAPE, ordinal),
(true, true) => (Self::LOW_ESCAPE, Self::WIDE_SCALE_LIMIT - 1 - ordinal),
(true, false) => (Self::HIGH_ESCAPE, Self::WIDE_SCALE_LIMIT - 1 - ordinal),
};
out.push(escape);
let mut digits = [0u8; Self::WIDE_SCALE_DIGITS];
let mut rest = ordinal;
for digit in digits.iter_mut().rev() {
*digit = (rest % 255) as u8 + 1;
rest /= 255;
}
out.extend_from_slice(&digits);
}
fn decode_header(bytes: &[u8]) -> Result<Option<FiniteHeader>> {
let Some(&marker) = bytes.first() else {
return Err(Error::Serialization("Cannot decode from empty buffer".to_string()).into());
};
let negative = match marker {
Self::ZERO_MARKER
| Self::INFINITE_NEGATIVE_MARKER
| Self::INFINITE_POSITIVE_MAKER
| Self::NAN_MARKER => return Ok(None),
Self::FINITE_NEGATIVE_MARKER => true,
Self::FINITE_POSITIVE_MARKER => false,
marker => {
return Err(Error::Serialization(format!("Invalid marker byte: {marker}")).into());
}
};
if bytes.len() < 3 {
return Err(Error::Serialization(format!("Buffer too short: {}", bytes.len())).into());
}
if matches!(bytes[1], Self::LOW_ESCAPE | Self::HIGH_ESCAPE) {
let ordinal_at = 2..2 + Self::WIDE_SCALE_DIGITS;
let Some(ordinal_bytes) = bytes.get(ordinal_at) else {
return Err(
Error::Serialization(format!("Buffer too short: {}", bytes.len())).into()
);
};
let mut ordinal = 0u64;
for &byte in ordinal_bytes {
ordinal = ordinal * 255 + u64::from(Self::exponent_byte(byte)?);
}
let above = (bytes[1] == Self::HIGH_ESCAPE) != negative;
let ordinal = if negative {
Self::WIDE_SCALE_LIMIT - 1 - ordinal
} else {
ordinal
};
let scale = if above {
i128::from(Self::MAX_TWO_BYTE_SCALE) + 1 + i128::from(ordinal)
} else {
i128::from(Self::MIN_TWO_BYTE_SCALE)
- 1 - (i128::from(Self::WIDE_SCALE_LIMIT) - 1 - i128::from(ordinal))
};
let scale = i64::try_from(scale)
.map_err(|_| Error::Serialization(format!("Exponent out of range: {scale}")))?;
return Ok(Some(FiniteHeader {
negative,
scale,
two_byte: false,
digits_at: 2 + Self::WIDE_SCALE_DIGITS,
}));
}
let biased_exponent = Self::exponent_byte(bytes[1])? * 255 + Self::exponent_byte(bytes[2])?;
let biased_exponent = if negative {
0xFFFF - biased_exponent
} else {
biased_exponent
};
let scale = i64::from(biased_exponent) - i64::from(Self::EXP_BIAS);
if !(Self::MIN_TWO_BYTE_SCALE..=Self::MAX_TWO_BYTE_SCALE).contains(&scale) {
return Err(Error::Serialization(format!("Exponent out of range: {scale}")).into());
}
Ok(Some(FiniteHeader {
negative,
scale,
two_byte: true,
digits_at: 3,
}))
}
pub fn decode_key(bytes: &[u8]) -> Result<KeyDecimal> {
let Some(header) = Self::decode_header(bytes)? else {
return Self::decode(bytes).map(KeyDecimal::Lex);
};
let flip = if header.negative {
0xFF
} else {
0x00
};
let digits = Self::unpack_digit_text(&bytes[header.digits_at..], flip)?;
if digits.is_empty() {
return Err(Error::Serialization("Empty mantissa".to_string()).into());
}
let exponent = header.scale.checked_sub(digits.len() as i64 - 1).ok_or_else(|| {
Error::Serialization(format!("Exponent out of range: {}", header.scale))
})?;
if header.two_byte && digits.len() <= 39 {
let coefficient =
std::str::from_utf8(&digits).ok().and_then(|text| text.parse::<u128>().ok());
if let Some(coefficient) = coefficient {
return Ok(KeyDecimal::Lex(LexDecimal::Finite {
negative: header.negative,
coefficient,
exponent: exponent as i32,
}));
}
}
Ok(KeyDecimal::Wide {
negative: header.negative,
digits,
exponent,
})
}
fn unpack_digit_text(buf: &[u8], flip: u8) -> Result<Vec<u8>> {
let mut digits = Vec::new();
for pack in buf {
let pack = *pack ^ flip;
for (half, nibble) in [("high", pack >> 4), ("low", pack & 0x0F)] {
if nibble == 0 {
return Ok(digits);
}
if !(1..=10).contains(&nibble) {
return Err(
Error::Serialization(format!("Invalid {half} nibble: {nibble}")).into()
);
}
if digits.len() >= Self::MAX_WIDE_DIGITS {
return Err(Error::Serialization("Mantissa too large".to_string()).into());
}
digits.push(b'0' + nibble - 1);
}
}
Ok(digits)
}
pub fn decode(bytes: &[u8]) -> Result<LexDecimal> {
let Some(header) = Self::decode_header(bytes)? else {
return Ok(match bytes.first().copied() {
Some(Self::INFINITE_NEGATIVE_MARKER) => LexDecimal::Infinite {
negative: true,
},
Some(Self::INFINITE_POSITIVE_MAKER) => LexDecimal::Infinite {
negative: false,
},
Some(Self::NAN_MARKER) => LexDecimal::NaN,
Some(_) | None => LexDecimal::ZERO,
});
};
if !header.two_byte {
return Err(
Error::Serialization(format!("Exponent out of range: {}", header.scale)).into()
);
}
let flip = if header.negative {
0xFF
} else {
0x00
};
let (coefficient, digit_count) = Self::unpack_digits(&bytes[header.digits_at..], flip)?;
if digit_count == 0 {
return Err(Error::Serialization("Empty mantissa".to_string()).into());
}
let scale = header.scale as i32;
Ok(LexDecimal::Finite {
negative: header.negative,
coefficient,
exponent: scale - (digit_count - 1),
})
}
fn exponent_byte(byte: u8) -> Result<u16> {
match byte.checked_sub(1) {
Some(byte) => Ok(u16::from(byte)),
None => Err(Error::Serialization("Invalid exponent byte: 0".to_string()).into()),
}
}
fn pack_digits(digits: &[u8], flip: u8, buf: &mut Vec<u8>) {
let mut iter = digits.chunks_exact(2);
for pair in &mut iter {
buf.push((((pair[0] + 1) << 4) | (pair[1] + 1)) ^ flip);
}
match iter.remainder().first() {
Some(d) => buf.push(((d + 1) << 4) ^ flip),
None => buf.push(0x0F ^ flip),
}
}
fn unpack_digits(buf: &[u8], flip: u8) -> Result<(u128, i32)> {
let mut m = 0u128;
let mut l = 0i32;
for pack in buf {
let pack = *pack ^ flip;
for (half, nibble) in [("high", pack >> 4), ("low", pack & 0x0F)] {
if nibble == 0 {
return Ok((m, l));
}
if !(1..=10).contains(&nibble) {
return Err(
Error::Serialization(format!("Invalid {half} nibble: {nibble}")).into()
);
}
m = m
.checked_mul(10)
.and_then(|m| m.checked_add((nibble - 1) as u128))
.ok_or_else(|| Error::Serialization("Mantissa too large".to_string()))?;
l += 1;
}
}
Ok((m, l))
}
}
fn coefficient_digits(mut n: u128, buf: &mut [u8; 39]) -> &[u8] {
const P19: u128 = 10_000_000_000_000_000_000;
let mut end = buf.len();
while n >= P19 {
let mut chunk = (n % P19) as u64;
n /= P19;
for _ in 0..19 {
end -= 1;
buf[end] = (chunk % 10) as u8;
chunk /= 10;
}
}
let mut rest = n as u64;
while rest > 0 {
end -= 1;
buf[end] = (rest % 10) as u8;
rest /= 10;
}
&buf[end..]
}
#[cfg(test)]
mod tests {
use super::*;
use crate::val::Number;
use crate::val::number::NumberKind;
fn finite(negative: bool, coefficient: u128, exponent: i32) -> LexDecimal {
LexDecimal::Finite {
negative,
coefficient,
exponent,
}
}
fn test_cases() -> [LexDecimal; 32] {
[
LexDecimal::from_f64(f64::NEG_INFINITY),
LexDecimal::from_f64(f64::MIN),
finite(true, i128::MIN.unsigned_abs(), 0),
LexDecimal::from_i64(i64::MIN),
LexDecimal::from_i64(-1001),
LexDecimal::from_i64(-1000),
LexDecimal::from_i64(-999),
LexDecimal::from_i64(-100),
LexDecimal::from_i64(-10),
LexDecimal::from_i64(-9),
LexDecimal::from_f64(-3.15),
LexDecimal::from_f64(-std::f64::consts::PI),
LexDecimal::from_f64(-1.5f64),
LexDecimal::from_i64(-1),
LexDecimal::ZERO,
LexDecimal::from_i64(1),
LexDecimal::from_f64(1.5f64),
LexDecimal::from_i64(2),
LexDecimal::from_f64(std::f64::consts::PI),
LexDecimal::from_f64(3.15),
LexDecimal::from_i64(9),
LexDecimal::from_i64(10),
LexDecimal::from_i64(100),
LexDecimal::from_i64(999),
LexDecimal::from_i64(1000),
LexDecimal::from_i64(1001),
LexDecimal::from_i64(i64::MAX),
finite(false, i128::MAX as u128, 0),
finite(false, u128::MAX, 0),
LexDecimal::from_f64(f64::MAX),
LexDecimal::from_f64(f64::INFINITY),
LexDecimal::from_f64(f64::NAN),
]
}
#[test]
fn test_encode_decode_roundtrip() {
for (i, case) in test_cases().into_iter().enumerate() {
let encoded = DecimalLexEncoder::encode(case);
let decoded = DecimalLexEncoder::decode(&encoded).expect("Decode should succeed");
assert_eq!(case, decoded, "Roundtrip failed for {i}: {case} != {decoded}");
}
}
#[test]
fn test_encode_terminate_with_zero() {
for (i, case) in test_cases().into_iter().enumerate() {
let encoded = DecimalLexEncoder::encode(case);
assert_eq!(
encoded.iter().filter(|&b| *b == 0x00).count(),
1,
"Encoded buffer should contains only one 0x00 - {i}: {case} {encoded:?}"
);
assert_eq!(
encoded.iter().position(|&b| b == 0x00).unwrap(),
encoded.len() - 1,
"Encoded buffer should terminate with 0x00 - {i}: {case} {encoded:?}"
);
}
}
#[test]
fn test_lexicographic_ordering() {
for (i, window) in test_cases().windows(2).enumerate() {
let b1 = DecimalLexEncoder::encode(window[0]);
let b2 = DecimalLexEncoder::encode(window[1]);
assert!(
b1 < b2,
"#{i} - {} < {} (after serialization) - {b1:?} < {b2:?}",
window[0],
window[1]
);
}
}
#[test]
fn test_decode_empty_buffer() {
let result = DecimalLexEncoder::decode(&[]);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("Cannot decode from empty buffer"));
}
#[test]
fn test_decode_buffer_too_short() {
let result = DecimalLexEncoder::decode(&[0xA0]);
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.to_string().contains("Buffer too short"), "{err:?}");
}
#[test]
fn test_decode_invalid_marker() {
let result = DecimalLexEncoder::decode(&[0x42, 0x00, 0x00, 0x00]);
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.to_string(), "Serialization error: Invalid marker byte: 66", "{err:?}");
}
#[test]
fn test_decode_empty_mantissa() {
let result = DecimalLexEncoder::decode(&[0xA0, 0x19, 0x19, 0x00]);
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.to_string().contains("Empty mantissa"), "{err:?}");
}
#[test]
fn test_decode_rejects_mantissa_overflow() {
let mut bytes = vec![0xA0, 0x19, 0x19];
bytes.extend(std::iter::repeat_n(0xAA, 20));
bytes.push(0x0F);
let err = DecimalLexEncoder::decode(&bytes).unwrap_err();
assert!(err.to_string().contains("Mantissa too large"), "{err:?}");
}
struct Frozen {
value: Number,
key: Vec<u8>,
decoded: String,
decoded_kind: String,
}
fn frozen() -> Vec<Frozen> {
fn hex(s: &str) -> Vec<u8> {
(0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect()
}
include_str!("decimal/number_keys.txt")
.lines()
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.map(|line| {
let fields: Vec<&str> = line.split(' ').collect();
let value: Vec<&str> = fields[0].split(':').collect();
let value = match value[0] {
"i" => Number::Int(value[1].parse().unwrap()),
"d" => Number::Decimal(Decimal::from_i128_with_scale(
value[1].parse().unwrap(),
value[2].parse().unwrap(),
)),
"f" => {
Number::Float(f64::from_bits(u64::from_str_radix(value[1], 16).unwrap()))
}
other => panic!("unknown kind {other}"),
};
Frozen {
value,
key: hex(fields[1]),
decoded: fields[2].to_string(),
decoded_kind: fields[3].to_string(),
}
})
.collect()
}
fn repr(r: &Result<Number>) -> String {
match r {
Ok(Number::Int(v)) => format!("i:{v}"),
Ok(Number::Float(v)) => format!("f:{:x}", v.to_bits()),
Ok(Number::Decimal(d)) => format!("d:{}:{}", d.mantissa(), d.scale()),
Ok(Number::BigDecimal(b)) => format!("b:{}:{}", b, b.scale()),
Ok(Number::I8(v)) => format!("i8:{v}"),
Ok(Number::U8(v)) => format!("u8:{v}"),
Ok(Number::I16(v)) => format!("i16:{v}"),
Ok(Number::U16(v)) => format!("u16:{v}"),
Ok(Number::I32(v)) => format!("i32:{v}"),
Ok(Number::U32(v)) => format!("u32:{v}"),
Ok(Number::U64(v)) => format!("u64:{v}"),
Ok(Number::F16(v)) => format!("f16:{:x}", v.to_bits()),
Ok(Number::F32(v)) => format!("f32:{:x}", v.to_bits()),
Err(_) => "err".to_string(),
}
}
#[test]
fn encodings_match_frozen_fixture() {
let cases = frozen();
assert!(cases.len() > 3000);
for c in cases {
assert_eq!(c.value.as_decimal_buf(), c.key, "key bytes changed for {:?}", c.value);
assert_eq!(
repr(&Number::from_decimal_buf(&c.key)),
c.decoded,
"decoding changed for {:?}",
c.value
);
let kind = match c.value {
Number::Int(_) => NumberKind::Int,
Number::Float(_) => NumberKind::Float,
Number::Decimal(_) => NumberKind::Decimal,
Number::BigDecimal(_) => NumberKind::BigDecimal,
Number::I8(_) => NumberKind::I8,
Number::U8(_) => NumberKind::U8,
Number::I16(_) => NumberKind::I16,
Number::U16(_) => NumberKind::U16,
Number::I32(_) => NumberKind::I32,
Number::U32(_) => NumberKind::U32,
Number::U64(_) => NumberKind::U64,
Number::F16(_) => NumberKind::F16,
Number::F32(_) => NumberKind::F32,
};
assert_eq!(
repr(&Number::from_decimal_buf_kind(&c.key, kind)),
c.decoded_kind,
"kind-tagged decoding changed for {:?}",
c.value
);
}
}
#[test]
fn frozen_keys_preserve_order() {
let cases = frozen();
let mut ints: Vec<_> = cases.iter().filter(|c| matches!(c.value, Number::Int(_))).collect();
let mut floats: Vec<_> = cases
.iter()
.filter(|c| matches!(c.value, Number::Float(f) if !f.is_nan() && f != 0.0))
.collect();
let mut decimals: Vec<_> =
cases.iter().filter(|c| matches!(c.value, Number::Decimal(_))).collect();
for set in [&mut ints, &mut floats, &mut decimals] {
set.sort_by_key(|a| a.value.clone());
for w in set.windows(2) {
if w[0].value < w[1].value {
assert!(
w[0].key < w[1].key,
"{:?} < {:?} but keys are not",
w[0].value,
w[1].value
);
}
}
}
}
#[test]
fn frozen_keys_preserve_order_across_kinds() {
fn comparable(a: &Number, b: &Number) -> bool {
let float = |n: &Number| matches!(n, Number::Float(_));
if float(a) == float(b) {
return true;
}
let (x, y) = (a.to_float(), b.to_float());
(x - y).abs() > 1e-12 * x.abs().max(y.abs())
}
let cases = frozen();
let (nan, rest): (Vec<_>, Vec<_>) =
cases.iter().partition(|c| matches!(c.value, Number::Float(f) if f.is_nan()));
assert!(!nan.is_empty());
for n in &nan {
assert_eq!(n.key, [0xFF, 0x00]);
assert!(rest.iter().all(|c| c.key < n.key), "NaN must sort last");
}
for c in &rest {
if c.value != Number::Float(f64::NEG_INFINITY) {
assert!(c.key > vec![0x20, 0x00], "-inf must sort first");
}
}
let mut compared = 0usize;
for a in &rest {
for b in &rest {
if a.value < b.value && comparable(&a.value, &b.value) {
compared += 1;
assert!(a.key < b.key, "{:?} < {:?} but keys are not", a.value, b.value);
}
}
}
assert!(compared > 1_000_000);
}
#[test]
fn test_display_matches_notation_thresholds() {
let cases = [
(finite(false, 15, -1), "1.5"),
(finite(true, 15, -1), "-1.5"),
(finite(false, 15, -6), "0.000015"),
(finite(false, 15, -7), "0.0000015"),
(finite(false, 15, -8), "1.5E-7"),
(finite(false, 1, -7), "1E-7"),
(finite(false, 1000, 0), "1000"),
(finite(false, 15, 15), "15000000000000000"),
(finite(false, 15, 16), "15e+16"),
(finite(false, 12345, -2), "123.45"),
(LexDecimal::ZERO, "0"),
(LexDecimal::NaN, "NaN"),
];
for (value, text) in cases {
assert_eq!(value.to_string(), text);
}
}
fn buffer(negative: bool, scale: i32, digits: &str) -> Vec<u8> {
let digits: Vec<u8> = digits.bytes().map(|b| b - b'0').collect();
let biased = (scale + DecimalLexEncoder::EXP_BIAS) as u16;
let shift = |e: u16| [(e / 255) as u8 + 1, (e % 255) as u8 + 1];
let mut buf = Vec::new();
if negative {
buf.push(DecimalLexEncoder::FINITE_NEGATIVE_MARKER);
buf.extend(shift(0xFFFF - biased));
DecimalLexEncoder::pack_digits(&digits, 0xFF, &mut buf);
} else {
buf.push(DecimalLexEncoder::FINITE_POSITIVE_MARKER);
buf.extend(shift(biased));
DecimalLexEncoder::pack_digits(&digits, 0x00, &mut buf);
}
buf.push(0x00);
buf
}
fn lex(text: &str) -> LexDecimal {
let (negative, rest) = match text.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, text),
};
let (mantissa, exp) = match rest.split_once(['E', 'e']) {
Some((mantissa, exp)) => (mantissa, exp.parse::<i32>().unwrap()),
None => (rest, 0),
};
let (int, frac) = mantissa.split_once('.').unwrap_or((mantissa, ""));
LexDecimal::Finite {
negative,
coefficient: format!("{int}{frac}").parse().unwrap(),
exponent: exp - frac.len() as i32,
}
}
#[test]
fn decode_refuses_a_scale_past_the_largest_encoded() {
for negative in [false, true] {
for scale in [-324, 0, 28, 308, 6177] {
let decoded = DecimalLexEncoder::decode(&buffer(negative, scale, "1"));
assert!(decoded.is_ok(), "scale {scale} negative={negative}: {decoded:?}");
}
for scale in [6178, 40_000, 50_000, 58_000] {
let err = DecimalLexEncoder::decode(&buffer(negative, scale, "1"))
.expect_err("a scale past the largest encoded one is not decodable");
assert!(err.to_string().contains("Exponent out of range"), "{err:?}");
}
}
}
#[test]
fn decode_refuses_a_zero_exponent_byte() {
for marker in
[DecimalLexEncoder::FINITE_POSITIVE_MARKER, DecimalLexEncoder::FINITE_NEGATIVE_MARKER]
{
for exponent in [[0x00, 0x80], [0x80, 0x00], [0x00, 0x00]] {
let err =
DecimalLexEncoder::decode(&[marker, exponent[0], exponent[1], 0x20, 0x00])
.expect_err("a zero exponent byte is not part of any encoding");
assert!(err.to_string().contains("Invalid exponent byte"), "{err:?}");
}
}
}
#[test]
fn decode_never_panics_over_every_exponent_byte_pair() {
for marker in
[DecimalLexEncoder::FINITE_POSITIVE_MARKER, DecimalLexEncoder::FINITE_NEGATIVE_MARKER]
{
let digits: [u8; 2] = if marker == DecimalLexEncoder::FINITE_POSITIVE_MARKER {
[0x20, 0x00]
} else {
[!0x20, 0xFF]
};
for hi in 0..=u8::MAX {
for lo in 0..=u8::MAX {
let decoded =
DecimalLexEncoder::decode(&[marker, hi, lo, digits[0], digits[1]]);
if hi == 0 || lo == 0 {
assert!(
decoded.is_err(),
"[{marker:#x}, {hi}, {lo}] decoded as {decoded:?}"
);
}
}
}
}
}
#[test]
fn decode_refuses_a_mantissa_wider_than_the_coefficient() {
let widest = u128::MAX.to_string();
for negative in [false, true] {
let decoded = DecimalLexEncoder::decode(&buffer(negative, 38, &widest))
.expect("the widest coefficient is decodable");
assert!(matches!(
decoded,
LexDecimal::Finite {
coefficient: u128::MAX,
..
}
));
for too_wide in [format!("{widest}0"), "9".repeat(39), "9".repeat(80)] {
let err = DecimalLexEncoder::decode(&buffer(negative, 38, &too_wide))
.expect_err("a coefficient past 2^128 is not decodable");
assert!(err.to_string().contains("Mantissa too large"), "{too_wide}: {err:?}");
}
}
}
#[test]
fn to_decimal_rounded_reads_a_value_the_way_a_decimal_parses_it() {
let cases: [(&str, &str); 17] = [
("0", "0"),
("1", "1"),
("-1.50", "-1.50"),
("100.000", "100.000"),
("0.000001", "0.000001"),
("-0.000001234", "-0.000001234"),
("79228162514264337593543950335", "79228162514264337593543950335"),
("7.9228162514264337593543950335", "7.9228162514264337593543950335"),
("0.1000000000000000055511151231257827", "0.1000000000000000055511151231"),
("0.12345678901234567890123456785", "0.1234567890123456789012345679"),
("-0.12345678901234567890123456785", "-0.1234567890123456789012345679"),
("0.12345678901234567890123456784999", "0.1234567890123456789012345678"),
("7.9228162514264337593543950336", "7.922816251426433759354395034"),
("7.92281625142643375935439503355", "7.922816251426433759354395034"),
("79228162514264337593543950335.4", "79228162514264337593543950335"),
("1E+28", "10000000000000000000000000000"),
("-3.4E+2", "-340"),
];
for (value, expected) in cases {
let expected: Decimal = expected.parse().unwrap();
let got =
lex(value).to_decimal_rounded().unwrap_or_else(|| panic!("{value} has a Decimal"));
assert_eq!(got.serialize(), expected.serialize(), "{value}: {got} != {expected}");
}
}
#[test]
fn to_decimal_rounded_has_nothing_for_what_a_decimal_does_not_stand_for() {
for text in [
"79228162514264337593543950336",
"-79228162514264337593543950336",
"79228162514264337593543950335.5",
"1E+29",
"1E+6144",
"9999999999999999455752309870428160",
"9.99999E-7",
"1E-7",
"-1.5E-10",
"9.9999999999999994515327145420957165173E-21",
"1E-6143",
] {
assert_eq!(lex(text).to_decimal_rounded(), None, "{text}");
}
for value in [
LexDecimal::NaN,
LexDecimal::Infinite {
negative: false,
},
LexDecimal::Infinite {
negative: true,
},
] {
assert_eq!(value.to_decimal_rounded(), None, "{value}");
}
}
#[test]
fn to_decimal_rounded_agrees_with_parsing_the_positional_text() {
use rand::{Rng, SeedableRng};
let mut rng = rand::rngs::StdRng::seed_from_u64(0xDEC1);
let mut positional = 0u32;
for _ in 0..200_000 {
let digits = rng.random_range(1..=39u32);
let coefficient = match 10u128.checked_pow(digits) {
Some(limit) => rng.random::<u128>() % limit,
None => rng.random::<u128>(),
};
let value = LexDecimal::Finite {
negative: rng.random(),
coefficient,
exponent: rng.random_range(-60..=2i32),
};
if coefficient == 0 || value.to_string().contains(['e', 'E']) {
continue;
}
positional += 1;
let parsed = value.to_decimal();
let rounded = value.to_decimal_rounded();
match (&parsed, &rounded) {
(Ok(parsed), Some(rounded)) => {
assert_eq!(
parsed.serialize(),
rounded.serialize(),
"{value}: {parsed} != {rounded}"
);
}
(Err(_), None) => {}
_ => panic!("{value}: parsed as {parsed:?}, rounded to {rounded:?}"),
}
}
assert!(positional > 50_000, "only {positional} positional values were generated");
}
#[test]
fn to_i64_positional_agrees_with_parsing_the_text() {
use rand::{Rng, SeedableRng};
let check = |value: LexDecimal| {
let parsed = value.to_string().parse::<i64>().ok();
assert_eq!(value.to_i64_positional().ok(), parsed, "{value}");
};
let coefficients = [
0,
1,
9,
10,
99,
i64::MAX as u128 / 10,
i64::MAX as u128 - 1,
i64::MAX as u128,
i64::MAX as u128 + 1,
i64::MAX as u128 + 2,
u64::MAX as u128,
1 << 96,
u128::MAX / 10,
u128::MAX,
];
for negative in [false, true] {
for coefficient in coefficients {
for exponent in -45..=45 {
check(LexDecimal::Finite {
negative,
coefficient,
exponent,
});
}
}
check(LexDecimal::Infinite {
negative,
});
}
check(LexDecimal::NaN);
let mut rng = rand::rngs::StdRng::seed_from_u64(0x1664);
let mut converted = 0u32;
for _ in 0..200_000 {
let digits = rng.random_range(1..=39u32);
let coefficient = match 10u128.checked_pow(digits) {
Some(limit) => rng.random::<u128>() % limit,
None => rng.random::<u128>(),
};
let value = LexDecimal::Finite {
negative: rng.random(),
coefficient,
exponent: rng.random_range(-4..=20i32),
};
converted += u32::from(value.to_i64_positional().is_ok());
check(value);
}
assert!(converted > 10_000, "only {converted} values were integers");
}
#[test]
fn to_decimal_exact_is_the_inverse_of_from_decimal() {
for text in [
"0",
"1",
"-1",
"1.50",
"100.000",
"0.0000001",
"-0.00000000000000000000000001",
"7.9228162514264337593543950335",
"79228162514264337593543950335",
"-79228162514264337593543950335",
] {
let d: Decimal = text.parse().unwrap();
let back = LexDecimal::from_decimal(d).to_decimal_exact().unwrap();
assert_eq!(back, d, "{text}");
assert_eq!(back.scale(), d.scale(), "{text} lost its scale");
}
}
#[test]
fn to_decimal_exact_reads_any_spelling_of_a_representable_value() {
let cases: [(&str, &str); 5] = [
("1E+3", "1000"),
("12E+27", "12000000000000000000000000000"),
("1200E-30", "0.0000000000000000000000000012"),
("5E-7", "0.0000005"),
("-0E+10", "0"),
];
for (value, decimal) in cases {
let decimal: Decimal = decimal.parse().unwrap();
assert_eq!(lex(value).to_decimal_exact().unwrap(), decimal, "{value}");
}
}
#[test]
fn to_decimal_exact_refuses_what_a_decimal_cannot_hold() {
for text in [
"1E+29",
"79228162514264337593543950336",
"9999999999999999999999999999999999",
"1E+6144",
"1E-29",
"123E-30",
"1E-6143",
] {
assert!(lex(text).to_decimal_exact().is_err(), "{text} is not a Decimal");
}
for value in [
LexDecimal::NaN,
LexDecimal::Infinite {
negative: false,
},
] {
assert!(value.to_decimal_exact().is_err(), "{value} is not a Decimal");
}
}
}