hermes_parser/number.rs
1//! Numeric-literal conversion primitives for the JS lexer, ported from
2//! include/hermes/Support/Conversions.h. The decimal/real path uses Rust std's
3//! correctly-rounded `str::parse::<f64>()` (the same fast_float algorithm the
4//! C++ lexer uses) — no FFI, no third-party crate.
5//!
6//! Public API:
7//! - [`parse_int_with_radix_digits`] — digit-by-digit radix parser (callback style).
8//! - [`parse_int_with_radix`] — full integer-radix parse with power-of-2 rounding path.
9//! - [`str_to_double`] — decimal/real path: pure-Rust, bit-identical to `fastStrToDouble`.
10
11/// Takes a letter (a-z or A-Z) and makes it lowercase.
12/// Port of `charLetterToLower` (Conversions.h:160).
13#[inline]
14fn char_letter_to_lower(c: u8) -> u8 {
15 c | 32
16}
17
18/// Takes a non-empty string (without the leading "0x" if hex) and parses it
19/// as radix `radix`, calling `digit` with the value of each digit,
20/// going from left to right.
21/// `allow_sep`: when true, allow '_' as a separator and ignore it when parsing.
22/// Returns true if the string was successfully parsed, false otherwise.
23/// Port of `parseIntWithRadixDigits` (Conversions.h:166).
24pub fn parse_int_with_radix_digits(
25 bytes: &[u8],
26 radix: u32,
27 allow_sep: bool,
28 mut digit: impl FnMut(u8),
29) -> bool {
30 debug_assert!((2..=36).contains(&radix), "Invalid radix passed to parseIntWithRadix");
31 debug_assert!(!bytes.is_empty(), "Empty string");
32 // Use i32 for arithmetic so `radix - 10` does not underflow for radix < 10.
33 let radix = radix as i32;
34 for (i, &c) in bytes.iter().enumerate() {
35 let c_low = char_letter_to_lower(c);
36 if c >= b'0' && c <= b'9' && (c as i32) < b'0' as i32 + radix {
37 digit(c - b'0');
38 } else if c_low >= b'a' && (c_low as i32) < b'a' as i32 + radix - 10 {
39 digit(c_low - b'a' + 0xa);
40 } else if allow_sep && c == b'_' {
41 // Ensure the '_' is in a valid location.
42 // It can only be between two existing digits.
43 if i == 0 || i == bytes.len() - 1 {
44 return false;
45 }
46 // Note that the previous character must not be '_' if the current
47 // character is '_', because we would have returned false.
48 // So just check if the next character is '_'.
49 if bytes[i + 1] == b'_' {
50 return false;
51 }
52 } else {
53 return false;
54 }
55 }
56 true
57}
58
59/// Takes a non-empty string (without the leading "0x" if hex) and parses it
60/// as radix `radix`.
61/// `allow_sep`: when true, allow '_' as a separator and ignore it when parsing.
62/// Returns the f64 that results on success, or None on error.
63/// Port of `parseIntWithRadix` (Conversions.h:204), including the >2^53
64/// power-of-two bit-by-bit rounding path (lines 222–328).
65pub fn parse_int_with_radix(bytes: &[u8], radix: u32, allow_sep: bool) -> Option<f64> {
66 let mut result: f64 = 0.0;
67 let success = parse_int_with_radix_digits(bytes, radix, allow_sep, |d| {
68 result *= radix as f64;
69 result += d as f64;
70 });
71 if !success {
72 return None;
73 }
74
75 // The largest value that fits in the 53-bit mantissa (2**53).
76 const MAX_MANTISSA: f64 = 9007199254740992.0;
77 if result >= MAX_MANTISSA && radix.is_power_of_two() {
78 // If the result is too high, manually reconstruct the double if
79 // the radix is 2, 4, 8, 16, 32.
80 // Go through the digits bit by bit, and manually round when necessary.
81 result = 0.0;
82
83 // Keep track of how far along parsing is using this enum.
84 #[derive(PartialEq)]
85 enum Mode {
86 LeadingZero, // Haven't seen a set bit yet.
87 Mantissa, // Lower bits that allow exact representation.
88 ExpLowBit, // Lowest bit of the exponent (determine rounding).
89 ExpLeadingZero, // Zeros in the exponent.
90 Exponent, // Seen a set bit in the exponent.
91 }
92
93 let mut remaining_mantissa: usize = 53;
94 let mut exp_factor: f64 = 0.0;
95 let mut cur_digit: usize = 0;
96
97 let mut last_mantissa_bit = false;
98 let mut lowest_exponent_bit = false;
99
100 let mut cur_mode = Mode::LeadingZero;
101 // Plain iterator (matches the C++ `auto itr = str.begin()`); we only ever
102 // advance with `next()`.
103 let mut itr = bytes.iter();
104 let mut bit_mask: u32 = 0;
105 loop {
106 if bit_mask == 0 {
107 // Only need to do this check every log2(radix) iterations.
108 match itr.next() {
109 None => break,
110 Some(&c) => {
111 let c = c as char;
112 if allow_sep && c == '_' {
113 // Skip separators; we already validated them.
114 continue;
115 }
116 let c_low = char_letter_to_lower(c as u8);
117 if c >= '0' && c <= '9' {
118 cur_digit = (c as u8 - b'0') as usize;
119 } else {
120 // Must be valid, else we would have returned None on first pass.
121 debug_assert!(
122 c_low >= b'a' && (c_low as i32) < b'a' as i32 + radix as i32 - 10
123 );
124 cur_digit = (c_low - b'a' + 0xa) as usize;
125 }
126 // Reset bitmask to look at the first bit.
127 bit_mask = radix >> 1;
128 }
129 }
130 }
131 let cur_bit = (cur_digit as u32 & bit_mask) != 0;
132 bit_mask >>= 1;
133
134 match cur_mode {
135 Mode::LeadingZero => {
136 // Go through the string until we hit a nonzero bit.
137 if cur_bit {
138 remaining_mantissa -= 1;
139 result = 1.0;
140 // No more leading zeros.
141 cur_mode = Mode::Mantissa;
142 }
143 }
144 Mode::Mantissa => {
145 // Read into the lower bits of the mantissa (plain binary).
146 result *= 2.0;
147 result += cur_bit as u8 as f64;
148 remaining_mantissa -= 1;
149 if remaining_mantissa == 0 {
150 // Out of bits, set the last bit and go to the next curMode.
151 last_mantissa_bit = cur_bit;
152 cur_mode = Mode::ExpLowBit;
153 }
154 }
155 Mode::ExpLowBit => {
156 lowest_exponent_bit = cur_bit;
157 exp_factor = 2.0;
158 cur_mode = Mode::ExpLeadingZero;
159 }
160 Mode::ExpLeadingZero => {
161 if cur_bit {
162 cur_mode = Mode::Exponent;
163 }
164 exp_factor *= 2.0;
165 }
166 Mode::Exponent => {
167 exp_factor *= 2.0;
168 }
169 }
170 }
171 match cur_mode {
172 Mode::LeadingZero | Mode::Mantissa | Mode::ExpLowBit => {
173 // Nothing to do here, already read those in.
174 }
175 Mode::ExpLeadingZero => {
176 // Rounding up.
177 result += (lowest_exponent_bit && last_mantissa_bit) as u8 as f64;
178 result *= exp_factor;
179 }
180 Mode::Exponent => {
181 // Rounding up.
182 result += lowest_exponent_bit as u8 as f64;
183 result *= exp_factor;
184 }
185 }
186 }
187 Some(result)
188}
189
190/// Parse a cleaned decimal/real numeric buffer (only `[0-9.eE+-]`, separators
191/// already stripped) to an f64. Returns the value if the WHOLE buffer parses,
192/// or None on invalid input — mirroring `fastStrToDouble`'s "consume all or
193/// fail" contract. Out-of-range inputs parse to +/-inf or 0.0 (as fast_float and
194/// Rust std both do). Rust std's parser is the same correctly-rounded algorithm
195/// as the lexer's `fast_float`, so results are bit-identical.
196pub fn str_to_double(bytes: &[u8]) -> Option<f64> {
197 // The buffer is pure ASCII; from_utf8 cannot fail, but handle defensively.
198 let s = std::str::from_utf8(bytes).ok()?;
199 s.parse::<f64>().ok()
200}
201
202#[cfg(test)]
203mod int_tests {
204 use super::*;
205
206 #[test]
207 fn small_exact() {
208 assert_eq!(parse_int_with_radix(b"ff", 16, true), Some(255.0));
209 assert_eq!(parse_int_with_radix(b"777", 8, true), Some(511.0));
210 assert_eq!(parse_int_with_radix(b"1010", 2, true), Some(10.0));
211 assert_eq!(parse_int_with_radix(b"123", 10, true), Some(123.0));
212 assert_eq!(parse_int_with_radix(b"z", 36, true), Some(35.0));
213 // Letters are rejected for radix <= 10 (no u32 underflow on radix-10).
214 assert_eq!(parse_int_with_radix(b"a", 10, true), None);
215 assert_eq!(parse_int_with_radix(b"8", 8, true), None);
216 }
217
218 #[test]
219 fn separators() {
220 assert_eq!(parse_int_with_radix(b"1_000", 10, true), Some(1000.0));
221 assert_eq!(
222 parse_int_with_radix(b"dead_beef", 16, true),
223 Some(0xdeadbeef_u32 as f64)
224 );
225 assert_eq!(parse_int_with_radix(b"_1", 10, true), None); // leading
226 assert_eq!(parse_int_with_radix(b"1_", 10, true), None); // trailing
227 assert_eq!(parse_int_with_radix(b"1__2", 10, true), None); // double
228 // When separators are disallowed, '_' is just an invalid digit.
229 assert_eq!(parse_int_with_radix(b"1_0", 10, false), None);
230 }
231
232 #[test]
233 fn invalid() {
234 assert_eq!(parse_int_with_radix(b"xyz", 16, true), None);
235 assert_eq!(parse_int_with_radix(b"12.3", 10, true), None);
236 }
237
238 // The power-of-2 high-precision path (result >= 2^53) must produce the
239 // correctly-rounded f64. Rust's `u128 as f64` is round-to-nearest-even, an
240 // independent correctly-rounded oracle for any value that fits in u128.
241 #[test]
242 fn large_power_of_two_rounding_matches_u128_oracle() {
243 let cases: &[(&[u8], u32)] = &[
244 (b"20000000000001", 16), // 2^53 + 1 region
245 (b"1fffffffffffff", 16), // 2^53 - 1 (exact)
246 (b"ffffffffffffffff", 16), // u64::MAX
247 (b"123456789abcdef0123", 16), // > 2^64, still < 2^128
248 (b"777777777777777777777", 8), // large octal
249 (b"1111111111111111111111111111111111111111111111111111111", 2),
250 (b"20000000000000", 16), // exactly 2^53 (>= boundary triggers the path)
251 (b"33333333333333333333333333333", 4), // large radix-4
252 (b"vvvvvvvvvvvv", 32), // large radix-32 (v = 31)
253 ];
254 for &(s, radix) in cases {
255 let txt = std::str::from_utf8(s).unwrap();
256 let expected = u128::from_str_radix(txt, radix).unwrap() as f64;
257 assert_eq!(
258 parse_int_with_radix(s, radix, true),
259 Some(expected),
260 "mismatch for {txt} radix {radix}"
261 );
262 }
263 }
264
265 // Radix 10 is NOT a power of two, so even above 2^53 it uses the plain f64
266 // accumulation (no bit-by-bit precision path). This is a sanity check that the
267 // accumulation agrees with the u128->f64 cast for such a value (both round to
268 // 2^53), not a test of the precision path.
269 #[test]
270 fn large_decimal() {
271 assert_eq!(
272 parse_int_with_radix(b"9007199254740993", 10, true),
273 Some(9007199254740993u128 as f64)
274 );
275 }
276}
277
278#[cfg(test)]
279mod double_tests {
280 use super::*;
281
282 fn bits(v: f64) -> u64 {
283 v.to_bits()
284 }
285
286 #[test]
287 fn known_bit_patterns() {
288 // These mirror the js-lexer-dump oracle's `bits=` output.
289 assert_eq!(str_to_double(b"5").map(bits), Some(0x4014000000000000));
290 assert_eq!(str_to_double(b"0.1").map(bits), Some(0x3fb999999999999a));
291 assert_eq!(str_to_double(b"255").map(bits), Some(0x406fe00000000000));
292 // (5, 0.1, 255 bit patterns were confirmed against the real C++ js-lexer-dump.)
293 // These two cross-check delegation to the std parser:
294 assert_eq!(str_to_double(b"1e10").map(bits), Some(1e10f64.to_bits()));
295 assert_eq!(str_to_double(b"12.5").map(bits), Some(12.5f64.to_bits()));
296 }
297
298 #[test]
299 fn must_consume_all() {
300 assert_eq!(str_to_double(b"12x"), None);
301 assert_eq!(str_to_double(b""), None);
302 assert_eq!(str_to_double(b"1.2.3"), None);
303 }
304
305 #[test]
306 fn leading_plus_and_exponent() {
307 assert_eq!(str_to_double(b"+5").map(bits), Some(5.0f64.to_bits()));
308 assert_eq!(str_to_double(b"5e+3").map(bits), Some(5000.0f64.to_bits()));
309 assert_eq!(str_to_double(b"5E-3").map(bits), Some(0.005f64.to_bits()));
310 }
311
312 #[test]
313 fn out_of_range() {
314 // Overflow -> +inf; underflow -> 0.0 (matches fast_float ignoring out-of-range).
315 assert_eq!(str_to_double(b"1e400"), Some(f64::INFINITY));
316 assert_eq!(str_to_double(b"1e-400"), Some(0.0));
317 }
318}