1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
use super::{Lexer, Token};
impl Lexer {
pub(in crate::lexer) fn read_number(&mut self) -> Token {
let int_only_after_field_dot = self.numeric_field_index_after_dot;
self.numeric_field_index_after_dot = false;
let mut num_str = String::new();
let mut is_float = false;
// Check for hex, binary, or octal prefix
if self.current_char == Some('0') {
if let Some(prefix) = self.peek(1) {
match prefix {
'x' | 'X' => {
// Hexadecimal: 0xFF, 0xDEADBEEF
self.advance(); // skip '0'
self.advance(); // skip 'x'
let mut hex_str = String::new();
while let Some(ch) = self.current_char {
if ch.is_ascii_hexdigit() {
hex_str.push(ch);
self.advance();
} else if ch == '_' {
self.advance(); // skip underscore separator
} else {
break;
}
}
let value = i64::from_str_radix(&hex_str, 16).expect("Invalid hex literal");
return Token::IntLiteral(value);
}
'b' | 'B' => {
// Binary: 0b1010, 0b1111_0000
self.advance(); // skip '0'
self.advance(); // skip 'b'
let mut bin_str = String::new();
while let Some(ch) = self.current_char {
if ch == '0' || ch == '1' {
bin_str.push(ch);
self.advance();
} else if ch == '_' {
self.advance(); // skip underscore separator
} else {
break;
}
}
let value =
i64::from_str_radix(&bin_str, 2).expect("Invalid binary literal");
return Token::IntLiteral(value);
}
'o' | 'O' => {
// Octal: 0o755, 0o644
self.advance(); // skip '0'
self.advance(); // skip 'o'
let mut oct_str = String::new();
while let Some(ch) = self.current_char {
if ('0'..='7').contains(&ch) {
oct_str.push(ch);
self.advance();
} else if ch == '_' {
self.advance(); // skip underscore separator
} else {
break;
}
}
let value =
i64::from_str_radix(&oct_str, 8).expect("Invalid octal literal");
return Token::IntLiteral(value);
}
_ => {
// Regular decimal number starting with 0
}
}
}
}
// Regular decimal number (or float)
while let Some(ch) = self.current_char {
if ch.is_ascii_digit() {
num_str.push(ch);
self.advance();
} else if ch == '_' {
// Skip underscores in numeric literals (e.g., 1_000_000)
self.advance();
} else if ch == '.'
&& !is_float
&& !int_only_after_field_dot
&& self.peek(1).is_some_and(|c| c.is_ascii_digit())
{
is_float = true;
num_str.push(ch);
self.advance();
} else {
break;
}
}
// Scientific notation: e.g. 1e10, 2.5e-3, 3E+2
if !int_only_after_field_dot
&& (self.current_char == Some('e') || self.current_char == Some('E'))
{
num_str.push(self.current_char.unwrap());
self.advance();
if self.current_char == Some('-') || self.current_char == Some('+') {
num_str.push(self.current_char.unwrap());
self.advance();
}
while let Some(ch) = self.current_char {
if ch.is_ascii_digit() {
num_str.push(ch);
self.advance();
} else {
break;
}
}
is_float = true;
}
// TDD FIX: Handle type suffixes for integer literals (0u64, 0i32, 0u32, etc.)
// Without this, "0u64" gets tokenized as IntLiteral(0) + Ident("u64"),
// which causes the "u64" to become a stray expression statement "u64;"
// resulting in E0423: expected value, found builtin type `u64`
if !is_float {
// Check for type suffix: u64, i64, u32, i32, u16, i16, u8, i8, usize, isize
let _type_suffix = if self.current_char == Some('u') || self.current_char == Some('i') {
let mut suffix = String::new();
while let Some(ch) = self.current_char {
if ch.is_ascii_alphanumeric() {
suffix.push(ch);
self.advance();
} else {
break;
}
}
// Validate it's a real type suffix
match suffix.as_str() {
s if crate::type_classification::is_numeric_suffix(s) => Some(suffix),
_ => {
// Not a valid type suffix, backtrack
// This handles cases like "0ux" which should be "0" + "ux" (identifier)
for _ in 0..suffix.len() {
self.position -= 1;
}
self.current_char = if self.position < self.input.len() {
Some(self.input[self.position])
} else {
None
};
None
}
}
} else {
None
};
if let Some(suffix) = _type_suffix {
Token::IntLiteralSuffixed(num_str.parse().unwrap(), suffix)
} else {
Token::IntLiteral(num_str.parse().unwrap())
}
} else {
Token::FloatLiteral(num_str.parse().unwrap())
}
}
}