1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
use std::fmt::Debug;
use bumpalo::Bump;
use bumpalo::collections::CollectIn;
use bumpalo::collections::Vec;
use mago_database::file::FileId;
use mago_database::file::HasFileId;
use mago_span::Position;
use mago_span::Span;
use mago_syntax_core::parser::LookaheadBuf;
use crate::ast::sequence::Sequence;
use crate::ast::trivia::Trivia;
use crate::ast::trivia::TriviaKind;
use crate::error::ParseError;
use crate::error::SyntaxError;
use crate::lexer::Lexer;
use crate::token::Token;
use crate::token::TokenKind;
#[derive(Debug)]
pub struct TokenStream<'input, 'arena> {
arena: &'arena Bump,
lexer: Lexer<'input>,
buffer: LookaheadBuf<Token<'input>, 16>,
trivia: Vec<'arena, Token<'input>>,
position: Position,
file_id: FileId,
}
impl<'input, 'arena> TokenStream<'input, 'arena> {
pub fn new(arena: &'arena Bump, lexer: Lexer<'input>) -> TokenStream<'input, 'arena> {
let position = lexer.current_position();
let file_id_cached = lexer.file_id();
TokenStream {
arena,
lexer,
buffer: LookaheadBuf::new(),
trivia: Vec::new_in(arena),
position,
file_id: file_id_cached,
}
}
/// Returns the current position of the stream within the source file.
///
/// This position represents the end location of the most recently
/// consumed significant token via `advance()` or `consume()`.
#[inline]
#[must_use]
pub const fn current_position(&self) -> Position {
self.position
}
/// Returns whether the stream has consumed all tokens up to EOF.
///
/// # Errors
///
/// Returns a [`SyntaxError`] if the lexer fails to produce the next token.
#[inline]
pub fn has_reached_eof(&mut self) -> Result<bool, SyntaxError> {
Ok(self.fill_buffer(1)?.is_none())
}
/// Consumes and returns the next significant token.
///
/// # Errors
///
/// Returns a [`ParseError`] if EOF is reached or a lexer error occurs.
#[inline]
pub fn consume(&mut self) -> Result<Token<'input>, ParseError> {
match self.advance() {
Some(Ok(token)) => Ok(token),
Some(Err(error)) => Err(error.into()),
None => Err(self.unexpected(None, &[])),
}
}
/// Consumes the next token only if it matches the expected kind.
///
/// Returns the token if it matches, otherwise returns an error.
///
/// # Errors
///
/// Returns a [`ParseError`] if the next token's kind does not match `kind`, or if EOF is reached.
#[inline]
pub fn eat(&mut self, kind: TokenKind) -> Result<Token<'input>, ParseError> {
// Fast path: head already buffered. Avoids the Result<Option<...>>
// round trip from `peek_kind` plus a follow-up `lookahead` on the
// happy path.
if let Some(token) = self.buffer.get(0) {
if token.kind == kind {
let _ = self.buffer.pop_front();
self.position = Position::new(token.start.offset + token.value.len() as u32);
return Ok(token);
}
return Err(self.unexpected(Some(token), &[kind]));
}
// Slow path: buffer empty, fill it.
let current_kind = self.peek_kind(0)?;
match current_kind {
Some(k) if k == kind => self.consume(),
Some(_) => {
// The kind we just peeked guarantees a token is buffered, so `lookahead(0)`
// must yield `Some`; if the lexer somehow disagrees we surface an EOF error
// rather than panicking.
match self.lookahead(0)? {
Some(token) => Err(self.unexpected(Some(token), &[kind])),
None => Err(self.unexpected(None, &[kind])),
}
}
None => Err(self.unexpected(None, &[kind])),
}
}
/// Consumes and returns the span of the next significant token.
///
/// This is a convenience method equivalent to `consume()?.span_for(file_id())`.
///
/// # Errors
///
/// Returns a [`ParseError`] if EOF is reached or a lexer error occurs.
#[inline]
pub fn consume_span(&mut self) -> Result<Span, ParseError> {
let file_id = self.file_id();
self.consume().map(|t| t.span_for(file_id))
}
/// Consumes the next token only if it matches the expected kind, returning its span.
///
/// This is a convenience method equivalent to `eat(kind)?.span_for(file_id())`.
///
/// # Errors
///
/// Returns a [`ParseError`] if the next token's kind does not match `kind`, or if EOF is reached.
#[inline]
pub fn eat_span(&mut self, kind: TokenKind) -> Result<Span, ParseError> {
let file_id = self.file_id();
self.eat(kind).map(|t| t.span_for(file_id))
}
/// Advances the stream to the next token in the input source code and returns it.
///
/// If the stream has already read the entire input source code, this method will return `None`.
///
/// # Returns
///
/// The next token in the input source code, or `None` if the lexer has reached the end of the input.
#[inline]
pub fn advance(&mut self) -> Option<Result<Token<'input>, SyntaxError>> {
match self.fill_buffer(1) {
Ok(Some(_)) => {
if let Some(token) = self.buffer.pop_front() {
// Compute end position from start + value length
self.position = Position::new(token.start.offset + token.value.len() as u32);
Some(Ok(token))
} else {
None
}
}
Ok(None) => None,
Err(error) => Some(Err(error)),
}
}
/// Checks if the next token matches the given kind without consuming it.
///
/// Returns `false` if at EOF.
///
/// # Errors
///
/// Returns a [`ParseError`] if the lexer fails to produce the next token.
#[inline]
pub fn is_at(&mut self, kind: TokenKind) -> Result<bool, ParseError> {
if let Some(token) = self.buffer.get(0) {
return Ok(token.kind == kind);
}
Ok(self.peek_kind(0)? == Some(kind))
}
/// Peeks at the nth (0-indexed) significant token ahead without consuming it.
///
/// Returns `Ok(None)` if EOF is reached before the nth token.
///
/// # Errors
///
/// Returns a [`ParseError`] if the lexer fails to produce a token while filling the lookahead buffer.
#[inline]
pub fn lookahead(&mut self, n: usize) -> Result<Option<Token<'input>>, ParseError> {
if n < self.buffer.len() {
return Ok(self.buffer.get(n));
}
match self.fill_buffer(n + 1) {
Ok(Some(_)) => Ok(self.buffer.get(n)),
Ok(None) => Ok(None),
Err(error) => Err(error.into()),
}
}
/// Peeks at the kind of the nth (0-indexed) significant token ahead.
///
/// More efficient than `lookahead(n)?.map(|t| t.kind)` as it avoids
/// copying the full token when only the kind is needed.
///
/// # Errors
///
/// Returns a [`ParseError`] if the lexer fails to produce a token while filling the lookahead buffer.
#[inline]
pub fn peek_kind(&mut self, n: usize) -> Result<Option<TokenKind>, ParseError> {
if n < self.buffer.len() {
return Ok(self.buffer.get(n).map(|t| t.kind));
}
match self.fill_buffer(n + 1) {
Ok(Some(_)) => Ok(self.buffer.get(n).map(|t| t.kind)),
Ok(None) => Ok(None),
Err(error) => Err(error.into()),
}
}
/// Creates a `ParseError` for an unexpected token or EOF.
#[inline]
#[must_use]
pub fn unexpected(&self, found: Option<Token<'_>>, expected: &[TokenKind]) -> ParseError {
let expected_kinds: Box<[TokenKind]> = expected.into();
if let Some(token) = found {
ParseError::UnexpectedToken(expected_kinds, token.kind, token.span_for(self.file_id()))
} else {
ParseError::UnexpectedEndOfFile(expected_kinds, self.file_id(), self.current_position())
}
}
/// Consumes the comments collected by the lexer and returns them.
#[inline]
pub fn get_trivia(&mut self) -> Sequence<'arena, Trivia<'arena>> {
let mut tokens = Vec::new_in(self.arena);
std::mem::swap(&mut self.trivia, &mut tokens);
let file_id = self.file_id();
Sequence::new(
tokens
.into_iter()
.filter_map(|token| {
let span = token.span_for(file_id);
let kind = match token.kind {
TokenKind::Whitespace => TriviaKind::WhiteSpace,
TokenKind::HashComment => TriviaKind::HashComment,
TokenKind::SingleLineComment => TriviaKind::SingleLineComment,
TokenKind::MultiLineComment => TriviaKind::MultiLineComment,
TokenKind::DocBlockComment => TriviaKind::DocBlockComment,
// Tokens collected into `self.trivia` are guaranteed by `fill_buffer_slow`
// to satisfy `kind.is_trivia()`; any non-trivia kind here is a parser bug
// and the safe response is to drop it rather than panic.
_ => return None,
};
Some(Trivia { kind, span, value: token.value })
})
.collect_in(self.arena),
)
}
/// Fills the token buffer until at least `n` tokens are available, unless the lexer returns EOF.
///
/// Trivia tokens are collected separately and are not stored in the main token buffer.
#[inline]
fn fill_buffer(&mut self, n: usize) -> Result<Option<usize>, SyntaxError> {
if self.buffer.len() >= n {
return Ok(Some(n));
}
self.fill_buffer_slow(n)
}
#[inline(never)]
fn fill_buffer_slow(&mut self, n: usize) -> Result<Option<usize>, SyntaxError> {
while self.buffer.len() < n {
match self.lexer.advance() {
Some(result) => {
let token = result?;
if token.kind.is_trivia() {
self.trivia.push(token);
continue;
}
self.buffer.push_back(token);
}
None => return Ok(None),
}
}
Ok(Some(n))
}
}
impl HasFileId for TokenStream<'_, '_> {
#[inline]
fn file_id(&self) -> FileId {
self.file_id
}
}