perl_parser_core/tokens/token_stream.rs
1//! Token stream adapter between `perl-lexer` output and the parser.
2//!
3//! Provides buffered lookahead, skips trivia tokens, and resets lexer mode at
4//! statement boundaries. This stream is optimized for parser consumption rather
5//! than full-fidelity token preservation.
6//!
7//! # Basic usage
8//!
9//! ```
10//! use perl_parser_core::tokens::token_stream::{TokenKind, TokenStream};
11//!
12//! let mut stream = TokenStream::new("my $x = 42;");
13//! assert!(matches!(stream.peek(), Ok(token) if token.kind == TokenKind::My));
14//!
15//! while let Ok(token) = stream.next() {
16//! if token.kind == TokenKind::Eof {
17//! break;
18//! }
19//! }
20//! ```
21//!
22//! # Pre-lexed token stream
23//!
24//! For incremental parsing, use [`TokenStream::from_vec`] to create a stream
25//! from pre-lexed tokens without re-lexing from source:
26//!
27//! ```
28//! use perl_parser_core::tokens::token_stream::{Token, TokenKind, TokenStream};
29//!
30//! let tokens = vec![
31//! Token::new(TokenKind::My, "my", 0, 2),
32//! Token::new(TokenKind::ScalarSigil, "$", 3, 4),
33//! Token::new(TokenKind::Identifier, "x", 4, 5),
34//! Token::new(TokenKind::Assign, "=", 6, 7),
35//! Token::new(TokenKind::Number, "1", 8, 9),
36//! Token::new(TokenKind::Semicolon, ";", 9, 10),
37//! Token::new(TokenKind::Eof, "", 10, 10),
38//! ];
39//! let mut stream = TokenStream::from_vec(tokens);
40//! assert!(matches!(stream.peek(), Ok(t) if t.kind == TokenKind::My));
41//! ```
42
43use crate::syntax::error::{ParseError, ParseResult};
44use perl_lexer::{LexerMode, PerlLexer, Token as LexerToken, TokenType as LexerTokenType};
45pub use perl_token::{Token, TokenKind};
46use std::collections::VecDeque;
47
48/// Backing source for the token stream — either a live lexer or pre-lexed tokens.
49enum TokenStreamInner<'a> {
50 /// Live lexer producing tokens on demand from source text.
51 ///
52 /// Boxed because `PerlLexer` is substantially larger than the `Buffered`
53 /// variant; without indirection the enum's size is dominated by this one
54 /// arm (clippy::large_enum_variant).
55 Lexer(Box<PerlLexer<'a>>),
56 /// Pre-lexed token buffer; used by [`TokenStream::from_vec`].
57 Buffered(VecDeque<Token>),
58}
59
60/// Token stream that wraps perl-lexer or a pre-lexed token buffer.
61///
62/// Provides three-token lookahead, transparent trivia skipping (in lexer mode),
63/// and statement-boundary state management used by the recursive-descent parser.
64pub struct TokenStream<'a> {
65 inner: TokenStreamInner<'a>,
66 buffered_eof_pos: usize,
67 peeked: Option<Token>,
68 peeked_second: Option<Token>,
69 peeked_third: Option<Token>,
70}
71
72impl<'a> TokenStream<'a> {
73 /// Create a new token stream from source code.
74 pub fn new(input: &'a str) -> Self {
75 TokenStream {
76 inner: TokenStreamInner::Lexer(Box::new(PerlLexer::new(input))),
77 buffered_eof_pos: input.len(),
78 peeked: None,
79 peeked_second: None,
80 peeked_third: None,
81 }
82 }
83
84 /// Create a token stream from a pre-lexed token list.
85 ///
86 /// This constructor skips lexing entirely and feeds tokens directly from the
87 /// provided `Vec`. It is intended for the incremental parsing pipeline where
88 /// tokens from a prior parse run can be reused for unchanged regions.
89 ///
90 /// # Behaviour differences from [`TokenStream::new`]
91 ///
92 /// - [`on_stmt_boundary`](Self::on_stmt_boundary): clears lookahead cache only;
93 /// no lexer mode reset (tokens are already classified).
94 /// - [`relex_as_term`](Self::relex_as_term): clears lookahead cache only;
95 /// no re-lexing (token kinds are fixed from the original lex pass).
96 /// - [`enter_format_mode`](Self::enter_format_mode): no-op.
97 ///
98 /// # Arguments
99 ///
100 /// * `tokens` — Pre-lexed tokens. An `Eof` token does **not** need to be
101 /// included; the stream synthesises one when the buffer is exhausted.
102 ///
103 /// # Examples
104 ///
105 /// ```rust
106 /// use perl_parser_core::tokens::token_stream::{Token, TokenKind, TokenStream};
107 ///
108 /// let tokens = vec![
109 /// Token::new(TokenKind::My, "my", 0, 2),
110 /// Token::new(TokenKind::Eof, "", 2, 2),
111 /// ];
112 /// let mut stream = TokenStream::from_vec(tokens);
113 /// assert!(matches!(stream.peek(), Ok(t) if t.kind == TokenKind::My));
114 /// ```
115 pub fn from_vec(tokens: Vec<Token>) -> Self {
116 let buffered_eof_pos = tokens
117 .last()
118 .map(|token| if token.kind == TokenKind::Eof { token.start } else { token.end })
119 .unwrap_or(0);
120
121 TokenStream {
122 inner: TokenStreamInner::Buffered(VecDeque::from(tokens)),
123 buffered_eof_pos,
124 peeked: None,
125 peeked_second: None,
126 peeked_third: None,
127 }
128 }
129
130 /// Convert a slice of raw [`LexerToken`]s to parser [`Token`]s, filtering out trivia.
131 ///
132 /// This is a convenience method for the incremental parsing pipeline where the
133 /// token cache stores raw lexer tokens (including whitespace and comments) and
134 /// needs to convert them to parser tokens before feeding to [`Self::from_vec`].
135 ///
136 /// Trivia token types (whitespace, newlines, comments, EOF) are discarded.
137 /// All other token types are converted using the same mapping as the live
138 /// [`TokenStream`] would apply.
139 ///
140 /// # Examples
141 ///
142 /// ```rust
143 /// use perl_parser_core::tokens::token_stream::{TokenKind, TokenStream};
144 /// use perl_lexer::{PerlLexer, TokenType};
145 ///
146 /// // Collect raw lexer tokens
147 /// let mut lexer = PerlLexer::new("my $x = 1;");
148 /// let mut raw = Vec::new();
149 /// while let Some(t) = lexer.next_token() {
150 /// if matches!(t.token_type, TokenType::EOF) { break; }
151 /// raw.push(t);
152 /// }
153 ///
154 /// // Convert to parser tokens and build a stream
155 /// let parser_tokens = TokenStream::lexer_tokens_to_parser_tokens(raw);
156 /// let mut stream = TokenStream::from_vec(parser_tokens);
157 /// assert!(matches!(stream.peek(), Ok(t) if t.kind == TokenKind::My));
158 /// ```
159 pub fn lexer_tokens_to_parser_tokens(tokens: Vec<LexerToken>) -> Vec<Token> {
160 tokens
161 .into_iter()
162 .filter(|t| {
163 !matches!(
164 t.token_type,
165 LexerTokenType::Whitespace | LexerTokenType::Newline | LexerTokenType::EOF
166 ) && !matches!(t.token_type, LexerTokenType::Comment(_))
167 })
168 .map(Self::convert_lexer_token)
169 .collect()
170 }
171
172 /// Peek at the next token without consuming it
173 pub fn peek(&mut self) -> ParseResult<&Token> {
174 if self.peeked.is_none() {
175 self.peeked = Some(self.next_token()?);
176 }
177 // Safe: we just ensured peeked is Some
178 self.peeked.as_ref().ok_or(ParseError::UnexpectedEof)
179 }
180
181 /// Consume and return the next token
182 #[allow(clippy::should_implement_trait)]
183 pub fn next(&mut self) -> ParseResult<Token> {
184 // If we have a peeked token, return it and shift the peek chain down
185
186 if let Some(token) = self.peeked.take() {
187 // Make EOF sticky - if we're returning EOF, put it back in the peek buffer
188 // so future peeks still see EOF instead of getting an error
189 if token.kind == TokenKind::Eof {
190 self.peeked = Some(token.clone());
191 } else {
192 self.peeked = self.peeked_second.take();
193 self.peeked_second = self.peeked_third.take();
194 }
195 Ok(token)
196 } else {
197 let token = self.next_token()?;
198 // Make EOF sticky for fresh tokens too
199 if token.kind == TokenKind::Eof {
200 self.peeked = Some(token.clone());
201 }
202 Ok(token)
203 }
204 }
205
206 /// Check if we're at the end of input
207 pub fn is_eof(&mut self) -> bool {
208 matches!(self.peek(), Ok(token) if token.kind == TokenKind::Eof)
209 }
210
211 /// Peek at the second token (two tokens ahead)
212 pub fn peek_second(&mut self) -> ParseResult<&Token> {
213 // First ensure we have a peeked token
214 self.peek()?;
215
216 // If we don't have a second peeked token, get it
217 if self.peeked_second.is_none() {
218 self.peeked_second = Some(self.next_token()?);
219 }
220
221 // Safe: we just ensured peeked_second is Some
222 self.peeked_second.as_ref().ok_or(ParseError::UnexpectedEof)
223 }
224
225 /// Peek at the third token (three tokens ahead)
226 pub fn peek_third(&mut self) -> ParseResult<&Token> {
227 // First ensure we have peeked and second peeked tokens
228 self.peek_second()?;
229
230 // If we don't have a third peeked token, get it
231 if self.peeked_third.is_none() {
232 self.peeked_third = Some(self.next_token()?);
233 }
234
235 // Safe: we just ensured peeked_third is Some
236 self.peeked_third.as_ref().ok_or(ParseError::UnexpectedEof)
237 }
238
239 /// Enter format body parsing mode in the lexer.
240 ///
241 /// No-op when operating in buffered (pre-lexed) mode — the tokens are
242 /// already fully classified.
243 pub fn enter_format_mode(&mut self) {
244 if let TokenStreamInner::Lexer(ref mut lexer) = self.inner {
245 lexer.enter_format_mode();
246 }
247 // Buffered mode: no-op — tokens are pre-classified.
248 }
249
250 /// Called at statement boundaries to reset lexer state and clear cached lookahead.
251 ///
252 /// In buffered mode only the lookahead cache is cleared; no lexer mode reset
253 /// is performed because the tokens are already fully classified.
254 pub fn on_stmt_boundary(&mut self) {
255 // Clear any cached lookahead tokens
256 self.peeked = None;
257 self.peeked_second = None;
258 self.peeked_third = None;
259
260 // Reset lexer to expect a term (start of new statement)
261 if let TokenStreamInner::Lexer(ref mut lexer) = self.inner {
262 lexer.set_mode(LexerMode::ExpectTerm);
263 }
264 // Buffered mode: no lexer mode reset needed — tokens are pre-classified.
265 }
266
267 /// Re-lex the current peeked token in `ExpectTerm` mode.
268 ///
269 /// This is needed for context-sensitive constructs like `split /regex/`
270 /// where the `/` was lexed as division (`Slash`) but should be a regex
271 /// delimiter. Rolls the lexer back to the peeked token's start position,
272 /// switches to `ExpectTerm` mode, and clears the peek cache so the next
273 /// `peek()` or `next()` re-lexes it as a regex.
274 ///
275 /// In buffered mode the peek cache is cleared but no re-lexing occurs —
276 /// token kinds are fixed from the original lex pass.
277 pub fn relex_as_term(&mut self) {
278 if let TokenStreamInner::Lexer(ref mut lexer) = self.inner {
279 if let Some(ref token) = self.peeked {
280 use perl_lexer::Checkpointable;
281 let pos = token.start;
282 // Build a checkpoint at the peeked token's position with ExpectTerm mode
283 let cp = perl_lexer::LexerCheckpoint::at_position(pos);
284 lexer.restore(&cp);
285 }
286 }
287 // Both modes: clear the peek cache.
288 self.peeked = None;
289 self.peeked_second = None;
290 self.peeked_third = None;
291 }
292
293 /// Pure peek cache invalidation - no mode changes
294 pub fn invalidate_peek(&mut self) {
295 self.peeked = None;
296 self.peeked_third = None;
297 self.peeked_second = None;
298 }
299
300 /// Convenience method for a one-shot fresh peek
301 pub fn peek_fresh_kind(&mut self) -> Option<TokenKind> {
302 self.invalidate_peek();
303 match self.peek() {
304 Ok(token) => Some(token.kind),
305 Err(_) => None,
306 }
307 }
308
309 /// Get the next token from the backing source.
310 fn next_token(&mut self) -> ParseResult<Token> {
311 match &mut self.inner {
312 TokenStreamInner::Lexer(lexer) => Self::next_token_from_lexer(lexer),
313 TokenStreamInner::Buffered(buf) => {
314 Self::next_token_from_buf(buf, &mut self.buffered_eof_pos)
315 }
316 }
317 }
318
319 /// Drain the next non-trivia token from the live lexer.
320 fn next_token_from_lexer(lexer: &mut PerlLexer<'_>) -> ParseResult<Token> {
321 // Skip whitespace and comments
322 loop {
323 let lexer_token = lexer.next_token().ok_or(ParseError::UnexpectedEof)?;
324
325 match &lexer_token.token_type {
326 LexerTokenType::Whitespace | LexerTokenType::Newline => continue,
327 LexerTokenType::Comment(_) => continue,
328 LexerTokenType::EOF => {
329 return Ok(Token {
330 kind: TokenKind::Eof,
331 text: String::new().into(),
332 start: lexer_token.start,
333 end: lexer_token.end,
334 });
335 }
336 _ => {
337 return Ok(Self::convert_lexer_token(lexer_token));
338 }
339 }
340 }
341 }
342
343 /// Return the next token from the pre-lexed buffer.
344 fn next_token_from_buf(
345 buf: &mut VecDeque<Token>,
346 buffered_eof_pos: &mut usize,
347 ) -> ParseResult<Token> {
348 match buf.pop_front() {
349 Some(token) => {
350 *buffered_eof_pos =
351 if token.kind == TokenKind::Eof { token.start } else { token.end };
352 Ok(token)
353 }
354 // Synthesise EOF at the most recently known source position.
355 None => Ok(Token::eof_at(*buffered_eof_pos)),
356 }
357 }
358
359 /// Convert a raw lexer token to the parser `Token` type.
360 ///
361 /// Extracted from `next_token_from_lexer` to keep the match arm readable.
362 fn convert_lexer_token(token: LexerToken) -> Token {
363 let kind = match &token.token_type {
364 // Keywords
365 LexerTokenType::Keyword(kw) => match kw.as_ref() {
366 "qw" => TokenKind::Identifier, // Keep as identifier but handle specially
367 keyword => TokenKind::from_keyword(keyword).unwrap_or(TokenKind::Identifier),
368 },
369
370 // Operators
371 LexerTokenType::Operator(op) => TokenKind::from_operator(op)
372 // Sigils may be surfaced as operator tokens in some contexts.
373 .or_else(|| TokenKind::from_sigil(op))
374 .unwrap_or(TokenKind::Unknown),
375
376 // Arrow tokens
377 LexerTokenType::Arrow => TokenKind::Arrow,
378 LexerTokenType::FatComma => TokenKind::FatArrow,
379
380 // Delimiters
381 LexerTokenType::LeftParen => TokenKind::LeftParen,
382 LexerTokenType::RightParen => TokenKind::RightParen,
383 LexerTokenType::LeftBrace => TokenKind::LeftBrace,
384 LexerTokenType::RightBrace => TokenKind::RightBrace,
385 LexerTokenType::LeftBracket => TokenKind::LeftBracket,
386 LexerTokenType::RightBracket => TokenKind::RightBracket,
387 LexerTokenType::Semicolon => TokenKind::Semicolon,
388 LexerTokenType::Comma => TokenKind::Comma,
389
390 // Division operator (important to handle before other tokens)
391 LexerTokenType::Division => TokenKind::Slash,
392
393 // Literals
394 LexerTokenType::Number(_) => TokenKind::Number,
395 LexerTokenType::StringLiteral | LexerTokenType::InterpolatedString(_) => {
396 TokenKind::String
397 }
398 LexerTokenType::RegexMatch | LexerTokenType::QuoteRegex => TokenKind::Regex,
399 LexerTokenType::Substitution => TokenKind::Substitution,
400 LexerTokenType::Transliteration => TokenKind::Transliteration,
401 LexerTokenType::QuoteSingle => TokenKind::QuoteSingle,
402 LexerTokenType::QuoteDouble => TokenKind::QuoteDouble,
403 LexerTokenType::QuoteWords => TokenKind::QuoteWords,
404 LexerTokenType::QuoteCommand => TokenKind::QuoteCommand,
405 LexerTokenType::HeredocStart => TokenKind::HeredocStart,
406 LexerTokenType::HeredocBody(_) => TokenKind::HeredocBody,
407 LexerTokenType::FormatBody(_) => TokenKind::FormatBody,
408 LexerTokenType::Version(_) => TokenKind::VString,
409 LexerTokenType::DataMarker(_) => TokenKind::DataMarker,
410 LexerTokenType::DataBody(_) => TokenKind::DataBody,
411 LexerTokenType::UnknownRest => TokenKind::UnknownRest,
412
413 // Identifiers
414 LexerTokenType::Identifier(text) => {
415 // The lexer emits bare sigil characters ('%', '&') as Identifier
416 // tokens in postfix-dereference contexts (e.g. `->%{key}`,
417 // `%{$ref}`). Those must map to sigil kinds, NOT operator kinds,
418 // so we check sigil priority first for the ambiguous cases.
419 // '*' is the exception: as a bare identifier it is multiplication.
420 match text.as_ref() {
421 "%" => TokenKind::HashSigil,
422 "&" => TokenKind::SubSigil,
423 _ => TokenKind::from_keyword(text)
424 .or_else(|| TokenKind::from_operator(text))
425 .or_else(|| TokenKind::from_sigil(text))
426 .unwrap_or(TokenKind::Identifier),
427 }
428 }
429
430 // Handle error tokens that might be valid syntax
431 LexerTokenType::Error(msg) => {
432 // Check if it's a specific error we want to handle specially
433 if msg.as_ref() == "Heredoc nesting too deep" {
434 TokenKind::HeredocDepthLimit
435 } else if msg.as_ref().starts_with("unclosed ") {
436 // Unclosed quote-like operator from the lexer (e.g. "unclosed qq delimiter '{'").
437 // Map to the corresponding quote token kind so the parser's quote-handler
438 // produces a proper "Unclosed delimiter" diagnostic rather than the generic
439 // "expected expression, found unknown token" error. q/qq/qw have
440 // unclosed-detection in their primary-expression arms. Substitution
441 // also has strict parser-side validation, as do transliteration
442 // operators, so route malformed `s///`, `tr///`, and `y///`
443 // tokens there instead of losing the lexer diagnostic as Unknown.
444 // Other operators (qr, qx, m) still fall through to Unknown until
445 // dedicated recovery is added.
446 let text = token.text.as_ref();
447 if text.starts_with("qq") {
448 TokenKind::QuoteDouble
449 } else if text.starts_with("qw") {
450 TokenKind::QuoteWords
451 } else if text
452 .strip_prefix('s')
453 .and_then(|rest| rest.chars().next())
454 .is_some_and(|ch| !ch.is_ascii_alphanumeric() && ch != '_')
455 {
456 TokenKind::Substitution
457 } else if text
458 .strip_prefix("tr")
459 .and_then(|rest| rest.chars().next())
460 .is_some_and(|ch| !ch.is_ascii_alphanumeric() && ch != '_')
461 || text
462 .strip_prefix('y')
463 .and_then(|rest| rest.chars().next())
464 .is_some_and(|ch| !ch.is_ascii_alphanumeric() && ch != '_')
465 {
466 TokenKind::Transliteration
467 } else if text
468 .strip_prefix('q')
469 .and_then(|rest| rest.chars().next())
470 .is_some_and(|ch| !ch.is_ascii_alphanumeric() && ch != '_')
471 {
472 TokenKind::QuoteSingle
473 } else {
474 TokenKind::Unknown
475 }
476 } else {
477 // Check if it's a brace that the lexer couldn't recognize
478 TokenKind::from_delimiter(token.text.as_ref()).unwrap_or(TokenKind::Unknown)
479 }
480 }
481
482 _ => TokenKind::Unknown,
483 };
484
485 Token { kind, text: token.text, start: token.start, end: token.end }
486 }
487}