harn_parser/parser/
state.rs

1use crate::ast::*;
2use crate::operators::OperatorGrammar;
3use harn_lexer::{Span, Token, TokenKind};
4
5use super::contextual::{contextual_statement_at, introduces_construct};
6use super::error::ParserError;
7
8/// Deepest syntax nesting the parser descends into before refusing the source
9/// with a "nesting depth exceeded" error.
10pub const MAX_NESTING_DEPTH: usize = 64;
11
12/// Recursive descent parser for Harn.
13pub struct Parser {
14    pub(super) tokens: Vec<Token>,
15    pub(super) pos: usize,
16    pub(super) errors: Vec<ParserError>,
17    pub(super) grammar: OperatorGrammar,
18    nesting_depth: usize,
19    #[cfg(test)]
20    pub(super) stack_extent: (usize, usize),
21    #[cfg(test)]
22    pub(super) deepest_nesting: usize,
23}
24
25impl Parser {
26    pub(super) fn at_module_scope(&self) -> bool {
27        self.nesting_depth == 0
28    }
29
30    pub fn new(tokens: Vec<Token>) -> Self {
31        Self {
32            tokens,
33            pos: 0,
34            errors: Vec::new(),
35            grammar: OperatorGrammar::CURRENT,
36            nesting_depth: 0,
37            #[cfg(test)]
38            stack_extent: (usize::MAX, 0),
39            #[cfg(test)]
40            deepest_nesting: 0,
41        }
42    }
43
44    /// Parse with a different operator grammar. Only the syntax migration
45    /// uses [`OperatorGrammar::LEGACY`]; everything else parses the language.
46    #[must_use]
47    pub fn with_operator_grammar(mut self, grammar: OperatorGrammar) -> Self {
48        self.grammar = grammar;
49        self
50    }
51
52    pub(super) fn check_token_nesting_limit(&self) -> Result<(), ParserError> {
53        let mut depth = 0usize;
54        for token in &self.tokens {
55            match token.kind {
56                TokenKind::LBrace | TokenKind::LBracket | TokenKind::LParen => {
57                    depth += 1;
58                    if depth > MAX_NESTING_DEPTH {
59                        return Err(ParserError::Unexpected {
60                            got: "source nesting depth exceeded".to_string(),
61                            expected: format!(
62                                "parser nesting depth within {MAX_NESTING_DEPTH} levels"
63                            ),
64                            span: token.span,
65                        });
66                    }
67                }
68                TokenKind::RBrace | TokenKind::RBracket | TokenKind::RParen => {
69                    depth = depth.saturating_sub(1);
70                }
71                _ => {}
72            }
73        }
74        Ok(())
75    }
76
77    pub(super) fn current_span(&self) -> Span {
78        self.tokens
79            .get(self.pos)
80            .map(|t| t.span)
81            .unwrap_or(Span::dummy())
82    }
83
84    pub(super) fn current_kind(&self) -> Option<&TokenKind> {
85        self.tokens.get(self.pos).map(|t| &t.kind)
86    }
87
88    pub(super) fn prev_span(&self) -> Span {
89        if self.pos > 0 {
90            self.tokens[self.pos - 1].span
91        } else {
92            Span::dummy()
93        }
94    }
95
96    /// Span of the most recently consumed *non-newline* token. Useful when
97    /// computing a node's end span after the parser has already consumed
98    /// trailing newlines (e.g. while looking ahead for an optional `else` /
99    /// `catch` / `finally` clause). Using `prev_span()` in that position
100    /// would report a newline token whose `end_line` is past the visual end
101    /// of the node, which downstream tools (notably the formatter) interpret
102    /// as belonging to the node.
103    pub(super) fn last_non_newline_span(&self) -> Span {
104        let mut i = self.pos;
105        while i > 0 {
106            i -= 1;
107            if self.tokens[i].kind != TokenKind::Newline {
108                return self.tokens[i].span;
109            }
110        }
111        Span::dummy()
112    }
113
114    /// Parse a complete .harn file. Reports multiple errors via recovery.
115    pub fn parse(&mut self) -> Result<Vec<SNode>, ParserError> {
116        crate::analysis::record_parse_run();
117        self.check_token_nesting_limit()?;
118
119        let mut nodes = Vec::new();
120        self.skip_newlines();
121
122        while !self.is_at_end() {
123            // Recovery may leave us pointing at a stray `}` at top level; skip it.
124            if self.check(&TokenKind::RBrace) {
125                self.advance();
126                self.skip_newlines();
127                continue;
128            }
129
130            let result = if self.check(&TokenKind::Import) {
131                self.parse_import()
132            } else if self.check(&TokenKind::At) {
133                self.parse_attributed_decl()
134            } else if self.check(&TokenKind::Pipeline) {
135                self.parse_pipeline()
136            } else if self.check_contextual_construct("eval_pack") {
137                self.parse_eval_pack_decl(false)
138            } else {
139                self.parse_statement()
140            };
141
142            match result {
143                Ok(node) => {
144                    let end_line = node.span.end_line;
145                    nodes.push(node);
146                    let consumed_sep = self.consume_statement_separator();
147                    if !consumed_sep && !self.is_at_end() {
148                        self.require_statement_separator(end_line, "top-level item")?;
149                    }
150                }
151                Err(err) => {
152                    self.errors.push(err);
153                    self.synchronize();
154                }
155            }
156        }
157
158        if let Some(first) = self.errors.first() {
159            return Err(first.clone());
160        }
161        Ok(nodes)
162    }
163
164    /// Return all accumulated parser errors (after `parse()` returns).
165    pub fn all_errors(&self) -> &[ParserError] {
166        &self.errors
167    }
168
169    /// The contextual statement keyword at the current position, if the
170    /// current identifier introduces one.
171    pub(super) fn current_contextual_statement(&self) -> Option<&'static str> {
172        contextual_statement_at(&self.tokens, self.pos)
173    }
174
175    /// Check if the current token is one that starts a statement.
176    pub(super) fn is_statement_start(&self) -> bool {
177        if matches!(self.current_kind(), Some(TokenKind::Identifier(_))) {
178            return self.current_contextual_statement().is_some();
179        }
180        matches!(
181            self.current_kind(),
182            Some(
183                TokenKind::Let
184                    | TokenKind::Const
185                    | TokenKind::Var
186                    | TokenKind::If
187                    | TokenKind::For
188                    | TokenKind::While
189                    | TokenKind::Match
190                    | TokenKind::Retry
191                    | TokenKind::Return
192                    | TokenKind::Throw
193                    | TokenKind::Fn
194                    | TokenKind::Pub
195                    | TokenKind::Try
196                    | TokenKind::Pipeline
197                    | TokenKind::Import
198                    | TokenKind::Parallel
199                    | TokenKind::Enum
200                    | TokenKind::Struct
201                    | TokenKind::Interface
202                    | TokenKind::Guard
203                    | TokenKind::Yield
204                    | TokenKind::Defer
205                    | TokenKind::Break
206                    | TokenKind::Continue
207                    | TokenKind::Impl
208            )
209        )
210    }
211
212    /// Advance past tokens until we reach a likely statement boundary.
213    pub(super) fn synchronize(&mut self) {
214        while !self.is_at_end() {
215            if self.check(&TokenKind::Semicolon) {
216                self.advance();
217                self.skip_newlines();
218                return;
219            }
220            if self.check(&TokenKind::Newline) {
221                self.advance();
222                if self.is_at_end() || self.is_statement_start() {
223                    return;
224                }
225                continue;
226            }
227            if self.check(&TokenKind::RBrace) {
228                return;
229            }
230            self.advance();
231        }
232    }
233
234    pub(super) fn is_at_end(&self) -> bool {
235        self.pos >= self.tokens.len()
236            || matches!(self.tokens.get(self.pos), Some(t) if t.kind == TokenKind::Eof)
237    }
238
239    pub(super) fn current(&self) -> Option<&Token> {
240        self.tokens.get(self.pos)
241    }
242
243    pub(super) fn peek_kind(&self) -> Option<&TokenKind> {
244        self.tokens.get(self.pos + 1).map(|t| &t.kind)
245    }
246
247    pub(super) fn peek_kind_at(&self, offset: usize) -> Option<&TokenKind> {
248        self.tokens.get(self.pos + offset).map(|t| &t.kind)
249    }
250
251    pub(super) fn check(&self, kind: &TokenKind) -> bool {
252        self.current()
253            .map(|t| std::mem::discriminant(&t.kind) == std::mem::discriminant(kind))
254            .unwrap_or(false)
255    }
256
257    /// Check for `kind`, skipping newlines first; used for binary operators
258    /// like `||` and `&&` that can span lines.
259    pub(super) fn check_skip_newlines(&mut self, kind: &TokenKind) -> bool {
260        let saved = self.pos;
261        self.skip_newlines();
262        if self.check(kind) {
263            true
264        } else {
265            self.pos = saved;
266            false
267        }
268    }
269
270    /// Check if current token is an identifier with the given name (without consuming it).
271    pub(super) fn check_identifier(&self, name: &str) -> bool {
272        matches!(self.current().map(|t| &t.kind), Some(TokenKind::Identifier(s)) if s == name)
273    }
274
275    /// `gen` is contextual so existing identifiers named `gen` keep working.
276    /// It starts a stream declaration only when followed by `fn`.
277    pub(super) fn check_contextual_gen_fn(&self) -> bool {
278        if !self.check_identifier("gen") {
279            return false;
280        }
281        matches!(
282            self.tokens.get(self.pos + 1).map(|t| &t.kind),
283            Some(TokenKind::Fn)
284        )
285    }
286
287    /// Whether the current identifier `name` introduces its contextual
288    /// construct rather than naming a value. See [`contextual_statement_at`].
289    pub(super) fn check_contextual_construct(&self, name: &str) -> bool {
290        introduces_construct(&self.tokens, self.pos, name)
291    }
292
293    pub(super) fn advance(&mut self) {
294        if self.pos < self.tokens.len() {
295            self.pos += 1;
296        }
297    }
298
299    pub(super) fn consume(
300        &mut self,
301        kind: &TokenKind,
302        expected: &str,
303    ) -> Result<Token, ParserError> {
304        self.skip_newlines();
305        let tok = self.current().ok_or_else(|| self.make_error(expected))?;
306        if std::mem::discriminant(&tok.kind) != std::mem::discriminant(kind) {
307            return Err(self.make_error(expected));
308        }
309        let tok = tok.clone();
310        self.advance();
311        Ok(tok)
312    }
313
314    pub(super) fn consume_identifier(&mut self, expected: &str) -> Result<String, ParserError> {
315        self.skip_newlines();
316        let tok = self.current().ok_or_else(|| self.make_error(expected))?;
317        if let TokenKind::Identifier(name) = &tok.kind {
318            let name = name.clone();
319            self.advance();
320            Ok(name)
321        } else {
322            // Distinguish reserved-keyword misuse (e.g. `for tool in list`) from
323            // a general unexpected token so the error is actionable.
324            let kw_name = harn_lexer::KEYWORDS
325                .iter()
326                .find(|&&kw| kw == tok.kind.to_string());
327            if let Some(kw) = kw_name {
328                Err(ParserError::Unexpected {
329                    got: format!("'{kw}' (reserved keyword)"),
330                    expected: expected.into(),
331                    span: tok.span,
332                })
333            } else {
334                Err(self.make_error(expected))
335            }
336        }
337    }
338
339    pub(super) fn consume_contextual_keyword(
340        &mut self,
341        name: &str,
342        expected: &str,
343    ) -> Result<Token, ParserError> {
344        self.skip_newlines();
345        let tok = self.current().ok_or_else(|| self.make_error(expected))?;
346        if matches!(&tok.kind, TokenKind::Identifier(id) if id == name) {
347            let tok = tok.clone();
348            self.advance();
349            Ok(tok)
350        } else {
351            Err(self.make_error(expected))
352        }
353    }
354
355    /// Like `consume_identifier`, but also accepts keywords as identifiers.
356    /// Used for property access (e.g., `obj.type`) and dict keys where
357    /// keywords are valid member names.
358    pub(super) fn consume_identifier_or_keyword(
359        &mut self,
360        expected: &str,
361    ) -> Result<String, ParserError> {
362        self.skip_newlines();
363        let tok = self.current().ok_or_else(|| self.make_error(expected))?;
364        if let TokenKind::Identifier(name) = &tok.kind {
365            let name = name.clone();
366            self.advance();
367            return Ok(name);
368        }
369        let name = match &tok.kind {
370            TokenKind::Pipeline => "pipeline",
371            TokenKind::Extends => "extends",
372            TokenKind::Override => "override",
373            TokenKind::Let => "let",
374            TokenKind::Const => "const",
375            TokenKind::Var => "var",
376            TokenKind::If => "if",
377            TokenKind::Else => "else",
378            TokenKind::For => "for",
379            TokenKind::In => "in",
380            TokenKind::Match => "match",
381            TokenKind::Retry => "retry",
382            TokenKind::Parallel => "parallel",
383            TokenKind::Return => "return",
384            TokenKind::Import => "import",
385            TokenKind::True => "true",
386            TokenKind::False => "false",
387            TokenKind::Nil => "nil",
388            TokenKind::Try => "try",
389            TokenKind::Catch => "catch",
390            TokenKind::Throw => "throw",
391            TokenKind::Finally => "finally",
392            TokenKind::Fn => "fn",
393            TokenKind::Spawn => "spawn",
394            TokenKind::While => "while",
395            TokenKind::TypeKw => "type",
396            TokenKind::Enum => "enum",
397            TokenKind::Struct => "struct",
398            TokenKind::Interface => "interface",
399            TokenKind::Pub => "pub",
400            TokenKind::Guard => "guard",
401            TokenKind::Defer => "defer",
402            TokenKind::Yield => "yield",
403            TokenKind::Break => "break",
404            TokenKind::Continue => "continue",
405            TokenKind::Impl => "impl",
406            _ => return Err(self.make_error(expected)),
407        };
408        let name = name.to_string();
409        self.advance();
410        Ok(name)
411    }
412
413    pub(super) fn skip_newlines(&mut self) {
414        while self.pos < self.tokens.len() && self.tokens[self.pos].kind == TokenKind::Newline {
415            self.pos += 1;
416        }
417    }
418
419    /// Consume an optional semicolon statement separator followed by any
420    /// number of newlines, or one-or-more newlines on their own.
421    ///
422    /// This is intentionally narrower than `skip_newlines()`: semicolons are
423    /// only legal between already-parsed list items, not in arbitrary parse
424    /// positions.
425    pub(super) fn consume_statement_separator(&mut self) -> bool {
426        let mut consumed = false;
427        if self.check(&TokenKind::Semicolon) {
428            self.advance();
429            consumed = true;
430        }
431        let start = self.pos;
432        self.skip_newlines();
433        consumed || self.pos != start
434    }
435
436    pub(super) fn require_statement_separator(
437        &self,
438        prev_end_line: usize,
439        expected_item: &str,
440    ) -> Result<(), ParserError> {
441        let Some(tok) = self.current() else {
442            return Ok(());
443        };
444        if tok.kind == TokenKind::Eof || tok.span.line != prev_end_line {
445            return Ok(());
446        }
447        Err(ParserError::Unexpected {
448            got: tok.kind.to_string(),
449            expected: format!("{expected_item} separator (`;` or newline)"),
450            span: tok.span,
451        })
452    }
453
454    pub(super) fn make_error(&self, expected: &str) -> ParserError {
455        if let Some(tok) = self.tokens.get(self.pos) {
456            if tok.kind == TokenKind::Eof {
457                return ParserError::UnexpectedEof {
458                    expected: expected.into(),
459                    span: tok.span,
460                };
461            }
462            ParserError::Unexpected {
463                got: tok.kind.to_string(),
464                expected: expected.into(),
465                span: tok.span,
466            }
467        } else {
468            ParserError::UnexpectedEof {
469                expected: expected.into(),
470                span: self.prev_span(),
471            }
472        }
473    }
474
475    pub(super) fn error(&self, expected: &str) -> ParserError {
476        self.make_error(expected)
477    }
478
479    pub(super) fn with_nesting<T>(
480        &mut self,
481        context: &'static str,
482        f: impl FnOnce(&mut Self) -> Result<T, ParserError>,
483    ) -> Result<T, ParserError> {
484        #[cfg(test)]
485        self.record_stack_position();
486        if self.nesting_depth >= MAX_NESTING_DEPTH {
487            return Err(self.nesting_limit_error(context));
488        }
489        self.nesting_depth += 1;
490        let result = f(self);
491        self.nesting_depth = self.nesting_depth.saturating_sub(1);
492        result
493    }
494
495    #[cold]
496    #[inline(never)]
497    fn nesting_limit_error(&self, context: &'static str) -> ParserError {
498        ParserError::Unexpected {
499            got: format!("{context} nesting depth exceeded"),
500            expected: format!("parser nesting depth within {MAX_NESTING_DEPTH} levels"),
501            span: self.current_span(),
502        }
503    }
504
505    #[cfg(test)]
506    #[inline(never)]
507    pub(super) fn record_stack_position(&mut self) {
508        let marker = 0u8;
509        let address = std::ptr::from_ref(std::hint::black_box(&marker)) as usize;
510        self.stack_extent.0 = self.stack_extent.0.min(address);
511        self.stack_extent.1 = self.stack_extent.1.max(address);
512        self.deepest_nesting = self.deepest_nesting.max(self.nesting_depth);
513    }
514}
harn_parser/parser/state.rs

harn_parser/parser/
state.rs
