aura_lang/lexer/
mod.rs

1pub mod token;
2
3use crate::error::Diagnostic;
4use crate::span::{SourceId, Span};
5pub use token::{StrPart, Token, TokenKind};
6
7pub struct Lexer<'a> {
8    src: &'a str,
9    pos: usize,
10    source: SourceId,
11    expect_import_path: bool,
12}
13
14impl<'a> Lexer<'a> {
15    pub fn new(src: &'a str, source: SourceId) -> Self {
16        Lexer {
17            src,
18            pos: 0,
19            source,
20            expect_import_path: false,
21        }
22    }
23
24    /// Fail-fast on the first lexical error (SPEC §2.6).
25    pub fn tokenize(mut self) -> Result<Vec<Token<'a>>, Diagnostic> {
26        let mut raw: Vec<Token<'a>> = Vec::new();
27        loop {
28            self.skip_trivia(&mut raw);
29            let start = self.pos;
30            let Some(c) = self.peek() else {
31                raw.push(Token {
32                    kind: TokenKind::Eof,
33                    span: self.span(start),
34                });
35                break;
36            };
37            let kind = match c {
38                b'"' => self.lex_string()?,
39                b'0'..=b'9' => self.lex_number()?,
40                b'a'..=b'z' | b'A'..=b'Z' | b'_' => {
41                    if self.expect_import_path {
42                        self.lex_import_path()?
43                    } else {
44                        let k = self.lex_ident_or_keyword();
45                        // D16 block strings: `key: text` / `x = text` immediately followed
46                        // by a newline opens a multi-line string captured verbatim until a
47                        // lone `end` at the opener line's indentation. Contextual so that a
48                        // property/field literally named `text` keeps working.
49                        if matches!(k, TokenKind::Ident("text"))
50                            && matches!(
51                                raw.last().map(|t| &t.kind),
52                                Some(TokenKind::Colon | TokenKind::Assign)
53                            )
54                            && self.block_string_follows()
55                        {
56                            self.lex_block_string(start)?
57                        } else {
58                            k
59                        }
60                    }
61                }
62                _ => self.lex_operator()?,
63            };
64            // Contextual import-path mode applies to exactly the next token (SPEC §2.4).
65            self.expect_import_path = matches!(kind, TokenKind::Import);
66            raw.push(Token {
67                kind,
68                span: self.span(start),
69            });
70        }
71        Ok(normalize(raw))
72    }
73
74    fn span(&self, start: usize) -> Span {
75        Span::new(self.source, start, self.pos)
76    }
77
78    fn peek(&self) -> Option<u8> {
79        self.src.as_bytes().get(self.pos).copied()
80    }
81
82    fn peek_at(&self, off: usize) -> Option<u8> {
83        self.src.as_bytes().get(self.pos + off).copied()
84    }
85
86    fn bump(&mut self) {
87        // Advance by a whole UTF-8 char; the ASCII fast path advances by a byte.
88        let step = self.src[self.pos..]
89            .chars()
90            .next()
91            .map_or(1, |c| c.len_utf8());
92        self.pos += step;
93    }
94
95    /// Whitespace, `\r`, `#...` comments, collapsing `\n+` into a single Newline (SPEC §2.5, rule 1).
96    fn skip_trivia(&mut self, out: &mut Vec<Token<'a>>) {
97        loop {
98            match self.peek() {
99                Some(b' ' | b'\t' | b'\r') => self.pos += 1,
100                Some(b'#') => {
101                    while !matches!(self.peek(), None | Some(b'\n')) {
102                        self.bump();
103                    }
104                }
105                Some(b'\n') => {
106                    let start = self.pos;
107                    self.pos += 1;
108                    if !matches!(out.last().map(|t| &t.kind), Some(TokenKind::Newline)) {
109                        out.push(Token {
110                            kind: TokenKind::Newline,
111                            span: self.span(start),
112                        });
113                    }
114                }
115                _ => break,
116            }
117        }
118    }
119
120    /// Number DFA (SPEC §2.2): Int(i64) | Float(f64); `12.foo` — rollback, `12.` — E0101.
121    fn lex_number(&mut self) -> Result<TokenKind<'a>, Diagnostic> {
122        let start = self.pos;
123        while matches!(self.peek(), Some(b'0'..=b'9')) {
124            self.pos += 1;
125        }
126        if self.peek() == Some(b'.') {
127            match self.peek_at(1) {
128                Some(b'0'..=b'9') => {
129                    self.pos += 1;
130                    while matches!(self.peek(), Some(b'0'..=b'9')) {
131                        self.pos += 1;
132                    }
133                    let text = &self.src[start..self.pos];
134                    return Ok(TokenKind::Float(
135                        text.parse().expect("DFA guarantees valid float"),
136                    ));
137                }
138                Some(b'a'..=b'z' | b'A'..=b'Z' | b'_') => {} // rollback: Int, then Dot, Ident
139                _ => {
140                    return Err(Diagnostic::error(
141                        "E0101",
142                        "digit expected after decimal point",
143                        Span::new(self.source, start, self.pos + 1),
144                        "incomplete float literal",
145                    ))
146                }
147            }
148        }
149        let text = &self.src[start..self.pos];
150        text.parse::<i64>().map(TokenKind::Int).map_err(|_| {
151            Diagnostic::error(
152                "E0103",
153                "integer literal overflows i64",
154                self.span(start),
155                "does not fit in i64",
156            )
157        })
158    }
159
160    /// String DFA (SPEC §2.3): a plain slice without escapes/interp; InterpStr for `#{...}`.
161    fn lex_string(&mut self) -> Result<TokenKind<'a>, Diagnostic> {
162        let quote_start = self.pos;
163        self.pos += 1; // opening quote
164        let content_start = self.pos;
165        let mut parts: Vec<StrPart<'a>> = Vec::new();
166        let mut lit_start = self.pos;
167        loop {
168            match self.peek() {
169                None | Some(b'\n') => {
170                    return Err(Diagnostic::error(
171                        "E0102",
172                        "unterminated string literal",
173                        self.span(quote_start),
174                        "string is not closed before end of line",
175                    ))
176                }
177                Some(b'"') => {
178                    let kind = if parts.is_empty() {
179                        TokenKind::Str(&self.src[content_start..self.pos])
180                    } else {
181                        if lit_start < self.pos {
182                            parts.push(StrPart::Lit(&self.src[lit_start..self.pos]));
183                        }
184                        TokenKind::InterpStr(parts)
185                    };
186                    self.pos += 1;
187                    return Ok(kind);
188                }
189                Some(b'\\') => {
190                    self.pos += 1;
191                    if matches!(self.peek(), None | Some(b'\n')) {
192                        return Err(Diagnostic::error(
193                            "E0102",
194                            "unterminated string literal",
195                            self.span(quote_start),
196                            "escape at end of line",
197                        ));
198                    }
199                    self.bump();
200                }
201                Some(b'#') if self.peek_at(1) == Some(b'{') => {
202                    if lit_start < self.pos {
203                        parts.push(StrPart::Lit(&self.src[lit_start..self.pos]));
204                    }
205                    self.pos += 2;
206                    let expr_start = self.pos;
207                    let mut depth = 1u32;
208                    loop {
209                        match self.peek() {
210                            None | Some(b'\n') => {
211                                return Err(Diagnostic::error(
212                                    "E0106",
213                                    "unterminated string interpolation",
214                                    Span::new(self.source, expr_start - 2, self.pos),
215                                    "missing closing '}'",
216                                ))
217                            }
218                            Some(b'{') => depth += 1,
219                            Some(b'}') => {
220                                depth -= 1;
221                                if depth == 0 {
222                                    break;
223                                }
224                            }
225                            _ => {}
226                        }
227                        self.bump();
228                    }
229                    parts.push(StrPart::Interp(&self.src[expr_start..self.pos]));
230                    self.pos += 1; // closing '}'
231                    lit_start = self.pos;
232                }
233                _ => self.bump(),
234            }
235        }
236    }
237
238    /// Lookahead for a D16 block-string opener: after `text`, only spaces/tabs
239    /// then a newline may follow. Does not consume.
240    fn block_string_follows(&self) -> bool {
241        let mut i = self.pos;
242        while matches!(self.src.as_bytes().get(i), Some(b' ' | b'\t')) {
243            i += 1;
244        }
245        matches!(self.src.as_bytes().get(i), Some(b'\n'))
246    }
247
248    /// D16: capture a `text … end` block. `text_start` is the offset of `text`.
249    /// The closing `end` sits at the opener line's indentation; deeper `end`s are
250    /// text. Common leading indentation is stripped. Interpolation `#{…}` and `\`
251    /// escapes work exactly as in quoted strings (applied lazily during eval).
252    fn lex_block_string(&mut self, text_start: usize) -> Result<TokenKind<'a>, Diagnostic> {
253        let bytes = self.src.as_bytes();
254        // Indentation of the opener line = leading whitespace before its first token.
255        let line_start = self.src[..text_start].rfind('\n').map_or(0, |i| i + 1);
256        let key_indent = self.src[line_start..text_start]
257            .bytes()
258            .take_while(|b| matches!(b, b' ' | b'\t'))
259            .count();
260        // Consume the rest of the opener line up to and including the newline.
261        while matches!(self.peek(), Some(b' ' | b'\t')) {
262            self.pos += 1;
263        }
264        self.pos += 1; // the '\n' guaranteed by block_string_follows
265
266        // Collect content line ranges; stop at the terminator `end`.
267        let mut lines: Vec<(usize, usize)> = Vec::new(); // (content-start, content-end) sans \r
268        loop {
269            let ls = self.pos;
270            let mut le = ls;
271            while !matches!(bytes.get(le), None | Some(b'\n')) {
272                le += 1;
273            }
274            let indent = self.src[ls..le]
275                .bytes()
276                .take_while(|b| matches!(b, b' ' | b'\t'))
277                .count();
278            let content_end = if le > ls && bytes[le - 1] == b'\r' {
279                le - 1
280            } else {
281                le
282            };
283            let trimmed = &self.src[ls + indent..content_end];
284            if indent <= key_indent && trimmed == "end" {
285                // Terminator: leave self.pos at its newline so the main loop emits it.
286                self.pos = le;
287                return Ok(self.build_block_string(&lines));
288            }
289            if bytes.get(le).is_none() {
290                return Err(Diagnostic::error(
291                    "E0107",
292                    "unterminated block string",
293                    Span::new(self.source, text_start, le),
294                    "missing closing `end` at the opener's indentation",
295                ));
296            }
297            lines.push((ls, content_end));
298            self.pos = le + 1;
299        }
300    }
301
302    /// Assemble the captured content lines into a string token: strip the common
303    /// leading indentation, join by newline, split each line on `#{…}`.
304    fn build_block_string(&self, lines: &[(usize, usize)]) -> TokenKind<'a> {
305        let common = lines
306            .iter()
307            .filter(|(s, e)| !self.src[*s..*e].trim().is_empty())
308            .map(|(s, e)| {
309                self.src[*s..*e]
310                    .bytes()
311                    .take_while(|b| matches!(b, b' ' | b'\t'))
312                    .count()
313            })
314            .min()
315            .unwrap_or(0);
316        let mut parts: Vec<StrPart<'a>> = Vec::new();
317        for (idx, &(ls, le)) in lines.iter().enumerate() {
318            if idx > 0 {
319                // A real '\n' from the source (the byte right before this line).
320                parts.push(StrPart::Lit(&self.src[ls - 1..ls]));
321            }
322            // Strip at most this line's own leading spaces/tabs: `common` is a byte
323            // count, and a line that is Unicode-blank (e.g. a lone NBSP) but has no
324            // ASCII indentation must not have `common` slice into a multi-byte char.
325            let line_ws = self.src[ls..le]
326                .bytes()
327                .take_while(|b| matches!(b, b' ' | b'\t'))
328                .count();
329            let start = ls + common.min(line_ws);
330            self.segment_parts(start, le, &mut parts);
331        }
332        match parts.as_slice() {
333            [] => TokenKind::Str(&self.src[self.pos..self.pos]), // empty block -> ""
334            [StrPart::Lit(s)] => TokenKind::Str(s),
335            _ => TokenKind::InterpStr(parts),
336        }
337    }
338
339    /// Split `[start,end)` into `Lit`/`Interp` parts, mirroring quoted-string rules:
340    /// `\x` escapes are left raw (unescaped by eval); `#{…}` becomes an `Interp` part.
341    fn segment_parts(&self, start: usize, end: usize, parts: &mut Vec<StrPart<'a>>) {
342        let bytes = self.src.as_bytes();
343        let mut i = start;
344        let mut lit_start = start;
345        while i < end {
346            match bytes[i] {
347                b'\\' => i += 2, // skip the escape pair; eval unescapes later
348                b'#' if bytes.get(i + 1) == Some(&b'{') => {
349                    if lit_start < i {
350                        parts.push(StrPart::Lit(&self.src[lit_start..i]));
351                    }
352                    let expr_start = i + 2;
353                    let mut depth = 1u32;
354                    let mut j = expr_start;
355                    while j < end && depth > 0 {
356                        match bytes[j] {
357                            b'{' => depth += 1,
358                            b'}' => depth -= 1,
359                            _ => {}
360                        }
361                        if depth == 0 {
362                            break;
363                        }
364                        j += 1;
365                    }
366                    parts.push(StrPart::Interp(&self.src[expr_start..j]));
367                    i = j + 1; // past the closing '}'
368                    lit_start = i;
369                }
370                _ => i += 1,
371            }
372        }
373        if lit_start < end {
374            parts.push(StrPart::Lit(&self.src[lit_start..end]));
375        }
376    }
377
378    fn lex_ident_or_keyword(&mut self) -> TokenKind<'a> {
379        let start = self.pos;
380        while matches!(
381            self.peek(),
382            Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_')
383        ) {
384            self.pos += 1;
385        }
386        let text = &self.src[start..self.pos];
387        TokenKind::keyword(text).unwrap_or(TokenKind::Ident(text))
388    }
389
390    /// SPEC §2.4 (D8): `path("/" path)* "@" "v" digits ("." digits){0,2}`; missing version — E0104.
391    fn lex_import_path(&mut self) -> Result<TokenKind<'a>, Diagnostic> {
392        let start = self.pos;
393        while matches!(
394            self.peek(),
395            Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-' | b'/')
396        ) {
397            self.pos += 1;
398        }
399        let path = &self.src[start..self.pos];
400        // A single slash-less identifier could be a variable, but the import grammar
401        // only allows a path or a string — treat it as a path and require a version.
402        if self.peek() != Some(b'@') {
403            return Err(Diagnostic::error(
404                "E0104",
405                "registry import requires a version",
406                self.span(start),
407                format!("add a version, e.g. `{path}@v1`"),
408            ));
409        }
410        self.pos += 1;
411        let ver_start = self.pos;
412        let ok = self.peek() == Some(b'v') && {
413            self.pos += 1;
414            let mut groups = 0;
415            loop {
416                let digits_start = self.pos;
417                while matches!(self.peek(), Some(b'0'..=b'9')) {
418                    self.pos += 1;
419                }
420                if self.pos == digits_start {
421                    break false;
422                }
423                groups += 1;
424                if groups == 3 || self.peek() != Some(b'.') {
425                    break true;
426                }
427                self.pos += 1;
428            }
429        };
430        if !ok {
431            return Err(Diagnostic::error(
432                "E0104",
433                "malformed import version",
434                Span::new(self.source, ver_start.saturating_sub(1), self.pos),
435                "expected `@vX`, `@vX.Y` or `@vX.Y.Z`",
436            ));
437        }
438        Ok(TokenKind::ImportPath {
439            path,
440            version: &self.src[ver_start..self.pos],
441        })
442    }
443
444    fn lex_operator(&mut self) -> Result<TokenKind<'a>, Diagnostic> {
445        let start = self.pos;
446        let two = |k| Some((2usize, k));
447        let pair = (self.peek().unwrap(), self.peek_at(1));
448        let m = match pair {
449            (b'-', Some(b'>')) => two(TokenKind::Arrow),
450            (b'=', Some(b'=')) => two(TokenKind::EqEq),
451            (b'!', Some(b'=')) => two(TokenKind::NotEq),
452            (b'<', Some(b'=')) => two(TokenKind::LtEq),
453            (b'>', Some(b'=')) => two(TokenKind::GtEq),
454            (b'&', Some(b'&')) => two(TokenKind::And),
455            (b'|', Some(b'|')) => two(TokenKind::Or),
456            (c, _) => {
457                let k = match c {
458                    b'(' => TokenKind::LParen,
459                    b')' => TokenKind::RParen,
460                    b'[' => TokenKind::LBracket,
461                    b']' => TokenKind::RBracket,
462                    b':' => TokenKind::Colon,
463                    b',' => TokenKind::Comma,
464                    b'.' => TokenKind::Dot,
465                    b'=' => TokenKind::Assign,
466                    b'?' => TokenKind::Question,
467                    b'+' => TokenKind::Plus,
468                    b'-' => TokenKind::Minus,
469                    b'*' => TokenKind::Star,
470                    b'/' => TokenKind::Slash,
471                    b'%' => TokenKind::Percent,
472                    b'<' => TokenKind::Lt,
473                    b'>' => TokenKind::Gt,
474                    b'!' => TokenKind::Not,
475                    _ => {
476                        self.bump();
477                        return Err(Diagnostic::error(
478                            "E0105",
479                            "unexpected character",
480                            self.span(start),
481                            "not a valid Aura token",
482                        ));
483                    }
484                };
485                Some((1, k))
486            }
487        };
488        let (len, kind) = m.unwrap();
489        self.pos += len;
490        Ok(kind)
491    }
492}
493
494/// Newline-normalizing post-filter (SPEC §2.5, rules 2–7) with a bracket stack.
495fn normalize(raw: Vec<Token<'_>>) -> Vec<Token<'_>> {
496    let mut out: Vec<Token<'_>> = Vec::with_capacity(raw.len());
497    let mut stack: Vec<u8> = Vec::new();
498    for i in 0..raw.len() {
499        let t = &raw[i];
500        if let TokenKind::Newline = t.kind {
501            let prev = out.last().map(|t| &t.kind);
502            let next = raw.get(i + 1).map(|t| &t.kind);
503            let keep = match stack.last() {
504                // Inside (...) newlines are suppressed entirely (rule 4).
505                Some(b'(') => false,
506                // Inside [...] a newline is an element separator, except at the edges and after ',' (rules 5–6, D2).
507                Some(b'[') => {
508                    !matches!(prev, None | Some(TokenKind::LBracket | TokenKind::Comma))
509                        && !matches!(next, None | Some(TokenKind::RBracket))
510                }
511                // Outside brackets: rules 2, 3, 7. `:` deliberately does NOT suppress — a newline
512                // after `key:` is significant, it opens an object block (SPEC §3.2).
513                _ => {
514                    let after = !matches!(
515                        prev,
516                        None | Some(
517                            TokenKind::Assign
518                                | TokenKind::Arrow
519                                | TokenKind::Question
520                                | TokenKind::Comma
521                                | TokenKind::Dot
522                                | TokenKind::Plus
523                                | TokenKind::Minus
524                                | TokenKind::Star
525                                | TokenKind::Slash
526                                | TokenKind::Percent
527                                | TokenKind::EqEq
528                                | TokenKind::NotEq
529                                | TokenKind::Lt
530                                | TokenKind::Gt
531                                | TokenKind::LtEq
532                                | TokenKind::GtEq
533                                | TokenKind::And
534                                | TokenKind::Or
535                        )
536                    );
537                    let before = !matches!(next, None | Some(TokenKind::Dot | TokenKind::Eof));
538                    after && before
539                }
540            };
541            if keep {
542                out.push(t.clone());
543            }
544            continue;
545        }
546        match t.kind {
547            TokenKind::LParen => stack.push(b'('),
548            TokenKind::LBracket => stack.push(b'['),
549            TokenKind::RParen | TokenKind::RBracket => {
550                stack.pop();
551            }
552            _ => {}
553        }
554        out.push(t.clone());
555    }
556    out
557}
558
559#[cfg(test)]
560mod tests {
561    use super::*;
562
563    fn kinds(src: &str) -> Vec<TokenKind<'_>> {
564        Lexer::new(src, 0)
565            .tokenize()
566            .expect("lex ok")
567            .into_iter()
568            .map(|t| t.kind)
569            .collect()
570    }
571
572    fn err(src: &str) -> &'static str {
573        Lexer::new(src, 0)
574            .tokenize()
575            .expect_err("lex must fail")
576            .code
577    }
578
579    use TokenKind::*;
580
581    #[test]
582    fn list_newline_is_element_separator_d2() {
583        // [a \n -b] — unambiguously two elements (SPEC D2)
584        assert_eq!(
585            kinds("[a\n-b\n]"),
586            vec![
587                LBracket,
588                Ident("a"),
589                Newline,
590                Minus,
591                Ident("b"),
592                RBracket,
593                Eof
594            ]
595        );
596    }
597
598    #[test]
599    fn newline_suppressed_after_binary_op_at_top_level() {
600        assert_eq!(
601            kinds("x = 1 +\n2"),
602            vec![Ident("x"), Assign, Int(1), Plus, Int(2), Eof]
603        );
604    }
605
606    #[test]
607    fn newline_kept_after_colon_for_object_blocks() {
608        // A newline after `key:` is significant — it opens an object block (SPEC §3.2)
609        assert_eq!(
610            kinds("security:\nx: 1\nend"),
611            vec![
612                Ident("security"),
613                Colon,
614                Newline,
615                Ident("x"),
616                Colon,
617                Int(1),
618                Newline,
619                End,
620                Eof
621            ]
622        );
623    }
624
625    #[test]
626    fn newlines_inside_parens_fully_suppressed() {
627        assert_eq!(
628            kinds("f(\n1,\n2\n)"),
629            vec![Ident("f"), LParen, Int(1), Comma, Int(2), RParen, Eof]
630        );
631    }
632
633    #[test]
634    fn newline_suppressed_before_dot_chain() {
635        assert_eq!(
636            kinds("a\n.b()"),
637            vec![Ident("a"), Dot, Ident("b"), LParen, RParen, Eof]
638        );
639    }
640
641    #[test]
642    fn import_path_with_version_d8() {
643        assert_eq!(
644            kinds("import github/actions/rust-cache@v1.2 as rust"),
645            vec![
646                Import,
647                ImportPath {
648                    path: "github/actions/rust-cache",
649                    version: "v1.2"
650                },
651                As,
652                Ident("rust"),
653                Eof
654            ]
655        );
656    }
657
658    #[test]
659    fn import_path_without_version_is_e0104() {
660        assert_eq!(err("import github/actions/rust-cache as rust"), "E0104");
661        assert_eq!(err("import a/b@1.2 as x"), "E0104"); // missing the 'v' prefix
662    }
663
664    #[test]
665    fn file_import_needs_no_version() {
666        assert_eq!(
667            kinds(r#"import "templates/x.aura" as d"#),
668            vec![Import, Str("templates/x.aura"), As, Ident("d"), Eof]
669        );
670    }
671
672    #[test]
673    fn numbers_int_float_rollback_and_errors() {
674        assert_eq!(kinds("12.5"), vec![Float(12.5), Eof]);
675        assert_eq!(kinds("12.foo"), vec![Int(12), Dot, Ident("foo"), Eof]);
676        assert_eq!(err("x = 12."), "E0101");
677        assert_eq!(err("x = 99999999999999999999"), "E0103");
678    }
679
680    #[test]
681    fn string_interpolation_parts() {
682        assert_eq!(
683            kinds(r#""company/#{name}:#{ver}""#),
684            vec![
685                InterpStr(vec![
686                    StrPart::Lit("company/"),
687                    StrPart::Interp("name"),
688                    StrPart::Lit(":"),
689                    StrPart::Interp("ver"),
690                ]),
691                Eof
692            ]
693        );
694        assert_eq!(err("\"unterminated"), "E0102");
695        assert_eq!(err("\"#{a + b\""), "E0106");
696    }
697
698    #[test]
699    fn comments_and_blank_lines_collapse_to_one_newline() {
700        assert_eq!(
701            kinds("a = 1\n# comment\n\n\nb = 2"),
702            vec![
703                Ident("a"),
704                Assign,
705                Int(1),
706                Newline,
707                Ident("b"),
708                Assign,
709                Int(2),
710                Eof
711            ]
712        );
713    }
714
715    #[test]
716    fn keywords_v12() {
717        assert_eq!(kinds("new assert shadow"), vec![New, Assert, Shadow, Eof]);
718    }
719
720    #[test]
721    fn block_string_d16_basic() {
722        // Common indent stripped; joined by '\n'; single Str when no interpolation.
723        assert_eq!(
724            kinds("s: text\n  a\n  b\nend"),
725            vec![
726                Ident("s"),
727                Colon,
728                InterpStr(vec![
729                    StrPart::Lit("a"),
730                    StrPart::Lit("\n"),
731                    StrPart::Lit("b"),
732                ]),
733                Eof
734            ]
735        );
736    }
737
738    #[test]
739    fn block_string_d16_interpolation_and_inner_end() {
740        // `#{}` works; a deeper `end` is content, not the terminator.
741        assert_eq!(
742            kinds("x = text\n    echo #{v}\n    if y; end\nend"),
743            vec![
744                Ident("x"),
745                Assign,
746                InterpStr(vec![
747                    StrPart::Lit("echo "),
748                    StrPart::Interp("v"),
749                    StrPart::Lit("\n"),
750                    StrPart::Lit("if y; end"),
751                ]),
752                Eof
753            ]
754        );
755    }
756
757    #[test]
758    fn text_as_property_key_is_not_a_block() {
759        // `text` left of `:` is an ordinary key, not a block opener.
760        assert_eq!(
761            kinds("text: \"hi\""),
762            vec![Ident("text"), Colon, Str("hi"), Eof]
763        );
764    }
765
766    #[test]
767    fn block_string_unterminated_is_e0107() {
768        assert_eq!(err("s: text\n  oops\n"), "E0107");
769    }
770
771    #[test]
772    fn block_string_unicode_blank_line_does_not_panic() {
773        // Fuzz regression: a Unicode-only-whitespace line (NBSP) counts as blank
774        // for `common`, but indent stripping is byte-based and must not slice into
775        // the multi-byte char. Previously panicked on a non-char-boundary slice.
776        let src = "a: text\n\u{a0}\n \\\nend\n";
777        assert!(Lexer::new(src, 0).tokenize().is_ok());
778    }
779}
aura_lang/lexer/mod.rs

aura_lang/lexer/
mod.rs
