luna_core/frontend/token.rs
1//! Lexical tokens produced by the lexer.
2
3use crate::frontend::span::Span;
4use crate::version::LuaVersion;
5
6/// One lexical token produced by the lexer.
7#[derive(Clone, PartialEq, Debug)]
8pub enum Token {
9 // keywords
10 /// `and` keyword.
11 And,
12 /// `break` keyword.
13 Break,
14 /// `do` keyword.
15 Do,
16 /// `else` keyword.
17 Else,
18 /// `elseif` keyword.
19 Elseif,
20 /// `end` keyword.
21 End,
22 /// `false` keyword.
23 False,
24 /// `for` keyword.
25 For,
26 /// `function` keyword.
27 Function,
28 /// 5.5 `global` keyword.
29 Global,
30 /// `goto` keyword.
31 Goto,
32 /// `if` keyword.
33 If,
34 /// `in` keyword.
35 In,
36 /// `local` keyword.
37 Local,
38 /// `nil` keyword.
39 Nil,
40 /// `not` keyword.
41 Not,
42 /// `or` keyword.
43 Or,
44 /// `repeat` keyword.
45 Repeat,
46 /// `return` keyword.
47 Return,
48 /// `then` keyword.
49 Then,
50 /// `true` keyword.
51 True,
52 /// `until` keyword.
53 Until,
54 /// `while` keyword.
55 While,
56 // symbols
57 /// `+` symbol.
58 Plus,
59 /// `-` symbol.
60 Minus,
61 /// `*` symbol.
62 Star,
63 /// `/` symbol.
64 Slash,
65 /// `//` symbol (floor division).
66 DSlash,
67 /// `%` symbol.
68 Percent,
69 /// `^` symbol.
70 Caret,
71 /// `#` symbol.
72 Hash,
73 /// `&` symbol.
74 Amp,
75 /// `~` symbol (bitwise xor / unary bnot).
76 Tilde,
77 /// `|` symbol.
78 Pipe,
79 /// `<<` symbol.
80 Shl,
81 /// `>>` symbol.
82 Shr,
83 /// `==` symbol.
84 Eq,
85 /// `~=` symbol.
86 Ne,
87 /// `<=` symbol.
88 Le,
89 /// `>=` symbol.
90 Ge,
91 /// `<` symbol.
92 Lt,
93 /// `>` symbol.
94 Gt,
95 /// `=` symbol (assignment).
96 Assign,
97 /// `(` symbol.
98 LParen,
99 /// `)` symbol.
100 RParen,
101 /// `{` symbol.
102 LBrace,
103 /// `}` symbol.
104 RBrace,
105 /// `[` symbol.
106 LBracket,
107 /// `]` symbol.
108 RBracket,
109 /// `::` symbol (label delimiter).
110 DColon,
111 /// `;` symbol.
112 Semi,
113 /// `:` symbol.
114 Colon,
115 /// `,` symbol.
116 Comma,
117 /// `.` symbol.
118 Dot,
119 /// `..` symbol (concatenation).
120 Concat,
121 /// `...` symbol (vararg).
122 Ellipsis,
123 // literals
124 /// Integer literal.
125 Int(
126 /// Decoded 64-bit signed value.
127 i64,
128 ),
129 /// Floating-point literal.
130 Float(
131 /// Decoded IEEE-754 double value.
132 f64,
133 ),
134 /// String literal (raw bytes; Lua strings are 8-bit clean).
135 Str(
136 /// Decoded byte contents.
137 Vec<u8>,
138 ),
139 /// Identifier.
140 Name(
141 /// Source text of the identifier.
142 Box<str>,
143 ),
144 /// MacroLua `@` sigil. Lexed only when
145 /// `version.is_macro_lua()`; PUC 5.1-5.5 sources continue to
146 /// error `unexpected symbol near '@'`.
147 At,
148 /// MacroLua explicit quote-block opener `@{`. Lexed only when
149 /// `version.is_macro_lua()`; pairs with [`Token::MacroBraceClose`].
150 MacroBraceOpen,
151 /// MacroLua explicit quote-block closer `}@`. Lexed only when
152 /// `version.is_macro_lua()`; pairs with [`Token::MacroBraceOpen`].
153 MacroBraceClose,
154 /// Synthetic token produced by the macro expander pre-pass: a
155 /// captured token run (the body of a `@quote{...}` or `@{...}@`
156 /// block). The lexer never emits this. After the expander runs
157 /// it splices these back into the stream as raw token sequences
158 /// before the parser proper sees them.
159 MacroQuote(
160 /// Captured token run.
161 Box<[TokenInfo]>,
162 ),
163 /// End-of-file marker.
164 Eof,
165}
166
167impl Token {
168 /// The near-token shown in `... near <tok>` error messages, rendered the
169 /// way the dialect's `txtToken` / `luaX_token2str` does (see
170 /// `near_text`). Lossy view of `Token::near_bytes`.
171 pub fn describe(&self, src: &[u8], span: Span, version: LuaVersion) -> String {
172 String::from_utf8_lossy(&self.near_bytes(src, span, version)).into_owned()
173 }
174
175 /// Raw-byte form of [`Token::describe`]: names, strings and numerals
176 /// are shown through PUC's lex buffer, which holds the token's text as
177 /// scanned (a string keeps its delimiters and holds its *decoded*
178 /// contents), so non-UTF-8 bytes survive.
179 pub(crate) fn near_bytes(&self, src: &[u8], span: Span, version: LuaVersion) -> Vec<u8> {
180 match self {
181 Token::Eof => near_text(version, Near::Eof),
182 Token::Str(content) => {
183 let raw = span.slice(src);
184 let mut text = Vec::with_capacity(content.len() + 4);
185 if raw.first() == Some(&b'[') {
186 let level = raw[1..].iter().take_while(|&&c| c == b'=').count();
187 text.push(b'[');
188 text.extend(std::iter::repeat_n(b'=', level));
189 text.push(b'[');
190 text.extend_from_slice(content);
191 text.push(b']');
192 text.extend(std::iter::repeat_n(b'=', level));
193 text.push(b']');
194 } else {
195 text.push(raw[0]);
196 text.extend_from_slice(content);
197 text.push(raw[0]);
198 }
199 near_text(version, Near::Text(&text))
200 }
201 Token::Name(_) | Token::Int(_) | Token::Float(_) => {
202 near_text(version, Near::Text(span.slice(src)))
203 }
204 _ => near_text(version, Near::Fixed(span.slice(src))),
205 }
206 }
207}
208
209/// What an error's `near` part refers to, in PUC's terms.
210#[derive(Clone, Copy)]
211pub(crate) enum Near<'a> {
212 /// `TK_EOS`.
213 Eof,
214 /// A single-byte token the lexer returned as itself (PUC returns every
215 /// byte it does not recognise that way, so the parser reports it).
216 Char(u8),
217 /// A reserved word or a symbol.
218 Fixed(&'a [u8]),
219 /// The lex buffer (names, strings, numerals, and lexer errors inside
220 /// them).
221 Text(&'a [u8]),
222}
223
224/// Render a near-token per dialect. 5.1 wraps whatever `luaX_token2str`
225/// produced in quotes (so `'<eof>'`, `'char(1)'`); 5.2+ quote only concrete
226/// tokens and leave the `<eof>` pseudo-token bare. A non-printable byte is
227/// `char(N)` up to 5.2 (5.1 decides by `iscntrl`, so a high byte goes out
228/// raw) and `'<\N>'` from 5.3.
229pub(crate) fn near_text(version: LuaVersion, near: Near<'_>) -> Vec<u8> {
230 let mut out = Vec::new();
231 let quoted = |out: &mut Vec<u8>, s: &[u8]| {
232 out.push(b'\'');
233 out.extend_from_slice(s);
234 out.push(b'\'');
235 };
236 if version <= LuaVersion::Lua51 {
237 let inner: Vec<u8> = match near {
238 Near::Eof => b"<eof>".to_vec(),
239 Near::Char(c) if c.is_ascii_control() => format!("char({c})").into_bytes(),
240 Near::Char(c) => vec![c],
241 Near::Fixed(s) | Near::Text(s) => c_str(s).to_vec(),
242 };
243 quoted(&mut out, &inner);
244 return out;
245 }
246 match near {
247 Near::Eof => out.extend_from_slice(b"<eof>"),
248 Near::Char(c) if (0x20..0x7f).contains(&c) => quoted(&mut out, &[c]),
249 Near::Char(c) if version == LuaVersion::Lua52 => {
250 out.extend_from_slice(format!("char({c})").as_bytes())
251 }
252 Near::Char(c) => quoted(&mut out, format!("<\\{c}>").as_bytes()),
253 Near::Fixed(s) | Near::Text(s) => quoted(&mut out, c_str(s)),
254 }
255 out
256}
257
258/// PUC splices the lex buffer in with `%s`, so a NUL inside a string
259/// literal ends the near-token there.
260fn c_str(s: &[u8]) -> &[u8] {
261 &s[..s.iter().position(|&c| c == 0).unwrap_or(s.len())]
262}
263
264/// A token plus where it came from.
265#[derive(Clone, Debug, PartialEq)]
266pub struct TokenInfo {
267 /// The lexical token.
268 pub tok: Token,
269 /// Byte range in source.
270 pub span: Span,
271 /// 1-based source line where the token starts.
272 pub line: u32,
273}