1use std::fmt;
2
3use crate::ascii::ascii_display;
4
5use crate::ast_names::{AstName, LexemeType};
6use crate::location::Location;
7
8#[derive(Debug, Clone, Copy, PartialEq)]
9pub enum Token<'source, 'ast> {
10 Eof,
11 Ident(AstName<'ast>),
12 Comment(&'source [u8]),
13 BlockComment(usize),
14 BrokenComment,
15 Reserved(ReservedWord),
16 InterpStringBegin(&'source [u8]),
17 InterpStringMid(&'source [u8]),
18 InterpStringEnd(&'source [u8]),
19 InterpStringSimple(&'source [u8]),
20 BrokenInterpDoubleBrace(&'source [u8]),
21 Number(&'source [u8]),
22 QuotedString {
23 value: &'source [u8],
24 quote_style: QuoteStyle,
25 },
26 RawString {
27 value: &'source [u8],
28 block_depth: usize,
29 },
30 BrokenString,
31 Error,
32 BrokenUnicode {
33 codepoint: u32,
34 },
35 Attribute(AstName<'ast>),
36 AttributeOpen,
37 Ampersand,
38 Bang,
39 Hash,
40 LeftBrace,
41 LeftBracket,
42 Plus,
43 PlusEqual,
44 Minus,
45 MinusEqual,
46 SkinnyArrow,
47 Star,
48 StarEqual,
49 Slash,
50 SlashEqual,
51 SlashSlash,
52 SlashSlashEqual,
53 Percent,
54 PercentEqual,
55 Question,
56 Caret,
57 CaretEqual,
58 Colon,
59 DoubleColon,
60 Comma,
61 Dot,
62 DotDot,
63 DotDotEqual,
64 Equal,
65 EqualEqual,
66 Ellipsis,
67 Greater,
68 GreaterEqual,
69 Less,
70 LessEqual,
71 LeftParen,
72 Pipe,
73 RightBrace,
74 RightBracket,
75 RightParen,
76 Semicolon,
77 TildeEqual,
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq)]
81pub enum TokenKind {
82 Eof,
83 Ident,
84 Comment,
85 BlockComment,
86 BrokenComment,
87 Reserved(ReservedWord),
88 InterpStringBegin,
89 InterpStringMid,
90 InterpStringEnd,
91 InterpStringSimple,
92 BrokenInterpDoubleBrace,
93 Number,
94 QuotedString,
95 RawString,
96 BrokenString,
97 Error,
98 BrokenUnicode,
99 Attribute,
100 AttributeOpen,
101 Ampersand,
102 Bang,
103 Hash,
104 LeftBrace,
105 LeftBracket,
106 Plus,
107 PlusEqual,
108 Minus,
109 MinusEqual,
110 SkinnyArrow,
111 Star,
112 StarEqual,
113 Slash,
114 SlashEqual,
115 SlashSlash,
116 SlashSlashEqual,
117 Percent,
118 PercentEqual,
119 Question,
120 Caret,
121 CaretEqual,
122 Colon,
123 DoubleColon,
124 Comma,
125 Dot,
126 DotDot,
127 DotDotEqual,
128 Equal,
129 EqualEqual,
130 Ellipsis,
131 Greater,
132 GreaterEqual,
133 Less,
134 LessEqual,
135 LeftParen,
136 Pipe,
137 RightBrace,
138 RightBracket,
139 RightParen,
140 Semicolon,
141 TildeEqual,
142}
143
144#[derive(Debug, Clone, Copy, PartialEq, Eq)]
145pub enum QuoteStyle {
146 Single,
147 Double,
148}
149
150impl QuoteStyle {
151 pub(super) const fn from_delimiter(delimiter: u8) -> Self {
152 match delimiter {
153 b'\'' => Self::Single,
154 b'"' => Self::Double,
155 _ => unreachable!(),
156 }
157 }
158}
159
160impl Token<'_, '_> {
161 #[inline(always)]
162 pub const fn kind(self) -> TokenKind {
163 match self {
164 Self::Eof => TokenKind::Eof,
165 Self::Ident(_) => TokenKind::Ident,
166 Self::Comment(_) => TokenKind::Comment,
167 Self::BlockComment(_) => TokenKind::BlockComment,
168 Self::BrokenComment => TokenKind::BrokenComment,
169 Self::Reserved(word) => TokenKind::Reserved(word),
170 Self::InterpStringBegin(_) => TokenKind::InterpStringBegin,
171 Self::InterpStringMid(_) => TokenKind::InterpStringMid,
172 Self::InterpStringEnd(_) => TokenKind::InterpStringEnd,
173 Self::InterpStringSimple(_) => TokenKind::InterpStringSimple,
174 Self::BrokenInterpDoubleBrace(_) => TokenKind::BrokenInterpDoubleBrace,
175 Self::Number(_) => TokenKind::Number,
176 Self::QuotedString { .. } => TokenKind::QuotedString,
177 Self::RawString { .. } => TokenKind::RawString,
178 Self::BrokenString => TokenKind::BrokenString,
179 Self::Error => TokenKind::Error,
180 Self::BrokenUnicode { .. } => TokenKind::BrokenUnicode,
181 Self::Attribute(_) => TokenKind::Attribute,
182 Self::AttributeOpen => TokenKind::AttributeOpen,
183 Self::Ampersand => TokenKind::Ampersand,
184 Self::Bang => TokenKind::Bang,
185 Self::Hash => TokenKind::Hash,
186 Self::LeftBrace => TokenKind::LeftBrace,
187 Self::LeftBracket => TokenKind::LeftBracket,
188 Self::Plus => TokenKind::Plus,
189 Self::PlusEqual => TokenKind::PlusEqual,
190 Self::Minus => TokenKind::Minus,
191 Self::MinusEqual => TokenKind::MinusEqual,
192 Self::SkinnyArrow => TokenKind::SkinnyArrow,
193 Self::Star => TokenKind::Star,
194 Self::StarEqual => TokenKind::StarEqual,
195 Self::Slash => TokenKind::Slash,
196 Self::SlashEqual => TokenKind::SlashEqual,
197 Self::SlashSlash => TokenKind::SlashSlash,
198 Self::SlashSlashEqual => TokenKind::SlashSlashEqual,
199 Self::Percent => TokenKind::Percent,
200 Self::PercentEqual => TokenKind::PercentEqual,
201 Self::Question => TokenKind::Question,
202 Self::Caret => TokenKind::Caret,
203 Self::CaretEqual => TokenKind::CaretEqual,
204 Self::Colon => TokenKind::Colon,
205 Self::DoubleColon => TokenKind::DoubleColon,
206 Self::Comma => TokenKind::Comma,
207 Self::Dot => TokenKind::Dot,
208 Self::DotDot => TokenKind::DotDot,
209 Self::DotDotEqual => TokenKind::DotDotEqual,
210 Self::Equal => TokenKind::Equal,
211 Self::EqualEqual => TokenKind::EqualEqual,
212 Self::Ellipsis => TokenKind::Ellipsis,
213 Self::Greater => TokenKind::Greater,
214 Self::GreaterEqual => TokenKind::GreaterEqual,
215 Self::Less => TokenKind::Less,
216 Self::LessEqual => TokenKind::LessEqual,
217 Self::LeftParen => TokenKind::LeftParen,
218 Self::Pipe => TokenKind::Pipe,
219 Self::RightBrace => TokenKind::RightBrace,
220 Self::RightBracket => TokenKind::RightBracket,
221 Self::RightParen => TokenKind::RightParen,
222 Self::Semicolon => TokenKind::Semicolon,
223 Self::TildeEqual => TokenKind::TildeEqual,
224 }
225 }
226
227 pub fn name(&self) -> &'static str {
228 match self {
229 Self::Eof => "end of input",
230 Self::Ident(_) => "identifier",
231 Self::Comment(_) => "comment",
232 Self::BlockComment(_) => "block comment",
233 Self::BrokenComment => "unfinished comment",
234 Self::Reserved(word) => word.as_str(),
235 Self::InterpStringBegin(_)
236 | Self::InterpStringMid(_)
237 | Self::InterpStringEnd(_)
238 | Self::InterpStringSimple(_) => "interpolated string",
239 Self::BrokenInterpDoubleBrace(_) => "broken interpolated string",
240 Self::Number(_) => "number",
241 Self::QuotedString { .. } => "string",
242 Self::RawString { .. } => "string",
243 Self::BrokenString => "unterminated string",
244 Self::Error => "invalid token",
245 Self::BrokenUnicode { .. } => "invalid unicode",
246 Self::Attribute(_) => "attribute",
247 Self::AttributeOpen => "@[",
248 Self::Ampersand => "&",
249 Self::Bang => "!",
250 Self::Hash => "#",
251 Self::LeftBrace => "{",
252 Self::LeftBracket => "[",
253 Self::Plus => "+",
254 Self::PlusEqual => "+=",
255 Self::Minus => "-",
256 Self::MinusEqual => "-=",
257 Self::SkinnyArrow => "->",
258 Self::Star => "*",
259 Self::StarEqual => "*=",
260 Self::Slash => "/",
261 Self::SlashEqual => "/=",
262 Self::SlashSlash => "//",
263 Self::SlashSlashEqual => "//=",
264 Self::Percent => "%",
265 Self::PercentEqual => "%=",
266 Self::Question => "?",
267 Self::Caret => "^",
268 Self::CaretEqual => "^=",
269 Self::Colon => ":",
270 Self::DoubleColon => "::",
271 Self::Comma => ",",
272 Self::Dot => ".",
273 Self::DotDot => "..",
274 Self::DotDotEqual => "..=",
275 Self::Equal => "=",
276 Self::EqualEqual => "==",
277 Self::Ellipsis => "...",
278 Self::Greater => ">",
279 Self::GreaterEqual => ">=",
280 Self::Less => "<",
281 Self::LessEqual => "<=",
282 Self::LeftParen => "(",
283 Self::Pipe => "|",
284 Self::RightBrace => "}",
285 Self::RightBracket => "]",
286 Self::RightParen => ")",
287 Self::Semicolon => ";",
288 Self::TildeEqual => "~=",
289 }
290 }
291}
292
293impl fmt::Display for Token<'_, '_> {
294 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
295 match self {
296 Token::Eof => formatter.write_str("<eof>"),
297 Token::Attribute(name) => write!(formatter, "'@{}'", ascii_display(name.bytes())),
298 Token::AttributeOpen => formatter.write_str("'@['"),
299 Token::Ident(value) => write!(formatter, "'{}'", ascii_display(value.bytes())),
300 Token::Number(value) => write!(formatter, "'{}'", ascii_display(value)),
301 Token::QuotedString { value, .. } | Token::RawString { value, .. } => {
302 write!(formatter, "\"{}\"", ascii_display(value))
303 }
304 Token::InterpStringBegin(_) => formatter.write_str("`{"),
305 Token::InterpStringMid(_) => formatter.write_str("}{"),
306 Token::InterpStringEnd(_) => formatter.write_str("}`"),
307 Token::InterpStringSimple(_) => formatter.write_str("`"),
308 Token::Reserved(word) => write!(formatter, "'{}'", word.as_str()),
309 Token::BrokenComment => formatter.write_str("unfinished comment"),
310 token => write!(formatter, "'{}'", token.name()),
311 }
312 }
313}
314
315#[derive(Debug, Clone, Copy, PartialEq, Eq)]
316pub enum ReservedWord {
317 And,
318 Break,
319 Do,
320 Else,
321 Elseif,
322 End,
323 False,
324 For,
325 Function,
326 If,
327 In,
328 Local,
329 Nil,
330 Not,
331 Or,
332 Repeat,
333 Return,
334 Then,
335 True,
336 Until,
337 While,
338}
339
340impl ReservedWord {
341 pub const fn as_str(self) -> &'static str {
342 match self {
343 Self::And => "and",
344 Self::Break => "break",
345 Self::Do => "do",
346 Self::Else => "else",
347 Self::Elseif => "elseif",
348 Self::End => "end",
349 Self::False => "false",
350 Self::For => "for",
351 Self::Function => "function",
352 Self::If => "if",
353 Self::In => "in",
354 Self::Local => "local",
355 Self::Nil => "nil",
356 Self::Not => "not",
357 Self::Or => "or",
358 Self::Repeat => "repeat",
359 Self::Return => "return",
360 Self::Then => "then",
361 Self::True => "true",
362 Self::Until => "until",
363 Self::While => "while",
364 }
365 }
366
367 pub(super) const fn from_lexeme_type(kind: LexemeType) -> Option<Self> {
368 Some(match kind {
369 LexemeType::ReservedAnd => Self::And,
370 LexemeType::ReservedBreak => Self::Break,
371 LexemeType::ReservedDo => Self::Do,
372 LexemeType::ReservedElse => Self::Else,
373 LexemeType::ReservedElseif => Self::Elseif,
374 LexemeType::ReservedEnd => Self::End,
375 LexemeType::ReservedFalse => Self::False,
376 LexemeType::ReservedFor => Self::For,
377 LexemeType::ReservedFunction => Self::Function,
378 LexemeType::ReservedIf => Self::If,
379 LexemeType::ReservedIn => Self::In,
380 LexemeType::ReservedLocal => Self::Local,
381 LexemeType::ReservedNil => Self::Nil,
382 LexemeType::ReservedNot => Self::Not,
383 LexemeType::ReservedOr => Self::Or,
384 LexemeType::ReservedRepeat => Self::Repeat,
385 LexemeType::ReservedReturn => Self::Return,
386 LexemeType::ReservedThen => Self::Then,
387 LexemeType::ReservedTrue => Self::True,
388 LexemeType::ReservedUntil => Self::Until,
389 LexemeType::ReservedWhile => Self::While,
390 _ => return None,
391 })
392 }
393}
394
395#[derive(Clone, Copy)]
396pub struct LexedToken<'source, 'ast> {
397 pub token: Token<'source, 'ast>,
398 pub start: usize,
399 pub end: usize,
400 pub location: Location,
401}