luau_lexer/token/
mod.rs

1//! The [`Token`] struct.
2
3mod r#impl;
4
5use smol_str::SmolStr;
6
7use crate::prelude::{ParseError, Position};
8
9crate_reexport!(literal, keyword, symbol, operator, comment);
10
11/// A single token. Every [`lexable`](crate::lexer::Lexable) item becomes
12/// a token in [`Lexer::next_token()`](crate::lexer::Lexer::next_token).
13#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
14#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
15pub struct Token {
16    /// The starting position of this token
17    pub start: Position,
18
19    /// The spaces before the token.
20    pub spaces_before: SmolStr,
21
22    /// The actual info of the token.
23    pub token_type: TokenType,
24
25    /// The spaces after the token.
26    pub spaces_after: SmolStr,
27
28    /// The ending position of this token.
29    pub end: Position,
30}
31
32impl Token {
33    /// The end of file constant token.
34    pub const END_OF_FILE: Self = Self::empty(TokenType::EndOfFile);
35
36    /// Creates an empty token with the specified type. This is only used when
37    /// creating tokens that don't have actual positions.
38    #[inline]
39    pub const fn empty(token_type: TokenType) -> Self {
40        Self {
41            start: Position::MAX,
42            spaces_before: SmolStr::new_inline(""),
43            token_type,
44            spaces_after: SmolStr::new_inline(""),
45            end: Position::MAX,
46        }
47    }
48}
49
50impl PartialEq<TokenType> for Token {
51    fn eq(&self, other: &TokenType) -> bool {
52        &self.token_type == other
53    }
54}
55
56/// All token types.
57#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
58#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
59pub enum TokenType {
60    /// An unknown type.
61    Error(ParseError),
62
63    /// A luau literal
64    Literal(Literal),
65
66    /// An identifier, like a variable name.
67    Identifier(SmolStr),
68
69    /// A comment
70    Comment(Comment),
71
72    /// A luau-reserved-keyword
73    Keyword(Keyword),
74
75    /// A word that can both be a keyword and an identifier.
76    PartialKeyword(PartialKeyword),
77
78    /// Symbols like `(` and `)`.
79    Symbol(Symbol),
80
81    /// Operators like `+` and `and`
82    Operator(Operator),
83
84    /// Compound operators like `+=` and `//=`
85    CompoundOperator(CompoundOperator),
86
87    /// The end of file token.
88    EndOfFile,
89}
90
91impl TokenType {
92    /// Turn this token type into a [`Token`] with the passed properties.
93    pub fn into_token(
94        self,
95        start: Position,
96        end: Position,
97        spaces_before: impl Into<SmolStr>,
98        spaces_after: impl Into<SmolStr>,
99    ) -> Token {
100        Token {
101            start,
102            spaces_before: spaces_before.into(),
103            token_type: self,
104            spaces_after: spaces_after.into(),
105            end,
106        }
107    }
108}
109
110impl TokenType {
111    /// Try converting this token type into a string.
112    pub fn try_as_string(&self) -> Option<String> {
113        match self {
114            TokenType::Literal(literal) => match literal {
115                Literal::Number(luau_number) => match luau_number {
116                    LuauNumber::Plain(smol_str)
117                    | LuauNumber::Binary(smol_str)
118                    | LuauNumber::Hex(smol_str) => Some(smol_str.to_string()),
119                },
120                Literal::String(luau_string) => match luau_string {
121                    LuauString::SingleQuotes(smol_str)
122                    | LuauString::DoubleQuotes(smol_str)
123                    | LuauString::Backticks(smol_str)
124                    | LuauString::MultiLine(smol_str) => Some(smol_str.to_string()),
125                },
126                Literal::Boolean(true) => Some("true".to_string()),
127                Literal::Boolean(false) => Some("false".to_string()),
128            },
129            TokenType::Identifier(smol_str) => Some(smol_str.to_string()),
130            TokenType::Comment(comment) => match comment {
131                Comment::MultiLine(smol_str) | Comment::SingleLine(smol_str) => {
132                    Some(smol_str.to_string())
133                }
134            },
135            TokenType::Keyword(keyword) => Some(keyword.to_string()),
136            TokenType::PartialKeyword(partial_keyword) => Some(partial_keyword.to_string()),
137            TokenType::Symbol(symbol) => Some(symbol.to_string()),
138            TokenType::Operator(operator) => Some(operator.to_string()),
139            TokenType::CompoundOperator(compound_operator) => Some(compound_operator.to_string()),
140            _ => None,
141        }
142    }
143}
144
145impl_from!(TokenType <= {
146    Error(ParseError),
147    Literal(Literal),
148    Keyword(Keyword),
149    PartialKeyword(PartialKeyword),
150    Symbol(Symbol),
151    Operator(Operator),
152    CompoundOperator(CompoundOperator),
153});
luau_lexer/token/mod.rs

luau_lexer/token/
mod.rs
