luau_lexer/token/
mod.rs

1//! The [`Token`] struct.
2
3mod r#impl;
4
5use smol_str::SmolStr;
6
7use crate::prelude::{ParseError, Position};
8
9crate_reexport!(literal, keyword, symbol, operator, comment);
10
11/// A single token. Every [`lexable`](crate::lexer::Lexable) item becomes
12/// a token in [`Lexer::next_token()`](crate::lexer::Lexer::next_token).
13#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
14#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
15pub struct Token {
16    /// The starting position of this token
17    pub start: Position,
18
19    /// The trivia before the token.
20    pub leading_trivia: Vec<Trivia>,
21
22    /// The actual info of the token.
23    pub token_type: TokenType,
24
25    /// The trivia after the token.
26    pub trailing_trivia: Vec<Trivia>,
27
28    /// The ending position of this token.
29    pub end: Position,
30}
31
32/// Trivia that can be before and after a token.
33#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
34#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
35pub enum Trivia {
36    /// Spaces, be it whitespace, tabs, new lines, etc.
37    Spaces(SmolStr),
38
39    /// Comment, single or multi line.
40    Comment(Comment),
41}
42
43impl Token {
44    /// Creates an empty token with the specified type. This is only used when
45    /// creating tokens that don't have actual positions.
46    #[inline]
47    pub const fn empty(token_type: TokenType) -> Self {
48        Self {
49            start: Position::MAX,
50            leading_trivia: Vec::new(),
51            token_type,
52            trailing_trivia: Vec::new(),
53            end: Position::MAX,
54        }
55    }
56}
57
58impl PartialEq<TokenType> for Token {
59    fn eq(&self, other: &TokenType) -> bool {
60        &self.token_type == other
61    }
62}
63
64/// All token types.
65#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
66#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
67pub enum TokenType {
68    /// An unknown type.
69    Error(ParseError),
70
71    /// A luau literal
72    Literal(Literal),
73
74    /// An identifier, like a variable name.
75    Identifier(SmolStr),
76
77    /// A comment
78    Comment(Comment),
79
80    /// A luau-reserved-keyword
81    Keyword(Keyword),
82
83    /// A word that can both be a keyword and an identifier.
84    PartialKeyword(PartialKeyword),
85
86    /// Symbols like `(` and `)`.
87    Symbol(Symbol),
88
89    /// Operators like `+` and `and`
90    Operator(Operator),
91
92    /// Compound operators like `+=` and `//=`
93    CompoundOperator(CompoundOperator),
94
95    /// The end of file token.
96    EndOfFile,
97}
98
99impl TokenType {
100    /// Turn this token type into a [`Token`] with the passed properties.
101    pub fn into_token(
102        self,
103        start: Position,
104        end: Position,
105        leading_trivia: Vec<Trivia>,
106        trailing_trivia: Vec<Trivia>,
107    ) -> Token {
108        Token {
109            start,
110            leading_trivia,
111            token_type: self,
112            trailing_trivia,
113            end,
114        }
115    }
116}
117
118impl TokenType {
119    /// Try converting this token type into a string.
120    pub fn try_as_string(&self) -> Option<String> {
121        match self {
122            TokenType::Literal(literal) => match literal {
123                Literal::Number(luau_number) => match luau_number {
124                    LuauNumber::Plain(smol_str)
125                    | LuauNumber::Binary(smol_str)
126                    | LuauNumber::Hex(smol_str) => Some(smol_str.to_string()),
127                },
128                Literal::String(luau_string) => match luau_string {
129                    LuauString::SingleQuotes(smol_str)
130                    | LuauString::DoubleQuotes(smol_str)
131                    | LuauString::Backticks(smol_str)
132                    | LuauString::MultiLine(smol_str) => Some(smol_str.to_string()),
133                },
134                Literal::Boolean(true) => Some("true".to_string()),
135                Literal::Boolean(false) => Some("false".to_string()),
136            },
137            TokenType::Identifier(smol_str) => Some(smol_str.to_string()),
138            TokenType::Comment(comment) => match comment {
139                Comment::MultiLine(smol_str) | Comment::SingleLine(smol_str) => {
140                    Some(smol_str.to_string())
141                }
142            },
143            TokenType::Keyword(keyword) => Some(keyword.to_string()),
144            TokenType::PartialKeyword(partial_keyword) => Some(partial_keyword.to_string()),
145            TokenType::Symbol(symbol) => Some(symbol.to_string()),
146            TokenType::Operator(operator) => Some(operator.to_string()),
147            TokenType::CompoundOperator(compound_operator) => Some(compound_operator.to_string()),
148            _ => None,
149        }
150    }
151}
152
153impl_from!(TokenType <= {
154    Error(ParseError),
155    Literal(Literal),
156    Keyword(Keyword),
157    PartialKeyword(PartialKeyword),
158    Symbol(Symbol),
159    Operator(Operator),
160    CompoundOperator(CompoundOperator),
161});
luau_lexer/token/mod.rs

luau_lexer/token/
mod.rs
