luau_lexer/token/
mod.rs

1//! The [`Token`] struct.
2
3mod r#impl;
4
5use smol_str::SmolStr;
6
7use crate::prelude::{ParseError, Position};
8
9crate_reexport!(literal, keyword, symbol, operator, comment);
10
11/// A single token. Every [`lexable`](crate::lexer::Lexable) item becomes
12/// a token in [`Lexer::next_token()`](crate::lexer::Lexer::next_token).
13#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
14#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
15pub struct Token {
16    /// The starting position of this token
17    pub start: Position,
18
19    /// The trivia before the token.
20    pub leading_trivia: Vec<Trivia>,
21
22    /// The actual info of the token.
23    pub token_type: TokenType,
24
25    /// The trivia after the token.
26    pub trailing_trivia: Vec<Trivia>,
27
28    /// The ending position of this token.
29    pub end: Position,
30}
31
32/// Trivia that can be before and after a token.
33#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
34#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
35pub enum Trivia {
36    /// Spaces, be it whitespace, tabs, new lines, etc.
37    Spaces(SmolStr),
38
39    /// Comment, single or multi line.
40    Comment(Comment),
41}
42
43impl Token {
44    /// The end of file constant token.
45    pub const END_OF_FILE: Self = Self::empty(TokenType::EndOfFile);
46
47    /// Creates an empty token with the specified type. This is only used when
48    /// creating tokens that don't have actual positions.
49    #[inline]
50    pub const fn empty(token_type: TokenType) -> Self {
51        Self {
52            start: Position::MAX,
53            leading_trivia: Vec::new(),
54            token_type,
55            trailing_trivia: Vec::new(),
56            end: Position::MAX,
57        }
58    }
59}
60
61impl PartialEq<TokenType> for Token {
62    fn eq(&self, other: &TokenType) -> bool {
63        &self.token_type == other
64    }
65}
66
67/// All token types.
68#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
69#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
70pub enum TokenType {
71    /// An unknown type.
72    Error(ParseError),
73
74    /// A luau literal
75    Literal(Literal),
76
77    /// An identifier, like a variable name.
78    Identifier(SmolStr),
79
80    /// A comment
81    Comment(Comment),
82
83    /// A luau-reserved-keyword
84    Keyword(Keyword),
85
86    /// A word that can both be a keyword and an identifier.
87    PartialKeyword(PartialKeyword),
88
89    /// Symbols like `(` and `)`.
90    Symbol(Symbol),
91
92    /// Operators like `+` and `and`
93    Operator(Operator),
94
95    /// Compound operators like `+=` and `//=`
96    CompoundOperator(CompoundOperator),
97
98    /// The end of file token.
99    EndOfFile,
100}
101
102impl TokenType {
103    /// Turn this token type into a [`Token`] with the passed properties.
104    pub fn into_token(
105        self,
106        start: Position,
107        end: Position,
108        leading_trivia: Vec<Trivia>,
109        trailing_trivia: Vec<Trivia>,
110    ) -> Token {
111        Token {
112            start,
113            leading_trivia,
114            token_type: self,
115            trailing_trivia,
116            end,
117        }
118    }
119}
120
121impl TokenType {
122    /// Try converting this token type into a string.
123    pub fn try_as_string(&self) -> Option<String> {
124        match self {
125            TokenType::Literal(literal) => match literal {
126                Literal::Number(luau_number) => match luau_number {
127                    LuauNumber::Plain(smol_str)
128                    | LuauNumber::Binary(smol_str)
129                    | LuauNumber::Hex(smol_str) => Some(smol_str.to_string()),
130                },
131                Literal::String(luau_string) => match luau_string {
132                    LuauString::SingleQuotes(smol_str)
133                    | LuauString::DoubleQuotes(smol_str)
134                    | LuauString::Backticks(smol_str)
135                    | LuauString::MultiLine(smol_str) => Some(smol_str.to_string()),
136                },
137                Literal::Boolean(true) => Some("true".to_string()),
138                Literal::Boolean(false) => Some("false".to_string()),
139            },
140            TokenType::Identifier(smol_str) => Some(smol_str.to_string()),
141            TokenType::Comment(comment) => match comment {
142                Comment::MultiLine(smol_str) | Comment::SingleLine(smol_str) => {
143                    Some(smol_str.to_string())
144                }
145            },
146            TokenType::Keyword(keyword) => Some(keyword.to_string()),
147            TokenType::PartialKeyword(partial_keyword) => Some(partial_keyword.to_string()),
148            TokenType::Symbol(symbol) => Some(symbol.to_string()),
149            TokenType::Operator(operator) => Some(operator.to_string()),
150            TokenType::CompoundOperator(compound_operator) => Some(compound_operator.to_string()),
151            _ => None,
152        }
153    }
154}
155
156impl_from!(TokenType <= {
157    Error(ParseError),
158    Literal(Literal),
159    Keyword(Keyword),
160    PartialKeyword(PartialKeyword),
161    Symbol(Symbol),
162    Operator(Operator),
163    CompoundOperator(CompoundOperator),
164});
luau_lexer/token/mod.rs

luau_lexer/token/
mod.rs
