luau_lexer/token/
mod.rs

1//! The [`Token`] struct.
2
3mod r#impl;
4
5use lsp_types::Position;
6use smol_str::SmolStr;
7
8use crate::prelude::{ParseError, PositionExt};
9
10crate_reexport!(literal, keyword, symbol, operator, comment);
11
12/// A single token. Every [`lexable`](crate::lexer::Lexable) item becomes
13/// a token in [`Lexer::next_token()`](crate::lexer::Lexer::next_token).
14#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
15#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
16pub struct Token {
17    /// The starting position of this token
18    pub start: Position,
19
20    /// The trivia before the token.
21    pub leading_trivia: Vec<Trivia>,
22
23    /// The actual info of the token.
24    pub token_type: TokenType,
25
26    /// The trivia after the token.
27    pub trailing_trivia: Vec<Trivia>,
28
29    /// The ending position of this token.
30    pub end: Position,
31}
32
33/// Trivia that can be before and after a token.
34#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
35#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
36pub enum Trivia {
37    /// Spaces, be it whitespace, tabs, new lines, etc.
38    Spaces(SmolStr),
39
40    /// Comment, single or multi line.
41    Comment(Comment),
42}
43
44impl Token {
45    /// Creates an empty token with the specified type. This is only used when
46    /// creating tokens that don't have actual positions.
47    #[inline]
48    pub const fn empty(token_type: TokenType) -> Self {
49        Self {
50            start: Position::MAX,
51            leading_trivia: Vec::new(),
52            token_type,
53            trailing_trivia: Vec::new(),
54            end: Position::MAX,
55        }
56    }
57}
58
59impl PartialEq<TokenType> for Token {
60    fn eq(&self, other: &TokenType) -> bool {
61        &self.token_type == other
62    }
63}
64
65/// All token types.
66#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
67#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
68pub enum TokenType {
69    /// An unknown type.
70    Error(ParseError),
71
72    /// A luau literal
73    Literal(Literal),
74
75    /// An identifier, like a variable name.
76    Identifier(SmolStr),
77
78    /// A comment
79    Comment(Comment),
80
81    /// A luau-reserved-keyword
82    Keyword(Keyword),
83
84    /// A word that can both be a keyword and an identifier.
85    PartialKeyword(PartialKeyword),
86
87    /// Symbols like `(` and `)`.
88    Symbol(Symbol),
89
90    /// Operators like `+` and `and`
91    Operator(Operator),
92
93    /// Compound operators like `+=` and `//=`
94    CompoundOperator(CompoundOperator),
95
96    /// The end of file token.
97    EndOfFile,
98}
99
100impl TokenType {
101    /// Turn this token type into a [`Token`] with the passed properties.
102    pub fn into_token(
103        self,
104        start: Position,
105        end: Position,
106        leading_trivia: Vec<Trivia>,
107        trailing_trivia: Vec<Trivia>,
108    ) -> Token {
109        Token {
110            start,
111            leading_trivia,
112            token_type: self,
113            trailing_trivia,
114            end,
115        }
116    }
117}
118
119impl TokenType {
120    /// Try converting this token type into a string.
121    pub fn try_as_string(&self) -> Option<String> {
122        match self {
123            TokenType::Literal(literal) => match literal {
124                Literal::Number(luau_number) => match luau_number {
125                    LuauNumber::Plain(smol_str)
126                    | LuauNumber::Binary(smol_str)
127                    | LuauNumber::Hex(smol_str) => Some(smol_str.to_string()),
128                },
129                Literal::String(luau_string) => match luau_string {
130                    LuauString::SingleQuotes(smol_str)
131                    | LuauString::DoubleQuotes(smol_str)
132                    | LuauString::Backticks(smol_str)
133                    | LuauString::MultiLine(smol_str) => Some(smol_str.to_string()),
134                },
135                Literal::Boolean(true) => Some("true".to_string()),
136                Literal::Boolean(false) => Some("false".to_string()),
137            },
138            TokenType::Identifier(smol_str) => Some(smol_str.to_string()),
139            TokenType::Comment(comment) => match comment {
140                Comment::MultiLine(smol_str) | Comment::SingleLine(smol_str) => {
141                    Some(smol_str.to_string())
142                }
143            },
144            TokenType::Keyword(keyword) => Some(keyword.to_string()),
145            TokenType::PartialKeyword(partial_keyword) => Some(partial_keyword.to_string()),
146            TokenType::Symbol(symbol) => Some(symbol.to_string()),
147            TokenType::Operator(operator) => Some(operator.to_string()),
148            TokenType::CompoundOperator(compound_operator) => Some(compound_operator.to_string()),
149            _ => None,
150        }
151    }
152}
153
154impl_from!(TokenType <= {
155    Error(ParseError),
156    Literal(Literal),
157    Keyword(Keyword),
158    PartialKeyword(PartialKeyword),
159    Symbol(Symbol),
160    Operator(Operator),
161    CompoundOperator(CompoundOperator),
162});
luau_lexer/token/mod.rs

luau_lexer/token/
mod.rs
