boa/syntax/lexer/
token.rs

1//! This module implements all of the [Token]s used in the JavaScript programing language.
2//!
3//! More information:
4//!  - [ECMAScript reference][spec]
5//!
6//! [spec]: https://tc39.es/ecma262/#sec-tokens
7
8use super::regex::RegExpFlags;
9
10use crate::{
11    syntax::ast::{Keyword, Punctuator, Span},
12    syntax::lexer::template::TemplateString,
13    JsBigInt,
14};
15use std::fmt::{self, Debug, Display, Formatter};
16
17#[cfg(feature = "deser")]
18use serde::{Deserialize, Serialize};
19
20/// This represents the smallest individual words, phrases, or characters that JavaScript can understand.
21///
22/// More information:
23///  - [ECMAScript reference][spec]
24///
25/// [spec]: https://tc39.es/ecma262/#sec-tokens
26#[cfg_attr(feature = "deser", derive(Serialize, Deserialize))]
27#[derive(Debug, Clone, PartialEq)]
28pub struct Token {
29    /// The token kind, which contains the actual data of the token.
30    kind: TokenKind,
31    /// The token position in the original source code.
32    span: Span,
33}
34
35impl Token {
36    /// Create a new detailed token from the token data, line number and column number
37    #[inline]
38    pub fn new(kind: TokenKind, span: Span) -> Self {
39        Self { kind, span }
40    }
41
42    /// Gets the kind of the token.
43    #[inline]
44    pub fn kind(&self) -> &TokenKind {
45        &self.kind
46    }
47
48    /// Gets the token span in the original source code.
49    #[inline]
50    pub fn span(&self) -> Span {
51        self.span
52    }
53}
54
55impl Display for Token {
56    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
57        write!(f, "{}", self.kind)
58    }
59}
60
61/// Represents the type differenct types of numeric literals.
62#[cfg_attr(feature = "deser", derive(Serialize, Deserialize))]
63#[derive(Clone, PartialEq, Debug)]
64pub enum Numeric {
65    /// A floating point number
66    Rational(f64),
67
68    /// An integer
69    Integer(i32),
70
71    // A BigInt
72    BigInt(JsBigInt),
73}
74
75impl From<f64> for Numeric {
76    #[inline]
77    fn from(n: f64) -> Self {
78        Self::Rational(n)
79    }
80}
81
82impl From<i32> for Numeric {
83    #[inline]
84    fn from(n: i32) -> Self {
85        Self::Integer(n)
86    }
87}
88
89impl From<JsBigInt> for Numeric {
90    #[inline]
91    fn from(n: JsBigInt) -> Self {
92        Self::BigInt(n)
93    }
94}
95
96/// Represents the type of Token and the data it has inside.
97#[cfg_attr(feature = "deser", derive(Serialize, Deserialize))]
98#[derive(Clone, PartialEq, Debug)]
99pub enum TokenKind {
100    /// A boolean literal, which is either `true` or `false`.
101    BooleanLiteral(bool),
102
103    /// The end of the file.
104    EOF,
105
106    /// An identifier.
107    Identifier(Box<str>),
108
109    /// A keyword.
110    ///
111    /// see: [`Keyword`](../keyword/enum.Keyword.html)
112    Keyword(Keyword),
113
114    /// A `null` literal.
115    NullLiteral,
116
117    /// A numeric literal.
118    NumericLiteral(Numeric),
119
120    /// A piece of punctuation
121    ///
122    /// see: [`Punctuator`](../punc/enum.Punctuator.html)
123    Punctuator(Punctuator),
124
125    /// A string literal.
126    StringLiteral(Box<str>),
127
128    /// A part of a template literal without substitution.
129    TemplateNoSubstitution(TemplateString),
130
131    /// The part of a template literal between substitutions
132    TemplateMiddle(TemplateString),
133
134    /// A regular expression, consisting of body and flags.
135    RegularExpressionLiteral(Box<str>, RegExpFlags),
136
137    /// Indicates the end of a line (`\n`).
138    LineTerminator,
139
140    /// Indicates a comment, the content isn't stored.
141    Comment,
142}
143
144impl From<bool> for TokenKind {
145    fn from(oth: bool) -> Self {
146        Self::BooleanLiteral(oth)
147    }
148}
149
150impl From<Keyword> for TokenKind {
151    fn from(kw: Keyword) -> Self {
152        Self::Keyword(kw)
153    }
154}
155
156impl From<Punctuator> for TokenKind {
157    fn from(punc: Punctuator) -> Self {
158        Self::Punctuator(punc)
159    }
160}
161
162impl From<Numeric> for TokenKind {
163    fn from(num: Numeric) -> Self {
164        Self::NumericLiteral(num)
165    }
166}
167
168impl TokenKind {
169    /// Creates a `BooleanLiteral` token kind.
170    pub fn boolean_literal(lit: bool) -> Self {
171        Self::BooleanLiteral(lit)
172    }
173
174    /// Creates an `EOF` token kind.
175    pub fn eof() -> Self {
176        Self::EOF
177    }
178
179    /// Creates an `Identifier` token type.
180    pub fn identifier<I>(ident: I) -> Self
181    where
182        I: Into<Box<str>>,
183    {
184        Self::Identifier(ident.into())
185    }
186
187    /// Creates a `Keyword` token kind.
188    pub fn keyword(keyword: Keyword) -> Self {
189        Self::Keyword(keyword)
190    }
191
192    /// Creates a `NumericLiteral` token kind.
193    pub fn numeric_literal<L>(lit: L) -> Self
194    where
195        L: Into<Numeric>,
196    {
197        Self::NumericLiteral(lit.into())
198    }
199
200    /// Creates a `Punctuator` token type.
201    pub fn punctuator(punc: Punctuator) -> Self {
202        Self::Punctuator(punc)
203    }
204
205    /// Creates a `StringLiteral` token type.
206    pub fn string_literal<S>(lit: S) -> Self
207    where
208        S: Into<Box<str>>,
209    {
210        Self::StringLiteral(lit.into())
211    }
212
213    pub fn template_middle(template_string: TemplateString) -> Self {
214        Self::TemplateMiddle(template_string)
215    }
216
217    pub fn template_no_substitution(template_string: TemplateString) -> Self {
218        Self::TemplateNoSubstitution(template_string)
219    }
220
221    /// Creates a `RegularExpressionLiteral` token kind.
222    pub fn regular_expression_literal<B, R>(body: B, flags: R) -> Self
223    where
224        B: Into<Box<str>>,
225        R: Into<RegExpFlags>,
226    {
227        Self::RegularExpressionLiteral(body.into(), flags.into())
228    }
229
230    /// Creates a `LineTerminator` token kind.
231    pub fn line_terminator() -> Self {
232        Self::LineTerminator
233    }
234
235    /// Creates a 'Comment' token kind.
236    pub fn comment() -> Self {
237        Self::Comment
238    }
239}
240
241impl Display for TokenKind {
242    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
243        match *self {
244            Self::BooleanLiteral(ref val) => write!(f, "{}", val),
245            Self::EOF => write!(f, "end of file"),
246            Self::Identifier(ref ident) => write!(f, "{}", ident),
247            Self::Keyword(ref word) => write!(f, "{}", word),
248            Self::NullLiteral => write!(f, "null"),
249            Self::NumericLiteral(Numeric::Rational(num)) => write!(f, "{}", num),
250            Self::NumericLiteral(Numeric::Integer(num)) => write!(f, "{}", num),
251            Self::NumericLiteral(Numeric::BigInt(ref num)) => write!(f, "{}n", num),
252            Self::Punctuator(ref punc) => write!(f, "{}", punc),
253            Self::StringLiteral(ref lit) => write!(f, "{}", lit),
254            Self::TemplateNoSubstitution(ref ts) => write!(f, "{}", ts.as_raw()),
255            Self::TemplateMiddle(ref ts) => write!(f, "{}", ts.as_raw()),
256            Self::RegularExpressionLiteral(ref body, ref flags) => write!(f, "/{}/{}", body, flags),
257            Self::LineTerminator => write!(f, "line terminator"),
258            Self::Comment => write!(f, "comment"),
259        }
260    }
261}
boa/syntax/lexer/token.rs

boa/syntax/lexer/
token.rs
