submilli_engine/
token.rs

1use crate::{RawDoc, Span};
2
3#[derive(Clone, Debug, PartialEq)]
4pub enum TokenKind {
5    NumberLiteral(f64),
6    /// No `n` suffix, no leading sign (unary `-` is at the AST level).
7    /// String because the value may exceed any fixed-width integer.
8    BigIntLiteral(String),
9    StringLiteral(String),
10    BooleanLiteral(bool),
11    NullLiteral,
12    /// No `${…}` interpolations; parser collapses to a plain string literal.
13    TemplateNoSubstitution(String),
14    /// Cooked text up to the first `${`. Lexer pushes a frame onto
15    /// `template_frames`; the next depth-0 `}` re-enters template scanning.
16    TemplateHead(String),
17    TemplateMiddle(String),
18    /// Cooked text from the last `}` to the closing backtick.
19    /// Pops the frame pushed by [`TemplateHead`](Self::TemplateHead).
20    TemplateTail(String),
21    /// `source` preserves escape sequences verbatim (the regex engine
22    /// interprets them, not the lexer). Context disambiguation
23    /// (literal vs. division) via `is_regex_context`.
24    RegexLiteral {
25        source: String,
26        flags: String,
27    },
28
29    // Identifier (span-only; text via &source[span.start..span.end])
30    Identifier,
31
32    Let,
33    Const,
34    Function,
35    If,
36    Else,
37    While,
38    Do,
39    For,
40    Break,
41    Continue,
42    Return,
43    Switch,
44    Case,
45    Default,
46    /// Phase-1 forgiveness: accepted as a leading modifier on top-level
47    /// declarations and ignored (there is no export consumer yet).
48    Export,
49    Void,
50    Interface,
51    Enum,
52    /// Also drives the `"field" in x` narrowing predicate.
53    In,
54    Typeof,
55    Import,
56    Instanceof,
57    New,
58    Try,
59    Catch,
60    Finally,
61    Throw,
62    Class,
63    Extends,
64    Implements,
65    Super,
66    This,
67
68    Plus,
69    Minus,
70    Star,
71    Slash,
72    Percent,
73    Equals,
74    PlusEquals,
75    MinusEquals,
76    StarEquals,
77    SlashEquals,
78    /// Lowered by the compound-assignment desugar to `x = x % y`.
79    PercentEquals,
80    /// Right-associative, higher precedence than `*`/`/`/`%`.
81    /// Lex order: `**=` matched before `**` before `*=`.
82    StarStar,
83    StarStarEquals,
84    /// Postfix-only (no prefix `++x`). Lex order: `++` matched before `+=`.
85    PlusPlus,
86    MinusMinus,
87    EqEqEq,
88    EqEq,
89    BangEqEq,
90    BangEq,
91    LessThan,
92    GreaterThan,
93    LessEquals,
94    GreaterEquals,
95    Bang,
96    AmpAmp,
97    Amp,
98    AmpEquals,
99    PipeEquals,
100    Caret,
101    CaretEquals,
102    Tilde,
103    Pipe,
104    PipePipe,
105    Dot,
106    /// Parser accepts only inside destructuring patterns (`{ … }` / `[ … ]`).
107    DotDotDot,
108    Arrow,
109    Question,
110    /// Single token so the postfix-chain parser dispatches on one peek.
111    QuestionDot,
112    QuestionQuestion,
113
114    LeftParen,
115    RightParen,
116    LeftBrace,
117    RightBrace,
118    LeftBracket,
119    RightBracket,
120    Comma,
121    Colon,
122    Semicolon,
123
124    Newline,
125    Eof,
126}
127
128#[derive(Clone, Debug, PartialEq)]
129pub struct Token {
130    pub kind: TokenKind,
131    pub span: Span,
132    /// `/** ... */` doc comment captured as out-of-band trivia;
133    /// consumed by the parser at declaration tokens.
134    pub leading_doc: Option<RawDoc>,
135}
136
137impl Token {
138    pub fn new(kind: TokenKind, span: Span) -> Self {
139        Self {
140            kind,
141            span,
142            leading_doc: None,
143        }
144    }
145}
146
147#[cfg(test)]
148mod tests {
149    use super::{Token, TokenKind};
150    use crate::Span;
151
152    #[test]
153    fn construct_literal_keyword_operator() {
154        let num = Token::new(
155            TokenKind::NumberLiteral(42.5),
156            Span::new(crate::FileId(0), 0, 4).unwrap(),
157        );
158        assert_eq!(num.kind, TokenKind::NumberLiteral(42.5));
159        assert_eq!(num.span, Span::new(crate::FileId(0), 0, 4).unwrap());
160        assert!(num.leading_doc.is_none());
161
162        let kw = Token::new(TokenKind::Let, Span::new(crate::FileId(0), 5, 8).unwrap());
163        assert_eq!(kw.kind, TokenKind::Let);
164
165        let op = Token::new(
166            TokenKind::EqEqEq,
167            Span::new(crate::FileId(0), 10, 13).unwrap(),
168        );
169        assert_eq!(op.kind, TokenKind::EqEqEq);
170    }
171
172    #[test]
173    fn clone_and_equality() {
174        let t = Token::new(
175            TokenKind::StringLiteral("hello".to_string()),
176            Span::new(crate::FileId(0), 0, 7).unwrap(),
177        );
178        assert_eq!(t.clone(), t);
179    }
180}
submilli_engine/token.rs

submilli_engine/
token.rs
