mxmlextrema_as3parser/parser/
token.rs

1use crate::ns::*;
2
3/// Represents a lexical token.
4#[derive(Clone, PartialEq, Debug)]
5pub enum Token {
6    Eof,
7    Identifier(String),
8    String(String),
9    /// Numeric literal token.
10    /// The numeric value is in character representation, which may be parsed
11    /// through data type specific methods such as [`NumericLiteral::parse_double()`].
12    Number(String, NumberSuffix),
13    RegExp {
14        body: String,
15        flags: String,
16    },
17
18    CssNumber {
19        value: f64,
20        unit: Option<String>,
21    },
22    CssHashWord(String),
23    CssBeginsWith,
24    CssEndsWith,
25    CssContains,
26    CssListMatch,
27    CssHreflangMatch,
28    CssAtNamespace,
29    CssAtMedia,
30    CssAtFontFace,
31    CssAtImport,
32    CssImportant,
33    CssSemicolons,
34
35    // Punctuator
36    ColonColon,
37    /// The `@` token.
38    Attribute,
39    /// The `..` token.
40    Descendants,
41    /// The `...` token.
42    Ellipsis,
43    ParenOpen,
44    ParenClose,
45    SquareOpen,
46    SquareClose,
47    BlockOpen,
48    BlockClose,
49    Dot,
50    Semicolon,
51    Comma,
52    Lt,
53    Gt,
54    /// `<=`
55    Le,
56    /// `>=`
57    Ge,
58    Equals,
59    NotEquals,
60    StrictEquals,
61    StrictNotEquals,
62    Plus,
63    Minus,
64    Times,
65    Div,
66    Percent,
67    Increment,
68    Decrement,
69    LeftShift,
70    RightShift,
71    UnsignedRightShift,
72    Ampersand,
73    Hat,
74    Pipe,
75    Tilde,
76    LogicalAnd,
77    LogicalXor,
78    LogicalOr,
79    Question,
80    Exclamation,
81    Colon,
82    Assign,
83    AddAssign,
84    SubtractAssign,
85    MultiplyAssign,
86    DivideAssign,
87    RemainderAssign,
88    LeftShiftAssign,
89    RightShiftAssign,
90    UnsignedRightShiftAssign,
91    BitwiseAndAssign,
92    BitwiseXorAssign,
93    BitwiseOrAssign,
94    LogicalAndAssign,
95    LogicalXorAssign,
96    LogicalOrAssign,
97    /// `**`
98    Power,
99    /// `**=`
100    PowerAssign,
101    /// `??`
102    NullCoalescing,
103    /// `??=`
104    NullCoalescingAssign,
105    /// `?.`
106    OptionalChaining,
107
108    // Reserved words
109    As,
110    Await,
111    Break,
112    Case,
113    Catch,
114    Class,
115    Const,
116    Continue,
117    Default,
118    Delete,
119    Do,
120    Else,
121    Extends,
122    False,
123    Finally,
124    For,
125    Function,
126    If,
127    Implements,
128    Import,
129    In,
130    Instanceof,
131    Interface,
132    Internal,
133    Is,
134    New,
135    Not,
136    Null,
137    Package,
138    Private,
139    Protected,
140    Public,
141    Return,
142    Super,
143    Switch,
144    This,
145    Throw,
146    True,
147    Try,
148    Typeof,
149    Use,
150    Var,
151    Void,
152    While,
153    With,
154    Yield,
155
156    XmlWhitespace,
157    XmlLtSlash,
158    XmlSlashGt,
159    XmlText(String),
160    XmlName(String),
161    XmlMarkup(String),
162    XmlAttributeValue(String),
163}
164
165impl ToString for Token {
166    /// Converts the token into a readable string.
167    ///
168    /// The method `Token::to_string` returns the following possible values:
169    /// 
170    /// * `"end of program"`
171    /// * `"identifier"`
172    /// * `"string"` for string literal
173    /// * `"number"` for numeric literal
174    /// * `"regular expression"` for regular expression literal
175    /// * `"'keyword'"` for reserved words
176    /// * `"'punctuator'"` for various punctuators
177    /// * `"punctuator"` for various punctuators
178    /// * `"XML whitespace"`
179    /// * `"'</'"`
180    /// * `"'/>'"`
181    /// * `"XML text"`
182    /// * `"XML name"`
183    /// * `"XML markup"`
184    /// * `"XML attribute value"`
185    fn to_string(&self) -> String {
186        (match self {
187            Token::Eof => "end-of-file",
188            Token::Identifier(_) => "identifier",
189            Token::String(_) => "string",
190            Token::Number(_, _) => "number",
191            Token::RegExp { .. } => "regular expression",
192
193            Token::CssNumber { .. } => "number",
194            Token::CssHashWord(_) => "hash-word",
195            Token::CssBeginsWith => "'^='",
196            Token::CssEndsWith => "'$='",
197            Token::CssContains => "'*='",
198            Token::CssListMatch => "'~='",
199            Token::CssHreflangMatch => "'|='",
200            Token::CssAtNamespace => "at-namespace",
201            Token::CssAtMedia => "at-media",
202            Token::CssAtFontFace => "at-font-face",
203            Token::CssAtImport => "at-import",
204            Token::CssImportant => "'!important'",
205            Token::CssSemicolons => "semicolon",
206
207            // Punctuators
208            Token::ColonColon => "colon-colon",
209            Token::Attribute => "'@'",
210            Token::Descendants => "'..'",
211            Token::Ellipsis => "'...'",
212            Token::ParenOpen => "paren-open",
213            Token::ParenClose => "paren-close",
214            Token::SquareOpen => "square-open",
215            Token::SquareClose => "square-close",
216            Token::BlockOpen => "block-open",
217            Token::BlockClose => "block-close",
218            Token::Dot => "dot",
219            Token::Semicolon => "semicolon",
220            Token::Comma => "comma",
221            Token::Lt => "less-than",
222            Token::Gt => "greater-than",
223            Token::Le => "'<='",
224            Token::Ge => "'>='",
225            Token::Equals => "'=='",
226            Token::NotEquals => "'!='",
227            Token::StrictEquals => "'==='",
228            Token::StrictNotEquals => "'!=='",
229            Token::Plus => "plus",
230            Token::Minus => "minus",
231            Token::Times => "times",
232            Token::Div => "slash",
233            Token::Percent => "percent",
234            Token::Increment => "'++'",
235            Token::Decrement => "'--'",
236            Token::LeftShift => "'<<'",
237            Token::RightShift => "'>>'",
238            Token::UnsignedRightShift => "'>>>'",
239            Token::Ampersand => "ampersand",
240            Token::Hat => "hat",
241            Token::Pipe => "pipe",
242            Token::Tilde => "tilde",
243            Token::LogicalAnd => "'&&'",
244            Token::LogicalXor => "'^^'",
245            Token::LogicalOr => "'||'",
246            Token::Question => "question-mark",
247            Token::Exclamation => "exclamation-mark",
248            Token::Colon => "colon",
249            Token::Assign => "'='",
250            Token::AddAssign => "'+='",
251            Token::SubtractAssign => "'-='",
252            Token::MultiplyAssign => "'*='",
253            Token::DivideAssign => "'/='",
254            Token::RemainderAssign => "'%='",
255            Token::LeftShiftAssign => "'<<='",
256            Token::RightShiftAssign => "'>>='",
257            Token::UnsignedRightShiftAssign => "'>>>='",
258            Token::BitwiseAndAssign => "'&='",
259            Token::BitwiseXorAssign => "'^='",
260            Token::BitwiseOrAssign => "'|='",
261            Token::LogicalAndAssign => "'&&='",
262            Token::LogicalXorAssign => "'^^='",
263            Token::LogicalOrAssign => "'||='",
264            Token::Power => "'**'",
265            Token::PowerAssign => "'**='",
266            Token::NullCoalescing => "'??'",
267            Token::NullCoalescingAssign => "'??='",
268            Token::OptionalChaining => "'?.'",
269
270            // Reserved words
271            Token::As => "'as'",
272            Token::Await => "'await'",
273            Token::Break => "'break'",
274            Token::Case => "'case'",
275            Token::Catch => "'catch'",
276            Token::Class => "'class'",
277            Token::Const => "'const'",
278            Token::Continue => "'continue'",
279            Token::Default => "'default'",
280            Token::Delete => "'delete'",
281            Token::Do => "'do'",
282            Token::Else => "'else'",
283            Token::Extends => "'extends'",
284            Token::False => "'false'",
285            Token::Finally => "'finally'",
286            Token::For => "'for'",
287            Token::Function => "'function'",
288            Token::If => "'if'",
289            Token::Implements => "'implements'",
290            Token::Import => "'import'",
291            Token::In => "'in'",
292            Token::Instanceof => "'instanceof'",
293            Token::Interface => "'interface'",
294            Token::Internal => "'internal'",
295            Token::Is => "'is'",
296            Token::New => "'new'",
297            Token::Not => "'not'",
298            Token::Null => "'null'",
299            Token::Package => "'package'",
300            Token::Private => "'private'",
301            Token::Protected => "'protected'",
302            Token::Public => "'public'",
303            Token::Return => "'return'",
304            Token::Super => "'super'",
305            Token::Switch => "'switch'",
306            Token::This => "'this'",
307            Token::Throw => "'throw'",
308            Token::True => "'true'",
309            Token::Try => "'try'",
310            Token::Typeof => "'typeof'",
311            Token::Use => "'use'",
312            Token::Var => "'var'",
313            Token::Void => "'void'",
314            Token::While => "'while'",
315            Token::With => "'with'",
316            Token::Yield => "'yield'",
317
318            Token::XmlWhitespace => "XML whitespace",
319            Token::XmlLtSlash => "'</'",
320            Token::XmlSlashGt => "'/>'",
321            Token::XmlText(_) => "XML text",
322            Token::XmlName(_) => "XML name",
323            Token::XmlMarkup(_) => "XML markup",
324            Token::XmlAttributeValue(_) => "XML attribute value",
325        }).into()
326    }
327}
328
329impl Token {
330    pub fn is_context_keyword(token: &(Token, Location), keyword: &str) -> bool {
331        if let Token::Identifier(name) = &token.0 {
332            name == keyword && token.1.character_count() == name.len()
333        } else {
334            false
335        }
336    }
337
338    /// Indicates whether the token is a reserved word.
339    pub fn is_reserved_word(&self) -> bool {
340        self.reserved_word_name().is_some()
341    }
342
343    pub fn is_identifier_name(&self) -> bool {
344        matches!(self, Token::Identifier(_)) || self.is_reserved_word()
345    }
346
347    /// Tests whether the token is a reserved word and returns
348    /// its *IdentifierName* string.
349    pub fn reserved_word_name(&self) -> Option<String> {
350        match *self {
351            Token::As => Some("as".into()),
352            Token::Await => Some("await".into()),
353            Token::Break => Some("break".into()),
354            Token::Case => Some("case".into()),
355            Token::Catch => Some("catch".into()),
356            Token::Class => Some("class".into()),
357            Token::Const => Some("const".into()),
358            Token::Continue => Some("continue".into()),
359            Token::Default => Some("default".into()),
360            Token::Delete => Some("delete".into()),
361            Token::Do => Some("do".into()),
362            Token::Else => Some("else".into()),
363            Token::Extends => Some("extends".into()),
364            Token::False => Some("false".into()),
365            Token::Finally => Some("finally".into()),
366            Token::For => Some("for".into()),
367            Token::Function => Some("function".into()),
368            Token::If => Some("if".into()),
369            Token::Implements => Some("implements".into()),
370            Token::Import => Some("import".into()),
371            Token::In => Some("in".into()),
372            Token::Instanceof => Some("instanceof".into()),
373            Token::Interface => Some("interface".into()),
374            Token::Internal => Some("internal".into()),
375            Token::Is => Some("is".into()),
376            Token::New => Some("new".into()),
377            Token::Not => Some("not".into()),
378            Token::Null => Some("null".into()),
379            Token::Package => Some("package".into()),
380            Token::Private => Some("private".into()),
381            Token::Protected => Some("protected".into()),
382            Token::Public => Some("public".into()),
383            Token::Return => Some("return".into()),
384            Token::Super => Some("super".into()),
385            Token::Switch => Some("switch".into()),
386            Token::This => Some("this".into()),
387            Token::Throw => Some("throw".into()),
388            Token::True => Some("true".into()),
389            Token::Try => Some("try".into()),
390            Token::Typeof => Some("typeof".into()),
391            Token::Use => Some("use".into()),
392            Token::Var => Some("var".into()),
393            Token::Void => Some("void".into()),
394            Token::While => Some("while".into()),
395            Token::With => Some("with".into()),
396            Token::Yield => Some("yield".into()),
397            _ => None,
398        }
399    }
400
401    /// Converts a compound assignment, a logical assignment, or a nullish coalescing assignment to an *Operator* value.
402    pub fn compound_assignment(&self) -> Option<Operator> {
403        match self {
404            Self::AddAssign => Some(Operator::Add),
405            Self::SubtractAssign => Some(Operator::Subtract),
406            Self::MultiplyAssign => Some(Operator::Multiply),
407            Self::DivideAssign => Some(Operator::Divide),
408            Self::RemainderAssign => Some(Operator::Remainder),
409            Self::PowerAssign => Some(Operator::Power),
410            Self::LeftShiftAssign => Some(Operator::ShiftLeft),
411            Self::RightShiftAssign => Some(Operator::ShiftRight),
412            Self::UnsignedRightShiftAssign => Some(Operator::ShiftRightUnsigned),
413            Self::BitwiseAndAssign => Some(Operator::BitwiseAnd),
414            Self::BitwiseXorAssign => Some(Operator::BitwiseXor),
415            Self::BitwiseOrAssign => Some(Operator::BitwiseOr),
416            Self::LogicalAndAssign => Some(Operator::LogicalAnd),
417            Self::LogicalXorAssign => Some(Operator::LogicalXor),
418            Self::LogicalOrAssign => Some(Operator::LogicalOr),
419            Self::NullCoalescingAssign => Some(Operator::NullCoalescing),
420            _ => None,
421        }
422    }
423
424    /// Converts this token into a binary operator, excluding
425    /// `not in`, and `is not`.
426    pub fn to_binary_operator(&self) -> Option<Operator> {
427        match self {
428            Self::Times => Some(Operator::Multiply),
429            Self::Div => Some(Operator::Divide),
430            Self::Percent => Some(Operator::Remainder),
431            Self::Plus => Some(Operator::Add),
432            Self::Minus => Some(Operator::Subtract),
433            Self::LeftShift => Some(Operator::ShiftLeft),
434            Self::RightShift => Some(Operator::ShiftRight),
435            Self::UnsignedRightShift => Some(Operator::ShiftRightUnsigned),
436            Self::Lt => Some(Operator::Lt),
437            Self::Gt => Some(Operator::Gt),
438            Self::Le => Some(Operator::Le),
439            Self::Ge => Some(Operator::Ge),
440            Self::As => Some(Operator::As),
441            Self::In => Some(Operator::In),
442            Self::Is => Some(Operator::Is),
443            Self::Instanceof => Some(Operator::Instanceof),
444            Self::Equals => Some(Operator::Equals),
445            Self::NotEquals => Some(Operator::NotEquals),
446            Self::StrictEquals => Some(Operator::StrictEquals),
447            Self::StrictNotEquals => Some(Operator::StrictNotEquals),
448            Self::Ampersand => Some(Operator::BitwiseAnd),
449            Self::Hat => Some(Operator::BitwiseXor),
450            Self::Pipe => Some(Operator::BitwiseOr),
451            Self::LogicalAnd => Some(Operator::LogicalAnd),
452            Self::LogicalXor => Some(Operator::LogicalXor),
453            Self::LogicalOr => Some(Operator::LogicalOr),
454            Self::NullCoalescing => Some(Operator::NullCoalescing),
455            Self::Power => Some(Operator::Power),
456            _  => None,
457        }
458    }
459    
460    pub(crate) fn to_attribute(&self, location: &Location) -> Option<Attribute> {
461        match self {
462            Self::Public => Some(Attribute::Public(location.clone())),
463            Self::Private => Some(Attribute::Private(location.clone())),
464            Self::Protected => Some(Attribute::Protected(location.clone())),
465            Self::Internal => Some(Attribute::Internal(location.clone())),
466            Self::Identifier(ref name) => {
467                Attribute::from_identifier_name(name, &location)
468            },
469            _ => None,
470        }
471    }
472}
mxmlextrema_as3parser/parser/token.rs

mxmlextrema_as3parser/parser/
token.rs
