mago_syntax/token/
mod.rs

1use serde::Deserialize;
2use serde::Serialize;
3use strum::Display;
4
5use mago_interner::StringIdentifier;
6use mago_span::Span;
7
8use crate::T;
9
10#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
11#[serde(tag = "type", content = "value")]
12#[repr(C)]
13pub enum DocumentKind {
14    Heredoc,
15    Nowdoc,
16}
17
18#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
19#[serde(tag = "type", content = "value")]
20#[repr(C)]
21pub enum Associativity {
22    NonAssociative,
23    Left,
24    Right,
25}
26
27#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
28#[serde(tag = "type", content = "value")]
29#[repr(C)]
30pub enum Precedence {
31    Lowest,
32    LowLogicalOr,
33    LowLogicalXor,
34    LowLogicalAnd,
35    Print,
36    Yield,
37    YieldFrom,
38    IncDec,
39    KeyOr,
40    KeyXor,
41    KeyAnd,
42    Assignment,
43    ElvisOrConditional,
44    NullCoalesce,
45    Or,
46    And,
47    BitwiseOr,
48    BitwiseXor,
49    BitwiseAnd,
50    Equality,
51    Comparison,
52    Concat,
53    BitShift,
54    AddSub,
55    MulDivMod,
56    Bang,
57    Instanceof,
58    Prefix,
59    Pow,
60    Clone,
61    CallDim,
62    New,
63    ArrayDim,
64    ObjectAccess,
65}
66
67pub trait GetPrecedence {
68    fn precedence(&self) -> Precedence;
69}
70
71#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
72#[serde(tag = "type", content = "value")]
73#[repr(C)]
74pub enum TokenKind {
75    Whitespace,                  // ` `
76    Eval,                        // `eval`
77    Die,                         // `die`
78    Self_,                       // `self`
79    Parent,                      // `parent`
80    Backtick,                    // `` ` ``
81    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
82    DocumentEnd,                 // `abc`
83    From,                        // `from`
84    Print,                       // `print`
85    Dollar,                      // `$`
86    HaltCompiler,                // `__halt_compiler`
87    Readonly,                    // `readonly`
88    Global,                      // `global`
89    Abstract,                    // `abstract`
90    Ampersand,                   // `&`
91    AmpersandEqual,              // `&=`
92    AmpersandAmpersand,          // `&&`
93    AmpersandAmpersandEqual,     // `&&=`
94    Array,                       // `array`
95    ArrayCast,                   // `(array)`
96    MinusGreaterThan,            // `->`
97    QuestionMinusGreaterThan,    // `?->`
98    At,                          // `@`
99    As,                          // `as`
100    Asterisk,                    // `*`
101    HashLeftBracket,             // `#[`
102    Bang,                        // `!`
103    BangEqual,                   // `!=`
104    LessThanGreaterThan,         // `<>`
105    BangEqualEqual,              // `!==`
106    LessThanEqualGreaterThan,    // `<=>`
107    BoolCast,                    // `(bool)`
108    BooleanCast,                 // `(boolean)`
109    And,                         // `and`
110    Or,                          // `or`
111    Break,                       // `break`
112    Callable,                    // `callable`
113    Caret,                       // `^`
114    CaretEqual,                  // `^=`
115    Case,                        // `case`
116    Catch,                       // `catch`
117    Class,                       // `class`
118    ClassConstant,               // `__CLASS__`
119    TraitConstant,               // `__TRAIT__`
120    FunctionConstant,            // `__FUNCTION__`
121    MethodConstant,              // `__METHOD__`
122    LineConstant,                // `__LINE__`
123    FileConstant,                // `__FILE__`
124    Clone,                       // `clone`
125    MinusEqual,                  // `-=`
126    CloseTag,                    // `?>`
127    QuestionQuestion,            // `??`
128    QuestionQuestionEqual,       // `??=`
129    AsteriskEqual,               // `*=`
130    Colon,                       // `:`
131    Comma,                       // `,`
132    SingleLineComment,           // `// comment`
133    HashComment,                 // `# comment`
134    MultiLineComment,            // `/* comment */`
135    DocBlockComment,             // `/** comment */`
136    Const,                       // `const`
137    PartialLiteralString,        // `"string` or `'string`, missing closing quote
138    LiteralString,               // `"string"` or `'string'`
139    Continue,                    // `continue`
140    Declare,                     // `declare`
141    MinusMinus,                  // `--`
142    Default,                     // `default`
143    DirConstant,                 // `__DIR__`
144    SlashEqual,                  // `/=`
145    Do,                          // `do`
146    DollarLeftBrace,             // `${`
147    Dot,                         // `.`
148    DotEqual,                    // `.=`
149    EqualGreaterThan,            // `=>`
150    DoubleCast,                  // `(double)`
151    RealCast,                    // `(real)`
152    FloatCast,                   // `(float)`
153    ColonColon,                  // `::`
154    EqualEqual,                  // `==`
155    DoubleQuote,                 // `"`
156    Else,                        // `else`
157    Echo,                        // `echo`
158    DotDotDot,                   // `...`
159    ElseIf,                      // `elseif`
160    Empty,                       // `empty`
161    EndDeclare,                  // `enddeclare`
162    EndFor,                      // `endfor`
163    EndForeach,                  // `endforeach`
164    EndIf,                       // `endif`
165    EndSwitch,                   // `endswitch`
166    EndWhile,                    // `endwhile`
167    Enum,                        // `enum`
168    Equal,                       // `=`
169    Extends,                     // `extends`
170    False,                       // `false`
171    Final,                       // `final`
172    Finally,                     // `finally`
173    LiteralFloat,                // `1.0`
174    Fn,                          // `fn`
175    For,                         // `for`
176    Foreach,                     // `foreach`
177    FullyQualifiedIdentifier,    // `\Namespace\Class`
178    Function,                    // `function`
179    Goto,                        // `goto`
180    GreaterThan,                 // `>`
181    GreaterThanEqual,            // `>=`
182    Identifier,                  // `name`
183    If,                          // `if`
184    Implements,                  // `implements`
185    Include,                     // `include`
186    IncludeOnce,                 // `include_once`
187    PlusPlus,                    // `++`
188    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
189    InlineShebang,               // `#!...`
190    Instanceof,                  // `instanceof`
191    Insteadof,                   // `insteadof`
192    Exit,                        // `exit`
193    Unset,                       // `unset`
194    Isset,                       // `isset`
195    List,                        // `list`
196    LiteralInteger,              // `1`
197    IntCast,                     // `(int)`
198    IntegerCast,                 // `(integer)`
199    Interface,                   // `interface`
200    LeftBrace,                   // `{`
201    LeftBracket,                 // `[`
202    LeftParenthesis,             // `(`
203    LeftShift,                   // `<<`
204    LeftShiftEqual,              // `<<=`
205    RightShift,                  // `>>`
206    RightShiftEqual,             // `>>=`
207    LessThan,                    // `<`
208    LessThanEqual,               // `<=`
209    Match,                       // `match`
210    Minus,                       // `-`
211    Namespace,                   // `namespace`
212    NamespaceSeparator,          // `\`
213    NamespaceConstant,           // `__NAMESPACE__`
214    New,                         // `new`
215    Null,                        // `null`
216    ObjectCast,                  // `(object)`
217    UnsetCast,                   // `(unset)`
218    OpenTag,                     // `<?php`
219    EchoTag,                     // `<?=`
220    ShortOpenTag,                // `<?`
221    Percent,                     // `%`
222    PercentEqual,                // `%=`
223    Pipe,                        // `|`
224    PipeEqual,                   // `|=`
225    Plus,                        // `+`
226    PlusEqual,                   // `+=`
227    AsteriskAsterisk,            // `**`
228    AsteriskAsteriskEqual,       // `**=`
229    Private,                     // `private`
230    PrivateSet,                  // `private(set)`
231    Protected,                   // `protected`
232    ProtectedSet,                // `protected(set)`
233    Public,                      // `public`
234    PublicSet,                   // `public(set)`
235    QualifiedIdentifier,         // `Namespace\Class`
236    Question,                    // `?`
237    QuestionColon,               // `?:`
238    Require,                     // `require`
239    RequireOnce,                 // `require_once`
240    Return,                      // `return`
241    RightBrace,                  // `}`
242    RightBracket,                // `]`
243    RightParenthesis,            // `)`
244    Semicolon,                   // `;`
245    Slash,                       // `/`
246    Static,                      // `static`
247    StringCast,                  // `(string)`
248    BinaryCast,                  // `(binary)`
249    VoidCast,                    // `(void)`
250    StringPart,                  // `string` inside a double-quoted string, or a document string
251    Switch,                      // `switch`
252    Throw,                       // `throw`
253    Trait,                       // `trait`
254    EqualEqualEqual,             // `===`
255    True,                        // `true`
256    Try,                         // `try`
257    Use,                         // `use`
258    Var,                         // `var`
259    Variable,                    // `$name`
260    Yield,                       // `yield`
261    While,                       // `while`
262    Tilde,                       // `~`
263    PipePipe,                    // `||`
264    Xor,                         // `xor`
265}
266
267#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
268#[repr(C)]
269pub struct Token {
270    pub kind: TokenKind,
271    pub value: StringIdentifier,
272    pub span: Span,
273}
274
275impl Precedence {
276    #[inline(always)]
277    pub fn infix(kind: &TokenKind) -> Precedence {
278        match kind {
279            T!["**"] => Precedence::Pow,
280            T!["instanceof"] => Precedence::Instanceof,
281            T!["*" | "/" | "%"] => Precedence::MulDivMod,
282            T!["+" | "-"] => Precedence::AddSub,
283            T!["<<"] | T![">>"] => Precedence::BitShift,
284            T!["."] => Precedence::Concat,
285            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
286            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
287            T!["&"] => Precedence::BitwiseAnd,
288            T!["^"] => Precedence::BitwiseXor,
289            T!["|"] => Precedence::BitwiseOr,
290            T!["&&"] => Precedence::And,
291            T!["||"] => Precedence::Or,
292            T!["??"] => Precedence::NullCoalesce,
293            T!["?" | "?:"] => Precedence::ElvisOrConditional,
294            T!["="
295                | "+="
296                | "-="
297                | "*="
298                | "**="
299                | "/="
300                | ".="
301                | "&&="
302                | "??="
303                | "%="
304                | "&="
305                | "|="
306                | "^="
307                | "<<="
308                | ">>="] => Precedence::Assignment,
309            T!["yield"] => Precedence::Yield,
310            T!["and"] => Precedence::KeyAnd,
311            T!["or"] => Precedence::KeyOr,
312            T!["xor"] => Precedence::KeyXor,
313            _ => Precedence::Lowest,
314        }
315    }
316
317    #[inline(always)]
318    pub fn postfix(kind: &TokenKind) -> Self {
319        match kind {
320            T!["++" | "--"] => Self::Prefix,
321            T!["("] => Self::CallDim,
322            T!["["] => Self::ArrayDim,
323            T!["->" | "?->" | "::"] => Self::ObjectAccess,
324            _ => Self::Lowest,
325        }
326    }
327
328    #[inline(always)]
329    pub fn associativity(&self) -> Option<Associativity> {
330        Some(match self {
331            Self::Instanceof
332            | Self::MulDivMod
333            | Self::AddSub
334            | Self::BitShift
335            | Self::Concat
336            | Self::BitwiseAnd
337            | Self::BitwiseOr
338            | Self::BitwiseXor
339            | Self::And
340            | Self::Or
341            | Self::KeyAnd
342            | Self::KeyOr
343            | Self::KeyXor => Associativity::Left,
344            Self::Pow | Self::NullCoalesce | Self::Assignment => Associativity::Right,
345            Self::ElvisOrConditional | Self::Equality | Self::Comparison => Associativity::NonAssociative,
346            _ => return None,
347        })
348    }
349}
350
351impl TokenKind {
352    #[inline(always)]
353    pub const fn is_keyword(&self) -> bool {
354        matches!(
355            self,
356            TokenKind::Eval
357                | TokenKind::Die
358                | TokenKind::Empty
359                | TokenKind::Isset
360                | TokenKind::Unset
361                | TokenKind::Exit
362                | TokenKind::EndDeclare
363                | TokenKind::EndSwitch
364                | TokenKind::EndWhile
365                | TokenKind::EndForeach
366                | TokenKind::EndFor
367                | TokenKind::EndIf
368                | TokenKind::From
369                | TokenKind::And
370                | TokenKind::Or
371                | TokenKind::Xor
372                | TokenKind::Print
373                | TokenKind::Readonly
374                | TokenKind::Global
375                | TokenKind::Match
376                | TokenKind::Abstract
377                | TokenKind::Array
378                | TokenKind::As
379                | TokenKind::Break
380                | TokenKind::Case
381                | TokenKind::Catch
382                | TokenKind::Class
383                | TokenKind::Clone
384                | TokenKind::Continue
385                | TokenKind::Const
386                | TokenKind::Declare
387                | TokenKind::Default
388                | TokenKind::Do
389                | TokenKind::Echo
390                | TokenKind::ElseIf
391                | TokenKind::Else
392                | TokenKind::Enum
393                | TokenKind::Extends
394                | TokenKind::False
395                | TokenKind::Finally
396                | TokenKind::Final
397                | TokenKind::Fn
398                | TokenKind::Foreach
399                | TokenKind::For
400                | TokenKind::Function
401                | TokenKind::Goto
402                | TokenKind::If
403                | TokenKind::IncludeOnce
404                | TokenKind::Include
405                | TokenKind::Implements
406                | TokenKind::Interface
407                | TokenKind::Instanceof
408                | TokenKind::Namespace
409                | TokenKind::New
410                | TokenKind::Null
411                | TokenKind::Private
412                | TokenKind::PrivateSet
413                | TokenKind::Protected
414                | TokenKind::Public
415                | TokenKind::RequireOnce
416                | TokenKind::Require
417                | TokenKind::Return
418                | TokenKind::Static
419                | TokenKind::Switch
420                | TokenKind::Throw
421                | TokenKind::Trait
422                | TokenKind::True
423                | TokenKind::Try
424                | TokenKind::Use
425                | TokenKind::Var
426                | TokenKind::Yield
427                | TokenKind::While
428                | TokenKind::Insteadof
429                | TokenKind::List
430                | TokenKind::Self_
431                | TokenKind::Parent
432                | TokenKind::DirConstant
433                | TokenKind::FileConstant
434                | TokenKind::LineConstant
435                | TokenKind::FunctionConstant
436                | TokenKind::ClassConstant
437                | TokenKind::MethodConstant
438                | TokenKind::TraitConstant
439                | TokenKind::NamespaceConstant
440                | TokenKind::HaltCompiler
441        )
442    }
443
444    #[inline(always)]
445    pub const fn is_infix(&self) -> bool {
446        matches!(
447            self,
448            T!["**"
449                | ">>="
450                | "<<="
451                | "^="
452                | "&="
453                | "|="
454                | "%="
455                | "**="
456                | "and"
457                | "or"
458                | "xor"
459                | "<=>"
460                | "<<"
461                | ">>"
462                | "&"
463                | "|"
464                | "^"
465                | "%"
466                | "instanceof"
467                | "*"
468                | "/"
469                | "+"
470                | "-"
471                | "."
472                | "<"
473                | ">"
474                | "<="
475                | ">="
476                | "=="
477                | "==="
478                | "!="
479                | "!=="
480                | "<>"
481                | "?"
482                | "?:"
483                | "&&"
484                | "||"
485                | "="
486                | "+="
487                | "-="
488                | ".="
489                | "??="
490                | "/="
491                | "*="
492                | "??"]
493        )
494    }
495
496    #[inline(always)]
497    pub const fn is_postfix(&self) -> bool {
498        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
499    }
500
501    #[inline(always)]
502    pub const fn is_visibility_modifier(&self) -> bool {
503        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
504    }
505
506    #[inline(always)]
507    pub const fn is_modifier(&self) -> bool {
508        matches!(
509            self,
510            T!["public"
511                | "protected"
512                | "private"
513                | "private(set)"
514                | "protected(set)"
515                | "public(set)"
516                | "static"
517                | "final"
518                | "abstract"
519                | "readonly"]
520        )
521    }
522
523    #[inline(always)]
524    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
525        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
526    }
527
528    #[inline(always)]
529    pub const fn is_identifier_maybe_reserved(&self) -> bool {
530        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
531    }
532
533    #[inline(always)]
534    pub const fn is_soft_reserved_identifier(&self) -> bool {
535        matches!(
536            self,
537            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
538        )
539    }
540
541    #[inline(always)]
542    pub const fn is_reserved_identifier(&self) -> bool {
543        if self.is_soft_reserved_identifier() {
544            return true;
545        }
546
547        matches!(
548            self,
549            T!["static"
550                | "abstract"
551                | "final"
552                | "for"
553                | "private"
554                | "private(set)"
555                | "protected"
556                | "protected(set)"
557                | "public"
558                | "public(set)"
559                | "include"
560                | "include_once"
561                | "eval"
562                | "require"
563                | "require_once"
564                | "or"
565                | "xor"
566                | "and"
567                | "instanceof"
568                | "new"
569                | "clone"
570                | "exit"
571                | "die"
572                | "if"
573                | "elseif"
574                | "else"
575                | "endif"
576                | "echo"
577                | "do"
578                | "while"
579                | "endwhile"
580                | "endfor"
581                | "foreach"
582                | "endforeach"
583                | "declare"
584                | "enddeclare"
585                | "as"
586                | "try"
587                | "catch"
588                | "finally"
589                | "throw"
590                | "use"
591                | "insteadof"
592                | "global"
593                | "var"
594                | "unset"
595                | "isset"
596                | "empty"
597                | "continue"
598                | "goto"
599                | "function"
600                | "const"
601                | "return"
602                | "print"
603                | "yield"
604                | "list"
605                | "switch"
606                | "endswitch"
607                | "case"
608                | "default"
609                | "break"
610                | "array"
611                | "callable"
612                | "extends"
613                | "implements"
614                | "namespace"
615                | "trait"
616                | "interface"
617                | "class"
618                | "__CLASS__"
619                | "__TRAIT__"
620                | "__FUNCTION__"
621                | "__METHOD__"
622                | "__LINE__"
623                | "__FILE__"
624                | "__DIR__"
625                | "__NAMESPACE__"
626                | "__halt_compiler"
627                | "fn"
628                | "match"]
629        )
630    }
631
632    #[inline(always)]
633    pub const fn is_literal(&self) -> bool {
634        matches!(
635            self,
636            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
637        )
638    }
639
640    #[inline(always)]
641    pub const fn is_magic_constant(&self) -> bool {
642        matches!(
643            self,
644            T!["__CLASS__"
645                | "__DIR__"
646                | "__FILE__"
647                | "__FUNCTION__"
648                | "__LINE__"
649                | "__METHOD__"
650                | "__NAMESPACE__"
651                | "__TRAIT__"]
652        )
653    }
654
655    #[inline(always)]
656    pub const fn is_cast(&self) -> bool {
657        matches!(
658            self,
659            T!["(string)"
660                | "(binary)"
661                | "(int)"
662                | "(integer)"
663                | "(float)"
664                | "(double)"
665                | "(real)"
666                | "(bool)"
667                | "(boolean)"
668                | "(array)"
669                | "(object)"
670                | "(unset)"
671                | "(void)"]
672        )
673    }
674
675    #[inline(always)]
676    pub const fn is_unary_prefix(&self) -> bool {
677        if self.is_cast() {
678            return true;
679        }
680
681        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
682    }
683
684    #[inline(always)]
685    pub const fn is_trivia(&self) -> bool {
686        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
687    }
688
689    #[inline(always)]
690    pub const fn is_comment(&self) -> bool {
691        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
692    }
693
694    #[inline(always)]
695    pub const fn is_construct(&self) -> bool {
696        matches!(
697            self,
698            T!["isset"
699                | "empty"
700                | "eval"
701                | "include"
702                | "include_once"
703                | "require"
704                | "require_once"
705                | "print"
706                | "unset"
707                | "exit"
708                | "die"]
709        )
710    }
711}
712
713impl Token {
714    pub fn new(kind: TokenKind, value: StringIdentifier, span: Span) -> Self {
715        Self { kind, value, span }
716    }
717}
718
719impl std::fmt::Display for Token {
720    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
721        write!(f, "{:?} {}", self.kind, self.span)
722    }
723}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
