mago_syntax/token/
mod.rs

1use serde::Serialize;
2use strum::Display;
3
4use mago_span::Span;
5
6use crate::T;
7
8#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
9#[serde(tag = "type", content = "value")]
10pub enum DocumentKind {
11    Heredoc,
12    Nowdoc,
13}
14
15#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
16#[serde(tag = "type", content = "value")]
17pub enum Associativity {
18    NonAssociative,
19    Left,
20    Right,
21}
22
23#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
24#[serde(tag = "type", content = "value")]
25pub enum Precedence {
26    Lowest,
27    Print,
28    YieldFrom,
29    Yield,
30    KeyOr,
31    KeyXor,
32    KeyAnd,
33    Assignment,
34    ElvisOrConditional,
35    NullCoalesce,
36    Or,
37    And,
38    BitwiseOr,
39    BitwiseXor,
40    BitwiseAnd,
41    Equality,
42    Comparison,
43    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
44    // clearly, the current precedence position handles the examples shown in the RFC,
45    // but will need to be verified with the actual implementation once its merged into php-src.
46    //
47    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
48    // PR: https://github.com/php/php-src/pull/17118
49    Pipe,
50    Concat,
51    BitShift,
52    AddSub,
53    MulDivMod,
54    Unary,
55    Instanceof,
56    ErrorControl,
57    Pow,
58    Clone,
59    IncDec,
60    CallDim,
61    New,
62    ArrayDim,
63    ObjectAccess,
64    Highest,
65}
66
67pub trait GetPrecedence {
68    fn precedence(&self) -> Precedence;
69}
70
71#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
72#[serde(tag = "type", content = "value")]
73pub enum TokenKind {
74    Whitespace,                  // ` `
75    Eval,                        // `eval`
76    Die,                         // `die`
77    Self_,                       // `self`
78    Parent,                      // `parent`
79    Backtick,                    // `` ` ``
80    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
81    DocumentEnd,                 // `abc`
82    From,                        // `from`
83    Print,                       // `print`
84    Dollar,                      // `$`
85    HaltCompiler,                // `__halt_compiler`
86    Readonly,                    // `readonly`
87    Global,                      // `global`
88    Abstract,                    // `abstract`
89    Ampersand,                   // `&`
90    AmpersandEqual,              // `&=`
91    AmpersandAmpersand,          // `&&`
92    AmpersandAmpersandEqual,     // `&&=`
93    Array,                       // `array`
94    ArrayCast,                   // `(array)`
95    MinusGreaterThan,            // `->`
96    QuestionMinusGreaterThan,    // `?->`
97    At,                          // `@`
98    As,                          // `as`
99    Asterisk,                    // `*`
100    HashLeftBracket,             // `#[`
101    Bang,                        // `!`
102    BangEqual,                   // `!=`
103    LessThanGreaterThan,         // `<>`
104    BangEqualEqual,              // `!==`
105    LessThanEqualGreaterThan,    // `<=>`
106    BoolCast,                    // `(bool)`
107    BooleanCast,                 // `(boolean)`
108    And,                         // `and`
109    Or,                          // `or`
110    Break,                       // `break`
111    Callable,                    // `callable`
112    Caret,                       // `^`
113    CaretEqual,                  // `^=`
114    Case,                        // `case`
115    Catch,                       // `catch`
116    Class,                       // `class`
117    ClassConstant,               // `__CLASS__`
118    TraitConstant,               // `__TRAIT__`
119    FunctionConstant,            // `__FUNCTION__`
120    MethodConstant,              // `__METHOD__`
121    LineConstant,                // `__LINE__`
122    FileConstant,                // `__FILE__`
123    Clone,                       // `clone`
124    MinusEqual,                  // `-=`
125    CloseTag,                    // `?>`
126    QuestionQuestion,            // `??`
127    QuestionQuestionEqual,       // `??=`
128    AsteriskEqual,               // `*=`
129    Colon,                       // `:`
130    Comma,                       // `,`
131    SingleLineComment,           // `// comment`
132    HashComment,                 // `# comment`
133    MultiLineComment,            // `/* comment */`
134    DocBlockComment,             // `/** comment */`
135    Const,                       // `const`
136    PartialLiteralString,        // `"string` or `'string`, missing closing quote
137    LiteralString,               // `"string"` or `'string'`
138    Continue,                    // `continue`
139    Declare,                     // `declare`
140    MinusMinus,                  // `--`
141    Default,                     // `default`
142    DirConstant,                 // `__DIR__`
143    SlashEqual,                  // `/=`
144    Do,                          // `do`
145    DollarLeftBrace,             // `${`
146    Dot,                         // `.`
147    DotEqual,                    // `.=`
148    EqualGreaterThan,            // `=>`
149    DoubleCast,                  // `(double)`
150    RealCast,                    // `(real)`
151    FloatCast,                   // `(float)`
152    ColonColon,                  // `::`
153    EqualEqual,                  // `==`
154    DoubleQuote,                 // `"`
155    Else,                        // `else`
156    Echo,                        // `echo`
157    DotDotDot,                   // `...`
158    ElseIf,                      // `elseif`
159    Empty,                       // `empty`
160    EndDeclare,                  // `enddeclare`
161    EndFor,                      // `endfor`
162    EndForeach,                  // `endforeach`
163    EndIf,                       // `endif`
164    EndSwitch,                   // `endswitch`
165    EndWhile,                    // `endwhile`
166    Enum,                        // `enum`
167    Equal,                       // `=`
168    Extends,                     // `extends`
169    False,                       // `false`
170    Final,                       // `final`
171    Finally,                     // `finally`
172    LiteralFloat,                // `1.0`
173    Fn,                          // `fn`
174    For,                         // `for`
175    Foreach,                     // `foreach`
176    FullyQualifiedIdentifier,    // `\Namespace\Class`
177    Function,                    // `function`
178    Goto,                        // `goto`
179    GreaterThan,                 // `>`
180    GreaterThanEqual,            // `>=`
181    Identifier,                  // `name`
182    If,                          // `if`
183    Implements,                  // `implements`
184    Include,                     // `include`
185    IncludeOnce,                 // `include_once`
186    PlusPlus,                    // `++`
187    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
188    InlineShebang,               // `#!...`
189    Instanceof,                  // `instanceof`
190    Insteadof,                   // `insteadof`
191    Exit,                        // `exit`
192    Unset,                       // `unset`
193    Isset,                       // `isset`
194    List,                        // `list`
195    LiteralInteger,              // `1`
196    IntCast,                     // `(int)`
197    IntegerCast,                 // `(integer)`
198    Interface,                   // `interface`
199    LeftBrace,                   // `{`
200    LeftBracket,                 // `[`
201    LeftParenthesis,             // `(`
202    LeftShift,                   // `<<`
203    LeftShiftEqual,              // `<<=`
204    RightShift,                  // `>>`
205    RightShiftEqual,             // `>>=`
206    LessThan,                    // `<`
207    LessThanEqual,               // `<=`
208    Match,                       // `match`
209    Minus,                       // `-`
210    Namespace,                   // `namespace`
211    NamespaceSeparator,          // `\`
212    NamespaceConstant,           // `__NAMESPACE__`
213    New,                         // `new`
214    Null,                        // `null`
215    ObjectCast,                  // `(object)`
216    UnsetCast,                   // `(unset)`
217    OpenTag,                     // `<?php`
218    EchoTag,                     // `<?=`
219    ShortOpenTag,                // `<?`
220    Percent,                     // `%`
221    PercentEqual,                // `%=`
222    Pipe,                        // `|`
223    PipeEqual,                   // `|=`
224    Plus,                        // `+`
225    PlusEqual,                   // `+=`
226    AsteriskAsterisk,            // `**`
227    AsteriskAsteriskEqual,       // `**=`
228    Private,                     // `private`
229    PrivateSet,                  // `private(set)`
230    Protected,                   // `protected`
231    ProtectedSet,                // `protected(set)`
232    Public,                      // `public`
233    PublicSet,                   // `public(set)`
234    QualifiedIdentifier,         // `Namespace\Class`
235    Question,                    // `?`
236    Require,                     // `require`
237    RequireOnce,                 // `require_once`
238    Return,                      // `return`
239    RightBrace,                  // `}`
240    RightBracket,                // `]`
241    RightParenthesis,            // `)`
242    Semicolon,                   // `;`
243    Slash,                       // `/`
244    Static,                      // `static`
245    StringCast,                  // `(string)`
246    BinaryCast,                  // `(binary)`
247    VoidCast,                    // `(void)`
248    StringPart,                  // `string` inside a double-quoted string, or a document string
249    Switch,                      // `switch`
250    Throw,                       // `throw`
251    Trait,                       // `trait`
252    EqualEqualEqual,             // `===`
253    True,                        // `true`
254    Try,                         // `try`
255    Use,                         // `use`
256    Var,                         // `var`
257    Variable,                    // `$name`
258    Yield,                       // `yield`
259    While,                       // `while`
260    Tilde,                       // `~`
261    PipePipe,                    // `||`
262    Xor,                         // `xor`
263    PipeGreaterThan,             // `|>`
264}
265
266#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord)]
267pub struct Token<'arena> {
268    pub kind: TokenKind,
269    pub value: &'arena str,
270    pub span: Span,
271}
272
273impl Precedence {
274    #[inline]
275    pub const fn infix(kind: &TokenKind) -> Precedence {
276        match kind {
277            T!["**"] => Precedence::Pow,
278            T!["instanceof"] => Precedence::Instanceof,
279            T!["*" | "/" | "%"] => Precedence::MulDivMod,
280            T!["+" | "-"] => Precedence::AddSub,
281            T!["<<"] | T![">>"] => Precedence::BitShift,
282            T!["."] => Precedence::Concat,
283            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
284            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
285            T!["&"] => Precedence::BitwiseAnd,
286            T!["^"] => Precedence::BitwiseXor,
287            T!["|"] => Precedence::BitwiseOr,
288            T!["&&"] => Precedence::And,
289            T!["||"] => Precedence::Or,
290            T!["??"] => Precedence::NullCoalesce,
291            T!["?"] => Precedence::ElvisOrConditional,
292            T!["="
293                | "+="
294                | "-="
295                | "*="
296                | "**="
297                | "/="
298                | ".="
299                | "&&="
300                | "??="
301                | "%="
302                | "&="
303                | "|="
304                | "^="
305                | "<<="
306                | ">>="] => Precedence::Assignment,
307            T!["yield"] => Precedence::Yield,
308            T!["and"] => Precedence::KeyAnd,
309            T!["or"] => Precedence::KeyOr,
310            T!["xor"] => Precedence::KeyXor,
311            T!["print"] => Precedence::Print,
312            T!["|>"] => Precedence::Pipe,
313            _ => Precedence::Lowest,
314        }
315    }
316
317    #[inline]
318    pub const fn postfix(kind: &TokenKind) -> Self {
319        match kind {
320            T!["++" | "--"] => Self::IncDec,
321            T!["("] => Self::CallDim,
322            T!["["] => Self::ArrayDim,
323            T!["->" | "?->" | "::"] => Self::ObjectAccess,
324            _ => Self::Lowest,
325        }
326    }
327
328    #[inline]
329    pub const fn associativity(&self) -> Option<Associativity> {
330        Some(match self {
331            Self::MulDivMod
332            | Self::AddSub
333            | Self::Concat
334            | Self::BitShift
335            | Self::BitwiseAnd
336            | Self::BitwiseOr
337            | Self::BitwiseXor
338            | Self::And
339            | Self::Or
340            | Self::KeyAnd
341            | Self::KeyXor
342            | Self::KeyOr
343            | Self::Pipe
344            | Self::ElvisOrConditional
345            | Self::ObjectAccess => Associativity::Left,
346            Self::Pow | Self::NullCoalesce | Self::Assignment | Self::Unary | Self::New => Associativity::Right,
347            Self::Equality | Self::Comparison | Self::Instanceof => Associativity::NonAssociative,
348            _ => return None,
349        })
350    }
351
352    #[inline]
353    pub const fn is_associative(&self) -> bool {
354        self.associativity().is_some()
355    }
356
357    #[inline]
358    pub const fn is_right_associative(&self) -> bool {
359        matches!(self.associativity(), Some(Associativity::Right))
360    }
361
362    #[inline]
363    pub const fn is_left_associative(&self) -> bool {
364        matches!(self.associativity(), Some(Associativity::Left))
365    }
366
367    #[inline]
368    pub const fn is_non_associative(&self) -> bool {
369        matches!(self.associativity(), Some(Associativity::NonAssociative))
370    }
371}
372
373impl TokenKind {
374    #[inline]
375    pub const fn is_keyword(&self) -> bool {
376        matches!(
377            self,
378            TokenKind::Eval
379                | TokenKind::Die
380                | TokenKind::Empty
381                | TokenKind::Isset
382                | TokenKind::Unset
383                | TokenKind::Exit
384                | TokenKind::EndDeclare
385                | TokenKind::EndSwitch
386                | TokenKind::EndWhile
387                | TokenKind::EndForeach
388                | TokenKind::EndFor
389                | TokenKind::EndIf
390                | TokenKind::From
391                | TokenKind::And
392                | TokenKind::Or
393                | TokenKind::Xor
394                | TokenKind::Print
395                | TokenKind::Readonly
396                | TokenKind::Global
397                | TokenKind::Match
398                | TokenKind::Abstract
399                | TokenKind::Array
400                | TokenKind::As
401                | TokenKind::Break
402                | TokenKind::Case
403                | TokenKind::Catch
404                | TokenKind::Class
405                | TokenKind::Clone
406                | TokenKind::Continue
407                | TokenKind::Const
408                | TokenKind::Declare
409                | TokenKind::Default
410                | TokenKind::Do
411                | TokenKind::Echo
412                | TokenKind::ElseIf
413                | TokenKind::Else
414                | TokenKind::Enum
415                | TokenKind::Extends
416                | TokenKind::False
417                | TokenKind::Finally
418                | TokenKind::Final
419                | TokenKind::Fn
420                | TokenKind::Foreach
421                | TokenKind::For
422                | TokenKind::Function
423                | TokenKind::Goto
424                | TokenKind::If
425                | TokenKind::IncludeOnce
426                | TokenKind::Include
427                | TokenKind::Implements
428                | TokenKind::Interface
429                | TokenKind::Instanceof
430                | TokenKind::Namespace
431                | TokenKind::New
432                | TokenKind::Null
433                | TokenKind::Private
434                | TokenKind::PrivateSet
435                | TokenKind::Protected
436                | TokenKind::Public
437                | TokenKind::RequireOnce
438                | TokenKind::Require
439                | TokenKind::Return
440                | TokenKind::Static
441                | TokenKind::Switch
442                | TokenKind::Throw
443                | TokenKind::Trait
444                | TokenKind::True
445                | TokenKind::Try
446                | TokenKind::Use
447                | TokenKind::Var
448                | TokenKind::Yield
449                | TokenKind::While
450                | TokenKind::Insteadof
451                | TokenKind::List
452                | TokenKind::Self_
453                | TokenKind::Parent
454                | TokenKind::DirConstant
455                | TokenKind::FileConstant
456                | TokenKind::LineConstant
457                | TokenKind::FunctionConstant
458                | TokenKind::ClassConstant
459                | TokenKind::MethodConstant
460                | TokenKind::TraitConstant
461                | TokenKind::NamespaceConstant
462                | TokenKind::HaltCompiler
463        )
464    }
465
466    #[inline]
467    pub const fn is_infix(&self) -> bool {
468        matches!(
469            self,
470            T!["**"
471                | ">>="
472                | "<<="
473                | "^="
474                | "&="
475                | "|="
476                | "%="
477                | "**="
478                | "and"
479                | "or"
480                | "xor"
481                | "<=>"
482                | "<<"
483                | ">>"
484                | "&"
485                | "|"
486                | "^"
487                | "%"
488                | "instanceof"
489                | "*"
490                | "/"
491                | "+"
492                | "-"
493                | "."
494                | "<"
495                | ">"
496                | "<="
497                | ">="
498                | "=="
499                | "==="
500                | "!="
501                | "!=="
502                | "<>"
503                | "?"
504                | "&&"
505                | "||"
506                | "="
507                | "+="
508                | "-="
509                | ".="
510                | "??="
511                | "/="
512                | "*="
513                | "??"
514                | "|>"]
515        )
516    }
517
518    #[inline]
519    pub const fn is_postfix(&self) -> bool {
520        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
521    }
522
523    #[inline]
524    pub const fn is_visibility_modifier(&self) -> bool {
525        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
526    }
527
528    #[inline]
529    pub const fn is_modifier(&self) -> bool {
530        matches!(
531            self,
532            T!["public"
533                | "protected"
534                | "private"
535                | "private(set)"
536                | "protected(set)"
537                | "public(set)"
538                | "static"
539                | "final"
540                | "abstract"
541                | "readonly"]
542        )
543    }
544
545    #[inline]
546    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
547        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
548    }
549
550    #[inline]
551    pub const fn is_identifier_maybe_reserved(&self) -> bool {
552        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
553    }
554
555    #[inline]
556    pub const fn is_soft_reserved_identifier(&self) -> bool {
557        matches!(
558            self,
559            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
560        )
561    }
562
563    #[inline]
564    pub const fn is_reserved_identifier(&self) -> bool {
565        if self.is_soft_reserved_identifier() {
566            return true;
567        }
568
569        matches!(
570            self,
571            T!["static"
572                | "abstract"
573                | "final"
574                | "for"
575                | "private"
576                | "private(set)"
577                | "protected"
578                | "protected(set)"
579                | "public"
580                | "public(set)"
581                | "include"
582                | "include_once"
583                | "eval"
584                | "require"
585                | "require_once"
586                | "or"
587                | "xor"
588                | "and"
589                | "instanceof"
590                | "new"
591                | "clone"
592                | "exit"
593                | "die"
594                | "if"
595                | "elseif"
596                | "else"
597                | "endif"
598                | "echo"
599                | "do"
600                | "while"
601                | "endwhile"
602                | "endfor"
603                | "foreach"
604                | "endforeach"
605                | "declare"
606                | "enddeclare"
607                | "as"
608                | "try"
609                | "catch"
610                | "finally"
611                | "throw"
612                | "use"
613                | "insteadof"
614                | "global"
615                | "var"
616                | "unset"
617                | "isset"
618                | "empty"
619                | "continue"
620                | "goto"
621                | "function"
622                | "const"
623                | "return"
624                | "print"
625                | "yield"
626                | "list"
627                | "switch"
628                | "endswitch"
629                | "case"
630                | "default"
631                | "break"
632                | "array"
633                | "callable"
634                | "extends"
635                | "implements"
636                | "namespace"
637                | "trait"
638                | "interface"
639                | "class"
640                | "__CLASS__"
641                | "__TRAIT__"
642                | "__FUNCTION__"
643                | "__METHOD__"
644                | "__LINE__"
645                | "__FILE__"
646                | "__DIR__"
647                | "__NAMESPACE__"
648                | "__halt_compiler"
649                | "fn"
650                | "match"]
651        )
652    }
653
654    #[inline]
655    pub const fn is_literal(&self) -> bool {
656        matches!(
657            self,
658            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
659        )
660    }
661
662    #[inline]
663    pub const fn is_magic_constant(&self) -> bool {
664        matches!(
665            self,
666            T!["__CLASS__"
667                | "__DIR__"
668                | "__FILE__"
669                | "__FUNCTION__"
670                | "__LINE__"
671                | "__METHOD__"
672                | "__NAMESPACE__"
673                | "__TRAIT__"]
674        )
675    }
676
677    #[inline]
678    pub const fn is_cast(&self) -> bool {
679        matches!(
680            self,
681            T!["(string)"
682                | "(binary)"
683                | "(int)"
684                | "(integer)"
685                | "(float)"
686                | "(double)"
687                | "(real)"
688                | "(bool)"
689                | "(boolean)"
690                | "(array)"
691                | "(object)"
692                | "(unset)"
693                | "(void)"]
694        )
695    }
696
697    #[inline]
698    pub const fn is_unary_prefix(&self) -> bool {
699        if self.is_cast() {
700            return true;
701        }
702
703        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
704    }
705
706    #[inline]
707    pub const fn is_trivia(&self) -> bool {
708        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
709    }
710
711    #[inline]
712    pub const fn is_comment(&self) -> bool {
713        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
714    }
715
716    #[inline]
717    pub const fn is_comma(&self) -> bool {
718        matches!(self, T![","])
719    }
720
721    #[inline]
722    pub const fn is_construct(&self) -> bool {
723        matches!(
724            self,
725            T!["isset"
726                | "empty"
727                | "eval"
728                | "include"
729                | "include_once"
730                | "require"
731                | "require_once"
732                | "print"
733                | "unset"
734                | "exit"
735                | "die"]
736        )
737    }
738}
739
740impl<'arena> Token<'arena> {
741    pub const fn new(kind: TokenKind, value: &'arena str, span: Span) -> Self {
742        Self { kind, value, span }
743    }
744}
745
746impl<'arena> std::fmt::Display for Token<'arena> {
747    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
748        write!(f, "{}({})", self.kind, self.value)
749    }
750}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
