mago_syntax/token/
mod.rs

1use serde::Serialize;
2use strum::Display;
3
4use mago_span::Span;
5
6use crate::T;
7
8#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
9#[serde(tag = "type", content = "value")]
10pub enum DocumentKind {
11    Heredoc,
12    Nowdoc,
13}
14
15#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
16#[serde(tag = "type", content = "value")]
17pub enum Associativity {
18    NonAssociative,
19    Left,
20    Right,
21}
22
23#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
24#[serde(tag = "type", content = "value")]
25pub enum Precedence {
26    Lowest,
27    LowLogicalOr,
28    LowLogicalXor,
29    LowLogicalAnd,
30    Print,
31    Yield,
32    YieldFrom,
33    IncDec,
34    KeyOr,
35    KeyXor,
36    KeyAnd,
37    Assignment,
38    ElvisOrConditional,
39    NullCoalesce,
40    Or,
41    And,
42    BitwiseOr,
43    BitwiseXor,
44    BitwiseAnd,
45    Equality,
46    Comparison,
47    Concat,
48    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
49    // clearly, the current precedence position handles the examples shown in the RFC,
50    // but will need to be verified with the actual implementation once its merged into php-src.
51    //
52    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
53    // PR: https://github.com/php/php-src/pull/17118
54    Pipe,
55    BitShift,
56    AddSub,
57    MulDivMod,
58    Bang,
59    Instanceof,
60    Prefix,
61    Pow,
62    Clone,
63    CallDim,
64    New,
65    ArrayDim,
66    ObjectAccess,
67}
68
69pub trait GetPrecedence {
70    fn precedence(&self) -> Precedence;
71}
72
73#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
74#[serde(tag = "type", content = "value")]
75pub enum TokenKind {
76    Whitespace,                  // ` `
77    Eval,                        // `eval`
78    Die,                         // `die`
79    Self_,                       // `self`
80    Parent,                      // `parent`
81    Backtick,                    // `` ` ``
82    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
83    DocumentEnd,                 // `abc`
84    From,                        // `from`
85    Print,                       // `print`
86    Dollar,                      // `$`
87    HaltCompiler,                // `__halt_compiler`
88    Readonly,                    // `readonly`
89    Global,                      // `global`
90    Abstract,                    // `abstract`
91    Ampersand,                   // `&`
92    AmpersandEqual,              // `&=`
93    AmpersandAmpersand,          // `&&`
94    AmpersandAmpersandEqual,     // `&&=`
95    Array,                       // `array`
96    ArrayCast,                   // `(array)`
97    MinusGreaterThan,            // `->`
98    QuestionMinusGreaterThan,    // `?->`
99    At,                          // `@`
100    As,                          // `as`
101    Asterisk,                    // `*`
102    HashLeftBracket,             // `#[`
103    Bang,                        // `!`
104    BangEqual,                   // `!=`
105    LessThanGreaterThan,         // `<>`
106    BangEqualEqual,              // `!==`
107    LessThanEqualGreaterThan,    // `<=>`
108    BoolCast,                    // `(bool)`
109    BooleanCast,                 // `(boolean)`
110    And,                         // `and`
111    Or,                          // `or`
112    Break,                       // `break`
113    Callable,                    // `callable`
114    Caret,                       // `^`
115    CaretEqual,                  // `^=`
116    Case,                        // `case`
117    Catch,                       // `catch`
118    Class,                       // `class`
119    ClassConstant,               // `__CLASS__`
120    TraitConstant,               // `__TRAIT__`
121    FunctionConstant,            // `__FUNCTION__`
122    MethodConstant,              // `__METHOD__`
123    LineConstant,                // `__LINE__`
124    FileConstant,                // `__FILE__`
125    Clone,                       // `clone`
126    MinusEqual,                  // `-=`
127    CloseTag,                    // `?>`
128    QuestionQuestion,            // `??`
129    QuestionQuestionEqual,       // `??=`
130    AsteriskEqual,               // `*=`
131    Colon,                       // `:`
132    Comma,                       // `,`
133    SingleLineComment,           // `// comment`
134    HashComment,                 // `# comment`
135    MultiLineComment,            // `/* comment */`
136    DocBlockComment,             // `/** comment */`
137    Const,                       // `const`
138    PartialLiteralString,        // `"string` or `'string`, missing closing quote
139    LiteralString,               // `"string"` or `'string'`
140    Continue,                    // `continue`
141    Declare,                     // `declare`
142    MinusMinus,                  // `--`
143    Default,                     // `default`
144    DirConstant,                 // `__DIR__`
145    SlashEqual,                  // `/=`
146    Do,                          // `do`
147    DollarLeftBrace,             // `${`
148    Dot,                         // `.`
149    DotEqual,                    // `.=`
150    EqualGreaterThan,            // `=>`
151    DoubleCast,                  // `(double)`
152    RealCast,                    // `(real)`
153    FloatCast,                   // `(float)`
154    ColonColon,                  // `::`
155    EqualEqual,                  // `==`
156    DoubleQuote,                 // `"`
157    Else,                        // `else`
158    Echo,                        // `echo`
159    DotDotDot,                   // `...`
160    ElseIf,                      // `elseif`
161    Empty,                       // `empty`
162    EndDeclare,                  // `enddeclare`
163    EndFor,                      // `endfor`
164    EndForeach,                  // `endforeach`
165    EndIf,                       // `endif`
166    EndSwitch,                   // `endswitch`
167    EndWhile,                    // `endwhile`
168    Enum,                        // `enum`
169    Equal,                       // `=`
170    Extends,                     // `extends`
171    False,                       // `false`
172    Final,                       // `final`
173    Finally,                     // `finally`
174    LiteralFloat,                // `1.0`
175    Fn,                          // `fn`
176    For,                         // `for`
177    Foreach,                     // `foreach`
178    FullyQualifiedIdentifier,    // `\Namespace\Class`
179    Function,                    // `function`
180    Goto,                        // `goto`
181    GreaterThan,                 // `>`
182    GreaterThanEqual,            // `>=`
183    Identifier,                  // `name`
184    If,                          // `if`
185    Implements,                  // `implements`
186    Include,                     // `include`
187    IncludeOnce,                 // `include_once`
188    PlusPlus,                    // `++`
189    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
190    InlineShebang,               // `#!...`
191    Instanceof,                  // `instanceof`
192    Insteadof,                   // `insteadof`
193    Exit,                        // `exit`
194    Unset,                       // `unset`
195    Isset,                       // `isset`
196    List,                        // `list`
197    LiteralInteger,              // `1`
198    IntCast,                     // `(int)`
199    IntegerCast,                 // `(integer)`
200    Interface,                   // `interface`
201    LeftBrace,                   // `{`
202    LeftBracket,                 // `[`
203    LeftParenthesis,             // `(`
204    LeftShift,                   // `<<`
205    LeftShiftEqual,              // `<<=`
206    RightShift,                  // `>>`
207    RightShiftEqual,             // `>>=`
208    LessThan,                    // `<`
209    LessThanEqual,               // `<=`
210    Match,                       // `match`
211    Minus,                       // `-`
212    Namespace,                   // `namespace`
213    NamespaceSeparator,          // `\`
214    NamespaceConstant,           // `__NAMESPACE__`
215    New,                         // `new`
216    Null,                        // `null`
217    ObjectCast,                  // `(object)`
218    UnsetCast,                   // `(unset)`
219    OpenTag,                     // `<?php`
220    EchoTag,                     // `<?=`
221    ShortOpenTag,                // `<?`
222    Percent,                     // `%`
223    PercentEqual,                // `%=`
224    Pipe,                        // `|`
225    PipeEqual,                   // `|=`
226    Plus,                        // `+`
227    PlusEqual,                   // `+=`
228    AsteriskAsterisk,            // `**`
229    AsteriskAsteriskEqual,       // `**=`
230    Private,                     // `private`
231    PrivateSet,                  // `private(set)`
232    Protected,                   // `protected`
233    ProtectedSet,                // `protected(set)`
234    Public,                      // `public`
235    PublicSet,                   // `public(set)`
236    QualifiedIdentifier,         // `Namespace\Class`
237    Question,                    // `?`
238    QuestionColon,               // `?:`
239    Require,                     // `require`
240    RequireOnce,                 // `require_once`
241    Return,                      // `return`
242    RightBrace,                  // `}`
243    RightBracket,                // `]`
244    RightParenthesis,            // `)`
245    Semicolon,                   // `;`
246    Slash,                       // `/`
247    Static,                      // `static`
248    StringCast,                  // `(string)`
249    BinaryCast,                  // `(binary)`
250    VoidCast,                    // `(void)`
251    StringPart,                  // `string` inside a double-quoted string, or a document string
252    Switch,                      // `switch`
253    Throw,                       // `throw`
254    Trait,                       // `trait`
255    EqualEqualEqual,             // `===`
256    True,                        // `true`
257    Try,                         // `try`
258    Use,                         // `use`
259    Var,                         // `var`
260    Variable,                    // `$name`
261    Yield,                       // `yield`
262    While,                       // `while`
263    Tilde,                       // `~`
264    PipePipe,                    // `||`
265    Xor,                         // `xor`
266    PipeGreaterThan,             // `|>`
267}
268
269#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord)]
270pub struct Token<'arena> {
271    pub kind: TokenKind,
272    pub value: &'arena str,
273    pub span: Span,
274}
275
276impl Precedence {
277    #[inline]
278    pub const fn infix(kind: &TokenKind) -> Precedence {
279        match kind {
280            T!["**"] => Precedence::Pow,
281            T!["instanceof"] => Precedence::Instanceof,
282            T!["*" | "/" | "%"] => Precedence::MulDivMod,
283            T!["+" | "-"] => Precedence::AddSub,
284            T!["<<"] | T![">>"] => Precedence::BitShift,
285            T!["."] => Precedence::Concat,
286            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
287            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
288            T!["&"] => Precedence::BitwiseAnd,
289            T!["^"] => Precedence::BitwiseXor,
290            T!["|"] => Precedence::BitwiseOr,
291            T!["&&"] => Precedence::And,
292            T!["||"] => Precedence::Or,
293            T!["??"] => Precedence::NullCoalesce,
294            T!["?" | "?:"] => Precedence::ElvisOrConditional,
295            T!["="
296                | "+="
297                | "-="
298                | "*="
299                | "**="
300                | "/="
301                | ".="
302                | "&&="
303                | "??="
304                | "%="
305                | "&="
306                | "|="
307                | "^="
308                | "<<="
309                | ">>="] => Precedence::Assignment,
310            T!["yield"] => Precedence::Yield,
311            T!["and"] => Precedence::KeyAnd,
312            T!["or"] => Precedence::KeyOr,
313            T!["xor"] => Precedence::KeyXor,
314            T!["|>"] => Precedence::Pipe,
315            _ => Precedence::Lowest,
316        }
317    }
318
319    #[inline]
320    pub const fn postfix(kind: &TokenKind) -> Self {
321        match kind {
322            T!["++" | "--"] => Self::Prefix,
323            T!["("] => Self::CallDim,
324            T!["["] => Self::ArrayDim,
325            T!["->" | "?->" | "::"] => Self::ObjectAccess,
326            _ => Self::Lowest,
327        }
328    }
329
330    #[inline]
331    pub const fn associativity(&self) -> Option<Associativity> {
332        Some(match self {
333            Self::Instanceof
334            | Self::MulDivMod
335            | Self::AddSub
336            | Self::BitShift
337            | Self::Concat
338            | Self::BitwiseAnd
339            | Self::BitwiseOr
340            | Self::BitwiseXor
341            | Self::And
342            | Self::Or
343            | Self::KeyAnd
344            | Self::KeyOr
345            | Self::KeyXor
346            | Self::Pipe => Associativity::Left,
347            Self::Pow | Self::NullCoalesce | Self::Assignment => Associativity::Right,
348            Self::ElvisOrConditional | Self::Equality | Self::Comparison => Associativity::NonAssociative,
349            _ => return None,
350        })
351    }
352
353    #[inline]
354    pub const fn is_associative(&self) -> bool {
355        self.associativity().is_some()
356    }
357
358    #[inline]
359    pub const fn is_right_associative(&self) -> bool {
360        matches!(self.associativity(), Some(Associativity::Right))
361    }
362
363    #[inline]
364    pub const fn is_left_associative(&self) -> bool {
365        matches!(self.associativity(), Some(Associativity::Left))
366    }
367
368    #[inline]
369    pub const fn is_non_associative(&self) -> bool {
370        matches!(self.associativity(), Some(Associativity::NonAssociative))
371    }
372}
373
374impl TokenKind {
375    #[inline]
376    pub const fn is_keyword(&self) -> bool {
377        matches!(
378            self,
379            TokenKind::Eval
380                | TokenKind::Die
381                | TokenKind::Empty
382                | TokenKind::Isset
383                | TokenKind::Unset
384                | TokenKind::Exit
385                | TokenKind::EndDeclare
386                | TokenKind::EndSwitch
387                | TokenKind::EndWhile
388                | TokenKind::EndForeach
389                | TokenKind::EndFor
390                | TokenKind::EndIf
391                | TokenKind::From
392                | TokenKind::And
393                | TokenKind::Or
394                | TokenKind::Xor
395                | TokenKind::Print
396                | TokenKind::Readonly
397                | TokenKind::Global
398                | TokenKind::Match
399                | TokenKind::Abstract
400                | TokenKind::Array
401                | TokenKind::As
402                | TokenKind::Break
403                | TokenKind::Case
404                | TokenKind::Catch
405                | TokenKind::Class
406                | TokenKind::Clone
407                | TokenKind::Continue
408                | TokenKind::Const
409                | TokenKind::Declare
410                | TokenKind::Default
411                | TokenKind::Do
412                | TokenKind::Echo
413                | TokenKind::ElseIf
414                | TokenKind::Else
415                | TokenKind::Enum
416                | TokenKind::Extends
417                | TokenKind::False
418                | TokenKind::Finally
419                | TokenKind::Final
420                | TokenKind::Fn
421                | TokenKind::Foreach
422                | TokenKind::For
423                | TokenKind::Function
424                | TokenKind::Goto
425                | TokenKind::If
426                | TokenKind::IncludeOnce
427                | TokenKind::Include
428                | TokenKind::Implements
429                | TokenKind::Interface
430                | TokenKind::Instanceof
431                | TokenKind::Namespace
432                | TokenKind::New
433                | TokenKind::Null
434                | TokenKind::Private
435                | TokenKind::PrivateSet
436                | TokenKind::Protected
437                | TokenKind::Public
438                | TokenKind::RequireOnce
439                | TokenKind::Require
440                | TokenKind::Return
441                | TokenKind::Static
442                | TokenKind::Switch
443                | TokenKind::Throw
444                | TokenKind::Trait
445                | TokenKind::True
446                | TokenKind::Try
447                | TokenKind::Use
448                | TokenKind::Var
449                | TokenKind::Yield
450                | TokenKind::While
451                | TokenKind::Insteadof
452                | TokenKind::List
453                | TokenKind::Self_
454                | TokenKind::Parent
455                | TokenKind::DirConstant
456                | TokenKind::FileConstant
457                | TokenKind::LineConstant
458                | TokenKind::FunctionConstant
459                | TokenKind::ClassConstant
460                | TokenKind::MethodConstant
461                | TokenKind::TraitConstant
462                | TokenKind::NamespaceConstant
463                | TokenKind::HaltCompiler
464        )
465    }
466
467    #[inline]
468    pub const fn is_infix(&self) -> bool {
469        matches!(
470            self,
471            T!["**"
472                | ">>="
473                | "<<="
474                | "^="
475                | "&="
476                | "|="
477                | "%="
478                | "**="
479                | "and"
480                | "or"
481                | "xor"
482                | "<=>"
483                | "<<"
484                | ">>"
485                | "&"
486                | "|"
487                | "^"
488                | "%"
489                | "instanceof"
490                | "*"
491                | "/"
492                | "+"
493                | "-"
494                | "."
495                | "<"
496                | ">"
497                | "<="
498                | ">="
499                | "=="
500                | "==="
501                | "!="
502                | "!=="
503                | "<>"
504                | "?"
505                | "?:"
506                | "&&"
507                | "||"
508                | "="
509                | "+="
510                | "-="
511                | ".="
512                | "??="
513                | "/="
514                | "*="
515                | "??"
516                | "|>"]
517        )
518    }
519
520    #[inline]
521    pub const fn is_postfix(&self) -> bool {
522        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
523    }
524
525    #[inline]
526    pub const fn is_visibility_modifier(&self) -> bool {
527        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
528    }
529
530    #[inline]
531    pub const fn is_modifier(&self) -> bool {
532        matches!(
533            self,
534            T!["public"
535                | "protected"
536                | "private"
537                | "private(set)"
538                | "protected(set)"
539                | "public(set)"
540                | "static"
541                | "final"
542                | "abstract"
543                | "readonly"]
544        )
545    }
546
547    #[inline]
548    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
549        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
550    }
551
552    #[inline]
553    pub const fn is_identifier_maybe_reserved(&self) -> bool {
554        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
555    }
556
557    #[inline]
558    pub const fn is_soft_reserved_identifier(&self) -> bool {
559        matches!(
560            self,
561            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
562        )
563    }
564
565    #[inline]
566    pub const fn is_reserved_identifier(&self) -> bool {
567        if self.is_soft_reserved_identifier() {
568            return true;
569        }
570
571        matches!(
572            self,
573            T!["static"
574                | "abstract"
575                | "final"
576                | "for"
577                | "private"
578                | "private(set)"
579                | "protected"
580                | "protected(set)"
581                | "public"
582                | "public(set)"
583                | "include"
584                | "include_once"
585                | "eval"
586                | "require"
587                | "require_once"
588                | "or"
589                | "xor"
590                | "and"
591                | "instanceof"
592                | "new"
593                | "clone"
594                | "exit"
595                | "die"
596                | "if"
597                | "elseif"
598                | "else"
599                | "endif"
600                | "echo"
601                | "do"
602                | "while"
603                | "endwhile"
604                | "endfor"
605                | "foreach"
606                | "endforeach"
607                | "declare"
608                | "enddeclare"
609                | "as"
610                | "try"
611                | "catch"
612                | "finally"
613                | "throw"
614                | "use"
615                | "insteadof"
616                | "global"
617                | "var"
618                | "unset"
619                | "isset"
620                | "empty"
621                | "continue"
622                | "goto"
623                | "function"
624                | "const"
625                | "return"
626                | "print"
627                | "yield"
628                | "list"
629                | "switch"
630                | "endswitch"
631                | "case"
632                | "default"
633                | "break"
634                | "array"
635                | "callable"
636                | "extends"
637                | "implements"
638                | "namespace"
639                | "trait"
640                | "interface"
641                | "class"
642                | "__CLASS__"
643                | "__TRAIT__"
644                | "__FUNCTION__"
645                | "__METHOD__"
646                | "__LINE__"
647                | "__FILE__"
648                | "__DIR__"
649                | "__NAMESPACE__"
650                | "__halt_compiler"
651                | "fn"
652                | "match"]
653        )
654    }
655
656    #[inline]
657    pub const fn is_literal(&self) -> bool {
658        matches!(
659            self,
660            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
661        )
662    }
663
664    #[inline]
665    pub const fn is_magic_constant(&self) -> bool {
666        matches!(
667            self,
668            T!["__CLASS__"
669                | "__DIR__"
670                | "__FILE__"
671                | "__FUNCTION__"
672                | "__LINE__"
673                | "__METHOD__"
674                | "__NAMESPACE__"
675                | "__TRAIT__"]
676        )
677    }
678
679    #[inline]
680    pub const fn is_cast(&self) -> bool {
681        matches!(
682            self,
683            T!["(string)"
684                | "(binary)"
685                | "(int)"
686                | "(integer)"
687                | "(float)"
688                | "(double)"
689                | "(real)"
690                | "(bool)"
691                | "(boolean)"
692                | "(array)"
693                | "(object)"
694                | "(unset)"
695                | "(void)"]
696        )
697    }
698
699    #[inline]
700    pub const fn is_unary_prefix(&self) -> bool {
701        if self.is_cast() {
702            return true;
703        }
704
705        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
706    }
707
708    #[inline]
709    pub const fn is_trivia(&self) -> bool {
710        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
711    }
712
713    #[inline]
714    pub const fn is_comment(&self) -> bool {
715        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
716    }
717
718    #[inline]
719    pub const fn is_comma(&self) -> bool {
720        matches!(self, T![","])
721    }
722
723    #[inline]
724    pub const fn is_construct(&self) -> bool {
725        matches!(
726            self,
727            T!["isset"
728                | "empty"
729                | "eval"
730                | "include"
731                | "include_once"
732                | "require"
733                | "require_once"
734                | "print"
735                | "unset"
736                | "exit"
737                | "die"]
738        )
739    }
740}
741
742impl<'arena> Token<'arena> {
743    pub const fn new(kind: TokenKind, value: &'arena str, span: Span) -> Self {
744        Self { kind, value, span }
745    }
746}
747
748impl<'arena> std::fmt::Display for Token<'arena> {
749    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
750        write!(f, "{}({})", self.kind, self.value)
751    }
752}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
