mago_syntax/token/
mod.rs

1use serde::Serialize;
2use strum::Display;
3
4use mago_span::Span;
5
6use crate::T;
7
8#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
9#[serde(tag = "type", content = "value")]
10pub enum DocumentKind {
11    Heredoc,
12    Nowdoc,
13}
14
15#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
16#[serde(tag = "type", content = "value")]
17pub enum Associativity {
18    NonAssociative,
19    Left,
20    Right,
21}
22
23#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
24#[serde(tag = "type", content = "value")]
25pub enum Precedence {
26    Lowest,
27    Print,
28    YieldFrom,
29    Yield,
30    KeyOr,
31    KeyXor,
32    KeyAnd,
33    Assignment,
34    ElvisOrConditional,
35    NullCoalesce,
36    Or,
37    And,
38    BitwiseOr,
39    BitwiseXor,
40    BitwiseAnd,
41    Equality,
42    Comparison,
43    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
44    // clearly, the current precedence position handles the examples shown in the RFC,
45    // but will need to be verified with the actual implementation once its merged into php-src.
46    //
47    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
48    // PR: https://github.com/php/php-src/pull/17118
49    Pipe,
50    Concat,
51    BitShift,
52    AddSub,
53    MulDivMod,
54    Unary,
55    Instanceof,
56    ErrorControl,
57    Pow,
58    Clone,
59    IncDec,
60    Reference,
61    CallDim,
62    New,
63    ArrayDim,
64    ObjectAccess,
65    Highest,
66}
67
68pub trait GetPrecedence {
69    fn precedence(&self) -> Precedence;
70}
71
72#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
73#[serde(tag = "type", content = "value")]
74pub enum TokenKind {
75    Whitespace,                  // ` `
76    Eval,                        // `eval`
77    Die,                         // `die`
78    Self_,                       // `self`
79    Parent,                      // `parent`
80    Backtick,                    // `` ` ``
81    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
82    DocumentEnd,                 // `abc`
83    From,                        // `from`
84    Print,                       // `print`
85    Dollar,                      // `$`
86    HaltCompiler,                // `__halt_compiler`
87    Readonly,                    // `readonly`
88    Global,                      // `global`
89    Abstract,                    // `abstract`
90    Ampersand,                   // `&`
91    AmpersandEqual,              // `&=`
92    AmpersandAmpersand,          // `&&`
93    AmpersandAmpersandEqual,     // `&&=`
94    Array,                       // `array`
95    ArrayCast,                   // `(array)`
96    MinusGreaterThan,            // `->`
97    QuestionMinusGreaterThan,    // `?->`
98    At,                          // `@`
99    As,                          // `as`
100    Asterisk,                    // `*`
101    HashLeftBracket,             // `#[`
102    Bang,                        // `!`
103    BangEqual,                   // `!=`
104    LessThanGreaterThan,         // `<>`
105    BangEqualEqual,              // `!==`
106    LessThanEqualGreaterThan,    // `<=>`
107    BoolCast,                    // `(bool)`
108    BooleanCast,                 // `(boolean)`
109    And,                         // `and`
110    Or,                          // `or`
111    Break,                       // `break`
112    Callable,                    // `callable`
113    Caret,                       // `^`
114    CaretEqual,                  // `^=`
115    Case,                        // `case`
116    Catch,                       // `catch`
117    Class,                       // `class`
118    ClassConstant,               // `__CLASS__`
119    TraitConstant,               // `__TRAIT__`
120    FunctionConstant,            // `__FUNCTION__`
121    MethodConstant,              // `__METHOD__`
122    LineConstant,                // `__LINE__`
123    FileConstant,                // `__FILE__`
124    Clone,                       // `clone`
125    MinusEqual,                  // `-=`
126    CloseTag,                    // `?>`
127    QuestionQuestion,            // `??`
128    QuestionQuestionEqual,       // `??=`
129    AsteriskEqual,               // `*=`
130    Colon,                       // `:`
131    Comma,                       // `,`
132    SingleLineComment,           // `// comment`
133    HashComment,                 // `# comment`
134    MultiLineComment,            // `/* comment */`
135    DocBlockComment,             // `/** comment */`
136    Const,                       // `const`
137    PartialLiteralString,        // `"string` or `'string`, missing closing quote
138    LiteralString,               // `"string"` or `'string'`
139    Continue,                    // `continue`
140    Declare,                     // `declare`
141    MinusMinus,                  // `--`
142    Default,                     // `default`
143    DirConstant,                 // `__DIR__`
144    SlashEqual,                  // `/=`
145    Do,                          // `do`
146    DollarLeftBrace,             // `${`
147    Dot,                         // `.`
148    DotEqual,                    // `.=`
149    EqualGreaterThan,            // `=>`
150    DoubleCast,                  // `(double)`
151    RealCast,                    // `(real)`
152    FloatCast,                   // `(float)`
153    ColonColon,                  // `::`
154    EqualEqual,                  // `==`
155    DoubleQuote,                 // `"`
156    Else,                        // `else`
157    Echo,                        // `echo`
158    DotDotDot,                   // `...`
159    ElseIf,                      // `elseif`
160    Empty,                       // `empty`
161    EndDeclare,                  // `enddeclare`
162    EndFor,                      // `endfor`
163    EndForeach,                  // `endforeach`
164    EndIf,                       // `endif`
165    EndSwitch,                   // `endswitch`
166    EndWhile,                    // `endwhile`
167    Enum,                        // `enum`
168    Equal,                       // `=`
169    Extends,                     // `extends`
170    False,                       // `false`
171    Final,                       // `final`
172    Finally,                     // `finally`
173    LiteralFloat,                // `1.0`
174    Fn,                          // `fn`
175    For,                         // `for`
176    Foreach,                     // `foreach`
177    FullyQualifiedIdentifier,    // `\Namespace\Class`
178    Function,                    // `function`
179    Goto,                        // `goto`
180    GreaterThan,                 // `>`
181    GreaterThanEqual,            // `>=`
182    Identifier,                  // `name`
183    If,                          // `if`
184    Implements,                  // `implements`
185    Include,                     // `include`
186    IncludeOnce,                 // `include_once`
187    PlusPlus,                    // `++`
188    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
189    InlineShebang,               // `#!...`
190    Instanceof,                  // `instanceof`
191    Insteadof,                   // `insteadof`
192    Exit,                        // `exit`
193    Unset,                       // `unset`
194    Isset,                       // `isset`
195    List,                        // `list`
196    LiteralInteger,              // `1`
197    IntCast,                     // `(int)`
198    IntegerCast,                 // `(integer)`
199    Interface,                   // `interface`
200    LeftBrace,                   // `{`
201    LeftBracket,                 // `[`
202    LeftParenthesis,             // `(`
203    LeftShift,                   // `<<`
204    LeftShiftEqual,              // `<<=`
205    RightShift,                  // `>>`
206    RightShiftEqual,             // `>>=`
207    LessThan,                    // `<`
208    LessThanEqual,               // `<=`
209    Match,                       // `match`
210    Minus,                       // `-`
211    Namespace,                   // `namespace`
212    NamespaceSeparator,          // `\`
213    NamespaceConstant,           // `__NAMESPACE__`
214    PropertyConstant,            // `__PROPERTY__`
215    New,                         // `new`
216    Null,                        // `null`
217    ObjectCast,                  // `(object)`
218    UnsetCast,                   // `(unset)`
219    OpenTag,                     // `<?php`
220    EchoTag,                     // `<?=`
221    ShortOpenTag,                // `<?`
222    Percent,                     // `%`
223    PercentEqual,                // `%=`
224    Pipe,                        // `|`
225    PipeEqual,                   // `|=`
226    Plus,                        // `+`
227    PlusEqual,                   // `+=`
228    AsteriskAsterisk,            // `**`
229    AsteriskAsteriskEqual,       // `**=`
230    Private,                     // `private`
231    PrivateSet,                  // `private(set)`
232    Protected,                   // `protected`
233    ProtectedSet,                // `protected(set)`
234    Public,                      // `public`
235    PublicSet,                   // `public(set)`
236    QualifiedIdentifier,         // `Namespace\Class`
237    Question,                    // `?`
238    Require,                     // `require`
239    RequireOnce,                 // `require_once`
240    Return,                      // `return`
241    RightBrace,                  // `}`
242    RightBracket,                // `]`
243    RightParenthesis,            // `)`
244    Semicolon,                   // `;`
245    Slash,                       // `/`
246    Static,                      // `static`
247    StringCast,                  // `(string)`
248    BinaryCast,                  // `(binary)`
249    VoidCast,                    // `(void)`
250    StringPart,                  // `string` inside a double-quoted string, or a document string
251    Switch,                      // `switch`
252    Throw,                       // `throw`
253    Trait,                       // `trait`
254    EqualEqualEqual,             // `===`
255    True,                        // `true`
256    Try,                         // `try`
257    Use,                         // `use`
258    Var,                         // `var`
259    Variable,                    // `$name`
260    Yield,                       // `yield`
261    While,                       // `while`
262    Tilde,                       // `~`
263    PipePipe,                    // `||`
264    Xor,                         // `xor`
265    PipeGreaterThan,             // `|>`
266}
267
268#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord)]
269pub struct Token<'arena> {
270    pub kind: TokenKind,
271    pub value: &'arena str,
272    pub span: Span,
273}
274
275impl Precedence {
276    #[inline]
277    #[must_use]
278    pub const fn infix(kind: &TokenKind) -> Precedence {
279        match kind {
280            T!["**"] => Precedence::Pow,
281            T!["instanceof"] => Precedence::Instanceof,
282            T!["*" | "/" | "%"] => Precedence::MulDivMod,
283            T!["+" | "-"] => Precedence::AddSub,
284            T!["<<"] | T![">>"] => Precedence::BitShift,
285            T!["."] => Precedence::Concat,
286            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
287            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
288            T!["&"] => Precedence::BitwiseAnd,
289            T!["^"] => Precedence::BitwiseXor,
290            T!["|"] => Precedence::BitwiseOr,
291            T!["&&"] => Precedence::And,
292            T!["||"] => Precedence::Or,
293            T!["??"] => Precedence::NullCoalesce,
294            T!["?"] => Precedence::ElvisOrConditional,
295            T!["="
296                | "+="
297                | "-="
298                | "*="
299                | "**="
300                | "/="
301                | ".="
302                | "&&="
303                | "??="
304                | "%="
305                | "&="
306                | "|="
307                | "^="
308                | "<<="
309                | ">>="] => Precedence::Assignment,
310            T!["yield"] => Precedence::Yield,
311            T!["and"] => Precedence::KeyAnd,
312            T!["or"] => Precedence::KeyOr,
313            T!["xor"] => Precedence::KeyXor,
314            T!["print"] => Precedence::Print,
315            T!["|>"] => Precedence::Pipe,
316            _ => Precedence::Lowest,
317        }
318    }
319
320    #[inline]
321    #[must_use]
322    pub const fn postfix(kind: &TokenKind) -> Self {
323        match kind {
324            T!["++" | "--"] => Self::IncDec,
325            T!["("] => Self::CallDim,
326            T!["["] => Self::ArrayDim,
327            T!["->" | "?->" | "::"] => Self::ObjectAccess,
328            _ => Self::Lowest,
329        }
330    }
331
332    #[inline]
333    #[must_use]
334    pub const fn associativity(&self) -> Option<Associativity> {
335        Some(match self {
336            Self::MulDivMod
337            | Self::AddSub
338            | Self::Concat
339            | Self::BitShift
340            | Self::BitwiseAnd
341            | Self::BitwiseOr
342            | Self::BitwiseXor
343            | Self::And
344            | Self::Or
345            | Self::KeyAnd
346            | Self::KeyXor
347            | Self::KeyOr
348            | Self::Pipe
349            | Self::ElvisOrConditional
350            | Self::ObjectAccess => Associativity::Left,
351            Self::Pow | Self::NullCoalesce | Self::Assignment | Self::Unary | Self::New => Associativity::Right,
352            Self::Equality | Self::Comparison | Self::Instanceof => Associativity::NonAssociative,
353            _ => return None,
354        })
355    }
356
357    #[inline]
358    #[must_use]
359    pub const fn is_associative(&self) -> bool {
360        self.associativity().is_some()
361    }
362
363    #[inline]
364    #[must_use]
365    pub const fn is_right_associative(&self) -> bool {
366        matches!(self.associativity(), Some(Associativity::Right))
367    }
368
369    #[inline]
370    #[must_use]
371    pub const fn is_left_associative(&self) -> bool {
372        matches!(self.associativity(), Some(Associativity::Left))
373    }
374
375    #[inline]
376    #[must_use]
377    pub const fn is_non_associative(&self) -> bool {
378        matches!(self.associativity(), Some(Associativity::NonAssociative))
379    }
380}
381
382impl TokenKind {
383    #[inline]
384    #[must_use]
385    pub const fn is_keyword(&self) -> bool {
386        matches!(
387            self,
388            TokenKind::Eval
389                | TokenKind::Die
390                | TokenKind::Empty
391                | TokenKind::Isset
392                | TokenKind::Unset
393                | TokenKind::Exit
394                | TokenKind::EndDeclare
395                | TokenKind::EndSwitch
396                | TokenKind::EndWhile
397                | TokenKind::EndForeach
398                | TokenKind::EndFor
399                | TokenKind::EndIf
400                | TokenKind::From
401                | TokenKind::And
402                | TokenKind::Or
403                | TokenKind::Xor
404                | TokenKind::Print
405                | TokenKind::Readonly
406                | TokenKind::Global
407                | TokenKind::Match
408                | TokenKind::Abstract
409                | TokenKind::Array
410                | TokenKind::As
411                | TokenKind::Break
412                | TokenKind::Case
413                | TokenKind::Catch
414                | TokenKind::Class
415                | TokenKind::Clone
416                | TokenKind::Continue
417                | TokenKind::Const
418                | TokenKind::Declare
419                | TokenKind::Default
420                | TokenKind::Do
421                | TokenKind::Echo
422                | TokenKind::ElseIf
423                | TokenKind::Else
424                | TokenKind::Enum
425                | TokenKind::Extends
426                | TokenKind::False
427                | TokenKind::Finally
428                | TokenKind::Final
429                | TokenKind::Fn
430                | TokenKind::Foreach
431                | TokenKind::For
432                | TokenKind::Function
433                | TokenKind::Goto
434                | TokenKind::If
435                | TokenKind::IncludeOnce
436                | TokenKind::Include
437                | TokenKind::Implements
438                | TokenKind::Interface
439                | TokenKind::Instanceof
440                | TokenKind::Namespace
441                | TokenKind::New
442                | TokenKind::Null
443                | TokenKind::Private
444                | TokenKind::PrivateSet
445                | TokenKind::Protected
446                | TokenKind::Public
447                | TokenKind::RequireOnce
448                | TokenKind::Require
449                | TokenKind::Return
450                | TokenKind::Static
451                | TokenKind::Switch
452                | TokenKind::Throw
453                | TokenKind::Trait
454                | TokenKind::True
455                | TokenKind::Try
456                | TokenKind::Use
457                | TokenKind::Var
458                | TokenKind::Yield
459                | TokenKind::While
460                | TokenKind::Insteadof
461                | TokenKind::List
462                | TokenKind::Self_
463                | TokenKind::Parent
464                | TokenKind::DirConstant
465                | TokenKind::FileConstant
466                | TokenKind::LineConstant
467                | TokenKind::FunctionConstant
468                | TokenKind::ClassConstant
469                | TokenKind::MethodConstant
470                | TokenKind::TraitConstant
471                | TokenKind::NamespaceConstant
472                | TokenKind::PropertyConstant
473                | TokenKind::HaltCompiler
474        )
475    }
476
477    #[inline]
478    #[must_use]
479    pub const fn is_infix(&self) -> bool {
480        matches!(
481            self,
482            T!["**"
483                | ">>="
484                | "<<="
485                | "^="
486                | "&="
487                | "|="
488                | "%="
489                | "**="
490                | "and"
491                | "or"
492                | "xor"
493                | "<=>"
494                | "<<"
495                | ">>"
496                | "&"
497                | "|"
498                | "^"
499                | "%"
500                | "instanceof"
501                | "*"
502                | "/"
503                | "+"
504                | "-"
505                | "."
506                | "<"
507                | ">"
508                | "<="
509                | ">="
510                | "=="
511                | "==="
512                | "!="
513                | "!=="
514                | "<>"
515                | "?"
516                | "&&"
517                | "||"
518                | "="
519                | "+="
520                | "-="
521                | ".="
522                | "??="
523                | "/="
524                | "*="
525                | "??"
526                | "|>"]
527        )
528    }
529
530    #[inline]
531    #[must_use]
532    pub const fn is_postfix(&self) -> bool {
533        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
534    }
535
536    #[inline]
537    #[must_use]
538    pub const fn is_visibility_modifier(&self) -> bool {
539        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
540    }
541
542    #[inline]
543    #[must_use]
544    pub const fn is_modifier(&self) -> bool {
545        matches!(
546            self,
547            T!["public"
548                | "protected"
549                | "private"
550                | "private(set)"
551                | "protected(set)"
552                | "public(set)"
553                | "static"
554                | "final"
555                | "abstract"
556                | "readonly"]
557        )
558    }
559
560    #[inline]
561    #[must_use]
562    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
563        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
564    }
565
566    #[inline]
567    #[must_use]
568    pub const fn is_identifier_maybe_reserved(&self) -> bool {
569        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
570    }
571
572    #[inline]
573    #[must_use]
574    pub const fn is_soft_reserved_identifier(&self) -> bool {
575        matches!(
576            self,
577            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
578        )
579    }
580
581    #[inline]
582    #[must_use]
583    pub const fn is_reserved_identifier(&self) -> bool {
584        if self.is_soft_reserved_identifier() {
585            return true;
586        }
587
588        matches!(
589            self,
590            T!["static"
591                | "abstract"
592                | "final"
593                | "for"
594                | "private"
595                | "private(set)"
596                | "protected"
597                | "protected(set)"
598                | "public"
599                | "public(set)"
600                | "include"
601                | "include_once"
602                | "eval"
603                | "require"
604                | "require_once"
605                | "or"
606                | "xor"
607                | "and"
608                | "instanceof"
609                | "new"
610                | "clone"
611                | "exit"
612                | "die"
613                | "if"
614                | "elseif"
615                | "else"
616                | "endif"
617                | "echo"
618                | "do"
619                | "while"
620                | "endwhile"
621                | "endfor"
622                | "foreach"
623                | "endforeach"
624                | "declare"
625                | "enddeclare"
626                | "as"
627                | "try"
628                | "catch"
629                | "finally"
630                | "throw"
631                | "use"
632                | "insteadof"
633                | "global"
634                | "var"
635                | "unset"
636                | "isset"
637                | "empty"
638                | "continue"
639                | "goto"
640                | "function"
641                | "const"
642                | "return"
643                | "print"
644                | "yield"
645                | "list"
646                | "switch"
647                | "endswitch"
648                | "case"
649                | "default"
650                | "break"
651                | "array"
652                | "callable"
653                | "extends"
654                | "implements"
655                | "namespace"
656                | "trait"
657                | "interface"
658                | "class"
659                | "__CLASS__"
660                | "__TRAIT__"
661                | "__FUNCTION__"
662                | "__METHOD__"
663                | "__LINE__"
664                | "__FILE__"
665                | "__DIR__"
666                | "__NAMESPACE__"
667                | "__PROPERTY__"
668                | "__halt_compiler"
669                | "fn"
670                | "match"]
671        )
672    }
673
674    #[inline]
675    #[must_use]
676    pub const fn is_literal(&self) -> bool {
677        matches!(
678            self,
679            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
680        )
681    }
682
683    #[inline]
684    #[must_use]
685    pub const fn is_magic_constant(&self) -> bool {
686        matches!(
687            self,
688            T!["__CLASS__"
689                | "__DIR__"
690                | "__FILE__"
691                | "__FUNCTION__"
692                | "__LINE__"
693                | "__METHOD__"
694                | "__NAMESPACE__"
695                | "__PROPERTY__"
696                | "__TRAIT__"]
697        )
698    }
699
700    #[inline]
701    #[must_use]
702    pub const fn is_cast(&self) -> bool {
703        matches!(
704            self,
705            T!["(string)"
706                | "(binary)"
707                | "(int)"
708                | "(integer)"
709                | "(float)"
710                | "(double)"
711                | "(real)"
712                | "(bool)"
713                | "(boolean)"
714                | "(array)"
715                | "(object)"
716                | "(unset)"
717                | "(void)"]
718        )
719    }
720
721    #[inline]
722    #[must_use]
723    pub const fn is_unary_prefix(&self) -> bool {
724        if self.is_cast() {
725            return true;
726        }
727
728        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
729    }
730
731    #[inline]
732    #[must_use]
733    pub const fn is_trivia(&self) -> bool {
734        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
735    }
736
737    #[inline]
738    #[must_use]
739    pub const fn is_comment(&self) -> bool {
740        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
741    }
742
743    #[inline]
744    #[must_use]
745    pub const fn is_comma(&self) -> bool {
746        matches!(self, T![","])
747    }
748
749    #[inline]
750    #[must_use]
751    pub const fn is_construct(&self) -> bool {
752        matches!(
753            self,
754            T!["isset"
755                | "empty"
756                | "eval"
757                | "include"
758                | "include_once"
759                | "require"
760                | "require_once"
761                | "print"
762                | "unset"
763                | "exit"
764                | "die"]
765        )
766    }
767}
768
769impl<'arena> Token<'arena> {
770    #[must_use]
771    pub const fn new(kind: TokenKind, value: &'arena str, span: Span) -> Self {
772        Self { kind, value, span }
773    }
774}
775
776impl std::fmt::Display for Token<'_> {
777    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
778        write!(f, "{}({})", self.kind, self.value)
779    }
780}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
