mago_syntax/token/
mod.rs

1use serde::Deserialize;
2use serde::Serialize;
3use strum::Display;
4
5use mago_interner::StringIdentifier;
6use mago_span::Span;
7
8use crate::T;
9
10#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
11#[serde(tag = "type", content = "value")]
12#[repr(C)]
13pub enum DocumentKind {
14    Heredoc,
15    Nowdoc,
16}
17
18#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
19#[serde(tag = "type", content = "value")]
20#[repr(C)]
21pub enum Associativity {
22    NonAssociative,
23    Left,
24    Right,
25}
26
27#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
28#[serde(tag = "type", content = "value")]
29#[repr(C)]
30pub enum Precedence {
31    Lowest,
32    LowLogicalOr,
33    LowLogicalXor,
34    LowLogicalAnd,
35    Print,
36    Yield,
37    YieldFrom,
38    IncDec,
39    KeyOr,
40    KeyXor,
41    KeyAnd,
42    Assignment,
43    ElvisOrConditional,
44    NullCoalesce,
45    Or,
46    And,
47    BitwiseOr,
48    BitwiseXor,
49    BitwiseAnd,
50    Equality,
51    Comparison,
52    Concat,
53    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
54    // clearly, the current precedence position handles the examples shown in the RFC,
55    // but will need to be verified with the actual implementation once its merged into php-src.
56    //
57    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
58    // PR: https://github.com/php/php-src/pull/17118
59    Pipe,
60    BitShift,
61    AddSub,
62    MulDivMod,
63    Bang,
64    Instanceof,
65    Prefix,
66    Pow,
67    Clone,
68    CallDim,
69    New,
70    ArrayDim,
71    ObjectAccess,
72}
73
74pub trait GetPrecedence {
75    fn precedence(&self) -> Precedence;
76}
77
78#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
79#[serde(tag = "type", content = "value")]
80#[repr(C)]
81pub enum TokenKind {
82    Whitespace,                  // ` `
83    Eval,                        // `eval`
84    Die,                         // `die`
85    Self_,                       // `self`
86    Parent,                      // `parent`
87    Backtick,                    // `` ` ``
88    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
89    DocumentEnd,                 // `abc`
90    From,                        // `from`
91    Print,                       // `print`
92    Dollar,                      // `$`
93    HaltCompiler,                // `__halt_compiler`
94    Readonly,                    // `readonly`
95    Global,                      // `global`
96    Abstract,                    // `abstract`
97    Ampersand,                   // `&`
98    AmpersandEqual,              // `&=`
99    AmpersandAmpersand,          // `&&`
100    AmpersandAmpersandEqual,     // `&&=`
101    Array,                       // `array`
102    ArrayCast,                   // `(array)`
103    MinusGreaterThan,            // `->`
104    QuestionMinusGreaterThan,    // `?->`
105    At,                          // `@`
106    As,                          // `as`
107    Asterisk,                    // `*`
108    HashLeftBracket,             // `#[`
109    Bang,                        // `!`
110    BangEqual,                   // `!=`
111    LessThanGreaterThan,         // `<>`
112    BangEqualEqual,              // `!==`
113    LessThanEqualGreaterThan,    // `<=>`
114    BoolCast,                    // `(bool)`
115    BooleanCast,                 // `(boolean)`
116    And,                         // `and`
117    Or,                          // `or`
118    Break,                       // `break`
119    Callable,                    // `callable`
120    Caret,                       // `^`
121    CaretEqual,                  // `^=`
122    Case,                        // `case`
123    Catch,                       // `catch`
124    Class,                       // `class`
125    ClassConstant,               // `__CLASS__`
126    TraitConstant,               // `__TRAIT__`
127    FunctionConstant,            // `__FUNCTION__`
128    MethodConstant,              // `__METHOD__`
129    LineConstant,                // `__LINE__`
130    FileConstant,                // `__FILE__`
131    Clone,                       // `clone`
132    MinusEqual,                  // `-=`
133    CloseTag,                    // `?>`
134    QuestionQuestion,            // `??`
135    QuestionQuestionEqual,       // `??=`
136    AsteriskEqual,               // `*=`
137    Colon,                       // `:`
138    Comma,                       // `,`
139    SingleLineComment,           // `// comment`
140    HashComment,                 // `# comment`
141    MultiLineComment,            // `/* comment */`
142    DocBlockComment,             // `/** comment */`
143    Const,                       // `const`
144    PartialLiteralString,        // `"string` or `'string`, missing closing quote
145    LiteralString,               // `"string"` or `'string'`
146    Continue,                    // `continue`
147    Declare,                     // `declare`
148    MinusMinus,                  // `--`
149    Default,                     // `default`
150    DirConstant,                 // `__DIR__`
151    SlashEqual,                  // `/=`
152    Do,                          // `do`
153    DollarLeftBrace,             // `${`
154    Dot,                         // `.`
155    DotEqual,                    // `.=`
156    EqualGreaterThan,            // `=>`
157    DoubleCast,                  // `(double)`
158    RealCast,                    // `(real)`
159    FloatCast,                   // `(float)`
160    ColonColon,                  // `::`
161    EqualEqual,                  // `==`
162    DoubleQuote,                 // `"`
163    Else,                        // `else`
164    Echo,                        // `echo`
165    DotDotDot,                   // `...`
166    ElseIf,                      // `elseif`
167    Empty,                       // `empty`
168    EndDeclare,                  // `enddeclare`
169    EndFor,                      // `endfor`
170    EndForeach,                  // `endforeach`
171    EndIf,                       // `endif`
172    EndSwitch,                   // `endswitch`
173    EndWhile,                    // `endwhile`
174    Enum,                        // `enum`
175    Equal,                       // `=`
176    Extends,                     // `extends`
177    False,                       // `false`
178    Final,                       // `final`
179    Finally,                     // `finally`
180    LiteralFloat,                // `1.0`
181    Fn,                          // `fn`
182    For,                         // `for`
183    Foreach,                     // `foreach`
184    FullyQualifiedIdentifier,    // `\Namespace\Class`
185    Function,                    // `function`
186    Goto,                        // `goto`
187    GreaterThan,                 // `>`
188    GreaterThanEqual,            // `>=`
189    Identifier,                  // `name`
190    If,                          // `if`
191    Implements,                  // `implements`
192    Include,                     // `include`
193    IncludeOnce,                 // `include_once`
194    PlusPlus,                    // `++`
195    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
196    InlineShebang,               // `#!...`
197    Instanceof,                  // `instanceof`
198    Insteadof,                   // `insteadof`
199    Exit,                        // `exit`
200    Unset,                       // `unset`
201    Isset,                       // `isset`
202    List,                        // `list`
203    LiteralInteger,              // `1`
204    IntCast,                     // `(int)`
205    IntegerCast,                 // `(integer)`
206    Interface,                   // `interface`
207    LeftBrace,                   // `{`
208    LeftBracket,                 // `[`
209    LeftParenthesis,             // `(`
210    LeftShift,                   // `<<`
211    LeftShiftEqual,              // `<<=`
212    RightShift,                  // `>>`
213    RightShiftEqual,             // `>>=`
214    LessThan,                    // `<`
215    LessThanEqual,               // `<=`
216    Match,                       // `match`
217    Minus,                       // `-`
218    Namespace,                   // `namespace`
219    NamespaceSeparator,          // `\`
220    NamespaceConstant,           // `__NAMESPACE__`
221    New,                         // `new`
222    Null,                        // `null`
223    ObjectCast,                  // `(object)`
224    UnsetCast,                   // `(unset)`
225    OpenTag,                     // `<?php`
226    EchoTag,                     // `<?=`
227    ShortOpenTag,                // `<?`
228    Percent,                     // `%`
229    PercentEqual,                // `%=`
230    Pipe,                        // `|`
231    PipeEqual,                   // `|=`
232    Plus,                        // `+`
233    PlusEqual,                   // `+=`
234    AsteriskAsterisk,            // `**`
235    AsteriskAsteriskEqual,       // `**=`
236    Private,                     // `private`
237    PrivateSet,                  // `private(set)`
238    Protected,                   // `protected`
239    ProtectedSet,                // `protected(set)`
240    Public,                      // `public`
241    PublicSet,                   // `public(set)`
242    QualifiedIdentifier,         // `Namespace\Class`
243    Question,                    // `?`
244    QuestionColon,               // `?:`
245    Require,                     // `require`
246    RequireOnce,                 // `require_once`
247    Return,                      // `return`
248    RightBrace,                  // `}`
249    RightBracket,                // `]`
250    RightParenthesis,            // `)`
251    Semicolon,                   // `;`
252    Slash,                       // `/`
253    Static,                      // `static`
254    StringCast,                  // `(string)`
255    BinaryCast,                  // `(binary)`
256    VoidCast,                    // `(void)`
257    StringPart,                  // `string` inside a double-quoted string, or a document string
258    Switch,                      // `switch`
259    Throw,                       // `throw`
260    Trait,                       // `trait`
261    EqualEqualEqual,             // `===`
262    True,                        // `true`
263    Try,                         // `try`
264    Use,                         // `use`
265    Var,                         // `var`
266    Variable,                    // `$name`
267    Yield,                       // `yield`
268    While,                       // `while`
269    Tilde,                       // `~`
270    PipePipe,                    // `||`
271    Xor,                         // `xor`
272    PipeGreaterThan,             // `|>`
273}
274
275#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
276#[repr(C)]
277pub struct Token {
278    pub kind: TokenKind,
279    pub value: StringIdentifier,
280    pub span: Span,
281}
282
283impl Precedence {
284    #[inline]
285    pub const fn infix(kind: &TokenKind) -> Precedence {
286        match kind {
287            T!["**"] => Precedence::Pow,
288            T!["instanceof"] => Precedence::Instanceof,
289            T!["*" | "/" | "%"] => Precedence::MulDivMod,
290            T!["+" | "-"] => Precedence::AddSub,
291            T!["<<"] | T![">>"] => Precedence::BitShift,
292            T!["."] => Precedence::Concat,
293            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
294            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
295            T!["&"] => Precedence::BitwiseAnd,
296            T!["^"] => Precedence::BitwiseXor,
297            T!["|"] => Precedence::BitwiseOr,
298            T!["&&"] => Precedence::And,
299            T!["||"] => Precedence::Or,
300            T!["??"] => Precedence::NullCoalesce,
301            T!["?" | "?:"] => Precedence::ElvisOrConditional,
302            T!["="
303                | "+="
304                | "-="
305                | "*="
306                | "**="
307                | "/="
308                | ".="
309                | "&&="
310                | "??="
311                | "%="
312                | "&="
313                | "|="
314                | "^="
315                | "<<="
316                | ">>="] => Precedence::Assignment,
317            T!["yield"] => Precedence::Yield,
318            T!["and"] => Precedence::KeyAnd,
319            T!["or"] => Precedence::KeyOr,
320            T!["xor"] => Precedence::KeyXor,
321            T!["|>"] => Precedence::Pipe,
322            _ => Precedence::Lowest,
323        }
324    }
325
326    #[inline]
327    pub const fn postfix(kind: &TokenKind) -> Self {
328        match kind {
329            T!["++" | "--"] => Self::Prefix,
330            T!["("] => Self::CallDim,
331            T!["["] => Self::ArrayDim,
332            T!["->" | "?->" | "::"] => Self::ObjectAccess,
333            _ => Self::Lowest,
334        }
335    }
336
337    #[inline]
338    pub const fn associativity(&self) -> Option<Associativity> {
339        Some(match self {
340            Self::Instanceof
341            | Self::MulDivMod
342            | Self::AddSub
343            | Self::BitShift
344            | Self::Concat
345            | Self::BitwiseAnd
346            | Self::BitwiseOr
347            | Self::BitwiseXor
348            | Self::And
349            | Self::Or
350            | Self::KeyAnd
351            | Self::KeyOr
352            | Self::KeyXor
353            | Self::Pipe => Associativity::Left,
354            Self::Pow | Self::NullCoalesce | Self::Assignment => Associativity::Right,
355            Self::ElvisOrConditional | Self::Equality | Self::Comparison => Associativity::NonAssociative,
356            _ => return None,
357        })
358    }
359}
360
361impl TokenKind {
362    #[inline]
363    pub const fn is_keyword(&self) -> bool {
364        matches!(
365            self,
366            TokenKind::Eval
367                | TokenKind::Die
368                | TokenKind::Empty
369                | TokenKind::Isset
370                | TokenKind::Unset
371                | TokenKind::Exit
372                | TokenKind::EndDeclare
373                | TokenKind::EndSwitch
374                | TokenKind::EndWhile
375                | TokenKind::EndForeach
376                | TokenKind::EndFor
377                | TokenKind::EndIf
378                | TokenKind::From
379                | TokenKind::And
380                | TokenKind::Or
381                | TokenKind::Xor
382                | TokenKind::Print
383                | TokenKind::Readonly
384                | TokenKind::Global
385                | TokenKind::Match
386                | TokenKind::Abstract
387                | TokenKind::Array
388                | TokenKind::As
389                | TokenKind::Break
390                | TokenKind::Case
391                | TokenKind::Catch
392                | TokenKind::Class
393                | TokenKind::Clone
394                | TokenKind::Continue
395                | TokenKind::Const
396                | TokenKind::Declare
397                | TokenKind::Default
398                | TokenKind::Do
399                | TokenKind::Echo
400                | TokenKind::ElseIf
401                | TokenKind::Else
402                | TokenKind::Enum
403                | TokenKind::Extends
404                | TokenKind::False
405                | TokenKind::Finally
406                | TokenKind::Final
407                | TokenKind::Fn
408                | TokenKind::Foreach
409                | TokenKind::For
410                | TokenKind::Function
411                | TokenKind::Goto
412                | TokenKind::If
413                | TokenKind::IncludeOnce
414                | TokenKind::Include
415                | TokenKind::Implements
416                | TokenKind::Interface
417                | TokenKind::Instanceof
418                | TokenKind::Namespace
419                | TokenKind::New
420                | TokenKind::Null
421                | TokenKind::Private
422                | TokenKind::PrivateSet
423                | TokenKind::Protected
424                | TokenKind::Public
425                | TokenKind::RequireOnce
426                | TokenKind::Require
427                | TokenKind::Return
428                | TokenKind::Static
429                | TokenKind::Switch
430                | TokenKind::Throw
431                | TokenKind::Trait
432                | TokenKind::True
433                | TokenKind::Try
434                | TokenKind::Use
435                | TokenKind::Var
436                | TokenKind::Yield
437                | TokenKind::While
438                | TokenKind::Insteadof
439                | TokenKind::List
440                | TokenKind::Self_
441                | TokenKind::Parent
442                | TokenKind::DirConstant
443                | TokenKind::FileConstant
444                | TokenKind::LineConstant
445                | TokenKind::FunctionConstant
446                | TokenKind::ClassConstant
447                | TokenKind::MethodConstant
448                | TokenKind::TraitConstant
449                | TokenKind::NamespaceConstant
450                | TokenKind::HaltCompiler
451        )
452    }
453
454    #[inline]
455    pub const fn is_infix(&self) -> bool {
456        matches!(
457            self,
458            T!["**"
459                | ">>="
460                | "<<="
461                | "^="
462                | "&="
463                | "|="
464                | "%="
465                | "**="
466                | "and"
467                | "or"
468                | "xor"
469                | "<=>"
470                | "<<"
471                | ">>"
472                | "&"
473                | "|"
474                | "^"
475                | "%"
476                | "instanceof"
477                | "*"
478                | "/"
479                | "+"
480                | "-"
481                | "."
482                | "<"
483                | ">"
484                | "<="
485                | ">="
486                | "=="
487                | "==="
488                | "!="
489                | "!=="
490                | "<>"
491                | "?"
492                | "?:"
493                | "&&"
494                | "||"
495                | "="
496                | "+="
497                | "-="
498                | ".="
499                | "??="
500                | "/="
501                | "*="
502                | "??"
503                | "|>"]
504        )
505    }
506
507    #[inline]
508    pub const fn is_postfix(&self) -> bool {
509        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
510    }
511
512    #[inline]
513    pub const fn is_visibility_modifier(&self) -> bool {
514        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
515    }
516
517    #[inline]
518    pub const fn is_modifier(&self) -> bool {
519        matches!(
520            self,
521            T!["public"
522                | "protected"
523                | "private"
524                | "private(set)"
525                | "protected(set)"
526                | "public(set)"
527                | "static"
528                | "final"
529                | "abstract"
530                | "readonly"]
531        )
532    }
533
534    #[inline]
535    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
536        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
537    }
538
539    #[inline]
540    pub const fn is_identifier_maybe_reserved(&self) -> bool {
541        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
542    }
543
544    #[inline]
545    pub const fn is_soft_reserved_identifier(&self) -> bool {
546        matches!(
547            self,
548            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
549        )
550    }
551
552    #[inline]
553    pub const fn is_reserved_identifier(&self) -> bool {
554        if self.is_soft_reserved_identifier() {
555            return true;
556        }
557
558        matches!(
559            self,
560            T!["static"
561                | "abstract"
562                | "final"
563                | "for"
564                | "private"
565                | "private(set)"
566                | "protected"
567                | "protected(set)"
568                | "public"
569                | "public(set)"
570                | "include"
571                | "include_once"
572                | "eval"
573                | "require"
574                | "require_once"
575                | "or"
576                | "xor"
577                | "and"
578                | "instanceof"
579                | "new"
580                | "clone"
581                | "exit"
582                | "die"
583                | "if"
584                | "elseif"
585                | "else"
586                | "endif"
587                | "echo"
588                | "do"
589                | "while"
590                | "endwhile"
591                | "endfor"
592                | "foreach"
593                | "endforeach"
594                | "declare"
595                | "enddeclare"
596                | "as"
597                | "try"
598                | "catch"
599                | "finally"
600                | "throw"
601                | "use"
602                | "insteadof"
603                | "global"
604                | "var"
605                | "unset"
606                | "isset"
607                | "empty"
608                | "continue"
609                | "goto"
610                | "function"
611                | "const"
612                | "return"
613                | "print"
614                | "yield"
615                | "list"
616                | "switch"
617                | "endswitch"
618                | "case"
619                | "default"
620                | "break"
621                | "array"
622                | "callable"
623                | "extends"
624                | "implements"
625                | "namespace"
626                | "trait"
627                | "interface"
628                | "class"
629                | "__CLASS__"
630                | "__TRAIT__"
631                | "__FUNCTION__"
632                | "__METHOD__"
633                | "__LINE__"
634                | "__FILE__"
635                | "__DIR__"
636                | "__NAMESPACE__"
637                | "__halt_compiler"
638                | "fn"
639                | "match"]
640        )
641    }
642
643    #[inline]
644    pub const fn is_literal(&self) -> bool {
645        matches!(
646            self,
647            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
648        )
649    }
650
651    #[inline]
652    pub const fn is_magic_constant(&self) -> bool {
653        matches!(
654            self,
655            T!["__CLASS__"
656                | "__DIR__"
657                | "__FILE__"
658                | "__FUNCTION__"
659                | "__LINE__"
660                | "__METHOD__"
661                | "__NAMESPACE__"
662                | "__TRAIT__"]
663        )
664    }
665
666    #[inline]
667    pub const fn is_cast(&self) -> bool {
668        matches!(
669            self,
670            T!["(string)"
671                | "(binary)"
672                | "(int)"
673                | "(integer)"
674                | "(float)"
675                | "(double)"
676                | "(real)"
677                | "(bool)"
678                | "(boolean)"
679                | "(array)"
680                | "(object)"
681                | "(unset)"
682                | "(void)"]
683        )
684    }
685
686    #[inline]
687    pub const fn is_unary_prefix(&self) -> bool {
688        if self.is_cast() {
689            return true;
690        }
691
692        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
693    }
694
695    #[inline]
696    pub const fn is_trivia(&self) -> bool {
697        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
698    }
699
700    #[inline]
701    pub const fn is_comment(&self) -> bool {
702        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
703    }
704
705    #[inline]
706    pub const fn is_construct(&self) -> bool {
707        matches!(
708            self,
709            T!["isset"
710                | "empty"
711                | "eval"
712                | "include"
713                | "include_once"
714                | "require"
715                | "require_once"
716                | "print"
717                | "unset"
718                | "exit"
719                | "die"]
720        )
721    }
722}
723
724impl Token {
725    pub const fn new(kind: TokenKind, value: StringIdentifier, span: Span) -> Self {
726        Self { kind, value, span }
727    }
728}
729
730impl std::fmt::Display for Token {
731    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
732        write!(f, "{:?} {}", self.kind, self.span)
733    }
734}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
