mago_syntax/token/
mod.rs

1use serde::Deserialize;
2use serde::Serialize;
3use strum::Display;
4
5use mago_interner::StringIdentifier;
6use mago_span::Span;
7
8use crate::T;
9
10#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
11#[serde(tag = "type", content = "value")]
12pub enum DocumentKind {
13    Heredoc,
14    Nowdoc,
15}
16
17#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
18#[serde(tag = "type", content = "value")]
19pub enum Associativity {
20    NonAssociative,
21    Left,
22    Right,
23}
24
25#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
26#[serde(tag = "type", content = "value")]
27pub enum Precedence {
28    Lowest,
29    LowLogicalOr,
30    LowLogicalXor,
31    LowLogicalAnd,
32    Print,
33    Yield,
34    YieldFrom,
35    IncDec,
36    KeyOr,
37    KeyXor,
38    KeyAnd,
39    Assignment,
40    ElvisOrConditional,
41    NullCoalesce,
42    Or,
43    And,
44    BitwiseOr,
45    BitwiseXor,
46    BitwiseAnd,
47    Equality,
48    Comparison,
49    Concat,
50    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
51    // clearly, the current precedence position handles the examples shown in the RFC,
52    // but will need to be verified with the actual implementation once its merged into php-src.
53    //
54    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
55    // PR: https://github.com/php/php-src/pull/17118
56    Pipe,
57    BitShift,
58    AddSub,
59    MulDivMod,
60    Bang,
61    Instanceof,
62    Prefix,
63    Pow,
64    Clone,
65    CallDim,
66    New,
67    ArrayDim,
68    ObjectAccess,
69}
70
71pub trait GetPrecedence {
72    fn precedence(&self) -> Precedence;
73}
74
75#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
76#[serde(tag = "type", content = "value")]
77pub enum TokenKind {
78    Whitespace,                  // ` `
79    Eval,                        // `eval`
80    Die,                         // `die`
81    Self_,                       // `self`
82    Parent,                      // `parent`
83    Backtick,                    // `` ` ``
84    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
85    DocumentEnd,                 // `abc`
86    From,                        // `from`
87    Print,                       // `print`
88    Dollar,                      // `$`
89    HaltCompiler,                // `__halt_compiler`
90    Readonly,                    // `readonly`
91    Global,                      // `global`
92    Abstract,                    // `abstract`
93    Ampersand,                   // `&`
94    AmpersandEqual,              // `&=`
95    AmpersandAmpersand,          // `&&`
96    AmpersandAmpersandEqual,     // `&&=`
97    Array,                       // `array`
98    ArrayCast,                   // `(array)`
99    MinusGreaterThan,            // `->`
100    QuestionMinusGreaterThan,    // `?->`
101    At,                          // `@`
102    As,                          // `as`
103    Asterisk,                    // `*`
104    HashLeftBracket,             // `#[`
105    Bang,                        // `!`
106    BangEqual,                   // `!=`
107    LessThanGreaterThan,         // `<>`
108    BangEqualEqual,              // `!==`
109    LessThanEqualGreaterThan,    // `<=>`
110    BoolCast,                    // `(bool)`
111    BooleanCast,                 // `(boolean)`
112    And,                         // `and`
113    Or,                          // `or`
114    Break,                       // `break`
115    Callable,                    // `callable`
116    Caret,                       // `^`
117    CaretEqual,                  // `^=`
118    Case,                        // `case`
119    Catch,                       // `catch`
120    Class,                       // `class`
121    ClassConstant,               // `__CLASS__`
122    TraitConstant,               // `__TRAIT__`
123    FunctionConstant,            // `__FUNCTION__`
124    MethodConstant,              // `__METHOD__`
125    LineConstant,                // `__LINE__`
126    FileConstant,                // `__FILE__`
127    Clone,                       // `clone`
128    MinusEqual,                  // `-=`
129    CloseTag,                    // `?>`
130    QuestionQuestion,            // `??`
131    QuestionQuestionEqual,       // `??=`
132    AsteriskEqual,               // `*=`
133    Colon,                       // `:`
134    Comma,                       // `,`
135    SingleLineComment,           // `// comment`
136    HashComment,                 // `# comment`
137    MultiLineComment,            // `/* comment */`
138    DocBlockComment,             // `/** comment */`
139    Const,                       // `const`
140    PartialLiteralString,        // `"string` or `'string`, missing closing quote
141    LiteralString,               // `"string"` or `'string'`
142    Continue,                    // `continue`
143    Declare,                     // `declare`
144    MinusMinus,                  // `--`
145    Default,                     // `default`
146    DirConstant,                 // `__DIR__`
147    SlashEqual,                  // `/=`
148    Do,                          // `do`
149    DollarLeftBrace,             // `${`
150    Dot,                         // `.`
151    DotEqual,                    // `.=`
152    EqualGreaterThan,            // `=>`
153    DoubleCast,                  // `(double)`
154    RealCast,                    // `(real)`
155    FloatCast,                   // `(float)`
156    ColonColon,                  // `::`
157    EqualEqual,                  // `==`
158    DoubleQuote,                 // `"`
159    Else,                        // `else`
160    Echo,                        // `echo`
161    DotDotDot,                   // `...`
162    ElseIf,                      // `elseif`
163    Empty,                       // `empty`
164    EndDeclare,                  // `enddeclare`
165    EndFor,                      // `endfor`
166    EndForeach,                  // `endforeach`
167    EndIf,                       // `endif`
168    EndSwitch,                   // `endswitch`
169    EndWhile,                    // `endwhile`
170    Enum,                        // `enum`
171    Equal,                       // `=`
172    Extends,                     // `extends`
173    False,                       // `false`
174    Final,                       // `final`
175    Finally,                     // `finally`
176    LiteralFloat,                // `1.0`
177    Fn,                          // `fn`
178    For,                         // `for`
179    Foreach,                     // `foreach`
180    FullyQualifiedIdentifier,    // `\Namespace\Class`
181    Function,                    // `function`
182    Goto,                        // `goto`
183    GreaterThan,                 // `>`
184    GreaterThanEqual,            // `>=`
185    Identifier,                  // `name`
186    If,                          // `if`
187    Implements,                  // `implements`
188    Include,                     // `include`
189    IncludeOnce,                 // `include_once`
190    PlusPlus,                    // `++`
191    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
192    InlineShebang,               // `#!...`
193    Instanceof,                  // `instanceof`
194    Insteadof,                   // `insteadof`
195    Exit,                        // `exit`
196    Unset,                       // `unset`
197    Isset,                       // `isset`
198    List,                        // `list`
199    LiteralInteger,              // `1`
200    IntCast,                     // `(int)`
201    IntegerCast,                 // `(integer)`
202    Interface,                   // `interface`
203    LeftBrace,                   // `{`
204    LeftBracket,                 // `[`
205    LeftParenthesis,             // `(`
206    LeftShift,                   // `<<`
207    LeftShiftEqual,              // `<<=`
208    RightShift,                  // `>>`
209    RightShiftEqual,             // `>>=`
210    LessThan,                    // `<`
211    LessThanEqual,               // `<=`
212    Match,                       // `match`
213    Minus,                       // `-`
214    Namespace,                   // `namespace`
215    NamespaceSeparator,          // `\`
216    NamespaceConstant,           // `__NAMESPACE__`
217    New,                         // `new`
218    Null,                        // `null`
219    ObjectCast,                  // `(object)`
220    UnsetCast,                   // `(unset)`
221    OpenTag,                     // `<?php`
222    EchoTag,                     // `<?=`
223    ShortOpenTag,                // `<?`
224    Percent,                     // `%`
225    PercentEqual,                // `%=`
226    Pipe,                        // `|`
227    PipeEqual,                   // `|=`
228    Plus,                        // `+`
229    PlusEqual,                   // `+=`
230    AsteriskAsterisk,            // `**`
231    AsteriskAsteriskEqual,       // `**=`
232    Private,                     // `private`
233    PrivateSet,                  // `private(set)`
234    Protected,                   // `protected`
235    ProtectedSet,                // `protected(set)`
236    Public,                      // `public`
237    PublicSet,                   // `public(set)`
238    QualifiedIdentifier,         // `Namespace\Class`
239    Question,                    // `?`
240    QuestionColon,               // `?:`
241    Require,                     // `require`
242    RequireOnce,                 // `require_once`
243    Return,                      // `return`
244    RightBrace,                  // `}`
245    RightBracket,                // `]`
246    RightParenthesis,            // `)`
247    Semicolon,                   // `;`
248    Slash,                       // `/`
249    Static,                      // `static`
250    StringCast,                  // `(string)`
251    BinaryCast,                  // `(binary)`
252    VoidCast,                    // `(void)`
253    StringPart,                  // `string` inside a double-quoted string, or a document string
254    Switch,                      // `switch`
255    Throw,                       // `throw`
256    Trait,                       // `trait`
257    EqualEqualEqual,             // `===`
258    True,                        // `true`
259    Try,                         // `try`
260    Use,                         // `use`
261    Var,                         // `var`
262    Variable,                    // `$name`
263    Yield,                       // `yield`
264    While,                       // `while`
265    Tilde,                       // `~`
266    PipePipe,                    // `||`
267    Xor,                         // `xor`
268    PipeGreaterThan,             // `|>`
269}
270
271#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
272pub struct Token {
273    pub kind: TokenKind,
274    pub value: StringIdentifier,
275    pub span: Span,
276}
277
278impl Precedence {
279    #[inline]
280    pub const fn infix(kind: &TokenKind) -> Precedence {
281        match kind {
282            T!["**"] => Precedence::Pow,
283            T!["instanceof"] => Precedence::Instanceof,
284            T!["*" | "/" | "%"] => Precedence::MulDivMod,
285            T!["+" | "-"] => Precedence::AddSub,
286            T!["<<"] | T![">>"] => Precedence::BitShift,
287            T!["."] => Precedence::Concat,
288            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
289            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
290            T!["&"] => Precedence::BitwiseAnd,
291            T!["^"] => Precedence::BitwiseXor,
292            T!["|"] => Precedence::BitwiseOr,
293            T!["&&"] => Precedence::And,
294            T!["||"] => Precedence::Or,
295            T!["??"] => Precedence::NullCoalesce,
296            T!["?" | "?:"] => Precedence::ElvisOrConditional,
297            T!["="
298                | "+="
299                | "-="
300                | "*="
301                | "**="
302                | "/="
303                | ".="
304                | "&&="
305                | "??="
306                | "%="
307                | "&="
308                | "|="
309                | "^="
310                | "<<="
311                | ">>="] => Precedence::Assignment,
312            T!["yield"] => Precedence::Yield,
313            T!["and"] => Precedence::KeyAnd,
314            T!["or"] => Precedence::KeyOr,
315            T!["xor"] => Precedence::KeyXor,
316            T!["|>"] => Precedence::Pipe,
317            _ => Precedence::Lowest,
318        }
319    }
320
321    #[inline]
322    pub const fn postfix(kind: &TokenKind) -> Self {
323        match kind {
324            T!["++" | "--"] => Self::Prefix,
325            T!["("] => Self::CallDim,
326            T!["["] => Self::ArrayDim,
327            T!["->" | "?->" | "::"] => Self::ObjectAccess,
328            _ => Self::Lowest,
329        }
330    }
331
332    #[inline]
333    pub const fn associativity(&self) -> Option<Associativity> {
334        Some(match self {
335            Self::Instanceof
336            | Self::MulDivMod
337            | Self::AddSub
338            | Self::BitShift
339            | Self::Concat
340            | Self::BitwiseAnd
341            | Self::BitwiseOr
342            | Self::BitwiseXor
343            | Self::And
344            | Self::Or
345            | Self::KeyAnd
346            | Self::KeyOr
347            | Self::KeyXor
348            | Self::Pipe => Associativity::Left,
349            Self::Pow | Self::NullCoalesce | Self::Assignment => Associativity::Right,
350            Self::ElvisOrConditional | Self::Equality | Self::Comparison => Associativity::NonAssociative,
351            _ => return None,
352        })
353    }
354}
355
356impl TokenKind {
357    #[inline]
358    pub const fn is_keyword(&self) -> bool {
359        matches!(
360            self,
361            TokenKind::Eval
362                | TokenKind::Die
363                | TokenKind::Empty
364                | TokenKind::Isset
365                | TokenKind::Unset
366                | TokenKind::Exit
367                | TokenKind::EndDeclare
368                | TokenKind::EndSwitch
369                | TokenKind::EndWhile
370                | TokenKind::EndForeach
371                | TokenKind::EndFor
372                | TokenKind::EndIf
373                | TokenKind::From
374                | TokenKind::And
375                | TokenKind::Or
376                | TokenKind::Xor
377                | TokenKind::Print
378                | TokenKind::Readonly
379                | TokenKind::Global
380                | TokenKind::Match
381                | TokenKind::Abstract
382                | TokenKind::Array
383                | TokenKind::As
384                | TokenKind::Break
385                | TokenKind::Case
386                | TokenKind::Catch
387                | TokenKind::Class
388                | TokenKind::Clone
389                | TokenKind::Continue
390                | TokenKind::Const
391                | TokenKind::Declare
392                | TokenKind::Default
393                | TokenKind::Do
394                | TokenKind::Echo
395                | TokenKind::ElseIf
396                | TokenKind::Else
397                | TokenKind::Enum
398                | TokenKind::Extends
399                | TokenKind::False
400                | TokenKind::Finally
401                | TokenKind::Final
402                | TokenKind::Fn
403                | TokenKind::Foreach
404                | TokenKind::For
405                | TokenKind::Function
406                | TokenKind::Goto
407                | TokenKind::If
408                | TokenKind::IncludeOnce
409                | TokenKind::Include
410                | TokenKind::Implements
411                | TokenKind::Interface
412                | TokenKind::Instanceof
413                | TokenKind::Namespace
414                | TokenKind::New
415                | TokenKind::Null
416                | TokenKind::Private
417                | TokenKind::PrivateSet
418                | TokenKind::Protected
419                | TokenKind::Public
420                | TokenKind::RequireOnce
421                | TokenKind::Require
422                | TokenKind::Return
423                | TokenKind::Static
424                | TokenKind::Switch
425                | TokenKind::Throw
426                | TokenKind::Trait
427                | TokenKind::True
428                | TokenKind::Try
429                | TokenKind::Use
430                | TokenKind::Var
431                | TokenKind::Yield
432                | TokenKind::While
433                | TokenKind::Insteadof
434                | TokenKind::List
435                | TokenKind::Self_
436                | TokenKind::Parent
437                | TokenKind::DirConstant
438                | TokenKind::FileConstant
439                | TokenKind::LineConstant
440                | TokenKind::FunctionConstant
441                | TokenKind::ClassConstant
442                | TokenKind::MethodConstant
443                | TokenKind::TraitConstant
444                | TokenKind::NamespaceConstant
445                | TokenKind::HaltCompiler
446        )
447    }
448
449    #[inline]
450    pub const fn is_infix(&self) -> bool {
451        matches!(
452            self,
453            T!["**"
454                | ">>="
455                | "<<="
456                | "^="
457                | "&="
458                | "|="
459                | "%="
460                | "**="
461                | "and"
462                | "or"
463                | "xor"
464                | "<=>"
465                | "<<"
466                | ">>"
467                | "&"
468                | "|"
469                | "^"
470                | "%"
471                | "instanceof"
472                | "*"
473                | "/"
474                | "+"
475                | "-"
476                | "."
477                | "<"
478                | ">"
479                | "<="
480                | ">="
481                | "=="
482                | "==="
483                | "!="
484                | "!=="
485                | "<>"
486                | "?"
487                | "?:"
488                | "&&"
489                | "||"
490                | "="
491                | "+="
492                | "-="
493                | ".="
494                | "??="
495                | "/="
496                | "*="
497                | "??"
498                | "|>"]
499        )
500    }
501
502    #[inline]
503    pub const fn is_postfix(&self) -> bool {
504        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
505    }
506
507    #[inline]
508    pub const fn is_visibility_modifier(&self) -> bool {
509        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
510    }
511
512    #[inline]
513    pub const fn is_modifier(&self) -> bool {
514        matches!(
515            self,
516            T!["public"
517                | "protected"
518                | "private"
519                | "private(set)"
520                | "protected(set)"
521                | "public(set)"
522                | "static"
523                | "final"
524                | "abstract"
525                | "readonly"]
526        )
527    }
528
529    #[inline]
530    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
531        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
532    }
533
534    #[inline]
535    pub const fn is_identifier_maybe_reserved(&self) -> bool {
536        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
537    }
538
539    #[inline]
540    pub const fn is_soft_reserved_identifier(&self) -> bool {
541        matches!(
542            self,
543            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
544        )
545    }
546
547    #[inline]
548    pub const fn is_reserved_identifier(&self) -> bool {
549        if self.is_soft_reserved_identifier() {
550            return true;
551        }
552
553        matches!(
554            self,
555            T!["static"
556                | "abstract"
557                | "final"
558                | "for"
559                | "private"
560                | "private(set)"
561                | "protected"
562                | "protected(set)"
563                | "public"
564                | "public(set)"
565                | "include"
566                | "include_once"
567                | "eval"
568                | "require"
569                | "require_once"
570                | "or"
571                | "xor"
572                | "and"
573                | "instanceof"
574                | "new"
575                | "clone"
576                | "exit"
577                | "die"
578                | "if"
579                | "elseif"
580                | "else"
581                | "endif"
582                | "echo"
583                | "do"
584                | "while"
585                | "endwhile"
586                | "endfor"
587                | "foreach"
588                | "endforeach"
589                | "declare"
590                | "enddeclare"
591                | "as"
592                | "try"
593                | "catch"
594                | "finally"
595                | "throw"
596                | "use"
597                | "insteadof"
598                | "global"
599                | "var"
600                | "unset"
601                | "isset"
602                | "empty"
603                | "continue"
604                | "goto"
605                | "function"
606                | "const"
607                | "return"
608                | "print"
609                | "yield"
610                | "list"
611                | "switch"
612                | "endswitch"
613                | "case"
614                | "default"
615                | "break"
616                | "array"
617                | "callable"
618                | "extends"
619                | "implements"
620                | "namespace"
621                | "trait"
622                | "interface"
623                | "class"
624                | "__CLASS__"
625                | "__TRAIT__"
626                | "__FUNCTION__"
627                | "__METHOD__"
628                | "__LINE__"
629                | "__FILE__"
630                | "__DIR__"
631                | "__NAMESPACE__"
632                | "__halt_compiler"
633                | "fn"
634                | "match"]
635        )
636    }
637
638    #[inline]
639    pub const fn is_literal(&self) -> bool {
640        matches!(
641            self,
642            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
643        )
644    }
645
646    #[inline]
647    pub const fn is_magic_constant(&self) -> bool {
648        matches!(
649            self,
650            T!["__CLASS__"
651                | "__DIR__"
652                | "__FILE__"
653                | "__FUNCTION__"
654                | "__LINE__"
655                | "__METHOD__"
656                | "__NAMESPACE__"
657                | "__TRAIT__"]
658        )
659    }
660
661    #[inline]
662    pub const fn is_cast(&self) -> bool {
663        matches!(
664            self,
665            T!["(string)"
666                | "(binary)"
667                | "(int)"
668                | "(integer)"
669                | "(float)"
670                | "(double)"
671                | "(real)"
672                | "(bool)"
673                | "(boolean)"
674                | "(array)"
675                | "(object)"
676                | "(unset)"
677                | "(void)"]
678        )
679    }
680
681    #[inline]
682    pub const fn is_unary_prefix(&self) -> bool {
683        if self.is_cast() {
684            return true;
685        }
686
687        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
688    }
689
690    #[inline]
691    pub const fn is_trivia(&self) -> bool {
692        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
693    }
694
695    #[inline]
696    pub const fn is_comment(&self) -> bool {
697        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
698    }
699
700    #[inline]
701    pub const fn is_comma(&self) -> bool {
702        matches!(self, T![","])
703    }
704
705    #[inline]
706    pub const fn is_construct(&self) -> bool {
707        matches!(
708            self,
709            T!["isset"
710                | "empty"
711                | "eval"
712                | "include"
713                | "include_once"
714                | "require"
715                | "require_once"
716                | "print"
717                | "unset"
718                | "exit"
719                | "die"]
720        )
721    }
722}
723
724impl Token {
725    pub const fn new(kind: TokenKind, value: StringIdentifier, span: Span) -> Self {
726        Self { kind, value, span }
727    }
728}
729
730impl std::fmt::Display for Token {
731    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
732        write!(f, "{:?} {}", self.kind, self.span)
733    }
734}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
