mago_syntax/token/
mod.rs

1use serde::Serialize;
2use strum::Display;
3
4use mago_span::Span;
5
6use crate::T;
7
8#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
9#[serde(tag = "type", content = "value")]
10pub enum DocumentKind {
11    Heredoc,
12    Nowdoc,
13}
14
15#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
16#[serde(tag = "type", content = "value")]
17pub enum Associativity {
18    NonAssociative,
19    Left,
20    Right,
21}
22
23#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
24#[serde(tag = "type", content = "value")]
25pub enum Precedence {
26    Lowest,
27    Print,
28    YieldFrom,
29    Yield,
30    KeyOr,
31    KeyXor,
32    KeyAnd,
33    Assignment,
34    ElvisOrConditional,
35    NullCoalesce,
36    Or,
37    And,
38    BitwiseOr,
39    BitwiseXor,
40    BitwiseAnd,
41    Equality,
42    Comparison,
43    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
44    // clearly, the current precedence position handles the examples shown in the RFC,
45    // but will need to be verified with the actual implementation once its merged into php-src.
46    //
47    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
48    // PR: https://github.com/php/php-src/pull/17118
49    Pipe,
50    Concat,
51    BitShift,
52    AddSub,
53    MulDivMod,
54    Unary,
55    Instanceof,
56    ErrorControl,
57    Pow,
58    Clone,
59    IncDec,
60    Reference,
61    CallDim,
62    New,
63    ArrayDim,
64    ObjectAccess,
65    Highest,
66}
67
68pub trait GetPrecedence {
69    fn precedence(&self) -> Precedence;
70}
71
72#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord, Display)]
73#[serde(tag = "type", content = "value")]
74pub enum TokenKind {
75    Whitespace,                  // ` `
76    Eval,                        // `eval`
77    Die,                         // `die`
78    Self_,                       // `self`
79    Parent,                      // `parent`
80    Backtick,                    // `` ` ``
81    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
82    DocumentEnd,                 // `abc`
83    From,                        // `from`
84    Print,                       // `print`
85    Dollar,                      // `$`
86    HaltCompiler,                // `__halt_compiler`
87    Readonly,                    // `readonly`
88    Global,                      // `global`
89    Abstract,                    // `abstract`
90    Ampersand,                   // `&`
91    AmpersandEqual,              // `&=`
92    AmpersandAmpersand,          // `&&`
93    AmpersandAmpersandEqual,     // `&&=`
94    Array,                       // `array`
95    ArrayCast,                   // `(array)`
96    MinusGreaterThan,            // `->`
97    QuestionMinusGreaterThan,    // `?->`
98    At,                          // `@`
99    As,                          // `as`
100    Asterisk,                    // `*`
101    HashLeftBracket,             // `#[`
102    Bang,                        // `!`
103    BangEqual,                   // `!=`
104    LessThanGreaterThan,         // `<>`
105    BangEqualEqual,              // `!==`
106    LessThanEqualGreaterThan,    // `<=>`
107    BoolCast,                    // `(bool)`
108    BooleanCast,                 // `(boolean)`
109    And,                         // `and`
110    Or,                          // `or`
111    Break,                       // `break`
112    Callable,                    // `callable`
113    Caret,                       // `^`
114    CaretEqual,                  // `^=`
115    Case,                        // `case`
116    Catch,                       // `catch`
117    Class,                       // `class`
118    ClassConstant,               // `__CLASS__`
119    TraitConstant,               // `__TRAIT__`
120    FunctionConstant,            // `__FUNCTION__`
121    MethodConstant,              // `__METHOD__`
122    LineConstant,                // `__LINE__`
123    FileConstant,                // `__FILE__`
124    Clone,                       // `clone`
125    MinusEqual,                  // `-=`
126    CloseTag,                    // `?>`
127    QuestionQuestion,            // `??`
128    QuestionQuestionEqual,       // `??=`
129    AsteriskEqual,               // `*=`
130    Colon,                       // `:`
131    Comma,                       // `,`
132    SingleLineComment,           // `// comment`
133    HashComment,                 // `# comment`
134    MultiLineComment,            // `/* comment */`
135    DocBlockComment,             // `/** comment */`
136    Const,                       // `const`
137    PartialLiteralString,        // `"string` or `'string`, missing closing quote
138    LiteralString,               // `"string"` or `'string'`
139    Continue,                    // `continue`
140    Declare,                     // `declare`
141    MinusMinus,                  // `--`
142    Default,                     // `default`
143    DirConstant,                 // `__DIR__`
144    SlashEqual,                  // `/=`
145    Do,                          // `do`
146    DollarLeftBrace,             // `${`
147    Dot,                         // `.`
148    DotEqual,                    // `.=`
149    EqualGreaterThan,            // `=>`
150    DoubleCast,                  // `(double)`
151    RealCast,                    // `(real)`
152    FloatCast,                   // `(float)`
153    ColonColon,                  // `::`
154    EqualEqual,                  // `==`
155    DoubleQuote,                 // `"`
156    Else,                        // `else`
157    Echo,                        // `echo`
158    DotDotDot,                   // `...`
159    ElseIf,                      // `elseif`
160    Empty,                       // `empty`
161    EndDeclare,                  // `enddeclare`
162    EndFor,                      // `endfor`
163    EndForeach,                  // `endforeach`
164    EndIf,                       // `endif`
165    EndSwitch,                   // `endswitch`
166    EndWhile,                    // `endwhile`
167    Enum,                        // `enum`
168    Equal,                       // `=`
169    Extends,                     // `extends`
170    False,                       // `false`
171    Final,                       // `final`
172    Finally,                     // `finally`
173    LiteralFloat,                // `1.0`
174    Fn,                          // `fn`
175    For,                         // `for`
176    Foreach,                     // `foreach`
177    FullyQualifiedIdentifier,    // `\Namespace\Class`
178    Function,                    // `function`
179    Goto,                        // `goto`
180    GreaterThan,                 // `>`
181    GreaterThanEqual,            // `>=`
182    Identifier,                  // `name`
183    If,                          // `if`
184    Implements,                  // `implements`
185    Include,                     // `include`
186    IncludeOnce,                 // `include_once`
187    PlusPlus,                    // `++`
188    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
189    InlineShebang,               // `#!...`
190    Instanceof,                  // `instanceof`
191    Insteadof,                   // `insteadof`
192    Exit,                        // `exit`
193    Unset,                       // `unset`
194    Isset,                       // `isset`
195    List,                        // `list`
196    LiteralInteger,              // `1`
197    IntCast,                     // `(int)`
198    IntegerCast,                 // `(integer)`
199    Interface,                   // `interface`
200    LeftBrace,                   // `{`
201    LeftBracket,                 // `[`
202    LeftParenthesis,             // `(`
203    LeftShift,                   // `<<`
204    LeftShiftEqual,              // `<<=`
205    RightShift,                  // `>>`
206    RightShiftEqual,             // `>>=`
207    LessThan,                    // `<`
208    LessThanEqual,               // `<=`
209    Match,                       // `match`
210    Minus,                       // `-`
211    Namespace,                   // `namespace`
212    NamespaceSeparator,          // `\`
213    NamespaceConstant,           // `__NAMESPACE__`
214    New,                         // `new`
215    Null,                        // `null`
216    ObjectCast,                  // `(object)`
217    UnsetCast,                   // `(unset)`
218    OpenTag,                     // `<?php`
219    EchoTag,                     // `<?=`
220    ShortOpenTag,                // `<?`
221    Percent,                     // `%`
222    PercentEqual,                // `%=`
223    Pipe,                        // `|`
224    PipeEqual,                   // `|=`
225    Plus,                        // `+`
226    PlusEqual,                   // `+=`
227    AsteriskAsterisk,            // `**`
228    AsteriskAsteriskEqual,       // `**=`
229    Private,                     // `private`
230    PrivateSet,                  // `private(set)`
231    Protected,                   // `protected`
232    ProtectedSet,                // `protected(set)`
233    Public,                      // `public`
234    PublicSet,                   // `public(set)`
235    QualifiedIdentifier,         // `Namespace\Class`
236    Question,                    // `?`
237    Require,                     // `require`
238    RequireOnce,                 // `require_once`
239    Return,                      // `return`
240    RightBrace,                  // `}`
241    RightBracket,                // `]`
242    RightParenthesis,            // `)`
243    Semicolon,                   // `;`
244    Slash,                       // `/`
245    Static,                      // `static`
246    StringCast,                  // `(string)`
247    BinaryCast,                  // `(binary)`
248    VoidCast,                    // `(void)`
249    StringPart,                  // `string` inside a double-quoted string, or a document string
250    Switch,                      // `switch`
251    Throw,                       // `throw`
252    Trait,                       // `trait`
253    EqualEqualEqual,             // `===`
254    True,                        // `true`
255    Try,                         // `try`
256    Use,                         // `use`
257    Var,                         // `var`
258    Variable,                    // `$name`
259    Yield,                       // `yield`
260    While,                       // `while`
261    Tilde,                       // `~`
262    PipePipe,                    // `||`
263    Xor,                         // `xor`
264    PipeGreaterThan,             // `|>`
265}
266
267#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, PartialOrd, Ord)]
268pub struct Token<'arena> {
269    pub kind: TokenKind,
270    pub value: &'arena str,
271    pub span: Span,
272}
273
274impl Precedence {
275    #[inline]
276    pub const fn infix(kind: &TokenKind) -> Precedence {
277        match kind {
278            T!["**"] => Precedence::Pow,
279            T!["instanceof"] => Precedence::Instanceof,
280            T!["*" | "/" | "%"] => Precedence::MulDivMod,
281            T!["+" | "-"] => Precedence::AddSub,
282            T!["<<"] | T![">>"] => Precedence::BitShift,
283            T!["."] => Precedence::Concat,
284            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
285            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
286            T!["&"] => Precedence::BitwiseAnd,
287            T!["^"] => Precedence::BitwiseXor,
288            T!["|"] => Precedence::BitwiseOr,
289            T!["&&"] => Precedence::And,
290            T!["||"] => Precedence::Or,
291            T!["??"] => Precedence::NullCoalesce,
292            T!["?"] => Precedence::ElvisOrConditional,
293            T!["="
294                | "+="
295                | "-="
296                | "*="
297                | "**="
298                | "/="
299                | ".="
300                | "&&="
301                | "??="
302                | "%="
303                | "&="
304                | "|="
305                | "^="
306                | "<<="
307                | ">>="] => Precedence::Assignment,
308            T!["yield"] => Precedence::Yield,
309            T!["and"] => Precedence::KeyAnd,
310            T!["or"] => Precedence::KeyOr,
311            T!["xor"] => Precedence::KeyXor,
312            T!["print"] => Precedence::Print,
313            T!["|>"] => Precedence::Pipe,
314            _ => Precedence::Lowest,
315        }
316    }
317
318    #[inline]
319    pub const fn postfix(kind: &TokenKind) -> Self {
320        match kind {
321            T!["++" | "--"] => Self::IncDec,
322            T!["("] => Self::CallDim,
323            T!["["] => Self::ArrayDim,
324            T!["->" | "?->" | "::"] => Self::ObjectAccess,
325            _ => Self::Lowest,
326        }
327    }
328
329    #[inline]
330    pub const fn associativity(&self) -> Option<Associativity> {
331        Some(match self {
332            Self::MulDivMod
333            | Self::AddSub
334            | Self::Concat
335            | Self::BitShift
336            | Self::BitwiseAnd
337            | Self::BitwiseOr
338            | Self::BitwiseXor
339            | Self::And
340            | Self::Or
341            | Self::KeyAnd
342            | Self::KeyXor
343            | Self::KeyOr
344            | Self::Pipe
345            | Self::ElvisOrConditional
346            | Self::ObjectAccess => Associativity::Left,
347            Self::Pow | Self::NullCoalesce | Self::Assignment | Self::Unary | Self::New => Associativity::Right,
348            Self::Equality | Self::Comparison | Self::Instanceof => Associativity::NonAssociative,
349            _ => return None,
350        })
351    }
352
353    #[inline]
354    pub const fn is_associative(&self) -> bool {
355        self.associativity().is_some()
356    }
357
358    #[inline]
359    pub const fn is_right_associative(&self) -> bool {
360        matches!(self.associativity(), Some(Associativity::Right))
361    }
362
363    #[inline]
364    pub const fn is_left_associative(&self) -> bool {
365        matches!(self.associativity(), Some(Associativity::Left))
366    }
367
368    #[inline]
369    pub const fn is_non_associative(&self) -> bool {
370        matches!(self.associativity(), Some(Associativity::NonAssociative))
371    }
372}
373
374impl TokenKind {
375    #[inline]
376    pub const fn is_keyword(&self) -> bool {
377        matches!(
378            self,
379            TokenKind::Eval
380                | TokenKind::Die
381                | TokenKind::Empty
382                | TokenKind::Isset
383                | TokenKind::Unset
384                | TokenKind::Exit
385                | TokenKind::EndDeclare
386                | TokenKind::EndSwitch
387                | TokenKind::EndWhile
388                | TokenKind::EndForeach
389                | TokenKind::EndFor
390                | TokenKind::EndIf
391                | TokenKind::From
392                | TokenKind::And
393                | TokenKind::Or
394                | TokenKind::Xor
395                | TokenKind::Print
396                | TokenKind::Readonly
397                | TokenKind::Global
398                | TokenKind::Match
399                | TokenKind::Abstract
400                | TokenKind::Array
401                | TokenKind::As
402                | TokenKind::Break
403                | TokenKind::Case
404                | TokenKind::Catch
405                | TokenKind::Class
406                | TokenKind::Clone
407                | TokenKind::Continue
408                | TokenKind::Const
409                | TokenKind::Declare
410                | TokenKind::Default
411                | TokenKind::Do
412                | TokenKind::Echo
413                | TokenKind::ElseIf
414                | TokenKind::Else
415                | TokenKind::Enum
416                | TokenKind::Extends
417                | TokenKind::False
418                | TokenKind::Finally
419                | TokenKind::Final
420                | TokenKind::Fn
421                | TokenKind::Foreach
422                | TokenKind::For
423                | TokenKind::Function
424                | TokenKind::Goto
425                | TokenKind::If
426                | TokenKind::IncludeOnce
427                | TokenKind::Include
428                | TokenKind::Implements
429                | TokenKind::Interface
430                | TokenKind::Instanceof
431                | TokenKind::Namespace
432                | TokenKind::New
433                | TokenKind::Null
434                | TokenKind::Private
435                | TokenKind::PrivateSet
436                | TokenKind::Protected
437                | TokenKind::Public
438                | TokenKind::RequireOnce
439                | TokenKind::Require
440                | TokenKind::Return
441                | TokenKind::Static
442                | TokenKind::Switch
443                | TokenKind::Throw
444                | TokenKind::Trait
445                | TokenKind::True
446                | TokenKind::Try
447                | TokenKind::Use
448                | TokenKind::Var
449                | TokenKind::Yield
450                | TokenKind::While
451                | TokenKind::Insteadof
452                | TokenKind::List
453                | TokenKind::Self_
454                | TokenKind::Parent
455                | TokenKind::DirConstant
456                | TokenKind::FileConstant
457                | TokenKind::LineConstant
458                | TokenKind::FunctionConstant
459                | TokenKind::ClassConstant
460                | TokenKind::MethodConstant
461                | TokenKind::TraitConstant
462                | TokenKind::NamespaceConstant
463                | TokenKind::HaltCompiler
464        )
465    }
466
467    #[inline]
468    pub const fn is_infix(&self) -> bool {
469        matches!(
470            self,
471            T!["**"
472                | ">>="
473                | "<<="
474                | "^="
475                | "&="
476                | "|="
477                | "%="
478                | "**="
479                | "and"
480                | "or"
481                | "xor"
482                | "<=>"
483                | "<<"
484                | ">>"
485                | "&"
486                | "|"
487                | "^"
488                | "%"
489                | "instanceof"
490                | "*"
491                | "/"
492                | "+"
493                | "-"
494                | "."
495                | "<"
496                | ">"
497                | "<="
498                | ">="
499                | "=="
500                | "==="
501                | "!="
502                | "!=="
503                | "<>"
504                | "?"
505                | "&&"
506                | "||"
507                | "="
508                | "+="
509                | "-="
510                | ".="
511                | "??="
512                | "/="
513                | "*="
514                | "??"
515                | "|>"]
516        )
517    }
518
519    #[inline]
520    pub const fn is_postfix(&self) -> bool {
521        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
522    }
523
524    #[inline]
525    pub const fn is_visibility_modifier(&self) -> bool {
526        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
527    }
528
529    #[inline]
530    pub const fn is_modifier(&self) -> bool {
531        matches!(
532            self,
533            T!["public"
534                | "protected"
535                | "private"
536                | "private(set)"
537                | "protected(set)"
538                | "public(set)"
539                | "static"
540                | "final"
541                | "abstract"
542                | "readonly"]
543        )
544    }
545
546    #[inline]
547    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
548        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
549    }
550
551    #[inline]
552    pub const fn is_identifier_maybe_reserved(&self) -> bool {
553        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
554    }
555
556    #[inline]
557    pub const fn is_soft_reserved_identifier(&self) -> bool {
558        matches!(
559            self,
560            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
561        )
562    }
563
564    #[inline]
565    pub const fn is_reserved_identifier(&self) -> bool {
566        if self.is_soft_reserved_identifier() {
567            return true;
568        }
569
570        matches!(
571            self,
572            T!["static"
573                | "abstract"
574                | "final"
575                | "for"
576                | "private"
577                | "private(set)"
578                | "protected"
579                | "protected(set)"
580                | "public"
581                | "public(set)"
582                | "include"
583                | "include_once"
584                | "eval"
585                | "require"
586                | "require_once"
587                | "or"
588                | "xor"
589                | "and"
590                | "instanceof"
591                | "new"
592                | "clone"
593                | "exit"
594                | "die"
595                | "if"
596                | "elseif"
597                | "else"
598                | "endif"
599                | "echo"
600                | "do"
601                | "while"
602                | "endwhile"
603                | "endfor"
604                | "foreach"
605                | "endforeach"
606                | "declare"
607                | "enddeclare"
608                | "as"
609                | "try"
610                | "catch"
611                | "finally"
612                | "throw"
613                | "use"
614                | "insteadof"
615                | "global"
616                | "var"
617                | "unset"
618                | "isset"
619                | "empty"
620                | "continue"
621                | "goto"
622                | "function"
623                | "const"
624                | "return"
625                | "print"
626                | "yield"
627                | "list"
628                | "switch"
629                | "endswitch"
630                | "case"
631                | "default"
632                | "break"
633                | "array"
634                | "callable"
635                | "extends"
636                | "implements"
637                | "namespace"
638                | "trait"
639                | "interface"
640                | "class"
641                | "__CLASS__"
642                | "__TRAIT__"
643                | "__FUNCTION__"
644                | "__METHOD__"
645                | "__LINE__"
646                | "__FILE__"
647                | "__DIR__"
648                | "__NAMESPACE__"
649                | "__halt_compiler"
650                | "fn"
651                | "match"]
652        )
653    }
654
655    #[inline]
656    pub const fn is_literal(&self) -> bool {
657        matches!(
658            self,
659            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
660        )
661    }
662
663    #[inline]
664    pub const fn is_magic_constant(&self) -> bool {
665        matches!(
666            self,
667            T!["__CLASS__"
668                | "__DIR__"
669                | "__FILE__"
670                | "__FUNCTION__"
671                | "__LINE__"
672                | "__METHOD__"
673                | "__NAMESPACE__"
674                | "__TRAIT__"]
675        )
676    }
677
678    #[inline]
679    pub const fn is_cast(&self) -> bool {
680        matches!(
681            self,
682            T!["(string)"
683                | "(binary)"
684                | "(int)"
685                | "(integer)"
686                | "(float)"
687                | "(double)"
688                | "(real)"
689                | "(bool)"
690                | "(boolean)"
691                | "(array)"
692                | "(object)"
693                | "(unset)"
694                | "(void)"]
695        )
696    }
697
698    #[inline]
699    pub const fn is_unary_prefix(&self) -> bool {
700        if self.is_cast() {
701            return true;
702        }
703
704        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
705    }
706
707    #[inline]
708    pub const fn is_trivia(&self) -> bool {
709        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
710    }
711
712    #[inline]
713    pub const fn is_comment(&self) -> bool {
714        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
715    }
716
717    #[inline]
718    pub const fn is_comma(&self) -> bool {
719        matches!(self, T![","])
720    }
721
722    #[inline]
723    pub const fn is_construct(&self) -> bool {
724        matches!(
725            self,
726            T!["isset"
727                | "empty"
728                | "eval"
729                | "include"
730                | "include_once"
731                | "require"
732                | "require_once"
733                | "print"
734                | "unset"
735                | "exit"
736                | "die"]
737        )
738    }
739}
740
741impl<'arena> Token<'arena> {
742    pub const fn new(kind: TokenKind, value: &'arena str, span: Span) -> Self {
743        Self { kind, value, span }
744    }
745}
746
747impl<'arena> std::fmt::Display for Token<'arena> {
748    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
749        write!(f, "{}({})", self.kind, self.value)
750    }
751}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
