mago_syntax/token/
mod.rs

1use serde::Deserialize;
2use serde::Serialize;
3use strum::Display;
4
5use mago_interner::StringIdentifier;
6use mago_span::Span;
7
8use crate::T;
9
10#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
11#[serde(tag = "type", content = "value")]
12#[repr(C)]
13pub enum DocumentKind {
14    Heredoc,
15    Nowdoc,
16}
17
18#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
19#[serde(tag = "type", content = "value")]
20#[repr(C)]
21pub enum Associativity {
22    NonAssociative,
23    Left,
24    Right,
25}
26
27#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
28#[serde(tag = "type", content = "value")]
29#[repr(C)]
30pub enum Precedence {
31    Lowest,
32    LowLogicalOr,
33    LowLogicalXor,
34    LowLogicalAnd,
35    Print,
36    Yield,
37    YieldFrom,
38    IncDec,
39    KeyOr,
40    KeyXor,
41    KeyAnd,
42    Assignment,
43    ElvisOrConditional,
44    NullCoalesce,
45    Or,
46    And,
47    BitwiseOr,
48    BitwiseXor,
49    BitwiseAnd,
50    Equality,
51    Comparison,
52    Concat,
53    BitShift,
54    AddSub,
55    MulDivMod,
56    Bang,
57    Instanceof,
58    Prefix,
59    Pow,
60    Clone,
61    CallDim,
62    New,
63    ArrayDim,
64    ObjectAccess,
65}
66
67pub trait GetPrecedence {
68    fn precedence(&self) -> Precedence;
69}
70
71#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord, Display)]
72#[serde(tag = "type", content = "value")]
73#[repr(C)]
74pub enum TokenKind {
75    Whitespace,                  // ` `
76    Eval,                        // `eval`
77    Die,                         // `die`
78    Self_,                       // `self`
79    Parent,                      // `parent`
80    Backtick,                    // `` ` ``
81    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
82    DocumentEnd,                 // `abc`
83    From,                        // `from`
84    Print,                       // `print`
85    Dollar,                      // `$`
86    HaltCompiler,                // `__halt_compiler`
87    Readonly,                    // `readonly`
88    Global,                      // `global`
89    Abstract,                    // `abstract`
90    Ampersand,                   // `&`
91    AmpersandEqual,              // `&=`
92    AmpersandAmpersand,          // `&&`
93    AmpersandAmpersandEqual,     // `&&=`
94    Array,                       // `array`
95    ArrayCast,                   // `(array)`
96    MinusGreaterThan,            // `->`
97    QuestionMinusGreaterThan,    // `?->`
98    At,                          // `@`
99    As,                          // `as`
100    Asterisk,                    // `*`
101    HashLeftBracket,             // `#[`
102    Bang,                        // `!`
103    BangEqual,                   // `!=`
104    LessThanGreaterThan,         // `<>`
105    BangEqualEqual,              // `!==`
106    LessThanEqualGreaterThan,    // `<=>`
107    BoolCast,                    // `(bool)`
108    BooleanCast,                 // `(boolean)`
109    And,                         // `and`
110    Or,                          // `or`
111    Break,                       // `break`
112    Callable,                    // `callable`
113    Caret,                       // `^`
114    CaretEqual,                  // `^=`
115    Case,                        // `case`
116    Catch,                       // `catch`
117    Class,                       // `class`
118    ClassConstant,               // `__CLASS__`
119    TraitConstant,               // `__TRAIT__`
120    FunctionConstant,            // `__FUNCTION__`
121    MethodConstant,              // `__METHOD__`
122    LineConstant,                // `__LINE__`
123    FileConstant,                // `__FILE__`
124    Clone,                       // `clone`
125    MinusEqual,                  // `-=`
126    CloseTag,                    // `?>`
127    QuestionQuestion,            // `??`
128    QuestionQuestionEqual,       // `??=`
129    AsteriskEqual,               // `*=`
130    Colon,                       // `:`
131    Comma,                       // `,`
132    SingleLineComment,           // `// comment`
133    HashComment,                 // `# comment`
134    MultiLineComment,            // `/* comment */`
135    DocBlockComment,             // `/** comment */`
136    Const,                       // `const`
137    PartialLiteralString,        // `"string` or `'string`, missing closing quote
138    LiteralString,               // `"string"` or `'string'`
139    Continue,                    // `continue`
140    Declare,                     // `declare`
141    MinusMinus,                  // `--`
142    Default,                     // `default`
143    DirConstant,                 // `__DIR__`
144    SlashEqual,                  // `/=`
145    Do,                          // `do`
146    DollarLeftBrace,             // `${`
147    Dot,                         // `.`
148    DotEqual,                    // `.=`
149    EqualGreaterThan,            // `=>`
150    DoubleCast,                  // `(double)`
151    RealCast,                    // `(real)`
152    FloatCast,                   // `(float)`
153    ColonColon,                  // `::`
154    EqualEqual,                  // `==`
155    DoubleQuote,                 // `"`
156    Else,                        // `else`
157    Echo,                        // `echo`
158    DotDotDot,                   // `...`
159    ElseIf,                      // `elseif`
160    Empty,                       // `empty`
161    EndDeclare,                  // `enddeclare`
162    EndFor,                      // `endfor`
163    EndForeach,                  // `endforeach`
164    EndIf,                       // `endif`
165    EndSwitch,                   // `endswitch`
166    EndWhile,                    // `endwhile`
167    Enum,                        // `enum`
168    Equal,                       // `=`
169    Extends,                     // `extends`
170    False,                       // `false`
171    Final,                       // `final`
172    Finally,                     // `finally`
173    LiteralFloat,                // `1.0`
174    Fn,                          // `fn`
175    For,                         // `for`
176    Foreach,                     // `foreach`
177    FullyQualifiedIdentifier,    // `\Namespace\Class`
178    Function,                    // `function`
179    Goto,                        // `goto`
180    GreaterThan,                 // `>`
181    GreaterThanEqual,            // `>=`
182    Identifier,                  // `name`
183    If,                          // `if`
184    Implements,                  // `implements`
185    Include,                     // `include`
186    IncludeOnce,                 // `include_once`
187    PlusPlus,                    // `++`
188    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
189    InlineShebang,               // `#!...`
190    Instanceof,                  // `instanceof`
191    Insteadof,                   // `insteadof`
192    Exit,                        // `exit`
193    Unset,                       // `unset`
194    Isset,                       // `isset`
195    List,                        // `list`
196    LiteralInteger,              // `1`
197    IntCast,                     // `(int)`
198    IntegerCast,                 // `(integer)`
199    Interface,                   // `interface`
200    LeftBrace,                   // `{`
201    LeftBracket,                 // `[`
202    LeftParenthesis,             // `(`
203    LeftShift,                   // `<<`
204    LeftShiftEqual,              // `<<=`
205    RightShift,                  // `>>`
206    RightShiftEqual,             // `>>=`
207    LessThan,                    // `<`
208    LessThanEqual,               // `<=`
209    Match,                       // `match`
210    Minus,                       // `-`
211    Namespace,                   // `namespace`
212    NamespaceSeparator,          // `\`
213    NamespaceConstant,           // `__NAMESPACE__`
214    New,                         // `new`
215    Null,                        // `null`
216    ObjectCast,                  // `(object)`
217    UnsetCast,                   // `(unset)`
218    OpenTag,                     // `<?php`
219    EchoTag,                     // `<?=`
220    ShortOpenTag,                // `<?`
221    Percent,                     // `%`
222    PercentEqual,                // `%=`
223    Pipe,                        // `|`
224    PipeEqual,                   // `|=`
225    Plus,                        // `+`
226    PlusEqual,                   // `+=`
227    AsteriskAsterisk,            // `**`
228    AsteriskAsteriskEqual,       // `**=`
229    Private,                     // `private`
230    PrivateSet,                  // `private(set)`
231    Protected,                   // `protected`
232    ProtectedSet,                // `protected(set)`
233    Public,                      // `public`
234    PublicSet,                   // `public(set)`
235    QualifiedIdentifier,         // `Namespace\Class`
236    Question,                    // `?`
237    QuestionColon,               // `?:`
238    Require,                     // `require`
239    RequireOnce,                 // `require_once`
240    Return,                      // `return`
241    RightBrace,                  // `}`
242    RightBracket,                // `]`
243    RightParenthesis,            // `)`
244    Semicolon,                   // `;`
245    Slash,                       // `/`
246    Static,                      // `static`
247    StringCast,                  // `(string)`
248    BinaryCast,                  // `(binary)`
249    StringPart,                  // `string` inside a double-quoted string, or a document string
250    Switch,                      // `switch`
251    Throw,                       // `throw`
252    Trait,                       // `trait`
253    EqualEqualEqual,             // `===`
254    True,                        // `true`
255    Try,                         // `try`
256    Use,                         // `use`
257    Var,                         // `var`
258    Variable,                    // `$name`
259    Yield,                       // `yield`
260    While,                       // `while`
261    Tilde,                       // `~`
262    PipePipe,                    // `||`
263    Xor,                         // `xor`
264}
265
266#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
267#[repr(C)]
268pub struct Token {
269    pub kind: TokenKind,
270    pub value: StringIdentifier,
271    pub span: Span,
272}
273
274impl Precedence {
275    #[inline(always)]
276    pub fn infix(kind: &TokenKind) -> Precedence {
277        match kind {
278            T!["**"] => Precedence::Pow,
279            T!["instanceof"] => Precedence::Instanceof,
280            T!["*" | "/" | "%"] => Precedence::MulDivMod,
281            T!["+" | "-"] => Precedence::AddSub,
282            T!["<<"] | T![">>"] => Precedence::BitShift,
283            T!["."] => Precedence::Concat,
284            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
285            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
286            T!["&"] => Precedence::BitwiseAnd,
287            T!["^"] => Precedence::BitwiseXor,
288            T!["|"] => Precedence::BitwiseOr,
289            T!["&&"] => Precedence::And,
290            T!["||"] => Precedence::Or,
291            T!["??"] => Precedence::NullCoalesce,
292            T!["?" | "?:"] => Precedence::ElvisOrConditional,
293            T!["="
294                | "+="
295                | "-="
296                | "*="
297                | "**="
298                | "/="
299                | ".="
300                | "&&="
301                | "??="
302                | "%="
303                | "&="
304                | "|="
305                | "^="
306                | "<<="
307                | ">>="] => Precedence::Assignment,
308            T!["yield"] => Precedence::Yield,
309            T!["and"] => Precedence::KeyAnd,
310            T!["or"] => Precedence::KeyOr,
311            T!["xor"] => Precedence::KeyXor,
312            _ => Precedence::Lowest,
313        }
314    }
315
316    #[inline(always)]
317    pub fn postfix(kind: &TokenKind) -> Self {
318        match kind {
319            T!["++" | "--"] => Self::Prefix,
320            T!["("] => Self::CallDim,
321            T!["["] => Self::ArrayDim,
322            T!["->" | "?->" | "::"] => Self::ObjectAccess,
323            _ => Self::Lowest,
324        }
325    }
326
327    #[inline(always)]
328    pub fn associativity(&self) -> Option<Associativity> {
329        Some(match self {
330            Self::Instanceof
331            | Self::MulDivMod
332            | Self::AddSub
333            | Self::BitShift
334            | Self::Concat
335            | Self::BitwiseAnd
336            | Self::BitwiseOr
337            | Self::BitwiseXor
338            | Self::And
339            | Self::Or
340            | Self::KeyAnd
341            | Self::KeyOr
342            | Self::KeyXor => Associativity::Left,
343            Self::Pow | Self::NullCoalesce | Self::Assignment => Associativity::Right,
344            Self::ElvisOrConditional | Self::Equality | Self::Comparison => Associativity::NonAssociative,
345            _ => return None,
346        })
347    }
348}
349
350impl TokenKind {
351    #[inline(always)]
352    pub const fn is_keyword(&self) -> bool {
353        matches!(
354            self,
355            TokenKind::Eval
356                | TokenKind::Die
357                | TokenKind::Empty
358                | TokenKind::Isset
359                | TokenKind::Unset
360                | TokenKind::Exit
361                | TokenKind::EndDeclare
362                | TokenKind::EndSwitch
363                | TokenKind::EndWhile
364                | TokenKind::EndForeach
365                | TokenKind::EndFor
366                | TokenKind::EndIf
367                | TokenKind::From
368                | TokenKind::And
369                | TokenKind::Or
370                | TokenKind::Xor
371                | TokenKind::Print
372                | TokenKind::Readonly
373                | TokenKind::Global
374                | TokenKind::Match
375                | TokenKind::Abstract
376                | TokenKind::Array
377                | TokenKind::As
378                | TokenKind::Break
379                | TokenKind::Case
380                | TokenKind::Catch
381                | TokenKind::Class
382                | TokenKind::Clone
383                | TokenKind::Continue
384                | TokenKind::Const
385                | TokenKind::Declare
386                | TokenKind::Default
387                | TokenKind::Do
388                | TokenKind::Echo
389                | TokenKind::ElseIf
390                | TokenKind::Else
391                | TokenKind::Enum
392                | TokenKind::Extends
393                | TokenKind::False
394                | TokenKind::Finally
395                | TokenKind::Final
396                | TokenKind::Fn
397                | TokenKind::Foreach
398                | TokenKind::For
399                | TokenKind::Function
400                | TokenKind::Goto
401                | TokenKind::If
402                | TokenKind::IncludeOnce
403                | TokenKind::Include
404                | TokenKind::Implements
405                | TokenKind::Interface
406                | TokenKind::Instanceof
407                | TokenKind::Namespace
408                | TokenKind::New
409                | TokenKind::Null
410                | TokenKind::Private
411                | TokenKind::PrivateSet
412                | TokenKind::Protected
413                | TokenKind::Public
414                | TokenKind::RequireOnce
415                | TokenKind::Require
416                | TokenKind::Return
417                | TokenKind::Static
418                | TokenKind::Switch
419                | TokenKind::Throw
420                | TokenKind::Trait
421                | TokenKind::True
422                | TokenKind::Try
423                | TokenKind::Use
424                | TokenKind::Var
425                | TokenKind::Yield
426                | TokenKind::While
427                | TokenKind::Insteadof
428                | TokenKind::List
429                | TokenKind::Self_
430                | TokenKind::Parent
431                | TokenKind::DirConstant
432                | TokenKind::FileConstant
433                | TokenKind::LineConstant
434                | TokenKind::FunctionConstant
435                | TokenKind::ClassConstant
436                | TokenKind::MethodConstant
437                | TokenKind::TraitConstant
438                | TokenKind::NamespaceConstant
439                | TokenKind::HaltCompiler
440        )
441    }
442
443    #[inline(always)]
444    pub const fn is_infix(&self) -> bool {
445        matches!(
446            self,
447            T!["**"
448                | ">>="
449                | "<<="
450                | "^="
451                | "&="
452                | "|="
453                | "%="
454                | "**="
455                | "and"
456                | "or"
457                | "xor"
458                | "<=>"
459                | "<<"
460                | ">>"
461                | "&"
462                | "|"
463                | "^"
464                | "%"
465                | "instanceof"
466                | "*"
467                | "/"
468                | "+"
469                | "-"
470                | "."
471                | "<"
472                | ">"
473                | "<="
474                | ">="
475                | "=="
476                | "==="
477                | "!="
478                | "!=="
479                | "<>"
480                | "?"
481                | "?:"
482                | "&&"
483                | "||"
484                | "="
485                | "+="
486                | "-="
487                | ".="
488                | "??="
489                | "/="
490                | "*="
491                | "??"]
492        )
493    }
494
495    #[inline(always)]
496    pub const fn is_postfix(&self) -> bool {
497        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
498    }
499
500    #[inline(always)]
501    pub const fn is_visibility_modifier(&self) -> bool {
502        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
503    }
504
505    #[inline(always)]
506    pub const fn is_modifier(&self) -> bool {
507        matches!(
508            self,
509            T!["public"
510                | "protected"
511                | "private"
512                | "private(set)"
513                | "protected(set)"
514                | "public(set)"
515                | "static"
516                | "final"
517                | "abstract"
518                | "readonly"]
519        )
520    }
521
522    #[inline(always)]
523    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
524        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
525    }
526
527    #[inline(always)]
528    pub const fn is_identifier_maybe_reserved(&self) -> bool {
529        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
530    }
531
532    #[inline(always)]
533    pub const fn is_soft_reserved_identifier(&self) -> bool {
534        matches!(
535            self,
536            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
537        )
538    }
539
540    #[inline(always)]
541    pub const fn is_reserved_identifier(&self) -> bool {
542        if self.is_soft_reserved_identifier() {
543            return true;
544        }
545
546        matches!(
547            self,
548            T!["static"
549                | "abstract"
550                | "final"
551                | "for"
552                | "private"
553                | "private(set)"
554                | "protected"
555                | "protected(set)"
556                | "public"
557                | "public(set)"
558                | "include"
559                | "include_once"
560                | "eval"
561                | "require"
562                | "require_once"
563                | "or"
564                | "xor"
565                | "and"
566                | "instanceof"
567                | "new"
568                | "clone"
569                | "exit"
570                | "die"
571                | "if"
572                | "elseif"
573                | "else"
574                | "endif"
575                | "echo"
576                | "do"
577                | "while"
578                | "endwhile"
579                | "endfor"
580                | "foreach"
581                | "endforeach"
582                | "declare"
583                | "enddeclare"
584                | "as"
585                | "try"
586                | "catch"
587                | "finally"
588                | "throw"
589                | "use"
590                | "insteadof"
591                | "global"
592                | "var"
593                | "unset"
594                | "isset"
595                | "empty"
596                | "continue"
597                | "goto"
598                | "function"
599                | "const"
600                | "return"
601                | "print"
602                | "yield"
603                | "list"
604                | "switch"
605                | "endswitch"
606                | "case"
607                | "default"
608                | "break"
609                | "array"
610                | "callable"
611                | "extends"
612                | "implements"
613                | "namespace"
614                | "trait"
615                | "interface"
616                | "class"
617                | "__CLASS__"
618                | "__TRAIT__"
619                | "__FUNCTION__"
620                | "__METHOD__"
621                | "__LINE__"
622                | "__FILE__"
623                | "__DIR__"
624                | "__NAMESPACE__"
625                | "__halt_compiler"
626                | "fn"
627                | "match"]
628        )
629    }
630
631    #[inline(always)]
632    pub const fn is_literal(&self) -> bool {
633        matches!(
634            self,
635            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
636        )
637    }
638
639    #[inline(always)]
640    pub const fn is_magic_constant(&self) -> bool {
641        matches!(
642            self,
643            T!["__CLASS__"
644                | "__DIR__"
645                | "__FILE__"
646                | "__FUNCTION__"
647                | "__LINE__"
648                | "__METHOD__"
649                | "__NAMESPACE__"
650                | "__TRAIT__"]
651        )
652    }
653
654    #[inline(always)]
655    pub const fn is_cast(&self) -> bool {
656        matches!(
657            self,
658            T!["(string)"
659                | "(binary)"
660                | "(int)"
661                | "(integer)"
662                | "(float)"
663                | "(double)"
664                | "(real)"
665                | "(bool)"
666                | "(boolean)"
667                | "(array)"
668                | "(object)"
669                | "(unset)"]
670        )
671    }
672
673    #[inline(always)]
674    pub const fn is_unary_prefix(&self) -> bool {
675        if self.is_cast() {
676            return true;
677        }
678
679        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--" | "&"])
680    }
681
682    #[inline(always)]
683    pub const fn is_trivia(&self) -> bool {
684        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
685    }
686
687    #[inline(always)]
688    pub const fn is_comment(&self) -> bool {
689        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
690    }
691
692    #[inline(always)]
693    pub const fn is_construct(&self) -> bool {
694        matches!(
695            self,
696            T!["isset"
697                | "empty"
698                | "eval"
699                | "include"
700                | "include_once"
701                | "require"
702                | "require_once"
703                | "print"
704                | "unset"
705                | "exit"
706                | "die"]
707        )
708    }
709}
710
711impl Token {
712    pub fn new(kind: TokenKind, value: StringIdentifier, span: Span) -> Self {
713        Self { kind, value, span }
714    }
715}
716
717impl std::fmt::Display for Token {
718    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
719        write!(f, "{:?} {}", self.kind, self.span)
720    }
721}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
