php_lexer/
token.rs

1/// All token types produced by the PHP lexer.
2///
3/// The variants are grouped by category in source order; the `#[repr(u8)]`
4/// guarantees a compact discriminant layout used by [`TokenKind::is_assignment_op`].
5#[repr(u8)]
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7#[non_exhaustive]
8pub enum TokenKind {
9    // -------------------------------------------------------------------------
10    // Literals
11    // -------------------------------------------------------------------------
12    /// Float in scientific notation: `1e10`, `1.5e-3`, `1E+10`.
13    FloatLiteral,
14
15    /// Float with digits on both sides of the decimal point: `1.5`, `0.0`.
16    FloatLiteralSimple,
17
18    /// Float starting with a leading dot: `.5`, `.001`.
19    FloatLiteralLeadingDot,
20
21    /// Hexadecimal integer literal: `0x1A`, `0xFF`.
22    HexIntLiteral,
23
24    /// Binary integer literal: `0b1010`, `0B0011`.
25    BinIntLiteral,
26
27    /// Octal integer literal in the new `0o` prefix form: `0o17` (PHP 8.1+).
28    OctIntLiteralNew,
29
30    /// Octal integer literal in the legacy leading-zero form: `017`.
31    OctIntLiteral,
32
33    /// Decimal integer literal: `0`, `42`, `1_000_000`.
34    IntLiteral,
35
36    /// Single-quoted string literal: `'hello'`. Optional `b`/`B` prefix allowed.
37    SingleQuotedString,
38
39    /// Double-quoted string literal: `"hello $name"`. Triggers interpolation processing.
40    DoubleQuotedString,
41
42    /// Backtick string (shell execution): `` `ls -la` ``.
43    BacktickString,
44
45    // -------------------------------------------------------------------------
46    // Variables
47    // -------------------------------------------------------------------------
48    /// A `$`-prefixed variable name: `$foo`, `$myVar`.
49    Variable,
50
51    /// A bare `$` not followed by a valid identifier start (variable-variable prefix).
52    Dollar,
53
54    // -------------------------------------------------------------------------
55    // Identifiers
56    // -------------------------------------------------------------------------
57    /// A bare identifier or keyword that has not yet been resolved.
58    /// PHP keywords are case-insensitive and are resolved from this token.
59    Identifier,
60
61    // -------------------------------------------------------------------------
62    // Arithmetic operators
63    // -------------------------------------------------------------------------
64    /// `+`
65    Plus,
66    /// `-`
67    Minus,
68    /// `*`
69    Star,
70    /// `/`
71    Slash,
72    /// `%`
73    Percent,
74    /// `**` — exponentiation.
75    StarStar,
76    /// `.` — string concatenation.
77    Dot,
78
79    // -------------------------------------------------------------------------
80    // Assignment operators
81    // -------------------------------------------------------------------------
82    // NOTE: Equals..=CoalesceEquals must remain contiguous; is_assignment_op relies on this.
83    /// `=`
84    Equals,
85    /// `+=`
86    PlusEquals,
87    /// `-=`
88    MinusEquals,
89    /// `*=`
90    StarEquals,
91    /// `/=`
92    SlashEquals,
93    /// `%=`
94    PercentEquals,
95    /// `**=`
96    StarStarEquals,
97    /// `.=`
98    DotEquals,
99    /// `&=`
100    AmpersandEquals,
101    /// `|=`
102    PipeEquals,
103    /// `^=`
104    CaretEquals,
105    /// `<<=`
106    ShiftLeftEquals,
107    /// `>>=`
108    ShiftRightEquals,
109    /// `??=`
110    CoalesceEquals,
111
112    // -------------------------------------------------------------------------
113    // Comparison operators
114    // -------------------------------------------------------------------------
115    /// `==` — loose equality.
116    EqualsEquals,
117    /// `!=` or `<>` — loose inequality.
118    BangEquals,
119    /// `===` — strict equality.
120    EqualsEqualsEquals,
121    /// `!==` — strict inequality.
122    BangEqualsEquals,
123    /// `<`
124    LessThan,
125    /// `>`
126    GreaterThan,
127    /// `<=`
128    LessThanEquals,
129    /// `>=`
130    GreaterThanEquals,
131    /// `<=>` — spaceship / three-way comparison.
132    Spaceship,
133
134    // -------------------------------------------------------------------------
135    // Logical / bitwise operators
136    // -------------------------------------------------------------------------
137    /// `&&` — short-circuit boolean AND.
138    AmpersandAmpersand,
139    /// `||` — short-circuit boolean OR.
140    PipePipe,
141    /// `!` — boolean NOT.
142    Bang,
143
144    /// `&` — bitwise AND (also reference marker in declarations).
145    Ampersand,
146    /// `|` — bitwise OR.
147    Pipe,
148    /// `^` — bitwise XOR.
149    Caret,
150    /// `~` — bitwise NOT.
151    Tilde,
152    /// `<<` — left bit-shift.
153    ShiftLeft,
154    /// `>>` — right bit-shift.
155    ShiftRight,
156
157    // -------------------------------------------------------------------------
158    // Increment / decrement
159    // -------------------------------------------------------------------------
160    /// `++`
161    PlusPlus,
162    /// `--`
163    MinusMinus,
164
165    // -------------------------------------------------------------------------
166    // Other operators
167    // -------------------------------------------------------------------------
168    /// `?` — ternary operator or nullable type-hint prefix.
169    Question,
170    /// `??` — null coalescing.
171    QuestionQuestion,
172    /// `:` — ternary separator, named argument separator, or return type separator.
173    Colon,
174
175    /// `=>` — key-value separator in arrays and `match` arms.
176    FatArrow,
177
178    /// `|>` — pipe operator (PHP 8.5+).
179    PipeArrow,
180
181    // -------------------------------------------------------------------------
182    // Delimiters & punctuation
183    // -------------------------------------------------------------------------
184    /// `(`
185    LeftParen,
186    /// `)`
187    RightParen,
188    /// `[`
189    LeftBracket,
190    /// `]`
191    RightBracket,
192    /// `{`
193    LeftBrace,
194    /// `}`
195    RightBrace,
196    /// `;`
197    Semicolon,
198    /// `,`
199    Comma,
200
201    /// `::` — static access separator.
202    DoubleColon,
203
204    /// `->` — object property / method access.
205    Arrow,
206
207    /// `?->` — nullsafe property / method access.
208    NullsafeArrow,
209
210    /// `\` — namespace separator.
211    Backslash,
212
213    /// `@` — error-suppression operator.
214    At,
215
216    /// `#[` — attribute syntax opener.
217    HashBracket,
218
219    /// `...` — splat / variadic operator.
220    Ellipsis,
221
222    // -------------------------------------------------------------------------
223    // Keywords (resolved from Identifier at lex time)
224    // -------------------------------------------------------------------------
225    /// `if`
226    If,
227    /// `else`
228    Else,
229    /// `elseif`
230    ElseIf,
231    /// `while`
232    While,
233    /// `do`
234    Do,
235    /// `for`
236    For,
237    /// `foreach`
238    Foreach,
239    /// `as`
240    As,
241    /// `function`
242    Function,
243    /// `return`
244    Return,
245    /// `echo`
246    Echo,
247    /// `print`
248    Print,
249    /// `true`
250    True,
251    /// `false`
252    False,
253    /// `null`
254    Null,
255    /// `and` — low-precedence boolean AND.
256    And,
257    /// `or` — low-precedence boolean OR.
258    Or,
259    /// `xor` — low-precedence boolean XOR.
260    Xor,
261    /// `break`
262    Break,
263    /// `continue`
264    Continue,
265    /// `switch`
266    Switch,
267    /// `case`
268    Case,
269    /// `default`
270    Default,
271    /// `endif` — alternative `if` syntax terminator.
272    EndIf,
273    /// `endwhile` — alternative `while` syntax terminator.
274    EndWhile,
275    /// `endfor` — alternative `for` syntax terminator.
276    EndFor,
277    /// `endforeach` — alternative `foreach` syntax terminator.
278    EndForeach,
279    /// `throw`
280    Throw,
281    /// `try`
282    Try,
283    /// `catch`
284    Catch,
285    /// `finally`
286    Finally,
287    /// `instanceof`
288    Instanceof,
289    /// `array` — as a keyword (e.g. `array(1, 2)` or `array` type hint).
290    Array,
291    /// `list` — destructuring construct: `list($a, $b) = ...`.
292    List,
293    /// `goto`
294    Goto,
295    /// `declare`
296    Declare,
297    /// `unset`
298    Unset,
299    /// `global`
300    Global,
301    /// `enddeclare` — alternative `declare` syntax terminator.
302    EndDeclare,
303    /// `endswitch` — alternative `switch` syntax terminator.
304    EndSwitch,
305    /// `isset`
306    Isset,
307    /// `empty`
308    Empty,
309    /// `include`
310    Include,
311    /// `include_once`
312    IncludeOnce,
313    /// `require`
314    Require,
315    /// `require_once`
316    RequireOnce,
317    /// `eval`
318    Eval,
319    /// `exit`
320    Exit,
321    /// `die` — alias for `exit`.
322    Die,
323    /// `clone`
324    Clone,
325
326    // OOP keywords
327    /// `new`
328    New,
329    /// `class`
330    Class,
331    /// `abstract`
332    Abstract,
333    /// `final`
334    Final,
335    /// `interface`
336    Interface,
337    /// `trait`
338    Trait,
339    /// `extends`
340    Extends,
341    /// `implements`
342    Implements,
343    /// `public`
344    Public,
345    /// `protected`
346    Protected,
347    /// `private`
348    Private,
349    /// `static`
350    Static,
351    /// `const`
352    Const,
353    /// `fn` — arrow function keyword.
354    Fn_,
355    /// `match` — match expression keyword (PHP 8.0+).
356    Match_,
357    /// `namespace`
358    Namespace,
359    /// `use`
360    Use,
361    /// `readonly` (PHP 8.1+).
362    Readonly,
363    /// `enum` (PHP 8.1+).
364    Enum_,
365    /// `yield`
366    Yield_,
367    /// `from` — used in `yield from`.
368    From,
369    /// `self`
370    Self_,
371    /// `parent`
372    Parent_,
373
374    // -------------------------------------------------------------------------
375    // Magic constants (resolved from Identifier at lex time)
376    // -------------------------------------------------------------------------
377    /// `__CLASS__`
378    MagicClass,
379    /// `__DIR__`
380    MagicDir,
381    /// `__FILE__`
382    MagicFile,
383    /// `__FUNCTION__`
384    MagicFunction,
385    /// `__LINE__`
386    MagicLine,
387    /// `__METHOD__`
388    MagicMethod,
389    /// `__NAMESPACE__`
390    MagicNamespace,
391    /// `__TRAIT__`
392    MagicTrait,
393    /// `__PROPERTY__` (PHP 8.4+)
394    MagicProperty,
395    /// `__halt_compiler` — stops the parser; remaining bytes are raw data.
396    HaltCompiler,
397
398    // -------------------------------------------------------------------------
399    // PHP tags & template output
400    // -------------------------------------------------------------------------
401    /// `<?php` or `<?=` opening tag.
402    OpenTag,
403
404    /// `?>` closing tag; switches the lexer back to inline-HTML mode.
405    CloseTag,
406
407    /// Raw HTML text between PHP tags (produced by the `Lexer` wrapper, not the inner scanner).
408    InlineHtml,
409
410    // -------------------------------------------------------------------------
411    // Heredoc / Nowdoc (produced by Lexer wrapper)
412    // -------------------------------------------------------------------------
413    /// `<<<LABEL … LABEL;` — heredoc string (supports interpolation).
414    Heredoc,
415    /// `<<<'LABEL' … LABEL;` — nowdoc string (no interpolation).
416    Nowdoc,
417
418    // -------------------------------------------------------------------------
419    // Error tokens
420    // -------------------------------------------------------------------------
421    /// An invalid numeric literal, e.g. `1_000_` (trailing underscore).
422    /// The lexer emits this token and records a [`LexerError`](crate::LexerError) for it.
423    InvalidNumericLiteral,
424
425    // -------------------------------------------------------------------------
426    // Comments (filtered into a side-table by the parser)
427    // -------------------------------------------------------------------------
428    /// `// …` single-line slash comment.
429    LineComment,
430    /// `# …` single-line hash comment.
431    HashComment,
432    /// `/* … */` block comment.
433    BlockComment,
434    /// `/** … */` doc-block comment (first non-whitespace char after `/*` is `*`).
435    DocComment,
436
437    // -------------------------------------------------------------------------
438    // End of input
439    // -------------------------------------------------------------------------
440    /// Sentinel token emitted once all source bytes have been consumed.
441    Eof,
442}
443
444impl TokenKind {
445    /// Returns `true` for the four comment-token variants.
446    #[inline]
447    pub fn is_comment(self) -> bool {
448        matches!(
449            self,
450            TokenKind::LineComment
451                | TokenKind::HashComment
452                | TokenKind::BlockComment
453                | TokenKind::DocComment
454        )
455    }
456}
457
458impl TokenKind {
459    /// Whether this is `=` or a compound assignment operator.
460    #[inline(always)]
461    pub fn is_assignment_op(self) -> bool {
462        // The assignment operators are contiguous in the enum definition:
463        // Equals..=CoalesceEquals. With #[repr(u8)], a single range check suffices.
464        (self as u8).wrapping_sub(TokenKind::Equals as u8)
465            <= (TokenKind::CoalesceEquals as u8 - TokenKind::Equals as u8)
466    }
467}
468
469/// Resolve a keyword from an identifier string. Returns the keyword TokenKind
470/// if the string is a keyword, or None if it's a plain identifier.
471pub fn resolve_keyword(text: &str) -> Option<TokenKind> {
472    // PHP keywords are case-insensitive; use eq_ignore_ascii_case to avoid allocation.
473    // Dispatch on length first to reduce comparisons.
474    let t = text;
475    match text.len() {
476        2 => {
477            if t.eq_ignore_ascii_case("if") {
478                return Some(TokenKind::If);
479            }
480            if t.eq_ignore_ascii_case("do") {
481                return Some(TokenKind::Do);
482            }
483            if t.eq_ignore_ascii_case("or") {
484                return Some(TokenKind::Or);
485            }
486            if t.eq_ignore_ascii_case("as") {
487                return Some(TokenKind::As);
488            }
489            if t.eq_ignore_ascii_case("fn") {
490                return Some(TokenKind::Fn_);
491            }
492        }
493        3 => {
494            if t.eq_ignore_ascii_case("for") {
495                return Some(TokenKind::For);
496            }
497            if t.eq_ignore_ascii_case("xor") {
498                return Some(TokenKind::Xor);
499            }
500            if t.eq_ignore_ascii_case("and") {
501                return Some(TokenKind::And);
502            }
503            if t.eq_ignore_ascii_case("new") {
504                return Some(TokenKind::New);
505            }
506            if t.eq_ignore_ascii_case("use") {
507                return Some(TokenKind::Use);
508            }
509            if t.eq_ignore_ascii_case("try") {
510                return Some(TokenKind::Try);
511            }
512            if t.eq_ignore_ascii_case("die") {
513                return Some(TokenKind::Die);
514            }
515        }
516        4 => {
517            if t.eq_ignore_ascii_case("else") {
518                return Some(TokenKind::Else);
519            }
520            if t.eq_ignore_ascii_case("echo") {
521                return Some(TokenKind::Echo);
522            }
523            if t.eq_ignore_ascii_case("true") {
524                return Some(TokenKind::True);
525            }
526            if t.eq_ignore_ascii_case("null") {
527                return Some(TokenKind::Null);
528            }
529            if t.eq_ignore_ascii_case("list") {
530                return Some(TokenKind::List);
531            }
532            if t.eq_ignore_ascii_case("goto") {
533                return Some(TokenKind::Goto);
534            }
535            if t.eq_ignore_ascii_case("case") {
536                return Some(TokenKind::Case);
537            }
538            if t.eq_ignore_ascii_case("self") {
539                return Some(TokenKind::Self_);
540            }
541            if t.eq_ignore_ascii_case("from") {
542                return Some(TokenKind::From);
543            }
544            if t.eq_ignore_ascii_case("enum") {
545                return Some(TokenKind::Enum_);
546            }
547            if t.eq_ignore_ascii_case("eval") {
548                return Some(TokenKind::Eval);
549            }
550            if t.eq_ignore_ascii_case("exit") {
551                return Some(TokenKind::Exit);
552            }
553        }
554        5 => {
555            if t.eq_ignore_ascii_case("while") {
556                return Some(TokenKind::While);
557            }
558            if t.eq_ignore_ascii_case("false") {
559                return Some(TokenKind::False);
560            }
561            if t.eq_ignore_ascii_case("array") {
562                return Some(TokenKind::Array);
563            }
564            if t.eq_ignore_ascii_case("unset") {
565                return Some(TokenKind::Unset);
566            }
567            if t.eq_ignore_ascii_case("isset") {
568                return Some(TokenKind::Isset);
569            }
570            if t.eq_ignore_ascii_case("empty") {
571                return Some(TokenKind::Empty);
572            }
573            if t.eq_ignore_ascii_case("print") {
574                return Some(TokenKind::Print);
575            }
576            if t.eq_ignore_ascii_case("throw") {
577                return Some(TokenKind::Throw);
578            }
579            if t.eq_ignore_ascii_case("catch") {
580                return Some(TokenKind::Catch);
581            }
582            if t.eq_ignore_ascii_case("break") {
583                return Some(TokenKind::Break);
584            }
585            if t.eq_ignore_ascii_case("yield") {
586                return Some(TokenKind::Yield_);
587            }
588            if t.eq_ignore_ascii_case("class") {
589                return Some(TokenKind::Class);
590            }
591            if t.eq_ignore_ascii_case("const") {
592                return Some(TokenKind::Const);
593            }
594            if t.eq_ignore_ascii_case("final") {
595                return Some(TokenKind::Final);
596            }
597            if t.eq_ignore_ascii_case("match") {
598                return Some(TokenKind::Match_);
599            }
600            if t.eq_ignore_ascii_case("trait") {
601                return Some(TokenKind::Trait);
602            }
603            if t.eq_ignore_ascii_case("clone") {
604                return Some(TokenKind::Clone);
605            }
606            if t.eq_ignore_ascii_case("endif") {
607                return Some(TokenKind::EndIf);
608            }
609        }
610        6 => {
611            if t.eq_ignore_ascii_case("elseif") {
612                return Some(TokenKind::ElseIf);
613            }
614            if t.eq_ignore_ascii_case("return") {
615                return Some(TokenKind::Return);
616            }
617            if t.eq_ignore_ascii_case("switch") {
618                return Some(TokenKind::Switch);
619            }
620            if t.eq_ignore_ascii_case("global") {
621                return Some(TokenKind::Global);
622            }
623            if t.eq_ignore_ascii_case("static") {
624                return Some(TokenKind::Static);
625            }
626            if t.eq_ignore_ascii_case("public") {
627                return Some(TokenKind::Public);
628            }
629            if t.eq_ignore_ascii_case("parent") {
630                return Some(TokenKind::Parent_);
631            }
632            if t.eq_ignore_ascii_case("endfor") {
633                return Some(TokenKind::EndFor);
634            }
635        }
636        7 => {
637            if t.eq_ignore_ascii_case("foreach") {
638                return Some(TokenKind::Foreach);
639            }
640            if t.eq_ignore_ascii_case("default") {
641                return Some(TokenKind::Default);
642            }
643            if t.eq_ignore_ascii_case("finally") {
644                return Some(TokenKind::Finally);
645            }
646            if t.eq_ignore_ascii_case("include") {
647                return Some(TokenKind::Include);
648            }
649            if t.eq_ignore_ascii_case("declare") {
650                return Some(TokenKind::Declare);
651            }
652            if t.eq_ignore_ascii_case("extends") {
653                return Some(TokenKind::Extends);
654            }
655            if t.eq_ignore_ascii_case("require") {
656                return Some(TokenKind::Require);
657            }
658            if t.eq_ignore_ascii_case("private") {
659                return Some(TokenKind::Private);
660            }
661            if t.eq_ignore_ascii_case("__dir__") {
662                return Some(TokenKind::MagicDir);
663            }
664        }
665        8 => {
666            if t.eq_ignore_ascii_case("function") {
667                return Some(TokenKind::Function);
668            }
669            if t.eq_ignore_ascii_case("abstract") {
670                return Some(TokenKind::Abstract);
671            }
672            if t.eq_ignore_ascii_case("readonly") {
673                return Some(TokenKind::Readonly);
674            }
675            if t.eq_ignore_ascii_case("continue") {
676                return Some(TokenKind::Continue);
677            }
678            if t.eq_ignore_ascii_case("endwhile") {
679                return Some(TokenKind::EndWhile);
680            }
681            if t.eq_ignore_ascii_case("__file__") {
682                return Some(TokenKind::MagicFile);
683            }
684            if t.eq_ignore_ascii_case("__line__") {
685                return Some(TokenKind::MagicLine);
686            }
687        }
688        9 => {
689            if t.eq_ignore_ascii_case("namespace") {
690                return Some(TokenKind::Namespace);
691            }
692            if t.eq_ignore_ascii_case("interface") {
693                return Some(TokenKind::Interface);
694            }
695            if t.eq_ignore_ascii_case("protected") {
696                return Some(TokenKind::Protected);
697            }
698            if t.eq_ignore_ascii_case("endswitch") {
699                return Some(TokenKind::EndSwitch);
700            }
701            if t.eq_ignore_ascii_case("__class__") {
702                return Some(TokenKind::MagicClass);
703            }
704            if t.eq_ignore_ascii_case("__trait__") {
705                return Some(TokenKind::MagicTrait);
706            }
707        }
708        10 => {
709            if t.eq_ignore_ascii_case("implements") {
710                return Some(TokenKind::Implements);
711            }
712            if t.eq_ignore_ascii_case("instanceof") {
713                return Some(TokenKind::Instanceof);
714            }
715            if t.eq_ignore_ascii_case("endforeach") {
716                return Some(TokenKind::EndForeach);
717            }
718            if t.eq_ignore_ascii_case("enddeclare") {
719                return Some(TokenKind::EndDeclare);
720            }
721            if t.eq_ignore_ascii_case("__method__") {
722                return Some(TokenKind::MagicMethod);
723            }
724        }
725        12 => {
726            if t.eq_ignore_ascii_case("include_once") {
727                return Some(TokenKind::IncludeOnce);
728            }
729            if t.eq_ignore_ascii_case("require_once") {
730                return Some(TokenKind::RequireOnce);
731            }
732            if t.eq_ignore_ascii_case("__function__") {
733                return Some(TokenKind::MagicFunction);
734            }
735            if t.eq_ignore_ascii_case("__property__") {
736                return Some(TokenKind::MagicProperty);
737            }
738        }
739        13 if t.eq_ignore_ascii_case("__namespace__") => {
740            return Some(TokenKind::MagicNamespace);
741        }
742        15 if t.eq_ignore_ascii_case("__halt_compiler") => {
743            return Some(TokenKind::HaltCompiler);
744        }
745        _ => {}
746    }
747    None
748}
749
750impl TokenKind {
751    /// A short human-readable description, as used in diagnostics
752    /// (e.g. `"';'"`, `"identifier"`).
753    pub fn description(self) -> &'static str {
754        match self {
755            TokenKind::IntLiteral => "integer",
756            TokenKind::HexIntLiteral => "hex integer",
757            TokenKind::BinIntLiteral => "binary integer",
758            TokenKind::OctIntLiteral | TokenKind::OctIntLiteralNew => "octal integer",
759            TokenKind::FloatLiteral
760            | TokenKind::FloatLiteralSimple
761            | TokenKind::FloatLiteralLeadingDot => "float",
762            TokenKind::SingleQuotedString | TokenKind::DoubleQuotedString => "string",
763            TokenKind::BacktickString => "backtick string",
764            TokenKind::Variable => "variable",
765            TokenKind::Dollar => "'$'",
766            TokenKind::Identifier => "identifier",
767            TokenKind::Plus => "'+'",
768            TokenKind::Minus => "'-'",
769            TokenKind::Star => "'*'",
770            TokenKind::Slash => "'/'",
771            TokenKind::Percent => "'%'",
772            TokenKind::StarStar => "'**'",
773            TokenKind::Dot => "'.'",
774            TokenKind::Equals => "'='",
775            TokenKind::PlusEquals => "'+='",
776            TokenKind::MinusEquals => "'-='",
777            TokenKind::StarEquals => "'*='",
778            TokenKind::SlashEquals => "'/='",
779            TokenKind::PercentEquals => "'%='",
780            TokenKind::StarStarEquals => "'**='",
781            TokenKind::DotEquals => "'.='",
782            TokenKind::AmpersandEquals => "'&='",
783            TokenKind::PipeEquals => "'|='",
784            TokenKind::CaretEquals => "'^='",
785            TokenKind::ShiftLeftEquals => "'<<='",
786            TokenKind::ShiftRightEquals => "'>>='",
787            TokenKind::CoalesceEquals => "'??='",
788            TokenKind::EqualsEquals => "'=='",
789            TokenKind::BangEquals => "'!='",
790            TokenKind::EqualsEqualsEquals => "'==='",
791            TokenKind::BangEqualsEquals => "'!=='",
792            TokenKind::LessThan => "'<'",
793            TokenKind::GreaterThan => "'>'",
794            TokenKind::LessThanEquals => "'<='",
795            TokenKind::GreaterThanEquals => "'>='",
796            TokenKind::Spaceship => "'<=>'",
797            TokenKind::AmpersandAmpersand => "'&&'",
798            TokenKind::PipePipe => "'||'",
799            TokenKind::Bang => "'!'",
800            TokenKind::Ampersand => "'&'",
801            TokenKind::Pipe => "'|'",
802            TokenKind::Caret => "'^'",
803            TokenKind::Tilde => "'~'",
804            TokenKind::ShiftLeft => "'<<'",
805            TokenKind::ShiftRight => "'>>'",
806            TokenKind::PlusPlus => "'++'",
807            TokenKind::MinusMinus => "'--'",
808            TokenKind::Question => "'?'",
809            TokenKind::QuestionQuestion => "'??'",
810            TokenKind::Colon => "':'",
811            TokenKind::FatArrow => "'=>'",
812            TokenKind::PipeArrow => "'|>'",
813            TokenKind::LeftParen => "'('",
814            TokenKind::RightParen => "')'",
815            TokenKind::LeftBracket => "'['",
816            TokenKind::RightBracket => "']'",
817            TokenKind::LeftBrace => "'{'",
818            TokenKind::RightBrace => "'}'",
819            TokenKind::Semicolon => "';'",
820            TokenKind::Comma => "','",
821            TokenKind::DoubleColon => "'::'",
822            TokenKind::Arrow => "'->'",
823            TokenKind::NullsafeArrow => "'?->'",
824            TokenKind::Backslash => "'\\'",
825            TokenKind::At => "'@'",
826            TokenKind::HashBracket => "'#['",
827            TokenKind::Ellipsis => "'...'",
828            TokenKind::If => "'if'",
829            TokenKind::Else => "'else'",
830            TokenKind::ElseIf => "'elseif'",
831            TokenKind::While => "'while'",
832            TokenKind::Do => "'do'",
833            TokenKind::For => "'for'",
834            TokenKind::Foreach => "'foreach'",
835            TokenKind::As => "'as'",
836            TokenKind::Function => "'function'",
837            TokenKind::Return => "'return'",
838            TokenKind::Echo => "'echo'",
839            TokenKind::Print => "'print'",
840            TokenKind::True => "'true'",
841            TokenKind::False => "'false'",
842            TokenKind::Null => "'null'",
843            TokenKind::And => "'and'",
844            TokenKind::Or => "'or'",
845            TokenKind::Xor => "'xor'",
846            TokenKind::Break => "'break'",
847            TokenKind::Continue => "'continue'",
848            TokenKind::Switch => "'switch'",
849            TokenKind::Case => "'case'",
850            TokenKind::Default => "'default'",
851            TokenKind::EndIf => "'endif'",
852            TokenKind::EndWhile => "'endwhile'",
853            TokenKind::EndFor => "'endfor'",
854            TokenKind::EndForeach => "'endforeach'",
855            TokenKind::Throw => "'throw'",
856            TokenKind::Try => "'try'",
857            TokenKind::Catch => "'catch'",
858            TokenKind::Finally => "'finally'",
859            TokenKind::Instanceof => "'instanceof'",
860            TokenKind::Array => "'array'",
861            TokenKind::List => "'list'",
862            TokenKind::Goto => "'goto'",
863            TokenKind::Declare => "'declare'",
864            TokenKind::Unset => "'unset'",
865            TokenKind::Global => "'global'",
866            TokenKind::EndDeclare => "'enddeclare'",
867            TokenKind::EndSwitch => "'endswitch'",
868            TokenKind::Isset => "'isset'",
869            TokenKind::Empty => "'empty'",
870            TokenKind::Include => "'include'",
871            TokenKind::IncludeOnce => "'include_once'",
872            TokenKind::Require => "'require'",
873            TokenKind::RequireOnce => "'require_once'",
874            TokenKind::Eval => "'eval'",
875            TokenKind::Exit => "'exit'",
876            TokenKind::Die => "'die'",
877            TokenKind::Clone => "'clone'",
878            TokenKind::New => "'new'",
879            TokenKind::Class => "'class'",
880            TokenKind::Abstract => "'abstract'",
881            TokenKind::Final => "'final'",
882            TokenKind::Interface => "'interface'",
883            TokenKind::Trait => "'trait'",
884            TokenKind::Extends => "'extends'",
885            TokenKind::Implements => "'implements'",
886            TokenKind::Public => "'public'",
887            TokenKind::Protected => "'protected'",
888            TokenKind::Private => "'private'",
889            TokenKind::Static => "'static'",
890            TokenKind::Const => "'const'",
891            TokenKind::Fn_ => "'fn'",
892            TokenKind::Match_ => "'match'",
893            TokenKind::Namespace => "'namespace'",
894            TokenKind::Use => "'use'",
895            TokenKind::Readonly => "'readonly'",
896            TokenKind::Enum_ => "'enum'",
897            TokenKind::Yield_ => "'yield'",
898            TokenKind::From => "'from'",
899            TokenKind::Self_ => "'self'",
900            TokenKind::Parent_ => "'parent'",
901            TokenKind::MagicClass => "'__CLASS__'",
902            TokenKind::MagicDir => "'__DIR__'",
903            TokenKind::MagicFile => "'__FILE__'",
904            TokenKind::MagicFunction => "'__FUNCTION__'",
905            TokenKind::MagicLine => "'__LINE__'",
906            TokenKind::MagicMethod => "'__METHOD__'",
907            TokenKind::MagicNamespace => "'__NAMESPACE__'",
908            TokenKind::MagicTrait => "'__TRAIT__'",
909            TokenKind::MagicProperty => "'__PROPERTY__'",
910            TokenKind::HaltCompiler => "'__halt_compiler'",
911            TokenKind::OpenTag => "'<?php'",
912            TokenKind::CloseTag => "'?>'",
913            TokenKind::InlineHtml => "inline HTML",
914            TokenKind::Heredoc => "heredoc",
915            TokenKind::Nowdoc => "nowdoc",
916            TokenKind::InvalidNumericLiteral => "invalid numeric literal",
917            TokenKind::LineComment => "line comment",
918            TokenKind::HashComment => "hash comment",
919            TokenKind::BlockComment => "block comment",
920            TokenKind::DocComment => "doc comment",
921            TokenKind::Eof => "end of file",
922        }
923    }
924}
925
926impl std::fmt::Display for TokenKind {
927    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
928        f.write_str(self.description())
929    }
930}
931
932#[cfg(test)]
933mod tests {
934    use super::*;
935
936    #[test]
937    fn test_resolve_keyword() {
938        assert_eq!(resolve_keyword("if"), Some(TokenKind::If));
939        assert_eq!(resolve_keyword("IF"), Some(TokenKind::If));
940        assert_eq!(resolve_keyword("If"), Some(TokenKind::If));
941        assert_eq!(resolve_keyword("function"), Some(TokenKind::Function));
942        assert_eq!(resolve_keyword("myFunc"), None);
943        assert_eq!(resolve_keyword("true"), Some(TokenKind::True));
944        assert_eq!(resolve_keyword("TRUE"), Some(TokenKind::True));
945        assert_eq!(resolve_keyword("null"), Some(TokenKind::Null));
946    }
947
948    #[test]
949    fn test_is_assignment_op() {
950        assert!(TokenKind::Equals.is_assignment_op());
951        assert!(TokenKind::PlusEquals.is_assignment_op());
952        assert!(TokenKind::DotEquals.is_assignment_op());
953        assert!(!TokenKind::Plus.is_assignment_op());
954        assert!(!TokenKind::EqualsEquals.is_assignment_op());
955    }
956}
php_lexer/token.rs

php_lexer/
token.rs
