php_lexer/
token.rs

1/// All token types produced by the PHP lexer.
2///
3/// The variants are grouped by category in source order; the `#[repr(u8)]`
4/// guarantees a compact discriminant layout used by [`TokenKind::is_assignment_op`].
5#[repr(u8)]
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7#[non_exhaustive]
8pub enum TokenKind {
9    // -------------------------------------------------------------------------
10    // Literals
11    // -------------------------------------------------------------------------
12    /// Float in scientific notation: `1e10`, `1.5e-3`, `1E+10`.
13    FloatLiteral,
14
15    /// Float with digits on both sides of the decimal point: `1.5`, `0.0`.
16    FloatLiteralSimple,
17
18    /// Float starting with a leading dot: `.5`, `.001`.
19    FloatLiteralLeadingDot,
20
21    /// Hexadecimal integer literal: `0x1A`, `0xFF`.
22    HexIntLiteral,
23
24    /// Binary integer literal: `0b1010`, `0B0011`.
25    BinIntLiteral,
26
27    /// Octal integer literal in the new `0o` prefix form: `0o17` (PHP 8.1+).
28    OctIntLiteralNew,
29
30    /// Octal integer literal in the legacy leading-zero form: `017`.
31    OctIntLiteral,
32
33    /// Decimal integer literal: `0`, `42`, `1_000_000`.
34    IntLiteral,
35
36    /// Single-quoted string literal: `'hello'`. Optional `b`/`B` prefix allowed.
37    SingleQuotedString,
38
39    /// Double-quoted string literal: `"hello $name"`. Triggers interpolation processing.
40    DoubleQuotedString,
41
42    /// Backtick string (shell execution): `` `ls -la` ``.
43    BacktickString,
44
45    // -------------------------------------------------------------------------
46    // Variables
47    // -------------------------------------------------------------------------
48    /// A `$`-prefixed variable name: `$foo`, `$myVar`.
49    Variable,
50
51    /// A bare `$` not followed by a valid identifier start (variable-variable prefix).
52    Dollar,
53
54    // -------------------------------------------------------------------------
55    // Identifiers
56    // -------------------------------------------------------------------------
57    /// A bare identifier or keyword that has not yet been resolved.
58    /// PHP keywords are case-insensitive and are resolved from this token.
59    Identifier,
60
61    // -------------------------------------------------------------------------
62    // Arithmetic operators
63    // -------------------------------------------------------------------------
64    /// `+`
65    Plus,
66    /// `-`
67    Minus,
68    /// `*`
69    Star,
70    /// `/`
71    Slash,
72    /// `%`
73    Percent,
74    /// `**` — exponentiation.
75    StarStar,
76    /// `.` — string concatenation.
77    Dot,
78
79    // -------------------------------------------------------------------------
80    // Assignment operators
81    // -------------------------------------------------------------------------
82    // NOTE: Equals..=CoalesceEquals must remain contiguous; is_assignment_op relies on this.
83    /// `=`
84    Equals,
85    /// `+=`
86    PlusEquals,
87    /// `-=`
88    MinusEquals,
89    /// `*=`
90    StarEquals,
91    /// `/=`
92    SlashEquals,
93    /// `%=`
94    PercentEquals,
95    /// `**=`
96    StarStarEquals,
97    /// `.=`
98    DotEquals,
99    /// `&=`
100    AmpersandEquals,
101    /// `|=`
102    PipeEquals,
103    /// `^=`
104    CaretEquals,
105    /// `<<=`
106    ShiftLeftEquals,
107    /// `>>=`
108    ShiftRightEquals,
109    /// `??=`
110    CoalesceEquals,
111
112    // -------------------------------------------------------------------------
113    // Comparison operators
114    // -------------------------------------------------------------------------
115    /// `==` — loose equality.
116    EqualsEquals,
117    /// `!=` or `<>` — loose inequality.
118    BangEquals,
119    /// `===` — strict equality.
120    EqualsEqualsEquals,
121    /// `!==` — strict inequality.
122    BangEqualsEquals,
123    /// `<`
124    LessThan,
125    /// `>`
126    GreaterThan,
127    /// `<=`
128    LessThanEquals,
129    /// `>=`
130    GreaterThanEquals,
131    /// `<=>` — spaceship / three-way comparison.
132    Spaceship,
133
134    // -------------------------------------------------------------------------
135    // Logical / bitwise operators
136    // -------------------------------------------------------------------------
137    /// `&&` — short-circuit boolean AND.
138    AmpersandAmpersand,
139    /// `||` — short-circuit boolean OR.
140    PipePipe,
141    /// `!` — boolean NOT.
142    Bang,
143
144    /// `&` — bitwise AND (also reference marker in declarations).
145    Ampersand,
146    /// `|` — bitwise OR.
147    Pipe,
148    /// `^` — bitwise XOR.
149    Caret,
150    /// `~` — bitwise NOT.
151    Tilde,
152    /// `<<` — left bit-shift.
153    ShiftLeft,
154    /// `>>` — right bit-shift.
155    ShiftRight,
156
157    // -------------------------------------------------------------------------
158    // Increment / decrement
159    // -------------------------------------------------------------------------
160    /// `++`
161    PlusPlus,
162    /// `--`
163    MinusMinus,
164
165    // -------------------------------------------------------------------------
166    // Other operators
167    // -------------------------------------------------------------------------
168    /// `?` — ternary operator or nullable type-hint prefix.
169    Question,
170    /// `??` — null coalescing.
171    QuestionQuestion,
172    /// `:` — ternary separator, named argument separator, or return type separator.
173    Colon,
174
175    /// `=>` — key-value separator in arrays and `match` arms.
176    FatArrow,
177
178    /// `|>` — pipe operator (PHP 8.5+).
179    PipeArrow,
180
181    // -------------------------------------------------------------------------
182    // Delimiters & punctuation
183    // -------------------------------------------------------------------------
184    /// `(`
185    LeftParen,
186    /// `)`
187    RightParen,
188    /// `[`
189    LeftBracket,
190    /// `]`
191    RightBracket,
192    /// `{`
193    LeftBrace,
194    /// `}`
195    RightBrace,
196    /// `;`
197    Semicolon,
198    /// `,`
199    Comma,
200
201    /// `::` — static access separator.
202    DoubleColon,
203
204    /// `->` — object property / method access.
205    Arrow,
206
207    /// `?->` — nullsafe property / method access.
208    NullsafeArrow,
209
210    /// `\` — namespace separator.
211    Backslash,
212
213    /// `@` — error-suppression operator.
214    At,
215
216    /// `#[` — attribute syntax opener.
217    HashBracket,
218
219    /// `...` — splat / variadic operator.
220    Ellipsis,
221
222    // -------------------------------------------------------------------------
223    // Keywords (resolved from Identifier at lex time)
224    // -------------------------------------------------------------------------
225    /// `if`
226    If,
227    /// `else`
228    Else,
229    /// `elseif`
230    ElseIf,
231    /// `while`
232    While,
233    /// `do`
234    Do,
235    /// `for`
236    For,
237    /// `foreach`
238    Foreach,
239    /// `as`
240    As,
241    /// `function`
242    Function,
243    /// `return`
244    Return,
245    /// `echo`
246    Echo,
247    /// `print`
248    Print,
249    /// `true`
250    True,
251    /// `false`
252    False,
253    /// `null`
254    Null,
255    /// `and` — low-precedence boolean AND.
256    And,
257    /// `or` — low-precedence boolean OR.
258    Or,
259    /// `xor` — low-precedence boolean XOR.
260    Xor,
261    /// `break`
262    Break,
263    /// `continue`
264    Continue,
265    /// `switch`
266    Switch,
267    /// `case`
268    Case,
269    /// `default`
270    Default,
271    /// `endif` — alternative `if` syntax terminator.
272    EndIf,
273    /// `endwhile` — alternative `while` syntax terminator.
274    EndWhile,
275    /// `endfor` — alternative `for` syntax terminator.
276    EndFor,
277    /// `endforeach` — alternative `foreach` syntax terminator.
278    EndForeach,
279    /// `throw`
280    Throw,
281    /// `try`
282    Try,
283    /// `catch`
284    Catch,
285    /// `finally`
286    Finally,
287    /// `instanceof`
288    Instanceof,
289    /// `array` — as a keyword (e.g. `array(1, 2)` or `array` type hint).
290    Array,
291    /// `list` — destructuring construct: `list($a, $b) = ...`.
292    List,
293    /// `goto`
294    Goto,
295    /// `declare`
296    Declare,
297    /// `unset`
298    Unset,
299    /// `global`
300    Global,
301    /// `enddeclare` — alternative `declare` syntax terminator.
302    EndDeclare,
303    /// `endswitch` — alternative `switch` syntax terminator.
304    EndSwitch,
305    /// `isset`
306    Isset,
307    /// `empty`
308    Empty,
309    /// `include`
310    Include,
311    /// `include_once`
312    IncludeOnce,
313    /// `require`
314    Require,
315    /// `require_once`
316    RequireOnce,
317    /// `eval`
318    Eval,
319    /// `exit`
320    Exit,
321    /// `die` — alias for `exit`.
322    Die,
323    /// `clone`
324    Clone,
325
326    // OOP keywords
327    /// `new`
328    New,
329    /// `class`
330    Class,
331    /// `abstract`
332    Abstract,
333    /// `final`
334    Final,
335    /// `interface`
336    Interface,
337    /// `trait`
338    Trait,
339    /// `extends`
340    Extends,
341    /// `implements`
342    Implements,
343    /// `public`
344    Public,
345    /// `protected`
346    Protected,
347    /// `private`
348    Private,
349    /// `static`
350    Static,
351    /// `const`
352    Const,
353    /// `fn` — arrow function keyword.
354    Fn_,
355    /// `match` — match expression keyword (PHP 8.0+).
356    Match_,
357    /// `namespace`
358    Namespace,
359    /// `use`
360    Use,
361    /// `readonly` (PHP 8.1+).
362    Readonly,
363    /// `enum` (PHP 8.1+).
364    Enum_,
365    /// `yield`
366    Yield_,
367    /// `self`
368    Self_,
369    /// `parent`
370    Parent_,
371
372    // -------------------------------------------------------------------------
373    // Magic constants (resolved from Identifier at lex time)
374    // -------------------------------------------------------------------------
375    /// `__CLASS__`
376    MagicClass,
377    /// `__DIR__`
378    MagicDir,
379    /// `__FILE__`
380    MagicFile,
381    /// `__FUNCTION__`
382    MagicFunction,
383    /// `__LINE__`
384    MagicLine,
385    /// `__METHOD__`
386    MagicMethod,
387    /// `__NAMESPACE__`
388    MagicNamespace,
389    /// `__TRAIT__`
390    MagicTrait,
391    /// `__PROPERTY__` (PHP 8.4+)
392    MagicProperty,
393    /// `__halt_compiler` — stops the parser; remaining bytes are raw data.
394    HaltCompiler,
395
396    // -------------------------------------------------------------------------
397    // PHP tags & template output
398    // -------------------------------------------------------------------------
399    /// `<?php` or `<?=` opening tag.
400    OpenTag,
401
402    /// `?>` closing tag; switches the lexer back to inline-HTML mode.
403    CloseTag,
404
405    /// Raw HTML text between PHP tags (produced by the `Lexer` wrapper, not the inner scanner).
406    InlineHtml,
407
408    // -------------------------------------------------------------------------
409    // Heredoc / Nowdoc (produced by Lexer wrapper)
410    // -------------------------------------------------------------------------
411    /// `<<<LABEL … LABEL;` — heredoc string (supports interpolation).
412    Heredoc,
413    /// `<<<'LABEL' … LABEL;` — nowdoc string (no interpolation).
414    Nowdoc,
415
416    // -------------------------------------------------------------------------
417    // Error tokens
418    // -------------------------------------------------------------------------
419    /// An invalid numeric literal, e.g. `1_000_` (trailing underscore).
420    /// The lexer emits this token and records a [`LexerError`](crate::LexerError) for it.
421    InvalidNumericLiteral,
422
423    // -------------------------------------------------------------------------
424    // Comments (filtered into a side-table by the parser)
425    // -------------------------------------------------------------------------
426    /// `// …` single-line slash comment.
427    LineComment,
428    /// `# …` single-line hash comment.
429    HashComment,
430    /// `/* … */` block comment.
431    BlockComment,
432    /// `/** … */` doc-block comment (first non-whitespace char after `/*` is `*`).
433    DocComment,
434
435    // -------------------------------------------------------------------------
436    // End of input
437    // -------------------------------------------------------------------------
438    /// Sentinel token emitted once all source bytes have been consumed.
439    Eof,
440}
441
442impl TokenKind {
443    /// Returns `true` for the four comment-token variants.
444    #[inline]
445    pub fn is_comment(self) -> bool {
446        matches!(
447            self,
448            TokenKind::LineComment
449                | TokenKind::HashComment
450                | TokenKind::BlockComment
451                | TokenKind::DocComment
452        )
453    }
454}
455
456impl TokenKind {
457    /// Whether this is `=` or a compound assignment operator.
458    #[inline(always)]
459    pub fn is_assignment_op(self) -> bool {
460        // The assignment operators are contiguous in the enum definition:
461        // Equals..=CoalesceEquals. With #[repr(u8)], a single range check suffices.
462        (self as u8).wrapping_sub(TokenKind::Equals as u8)
463            <= (TokenKind::CoalesceEquals as u8 - TokenKind::Equals as u8)
464    }
465}
466
467/// Resolve a keyword from an identifier string. Returns the keyword TokenKind
468/// if the string is a keyword, or None if it's a plain identifier.
469pub fn resolve_keyword(text: &str) -> Option<TokenKind> {
470    // PHP keywords are case-insensitive; use eq_ignore_ascii_case to avoid allocation.
471    // Dispatch on length first to reduce comparisons.
472    let t = text;
473    match text.len() {
474        2 => {
475            if t.eq_ignore_ascii_case("if") {
476                return Some(TokenKind::If);
477            }
478            if t.eq_ignore_ascii_case("do") {
479                return Some(TokenKind::Do);
480            }
481            if t.eq_ignore_ascii_case("or") {
482                return Some(TokenKind::Or);
483            }
484            if t.eq_ignore_ascii_case("as") {
485                return Some(TokenKind::As);
486            }
487            if t.eq_ignore_ascii_case("fn") {
488                return Some(TokenKind::Fn_);
489            }
490        }
491        3 => {
492            if t.eq_ignore_ascii_case("for") {
493                return Some(TokenKind::For);
494            }
495            if t.eq_ignore_ascii_case("xor") {
496                return Some(TokenKind::Xor);
497            }
498            if t.eq_ignore_ascii_case("and") {
499                return Some(TokenKind::And);
500            }
501            if t.eq_ignore_ascii_case("new") {
502                return Some(TokenKind::New);
503            }
504            if t.eq_ignore_ascii_case("use") {
505                return Some(TokenKind::Use);
506            }
507            if t.eq_ignore_ascii_case("try") {
508                return Some(TokenKind::Try);
509            }
510            if t.eq_ignore_ascii_case("die") {
511                return Some(TokenKind::Die);
512            }
513        }
514        4 => {
515            if t.eq_ignore_ascii_case("else") {
516                return Some(TokenKind::Else);
517            }
518            if t.eq_ignore_ascii_case("echo") {
519                return Some(TokenKind::Echo);
520            }
521            if t.eq_ignore_ascii_case("true") {
522                return Some(TokenKind::True);
523            }
524            if t.eq_ignore_ascii_case("null") {
525                return Some(TokenKind::Null);
526            }
527            if t.eq_ignore_ascii_case("list") {
528                return Some(TokenKind::List);
529            }
530            if t.eq_ignore_ascii_case("goto") {
531                return Some(TokenKind::Goto);
532            }
533            if t.eq_ignore_ascii_case("case") {
534                return Some(TokenKind::Case);
535            }
536            if t.eq_ignore_ascii_case("self") {
537                return Some(TokenKind::Self_);
538            }
539            if t.eq_ignore_ascii_case("enum") {
540                return Some(TokenKind::Enum_);
541            }
542            if t.eq_ignore_ascii_case("eval") {
543                return Some(TokenKind::Eval);
544            }
545            if t.eq_ignore_ascii_case("exit") {
546                return Some(TokenKind::Exit);
547            }
548        }
549        5 => {
550            if t.eq_ignore_ascii_case("while") {
551                return Some(TokenKind::While);
552            }
553            if t.eq_ignore_ascii_case("false") {
554                return Some(TokenKind::False);
555            }
556            if t.eq_ignore_ascii_case("array") {
557                return Some(TokenKind::Array);
558            }
559            if t.eq_ignore_ascii_case("unset") {
560                return Some(TokenKind::Unset);
561            }
562            if t.eq_ignore_ascii_case("isset") {
563                return Some(TokenKind::Isset);
564            }
565            if t.eq_ignore_ascii_case("empty") {
566                return Some(TokenKind::Empty);
567            }
568            if t.eq_ignore_ascii_case("print") {
569                return Some(TokenKind::Print);
570            }
571            if t.eq_ignore_ascii_case("throw") {
572                return Some(TokenKind::Throw);
573            }
574            if t.eq_ignore_ascii_case("catch") {
575                return Some(TokenKind::Catch);
576            }
577            if t.eq_ignore_ascii_case("break") {
578                return Some(TokenKind::Break);
579            }
580            if t.eq_ignore_ascii_case("yield") {
581                return Some(TokenKind::Yield_);
582            }
583            if t.eq_ignore_ascii_case("class") {
584                return Some(TokenKind::Class);
585            }
586            if t.eq_ignore_ascii_case("const") {
587                return Some(TokenKind::Const);
588            }
589            if t.eq_ignore_ascii_case("final") {
590                return Some(TokenKind::Final);
591            }
592            if t.eq_ignore_ascii_case("match") {
593                return Some(TokenKind::Match_);
594            }
595            if t.eq_ignore_ascii_case("trait") {
596                return Some(TokenKind::Trait);
597            }
598            if t.eq_ignore_ascii_case("clone") {
599                return Some(TokenKind::Clone);
600            }
601            if t.eq_ignore_ascii_case("endif") {
602                return Some(TokenKind::EndIf);
603            }
604        }
605        6 => {
606            if t.eq_ignore_ascii_case("elseif") {
607                return Some(TokenKind::ElseIf);
608            }
609            if t.eq_ignore_ascii_case("return") {
610                return Some(TokenKind::Return);
611            }
612            if t.eq_ignore_ascii_case("switch") {
613                return Some(TokenKind::Switch);
614            }
615            if t.eq_ignore_ascii_case("global") {
616                return Some(TokenKind::Global);
617            }
618            if t.eq_ignore_ascii_case("static") {
619                return Some(TokenKind::Static);
620            }
621            if t.eq_ignore_ascii_case("public") {
622                return Some(TokenKind::Public);
623            }
624            if t.eq_ignore_ascii_case("parent") {
625                return Some(TokenKind::Parent_);
626            }
627            if t.eq_ignore_ascii_case("endfor") {
628                return Some(TokenKind::EndFor);
629            }
630        }
631        7 => {
632            if t.eq_ignore_ascii_case("foreach") {
633                return Some(TokenKind::Foreach);
634            }
635            if t.eq_ignore_ascii_case("default") {
636                return Some(TokenKind::Default);
637            }
638            if t.eq_ignore_ascii_case("finally") {
639                return Some(TokenKind::Finally);
640            }
641            if t.eq_ignore_ascii_case("include") {
642                return Some(TokenKind::Include);
643            }
644            if t.eq_ignore_ascii_case("declare") {
645                return Some(TokenKind::Declare);
646            }
647            if t.eq_ignore_ascii_case("extends") {
648                return Some(TokenKind::Extends);
649            }
650            if t.eq_ignore_ascii_case("require") {
651                return Some(TokenKind::Require);
652            }
653            if t.eq_ignore_ascii_case("private") {
654                return Some(TokenKind::Private);
655            }
656            if t.eq_ignore_ascii_case("__dir__") {
657                return Some(TokenKind::MagicDir);
658            }
659        }
660        8 => {
661            if t.eq_ignore_ascii_case("function") {
662                return Some(TokenKind::Function);
663            }
664            if t.eq_ignore_ascii_case("abstract") {
665                return Some(TokenKind::Abstract);
666            }
667            if t.eq_ignore_ascii_case("readonly") {
668                return Some(TokenKind::Readonly);
669            }
670            if t.eq_ignore_ascii_case("continue") {
671                return Some(TokenKind::Continue);
672            }
673            if t.eq_ignore_ascii_case("endwhile") {
674                return Some(TokenKind::EndWhile);
675            }
676            if t.eq_ignore_ascii_case("__file__") {
677                return Some(TokenKind::MagicFile);
678            }
679            if t.eq_ignore_ascii_case("__line__") {
680                return Some(TokenKind::MagicLine);
681            }
682        }
683        9 => {
684            if t.eq_ignore_ascii_case("namespace") {
685                return Some(TokenKind::Namespace);
686            }
687            if t.eq_ignore_ascii_case("interface") {
688                return Some(TokenKind::Interface);
689            }
690            if t.eq_ignore_ascii_case("protected") {
691                return Some(TokenKind::Protected);
692            }
693            if t.eq_ignore_ascii_case("endswitch") {
694                return Some(TokenKind::EndSwitch);
695            }
696            if t.eq_ignore_ascii_case("__class__") {
697                return Some(TokenKind::MagicClass);
698            }
699            if t.eq_ignore_ascii_case("__trait__") {
700                return Some(TokenKind::MagicTrait);
701            }
702        }
703        10 => {
704            if t.eq_ignore_ascii_case("implements") {
705                return Some(TokenKind::Implements);
706            }
707            if t.eq_ignore_ascii_case("instanceof") {
708                return Some(TokenKind::Instanceof);
709            }
710            if t.eq_ignore_ascii_case("endforeach") {
711                return Some(TokenKind::EndForeach);
712            }
713            if t.eq_ignore_ascii_case("enddeclare") {
714                return Some(TokenKind::EndDeclare);
715            }
716            if t.eq_ignore_ascii_case("__method__") {
717                return Some(TokenKind::MagicMethod);
718            }
719        }
720        12 => {
721            if t.eq_ignore_ascii_case("include_once") {
722                return Some(TokenKind::IncludeOnce);
723            }
724            if t.eq_ignore_ascii_case("require_once") {
725                return Some(TokenKind::RequireOnce);
726            }
727            if t.eq_ignore_ascii_case("__function__") {
728                return Some(TokenKind::MagicFunction);
729            }
730            if t.eq_ignore_ascii_case("__property__") {
731                return Some(TokenKind::MagicProperty);
732            }
733        }
734        13 if t.eq_ignore_ascii_case("__namespace__") => {
735            return Some(TokenKind::MagicNamespace);
736        }
737        15 if t.eq_ignore_ascii_case("__halt_compiler") => {
738            return Some(TokenKind::HaltCompiler);
739        }
740        _ => {}
741    }
742    None
743}
744
745impl TokenKind {
746    /// A short human-readable description, as used in diagnostics
747    /// (e.g. `"';'"`, `"identifier"`).
748    pub fn description(self) -> &'static str {
749        match self {
750            TokenKind::IntLiteral => "integer",
751            TokenKind::HexIntLiteral => "hex integer",
752            TokenKind::BinIntLiteral => "binary integer",
753            TokenKind::OctIntLiteral | TokenKind::OctIntLiteralNew => "octal integer",
754            TokenKind::FloatLiteral
755            | TokenKind::FloatLiteralSimple
756            | TokenKind::FloatLiteralLeadingDot => "float",
757            TokenKind::SingleQuotedString | TokenKind::DoubleQuotedString => "string",
758            TokenKind::BacktickString => "backtick string",
759            TokenKind::Variable => "variable",
760            TokenKind::Dollar => "'$'",
761            TokenKind::Identifier => "identifier",
762            TokenKind::Plus => "'+'",
763            TokenKind::Minus => "'-'",
764            TokenKind::Star => "'*'",
765            TokenKind::Slash => "'/'",
766            TokenKind::Percent => "'%'",
767            TokenKind::StarStar => "'**'",
768            TokenKind::Dot => "'.'",
769            TokenKind::Equals => "'='",
770            TokenKind::PlusEquals => "'+='",
771            TokenKind::MinusEquals => "'-='",
772            TokenKind::StarEquals => "'*='",
773            TokenKind::SlashEquals => "'/='",
774            TokenKind::PercentEquals => "'%='",
775            TokenKind::StarStarEquals => "'**='",
776            TokenKind::DotEquals => "'.='",
777            TokenKind::AmpersandEquals => "'&='",
778            TokenKind::PipeEquals => "'|='",
779            TokenKind::CaretEquals => "'^='",
780            TokenKind::ShiftLeftEquals => "'<<='",
781            TokenKind::ShiftRightEquals => "'>>='",
782            TokenKind::CoalesceEquals => "'??='",
783            TokenKind::EqualsEquals => "'=='",
784            TokenKind::BangEquals => "'!='",
785            TokenKind::EqualsEqualsEquals => "'==='",
786            TokenKind::BangEqualsEquals => "'!=='",
787            TokenKind::LessThan => "'<'",
788            TokenKind::GreaterThan => "'>'",
789            TokenKind::LessThanEquals => "'<='",
790            TokenKind::GreaterThanEquals => "'>='",
791            TokenKind::Spaceship => "'<=>'",
792            TokenKind::AmpersandAmpersand => "'&&'",
793            TokenKind::PipePipe => "'||'",
794            TokenKind::Bang => "'!'",
795            TokenKind::Ampersand => "'&'",
796            TokenKind::Pipe => "'|'",
797            TokenKind::Caret => "'^'",
798            TokenKind::Tilde => "'~'",
799            TokenKind::ShiftLeft => "'<<'",
800            TokenKind::ShiftRight => "'>>'",
801            TokenKind::PlusPlus => "'++'",
802            TokenKind::MinusMinus => "'--'",
803            TokenKind::Question => "'?'",
804            TokenKind::QuestionQuestion => "'??'",
805            TokenKind::Colon => "':'",
806            TokenKind::FatArrow => "'=>'",
807            TokenKind::PipeArrow => "'|>'",
808            TokenKind::LeftParen => "'('",
809            TokenKind::RightParen => "')'",
810            TokenKind::LeftBracket => "'['",
811            TokenKind::RightBracket => "']'",
812            TokenKind::LeftBrace => "'{'",
813            TokenKind::RightBrace => "'}'",
814            TokenKind::Semicolon => "';'",
815            TokenKind::Comma => "','",
816            TokenKind::DoubleColon => "'::'",
817            TokenKind::Arrow => "'->'",
818            TokenKind::NullsafeArrow => "'?->'",
819            TokenKind::Backslash => "'\\'",
820            TokenKind::At => "'@'",
821            TokenKind::HashBracket => "'#['",
822            TokenKind::Ellipsis => "'...'",
823            TokenKind::If => "'if'",
824            TokenKind::Else => "'else'",
825            TokenKind::ElseIf => "'elseif'",
826            TokenKind::While => "'while'",
827            TokenKind::Do => "'do'",
828            TokenKind::For => "'for'",
829            TokenKind::Foreach => "'foreach'",
830            TokenKind::As => "'as'",
831            TokenKind::Function => "'function'",
832            TokenKind::Return => "'return'",
833            TokenKind::Echo => "'echo'",
834            TokenKind::Print => "'print'",
835            TokenKind::True => "'true'",
836            TokenKind::False => "'false'",
837            TokenKind::Null => "'null'",
838            TokenKind::And => "'and'",
839            TokenKind::Or => "'or'",
840            TokenKind::Xor => "'xor'",
841            TokenKind::Break => "'break'",
842            TokenKind::Continue => "'continue'",
843            TokenKind::Switch => "'switch'",
844            TokenKind::Case => "'case'",
845            TokenKind::Default => "'default'",
846            TokenKind::EndIf => "'endif'",
847            TokenKind::EndWhile => "'endwhile'",
848            TokenKind::EndFor => "'endfor'",
849            TokenKind::EndForeach => "'endforeach'",
850            TokenKind::Throw => "'throw'",
851            TokenKind::Try => "'try'",
852            TokenKind::Catch => "'catch'",
853            TokenKind::Finally => "'finally'",
854            TokenKind::Instanceof => "'instanceof'",
855            TokenKind::Array => "'array'",
856            TokenKind::List => "'list'",
857            TokenKind::Goto => "'goto'",
858            TokenKind::Declare => "'declare'",
859            TokenKind::Unset => "'unset'",
860            TokenKind::Global => "'global'",
861            TokenKind::EndDeclare => "'enddeclare'",
862            TokenKind::EndSwitch => "'endswitch'",
863            TokenKind::Isset => "'isset'",
864            TokenKind::Empty => "'empty'",
865            TokenKind::Include => "'include'",
866            TokenKind::IncludeOnce => "'include_once'",
867            TokenKind::Require => "'require'",
868            TokenKind::RequireOnce => "'require_once'",
869            TokenKind::Eval => "'eval'",
870            TokenKind::Exit => "'exit'",
871            TokenKind::Die => "'die'",
872            TokenKind::Clone => "'clone'",
873            TokenKind::New => "'new'",
874            TokenKind::Class => "'class'",
875            TokenKind::Abstract => "'abstract'",
876            TokenKind::Final => "'final'",
877            TokenKind::Interface => "'interface'",
878            TokenKind::Trait => "'trait'",
879            TokenKind::Extends => "'extends'",
880            TokenKind::Implements => "'implements'",
881            TokenKind::Public => "'public'",
882            TokenKind::Protected => "'protected'",
883            TokenKind::Private => "'private'",
884            TokenKind::Static => "'static'",
885            TokenKind::Const => "'const'",
886            TokenKind::Fn_ => "'fn'",
887            TokenKind::Match_ => "'match'",
888            TokenKind::Namespace => "'namespace'",
889            TokenKind::Use => "'use'",
890            TokenKind::Readonly => "'readonly'",
891            TokenKind::Enum_ => "'enum'",
892            TokenKind::Yield_ => "'yield'",
893            TokenKind::Self_ => "'self'",
894            TokenKind::Parent_ => "'parent'",
895            TokenKind::MagicClass => "'__CLASS__'",
896            TokenKind::MagicDir => "'__DIR__'",
897            TokenKind::MagicFile => "'__FILE__'",
898            TokenKind::MagicFunction => "'__FUNCTION__'",
899            TokenKind::MagicLine => "'__LINE__'",
900            TokenKind::MagicMethod => "'__METHOD__'",
901            TokenKind::MagicNamespace => "'__NAMESPACE__'",
902            TokenKind::MagicTrait => "'__TRAIT__'",
903            TokenKind::MagicProperty => "'__PROPERTY__'",
904            TokenKind::HaltCompiler => "'__halt_compiler'",
905            TokenKind::OpenTag => "'<?php'",
906            TokenKind::CloseTag => "'?>'",
907            TokenKind::InlineHtml => "inline HTML",
908            TokenKind::Heredoc => "heredoc",
909            TokenKind::Nowdoc => "nowdoc",
910            TokenKind::InvalidNumericLiteral => "invalid numeric literal",
911            TokenKind::LineComment => "line comment",
912            TokenKind::HashComment => "hash comment",
913            TokenKind::BlockComment => "block comment",
914            TokenKind::DocComment => "doc comment",
915            TokenKind::Eof => "end of file",
916        }
917    }
918}
919
920impl std::fmt::Display for TokenKind {
921    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
922        f.write_str(self.description())
923    }
924}
925
926#[cfg(test)]
927mod tests {
928    use super::*;
929
930    #[test]
931    fn test_resolve_keyword() {
932        assert_eq!(resolve_keyword("if"), Some(TokenKind::If));
933        assert_eq!(resolve_keyword("IF"), Some(TokenKind::If));
934        assert_eq!(resolve_keyword("If"), Some(TokenKind::If));
935        assert_eq!(resolve_keyword("function"), Some(TokenKind::Function));
936        assert_eq!(resolve_keyword("myFunc"), None);
937        assert_eq!(resolve_keyword("true"), Some(TokenKind::True));
938        assert_eq!(resolve_keyword("TRUE"), Some(TokenKind::True));
939        assert_eq!(resolve_keyword("null"), Some(TokenKind::Null));
940    }
941
942    #[test]
943    fn test_is_assignment_op() {
944        assert!(TokenKind::Equals.is_assignment_op());
945        assert!(TokenKind::PlusEquals.is_assignment_op());
946        assert!(TokenKind::DotEquals.is_assignment_op());
947        assert!(!TokenKind::Plus.is_assignment_op());
948        assert!(!TokenKind::EqualsEquals.is_assignment_op());
949    }
950}
php_lexer/token.rs

php_lexer/
token.rs
