mago_syntax/token/
mod.rs

1use strum::Display;
2
3use mago_database::file::FileId;
4use mago_span::HasPosition;
5use mago_span::Position;
6use mago_span::Span;
7
8use crate::T;
9
10#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord, Display)]
11#[cfg_attr(feature = "serde", derive(serde::Serialize))]
12#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))]
13pub enum DocumentKind {
14    Heredoc,
15    Nowdoc,
16}
17
18#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord, Display)]
19#[cfg_attr(feature = "serde", derive(serde::Serialize))]
20#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))]
21pub enum Associativity {
22    NonAssociative,
23    Left,
24    Right,
25}
26
27#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord, Display)]
28#[cfg_attr(feature = "serde", derive(serde::Serialize))]
29#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))]
30pub enum Precedence {
31    Lowest,
32    KeyOr,
33    KeyXor,
34    KeyAnd,
35    Print,
36    Yield,
37    YieldFrom,
38    Assignment,
39    ElvisOrConditional,
40    NullCoalesce,
41    Or,
42    And,
43    BitwiseOr,
44    BitwiseXor,
45    BitwiseAnd,
46    Equality,
47    Comparison,
48    // NOTE(azjezz): the RFC does not really specify the precedence of the `|>` operator
49    // clearly, the current precedence position handles the examples shown in the RFC,
50    // but will need to be verified with the actual implementation once its merged into php-src.
51    //
52    // RFC: https://wiki.php.net/rfc/pipe-operator-v3
53    // PR: https://github.com/php/php-src/pull/17118
54    Pipe,
55    Concat,
56    BitShift,
57    AddSub,
58    MulDivMod,
59    Unary,
60    Instanceof,
61    ErrorControl,
62    Pow,
63    Clone,
64    IncDec,
65    Reference,
66    CallDim,
67    New,
68    ArrayDim,
69    ObjectAccess,
70    Highest,
71}
72
73pub trait GetPrecedence {
74    fn precedence(&self) -> Precedence;
75}
76
77#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord, Display)]
78#[cfg_attr(feature = "serde", derive(serde::Serialize))]
79#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))]
80pub enum TokenKind {
81    Whitespace,                  // ` `
82    Eval,                        // `eval`
83    Die,                         // `die`
84    Self_,                       // `self`
85    Parent,                      // `parent`
86    Backtick,                    // `` ` ``
87    DocumentStart(DocumentKind), // `<<<abc`, or `<<<'abc'`
88    DocumentEnd,                 // `abc`
89    From,                        // `from`
90    Print,                       // `print`
91    Dollar,                      // `$`
92    HaltCompiler,                // `__halt_compiler`
93    Readonly,                    // `readonly`
94    Global,                      // `global`
95    Abstract,                    // `abstract`
96    Ampersand,                   // `&`
97    AmpersandEqual,              // `&=`
98    AmpersandAmpersand,          // `&&`
99    AmpersandAmpersandEqual,     // `&&=`
100    Array,                       // `array`
101    ArrayCast,                   // `(array)`
102    MinusGreaterThan,            // `->`
103    QuestionMinusGreaterThan,    // `?->`
104    At,                          // `@`
105    As,                          // `as`
106    Asterisk,                    // `*`
107    HashLeftBracket,             // `#[`
108    Bang,                        // `!`
109    BangEqual,                   // `!=`
110    LessThanGreaterThan,         // `<>`
111    BangEqualEqual,              // `!==`
112    LessThanEqualGreaterThan,    // `<=>`
113    BoolCast,                    // `(bool)`
114    BooleanCast,                 // `(boolean)`
115    And,                         // `and`
116    Or,                          // `or`
117    Break,                       // `break`
118    Callable,                    // `callable`
119    Caret,                       // `^`
120    CaretEqual,                  // `^=`
121    Case,                        // `case`
122    Catch,                       // `catch`
123    Class,                       // `class`
124    ClassConstant,               // `__CLASS__`
125    TraitConstant,               // `__TRAIT__`
126    FunctionConstant,            // `__FUNCTION__`
127    MethodConstant,              // `__METHOD__`
128    LineConstant,                // `__LINE__`
129    FileConstant,                // `__FILE__`
130    Clone,                       // `clone`
131    MinusEqual,                  // `-=`
132    CloseTag,                    // `?>`
133    QuestionQuestion,            // `??`
134    QuestionQuestionEqual,       // `??=`
135    AsteriskEqual,               // `*=`
136    Colon,                       // `:`
137    Comma,                       // `,`
138    SingleLineComment,           // `// comment`
139    HashComment,                 // `# comment`
140    MultiLineComment,            // `/* comment */`
141    DocBlockComment,             // `/** comment */`
142    Const,                       // `const`
143    PartialLiteralString,        // `"string` or `'string`, missing closing quote
144    LiteralString,               // `"string"` or `'string'`
145    Continue,                    // `continue`
146    Declare,                     // `declare`
147    MinusMinus,                  // `--`
148    Default,                     // `default`
149    DirConstant,                 // `__DIR__`
150    SlashEqual,                  // `/=`
151    Do,                          // `do`
152    DollarLeftBrace,             // `${`
153    Dot,                         // `.`
154    DotEqual,                    // `.=`
155    EqualGreaterThan,            // `=>`
156    DoubleCast,                  // `(double)`
157    RealCast,                    // `(real)`
158    FloatCast,                   // `(float)`
159    ColonColon,                  // `::`
160    EqualEqual,                  // `==`
161    DoubleQuote,                 // `"`
162    Else,                        // `else`
163    Echo,                        // `echo`
164    DotDotDot,                   // `...`
165    ElseIf,                      // `elseif`
166    Empty,                       // `empty`
167    EndDeclare,                  // `enddeclare`
168    EndFor,                      // `endfor`
169    EndForeach,                  // `endforeach`
170    EndIf,                       // `endif`
171    EndSwitch,                   // `endswitch`
172    EndWhile,                    // `endwhile`
173    Enum,                        // `enum`
174    Equal,                       // `=`
175    Extends,                     // `extends`
176    False,                       // `false`
177    Final,                       // `final`
178    Finally,                     // `finally`
179    LiteralFloat,                // `1.0`
180    Fn,                          // `fn`
181    For,                         // `for`
182    Foreach,                     // `foreach`
183    FullyQualifiedIdentifier,    // `\Namespace\Class`
184    Function,                    // `function`
185    Goto,                        // `goto`
186    GreaterThan,                 // `>`
187    GreaterThanEqual,            // `>=`
188    Identifier,                  // `name`
189    If,                          // `if`
190    Implements,                  // `implements`
191    Include,                     // `include`
192    IncludeOnce,                 // `include_once`
193    PlusPlus,                    // `++`
194    InlineText,                  // inline text outside of PHP tags, also referred to as "HTML"
195    InlineShebang,               // `#!...`
196    Instanceof,                  // `instanceof`
197    Insteadof,                   // `insteadof`
198    Exit,                        // `exit`
199    Unset,                       // `unset`
200    Isset,                       // `isset`
201    List,                        // `list`
202    LiteralInteger,              // `1`
203    OffsetNumber,                // `5`/`0x0` offset in `$a[...]` (PHP `T_NUM_STRING`)
204    OffsetString,                // `bar`/`true` offset in `$a[...]` (PHP `T_STRING`)
205    IntCast,                     // `(int)`
206    IntegerCast,                 // `(integer)`
207    Interface,                   // `interface`
208    LeftBrace,                   // `{`
209    LeftBracket,                 // `[`
210    LeftParenthesis,             // `(`
211    LeftShift,                   // `<<`
212    LeftShiftEqual,              // `<<=`
213    RightShift,                  // `>>`
214    RightShiftEqual,             // `>>=`
215    LessThan,                    // `<`
216    LessThanEqual,               // `<=`
217    Match,                       // `match`
218    Minus,                       // `-`
219    Namespace,                   // `namespace`
220    NamespaceSeparator,          // `\`
221    NamespaceConstant,           // `__NAMESPACE__`
222    PropertyConstant,            // `__PROPERTY__`
223    New,                         // `new`
224    Null,                        // `null`
225    ObjectCast,                  // `(object)`
226    UnsetCast,                   // `(unset)`
227    OpenTag,                     // `<?php`
228    EchoTag,                     // `<?=`
229    ShortOpenTag,                // `<?`
230    Percent,                     // `%`
231    PercentEqual,                // `%=`
232    Pipe,                        // `|`
233    PipeEqual,                   // `|=`
234    Plus,                        // `+`
235    PlusEqual,                   // `+=`
236    AsteriskAsterisk,            // `**`
237    AsteriskAsteriskEqual,       // `**=`
238    Private,                     // `private`
239    PrivateSet,                  // `private(set)`
240    Protected,                   // `protected`
241    ProtectedSet,                // `protected(set)`
242    Public,                      // `public`
243    PublicSet,                   // `public(set)`
244    QualifiedIdentifier,         // `Namespace\Class`
245    Question,                    // `?`
246    Require,                     // `require`
247    RequireOnce,                 // `require_once`
248    Return,                      // `return`
249    RightBrace,                  // `}`
250    RightBracket,                // `]`
251    RightParenthesis,            // `)`
252    Semicolon,                   // `;`
253    Slash,                       // `/`
254    Static,                      // `static`
255    StringCast,                  // `(string)`
256    BinaryCast,                  // `(binary)`
257    VoidCast,                    // `(void)`
258    StringPart,                  // `string` inside a double-quoted string, or a document string
259    StringVariableName,          // `foo` in the immediate `${foo}` interpolation form (PHP `T_STRING_VARNAME`)
260    Switch,                      // `switch`
261    Throw,                       // `throw`
262    Trait,                       // `trait`
263    EqualEqualEqual,             // `===`
264    True,                        // `true`
265    Try,                         // `try`
266    Use,                         // `use`
267    Var,                         // `var`
268    Variable,                    // `$name`
269    Yield,                       // `yield`
270    While,                       // `while`
271    Tilde,                       // `~`
272    PipePipe,                    // `||`
273    Xor,                         // `xor`
274    PipeGreaterThan,             // `|>`
275}
276
277#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord)]
278#[cfg_attr(feature = "serde", derive(serde::Serialize))]
279pub struct Token<'src> {
280    pub kind: TokenKind,
281    pub start: Position,
282    pub value: &'src [u8],
283}
284
285impl HasPosition for Token<'_> {
286    #[inline]
287    fn position(&self) -> Position {
288        self.start
289    }
290}
291
292impl Precedence {
293    #[inline]
294    #[must_use]
295    pub const fn infix(kind: &TokenKind) -> Precedence {
296        match kind {
297            T!["**"] => Precedence::Pow,
298            T!["instanceof"] => Precedence::Instanceof,
299            T!["*" | "/" | "%"] => Precedence::MulDivMod,
300            T!["+" | "-"] => Precedence::AddSub,
301            T!["<<"] | T![">>"] => Precedence::BitShift,
302            T!["."] => Precedence::Concat,
303            T!["<" | "<=" | ">" | ">="] => Precedence::Comparison,
304            T!["==" | "!=" | "===" | "!==" | "<>" | "<=>"] => Precedence::Equality,
305            T!["&"] => Precedence::BitwiseAnd,
306            T!["^"] => Precedence::BitwiseXor,
307            T!["|"] => Precedence::BitwiseOr,
308            T!["&&"] => Precedence::And,
309            T!["||"] => Precedence::Or,
310            T!["??"] => Precedence::NullCoalesce,
311            T!["?"] => Precedence::ElvisOrConditional,
312            T!["="
313                | "+="
314                | "-="
315                | "*="
316                | "**="
317                | "/="
318                | ".="
319                | "&&="
320                | "??="
321                | "%="
322                | "&="
323                | "|="
324                | "^="
325                | "<<="
326                | ">>="] => Precedence::Assignment,
327            T!["yield"] => Precedence::Yield,
328            T!["and"] => Precedence::KeyAnd,
329            T!["or"] => Precedence::KeyOr,
330            T!["xor"] => Precedence::KeyXor,
331            T!["print"] => Precedence::Print,
332            T!["|>"] => Precedence::Pipe,
333            _ => Precedence::Lowest,
334        }
335    }
336
337    #[inline]
338    #[must_use]
339    pub const fn postfix(kind: &TokenKind) -> Self {
340        match kind {
341            T!["++" | "--"] => Self::IncDec,
342            T!["("] => Self::CallDim,
343            T!["["] => Self::ArrayDim,
344            T!["->" | "?->" | "::"] => Self::ObjectAccess,
345            _ => Self::Lowest,
346        }
347    }
348
349    #[inline]
350    #[must_use]
351    pub const fn associativity(&self) -> Option<Associativity> {
352        Some(match self {
353            Self::MulDivMod
354            | Self::AddSub
355            | Self::Concat
356            | Self::BitShift
357            | Self::BitwiseAnd
358            | Self::BitwiseOr
359            | Self::BitwiseXor
360            | Self::And
361            | Self::Or
362            | Self::KeyAnd
363            | Self::KeyXor
364            | Self::KeyOr
365            | Self::Pipe
366            | Self::ElvisOrConditional
367            | Self::ObjectAccess => Associativity::Left,
368            Self::Pow | Self::NullCoalesce | Self::Assignment | Self::Unary | Self::New => Associativity::Right,
369            Self::Equality | Self::Comparison | Self::Instanceof => Associativity::NonAssociative,
370            _ => return None,
371        })
372    }
373
374    #[inline]
375    #[must_use]
376    pub const fn is_associative(&self) -> bool {
377        self.associativity().is_some()
378    }
379
380    #[inline]
381    #[must_use]
382    pub const fn is_right_associative(&self) -> bool {
383        matches!(self.associativity(), Some(Associativity::Right))
384    }
385
386    #[inline]
387    #[must_use]
388    pub const fn is_left_associative(&self) -> bool {
389        matches!(self.associativity(), Some(Associativity::Left))
390    }
391
392    #[inline]
393    #[must_use]
394    pub const fn is_non_associative(&self) -> bool {
395        matches!(self.associativity(), Some(Associativity::NonAssociative))
396    }
397}
398
399impl TokenKind {
400    #[inline]
401    #[must_use]
402    pub const fn is_keyword(&self) -> bool {
403        matches!(
404            self,
405            TokenKind::Eval
406                | TokenKind::Die
407                | TokenKind::Empty
408                | TokenKind::Isset
409                | TokenKind::Unset
410                | TokenKind::Exit
411                | TokenKind::EndDeclare
412                | TokenKind::EndSwitch
413                | TokenKind::EndWhile
414                | TokenKind::EndForeach
415                | TokenKind::EndFor
416                | TokenKind::EndIf
417                | TokenKind::From
418                | TokenKind::And
419                | TokenKind::Or
420                | TokenKind::Xor
421                | TokenKind::Print
422                | TokenKind::Readonly
423                | TokenKind::Global
424                | TokenKind::Match
425                | TokenKind::Abstract
426                | TokenKind::Array
427                | TokenKind::As
428                | TokenKind::Break
429                | TokenKind::Case
430                | TokenKind::Catch
431                | TokenKind::Class
432                | TokenKind::Clone
433                | TokenKind::Continue
434                | TokenKind::Const
435                | TokenKind::Declare
436                | TokenKind::Default
437                | TokenKind::Do
438                | TokenKind::Echo
439                | TokenKind::ElseIf
440                | TokenKind::Else
441                | TokenKind::Enum
442                | TokenKind::Extends
443                | TokenKind::False
444                | TokenKind::Finally
445                | TokenKind::Final
446                | TokenKind::Fn
447                | TokenKind::Foreach
448                | TokenKind::For
449                | TokenKind::Function
450                | TokenKind::Goto
451                | TokenKind::If
452                | TokenKind::IncludeOnce
453                | TokenKind::Include
454                | TokenKind::Implements
455                | TokenKind::Interface
456                | TokenKind::Instanceof
457                | TokenKind::Namespace
458                | TokenKind::New
459                | TokenKind::Null
460                | TokenKind::Private
461                | TokenKind::PrivateSet
462                | TokenKind::Protected
463                | TokenKind::Public
464                | TokenKind::RequireOnce
465                | TokenKind::Require
466                | TokenKind::Return
467                | TokenKind::Static
468                | TokenKind::Switch
469                | TokenKind::Throw
470                | TokenKind::Trait
471                | TokenKind::True
472                | TokenKind::Try
473                | TokenKind::Use
474                | TokenKind::Var
475                | TokenKind::Yield
476                | TokenKind::While
477                | TokenKind::Insteadof
478                | TokenKind::List
479                | TokenKind::Self_
480                | TokenKind::Parent
481                | TokenKind::DirConstant
482                | TokenKind::FileConstant
483                | TokenKind::LineConstant
484                | TokenKind::FunctionConstant
485                | TokenKind::ClassConstant
486                | TokenKind::MethodConstant
487                | TokenKind::TraitConstant
488                | TokenKind::NamespaceConstant
489                | TokenKind::PropertyConstant
490                | TokenKind::HaltCompiler
491        )
492    }
493
494    #[inline]
495    #[must_use]
496    pub const fn is_infix(&self) -> bool {
497        matches!(
498            self,
499            T!["**"
500                | ">>="
501                | "<<="
502                | "^="
503                | "&="
504                | "|="
505                | "%="
506                | "**="
507                | "and"
508                | "or"
509                | "xor"
510                | "<=>"
511                | "<<"
512                | ">>"
513                | "&"
514                | "|"
515                | "^"
516                | "%"
517                | "instanceof"
518                | "*"
519                | "/"
520                | "+"
521                | "-"
522                | "."
523                | "<"
524                | ">"
525                | "<="
526                | ">="
527                | "=="
528                | "==="
529                | "!="
530                | "!=="
531                | "<>"
532                | "?"
533                | "&&"
534                | "||"
535                | "="
536                | "+="
537                | "-="
538                | ".="
539                | "??="
540                | "/="
541                | "*="
542                | "??"
543                | "|>"]
544        )
545    }
546
547    #[inline]
548    #[must_use]
549    pub const fn is_postfix(&self) -> bool {
550        matches!(self, T!["++" | "--" | "(" | "[" | "->" | "?->" | "::"])
551    }
552
553    #[inline]
554    #[must_use]
555    pub const fn is_visibility_modifier(&self) -> bool {
556        matches!(self, T!["public" | "protected" | "private" | "private(set)" | "protected(set)" | "public(set)"])
557    }
558
559    #[inline]
560    #[must_use]
561    pub const fn is_modifier(&self) -> bool {
562        matches!(
563            self,
564            T!["public"
565                | "protected"
566                | "private"
567                | "private(set)"
568                | "protected(set)"
569                | "public(set)"
570                | "static"
571                | "final"
572                | "abstract"
573                | "readonly"]
574        )
575    }
576
577    #[inline]
578    #[must_use]
579    pub const fn is_identifier_maybe_soft_reserved(&self) -> bool {
580        if let TokenKind::Identifier = self { true } else { self.is_soft_reserved_identifier() }
581    }
582
583    #[inline]
584    #[must_use]
585    pub const fn is_identifier_maybe_reserved(&self) -> bool {
586        if let TokenKind::Identifier = self { true } else { self.is_reserved_identifier() }
587    }
588
589    #[inline]
590    #[must_use]
591    pub const fn is_soft_reserved_identifier(&self) -> bool {
592        matches!(
593            self,
594            T!["parent" | "self" | "true" | "false" | "list" | "null" | "enum" | "from" | "readonly" | "match"]
595        )
596    }
597
598    #[inline]
599    #[must_use]
600    pub const fn is_reserved_identifier(&self) -> bool {
601        if self.is_soft_reserved_identifier() {
602            return true;
603        }
604
605        matches!(
606            self,
607            T!["static"
608                | "abstract"
609                | "final"
610                | "for"
611                | "private"
612                | "private(set)"
613                | "protected"
614                | "protected(set)"
615                | "public"
616                | "public(set)"
617                | "include"
618                | "include_once"
619                | "eval"
620                | "require"
621                | "require_once"
622                | "or"
623                | "xor"
624                | "and"
625                | "instanceof"
626                | "new"
627                | "clone"
628                | "exit"
629                | "die"
630                | "if"
631                | "elseif"
632                | "else"
633                | "endif"
634                | "echo"
635                | "do"
636                | "while"
637                | "endwhile"
638                | "endfor"
639                | "foreach"
640                | "endforeach"
641                | "declare"
642                | "enddeclare"
643                | "as"
644                | "try"
645                | "catch"
646                | "finally"
647                | "throw"
648                | "use"
649                | "insteadof"
650                | "global"
651                | "var"
652                | "unset"
653                | "isset"
654                | "empty"
655                | "continue"
656                | "goto"
657                | "function"
658                | "const"
659                | "return"
660                | "print"
661                | "yield"
662                | "list"
663                | "switch"
664                | "endswitch"
665                | "case"
666                | "default"
667                | "break"
668                | "array"
669                | "callable"
670                | "extends"
671                | "implements"
672                | "namespace"
673                | "trait"
674                | "interface"
675                | "class"
676                | "__CLASS__"
677                | "__TRAIT__"
678                | "__FUNCTION__"
679                | "__METHOD__"
680                | "__LINE__"
681                | "__FILE__"
682                | "__DIR__"
683                | "__NAMESPACE__"
684                | "__PROPERTY__"
685                | "__halt_compiler"
686                | "fn"
687                | "match"]
688        )
689    }
690
691    #[inline]
692    #[must_use]
693    pub const fn is_literal(&self) -> bool {
694        matches!(
695            self,
696            T!["true" | "false" | "null" | LiteralFloat | LiteralInteger | LiteralString | PartialLiteralString]
697        )
698    }
699
700    #[inline]
701    #[must_use]
702    pub const fn is_magic_constant(&self) -> bool {
703        matches!(
704            self,
705            T!["__CLASS__"
706                | "__DIR__"
707                | "__FILE__"
708                | "__FUNCTION__"
709                | "__LINE__"
710                | "__METHOD__"
711                | "__NAMESPACE__"
712                | "__PROPERTY__"
713                | "__TRAIT__"]
714        )
715    }
716
717    #[inline]
718    #[must_use]
719    pub const fn is_cast(&self) -> bool {
720        matches!(
721            self,
722            T!["(string)"
723                | "(binary)"
724                | "(int)"
725                | "(integer)"
726                | "(float)"
727                | "(double)"
728                | "(real)"
729                | "(bool)"
730                | "(boolean)"
731                | "(array)"
732                | "(object)"
733                | "(unset)"
734                | "(void)"]
735        )
736    }
737
738    #[inline]
739    #[must_use]
740    pub const fn is_unary_prefix(&self) -> bool {
741        if self.is_cast() {
742            return true;
743        }
744
745        matches!(self, T!["@" | "!" | "~" | "-" | "+" | "++" | "--"])
746    }
747
748    #[inline]
749    #[must_use]
750    pub const fn is_trivia(&self) -> bool {
751        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment | Whitespace])
752    }
753
754    #[inline]
755    #[must_use]
756    pub const fn is_comment(&self) -> bool {
757        matches!(self, T![SingleLineComment | MultiLineComment | DocBlockComment | HashComment])
758    }
759
760    #[inline]
761    #[must_use]
762    pub const fn is_comma(&self) -> bool {
763        matches!(self, T![","])
764    }
765
766    #[inline]
767    #[must_use]
768    pub const fn is_construct(&self) -> bool {
769        matches!(
770            self,
771            T!["isset"
772                | "empty"
773                | "eval"
774                | "include"
775                | "include_once"
776                | "require"
777                | "require_once"
778                | "print"
779                | "unset"
780                | "exit"
781                | "die"]
782        )
783    }
784}
785
786impl<'arena> Token<'arena> {
787    #[inline]
788    #[must_use]
789    pub const fn new(kind: TokenKind, value: &'arena [u8], start: Position) -> Self {
790        Self { kind, start, value }
791    }
792
793    /// Constructs a `Span` for this token given the file ID.
794    ///
795    /// The span is computed from the token's start position and its value length.
796    #[inline]
797    #[must_use]
798    pub const fn span_for(&self, file_id: FileId) -> Span {
799        let end = Position::new(self.start.offset + self.value.len() as u32);
800        Span::new(file_id, self.start, end)
801    }
802}
803
804impl std::fmt::Display for Token<'_> {
805    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
806        write!(f, "{}({})", self.kind, String::from_utf8_lossy(self.value))
807    }
808}
mago_syntax/token/mod.rs

mago_syntax/token/
mod.rs
