php_parser/lexer/
token.rs

1use crate::span::Span;
2use serde::Serialize;
3
4#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
5pub struct Token {
6    pub kind: TokenKind,
7    pub span: Span,
8}
9
10impl Token {
11    pub fn text<'a>(&self, source: &'a [u8]) -> &'a [u8] {
12        self.span.as_str(source)
13    }
14
15    pub fn line(&self, source: &[u8]) -> usize {
16        source
17            .get(..self.span.start)
18            .unwrap_or_default()
19            .iter()
20            .filter(|&&b| b == b'\n')
21            .count()
22            + 1
23    }
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Copy, Serialize)]
27pub enum TokenKind {
28    // Keywords
29    Function,
30    Class,
31    Interface,
32    Trait,
33    Extends,
34    Implements,
35    Enum,
36    If,
37    Else,
38    ElseIf,
39    EndIf,
40    Return,
41    Echo,
42    Print,
43    While,
44    Do,
45    For,
46    Foreach,
47    EndWhile,
48    EndFor,
49    EndForeach,
50    As,
51    Switch,
52    EndSwitch,
53    Case,
54    Default,
55    Break,
56    Continue,
57    Goto,
58    Insteadof,
59    Try,
60    Catch,
61    Finally,
62    Throw,
63    Public,
64    Protected,
65    Private,
66    Static,
67    Abstract,
68    Final,
69    Readonly,
70    PublicSet,
71    ProtectedSet,
72    PrivateSet,
73    Namespace,
74    Use,
75    Global,
76    New,
77    Clone,
78    InstanceOf,
79    Array,
80    Const,
81    Include,
82    IncludeOnce,
83    Require,
84    RequireOnce,
85    Eval,
86    Exit,
87    Die,
88    Empty,
89    Isset,
90    Unset,
91    List,
92    Yield,
93    YieldFrom,
94    Declare,
95    EndDeclare,
96    Match,
97    Fn,
98    HaltCompiler, // __halt_compiler
99    Attribute,    // #[
100
101    // Magic Constants
102    Line,
103    File,
104    Dir,
105    ClassC,
106    TraitC,
107    MethodC,
108    FuncC,
109    NsC,
110    PropertyC,
111
112    // Types (for type hints)
113    TypeBool,
114    TypeInt,
115    TypeFloat,
116    TypeString,
117    TypeObject,
118    TypeVoid,
119    TypeIterable,
120    TypeCallable,
121    TypeMixed,
122    TypeNever,
123    TypeNull,
124    TypeFalse,
125    TypeTrue,
126
127    // Casts
128    IntCast,
129    FloatCast,
130    StringCast,
131    ArrayCast,
132    ObjectCast,
133    BoolCast,
134    UnsetCast,
135    VoidCast,
136
137    // Identifiers & Literals
138    Identifier,
139    LNumber,
140    DNumber,
141    StringLiteral,
142    NumString,     // For array offset in string
143    StringVarname, // For ${var} in string
144    Variable,
145    InlineHtml,
146    EncapsedAndWhitespace,
147    DollarOpenCurlyBraces, // ${
148    CurlyOpen,             // {$
149    Backtick,              // `
150    DoubleQuote,           // "
151    StartHeredoc,          // <<<
152    EndHeredoc,            // The closing identifier
153    Dollar,                // $ (for variable variables like $$a)
154    NsSeparator,           // \
155
156    // Comments
157    Comment,
158    DocComment,
159
160    // Symbols
161    Arrow,         // ->
162    NullSafeArrow, // ?->
163    DoubleArrow,   // =>
164    DoubleColon,   // ::
165    Ellipsis,      // ...
166
167    Plus,
168    Minus,
169    Asterisk,
170    Slash,
171    Percent,
172    Dot,
173    Pow, // **
174    Inc,
175    Dec, // ++, --
176
177    Eq, // =
178    PlusEq,
179    MinusEq,
180    MulEq,
181    DivEq,
182    ModEq,
183    ConcatEq,
184    PowEq,
185    AndEq,
186    OrEq,
187    XorEq,
188    SlEq,
189    SrEq,
190    CoalesceEq,
191
192    EqEq,      // ==
193    EqEqEq,    // ===
194    Bang,      // !
195    BangEq,    // !=
196    BangEqEq,  // !==
197    Lt,        // <
198    LtEq,      // <=
199    Gt,        // >
200    GtEq,      // >=
201    Spaceship, // <=>
202
203    Ampersand, // &
204    AmpersandFollowedByVarOrVararg,
205    AmpersandNotFollowedByVarOrVararg,
206    Pipe,   // |
207    Caret,  // ^
208    BitNot, // ~
209    Sl,     // <<
210    Sr,     // >>
211
212    AmpersandAmpersand, // &&
213    PipePipe,           // ||
214    LogicalAnd,         // and
215    LogicalOr,          // or
216    LogicalXor,         // xor
217    Question,           // ?
218    Coalesce,           // ??
219    At,                 // @
220
221    SemiColon,
222    Colon,
223    Comma,
224    OpenBrace,
225    CloseBrace,
226    OpenParen,
227    CloseParen,
228    OpenBracket,
229    CloseBracket,
230
231    OpenTag,     // <?php
232    OpenTagEcho, // <?=
233    CloseTag,    // ?>
234
235    Eof,
236
237    // Error token for lexing failures
238    Error,
239    AmpersandFollowedByVar,
240    AmpersandNotFollowedByVar,
241}
242
243impl TokenKind {
244    pub fn is_semi_reserved(self) -> bool {
245        matches!(
246            self,
247            TokenKind::New
248                | TokenKind::Static
249                | TokenKind::Class
250                | TokenKind::Trait
251                | TokenKind::Interface
252                | TokenKind::Extends
253                | TokenKind::Implements
254                | TokenKind::Enum
255                | TokenKind::Namespace
256                | TokenKind::TypeInt
257                | TokenKind::TypeFloat
258                | TokenKind::TypeBool
259                | TokenKind::TypeString
260                | TokenKind::TypeVoid
261                | TokenKind::TypeNever
262                | TokenKind::TypeNull
263                | TokenKind::TypeFalse
264                | TokenKind::TypeTrue
265                | TokenKind::TypeMixed
266                | TokenKind::TypeIterable
267                | TokenKind::TypeObject
268                | TokenKind::TypeCallable
269                | TokenKind::LogicalOr
270                | TokenKind::Insteadof
271                | TokenKind::LogicalAnd
272                | TokenKind::LogicalXor
273                | TokenKind::As
274                | TokenKind::Empty
275                | TokenKind::Isset
276                | TokenKind::Default
277                | TokenKind::Switch
278                | TokenKind::Case
279                | TokenKind::For
280                | TokenKind::Foreach
281                | TokenKind::While
282                | TokenKind::Do
283                | TokenKind::If
284                | TokenKind::Else
285                | TokenKind::ElseIf
286                | TokenKind::EndIf
287                | TokenKind::Try
288                | TokenKind::Catch
289                | TokenKind::Finally
290                | TokenKind::Throw
291                | TokenKind::Return
292                | TokenKind::Break
293                | TokenKind::Continue
294                | TokenKind::Goto
295                | TokenKind::Echo
296                | TokenKind::Print
297                | TokenKind::List
298                | TokenKind::Clone
299                | TokenKind::Include
300                | TokenKind::IncludeOnce
301                | TokenKind::Require
302                | TokenKind::RequireOnce
303                | TokenKind::Global
304                | TokenKind::Unset
305                | TokenKind::Exit
306                | TokenKind::Die
307                | TokenKind::Eval
308                | TokenKind::Yield
309                | TokenKind::YieldFrom
310                | TokenKind::Declare
311                | TokenKind::EndDeclare
312                | TokenKind::Match
313                | TokenKind::Fn
314                | TokenKind::Const
315                | TokenKind::Use
316                | TokenKind::Public
317                | TokenKind::Protected
318                | TokenKind::Private
319                | TokenKind::Abstract
320                | TokenKind::Final
321                | TokenKind::Readonly
322                | TokenKind::Array
323                | TokenKind::Function
324                | TokenKind::HaltCompiler
325                | TokenKind::InstanceOf
326                | TokenKind::Line
327                | TokenKind::File
328                | TokenKind::Dir
329                | TokenKind::ClassC
330                | TokenKind::PropertyC
331                | TokenKind::TraitC
332                | TokenKind::MethodC
333                | TokenKind::FuncC
334                | TokenKind::NsC
335        )
336    }
337}
php_parser/lexer/token.rs

php_parser/lexer/
token.rs
