php_parser_rs/lexer/
token.rs

1use schemars::JsonSchema;
2use serde::Deserialize;
3use serde::Serialize;
4
5use std::fmt::Display;
6
7use crate::lexer::byte_string::ByteString;
8
9#[derive(Debug, PartialEq, Eq, Clone, Copy, Deserialize, Serialize, JsonSchema)]
10
11pub struct Span {
12    pub line: usize,
13    pub column: usize,
14    pub position: usize,
15}
16
17impl Span {
18    pub fn new(line: usize, column: usize, position: usize) -> Self {
19        Self {
20            line,
21            column,
22            position,
23        }
24    }
25}
26
27#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
28
29pub enum OpenTagKind {
30    Full,  // `<?php`
31    Short, // `<?`
32    Echo,  // `<?=`
33}
34
35#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
36
37pub enum DocStringKind {
38    Heredoc,
39    Nowdoc,
40}
41
42pub type DocStringIndentationAmount = usize;
43
44#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
45
46pub enum DocStringIndentationKind {
47    Space,
48    Tab,
49    None,
50    Both,
51}
52
53impl From<u8> for DocStringIndentationKind {
54    fn from(byte: u8) -> Self {
55        match byte {
56            b' ' => Self::Space,
57            b'\t' => Self::Tab,
58            _ => unreachable!(),
59        }
60    }
61}
62
63impl From<DocStringIndentationKind> for u8 {
64    fn from(kind: DocStringIndentationKind) -> Self {
65        match kind {
66            DocStringIndentationKind::Space => b' ',
67            DocStringIndentationKind::Tab => b'\t',
68            _ => unreachable!(),
69        }
70    }
71}
72
73#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
74
75pub enum TokenKind {
76    Die,
77    // Can't use `Self` as a name here, so suffixing with an underscore.
78    Self_,
79    Parent,
80    Backtick,
81    StartDocString(DocStringKind),
82    EndDocString(DocStringIndentationKind, usize),
83    From,
84    Print,
85    Dollar,
86    HaltCompiler,
87    Readonly,
88    Global,
89    Abstract,
90    Ampersand,
91    AmpersandEquals,
92    And,
93    AndEquals,
94    Array,
95    ArrayCast,
96    Arrow,
97    QuestionArrow,
98    At,
99    As,
100    Asterisk,
101    Attribute,
102    Bang,
103    BangEquals,
104    AngledLeftRight,
105    BangDoubleEquals,
106    Spaceship,
107    BoolCast,
108    BooleanCast,
109    BooleanAnd,
110    BooleanOr,
111    Break,
112    Callable,
113    Caret,
114    CaretEquals,
115    Case,
116    Catch,
117    Class,
118    ClassConstant,
119    TraitConstant,
120    FunctionConstant,
121    MethodConstant,
122    LineConstant,
123    FileConstant,
124    Clone,
125    MinusEquals,
126    CloseTag,
127    DoubleQuestion,
128    DoubleQuestionEquals,
129    AsteriskEquals,
130    Colon,
131    Comma,
132    SingleLineComment,
133    HashMarkComment,
134    MultiLineComment,
135    DocumentComment,
136    Const,
137    LiteralSingleQuotedString,
138    LiteralDoubleQuotedString,
139    Continue,
140    CurlyOpen,
141    Declare,
142    Decrement,
143    Default,
144    DirConstant,
145    DivEquals,
146    Do,
147    DollarLeftBrace,
148    Dot,
149    DotEquals,
150    DoubleArrow,
151    DoubleCast,
152    RealCast,
153    FloatCast,
154    DoubleColon,
155    DoubleEquals,
156    DoubleQuote,
157    Echo,
158    Ellipsis,
159    Else,
160    ElseIf,
161    Empty,
162    EndDeclare,
163    EndFor,
164    EndForeach,
165    EndIf,
166    EndSwitch,
167    EndWhile,
168    Enum,
169    Eof,
170    Equals,
171    Extends,
172    False,
173    Final,
174    Finally,
175    LiteralFloat,
176    Fn,
177    For,
178    Foreach,
179    FullyQualifiedIdentifier,
180    Function,
181    Goto,
182    GreaterThan,
183    GreaterThanEquals,
184    Identifier,
185    If,
186    Implements,
187    Include,
188    IncludeOnce,
189    Increment,
190    InlineHtml,
191    Instanceof,
192    Insteadof,
193    Eval,
194    Exit,
195    Unset,
196    Isset,
197    List,
198    LiteralInteger,
199    IntCast,
200    IntegerCast,
201    Interface,
202    LeftBrace,
203    LeftBracket,
204    LeftParen,
205    LeftShift,
206    LeftShiftEquals,
207    RightShift,
208    RightShiftEquals,
209    LessThan,
210    LessThanEquals,
211    Match,
212    Minus,
213    Namespace,
214    NamespaceSeparator,
215    NamespaceConstant,
216    CompilerHaltOffsetConstant,
217    New,
218    Null,
219    ObjectCast,
220    UnsetCast,
221    OpenTag(OpenTagKind),
222    Percent,
223    PercentEquals,
224    Pipe,
225    PipeEquals,
226    Plus,
227    PlusEquals,
228    Pow,
229    PowEquals,
230    Private,
231    Protected,
232    Public,
233    QualifiedIdentifier,
234    Question,
235    QuestionColon,
236    Require,
237    RequireOnce,
238    Return,
239    RightBrace,
240    RightBracket,
241    RightParen,
242    SemiColon,
243    Slash,
244    SlashEquals,
245    Static,
246    StringCast,
247    BinaryCast,
248    StringPart,
249    Switch,
250    Throw,
251    Trait,
252    TripleEquals,
253    True,
254    Try,
255    Use,
256    Var,
257    Variable,
258    Yield,
259    While,
260    BitwiseNot,
261    LogicalAnd,
262    LogicalOr,
263    LogicalXor,
264}
265
266#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize, JsonSchema)]
267
268pub struct Token {
269    pub kind: TokenKind,
270    pub span: Span,
271    pub value: ByteString,
272}
273
274impl Default for Token {
275    fn default() -> Self {
276        Self {
277            kind: TokenKind::Eof,
278            span: Span::new(0, 0, 0),
279            value: ByteString::default(),
280        }
281    }
282}
283
284impl Display for Token {
285    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
286        write!(f, "{}", self.value)
287    }
288}
289
290impl Display for TokenKind {
291    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
292        let s = match self {
293            Self::CompilerHaltOffsetConstant => "__COMPILER_HALT_OFFSET__",
294            Self::Die => "die",
295            Self::Self_ => "self",
296            Self::Parent => "parent",
297            Self::Backtick => "`",
298            Self::StartDocString(_) => todo!(),
299            Self::EndDocString(..) => todo!(),
300            Self::BangEquals => "!=",
301            Self::From => "from",
302            Self::Print => "print",
303            Self::BitwiseNot => "~",
304            Self::Dollar => "$",
305            Self::HaltCompiler => "__halt_compiler",
306            Self::Readonly => "readonly",
307            Self::AsteriskEquals => "*=",
308            Self::ObjectCast => "(object)",
309            Self::UnsetCast => "(unset)",
310            Self::Abstract => "abstract",
311            Self::Ampersand => "&",
312            Self::And => "&&",
313            Self::AndEquals => "&=",
314            Self::Arrow => "->",
315            Self::QuestionArrow => "?->",
316            Self::Array => "array",
317            Self::ArrayCast => "(array)",
318            Self::As => "as",
319            Self::Asterisk => "*",
320            Self::Attribute => "#[",
321            Self::Bang => "!",
322            Self::BoolCast => "(bool)",
323            Self::BooleanCast => "(boolean)",
324            Self::BooleanAnd => "&&",
325            Self::BooleanOr => "||",
326            Self::Break => "break",
327            Self::Callable => "callable",
328            Self::Caret => "^",
329            Self::CaretEquals => "^=",
330            Self::Case => "case",
331            Self::Catch => "catch",
332            Self::Class => "class",
333            Self::ClassConstant => "__CLASS__",
334            Self::Clone => "clone",
335            Self::CloseTag => "?>",
336            Self::DoubleQuestion => "??",
337            Self::DoubleQuestionEquals => "??=",
338            Self::Colon => ":",
339            Self::Comma => ",",
340            Self::Const => "const",
341            Self::Continue => "continue",
342            Self::IntCast => "(int)",
343            Self::IntegerCast => "(integer)",
344            Self::CurlyOpen => "{$",
345            Self::Declare => "declare",
346            Self::Decrement => "--",
347            Self::Default => "default",
348            Self::DirConstant => "__DIR__",
349            Self::DivEquals => "/=",
350            Self::Do => "do",
351            Self::Dot => ".",
352            Self::DotEquals => ".=",
353            Self::DoubleArrow => "=>",
354            Self::DoubleCast => "(double)",
355            Self::RealCast => "(real)",
356            Self::FloatCast => "(float)",
357            Self::DoubleColon => "::",
358            Self::DoubleEquals => "==",
359            Self::Echo => "echo",
360            Self::Ellipsis => "...",
361            Self::Else => "else",
362            Self::ElseIf => "elseif",
363            Self::Empty => "empty",
364            Self::EndDeclare => "enddeclare",
365            Self::EndFor => "endfor",
366            Self::EndForeach => "endforeach",
367            Self::EndIf => "endif",
368            Self::EndSwitch => "endswitch",
369            Self::EndWhile => "endwhile",
370            Self::Enum => "enum",
371            Self::Eof => "[end of file]",
372            Self::Equals => "=",
373            Self::Extends => "extends",
374            Self::False => "false",
375            Self::Final => "final",
376            Self::Finally => "finally",
377            Self::LiteralFloat => return write!(f, "float literal"),
378            Self::Fn => "fn",
379            Self::For => "for",
380            Self::Function => "function",
381            Self::Goto => "goto",
382            Self::GreaterThan => ">",
383            Self::GreaterThanEquals => ">=",
384            Self::If => "if",
385            Self::Implements => "implements",
386            Self::Increment => "++",
387            Self::InlineHtml => "InlineHtml",
388            Self::LiteralInteger => return write!(f, "integer literal"),
389            Self::LeftBrace => "{",
390            Self::LeftBracket => "[",
391            Self::LeftParen => "(",
392            Self::LeftShift => "<<",
393            Self::LeftShiftEquals => "<<=",
394            Self::RightShift => ">>",
395            Self::RightShiftEquals => ">>=",
396            Self::LessThan => "<",
397            Self::LessThanEquals => "<=",
398            Self::Match => "match",
399            Self::Minus => "-",
400            Self::MinusEquals => "-=",
401            Self::Namespace => "namespace",
402            Self::NamespaceSeparator => "\\",
403            Self::New => "new",
404            Self::Null => "null",
405            Self::OpenTag(kind) => match kind {
406                OpenTagKind::Full => "<?php",
407                OpenTagKind::Short => "<?",
408                OpenTagKind::Echo => "<?=",
409            },
410            Self::Percent => "%",
411            Self::PercentEquals => "%=",
412            Self::Pipe => "|",
413            Self::PipeEquals => "|=",
414            Self::Plus => "+",
415            Self::PlusEquals => "+=",
416            Self::Pow => "**",
417            Self::Private => "private",
418            Self::Protected => "protected",
419            Self::Public => "public",
420            Self::Question => "?",
421            Self::QuestionColon => "?:",
422            Self::Require => "require",
423            Self::RequireOnce => "require_once",
424            Self::Return => "return",
425            Self::RightBrace => "}",
426            Self::RightBracket => "]",
427            Self::RightParen => ")",
428            Self::SemiColon => ";",
429            Self::Slash => "/",
430            Self::SlashEquals => "/=",
431            Self::Static => "static",
432            Self::StringCast => "(string)",
433            Self::BinaryCast => "(binary)",
434            Self::Switch => "switch",
435            Self::Throw => "throw",
436            Self::Trait => "trait",
437            Self::TripleEquals => "===",
438            Self::True => "true",
439            Self::Try => "try",
440            Self::Use => "use",
441            Self::Var => "var",
442            Self::Yield => "yield",
443            Self::While => "while",
444            Self::Global => "global",
445            Self::AngledLeftRight => "<>",
446            Self::Spaceship => "<=>",
447            Self::LogicalAnd => "and",
448            Self::LogicalOr => "or",
449            Self::LogicalXor => "xor",
450            Self::Foreach => "foreach",
451            Self::AmpersandEquals => "&=",
452            Self::At => "at",
453            Self::BangDoubleEquals => "!==",
454            Self::TraitConstant => "__TRAIT__",
455            Self::FunctionConstant => "__FUNCTION__",
456            Self::MethodConstant => "__METHOD__",
457            Self::LineConstant => "__LINE__",
458            Self::FileConstant => "__FILE__",
459            Self::DollarLeftBrace => "${",
460            Self::DoubleQuote => "\"",
461            Self::Include => "include",
462            Self::IncludeOnce => "include_once",
463            Self::Instanceof => "instanceof",
464            Self::Insteadof => "insteadof",
465            Self::Eval => "eval",
466            Self::Exit => "exit",
467            Self::Unset => "unset",
468            Self::Isset => "isset",
469            Self::List => "list",
470            Self::Interface => "interface",
471            Self::NamespaceConstant => "__NAMESPACE__",
472            Self::PowEquals => "**=",
473            Self::StringPart
474            | Self::Variable
475            | Self::QualifiedIdentifier
476            | Self::Identifier
477            | Self::FullyQualifiedIdentifier
478            | Self::LiteralSingleQuotedString
479            | Self::LiteralDoubleQuotedString
480            | Self::SingleLineComment
481            | Self::MultiLineComment
482            | Self::HashMarkComment
483            | Self::DocumentComment => {
484                return write!(f, "{:?}", self);
485            }
486        };
487
488        write!(f, "{}", s)
489    }
490}
php_parser_rs/lexer/token.rs

php_parser_rs/lexer/
token.rs
