solar_ast/
token.rs

1//! Solidity source code token.
2
3use crate::{
4    ast::{BinOp, BinOpKind, UnOp, UnOpKind},
5    DocComment,
6};
7use solar_interface::{diagnostics::ErrorGuaranteed, Ident, Span, Symbol};
8use std::{borrow::Cow, fmt};
9
10/// The type of a comment.
11#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
12pub enum CommentKind {
13    /// `// ...`, `/// ...`
14    Line,
15    /// `/* ... */`, `/** ... */`
16    Block,
17}
18
19/// A binary operation token.
20///
21/// Note that this enum contains only binary operators that can also be used in assignments.
22#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
23pub enum BinOpToken {
24    /// `+`
25    Plus,
26    /// `-`
27    Minus,
28    /// `*`
29    Star,
30    /// `/`
31    Slash,
32    /// `%`
33    Percent,
34    /// `^`
35    Caret,
36    /// `&`
37    And,
38    /// `|`
39    Or,
40    /// `<<`
41    Shl,
42    /// `>>`
43    Shr,
44    /// `>>>`
45    Sar,
46}
47
48impl fmt::Display for BinOpToken {
49    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50        f.write_str(self.to_str())
51    }
52}
53
54impl BinOpToken {
55    /// Returns the string representation of the binary operator token.
56    pub const fn to_str(self) -> &'static str {
57        match self {
58            Self::Plus => "+",
59            Self::Minus => "-",
60            Self::Star => "*",
61            Self::Slash => "/",
62            Self::Percent => "%",
63            Self::Caret => "^",
64            Self::And => "&",
65            Self::Or => "|",
66            Self::Shl => "<<",
67            Self::Shr => ">>",
68            Self::Sar => ">>>",
69        }
70    }
71
72    /// Returns the string representation of the binary operator token followed by an equals sign
73    /// (`=`).
74    pub const fn to_str_with_eq(self) -> &'static str {
75        match self {
76            Self::Plus => "+=",
77            Self::Minus => "-=",
78            Self::Star => "*=",
79            Self::Slash => "/=",
80            Self::Percent => "%=",
81            Self::Caret => "^=",
82            Self::And => "&=",
83            Self::Or => "|=",
84            Self::Shl => "<<=",
85            Self::Shr => ">>=",
86            Self::Sar => ">>>=",
87        }
88    }
89
90    /// Returns the binary operator kind.
91    #[inline]
92    pub const fn as_binop(self) -> BinOpKind {
93        match self {
94            Self::Plus => BinOpKind::Add,
95            Self::Minus => BinOpKind::Sub,
96            Self::Star => BinOpKind::Mul,
97            Self::Slash => BinOpKind::Div,
98            Self::Percent => BinOpKind::Rem,
99            Self::Caret => BinOpKind::BitXor,
100            Self::And => BinOpKind::BitAnd,
101            Self::Or => BinOpKind::BitOr,
102            Self::Shl => BinOpKind::Shl,
103            Self::Shr => BinOpKind::Shr,
104            Self::Sar => BinOpKind::Sar,
105        }
106    }
107}
108
109/// Describes how a sequence of token trees is delimited.
110#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
111pub enum Delimiter {
112    /// `( ... )`
113    Parenthesis,
114    /// `{ ... }`
115    Brace,
116    /// `[ ... ]`
117    Bracket,
118}
119
120impl Delimiter {
121    /// Returns the string representation of the opening delimiter.
122    pub const fn to_open_str(self) -> &'static str {
123        match self {
124            Self::Parenthesis => "(",
125            Self::Brace => "{",
126            Self::Bracket => "[",
127        }
128    }
129
130    /// Returns the string representation of the closing delimiter.
131    pub const fn to_close_str(self) -> &'static str {
132        match self {
133            Self::Parenthesis => ")",
134            Self::Brace => "}",
135            Self::Bracket => "]",
136        }
137    }
138}
139
140/// A literal token. Different from an AST literal as this is unparsed and only contains the raw
141/// contents.
142#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
143pub struct TokenLit {
144    /// The symbol of the literal token, excluding any quotes.
145    pub symbol: Symbol,
146    /// The literal kind.
147    pub kind: TokenLitKind,
148}
149
150impl fmt::Display for TokenLit {
151    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
152        let &Self { kind, symbol } = self;
153        match kind {
154            TokenLitKind::Str => write!(f, "\"{symbol}\""),
155            TokenLitKind::UnicodeStr => write!(f, "unicode\"{symbol}\""),
156            TokenLitKind::HexStr => write!(f, "hex\"{symbol}\""),
157            TokenLitKind::Integer | TokenLitKind::Rational | TokenLitKind::Err(_) => {
158                write!(f, "{symbol}")
159            }
160        }
161    }
162}
163
164impl TokenLit {
165    /// Creates a new literal token.
166    #[inline]
167    pub const fn new(kind: TokenLitKind, symbol: Symbol) -> Self {
168        Self { kind, symbol }
169    }
170
171    /// Returns a description of the literal.
172    pub const fn description(self) -> &'static str {
173        self.kind.description()
174    }
175}
176
177/// A kind of literal token.
178#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
179pub enum TokenLitKind {
180    /// An integer literal token.
181    Integer,
182    /// A rational literal token.
183    Rational,
184    /// A string literal token.
185    Str,
186    /// A unicode string literal token.
187    UnicodeStr,
188    /// A hex string literal token.
189    HexStr,
190    /// An error occurred while lexing the literal token.
191    Err(ErrorGuaranteed),
192}
193
194impl TokenLitKind {
195    /// Returns the description of the literal kind.
196    pub const fn description(self) -> &'static str {
197        match self {
198            Self::Integer => "integer",
199            Self::Rational => "rational",
200            Self::Str => "string",
201            Self::UnicodeStr => "unicode string",
202            Self::HexStr => "hex string",
203            Self::Err(_) => "error",
204        }
205    }
206}
207
208/// A kind of token.
209#[derive(Clone, Copy, Debug, PartialEq, Eq)]
210pub enum TokenKind {
211    // Expression-operator symbols.
212    /// `=`
213    Eq,
214    /// `<`
215    Lt,
216    /// `<=`
217    Le,
218    /// `==`
219    EqEq,
220    /// `!=`
221    Ne,
222    /// `>=`
223    Ge,
224    /// `>`
225    Gt,
226    /// `&&`
227    AndAnd,
228    /// `||`
229    OrOr,
230    /// `!`
231    Not,
232    /// `~`
233    Tilde,
234    /// `:=`
235    Walrus,
236    /// `++`
237    PlusPlus,
238    /// `--`
239    MinusMinus,
240    /// `**`
241    StarStar,
242    /// A binary operator token.
243    BinOp(BinOpToken),
244    /// A binary operator token, followed by an equals sign (`=`).
245    BinOpEq(BinOpToken),
246
247    // Structural symbols.
248    /// `@`
249    At,
250    /// `.`
251    Dot,
252    /// `,`
253    Comma,
254    /// `;`
255    Semi,
256    /// `:`
257    Colon,
258    /// `->`
259    Arrow,
260    /// `=>`
261    FatArrow,
262    /// `?`
263    Question,
264    /// An opening delimiter (e.g., `{`).
265    OpenDelim(Delimiter),
266    /// A closing delimiter (e.g., `}`).
267    CloseDelim(Delimiter),
268
269    // Literals.
270    /// A literal token.
271    ///
272    /// Note that this does not include boolean literals.
273    ///
274    /// `Symbol` is the literal's parsed data. In string literals, this is the unescaped value, and
275    /// excludes its quotes (`"`, `'`) and prefix (`hex`, `unicode`).
276    Literal(TokenLitKind, Symbol),
277
278    /// Identifier token.
279    Ident(Symbol),
280
281    /// A comment or doc-comment token.
282    ///
283    /// `Symbol` is the comment's data excluding its "quotes" (`//`, `/**`)
284    /// similarly to symbols in string literal tokens.
285    Comment(bool /* is_doc */, CommentKind, Symbol),
286
287    /// End of file marker.
288    Eof,
289}
290
291impl fmt::Display for TokenKind {
292    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
293        f.write_str(&self.description())
294    }
295}
296
297impl TokenKind {
298    /// Creates a new literal token kind.
299    pub fn lit(kind: TokenLitKind, symbol: Symbol) -> Self {
300        Self::Literal(kind, symbol)
301    }
302
303    /// Returns the string representation of the token kind.
304    pub fn as_str(&self) -> &str {
305        match self {
306            Self::Eq => "=",
307            Self::Lt => "<",
308            Self::Le => "<=",
309            Self::EqEq => "==",
310            Self::Ne => "!=",
311            Self::Ge => ">=",
312            Self::Gt => ">",
313            Self::AndAnd => "&&",
314            Self::OrOr => "||",
315            Self::Not => "!",
316            Self::Tilde => "~",
317            Self::Walrus => ":=",
318            Self::PlusPlus => "++",
319            Self::MinusMinus => "--",
320            Self::StarStar => "**",
321            Self::BinOp(op) => op.to_str(),
322            Self::BinOpEq(op) => op.to_str_with_eq(),
323
324            Self::At => "@",
325            Self::Dot => ".",
326            Self::Comma => ",",
327            Self::Semi => ";",
328            Self::Colon => ":",
329            Self::Arrow => "->",
330            Self::FatArrow => "=>",
331            Self::Question => "?",
332            Self::OpenDelim(d) => d.to_open_str(),
333            Self::CloseDelim(d) => d.to_close_str(),
334
335            Self::Literal(.., symbol) | Self::Ident(.., symbol) | Self::Comment(.., symbol) => {
336                symbol.as_str()
337            }
338
339            Self::Eof => "<eof>",
340        }
341    }
342
343    /// Returns the description of the token kind.
344    pub fn description(&self) -> Cow<'_, str> {
345        match self {
346            Self::Literal(kind, _) => return format!("<{}>", kind.description()).into(),
347            Self::Ident(symbol) => return symbol.to_string().into(),
348            Self::Comment(false, CommentKind::Block, _) => "<block comment>",
349            Self::Comment(true, CommentKind::Block, _) => "<block doc-comment>",
350            Self::Comment(false, CommentKind::Line, _) => "<line comment>",
351            Self::Comment(true, CommentKind::Line, _) => "<line doc-comment>",
352            _ => self.as_str(),
353        }
354        .into()
355    }
356
357    /// Returns `true` if the token kind is an operator.
358    pub const fn is_op(&self) -> bool {
359        use TokenKind::*;
360        match self {
361            Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | Walrus
362            | PlusPlus | MinusMinus | StarStar | BinOp(_) | BinOpEq(_) | At | Dot | Comma
363            | Colon | Arrow | FatArrow | Question => true,
364
365            OpenDelim(..) | CloseDelim(..) | Literal(..) | Comment(..) | Ident(..) | Semi | Eof => {
366                false
367            }
368        }
369    }
370
371    /// Returns the token kind as a unary operator, if any.
372    pub fn as_unop(&self, is_postfix: bool) -> Option<UnOpKind> {
373        let kind = if is_postfix {
374            match self {
375                Self::PlusPlus => UnOpKind::PostInc,
376                Self::MinusMinus => UnOpKind::PostDec,
377                _ => return None,
378            }
379        } else {
380            match self {
381                Self::PlusPlus => UnOpKind::PreInc,
382                Self::MinusMinus => UnOpKind::PreDec,
383                Self::Not => UnOpKind::Not,
384                Self::Tilde => UnOpKind::BitNot,
385                Self::BinOp(BinOpToken::Minus) => UnOpKind::Neg,
386                _ => return None,
387            }
388        };
389        debug_assert_eq!(kind.is_postfix(), is_postfix);
390        Some(kind)
391    }
392
393    /// Returns the token kind as a binary operator, if any.
394    #[inline]
395    pub fn as_binop(&self) -> Option<BinOpKind> {
396        match self {
397            Self::Eq => Some(BinOpKind::Eq),
398            Self::Lt => Some(BinOpKind::Lt),
399            Self::Le => Some(BinOpKind::Le),
400            Self::EqEq => Some(BinOpKind::Eq),
401            Self::Ne => Some(BinOpKind::Ne),
402            Self::Ge => Some(BinOpKind::Ge),
403            Self::Gt => Some(BinOpKind::Gt),
404            Self::AndAnd => Some(BinOpKind::And),
405            Self::OrOr => Some(BinOpKind::Or),
406            Self::StarStar => Some(BinOpKind::Pow),
407            Self::BinOp(op) => Some(op.as_binop()),
408            _ => None,
409        }
410    }
411
412    /// Returns the token kind as a binary operator, if any.
413    #[inline]
414    pub fn as_binop_eq(&self) -> Option<BinOpKind> {
415        match self {
416            Self::BinOpEq(op) => Some(op.as_binop()),
417            _ => None,
418        }
419    }
420
421    /// Returns `true` if the token kind is a normal comment (not a doc-comment).
422    #[inline]
423    pub const fn is_comment(&self) -> bool {
424        matches!(self, Self::Comment(false, ..))
425    }
426
427    /// Returns `true` if the token kind is a comment or doc-comment.
428    #[inline]
429    pub const fn is_comment_or_doc(&self) -> bool {
430        matches!(self, Self::Comment(..))
431    }
432
433    /// Glues two token kinds together.
434    pub const fn glue(self, other: Self) -> Option<Self> {
435        use BinOpToken::*;
436        use TokenKind::*;
437        Some(match self {
438            Eq => match other {
439                Eq => EqEq,
440                Gt => FatArrow,
441                _ => return None,
442            },
443            Lt => match other {
444                Eq => Le,
445                Lt => BinOp(Shl),
446                Le => BinOpEq(Shl),
447                _ => return None,
448            },
449            Gt => match other {
450                Eq => Ge,
451                Gt => BinOp(Shr),
452                Ge => BinOpEq(Shr),
453                BinOp(Shr) => BinOp(Sar),
454                BinOpEq(Shr) => BinOpEq(Sar),
455                _ => return None,
456            },
457            Not => match other {
458                Eq => Ne,
459                _ => return None,
460            },
461            Colon => match other {
462                Eq => Walrus,
463                _ => return None,
464            },
465            BinOp(op) => match (op, other) {
466                (op, Eq) => BinOpEq(op),
467                (And, BinOp(And)) => AndAnd,
468                (Or, BinOp(Or)) => OrOr,
469                (Minus, Gt) => Arrow,
470                (Shr, Gt) => BinOp(Sar),
471                (Shr, Ge) => BinOpEq(Sar),
472                (Plus, BinOp(Plus)) => PlusPlus,
473                (Minus, BinOp(Minus)) => MinusMinus,
474                (Star, BinOp(Star)) => StarStar,
475                _ => return None,
476            },
477
478            Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | Walrus | PlusPlus | MinusMinus
479            | StarStar | BinOpEq(_) | At | Dot | Comma | Semi | Arrow | FatArrow | Question
480            | OpenDelim(_) | CloseDelim(_) | Literal(..) | Ident(_) | Comment(..) | Eof => {
481                return None
482            }
483        })
484    }
485}
486
487/// A single token.
488#[derive(Clone, Copy, Debug, PartialEq, Eq)]
489pub struct Token {
490    /// The kind of the token.
491    pub kind: TokenKind,
492    /// The full span of the token.
493    pub span: Span,
494}
495
496impl From<Ident> for Token {
497    #[inline]
498    fn from(ident: Ident) -> Self {
499        Self::from_ast_ident(ident)
500    }
501}
502
503impl Token {
504    /// The [EOF](TokenKind::Eof) token.
505    pub const EOF: Self = Self::new(TokenKind::Eof, Span::DUMMY);
506
507    /// A dummy token that will be thrown away later.
508    pub const DUMMY: Self = Self::new(TokenKind::Question, Span::DUMMY);
509
510    /// Creates a new token.
511    #[inline]
512    pub const fn new(kind: TokenKind, span: Span) -> Self {
513        Self { kind, span }
514    }
515
516    /// Recovers a `Token` from an `Ident`.
517    #[inline]
518    pub fn from_ast_ident(ident: Ident) -> Self {
519        Self::new(TokenKind::Ident(ident.name), ident.span)
520    }
521
522    /// Creates a new identifier if the kind is [`TokenKind::Ident`].
523    #[inline]
524    pub const fn ident(&self) -> Option<Ident> {
525        match self.kind {
526            TokenKind::Ident(ident) => Some(Ident::new(ident, self.span)),
527            _ => None,
528        }
529    }
530
531    /// Returns the literal if the kind is [`TokenKind::Literal`].
532    #[inline]
533    pub const fn lit(&self) -> Option<TokenLit> {
534        match self.kind {
535            TokenKind::Literal(kind, symbol) => Some(TokenLit::new(kind, symbol)),
536            _ => None,
537        }
538    }
539
540    /// Returns this token's literal kind, if any.
541    #[inline]
542    pub const fn lit_kind(&self) -> Option<TokenLitKind> {
543        match self.kind {
544            TokenKind::Literal(kind, _) => Some(kind),
545            _ => None,
546        }
547    }
548
549    /// Returns the comment if the kind is [`TokenKind::Comment`], and whether it's a doc-comment.
550    #[inline]
551    pub const fn comment(&self) -> Option<(bool, DocComment)> {
552        match self.kind {
553            TokenKind::Comment(is_doc, kind, symbol) => {
554                Some((is_doc, DocComment { span: self.span, kind, symbol }))
555            }
556            _ => None,
557        }
558    }
559
560    /// Returns the comment if the kind is [`TokenKind::Comment`].
561    ///
562    /// Does not check that `is_doc` is `true`.
563    #[inline]
564    pub const fn doc(&self) -> Option<DocComment> {
565        match self.kind {
566            TokenKind::Comment(_, kind, symbol) => {
567                Some(DocComment { span: self.span, kind, symbol })
568            }
569            _ => None,
570        }
571    }
572
573    /// Returns `true` if the token is an operator.
574    #[inline]
575    pub const fn is_op(&self) -> bool {
576        self.kind.is_op()
577    }
578
579    /// Returns the token as a unary operator, if any.
580    #[inline]
581    pub fn as_unop(&self, is_postfix: bool) -> Option<UnOp> {
582        self.kind.as_unop(is_postfix).map(|kind| UnOp { span: self.span, kind })
583    }
584
585    /// Returns the token as a binary operator, if any.
586    #[inline]
587    pub fn as_binop(&self) -> Option<BinOp> {
588        self.kind.as_binop().map(|kind| BinOp { span: self.span, kind })
589    }
590
591    /// Returns the token as a binary operator, if any.
592    #[inline]
593    pub fn as_binop_eq(&self) -> Option<BinOp> {
594        self.kind.as_binop_eq().map(|kind| BinOp { span: self.span, kind })
595    }
596
597    /// Returns `true` if the token is an identifier.
598    #[inline]
599    pub const fn is_ident(&self) -> bool {
600        matches!(self.kind, TokenKind::Ident(_))
601    }
602
603    /// Returns `true` if the token is a literal. Includes `bool` literals.
604    #[inline]
605    pub fn is_lit(&self) -> bool {
606        matches!(self.kind, TokenKind::Literal(..)) || self.is_bool_lit()
607    }
608
609    /// Returns `true` if the token is a given keyword, `kw`.
610    #[inline]
611    pub fn is_keyword(&self, kw: Symbol) -> bool {
612        self.is_ident_where(|id| id.name == kw)
613    }
614
615    /// Returns `true` if the token is any of the given keywords.
616    #[inline]
617    pub fn is_keyword_any(&self, kws: &[Symbol]) -> bool {
618        self.is_ident_where(|id| kws.contains(&id.name))
619    }
620
621    /// Returns `true` if the token is a keyword used in the language.
622    #[inline]
623    pub fn is_used_keyword(&self) -> bool {
624        self.is_ident_where(Ident::is_used_keyword)
625    }
626
627    /// Returns `true` if the token is a keyword reserved for possible future use.
628    #[inline]
629    pub fn is_unused_keyword(&self) -> bool {
630        self.is_ident_where(Ident::is_unused_keyword)
631    }
632
633    /// Returns `true` if the token is a keyword.
634    #[inline]
635    pub fn is_reserved_ident(&self, yul: bool) -> bool {
636        self.is_ident_where(|i| i.is_reserved(yul))
637    }
638
639    /// Returns `true` if the token is an identifier, but not a keyword.
640    #[inline]
641    pub fn is_non_reserved_ident(&self, yul: bool) -> bool {
642        self.is_ident_where(|i| i.is_non_reserved(yul))
643    }
644
645    /// Returns `true` if the token is an elementary type name.
646    ///
647    /// Note that this does not include `[u]fixedMxN` types.
648    #[inline]
649    pub fn is_elementary_type(&self) -> bool {
650        self.is_ident_where(Ident::is_elementary_type)
651    }
652
653    /// Returns `true` if the token is the identifier `true` or `false`.
654    #[inline]
655    pub fn is_bool_lit(&self) -> bool {
656        self.is_ident_where(|id| id.name.is_bool_lit())
657    }
658
659    /// Returns `true` if the token is a numeric literal.
660    #[inline]
661    pub fn is_numeric_lit(&self) -> bool {
662        matches!(self.kind, TokenKind::Literal(TokenLitKind::Integer | TokenLitKind::Rational, _))
663    }
664
665    /// Returns `true` if the token is the integer literal.
666    #[inline]
667    pub fn is_integer_lit(&self) -> bool {
668        matches!(self.kind, TokenKind::Literal(TokenLitKind::Integer, _))
669    }
670
671    /// Returns `true` if the token is the rational literal.
672    #[inline]
673    pub fn is_rational_lit(&self) -> bool {
674        matches!(self.kind, TokenKind::Literal(TokenLitKind::Rational, _))
675    }
676
677    /// Returns `true` if the token is a string literal.
678    #[inline]
679    pub fn is_str_lit(&self) -> bool {
680        matches!(self.kind, TokenKind::Literal(TokenLitKind::Str, _))
681    }
682
683    /// Returns `true` if the token is an identifier for which `pred` holds.
684    #[inline]
685    pub fn is_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
686        self.ident().map(pred).unwrap_or(false)
687    }
688
689    /// Returns `true` if the token is an end-of-file marker.
690    #[inline]
691    pub const fn is_eof(&self) -> bool {
692        matches!(self.kind, TokenKind::Eof)
693    }
694
695    /// Returns `true` if the token is the given open delimiter.
696    #[inline]
697    pub fn is_open_delim(&self, d: Delimiter) -> bool {
698        self.kind == TokenKind::OpenDelim(d)
699    }
700
701    /// Returns `true` if the token is the given close delimiter.
702    #[inline]
703    pub fn is_close_delim(&self, d: Delimiter) -> bool {
704        self.kind == TokenKind::CloseDelim(d)
705    }
706
707    /// Returns `true` if the token is a normal comment (not a doc-comment).
708    #[inline]
709    pub const fn is_comment(&self) -> bool {
710        self.kind.is_comment()
711    }
712
713    /// Returns `true` if the token is a comment or doc-comment.
714    #[inline]
715    pub const fn is_comment_or_doc(&self) -> bool {
716        self.kind.is_comment_or_doc()
717    }
718
719    /// Returns `true` if the token is a location specifier.
720    #[inline]
721    pub fn is_location_specifier(&self) -> bool {
722        self.is_ident_where(Ident::is_location_specifier)
723    }
724
725    /// Returns `true` if the token is a mutability specifier.
726    #[inline]
727    pub fn is_mutability_specifier(&self) -> bool {
728        self.is_ident_where(Ident::is_mutability_specifier)
729    }
730
731    /// Returns `true` if the token is a visibility specifier.
732    #[inline]
733    pub fn is_visibility_specifier(&self) -> bool {
734        self.is_ident_where(Ident::is_visibility_specifier)
735    }
736
737    /// Returns this token's full description: `{self.description()} '{self.kind}'`.
738    pub fn full_description(&self) -> impl fmt::Display + '_ {
739        // https://github.com/rust-lang/rust/blob/44bf2a32a52467c45582c3355a893400e620d010/compiler/rustc_parse/src/parser/mod.rs#L378
740        if let Some(description) = self.description() {
741            format!("{description} `{}`", self.kind)
742        } else {
743            format!("`{}`", self.kind)
744        }
745    }
746
747    /// Returns the string representation of the token.
748    pub fn as_str(&self) -> &str {
749        self.kind.as_str()
750    }
751
752    /// Returns this token's description, if any.
753    #[inline]
754    pub fn description(self) -> Option<TokenDescription> {
755        TokenDescription::from_token(self)
756    }
757
758    /// Glues two tokens together.
759    pub fn glue(self, other: Self) -> Option<Self> {
760        self.kind.glue(other.kind).map(|kind| Self::new(kind, self.span.to(other.span)))
761    }
762}
763
764/// A description of a token.
765///
766/// Precedes the token string in error messages like `keyword 'for'` in `expected identifier, found
767/// keyword 'for'`. See [`full_description`](Token::full_description).
768#[derive(Clone, Copy, Debug, PartialEq, Eq)]
769pub enum TokenDescription {
770    /// A keyword.
771    Keyword,
772    /// A reserved keyword.
773    ReservedKeyword,
774    /// A Yul keyword.
775    YulKeyword,
776    /// A Yul EVM builtin.
777    YulEvmBuiltin,
778}
779
780impl fmt::Display for TokenDescription {
781    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
782        f.write_str(self.to_str())
783    }
784}
785
786impl TokenDescription {
787    /// Returns the description of the given token.
788    pub fn from_token(token: Token) -> Option<Self> {
789        match token.kind {
790            _ if token.is_used_keyword() => Some(Self::Keyword),
791            _ if token.is_unused_keyword() => Some(Self::ReservedKeyword),
792            _ if token.is_ident_where(|id| id.is_yul_keyword()) => Some(Self::YulKeyword),
793            _ if token.is_ident_where(|id| id.is_yul_evm_builtin()) => Some(Self::YulEvmBuiltin),
794            _ => None,
795        }
796    }
797
798    /// Returns the string representation of the token description.
799    pub const fn to_str(self) -> &'static str {
800        match self {
801            Self::Keyword => "keyword",
802            Self::ReservedKeyword => "reserved keyword",
803            Self::YulKeyword => "Yul keyword",
804            Self::YulEvmBuiltin => "Yul EVM builtin keyword",
805        }
806    }
807}
solar_ast/token.rs

solar_ast/
token.rs
